dev changes deployment

Author: shushilgirish

.github/workflows/snowpark-ci-cd.yml

@@ -151,13 +151,13 @@ jobs:
         run: |
           mkdir -p ~/.snowflake/keys
           
-          # Debug the Snowflake account format
-          echo "Using Snowflake account: ${SNOWFLAKE_ACCOUNT}"
-          
           # Store private key correctly from GitHub secrets
           echo "${{ secrets.SNOWFLAKE_PRIVATE_KEY }}" > ~/.snowflake/keys/rsa_key.p8
           chmod 600 ~/.snowflake/keys/rsa_key.p8
           
+          # Debug the Snowflake account format
+          echo "Using Snowflake account: ${SNOWFLAKE_ACCOUNT}"
+          
           # Install required packages
           pip install cryptography snowflake-connector-python
           
@@ -172,6 +172,7 @@ jobs:
           role = "CO2_ROLE_DEV"
           database = "CO2_DB_DEV"
           schema = "RAW_CO2"
+          client_request_mfa_token = false
           
           [prod]
           account = "$SNOWFLAKE_ACCOUNT"
@@ -182,6 +183,7 @@ jobs:
           role = "CO2_ROLE_PROD"
           database = "CO2_DB_PROD"
           schema = "RAW_CO2"
+          client_request_mfa_token = false
           EOF
           
           chmod 600 ~/.snowflake/connections.toml
@@ -271,59 +273,23 @@ jobs:
             
             echo "🔍 Checking for changes in $component_path..."
             
-            # Default to not deploying
-            should_deploy=false
-            
-            # For pull requests, always deploy components
-            if [[ "${{ github.event_name }}" == "pull_request" ]]; then
-              echo "Pull request detected - always deploying component"
-              should_deploy=true
-            # For workflow_dispatch, always deploy components
-            elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
-              echo "Manual workflow dispatch detected - always deploying component"
-              should_deploy=true
+            # For pull requests or workflow_dispatch, always deploy components
+            if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+              echo "Pull request or manual dispatch detected - deploying component"
+              # Use our Python deployment script without checking changes
+              PYTHONPATH=$PYTHONPATH:$(pwd) python -u scripts/deployment_files/snowflake_deployer.py deploy --profile $CONN_PROFILE --path "$component_path" --name "$component_name" --type "$component_type"
             else
-              # Check for force_deploy file
-              if [[ -f "$component_path/.force_deploy" ]]; then
-                echo "🔥 Force deploy file found - forcing deployment"
-                should_deploy=true
-              else
-                # For push events, check if component changed in this commit
-                # Get changed files in the most recent commit
-                changed_files=$(git diff --name-only HEAD HEAD~1)
-                echo "Changed files in latest commit:"
-                echo "$changed_files"
-                
-                # Check if any files in the component path changed
-                if echo "$changed_files" | grep -q "$component_path"; then
-                  echo "✅ Changes detected in component"
-                  should_deploy=true
-                else
-                  echo "❌ No changes detected in component"
-                fi
-              fi
+              # For push events, only deploy if component changed
+              echo "Push event detected - only deploying changed components"
+              # Use our Python deployment script with change detection
+              PYTHONPATH=$PYTHONPATH:$(pwd) python -u scripts/deployment_files/snowflake_deployer.py deploy --profile $CONN_PROFILE --path "$component_path" --name "$component_name" --type "$component_type" --check-changes
             fi
             
-            if $should_deploy; then
-              echo "⏳ Deploying $component_name..."
-              
-              # Use our Python deployment script with additional logging
-              PYTHONPATH=$PYTHONPATH:$(pwd) python -u scripts/deployment_files/snowflake_deployer.py deploy --profile $CONN_PROFILE --path "$component_path" --name "$component_name" --type "$component_type"
-              
-              if [ $? -eq 0 ]; then
-                echo "✅ Successfully deployed $component_name"
-                
-                # If force deploy file exists, remove it after successful deployment
-                if [[ -f "$component_path/.force_deploy" ]]; then
-                  echo "Removing force deploy marker"
-                  rm -f "$component_path/.force_deploy"
-                fi
-              else
-                echo "❌ Deploy failed for $component_name"
-                exit 1
-              fi
+            if [ $? -eq 0 ]; then
+              echo "✅ Component $component_name processed successfully"
             else
-              echo "⏭️ Skipping $component_name (no changes detected)"
+              echo "❌ Processing failed for $component_name"
+              exit 1
             fi
           }
           

deployment_and_key_workflow.md

@@ -0,0 +1,93 @@
+# Snowflake Deployment and RSA Key Authentication Flow
+
+## Overview of the Deployment Architecture
+
+The deployment process uses RSA key-based authentication to deploy UDFs and stored procedures to Snowflake without requiring MFA prompts. Here's how the components work together:
+
+## 1. RSA Key Generation and Authentication
+
+### Key Files:
+- **rsa_key_pair_generator.md** - Documentation with manual steps for key generation
+- **scripts/rsa_key_pair_authentication/generate_snowflake_keys.py** - Script to generate RSA keys
+- **scripts/rsa_key_pair_authentication/check_snowflake_key_auth.py** - Validates key formatting
+
+### Purpose:
+These files help create and validate the RSA key pair needed for passwordless authentication with Snowflake. The process is:
+
+1. Generate a private key (rsa_key.p8) and public key (rsa_key.pub)
+2. Register the public key with your Snowflake user account
+3. Store the private key securely locally for development and in GitHub secrets for CI/CD
+
+## 2. Connection Configuration
+
+### Key Files:
+- **~/.snowflake/connections.toml** - Stores connection profiles (local development)
+- **scripts/deployment_files/check_connections_file.py** - Diagnoses issues with connections.toml
+- **scripts/deployment_files/test_key_auth.py** - Tests key authentication
+
+### Purpose:
+These files manage the connection details for different environments:
+
+1. The connections.toml needs proper format with `private_key_path` pointing to your local key
+2. test_key_auth.py validates the key works before attempting deployment
+3. check_connections_file.py helps diagnose TOML parsing issues
+
+## 3. Deployment Process
+
+### Key Files:
+- **scripts/deployment_files/snowflake_deployer.py** - Main deployment logic
+- **.github/workflows/snowpark-ci-cd.yml** - CI/CD pipeline configuration
+
+### Purpose:
+These files handle the actual deployment:
+
+1. GitHub workflow sets up authentication in CI/CD environment
+2. snowflake_deployer.py:
+   - Analyzes UDF function signatures
+   - Packages code into ZIP files
+   - Uploads to Snowflake stage
+   - Creates UDFs/procedures with proper SQL DDL
+
+## 4. UDF Parameter Handling
+
+### Key Files:
+- **scripts/check_and_fix_udf.py** - Fixes UDF parameter mismatches
+- **udfs_and_spoc/*/function.py** - The actual UDF/procedure code
+
+### Purpose:
+The UDF files have different parameter signatures:
+1. CO2_VOLATILITY and DAILY_CO2_CHANGES have 2 parameters (current_value, previous_value)
+2. Stored procedures take session parameters
+3. The deployment process must adapt SQL definitions to match these signatures
+
+## Interrelationships Between Files
+
+1. **Key Authentication Chain:**
+   ```
+   generate_snowflake_keys.py -> rsa_key.p8/.pub -> connections.toml -> snowflake_deployer.py
+   ```
+
+2. **Deployment Chain:**
+   ```
+   CI/CD workflow -> snowflake_deployer.py -> analyze_function_signature() -> Customized SQL DDL -> Snowflake
+   ```
+
+3. **Parameter Handling Chain:**
+   ```
+   UDF function.py -> analyze_function_signature() -> SQL with matching parameters
+   ```
+
+## Current Issues and Solutions
+
+The issues you're experiencing are related to:
+
+1. **TOML File Syntax**: The connections.toml file has `//` comments instead of `#` comments
+2. **Missing Key Path**: Your connections.toml doesn't have `private_key_path` configured
+3. **Authentication Flow**: The deployment tries password auth and triggers MFA
+
+To fix:
+1. Update connections.toml with proper TOML syntax (use # for comments)
+2. Add private_key_path pointing to your local key
+3. Set `client_request_mfa_token = false` to explicitly disable MFA
+
+This workflow ensures secure, MFA-free deployments and handles different function signatures appropriately.

poetry.lock

@@ -16,6 +16,9 @@ jinja2 = "3.1.4"
 pyyaml = "^6.0.2"
 boto3 = "^1.37.3"
 cryptography = "^44.0.2"
+snowflake-connector-python = ">=3.0.0"
+logging = "^0.4.9.6"
+pyarrow = ">=7.0.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

pyproject.toml

@@ -1,4 +1,7 @@
 snowflake-snowpark-python[pandas]
+snowflake-connector-python>=3.0.0
+logging
+pyarrow>=7.0.0
 snowflake-cli-labs
 python-dotenv
 pytest

requirements.txt

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""
+Check the connections.toml file and output its structure to diagnose issues.
+"""
+import os
+import sys
+import toml
+
+def main():
+    config_path = os.path.expanduser("~/.snowflake/connections.toml")
+    print(f"Looking for connections.toml at: {config_path}")
+    
+    if not os.path.exists(config_path):
+        print(f"ERROR: File does not exist: {config_path}")
+        return False
+    
+    print(f"File exists, size: {os.path.getsize(config_path)} bytes")
+    print(f"File permissions: {oct(os.stat(config_path).st_mode)[-3:]}")
+    
+    try:
+        with open(config_path, 'r') as f:
+            content = f.read()
+            print("\nFile content preview (first 200 chars):")
+            print(content[:200])
+            
+        # Try loading with toml
+        print("\nParsing with toml:")
+        config = toml.load(config_path)
+        print(f"Profiles found: {list(config.keys())}")
+        
+        # Check each profile
+        for profile, settings in config.items():
+            print(f"\nProfile: {profile}")
+            for key, value in settings.items():
+                # Don't print sensitive values
+                if key in ['password', 'private_key']:
+                    print(f"  {key}: [REDACTED]")
+                else:
+                    print(f"  {key}: {value}")
+        
+        return True
+    except Exception as e:
+        print(f"ERROR parsing file: {str(e)}")
+        
+        # Try to read the raw file content
+        try:
+            with open(config_path, 'rb') as f:
+                binary_content = f.read(100)
+                print(f"\nBinary content (first 100 bytes): {binary_content}")
+        except Exception as e2:
+            print(f"ERROR reading raw file: {str(e2)}")
+        
+        return False
+
+if __name__ == "__main__":
+    success = main()
+    sys.exit(0 if success else 1)