dev changes to deployment

Author: shushilgirish

scripts/02_create_rawco2data_stream.py renamed to scripts/raw data loading and stream creation/02_create_rawco2data_stream.py

@@ -180,81 +180,10 @@ def deploy_component(profile_name, component_path, component_name, component_typ
                     code_dir = os.path.join(component_path, subdirs[0])
                     logger.info(f"Using actual code directory: {code_dir}")
 
-            # Check for UDF function signature to automatically fix issues
-            function_file = os.path.join(code_dir, "function.py")
-            has_session_param = False
-            
-            if os.path.exists(function_file):
-                try:
-                    with open(function_file, 'r') as f:
-                        content = f.read()
-                    
-                    # Check if we need to fix the function signature for UDFs with session parameters
-                    if component_type.lower() == "udf" and "def main(session, input_data" in content:
-                        logger.info("Detected UDF with session parameter - applying automatic fix")
-                        has_session_param = True
-                        
-                        # Check if we need to automatically modify UDF with wrapper function
-                        if "def main_with_session(" not in content:
-                            # Create a backup of the original file 
-                            backup_file = function_file + ".bak"
-                            with open(backup_file, 'w') as f:
-                                f.write(content)
-                            
-                            # Modify the content to include a wrapper
-                            modified_content = content.replace(
-                                "def main(session, input_data",
-                                """# Original function 
-def main_with_session(session, input_data
-
-# Wrapper function that Snowflake calls directly
-def main(input_data"""
-                            )
-                            
-                            # Add the wrapper implementation at the end
-                            indent = "    "  # Default indentation
-                            lines = modified_content.split("\n")
-                            # Find where to insert wrapper code
-                            insert_pos = len(lines)  # Default to end of file
-                            for i in range(len(lines)):
-                                if lines[i].strip() == "def main(input_data":
-                                    # Find indentation level and next non-empty line
-                                    for j in range(i+1, len(lines)):
-                                        if lines[j].strip():
-                                            indent = lines[j].split(lines[j].lstrip())[0]
-                                            break
-                            
-                            # Add wrapper implementation
-                            wrapper_code = [
-                                f"{indent}# Get session from Snowflake context",
-                                f"{indent}from snowflake.snowpark.context import get_active_session",
-                                f"{indent}session = get_active_session()",
-                                f"{indent}# Call the original function with session",
-                                f"{indent}return main_with_session(session, input_data)"
-                            ]
-                            
-                            # Find the position to insert the wrapper code
-                            for i in range(len(lines)-1, -1, -1):
-                                if "def main(input_data" in lines[i]:
-                                    # Find where the function body ends
-                                    func_indent = lines[i+1].split(lines[i+1].lstrip())[0]
-                                    for j in range(i+1, len(lines)):
-                                        if j == len(lines)-1 or (lines[j] and not lines[j].startswith(func_indent)):
-                                            insert_pos = j
-                                            break
-                            
-                            # Insert wrapper code at appropriate position
-                            for code_line in wrapper_code:
-                                lines.insert(insert_pos, code_line)
-                                insert_pos += 1
-                            
-                            # Write modified content back
-                            with open(function_file, 'w') as f:
-                                f.write("\n".join(lines))
-                                
-                            logger.info(f"Modified UDF function to handle session parameter")
-                except Exception as e:
-                    logger.warning(f"Could not check/fix function signature: {str(e)}")
+            # Check and fix UDF function signature if necessary
+            if component_type.lower() == "udf":
+                logger.info(f"Checking and fixing UDF function signature for {component_name}")
+                os.system(f"python scripts/check_and_fix_udf.py {code_dir}")
 
             # Log directory contents
             logger.info(f"Component directory structure:")
@@ -266,19 +195,6 @@ def main(input_data"""
                 for f in files:
                     logger.info(f"{sub_indent}{f}")
 
-            # Check if UDF requires session parameter by examining function.py
-            has_session_param = False
-            try:
-                function_file = os.path.join(code_dir, "function.py")
-                if os.path.exists(function_file):
-                    with open(function_file, 'r') as f:
-                        content = f.read()
-                        if "def main(session, input_data" in content:
-                            logger.info("Detected UDF with session parameter")
-                            has_session_param = True
-            except Exception as e:
-                logger.warning(f"Could not check function signature: {str(e)}")
-            
             # Zip the directory
             zip_directory(code_dir, zip_path)
             logger.info(f"Created zip file: {zip_path}")
@@ -294,28 +210,15 @@ def main(input_data"""
 
             if component_type.lower() == "udf":
                 # For Snowpark UDFs - different SQL based on parameter signature
-                if has_session_param:
-                    # For UDFs with session parameter (Snowpark style)
-                    sql = f"""
-                    CREATE OR REPLACE FUNCTION {component_name.replace(' ', '_')}(input_data VARIANT)
-                    RETURNS VARIANT
-                    LANGUAGE PYTHON
-                    RUNTIME_VERSION=3.8
-                    PACKAGES = ('snowflake-snowpark-python')
-                    HANDLER = 'function.main'
-                    IMPORTS = ('{import_path}')
-                    """
-                else:
-                    # For basic UDFs without session parameter
-                    sql = f"""
-                    CREATE OR REPLACE FUNCTION {component_name.replace(' ', '_')}(input_data VARIANT)
-                    RETURNS VARIANT
-                    LANGUAGE PYTHON
-                    RUNTIME_VERSION=3.8
-                    PACKAGES = ('snowflake-snowpark-python')
-                    IMPORTS = ('{import_path}')
-                    HANDLER = 'function.main'
-                    """
+                sql = f"""
+                CREATE OR REPLACE FUNCTION {component_name.replace(' ', '_')}(input_data VARIANT)
+                RETURNS VARIANT
+                LANGUAGE PYTHON
+                RUNTIME_VERSION=3.8
+                PACKAGES = ('snowflake-snowpark-python')
+                HANDLER = 'function.main'
+                IMPORTS = ('{import_path}')
+                """
             else:  # procedure
                 sql = f"""
                 CREATE OR REPLACE PROCEDURE {component_name.replace(' ', '_')}()

scripts/incremental_raw_co2data.py renamed to scripts/raw data loading and stream creation/incremental_raw_co2data.py

@@ -5,6 +5,7 @@
 import json
 import sys
 
+
 # Determine if we're running in Snowflake or locally
 is_running_in_snowflake = 'SNOWFLAKE_PYTHON_INTERPRETER' in os.environ