Skip to content

Commit eb2982a

Browse files
committed
dev changes deployment
1 parent 23b14bc commit eb2982a

File tree

2 files changed

+126
-58
lines changed

2 files changed

+126
-58
lines changed

.github/workflows/snowpark-ci-cd.yml

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -25,16 +25,13 @@ jobs:
2525
- name: Install dependencies
2626
run: |
2727
python -m pip install --upgrade pip
28-
29-
# Install Poetry for better dependency management
30-
pip install poetry
31-
28+
3229
# Install from requirements.txt
3330
pip install -r requirements.txt
3431
35-
# Verify Snow CLI installation
36-
snow --version
37-
snow help | grep snowpark
32+
# Verify Snow CLI installation - use simpler command that works with older versions
33+
snow --version || echo "Snow CLI version check failed"
34+
snow object list --help || echo "Snow CLI command check failed"
3835
3936
- name: Set environment variables based on branch
4037
run: |
@@ -116,10 +113,7 @@ jobs:
116113
- name: Install dependencies
117114
run: |
118115
python -m pip install --upgrade pip
119-
120-
# Install Poetry for better dependency management
121-
pip install poetry
122-
116+
123117
# Install from requirements.txt
124118
pip install -r requirements.txt
125119

scripts/deployment_files/snowflake_deployer.py

Lines changed: 121 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -410,33 +410,52 @@ def verify_snow_cli_installation():
410410
try:
411411
# Try to check if snow CLI is installed
412412
result = subprocess.run(["snow", "--version"], capture_output=True, text=True)
413-
logger.info(f"Snow CLI is already installed: {result.stdout.strip()}")
413+
version_output = result.stdout.strip()
414+
logger.info(f"Snow CLI is installed: {version_output}")
414415

415-
# Check if snowpark command is available
416-
help_output = subprocess.run(["snow", "help"], capture_output=True, text=True).stdout
417-
if "snowpark" not in help_output:
418-
logger.warning("Snow CLI is installed but missing 'snowpark' command. Installing Snowpark extension...")
419-
# Install Snowpark extension
420-
subprocess.run(["snow", "extension", "install", "snowpark"], check=True)
421-
logger.info("Snowpark extension installed successfully")
416+
# Parse the version number to determine command availability
417+
version_str = version_output.split(": ")[-1] if ": " in version_output else version_output
418+
version_parts = version_str.split('.')
422419

420+
try:
421+
major = int(version_parts[0])
422+
minor = int(version_parts[1]) if len(version_parts) > 1 else 0
423+
424+
logger.info(f"Snow CLI version parsed: {major}.{minor}")
425+
426+
# Use appropriate commands based on version
427+
if major == 0 and minor < 3:
428+
# For older versions like 0.2.9, check if we can list objects
429+
check_cmd = ["snow", "object", "list", "--help"]
430+
logger.info("Using Snow CLI legacy commands (v0.2.x)")
431+
else:
432+
# For newer versions, we expect snowpark commands
433+
check_cmd = ["snow", "--help"]
434+
logger.info("Using Snow CLI modern commands (v0.3+)")
435+
436+
# Check if basic commands work
437+
help_result = subprocess.run(check_cmd, capture_output=True, text=True)
438+
if help_result.returncode != 0:
439+
logger.warning(f"Snow CLI command check failed: {help_result.stderr}")
440+
else:
441+
logger.info("Snow CLI command check passed")
442+
except (ValueError, IndexError):
443+
logger.warning(f"Could not parse Snow CLI version: {version_str}")
444+
423445
return True
446+
424447
except (subprocess.SubprocessError, FileNotFoundError):
425448
logger.warning("Snow CLI not found. Attempting to install...")
426449

427450
try:
428451
# Install Snow CLI using pip
429-
subprocess.run(["pip", "install", "--upgrade", "snowflake-cli", "snowflake-cli-labs"], check=True)
452+
subprocess.run(["pip", "install", "--upgrade", "snowflake-cli==0.3.0", "snowflake-cli-labs==0.2.0"], check=True)
430453
logger.info("Successfully installed Snow CLI via pip")
431454

432455
# Verify installation
433456
result = subprocess.run(["snow", "--version"], capture_output=True, text=True)
434457
logger.info(f"Verified Snow CLI installation: {result.stdout.strip()}")
435458

436-
# Install Snowpark extension
437-
subprocess.run(["snow", "extension", "install", "snowpark"], check=True)
438-
logger.info("Snowpark extension installed successfully")
439-
440459
return True
441460
except subprocess.SubprocessError as e:
442461
logger.error(f"Failed to install Snow CLI: {str(e)}")
@@ -468,6 +487,27 @@ def deploy_snowpark_projects(root_directory, profile_name, check_git_changes=Fal
468487
os.environ["SNOWFLAKE_PASSWORD"] = conn_config.get('password', '')
469488
elif 'private_key_path' in conn_config:
470489
os.environ["SNOWFLAKE_PRIVATE_KEY_PATH"] = conn_config.get('private_key_path', '')
490+
491+
# Try to get the Snow CLI version to determine command format
492+
try:
493+
version_result = subprocess.run(["snow", "--version"], capture_output=True, text=True)
494+
version_output = version_result.stdout.strip()
495+
version_str = version_output.split(": ")[-1] if ": " in version_output else version_output
496+
version_parts = version_str.split('.')
497+
major = int(version_parts[0])
498+
minor = int(version_parts[1]) if len(version_parts) > 1 else 0
499+
500+
# Set command format based on version
501+
if major == 0 and minor < 3:
502+
# Old Snow CLI (0.2.x) uses different commands
503+
use_legacy_commands = True
504+
logger.info("Using legacy Snow CLI commands (v0.2.x)")
505+
else:
506+
use_legacy_commands = False
507+
logger.info("Using modern Snow CLI commands (v0.3+)")
508+
except Exception as e:
509+
logger.warning(f"Could not determine Snow CLI version: {str(e)}. Will use fallback.")
510+
use_legacy_commands = True
471511

472512
# Stats for summary
473513
projects_found = 0
@@ -528,6 +568,9 @@ def deploy_snowpark_projects(root_directory, profile_name, check_git_changes=Fal
528568
# Save current directory
529569
current_dir = os.getcwd()
530570

571+
# Decide deployment method based on CLI version
572+
deployment_successful = False
573+
531574
try:
532575
# Change to project directory
533576
os.chdir(f"{directory_path}")
@@ -537,41 +580,72 @@ def deploy_snowpark_projects(root_directory, profile_name, check_git_changes=Fal
537580
for item in os.listdir('.'):
538581
logger.info(f" - {item}")
539582

540-
# Build the project
541-
logger.info("Building project with Snow CLI...")
542-
build_cmd = ["snow", "snowpark", "build", "--temporary-connection",
543-
"--account", os.environ["SNOWFLAKE_ACCOUNT"],
544-
"--user", os.environ["SNOWFLAKE_USER"],
545-
"--role", os.environ["SNOWFLAKE_ROLE"],
546-
"--warehouse", os.environ["SNOWFLAKE_WAREHOUSE"],
547-
"--database", os.environ["SNOWFLAKE_DATABASE"]]
548-
549-
logger.info(f"Executing: {' '.join(build_cmd)}")
550-
build_result = subprocess.run(build_cmd, capture_output=True, text=True)
551-
logger.info(f"Build STDOUT: {build_result.stdout}")
552-
logger.info(f"Build STDERR: {build_result.stderr}")
553-
554-
# Deploy the project
555-
logger.info("Deploying project with Snow CLI...")
556-
deploy_cmd = ["snow", "snowpark", "deploy", "--replace", "--temporary-connection",
557-
"--account", os.environ["SNOWFLAKE_ACCOUNT"],
558-
"--user", os.environ["SNOWFLAKE_USER"],
559-
"--role", os.environ["SNOWFLAKE_ROLE"],
560-
"--warehouse", os.environ["SNOWFLAKE_WAREHOUSE"],
561-
"--database", os.environ["SNOWFLAKE_DATABASE"]]
562-
563-
logger.info(f"Executing: {' '.join(deploy_cmd)}")
564-
deploy_result = subprocess.run(deploy_cmd, capture_output=True, text=True)
565-
logger.info(f"Deploy STDOUT: {deploy_result.stdout}")
566-
logger.info(f"Deploy STDERR: {deploy_result.stderr}")
567-
568-
if build_result.returncode != 0 or deploy_result.returncode != 0:
569-
logger.error(f"Failed to deploy project in {directory_path}")
570-
success = False
583+
if use_legacy_commands:
584+
# Legacy Snow CLI (0.2.x) - use native Python UDF deployment
585+
logger.info("Using fallback deployment method for Snow CLI 0.2.x")
586+
587+
# Get function information from project config
588+
if 'functions' in project_settings.get('snowpark', {}) and project_settings['snowpark']['functions']:
589+
function_config = project_settings['snowpark']['functions'][0]
590+
function_name = function_config.get('name', component_name)
591+
592+
# Use fallback deployment
593+
if fallback_deploy_udf(conn_config, directory_path, function_name, project_settings):
594+
logger.info(f"Successfully deployed {project_name} using fallback method")
595+
deployment_successful = True
596+
projects_deployed += 1
597+
else:
598+
logger.error("No function definition found in project config")
571599
else:
572-
logger.info(f"Successfully deployed project in {directory_path}")
573-
projects_deployed += 1
574-
600+
# Modern Snow CLI (0.3+) - use snowpark commands
601+
# Build the project
602+
logger.info("Building project with Snow CLI...")
603+
build_cmd = ["snow", "snowpark", "build", "--temporary-connection",
604+
"--account", os.environ["SNOWFLAKE_ACCOUNT"],
605+
"--user", os.environ["SNOWFLAKE_USER"],
606+
"--role", os.environ["SNOWFLAKE_ROLE"],
607+
"--warehouse", os.environ["SNOWFLAKE_WAREHOUSE"],
608+
"--database", os.environ["SNOWFLAKE_DATABASE"]]
609+
610+
logger.info(f"Executing: {' '.join(build_cmd)}")
611+
build_result = subprocess.run(build_cmd, capture_output=True, text=True)
612+
logger.info(f"Build STDOUT: {build_result.stdout}")
613+
logger.info(f"Build STDERR: {build_result.stderr}")
614+
615+
# Deploy the project
616+
logger.info("Deploying project with Snow CLI...")
617+
deploy_cmd = ["snow", "snowpark", "deploy", "--replace", "--temporary-connection",
618+
"--account", os.environ["SNOWFLAKE_ACCOUNT"],
619+
"--user", os.environ["SNOWFLAKE_USER"],
620+
"--role", os.environ["SNOWFLAKE_ROLE"],
621+
"--warehouse", os.environ["SNOWFLAKE_WAREHOUSE"],
622+
"--database", os.environ["SNOWFLAKE_DATABASE"]]
623+
624+
logger.info(f"Executing: {' '.join(deploy_cmd)}")
625+
deploy_result = subprocess.run(deploy_cmd, capture_output=True, text=True)
626+
logger.info(f"Deploy STDOUT: {deploy_result.stdout}")
627+
logger.info(f"Deploy STDERR: {deploy_result.stderr}")
628+
629+
if build_result.returncode != 0 or deploy_result.returncode != 0:
630+
logger.error(f"Failed to deploy project in {directory_path}")
631+
# Try fallback method
632+
logger.info(f"Attempting fallback deployment for {project_name}")
633+
function_name = project_name
634+
if 'functions' in project_settings.get('snowpark', {}) and project_settings['snowpark']['functions']:
635+
function_name = project_settings['snowpark']['functions'][0].get('name', project_name)
636+
637+
if fallback_deploy_udf(conn_config, directory_path, function_name, project_settings):
638+
logger.info(f"Fallback deployment successful for {project_name}")
639+
deployment_successful = True
640+
projects_deployed += 1
641+
else:
642+
logger.info(f"Successfully deployed project in {directory_path}")
643+
deployment_successful = True
644+
projects_deployed += 1
645+
646+
except Exception as e:
647+
logger.error(f"Error processing project in {directory_path}: {str(e)}")
648+
success = False
575649
finally:
576650
# Restore original directory
577651
os.chdir(current_dir)

0 commit comments

Comments
 (0)