Skip to content

Commit f48e2e2

Browse files
khavnekar-ykhavnekar-y
committed
2 parents ccba923 + f262e40 commit f48e2e2

File tree

134 files changed

+42796
-429
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

134 files changed

+42796
-429
lines changed

‎.github/workflows/snowpark-ci-cd.yml

Lines changed: 63 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -22,30 +22,14 @@ jobs:
2222
pip install -r requirements.txt
2323
- name: Debug Environment Variables
2424
run: |
25-
echo "SNOWFLAKE_ACCOUNT=${{ secrets.SNOWFLAKE_ACCOUNT }}"
26-
echo "SNOWFLAKE_USER=${{ secrets.SNOWFLAKE_USER }}"
27-
echo "SNOWFLAKE_PASSWORD=${{ secrets.SNOWFLAKE_PASSWORD }}"
28-
echo "SNOWFLAKE_WAREHOUSE=${{ secrets.SNOWFLAKE_WAREHOUSE }}"
29-
echo "AWS_ACCESS_KEY=${{ secrets.AWS_ACCESS_KEY }}"
30-
echo "AWS_SECRET_KEY=${{ secrets.AWS_SECRET_KEY }}"
31-
- name: Run tests
32-
env:
33-
SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
34-
SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
35-
SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
36-
SNOWFLAKE_ROLE: CO2_ROLE # Default role
37-
SNOWFLAKE_WAREHOUSE: CO2_WH
38-
SNOWFLAKE_SCHEMA: RAW_CO2
39-
AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
40-
AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
41-
run: |
42-
cd tests
43-
pytest testing_snowflake_connection.py
44-
pytest testing_s3_connection.py
45-
pytest testing_volatility_co2ppm.py
46-
pytest testing_daily_percent_change.py
47-
pytest testing_weekly_changes.py
48-
cd ../..
25+
pytest tests/testing_snowflake_connection.py
26+
pytest tests/test_s3_connection.py
27+
pytest tests/test_volatility_co2ppm.py
28+
pytest tests/test_daily_co2_changes.py
29+
pytest tests/test_weekly_co2_changes.py
30+
pytest tests/test_loading_co2_data_sp.py
31+
pytest tests/test_co2_harmonized_sp.py
32+
pytest tests/test_co2_analytical_sp.py
4933
5034
deploy:
5135
needs: test
@@ -71,86 +55,78 @@ jobs:
7155
run: |
7256
python -m pip install --upgrade pip
7357
pip install -r requirements.txt
74-
58+
7559
- name: Set environment variables
7660
run: |
7761
echo "AWS_ACCESS_KEY=${{ secrets.AWS_ACCESS_KEY }}" >> $GITHUB_ENV
7862
echo "AWS_SECRET_KEY=${{ secrets.AWS_SECRET_KEY }}" >> $GITHUB_ENV
7963
if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
8064
echo "ENVIRONMENT=prod" >> $GITHUB_ENV
8165
echo "DATABASE_NAME=CO2_DB_PROD" >> $GITHUB_ENV
82-
echo "SNOWFLAKE_ROLE=CO2_ROLE_PROD" >> $GITHUB_ENV # Set prod role
66+
echo "CONN_PROFILE=prod" >> $GITHUB_ENV
8367
else
8468
echo "ENVIRONMENT=dev" >> $GITHUB_ENV
8569
echo "DATABASE_NAME=CO2_DB_DEV" >> $GITHUB_ENV
86-
echo "SNOWFLAKE_ROLE=CO2_ROLE_DEV" >> $GITHUB_ENV # Set dev role
87-
fi
88-
89-
- name: Debug Environment Variables Before Deployment
90-
run: |
91-
echo "ENVIRONMENT=$ENVIRONMENT"
92-
echo "DATABASE_NAME=$DATABASE_NAME"
93-
echo "SNOWFLAKE_ROLE=$SNOWFLAKE_ROLE"
94-
echo "SNOWFLAKE_ACCOUNT=$SNOWFLAKE_ACCOUNT"
95-
echo "SNOWFLAKE_USER=$SNOWFLAKE_USER"
96-
echo "SNOWFLAKE_WAREHOUSE=$SNOWFLAKE_WAREHOUSE"
97-
70+
echo "CONN_PROFILE=dev" >> $GITHUB_ENV
71+
9872
- name: Generate configuration and SQL files
9973
run: |
10074
echo "AWS_ACCESS_KEY=$AWS_ACCESS_KEY" > .env
10175
echo "AWS_SECRET_KEY=$AWS_SECRET_KEY" >> .env
10276
python scripts/render_yaml.py $ENVIRONMENT
77+
78+
# Run script to generate setup and orchestration SQL files
10379
python scripts/render_setup.py $ENVIRONMENT
104-
105-
- name: Check Snowflake CLI Version
106-
run: |
107-
snow --version
108-
109-
- name: Setup Snowflake
110-
run: |
111-
echo "Running setup SQL script..."
112-
snow sql -f scripts/setup_${ENVIRONMENT}.sql \
113-
--accountname $SNOWFLAKE_ACCOUNT \
114-
--user $SNOWFLAKE_USER \
115-
--role $SNOWFLAKE_ROLE \
116-
--warehouse $SNOWFLAKE_WAREHOUSE \
117-
--database $DATABASE_NAME \
118-
--schema $SCHEMA_NAME \
119-
--temporary-connection
120-
80+
12181
- name: Deploy UDFs and Stored Procedures
12282
run: |
123-
echo "Deploying UDFs and Stored Procedures..."
124-
# Deploy using temporary connection
125-
for dir in udfs_and_spoc/*; do
126-
if [ -d "$dir" ]; then
127-
echo "Deploying project in $dir"
128-
cd "$dir"
129-
snow snowpark build --temporary-connection \
130-
--account $SNOWFLAKE_ACCOUNT \
131-
--user $SNOWFLAKE_USER \
132-
--role $SNOWFLAKE_ROLE \
133-
--warehouse $SNOWFLAKE_WAREHOUSE \
134-
--database $DATABASE_NAME
135-
snow snowpark deploy --replace --temporary-connection \
136-
--account $SNOWFLAKE_ACCOUNT \
137-
--user $SNOWFLAKE_USER \
138-
--role $SNOWFLAKE_ROLE \
139-
--warehouse $SNOWFLAKE_WAREHOUSE \
140-
--database $DATABASE_NAME
141-
cd -
142-
fi
143-
done
144-
83+
# 1. Deploy CO2 Volatility UDF
84+
cd udfs_and_spoc/python_udf
85+
echo "Deploying CO2 Volatility UDF..."
86+
snow snowpark build --connection-name $CONN_PROFILE
87+
snow snowpark deploy --replace --connection-name $CONN_PROFILE
88+
cd ../..
89+
90+
# 2. Deploy Daily CO2 Changes UDF
91+
cd udfs_and_spoc/daily_co2_changes
92+
echo "Deploying Daily CO2 Changes UDF..."
93+
snow snowpark build --connection-name $CONN_PROFILE
94+
snow snowpark deploy --replace --connection-name $CONN_PROFILE
95+
cd ../..
96+
97+
# 3. Deploy Weekly CO2 Changes UDF
98+
cd udfs_and_spoc/weekly_co2_changes
99+
echo "Deploying Weekly CO2 Changes UDF..."
100+
snow snowpark build --connection-name $CONN_PROFILE
101+
snow snowpark deploy --replace --connection-name $CONN_PROFILE
102+
cd ../..
103+
104+
# 4. Deploy CO2 Loading SP
105+
cd udfs_and_spoc/loading_co2_data_sp
106+
echo "Deploying CO2 Loading Stored Procedure..."
107+
snow snowpark build --connection-name $CONN_PROFILE
108+
snow snowpark deploy --replace --connection-name $CONN_PROFILE
109+
cd ../..
110+
111+
# 5. Deploy CO2 Harmonized SP
112+
cd udfs_and_spoc/co2_harmonized_sp
113+
echo "Deploying CO2 Harmonized Stored Procedure..."
114+
snow snowpark build --connection-name $CONN_PROFILE
115+
snow snowpark deploy --replace --connection-name $CONN_PROFILE
116+
cd ../..
117+
118+
# 6. Deploy CO2 Analytics SP
119+
cd udfs_and_spoc/co2_analytical_sp
120+
echo "Deploying CO2 Analytics Stored Procedure..."
121+
snow snowpark build --connection-name $CONN_PROFILE
122+
snow snowpark deploy --replace --connection-name $CONN_PROFILE
123+
cd ../..
124+
145125
- name: Orchestrate Tasks
146-
run: |
126+
run: |
127+
# orchestrate the tasks
147128
echo "Orchestrating tasks for $ENVIRONMENT environment..."
148-
snow sql --accountname $SNOWFLAKE_ACCOUNT \
149-
--user $SNOWFLAKE_USER \
150-
--role $SNOWFLAKE_ROLE \
151-
--warehouse $SNOWFLAKE_WAREHOUSE \
152-
--database $DATABASE_NAME \
153-
--schema $SCHEMA_NAME \
154-
-f scripts/orchestrate_tasks_${ENVIRONMENT}.sql
155-
156-
echo "Deployment complete. Tasks successfully orchestrated."
129+
snow sql -f scripts/orchestrate_tasks_${ENVIRONMENT}.sql --connection-name $CONN_PROFILE
130+
131+
# Log task execution status
132+
echo "Deployment complete. Tasks successfully orchestrated."

‎.gitignore

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,6 @@ setup_prod.sql
1212
src/data_masker/.vscode/settings.json
1313
.zip
1414
orchestrate_tasks_dev.sql
15-
orchestrate_tasks_prod.sql
15+
orchestrate_tasks_prod.sql
16+
environment.json
17+
lambda_function.zip

‎lambda_package/bin/normalizer.exe

106 KB
Binary file not shown.

‎lambda_package/certifi-2025.1.31.dist-info/INSTALLER

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
pip

‎lambda_package/certifi-2025.1.31.dist-info/LICENSE

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
This package contains a modified version of ca-bundle.crt:
2+
3+
ca-bundle.crt -- Bundle of CA Root Certificates
4+
5+
This is a bundle of X.509 certificates of public Certificate Authorities
6+
(CA). These were automatically extracted from Mozilla's root certificates
7+
file (certdata.txt). This file can be found in the mozilla source tree:
8+
https://hg.mozilla.org/mozilla-central/file/tip/security/nss/lib/ckfw/builtins/certdata.txt
9+
It contains the certificates in PEM format and therefore
10+
can be directly used with curl / libcurl / php_curl, or with
11+
an Apache+mod_ssl webserver for SSL client authentication.
12+
Just configure this file as the SSLCACertificateFile.#
13+
14+
***** BEGIN LICENSE BLOCK *****
15+
This Source Code Form is subject to the terms of the Mozilla Public License,
16+
v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain
17+
one at http://mozilla.org/MPL/2.0/.
18+
19+
***** END LICENSE BLOCK *****
20+
@(#) $RCSfile: certdata.txt,v $ $Revision: 1.80 $ $Date: 2011/11/03 15:11:58 $

‎lambda_package/certifi-2025.1.31.dist-info/METADATA

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
Metadata-Version: 2.2
2+
Name: certifi
3+
Version: 2025.1.31
4+
Summary: Python package for providing Mozilla's CA Bundle.
5+
Home-page: https://github.com/certifi/python-certifi
6+
Author: Kenneth Reitz
7+
Author-email: me@kennethreitz.com
8+
License: MPL-2.0
9+
Project-URL: Source, https://github.com/certifi/python-certifi
10+
Classifier: Development Status :: 5 - Production/Stable
11+
Classifier: Intended Audience :: Developers
12+
Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
13+
Classifier: Natural Language :: English
14+
Classifier: Programming Language :: Python
15+
Classifier: Programming Language :: Python :: 3
16+
Classifier: Programming Language :: Python :: 3 :: Only
17+
Classifier: Programming Language :: Python :: 3.6
18+
Classifier: Programming Language :: Python :: 3.7
19+
Classifier: Programming Language :: Python :: 3.8
20+
Classifier: Programming Language :: Python :: 3.9
21+
Classifier: Programming Language :: Python :: 3.10
22+
Classifier: Programming Language :: Python :: 3.11
23+
Classifier: Programming Language :: Python :: 3.12
24+
Classifier: Programming Language :: Python :: 3.13
25+
Requires-Python: >=3.6
26+
License-File: LICENSE
27+
Dynamic: author
28+
Dynamic: author-email
29+
Dynamic: classifier
30+
Dynamic: description
31+
Dynamic: home-page
32+
Dynamic: license
33+
Dynamic: project-url
34+
Dynamic: requires-python
35+
Dynamic: summary
36+
37+
Certifi: Python SSL Certificates
38+
================================
39+
40+
Certifi provides Mozilla's carefully curated collection of Root Certificates for
41+
validating the trustworthiness of SSL certificates while verifying the identity
42+
of TLS hosts. It has been extracted from the `Requests`_ project.
43+
44+
Installation
45+
------------
46+
47+
``certifi`` is available on PyPI. Simply install it with ``pip``::
48+
49+
$ pip install certifi
50+
51+
Usage
52+
-----
53+
54+
To reference the installed certificate authority (CA) bundle, you can use the
55+
built-in function::
56+
57+
>>> import certifi
58+
59+
>>> certifi.where()
60+
'/usr/local/lib/python3.7/site-packages/certifi/cacert.pem'
61+
62+
Or from the command line::
63+
64+
$ python -m certifi
65+
/usr/local/lib/python3.7/site-packages/certifi/cacert.pem
66+
67+
Enjoy!
68+
69+
.. _`Requests`: https://requests.readthedocs.io/en/master/
70+
71+
Addition/Removal of Certificates
72+
--------------------------------
73+
74+
Certifi does not support any addition/removal or other modification of the
75+
CA trust store content. This project is intended to provide a reliable and
76+
highly portable root of trust to python deployments. Look to upstream projects
77+
for methods to use alternate trust.

‎lambda_package/certifi-2025.1.31.dist-info/RECORD

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
certifi-2025.1.31.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
2+
certifi-2025.1.31.dist-info/LICENSE,sha256=6TcW2mucDVpKHfYP5pWzcPBpVgPSH2-D8FPkLPwQyvc,989
3+
certifi-2025.1.31.dist-info/METADATA,sha256=t5kcT5aGu0dQ6_psUNZYTqnC0uCRnponewm3uYjeHbg,2451
4+
certifi-2025.1.31.dist-info/RECORD,,
5+
certifi-2025.1.31.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
6+
certifi-2025.1.31.dist-info/top_level.txt,sha256=KMu4vUCfsjLrkPbSNdgdekS-pVJzBAJFO__nI8NF6-U,8
7+
certifi/__init__.py,sha256=neIaAf7BM36ygmQCmy-ZsSyjnvjWghFeu13wwEAnjj0,94
8+
certifi/__main__.py,sha256=xBBoj905TUWBLRGANOcf7oi6e-3dMP4cEoG9OyMs11g,243
9+
certifi/__pycache__/__init__.cpython-310.pyc,,
10+
certifi/__pycache__/__main__.cpython-310.pyc,,
11+
certifi/__pycache__/core.cpython-310.pyc,,
12+
certifi/cacert.pem,sha256=xVsh-Qf3-G1IrdCTVS-1ZRdJ_1-GBQjMu0I9bB-9gMc,297255
13+
certifi/core.py,sha256=qRDDFyXVJwTB_EmoGppaXU_R9qCZvhl-EzxPMuV3nTA,4426
14+
certifi/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0

‎lambda_package/certifi-2025.1.31.dist-info/WHEEL

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Wheel-Version: 1.0
2+
Generator: setuptools (75.8.0)
3+
Root-Is-Purelib: true
4+
Tag: py3-none-any
5+

‎lambda_package/certifi-2025.1.31.dist-info/top_level.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
certifi

‎lambda_package/certifi/__init__.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
from .core import contents, where
2+
3+
__all__ = ["contents", "where"]
4+
__version__ = "2025.01.31"

0 commit comments

Comments
 (0)