Merge pull request #4 from kaarthik108/chore/refractor-incremental-ingest-chromadb

chore/refractor-incremental-ingest-chromadb

Author: kaarthik108

.example.env

@@ -6,4 +6,5 @@ DATABASE=
 SCHEMA=
 ROLE=
 
-OPENAI_API_KEY=
+OPENAI_API_KEY=
+LLM_MODEL=gpt-3.5-turbo-16k

.github/workflows/snowflake_deploy.yml

@@ -0,0 +1,94 @@
+name: Deploy to Snowflake
+
+on:
+  push:
+    branches:
+      - prod
+    paths:
+      - "src/**"
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment: ${{ github.ref_name }}
+
+    env:
+      ACCOUNT: ${{ vars.SNOWFLAKE_ACCOUNT }}
+      USER_NAME: ${{ vars.SNOWFLAKE_USER }}
+      PASSWORD: ${{ secrets.SNOWFLAKE_PWD }}
+      ROLE: ${{ vars.SNOWFLAKE_ROLE }}
+      DATABASE: ${{ vars.SNOWFLAKE_DATABASE }}
+      SCHEMA: ${{ vars.SNOWFLAKE_SCHEMA }}
+      WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Cache dependencies
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/pypoetry
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+
+      - name: Install dependencies
+        run: |
+          poetry install
+
+      - name: Build
+        run: |
+          poetry build
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v37
+
+      - name: Deploy to Snowflake
+        run: |
+          IFS=$'\n'
+          for file in ${{ steps.changed-files.outputs.all_modified_files }}; do
+              # Skip if path is not under src
+              if [[ $file != src/* ]]; then
+              continue
+              fi
+              # Extract the path relative to src
+              rel_path="${file#src/}"
+              component_type=$(echo $rel_path | cut -d'/' -f1)
+              component_name=$(echo $rel_path | cut -d'/' -f2)
+              component_path="src/$component_type/$component_name"
+              
+              # Check if the component directory exists
+              if [ ! -d "$component_path" ]; then
+              echo "Directory $component_path does not exist. Skipping..."
+              continue
+              fi
+              
+              echo "Component Type: $component_type"
+              echo "Component Name: $component_name"
+              case $component_type in
+                  task)
+                  poetry run snowdev deploy --task $component_name || echo "Failed to deploy task $component_name"
+                  ;;
+                  streamlit)
+                  poetry run snowdev deploy --streamlit $component_name || echo "Failed to deploy streamlit $component_name"
+                  ;;
+                  udf)
+                  poetry run snowdev deploy --udf $component_name || echo "Failed to deploy udf $component_name"
+                  ;;
+                  sproc)
+                  poetry run snowdev deploy --sproc $component_name || echo "Failed to deploy sproc $component_name"
+                  ;;
+              esac
+          done

.gitignore

@@ -155,4 +155,12 @@ install-snowpark-ml-1.0.2.sh
 !examples/src/
 !examples/static/
 
-chroma_db/
+.vscode/
+
+chroma_db/
+
+# checksums for chromadb
+checksums.json
+
+# compiled task scripts
+compiled/

README.md

@@ -4,6 +4,16 @@
 
 SnowDev is a command-line utility designed for deploying various components related to Snowflake such as UDFs, stored procedures, and Streamlit applications using **Snowpark**. This tool streamlines tasks like initializing directories, local testing, uploading, and auto create components code using AI.
 
+## Features
+
+SnowDev currently supports the following components:
+
+- **UDF (User-Defined Function)**
+- **Stored Procedure**
+- **Streamlit**
+- **Tasks**
+
+
 ## Setup
 
 ```bash
@@ -55,6 +65,7 @@ snowdev <command> [options]
 - `--udf <udf_name>`: Name or identifier for the UDF you want to deploy.
 - `--sproc <sproc_name>`: Name or identifier for the Stored Procedure you want to deploy.
 - `--streamlit <streamlit_name>`: Name or identifier for the Streamlit application you want to deploy. (This is still in PrPr)
+- `--task <task_name>`: Name of the snowflake task you want to deploy.
 - `--upload <upload_item>`: Specifies what to upload. Currently supported options: `static`.
 - `--package <package_name>`: Specifies the name of the package to zip and upload to the static folder.
 - `--embed`: Used with the `ai` command to run embeddings.
@@ -77,10 +88,11 @@ snowdev <command> [options]
 - [x] Support for UDFs and Stored Procedures
 - [x] Support for Streamlit
 - [x] AI interactions for embedding and suggestions
+- [x] Support for snowflake Tasks
+- [x] AI to add the packages to toml file
+- [ ] Support for multiple python scripts in stored procedures
 - [ ] Use AI to modify existing code for optimization
-- [ ] AI to add the packages to toml file
 - [ ] Adding more granularity for AI commands
-- [ ] Support for snowflake Tasks
 
 
 

docs/cli.md

@@ -0,0 +1,57 @@
+# SnowDev CLI Documentation
+
+SnowDev is a CLI tool designed to deploy Snowflake components such as UDFs, Stored Procedures, Streamlit apps, and tasks. Below is the detailed documentation of all the commands available in the SnowDev CLI.
+
+## 1. `init`
+- **Description**: Initialize the project structure.
+- **Usage**: `snowdev init`
+
+## 2. `new`
+- **Description**: Create a new component.
+- **Usage**: `snowdev new [OPTIONS]`
+- **Options**:
+    - `--udf <udf_name>`: The name of the UDF.
+    - `--sproc <sproc_name>`: The name of the stored procedure.
+    - `--streamlit <streamlit_name>`: The name of the Streamlit app.
+    - `--task <task_name>`: The name of the task.
+
+## 3. `test`
+- **Description**: Test the deployment.
+- **Usage**: `snowdev test [OPTIONS]`
+- **Options**:
+    - `--udf <udf_name>`: The name of the UDF.
+    - `--sproc <sproc_name>`: The name of the stored procedure.
+
+## 4. `upload`
+- **Description**: Upload static content to stage, only for zipped external packages.
+- **Usage**: `snowdev upload`
+
+## 5. `add`
+- **Description**: Add a package and optionally upload.
+- **Usage**: `snowdev add --package <package_name>`
+
+## 6. `ai`
+- **Description**: Interact with AI components. Run embeddings or create new AI components.
+- **Usage**: `snowdev ai [OPTIONS]`
+- **Options**:
+    - `--udf <udf_name>`: The name of the UDF.
+    - `--sproc <sproc_name>`: The name of the stored procedure.
+    - `--streamlit <streamlit_name>`: The name of the Streamlit app.
+    - `--embed`: Run the embeddings.
+    - `--task <task_name>`: The name of the task.
+
+## 7. `deploy`
+- **Description**: Deploy components.
+- **Usage**: `snowdev deploy [OPTIONS]`
+- **Options**:
+    - `--udf <udf_name>`: The name of the UDF.
+    - `--sproc <sproc_name>`: The name of the stored procedure.
+    - `--streamlit <streamlit_name>`: The name of the Streamlit app.
+    - `--task <task_name>`: The name of the task.
+
+## 8. `task`
+- **Description**: Commands for tasks. Actions: resume, suspend, execute.
+- **Usage**: `snowdev task --name <task_name> --action <action>`
+- **Options**:
+    - `--name <task_name>`: The name of the task. (Required)
+    - `--action <action>`: The action to be performed on the task. Choices: resume, suspend, execute. (Required)

docs/quickstart.md

@@ -105,6 +105,35 @@ Now that you have SnowDev installed and initialized, it's time to see it in acti
     snowdev deploy --streamlit "order_chart_app"
     ```
 
+---
+### Deploying Tasks
+
+1. **Add a New snowflake Task**
+
+    Create a new task application named `sample_task`:
+
+    ```bash
+    snowdev new --task "sample_task"
+    ```
+
+    Modify the sql in the src/task/sample_task/app.sql
+
+2. **Deploy the task Application to Production**
+
+    Ready to go live? Deploy the application to production:
+
+    ```bash
+    snowdev deploy --task "sample_task"
+    ```
+
+3. **Resume the Task**
+
+    Resume the task so it starts the schedule:
+
+    ```bash
+    snowdev task --name "sample_task" --action resume
+    ```
+
 ---
 
 ### Deploying using AI

examples/src/sproc/test_sproc/app.toml

@@ -5,4 +5,4 @@ role = ""
 
 [tool.poetry.dependencies]
 python = "3.10.0"
-snowflake-snowpark-python = "1.5.1"
+snowflake-snowpark-python = "1.6.1"

examples/src/sproc/test_sproc/imports.txt

@@ -0,0 +1 @@
+import any external package here

examples/src/streamlit/test_script/environment.yml

@@ -1,9 +1,5 @@
 name: snowflake-test
 channels:
   - snowflake
-snowflake:
-  database: 
-  schema: 
-  role:
 dependencies:
-  - pyyaml
+  - pyyaml