From af9eca68aad43678c6bf596a7b512e8be2bfcaeb Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 28 May 2024 16:28:44 -0500 Subject: [PATCH 01/15] Try to fix hub installs for notebooks --- .github/workflows/examples_check.yml | 33 ++++++++++++++-------- .github/workflows/install_from_hub.yml | 2 +- .github/workflows/scripts/run_notebooks.sh | 4 +++ 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml index 9be015077..cf13cc4fd 100644 --- a/.github/workflows/examples_check.yml +++ b/.github/workflows/examples_check.yml @@ -37,26 +37,35 @@ jobs: # with: # path: ~/.cache/pypoetry # key: poetry-cache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ env.POETRY_VERSION }} - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true + # - name: Install Poetry + # uses: snok/install-poetry@v1 + # with: + # virtualenvs-create: true + # virtualenvs-in-project: true + # installer-parallel: true - name: Install dependencies run: | - make full; - poetry add "openai>=1.2.4" jupyter nbconvert cohere==5.3.2; - - name: Check for pypdfium2 - run: poetry run pip show pypdfium2 + # Legacy run script + # make full; + # poetry add "openai>=1.2.4" jupyter nbconvert cohere==5.3.2; + + # Setup Virtual Environment + python3 -m venv ./.venv + source .venv/bin/activate + + # Install the current branch + pip install . + + # Install extra stuff for notebook runs + pip install jupyter nbconvert cohere==5.3.2 - name: Huggingface Hub Login run: poetry run huggingface-cli login --token $HUGGINGFACE_API_KEY - name: download nltk data run: | mkdir /tmp/nltk_data; poetry run python -m nltk.downloader -d /tmp/nltk_data punkt; - - name: Use venv - run: source .venv/bin/activate + # - name: Use venv + # run: source .venv/bin/activate - name: Execute notebooks and check for errors run: bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }} diff --git a/.github/workflows/install_from_hub.yml b/.github/workflows/install_from_hub.yml index c39cc8502..1e6483547 100644 --- a/.github/workflows/install_from_hub.yml +++ b/.github/workflows/install_from_hub.yml @@ -1,4 +1,4 @@ -name: Notebook Execution and Error Check +name: Install from Hub on: push: diff --git a/.github/workflows/scripts/run_notebooks.sh b/.github/workflows/scripts/run_notebooks.sh index e32932948..e35dba6b5 100755 --- a/.github/workflows/scripts/run_notebooks.sh +++ b/.github/workflows/scripts/run_notebooks.sh @@ -1,6 +1,10 @@ #!/bin/bash export NLTK_DATA=/tmp/nltk_data; +# Remove the local guardrails directory and use the installed version +rm -rf ./guardrails + +# Navigate to notebooks cd docs/examples # Get the notebook name from the matrix variable From 6f99ef172df7075b074a4452d9ce36736d39ebe2 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 28 May 2024 16:31:00 -0500 Subject: [PATCH 02/15] remove poetry references --- .github/workflows/examples_check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml index cf13cc4fd..e1912bc6b 100644 --- a/.github/workflows/examples_check.yml +++ b/.github/workflows/examples_check.yml @@ -59,11 +59,11 @@ jobs: # Install extra stuff for notebook runs pip install jupyter nbconvert cohere==5.3.2 - name: Huggingface Hub Login - run: poetry run huggingface-cli login --token $HUGGINGFACE_API_KEY + run: huggingface-cli login --token $HUGGINGFACE_API_KEY - name: download nltk data run: | mkdir /tmp/nltk_data; - poetry run python -m nltk.downloader -d /tmp/nltk_data punkt; + python -m nltk.downloader -d /tmp/nltk_data punkt; # - name: Use venv # run: source .venv/bin/activate - name: Execute notebooks and check for errors From c022e349011eb02ea7a938064c7e13e1df194c64 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 28 May 2024 16:43:30 -0500 Subject: [PATCH 03/15] install huggingface cli --- .github/workflows/examples_check.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml index e1912bc6b..f03007db1 100644 --- a/.github/workflows/examples_check.yml +++ b/.github/workflows/examples_check.yml @@ -58,6 +58,7 @@ jobs: # Install extra stuff for notebook runs pip install jupyter nbconvert cohere==5.3.2 + pip install huggingface_hub[cli] - name: Huggingface Hub Login run: huggingface-cli login --token $HUGGINGFACE_API_KEY - name: download nltk data From 7a9b31abcf1dde86eab031fc1bda66317246f21d Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 28 May 2024 16:44:19 -0500 Subject: [PATCH 04/15] reduce to one notebook for debugging --- .github/workflows/examples_check.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml index f03007db1..ad448dad8 100644 --- a/.github/workflows/examples_check.yml +++ b/.github/workflows/examples_check.yml @@ -12,10 +12,10 @@ on: jobs: execute_notebooks: runs-on: ubuntu-latest - strategy: - matrix: - # this line is automatically generated by the script in .github/workflows/scripts/update_notebook_matrix.sh - notebook: ["bug_free_python_code.ipynb","check_for_pii.ipynb","competitors_check.ipynb","extracting_entities.ipynb","generate_structured_data.ipynb","generate_structured_data_cohere.ipynb","guardrails_with_chat_models.ipynb","input_validation.ipynb","llamaindex-output-parsing.ipynb","no_secrets_in_generated_text.ipynb","provenance.ipynb","recipe_generation.ipynb","regex_validation.ipynb","response_is_on_topic.ipynb","secrets_detection.ipynb","select_choice_based_on_action.ipynb","streaming.ipynb","syntax_error_free_sql.ipynb","text_summarization_quality.ipynb","toxic_language.ipynb","translation_to_specific_language.ipynb","translation_with_quality_check.ipynb","valid_chess_moves.ipynb","value_within_distribution.ipynb"] + # strategy: + # matrix: + # # this line is automatically generated by the script in .github/workflows/scripts/update_notebook_matrix.sh + # notebook: ["bug_free_python_code.ipynb","check_for_pii.ipynb","competitors_check.ipynb","extracting_entities.ipynb","generate_structured_data.ipynb","generate_structured_data_cohere.ipynb","guardrails_with_chat_models.ipynb","input_validation.ipynb","llamaindex-output-parsing.ipynb","no_secrets_in_generated_text.ipynb","provenance.ipynb","recipe_generation.ipynb","regex_validation.ipynb","response_is_on_topic.ipynb","secrets_detection.ipynb","select_choice_based_on_action.ipynb","streaming.ipynb","syntax_error_free_sql.ipynb","text_summarization_quality.ipynb","toxic_language.ipynb","translation_to_specific_language.ipynb","translation_with_quality_check.ipynb","valid_chess_moves.ipynb","value_within_distribution.ipynb"] env: COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -68,5 +68,6 @@ jobs: # - name: Use venv # run: source .venv/bin/activate - name: Execute notebooks and check for errors - run: bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }} + # run: bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }} + run: bash ./.github/workflows/scripts/run_notebooks.sh bug_free_python_code.ipynb From 8664cecedca29a36d15d9d2c251448a39ffb04e1 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 28 May 2024 16:48:14 -0500 Subject: [PATCH 05/15] use bin processs --- .github/workflows/examples_check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml index ad448dad8..c14765c2f 100644 --- a/.github/workflows/examples_check.yml +++ b/.github/workflows/examples_check.yml @@ -58,9 +58,9 @@ jobs: # Install extra stuff for notebook runs pip install jupyter nbconvert cohere==5.3.2 - pip install huggingface_hub[cli] + pip install "huggingface_hub[cli]" - name: Huggingface Hub Login - run: huggingface-cli login --token $HUGGINGFACE_API_KEY + run: ./.venv/bin/huggingface-cli login --token $HUGGINGFACE_API_KEY - name: download nltk data run: | mkdir /tmp/nltk_data; From 072bd361a2f45284739e1e7be75afdfd07b8fca6 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 28 May 2024 16:50:22 -0500 Subject: [PATCH 06/15] install nltk --- .github/workflows/examples_check.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml index c14765c2f..46f223501 100644 --- a/.github/workflows/examples_check.yml +++ b/.github/workflows/examples_check.yml @@ -57,8 +57,7 @@ jobs: pip install . # Install extra stuff for notebook runs - pip install jupyter nbconvert cohere==5.3.2 - pip install "huggingface_hub[cli]" + pip install "huggingface_hub[cli]" nltk jupyter nbconvert cohere==5.3.2 - name: Huggingface Hub Login run: ./.venv/bin/huggingface-cli login --token $HUGGINGFACE_API_KEY - name: download nltk data From eefb449419fb17e07b08104e0625898307ad9609 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 28 May 2024 16:52:41 -0500 Subject: [PATCH 07/15] show nltk --- .github/workflows/examples_check.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml index 46f223501..a500289f6 100644 --- a/.github/workflows/examples_check.yml +++ b/.github/workflows/examples_check.yml @@ -58,10 +58,12 @@ jobs: # Install extra stuff for notebook runs pip install "huggingface_hub[cli]" nltk jupyter nbconvert cohere==5.3.2 + pip show nltk - name: Huggingface Hub Login run: ./.venv/bin/huggingface-cli login --token $HUGGINGFACE_API_KEY - name: download nltk data run: | + pip show nltk mkdir /tmp/nltk_data; python -m nltk.downloader -d /tmp/nltk_data punkt; # - name: Use venv From fa70f56dfcdf265a1aabd4b0b3120f9c71b41d68 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 28 May 2024 16:54:23 -0500 Subject: [PATCH 08/15] which pip --- .github/workflows/examples_check.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml index a500289f6..ae50d049e 100644 --- a/.github/workflows/examples_check.yml +++ b/.github/workflows/examples_check.yml @@ -57,12 +57,15 @@ jobs: pip install . # Install extra stuff for notebook runs - pip install "huggingface_hub[cli]" nltk jupyter nbconvert cohere==5.3.2 + pip install "huggingface_hub[cli]" jupyter nbconvert cohere==5.3.2 + pip install nltk pip show nltk + which pip - name: Huggingface Hub Login run: ./.venv/bin/huggingface-cli login --token $HUGGINGFACE_API_KEY - name: download nltk data run: | + which pip pip show nltk mkdir /tmp/nltk_data; python -m nltk.downloader -d /tmp/nltk_data punkt; From 0291e0de7dabc9ae5b8c9783293f423f83f3a661 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 28 May 2024 17:01:57 -0500 Subject: [PATCH 09/15] alter path, debug --- .github/workflows/examples_check.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml index ae50d049e..955aff73e 100644 --- a/.github/workflows/examples_check.yml +++ b/.github/workflows/examples_check.yml @@ -51,7 +51,8 @@ jobs: # Setup Virtual Environment python3 -m venv ./.venv - source .venv/bin/activate + . .venv/bin/activate + echo PATH=$PATH >> $GITHUB_ENV # Install the current branch pip install . @@ -59,14 +60,17 @@ jobs: # Install extra stuff for notebook runs pip install "huggingface_hub[cli]" jupyter nbconvert cohere==5.3.2 pip install nltk - pip show nltk - which pip + echo "pip show nltk: $(pip show nltk)" + echo "which pip $(which pip)" - name: Huggingface Hub Login - run: ./.venv/bin/huggingface-cli login --token $HUGGINGFACE_API_KEY + run: | + echo "which python: $(which python)" + echo "which pip $(which pip)" + huggingface-cli login --token $HUGGINGFACE_API_KEY - name: download nltk data run: | - which pip - pip show nltk + echo "which pip $(which pip)" + echo "pip show nltk: $(pip show nltk)" mkdir /tmp/nltk_data; python -m nltk.downloader -d /tmp/nltk_data punkt; # - name: Use venv From 656654853b4d4d7037a8f70177819f2d7bf4dd1a Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 28 May 2024 17:03:46 -0500 Subject: [PATCH 10/15] which python --- .github/workflows/examples_check.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml index 955aff73e..5c82a3028 100644 --- a/.github/workflows/examples_check.yml +++ b/.github/workflows/examples_check.yml @@ -60,8 +60,9 @@ jobs: # Install extra stuff for notebook runs pip install "huggingface_hub[cli]" jupyter nbconvert cohere==5.3.2 pip install nltk - echo "pip show nltk: $(pip show nltk)" + echo "which python $(which python)" echo "which pip $(which pip)" + echo "pip show nltk: $(pip show nltk)" - name: Huggingface Hub Login run: | echo "which python: $(which python)" From 4d9f6f0d032c9f3147f9521513dd1fdec6b60004 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 28 May 2024 17:07:58 -0500 Subject: [PATCH 11/15] source in every step --- .github/workflows/examples_check.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml index 5c82a3028..09bdc112a 100644 --- a/.github/workflows/examples_check.yml +++ b/.github/workflows/examples_check.yml @@ -51,8 +51,7 @@ jobs: # Setup Virtual Environment python3 -m venv ./.venv - . .venv/bin/activate - echo PATH=$PATH >> $GITHUB_ENV + source .venv/bin/activate # Install the current branch pip install . @@ -65,11 +64,13 @@ jobs: echo "pip show nltk: $(pip show nltk)" - name: Huggingface Hub Login run: | + source .venv/bin/activate echo "which python: $(which python)" echo "which pip $(which pip)" huggingface-cli login --token $HUGGINGFACE_API_KEY - name: download nltk data run: | + source .venv/bin/activate echo "which pip $(which pip)" echo "pip show nltk: $(pip show nltk)" mkdir /tmp/nltk_data; @@ -78,5 +79,7 @@ jobs: # run: source .venv/bin/activate - name: Execute notebooks and check for errors # run: bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }} - run: bash ./.github/workflows/scripts/run_notebooks.sh bug_free_python_code.ipynb + run: | + source .venv/bin/activate + bash ./.github/workflows/scripts/run_notebooks.sh bug_free_python_code.ipynb From 675b711b2be971f67148f49dd02354ef6412a8a4 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 28 May 2024 17:10:06 -0500 Subject: [PATCH 12/15] don't use poetry in script --- .github/workflows/examples_check.yml | 2 +- .github/workflows/scripts/run_notebooks.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml index 09bdc112a..606b46d60 100644 --- a/.github/workflows/examples_check.yml +++ b/.github/workflows/examples_check.yml @@ -81,5 +81,5 @@ jobs: # run: bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }} run: | source .venv/bin/activate - bash ./.github/workflows/scripts/run_notebooks.sh bug_free_python_code.ipynb + bash ./.github/workflows/scripts/run_notebooks.sh bug_free_python_code.ipynb || exit 1 diff --git a/.github/workflows/scripts/run_notebooks.sh b/.github/workflows/scripts/run_notebooks.sh index e35dba6b5..7a401e907 100755 --- a/.github/workflows/scripts/run_notebooks.sh +++ b/.github/workflows/scripts/run_notebooks.sh @@ -14,7 +14,8 @@ notebook="$1" invalid_notebooks=("valid_chess_moves.ipynb" "llamaindex-output-parsing.ipynb" "competitors_check.ipynb") if [[ ! " ${invalid_notebooks[@]} " =~ " ${notebook} " ]]; then echo "Processing $notebook..." - poetry run jupyter nbconvert --to notebook --execute "$notebook" + # poetry run jupyter nbconvert --to notebook --execute "$notebook" + jupyter nbconvert --to notebook --execute "$notebook" if [ $? -ne 0 ]; then echo "Error found in $notebook" echo "Error in $notebook. See logs for details." >> errors.txt From 64ef9c4634ed4299eedb99d07f036d4ec46e2c81 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 28 May 2024 17:12:49 -0500 Subject: [PATCH 13/15] run all notebooks --- .github/workflows/examples_check.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml index 606b46d60..caecbc055 100644 --- a/.github/workflows/examples_check.yml +++ b/.github/workflows/examples_check.yml @@ -12,10 +12,10 @@ on: jobs: execute_notebooks: runs-on: ubuntu-latest - # strategy: - # matrix: - # # this line is automatically generated by the script in .github/workflows/scripts/update_notebook_matrix.sh - # notebook: ["bug_free_python_code.ipynb","check_for_pii.ipynb","competitors_check.ipynb","extracting_entities.ipynb","generate_structured_data.ipynb","generate_structured_data_cohere.ipynb","guardrails_with_chat_models.ipynb","input_validation.ipynb","llamaindex-output-parsing.ipynb","no_secrets_in_generated_text.ipynb","provenance.ipynb","recipe_generation.ipynb","regex_validation.ipynb","response_is_on_topic.ipynb","secrets_detection.ipynb","select_choice_based_on_action.ipynb","streaming.ipynb","syntax_error_free_sql.ipynb","text_summarization_quality.ipynb","toxic_language.ipynb","translation_to_specific_language.ipynb","translation_with_quality_check.ipynb","valid_chess_moves.ipynb","value_within_distribution.ipynb"] + strategy: + matrix: + # this line is automatically generated by the script in .github/workflows/scripts/update_notebook_matrix.sh + notebook: ["bug_free_python_code.ipynb","check_for_pii.ipynb","competitors_check.ipynb","extracting_entities.ipynb","generate_structured_data.ipynb","generate_structured_data_cohere.ipynb","guardrails_with_chat_models.ipynb","input_validation.ipynb","llamaindex-output-parsing.ipynb","no_secrets_in_generated_text.ipynb","provenance.ipynb","recipe_generation.ipynb","regex_validation.ipynb","response_is_on_topic.ipynb","secrets_detection.ipynb","select_choice_based_on_action.ipynb","streaming.ipynb","syntax_error_free_sql.ipynb","text_summarization_quality.ipynb","toxic_language.ipynb","translation_to_specific_language.ipynb","translation_with_quality_check.ipynb","valid_chess_moves.ipynb","value_within_distribution.ipynb"] env: COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -81,5 +81,5 @@ jobs: # run: bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }} run: | source .venv/bin/activate - bash ./.github/workflows/scripts/run_notebooks.sh bug_free_python_code.ipynb || exit 1 + bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }} || exit 1 From 251fa5adce7c3658d846de24a366162f3dc53e88 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 28 May 2024 17:14:07 -0500 Subject: [PATCH 14/15] exit 1 on notebook failure --- .github/workflows/scripts/run_notebooks.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/scripts/run_notebooks.sh b/.github/workflows/scripts/run_notebooks.sh index 7a401e907..cd854ceb6 100755 --- a/.github/workflows/scripts/run_notebooks.sh +++ b/.github/workflows/scripts/run_notebooks.sh @@ -19,6 +19,7 @@ if [[ ! " ${invalid_notebooks[@]} " =~ " ${notebook} " ]]; then if [ $? -ne 0 ]; then echo "Error found in $notebook" echo "Error in $notebook. See logs for details." >> errors.txt + exit 1 fi fi From bd45f2fd2ebd5e9569a47f365dd7b226112e3447 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Tue, 28 May 2024 17:14:52 -0500 Subject: [PATCH 15/15] remove double bar --- .github/workflows/examples_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml index caecbc055..464cb0b6d 100644 --- a/.github/workflows/examples_check.yml +++ b/.github/workflows/examples_check.yml @@ -81,5 +81,5 @@ jobs: # run: bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }} run: | source .venv/bin/activate - bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }} || exit 1 + bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }}