feat/Leverage uvicorn trace level to log body (#22)

rbiseck3 · web-flow · commit 7afdef30ae00 · 2024-09-24T11:38:18.000-04:00
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -4,8 +4,6 @@ on:
   release:
     types:
       - published
-      - created
-      - released
 
 env:
   PYTHON_VERSION: "3.10"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,8 +1,12 @@
+## 0.0.5
+
+* **Improve logging to hide body in case of sensitive data unless TRACE level**
+
 ## 0.0.4
 
 ### Fixes
 
-* Do not block event loop when running plugin code
+* **Do not block event loop when running plugin code**
 
 ## 0.0.3
 
diff --git a/requirements/cli.txt b/requirements/cli.txt
@@ -2,17 +2,17 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile requirements//cli.in
+#    pip-compile ./cli.in
 #
 annotated-types==0.7.0
     # via pydantic
-anyio==4.4.0
+anyio==4.6.0
     # via starlette
 asgiref==3.8.1
     # via opentelemetry-instrumentation-asgi
 click==8.1.7
     # via
-    #   -r requirements//cli.in
+    #   -r ./cli.in
     #   unstructured-ingest
     #   uvicorn
 dataclasses-json==0.6.7
@@ -24,86 +24,86 @@ deprecated==1.2.14
     #   opentelemetry-semantic-conventions
 exceptiongroup==1.2.2
     # via anyio
-fastapi==0.112.2
-    # via -r requirements//cli.in
+fastapi==0.115.0
+    # via -r ./cli.in
 googleapis-common-protos==1.65.0
     # via opentelemetry-exporter-otlp-proto-grpc
-grpcio==1.66.0
+grpcio==1.66.1
     # via opentelemetry-exporter-otlp-proto-grpc
 h11==0.14.0
     # via uvicorn
-idna==3.8
+idna==3.10
     # via anyio
-importlib-metadata==8.0.0
+importlib-metadata==8.4.0
     # via opentelemetry-api
 marshmallow==3.22.0
     # via dataclasses-json
 mypy-extensions==1.0.0
     # via typing-inspect
-numpy==2.1.0
+numpy==2.1.1
     # via pandas
-opentelemetry-api==1.26.0
+opentelemetry-api==1.27.0
     # via
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-instrumentation
     #   opentelemetry-instrumentation-asgi
     #   opentelemetry-instrumentation-fastapi
     #   opentelemetry-sdk
     #   opentelemetry-semantic-conventions
-opentelemetry-exporter-otlp-proto-common==1.26.0
+opentelemetry-exporter-otlp-proto-common==1.27.0
     # via opentelemetry-exporter-otlp-proto-grpc
-opentelemetry-exporter-otlp-proto-grpc==1.26.0
-    # via -r requirements//cli.in
-opentelemetry-instrumentation==0.47b0
+opentelemetry-exporter-otlp-proto-grpc==1.27.0
+    # via -r ./cli.in
+opentelemetry-instrumentation==0.48b0
     # via
     #   opentelemetry-instrumentation-asgi
     #   opentelemetry-instrumentation-fastapi
-opentelemetry-instrumentation-asgi==0.47b0
+opentelemetry-instrumentation-asgi==0.48b0
     # via opentelemetry-instrumentation-fastapi
-opentelemetry-instrumentation-fastapi==0.47b0
-    # via -r requirements//cli.in
-opentelemetry-proto==1.26.0
+opentelemetry-instrumentation-fastapi==0.48b0
+    # via -r ./cli.in
+opentelemetry-proto==1.27.0
     # via
     #   opentelemetry-exporter-otlp-proto-common
     #   opentelemetry-exporter-otlp-proto-grpc
-opentelemetry-sdk==1.26.0
+opentelemetry-sdk==1.27.0
     # via
     #   opentelemetry-exporter-otlp-proto-grpc
     #   unstructured-ingest
-opentelemetry-semantic-conventions==0.47b0
+opentelemetry-semantic-conventions==0.48b0
     # via
     #   opentelemetry-instrumentation-asgi
     #   opentelemetry-instrumentation-fastapi
     #   opentelemetry-sdk
-opentelemetry-util-http==0.47b0
+opentelemetry-util-http==0.48b0
     # via
     #   opentelemetry-instrumentation-asgi
     #   opentelemetry-instrumentation-fastapi
 packaging==24.1
     # via marshmallow
-pandas==2.2.2
+pandas==2.2.3
     # via unstructured-ingest
-protobuf==4.25.4
+protobuf==4.25.5
     # via
     #   googleapis-common-protos
     #   opentelemetry-proto
-pydantic==2.8.2
+pydantic==2.9.2
     # via
     #   fastapi
     #   unstructured-ingest
-pydantic-core==2.20.1
+pydantic-core==2.23.4
     # via pydantic
 python-dateutil==2.9.0.post0
     # via
     #   pandas
     #   unstructured-ingest
-pytz==2024.1
+pytz==2024.2
     # via pandas
 six==1.16.0
     # via python-dateutil
 sniffio==1.3.1
     # via anyio
-starlette==0.38.2
+starlette==0.38.6
     # via fastapi
 tqdm==4.66.5
     # via unstructured-ingest
@@ -119,17 +119,17 @@ typing-extensions==4.12.2
     #   uvicorn
 typing-inspect==0.9.0
     # via dataclasses-json
-tzdata==2024.1
+tzdata==2024.2
     # via pandas
-unstructured-ingest==0.0.8
-    # via -r requirements//cli.in
+unstructured-ingest==0.0.18
+    # via -r ./cli.in
 uvicorn==0.30.6
-    # via -r requirements//cli.in
+    # via -r ./cli.in
 wrapt==1.16.0
     # via
     #   deprecated
     #   opentelemetry-instrumentation
-zipp==3.20.1
+zipp==3.20.2
     # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
diff --git a/requirements/constraints.txt b/requirements/constraints.txt
@@ -1 +1 @@
-unstructured-ingest==0.0.8
+unstructured-ingest>=0.0.18
diff --git a/requirements/lint.txt b/requirements/lint.txt
@@ -2,24 +2,24 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile requirements//lint.in
+#    pip-compile ./lint.in
 #
 autoflake==2.3.1
-    # via -r requirements//lint.in
+    # via -r ./lint.in
 black==24.8.0
-    # via -r requirements//lint.in
+    # via -r ./lint.in
 click==8.1.7
     # via black
 flake8==7.1.1
     # via
-    #   -r requirements//lint.in
+    #   -r ./lint.in
     #   flake8-print
 flake8-print==5.0.0
-    # via -r requirements//lint.in
+    # via -r ./lint.in
 mccabe==0.7.0
     # via flake8
 mypy==1.11.2
-    # via -r requirements//lint.in
+    # via -r ./lint.in
 mypy-extensions==1.0.0
     # via
     #   black
@@ -28,7 +28,7 @@ packaging==24.1
     # via black
 pathspec==0.12.1
     # via black
-platformdirs==4.2.2
+platformdirs==4.3.6
     # via black
 pycodestyle==2.12.1
     # via
@@ -38,8 +38,8 @@ pyflakes==3.2.0
     # via
     #   autoflake
     #   flake8
-ruff==0.6.2
-    # via -r requirements//lint.in
+ruff==0.6.7
+    # via -r ./lint.in
 tomli==2.0.1
     # via
     #   autoflake
diff --git a/requirements/release.txt b/requirements/release.txt
@@ -2,19 +2,19 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile requirements//release.in
+#    pip-compile ./release.in
 #
 backports-tarfile==1.2.0
     # via jaraco-context
-certifi==2024.7.4
+certifi==2024.8.30
     # via requests
 charset-normalizer==3.3.2
     # via requests
 docutils==0.21.2
     # via readme-renderer
-idna==3.8
+idna==3.10
     # via requests
-importlib-metadata==8.4.0
+importlib-metadata==8.5.0
     # via
     #   keyring
     #   twine
@@ -24,13 +24,13 @@ jaraco-context==6.0.1
     # via keyring
 jaraco-functools==4.0.2
     # via keyring
-keyring==25.3.0
+keyring==25.4.1
     # via twine
 markdown-it-py==3.0.0
     # via rich
 mdurl==0.1.2
     # via markdown-it-py
-more-itertools==10.4.0
+more-itertools==10.5.0
     # via
     #   jaraco-classes
     #   jaraco-functools
@@ -52,15 +52,15 @@ requests-toolbelt==1.0.0
     # via twine
 rfc3986==2.0.0
     # via twine
-rich==13.8.0
+rich==13.8.1
     # via twine
 twine==5.1.1
-    # via -r requirements//release.in
-urllib3==2.2.2
+    # via -r ./release.in
+urllib3==2.2.3
     # via
     #   requests
     #   twine
 wheel==0.44.0
-    # via -r requirements//release.in
-zipp==3.20.1
+    # via -r ./release.in
+zipp==3.20.2
     # via importlib-metadata
diff --git a/requirements/test.txt b/requirements/test.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile requirements//test.in
+#    pip-compile ./test.in
 #
 exceptiongroup==1.2.2
     # via pytest
@@ -12,7 +12,7 @@ packaging==24.1
     # via pytest
 pluggy==1.5.0
     # via pytest
-pytest==8.3.2
-    # via -r requirements//test.in
+pytest==8.3.3
+    # via -r ./test.in
 tomli==2.0.1
     # via pytest
diff --git a/requirements/validate.txt b/requirements/validate.txt
@@ -2,17 +2,17 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile requirements//validate.in
+#    pip-compile ./validate.in
 #
-certifi==2024.7.4
+certifi==2024.8.30
     # via requests
 charset-normalizer==3.3.2
     # via requests
 click==8.1.7
-    # via -r requirements//validate.in
-idna==3.8
+    # via -r ./validate.in
+idna==3.10
     # via requests
 requests==2.32.3
-    # via -r requirements//validate.in
-urllib3==2.2.2
+    # via -r ./validate.in
+urllib3==2.2.3
     # via requests
diff --git a/scripts/pip-compile.sh b/scripts/pip-compile.sh
@@ -1,18 +1,27 @@
 #!/usr/bin/env bash
 
+pushd ./requirements || exit
+
+find . -type f -name "*.txt" ! -name "constraints.txt" -exec rm '{}' ';'
+find . -type f -name "*.in" -maxdepth 1 -exec pip-compile --upgrade '{}' ';'
+
+popd || exit
+
+# Check python version
 # python version must match lowest supported (3.10)
 major=3
 minor=10
-if ! python -c "import sys; assert sys.version_info.major == $major and sys.version_info.minor == $minor"; then
-	echo "python version not equal to expected $major.$minor: $(python --version)"
+
+versions=$(cat requirements/* | grep "This file is autogenerated by pip-compile with Python" | awk '{print $NF}' | sort | uniq)
+if [[ $(echo $versions | wc -w) -ne 1 ]]; then
+	echo "Files generated with multiple python version: $versions"
 	exit 1
 fi
 
-while IFS= read -r -d '' file; do
-	filename=${file%.in}
-	txtfilename="${filename}.txt"
-	echo "Removing $txtfilename"
-	rm -f $txtfilename
-	echo "Running pip-compile on file $file"
-	pip-compile --upgrade --verbose "$file"
-done < <(find requirements/ -type f -name "*.in" -maxdepth 1 -print0)
+version_major=$(echo $versions | awk -F"." '{print $1}')
+version_minor=$(echo $versions | awk -F"." '{print $2}')
+
+if [[ $major -ne $version_major || $minor -ne $version_minor ]]; then
+	echo "python version not equal to expected $major.$minor: $versions"
+	exit 1
+fi
diff --git a/test/test_schema.py b/test/test_schema.py
@@ -566,6 +566,10 @@ def fn(a: FileData) -> list[FileData]:
                         "items": {"key": {"type": "string"}, "value": {}},
                     },
                     "reprocess": {"type": "boolean", "default": False},
+                    "local_download_path": {
+                        "anyOf": [{"type": "string"}, {"type": "null"}],
+                        "default": None,
+                    },
                 },
                 "required": ["identifier", "connector_type", "metadata", "additional_metadata"],
             }
@@ -659,6 +663,10 @@ def fn(a: FileData) -> list[FileData]:
                     "items": {"key": {"type": "string"}, "value": {}},
                 },
                 "reprocess": {"type": "boolean", "default": False},
+                "local_download_path": {
+                    "anyOf": [{"type": "string"}, {"type": "null"}],
+                    "default": None,
+                },
             },
             "required": ["identifier", "connector_type", "metadata", "additional_metadata"],
         },
diff --git a/unstructured_platform_plugins/__version__.py b/unstructured_platform_plugins/__version__.py
@@ -1 +1 @@
-__version__ = "0.0.4"  # pragma: no cover
+__version__ = "0.0.5"  # pragma: no cover
diff --git a/unstructured_platform_plugins/etl_uvicorn/api_generator.py b/unstructured_platform_plugins/etl_uvicorn/api_generator.py

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-unstructured-ingest==0.0.8`
	`1`	`+unstructured-ingest>=0.0.18`
Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = "0.0.4" # pragma: no cover`
	`1`	`+__version__ = "0.0.5" # pragma: no cover`