diff --git a/.gitignore b/.gitignore index 5b32331..546cf9a 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,11 @@ helm-chart/charts/rag-0.0.1.tgz pyrightconfig.json notes*.md notes.md +*todo*.md + +# macOS Finder metadata +.DS_Store +**/.DS_Store # Node Modules node_modules/ diff --git a/.vscode/launch.json b/.vscode/launch.json index 64d934d..80125b8 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,6 +1,21 @@ { "version": "0.2.0", "configurations": [ + { + "name": "Debug Pytest (current file)", + "type": "python", + "request": "launch", + // run pytest as a module + "module": "pytest", + "args": [ + "--maxfail=1", + "--disable-warnings", + "-q", + "${file}" + ], + "console": "integratedTerminal", + "justMyCode": false, + }, { "name": "rag_backend", "type": "python", @@ -94,7 +109,5 @@ } ] } - - ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 94ff2e2..151bbe2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,6 +6,8 @@ "./rag-core-library/rag-core-api/src", "./rag-core-library/rag-core-lib/src", "./rag-core-library/extractor-api-lib/src", + "./admin-backend", + "./rag-backend" ], "[yaml]": { "editor.tabSize": 2, @@ -13,4 +15,15 @@ "editor.formatOnType": true, "editor.autoIndent": "advanced" }, + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, + "python.testing.pytestArgs": ["--import-mode","importlib"], + "python.testing.autoTestDiscoverOnSaveEnabled": true, + "python.envFile": "${workspaceFolder}/.env", + "python-envs.defaultEnvManager": "ms-python.python:venv", + "python-envs.defaultPackageManager": "ms-python.python:pip", + "python-envs.pythonProjects": [], } + + + diff --git a/Tiltfile b/Tiltfile index 93b57ae..de6360c 100644 --- a/Tiltfile +++ b/Tiltfile @@ -132,7 +132,6 @@ local_resource( allow_parallel=True, ) - ######################################################################################################################## ################################## build backend_rag image and do live update ########################################## ######################################################################################################################## @@ -355,6 +354,9 @@ if has_confluence_config(): if os.environ.get("CONFLUENCE_DOCUMENT_NAME"): document_names = os.environ["CONFLUENCE_DOCUMENT_NAME"].replace(",", "\\,") confluence_settings.append("adminBackend.envs.confluenceLoader.CONFLUENCE_DOCUMENT_NAME=%s" % document_names) + if os.environ.get("CONFLUENCE_MAX_PAGES"): + max_pages = os.environ["CONFLUENCE_MAX_PAGES"].replace(",", "\\,") + confluence_settings.append("adminBackend.envs.confluenceLoader.CONFLUENCE_MAX_PAGES=%s" % max_pages) value_override.extend(confluence_settings) if os.environ.get("STACKIT_VLLM_API_KEY", False): @@ -373,7 +375,7 @@ if os.environ.get("STACKIT_EMBEDDER_API_KEY", False): yaml = helm( "./rag-infrastructure/rag", name="rag", - namespace="rag", + namespace=namespace, values=[ "./rag-infrastructure/rag/values.yaml", ], diff --git a/admin-backend/Dockerfile b/admin-backend/Dockerfile index 35aa741..4f4b1c5 100644 --- a/admin-backend/Dockerfile +++ b/admin-backend/Dockerfile @@ -19,7 +19,7 @@ COPY admin-backend/pyproject.toml admin-backend/poetry.lock ./ RUN mkdir log && chmod 700 log RUN touch /app/admin-backend/log/logfile.log && chmod 600 /app/admin-backend/log/logfile.log -RUN poetry config virtualenvs.create false &&\ +RUN poetry config virtualenvs.create false && \ if [ "$dev" = "1" ]; then \ poetry install --no-interaction --no-ansi --no-root --with dev; \ else \ diff --git a/admin-backend/pyproject.toml b/admin-backend/pyproject.toml index 0098e8b..338e57b 100644 --- a/admin-backend/pyproject.toml +++ b/admin-backend/pyproject.toml @@ -1,3 +1,45 @@ +[tool.poetry] +name = "admin_backend" +version = "1.0.0" +description = "The admin backend is responsible for the document management. This includes deletion, upload and getting particular documents or document lists." +authors = ["STACKIT Data and AI Consulting "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.11" +admin-api-lib = {path = "../rag-core-library/admin-api-lib", develop = true} + +[tool.poetry.group.dev.dependencies] +debugpy = "^1.8.1" +pytest = "^8.2.1" +flake8 = "^7.1.0" +flake8-black = "^0.3.6" +flake8-pyproject = "^1.2.3" +coverage = "^7.5.4" +flake8-quotes = "^3.4.0" +flake8-return = "^1.2.0" +flake8-annotations-complexity = "^0.0.8" +flake8-bandit = "^4.1.1" +flake8-bugbear = "^24.8.19" +flake8-builtins = "^2.5.0" +flake8-comprehensions = "^3.15.0" +flake8-eradicate = "^1.5.0" +flake8-expression-complexity = "^0.0.11" +# flake8-logging-format = "^2024.24.12" +# flake8-docstrings = "^1.7.0" +flake8-pytest-style = "^2.0.0" +pep8-naming = "^0.14.1" +flake8-eol = "^0.0.8" +flake8-exceptions = "^0.0.1a0" +flake8-simplify = "^0.21.0" +flake8-wot = "^0.2.0" +flake8-function-order = "^0.0.5" +flake8-tidy-imports = "^4.10.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + [tool.flake8] exclude= [".eggs", "./rag-core-library/*", "./src/admin_backend/models/*", "./src/admin_backend/rag_backend_client/*", "./src/admin_backend/document_extractor_client/*", ".git", ".hg", ".mypy_cache", ".tox", ".venv", ".devcontainer", "venv", "_build", "buck-out", "build", "dist"] statistics = true @@ -49,45 +91,3 @@ skip_gitignore = true [tool.pylint] max-line-length = 120 - -[tool.poetry] -name = "admin_backend" -version = "0.0.1" -description = "The admin backend is responsible for the document management. This includes deletion, upload and getting particular documents or document lists." -authors = ["STACKIT Data and AI Consulting "] -readme = "README.md" - -[tool.poetry.group.dev.dependencies] -debugpy = "^1.8.1" -pytest = "^8.2.1" -flake8 = "^7.1.0" -flake8-black = "^0.3.6" -flake8-pyproject = "^1.2.3" -coverage = "^7.5.4" -flake8-quotes = "^3.4.0" -flake8-return = "^1.2.0" -flake8-annotations-complexity = "^0.0.8" -flake8-bandit = "^4.1.1" -flake8-bugbear = "^24.8.19" -flake8-builtins = "^2.5.0" -flake8-comprehensions = "^3.15.0" -flake8-eradicate = "^1.5.0" -flake8-expression-complexity = "^0.0.11" -# flake8-logging-format = "^2024.24.12" -# flake8-docstrings = "^1.7.0" -flake8-pytest-style = "^2.0.0" -pep8-naming = "^0.14.1" -flake8-eol = "^0.0.8" -flake8-exceptions = "^0.0.1a0" -flake8-simplify = "^0.21.0" -flake8-wot = "^0.2.0" -flake8-function-order = "^0.0.5" -flake8-tidy-imports = "^4.10.0" - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" - -[tool.poetry.dependencies] -python = "^3.11" -admin-api-lib = {path = "../rag-core-library/admin-api-lib", develop = true} diff --git a/admin-backend/tests/dummy_test.py b/admin-backend/tests/dummy1_test.py similarity index 60% rename from admin-backend/tests/dummy_test.py rename to admin-backend/tests/dummy1_test.py index 1428394..7ca7f1b 100644 --- a/admin-backend/tests/dummy_test.py +++ b/admin-backend/tests/dummy1_test.py @@ -1,3 +1,3 @@ -def test_dummy() -> None: +def test_dummy1() -> None: print("Dummy test.") assert True diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..e2772f2 --- /dev/null +++ b/conftest.py @@ -0,0 +1,15 @@ +import sys +import os +from pathlib import Path + +# Add project root and specific directories to Python path +project_root = Path(__file__).parent +sys.path.insert(0, str(project_root)) +sys.path.insert(0, str(project_root / "admin-backend")) +sys.path.insert(0, str(project_root / "rag-backend")) +sys.path.insert(0, str(project_root / "document-extractor")) + +# point at each rag-core library's src folder so their packages (admin_api_lib, rag_core_api, etc.) are importable +lib_root = project_root / "rag-core-library" +for lib in ["admin-api-lib", "rag-core-api", "rag-core-lib", "extractor-api-lib"]: + sys.path.insert(0, str(lib_root / lib / "src")) diff --git a/document-extractor/tests/dummy_test.py b/document-extractor/tests/dummy3_test.py similarity index 100% rename from document-extractor/tests/dummy_test.py rename to document-extractor/tests/dummy3_test.py diff --git a/rag-backend/pyproject.toml b/rag-backend/pyproject.toml index 98ef321..7dcba24 100644 --- a/rag-backend/pyproject.toml +++ b/rag-backend/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] -name = "rag-usecase-example" -version = "0.1.0" -description = "" +name = "rag-backend" +version = "1.0.0" +description = "The RAG backend is responsible for handling the interaction with the RAG system." authors = ["STACKIT Data and AI Consulting "] [tool.poetry.dependencies] @@ -39,7 +39,6 @@ flake8-tidy-imports = "^4.10.0" requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" - [tool.flake8] exclude= [".eggs", "./rag-core-library/*", ".git", ".hg", ".mypy_cache", ".tox", ".venv", ".devcontainer", "venv", "_build", "buck-out", "build", "dist"] statistics = true @@ -58,7 +57,6 @@ per-file-ignores = """ ./tests/*: S101, """ - [tool.black] line-length = 120 exclude = """ @@ -80,7 +78,6 @@ exclude = """ )/ """ - [tool.isort] profile = "black" skip = ['.eggs', '.git', '.hg', '.mypy_cache', '.nox', '.pants.d', '.tox', '.venv', '_build', 'buck-out', 'build', 'dist', 'node_modules', 'venv'] @@ -88,4 +85,3 @@ skip_gitignore = true [tool.pylint] max-line-length = 120 - diff --git a/rag-backend/tests/dummy_test.py b/rag-backend/tests/dummy2_test.py similarity index 66% rename from rag-backend/tests/dummy_test.py rename to rag-backend/tests/dummy2_test.py index 6e09897..7d002b3 100644 --- a/rag-backend/tests/dummy_test.py +++ b/rag-backend/tests/dummy2_test.py @@ -1,3 +1,3 @@ -def test_dummy() -> None: +def test_dummy2() -> None: print("Dummy test.") assert True # noqa S101 diff --git a/rag-core-library b/rag-core-library index cf78325..74f45e7 160000 --- a/rag-core-library +++ b/rag-core-library @@ -1 +1 @@ -Subproject commit cf78325c59920fd31170c20b1244092b572d469d +Subproject commit 74f45e729d71627008d19903874238af2e3aca35 diff --git a/rag-infrastructure b/rag-infrastructure index 1dc80f9..29b455d 160000 --- a/rag-infrastructure +++ b/rag-infrastructure @@ -1 +1 @@ -Subproject commit 1dc80f916c67224901e4fa9dc82a744b065335c2 +Subproject commit 29b455d8a2d1b433c5076881751b7e7df9f9c7f2