feat: Add confluence update feature of rag-core-library

.gitignore

@@ -6,6 +6,11 @@ helm-chart/charts/rag-0.0.1.tgz
 pyrightconfig.json
 notes*.md
 notes.md
+*todo*.md
+
+# macOS Finder metadata
+.DS_Store
+**/.DS_Store
 
 # Node Modules
 node_modules/

.vscode/launch.json

@@ -1,6 +1,21 @@
 {
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "Debug Pytest (current file)",
+            "type": "python",
+            "request": "launch",
+            // run pytest as a module
+            "module": "pytest",
+            "args": [
+              "--maxfail=1",
+              "--disable-warnings",
+              "-q",
+              "${file}"
+            ],
+            "console": "integratedTerminal",
+            "justMyCode": false,
+        },
         {
             "name": "rag_backend",
             "type": "python",
@@ -94,7 +109,5 @@
                 }
             ]
         }
-
-
     ]
 }

.vscode/settings.json

@@ -6,11 +6,24 @@
         "./rag-core-library/rag-core-api/src",
         "./rag-core-library/rag-core-lib/src",
         "./rag-core-library/extractor-api-lib/src",
+        "./admin-backend",
+        "./rag-backend"
     ],
     "[yaml]": {
         "editor.tabSize": 2,
         "editor.insertSpaces": true,
         "editor.formatOnType": true,
         "editor.autoIndent": "advanced"
     },
+    "python.testing.pytestEnabled": true,
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestArgs": ["--import-mode","importlib"],
+    "python.testing.autoTestDiscoverOnSaveEnabled": true,
+    "python.envFile": "${workspaceFolder}/.env",
+    "python-envs.defaultEnvManager": "ms-python.python:venv",
+    "python-envs.defaultPackageManager": "ms-python.python:pip",
+    "python-envs.pythonProjects": [],
 }
+
+
+

Tiltfile

@@ -132,7 +132,6 @@ local_resource(
     allow_parallel=True,
 )
 
-
 ########################################################################################################################
 ################################## build backend_rag image and do live update ##########################################
 ########################################################################################################################
@@ -355,6 +354,9 @@ if has_confluence_config():
     if os.environ.get("CONFLUENCE_DOCUMENT_NAME"):
         document_names = os.environ["CONFLUENCE_DOCUMENT_NAME"].replace(",", "\\,")
         confluence_settings.append("adminBackend.envs.confluenceLoader.CONFLUENCE_DOCUMENT_NAME=%s" % document_names)
+    if os.environ.get("CONFLUENCE_MAX_PAGES"):
+        max_pages = os.environ["CONFLUENCE_MAX_PAGES"].replace(",", "\\,")
+        confluence_settings.append("adminBackend.envs.confluenceLoader.CONFLUENCE_MAX_PAGES=%s" % max_pages)
     value_override.extend(confluence_settings)
 
 if os.environ.get("STACKIT_VLLM_API_KEY", False):
@@ -373,7 +375,7 @@ if os.environ.get("STACKIT_EMBEDDER_API_KEY", False):
 yaml = helm(
     "./rag-infrastructure/rag",
     name="rag",
-    namespace="rag",
+    namespace=namespace,
     values=[
         "./rag-infrastructure/rag/values.yaml",
     ],

admin-backend/Dockerfile

@@ -19,7 +19,7 @@ COPY admin-backend/pyproject.toml admin-backend/poetry.lock ./
 RUN mkdir log && chmod 700 log
 RUN touch /app/admin-backend/log/logfile.log && chmod 600 /app/admin-backend/log/logfile.log
 
-RUN poetry config virtualenvs.create false &&\
+RUN poetry config virtualenvs.create false && \
     if [ "$dev" = "1" ]; then \
         poetry install --no-interaction --no-ansi --no-root --with dev; \
     else \

admin-backend/pyproject.toml

@@ -1,3 +1,45 @@
+[tool.poetry]
+name = "admin_backend"
+version = "1.0.0"
+description = "The admin backend is responsible for the document management. This includes deletion, upload and getting particular documents or document lists."
+authors = ["STACKIT Data and AI Consulting <data-ai-consulting@stackit.cloud>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+admin-api-lib = {path = "../rag-core-library/admin-api-lib", develop = true}
+
+[tool.poetry.group.dev.dependencies]
+debugpy = "^1.8.1"
+pytest = "^8.2.1"
+flake8 = "^7.1.0"
+flake8-black = "^0.3.6"
+flake8-pyproject = "^1.2.3"
+coverage = "^7.5.4"
+flake8-quotes = "^3.4.0"
+flake8-return = "^1.2.0"
+flake8-annotations-complexity = "^0.0.8"
+flake8-bandit = "^4.1.1"
+flake8-bugbear = "^24.8.19"
+flake8-builtins = "^2.5.0"
+flake8-comprehensions = "^3.15.0"
+flake8-eradicate = "^1.5.0"
+flake8-expression-complexity = "^0.0.11"
+# flake8-logging-format = "^2024.24.12"
+# flake8-docstrings = "^1.7.0"
+flake8-pytest-style = "^2.0.0"
+pep8-naming = "^0.14.1"
+flake8-eol = "^0.0.8"
+flake8-exceptions = "^0.0.1a0"
+flake8-simplify = "^0.21.0"
+flake8-wot = "^0.2.0"
+flake8-function-order = "^0.0.5"
+flake8-tidy-imports = "^4.10.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
 [tool.flake8]
 exclude= [".eggs", "./rag-core-library/*", "./src/admin_backend/models/*", "./src/admin_backend/rag_backend_client/*", "./src/admin_backend/document_extractor_client/*", ".git", ".hg", ".mypy_cache", ".tox", ".venv", ".devcontainer", "venv", "_build", "buck-out", "build", "dist"]
 statistics = true
@@ -49,45 +91,3 @@ skip_gitignore = true
 
 [tool.pylint]
 max-line-length = 120
-
-[tool.poetry]
-name = "admin_backend"
-version = "0.0.1"
-description = "The admin backend is responsible for the document management. This includes deletion, upload and getting particular documents or document lists."
-authors = ["STACKIT Data and AI Consulting <data-ai-consulting@stackit.cloud>"]
-readme = "README.md"
-
-[tool.poetry.group.dev.dependencies]
-debugpy = "^1.8.1"
-pytest = "^8.2.1"
-flake8 = "^7.1.0"
-flake8-black = "^0.3.6"
-flake8-pyproject = "^1.2.3"
-coverage = "^7.5.4"
-flake8-quotes = "^3.4.0"
-flake8-return = "^1.2.0"
-flake8-annotations-complexity = "^0.0.8"
-flake8-bandit = "^4.1.1"
-flake8-bugbear = "^24.8.19"
-flake8-builtins = "^2.5.0"
-flake8-comprehensions = "^3.15.0"
-flake8-eradicate = "^1.5.0"
-flake8-expression-complexity = "^0.0.11"
-# flake8-logging-format = "^2024.24.12"
-# flake8-docstrings = "^1.7.0"
-flake8-pytest-style = "^2.0.0"
-pep8-naming = "^0.14.1"
-flake8-eol = "^0.0.8"
-flake8-exceptions = "^0.0.1a0"
-flake8-simplify = "^0.21.0"
-flake8-wot = "^0.2.0"
-flake8-function-order = "^0.0.5"
-flake8-tidy-imports = "^4.10.0"
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
-
-[tool.poetry.dependencies]
-python = "^3.11"
-admin-api-lib = {path = "../rag-core-library/admin-api-lib", develop = true}

admin-backend/tests/dummy_test.py → admin-backend/tests/dummy1_test.py

@@ -1,3 +1,3 @@
-def test_dummy() -> None:
+def test_dummy1() -> None:
     print("Dummy test.")
     assert True

conftest.py

@@ -0,0 +1,15 @@
+import sys
+import os
+from pathlib import Path
+
+# Add project root and specific directories to Python path
+project_root = Path(__file__).parent
+sys.path.insert(0, str(project_root))
+sys.path.insert(0, str(project_root / "admin-backend"))
+sys.path.insert(0, str(project_root / "rag-backend"))
+sys.path.insert(0, str(project_root / "document-extractor"))
+
+# point at each rag-core library's src folder so their packages (admin_api_lib, rag_core_api, etc.) are importable
+lib_root = project_root / "rag-core-library"
+for lib in ["admin-api-lib", "rag-core-api", "rag-core-lib", "extractor-api-lib"]:
+    sys.path.insert(0, str(lib_root / lib / "src"))

rag-backend/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
-name = "rag-usecase-example"
-version = "0.1.0"
-description = ""
+name = "rag-backend"
+version = "1.0.0"
+description = "The RAG backend is responsible for handling the interaction with the RAG system."
 authors = ["STACKIT Data and AI Consulting <data-ai-consulting@stackit.cloud>"]
 
 [tool.poetry.dependencies]
@@ -39,7 +39,6 @@ flake8-tidy-imports = "^4.10.0"
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
-
 [tool.flake8]
 exclude= [".eggs", "./rag-core-library/*", ".git", ".hg", ".mypy_cache", ".tox", ".venv", ".devcontainer", "venv", "_build", "buck-out", "build", "dist"]
 statistics = true
@@ -58,7 +57,6 @@ per-file-ignores = """
   ./tests/*: S101,
 """
 
-
 [tool.black]
 line-length = 120
 exclude = """
@@ -80,12 +78,10 @@ exclude = """
 )/
 """
 
-
 [tool.isort]
 profile = "black"
 skip = ['.eggs', '.git', '.hg', '.mypy_cache', '.nox', '.pants.d', '.tox', '.venv', '_build', 'buck-out', 'build', 'dist', 'node_modules', 'venv']
 skip_gitignore = true
 
 [tool.pylint]
 max-line-length = 120
-

rag-backend/tests/dummy_test.py → rag-backend/tests/dummy2_test.py

@@ -1,3 +1,3 @@
-def test_dummy() -> None:
+def test_dummy2() -> None:
     print("Dummy test.")
     assert True  # noqa S101