From e7c28ac283a0beeb8a60731f68ac895119c479fa Mon Sep 17 00:00:00 2001 From: Andreas Klos Date: Fri, 23 May 2025 14:36:40 +0200 Subject: [PATCH 01/22] feat: add Confluence integration with configurable parameters for document loading --- .../data-access/+state/documents.store.ts | 7 +++-- .../admin-app/data-access/document.api.ts | 27 ++++++++++++++--- .../DocumentUploadContainer.vue | 29 +++++++++++++++++-- frontend/libs/i18n/admin/en.json | 2 +- rag-core-library | 2 +- rag-infrastructure | 2 +- 6 files changed, 56 insertions(+), 13 deletions(-) diff --git a/frontend/libs/admin-app/data-access/+state/documents.store.ts b/frontend/libs/admin-app/data-access/+state/documents.store.ts index e73151c..383da5d 100644 --- a/frontend/libs/admin-app/data-access/+state/documents.store.ts +++ b/frontend/libs/admin-app/data-access/+state/documents.store.ts @@ -3,7 +3,7 @@ import { ref } from 'vue'; import { DocumentModel } from "../../models/document.model.ts"; import { ErrorType } from "../../models/error-type"; import { UploadedDocument, mapToUploadDocument } from "../../models/uploaded-document.model"; -import { DocumentAPI } from "../document.api"; +import { DocumentAPI, ConfluenceConfig } from "../document.api"; export const useDocumentsStore = defineStore('chat', () => { const uploadedDocuments = ref([]); @@ -52,11 +52,12 @@ export const useDocumentsStore = defineStore('chat', () => { } }; - const loadConfluence = async () => { + const loadConfluence = async (config: ConfluenceConfig) => { isLoadingConfluence.value = true; error.value = null; try { - await DocumentAPI.loadConfluence(); + // provide confluence configuration from frontend + await DocumentAPI.loadConfluence(config); await loadDocuments(); // Refresh the document list after uploading } catch(err) { if (err.response && err.response.status === 501) { diff --git a/frontend/libs/admin-app/data-access/document.api.ts b/frontend/libs/admin-app/data-access/document.api.ts index ef058ab..3635f95 100644 --- a/frontend/libs/admin-app/data-access/document.api.ts +++ b/frontend/libs/admin-app/data-access/document.api.ts @@ -7,6 +7,15 @@ axios.defaults.auth = { password: import.meta.env.VITE_AUTH_PASSWORD }; +// confluence configuration interface +export interface ConfluenceConfig { + spaceKey: string; + token: string; + url: string; + maxPages: number; + name: string; +} + export class DocumentAPI { static async loadDocuments(): Promise { try { @@ -20,9 +29,9 @@ export class DocumentAPI { static async uploadDocument(file: File, onUploadProgress: (progressEvent: AxiosProgressEvent) => void): Promise { try { const formData = new FormData(); - formData.append('body', file); + formData.append('file', file); - const response = await axios.post('/upload_documents', formData, { + const response = await axios.post('/upload_file', formData, { headers: { 'Content-Type': 'multipart/form-data' }, @@ -35,9 +44,19 @@ export class DocumentAPI { } } - static async loadConfluence(): Promise { + static async loadConfluence(config: ConfluenceConfig): Promise { try { - await axios.post('/load_confluence'); + // convert config to list of key/value items for backend + const payload = [ + { key: 'url', value: config.url }, + { key: 'token', value: config.token }, + { key: 'space_key', value: config.spaceKey }, + { key: 'max_pages', value: String(config.maxPages) } + ]; + // include required query parameters + await axios.post('/upload_source', payload, { + params: { type: 'confluence', name: config.name } + }); } catch(error) { this.handleError(error); } diff --git a/frontend/libs/admin-app/feature-document/DocumentUploadContainer.vue b/frontend/libs/admin-app/feature-document/DocumentUploadContainer.vue index 949e976..089ef42 100644 --- a/frontend/libs/admin-app/feature-document/DocumentUploadContainer.vue +++ b/frontend/libs/admin-app/feature-document/DocumentUploadContainer.vue @@ -15,6 +15,14 @@ const isInvalidFileType = ref(false); const allowedFileTypes = ['application/pdf', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/vnd.openxmlformats-officedocument.presentationml.presentation', 'text/xml']; const uploadMethod = ref<'file' | 'confluence'>('file'); + +// confluence configuration refs +const confluenceName = ref(''); +const spaceKey = ref(''); +const confluenceToken = ref(''); +const confluenceUrl = ref(''); +const maxPages = ref(1000); + const error = computed(() => store.error); const uploadDocuments = (files: File[]) => { @@ -55,7 +63,14 @@ const onRemoveDocument = (documentId: string) => { } const handleConfluenceUpload = () => { - store.loadConfluence(); + // send configured parameters to backend + store.loadConfluence({ + name: confluenceName.value, + spaceKey: spaceKey.value, + token: confluenceToken.value, + url: confluenceUrl.value, + maxPages: maxPages.value + }); } const clearError = () => { @@ -113,7 +128,7 @@ const getErrorMessage = (errorType: string) => {
@@ -130,10 +145,18 @@ const getErrorMessage = (errorType: string) => {
+ class="flex flex-col m-auto justify-center items-center w-full h-112 bg-base-100 rounded-box border border-base-300">

{{ t('documents.confluenceLoadTitle') }}

+ +
+ + + + + +

{{ t('documents.confluenceLoadDescription') }}

@@ -144,7 +168,7 @@ const getErrorMessage = (errorType: string) => {
-
@@ -169,6 +193,28 @@ const getErrorMessage = (errorType: string) => {
+ +
+
+ +

{{ t('documents.sitemapLoadTitle') }}

+ +
+ + + + + + +
+

{{ t('documents.sitemapLoadDescription') }}

+ +
+
+
diff --git a/frontend/libs/i18n/admin/en.json b/frontend/libs/i18n/admin/en.json index d5ad81b..b1d145f 100644 --- a/frontend/libs/i18n/admin/en.json +++ b/frontend/libs/i18n/admin/en.json @@ -15,6 +15,10 @@ "confluenceLoadTitle": "Load all Confluence pages from a space", "confluenceLoadDescription": "Click the button below to load pages from Confluence", "loadConfluence": "Load Confluence", + "sitemapUpload": "Sitemap", + "sitemapLoadTitle": "Load content from a sitemap", + "sitemapLoadDescription": "Enter a sitemap URL to extract and load content from all linked pages", + "loadSitemap": "Load Sitemap", "select": "Select", "chat": "Start chat", "uploadDocumentFailed": "Upload failed", @@ -25,6 +29,9 @@ "confluenceError": "Failed to load from Confluence", "confluenceNotConfigured": "Confluence is not configured", "confluenceLocked": "Confluence Loader is busy. Please try again later", + "sitemapError": "Failed to load from sitemap", + "sitemapNotConfigured": "Sitemap is not configured", + "sitemapLocked": "Sitemap Loader is busy. Please try again later", "unknownError": "An unknown error occurred" } } diff --git a/rag-core-library b/rag-core-library index 16a7309..e8e55fc 160000 --- a/rag-core-library +++ b/rag-core-library @@ -1 +1 @@ -Subproject commit 16a730959b4ee389f2ee811ed0f2474ae11031fa +Subproject commit e8e55fcda605efee280d05e34869abb7586c0483 From 5702fa7648abfd53b2f3088da6ff6147f5d7417b Mon Sep 17 00:00:00 2001 From: Andreas Klos Date: Wed, 4 Jun 2025 15:41:57 +0200 Subject: [PATCH 08/22] feat: add header template support for sitemap upload with JSON validation --- frontend/libs/admin-app/data-access/document.api.ts | 12 ++++++++++++ .../feature-document/DocumentUploadContainer.vue | 6 +++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/frontend/libs/admin-app/data-access/document.api.ts b/frontend/libs/admin-app/data-access/document.api.ts index e0d45b8..04b6360 100644 --- a/frontend/libs/admin-app/data-access/document.api.ts +++ b/frontend/libs/admin-app/data-access/document.api.ts @@ -20,6 +20,7 @@ export interface ConfluenceConfig { export interface SitemapConfig { webPath: string; filterUrls: string; + headerTemplate: string; name: string; } @@ -89,6 +90,17 @@ export class DocumentAPI { } } + // add header_template only if provided + if (config.headerTemplate && config.headerTemplate.trim()) { + try { + // Validate JSON format + JSON.parse(config.headerTemplate); + payload.push({ key: 'header_template', value: config.headerTemplate }); + } catch (jsonError) { + throw new Error('Header template must be valid JSON format'); + } + } + // include required query parameters await axios.post('/upload_source', payload, { params: { source_type: 'sitemap', name: config.name } diff --git a/frontend/libs/admin-app/feature-document/DocumentUploadContainer.vue b/frontend/libs/admin-app/feature-document/DocumentUploadContainer.vue index 5bf90b0..7bf66bd 100644 --- a/frontend/libs/admin-app/feature-document/DocumentUploadContainer.vue +++ b/frontend/libs/admin-app/feature-document/DocumentUploadContainer.vue @@ -27,6 +27,7 @@ const maxPages = ref(); const sitemapName = ref(''); const sitemapWebPath = ref(''); const sitemapFilterUrls = ref(''); +const sitemapHeaderTemplate = ref(''); const error = computed(() => store.error); @@ -83,7 +84,8 @@ const handleSitemapUpload = () => { store.loadSitemap({ name: sitemapName.value, webPath: sitemapWebPath.value, - filterUrls: sitemapFilterUrls.value + filterUrls: sitemapFilterUrls.value, + headerTemplate: sitemapHeaderTemplate.value }); } @@ -207,6 +209,8 @@ const getErrorMessage = (errorType: string) => { + +

{{ t('documents.sitemapLoadDescription') }}