From 0f1dff0b8ebc354c0f16dc35c9ea37ec6b3b36cd Mon Sep 17 00:00:00 2001 From: Himansh Varma Date: Wed, 3 Sep 2025 23:21:51 +0530 Subject: [PATCH 1/8] added multi file attachment support --- frontend/src/components/AttachmentGallery.tsx | 8 +- frontend/src/components/AttachmentPreview.tsx | 98 +++----------- frontend/src/components/ChatBox.tsx | 95 +++++++++++--- frontend/src/utils/fileUtils.ts | 35 +++++ server/ai/provider/vertex_ai.ts | 49 +++++-- server/api/chat/agents.ts | 114 ++++++++--------- server/api/chat/chat.ts | 47 ++++--- server/api/chat/utils.ts | 2 +- server/api/files.ts | 121 +++++++++++++----- server/docxChunks.ts | 7 +- server/integrations/dataSource/index.ts | 2 +- server/pdfChunks.ts | 9 +- server/pptChunks.ts | 7 +- server/services/fileProcessor.ts | 19 ++- 14 files changed, 376 insertions(+), 237 deletions(-) diff --git a/frontend/src/components/AttachmentGallery.tsx b/frontend/src/components/AttachmentGallery.tsx index a185def5d..6b3d0b446 100644 --- a/frontend/src/components/AttachmentGallery.tsx +++ b/frontend/src/components/AttachmentGallery.tsx @@ -122,16 +122,16 @@ export const AttachmentGallery: React.FC = ({ {/* Other Files */} {otherFiles.length > 0 && ( -
-

+
+

Files ({otherFiles.length})

-
+
{otherFiles.map((file) => ( ))}
diff --git a/frontend/src/components/AttachmentPreview.tsx b/frontend/src/components/AttachmentPreview.tsx index 3c473b777..fe744cc0f 100644 --- a/frontend/src/components/AttachmentPreview.tsx +++ b/frontend/src/components/AttachmentPreview.tsx @@ -1,14 +1,7 @@ import React, { useState } from "react" import { AttachmentMetadata } from "shared/types" import { - Download, Eye, - FileText, - Image, - Video, - Music, - Archive, - File, } from "lucide-react" import { Button } from "@/components/ui/button" import { @@ -17,27 +10,14 @@ import { DialogHeader, DialogTitle, } from "@/components/ui/dialog" -import { authFetch } from "@/utils/authFetch" +import { getFileType } from "@/utils/fileUtils" +import { getFileIcon } from "@/components/ChatBox" interface AttachmentPreviewProps { attachment: AttachmentMetadata className?: string } -const getFileIcon = (fileType: string) => { - if (fileType.startsWith("image/")) return Image - if (fileType.startsWith("video/")) return Video - if (fileType.startsWith("audio/")) return Music - if (fileType.includes("pdf") || fileType.includes("document")) return FileText - if ( - fileType.includes("zip") || - fileType.includes("tar") || - fileType.includes("gz") - ) - return Archive - return File -} - const formatFileSize = (bytes: number): string => { if (bytes === 0) return "0 Bytes" const k = 1024 @@ -53,46 +33,11 @@ export const AttachmentPreview: React.FC = ({ const [showImageModal, setShowImageModal] = useState(false) const [imageError, setImageError] = useState(false) - const FileIcon = getFileIcon(attachment.fileType) const isImage = attachment.isImage && !imageError const thumbnailUrl = attachment.thumbnailPath ? `/api/v1/attachments/${attachment.fileId}/thumbnail` : null - const handleDownload = async () => { - let url: string | null = null - try { - const response = await authFetch( - `/api/v1/attachments/${attachment.fileId}`, - { - credentials: "include", - }, - ) - if (!response.ok) { - if (response.status === 401) { - throw new Error("Please log in to download attachments") - } - throw new Error(`Download failed: ${response.statusText}`) - } - - const blob = await response.blob() - url = window.URL.createObjectURL(blob) - const a = document.createElement("a") - a.href = url - a.download = attachment.fileName - document.body.appendChild(a) - a.click() - document.body.removeChild(a) - } catch (error) { - console.error("Download failed:", error) - alert(error instanceof Error ? error.message : "Download failed") - } finally { - if (url) { - window.URL.revokeObjectURL(url) - } - } - } - const handleImageView = () => { if (isImage) { setShowImageModal(true) @@ -126,7 +71,7 @@ export const AttachmentPreview: React.FC = ({
) : (
- + {getFileIcon(getFileType({type: attachment.fileType, name: attachment.fileName}))}
)}
@@ -137,33 +82,24 @@ export const AttachmentPreview: React.FC = ({ {attachment.fileName}

- {formatFileSize(attachment.fileSize)} • {attachment.fileType} + {formatFileSize(attachment.fileSize)} • {getFileType({type: attachment.fileType, name: attachment.fileName})}

{/* Actions */} -
- {isImage && ( - - )} - -
+ {isImage && ( +
+ +
+ )} {/* Image Modal */} {isImage && ( diff --git a/frontend/src/components/ChatBox.tsx b/frontend/src/components/ChatBox.tsx index f5cfff02a..6d1e28d9d 100644 --- a/frontend/src/components/ChatBox.tsx +++ b/frontend/src/components/ChatBox.tsx @@ -28,6 +28,10 @@ import { X, File, Loader2, + FileText, + FileSpreadsheet, + Presentation, + FileImage, } from "lucide-react" import Attach from "@/assets/attach.svg?react" import { @@ -72,6 +76,7 @@ import { createToastNotifier, createImagePreview, cleanupPreviewUrls, + getFileType, } from "@/utils/fileUtils" import { authFetch } from "@/utils/authFetch" @@ -82,6 +87,26 @@ interface SelectedFile { uploading?: boolean uploadError?: string preview?: string // URL for image preview + fileType?: string +} + +export const getFileIcon = (fileType: string | undefined) => { + switch (fileType) { + case "Image": + return + case "Document": + return + case "Spreadsheet": + return + case "Presentation": + return + case "PDF": + return + case "Text": + return + default: + return + } } // Add attachment limit constant @@ -498,6 +523,7 @@ export const ChatBox = React.forwardRef( id: generateFileId(), uploading: false, preview: createImagePreview(file), + fileType: getFileType(file), })) setSelectedFiles((prev) => { @@ -611,6 +637,26 @@ export const ChatBox = React.forwardRef( return ext || "file" } + // Get appropriate icon for file type + const getFileIcon = (fileType: string | undefined) => { + switch (fileType) { + case "Image": + return + case "Document": + return + case "Spreadsheet": + return + case "Presentation": + return + case "PDF": + return + case "Text": + return + default: + return + } + } + const removeFile = useCallback((id: string) => { setSelectedFiles((prev) => { const fileToRemove = prev.find((f) => f.id === id) @@ -2097,10 +2143,10 @@ export const ChatBox = React.forwardRef( const items = e.clipboardData?.items if (items) { - // Handle image paste + // Handle file paste (all supported file types) for (let i = 0; i < items.length; i++) { const item = items[i] - if (item.type.indexOf("image") !== -1) { + if (item.kind === "file") { // Check attachment limit before processing if (selectedFiles.length >= MAX_ATTACHMENTS) { showToast( @@ -2113,15 +2159,27 @@ export const ChatBox = React.forwardRef( const file = item.getAsFile() if (file) { - // Process the pasted image file directly - // The file will be processed with a default name by the processFiles function - processFiles([file]) - showToast( - "Image pasted", - "Image has been added to your message.", - false, - ) - return // Exit early since we handled the image + // Check if the file type is supported + const isValid = validateAndDeduplicateFiles([file], showToast) + if (isValid.length > 0) { + // Process the pasted file + processFiles([file]) + const fileType = getFileType(file) + + showToast( + "File pasted", + `${fileType} has been added to your message.`, + false, + ) + return // Exit early since we handled the file + } else { + showToast( + "Unsupported file type", + "This file type is not supported for attachments.", + true, + ) + return + } } } } @@ -2450,14 +2508,14 @@ export const ChatBox = React.forwardRef( ? `${selectedFile.file.name.substring(0, 9)}...` : selectedFile.file.name} + + {selectedFile.fileType} + ) : (
- + {getFileIcon(selectedFile.fileType)}
( className="text-xs text-gray-500 dark:text-gray-400 truncate block max-w-[120px]" title={getExtension(selectedFile.file)} > - {getExtension(selectedFile.file)} + {selectedFile.fileType}
@@ -2533,7 +2591,7 @@ export const ChatBox = React.forwardRef( title={ selectedFiles.length >= MAX_ATTACHMENTS ? `Maximum ${MAX_ATTACHMENTS} attachments allowed` - : "Attach files" + : "Attach files (images, documents, spreadsheets, presentations, PDFs, text files)" } /> {showAdvancedOptions && ( @@ -3299,7 +3357,8 @@ export const ChatBox = React.forwardRef( multiple className="hidden" onChange={handleFileChange} - accept="image/jpeg,image/jpg,image/png,image/gif,image/webp" + accept="image/jpeg,image/jpg,image/png,image/gif,image/webp,text/plain,text/csv,application/pdf,application/msword,application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/vnd.ms-powerpoint,application/vnd.openxmlformats-officedocument.presentationml.presentation,text/markdown,.txt,.csv,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx,.md" + title="Supported file types: Images (JPEG, PNG, GIF, WebP), Documents (DOC, DOCX), Spreadsheets (XLS, XLSX, CSV), Presentations (PPT, PPTX), PDFs, and Text files (TXT, MD)" />
) diff --git a/frontend/src/utils/fileUtils.ts b/frontend/src/utils/fileUtils.ts index 5ed85c565..fa9ec7680 100644 --- a/frontend/src/utils/fileUtils.ts +++ b/frontend/src/utils/fileUtils.ts @@ -19,6 +19,41 @@ const isImageFile = (file: File): boolean => { ) } +// Get file type category for display purposes +export const getFileType = (file: File | { type: string, name: string }): string => { + if (file instanceof File && isImageFile(file)) { + return "Image" + } + + // Check for document types + if (file.type.includes("word") || file.name.toLowerCase().match(/\.(doc|docx)$/)) { + return "Document" + } + + // Check for spreadsheet types + if (file.type.includes("excel") || file.type.includes("spreadsheet") || file.name.toLowerCase().match(/\.(xls|xlsx|csv)$/)) { + return "Spreadsheet" + } + + // Check for presentation types + if (file.type.includes("powerpoint") || file.type.includes("presentation") || file.name.toLowerCase().match(/\.(ppt|pptx)$/)) { + return "Presentation" + } + + // Check for PDF + if (file.type === "application/pdf" || file.name.toLowerCase().endsWith(".pdf")) { + return "PDF" + } + + // Check for text files + if (file.type.startsWith("text/") || file.name.toLowerCase().match(/\.(txt|md)$/)) { + return "Text" + } + + // Default fallback + return "File" +} + // Create preview URL for image files export const createImagePreview = (file: File): string | undefined => { if (isImageFile(file)) { diff --git a/server/ai/provider/vertex_ai.ts b/server/ai/provider/vertex_ai.ts index b757de0f0..efad38859 100644 --- a/server/ai/provider/vertex_ai.ts +++ b/server/ai/provider/vertex_ai.ts @@ -32,20 +32,51 @@ const buildVertexAIImageParts = async (imagePaths: string[]) => { ) const imagePromises = imagePaths.map(async (imgPath) => { - const match = imgPath.match(/^(.+)_([0-9]+)$/) - if (!match) throw new Error(`Invalid image path: ${imgPath}`) - const docId = match[1] + // format: docIndex_docId_imageNumber + const match = imgPath.match(/^([0-9]+)_(.+)_([0-9]+)$/) + if (!match) { + Logger.error( + `Invalid image path format: ${imgPath}. Expected format: docIndex_docId_imageNumber`, + ) + throw new Error(`Invalid image path: ${imgPath}`) + } + + const docIndex = match[1] + const docId = match[2] + const imageNumber = match[3] + + if (docId.includes("..") || docId.includes("/") || docId.includes("\\")) { + Logger.error(`Invalid docId containing path traversal: ${docId}`) + throw new Error(`Invalid docId: ${docId}`) + } + const imageDir = path.join(baseDir, docId) - const absolutePath = findImageByName(imageDir, match[2]) - const ext = path.extname(absolutePath).toLowerCase() - const mimeMap: Record = { + const absolutePath = findImageByName(imageDir, imageNumber) + const extension = path.extname(absolutePath).toLowerCase() + + // Map file extensions to Bedrock format values + const formatMap: Record = { ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", + ".gif": "image/gif", ".webp": "image/webp", } - const mimeType = mimeMap[ext] - if (!mimeType) return null + + const format = formatMap[extension] + if (!format) { + Logger.warn( + `Unsupported image format: ${extension}. Skipping image: ${absolutePath}`, + ) + return null + } + + // Ensure the resolved path is within baseDir + const resolvedPath = path.resolve(imageDir) + if (!resolvedPath.startsWith(baseDir)) { + Logger.error(`Path traversal attempt detected: ${imageDir}`) + throw new Error(`Invalid path: ${imageDir}`) + } try { await fs.promises.access(absolutePath, fs.constants.F_OK) @@ -54,7 +85,7 @@ const buildVertexAIImageParts = async (imagePaths: string[]) => { const base64 = imgBuffer.toString("base64") return { type: "image", - source: { type: "base64", media_type: mimeType, data: base64 }, + source: { type: "base64", media_type: format, data: base64 }, } } catch (err) { Logger.error(`Failed to read image: ${absolutePath}`) diff --git a/server/api/chat/agents.ts b/server/api/chat/agents.ts index 9fbf73e79..d6ef63bf3 100644 --- a/server/api/chat/agents.ts +++ b/server/api/chat/agents.ts @@ -652,9 +652,12 @@ export const MessageWithToolsApi = async (c: Context) => { agentId, }: MessageReqType = body const attachmentMetadata = parseAttachmentMetadata(c) - const attachmentFileIds = attachmentMetadata.map( - (m: AttachmentMetadata) => m.fileId, - ) + const NonImageAttachmentFileIds = attachmentMetadata.map( + (m: AttachmentMetadata) => (m.isImage ? null : m.fileId), + ).filter((m: string | null) => m !== null) + const ImageAttachmentFileIds = attachmentMetadata.map( + (m: AttachmentMetadata) => (m.isImage ? m.fileId : null), + ).filter((m: string | null) => m !== null) const agentPromptValue = agentId && isCuid(agentId) ? agentId : undefined // const userRequestsReasoning = isReasoningEnabled // Addressed: Will be used below let attachmentStorageError: Error | null = null @@ -665,7 +668,10 @@ export const MessageWithToolsApi = async (c: Context) => { totalValidFileIdsFromLinkCount: 0, fileIds: [], } - const fileIds = extractedInfo?.fileIds + let fileIds = extractedInfo?.fileIds + if (NonImageAttachmentFileIds && NonImageAttachmentFileIds.length > 0) { + fileIds = [...fileIds, ...NonImageAttachmentFileIds] + } const totalValidFileIdsFromLinkCount = extractedInfo?.totalValidFileIdsFromLinkCount const hasReferencedContext = fileIds && fileIds.length > 0 @@ -1396,6 +1402,18 @@ export const MessageWithToolsApi = async (c: Context) => { } } planningContext = cleanContext(resolvedContexts?.join("\n")) + const { imageFileNames } = extractImageFileNames( + planningContext, + [...results.root.children, ...chatContexts, ...threadContexts], + ) + + const finalImageFileNames = imageFileNames || [] + + if (ImageAttachmentFileIds?.length) { + finalImageFileNames.push( + ...ImageAttachmentFileIds.map((fileid, index) => `${index}_${fileid}_${0}`), + ) + } if (chatContexts.length > 0) { planningContext += "\n" + buildContext(chatContexts, 10) } @@ -1429,7 +1447,7 @@ export const MessageWithToolsApi = async (c: Context) => { messagesWithNoErrResponse, logAndStreamReasoning, sub, - attachmentFileIds, + ImageAttachmentFileIds, ) await logAndStreamReasoning({ type: AgentReasoningStepType.LogMessage, @@ -2021,7 +2039,7 @@ export const MessageWithToolsApi = async (c: Context) => { messagesWithNoErrResponse, logAndStreamReasoning, sub, - attachmentFileIds, + ImageAttachmentFileIds, ) await logAndStreamReasoning({ @@ -2208,7 +2226,7 @@ export const MessageWithToolsApi = async (c: Context) => { messagesWithNoErrResponse, logAndStreamReasoning, sub, - attachmentFileIds, + ImageAttachmentFileIds, ) await logAndStreamReasoning({ type: AgentReasoningStepType.LogMessage, @@ -2253,7 +2271,7 @@ export const MessageWithToolsApi = async (c: Context) => { agentPromptForLLM, messagesWithNoErrResponse, fallbackReasoning, - attachmentFileIds, + ImageAttachmentFileIds, email, ) for await (const chunk of continuationIterator) { @@ -2656,7 +2674,6 @@ async function* nonRagIterator( agentPrompt?: string, messages: Message[] = [], imageFileNames: string[] = [], - attachmentFileIds?: string[], email?: string, isReasoning = true, ): AsyncIterableIterator< @@ -2792,10 +2809,8 @@ export const AgentMessageApiRagOff = async (c: Context) => { rootSpan.setAttribute("email", email) rootSpan.setAttribute("workspaceId", workspaceId) - const attachmentMetadata = parseAttachmentMetadata(c) - const attachmentFileIds = attachmentMetadata.map( - (m: AttachmentMetadata) => m.fileId, - ) + // @ts-ignore + const body = c.req.valid("query") let { message, chatId, @@ -2864,7 +2879,6 @@ export const AgentMessageApiRagOff = async (c: Context) => { stream: false, }) title = titleResp.title - let attachmentStorageError: Error | null = null const cost = titleResp.cost if (cost) { costArr.push(cost) @@ -2898,24 +2912,6 @@ export const AgentMessageApiRagOff = async (c: Context) => { modelId, fileIds: fileIds, }) - - if (attachmentMetadata && attachmentMetadata.length > 0) { - try { - await storeAttachmentMetadata( - tx, - insertedMsg.externalId, - attachmentMetadata, - email, - ) - } catch (error) { - attachmentStorageError = error as Error - loggerWithChild({ email: email }).error( - error, - `Failed to store attachment metadata for user message ${insertedMsg.externalId}`, - ) - } - } - return [chat, insertedMsg] }, ) @@ -3057,13 +3053,6 @@ export const AgentMessageApiRagOff = async (c: Context) => { // context = initialContext; } } - if (attachmentFileIds?.length) { - finalImageFileNames.push( - ...attachmentFileIds.map( - (fileid, index) => `${index}_${fileid}_${0}`, - ), - ) - } const ragOffIterator = nonRagIterator( message, @@ -3073,7 +3062,6 @@ export const AgentMessageApiRagOff = async (c: Context) => { agentPromptForLLM, messagesWithNoErrResponse, finalImageFileNames, - attachmentFileIds, email, isReasoningEnabled, ) @@ -3333,13 +3321,6 @@ export const AgentMessageApiRagOff = async (c: Context) => { ) finalImageFileNames = imageFileNames || [] } - if (attachmentFileIds?.length) { - finalImageFileNames.push( - ...attachmentFileIds.map( - (fileid, index) => `${index}_${fileid}_${0}`, - ), - ) - } // Helper: persist & return JSON once ---------------------------------------- const finalizeAndRespond = async (params: { @@ -3397,7 +3378,6 @@ export const AgentMessageApiRagOff = async (c: Context) => { agentPromptForLLM, messagesWithNoErrResponse, finalImageFileNames, - attachmentFileIds, email, isReasoningEnabled, ) @@ -3493,9 +3473,12 @@ export const AgentMessageApi = async (c: Context) => { rootSpan.setAttribute("workspaceId", workspaceId) const attachmentMetadata = parseAttachmentMetadata(c) - const attachmentFileIds = attachmentMetadata.map( - (m: AttachmentMetadata) => m.fileId, - ) + const ImageAttachmentFileIds = attachmentMetadata.map( + (m: AttachmentMetadata) => (m.isImage ? m.fileId : null), + ).filter((m: string | null) => m !== null) + const NonImageAttachmentFileIds = attachmentMetadata.map( + (m: AttachmentMetadata) => (m.isImage ? null : m.fileId), + ).filter((m: string | null) => m !== null) let attachmentStorageError: Error | null = null let { message, @@ -3530,7 +3513,7 @@ export const AgentMessageApi = async (c: Context) => { }) } agentPromptForLLM = JSON.stringify(agentForDb) - if (config.ragOffFeature && agentForDb.isRagOn === false) { + if (config.ragOffFeature && agentForDb.isRagOn === false && !(attachmentMetadata && attachmentMetadata.length > 0)) { return AgentMessageApiRagOff(c) } } @@ -3548,15 +3531,20 @@ export const AgentMessageApi = async (c: Context) => { if (path) { ids = await getRecordBypath(path, db) } - const isMsgWithContext = isMessageWithContext(message) || (path && ids) + let isMsgWithContext = isMessageWithContext(message) const extractedInfo = isMsgWithContext || (path && ids) - ? await extractFileIdsFromMessage(message, email, ids) - : { - totalValidFileIdsFromLinkCount: 0, - fileIds: [], - } - const fileIds = extractedInfo?.fileIds + ? await extractFileIdsFromMessage(message, email, ids) + : { + totalValidFileIdsFromLinkCount: 0, + fileIds: [], + } + isMsgWithContext = isMsgWithContext || (NonImageAttachmentFileIds && NonImageAttachmentFileIds.length > 0) + let fileIds = extractedInfo?.fileIds + if (NonImageAttachmentFileIds && NonImageAttachmentFileIds.length > 0) { + fileIds = [...fileIds, ...NonImageAttachmentFileIds] + } + const agentDocs = agentForDb?.docIds || [] //add docIds of agents here itself @@ -3747,7 +3735,7 @@ export const AgentMessageApi = async (c: Context) => { if ( (isMsgWithContext && fileIds && fileIds?.length > 0) || - (attachmentFileIds && attachmentFileIds?.length > 0) + (ImageAttachmentFileIds && ImageAttachmentFileIds?.length > 0) ) { Logger.info( "User has selected some context with query, answering only based on that given context", @@ -3779,7 +3767,7 @@ export const AgentMessageApi = async (c: Context) => { userRequestsReasoning, understandSpan, [], - attachmentFileIds, + ImageAttachmentFileIds, agentPromptForLLM, ) stream.writeSSE({ @@ -4687,7 +4675,7 @@ export const AgentMessageApi = async (c: Context) => { // Path A: user provided explicit context (fileIds / attachments) if ( (isMsgWithContext && fileIds && fileIds.length > 0) || - (attachmentFileIds && attachmentFileIds.length > 0) + (ImageAttachmentFileIds && ImageAttachmentFileIds.length > 0) ) { const ragSpan = streamSpan.startSpan("rag_processing") const understandSpan = ragSpan.startSpan("understand_message") @@ -4701,7 +4689,7 @@ export const AgentMessageApi = async (c: Context) => { userRequestsReasoning, understandSpan, [], - attachmentFileIds, + ImageAttachmentFileIds, agentPromptForLLM, ) diff --git a/server/api/chat/chat.ts b/server/api/chat/chat.ts index 3d03e1ac3..486eadc1a 100644 --- a/server/api/chat/chat.ts +++ b/server/api/chat/chat.ts @@ -2122,7 +2122,7 @@ async function* generateAnswerFromGivenContext( modelId: defaultBestModel, reasoning: config.isReasoning && userRequestsReasoning, agentPrompt, - imageFileNames, + imageFileNames: finalImageFileNames, }, true, ) @@ -2139,6 +2139,12 @@ async function* generateAnswerFromGivenContext( generateAnswerSpan?.end() return } else if (!answer) { + if(attachmentFileIds && attachmentFileIds.length > 0) { + yield { + text: "From the selected context, I could not find any information to answer it, please change your query", + } + return + } // If we give the whole context then also if there's no answer then we can just search once and get the best matching chunks with the query and then make context try answering loggerWithChild({ email: email }).info( "No answer was found when all chunks were given, trying to answer after searching vespa now", @@ -4024,9 +4030,12 @@ export const MessageApi = async (c: Context) => { return MessageWithToolsApi(c) } const attachmentMetadata = parseAttachmentMetadata(c) - const attachmentFileIds = attachmentMetadata.map( - (m: AttachmentMetadata) => m.fileId, - ) + const ImageAttachmentFileIds = attachmentMetadata.map( + (m: AttachmentMetadata) => m.isImage ? m.fileId : null, + ).filter((m: string | null) => m !== null) + const NonImageAttachmentFileIds = attachmentMetadata.map( + (m: AttachmentMetadata) => m.isImage ? null : m.fileId, + ).filter((m: string | null) => m !== null) if (agentPromptValue) { const userAndWorkspaceCheck = await getUserAndWorkspaceByEmail( @@ -4055,7 +4064,7 @@ export const MessageApi = async (c: Context) => { message = decodeURIComponent(message) rootSpan.setAttribute("message", message) - const isMsgWithContext = isMessageWithContext(message) + let isMsgWithContext = isMessageWithContext(message) const extractedInfo = isMsgWithContext ? await extractFileIdsFromMessage(message, email) : { @@ -4063,7 +4072,11 @@ export const MessageApi = async (c: Context) => { fileIds: [], threadIds: [], } - const fileIds = extractedInfo?.fileIds + isMsgWithContext = isMsgWithContext || (NonImageAttachmentFileIds && NonImageAttachmentFileIds.length > 0) + let fileIds = extractedInfo?.fileIds + if (NonImageAttachmentFileIds && NonImageAttachmentFileIds.length > 0) { + fileIds = fileIds.concat(NonImageAttachmentFileIds) + } const threadIds = extractedInfo?.threadIds || [] const totalValidFileIdsFromLinkCount = extractedInfo?.totalValidFileIdsFromLinkCount @@ -4274,7 +4287,7 @@ export const MessageApi = async (c: Context) => { } if ( (isMsgWithContext && fileIds && fileIds?.length > 0) || - (attachmentFileIds && attachmentFileIds?.length > 0) + (ImageAttachmentFileIds && ImageAttachmentFileIds?.length > 0) ) { let answer = "" let citations = [] @@ -4301,7 +4314,7 @@ export const MessageApi = async (c: Context) => { userRequestsReasoning, understandSpan, threadIds, - attachmentFileIds, + ImageAttachmentFileIds, ) stream.writeSSE({ event: ChatSSEvents.Start, @@ -5451,14 +5464,14 @@ export const MessageRetryApi = async (c: Context) => { // If it's an assistant message, we need to get attachments from the previous user message let attachmentMetadata: AttachmentMetadata[] = [] - let attachmentFileIds: string[] = [] + let ImageAttachmentFileIds: string[] = [] if (isUserMessage) { // If retrying a user message, get attachments from that message attachmentMetadata = await getAttachmentsByMessageId(db, messageId, email) - attachmentFileIds = attachmentMetadata.map( - (m: AttachmentMetadata) => m.fileId, - ) + ImageAttachmentFileIds = attachmentMetadata.map( + (m: AttachmentMetadata) => m.isImage ? m.fileId : null, + ).filter((m: string | null) => m !== null) } rootSpan.setAttribute("email", email) @@ -5512,9 +5525,9 @@ export const MessageRetryApi = async (c: Context) => { prevUserMessage.externalId, email, ) - attachmentFileIds = attachmentMetadata.map( - (m: AttachmentMetadata) => m.fileId, - ) + ImageAttachmentFileIds = attachmentMetadata.map( + (m: AttachmentMetadata) => m.isImage ? m.fileId : null, + ).filter((m: string | null) => m !== null) } } @@ -5592,7 +5605,7 @@ export const MessageRetryApi = async (c: Context) => { let message = prevUserMessage.message if ( (fileIds && fileIds?.length > 0) || - (attachmentFileIds && attachmentFileIds?.length > 0) + (ImageAttachmentFileIds && ImageAttachmentFileIds?.length > 0) ) { loggerWithChild({ email: email }).info( "[RETRY] User has selected some context with query, answering only based on that given context", @@ -5623,7 +5636,7 @@ export const MessageRetryApi = async (c: Context) => { userRequestsReasoning, understandSpan, threadIds, - attachmentFileIds, + ImageAttachmentFileIds, ) stream.writeSSE({ event: ChatSSEvents.Start, diff --git a/server/api/chat/utils.ts b/server/api/chat/utils.ts index a62b6b7df..f1e67e3bc 100644 --- a/server/api/chat/utils.ts +++ b/server/api/chat/utils.ts @@ -318,7 +318,7 @@ export const extractImageFileNames = ( let imageContent = match[1].trim() try { if (imageContent) { - const docId = imageContent.split("_")[0] + const docId = imageContent.substring(0, imageContent.lastIndexOf("_")) // const docIndex = // results?.findIndex((c) => (c.fields as any).docId === docId) || -1 const docIndex = diff --git a/server/api/files.ts b/server/api/files.ts index 55e46d419..9e7ae4e66 100644 --- a/server/api/files.ts +++ b/server/api/files.ts @@ -12,6 +12,8 @@ import { db } from "@/db/client" import { checkIfDataSourceFileExistsByNameAndId, getDataSourceByNameAndCreator, + insert, + NAMESPACE, } from "../search/vespa" import { NoUserFound } from "@/errors" import config from "@/config" @@ -19,9 +21,12 @@ import { HTTPException } from "hono/http-exception" import { isValidFile } from "../../shared/filesutils" import { generateThumbnail, isImageFile, getThumbnailPath } from "@/utils/image" import type { AttachmentMetadata } from "@/shared/types" +import { FileProcessorService } from "@/services/fileProcessor" +import { Apps, dataSourceFileSchema, datasourceSchema } from "@/search/types" +import type { VespaDataSourceFile } from "@/search/types" +import { createFileMetadata } from "@/integrations/dataSource" const { JwtPayloadKey } = config -const Logger = getLogger(Subsystem.Api).child({ module: "newApps" }) const loggerWithChild = getLoggerWithChild(Subsystem.Api, { module: "newApps" }) const DOWNLOADS_DIR = join(process.cwd(), "downloads") @@ -216,28 +221,73 @@ export const handleAttachmentUpload = async (c: Context) => { for (const file of files) { const fileBuffer = await file.arrayBuffer() - const fileId = crypto.randomUUID() + const fileId = `att_${crypto.randomUUID()}` const ext = file.name.split(".").pop()?.toLowerCase() || "" const fullFileName = `${0}.${ext}` - const baseDir = isImageFile(file.type) - ? path.resolve(process.env.IMAGE_DIR || "downloads/xyne_images_db") - : path.resolve( - process.env.ATTACHMENTS_DIR || "downloads/xyne_attachments", - ) - const outputDir = path.join(baseDir, fileId) + const isImage = isImageFile(file.type) + let thumbnailPath: string | undefined + let outputDir: string | undefined try { - await mkdir(outputDir, { recursive: true }) - const filePath = path.join(outputDir, fullFileName) - await Bun.write(filePath, new Uint8Array(fileBuffer)) - - const isImage = isImageFile(file.type) - let thumbnailPath: string | undefined - - // Generate thumbnail for images if (isImage) { + // For images: save to disk and generate thumbnail + const baseDir = path.resolve(process.env.IMAGE_DIR || "downloads/xyne_images_db") + outputDir = path.join(baseDir, fileId) + + await mkdir(outputDir, { recursive: true }) + const filePath = path.join(outputDir, fullFileName) + await Bun.write(filePath, new Uint8Array(fileBuffer)) + + // Generate thumbnail for images thumbnailPath = getThumbnailPath(outputDir, fileId) await generateThumbnail(Buffer.from(fileBuffer), thumbnailPath) + } else { + // For non-images: process through FileProcessorService and ingest into Vespa + + // Process the file content using FileProcessorService + const processingResult = await FileProcessorService.processFile( + Buffer.from(fileBuffer), + file.type, + file.name, + fileId, + undefined, + true, + false, + ) + + // TODO: Ingest the processed content into Vespa + // This would typically involve calling your Vespa ingestion service + // For now, we'll log the processing result + loggerWithChild({ email }).info( + `Processed non-image file "${file.name}" with ${processingResult.chunks.length} text chunks and ${processingResult.image_chunks.length} image chunks` + ) + + const { chunks, chunks_pos, image_chunks, image_chunks_pos } = processingResult + + const vespaDoc: VespaDataSourceFile = { + docId: fileId, + description: `File: ${file.name} for Attachment`, + app: Apps.DataSource, + fileName: file.name, + fileSize: file.size, + chunks: chunks, + image_chunks: image_chunks || [], + chunks_pos: chunks_pos || [], + image_chunks_pos: image_chunks_pos || [], + uploadedBy: email, + mimeType: file.type, + createdAt: Date.now(), + updatedAt: Date.now(), + dataSourceRef: `id:${NAMESPACE}:${datasourceSchema}::${fileId}`, + metadata: createFileMetadata( + file.name, + email, + chunks.length, + "file", + ), + } + + await insert(vespaDoc, dataSourceFileSchema) } // Create attachment metadata @@ -247,9 +297,9 @@ export const handleAttachmentUpload = async (c: Context) => { fileType: file.type, fileSize: file.size, isImage, - thumbnailPath: thumbnailPath - ? path.relative(baseDir, thumbnailPath) - : undefined, + thumbnailPath: (thumbnailPath && outputDir) + ? path.relative(outputDir, thumbnailPath) + : "", createdAt: new Date(), url: `/api/v1/attachments/${fileId}`, } @@ -257,20 +307,22 @@ export const handleAttachmentUpload = async (c: Context) => { attachmentMetadata.push(metadata) loggerWithChild({ email }).info( - `Attachment "${file.name}" stored with ID ${fileId}${isImage ? " (thumbnail generated)" : ""}`, + `Attachment "${file.name}" processed with ID ${fileId}${isImage ? " (saved to disk with thumbnail)" : " (processed and ingested into Vespa)"}`, ) } catch (error) { - // Cleanup: remove the directory if file write fails - try { - await rm(outputDir, { recursive: true, force: true }) - loggerWithChild({ email }).warn( - `Cleaned up directory ${outputDir} after failed file write`, - ) - } catch (cleanupError) { - loggerWithChild({ email }).error( - cleanupError, - `Failed to cleanup directory ${outputDir} after file write error`, - ) + // Cleanup: remove the directory if file write fails (only for images) + if (isImage && outputDir) { + try { + await rm(outputDir, { recursive: true, force: true }) + loggerWithChild({ email }).warn( + `Cleaned up directory ${outputDir} after failed file write`, + ) + } catch (cleanupError) { + loggerWithChild({ email }).error( + cleanupError, + `Failed to cleanup directory ${outputDir} after file write error`, + ) + } } throw error } @@ -303,7 +355,7 @@ export const handleAttachmentServe = async (c: Context) => { throw new HTTPException(400, { message: "File ID is required" }) } - // First, try the legacy path structure + // First, try the legacy path structure (for images) const legacyBaseDir = path.resolve( process.env.IMAGE_DIR || "downloads/xyne_images_db", ) @@ -330,7 +382,10 @@ export const handleAttachmentServe = async (c: Context) => { // Check if file exists const file = Bun.file(filePath || "") if (!(await file.exists())) { - throw new HTTPException(404, { message: "File not found on disk" }) + // File not found on disk - it might be a non-image file processed through Vespa + throw new HTTPException(404, { + message: "File not found. Non-image files are processed through Vespa and not stored on disk." + }) } loggerWithChild({ email }).info( diff --git a/server/docxChunks.ts b/server/docxChunks.ts index bd2bb748b..35ee56406 100644 --- a/server/docxChunks.ts +++ b/server/docxChunks.ts @@ -2413,6 +2413,7 @@ export async function extractTextAndImagesWithChunksFromDocx( data: Uint8Array, docid: string = crypto.randomUUID(), extractImages: boolean = false, + describeImages: boolean = true, logWarnings: boolean = false, ): Promise { Logger.info(`Starting DOCX processing for document: ${docid}`) @@ -2623,7 +2624,11 @@ export async function extractTextAndImagesWithChunksFromDocx( `Reusing description for repeated image ${imagePath}`, ) } else { - description = await describeImageWithllm(imageBuffer) + if(describeImages) { + description = await describeImageWithllm(imageBuffer) + } else { + description = "This is an image." + } if ( description === "No description returned." || description === "Image is not worth describing." diff --git a/server/integrations/dataSource/index.ts b/server/integrations/dataSource/index.ts index 1808b5122..21a65b443 100644 --- a/server/integrations/dataSource/index.ts +++ b/server/integrations/dataSource/index.ts @@ -93,7 +93,7 @@ const checkFileSize = (file: File, maxFileSizeMB: number): void => { } } -const createFileMetadata = ( +export const createFileMetadata = ( fileName: string, userEmail: string, chunksCount: number, diff --git a/server/pdfChunks.ts b/server/pdfChunks.ts index 440a12328..c8ab942b9 100644 --- a/server/pdfChunks.ts +++ b/server/pdfChunks.ts @@ -124,7 +124,8 @@ function processTextParagraphs( export async function extractTextAndImagesWithChunksFromPDF( data: Uint8Array, docid: string = crypto.randomUUID(), - extractImages: boolean = true, + extractImages: boolean = false, + describeImages: boolean = true, ): Promise<{ text_chunks: string[] image_chunks: string[] @@ -457,7 +458,11 @@ export async function extractTextAndImagesWithChunksFromPDF( `Reusing description for repeated image ${imageName} on page ${pageNum}`, ) } else { - description = await describeImageWithllm(buffer) + if(describeImages) { + description = await describeImageWithllm(buffer) + } else { + description = "This is an image." + } if ( description === "No description returned." || description === "Image is not worth describing." diff --git a/server/pptChunks.ts b/server/pptChunks.ts index 7d35d029e..be52842f5 100644 --- a/server/pptChunks.ts +++ b/server/pptChunks.ts @@ -648,6 +648,7 @@ export async function extractTextAndImagesWithChunksFromPptx( data: Uint8Array, docid: string = crypto.randomUUID(), extractImages: boolean = false, + describeImages: boolean = true, ): Promise { Logger.info(`Starting PPTX processing for document: ${docid}`) let totalTextLength = 0 @@ -861,7 +862,11 @@ export async function extractTextAndImagesWithChunksFromPptx( `Reusing description for repeated image ${imagePath} in slide ${slideNumber}`, ) } else { - description = await describeImageWithllm(imageBuffer) + if(describeImages) { + description = await describeImageWithllm(imageBuffer) + } else { + description = "This is an image." + } if ( description === "No description returned." || description === "Image is not worth describing." diff --git a/server/services/fileProcessor.ts b/server/services/fileProcessor.ts index b4ce497c5..4690ed5f7 100644 --- a/server/services/fileProcessor.ts +++ b/server/services/fileProcessor.ts @@ -25,7 +25,9 @@ export class FileProcessorService { mimeType: string, fileName: string, vespaDocId: string, - storagePath?: string + storagePath?: string, + extractImages: boolean = false, + describeImages: boolean = false, ): Promise { const baseMimeType = getBaseMimeType(mimeType || "text/plain") let chunks: string[] = [] @@ -39,7 +41,8 @@ export class FileProcessorService { const result = await extractTextAndImagesWithChunksFromPDF( new Uint8Array(buffer), vespaDocId, - false, + extractImages, + describeImages, ) chunks = result.text_chunks chunks_pos = result.text_chunk_pos @@ -50,7 +53,8 @@ export class FileProcessorService { const result = await extractTextAndImagesWithChunksFromDocx( new Uint8Array(buffer), vespaDocId, - false, + extractImages, + describeImages, ) chunks = result.text_chunks chunks_pos = result.text_chunk_pos @@ -61,7 +65,8 @@ export class FileProcessorService { const result = await extractTextAndImagesWithChunksFromPptx( new Uint8Array(buffer), vespaDocId, - false, + extractImages, + describeImages, ) chunks = result.text_chunks chunks_pos = result.text_chunk_pos @@ -69,10 +74,12 @@ export class FileProcessorService { image_chunks_pos = result.image_chunk_pos || [] } else if (isSheetFile(baseMimeType)) { // Process spreadsheet + let workbook: XLSX.WorkBook if (!storagePath) { - throw new Error("Storage path required for spreadsheet processing") + workbook = XLSX.read(buffer, { type: 'buffer' }) + } else { + workbook = XLSX.readFile(storagePath) } - const workbook = XLSX.readFile(storagePath) const allChunks: string[] = [] for (const sheetName of workbook.SheetNames) { From d1aece6868bacc63ef09438981447b9faece7602 Mon Sep 17 00:00:00 2001 From: Himansh Varma Date: Wed, 3 Sep 2025 23:31:28 +0530 Subject: [PATCH 2/8] fixed tests --- server/api/chat/agents.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/api/chat/agents.ts b/server/api/chat/agents.ts index d6ef63bf3..c051042b5 100644 --- a/server/api/chat/agents.ts +++ b/server/api/chat/agents.ts @@ -2809,8 +2809,6 @@ export const AgentMessageApiRagOff = async (c: Context) => { rootSpan.setAttribute("email", email) rootSpan.setAttribute("workspaceId", workspaceId) - // @ts-ignore - const body = c.req.valid("query") let { message, chatId, From 9390f01670b8b63b3a1f5b107b7079e1fb11b475 Mon Sep 17 00:00:00 2001 From: Himansh Varma Date: Wed, 3 Sep 2025 23:38:27 +0530 Subject: [PATCH 3/8] resolved ai comments --- frontend/src/components/ChatBox.tsx | 20 -------------------- server/api/chat/agents.ts | 16 ++++------------ server/api/chat/chat.ts | 12 +++--------- 3 files changed, 7 insertions(+), 41 deletions(-) diff --git a/frontend/src/components/ChatBox.tsx b/frontend/src/components/ChatBox.tsx index 6d1e28d9d..def2e25ba 100644 --- a/frontend/src/components/ChatBox.tsx +++ b/frontend/src/components/ChatBox.tsx @@ -637,26 +637,6 @@ export const ChatBox = React.forwardRef( return ext || "file" } - // Get appropriate icon for file type - const getFileIcon = (fileType: string | undefined) => { - switch (fileType) { - case "Image": - return - case "Document": - return - case "Spreadsheet": - return - case "Presentation": - return - case "PDF": - return - case "Text": - return - default: - return - } - } - const removeFile = useCallback((id: string) => { setSelectedFiles((prev) => { const fileToRemove = prev.find((f) => f.id === id) diff --git a/server/api/chat/agents.ts b/server/api/chat/agents.ts index c051042b5..7b836cab2 100644 --- a/server/api/chat/agents.ts +++ b/server/api/chat/agents.ts @@ -652,12 +652,8 @@ export const MessageWithToolsApi = async (c: Context) => { agentId, }: MessageReqType = body const attachmentMetadata = parseAttachmentMetadata(c) - const NonImageAttachmentFileIds = attachmentMetadata.map( - (m: AttachmentMetadata) => (m.isImage ? null : m.fileId), - ).filter((m: string | null) => m !== null) - const ImageAttachmentFileIds = attachmentMetadata.map( - (m: AttachmentMetadata) => (m.isImage ? m.fileId : null), - ).filter((m: string | null) => m !== null) + const ImageAttachmentFileIds = attachmentMetadata.filter(m => m.isImage).map(m => m.fileId) + const NonImageAttachmentFileIds = attachmentMetadata.filter(m => !m.isImage).map(m => m.fileId) const agentPromptValue = agentId && isCuid(agentId) ? agentId : undefined // const userRequestsReasoning = isReasoningEnabled // Addressed: Will be used below let attachmentStorageError: Error | null = null @@ -3471,12 +3467,8 @@ export const AgentMessageApi = async (c: Context) => { rootSpan.setAttribute("workspaceId", workspaceId) const attachmentMetadata = parseAttachmentMetadata(c) - const ImageAttachmentFileIds = attachmentMetadata.map( - (m: AttachmentMetadata) => (m.isImage ? m.fileId : null), - ).filter((m: string | null) => m !== null) - const NonImageAttachmentFileIds = attachmentMetadata.map( - (m: AttachmentMetadata) => (m.isImage ? null : m.fileId), - ).filter((m: string | null) => m !== null) + const ImageAttachmentFileIds = attachmentMetadata.filter(m => m.isImage).map(m => m.fileId) + const NonImageAttachmentFileIds = attachmentMetadata.filter(m => !m.isImage).map(m => m.fileId) let attachmentStorageError: Error | null = null let { message, diff --git a/server/api/chat/chat.ts b/server/api/chat/chat.ts index 486eadc1a..82b7ff89f 100644 --- a/server/api/chat/chat.ts +++ b/server/api/chat/chat.ts @@ -4030,12 +4030,8 @@ export const MessageApi = async (c: Context) => { return MessageWithToolsApi(c) } const attachmentMetadata = parseAttachmentMetadata(c) - const ImageAttachmentFileIds = attachmentMetadata.map( - (m: AttachmentMetadata) => m.isImage ? m.fileId : null, - ).filter((m: string | null) => m !== null) - const NonImageAttachmentFileIds = attachmentMetadata.map( - (m: AttachmentMetadata) => m.isImage ? null : m.fileId, - ).filter((m: string | null) => m !== null) + const ImageAttachmentFileIds = attachmentMetadata.filter(m => m.isImage).map(m => m.fileId) + const NonImageAttachmentFileIds = attachmentMetadata.filter(m => !m.isImage).map(m => m.fileId) if (agentPromptValue) { const userAndWorkspaceCheck = await getUserAndWorkspaceByEmail( @@ -5469,9 +5465,7 @@ export const MessageRetryApi = async (c: Context) => { if (isUserMessage) { // If retrying a user message, get attachments from that message attachmentMetadata = await getAttachmentsByMessageId(db, messageId, email) - ImageAttachmentFileIds = attachmentMetadata.map( - (m: AttachmentMetadata) => m.isImage ? m.fileId : null, - ).filter((m: string | null) => m !== null) + ImageAttachmentFileIds = attachmentMetadata.filter(m => m.isImage).map(m => m.fileId) } rootSpan.setAttribute("email", email) From 87b1ef2e8f40f525b36fa089ec2902f7ac62cc13 Mon Sep 17 00:00:00 2001 From: Himansh Varma Date: Thu, 4 Sep 2025 12:18:13 +0530 Subject: [PATCH 4/8] fixed docId bug in utils.ts --- server/api/chat/utils.ts | 48 +++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/server/api/chat/utils.ts b/server/api/chat/utils.ts index f1e67e3bc..8617160e4 100644 --- a/server/api/chat/utils.ts +++ b/server/api/chat/utils.ts @@ -318,31 +318,33 @@ export const extractImageFileNames = ( let imageContent = match[1].trim() try { if (imageContent) { - const docId = imageContent.substring(0, imageContent.lastIndexOf("_")) - // const docIndex = - // results?.findIndex((c) => (c.fields as any).docId === docId) || -1 - const docIndex = - results?.findIndex((c) => (c.fields as any).docId === docId) ?? -1 - - if (docIndex === -1) { - console.warn( - `No matching document found for docId: ${docId} in results for image content extraction.`, - ) - continue - } - - // Split by newlines and filter out empty strings - const fileNames = imageContent - .split("\n") + // Split by newlines and spaces to handle various formatting + const individualFileNames = imageContent + .split(/\s+/) .map((name) => name.trim()) .filter((name) => name.length > 0) - // Additional safety: split by spaces and filter out empty strings - // in case multiple filenames are on the same line - .flatMap((name) => - name.split(/\s+/).filter((part) => part.length > 0), - ) - .map((name) => `${docIndex}_${name}`) - imageFileNames.push(...fileNames) + + for (const fileName of individualFileNames) { + const lastUnderscoreIndex = fileName.lastIndexOf("_") + if (lastUnderscoreIndex === -1) { + console.warn(`Invalid image file name format: ${fileName}`) + continue + } + + const docId = fileName.substring(0, lastUnderscoreIndex) + + const docIndex = + results?.findIndex((c) => (c.fields as any).docId === docId) ?? -1 + + if (docIndex === -1) { + console.warn( + `No matching document found for docId: ${docId} in results for image content extraction.`, + ) + continue + } + + imageFileNames.push(`${docIndex}_${fileName}`) + } } } catch (error) { console.error( From bff31f651629ea76cf5dc254b5df2445eef0a699 Mon Sep 17 00:00:00 2001 From: Himansh Varma Date: Thu, 4 Sep 2025 14:44:49 +0530 Subject: [PATCH 5/8] now attachments will use kb schema --- server/api/chat/agents.ts | 6 ++++++ server/api/chat/chat.ts | 5 +++-- server/api/chat/utils.ts | 2 +- server/api/files.ts | 45 ++++++++++++++++++++++----------------- server/search/types.ts | 4 ++++ 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/server/api/chat/agents.ts b/server/api/chat/agents.ts index 7b836cab2..6a8578798 100644 --- a/server/api/chat/agents.ts +++ b/server/api/chat/agents.ts @@ -142,6 +142,7 @@ import { SystemEntity, VespaSearchResultsSchema, type VespaSearchResult, + KnowledgeBaseEntity, } from "@/search/types" import { APIError } from "openai" import { @@ -294,6 +295,11 @@ const checkAndYieldCitationsForAgent = async function* ( continue } + // we dont want citations for attachments in the chat + if(item.source.entity === KnowledgeBaseEntity.Attachment) { + continue + } + yield { citation: { index: citationIndex, diff --git a/server/api/chat/chat.ts b/server/api/chat/chat.ts index 82b7ff89f..e303f6d17 100644 --- a/server/api/chat/chat.ts +++ b/server/api/chat/chat.ts @@ -155,6 +155,7 @@ import { type VespaSearchResults, type VespaSearchResultsSchema, type VespaUser, + KnowledgeBaseEntity, } from "@/search/types" import { APIError } from "openai" import { SearchVespaThreads } from "@/search/vespa" @@ -436,8 +437,8 @@ const checkAndYieldCitations = async function* ( const item = results[citationIndex - baseIndex] if (item) { // TODO: fix this properly, empty citations making streaming broke - if (item.fields.sddocname === dataSourceFileSchema) { - // Skip datasource and collection files from citations + if (item.fields.sddocname === dataSourceFileSchema || item.fields.entity === KnowledgeBaseEntity.Attachment) { + // Skip datasource and attachment files from citations continue } yield { diff --git a/server/api/chat/utils.ts b/server/api/chat/utils.ts index 8617160e4..c38250593 100644 --- a/server/api/chat/utils.ts +++ b/server/api/chat/utils.ts @@ -436,7 +436,7 @@ export const searchToCitation = (result: VespaSearchResults): Citation => { title: clFields.fileName || "Collection File", url: `/cl/${clFields.clId}`, app: Apps.KnowledgeBase, - entity: SystemEntity.SystemInfo, + entity: clFields.entity, itemId: clFields.itemId, clId: clFields.clId, } diff --git a/server/api/files.ts b/server/api/files.ts index 9e7ae4e66..76e7fe989 100644 --- a/server/api/files.ts +++ b/server/api/files.ts @@ -22,9 +22,8 @@ import { isValidFile } from "../../shared/filesutils" import { generateThumbnail, isImageFile, getThumbnailPath } from "@/utils/image" import type { AttachmentMetadata } from "@/shared/types" import { FileProcessorService } from "@/services/fileProcessor" -import { Apps, dataSourceFileSchema, datasourceSchema } from "@/search/types" -import type { VespaDataSourceFile } from "@/search/types" -import { createFileMetadata } from "@/integrations/dataSource" +import { Apps, KbItemsSchema, KnowledgeBaseEntity } from "@/search/types" +import { getBaseMimeType } from "@/integrations/dataSource/config" const { JwtPayloadKey } = config const loggerWithChild = getLoggerWithChild(Subsystem.Api, { module: "newApps" }) @@ -264,30 +263,36 @@ export const handleAttachmentUpload = async (c: Context) => { const { chunks, chunks_pos, image_chunks, image_chunks_pos } = processingResult - const vespaDoc: VespaDataSourceFile = { + const vespaDoc = { docId: fileId, - description: `File: ${file.name} for Attachment`, - app: Apps.DataSource, + clId: "attachment", + itemId: fileId, fileName: file.name, - fileSize: file.size, + app: Apps.KnowledgeBase as const, + entity: KnowledgeBaseEntity.Attachment, + description: "", + storagePath: "", chunks: chunks, - image_chunks: image_chunks || [], - chunks_pos: chunks_pos || [], - image_chunks_pos: image_chunks_pos || [], - uploadedBy: email, - mimeType: file.type, + chunks_pos: chunks_pos, + image_chunks: image_chunks, + image_chunks_pos: image_chunks_pos, + metadata: JSON.stringify({ + originalFileName: file.name, + uploadedBy: email, + chunksCount: chunks.length, + imageChunksCount: image_chunks.length, + processingMethod: getBaseMimeType(file.type || "text/plain"), + lastModified: Date.now(), + }), + createdBy: email, + duration: 0, + mimeType: getBaseMimeType(file.type || "text/plain"), + fileSize: file.size, createdAt: Date.now(), updatedAt: Date.now(), - dataSourceRef: `id:${NAMESPACE}:${datasourceSchema}::${fileId}`, - metadata: createFileMetadata( - file.name, - email, - chunks.length, - "file", - ), } - await insert(vespaDoc, dataSourceFileSchema) + await insert(vespaDoc, KbItemsSchema) } // Create attachment metadata diff --git a/server/search/types.ts b/server/search/types.ts index 2d5834aca..1ae199cf1 100644 --- a/server/search/types.ts +++ b/server/search/types.ts @@ -177,6 +177,7 @@ export enum KnowledgeBaseEntity { Folder = "folder", // Folders within collections Collection = "collection", // Collections (main containers) KnowledgeBase = "knowledgebase", // Legacy alias for collection + Attachment = "attachment", } export const isMailAttachment = (entity: Entity): boolean => @@ -196,6 +197,7 @@ export const FileEntitySchema = z.nativeEnum(DriveEntity) export const MailEntitySchema = z.nativeEnum(MailEntity) export const MailAttachmentEntitySchema = z.nativeEnum(MailAttachmentEntity) export const EventEntitySchema = z.nativeEnum(CalendarEntity) +export const KnowledgeBaseEntitySchema = z.nativeEnum(KnowledgeBaseEntity) const NotionEntitySchema = z.nativeEnum(NotionEntity) @@ -225,6 +227,7 @@ export const entitySchema = z.union([ ChatEntitySchema, DataSourceEntitySchema, WebSearchEntitySchema, + KnowledgeBaseEntitySchema, ]) export type Entity = @@ -238,6 +241,7 @@ export type Entity = | SlackEntity | DataSourceEntity | WebSearchEntity + | KnowledgeBaseEntity export type WorkspaceEntity = DriveEntity From 978aee0a5315b26de708739eca481e6ac5319836 Mon Sep 17 00:00:00 2001 From: Himansh Varma Date: Mon, 8 Sep 2025 13:33:17 +0530 Subject: [PATCH 6/8] solved types --- frontend/src/components/ChatBox.tsx | 23 ++++----- frontend/src/components/FileUpload.tsx | 2 +- frontend/src/utils/fileUtils.ts | 65 ++++++++------------------ server/ai/provider/base.ts | 3 ++ server/ai/provider/bedrock.ts | 5 +- server/ai/provider/gemini.ts | 4 +- server/ai/provider/openai.ts | 17 ++++--- server/ai/provider/vertex_ai.ts | 5 +- server/api/chat/agents.ts | 12 ----- server/api/chat/chat.ts | 3 +- server/api/files.ts | 4 +- server/shared/fileUtils.ts | 46 ++++++++++++++++++ server/shared/types.ts | 52 +++++++++++++++++++++ server/utils/image.ts | 11 ----- shared/filesutils.ts | 52 --------------------- 15 files changed, 154 insertions(+), 150 deletions(-) create mode 100644 server/shared/fileUtils.ts delete mode 100644 shared/filesutils.ts diff --git a/frontend/src/components/ChatBox.tsx b/frontend/src/components/ChatBox.tsx index 356450dd0..9991828dc 100644 --- a/frontend/src/components/ChatBox.tsx +++ b/frontend/src/components/ChatBox.tsx @@ -47,6 +47,7 @@ import { UserRole, DataSourceEntity, AttachmentMetadata, + FileType, } from "shared/types" // Add SelectPublicAgent, PublicUser import { DropdownMenu, @@ -92,19 +93,19 @@ interface SelectedFile { fileType?: string } -export const getFileIcon = (fileType: string | undefined) => { +export const getFileIcon = (fileType: FileType | string | undefined) => { switch (fileType) { - case "Image": + case FileType.IMAGE: return - case "Document": + case FileType.DOCUMENT: return - case "Spreadsheet": + case FileType.SPREADSHEET: return - case "Presentation": + case FileType.PRESENTATION: return - case "PDF": + case FileType.PDF: return - case "Text": + case FileType.TEXT: return default: return @@ -733,7 +734,7 @@ export const ChatBox = React.forwardRef( id: generateFileId(), uploading: false, preview: createImagePreview(file), - fileType: getFileType(file), + fileType: getFileType({ type: file.type, name: file.name }), })) setSelectedFiles((prev) => { @@ -2372,8 +2373,8 @@ export const ChatBox = React.forwardRef( if (isValid.length > 0) { // Process the pasted file processFiles([file]) - const fileType = getFileType(file) - + const fileType = getFileType({ type: file.type, name: file.name }) + showToast( "File pasted", `${fileType} has been added to your message.`, @@ -3790,4 +3791,4 @@ export const ChatBox = React.forwardRef(
) }, -) \ No newline at end of file +) diff --git a/frontend/src/components/FileUpload.tsx b/frontend/src/components/FileUpload.tsx index 07c8456a0..7b4131f5f 100644 --- a/frontend/src/components/FileUpload.tsx +++ b/frontend/src/components/FileUpload.tsx @@ -12,7 +12,7 @@ import { createImagePreview, cleanupPreviewUrls, } from "@/utils/fileUtils" -import { isValidFile } from "../../../shared/filesutils" +import { isValidFile } from "shared/fileUtils" import { authFetch } from "@/utils/authFetch" interface SelectedFile { diff --git a/frontend/src/utils/fileUtils.ts b/frontend/src/utils/fileUtils.ts index fa9ec7680..20b53b7ee 100644 --- a/frontend/src/utils/fileUtils.ts +++ b/frontend/src/utils/fileUtils.ts @@ -1,62 +1,37 @@ -import { isValidFile } from "../../../shared/filesutils" +import { isValidFile, isImageFile } from "shared/fileUtils" import { SelectedFile } from "@/components/ClFileUpload" import { authFetch } from "./authFetch" +import { FileType, MIME_TYPE_MAPPINGS, EXTENSION_MAPPINGS } from "shared/types" // Generate unique ID for files export const generateFileId = () => Math.random().toString(36).substring(2, 9) -// Check if file is an image -const isImageFile = (file: File): boolean => { - return ( - file.type.startsWith("image/") && - [ - "image/jpeg", - "image/jpg", - "image/png", - "image/gif", - "image/webp", - ].includes(file.type) - ) -} +export const getFileType = ({ type, name }: { type: string, name: string }): FileType => { + const fileName = name.toLowerCase() + const mimeType = type.toLowerCase() -// Get file type category for display purposes -export const getFileType = (file: File | { type: string, name: string }): string => { - if (file instanceof File && isImageFile(file)) { - return "Image" - } - - // Check for document types - if (file.type.includes("word") || file.name.toLowerCase().match(/\.(doc|docx)$/)) { - return "Document" - } - - // Check for spreadsheet types - if (file.type.includes("excel") || file.type.includes("spreadsheet") || file.name.toLowerCase().match(/\.(xls|xlsx|csv)$/)) { - return "Spreadsheet" - } - - // Check for presentation types - if (file.type.includes("powerpoint") || file.type.includes("presentation") || file.name.toLowerCase().match(/\.(ppt|pptx)$/)) { - return "Presentation" - } - - // Check for PDF - if (file.type === "application/pdf" || file.name.toLowerCase().endsWith(".pdf")) { - return "PDF" + // Check each file type category using the mappings + for (const [fileType, mimeTypes] of Object.entries(MIME_TYPE_MAPPINGS)) { + // Check MIME type first (more reliable) + if (mimeTypes.some(mime => mimeType === mime || mimeType.includes(mime))) { + return fileType as FileType + } } - - // Check for text files - if (file.type.startsWith("text/") || file.name.toLowerCase().match(/\.(txt|md)$/)) { - return "Text" + + // Fallback to extension-based detection + for (const [fileType, extensions] of Object.entries(EXTENSION_MAPPINGS)) { + if (extensions.some(ext => fileName.endsWith(ext))) { + return fileType as FileType + } } - + // Default fallback - return "File" + return FileType.FILE } // Create preview URL for image files export const createImagePreview = (file: File): string | undefined => { - if (isImageFile(file)) { + if (isImageFile(file.type)) { return URL.createObjectURL(file) } return undefined diff --git a/server/ai/provider/base.ts b/server/ai/provider/base.ts index 16ae8e982..10368b2cf 100644 --- a/server/ai/provider/base.ts +++ b/server/ai/provider/base.ts @@ -44,6 +44,9 @@ abstract class Provider implements LLMProvider { ): AsyncIterableIterator } +// format: docIndex_docId_imageNumber +export const regex = /^([0-9]+)_(.+)_([0-9]+)$/ + export function findImageByName(directory: string, imageName: string) { const files = fs.readdirSync(directory) const match = files.find((file) => path.parse(file).name === imageName) diff --git a/server/ai/provider/bedrock.ts b/server/ai/provider/bedrock.ts index 86b018754..59afc9bdc 100644 --- a/server/ai/provider/bedrock.ts +++ b/server/ai/provider/bedrock.ts @@ -8,7 +8,7 @@ import { import { modelDetailsMap } from "@/ai/mappers" import type { ConverseResponse, ModelParams } from "@/ai/types" import { AIProviders, Models } from "@/ai/types" -import BaseProvider from "@/ai/provider/base" +import BaseProvider, { regex } from "@/ai/provider/base" import { calculateCost } from "@/utils/index" import { getLogger } from "@/logger" import { Subsystem } from "@/types" @@ -30,8 +30,7 @@ const buildBedrockImageParts = async ( ) const imagePromises = imagePaths.map(async (imgPath) => { - // format: docIndex_docId_imageNumber - const match = imgPath.match(/^([0-9]+)_(.+)_([0-9]+)$/) + const match = imgPath.match(regex) if (!match) { Logger.error( `Invalid image path format: ${imgPath}. Expected format: docIndex_docId_imageNumber`, diff --git a/server/ai/provider/gemini.ts b/server/ai/provider/gemini.ts index 36aa42680..892d1bd0c 100644 --- a/server/ai/provider/gemini.ts +++ b/server/ai/provider/gemini.ts @@ -4,7 +4,7 @@ import { type GenerateContentConfig, type ThinkingConfig, } from "@google/genai" -import BaseProvider from "@/ai/provider/base" +import BaseProvider, { regex } from "@/ai/provider/base" import type { Message } from "@aws-sdk/client-bedrock-runtime" import { type ModelParams, @@ -31,7 +31,7 @@ async function buildGeminiImageParts( const imagePromises = imagePaths.map(async (imgPath) => { // Check if the file already has an extension, if not add .png - const match = imgPath.match(/^([0-9]+)_(.+)_([0-9]+)$/) + const match = imgPath.match(regex) if (!match) { Logger.error(`Invalid image path: ${imgPath}`) throw new Error(`Invalid image path: ${imgPath}`) diff --git a/server/ai/provider/openai.ts b/server/ai/provider/openai.ts index a362ca162..87925fe40 100644 --- a/server/ai/provider/openai.ts +++ b/server/ai/provider/openai.ts @@ -3,7 +3,7 @@ import OpenAI from "openai" import { isDeepResearchModel, modelDetailsMap } from "@/ai/mappers" import type { ConverseResponse, ModelParams } from "@/ai/types" import { AIProviders, Models } from "@/ai/types" -import BaseProvider from "@/ai/provider/base" +import BaseProvider, { regex } from "@/ai/provider/base" import { calculateCost } from "@/utils/index" import { getLogger } from "@/logger" import { Subsystem } from "@/types" @@ -20,22 +20,25 @@ const buildOpenAIImageParts = async (imagePaths: string[]) => { ) const imagePromises = imagePaths.map(async (imgPath) => { - // Check if the file already has an extension, if not add .png - const match = imgPath.match(/^(.+)_([0-9]+)$/) + const match = imgPath.match(regex) if (!match) { - Logger.error(`Invalid image path: ${imgPath}`) + Logger.error( + `Invalid image path format: ${imgPath}. Expected format: docIndex_docId_imageNumber`, + ) throw new Error(`Invalid image path: ${imgPath}`) } - // Validate that the docId doesn't contain path traversal characters - const docId = match[1] + const docIndex = match[1] + const docId = match[2] + const imageNumber = match[3] + if (docId.includes("..") || docId.includes("/") || docId.includes("\\")) { Logger.error(`Invalid docId containing path traversal: ${docId}`) throw new Error(`Invalid docId: ${docId}`) } const imageDir = path.join(baseDir, docId) - const absolutePath = findImageByName(imageDir, match[2]) + const absolutePath = findImageByName(imageDir, imageNumber) const extension = path.extname(absolutePath).toLowerCase() // Map file extensions to MIME types for OpenAI diff --git a/server/ai/provider/vertex_ai.ts b/server/ai/provider/vertex_ai.ts index 9f320ca73..68facf359 100644 --- a/server/ai/provider/vertex_ai.ts +++ b/server/ai/provider/vertex_ai.ts @@ -12,7 +12,7 @@ import { type ModelParams, type WebSearchSource, } from "@/ai/types" -import BaseProvider, { findImageByName } from "@/ai/provider/base" +import BaseProvider, { findImageByName, regex } from "@/ai/provider/base" import { Subsystem } from "@/types" import config from "@/config" import { createLabeledImageContent } from "../utils" @@ -32,8 +32,7 @@ const buildVertexAIImageParts = async (imagePaths: string[]) => { ) const imagePromises = imagePaths.map(async (imgPath) => { - // format: docIndex_docId_imageNumber - const match = imgPath.match(/^([0-9]+)_(.+)_([0-9]+)$/) + const match = imgPath.match(regex) if (!match) { Logger.error( `Invalid image path format: ${imgPath}. Expected format: docIndex_docId_imageNumber`, diff --git a/server/api/chat/agents.ts b/server/api/chat/agents.ts index 0a1bdff5e..7fafbbe48 100644 --- a/server/api/chat/agents.ts +++ b/server/api/chat/agents.ts @@ -1489,18 +1489,6 @@ export const MessageWithToolsApi = async (c: Context) => { } } planningContext = cleanContext(resolvedContexts?.join("\n")) - const { imageFileNames } = extractImageFileNames( - planningContext, - [...results.root.children, ...chatContexts, ...threadContexts], - ) - - const finalImageFileNames = imageFileNames || [] - - if (ImageAttachmentFileIds?.length) { - finalImageFileNames.push( - ...ImageAttachmentFileIds.map((fileid, index) => `${index}_${fileid}_${0}`), - ) - } if (chatContexts.length > 0) { planningContext += "\n" + buildContext(chatContexts, 10) } diff --git a/server/api/chat/chat.ts b/server/api/chat/chat.ts index 2e248289d..00af654d9 100644 --- a/server/api/chat/chat.ts +++ b/server/api/chat/chat.ts @@ -178,7 +178,7 @@ import { } from "@/db/attachment" import type { AttachmentMetadata } from "@/shared/types" import { parseAttachmentMetadata } from "@/utils/parseAttachment" -import { isImageFile } from "@/utils/image" +import { isImageFile } from "shared/fileUtils" import { promises as fs } from "node:fs" import path from "node:path" import { @@ -2170,6 +2170,7 @@ async function* generateAnswerFromGivenContext( yield { text: "From the selected context, I could not find any information to answer it, please change your query", } + generateAnswerSpan?.end() return } // If we give the whole context then also if there's no answer then we can just search once and get the best matching chunks with the query and then make context try answering diff --git a/server/api/files.ts b/server/api/files.ts index 76e7fe989..4d8f81fd7 100644 --- a/server/api/files.ts +++ b/server/api/files.ts @@ -18,8 +18,8 @@ import { import { NoUserFound } from "@/errors" import config from "@/config" import { HTTPException } from "hono/http-exception" -import { isValidFile } from "../../shared/filesutils" -import { generateThumbnail, isImageFile, getThumbnailPath } from "@/utils/image" +import { isValidFile, isImageFile } from "shared/fileUtils" +import { generateThumbnail, getThumbnailPath } from "@/utils/image" import type { AttachmentMetadata } from "@/shared/types" import { FileProcessorService } from "@/services/fileProcessor" import { Apps, KbItemsSchema, KnowledgeBaseEntity } from "@/search/types" diff --git a/server/shared/fileUtils.ts b/server/shared/fileUtils.ts new file mode 100644 index 000000000..de0040837 --- /dev/null +++ b/server/shared/fileUtils.ts @@ -0,0 +1,46 @@ +import { MIME_TYPE_MAPPINGS, EXTENSION_MAPPINGS, FileType } from "./types" + +// Check if file is an image +export const isImageFile = (fileType: string): boolean => { + return ( + (MIME_TYPE_MAPPINGS[FileType.IMAGE] as readonly string[]).includes(fileType) + ) +} + +export const isValidFile = (file: File) => { + // Set size limits + const maxGeneralSize = 40 * 1024 * 1024 // 40MB + const maxImageSize = 5 * 1024 * 1024 // 5MB + + // Get all allowed MIME types from the centralized mappings + const allowedMimeTypes = [ + ...MIME_TYPE_MAPPINGS[FileType.TEXT], + ...MIME_TYPE_MAPPINGS[FileType.PDF], + ...MIME_TYPE_MAPPINGS[FileType.DOCUMENT], + ...MIME_TYPE_MAPPINGS[FileType.SPREADSHEET], + ...MIME_TYPE_MAPPINGS[FileType.PRESENTATION], + ...MIME_TYPE_MAPPINGS[FileType.IMAGE], + ] as readonly string[] + + // Get all allowed extensions from the centralized mappings + const allowedExtensions = [ + ...EXTENSION_MAPPINGS[FileType.TEXT], + ...EXTENSION_MAPPINGS[FileType.PDF], + ...EXTENSION_MAPPINGS[FileType.DOCUMENT], + ...EXTENSION_MAPPINGS[FileType.SPREADSHEET], + ...EXTENSION_MAPPINGS[FileType.PRESENTATION], + ...EXTENSION_MAPPINGS[FileType.IMAGE], + ] as readonly string[] + + // Check if file is an image using the centralized mapping + const isImage = (MIME_TYPE_MAPPINGS[FileType.IMAGE] as readonly string[]).includes(file.type) + + // Check if file type is allowed by MIME type or extension + const isAllowedType = + allowedMimeTypes.includes(file.type) || + allowedExtensions.some((ext) => file.name.toLowerCase().endsWith(ext)) + + const sizeLimit = isImage ? maxImageSize : maxGeneralSize + + return file.size <= sizeLimit && isAllowedType +} diff --git a/server/shared/types.ts b/server/shared/types.ts index a4a3c1bdf..5fa2f159e 100644 --- a/server/shared/types.ts +++ b/server/shared/types.ts @@ -109,6 +109,58 @@ export enum OpenAIError { InvalidAPIKey = "invalid_api_key", } +// File type categories enum for better type safety and consistency +export enum FileType { + IMAGE = "Image", + DOCUMENT = "Document", + SPREADSHEET = "Spreadsheet", + PRESENTATION = "Presentation", + PDF = "PDF", + TEXT = "Text", + FILE = "File" // Default fallback +} + +// MIME type mappings for better organization +export const MIME_TYPE_MAPPINGS = { + [FileType.IMAGE]: [ + "image/jpeg", + "image/jpg", + "image/png", + "image/gif", + "image/webp" + ], + [FileType.DOCUMENT]: [ + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + ], + [FileType.SPREADSHEET]: [ + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "text/csv" + ], + [FileType.PRESENTATION]: [ + "application/vnd.ms-powerpoint", + "application/vnd.openxmlformats-officedocument.presentationml.presentation" + ], + [FileType.PDF]: [ + "application/pdf" + ], + [FileType.TEXT]: [ + "text/plain", + "text/markdown" + ] +} as const; + +// File extension mappings for fallback detection +export const EXTENSION_MAPPINGS = { + [FileType.IMAGE]: [".jpg", ".jpeg", ".png", ".gif", ".webp"], + [FileType.DOCUMENT]: [".doc", ".docx"], + [FileType.SPREADSHEET]: [".xls", ".xlsx", ".csv"], + [FileType.PRESENTATION]: [".ppt", ".pptx"], + [FileType.PDF]: [".pdf"], + [FileType.TEXT]: [".txt", ".md"] +} as const; + export const AutocompleteFileSchema = z .object({ type: z.literal(fileSchema), diff --git a/server/utils/image.ts b/server/utils/image.ts index 5050ed3a7..fef0c7f26 100644 --- a/server/utils/image.ts +++ b/server/utils/image.ts @@ -46,17 +46,6 @@ export const generateThumbnail = async ( } } -export const isImageFile = (mimeType: string): boolean => { - const imageTypes = [ - "image/jpeg", - "image/jpg", - "image/png", - "image/gif", - "image/webp", - ] - return imageTypes.includes(mimeType) -} - export const getThumbnailPath = (baseDir: string, fileId: string): string => { return path.join(baseDir, `${fileId}_thumbnail.jpeg`) } diff --git a/shared/filesutils.ts b/shared/filesutils.ts deleted file mode 100644 index f0611ec94..000000000 --- a/shared/filesutils.ts +++ /dev/null @@ -1,52 +0,0 @@ -export const isValidFile = (file: File) => { - // Set size limits - const maxGeneralSize = 40 * 1024 * 1024 // 40MB - const maxImageSize = 5 * 1024 * 1024 // 5MB - - // Allowed MIME types - const allowedTypes = [ - "text/plain", - "text/csv", - "application/pdf", - "application/msword", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "application/vnd.ms-excel", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "application/vnd.ms-powerpoint", - "application/vnd.openxmlformats-officedocument.presentationml.presentation", - "text/markdown", - ] - - // Allowed extensions (for fallback) - const allowedExtensions = [ - ".txt", - ".csv", - ".pdf", - ".doc", - ".docx", - ".xls", - ".xlsx", - ".ppt", - ".pptx", - ".md", - ] - - const allowedImageTypes = [ - "image/jpeg", - "image/jpg", - "image/png", - "image/gif", - "image/webp", - ] - - // Check by MIME type or extension - const isImage = allowedImageTypes.includes(file.type) - const isAllowedType = - allowedTypes.includes(file.type) || - allowedExtensions.some((ext) => file.name.toLowerCase().endsWith(ext)) || - isImage - - const sizeLimit = isImage ? maxImageSize : maxGeneralSize - - return file.size <= sizeLimit && isAllowedType -} From b16b461065187439db6067981feeba8f365027db Mon Sep 17 00:00:00 2001 From: Himansh Varma Date: Mon, 8 Sep 2025 14:14:51 +0530 Subject: [PATCH 7/8] solved ai comments --- frontend/src/components/ChatBox.tsx | 2 +- frontend/src/utils/fileUtils.ts | 3 ++- server/api/chat/agents.ts | 34 ++++++++++++++--------------- server/api/chat/chat.ts | 18 ++++++++------- 4 files changed, 30 insertions(+), 27 deletions(-) diff --git a/frontend/src/components/ChatBox.tsx b/frontend/src/components/ChatBox.tsx index 9991828dc..8e1cd6cff 100644 --- a/frontend/src/components/ChatBox.tsx +++ b/frontend/src/components/ChatBox.tsx @@ -90,7 +90,7 @@ interface SelectedFile { uploading?: boolean uploadError?: string preview?: string // URL for image preview - fileType?: string + fileType?: FileType } export const getFileIcon = (fileType: FileType | string | undefined) => { diff --git a/frontend/src/utils/fileUtils.ts b/frontend/src/utils/fileUtils.ts index 20b53b7ee..d9594009c 100644 --- a/frontend/src/utils/fileUtils.ts +++ b/frontend/src/utils/fileUtils.ts @@ -9,11 +9,12 @@ export const generateFileId = () => Math.random().toString(36).substring(2, 9) export const getFileType = ({ type, name }: { type: string, name: string }): FileType => { const fileName = name.toLowerCase() const mimeType = type.toLowerCase() + const baseMime = mimeType.split(";")[0] // Check each file type category using the mappings for (const [fileType, mimeTypes] of Object.entries(MIME_TYPE_MAPPINGS)) { // Check MIME type first (more reliable) - if (mimeTypes.some(mime => mimeType === mime || mimeType.includes(mime))) { + if (mimeTypes.some(mime => baseMime === mime)) { return fileType as FileType } } diff --git a/server/api/chat/agents.ts b/server/api/chat/agents.ts index 7fafbbe48..48329f2d1 100644 --- a/server/api/chat/agents.ts +++ b/server/api/chat/agents.ts @@ -741,8 +741,8 @@ export const MessageWithToolsApi = async (c: Context) => { } const attachmentMetadata = parseAttachmentMetadata(c) - const ImageAttachmentFileIds = attachmentMetadata.filter(m => m.isImage).map(m => m.fileId) - const NonImageAttachmentFileIds = attachmentMetadata.filter(m => !m.isImage).map(m => m.fileId) + const imageAttachmentFileIds = attachmentMetadata.filter(m => m.isImage).map(m => m.fileId) + const nonImageAttachmentFileIds = attachmentMetadata.filter(m => !m.isImage).map(m => m.fileId) const agentPromptValue = agentId && isCuid(agentId) ? agentId : undefined // const userRequestsReasoning = isReasoningEnabled // Addressed: Will be used below let attachmentStorageError: Error | null = null @@ -754,8 +754,8 @@ export const MessageWithToolsApi = async (c: Context) => { fileIds: [], } let fileIds = extractedInfo?.fileIds - if (NonImageAttachmentFileIds && NonImageAttachmentFileIds.length > 0) { - fileIds = [...fileIds, ...NonImageAttachmentFileIds] + if (nonImageAttachmentFileIds && nonImageAttachmentFileIds.length > 0) { + fileIds = [...fileIds, ...nonImageAttachmentFileIds] } const totalValidFileIdsFromLinkCount = extractedInfo?.totalValidFileIdsFromLinkCount @@ -1522,7 +1522,7 @@ export const MessageWithToolsApi = async (c: Context) => { messagesWithNoErrResponse, logAndStreamReasoning, sub, - ImageAttachmentFileIds, + imageAttachmentFileIds, actualModelId || undefined, ) await logAndStreamReasoning({ @@ -2115,7 +2115,7 @@ export const MessageWithToolsApi = async (c: Context) => { messagesWithNoErrResponse, logAndStreamReasoning, sub, - ImageAttachmentFileIds, + imageAttachmentFileIds, actualModelId || undefined, ) @@ -2303,7 +2303,7 @@ export const MessageWithToolsApi = async (c: Context) => { messagesWithNoErrResponse, logAndStreamReasoning, sub, - ImageAttachmentFileIds, + imageAttachmentFileIds, actualModelId || undefined, ) await logAndStreamReasoning({ @@ -2349,7 +2349,7 @@ export const MessageWithToolsApi = async (c: Context) => { agentPromptForLLM, messagesWithNoErrResponse, fallbackReasoning, - ImageAttachmentFileIds, + imageAttachmentFileIds, email, actualModelId || undefined, ) @@ -3588,8 +3588,8 @@ export const AgentMessageApi = async (c: Context) => { rootSpan.setAttribute("workspaceId", workspaceId) const attachmentMetadata = parseAttachmentMetadata(c) - const ImageAttachmentFileIds = attachmentMetadata.filter(m => m.isImage).map(m => m.fileId) - const NonImageAttachmentFileIds = attachmentMetadata.filter(m => !m.isImage).map(m => m.fileId) + const imageAttachmentFileIds = attachmentMetadata.filter(m => m.isImage).map(m => m.fileId) + const nonImageAttachmentFileIds = attachmentMetadata.filter(m => !m.isImage).map(m => m.fileId) let attachmentStorageError: Error | null = null let { message, @@ -3726,10 +3726,10 @@ export const AgentMessageApi = async (c: Context) => { totalValidFileIdsFromLinkCount: 0, fileIds: [], } - isMsgWithContext = isMsgWithContext || (NonImageAttachmentFileIds && NonImageAttachmentFileIds.length > 0) + isMsgWithContext = isMsgWithContext || (nonImageAttachmentFileIds && nonImageAttachmentFileIds.length > 0) let fileIds = extractedInfo?.fileIds - if (NonImageAttachmentFileIds && NonImageAttachmentFileIds.length > 0) { - fileIds = [...fileIds, ...NonImageAttachmentFileIds] + if (nonImageAttachmentFileIds && nonImageAttachmentFileIds.length > 0) { + fileIds = [...fileIds, ...nonImageAttachmentFileIds] } const agentDocs = agentForDb?.docIds || [] @@ -3922,7 +3922,7 @@ export const AgentMessageApi = async (c: Context) => { if ( (isMsgWithContext && fileIds && fileIds?.length > 0) || - (ImageAttachmentFileIds && ImageAttachmentFileIds?.length > 0) + (imageAttachmentFileIds && imageAttachmentFileIds?.length > 0) ) { Logger.info( "User has selected some context with query, answering only based on that given context", @@ -3954,7 +3954,7 @@ export const AgentMessageApi = async (c: Context) => { userRequestsReasoning, understandSpan, [], - ImageAttachmentFileIds, + imageAttachmentFileIds, agentPromptForLLM, ) stream.writeSSE({ @@ -4925,7 +4925,7 @@ export const AgentMessageApi = async (c: Context) => { // Path A: user provided explicit context (fileIds / attachments) if ( (isMsgWithContext && fileIds && fileIds.length > 0) || - (ImageAttachmentFileIds && ImageAttachmentFileIds.length > 0) + (imageAttachmentFileIds && imageAttachmentFileIds.length > 0) ) { const ragSpan = streamSpan.startSpan("rag_processing") const understandSpan = ragSpan.startSpan("understand_message") @@ -4939,7 +4939,7 @@ export const AgentMessageApi = async (c: Context) => { userRequestsReasoning, understandSpan, [], - ImageAttachmentFileIds, + imageAttachmentFileIds, agentPromptForLLM, ) diff --git a/server/api/chat/chat.ts b/server/api/chat/chat.ts index 00af654d9..52c2ae605 100644 --- a/server/api/chat/chat.ts +++ b/server/api/chat/chat.ts @@ -462,7 +462,8 @@ const checkAndYieldCitations = async function* ( const item = results[citationIndex - baseIndex] if (item) { // TODO: fix this properly, empty citations making streaming broke - if (item.fields.sddocname === dataSourceFileSchema || item.fields.entity === KnowledgeBaseEntity.Attachment) { + const f = (item as any)?.fields + if (f?.sddocname === dataSourceFileSchema || f?.entity === KnowledgeBaseEntity.Attachment) { // Skip datasource and attachment files from citations continue } @@ -2170,6 +2171,7 @@ async function* generateAnswerFromGivenContext( yield { text: "From the selected context, I could not find any information to answer it, please change your query", } + generateAnswerSpan?.setAttribute("answer_found", false) generateAnswerSpan?.end() return } @@ -4067,8 +4069,8 @@ export const MessageApi = async (c: Context) => { return MessageWithToolsApi(c) } const attachmentMetadata = parseAttachmentMetadata(c) - const ImageAttachmentFileIds = attachmentMetadata.filter(m => m.isImage).map(m => m.fileId) - const NonImageAttachmentFileIds = attachmentMetadata.filter(m => !m.isImage).map(m => m.fileId) + const imageAttachmentFileIds = attachmentMetadata.filter(m => m.isImage).map(m => m.fileId) + const nonImageAttachmentFileIds = attachmentMetadata.filter(m => !m.isImage).map(m => m.fileId) if (agentPromptValue) { const userAndWorkspaceCheck = await getUserAndWorkspaceByEmail( @@ -4110,10 +4112,10 @@ export const MessageApi = async (c: Context) => { fileIds: [], threadIds: [], } - isMsgWithContext = isMsgWithContext || (NonImageAttachmentFileIds && NonImageAttachmentFileIds.length > 0) + isMsgWithContext = isMsgWithContext || (nonImageAttachmentFileIds && nonImageAttachmentFileIds.length > 0) let fileIds = extractedInfo?.fileIds - if (NonImageAttachmentFileIds && NonImageAttachmentFileIds.length > 0) { - fileIds = fileIds.concat(NonImageAttachmentFileIds) + if (nonImageAttachmentFileIds && nonImageAttachmentFileIds.length > 0) { + fileIds = fileIds.concat(nonImageAttachmentFileIds) } const threadIds = extractedInfo?.threadIds || [] const totalValidFileIdsFromLinkCount = @@ -4325,7 +4327,7 @@ export const MessageApi = async (c: Context) => { } if ( (isMsgWithContext && fileIds && fileIds?.length > 0) || - (ImageAttachmentFileIds && ImageAttachmentFileIds?.length > 0) + (imageAttachmentFileIds && imageAttachmentFileIds?.length > 0) ) { let answer = "" let citations = [] @@ -4352,7 +4354,7 @@ export const MessageApi = async (c: Context) => { userRequestsReasoning, understandSpan, threadIds, - ImageAttachmentFileIds, + imageAttachmentFileIds, agentPromptValue, actualModelId || config.defaultBestModel, ) From 347d16f74ed7f267a5b70d82ce4c16def4d786ee Mon Sep 17 00:00:00 2001 From: Himansh Varma Date: Mon, 8 Sep 2025 14:49:32 +0530 Subject: [PATCH 8/8] added attachment deletion logic for chat delete --- server/api/chat/chat.ts | 43 ++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/server/api/chat/chat.ts b/server/api/chat/chat.ts index 52c2ae605..4edcd054b 100644 --- a/server/api/chat/chat.ts +++ b/server/api/chat/chat.ts @@ -93,6 +93,7 @@ import { searchVespaAgent, GetDocument, SearchEmailThreads, + DeleteDocument, } from "@/search/vespa" import { Apps, @@ -119,6 +120,7 @@ import { type VespaSearchResults, type VespaSearchResultsSchema, KnowledgeBaseEntity, + KbItemsSchema, } from "@/search/types" import { APIError } from "openai" import { SearchVespaThreads } from "@/search/vespa" @@ -763,11 +765,7 @@ export const ChatDeleteApi = async (c: Context) => { if (isImageAttachment) { imageAttachmentFileIds.push(attachment.fileId) } else { - // TODO: Handle non-image attachments in future implementation nonImageAttachmentFileIds.push(attachment.fileId) - loggerWithChild({ email: email }).info( - `Non-image attachment ${attachment.fileId} (${attachment.fileType}) found - TODO: implement deletion logic for non-image attachments`, - ) } } } @@ -819,20 +817,33 @@ export const ChatDeleteApi = async (c: Context) => { } } - // TODO: Implement deletion logic for non-image attachments + // Delete non-image attachments from Vespa kb_items schema if (nonImageAttachmentFileIds.length > 0) { loggerWithChild({ email: email }).info( - `Found ${nonImageAttachmentFileIds.length} non-image attachments that need deletion logic implementation`, + `Deleting ${nonImageAttachmentFileIds.length} non-image attachments from Vespa kb_items schema for chat ${chatId}`, ) - // TODO: Add specific deletion logic for different types of non-image attachments - // This could include: - // - PDFs: Delete from document storage directories - // - Documents (DOCX, DOC): Delete from document storage directories - // - Spreadsheets (XLSX, XLS): Delete from document storage directories - // - Presentations (PPTX, PPT): Delete from document storage directories - // - Text files: Delete from text storage directories - // - Other file types: Implement based on file type and storage location - // For now, we just log that we found them but don't delete them to avoid data loss + + for (const fileId of nonImageAttachmentFileIds) { + try { + // Delete from Vespa kb_items schema using the proper Vespa function + await DeleteDocument(fileId, KbItemsSchema) + loggerWithChild({ email: email }).info( + `Successfully deleted non-image attachment ${fileId} from Vespa kb_items schema`, + ) + } catch (error) { + const errorMessage = getErrorMessage(error) + if (errorMessage.includes("404 Not Found")) { + loggerWithChild({ email: email }).warn( + `Non-image attachment ${fileId} not found in Vespa kb_items schema (may have been already deleted)`, + ) + } else { + loggerWithChild({ email: email }).error( + error, + `Failed to delete non-image attachment ${fileId} from Vespa kb_items schema: ${errorMessage}`, + ) + } + } + } } // Delete shared chats associated with this chat @@ -7066,4 +7077,4 @@ export const GetAvailableModelsApi = async (c: Context) => { message: "Could not fetch available models" }) } -} \ No newline at end of file +}