Skip to content

Commit 73db81e

Browse files
mishig25mishig25
committed
Implement PDF-chat feature
1 parent f8d73d1 commit 73db81e

29 files changed

+882
-68
lines changed

‎package-lock.json

+366-5
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
"nanoid": "^4.0.2",
6262
"openid-client": "^5.4.2",
6363
"parquetjs": "^0.11.2",
64+
"pdfjs-dist": "^4.0.269",
6465
"postcss": "^8.4.31",
6566
"serpapi": "^1.1.1",
6667
"tailwind-scrollbar": "^3.0.0",

‎src/lib/buildPrompt.ts

+30-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ import type { BackendModel } from "./server/models";
22
import type { Message } from "./types/Message";
33
import { format } from "date-fns";
44
import type { WebSearch } from "./types/WebSearch";
5-
import { downloadFile } from "./server/files/downloadFile";
5+
import type { PdfSearch } from "./types/PdfChat";
6+
import { downloadImgFile } from "./server/files/downloadFile";
67
import type { Conversation } from "./types/Conversation";
78

89
interface buildPromptOptions {
@@ -11,6 +12,7 @@ interface buildPromptOptions {
1112
model: BackendModel;
1213
locals?: App.Locals;
1314
webSearch?: WebSearch;
15+
pdfSearch?: PdfSearch;
1416
preprompt?: string;
1517
files?: File[];
1618
}
@@ -19,6 +21,7 @@ export async function buildPrompt({
1921
messages,
2022
model,
2123
webSearch,
24+
pdfSearch,
2225
preprompt,
2326
id,
2427
}: buildPromptOptions): Promise<string> {
@@ -47,6 +50,31 @@ export async function buildPrompt({
4750
`,
4851
},
4952
];
53+
}else if (pdfSearch && pdfSearch.context) {
54+
const lastMsg = messages.slice(-1)[0];
55+
const messagesWithoutLastUsrMsg = messages.slice(0, -1);
56+
const previousUserMessages = messages.filter((el) => el.from === "user").slice(0, -1);
57+
58+
const previousQuestions =
59+
previousUserMessages.length > 0
60+
? `Previous questions: \n${previousUserMessages
61+
.map(({ content }) => `- ${content}`)
62+
.join("\n")}`
63+
: "";
64+
65+
messages = [
66+
...messagesWithoutLastUsrMsg,
67+
{
68+
from: "user",
69+
content: `Below are the information I extracted from a PDF file that might be useful:
70+
=====================
71+
${pdfSearch.context}
72+
=====================
73+
${previousQuestions}
74+
Answer the question: ${lastMsg.content}
75+
`,
76+
},
77+
];
5078
}
5179

5280
// section to handle potential files input
@@ -60,7 +88,7 @@ export async function buildPrompt({
6088
const markdowns = await Promise.all(
6189
el.files.map(async (hash) => {
6290
try {
63-
const { content: image, mime } = await downloadFile(hash, id);
91+
const { content: image, mime } = await downloadImgFile(hash, id);
6492
const b64 = image.toString("base64");
6593
return `![](data:${mime};base64,${b64})})`;
6694
} catch (e) {

‎src/lib/components/OpenPdfSearchResults.svelte

+114
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
<script lang="ts">
2+
import type { PdfSearchUpdate } from "$lib/types/MessageUpdate";
3+
import CarbonCaretRight from "~icons/carbon/caret-right";
4+
5+
import CarbonCheckmark from "~icons/carbon/checkmark-filled";
6+
import CarbonError from "~icons/carbon/error-filled";
7+
8+
import EosIconsLoading from "~icons/eos-icons/loading";
9+
10+
export let loading = false;
11+
export let classNames = "";
12+
export let pdfSearchMessages: PdfSearchUpdate[] = [];
13+
14+
let detailsOpen: boolean;
15+
let error: boolean;
16+
$: error = pdfSearchMessages[pdfSearchMessages.length - 1]?.messageType === "error";
17+
</script>
18+
19+
<details
20+
class="flex w-fit rounded-xl border border-gray-200 bg-white shadow-sm dark:border-gray-800 dark:bg-gray-900 {classNames} max-w-full"
21+
bind:open={detailsOpen}
22+
>
23+
<summary
24+
class="align-center flex cursor-pointer select-none list-none py-1 pl-2.5 pr-2 align-text-top transition-all"
25+
>
26+
{#if error}
27+
<CarbonError class="my-auto text-red-700 dark:text-red-500" />
28+
{:else if loading}
29+
<EosIconsLoading class="my-auto text-gray-500" />
30+
{:else}
31+
<CarbonCheckmark class="my-auto text-gray-500" />
32+
{/if}
33+
<span class="px-2 font-medium" class:text-red-700={error} class:dark:text-red-500={error}
34+
>PDF search
35+
</span>
36+
<div class="my-auto transition-all" class:rotate-90={detailsOpen}>
37+
<CarbonCaretRight />
38+
</div>
39+
</summary>
40+
41+
<div class="content px-5 pb-5 pt-4">
42+
{#if pdfSearchMessages.length === 0}
43+
<div class="mx-auto w-fit">
44+
<EosIconsLoading class="mb-3 h-4 w-4" />
45+
</div>
46+
{:else}
47+
<ol>
48+
{#each pdfSearchMessages as message}
49+
{#if message.messageType === "update"}
50+
<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
51+
<div class="flex items-start">
52+
<div
53+
class="-ml-1.5 h-3 w-3 flex-none rounded-full bg-gray-200 dark:bg-gray-600 {loading
54+
? 'group-last:animate-pulse group-last:bg-gray-300 group-last:dark:bg-gray-500'
55+
: ''}"
56+
/>
57+
<h3 class="text-md -mt-1.5 pl-2.5 text-gray-800 dark:text-gray-100">
58+
{message.message}
59+
</h3>
60+
</div>
61+
{#if message.args}
62+
<p class="mt-1.5 pl-4 text-gray-500 dark:text-gray-400">
63+
{message.args}
64+
</p>
65+
{/if}
66+
</li>
67+
{:else if message.messageType === "error"}
68+
<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
69+
<div class="flex items-start">
70+
<CarbonError
71+
class="-ml-1.5 h-3 w-3 flex-none scale-110 text-red-700 dark:text-red-500"
72+
/>
73+
<h3 class="text-md -mt-1.5 pl-2.5 text-red-700 dark:text-red-500">
74+
{message.message}
75+
</h3>
76+
</div>
77+
{#if message.args}
78+
<p class="mt-1.5 pl-4 text-gray-500 dark:text-gray-400">
79+
{message.args}
80+
</p>
81+
{/if}
82+
</li>
83+
{/if}
84+
{/each}
85+
</ol>
86+
{/if}
87+
</div>
88+
</details>
89+
90+
<style>
91+
@keyframes grow {
92+
0% {
93+
font-size: 0;
94+
opacity: 0;
95+
}
96+
30% {
97+
font-size: 1em;
98+
opacity: 0;
99+
}
100+
100% {
101+
opacity: 1;
102+
}
103+
}
104+
105+
details[open] .content {
106+
animation-name: grow;
107+
animation-duration: 300ms;
108+
animation-delay: 0ms;
109+
}
110+
111+
details summary::-webkit-details-marker {
112+
display: none;
113+
}
114+
</style>

‎src/lib/components/UploadBtn.svelte

+55-6
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,72 @@
11
<script lang="ts">
2+
import { PdfUploadStatus } from "$lib/types/PdfChat";
3+
import {createEventDispatcher, onDestroy} from "svelte";
24
import CarbonUpload from "~icons/carbon/upload";
35
46
export let classNames = "";
7+
export let multimodal = false;
58
export let files: File[];
6-
let filelist: FileList;
9+
export let uploadPdfStatus: PdfUploadStatus;
10+
11+
const accept = multimodal ? "image/*,.pdf" : ".pdf";
12+
const label = multimodal ? "Upload image or PDF" : "Upload PDF";
13+
let fileInput: HTMLInputElement;
14+
let interval: ReturnType<typeof setInterval>;
715
8-
$: if (filelist) {
9-
files = Array.from(filelist);
16+
$: uploading = uploadPdfStatus === PdfUploadStatus.Uploading;
17+
$:{
18+
if(uploadPdfStatus === PdfUploadStatus.Uploaded){
19+
interval = setInterval(() => {
20+
uploadPdfStatus = PdfUploadStatus.Ready;
21+
}, 1500);
22+
}
1023
}
24+
25+
const dispatch = createEventDispatcher<{
26+
uploadpdf: File;
27+
}>();
28+
29+
function onChange() {
30+
if(!fileInput.files){
31+
return;
32+
}
33+
34+
const file = fileInput.files?.[0];
35+
if (file?.type === "application/pdf") {
36+
// pdf upload
37+
dispatch("uploadpdf", file);
38+
}else{
39+
// image files for multimodal models
40+
files = Array.from(fileInput.files);
41+
}
42+
}
43+
44+
onDestroy(() => {
45+
if(interval){
46+
clearInterval(interval);
47+
}
48+
})
1149
</script>
1250

1351
<button
1452
class="btn relative h-8 rounded-lg border bg-white px-3 py-1 text-sm text-gray-500 shadow-sm transition-all hover:bg-gray-100 dark:border-gray-600 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600 {classNames}"
53+
class:animate-pulse={uploading}
54+
class:pointer-events-none={uploading}
1555
>
1656
<input
17-
bind:files={filelist}
57+
bind:this={fileInput}
58+
on:change={onChange}
1859
class="absolute w-full cursor-pointer opacity-0"
1960
type="file"
20-
accept="image/*"
61+
{accept}
62+
disabled={uploading}
2163
/>
22-
<CarbonUpload class="mr-2 text-xs " /> Upload image
64+
<CarbonUpload class="mr-2 text-xs " />
65+
{#if uploadPdfStatus === PdfUploadStatus.Uploaded}
66+
PDF Uploaded ✅
67+
{:else if uploading}
68+
Processing PDF file
69+
{:else}
70+
{label}
71+
{/if}
2372
</button>

‎src/lib/components/chat/ChatMessage.svelte

+19-5
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@
1717
import type { Model } from "$lib/types/Model";
1818
1919
import OpenWebSearchResults from "../OpenWebSearchResults.svelte";
20-
import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
20+
import OpenPdfSearchResults from "../OpenPdfSearchResults.svelte";
21+
import type { RAGUpdate, WebSearchUpdate, PdfSearchUpdate } from "$lib/types/MessageUpdate";
2122
2223
function sanitizeMd(md: string) {
2324
let ret = md
@@ -49,7 +50,7 @@
4950
export let readOnly = false;
5051
export let isTapped = false;
5152
52-
export let webSearchMessages: WebSearchUpdate[];
53+
export let RAGMessages: RAGUpdate[];
5354
5455
const dispatch = createEventDispatcher<{
5556
retry: { content: string; id: Message["id"] };
@@ -108,9 +109,15 @@
108109
109110
let searchUpdates: WebSearchUpdate[] = [];
110111
111-
$: searchUpdates = ((webSearchMessages.length > 0
112-
? webSearchMessages
112+
$: searchUpdates = ((RAGMessages.filter(({type}) => type === "webSearch").length > 0
113+
? RAGMessages.filter(({type}) => type === "webSearch")
113114
: message.updates?.filter(({ type }) => type === "webSearch")) ?? []) as WebSearchUpdate[];
115+
116+
let pdfUpdates: PdfSearchUpdate[] = [];
117+
118+
$: pdfUpdates = ((RAGMessages.filter(({type}) => type === "pdfSearch").length > 0
119+
? RAGMessages.filter(({type}) => type === "pdfSearch")
120+
: message.updates?.filter(({ type }) => type === "pdfSearch")) ?? []) as PdfSearchUpdate[];
114121
115122
$: downloadLink =
116123
message.from === "user" ? `${$page.url.pathname}/message/${message.id}/prompt` : undefined;
@@ -153,7 +160,14 @@
153160
loading={!(searchUpdates[searchUpdates.length - 1]?.messageType === "sources")}
154161
/>
155162
{/if}
156-
{#if !message.content && (webSearchIsDone || (webSearchMessages && webSearchMessages.length === 0))}
163+
{#if pdfUpdates && pdfUpdates.length > 0}
164+
<OpenPdfSearchResults
165+
classNames={tokens.length ? "mb-3.5" : ""}
166+
pdfSearchMessages={pdfUpdates}
167+
loading={!(pdfUpdates[pdfUpdates.length - 1]?.messageType === "done")}
168+
/>
169+
{/if}
170+
{#if !message.content && (webSearchIsDone || (RAGMessages && RAGMessages.length === 0))}
157171
<IconLoading />
158172
{/if}
159173

‎src/lib/components/chat/ChatMessages.svelte

+5-5
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import type { Model } from "$lib/types/Model";
88
import ChatIntroduction from "./ChatIntroduction.svelte";
99
import ChatMessage from "./ChatMessage.svelte";
10-
import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
10+
import type { RAGUpdate } from "$lib/types/MessageUpdate";
1111
import { browser } from "$app/environment";
1212
import SystemPromptModal from "../SystemPromptModal.svelte";
1313
@@ -22,7 +22,7 @@
2222
2323
let chatContainer: HTMLElement;
2424
25-
export let webSearchMessages: WebSearchUpdate[] = [];
25+
export let RAGMessages: RAGUpdate[] = [];
2626
2727
async function scrollToBottom() {
2828
await tick();
@@ -37,7 +37,7 @@
3737

3838
<div
3939
class="scrollbar-custom mr-1 h-full overflow-y-auto"
40-
use:snapScrollToBottom={messages.length ? [...messages, ...webSearchMessages] : false}
40+
use:snapScrollToBottom={messages.length ? [...messages, ...RAGMessages] : false}
4141
bind:this={chatContainer}
4242
>
4343
<div class="mx-auto flex h-full max-w-3xl flex-col gap-6 px-5 pt-6 sm:gap-8 xl:max-w-4xl">
@@ -51,7 +51,7 @@
5151
{isAuthor}
5252
{readOnly}
5353
model={currentModel}
54-
webSearchMessages={i === messages.length - 1 ? webSearchMessages : []}
54+
RAGMessages={i === messages.length - 1 ? RAGMessages : []}
5555
on:retry
5656
on:vote
5757
/>
@@ -62,7 +62,7 @@
6262
<ChatMessage
6363
message={{ from: "assistant", content: "", id: randomUUID() }}
6464
model={currentModel}
65-
{webSearchMessages}
65+
{RAGMessages}
6666
/>
6767
{/if}
6868
<div class="h-44 flex-none" />

0 commit comments

Comments
 (0)