Bug Fix: Json Decoder aggessively pulls json (#867)

* wait for newline char b4 decoding json

* remove noisy console log

* linters

---------

Co-authored-by: Nathan Sarrazin <sarrazin.nathan@gmail.com>

Author: MDCurrent

src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts

@@ -55,10 +55,13 @@ export function endpointLlamacpp(
 			let stop = false;
 			let generatedText = "";
 			let tokenId = 0;
+			let accumulatedData = ""; // Buffer to accumulate data chunks
+
 			while (!stop) {
-				// read the stream and log the outputs to console
+				// Read the stream and log the outputs to console
 				const out = (await reader?.read()) ?? { done: false, value: undefined };
-				// we read, if it's done we cancel
+
+				// If it's done, we cancel
 				if (out.done) {
 					reader?.cancel();
 					return;
@@ -68,31 +71,49 @@ export function endpointLlamacpp(
 					return;
 				}
 
-				if (out.value.startsWith("data: ")) {
-					let data = null;
-					try {
-						data = JSON.parse(out.value.slice(6));
-					} catch (e) {
-						return;
-					}
-					if (data.content || data.stop) {
-						generatedText += data.content;
-						const output: TextGenerationStreamOutput = {
-							token: {
-								id: tokenId++,
-								text: data.content ?? "",
-								logprob: 0,
-								special: false,
-							},
-							generated_text: data.stop ? generatedText : null,
-							details: null,
-						};
-						if (data.stop) {
-							stop = true;
-							reader?.cancel();
+				// Accumulate the data chunk
+				accumulatedData += out.value;
+
+				// Process each complete JSON object in the accumulated data
+				while (accumulatedData.includes("\n")) {
+					// Assuming each JSON object ends with a newline
+					const endIndex = accumulatedData.indexOf("\n");
+					let jsonString = accumulatedData.substring(0, endIndex).trim();
+
+					// Remove the processed part from the buffer
+					accumulatedData = accumulatedData.substring(endIndex + 1);
+
+					if (jsonString.startsWith("data: ")) {
+						jsonString = jsonString.slice(6);
+						let data = null;
+
+						try {
+							data = JSON.parse(jsonString);
+						} catch (e) {
+							console.error("Failed to parse JSON", e);
+							console.error("Problematic JSON string:", jsonString);
+							continue; // Skip this iteration and try the next chunk
+						}
+
+						// Handle the parsed data
+						if (data.content || data.stop) {
+							generatedText += data.content;
+							const output: TextGenerationStreamOutput = {
+								token: {
+									id: tokenId++,
+									text: data.content ?? "",
+									logprob: 0,
+									special: false,
+								},
+								generated_text: data.stop ? generatedText : null,
+								details: null,
+							};
+							if (data.stop) {
+								stop = true;
+								reader?.cancel();
+							}
+							yield output;
 						}
-						yield output;
-						// take the data.content value and yield it
 					}
 				}
 			}