Fix Unicode character handling in API responses

restful-service/src/index.js

@@ -76,7 +76,10 @@ app.post('/generateQuerySummary', verifyClientSecret, async (req, res) => {
     try {
         // Replace this with your Vertex AI summarization logic
         const summary = await generateQuerySummary(generativeModel, query, description, nextStepsInstructions);
-        res.json({ summary });
+        // Encode response as base64 to handle Unicode characters (like Japanese)
+        // This prevents character encoding issues when the response passes through Looker proxy
+        const base64Summary = Buffer.from(JSON.stringify({ summary })).toString('base64');
+        res.json({ summary: base64Summary });
     } catch (e) {
         console.log('There was an error processing the individual query summary: ', e);
         res.status(500).send('Internal Server Error');
@@ -87,7 +90,10 @@ app.post('/generateSummary', verifyClientSecret, async (req, res) => {
     try {
         // Generate dashboard summary
         const summary = await generateSummary(generativeModel, querySummaries, nextStepsInstructions);
-        res.json({ summary });
+        // Encode response as base64 to handle Unicode characters (like Japanese)
+        // This prevents character encoding issues when the response passes through Looker proxy
+        const base64Summary = Buffer.from(JSON.stringify({ summary })).toString('base64');
+        res.json({ summary: base64Summary });
     } catch (e) {
         console.log('There was an error processing the dashboard summary: ', e);
         res.status(500).send('Internal Server Error');
@@ -98,7 +104,10 @@ app.post('/generateQuerySuggestions', verifyClientSecret, async (req, res) => {
     try {
         // Generate query suggestions
         const suggestions = await generateQuerySuggestions(generativeModel, queryResults, querySummaries, nextStepsInstructions);
-        res.json({ suggestions }); // Correct the response key to suggestions
+        // Encode response as base64 to handle Unicode characters (like Japanese)
+        // This prevents character encoding issues when the response passes through Looker proxy
+        const base64Suggestions = Buffer.from(JSON.stringify({ suggestions })).toString('base64');
+        res.json({ suggestions: base64Suggestions });
     } catch (e) {
         console.log('There was an error processing the query suggestions: ', e);
         res.status(500).send('Internal Server Error');

src/utils/base64Helper.ts

@@ -0,0 +1,27 @@
+/**
+ * Decode base64 encoded string with support for Unicode characters
+ *
+ * Base64 encoding is necessary because:
+ * 1. The Looker proxy has issues handling Unicode characters (like Japanese)
+ *    in JSON responses, resulting in 500 errors
+ * 2. By encoding responses as base64 on the backend and decoding them on
+ *    the frontend, we can safely transmit any Unicode characters
+ * 3. The TextDecoder ensures proper UTF-8 handling for multi-byte characters
+ *
+ * @param base64String - Base64 encoded string to decode
+ * @returns Decoded UTF-8 string
+ */
+export const decodeBase64String = (base64String: string): string => {
+  try {
+    // Convert base64 to binary data using the approach from MDN
+    // https://developer.mozilla.org/en-US/docs/Web/API/Window/btoa#unicode_strings
+    const binString = atob(base64String);
+    const bytes = Uint8Array.from(binString, (m) => m.charCodeAt(0));
+    // Decode as UTF-8
+    const decoded = new TextDecoder('utf-8').decode(bytes);
+    return decoded;
+  } catch (error: any) {
+    console.error('Error decoding base64 string:', error);
+    throw new Error(`Failed to decode base64 string: ${error.message}`);
+  }
+};

src/utils/fetchQuerySummary.ts

@@ -1,4 +1,5 @@
 import { DashboardMetadata } from '../types';
+import { decodeBase64String } from './base64Helper';
 
 export const fetchQuerySummary = async (
   queryResult: any,
@@ -25,7 +26,7 @@ export const fetchQuerySummary = async (
     console.log('fetchquerysummary response', response);
     if (response.ok) {
       const data = await response.body.summary;
-      return data;
+      return decodeBase64String(data);
     } else {
       console.error('Error generating query summary:', response.statusText);
       return null;

src/utils/generateFinalSummary.ts

@@ -1,3 +1,4 @@
+import { decodeBase64String } from './base64Helper';
 
 export const generateFinalSummary = async (
     querySummaries: any[],
@@ -24,7 +25,7 @@ export const generateFinalSummary = async (
         console.log('generateFinalSummary request querySummaries and instructions', querySummaries, nextStepsInstructions);
         console.log('generateFinalSummary response', response);
         const data = await response.body;
-        setFormattedData(data.summary);
+        setFormattedData(decodeBase64String(data.summary));
       } else {
         console.error('Error generating summary:', response.statusText);
       }

src/utils/generateQuerySuggestions.ts

@@ -1,4 +1,5 @@
 import { look, Looker40SDK } from "@looker/sdk";
+import { decodeBase64String } from "./base64Helper";
 
 export const generateQuerySuggestions = async (
   querySummaries: any[],
@@ -28,7 +29,7 @@ export const generateQuerySuggestions = async (
       try {
         const data = await response.json();
         console.log("Suggestions: ", data)
-        setQuerySuggestions(data.suggestions);
+        setQuerySuggestions(decodeBase64String(data.suggestions));
       } catch (error) {
         // If parsing fails, assume it's Markdown
         setQuerySuggestions(response);