Enable prompt caching for Amazon Nova by default (#1068)

Author: toshikwa

packages/cdk/lambda/utils/models.ts

@@ -190,6 +190,13 @@ const NOVA_DEFAULT_PARAMS: ConverseInferenceParams = {
     temperature: 0.7,
     topP: 0.9,
   },
+  // There are no additional costs for cache writes with Amazon Nova models
+  promptCachingConfig: {
+    autoCacheFields: {
+      system: true,
+      messages: true,
+    },
+  },
 };
 
 const DEEPSEEK_DEFAULT_PARAMS: ConverseInferenceParams = {
@@ -211,17 +218,42 @@ const PALMYRA_DEFAULT_PARAMS: ConverseInferenceParams = {
 const USECASE_DEFAULT_PARAMS: UsecaseConverseInferenceParams = {
   '/chat': {
     promptCachingConfig: {
-      autoCacheFields: ['system', 'messages'],
+      autoCacheFields: {
+        system: true,
+        messages: true,
+      },
     },
   },
   '/rag': {
     inferenceConfig: {
       temperature: 0.0,
     },
+    promptCachingConfig: {
+      autoCacheFields: {
+        system: false,
+      },
+    },
   },
   '/diagram': {
     promptCachingConfig: {
-      autoCacheFields: ['system'],
+      autoCacheFields: {
+        system: true,
+      },
+    },
+  },
+  '/use-case-builder': {
+    promptCachingConfig: {
+      autoCacheFields: {
+        messages: false,
+      },
+    },
+  },
+  '/title': {
+    promptCachingConfig: {
+      autoCacheFields: {
+        system: false,
+        messages: false,
+      },
     },
   },
 };
@@ -263,6 +295,11 @@ const createGuardrailStreamConfig = ():
 const idTransformationRules = [
   // Chat history -> Chat
   { pattern: /^\/chat\/.+/, replacement: '/chat' },
+  // Use case builder (/new and /execute/*)
+  {
+    pattern: /^\/use-case-builder\/.+/,
+    replacement: '/use-case-builder',
+  },
 ];
 
 // ID conversion
@@ -273,6 +310,23 @@ function normalizeId(id: string): string {
   return ret;
 }
 
+const mergeConverseInferenceParams = (
+  a: ConverseInferenceParams,
+  b: ConverseInferenceParams
+) =>
+  ({
+    inferenceConfig: {
+      ...a.inferenceConfig,
+      ...b.inferenceConfig,
+    },
+    promptCachingConfig: {
+      autoCacheFields: {
+        ...a.promptCachingConfig?.autoCacheFields,
+        ...b.promptCachingConfig?.autoCacheFields,
+      },
+    },
+  }) as ConverseInferenceParams;
+
 // API call, extract string from output, etc.
 
 const createConverseCommandInput = (
@@ -351,14 +405,17 @@ const createConverseCommandInput = (
 
   // Merge model's default params with use-case specific ones
   const usecaseParams = usecaseConverseInferenceParams[normalizeId(id)] || {};
-  const params = { ...defaultConverseInferenceParams, ...usecaseParams };
+  const params = mergeConverseInferenceParams(
+    defaultConverseInferenceParams,
+    usecaseParams
+  );
 
   // Apply prompt caching
-  const autoCacheFields = params.promptCachingConfig?.autoCacheFields || [];
-  const conversationWithCache = autoCacheFields.includes('messages')
+  const autoCacheFields = params.promptCachingConfig?.autoCacheFields || {};
+  const conversationWithCache = autoCacheFields['messages']
     ? applyAutoCacheToMessages(conversation, model.modelId)
     : conversation;
-  const systemContextWithCache = autoCacheFields.includes('system')
+  const systemContextWithCache = autoCacheFields['system']
     ? applyAutoCacheToSystem(systemContext, model.modelId)
     : systemContext;
 
@@ -377,7 +434,7 @@ const createConverseCommandInput = (
     model.modelParameters?.reasoningConfig?.type === 'enabled'
   ) {
     converseCommandInput.inferenceConfig = {
-      ...(params.inferenceConfig || {}),
+      ...params.inferenceConfig,
       temperature: 1, // reasoning requires temperature to be 1
       topP: undefined, // reasoning does not require topP
       maxTokens:

packages/cdk/lambda/utils/promptCache.ts

@@ -32,12 +32,16 @@ export const applyAutoCacheToMessages = (
   const isToolsSupported = cacheFields.includes('tools');
   const cachableIndices = messages
     .map((message, index) => ({ message, index }))
-    .filter(({ message }) => message.role === 'user')
+    .filter(
+      ({ message }) =>
+        message.role === 'user' &&
+        !message.content?.some((block) => block.document || block.video)
+    )
     .filter(
       ({ message }) =>
         isToolsSupported ||
         // For Amazon Nova, placing cachePoint after toolResult is not supported
-        !message.content?.some((content) => content.toolResult)
+        !message.content?.some((block) => block.toolResult)
     )
     .slice(-2)
     .map(({ index }) => index);

packages/types/src/text.d.ts

@@ -4,7 +4,9 @@ import { InferenceConfiguration } from '@aws-sdk/client-bedrock-runtime';
 
 export type PromptCacheField = 'messages' | 'system' | 'tools';
 export type PromptCachingConfig = {
-  autoCacheFields: PromptCacheField[];
+  autoCacheFields: {
+    [key in PromptCacheField]?: boolean;
+  };
 };
 
 // https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html#API_runtime_Converse_RequestSyntax