feat: support image editing by nova canvas and cost display

Author: zhu-xiaowei

README.md

@@ -26,6 +26,8 @@ macOS platforms.
 - Understand images, documents and videos with Nova Lite and Pro
 - Record 30-second videos directly on Android and iOS for Nova analysis
 - Upload large videos (1080p/4K) beyond 8MB with auto compression
+- Support using natural language to make Nova Canvas generate images, remove backgrounds, replace backgrounds, and
+  create images in similar styles.
 
 ## Architecture
 
@@ -92,8 +94,8 @@ can find the **API URL** which looks like: `https://xxx.xxx.awsapprunner.com` or
 ### Step 3: Download the app and setup with API URL and API Key
 
 1. Download the App
-    - Android App click to [Download](https://github.com/aws-samples/swift-chat/releases/download/1.6.0/SwiftChat.apk)
-    - macOS App click to [Download](https://github.com/aws-samples/swift-chat/releases/download/1.6.0/SwiftChat.dmg)
+    - Android App click to [Download](https://github.com/aws-samples/swift-chat/releases/download/1.7.0/SwiftChat.apk)
+    - macOS App click to [Download](https://github.com/aws-samples/swift-chat/releases/download/1.7.0/SwiftChat.dmg)
     - iOS (Currently we do not provide the iOS version, you can build it locally with Xcode)
 
 2. Launch the App, open the drawer menu, and tap **Settings**.

README_CN.md

@@ -25,6 +25,7 @@ macOS 等多个平台。
 - 支持 Nova Lite 和 Pro 对图片、文档及视频内容的理解
 - 支持直接在安卓和 iOS 设备上录制最长 30 秒的视频供 Nova 分析
 - 支持自动压缩上传超过8MB的高清视频（1080p/4K）
+- 支持通过自然语言让 Nova Canvas 生成图片，去除背景，替换背景，以及生成类似风格的图片。
 
 ## 架构
 
@@ -83,8 +84,8 @@ macOS 等多个平台。
 ### 第3步: 下载应用并设置 API URL 和 API Key
 
 1. 下载应用
-    - Android 应用点击 [下载](https://github.com/aws-samples/swift-chat/releases/download/1.6.0/SwiftChat.apk)
-    - macOS 应用点击 [下载](https://github.com/aws-samples/swift-chat/releases/download/1.6.0/SwiftChat.dmg)
+    - Android 应用点击 [下载](https://github.com/aws-samples/swift-chat/releases/download/1.7.0/SwiftChat.apk)
+    - macOS 应用点击 [下载](https://github.com/aws-samples/swift-chat/releases/download/1.7.0/SwiftChat.dmg)
     - iOS (目前不提供 iOS 版本，您可以使用 Xcode 在本地构建)
 
 2. 启动应用，点击左侧菜单按钮，并点击底部的 **Settings**。

react-native/android/app/build.gradle

@@ -79,8 +79,8 @@ android {
         applicationId "com.aws.swiftchat"
         minSdkVersion rootProject.ext.minSdkVersion
         targetSdkVersion rootProject.ext.targetSdkVersion
-        versionCode 11
-        versionName "1.6.0"
+        versionCode 12
+        versionName "1.7.0"
         ndk {
             //noinspection ChromeOsAbiSupport
             abiFilters 'arm64-v8a'

react-native/ios/Podfile.lock

@@ -982,7 +982,7 @@ PODS:
     - Yoga
   - react-native-get-random-values (1.11.0):
     - React-Core
-  - react-native-image-picker (7.2.2):
+  - react-native-image-picker (7.2.3):
     - DoubleConversion
     - glog
     - hermes-engine
@@ -1591,7 +1591,7 @@ SPEC CHECKSUMS:
   react-native-compressor: 2ae9013718fb351264fcfcdf232eccbbf3d280a2
   react-native-document-picker: c4f197741c327270453aa9840932098e0064fd52
   react-native-get-random-values: 21325b2244dfa6b58878f51f9aa42821e7ba3d06
-  react-native-image-picker: dd85e2530d366acf77745830b053294afed66339
+  react-native-image-picker: fb0c2b3adc3eff6caa3cd6a507a34b9dcc9238dd
   react-native-mmkv: 8c9a677e64a1ac89b0c6cf240feea528318b3074
   react-native-safe-area-context: b7daa1a8df36095a032dff095a1ea8963cb48371
   React-nativeconfig: b0073a590774e8b35192fead188a36d1dca23dec

react-native/ios/SwiftChat.xcodeproj/project.pbxproj

@@ -485,7 +485,7 @@
 				CODE_SIGN_ENTITLEMENTS = SwiftChat/SwiftChat.entitlements;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 11;
+				CURRENT_PROJECT_VERSION = 12;
 				DEVELOPMENT_TEAM = BUA6W9H7T3;
 				ENABLE_BITCODE = NO;
 				"ENABLE_HARDENED_RUNTIME[sdk=macosx*]" = YES;
@@ -497,7 +497,7 @@
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
-				MARKETING_VERSION = 1.6.0;
+				MARKETING_VERSION = 1.7.0;
 				OTHER_LDFLAGS = (
 					"$(inherited)",
 					"-ObjC",
@@ -526,7 +526,7 @@
 				CODE_SIGN_ENTITLEMENTS = SwiftChat/SwiftChat.entitlements;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 11;
+				CURRENT_PROJECT_VERSION = 12;
 				DEVELOPMENT_TEAM = BUA6W9H7T3;
 				"ENABLE_HARDENED_RUNTIME[sdk=macosx*]" = YES;
 				INFOPLIST_FILE = SwiftChat/Info.plist;
@@ -537,7 +537,7 @@
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
-				MARKETING_VERSION = 1.6.0;
+				MARKETING_VERSION = 1.7.0;
 				OTHER_LDFLAGS = (
 					"$(inherited)",
 					"-ObjC",

react-native/package-lock.json

@@ -1,7 +1,7 @@
 {
   "name": "swift-chat",
   "description": "Sample Bedrock Cross-platform App - SwiftChat",
-  "version": "1.6.0",
+  "version": "1.7.0",
   "private": true,
   "scripts": {
     "android": "react-native run-android",
@@ -30,7 +30,7 @@
     "react-native-get-random-values": "^1.11.0",
     "react-native-gifted-chat": "^2.4.0",
     "react-native-haptic-feedback": "^2.2.0",
-    "react-native-image-picker": "^7.2.2",
+    "react-native-image-picker": "^7.2.3",
     "react-native-image-viewing": "^0.2.2",
     "react-native-marked": "^6.0.4",
     "react-native-mmkv": "^2.12.2",

react-native/package.json

@@ -16,6 +16,8 @@ import {
 import { saveImageToLocal } from '../chat/util/FileUtils.ts';
 import {
   BedrockMessage,
+  ImageContent,
+  ImageInfo,
   TextContent,
 } from '../chat/util/BedrockMessageConvertor.ts';
 
@@ -59,7 +61,7 @@ export const invokeBedrockWithCallBack = async (
     const url = getApiPrefix() + '/converse';
     let intervalId: ReturnType<typeof setInterval>;
     let completeMessage = '';
-    const timeoutId = setTimeout(() => controller.abort(), 20000);
+    const timeoutId = setTimeout(() => controller.abort(), 60000);
     fetch(url!, options)
       .then(response => {
         return response.body;
@@ -111,7 +113,7 @@ export const invokeBedrockWithCallBack = async (
           if (errorMsg.endsWith('AbortError: Aborted')) {
             errorMsg = 'Timed out';
           }
-          if (errorMsg.indexOf('Unable to resolve host')) {
+          if (errorMsg.indexOf('http') >= 0) {
             errorMsg = 'Unable to resolve host';
           }
           const errorInfo = 'Request error: ' + errorMsg;
@@ -122,16 +124,31 @@ export const invokeBedrockWithCallBack = async (
   } else {
     const prompt = (messages[messages.length - 1].content[0] as TextContent)
       .text;
-    const imageRes = await genImage(prompt, controller);
+    let image: ImageInfo | undefined;
+    if (messages[messages.length - 1].content[1]) {
+      image = (messages[messages.length - 1].content[1] as ImageContent).image;
+    }
+
+    const imageRes = await genImage(prompt, controller, image);
     if (imageRes.image.length > 0) {
       const localFilePath = await saveImageToLocal(imageRes.image);
+      const imageSize = getImageSize().split('x')[0].trim();
       const usage: Usage = {
         modelName: getImageModel().modelName,
         inputTokens: 0,
         outputTokens: 0,
         totalTokens: 0,
-        imageCount: 1,
+        smallImageCount: 0,
+        imageCount: 0,
+        largeImageCount: 0,
       };
+      if (imageSize === '512') {
+        usage.smallImageCount = 1;
+      } else if (imageSize === '1024') {
+        usage.imageCount = 1;
+      } else if (imageSize === '2048') {
+        usage.largeImageCount = 1;
+      }
       if (localFilePath) {
         callback(`![](${localFilePath})`, true, false, usage);
       }
@@ -143,7 +160,7 @@ export const invokeBedrockWithCallBack = async (
           imageRes.error = 'Request timed out';
         }
       }
-      if (imageRes.error.indexOf('Unable to resolve host')) {
+      if (imageRes.error.indexOf('http') >= 0) {
         imageRes.error = 'Request error: Unable to resolve host';
       }
       callback(imageRes.error, true, true);
@@ -210,7 +227,8 @@ export const requestUpgradeInfo = async (
 
 export const genImage = async (
   prompt: string,
-  controller: AbortController
+  controller: AbortController,
+  image?: ImageInfo
 ): Promise<ImageRes> => {
   if (!isConfigured()) {
     return {
@@ -224,6 +242,7 @@ export const genImage = async (
   const height = imageSize[1].trim();
   const bodyObject = {
     prompt: prompt,
+    refImages: image ? [image] : undefined,
     modelId: getImageModel().modelId,
     region: getRegion(),
     width: width,
@@ -242,7 +261,7 @@ export const genImage = async (
   };
 
   try {
-    const timeoutMs = parseInt(width, 10) >= 1024 ? 90000 : 60000;
+    const timeoutMs = parseInt(width, 10) >= 1024 ? 120000 : 90000;
     const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
     const response = await fetch(url, options);
     if (!response.ok) {

react-native/src/api/bedrock-api.ts

@@ -75,13 +75,14 @@ const createBotMessage = (mode: string) => {
 const imagePlaceholder = '![](bedrock://imgProgress)';
 const textPlaceholder = '...';
 type ChatScreenRouteProp = RouteProp<RouteParamList, 'Bedrock'>;
+let currentMode = ChatMode.Text;
 
 function ChatScreen(): React.JSX.Element {
   const navigation = useNavigation();
   const route = useRoute<ChatScreenRouteProp>();
   const initialSessionId = route.params?.sessionId;
   const tapIndex = route.params?.tapIndex;
-  const mode = route.params?.mode ?? ChatMode.Text;
+  const mode = route.params?.mode ?? currentMode;
   const modeRef = useRef(mode);
 
   const [messages, setMessages] = useState<IMessage[]>([]);
@@ -130,6 +131,7 @@ function ChatScreen(): React.JSX.Element {
 
   // header text and right button click
   React.useLayoutEffect(() => {
+    currentMode = mode;
     const headerOptions: HeaderOptions = {
       // eslint-disable-next-line react/no-unstable-nested-components
       headerTitle: () => (
@@ -499,10 +501,10 @@ function ChatScreen(): React.JSX.Element {
                 if (isUpdate) {
                   setSelectedFiles(files);
                 } else {
-                  console.log('handleNewFileSelected');
                   handleNewFileSelected(files);
                 }
               }}
+              chatMode={modeRef.current}
             />
           )
         }
@@ -528,9 +530,9 @@ function ChatScreen(): React.JSX.Element {
           if (
             isMac &&
             inputTexRef.current.length > 0 &&
+            text[text.length - 1] === '\n' &&
+            text[text.length - 2] !== ' ' &&
             text.length - inputTexRef.current.length === 1 &&
-            !text.endsWith(' \n') &&
-            text.endsWith('\n') &&
             chatStatusRef.current !== ChatStatus.Running
           ) {
             setTimeout(() => {