Skip to content

Commit 551206f

Browse files
tzolovtzolov
authored
fix(bedrock-converse): Update tool use handling and add usage aggregation tests (#1777)
- Replace hardcoded "tool_use" with StopReason enum value - Add tests for token usage aggregation with tool calls - Add handling for null response metadata
1 parent 00ede3f commit 551206f

File tree

7 files changed

+250
-9
lines changed

7 files changed

+250
-9
lines changed

‎models/spring-ai-bedrock-converse/src/main/java/org/springframework/ai/bedrock/converse/BedrockProxyChatModel.java

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import software.amazon.awssdk.services.bedrockruntime.model.ImageSource;
5656
import software.amazon.awssdk.services.bedrockruntime.model.InferenceConfiguration;
5757
import software.amazon.awssdk.services.bedrockruntime.model.Message;
58+
import software.amazon.awssdk.services.bedrockruntime.model.StopReason;
5859
import software.amazon.awssdk.services.bedrockruntime.model.SystemContentBlock;
5960
import software.amazon.awssdk.services.bedrockruntime.model.Tool;
6061
import software.amazon.awssdk.services.bedrockruntime.model.ToolConfiguration;
@@ -189,6 +190,8 @@ private ChatResponse internalCall(Prompt prompt, ChatResponse perviousChatRespon
189190

190191
ConverseResponse converseResponse = this.bedrockRuntimeClient.converse(converseRequest);
191192

193+
logger.debug("ConverseResponse: {}", converseResponse);
194+
192195
var response = this.toChatResponse(converseResponse, perviousChatResponse);
193196

194197
observationContext.setResponse(response);
@@ -197,7 +200,7 @@ private ChatResponse internalCall(Prompt prompt, ChatResponse perviousChatRespon
197200
});
198201

199202
if (!this.isProxyToolCalls(prompt, this.defaultOptions) && chatResponse != null
200-
&& this.isToolCall(chatResponse, Set.of("tool_use"))) {
203+
&& this.isToolCall(chatResponse, Set.of(StopReason.TOOL_USE.toString()))) {
201204
var toolCallConversation = this.handleToolCalls(prompt, chatResponse);
202205
return this.internalCall(new Prompt(toolCallConversation, prompt.getOptions()), chatResponse);
203206
}
@@ -471,7 +474,7 @@ private ChatResponse toChatResponse(ConverseResponse response, ChatResponse perv
471474
ConverseMetrics metrics = response.metrics();
472475

473476
var chatResponseMetaData = ChatResponseMetadata.builder()
474-
.withId(response.responseMetadata().requestId())
477+
.withId(response.responseMetadata() != null ? response.responseMetadata().requestId() : "Unknown")
475478
.withUsage(usage)
476479
.build();
477480

@@ -525,7 +528,8 @@ private Flux<ChatResponse> internalStream(Prompt prompt, ChatResponse perviousCh
525528

526529
Flux<ChatResponse> chatResponseFlux = chatResponses.switchMap(chatResponse -> {
527530
if (!this.isProxyToolCalls(prompt, this.defaultOptions) && chatResponse != null
528-
&& this.isToolCall(chatResponse, Set.of("tool_use"))) {
531+
&& this.isToolCall(chatResponse, Set.of(StopReason.TOOL_USE.toString()))) {
532+
529533
var toolCallConversation = this.handleToolCalls(prompt, chatResponse);
530534
return this.internalStream(new Prompt(toolCallConversation, prompt.getOptions()), chatResponse);
531535
}

‎models/spring-ai-bedrock-converse/src/test/java/org/springframework/ai/bedrock/converse/BedrockConverseChatClientIT.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@
4949
import org.springframework.util.MimeTypeUtils;
5050

5151
import static org.assertj.core.api.Assertions.assertThat;
52-
import static org.mockito.ArgumentMatchers.matches;
5352

5453
@SpringBootTest(classes = BedrockConverseTestConfiguration.class)
5554
@EnabledIfEnvironmentVariable(named = "AWS_ACCESS_KEY_ID", matches = ".*")

‎models/spring-ai-bedrock-converse/src/test/java/org/springframework/ai/bedrock/converse/BedrockConverseUsageAggregationTests.java

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
/*
2+
* Copyright 2024-2024 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.ai.bedrock.converse;
18+
19+
import java.util.List;
20+
21+
import io.micrometer.observation.ObservationRegistry;
22+
import org.junit.jupiter.api.BeforeEach;
23+
import org.junit.jupiter.api.Test;
24+
import org.junit.jupiter.api.extension.ExtendWith;
25+
import org.mockito.Mock;
26+
import org.mockito.junit.jupiter.MockitoExtension;
27+
import software.amazon.awssdk.core.document.internal.MapDocument;
28+
import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeAsyncClient;
29+
import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClient;
30+
import software.amazon.awssdk.services.bedrockruntime.model.ContentBlock;
31+
import software.amazon.awssdk.services.bedrockruntime.model.ConversationRole;
32+
import software.amazon.awssdk.services.bedrockruntime.model.ConverseMetrics;
33+
import software.amazon.awssdk.services.bedrockruntime.model.ConverseOutput;
34+
import software.amazon.awssdk.services.bedrockruntime.model.ConverseRequest;
35+
import software.amazon.awssdk.services.bedrockruntime.model.ConverseResponse;
36+
import software.amazon.awssdk.services.bedrockruntime.model.Message;
37+
import software.amazon.awssdk.services.bedrockruntime.model.StopReason;
38+
import software.amazon.awssdk.services.bedrockruntime.model.TokenUsage;
39+
import software.amazon.awssdk.services.bedrockruntime.model.ToolUseBlock;
40+
41+
import org.springframework.ai.chat.prompt.Prompt;
42+
import org.springframework.ai.model.function.FunctionCallback;
43+
import org.springframework.ai.model.function.FunctionCallingOptions;
44+
import org.springframework.ai.model.function.FunctionCallingOptionsBuilder.PortableFunctionCallingOptions;
45+
46+
import static org.assertj.core.api.Assertions.assertThat;
47+
import static org.mockito.ArgumentMatchers.isA;
48+
import static org.mockito.BDDMockito.given;
49+
50+
/**
51+
* @author Christian Tzolov
52+
*/
53+
@ExtendWith(MockitoExtension.class)
54+
public class BedrockConverseUsageAggregationTests {
55+
56+
private @Mock BedrockRuntimeClient bedrockRuntimeClient;
57+
58+
private @Mock BedrockRuntimeAsyncClient bedrockRuntimeAsyncClient;
59+
60+
private BedrockProxyChatModel chatModel;
61+
62+
@BeforeEach
63+
public void beforeEach() {
64+
chatModel = new BedrockProxyChatModel(this.bedrockRuntimeClient, this.bedrockRuntimeAsyncClient,
65+
FunctionCallingOptions.builder().build(), null, List.of(), ObservationRegistry.NOOP);
66+
}
67+
68+
@Test
69+
public void call() {
70+
ConverseResponse converseResponse = ConverseResponse.builder()
71+
72+
.output(ConverseOutput.builder()
73+
.message(Message.builder()
74+
.role(ConversationRole.ASSISTANT)
75+
.content(ContentBlock.fromText("Response Content Block"))
76+
.build())
77+
.build())
78+
.usage(TokenUsage.builder().inputTokens(16).outputTokens(14).totalTokens(30).build())
79+
.build();
80+
81+
given(this.bedrockRuntimeClient.converse(isA(ConverseRequest.class))).willReturn(converseResponse);
82+
83+
var result = this.chatModel.call(new Prompt("text"));
84+
85+
assertThat(result).isNotNull();
86+
assertThat(result.getResult().getOutput().getContent()).isSameAs("Response Content Block");
87+
88+
assertThat(result.getMetadata().getUsage().getPromptTokens()).isEqualTo(16);
89+
assertThat(result.getMetadata().getUsage().getGenerationTokens()).isEqualTo(14);
90+
assertThat(result.getMetadata().getUsage().getTotalTokens()).isEqualTo(30);
91+
}
92+
93+
public record Request(String location, String unit) {
94+
}
95+
96+
@Test
97+
public void callWithToolUse() {
98+
99+
ConverseResponse converseResponseToolUse = ConverseResponse.builder()
100+
.output(ConverseOutput.builder()
101+
.message(Message.builder()
102+
.role(ConversationRole.ASSISTANT)
103+
.content(ContentBlock.fromText(
104+
"Certainly! I'd be happy to check the current weather in Paris for you, with the temperature in Celsius. To get this information, I'll use the getCurrentWeather function. Let me fetch that for you right away."),
105+
ContentBlock.fromToolUse(ToolUseBlock.builder()
106+
.toolUseId("tooluse_2SZuiUDkRbeGysun8O2Wag")
107+
.name("getCurrentWeather")
108+
.input(MapDocument.mapBuilder()
109+
.putString("location", "Paris, France")
110+
.putString("unit", "C")
111+
.build())
112+
.build()))
113+
114+
.build())
115+
.build())
116+
.usage(TokenUsage.builder().inputTokens(445).outputTokens(119).totalTokens(564).build())
117+
.stopReason(StopReason.TOOL_USE)
118+
.metrics(ConverseMetrics.builder().latencyMs(3435L).build())
119+
.build();
120+
121+
ConverseResponse converseResponseFinal = ConverseResponse.builder()
122+
.output(ConverseOutput.builder()
123+
.message(Message.builder()
124+
.role(ConversationRole.ASSISTANT)
125+
.content(ContentBlock.fromText(
126+
"""
127+
Based on the information from the weather tool, the current temperature in Paris, France is 15.0°C (Celsius).
128+
129+
Please note that weather conditions can change throughout the day, so this temperature represents the current
130+
reading at the time of the request. If you need more detailed information about the weather in Paris, such as
131+
humidity, wind speed, or forecast for the coming days, please let me know, and I'll be happy to provide more
132+
details if that information is available through our weather service.
133+
"""))
134+
.build())
135+
.build())
136+
.usage(TokenUsage.builder().inputTokens(540).outputTokens(106).totalTokens(646).build())
137+
.stopReason(StopReason.END_TURN)
138+
.metrics(ConverseMetrics.builder().latencyMs(3435L).build())
139+
.build();
140+
141+
given(this.bedrockRuntimeClient.converse(isA(ConverseRequest.class))).willReturn(converseResponseToolUse)
142+
.willReturn(converseResponseFinal);
143+
144+
FunctionCallback functionCallback = FunctionCallback.builder()
145+
.description("Gets the weather in location")
146+
.function("getCurrentWeather", (Request request) -> "15.0°C")
147+
.inputType(Request.class)
148+
.build();
149+
150+
var result = this.chatModel.call(new Prompt("What is the weather in Paris?",
151+
PortableFunctionCallingOptions.builder().withFunctionCallbacks(functionCallback).build()));
152+
153+
assertThat(result).isNotNull();
154+
assertThat(result.getResult().getOutput().getContent())
155+
.isSameAs(converseResponseFinal.output().message().content().get(0).text());
156+
157+
assertThat(result.getMetadata().getUsage().getPromptTokens()).isEqualTo(445 + 540);
158+
assertThat(result.getMetadata().getUsage().getGenerationTokens()).isEqualTo(119 + 106);
159+
assertThat(result.getMetadata().getUsage().getTotalTokens()).isEqualTo(564 + 646);
160+
}
161+
162+
@Test
163+
public void streamWithToolUse() {
164+
// TODO: Implement the test
165+
}
166+
167+
}

‎models/spring-ai-bedrock-converse/src/test/java/org/springframework/ai/bedrock/converse/experiements/BedrockConverseChatModelMain.java renamed to ‎models/spring-ai-bedrock-converse/src/test/java/org/springframework/ai/bedrock/converse/experiments/BedrockConverseChatModelMain.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* limitations under the License.
1515
*/
1616

17-
package org.springframework.ai.bedrock.converse.experiements;
17+
package org.springframework.ai.bedrock.converse.experiments;
1818

1919
import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
2020
import software.amazon.awssdk.regions.Region;

‎models/spring-ai-bedrock-converse/src/test/java/org/springframework/ai/bedrock/converse/experiements/BedrockConverseChatModelMain2.java renamed to ‎models/spring-ai-bedrock-converse/src/test/java/org/springframework/ai/bedrock/converse/experiments/BedrockConverseChatModelMain2.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* limitations under the License.
1515
*/
1616

17-
package org.springframework.ai.bedrock.converse.experiements;
17+
package org.springframework.ai.bedrock.converse.experiments;
1818

1919
import java.util.List;
2020

‎models/spring-ai-bedrock-converse/src/test/java/org/springframework/ai/bedrock/converse/experiments/BedrockConverseChatModelMain3.java

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
* Copyright 2023-2024 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.ai.bedrock.converse.experiments;
18+
19+
import java.util.List;
20+
21+
import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
22+
import software.amazon.awssdk.regions.Region;
23+
24+
import org.springframework.ai.bedrock.converse.BedrockProxyChatModel;
25+
import org.springframework.ai.bedrock.converse.MockWeatherService;
26+
import org.springframework.ai.chat.prompt.Prompt;
27+
import org.springframework.ai.model.function.FunctionCallback;
28+
import org.springframework.ai.model.function.FunctionCallingOptionsBuilder.PortableFunctionCallingOptions;
29+
30+
/**
31+
* Used for reverse engineering the protocol
32+
*/
33+
public final class BedrockConverseChatModelMain3 {
34+
35+
private BedrockConverseChatModelMain3() {
36+
37+
}
38+
39+
public static void main(String[] args) {
40+
41+
// String modelId = "anthropic.claude-3-5-sonnet-20240620-v1:0";
42+
// String modelId = "ai21.jamba-1-5-large-v1:0";
43+
String modelId = "anthropic.claude-3-5-sonnet-20240620-v1:0";
44+
45+
// var prompt = new Prompt("Tell me a joke?",
46+
// ChatOptionsBuilder.builder().withModel(modelId).build());
47+
var prompt = new Prompt(
48+
// "What's the weather like in San Francisco, Tokyo, and Paris? Return the
49+
// temperature in Celsius.",
50+
"What's the weather like in Paris? Return the temperature in Celsius.",
51+
PortableFunctionCallingOptions.builder()
52+
.withModel(modelId)
53+
.withFunctionCallbacks(List.of(FunctionCallback.builder()
54+
.description("Get the weather in location")
55+
.function("getCurrentWeather", new MockWeatherService())
56+
.inputType(MockWeatherService.Request.class)
57+
.build()))
58+
.build());
59+
60+
BedrockProxyChatModel chatModel = BedrockProxyChatModel.builder()
61+
.withCredentialsProvider(EnvironmentVariableCredentialsProvider.create())
62+
.withRegion(Region.US_EAST_1)
63+
.build();
64+
65+
var response = chatModel.call(prompt);
66+
67+
System.out.println(response);
68+
69+
}
70+
71+
}

‎spring-ai-core/src/main/java/org/springframework/ai/model/function/FunctionCallingOptionsBuilder.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,9 @@ public FunctionCallingOptionsBuilder withFunctionCallbacks(List<FunctionCallback
5151
return this;
5252
}
5353

54-
public FunctionCallingOptionsBuilder withFunctionCallback(FunctionCallback functionCallback) {
55-
Assert.notNull(functionCallback, "FunctionCallback must not be null");
56-
this.options.getFunctionCallbacks().add(functionCallback);
54+
public FunctionCallingOptionsBuilder withFunctionCallbacks(FunctionCallback... functionCallbacks) {
55+
Assert.notNull(functionCallbacks, "FunctionCallbacks must not be null");
56+
this.options.setFunctionCallbacks(List.of(functionCallbacks));
5757
return this;
5858
}
5959

0 commit comments

Comments
 (0)