diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5d664e72..967b15fc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -149,6 +149,7 @@ jobs: golem-cli worker invoke test:llm/ollama-1 test5 golem-cli worker invoke test:llm/ollama-1 test6 golem-cli worker invoke test:llm/ollama-1 test7 + golem-cli worker invoke test:llm/ollama-1 test8 publish-all: needs: - tests diff --git a/llm/anthropic/src/conversions.rs b/llm/anthropic/src/conversions.rs index e332f139..e7d3175a 100644 --- a/llm/anthropic/src/conversions.rs +++ b/llm/anthropic/src/conversions.rs @@ -130,7 +130,7 @@ pub fn process_response(response: MessagesResponse) -> ChatEvent { Err(e) => { return ChatEvent::Error(Error { code: ErrorCode::InvalidRequest, - message: format!("Failed to decode base64 image data: {}", e), + message: format!("Failed to decode base64 image data: {e}"), provider_error_json: None, }); } diff --git a/llm/grok/src/conversions.rs b/llm/grok/src/conversions.rs index 68a5d570..129c128a 100644 --- a/llm/grok/src/conversions.rs +++ b/llm/grok/src/conversions.rs @@ -183,7 +183,7 @@ fn convert_content_parts(contents: Vec) -> crate::client::Content { let media_type = &image_source.mime_type; // This is already a string result.push(crate::client::ContentPart::ImageInput { image_url: crate::client::ImageUrl { - url: format!("data:{};base64,{}", media_type, base64_data), + url: format!("data:{media_type};base64,{base64_data}"), detail: image_source.detail.map(|d| d.into()), }, }); diff --git a/llm/llm/src/event_source/ndjson_stream.rs b/llm/llm/src/event_source/ndjson_stream.rs index e2f4cc1b..1b8ef377 100644 --- a/llm/llm/src/event_source/ndjson_stream.rs +++ b/llm/llm/src/event_source/ndjson_stream.rs @@ -126,7 +126,7 @@ fn try_parse_line( return Ok(None); } - trace!("Parsed NDJSON line: {}", line); + trace!("Parsed NDJSON line: {line}"); // Create a MessageEvent with the JSON line as data let event = MessageEvent { diff --git a/llm/llm/src/event_source/stream.rs b/llm/llm/src/event_source/stream.rs index 8f293367..13a5eeb5 100644 --- a/llm/llm/src/event_source/stream.rs +++ b/llm/llm/src/event_source/stream.rs @@ -56,9 +56,9 @@ where { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Utf8(err) => f.write_fmt(format_args!("UTF8 error: {}", err)), - Self::Parser(err) => f.write_fmt(format_args!("Parse error: {}", err)), - Self::Transport(err) => f.write_fmt(format_args!("Transport error: {}", err)), + Self::Utf8(err) => f.write_fmt(format_args!("UTF8 error: {err}")), + Self::Parser(err) => f.write_fmt(format_args!("Parse error: {err}")), + Self::Transport(err) => f.write_fmt(format_args!("Transport error: {err}")), } } } diff --git a/llm/ollama/src/client.rs b/llm/ollama/src/client.rs index e9514a8d..e2901e70 100644 --- a/llm/ollama/src/client.rs +++ b/llm/ollama/src/client.rs @@ -335,7 +335,7 @@ pub fn image_to_base64(source: &str) -> Result Error { Error { code: ErrorCode::InternalError, - message: format!("{}: {}", context, err), + message: format!("{context}: {err}"), provider_error_json: None, } } diff --git a/llm/ollama/src/conversions.rs b/llm/ollama/src/conversions.rs index b1db65c6..8d64e954 100644 --- a/llm/ollama/src/conversions.rs +++ b/llm/ollama/src/conversions.rs @@ -214,7 +214,7 @@ pub fn process_response(response: CompletionsResponse) -> ChatEvent { }; ChatEvent::Message(CompleteResponse { - id: format!("ollama-{}", timestamp), + id: format!("ollama-{timestamp}"), content, tool_calls, metadata, diff --git a/llm/openai/src/client.rs b/llm/openai/src/client.rs index 688939a3..afa66536 100644 --- a/llm/openai/src/client.rs +++ b/llm/openai/src/client.rs @@ -202,6 +202,8 @@ pub enum InnerInput { pub enum InnerInputItem { #[serde(rename = "input_text")] TextInput { text: String }, + #[serde(rename = "output_text")] + TextOutput { text: String }, #[serde(rename = "input_image")] ImageInput { image_url: String, diff --git a/llm/openai/src/conversions.rs b/llm/openai/src/conversions.rs index 43694c0f..19f501ec 100644 --- a/llm/openai/src/conversions.rs +++ b/llm/openai/src/conversions.rs @@ -43,16 +43,7 @@ pub fn create_request( pub fn messages_to_input_items(messages: Vec) -> Vec { let mut items = Vec::new(); for message in messages { - let role = to_openai_role_name(message.role).to_string(); - let mut input_items = Vec::new(); - for content_part in message.content { - input_items.push(content_part_to_inner_input_item(content_part)); - } - - items.push(InputItem::InputMessage { - role, - content: InnerInput::List(input_items), - }); + items.push(llm_message_to_openai_message(message)); } items } @@ -122,35 +113,48 @@ pub fn to_openai_role_name(role: Role) -> &'static str { } } -pub fn content_part_to_inner_input_item(content_part: ContentPart) -> InnerInputItem { - match content_part { - ContentPart::Text(msg) => InnerInputItem::TextInput { text: msg }, - ContentPart::Image(image_reference) => match image_reference { - ImageReference::Url(image_url) => InnerInputItem::ImageInput { - image_url: image_url.url, - detail: match image_url.detail { - Some(ImageDetail::Auto) => Detail::Auto, - Some(ImageDetail::Low) => Detail::Low, - Some(ImageDetail::High) => Detail::High, - None => Detail::default(), - }, +pub fn llm_message_to_openai_message(message: Message) -> InputItem { + let mut items = Vec::new(); + + for content_part in message.content { + let item = match content_part { + ContentPart::Text(msg) => match message.role { + Role::Assistant => InnerInputItem::TextOutput { text: msg }, + _ => InnerInputItem::TextInput { text: msg }, }, - ImageReference::Inline(image_source) => { - let base64_data = general_purpose::STANDARD.encode(&image_source.data); - let mime_type = &image_source.mime_type; // This is already a string - let data_url = format!("data:{};base64,{}", mime_type, base64_data); - - InnerInputItem::ImageInput { - image_url: data_url, - detail: match image_source.detail { + ContentPart::Image(image_reference) => match image_reference { + ImageReference::Url(image_url) => InnerInputItem::ImageInput { + image_url: image_url.url, + detail: match image_url.detail { Some(ImageDetail::Auto) => Detail::Auto, Some(ImageDetail::Low) => Detail::Low, Some(ImageDetail::High) => Detail::High, None => Detail::default(), }, + }, + ImageReference::Inline(image_source) => { + let base64_data = general_purpose::STANDARD.encode(&image_source.data); + let mime_type = &image_source.mime_type; // This is already a string + let data_url = format!("data:{mime_type};base64,{base64_data}"); + + InnerInputItem::ImageInput { + image_url: data_url, + detail: match image_source.detail { + Some(ImageDetail::Auto) => Detail::Auto, + Some(ImageDetail::Low) => Detail::Low, + Some(ImageDetail::High) => Detail::High, + None => Detail::default(), + }, + } } - } - }, + }, + }; + items.push(item); + } + + InputItem::InputMessage { + role: to_openai_role_name(message.role).to_string(), + content: InnerInput::List(items), } } diff --git a/llm/openrouter/src/conversions.rs b/llm/openrouter/src/conversions.rs index d4db2d34..61b5f973 100644 --- a/llm/openrouter/src/conversions.rs +++ b/llm/openrouter/src/conversions.rs @@ -184,7 +184,7 @@ fn convert_content_parts(contents: Vec) -> crate::client::Content { let media_type = &image_source.mime_type; // This is already a string result.push(crate::client::ContentPart::ImageInput { image_url: crate::client::ImageUrl { - url: format!("data:{};base64,{}", media_type, base64_data), + url: format!("data:{media_type};base64,{base64_data}"), detail: image_source.detail.map(|d| d.into()), }, }); diff --git a/test/components-rust/test-llm/Cargo.toml b/test/components-rust/test-llm/Cargo.toml index 7f624287..ca8b9eb7 100644 --- a/test/components-rust/test-llm/Cargo.toml +++ b/test/components-rust/test-llm/Cargo.toml @@ -37,8 +37,8 @@ path = "wit-generated" [package.metadata.component.target.dependencies] "golem:llm" = { path = "wit-generated/deps/golem-llm" } -"wasi:clocks" = { path = "wit-generated/deps/clocks" } "wasi:io" = { path = "wit-generated/deps/io" } +"wasi:clocks" = { path = "wit-generated/deps/clocks" } "golem:rpc" = { path = "wit-generated/deps/golem-rpc" } "test:helper-client" = { path = "wit-generated/deps/test_helper-client" } "test:llm-exports" = { path = "wit-generated/deps/test_llm-exports" } diff --git a/test/components-rust/test-llm/src/lib.rs b/test/components-rust/test-llm/src/lib.rs index fa11684d..a99419f6 100644 --- a/test/components-rust/test-llm/src/lib.rs +++ b/test/components-rust/test-llm/src/lib.rs @@ -1,11 +1,13 @@ #[allow(static_mut_refs)] mod bindings; -use golem_rust::atomically; use crate::bindings::exports::test::llm_exports::test_llm_api::*; use crate::bindings::golem::llm::llm; use crate::bindings::golem::llm::llm::StreamEvent; use crate::bindings::test::helper_client::test_helper_client::TestHelperApi; +use golem_rust::atomically; + +mod utils; struct Component; @@ -17,7 +19,7 @@ const MODEL: &'static str = "claude-3-7-sonnet-20250219"; const MODEL: &'static str = "grok-3-beta"; #[cfg(feature = "openrouter")] const MODEL: &'static str = "openrouter/auto"; -#[cfg(feature = "ollama")] +#[cfg(feature = "ollama")] const MODEL: &'static str = "qwen3:1.7b"; #[cfg(feature = "openai")] @@ -28,7 +30,7 @@ const IMAGE_MODEL: &'static str = "claude-3-7-sonnet-20250219"; const IMAGE_MODEL: &'static str = "grok-2-vision-latest"; #[cfg(feature = "openrouter")] const IMAGE_MODEL: &'static str = "openrouter/auto"; -#[cfg(feature = "ollama")] +#[cfg(feature = "ollama")] const IMAGE_MODEL: &'static str = "gemma3:4b"; impl Guest for Component { @@ -67,9 +69,14 @@ impl Guest for Component { .map(|content| match content { llm::ContentPart::Text(txt) => txt, llm::ContentPart::Image(image_ref) => match image_ref { - llm::ImageReference::Url(url_data) => format!("[IMAGE URL: {}]", url_data.url), - llm::ImageReference::Inline(inline_data) => format!("[INLINE IMAGE: {} bytes, mime: {}]", inline_data.data.len(), inline_data.mime_type), - } + llm::ImageReference::Url(url_data) => + format!("[IMAGE URL: {}]", url_data.url), + llm::ImageReference::Inline(inline_data) => format!( + "[INLINE IMAGE: {} bytes, mime: {}]", + inline_data.data.len(), + inline_data.mime_type + ), + }, }) .collect::>() .join(", ") @@ -154,7 +161,7 @@ impl Guest for Component { vec![] } }; - + if !tool_request.is_empty() { let mut calls = Vec::new(); for call in tool_request { @@ -385,9 +392,14 @@ impl Guest for Component { .map(|content| match content { llm::ContentPart::Text(txt) => txt, llm::ContentPart::Image(image_ref) => match image_ref { - llm::ImageReference::Url(url_data) => format!("[IMAGE URL: {}]", url_data.url), - llm::ImageReference::Inline(inline_data) => format!("[INLINE IMAGE: {} bytes, mime: {}]", inline_data.data.len(), inline_data.mime_type), - } + llm::ImageReference::Url(url_data) => + format!("[IMAGE URL: {}]", url_data.url), + llm::ImageReference::Inline(inline_data) => format!( + "[INLINE IMAGE: {} bytes, mime: {}]", + inline_data.data.len(), + inline_data.mime_type + ), + }, }) .collect::>() .join(", ") @@ -407,7 +419,7 @@ impl Guest for Component { } } - /// test6 simulates a crash during a streaming LLM response, but only first time. + /// test6 simulates a crash during a streaming LLM response, but only first time. /// after the automatic recovery it will continue and finish the request successfully. fn test6() -> String { let config = llm::Config { @@ -456,12 +468,20 @@ impl Guest for Component { } llm::ContentPart::Image(image_ref) => match image_ref { llm::ImageReference::Url(url_data) => { - result.push_str(&format!("IMAGE URL: {} ({:?})\n", url_data.url, url_data.detail)); + result.push_str(&format!( + "IMAGE URL: {} ({:?})\n", + url_data.url, url_data.detail + )); } llm::ImageReference::Inline(inline_data) => { - result.push_str(&format!("INLINE IMAGE: {} bytes, mime: {}, detail: {:?}\n", inline_data.data.len(), inline_data.mime_type, inline_data.detail)); + result.push_str(&format!( + "INLINE IMAGE: {} bytes, mime: {}, detail: {:?}\n", + inline_data.data.len(), + inline_data.mime_type, + inline_data.detail + )); } - } + }, } } } @@ -528,7 +548,10 @@ impl Guest for Component { role: llm::Role::User, name: None, content: vec![ - llm::ContentPart::Text("Please describe this cat image in detail. What breed might it be?".to_string()), + llm::ContentPart::Text( + "Please describe this cat image in detail. What breed might it be?" + .to_string(), + ), llm::ContentPart::Image(llm::ImageReference::Inline(llm::ImageSource { data: buffer, mime_type: "image/png".to_string(), @@ -549,9 +572,14 @@ impl Guest for Component { .map(|content| match content { llm::ContentPart::Text(txt) => txt, llm::ContentPart::Image(image_ref) => match image_ref { - llm::ImageReference::Url(url_data) => format!("[IMAGE URL: {}]", url_data.url), - llm::ImageReference::Inline(inline_data) => format!("[INLINE IMAGE: {} bytes, mime: {}]", inline_data.data.len(), inline_data.mime_type), - } + llm::ImageReference::Url(url_data) => + format!("[IMAGE URL: {}]", url_data.url), + llm::ImageReference::Inline(inline_data) => format!( + "[INLINE IMAGE: {} bytes, mime: {}]", + inline_data.data.len(), + inline_data.mime_type + ), + }, }) .collect::>() .join(", ") @@ -570,6 +598,84 @@ impl Guest for Component { } } } + fn test8() -> String { + let config = llm::Config { + model: MODEL.to_string(), + temperature: Some(0.2), + max_tokens: None, + stop_sequences: None, + tools: vec![], + tool_choice: None, + provider_options: vec![], + }; + + let mut messages = vec![llm::Message { + role: llm::Role::User, + name: Some("vigoo".to_string()), + content: vec![llm::ContentPart::Text( + "Do you know what a haiku is?".to_string(), + )], + }]; + + let stream = llm::stream(&messages, &config); + + let mut result = String::new(); + + loop { + match utils::consume_next_event(&stream) { + Some(delta) => { + result.push_str(&delta); + } + None => break, + } + } + + messages.push(llm::Message { + role: llm::Role::Assistant, + name: Some("assistant".to_string()), + content: vec![llm::ContentPart::Text(result)], + }); + + messages.push(llm::Message { + role: llm::Role::User, + name: Some("vigoo".to_string()), + content: vec![llm::ContentPart::Text( + "Can you write one for me?".to_string(), + )], + }); + + println!("Message: {messages:?}"); + + let stream = llm::stream(&messages, &config); + + let mut result = String::new(); + + let name = std::env::var("GOLEM_WORKER_NAME").unwrap(); + let mut round = 0; + + loop { + match utils::consume_next_event(&stream) { + Some(delta) => { + result.push_str(&delta); + } + None => break, + } + + if round == 2 { + atomically(|| { + let client = TestHelperApi::new(&name); + let answer = client.blocking_inc_and_get(); + if answer == 1 { + panic!("Simulating crash") + } + }); + } + + round += 1; + } + + result + } } bindings::export!(Component with_types_in bindings); diff --git a/test/components-rust/test-llm/src/utils.rs b/test/components-rust/test-llm/src/utils.rs new file mode 100644 index 00000000..7422b8f8 --- /dev/null +++ b/test/components-rust/test-llm/src/utils.rs @@ -0,0 +1,54 @@ +use crate::bindings::golem::llm::llm; + +pub fn consume_next_event(stream: &llm::ChatStream) -> Option { + let events = stream.blocking_get_next(); + + if events.is_empty() { + return None; + } + + let mut result = String::new(); + + for event in events { + println!("Received {event:?}"); + + match event { + llm::StreamEvent::Delta(delta) => { + for content in delta.content.unwrap_or_default() { + match content { + llm::ContentPart::Text(txt) => { + result.push_str(&txt); + } + llm::ContentPart::Image(image_ref) => match image_ref { + llm::ImageReference::Url(url_data) => { + result.push_str(&format!( + "IMAGE URL: {} ({:?})\n", + url_data.url, url_data.detail + )); + } + llm::ImageReference::Inline(inline_data) => { + result.push_str(&format!( + "INLINE IMAGE: {} bytes, mime: {}, detail: {:?}\n", + inline_data.data.len(), + inline_data.mime_type, + inline_data.detail + )); + } + }, + } + } + } + llm::StreamEvent::Finish(..) => {} + llm::StreamEvent::Error(error) => { + result.push_str(&format!( + "\nERROR: {:?} {} ({})\n", + error.code, + error.message, + error.provider_error_json.unwrap_or_default() + )); + } + } + } + + Some(result) +} diff --git a/test/components-rust/test-llm/wit/test-llm.wit b/test/components-rust/test-llm/wit/test-llm.wit index 37b4f419..97edad98 100644 --- a/test/components-rust/test-llm/wit/test-llm.wit +++ b/test/components-rust/test-llm/wit/test-llm.wit @@ -10,6 +10,7 @@ interface test-llm-api { test5: func() -> string; test6: func() -> string; test7: func() -> string; + test8: func() -> string; } world test-llm {