wip

Author: Rutik7066

Cargo.lock

@@ -1,7 +1,7 @@
 [workspace]
 resolver = "2"
 
-members = [  "golem-llm/llm", "golem-llm/llm-anthropic", "golem-llm/llm-grok", "golem-llm/llm-ollama", "golem-llm/llm-openai", "golem-llm/llm-openrouter", "golem-web-search/web-search", "golem-web-search/websearch-google"]
+members = [  "golem-llm/llm", "golem-llm/llm-anthropic", "golem-llm/llm-grok", "golem-llm/llm-ollama", "golem-llm/llm-openai", "golem-llm/llm-openrouter", "golem-web-search/web-search", "golem-web-search/websearch-brave", "golem-web-search/websearch-google", "golem-web-search/websearch-serper", "golem-web-search/websearch-tavily"]
 
 [profile.release]
 debug = false

Cargo.toml

@@ -20,9 +20,20 @@ install_crate = { crate_name = "cargo-component", version = "0.20.0" }
 command = "cargo-component"
 args = ["build", "-p", "golem-websearch-google", "--no-default-features"]
 
+[tasks.build-brave]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-websearch-brave"]
+
+[tasks.build-brave-portable]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-websearch-brave", "--no-default-features"]
+
 [tasks.build]
 dependencies = [
     "build-google",
+    "build-brave",
 ]
 
 [tasks.build-portable]

golem-web-search/Makefile.toml

@@ -10,7 +10,10 @@ required-features = []
 
 [features]
 default = ["google"]
-google =[]
+google = []
+brave = []
+tavily = []
+serper = []
 
 [dependencies]
 # To use common shared libs, use the following:

golem-web-search/test/components-rust/test-web-search/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "golem-web-search"
+version = "0.0.0"
+edition = "2021"
+license = "Apache-2.0"
+homepage = "https://golem.cloud"
+repository = "https://github.com/golemcloud/golem-llm"
+description = "WebAssembly components for working with AI models and providers APIs, with special support for Golem Cloud"
+
+[lib]
+path = "src/lib.rs"
+crate-type = ["rlib"]
+
+[dependencies]
+golem-rust = { workspace = true }
+log = { workspace = true }
+mime = "0.3.17"
+nom = { version = "7.1", default-features = false }
+reqwest = { workspace = true }
+thiserror = "2.0.12"
+wasi-logger = "0.1.2"
+wit-bindgen = { version = "0.40.0" }
+
+[features]
+default = ["durability"]
+durability = ["golem-rust/durability"]

golem-web-search/web-search/Cargo.toml

@@ -0,0 +1,24 @@
+use crate::exports::golem::web_search::web_search::{SearchError};
+use std::{collections::HashMap, ffi::OsStr};
+
+/// Gets an expected configuration value from the environment, and fails if its is not found
+/// using the `fail` function. Otherwise, it runs `succeed` with the configuration value.
+pub fn with_config_key<R>(
+    keys: &[impl AsRef<OsStr>],
+    fail: impl FnOnce(SearchError) -> R,
+    succeed: impl FnOnce(HashMap<String,String>) -> R,
+) -> R {
+    let mut hashmap = HashMap::new();
+    for key in keys {
+        match std::env::var(key.as_ref()) {
+            Ok(value) => {
+                hashmap.insert(key.as_ref().to_string_lossy().to_string(), value);
+            }
+            Err(_) => {
+                let error = SearchError::BackendError(format!("Missing config key: {}", key.as_ref().to_string_lossy()));
+                return fail(error); 
+            }
+        }
+    }
+    succeed(hashmap)
+}

golem-web-search/web-search/src/config.rs

@@ -0,0 +1,111 @@
+use std::marker::PhantomData;
+
+/// Wraps a web search implementation with custom durability
+pub struct DurableWebSearch<Impl> {
+    phantom: PhantomData<Impl>,
+}
+
+/// When the durability feature flag is off, wrapping with `DurableWebSearch` is just a passthrough
+#[cfg(not(feature = "durability"))]
+mod passthrough_impl {
+    use crate::durability::DurableWebSearch;
+    use crate::exports::golem::web_search::web_search::{
+        Guest, SearchError, SearchMetadata, SearchParams, SearchResult, SearchSession,
+    };
+
+    impl<Impl: Guest> Guest for DurableWebSearch<Impl> {
+        type SearchSession = Impl::SearchSession;
+
+        fn start_search(params: SearchParams) -> Result<SearchSession, SearchError> {
+            Impl::start_search(params)
+        }
+
+        fn search_once(
+            params: SearchParams,
+        ) -> Result<(Vec<SearchResult>, Option<SearchMetadata>), SearchError> {
+            Impl::search_once(params)
+        }
+    }
+}
+
+/// When the durability feature flag is on, wrapping with `DurableWebSearch` adds custom durability
+/// on top of the provider-specific web search implementation using Golem's special host functions and
+/// the `golem-rust` helper library.
+///
+/// There will be custom durability entries saved in the oplog, with the full web search request and configuration
+/// stored as input, and the full response stored as output. To serialize these in a way it is
+/// observable by oplog consumers, each relevant data type has to be converted to/from `ValueAndType`
+/// which is implemented using the type classes and builder in the `golem-rust` library.
+#[cfg(feature = "durability")]
+mod durable_impl {
+    use crate::durability::DurableWebSearch;
+    use crate::exports::golem::web_search::web_search::{
+        Guest, SearchError, SearchMetadata, SearchParams, SearchResult, SearchSession,
+    };
+    use golem_rust::bindings::golem::durability::durability::DurableFunctionType;
+    use golem_rust::durability::Durability;
+    use golem_rust::{with_persistence_level, FromValueAndType, IntoValue, PersistenceLevel};
+    use std::fmt::{Display, Formatter};
+
+    impl<Impl: Guest> Guest for DurableWebSearch<Impl> {
+        type SearchSession = Impl::SearchSession;
+
+        fn start_search(params: SearchParams) -> Result<SearchSession, SearchError> {
+            let durability = Durability::<NoOutput, UnusedError>::new(
+                "golem_web_search",
+                "start_search",
+                DurableFunctionType::WriteRemote,
+            );
+            if durability.is_live() {
+                let result = with_persistence_level(PersistenceLevel::PersistNothing, || {
+                    Impl::start_search(params.clone())
+                });
+                match result {
+                    Ok(session) => {
+                        let _ = durability.persist_infallible(params, NoOutput);
+                        Ok(session)
+                    }
+                    Err(err) => Err(err),
+                }
+            } else {
+                let _: NoOutput = durability.replay_infallible();
+                Impl::start_search(params)
+            }
+        }
+
+        fn search_once(
+            params: SearchParams,
+        ) -> Result<(Vec<SearchResult>, Option<SearchMetadata>), SearchError> {
+            let durability = Durability::<(Vec<SearchResult>, Option<SearchMetadata>), UnusedError>::new(
+                "golem_web_search",
+                "search_once",
+                DurableFunctionType::WriteRemote,
+            );
+            if durability.is_live() {
+                let result = with_persistence_level(PersistenceLevel::PersistNothing, || {
+                    Impl::search_once(params.clone())
+                });
+                match result {
+                    Ok(success) => Ok(durability.persist_infallible(params, success)),
+                    Err(err) => Err(err),
+                }
+            } else {
+                let result: (Vec<SearchResult>, Option<SearchMetadata>) =
+                    durability.replay_infallible();
+                Ok(result)
+            }
+        }
+    }
+
+    #[derive(Debug, Clone, IntoValue, FromValueAndType)]
+    struct NoOutput;
+
+    #[derive(Debug, IntoValue, FromValueAndType)]
+    struct UnusedError;
+
+    impl Display for UnusedError {
+        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            write!(f, "UnusedError")
+        }
+    }
+}

golem-web-search/web-search/src/durability.rs

@@ -0,0 +1,12 @@
+
+use crate::exports::golem::web_search::web_search::SearchError;
+
+/// Creates an `Error` value representing that something is unsuported
+pub fn unsupported(what: impl AsRef<str>) -> SearchError {
+    SearchError::UnsupportedFeature(format!("Unsupported: {}", what.as_ref()))
+}
+
+pub fn from_reqwest_error(details: impl AsRef<str>, err: reqwest::Error) -> SearchError {
+    SearchError::BackendError(format!("{}: {err}", details.as_ref()))
+}
+

golem-web-search/web-search/src/error.rs

@@ -0,0 +1,43 @@
+pub mod config;
+pub mod durability;
+pub mod error;
+
+
+wit_bindgen::generate!({
+    path: "../wit",
+    world: "web-search-library",
+    generate_all,
+    generate_unused_types: true,
+    additional_derives: [PartialEq, golem_rust::FromValueAndType, golem_rust::IntoValue],
+    pub_export_macro: true,
+});
+
+pub use crate::exports::golem;
+pub use __export_web_search_library_impl as export_web_search;
+use std::cell::RefCell;
+use std::str::FromStr;
+
+pub struct LoggingState {
+    logging_initialized: bool,
+}
+
+impl LoggingState {
+    /// Initializes WASI logging based on the `GOLEM_WEB_SEARCH_LOG` environment variable.
+    pub fn init(&mut self) {
+        if !self.logging_initialized {
+            let _ = wasi_logger::Logger::install();
+            let max_level: log::LevelFilter =
+                log::LevelFilter::from_str(&std::env::var("GOLEM_WEB_SEARCH_LOG").unwrap_or_default())
+                    .unwrap_or(log::LevelFilter::Info);
+            log::set_max_level(max_level);
+            self.logging_initialized = true;
+        }
+    }
+}
+
+thread_local! {
+    /// This holds the state of our application.
+    pub static LOGGING_STATE: RefCell<LoggingState> = const { RefCell::new(LoggingState {
+        logging_initialized: false,
+    }) };
+}

golem-web-search/web-search/src/lib.rs

@@ -0,0 +1,15 @@
+[package]
+name = "websearch-brave"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+wit-bindgen-rt = { workspace = true, features = ["bitflags"] }
+
+[lib]
+crate-type = ["cdylib"]
+
+[package.metadata.component]
+package = "component:websearch-brave"
+
+[package.metadata.component.dependencies]