feat: added common embeddings providers components

.github/workflows/ci.yaml

@@ -137,7 +137,7 @@ jobs:
       - name: Build and test Ollama integration
         run: |
           set -e
-          cargo make --cwd llm build-ollama
+          cargo make build
           cd test
           golem-cli app build -b ollama-debug
           golem-cli app deploy -b ollama-debug
@@ -183,4 +183,4 @@ jobs:
       - name: Login GH CLI
         shell: bash
         run: gh auth login --with-token < <(echo ${{ secrets.GITHUB_TOKEN }})
-      - run: gh release upload -R golemcloud/golem-llm --clobber ${{ github.ref_name }} components/release/*.wasm
+      - run: gh release upload -R golemcloud/golem-llm --clobber ${{ github.ref_name }} components/release/*.wasm

Cargo.toml

@@ -8,6 +8,11 @@ members = [
     "llm/ollama",
     "llm/openai",
     "llm/openrouter",
+    "embed/embed",
+    "embed/cohere",
+    "embed/hugging-face",
+    "embed/openai", 
+    "embed/voyageai",
 ]
 
 [profile.release]

Makefile.toml

@@ -6,14 +6,20 @@ skip_core_tasks = true
 command = "cargo"
 args = ["clean"]
 
+[tasks.clean-all]
+script_runner = "@duckscript"
+script = '''
+exec find . -type d \( -name "target" -o -name "wit-generated" -o -name "golem-temp" \) -exec rm -rf {} +
+'''
+
 [tasks.unit-tests]
 command = "cargo"
 args = ["test"]
 
 [tasks.build]
 script_runner = "@duckscript"
 script = '''
-domains = array llm
+domains = array llm embed
 
 # if there is no domain passed run for every domain
 if is_empty ${1}
@@ -28,7 +34,7 @@ end
 [tasks.release-build]
 script_runner = "@duckscript"
 script = '''
-domains = array llm
+domains = array llm embed
 
 # if there is no domain passed run for every domain
 if is_empty ${1}
@@ -44,7 +50,7 @@ end
 script_runner = "@duckscript"
 script = '''
 #!/bin/bash
-domains = array llm
+domains = array llm embed
 
 # if there is no domain passed run for every domain
 if is_empty ${1}
@@ -60,7 +66,7 @@ end
 script_runner = "@duckscript"
 script = '''
 #!/bin/bash
-domains = array llm
+domains = array llm embed
 
 # if there is no domain passed run for every domain
 if is_empty ${1}
@@ -75,7 +81,7 @@ end
 [tasks.wit]
 script_runner = "@duckscript"
 script = '''
-domains = array llm
+domains = array llm embed
 
 # if there is no domain passed run for every domain
 if is_empty ${1}
@@ -91,7 +97,7 @@ end
 description = "Builds all test components with golem-cli"
 script_runner = "@duckscript"
 script = '''
-domains = array llm
+domains = array llm embed
 
 # if there is no domain passed run for every domain
 if is_empty ${1}
@@ -137,7 +143,7 @@ script = '''
 
 is_portable = eq ${1} "--portable"
 
-targets = array llm_openai llm_anthropic llm_grok llm_openrouter llm_ollama
+targets = array llm_openai llm_anthropic llm_grok llm_openrouter llm_ollama embed_openai embed_cohere embed_hugging_face embed_voyageai
 for target in ${targets}
     if is_portable
         cp target/wasm32-wasip1/debug/golem_${target}.wasm components/debug/golem_${target}-portable.wasm
@@ -153,7 +159,7 @@ script = '''
 
 is_portable = eq ${1} "--portable"
 
-targets = array llm_openai llm_anthropic llm_grok llm_openrouter llm_ollama
+targets = array llm_openai llm_anthropic llm_grok llm_openrouter llm_ollama embed_openai embed_cohere embed_hugging_face embed_voyageai
 for target in ${targets}
     if is_portable
         cp target/wasm32-wasip1/release/golem_${target}.wasm components/release/golem_${target}-portable.wasm
@@ -229,4 +235,4 @@ foreach ($file in $cargoFiles)
     Foreach-Object { $_ -replace "0.0.0", $Env:VERSION } |
     Set-Content $file.PSPath
 }
-'''
+'''

embed/Makefile.toml

@@ -0,0 +1,169 @@
+[config]
+default_to_workspace = false
+skip_core_tasks = true
+
+[tasks.build]
+run_task = { name = [
+    "build-openai",
+    "build-cohere",
+    "build-hugging-face",
+    "build-voyageai",
+] }
+
+[tasks.build-portable]
+run_task = { name = [
+    "build-openai-portable",
+    "build-cohere-portable",
+    "build-hugging-face-portable",
+    "build-voyageai-portable",
+] }
+
+[tasks.release-build]
+run_task = { name = [
+    "release-build-openai",
+    "release-build-cohere",
+    "release-build-hugging-face",
+    "release-build-voyageai",
+] }
+
+[tasks.release-build-portable]
+run_task = { name = [
+    "release-build-openai-portable",
+    "release-build-cohere-portable",
+    "release-build-hugging-face-portable",
+    "release-build-voyageai-portable",
+] }
+
+[tasks.build-openai]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-openai"]
+
+[tasks.build-openai-portable]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-openai", "--no-default-features"]
+
+[tasks.build-cohere]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-cohere"]
+
+[tasks.build-cohere-portable]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-cohere", "--no-default-features"]
+
+[tasks.build-hugging-face]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-hugging-face"]
+
+[tasks.build-hugging-face-portable]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-hugging-face", "--no-default-features"]
+
+[tasks.build-voyageai]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-voyageai"]
+
+[tasks.build-voyageai-portable]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-voyageai", "--no-default-features"]
+
+[tasks.release-build-openai]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-openai", "--release"]
+
+[tasks.release-build-openai-portable]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-openai", "--release", "--no-default-features"]
+
+[tasks.release-build-cohere]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-cohere", "--release"]
+
+[tasks.release-build-cohere-portable]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-cohere", "--release", "--no-default-features"]
+
+[tasks.release-build-hugging-face]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-hugging-face", "--release"]
+
+[tasks.release-build-hugging-face-portable]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-hugging-face", "--release", "--no-default-features"]
+
+[tasks.release-build-voyageai]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-voyageai", "--release"]
+
+[tasks.release-build-voyageai-portable]
+install_crate = { crate_name = "cargo-component", version = "0.20.0" }
+command = "cargo-component"
+args = ["build", "-p", "golem-embed-voyageai", "--release", "--no-default-features"]
+
+[tasks.wit-update]
+install_crate = { crate_name = "wit-deps-cli" }
+command = "wit-deps"
+args = ["update"]
+
+[tasks.wit]
+dependencies = ["wit-update"]
+
+script_runner = "@duckscript"
+script = """
+modules = array embed openai cohere hugging-face voyageai
+
+for module in ${modules}
+    rm -r ${module}/wit/deps
+    mkdir ${module}/wit/deps/golem-embed
+    cp wit/golem-embed.wit ${module}/wit/deps/golem-embed/golem-embed.wit
+    cp wit/deps/wasi:io ${module}/wit/deps
+
+    echo "Copied WIT for module embed::${module}"
+end
+
+# Copy WIT files for integration tests
+rm -r test/wit
+mkdir test/wit/deps/golem-embed
+mkdir test/wit/deps/io
+cp wit/golem-embed.wit test/wit/deps/golem-embed/golem-embed.wit
+cp wit/deps/wasi:io/error.wit test/wit/deps/io/error.wit
+cp wit/deps/wasi:io/poll.wit test/wit/deps/io/poll.wit
+cp wit/deps/wasi:io/streams.wit test/wit/deps/io/streams.wit
+cp wit/deps/wasi:io/world.wit test/wit/deps/io/world.wit
+
+echo "Copied WIT for module test"
+"""
+
+[tasks.build-test-components]
+dependencies = ["build"]
+install_crate = "cargo-binstall"
+description = "Builds embed test components with golem-cli"
+script = '''
+cargo-binstall golem-cli@1.2.2-dev.11 --locked --no-confirm
+cargo-binstall wac-cli --locked --no-confirm
+cd test
+
+golem-cli --version
+golem-cli app clean
+golem-cli app build -b openai-debug
+golem-cli app clean
+golem-cli app build -b cohere-debug
+golem-cli app clean
+golem-cli app build -b hugging-face-debug
+golem-cli app clean
+golem-cli app build -b voyageai-debug
+'''