Skip to content

Commit 2c55648

Browse files
paraltapbalogh-sa
andauthored
feat: add build config samples (#14)
* feat: inital samples build test --------- Signed-off-by: Peter Balogh <p.balogh.sa@gmail.com> Co-authored-by: Peter Balogh <p.balogh.sa@gmail.com>
1 parent 5523449 commit 2c55648

File tree

11 files changed

+679
-4
lines changed

11 files changed

+679
-4
lines changed

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,3 +123,6 @@ integrations/**/charts/*
123123

124124
# Kubeconfig
125125
integrations/kubeconfig.yaml
126+
127+
samples/**/report.md
128+
samples/evaluation/.deepeval*

integrations/Taskfile.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,12 @@ tasks:
121121
- docker pull {{.IMAGE_REPO}}/dir-ctl:{{.DIRECTORY_IMAGE_TAG}}
122122
- REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} IMAGE_REPO={{.IMAGE_REPO}} DIRECTORY_IMAGE_TAG={{.DIRECTORY_IMAGE_TAG}} go test ./agntcy-dir/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 10m -timeout 10m -ginkgo.v -ginkgo.focus "agent compilation"
123123

124+
test:directory:compile:samples:
125+
desc: Agntcy compiler test in samples
126+
cmds:
127+
- docker pull {{.IMAGE_REPO}}/dir-ctl:{{.DIRECTORY_IMAGE_TAG}}
128+
- REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} IMAGE_REPO={{.IMAGE_REPO}} DIRECTORY_IMAGE_TAG={{.DIRECTORY_IMAGE_TAG}} go test ./agntcy-dir/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 10m -timeout 10m -ginkgo.v -ginkgo.focus "Samples build test"
129+
124130
test:directory:push:
125131
desc: Directory agent push test
126132
cmds:

integrations/agntcy-dir/tests/compiler_test.go renamed to integrations/agntcy-dir/tests/compiler_sanity_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import (
1616
"github.com/agntcy/csit/integrations/testutils"
1717
)
1818

19-
var _ = ginkgo.Describe("Agntcy compiler tests", func() {
19+
var _ = ginkgo.Describe("Agntcy compiler sanity tests", func() {
2020
var (
2121
tempAgentPath string
2222
dockerImage string
@@ -42,7 +42,6 @@ var _ = ginkgo.Describe("Agntcy compiler tests", func() {
4242

4343
ginkgo.Context("agent compilation", func() {
4444
ginkgo.It("should compile an agent", func() {
45-
4645
dirctlArgs := []string{
4746
"build",
4847
"--config",
@@ -80,15 +79,16 @@ var _ = ginkgo.Describe("Agntcy compiler tests", func() {
8079
// Ensure the path is deep enough
8180
if len(p) >= 3 {
8281
if mapStep, ok := p[len(p)-3].(cmp.MapIndex); ok {
83-
if key, ok := mapStep.Key().Interface().(string); ok && key == "created_at" {
82+
if key, ok := mapStep.Key().Interface().(string); ok && key == "created_at" || key == "extensions" {
8483
return true // Ignore these paths
8584
}
8685
}
8786
}
8887
return false // Include all other paths
8988
}, cmp.Ignore())
9089

91-
gomega.Expect(expected).To(gomega.BeComparableTo(compiled, filter))
90+
gomega.Expect(expected).Should(gomega.BeComparableTo(compiled, filter))
91+
gomega.Expect(expected["extensions"]).Should(gomega.ConsistOf(compiled["extensions"]))
9292
})
9393
})
9494
})
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2025 Cisco and/or its affiliates.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package tests
5+
6+
import (
7+
"fmt"
8+
"os"
9+
"path/filepath"
10+
)
11+
12+
func FindFilePairs(rootDir, buildConfigName, expectedModelName string) ([]string, error) {
13+
directories := make(map[string]int)
14+
15+
err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
16+
if err != nil {
17+
return err
18+
}
19+
if !info.IsDir() {
20+
dir := filepath.Dir(path)
21+
if info.Name() == buildConfigName || info.Name() == expectedModelName {
22+
directories[dir]++
23+
}
24+
}
25+
26+
return nil
27+
})
28+
29+
if err != nil {
30+
return nil, fmt.Errorf("error walking the path %q: %v", rootDir, err)
31+
}
32+
33+
dirs := make([]string, 0)
34+
for dir, count := range directories {
35+
if count >= 2 {
36+
dirs = append(dirs, dir)
37+
}
38+
}
39+
40+
return dirs, nil
41+
}
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2025 Cisco and/or its affiliates.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package tests
5+
6+
import (
7+
"encoding/json"
8+
"fmt"
9+
"os"
10+
"path/filepath"
11+
"strings"
12+
13+
"github.com/agntcy/csit/integrations/testutils"
14+
"github.com/google/go-cmp/cmp"
15+
ginkgo "github.com/onsi/ginkgo/v2"
16+
"github.com/onsi/gomega"
17+
)
18+
19+
const (
20+
buildConfigName = "build.config.yaml"
21+
expectedModelName = "model.json"
22+
samplesPath = "../../../samples"
23+
)
24+
25+
var _ = ginkgo.Describe("Samples build test", func() {
26+
var (
27+
dockerImage string
28+
samples []string
29+
)
30+
31+
samplesDir, err := filepath.Abs(samplesPath)
32+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
33+
samples, err = FindFilePairs(samplesDir, buildConfigName, expectedModelName)
34+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
35+
36+
dockerImage = fmt.Sprintf("%s/dir-ctl:%s", os.Getenv("IMAGE_REPO"), os.Getenv("DIRECTORY_IMAGE_TAG"))
37+
38+
_, err = fmt.Fprintf(ginkgo.GinkgoWriter, "samples: %v\n", samples)
39+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
40+
41+
for _, entry := range samples {
42+
entry := entry
43+
ginkgo.Context(entry, func() {
44+
var (
45+
tempAgentPath string
46+
mountDest string
47+
mountString string
48+
modelConfigFilePath string
49+
expectedAgentModelFile string
50+
)
51+
52+
ginkgo.BeforeEach(func() {
53+
mountDest := fmt.Sprintf("/%s", filepath.Base(entry))
54+
mountString = fmt.Sprintf("%s:%s", entry, mountDest)
55+
modelConfigFilePath = filepath.Join(mountDest, buildConfigName)
56+
expectedAgentModelFile = filepath.Join(entry, expectedModelName)
57+
tempFileName := fmt.Sprintf("%s.json", strings.ReplaceAll(entry, "/", "-"))
58+
tempAgentPath = filepath.Join(os.TempDir(), tempFileName)
59+
})
60+
61+
ginkgo.It("Should compile", func() {
62+
_, err := fmt.Fprintf(ginkgo.GinkgoWriter, "Compiling agent model: %v\n", entry)
63+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
64+
_, err = fmt.Fprintf(ginkgo.GinkgoWriter, "tempagent path: %v\n", tempAgentPath)
65+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
66+
67+
dirctlArgs := []string{
68+
"build",
69+
"--config",
70+
modelConfigFilePath,
71+
mountDest,
72+
}
73+
runner := testutils.NewDockerRunner(dockerImage, mountString, nil)
74+
outputBuffer, err := runner.Run(dirctlArgs...)
75+
gomega.Expect(err).NotTo(gomega.HaveOccurred(), outputBuffer.String())
76+
77+
err = os.WriteFile(tempAgentPath, outputBuffer.Bytes(), 0644)
78+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
79+
})
80+
81+
ginkgo.It("Agent model should be the expected", func() {
82+
var expected, compiled map[string]any
83+
84+
_, err = fmt.Fprintf(ginkgo.GinkgoWriter, "tempagent path: %v\n", tempAgentPath)
85+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
86+
87+
expactedModelJSON, err := os.ReadFile(expectedAgentModelFile)
88+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
89+
90+
// Unmarshal or Decode the JSON to the interface.
91+
err = json.Unmarshal([]byte(expactedModelJSON), &expected)
92+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
93+
94+
compiledModelJSON, err := os.ReadFile(tempAgentPath)
95+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
96+
97+
// Unmarshal or Decode the JSON to the interface.
98+
err = json.Unmarshal([]byte(compiledModelJSON), &compiled)
99+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
100+
101+
// Filter "created_at" field and extensions
102+
filter := cmp.FilterPath(func(p cmp.Path) bool {
103+
// Ensure the path is deep enough
104+
if len(p) >= 3 {
105+
if mapStep, ok := p[len(p)-3].(cmp.MapIndex); ok {
106+
if key, ok := mapStep.Key().Interface().(string); ok && key == "created_at" || key == "extensions" {
107+
return true // Ignore these paths
108+
}
109+
}
110+
}
111+
return false // Include all other paths
112+
}, cmp.Ignore())
113+
114+
gomega.Expect(expected).Should(gomega.BeComparableTo(compiled, filter))
115+
gomega.Expect(expected["extensions"]).Should(gomega.ConsistOf(compiled["extensions"]))
116+
})
117+
})
118+
}
119+
})
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
model:
2+
name: "research_crew"
3+
4+
version: "v0.1.18"
5+
6+
authors:
7+
- "Your Name <you@example.com>"
8+
9+
skills:
10+
- "Natural Language Understanding"
11+
- "Natural Language Generation"
12+
- "Information Retrieval and Synthesis"
13+
- "Fact Extraction"
14+
- "Knowledge Synthesis"
15+
16+
locators:
17+
- type: "source-code"
18+
url: "https://github.com/agntcy/csit/tree/main/samples/crewai/simple_crew"
19+
20+
extensions:
21+
- name: "oasf.agntcy.org/features/runtime/io-mapper"
22+
version: "v1.0.0"
23+
specs:
24+
input_name: "topic"
25+
input_type: "string"
26+
output_name: "report"
27+
output_type: "file"
28+
output_description: "A fully fledge reports with the mains topics, each with a full section of information. Formatted as markdown without '```'"
29+
- name: "oasf.agntcy.org/features/observability/logging"
30+
version: "v1.0.0"
31+
specs:
32+
type: "stdout"
33+
format: "<string>"
34+
- name: "oasf.agntcy.org/features/observability/metrics"
35+
version: "v1.0.0"
36+
specs:
37+
token_usage:
38+
- "total_tokens"
39+
- "prompt_tokens"
40+
- "cached_prompt_tokens"
41+
- "completion_tokens"
42+
- "successful_requests"
43+
task_duration: "task_duration"
44+
- name: "oasf.agntcy.org/features/framework/llm"
45+
version: "v1.0.0"
46+
specs:
47+
model: "ollama/llama3.1"
48+
base_url: "http://localhost:11434"
49+
- name: "oasf.agntcy.org/features/framework/evaluation"
50+
version: "v1.0.0"
51+
specs:
52+
provider: "local"
53+
type: "evaluator agent"
54+
- name: "oasf.agntcy.org/features/framework/orchestration"
55+
version: "v1.0.0"
56+
specs:
57+
type: "sequential"
58+
- name: "oasf.agntcy.org/features/framework/memory"
59+
version: "v1.0.0"
60+
specs:
61+
enabled: false
62+
63+
builder:
64+
source: "/simple_crew" # TODO(pbalogh-sa) please update
65+
source-ignore:
66+
- ".venv/"
67+
68+
llmanalyzer: false
69+
crewai: true
70+
runtime: true

0 commit comments

Comments
 (0)