Skip to content

Commit b6eea25

Browse files
authored
Local runner feature for integration tests (#48)
* feat(integrations): add different runner options Signed-off-by: Árpád Csepi <csepi.arpad@outlook.com> * test(integrations): testcase for local and docker runner Signed-off-by: Árpád Csepi <csepi.arpad@outlook.com> * fix(integrations): align test cases with new runner Signed-off-by: Árpád Csepi <csepi.arpad@outlook.com> * feat(integrations): add download task for dirctl Signed-off-by: Árpád Csepi <csepi.arpad@outlook.com> * fix(integrations): use new runner in agp tests Signed-off-by: Árpád Csepi <csepi.arpad@outlook.com> * fix(integrations): directory creation for bins Signed-off-by: Árpád Csepi <csepi.arpad@outlook.com> --------- Signed-off-by: Árpád Csepi <csepi.arpad@outlook.com>
1 parent 08eb7c2 commit b6eea25

File tree

11 files changed

+468
-91
lines changed

11 files changed

+468
-91
lines changed

.gitignore

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,4 +125,7 @@ integrations/**/charts/*
125125
integrations/kubeconfig.yaml
126126

127127
samples/**/report.md
128-
samples/evaluation/.deepeval*
128+
samples/evaluation/.deepeval*
129+
130+
# Binaries for testing
131+
integrations/**/.bin

integrations/Taskfile.yaml

Lines changed: 56 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,32 @@ vars:
2424

2525
REMOVE_CONTAINERS: '{{ .REMOVE_CONTAINERS | default "true" }}'
2626

27+
RUNNER_TYPE: '{{ .RUNNER_TYPE | default "docker" }}'
28+
29+
ARCH:
30+
sh: |
31+
if [ "$ARCH" == "" ]; then
32+
uname -m
33+
else
34+
printf $ARCH
35+
fi
36+
OS:
37+
sh: |
38+
if [ "$OS" == "" ]; then
39+
uname -s
40+
else
41+
printf $OS
42+
fi
43+
DIRCTL_BIN_PATH:
44+
sh: |
45+
if [ "$DIRCTL_BIN_PATH" == "" ]; then
46+
mkdir -p $PWD/agntcy-dir/tests/.bin
47+
printf $PWD/agntcy-dir/tests/.bin
48+
else
49+
printf $DIRCTL_BIN_PATH
50+
fi
51+
DIRCTL_BIN_VERSION: '{{ .DIRCTL_BIN_VERSION | default "v0.2.0" }}'
52+
2753
tasks:
2854
kind:create:
2955
desc: Create kind cluster
@@ -113,35 +139,35 @@ tasks:
113139
- task: k8s:port-forward:setup:directory
114140
- defer: { task: k8s:port-forward:teardown:directory }
115141
- defer: { task: manifests:cleanup }
116-
- REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} IMAGE_REPO={{.IMAGE_REPO}} DIRECTORY_IMAGE_TAG={{.DIRECTORY_IMAGE_TAG}} go test ./agntcy-dir/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 30m -timeout 30m -ginkgo.v
142+
- REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} RUNNER_TYPE={{.RUNNER_TYPE}} PATH=$PATH:{{.DIRCTL_BIN_PATH}} IMAGE_REPO={{.IMAGE_REPO}} DIRECTORY_IMAGE_TAG={{.DIRECTORY_IMAGE_TAG}} go test ./agntcy-dir/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 30m -timeout 30m -ginkgo.v
117143

118144
test:directory:compiler:
119145
desc: Agntcy compiler test
120146
cmds:
121147
- docker pull {{.IMAGE_REPO}}/dir-ctl:{{.DIRECTORY_IMAGE_TAG}}
122-
- REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} IMAGE_REPO={{.IMAGE_REPO}} DIRECTORY_IMAGE_TAG={{.DIRECTORY_IMAGE_TAG}} go test ./agntcy-dir/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 10m -timeout 10m -ginkgo.v -ginkgo.focus "agent compilation"
148+
- REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} RUNNER_TYPE={{.RUNNER_TYPE}} PATH=$PATH:{{.DIRCTL_BIN_PATH}} IMAGE_REPO={{.IMAGE_REPO}} DIRECTORY_IMAGE_TAG={{.DIRECTORY_IMAGE_TAG}} go test ./agntcy-dir/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 10m -timeout 10m -ginkgo.v -ginkgo.focus "agent compilation"
123149

124150
test:directory:compile:samples:
125151
desc: Agntcy compiler test in samples
126152
cmds:
127153
- docker pull {{.IMAGE_REPO}}/dir-ctl:{{.DIRECTORY_IMAGE_TAG}}
128-
- REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} IMAGE_REPO={{.IMAGE_REPO}} DIRECTORY_IMAGE_TAG={{.DIRECTORY_IMAGE_TAG}} go test ./agntcy-dir/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 10m -timeout 10m -ginkgo.v -ginkgo.focus "Samples build test"
154+
- REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} RUNNER_TYPE={{.RUNNER_TYPE}} PATH=$PATH:{{.DIRCTL_BIN_PATH}} IMAGE_REPO={{.IMAGE_REPO}} DIRECTORY_IMAGE_TAG={{.DIRECTORY_IMAGE_TAG}} go test ./agntcy-dir/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 10m -timeout 10m -ginkgo.v -ginkgo.focus "Samples build test"
129155

130156
test:directory:push:
131157
desc: Directory agent push test
132158
cmds:
133159
- task: k8s:port-forward:setup:directory
134160
- defer: { task: k8s:port-forward:teardown:directory }
135161
- defer: { task: manifests:cleanup }
136-
- REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} IMAGE_REPO={{.IMAGE_REPO}} DIRECTORY_IMAGE_TAG={{.DIRECTORY_IMAGE_TAG}} go test ./agntcy-dir/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 30m -timeout 30m -ginkgo.v -ginkgo.focus "agent push and pull"
162+
- REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} RUNNER_TYPE={{.RUNNER_TYPE}} PATH=$PATH:{{.DIRCTL_BIN_PATH}} IMAGE_REPO={{.IMAGE_REPO}} DIRECTORY_IMAGE_TAG={{.DIRECTORY_IMAGE_TAG}} go test ./agntcy-dir/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 30m -timeout 30m -ginkgo.v -ginkgo.focus "agent push and pull"
137163

138164
test:directory:list:
139165
desc: Directory agent list test
140166
cmds:
141167
- task: k8s:port-forward:setup:directory
142168
- defer: { task: k8s:port-forward:teardown:directory }
143169
- defer: { task: manifests:cleanup }
144-
- REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} IMAGE_REPO={{.IMAGE_REPO}} DIRECTORY_IMAGE_TAG={{.DIRECTORY_IMAGE_TAG}} go test ./agntcy-dir/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 30m -timeout 30m -ginkgo.v -ginkgo.focus "Agntcy agent list tests"
170+
- REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} RUNNER_TYPE={{.RUNNER_TYPE}} PATH=$PATH:{{.DIRCTL_BIN_PATH}} IMAGE_REPO={{.IMAGE_REPO}} DIRECTORY_IMAGE_TAG={{.DIRECTORY_IMAGE_TAG}} go test ./agntcy-dir/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 30m -timeout 30m -ginkgo.v -ginkgo.focus "Agntcy agent list tests"
145171

146172
test:autogen-agent:run:
147173
internal: true
@@ -183,9 +209,33 @@ tasks:
183209
- task: test:autogen-agent:run
184210
- defer: { task: test:autogen-agent:remove }
185211
- docker pull {{ .IMAGE_REPO }}/csit/test-langchain-agent:{{ .TEST_APP_TAG }}
186-
- REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} IMAGE_REPO={{.IMAGE_REPO}} TEST_APP_TAG={{.TEST_APP_TAG}} go test ./agntcy-agp/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 30m -timeout 30m -ginkgo.v
212+
- REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} RUNNER_TYPE={{.RUNNER_TYPE}} IMAGE_REPO={{.IMAGE_REPO}} TEST_APP_TAG={{.TEST_APP_TAG}} go test ./agntcy-agp/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 30m -timeout 30m -ginkgo.v
187213

188214
version:
189215
desc: Get version
190216
cmds:
191217
- git describe --tags --match "v*" | cut -c 2-
218+
219+
download:dirctl-bin:
220+
desc: Get dirctl binary from GitHub
221+
vars:
222+
ARCH:
223+
sh: |
224+
printf "%s" "{{.ARCH}}" | \
225+
sed -E 's/^(aarch64|aarch64_be|armv6l|armv7l|armv8b|armv8l)$$/arm64/g' | \
226+
sed -E 's/^x86_64$$/amd64/g'
227+
OS:
228+
sh: printf "%s" "{{.OS}}" | tr '[:upper:]' '[:lower:]'
229+
cmds:
230+
- |
231+
URL=https://github.com/agntcy/dir/releases/download/{{.DIRCTL_BIN_VERSION}}/dirctl-{{.OS}}-{{.ARCH}}
232+
echo $URL
233+
curl \
234+
--fail \
235+
--show-error \
236+
--location $URL \
237+
--output "{{.DIRCTL_BIN_PATH}}/dirctl" && \
238+
chmod +x "{{.DIRCTL_BIN_PATH}}/dirctl"
239+
240+
# NOTE: The attribute may removed already, silent errors
241+
xattr -d com.apple.quarantine "{{.DIRCTL_BIN_PATH}}/dirctl" &> /dev/null || true

integrations/agntcy-agp/tests/simple_test.go

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package tests
66
import (
77
"fmt"
88
"os"
9+
"os/exec"
910
"runtime"
1011

1112
ginkgo "github.com/onsi/ginkgo/v2"
@@ -19,6 +20,7 @@ var _ = ginkgo.Describe("Agntcy gateway tests", func() {
1920
dockerImage string
2021
azure_openapi_api_key string
2122
azure_openapi_endpoint string
23+
runner testutils.Runner
2224
)
2325

2426
ginkgo.BeforeEach(func() {
@@ -30,7 +32,6 @@ var _ = ginkgo.Describe("Agntcy gateway tests", func() {
3032
ginkgo.Context("agent gateway", func() {
3133
ginkgo.It("simple agent gateway test", func() {
3234
langchainAgentArgs := []string{
33-
"poetry",
3435
"run",
3536
"python",
3637
"langchain_agent.py",
@@ -50,9 +51,33 @@ var _ = ginkgo.Describe("Agntcy gateway tests", func() {
5051
"AZURE_OPENAI_API_KEY": azure_openapi_api_key,
5152
"AZURE_OPENAI_ENDPOINT": azure_openapi_endpoint,
5253
}
53-
runner := testutils.NewDockerRunner(dockerImage, "", envVars)
54-
outputBuffer, err := runner.Run(langchainAgentArgs...)
55-
gomega.Expect(err).NotTo(gomega.HaveOccurred(), outputBuffer.String())
54+
55+
var err error
56+
57+
switch os.Getenv("RUNNER_TYPE") {
58+
// NOTE: No binary release for agp yet
59+
// case "local":
60+
// runner, err = testutils.NewRunner(testutils.RunnerTypeLocal, testutils.WithEnvVars(envVars))
61+
default:
62+
runner, err = testutils.NewRunner(testutils.RunnerTypeDocker,
63+
testutils.WithDockerCmd("docker"),
64+
testutils.WithDockerImage(dockerImage),
65+
testutils.WithDockerArgs([]string{"run"}),
66+
testutils.WithEnvVars(envVars),
67+
)
68+
}
69+
70+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
71+
72+
_, err = runner.Run("poetry", langchainAgentArgs...)
73+
if err != nil {
74+
exitErr, ok := err.(*exec.ExitError)
75+
if ok {
76+
err = fmt.Errorf("%s, stderr:%s", exitErr.String(), string(exitErr.Stderr))
77+
}
78+
}
79+
80+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
5681
})
5782
})
5883
})

integrations/agntcy-dir/tests/compiler_sanity_test.go

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"encoding/json"
88
"fmt"
99
"os"
10+
"os/exec"
1011
"path/filepath"
1112
"strings"
1213

@@ -25,6 +26,7 @@ var _ = ginkgo.Describe("Agntcy compiler sanity tests", func() {
2526
mountString string
2627
modelConfigFilePath string
2728
expectedAgentModelFile string
29+
runner testutils.Runner
2830
)
2931

3032
ginkgo.BeforeEach(func() {
@@ -34,8 +36,13 @@ var _ = ginkgo.Describe("Agntcy compiler sanity tests", func() {
3436

3537
tempAgentPath = filepath.Join(os.TempDir(), "agent.json")
3638
dockerImage = fmt.Sprintf("%s/dir-ctl:%s", os.Getenv("IMAGE_REPO"), os.Getenv("DIRECTORY_IMAGE_TAG"))
37-
mountDest = "/testdata"
38-
mountString = fmt.Sprintf("%s:%s", testDataPath, mountDest)
39+
40+
if os.Getenv("RUNNER_TYPE") == "local" {
41+
mountDest = testDataPath
42+
} else {
43+
mountDest = "/testdata"
44+
mountString = fmt.Sprintf("%s:%s", testDataPath, mountDest)
45+
}
3946

4047
modelConfigFilePath = filepath.Join(mountDest, "build.config.yml")
4148
expectedAgentModelFile = filepath.Join(testDataPath, "agent.json")
@@ -50,15 +57,35 @@ var _ = ginkgo.Describe("Agntcy compiler sanity tests", func() {
5057
mountDest,
5158
}
5259

53-
runner := testutils.NewDockerRunner(dockerImage, mountString, nil)
60+
var err error
61+
62+
switch os.Getenv("RUNNER_TYPE") {
63+
case "local":
64+
runner, err = testutils.NewRunner(testutils.RunnerTypeLocal, nil)
65+
default:
66+
runner, err = testutils.NewRunner(testutils.RunnerTypeDocker,
67+
testutils.WithDockerCmd("docker"),
68+
testutils.WithDockerArgs([]string{"run", "-v", mountString}),
69+
testutils.WithDockerImage(dockerImage),
70+
)
71+
}
72+
73+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
5474

55-
_, err := fmt.Fprintf(ginkgo.GinkgoWriter, "dirctl command: %v %s\n", runner.GetCommandArgs(), strings.Join(dirctlArgs, " "))
75+
_, err = fmt.Fprintf(ginkgo.GinkgoWriter, "dirctl command: %v %s\n", runner.GetDockerCommandAndArgs(), strings.Join(dirctlArgs, " "))
5676
gomega.Expect(err).NotTo(gomega.HaveOccurred())
5777

58-
outputBuffer, err := runner.Run(dirctlArgs...)
59-
gomega.Expect(err).NotTo(gomega.HaveOccurred(), outputBuffer.String())
78+
cmdOutput, err := runner.Run("dirctl", dirctlArgs...)
79+
if err != nil {
80+
exitErr, ok := err.(*exec.ExitError)
81+
if ok {
82+
err = fmt.Errorf("%s, stderr:%s", exitErr.String(), string(exitErr.Stderr))
83+
}
84+
}
85+
86+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
6087

61-
err = os.WriteFile(tempAgentPath, outputBuffer.Bytes(), 0644)
88+
err = os.WriteFile(tempAgentPath, []byte(cmdOutput), 0644)
6289
gomega.Expect(err).NotTo(gomega.HaveOccurred())
6390
})
6491

integrations/agntcy-dir/tests/list_agent_test.go

Lines changed: 61 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package tests
66
import (
77
"fmt"
88
"os"
9+
"os/exec"
910
"path/filepath"
1011
"runtime"
1112
"strings"
@@ -27,6 +28,7 @@ var _ = ginkgo.Describe("Agntcy agent list tests", func() {
2728
mountDest string
2829
mountString string
2930
agents []*agent
31+
runner testutils.Runner
3032
)
3133

3234
ginkgo.Context("agents push for listing", func() {
@@ -36,8 +38,13 @@ var _ = ginkgo.Describe("Agntcy agent list tests", func() {
3638
gomega.Expect(err).NotTo(gomega.HaveOccurred())
3739

3840
dockerImage = fmt.Sprintf("%s/dir-ctl:%s", os.Getenv("IMAGE_REPO"), os.Getenv("DIRECTORY_IMAGE_TAG"))
39-
mountDest = "/testdata"
40-
mountString = fmt.Sprintf("%s:%s", testDataPath, mountDest)
41+
42+
if os.Getenv("RUNNER_TYPE") == "local" {
43+
mountDest = testDataPath
44+
} else {
45+
mountDest = "/testdata"
46+
mountString = fmt.Sprintf("%s:%s", testDataPath, mountDest)
47+
}
4148

4249
agents = append(agents, &agent{modelFile: filepath.Join(mountDest, "crewai.agent.json")})
4350
agents = append(agents, &agent{modelFile: filepath.Join(mountDest, "langgraph.agent.json")})
@@ -49,35 +56,65 @@ var _ = ginkgo.Describe("Agntcy agent list tests", func() {
4956
agent.modelFile,
5057
}
5158

52-
if runtime.GOOS != "linux" {
59+
if runtime.GOOS != "linux" && os.Getenv("RUNNER_TYPE") != "local" {
5360
dirctlArgs = append(dirctlArgs,
5461
"--server-addr",
5562
"host.docker.internal:8888",
5663
)
5764
}
5865

59-
runner := testutils.NewDockerRunner(dockerImage, mountString, nil)
60-
outputBuffer, err := runner.Run(dirctlArgs...)
61-
gomega.Expect(err).NotTo(gomega.HaveOccurred(), outputBuffer.String())
66+
var err error
67+
68+
switch os.Getenv("RUNNER_TYPE") {
69+
case "local":
70+
runner, err = testutils.NewRunner(testutils.RunnerTypeLocal, nil)
71+
default:
72+
runner, err = testutils.NewRunner(testutils.RunnerTypeDocker,
73+
testutils.WithDockerCmd("docker"),
74+
testutils.WithDockerImage(dockerImage),
75+
testutils.WithDockerArgs([]string{"run", "-v" + mountString}),
76+
)
77+
}
78+
79+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
80+
81+
cmdOutput, err := runner.Run("dirctl", dirctlArgs...)
82+
83+
if err != nil {
84+
exitErr, ok := err.(*exec.ExitError)
85+
if ok {
86+
err = fmt.Errorf("%s, stderr:%s", exitErr.String(), string(exitErr.Stderr))
87+
}
88+
}
6289

63-
agent.digest = strings.Trim(outputBuffer.String(), "\n")
90+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
91+
92+
agent.digest = strings.Trim(cmdOutput, "\n")
6493
_, err = fmt.Fprintf(ginkgo.GinkgoWriter, "DIGEST: %v\n", agent.digest)
6594

95+
gomega.Expect(err).NotTo(gomega.HaveOccurred(), cmdOutput)
96+
6697
dirctlArgs = []string{
6798
"publish",
6899
agent.digest,
69100
}
70101

71-
if runtime.GOOS != "linux" {
102+
if runtime.GOOS != "linux" && os.Getenv("RUNNER_TYPE") != "local" {
72103
dirctlArgs = append(dirctlArgs,
73104
"--server-addr",
74105
"host.docker.internal:8888",
75106
)
76107
}
77108

78-
runner = testutils.NewDockerRunner(dockerImage, mountString, nil)
79-
outputBuffer, err = runner.Run(dirctlArgs...)
80-
gomega.Expect(err).NotTo(gomega.HaveOccurred(), outputBuffer.String())
109+
_, err = runner.Run("dirctl", dirctlArgs...)
110+
if err != nil {
111+
exitErr, ok := err.(*exec.ExitError)
112+
if ok {
113+
err = fmt.Errorf("%s, stderr:%s", exitErr.String(), string(exitErr.Stderr))
114+
}
115+
}
116+
117+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
81118
}
82119
})
83120

@@ -95,7 +132,7 @@ var _ = ginkgo.Describe("Agntcy agent list tests", func() {
95132

96133
dirctlArgs = append(dirctlArgs, labels...)
97134

98-
if runtime.GOOS != "linux" {
135+
if runtime.GOOS != "linux" && os.Getenv("RUNNER_TYPE") != "local" {
99136
dirctlArgs = append(dirctlArgs,
100137
"--server-addr",
101138
"host.docker.internal:8888",
@@ -105,16 +142,23 @@ var _ = ginkgo.Describe("Agntcy agent list tests", func() {
105142
_, err := fmt.Fprintf(ginkgo.GinkgoWriter, "dirctl args: %v\n", dirctlArgs)
106143
gomega.Expect(err).NotTo(gomega.HaveOccurred())
107144

108-
runner := testutils.NewDockerRunner(dockerImage, mountString, nil)
109-
outputBuffer, err := runner.Run(dirctlArgs...)
110-
gomega.Expect(err).NotTo(gomega.HaveOccurred(), outputBuffer.String())
145+
cmdOutput, err := runner.Run("dirctl", dirctlArgs...)
146+
147+
if err != nil {
148+
exitErr, ok := err.(*exec.ExitError)
149+
if ok {
150+
err = fmt.Errorf("%s, stderr:%s", exitErr.String(), string(exitErr.Stderr))
151+
}
152+
}
153+
154+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
111155

112156
if expectFound {
113157
for _, agent := range agents {
114-
gomega.Expect(outputBuffer.String()).To(gomega.ContainSubstring(agent.digest))
158+
gomega.Expect(cmdOutput).To(gomega.ContainSubstring(agent.digest))
115159
}
116160
} else {
117-
gomega.Expect(outputBuffer.String()).To(gomega.BeEmpty())
161+
gomega.Expect(cmdOutput).To(gomega.BeEmpty())
118162
}
119163

120164
},

0 commit comments

Comments
 (0)