Skip to content

Commit a8a398d

Browse files
neoaggelosneoaggelos
authored
Add e2e tests for kube-vip and OVN network load balancer (#36)
* configure OVN in setup-lxd.sh and setup-incus.sh scripts * add kube-vip e2e test * add ovn load balancer e2e test
1 parent 957a234 commit a8a398d

File tree

7 files changed

+219
-3
lines changed

7 files changed

+219
-3
lines changed

‎hack/scripts/ci/setup-e2e.sh‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/bin/bash -xeu
2+
3+
########################################################################
4+
### Usage:
5+
### $ CLI=incus ./common-e2e.sh
6+
### $ CLI=lxc ./common-e2e.sh
7+
8+
# instance profiles
9+
LXC_PROFILE_NAME=default
10+
11+
# local bridge network (10.200.1.0/24)
12+
LXC_NETWORK_NAME="testbr0"
13+
LXC_NETWORK_IPV6="none"
14+
LXC_NETWORK_IPV4="10.200.1.1/24"
15+
LXC_NETWORK_IPV4_DHCP="10.200.1.10-10.200.1.100"
16+
LXC_NETWORK_IPV4_OVN="10.200.1.101-10.200.1.150"
17+
LXC_NETWORK_IPV4_KUBE_VIP="10.200.1.151"
18+
19+
# local OVN network (192.168.200.0/24)
20+
LXC_OVN_NETWORK_NAME="testovn0"
21+
LXC_OVN_NETWORK_IPV6="none"
22+
LXC_OVN_NETWORK_IPV4="192.168.200.1/24"
23+
LXC_OVN_NETWORK_IPV4_LB="10.200.1.201"
24+
25+
########################################################################
26+
27+
# install and configure OVN
28+
sudo apt install ovn-host ovn-central -y
29+
sudo ovs-vsctl set open_vswitch . external_ids:ovn-encap-ip="127.0.0.1"
30+
sudo ovs-vsctl set open_vswitch . external_ids:ovn-encap-type="geneve"
31+
sudo ovs-vsctl set open_vswitch . external_ids:ovn-remote="unix:/run/ovn/ovnsb_db.sock"
32+
33+
########################################################################
34+
35+
# configure default network. the user.capl.e2e.kube-vip-address annotation is used by "QuickStart KubeVIP"
36+
if ! "${CLI}" network show "${LXC_NETWORK_NAME}" 2> /dev/null; then
37+
"${CLI}" network create "${LXC_NETWORK_NAME}" --type=bridge \
38+
ipv4.address="${LXC_NETWORK_IPV4}" ipv4.nat=true \
39+
ipv6.address="${LXC_NETWORK_IPV6}" ipv6.nat=true \
40+
ipv4.dhcp.ranges="${LXC_NETWORK_IPV4_DHCP}" \
41+
ipv4.ovn.ranges="${LXC_NETWORK_IPV4_OVN}" \
42+
user.capl.e2e.kube-vip-address="${LXC_NETWORK_IPV4_KUBE_VIP}"
43+
fi
44+
45+
# configure ovn network. the user.capl.e2e.ovn-lb-address annotation is used by "QuickStart OVN"
46+
if ! "${CLI}" network show "${LXC_OVN_NETWORK_NAME}" 2> /dev/null; then
47+
"${CLI}" network create "${LXC_OVN_NETWORK_NAME}" --type=ovn \
48+
network="${LXC_NETWORK_NAME}" \
49+
ipv4.address="${LXC_OVN_NETWORK_IPV4}" ipv4.nat=true \
50+
ipv6.address="${LXC_OVN_NETWORK_IPV6}" ipv6.nat=true \
51+
user.capl.e2e.ovn-lb-address="${LXC_OVN_NETWORK_IPV4_LB}"
52+
fi
53+
54+
# configure default profile
55+
"${CLI}" profile device set "${LXC_PROFILE_NAME}" eth0 type=nic network="${LXC_NETWORK_NAME}"
56+
"${CLI}" profile show default

‎hack/scripts/ci/setup-incus.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,4 +37,7 @@ data:
3737
server-crt: '$(cat ~/.config/incus/servercerts/local-https.crt | base64 -w0)'
3838
client-crt: '$(cat ~/.config/incus/client.crt | base64 -w0)'
3939
client-key: '$(cat ~/.config/incus/client.key | base64 -w0)'
40-
" | tee "${DIR}/../../lxc-secret.yaml"
40+
" | tee "${DIR}/../../../lxc-secret.yaml"
41+
42+
# Setup local Incus daemon for e2e tests
43+
CLI=incus "${DIR}/setup-e2e.sh"

‎hack/scripts/ci/setup-lxd.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,7 @@ data:
4040
server-crt: '$(cat ~/snap/lxd/common/config/servercerts/local-https.crt | base64 -w0)'
4141
client-crt: '$(cat ~/snap/lxd/common/config/client.crt | base64 -w0)'
4242
client-key: '$(cat ~/snap/lxd/common/config/client.key | base64 -w0)'
43-
" | tee "${DIR}/../../lxc-secret.yaml"
43+
" | tee "${DIR}/../../../lxc-secret.yaml"
44+
45+
# Setup local LXD daemon for e2e tests
46+
CLI=lxc "${DIR}/setup-e2e.sh"

‎test/e2e/shared/defaults.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ const (
4040
// Name of secret for LXC credentials
4141
LXCSecretName = "LXC_SECRET_NAME"
4242

43+
// KubeVIP address to use for kube-vip tests
44+
KubeVIPAddress = "KUBE_VIP_ADDRESS"
45+
4346
FlavorDefault = ""
4447
FlavorDevelopment = "development"
4548
FlavorAutoscaler = "autoscaler"

‎test/e2e/suites/e2e/quick_start_kube_vip_test.go‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
//go:build e2e
2+
3+
package e2e
4+
5+
import (
6+
"context"
7+
"fmt"
8+
9+
"sigs.k8s.io/cluster-api/test/e2e"
10+
11+
"github.com/neoaggelos/cluster-api-provider-lxc/internal/incus"
12+
"github.com/neoaggelos/cluster-api-provider-lxc/internal/ptr"
13+
"github.com/neoaggelos/cluster-api-provider-lxc/test/e2e/shared"
14+
15+
. "github.com/onsi/ginkgo/v2"
16+
. "github.com/onsi/gomega"
17+
)
18+
19+
var _ = Describe("QuickStart", func() {
20+
Context("KubeVIP", Label("PRBlocking"), func() {
21+
var (
22+
clusterctlVariables map[string]string
23+
)
24+
BeforeEach(func(ctx context.Context) {
25+
if v := e2eCtx.E2EConfig.GetVariableBestEffort(shared.KubeVIPAddress); v != "" {
26+
shared.Logf("Using kube-vip address %q (from environment variable KUBE_VIP_ADDRESS)", v)
27+
clusterctlVariables = map[string]string{
28+
"LOAD_BALANCER": fmt.Sprintf("kube-vip: {host: %q}", v),
29+
}
30+
return
31+
}
32+
33+
// KUBE_VIP_ADDRESS is not set, look for a network
34+
client, err := incus.New(ctx, e2eCtx.Settings.LXCClientOptions)
35+
Expect(err).ToNot(HaveOccurred())
36+
networks, err := client.Client.GetNetworks()
37+
Expect(err).ToNot(HaveOccurred())
38+
39+
// find network with the annotations below
40+
// -- user.capl.e2e.kube-vip-address = "<ip address>"
41+
for _, network := range networks {
42+
if v, ok := network.Config["user.capl.e2e.kube-vip-address"]; ok {
43+
shared.Logf("Using kube-vip address %q (from network %q)", v, network.Name)
44+
clusterctlVariables = map[string]string{
45+
"LOAD_BALANCER": fmt.Sprintf("kube-vip: {host: '%s'}", v),
46+
"CONTROL_PLANE_MACHINE_DEVICES": fmt.Sprintf("['eth0,type=nic,network=%s']", network.Name),
47+
"WORKER_MACHINE_DEVICES": fmt.Sprintf("['eth0,type=nic,network=%s']", network.Name),
48+
}
49+
return
50+
}
51+
}
52+
53+
Skip("Did not find any network with configuration 'user.capl.e2e.kube-vip-address', and KUBE_VIP_ADDRESS is not set")
54+
})
55+
56+
e2e.QuickStartSpec(context.TODO(), func() e2e.QuickStartSpecInput {
57+
return e2e.QuickStartSpecInput{
58+
E2EConfig: e2eCtx.E2EConfig,
59+
ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath,
60+
BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy,
61+
ArtifactFolder: e2eCtx.Settings.ArtifactFolder,
62+
SkipCleanup: e2eCtx.Settings.SkipCleanup,
63+
PostNamespaceCreated: e2eCtx.DefaultPostNamespaceCreated(),
64+
ControlPlaneWaiters: e2eCtx.DefaultControlPlaneWaiters(),
65+
InfrastructureProvider: ptr.To("lxc:v0.88.99"),
66+
67+
Flavor: ptr.To(shared.FlavorDefault),
68+
ControlPlaneMachineCount: ptr.To[int64](3),
69+
WorkerMachineCount: ptr.To[int64](0),
70+
71+
ClusterctlVariables: clusterctlVariables,
72+
}
73+
})
74+
})
75+
})

‎test/e2e/suites/e2e/quick_start_oci_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ package e2e
44

55
import (
66
"context"
7+
"fmt"
78

89
"sigs.k8s.io/cluster-api/test/e2e"
910

@@ -24,7 +25,7 @@ var _ = Describe("QuickStart", func() {
2425
err = client.SupportsInstanceOCI()
2526
Expect(err).To(Or(Succeed(), MatchError(incus.IsTerminalError, "IsTerminalError")))
2627
if err != nil {
27-
Skip("Server does not support OCI instances")
28+
Skip(fmt.Sprintf("Server does not support OCI instances: %v", err))
2829
}
2930
})
3031

‎test/e2e/suites/e2e/quick_start_ovn_test.go‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
//go:build e2e
2+
3+
package e2e
4+
5+
import (
6+
"context"
7+
"fmt"
8+
9+
"sigs.k8s.io/cluster-api/test/e2e"
10+
11+
"github.com/neoaggelos/cluster-api-provider-lxc/internal/incus"
12+
"github.com/neoaggelos/cluster-api-provider-lxc/internal/ptr"
13+
"github.com/neoaggelos/cluster-api-provider-lxc/test/e2e/shared"
14+
15+
. "github.com/onsi/ginkgo/v2"
16+
. "github.com/onsi/gomega"
17+
)
18+
19+
var _ = Describe("QuickStart", func() {
20+
Context("OVN", Label("PRBlocking"), func() {
21+
var (
22+
lbAddress string
23+
networkName string
24+
)
25+
BeforeEach(func(ctx context.Context) {
26+
client, err := incus.New(ctx, e2eCtx.Settings.LXCClientOptions)
27+
Expect(err).ToNot(HaveOccurred())
28+
29+
err = client.SupportsNetworkLoadBalancer()
30+
Expect(err).To(Or(Succeed(), MatchError(incus.IsTerminalError, "IsTerminalError")))
31+
if err != nil {
32+
Skip(fmt.Sprintf("Server does not support network load balancer: %v", err))
33+
}
34+
35+
networks, err := client.Client.GetNetworks()
36+
Expect(err).ToNot(HaveOccurred())
37+
38+
// find network with the annotations below
39+
// -- user.capl.e2e.ovn-lb-address = "<ip address>"
40+
for _, network := range networks {
41+
if v, ok := network.Config["user.capl.e2e.ovn-lb-address"]; ok {
42+
networkName = network.Name
43+
lbAddress = v
44+
shared.Logf("Using OVN network %q with LoadBalancer address %q", networkName, lbAddress)
45+
return
46+
}
47+
}
48+
49+
Skip("Did not find any network with configuration 'user.capl.e2e.ovn-lb-address'")
50+
})
51+
52+
e2e.QuickStartSpec(context.TODO(), func() e2e.QuickStartSpecInput {
53+
return e2e.QuickStartSpecInput{
54+
E2EConfig: e2eCtx.E2EConfig,
55+
ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath,
56+
BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy,
57+
ArtifactFolder: e2eCtx.Settings.ArtifactFolder,
58+
SkipCleanup: e2eCtx.Settings.SkipCleanup,
59+
PostNamespaceCreated: e2eCtx.DefaultPostNamespaceCreated(),
60+
ControlPlaneWaiters: e2eCtx.DefaultControlPlaneWaiters(),
61+
InfrastructureProvider: ptr.To("lxc:v0.88.99"),
62+
63+
Flavor: ptr.To(shared.FlavorDefault),
64+
ControlPlaneMachineCount: ptr.To[int64](3),
65+
WorkerMachineCount: ptr.To[int64](0),
66+
67+
ClusterctlVariables: map[string]string{
68+
"LOAD_BALANCER": fmt.Sprintf("ovn: {host: '%s', networkName: '%s'}", lbAddress, networkName),
69+
"CONTROL_PLANE_MACHINE_DEVICES": fmt.Sprintf("['eth0,type=nic,network=%s']", networkName),
70+
"WORKER_MACHINE_DEVICES": fmt.Sprintf("['eth0,type=nic,network=%s']", networkName),
71+
},
72+
}
73+
})
74+
})
75+
})

0 commit comments

Comments
 (0)