Skip to content

Commit 8c63109

Browse files
authored
Run tests on real KinD cluster (#133)
* Run tests on real KinD cluster
1 parent 83deb06 commit 8c63109

File tree

5 files changed

+206
-114
lines changed

5 files changed

+206
-114
lines changed

.github/workflows/test.yml

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,30 +17,30 @@ jobs:
1717
name: controllers_testing
1818
runs-on: ubuntu-latest
1919
needs: lint
20+
env:
21+
KIND_CLUSTER_NAME: kind-cluster
2022

2123
strategy:
2224
matrix:
23-
kubebuilder-version: [1.16.4, 1.22.1]
25+
kubernetes-version: [v1.16.4, v1.19.4, v1.22.4, v1.23.4]
2426

2527
steps:
26-
- uses: actions/checkout@v1
28+
- uses: actions/checkout@v3
2729

2830
- name: Set up Go 1.18
29-
uses: actions/setup-go@v1
31+
uses: actions/setup-go@v3
3032
with:
3133
go-version: 1.18
3234
id: go
3335

34-
- name: Set up Kubebuilder
35-
run: |
36-
OS=$(uname -s | tr '[:upper:]' '[:lower:]')
37-
ARCH=$(uname -m | sed 's/x86_64/amd64/')
38-
curl -fsL "https://storage.googleapis.com/kubebuilder-tools/kubebuilder-tools-${{matrix.kubebuilder-version}}-${OS}-${ARCH}.tar.gz" -o kubebuilder-tools
39-
tar -zvxf kubebuilder-tools
40-
sudo mv kubebuilder/ /usr/local/kubebuilder
36+
- name: Set up Kind Cluster
37+
uses: helm/kind-action@v1.2.0
38+
with:
39+
node_image: kindest/node:${{ matrix.kubernetes-version }}
40+
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
4141

4242
- name: Test
43-
run: make test ENVTEST_K8S_VERSION=${{matrix.kubebuilder-version}}
43+
run: make gotest
4444

4545
e2e_testing:
4646
name: e2e_testing

Makefile

Lines changed: 36 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,6 @@ BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION)
3838
# Image URL to use all building/pushing image targets
3939
REPO ?= tarantool-operator
4040
IMG ?= $(REPO):$(VERSION)
41-
# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
42-
ENVTEST_K8S_VERSION ?= 1.22
4341

4442
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
4543
ifeq (,$(shell go env GOBIN))
@@ -86,8 +84,42 @@ fmt: ## Run go fmt against code.
8684
vet: ## Run go vet against code.
8785
go vet ./...
8886

89-
test: manifests generate fmt vet envtest ## Run tests.
90-
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out
87+
gotest:
88+
go test ./... -coverprofile cover.out ;
89+
90+
test: manifests generate fmt vet kind-start
91+
echo "Run tests" ; \
92+
$(MAKE) gotest KUBECONFIG=$(KIND_KUBECONFIG) || ($(MAKE) kind-stop || true; exit 1); \
93+
$(MAKE) kind-stop || true ;
94+
95+
##@ KIND Cluster. More info: https://kind.sigs.k8s.io/docs
96+
KIND_VERSION ?= v0.12.0
97+
KIND_CLUSTER_NAME ?= "tarantool-operator-testing"
98+
KIND=$(shell pwd)/bin/kind
99+
KIND_IMAGE=kindest/node:v1.23.4@sha256:0e34f0d0fd448aa2f2819cfd74e99fe5793a6e4938b328f657c8e3f81ee0dfb9
100+
KIND_KUBECONFIG=$(shell pwd)/bin/kubeconfig.yml
101+
102+
kind-install:
103+
@[ -f $(KIND) ] || { \
104+
OS=$(shell go env GOOS) ; \
105+
ARCH=$(shell go env GOARCH) ; \
106+
URL=https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$${OS}-$${ARCH} ; \
107+
echo "Downloading kind $${URL}" ; \
108+
curl -sSLo ${KIND} $${URL} ; \
109+
chmod +x $(KIND) ; \
110+
}
111+
112+
kind-start: kind-install
113+
EXISTS=$$($(KIND) get clusters | grep -c $(KIND_CLUSTER_NAME)) ; \
114+
if [ $$EXISTS -eq "0" ] ; then \
115+
$(KIND) create cluster --image=$(KIND_IMAGE) --name=$(KIND_CLUSTER_NAME) --kubeconfig=$(KIND_KUBECONFIG) || exit 1; \
116+
fi; \
117+
KUBECONFIG="$(KIND_KUBECONFIG)" kubectl cluster-info ; \
118+
echo "Kind cluster ready" ;
119+
120+
kind-stop: kind-install
121+
$(KIND) delete cluster --name=$(KIND_CLUSTER_NAME) ; \
122+
rm -f $(KIND_KUBECONFIG) ;
91123

92124
##@ Build
93125

@@ -174,10 +206,6 @@ KUSTOMIZE = $(shell pwd)/bin/kustomize
174206
kustomize: ## Download kustomize locally if necessary.
175207
$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3,v3.10.0)
176208

177-
ENVTEST = $(shell pwd)/bin/setup-envtest
178-
envtest: ## Download envtest-setup locally if necessary.
179-
$(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,latest)
180-
181209
# go-get-tool will 'go get' any package $2 in version $3 and install it to $1.
182210
PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
183211
define go-get-tool

controllers/cluster_controller_test.go

Lines changed: 59 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import (
1414
corev1 "k8s.io/api/core/v1"
1515
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1616

17-
tarantooliov1alpha1 "github.com/tarantool/tarantool-operator/api/v1alpha1"
1817
"sigs.k8s.io/controller-runtime/pkg/client"
1918
)
2019

@@ -30,107 +29,78 @@ func RandStringRunes(n int) string {
3029

3130
var _ = Describe("cluster_controller unit testing", func() {
3231
var (
33-
namespace = "default"
34-
ctx = context.TODO()
35-
36-
roleName = "" // setup for every spec in hook
37-
rsTemplateName = ""
38-
32+
ctx = context.TODO()
33+
namespace = "test"
3934
clusterName = "test"
40-
clusterId = clusterName
41-
42-
defaultRolesToAssign = "[\"A\",\"B\"]"
35+
clusterId = "test"
36+
ns = &corev1.Namespace{
37+
ObjectMeta: metav1.ObjectMeta{
38+
Name: namespace,
39+
},
40+
}
41+
cartridge = helpers.NewCartridge(helpers.CartridgeParams{
42+
Namespace: namespace,
43+
ClusterName: clusterName,
44+
ClusterID: clusterId,
45+
})
4346
)
4447

4548
Describe("cluster_controller manage cluster resources", func() {
4649
BeforeEach(func() {
47-
// setup variables for each spec
48-
roleName = fmt.Sprintf("test-role-%s", RandStringRunes(4))
49-
rsTemplateName = fmt.Sprintf("test-rs-%s", RandStringRunes(4))
50-
51-
By("create new Role " + roleName)
52-
role := helpers.NewRole(helpers.RoleParams{
53-
Name: roleName,
54-
Namespace: namespace,
55-
RolesToAssign: defaultRolesToAssign,
56-
RsNum: int32(1),
57-
RsTemplateName: rsTemplateName,
58-
ClusterId: clusterId,
59-
})
60-
// mock owner reference
61-
role.SetOwnerReferences([]metav1.OwnerReference{
62-
{
63-
APIVersion: "v0",
64-
Kind: "mockRef",
65-
Name: "mockRef",
66-
UID: "-",
67-
},
68-
})
69-
Expect(k8sClient.Create(ctx, &role)).NotTo(HaveOccurred(), "failed to create Role")
70-
71-
By("create new Cluster " + clusterName)
72-
cluster := helpers.NewCluster(helpers.ClusterParams{
73-
Name: clusterName,
74-
Namespace: namespace,
75-
Id: clusterId,
76-
})
77-
Expect(k8sClient.Create(ctx, &cluster)).NotTo(HaveOccurred(), "failed to create Cluster")
50+
Expect(k8sClient.Create(ctx, ns)).
51+
NotTo(
52+
HaveOccurred(),
53+
fmt.Sprintf("failed to create Namespace %s", ns.GetName()),
54+
)
55+
56+
Expect(k8sClient.Create(ctx, cartridge.Cluster)).
57+
NotTo(
58+
HaveOccurred(),
59+
fmt.Sprintf("failed to create Cluster %s", cartridge.Cluster.GetName()),
60+
)
61+
62+
for _, role := range cartridge.Roles {
63+
Expect(k8sClient.Create(ctx, role)).
64+
NotTo(
65+
HaveOccurred(),
66+
fmt.Sprintf("failed to create Role %s", role.GetName()),
67+
)
68+
}
69+
70+
for _, rs := range cartridge.ReplicasetTemplates {
71+
Expect(k8sClient.Create(ctx, rs)).
72+
NotTo(
73+
HaveOccurred(),
74+
fmt.Sprintf("failed to create ReplicasetTemplate %s", rs.GetName()),
75+
)
76+
}
77+
78+
for _, svc := range cartridge.Services {
79+
Expect(k8sClient.Create(ctx, svc)).
80+
NotTo(
81+
HaveOccurred(),
82+
fmt.Sprintf("failed to create Service %s", svc.GetName()),
83+
)
84+
}
7885
})
7986

8087
AfterEach(func() {
81-
By("remove role object " + roleName)
82-
role := &tarantooliov1alpha1.Role{}
83-
Expect(
84-
k8sClient.Get(ctx, client.ObjectKey{Name: roleName, Namespace: namespace}, role),
85-
).NotTo(HaveOccurred(), "failed to get Role")
86-
87-
Expect(k8sClient.Delete(ctx, role)).NotTo(HaveOccurred(), "failed to delete Role")
88-
89-
By("remove Cluster object " + clusterName)
90-
cluster := &tarantooliov1alpha1.Cluster{}
88+
By("remove Namespace object " + namespace)
89+
ns := &corev1.Namespace{}
9190
Expect(
92-
k8sClient.Get(ctx, client.ObjectKey{Name: clusterName, Namespace: namespace}, cluster),
93-
).NotTo(HaveOccurred(), "failed to get Cluster")
91+
k8sClient.Get(ctx, client.ObjectKey{Name: namespace}, ns),
92+
).NotTo(HaveOccurred(), "failed to get Namespace")
9493

95-
Expect(k8sClient.Delete(ctx, cluster)).NotTo(HaveOccurred(), "failed to delete Cluster")
94+
Expect(k8sClient.Delete(ctx, ns)).NotTo(HaveOccurred(), "failed to delete Namespace")
9695
})
9796

9897
Context("manage cluster leader: tarantool instance accepting admin requests", func() {
99-
BeforeEach(func() {
100-
By("create cluster endpoints")
101-
ep := corev1.Endpoints{
102-
ObjectMeta: metav1.ObjectMeta{
103-
Name: clusterId,
104-
Namespace: namespace,
105-
},
106-
Subsets: []corev1.EndpointSubset{
107-
{
108-
Addresses: []corev1.EndpointAddress{
109-
{IP: "1.1.1.1"},
110-
{IP: "2.2.2.2"},
111-
{IP: "3.3.3.3"},
112-
},
113-
},
114-
},
115-
}
116-
Expect(k8sClient.Create(ctx, &ep)).NotTo(HaveOccurred(), "failed to create cluster endpoints")
117-
})
118-
119-
AfterEach(func() {
120-
ep := corev1.Endpoints{}
121-
Expect(
122-
k8sClient.Get(ctx, client.ObjectKey{Name: clusterId, Namespace: namespace}, &ep),
123-
).NotTo(HaveOccurred(), "failed to get cluster endpoints")
124-
125-
Expect(k8sClient.Delete(ctx, &ep)).NotTo(HaveOccurred(), "failed to delete endpoints")
126-
})
127-
12898
It("change the leader if the previous one does not exist", func() {
12999
By("get the chosen leader")
130100
ep := corev1.Endpoints{}
131101
Eventually(
132102
func() bool {
133-
err := k8sClient.Get(ctx, client.ObjectKey{Name: clusterId, Namespace: namespace}, &ep)
103+
err := k8sClient.Get(ctx, client.ObjectKey{Name: clusterName, Namespace: namespace}, &ep)
134104
if err != nil {
135105
return false
136106
}
@@ -141,7 +111,8 @@ var _ = Describe("cluster_controller unit testing", func() {
141111

142112
return false
143113
},
144-
time.Second*100, time.Millisecond*500,
114+
2*time.Minute,
115+
500*time.Millisecond,
145116
).Should(BeTrue())
146117

147118
By("save old leader")
@@ -172,13 +143,14 @@ var _ = Describe("cluster_controller unit testing", func() {
172143
}
173144
return false
174145
},
175-
time.Second*10, time.Millisecond*500,
146+
2*time.Minute,
147+
500*time.Millisecond,
176148
).Should(BeTrue())
177149
})
178150
})
179151
})
180152

181-
Describe("cluster_contriller unit testing functions", func() {
153+
Describe("cluster_controller unit testing functions", func() {
182154
Describe("function IsLeaderExists must check for existence of leader in annotation of cluster Endpoints", func() {
183155
Context("positive cases (leader exist)", func() {
184156
It("should return True if leader assigned and exist", func() {

controllers/suite_test.go

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ import (
4040
ctrl "sigs.k8s.io/controller-runtime"
4141
"sigs.k8s.io/controller-runtime/pkg/client"
4242
"sigs.k8s.io/controller-runtime/pkg/envtest"
43-
"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
4443
logf "sigs.k8s.io/controller-runtime/pkg/log"
4544
"sigs.k8s.io/controller-runtime/pkg/log/zap"
4645

@@ -62,9 +61,7 @@ var (
6261
func TestAPIs(t *testing.T) {
6362
RegisterFailHandler(Fail)
6463

65-
RunSpecsWithDefaultAndCustomReporters(t,
66-
"Controller Suite",
67-
[]Reporter{printer.NewlineReporter{}})
64+
RunSpecs(t, "Controller Suite")
6865
}
6966

7067
var _ = BeforeSuite(func() {
@@ -73,11 +70,11 @@ var _ = BeforeSuite(func() {
7370
ctx, cancel = context.WithCancel(context.TODO())
7471

7572
By("bootstrapping test environment")
76-
USE_EXISTING_CLUSTER := false
73+
useExistingCluster := true
7774
testEnv = &envtest.Environment{
7875
CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")},
7976
ErrorIfCRDPathMissing: true,
80-
UseExistingCluster: &USE_EXISTING_CLUSTER,
77+
UseExistingCluster: &useExistingCluster,
8178
}
8279

8380
var err error
@@ -99,10 +96,11 @@ var _ = BeforeSuite(func() {
9996
mgr, err := ctrl.NewManager(cfg, ctrl.Options{})
10097
Expect(err).NotTo(HaveOccurred(), "failed to create manager")
10198

102-
err = (&ClusterReconciler{
99+
clusterReconciler := &ClusterReconciler{
103100
Client: mgr.GetClient(),
104101
Scheme: scheme.Scheme,
105-
}).SetupWithManager(mgr)
102+
}
103+
err = clusterReconciler.SetupWithManager(mgr)
106104
Expect(err).ToNot(HaveOccurred())
107105

108106
err = (&RoleReconciler{

0 commit comments

Comments
 (0)