Skip to content

Commit f086821

Browse files
bastjanbastjan
committed
Add controller to deploy machine-api-controllers for full functionality
The original deployment is done by [machine-api-operator](https://github.com/openshift/machine-api-operator/blob/9c3e4a04009ae84958c25b4cbb380a24e7260761/pkg/operator/sync.go#L70-L164), but it there is no possibility of using this on with non-inlined providers. The controller refuses to create a deployment if the upstream one exists and uses jsonnet for easier rendering. Upstream images are taken from the upstream image configmap.
1 parent d4e22d7 commit f086821

6 files changed

+791
-2
lines changed

‎Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,9 @@ clean: ## Cleans up the generated resources
5757
rm -rf .tmpvendor
5858

5959
.PHONY: run
60+
RUN_TARGET ?= manager
6061
run: generate fmt vet ## Run a controller from your host.
61-
go run ./main.go
62+
go run ./main.go "-target=$(RUN_TARGET)"
6263

6364
###
6465
### Assets

‎controllers/machine_api_controllers_controller.go

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
package controllers
2+
3+
import (
4+
"context"
5+
_ "embed"
6+
"encoding/json"
7+
"fmt"
8+
9+
"github.com/google/go-jsonnet"
10+
appsv1 "k8s.io/api/apps/v1"
11+
corev1 "k8s.io/api/core/v1"
12+
apierrors "k8s.io/apimachinery/pkg/api/errors"
13+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14+
"k8s.io/apimachinery/pkg/runtime"
15+
ctrl "sigs.k8s.io/controller-runtime"
16+
"sigs.k8s.io/controller-runtime/pkg/client"
17+
"sigs.k8s.io/controller-runtime/pkg/log"
18+
)
19+
20+
const (
21+
imagesConfigMapName = "machine-api-operator-images"
22+
originalUpstreamDeploymentName = "machine-api-controllers"
23+
imageKey = "images.json"
24+
25+
caBundleConfigMapName = "appuio-machine-api-ca-bundle"
26+
)
27+
28+
//go:embed machine_api_controllers_deployment.jsonnet
29+
var deploymentTemplate string
30+
31+
// MachineAPIControllersReconciler creates a appuio-machine-api-controllers deployment based on the images.json ConfigMap
32+
// if the upstream machine-api-controllers does not exist.
33+
type MachineAPIControllersReconciler struct {
34+
client.Client
35+
Scheme *runtime.Scheme
36+
37+
Namespace string
38+
}
39+
40+
func (r *MachineAPIControllersReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
41+
if req.Name != imagesConfigMapName {
42+
return ctrl.Result{}, nil
43+
}
44+
45+
l := log.FromContext(ctx).WithName("UpstreamDeploymentReconciler.Reconcile")
46+
l.Info("Reconciling")
47+
48+
var imageCM corev1.ConfigMap
49+
if err := r.Get(ctx, req.NamespacedName, &imageCM); err != nil {
50+
return ctrl.Result{}, client.IgnoreNotFound(err)
51+
}
52+
53+
ij, ok := imageCM.Data[imageKey]
54+
if !ok {
55+
return ctrl.Result{}, fmt.Errorf("images.json key not found in ConfigMap %q", imagesConfigMapName)
56+
}
57+
images := make(map[string]string)
58+
if err := json.Unmarshal([]byte(ij), &images); err != nil {
59+
return ctrl.Result{}, fmt.Errorf("failed to unmarshal %q from %q: %w", imageKey, imagesConfigMapName, err)
60+
}
61+
62+
// Check that the original upstream deployment does not exist
63+
// If it does, we should not create the new deployment
64+
var upstreamDeployment appsv1.Deployment
65+
err := r.Get(ctx, client.ObjectKey{
66+
Name: originalUpstreamDeploymentName,
67+
Namespace: r.Namespace,
68+
}, &upstreamDeployment)
69+
if err == nil {
70+
return ctrl.Result{}, fmt.Errorf("original upstream deployment %s already exists", originalUpstreamDeploymentName)
71+
} else if !apierrors.IsNotFound(err) {
72+
return ctrl.Result{}, fmt.Errorf("failed to check for original upstream deployment %s: %w", originalUpstreamDeploymentName, err)
73+
}
74+
75+
caBundleConfigMap := corev1.ConfigMap{
76+
TypeMeta: metav1.TypeMeta{
77+
APIVersion: "v1",
78+
Kind: "ConfigMap",
79+
},
80+
ObjectMeta: metav1.ObjectMeta{
81+
Name: caBundleConfigMapName,
82+
Namespace: r.Namespace,
83+
Labels: map[string]string{
84+
"config.openshift.io/inject-trusted-cabundle": "true",
85+
},
86+
},
87+
}
88+
if err := r.Client.Patch(ctx, &caBundleConfigMap, client.Apply, client.FieldOwner("upstream-deployment-controller")); err != nil {
89+
return ctrl.Result{}, fmt.Errorf("failed to apply ConfigMap %q: %w", caBundleConfigMapName, err)
90+
}
91+
92+
vm, err := jsonnetVMWithContext(images, caBundleConfigMap)
93+
if err != nil {
94+
return ctrl.Result{}, fmt.Errorf("failed to create jsonnet VM: %w", err)
95+
}
96+
97+
ud, err := vm.EvaluateAnonymousSnippet("controllers_deployment.jsonnet", deploymentTemplate)
98+
if err != nil {
99+
return ctrl.Result{}, fmt.Errorf("failed to evaluate jsonnet: %w", err)
100+
}
101+
102+
// TODO(bastjan) this could be way more generic and support any kind of object.
103+
// We don't need any other object types right now, so we're keeping it simple.
104+
var toDeploy appsv1.Deployment
105+
if err := json.Unmarshal([]byte(ud), &toDeploy); err != nil {
106+
return ctrl.Result{}, fmt.Errorf("failed to unmarshal jsonnet output: %w", err)
107+
}
108+
if toDeploy.APIVersion != "apps/v1" || toDeploy.Kind != "Deployment" {
109+
return ctrl.Result{}, fmt.Errorf("expected Deployment, got %s/%s", toDeploy.APIVersion, toDeploy.Kind)
110+
}
111+
toDeploy.Namespace = r.Namespace
112+
113+
if err := r.Client.Patch(ctx, &toDeploy, client.Apply, client.FieldOwner("upstream-deployment-controller")); err != nil {
114+
return ctrl.Result{}, fmt.Errorf("failed to apply Deployment %q: %w", toDeploy.GetName(), err)
115+
}
116+
117+
return ctrl.Result{}, nil
118+
}
119+
120+
// SetupWithManager sets up the controller with the Manager.
121+
func (r *MachineAPIControllersReconciler) SetupWithManager(mgr ctrl.Manager) error {
122+
return ctrl.NewControllerManagedBy(mgr).
123+
For(&corev1.ConfigMap{}).
124+
Owns(&appsv1.Deployment{}).
125+
Owns(&corev1.ConfigMap{}).
126+
Complete(r)
127+
}
128+
129+
func jsonnetVMWithContext(images map[string]string, cabundle corev1.ConfigMap) (*jsonnet.VM, error) {
130+
jcr, err := json.Marshal(map[string]any{
131+
"images": images,
132+
"cabundle": cabundle,
133+
})
134+
if err != nil {
135+
return nil, fmt.Errorf("unable to marshal jsonnet context: %w", err)
136+
}
137+
jvm := jsonnet.MakeVM()
138+
jvm.ExtCode("context", string(jcr))
139+
// Don't allow imports
140+
jvm.Importer(&jsonnet.MemoryImporter{})
141+
return jvm, nil
142+
}

‎controllers/machine_api_controllers_controller_test.go

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
package controllers
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"errors"
7+
"fmt"
8+
"testing"
9+
10+
"github.com/stretchr/testify/assert"
11+
"github.com/stretchr/testify/require"
12+
appsv1 "k8s.io/api/apps/v1"
13+
corev1 "k8s.io/api/core/v1"
14+
apierrors "k8s.io/apimachinery/pkg/api/errors"
15+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16+
"k8s.io/apimachinery/pkg/runtime"
17+
"k8s.io/apimachinery/pkg/types"
18+
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
19+
ctrl "sigs.k8s.io/controller-runtime"
20+
"sigs.k8s.io/controller-runtime/pkg/client"
21+
"sigs.k8s.io/controller-runtime/pkg/client/fake"
22+
)
23+
24+
func Test_MachineAPIControllersReconciler_Reconcile(t *testing.T) {
25+
t.Parallel()
26+
27+
ctx := context.Background()
28+
29+
const namespace = "openshift-machine-api"
30+
31+
scheme := runtime.NewScheme()
32+
require.NoError(t, clientgoscheme.AddToScheme(scheme))
33+
34+
images := map[string]string{
35+
"machineAPIOperator": "registry.io/machine-api-operator:v1.0.0",
36+
"kubeRBACProxy": "registry.io/kube-rbac-proxy:v1.0.0",
37+
}
38+
imagesJSON, err := json.Marshal(images)
39+
require.NoError(t, err)
40+
41+
ucm := &corev1.ConfigMap{
42+
ObjectMeta: metav1.ObjectMeta{
43+
Name: imagesConfigMapName,
44+
Namespace: namespace,
45+
},
46+
Data: map[string]string{
47+
imageKey: string(imagesJSON),
48+
},
49+
}
50+
51+
c := &fakeSSA{
52+
fake.NewClientBuilder().
53+
WithScheme(scheme).
54+
WithRuntimeObjects(ucm).
55+
Build(),
56+
}
57+
58+
r := &MachineAPIControllersReconciler{
59+
Client: c,
60+
Scheme: scheme,
61+
62+
Namespace: namespace,
63+
}
64+
65+
_, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(ucm)})
66+
require.NoError(t, err)
67+
68+
var deployment appsv1.Deployment
69+
require.NoError(t, c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: "appuio-" + originalUpstreamDeploymentName}, &deployment))
70+
71+
assert.Equal(t, "system-node-critical", deployment.Spec.Template.Spec.PriorityClassName)
72+
for _, c := range deployment.Spec.Template.Spec.Containers {
73+
if c.Image == images["machineAPIOperator"] || c.Image == images["kubeRBACProxy"] {
74+
continue
75+
}
76+
t.Errorf("expected image %q or %q, got %q", images["machineAPIOperator"], images["kubeRBACProxy"], c.Image)
77+
}
78+
}
79+
80+
func Test_MachineAPIControllersReconciler_OriginalDeploymentExists(t *testing.T) {
81+
t.Parallel()
82+
83+
ctx := context.Background()
84+
85+
const namespace = "openshift-machine-api"
86+
87+
scheme := runtime.NewScheme()
88+
require.NoError(t, clientgoscheme.AddToScheme(scheme))
89+
90+
images := map[string]string{
91+
"machineAPIOperator": "registry.io/machine-api-operator:v1.0.0",
92+
"kubeRBACProxy": "registry.io/kube-rbac-proxy:v1.0.0",
93+
}
94+
imagesJSON, err := json.Marshal(images)
95+
require.NoError(t, err)
96+
97+
ucm := &corev1.ConfigMap{
98+
ObjectMeta: metav1.ObjectMeta{
99+
Name: imagesConfigMapName,
100+
Namespace: namespace,
101+
},
102+
Data: map[string]string{
103+
imageKey: string(imagesJSON),
104+
},
105+
}
106+
107+
origDeploy := &appsv1.Deployment{
108+
ObjectMeta: metav1.ObjectMeta{
109+
Name: originalUpstreamDeploymentName,
110+
Namespace: namespace,
111+
},
112+
}
113+
114+
c := &fakeSSA{
115+
fake.NewClientBuilder().
116+
WithScheme(scheme).
117+
WithRuntimeObjects(ucm, origDeploy).
118+
Build(),
119+
}
120+
121+
r := &MachineAPIControllersReconciler{
122+
Client: c,
123+
Scheme: scheme,
124+
125+
Namespace: namespace,
126+
}
127+
128+
_, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(ucm)})
129+
require.ErrorContains(t, err, "machine-api-controllers already exists")
130+
}
131+
132+
// fakeSSA is a fake client that approximates SSA.
133+
// It creates objects that don't exist yet and _updates_ them if they exist.
134+
// This is completely kaputt since the object is overwritten with the new object.
135+
// See https://github.com/kubernetes-sigs/controller-runtime/issues/2341
136+
type fakeSSA struct {
137+
client.WithWatch
138+
}
139+
140+
// Patch approximates SSA by creating objects that don't exist yet.
141+
func (f *fakeSSA) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
142+
// Apply patches are supposed to upsert, but fake client fails if the object doesn't exist,
143+
// if an apply patch occurs for an object that doesn't yet exist, create it.
144+
if patch.Type() != types.ApplyPatchType {
145+
return f.WithWatch.Patch(ctx, obj, patch, opts...)
146+
}
147+
check, ok := obj.DeepCopyObject().(client.Object)
148+
if !ok {
149+
return errors.New("could not check for object in fake client")
150+
}
151+
if err := f.WithWatch.Get(ctx, client.ObjectKeyFromObject(obj), check); apierrors.IsNotFound(err) {
152+
if err := f.WithWatch.Create(ctx, check); err != nil {
153+
return fmt.Errorf("could not inject object creation for fake: %w", err)
154+
}
155+
} else if err != nil {
156+
return fmt.Errorf("could not check for object in fake client: %w", err)
157+
}
158+
return f.WithWatch.Update(ctx, obj)
159+
}

0 commit comments

Comments
 (0)