Skip to content

Commit 0519aff

Browse files
dependabot[bot]cprivitere
dependabot[bot]
authored and
cprivitere
committed
feat: Add Equinix Metal Load Balancer support
- Add internal package for interacting with Equinix Metal Load Balancer API - packetcluster_controller creates load balancer and listener port and stores their ids in packetCluster annotations. - packetmachine_controller creates an origin pool and origin port for each machine and stores their IDs in the packetMachine annotations. - CPEMLBConfig and EMLBID added to the packet cloud client package to be able to provide a config for the CPEM loadBalancer setting in the emlb templates. - Memory request for the Cluster API Provider Packet controller increased to 300Mi to avoid OOMing while debugging. - EMLB added as a valid VIPManager enum type. Signed-off-by: Chris Privitere <23177737+cprivitere@users.noreply.github.com>
1 parent 45c53b4 commit 0519aff

File tree

57 files changed

+10436
-97
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+10436
-97
lines changed

‎.github/workflows/release.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ jobs:
126126
REPO: ${{ github.event.repository.name }}
127127

128128
- name: Create Release
129-
uses: softprops/action-gh-release@v1
129+
uses: softprops/action-gh-release@v2
130130
if: startsWith(github.ref, 'refs/tags/')
131131
with:
132132
files: out/release/*

‎.golangci.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,6 @@ linters-settings:
164164
alias: infraexpv1
165165
nolintlint:
166166
allow-unused: false
167-
allow-leading-space: false
168167
require-specific: true
169168
revive:
170169
rules:
@@ -314,3 +313,7 @@ issues:
314313
- gocritic
315314
text: "deferInLoop: Possible resource leak, 'defer' is called in the 'for' loop"
316315
path: _test\.go
316+
- linters:
317+
- bodyclose
318+
path: .*(internal)/emlb/emlb.go
319+
text: "response body must be closed"

‎api/v1beta1/packetcluster_types.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ const (
2626
NetworkInfrastructureReadyCondition clusterv1.ConditionType = "NetworkInfrastructureReady"
2727
)
2828

29-
// VIPManagerType describes if the VIP will be managed by CPEM or kube-vip.
29+
// VIPManagerType describes if the VIP will be managed by CPEM or kube-vip or Equinix Metal Load Balancer.
3030
type VIPManagerType string
3131

3232
// PacketClusterSpec defines the desired state of PacketCluster.
@@ -46,9 +46,9 @@ type PacketClusterSpec struct {
4646
// +optional
4747
ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"`
4848

49-
// VIPManager represents whether this cluster uses CPEM or kube-vip to
49+
// VIPManager represents whether this cluster uses CPEM or kube-vip or Equinix Metal Load Balancer to
5050
// manage its vip for the api server IP
51-
// +kubebuilder:validation:Enum=CPEM;KUBE_VIP
51+
// +kubebuilder:validation:Enum=CPEM;KUBE_VIP;EMLB
5252
// +kubebuilder:default:=CPEM
5353
VIPManager VIPManagerType `json:"vipManager"`
5454
}

‎clusterctl-settings.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"name": "infrastructure-packet",
33
"config": {
44
"componentsFile": "infrastructure-components.yaml",
5-
"nextVersion": "v0.6.99"
5+
"nextVersion": "v0.8.99"
66
}
77
}
88

‎config/crd/bases/infrastructure.cluster.x-k8s.io_packetclusters.yaml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,10 +75,12 @@ spec:
7575
vipManager:
7676
default: CPEM
7777
description: VIPManager represents whether this cluster uses CPEM
78-
or kube-vip to manage its vip for the api server IP
78+
or kube-vip or Equinix Metal Load Balancer to manage its vip for
79+
the api server IP
7980
enum:
8081
- CPEM
8182
- KUBE_VIP
83+
- EMLB
8284
type: string
8385
required:
8486
- projectID

‎config/manager/manager.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,10 @@ spec:
4242
port: healthz
4343
resources:
4444
limits:
45-
memory: 200Mi
45+
memory: 300Mi
4646
requests:
4747
cpu: 100m
48-
memory: 200Mi
48+
memory: 300Mi
4949
securityContext:
5050
allowPrivilegeEscalation: false
5151
capabilities:

‎controllers/packetcluster_controller.go

Lines changed: 48 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import (
3434
"sigs.k8s.io/controller-runtime/pkg/handler"
3535

3636
infrav1 "sigs.k8s.io/cluster-api-provider-packet/api/v1beta1"
37+
"sigs.k8s.io/cluster-api-provider-packet/internal/emlb"
3738
packet "sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet"
3839
"sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet/scope"
3940
)
@@ -114,49 +115,63 @@ func (r *PacketClusterReconciler) reconcileNormal(ctx context.Context, clusterSc
114115

115116
packetCluster := clusterScope.PacketCluster
116117

117-
ipReserv, err := r.PacketClient.GetIPByClusterIdentifier(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID)
118118
switch {
119-
case errors.Is(err, packet.ErrControlPlanEndpointNotFound):
120-
// Parse metro and facility from the cluster spec
121-
var metro, facility string
122-
123-
facility = packetCluster.Spec.Facility
124-
metro = packetCluster.Spec.Metro
125-
126-
// If both specified, metro takes precedence over facility
127-
if metro != "" {
128-
facility = ""
119+
case packetCluster.Spec.VIPManager == "EMLB":
120+
if !packetCluster.Spec.ControlPlaneEndpoint.IsValid() {
121+
// Create new EMLB object
122+
lb := emlb.NewEMLB(r.PacketClient.GetConfig().DefaultHeader["X-Auth-Token"], packetCluster.Spec.ProjectID, packetCluster.Spec.Metro)
123+
124+
if err := lb.ReconcileLoadBalancer(ctx, clusterScope); err != nil {
125+
log.Error(err, "Error Reconciling EMLB")
126+
return err
127+
}
129128
}
130-
131-
// There is not an ElasticIP with the right tags, at this point we can create one
132-
ip, err := r.PacketClient.CreateIP(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID, facility, metro)
133-
if err != nil {
134-
log.Error(err, "error reserving an ip")
129+
case packetCluster.Spec.VIPManager == "KUBE_VIP":
130+
log.Info("KUBE_VIP VIPManager Detected")
131+
if err := r.PacketClient.EnableProjectBGP(ctx, packetCluster.Spec.ProjectID); err != nil {
132+
log.Error(err, "error enabling bgp for project")
135133
return err
136134
}
137-
clusterScope.PacketCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
138-
Host: ip.To4().String(),
139-
Port: 6443,
140-
}
141-
case err != nil:
142-
log.Error(err, "error getting cluster IP")
143-
return err
144-
default:
145-
// If there is an ElasticIP with the right tag just use it again
146-
clusterScope.PacketCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
147-
Host: ipReserv.GetAddress(),
148-
Port: 6443,
149-
}
150135
}
151136

152-
if clusterScope.PacketCluster.Spec.VIPManager == "KUBE_VIP" {
153-
if err := r.PacketClient.EnableProjectBGP(ctx, packetCluster.Spec.ProjectID); err != nil {
154-
log.Error(err, "error enabling bgp for project")
137+
if packetCluster.Spec.VIPManager != "EMLB" {
138+
ipReserv, err := r.PacketClient.GetIPByClusterIdentifier(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID)
139+
switch {
140+
case errors.Is(err, packet.ErrControlPlanEndpointNotFound):
141+
// Parse metro and facility from the cluster spec
142+
var metro, facility string
143+
144+
facility = packetCluster.Spec.Facility
145+
metro = packetCluster.Spec.Metro
146+
147+
// If both specified, metro takes precedence over facility
148+
if metro != "" {
149+
facility = ""
150+
}
151+
152+
// There is not an ElasticIP with the right tags, at this point we can create one
153+
ip, err := r.PacketClient.CreateIP(ctx, clusterScope.Namespace(), clusterScope.Name(), packetCluster.Spec.ProjectID, facility, metro)
154+
if err != nil {
155+
log.Error(err, "error reserving an ip")
156+
return err
157+
}
158+
packetCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
159+
Host: ip.To4().String(),
160+
Port: 6443,
161+
}
162+
case err != nil:
163+
log.Error(err, "error getting cluster IP")
155164
return err
165+
default:
166+
// If there is an ElasticIP with the right tag just use it again
167+
packetCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
168+
Host: ipReserv.GetAddress(),
169+
Port: 6443,
170+
}
156171
}
157172
}
158173

159-
clusterScope.PacketCluster.Status.Ready = true
174+
packetCluster.Status.Ready = true
160175
conditions.MarkTrue(packetCluster, infrav1.NetworkInfrastructureReadyCondition)
161176

162177
return nil

‎controllers/packetmachine_controller.go

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import (
4343
"sigs.k8s.io/controller-runtime/pkg/reconcile"
4444

4545
infrav1 "sigs.k8s.io/cluster-api-provider-packet/api/v1beta1"
46+
"sigs.k8s.io/cluster-api-provider-packet/internal/emlb"
4647
packet "sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet"
4748
"sigs.k8s.io/cluster-api-provider-packet/pkg/cloud/packet/scope"
4849
clog "sigs.k8s.io/cluster-api/util/log"
@@ -348,23 +349,33 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
348349
// when a node is a control plane node we need the elastic IP
349350
// to template out the kube-vip deployment
350351
if machineScope.IsControlPlane() {
351-
controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
352-
ctx,
353-
machineScope.Cluster.Namespace,
354-
machineScope.Cluster.Name,
355-
machineScope.PacketCluster.Spec.ProjectID)
356-
if machineScope.PacketCluster.Spec.VIPManager == "CPEM" {
352+
var controlPlaneEndpointAddress string
353+
var cpemLBConfig string
354+
var emlbID string
355+
switch {
356+
case machineScope.PacketCluster.Spec.VIPManager == "CPEM":
357+
controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
358+
ctx,
359+
machineScope.Cluster.Namespace,
360+
machineScope.Cluster.Name,
361+
machineScope.PacketCluster.Spec.ProjectID)
357362
if len(controlPlaneEndpoint.Assignments) == 0 {
358363
a := corev1.NodeAddress{
359364
Type: corev1.NodeExternalIP,
360365
Address: controlPlaneEndpoint.GetAddress(),
361366
}
362367
addrs = append(addrs, a)
363368
}
369+
controlPlaneEndpointAddress = controlPlaneEndpoint.GetAddress()
370+
case machineScope.PacketCluster.Spec.VIPManager == "EMLB":
371+
controlPlaneEndpointAddress = machineScope.Cluster.Spec.ControlPlaneEndpoint.Host
372+
cpemLBConfig = "emlb:///" + machineScope.PacketCluster.Spec.Metro
373+
emlbID = machineScope.PacketCluster.Annotations["equinix.com/loadbalancerID"]
364374
}
365-
createDeviceReq.ControlPlaneEndpoint = controlPlaneEndpoint.GetAddress()
375+
createDeviceReq.ControlPlaneEndpoint = controlPlaneEndpointAddress
376+
createDeviceReq.CPEMLBConfig = cpemLBConfig
377+
createDeviceReq.EMLBID = emlbID
366378
}
367-
368379
dev, err = r.PacketClient.NewDevice(ctx, createDeviceReq)
369380

370381
switch {
@@ -413,7 +424,8 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
413424
case infrav1.PacketResourceStatusRunning:
414425
log.Info("Machine instance is active", "instance-id", machineScope.ProviderID())
415426

416-
if machineScope.PacketCluster.Spec.VIPManager == "CPEM" {
427+
switch {
428+
case machineScope.PacketCluster.Spec.VIPManager == "CPEM":
417429
controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
418430
ctx,
419431
machineScope.Cluster.Namespace,
@@ -428,6 +440,15 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
428440
return ctrl.Result{RequeueAfter: time.Second * 20}, nil
429441
}
430442
}
443+
case machineScope.PacketCluster.Spec.VIPManager == "EMLB":
444+
if machineScope.IsControlPlane() {
445+
// Create new EMLB object
446+
lb := emlb.NewEMLB(r.PacketClient.GetConfig().DefaultHeader["X-Auth-Token"], machineScope.PacketCluster.Spec.ProjectID, machineScope.PacketCluster.Spec.Metro)
447+
448+
if err := lb.ReconcileVIPOrigin(ctx, machineScope, deviceAddr); err != nil {
449+
return ctrl.Result{}, err
450+
}
451+
}
431452
}
432453

433454
machineScope.SetReady()

‎go.mod

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,16 @@ module sigs.k8s.io/cluster-api-provider-packet
33
go 1.20
44

55
require (
6-
github.com/equinix/equinix-sdk-go v0.32.0
6+
github.com/equinix/equinix-sdk-go v0.35.0
77
github.com/onsi/gomega v1.30.0
88
github.com/pkg/errors v0.9.1
99
github.com/spf13/cobra v1.8.0
1010
github.com/spf13/pflag v1.0.5
11-
k8s.io/api v0.28.6
12-
k8s.io/apimachinery v0.28.6
13-
k8s.io/client-go v0.28.6
14-
k8s.io/component-base v0.28.6
11+
golang.org/x/oauth2 v0.14.0
12+
k8s.io/api v0.28.7
13+
k8s.io/apimachinery v0.28.7
14+
k8s.io/client-go v0.28.7
15+
k8s.io/component-base v0.28.7
1516
k8s.io/klog/v2 v2.100.1
1617
k8s.io/utils v0.0.0-20231127182322-b307cd553661
1718
sigs.k8s.io/cluster-api v1.6.0
@@ -81,7 +82,6 @@ require (
8182
golang.org/x/crypto v0.17.0 // indirect
8283
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
8384
golang.org/x/net v0.19.0 // indirect
84-
golang.org/x/oauth2 v0.14.0 // indirect
8585
golang.org/x/sync v0.4.0 // indirect
8686
golang.org/x/sys v0.15.0 // indirect
8787
golang.org/x/term v0.15.0 // indirect

‎go.sum

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
3838
github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
3939
github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
4040
github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA=
41-
github.com/equinix/equinix-sdk-go v0.32.0 h1:zUn0Em5FJe6f6bntftrDBpO9L+XhbpFMPuQ7RKEOgXM=
42-
github.com/equinix/equinix-sdk-go v0.32.0/go.mod h1:qnpdRzVftHFNaJFk1VSIrAOTLrIoeDrxzUr3l8ARyvQ=
41+
github.com/equinix/equinix-sdk-go v0.35.0 h1:p/uwA8QPBAuNnKGc3mQkwjw+6++qUadLNnKFQ8jw+wg=
42+
github.com/equinix/equinix-sdk-go v0.35.0/go.mod h1:hEb3XLaedz7xhl/dpPIS6eOIiXNPeqNiVoyDrT6paIg=
4343
github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
4444
github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
4545
github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc=
@@ -165,7 +165,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
165165
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
166166
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
167167
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
168-
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
168+
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
169169
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=
170170
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
171171
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -283,20 +283,20 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
283283
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
284284
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
285285
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
286-
k8s.io/api v0.28.6 h1:yy6u9CuIhmg55YvF/BavPBBXB+5QicB64njJXxVnzLo=
287-
k8s.io/api v0.28.6/go.mod h1:AM6Ys6g9MY3dl/XNaNfg/GePI0FT7WBGu8efU/lirAo=
286+
k8s.io/api v0.28.7 h1:YKIhBxjXKaxuxWJnwohV0aGjRA5l4IU0Eywf/q19AVI=
287+
k8s.io/api v0.28.7/go.mod h1:y4RbcjCCMff1930SG/TcP3AUKNfaJUgIeUp58e/2vyY=
288288
k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6yAJvU=
289289
k8s.io/apiextensions-apiserver v0.28.4/go.mod h1:pgQIZ1U8eJSMQcENew/0ShUTlePcSGFq6dxSxf2mwPM=
290-
k8s.io/apimachinery v0.28.6 h1:RsTeR4z6S07srPg6XYrwXpTJVMXsjPXn0ODakMytSW0=
291-
k8s.io/apimachinery v0.28.6/go.mod h1:QFNX/kCl/EMT2WTSz8k4WLCv2XnkOLMaL8GAVRMdpsA=
290+
k8s.io/apimachinery v0.28.7 h1:2Z38/XRAOcpb+PonxmBEmjG7hBfmmr41xnr0XvpTnB4=
291+
k8s.io/apimachinery v0.28.7/go.mod h1:QFNX/kCl/EMT2WTSz8k4WLCv2XnkOLMaL8GAVRMdpsA=
292292
k8s.io/apiserver v0.28.4 h1:BJXlaQbAU/RXYX2lRz+E1oPe3G3TKlozMMCZWu5GMgg=
293293
k8s.io/apiserver v0.28.4/go.mod h1:Idq71oXugKZoVGUUL2wgBCTHbUR+FYTWa4rq9j4n23w=
294-
k8s.io/client-go v0.28.6 h1:Gge6ziyIdafRchfoBKcpaARuz7jfrK1R1azuwORIsQI=
295-
k8s.io/client-go v0.28.6/go.mod h1:+nu0Yp21Oeo/cBCsprNVXB2BfJTV51lFfe5tXl2rUL8=
294+
k8s.io/client-go v0.28.7 h1:3L6402+tjmOl8twX3fjUQ/wsYAkw6UlVNDVP+rF6YGA=
295+
k8s.io/client-go v0.28.7/go.mod h1:xIoEaDewZ+EwWOo1/F1t0IOKMPe1rwBZhLu9Es6y0tE=
296296
k8s.io/cluster-bootstrap v0.28.4 h1:4MKNy1Qd9QY7pl47rSMGIORF+tm3CUaqC1M8U9bjn4Q=
297297
k8s.io/cluster-bootstrap v0.28.4/go.mod h1:/c4ro/R4yf4EtJgFgFtvnHkbDOHwubeKJXh5R1c89Bc=
298-
k8s.io/component-base v0.28.6 h1:G4T8VrcQ7xZou3by/fY5NU5mfxOBlWaivS2lPrEltAo=
299-
k8s.io/component-base v0.28.6/go.mod h1:Dg62OOG3ALu2P4nAG00UdsuHoNLQJ5VsUZKQlLDcS+E=
298+
k8s.io/component-base v0.28.7 h1:Cq5aQ52N0CTaOMiary4rXzR4RoTP77Z3ll4qSg4qH7s=
299+
k8s.io/component-base v0.28.7/go.mod h1:RrtNBKrSuckksSQ3fV9PhwBSHO/ZbwJXM2Z0OPx+UJk=
300300
k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
301301
k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
302302
k8s.io/kms v0.28.4 h1:PMgY/3CQTWP9eIKmNQiTgjLIZ0ns6O+voagzD2/4mSg=

0 commit comments

Comments
 (0)