Skip to content

Commit 14480c8

Browse files
aramasearamase
committed
feat: create separate cache for nodepublishsecretref in rotation
Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>
1 parent 68e8d9b commit 14480c8

File tree

5 files changed

+226
-157
lines changed

5 files changed

+226
-157
lines changed

‎cmd/secrets-store-csi-driver/main.go

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,14 @@ import (
2525
"strings"
2626
"time"
2727

28-
"k8s.io/apimachinery/pkg/runtime/schema"
29-
3028
"sigs.k8s.io/secrets-store-csi-driver/pkg/cache"
3129
"sigs.k8s.io/secrets-store-csi-driver/pkg/metrics"
3230
"sigs.k8s.io/secrets-store-csi-driver/pkg/rotation"
3331
"sigs.k8s.io/secrets-store-csi-driver/pkg/version"
3432

3533
"k8s.io/apimachinery/pkg/fields"
3634
"k8s.io/apimachinery/pkg/runtime"
35+
"k8s.io/apimachinery/pkg/runtime/schema"
3736
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
3837
json "k8s.io/component-base/logs/json"
3938
"k8s.io/klog/v2"
@@ -63,11 +62,10 @@ var (
6362
enableProfile = flag.Bool("enable-pprof", false, "enable pprof profiling")
6463
profilePort = flag.Int("pprof-port", 6065, "port for pprof profiling")
6564

66-
// enable filtered watch for secrets. The filtering is done on the csi driver label: secrets-store.csi.k8s.io/managed=true
67-
// this label is set for all Kubernetes secrets created by the CSI driver. For Kubernetes secrets used to provide credentials
68-
// for use with the CSI driver, set the label by running: kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/managed=true
65+
// enable filtered watch for NodePublishSecretRef secrets. The filtering is done on the csi driver label: secrets-store.csi.k8s.io/used=true
66+
// For Kubernetes secrets used to provide credentials for use with the CSI driver, set the label by running: kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true
6967
// This feature flag will be enabled by default after n+2 releases giving time for users to label all their existing credential secrets.
70-
filteredWatchSecret = flag.Bool("filtered-watch-secret", false, "enable filtered watch for secrets with label secrets-store.csi.k8s.io/managed=true")
68+
filteredWatchSecret = flag.Bool("filtered-watch-secret", false, "enable filtered watch for NodePublishSecretRef secrets with label secrets-store.csi.k8s.io/used=true")
7169

7270
scheme = runtime.NewScheme()
7371
)
@@ -98,7 +96,7 @@ func main() {
9896
}()
9997
}
10098
if *filteredWatchSecret {
101-
klog.Infof("Filtered watch for secret based on secrets-store.csi.k8s.io/managed=true label enabled")
99+
klog.Infof("Filtered watch for nodePublishSecretRef secret based on secrets-store.csi.k8s.io/used=true label enabled")
102100
}
103101

104102
// initialize metrics exporter before creating measurements
@@ -115,15 +113,13 @@ func main() {
115113
fieldSelectorByResource := map[schema.GroupResource]string{
116114
{Group: "", Resource: "pods"}: fields.OneTermEqualSelector("spec.nodeName", *nodeID).String(),
117115
}
118-
// this enables filtered watch of secretproviderclasspodstatuses based on the internal node label
119-
// internal.secrets-store.csi.k8s.io/node-name=<node name> added by csi driver
120116
labelSelectorByResource := map[schema.GroupResource]string{
121-
{Group: "", Resource: "secretproviderclasspodstatuses"}: fmt.Sprintf("%s=%s", v1alpha1.InternalNodeLabel, *nodeID),
122-
}
123-
// this enables filtered watch of secrets based on the label (secrets-store.csi.k8s.io/managed=true)
124-
// added to the secrets created by the CSI driver
125-
if *filteredWatchSecret {
126-
labelSelectorByResource[schema.GroupResource{Group: "", Resource: "secrets"}] = fmt.Sprintf("%s=true", controllers.SecretManagedLabel)
117+
// this enables filtered watch of secretproviderclasspodstatuses based on the internal node label
118+
// internal.secrets-store.csi.k8s.io/node-name=<node name> added by csi driver
119+
{Group: v1alpha1.GroupVersion.Group, Resource: "secretproviderclasspodstatuses"}: fmt.Sprintf("%s=%s", v1alpha1.InternalNodeLabel, *nodeID),
120+
// this enables filtered watch of secrets based on the label (secrets-store.csi.k8s.io/managed=true)
121+
// added to the secrets created by the CSI driver
122+
{Group: "", Resource: "secrets"}: fmt.Sprintf("%s=true", controllers.SecretManagedLabel),
127123
}
128124

129125
mgr, err := ctrl.NewManager(cfg, ctrl.Options{

‎controllers/secretproviderclasspodstatus_controller.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ import (
5555

5656
const (
5757
SecretManagedLabel = "secrets-store.csi.k8s.io/managed"
58+
SecretUsedLabel = "secrets-store.csi.k8s.io/used"
5859
secretCreationFailedReason = "FailedToCreateSecret"
5960
)
6061

@@ -123,7 +124,7 @@ func (r *SecretProviderClassPodStatusReconciler) Patcher(ctx context.Context) er
123124
spc := &v1alpha1.SecretProviderClass{}
124125
namespace := spcPodStatuses[i].Namespace
125126

126-
if val, exists := spcMap[spcPodStatuses[i].Namespace+"/"+spcName]; exists {
127+
if val, exists := spcMap[namespace+"/"+spcName]; exists {
127128
spc = &val
128129
} else {
129130
if err := r.reader.Get(ctx, client.ObjectKey{Namespace: namespace, Name: spcName}, spc); err != nil {
@@ -165,7 +166,7 @@ func (r *SecretProviderClassPodStatusReconciler) Patcher(ctx context.Context) er
165166
}
166167

167168
for _, secret := range spc.Spec.SecretObjects {
168-
key := types.NamespacedName{Name: secret.SecretName, Namespace: spcPodStatuses[i].Namespace}
169+
key := types.NamespacedName{Name: secret.SecretName, Namespace: namespace}
169170
val, exists := secretOwnerMap[key]
170171
if exists {
171172
secretOwnerMap[key] = append(val, ownerRefs...)

‎pkg/k8s/store.go

Lines changed: 40 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ import (
4040
type Informer struct {
4141
Pod cache.SharedIndexInformer
4242
Secret cache.SharedIndexInformer
43+
NodePublishSecretRefSecret cache.SharedIndexInformer
4344
SecretProviderClass cache.SharedIndexInformer
4445
SecretProviderClassPodStatus cache.SharedIndexInformer
4546
}
@@ -48,6 +49,7 @@ type Informer struct {
4849
type Lister struct {
4950
Pod PodLister
5051
Secret SecretLister
52+
NodePublishSecretRefSecret SecretLister
5153
SecretProviderClass SecretProviderClassLister
5254
SecretProviderClassPodStatus SecretProviderClassPodStatusLister
5355
}
@@ -57,6 +59,8 @@ type Store interface {
5759
GetPod(name, namespace string) (*v1.Pod, error)
5860
// GetSecret returns the secret matching name and namespace
5961
GetSecret(name, namespace string) (*v1.Secret, error)
62+
// GetNodePublishSecretRefSecret returns the NodePublishSecretRef secret matching name and namespace
63+
GetNodePublishSecretRefSecret(name, namespace string) (*v1.Secret, error)
6064
// GetSecretProviderClass returns the secret provider class matching name and namespace
6165
GetSecretProviderClass(name, namespace string) (*v1alpha1.SecretProviderClass, error)
6266
// GetSecretProviderClassPodStatus returns the secret provider class pod status matching key
@@ -81,9 +85,12 @@ func New(kubeClient kubernetes.Interface, crdClient secretsStoreClient.Interface
8185
store.informers.Pod = newPodInformer(kubeClient, resyncPeriod, nodeName)
8286
store.listers.Pod.Store = store.informers.Pod.GetStore()
8387

84-
store.informers.Secret = newSecretInformer(kubeClient, resyncPeriod, filteredWatchSecret)
88+
store.informers.Secret = newSecretInformer(kubeClient, resyncPeriod)
8589
store.listers.Secret.Store = store.informers.Secret.GetStore()
8690

91+
store.informers.NodePublishSecretRefSecret = newNodePublishSecretRefSecretInformer(kubeClient, resyncPeriod, filteredWatchSecret)
92+
store.listers.NodePublishSecretRefSecret.Store = store.informers.NodePublishSecretRefSecret.GetStore()
93+
8794
store.informers.SecretProviderClass = newSPCInformer(crdClient, resyncPeriod)
8895
store.listers.SecretProviderClass.Store = store.informers.SecretProviderClass.GetStore()
8996

@@ -108,6 +115,11 @@ func (s k8sStore) GetSecret(name, namespace string) (*v1.Secret, error) {
108115
return s.listers.Secret.GetWithKey(getStoreKey(name, namespace))
109116
}
110117

118+
// GetNodePublishSecretRefSecret returns the NodePublishSecretRef secret matching name and namespace
119+
func (s k8sStore) GetNodePublishSecretRefSecret(name, namespace string) (*v1.Secret, error) {
120+
return s.listers.NodePublishSecretRefSecret.GetWithKey(getStoreKey(name, namespace))
121+
}
122+
111123
// GetSecretProviderClass returns the secret provider class matching name and namespace
112124
func (s k8sStore) GetSecretProviderClass(name, namespace string) (*v1alpha1.SecretProviderClass, error) {
113125
return s.listers.SecretProviderClass.GetWithKey(getStoreKey(name, namespace))
@@ -135,10 +147,11 @@ func (s k8sStore) GetSecretProviderClassPodStatus(key string) (*v1alpha1.SecretP
135147
func (i *Informer) run(stopCh <-chan struct{}) error {
136148
go i.Pod.Run(stopCh)
137149
go i.Secret.Run(stopCh)
150+
go i.NodePublishSecretRefSecret.Run(stopCh)
138151
go i.SecretProviderClass.Run(stopCh)
139152
go i.SecretProviderClassPodStatus.Run(stopCh)
140153

141-
synced := []cache.InformerSynced{i.Pod.HasSynced, i.Secret.HasSynced, i.SecretProviderClass.HasSynced, i.SecretProviderClassPodStatus.HasSynced}
154+
synced := []cache.InformerSynced{i.Pod.HasSynced, i.Secret.HasSynced, i.NodePublishSecretRefSecret.HasSynced, i.SecretProviderClass.HasSynced, i.SecretProviderClassPodStatus.HasSynced}
142155
if !cache.WaitForCacheSync(stopCh, synced...) {
143156
return fmt.Errorf("failed to sync informer caches")
144157
}
@@ -158,22 +171,28 @@ func newPodInformer(kubeClient kubernetes.Interface, resyncPeriod time.Duration,
158171
}
159172

160173
// newSecretInformer returns a secret informer
161-
func newSecretInformer(kubeClient kubernetes.Interface, resyncPeriod time.Duration, filteredWatchSecret bool) cache.SharedIndexInformer {
174+
func newSecretInformer(kubeClient kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
175+
return coreInformers.NewFilteredSecretInformer(
176+
kubeClient,
177+
v1.NamespaceAll,
178+
resyncPeriod,
179+
cache.Indexers{},
180+
managedFilterForSecret(),
181+
)
182+
}
183+
184+
// newNodePublishSecretRefSecretInformer returns a NodePublishSecretRef informer
185+
func newNodePublishSecretRefSecretInformer(kubeClient kubernetes.Interface, resyncPeriod time.Duration, filteredWatchSecret bool) cache.SharedIndexInformer {
186+
var tweakListOptionsFunc internalinterfaces.TweakListOptionsFunc
162187
if filteredWatchSecret {
163-
return coreInformers.NewFilteredSecretInformer(
164-
kubeClient,
165-
v1.NamespaceAll,
166-
resyncPeriod,
167-
cache.Indexers{},
168-
managedFilterForSecret(),
169-
)
188+
tweakListOptionsFunc = usedFilterForSecret()
170189
}
171190
return coreInformers.NewFilteredSecretInformer(
172191
kubeClient,
173192
v1.NamespaceAll,
174193
resyncPeriod,
175194
cache.Indexers{},
176-
nil,
195+
tweakListOptionsFunc,
177196
)
178197
}
179198

@@ -221,9 +240,18 @@ func getStoreKey(name, namespace string) string {
221240
return fmt.Sprintf("%s/%s", namespace, name)
222241
}
223242

224-
// managedFilterForSecret returns tweak options to filter using managed label.
243+
// managedFilterForSecret returns tweak options to filter using managed label (secrets-store.csi.k8s.io/managed=true).
244+
// this label is configured for all secrets created by the driver.
225245
func managedFilterForSecret() internalinterfaces.TweakListOptionsFunc {
226246
return func(options *metav1.ListOptions) {
227247
options.LabelSelector = fmt.Sprintf("%s=true", controllers.SecretManagedLabel)
228248
}
229249
}
250+
251+
// usedFilterForSecret returns tweak options to filter using used label (secrets-store.csi.k8s.io/used=true).
252+
// this label will need to be configured by user for NodePublishSecretRef secrets.
253+
func usedFilterForSecret() internalinterfaces.TweakListOptionsFunc {
254+
return func(options *metav1.ListOptions) {
255+
options.LabelSelector = fmt.Sprintf("%s=true", controllers.SecretUsedLabel)
256+
}
257+
}

‎pkg/rotation/reconciler.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ func (r *Reconciler) reconcile(ctx context.Context, spcps *v1alpha1.SecretProvid
245245
secretNamespace := spcps.Namespace
246246

247247
// read secret from the informer cache
248-
secret, err := r.store.GetSecret(secretName, secretNamespace)
248+
secret, err := r.store.GetNodePublishSecretRefSecret(secretName, secretNamespace)
249249
if err != nil {
250250
errorReason = internalerrors.NodePublishSecretRefNotFound
251251
r.generateEvent(pod, v1.EventTypeWarning, mountRotationFailedReason, fmt.Sprintf("failed to get node publish secret %s/%s, err: %+v", secretNamespace, secretName, err))

0 commit comments

Comments
 (0)