Merge pull request #512 from tam7t/tam7t/config-message-size

feat: allow maxCallRecvMsgSize to be increased for large secret mounts

Author: k8s-ci-robot

cmd/secrets-store-csi-driver/main.go

@@ -29,13 +29,13 @@ import (
 	"sigs.k8s.io/secrets-store-csi-driver/pkg/rotation"
 	"sigs.k8s.io/secrets-store-csi-driver/pkg/version"
 
+	"google.golang.org/grpc"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	json "k8s.io/component-base/logs/json"
 	"k8s.io/klog/v2"
-
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	"sigs.k8s.io/secrets-store-csi-driver/apis/v1alpha1"
@@ -58,6 +58,7 @@ var (
 	rotationPollInterval = flag.Duration("rotation-poll-interval", 2*time.Minute, "Secret rotation poll interval duration")
 	enableProfile        = flag.Bool("enable-pprof", false, "enable pprof profiling")
 	profilePort          = flag.Int("pprof-port", 6065, "port for pprof profiling")
+	maxCallRecvMsgSize   = flag.Int("max-call-recv-msg-size", 1024*1024*4, "maximum size in bytes of gRPC response from plugins")
 
 	// enable filtered watch for NodePublishSecretRef secrets. The filtering is done on the csi driver label: secrets-store.csi.k8s.io/used=true
 	// For Kubernetes secrets used to provide credentials for use with the CSI driver, set the label by running: kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true
@@ -148,7 +149,7 @@ func main() {
 	ctx := withShutdownSignal(context.Background())
 
 	// create provider clients
-	providerClients := secretsstore.NewPluginClientBuilder(*providerVolumePath)
+	providerClients := secretsstore.NewPluginClientBuilder(*providerVolumePath, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(*maxCallRecvMsgSize)))
 	defer providerClients.Cleanup()
 
 	// enable provider health check

docs/book/src/troubleshooting.md

@@ -53,4 +53,13 @@ Secrets Store CSI Driver is deployed as a *DaemonSet*. The above error indicates
   NAME                       ATTACHREQUIRED   PODINFOONMOUNT   MODES       AGE
   secrets-store.csi.k8s.io   false            true             Ephemeral   110m
   ```
-  
+
+### Mount fails with `grpc: received message larger than max`
+
+If the files pulled in by the `SecretProviderClass` are larger than 4MiB you may observe `FailedMount` warnings with a
+message that includes `grpc: received message larger than max`. You can configure the driver to accept responses larger
+than 4MiB by specifying the `--max-call-recv-msg-size=<size in bytes>` argument to the `secrets-store` container in the
+`csi-secrets-store` DaemonSet.
+
+Note that this may also increase memory resource consumption of the `secrets-store` container, so you should also
+consider increasing the memory limit as well.

manifest_staging/charts/secrets-store-csi-driver/README.md

@@ -79,6 +79,7 @@ The following table lists the configurable parameters of the csi-secrets-store-p
 | `logFormatJSON`                         | Use JSON logging format                                                                                                           | `false`                                             |
 | `livenessProbe.port`                    | Liveness probe port                                                                                                               | `9808`                                              |
 | `livenessProbe.logLevel`                | Liveness probe container logging verbosity level                                                                                  | `2`                                                 |
+| `maxCallRecvMsgSize`                    | Maximum size in bytes of gRPC response from plugins                                                                               | `4194304`                                           |
 | `rbac.install`                          | Install default rbac roles and bindings                                                                                           | true                                                |
 | `rbac.pspEnabled`                       | If `true`, create and use a restricted pod security policy for Secrets Store CSI Driver pod(s)                                    | `false`                                                                                             |
 | `syncSecret.enabled`                    | Enable rbac roles and bindings required for syncing to Kubernetes native secrets (the default will change to false after v0.0.14) | true                                                |

manifest_staging/charts/secrets-store-csi-driver/templates/secrets-store-csi-driver-windows.yaml

@@ -84,6 +84,9 @@ spec:
             {{- if and (semverCompare ">= v0.0.22-0" .Values.windows.image.tag) .Values.providerHealthCheckInterval }}
             - "--provider-health-check-interval={{ .Values.providerHealthCheckInterval }}"
             {{- end }}
+            {{- if .Values.maxCallRecvMsgSize }}
+            - "--max-call-recv-msg-size={{ .Values.maxCallRecvMsgSize | int64 }}"
+            {{- end }}
           env:
           {{- with .Values.windows.env }}
             {{- toYaml . | nindent 10 }}

manifest_staging/charts/secrets-store-csi-driver/templates/secrets-store-csi-driver.yaml

@@ -84,6 +84,9 @@ spec:
             {{- if and (semverCompare ">= v0.0.22-0" .Values.linux.image.tag) .Values.providerHealthCheckInterval }}
             - "--provider-health-check-interval={{ .Values.providerHealthCheckInterval }}"
             {{- end }}
+            {{- if .Values.maxCallRecvMsgSize }}
+            - "--max-call-recv-msg-size={{ .Values.maxCallRecvMsgSize | int64 }}"
+            {{- end }}
           env:
           {{- with .Values.linux.env }}
             {{- toYaml . | nindent 10 }}

manifest_staging/charts/secrets-store-csi-driver/values.yaml

@@ -153,6 +153,9 @@ livenessProbe:
   port: 9808
   logLevel: 2
 
+## Maximum size in bytes of gRPC response from plugins
+maxCallRecvMsgSize: 4194304
+
 ## Install Default RBAC roles and bindings
 rbac:
   install: true

pkg/secrets-store/provider_client.go

@@ -73,6 +73,7 @@ type PluginClientBuilder struct {
 	conns      map[string]*grpc.ClientConn
 	socketPath string
 	lock       sync.RWMutex
+	opts       []grpc.DialOption
 }
 
 // NewPluginClientBuilder creates a PluginClientBuilder that will connect to
@@ -82,12 +83,23 @@ type PluginClientBuilder struct {
 // 		<path>/<plugin_name>.sock
 //
 // where <plugin_name> must match the PluginNameRe regular expression.
-func NewPluginClientBuilder(path string) *PluginClientBuilder {
+//
+// Additional grpc dial options can also be set through opts and will be used
+// when creating all clients.
+func NewPluginClientBuilder(path string, opts ...grpc.DialOption) *PluginClientBuilder {
 	return &PluginClientBuilder{
 		clients:    make(map[string]v1alpha1.CSIDriverProviderClient),
 		conns:      make(map[string]*grpc.ClientConn),
 		socketPath: path,
 		lock:       sync.RWMutex{},
+		opts: append(opts, []grpc.DialOption{
+			grpc.WithInsecure(), // the interface is only secured through filesystem ACLs
+			grpc.WithContextDialer(func(ctx context.Context, target string) (net.Conn, error) {
+				return (&net.Dialer{}).DialContext(ctx, "unix", target)
+			}),
+			grpc.WithDefaultServiceConfig(ServiceConfig),
+		}...,
+		),
 	}
 }
 
@@ -115,11 +127,7 @@ func (p *PluginClientBuilder) Get(ctx context.Context, provider string) (v1alpha
 
 	conn, err := grpc.Dial(
 		fmt.Sprintf("%s/%s.sock", p.socketPath, provider),
-		grpc.WithInsecure(), // the interface is only secured through filesystem ACLs
-		grpc.WithContextDialer(func(ctx context.Context, target string) (net.Conn, error) {
-			return (&net.Dialer{}).DialContext(ctx, "unix", target)
-		}),
-		grpc.WithDefaultServiceConfig(ServiceConfig),
+		p.opts...,
 	)
 	if err != nil {
 		return nil, err

pkg/secrets-store/provider_client_test.go

@@ -28,6 +28,9 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 
 	"sigs.k8s.io/secrets-store-csi-driver/pkg/test_utils/tmpdir"
 	"sigs.k8s.io/secrets-store-csi-driver/provider/fake"
@@ -157,6 +160,45 @@ func TestMountContent(t *testing.T) {
 	}
 }
 
+func TestMountContent_TooLarge(t *testing.T) {
+	socketPath := tmpdir.New(t, "", "ut")
+	targetPath := tmpdir.New(t, "", "ut")
+
+	// set a very small max message size
+	pool := NewPluginClientBuilder(socketPath, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(5)))
+	defer pool.Cleanup()
+
+	server, cleanup := fakeServer(t, socketPath, "provider1")
+	defer cleanup()
+
+	server.SetObjects(map[string]string{"foo": "v1"})
+	server.SetFiles([]*v1alpha1.File{
+		{
+			Path:     "foo",
+			Mode:     0644,
+			Contents: []byte("foo"),
+		},
+	})
+	server.Start()
+
+	client, err := pool.Get(context.Background(), "provider1")
+	if err != nil {
+		t.Fatalf("expected err to be nil, got: %+v", err)
+	}
+
+	// rpc error: code = ResourceExhausted desc = grpc: received message larger than max (28 vs. 5)
+	_, errorCode, err := MountContent(context.TODO(), client, "{}", "{}", targetPath, "777", nil)
+	if err == nil {
+		t.Errorf("expected err to be not nil")
+	}
+	if want := codes.ResourceExhausted; status.Code(err) != want {
+		t.Errorf("expected error code: %v, got: %+v", want, status.Code(err))
+	}
+	if want := "GRPCProviderError"; errorCode != want {
+		t.Errorf("expected error code: %v, got: %+v", want, errorCode)
+	}
+}
+
 func TestMountContentError(t *testing.T) {
 	cases := []struct {
 		name                  string