Add bufpolicystore and bufpolicycache for Policies

private/bufpkg/bufpolicy/bufpolicycache/bufpolicycache.go

@@ -0,0 +1,15 @@
+// Copyright 2020-2025 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bufpolicycache

private/bufpkg/bufpolicy/bufpolicycache/policy_data_provider.go

@@ -0,0 +1,111 @@
+// Copyright 2020-2025 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bufpolicycache
+
+import (
+	"context"
+	"log/slog"
+	"sync/atomic"
+
+	"github.com/bufbuild/buf/private/bufpkg/bufpolicy"
+	"github.com/bufbuild/buf/private/bufpkg/bufpolicy/bufpolicystore"
+	"github.com/bufbuild/buf/private/pkg/slicesext"
+	"github.com/bufbuild/buf/private/pkg/syserror"
+	"github.com/bufbuild/buf/private/pkg/uuidutil"
+	"github.com/google/uuid"
+)
+
+// NewPolicyDataProvider returns a new PolicyDataProvider that caches the results of the delegate.
+//
+// The PolicyDataStore is used as a cache.
+func NewPolicyDataProvider(
+	logger *slog.Logger,
+	delegate bufpolicy.PolicyDataProvider,
+	store bufpolicystore.PolicyDataStore,
+) bufpolicy.PolicyDataProvider {
+	return newPolicyDataProvider(logger, delegate, store)
+}
+
+/// *** PRIVATE ***
+
+type policyDataProvider struct {
+	logger   *slog.Logger
+	delegate bufpolicy.PolicyDataProvider
+	store    bufpolicystore.PolicyDataStore
+
+	keysRetrieved atomic.Int64
+	keysHit       atomic.Int64
+}
+
+func newPolicyDataProvider(
+	logger *slog.Logger,
+	delegate bufpolicy.PolicyDataProvider,
+	store bufpolicystore.PolicyDataStore,
+) *policyDataProvider {
+	return &policyDataProvider{
+		logger:   logger,
+		delegate: delegate,
+		store:    store,
+	}
+}
+
+func (p *policyDataProvider) GetPolicyDatasForPolicyKeys(
+	ctx context.Context,
+	policyKeys []bufpolicy.PolicyKey,
+) ([]bufpolicy.PolicyData, error) {
+	foundValues, notFoundKeys, err := p.store.GetPolicyDatasForPolicyKeys(ctx, policyKeys)
+	if err != nil {
+		return nil, err
+	}
+
+	delegateValues, err := p.delegate.GetPolicyDatasForPolicyKeys(ctx, notFoundKeys)
+	if err != nil {
+		return nil, err
+	}
+	if err := p.store.PutPolicyDatas(ctx, delegateValues); err != nil {
+		return nil, err
+	}
+
+	p.keysRetrieved.Add(int64(len(policyKeys)))
+	p.keysHit.Add(int64(len(foundValues)))
+
+	commitIDToIndexedKey, err := slicesext.ToUniqueIndexedValuesMap(
+		policyKeys,
+		func(policyKey bufpolicy.PolicyKey) uuid.UUID {
+			return policyKey.CommitID()
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+	indexedValues, err := slicesext.MapError(
+		append(foundValues, delegateValues...),
+		func(value bufpolicy.PolicyData) (slicesext.Indexed[bufpolicy.PolicyData], error) {
+			commitID := value.PolicyKey().CommitID()
+			indexedKey, ok := commitIDToIndexedKey[commitID]
+			if !ok {
+				return slicesext.Indexed[bufpolicy.PolicyData]{}, syserror.Newf("did not get value from store with commitID %q", uuidutil.ToDashless(commitID))
+			}
+			return slicesext.Indexed[bufpolicy.PolicyData]{
+				Value: value,
+				Index: indexedKey.Index,
+			}, nil
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+	return slicesext.IndexedToSortedValues(indexedValues), nil
+}

private/bufpkg/bufpolicy/bufpolicystore/buf_policy_store.go

@@ -0,0 +1,164 @@
+// Copyright 2020-2025 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bufpolicystore
+
+import (
+	"context"
+	"errors"
+	"io/fs"
+	"log/slog"
+
+	"github.com/bufbuild/buf/private/bufpkg/bufpolicy"
+	"github.com/bufbuild/buf/private/pkg/normalpath"
+	"github.com/bufbuild/buf/private/pkg/storage"
+	"github.com/bufbuild/buf/private/pkg/uuidutil"
+)
+
+// PolicyDataStore reads and writes PolicysDatas.
+type PolicyDataStore interface {
+	// GetPolicyDatasForPolicyKeys gets the PolicyDatas from the store for the PolicyKeys.
+	//
+	// Returns the found PolicyDatas, and the input PolicyKeys that were not found, each
+	// ordered by the order of the input PolicyKeys.
+	GetPolicyDatasForPolicyKeys(context.Context, []bufpolicy.PolicyKey) (
+		foundPolicyDatas []bufpolicy.PolicyData,
+		notFoundPolicyKeys []bufpolicy.PolicyKey,
+		err error,
+	)
+	// PutPolicyDatas puts the PolicyDatas to the store.
+	PutPolicyDatas(ctx context.Context, moduleDatas []bufpolicy.PolicyData) error
+}
+
+// NewPolicyDataStore returns a new PolicyDataStore for the given bucket.
+//
+// It is assumed that the PolicyDataStore has complete control of the bucket.
+//
+// This is typically used to interact with a cache directory.
+func NewPolicyDataStore(
+	logger *slog.Logger,
+	bucket storage.ReadWriteBucket,
+) PolicyDataStore {
+	return newPolicyDataStore(logger, bucket)
+}
+
+/// *** PRIVATE ***
+
+type policyDataStore struct {
+	logger *slog.Logger
+	bucket storage.ReadWriteBucket
+}
+
+func newPolicyDataStore(
+	logger *slog.Logger,
+	bucket storage.ReadWriteBucket,
+) *policyDataStore {
+	return &policyDataStore{
+		logger: logger,
+		bucket: bucket,
+	}
+}
+
+func (p *policyDataStore) GetPolicyDatasForPolicyKeys(
+	ctx context.Context,
+	policyKeys []bufpolicy.PolicyKey,
+) ([]bufpolicy.PolicyData, []bufpolicy.PolicyKey, error) {
+	var foundPolicyDatas []bufpolicy.PolicyData
+	var notFoundPolicyKeys []bufpolicy.PolicyKey
+	for _, policyKey := range policyKeys {
+		policyData, err := p.getPolicyDataForPolicyKey(ctx, policyKey)
+		if err != nil {
+			if !errors.Is(err, fs.ErrNotExist) {
+				return nil, nil, err
+			}
+			notFoundPolicyKeys = append(notFoundPolicyKeys, policyKey)
+		} else {
+			foundPolicyDatas = append(foundPolicyDatas, policyData)
+		}
+	}
+	return foundPolicyDatas, notFoundPolicyKeys, nil
+}
+
+func (p *policyDataStore) PutPolicyDatas(
+	ctx context.Context,
+	policyDatas []bufpolicy.PolicyData,
+) error {
+	for _, policyData := range policyDatas {
+		if err := p.putPolicyData(ctx, policyData); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// getPolicyDataForPolicyKey reads the policy data for the policy key from the cache.
+func (p *policyDataStore) getPolicyDataForPolicyKey(
+	ctx context.Context,
+	policyKey bufpolicy.PolicyKey,
+) (bufpolicy.PolicyData, error) {
+	policyDataStorePath, err := getPolicyDataStorePath(policyKey)
+	if err != nil {
+		return nil, err
+	}
+	if exists, err := storage.Exists(ctx, p.bucket, policyDataStorePath); err != nil {
+		return nil, err
+	} else if !exists {
+		return nil, fs.ErrNotExist
+	}
+	return bufpolicy.NewPolicyData(
+		ctx,
+		policyKey,
+		func() ([]byte, error) {
+			return storage.ReadPath(ctx, p.bucket, policyDataStorePath)
+		},
+	)
+}
+
+// putPolicyData puts the policy data into the policy cache.
+func (p *policyDataStore) putPolicyData(
+	ctx context.Context,
+	policyData bufpolicy.PolicyData,
+) error {
+	policyKey := policyData.PolicyKey()
+	policyDataStorePath, err := getPolicyDataStorePath(policyKey)
+	if err != nil {
+		return err
+	}
+	data, err := policyData.Data()
+	if err != nil {
+		return err
+	}
+	// Data is stored uncompressed.
+	return storage.PutPath(ctx, p.bucket, policyDataStorePath, data)
+}
+
+// getPolicyDataStorePath returns the path for the policy data store for the policy key.
+//
+// This is "digestType/registry/owner/name/dashlessCommitID", e.g. the policy
+// "buf.build/acme/check-policy" with commit "12345-abcde" and digest type "o1"
+// will return "o1/buf.build/acme/check-policy/12345abcde.yaml".
+func getPolicyDataStorePath(policyKey bufpolicy.PolicyKey) (string, error) {
+	digest, err := policyKey.Digest()
+	if err != nil {
+		return "", err
+	}
+	fullName := policyKey.FullName()
+	return normalpath.Join(
+		digest.Type().String(),
+		fullName.Registry(),
+		fullName.Owner(),
+		fullName.Name(),
+		uuidutil.ToDashless(policyKey.CommitID())+".yaml",
+	), nil
+}

private/bufpkg/bufpolicy/bufpolicystore/bufpolicystore.go

@@ -0,0 +1,15 @@
+// Copyright 2020-2025 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bufpolicystore