Replace GetContainingDigests() with VirtualApply()

Right now we have completely different code paths for computing the
containing digests for files and directories. For files we call
LinkableLeaf.VirtualApply(), while for directories we call
InitialContentsFetcher.GetContainingDigests().

The downside of the current approach is that generic virtual file system
APIs are strongly coupled against REv2. By giving InitialContentsFetcher
a VirtualApply(), we get rid of the coupling. In addition to that, it's
now possible to call InitialChild.GetNode().VirtualApply() to obtain the
set of containing digests in a file type oblivious way.

Author: EdSchouten

pkg/filesystem/virtual/cas_initial_contents_fetcher.go

@@ -127,23 +127,29 @@ func (icf *casInitialContentsFetcher) FetchContents(fileReadMonitorFactory FileR
 	return children, nil
 }
 
-func (icf *casInitialContentsFetcher) GetContainingDigests(ctx context.Context) (digest.Set, error) {
-	gatherer := casContainingDigestsGatherer{
-		context:             ctx,
-		digestFunction:      icf.options.digestFunction,
-		digests:             digest.NewSetBuilder(),
-		directoriesGathered: map[digest.Digest]struct{}{},
-	}
-	err := gatherer.traverse(icf.directoryWalker)
-	if err != nil {
-		return digest.EmptySet, err
+func (icf *casInitialContentsFetcher) VirtualApply(data any) bool {
+	switch p := data.(type) {
+	case *ApplyGetContainingDigests:
+		gatherer := casContainingDigestsGatherer{
+			context:             p.Context,
+			digestFunction:      icf.options.digestFunction,
+			digests:             digest.NewSetBuilder(),
+			directoriesGathered: map[digest.Digest]struct{}{},
+		}
+		if err := gatherer.traverse(icf.directoryWalker); err == nil {
+			p.ContainingDigests = gatherer.digests.Build()
+		} else {
+			p.Err = err
+		}
+	default:
+		return false
 	}
-	return gatherer.digests.Build(), nil
+	return true
 }
 
 // casContainingDigestsGatherer is used by casInitialContentsFetcher's
-// GetContainingDigests() to compute the transitive closure of digests
-// referenced by a hierarchy of Directory objects.
+// implementation of ApplyGetContainingDigests to compute the transitive
+// closure of digests referenced by a hierarchy of Directory objects.
 type casContainingDigestsGatherer struct {
 	context             context.Context
 	digestFunction      digest.Function

pkg/filesystem/virtual/cas_initial_contents_fetcher_test.go

@@ -257,8 +257,11 @@ func TestCASInitialContentsFetcherGetContainingDigests(t *testing.T) {
 			Return(nil, status.Error(codes.Internal, "Server failure"))
 		directoryWalker.EXPECT().GetDescription().Return("Root directory")
 
-		_, err := initialContentsFetcher.GetContainingDigests(ctx)
-		testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Root directory: Server failure"), err)
+		p := virtual.ApplyGetContainingDigests{
+			Context: ctx,
+		}
+		require.True(t, initialContentsFetcher.VirtualApply(&p))
+		testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Root directory: Server failure"), p.Err)
 	})
 
 	t.Run("ChildDirectoryInvalidDigest", func(t *testing.T) {
@@ -277,8 +280,11 @@ func TestCASInitialContentsFetcherGetContainingDigests(t *testing.T) {
 		}, nil)
 		directoryWalker.EXPECT().GetDescription().Return("Root directory")
 
-		_, err := initialContentsFetcher.GetContainingDigests(ctx)
-		testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Root directory: Failed to obtain digest for directory \"hello\": Hash has length 18, while 32 characters were expected"), err)
+		p := virtual.ApplyGetContainingDigests{
+			Context: ctx,
+		}
+		require.True(t, initialContentsFetcher.VirtualApply(&p))
+		testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Root directory: Failed to obtain digest for directory \"hello\": Hash has length 18, while 32 characters were expected"), p.Err)
 	})
 
 	t.Run("ChildFileInvalidDigest", func(t *testing.T) {
@@ -297,8 +303,11 @@ func TestCASInitialContentsFetcherGetContainingDigests(t *testing.T) {
 		}, nil)
 		directoryWalker.EXPECT().GetDescription().Return("Root directory")
 
-		_, err := initialContentsFetcher.GetContainingDigests(ctx)
-		testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Root directory: Failed to obtain digest for file \"hello\": Hash has length 18, while 32 characters were expected"), err)
+		p := virtual.ApplyGetContainingDigests{
+			Context: ctx,
+		}
+		require.True(t, initialContentsFetcher.VirtualApply(&p))
+		testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Root directory: Failed to obtain digest for file \"hello\": Hash has length 18, while 32 characters were expected"), p.Err)
 	})
 
 	t.Run("Success", func(t *testing.T) {
@@ -357,8 +366,11 @@ func TestCASInitialContentsFetcherGetContainingDigests(t *testing.T) {
 			},
 		}, nil)
 
-		digests, err := initialContentsFetcher.GetContainingDigests(ctx)
-		require.NoError(t, err)
+		p := virtual.ApplyGetContainingDigests{
+			Context: ctx,
+		}
+		require.True(t, initialContentsFetcher.VirtualApply(&p))
+		require.NoError(t, p.Err)
 		require.Equal(
 			t,
 			digest.NewSetBuilder().
@@ -367,6 +379,6 @@ func TestCASInitialContentsFetcherGetContainingDigests(t *testing.T) {
 				Add(digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "c0607941dd5b3ca8e175a1bfbfd1c0ea", 789)).
 				Add(digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "19dc69325bd8dfcd75cefbb6144ea3bb", 42)).
 				Build(),
-			digests)
+			p.ContainingDigests)
 	})
 }

pkg/filesystem/virtual/empty_initial_contents_fetcher.go

@@ -1,9 +1,6 @@
 package virtual
 
 import (
-	"context"
-
-	"github.com/buildbarn/bb-storage/pkg/digest"
 	"github.com/buildbarn/bb-storage/pkg/filesystem/path"
 )
 
@@ -13,8 +10,8 @@ func (f emptyInitialContentsFetcher) FetchContents(fileReadMonitorFactory FileRe
 	return map[path.Component]InitialChild{}, nil
 }
 
-func (f emptyInitialContentsFetcher) GetContainingDigests(ctx context.Context) (digest.Set, error) {
-	return digest.EmptySet, nil
+func (f emptyInitialContentsFetcher) VirtualApply(data any) bool {
+	return false
 }
 
 // EmptyInitialContentsFetcher is an instance of InitialContentsFetcher

pkg/filesystem/virtual/initial_contents_fetcher.go

@@ -1,16 +1,22 @@
 package virtual
 
 import (
-	"context"
-
-	"github.com/buildbarn/bb-storage/pkg/digest"
 	"github.com/buildbarn/bb-storage/pkg/filesystem/path"
 )
 
+// InitialNode contains the common set of operations that can be applied
+// against files and directories returned by
+// InitialContentsFetcher.FetchContents().
+type InitialNode interface {
+	// VirtualApply can be used to perform implementation defined
+	// operations against files and directories.
+	VirtualApply(data any) bool
+}
+
 // InitialChild is the value type of the map of directory entries
 // returned by InitialContentsFetcher.FetchContents(). Either Directory
 // or Leaf is set, but not both.
-type InitialChild = Child[InitialContentsFetcher, LinkableLeaf, any]
+type InitialChild = Child[InitialContentsFetcher, LinkableLeaf, InitialNode]
 
 // FileReadMonitor is used by the regular files created through the
 // InitialContentsFetcher to indicate that one or more calls against
@@ -35,21 +41,7 @@ type FileReadMonitorFactory func(name path.Component) FileReadMonitor
 // may be possible FetchContents() is never called. This may happen if
 // the directory in question is never accessed.
 type InitialContentsFetcher interface {
-	FetchContents(fileReadMonitorFactory FileReadMonitorFactory) (map[path.Component]InitialChild, error)
+	InitialNode
 
-	// GetContainingDigests() returns a set of digests of objects in
-	// the Content Addressable Storage that back the directories and
-	// leaf nodes yielded by this InitialContentsFetcher.
-	//
-	// The set returned by this function may be passed to
-	// ContentAddressableStorage.FindMissingBlobs() to check whether
-	// the all files underneath this directory still exist, and to
-	// prevent them from being removed in the nearby future.
-	//
-	// This API assumes that the resulting set is small enough to
-	// fit in memory. For hierarchies backed by Tree objects, this
-	// will generally hold. It may not be safe to call this method
-	// on InitialContentsFetchers that expand to infinitely big
-	// hierarchies.
-	GetContainingDigests(ctx context.Context) (digest.Set, error)
+	FetchContents(fileReadMonitorFactory FileReadMonitorFactory) (map[path.Component]InitialChild, error)
 }

pkg/filesystem/virtual/linkable_leaf.go

@@ -4,6 +4,7 @@ package virtual
 // PrepopulatedDirectory.
 type LinkableLeaf interface {
 	Leaf
+	InitialNode
 
 	// Operations called into by implementations of
 	// PrepopulatedDirectory. The Link() operation may fail, for the

pkg/filesystem/virtual/node.go

@@ -68,8 +68,12 @@ type ApplyUploadFile struct {
 // the file still exists in its entirety, and to prevent that
 // the file is removed in the nearby future.
 type ApplyGetContainingDigests struct {
+	// Inputs.
+	Context context.Context
+
 	// Outputs.
 	ContainingDigests digest.Set
+	Err               error
 }
 
 // ApplyGetBazelOutputServiceStat is an operation for VirtualApply that