Added local file cache for postgres data

Author: akclace

.gitignore

@@ -8,6 +8,7 @@ openrun.toml
 logs/
 .DS_Store
 run/
+app_src/
 config/
 internal/server/appspecs/
 internal/server/appspecs/dummy/README

internal/metadata/filestore.go

@@ -39,10 +39,19 @@ type FileStore struct {
 	db       *sql.DB
 	initTx   types.Transaction // This is the transaction for the initial setup of the app, before it is committed to the database.
 	// After app is committed to database, this is not used, auto-commit transactions are used for reads
+	fileCache *FileCache
 }
 
-func NewFileStore(appId types.AppId, version int, metadata *Metadata, tx types.Transaction) *FileStore {
-	return &FileStore{appId: appId, version: version, metadata: metadata, db: metadata.db, initTx: tx}
+func NewFileStore(appId types.AppId, version int, metadata *Metadata, tx types.Transaction) (*FileStore, error) {
+	var fileCache *FileCache
+	var err error
+	if metadata.dbType != system.DB_TYPE_SQLITE {
+		fileCache, err = InitFileCache(metadata.Logger, metadata.config)
+		if err != nil {
+			return nil, fmt.Errorf("error initializing file cache: %w", err)
+		}
+	}
+	return &FileStore{appId: appId, version: version, metadata: metadata, db: metadata.db, initTx: tx, fileCache: fileCache}, nil
 }
 
 func (f *FileStore) IncrementAppVersion(ctx context.Context, tx types.Transaction, metadata *types.AppMetadata) error {
@@ -184,6 +193,17 @@ func (f *FileStore) GetFileBySha(sha string) ([]byte, string, error) {
 }
 
 func (f *FileStore) GetFileByShaTx(ctx context.Context, tx types.Transaction, sha string) ([]byte, string, error) {
+	if f.fileCache != nil {
+		content, compressionType, err := f.fileCache.GetCachedFile(ctx, sha)
+		if err == nil {
+			f.metadata.Trace().Msgf("Got file from cache: %s", sha)
+			return content, compressionType, nil
+		}
+		if err != nil && err != sql.ErrNoRows {
+			return nil, "", fmt.Errorf("error getting cached file: %w", err)
+		}
+	}
+
 	stmt, err := tx.PrepareContext(ctx, system.RebindQuery(f.metadata.dbType, "SELECT compression_type, content FROM files where sha = ?"))
 	if err != nil {
 		return nil, "", fmt.Errorf("error preparing statement: %w", err)
@@ -197,6 +217,14 @@ func (f *FileStore) GetFileByShaTx(ctx context.Context, tx types.Transaction, sh
 		return nil, "", fmt.Errorf("error querying file table: %w", err)
 	}
 
+	if f.fileCache != nil {
+		f.metadata.Trace().Msgf("Adding file to cache: %s", sha)
+		err = f.fileCache.AddCache(ctx, sha, compressionType, content)
+		if err != nil {
+			return nil, "", fmt.Errorf("error adding file to cache: %w", err)
+		}
+	}
+
 	return content, compressionType, nil
 }
 

internal/server/app_apis.go

@@ -325,7 +325,10 @@ func (s *Server) setupApp(appEntry *types.AppEntry, tx types.Transaction) (*app.
 	var sourceFS *appfs.SourceFs
 	if !appEntry.IsDev {
 		// Prod mode, use DB as source
-		fileStore := metadata.NewFileStore(appEntry.Id, appEntry.Metadata.VersionMetadata.Version, s.db, tx)
+		fileStore, err := metadata.NewFileStore(appEntry.Id, appEntry.Metadata.VersionMetadata.Version, s.db, tx)
+		if err != nil {
+			return nil, err
+		}
 		dbFs, err := metadata.NewDbFs(s.Logger, fileStore, *appEntry.Metadata.SpecFiles)
 		if err != nil {
 			return nil, err
@@ -795,7 +798,10 @@ func (s *Server) loadSourceFromGit(ctx context.Context, tx types.Transaction, ap
 
 	s.Info().Msgf("Loading app sources from %s", checkoutFolder)
 	// Walk the local directory and add all files to the database
-	fileStore := metadata.NewFileStore(appEntry.Id, appEntry.Metadata.VersionMetadata.Version, s.db, tx)
+	fileStore, err := metadata.NewFileStore(appEntry.Id, appEntry.Metadata.VersionMetadata.Version, s.db, tx)
+	if err != nil {
+		return err
+	}
 	highestVersion, err := fileStore.GetHighestVersion(ctx, tx, appEntry.Id)
 	if err != nil {
 		return err
@@ -820,7 +826,10 @@ func (s *Server) loadSourceFromDisk(ctx context.Context, tx types.Transaction, a
 	appEntry.Settings.GitAuthName = ""
 	appEntry.Metadata.VersionMetadata.GitMessage = ""
 
-	fileStore := metadata.NewFileStore(appEntry.Id, appEntry.Metadata.VersionMetadata.Version, s.db, tx)
+	fileStore, err := metadata.NewFileStore(appEntry.Id, appEntry.Metadata.VersionMetadata.Version, s.db, tx)
+	if err != nil {
+		return err
+	}
 	highestVersion, err := fileStore.GetHighestVersion(ctx, tx, appEntry.Id)
 	if err != nil {
 		return fmt.Errorf("error getting highest version: %w", err)

internal/server/app_updates.go

@@ -264,8 +264,11 @@ func (s *Server) StagedUpdateAppsTx(ctx context.Context, tx types.Transaction, a
 				return nil, nil, nil, err
 			}
 
-			stagingFileStore := metadata.NewFileStore(appEntry.Id, appEntry.Metadata.VersionMetadata.Version, s.db, tx)
-			err := stagingFileStore.IncrementAppVersion(ctx, tx, &appEntry.Metadata)
+			stagingFileStore, err := metadata.NewFileStore(appEntry.Id, appEntry.Metadata.VersionMetadata.Version, s.db, tx)
+			if err != nil {
+				return nil, nil, nil, fmt.Errorf("error initializing staging file store: %w", err)
+			}
+			err = stagingFileStore.IncrementAppVersion(ctx, tx, &appEntry.Metadata)
 			if err != nil {
 				return nil, nil, nil, fmt.Errorf("error incrementing app version: %w", err)
 			}
@@ -368,8 +371,14 @@ func (s *Server) PromoteApps(ctx context.Context, appPathGlob string, dryRun boo
 }
 
 func (s *Server) promoteApp(ctx context.Context, tx types.Transaction, stagingApp *types.AppEntry, prodApp *types.AppEntry) error {
-	stagingFileStore := metadata.NewFileStore(stagingApp.Id, stagingApp.Metadata.VersionMetadata.Version, s.db, tx)
-	prodFileStore := metadata.NewFileStore(prodApp.Id, prodApp.Metadata.VersionMetadata.Version, s.db, tx)
+	stagingFileStore, err := metadata.NewFileStore(stagingApp.Id, stagingApp.Metadata.VersionMetadata.Version, s.db, tx)
+	if err != nil {
+		return fmt.Errorf("error initializing staging file store: %w", err)
+	}
+	prodFileStore, err := metadata.NewFileStore(prodApp.Id, prodApp.Metadata.VersionMetadata.Version, s.db, tx)
+	if err != nil {
+		return fmt.Errorf("error initializing prod file store: %w", err)
+	}
 	prevVersion := prodApp.Metadata.VersionMetadata.Version
 	newVersion := stagingApp.Metadata.VersionMetadata.Version
 

internal/server/apply.go

@@ -600,8 +600,12 @@ func (s *Server) applyAppUpdate(ctx context.Context, tx types.Transaction, appPa
 		promoteApp = len(reloadResult.PromoteResults) > 0
 	} else if updated {
 		// No reload, increment version and promote (if enabled)
-		stagingFileStore := metadata.NewFileStore(liveApp.Id, liveApp.Metadata.VersionMetadata.Version, s.db, tx)
-		err := stagingFileStore.IncrementAppVersion(ctx, tx, &liveApp.Metadata)
+		stagingFileStore, err := metadata.NewFileStore(liveApp.Id, liveApp.Metadata.VersionMetadata.Version, s.db, tx)
+		if err != nil {
+			return nil, fmt.Errorf("error initializing staging file store: %w", err)
+		}
+
+		err = stagingFileStore.IncrementAppVersion(ctx, tx, &liveApp.Metadata)
 		if err != nil {
 			return nil, fmt.Errorf("error incrementing app version: %w", err)
 		}

internal/server/version_apis.go

@@ -34,7 +34,10 @@ func (s *Server) VersionList(ctx context.Context, mainAppPath string) (*types.Ap
 		return nil, fmt.Errorf("version commands not supported for dev app")
 	}
 
-	fileStore := metadata.NewFileStore(appEntry.Id, appEntry.Metadata.VersionMetadata.Version, s.db, tx)
+	fileStore, err := metadata.NewFileStore(appEntry.Id, appEntry.Metadata.VersionMetadata.Version, s.db, tx)
+	if err != nil {
+		return nil, err
+	}
 	versions, err := fileStore.GetAppVersions(ctx, tx)
 	if err != nil {
 		return nil, err
@@ -80,7 +83,10 @@ func (s *Server) VersionFiles(ctx context.Context, mainAppPath, version string)
 		}
 	}
 
-	fileStore := metadata.NewFileStore(appEntry.Id, versionInt, s.db, tx)
+	fileStore, err := metadata.NewFileStore(appEntry.Id, versionInt, s.db, tx)
+	if err != nil {
+		return nil, err
+	}
 	files, err := fileStore.GetAppFiles(ctx, tx)
 	if err != nil {
 		return nil, err
@@ -110,7 +116,10 @@ func (s *Server) VersionSwitch(ctx context.Context, mainAppPath string, dryRun b
 		return nil, fmt.Errorf("version commands not supported for dev app")
 	}
 	var versionInt int
-	fileStore := metadata.NewFileStore(appEntry.Id, appEntry.Metadata.VersionMetadata.Version, s.db, tx)
+	fileStore, err := metadata.NewFileStore(appEntry.Id, appEntry.Metadata.VersionMetadata.Version, s.db, tx)
+	if err != nil {
+		return nil, err
+	}
 
 	versionLower := strings.ToLower(version)
 	switch versionLower {

internal/system/openrun.default.toml

@@ -54,6 +54,7 @@ access_logging = true
 db_connection = "sqlite:$OPENRUN_HOME/metadata/clace_metadata.db"
 auto_upgrade = true
 audit_db_connection = "sqlite:$OPENRUN_HOME/metadata/clace_audit.db"
+file_cache_connection = "sqlite:$OPENRUN_HOME/metadata/file_cache.db"
 
 [system]
 tailwindcss_command = "tailwindcss"

internal/types/types.go

@@ -198,6 +198,7 @@ type MetadataConfig struct {
 	AutoUpgrade         bool   `toml:"auto_upgrade"`
 	AuditDBConnection   string `toml:"audit_db_connection"`
 	IgnoreHigherVersion bool   `toml:"ignore_higher_version"` // If true, ignore higher version of the metadata schema
+	FileCacheConnection string `toml:"file_cache_connection"` // The connection string for the file cache database
 }
 
 // LogConfig is the configuration for the Logger