Add support for creating dev mode app from git source

akclace · akclace · commit 183d4eda2d09 · 2025-08-27T16:35:23.000-07:00
diff --git a/cmd/openrun/app_cmds.go b/cmd/openrun/app_cmds.go
@@ -229,6 +229,9 @@ func printCreateResult(cCtx *cli.Context, createResult types.AppCreateResponse)
 	if createResult.HttpsUrl != "" {
 		fmt.Printf("HTTPS Url: %s\n", createResult.HttpsUrl)
 	}
+	if createResult.OrigSourceUrl != "" {
+		fmt.Printf("   Source: %s (created from %s)\n", createResult.SourceUrl, createResult.OrigSourceUrl)
+	}
 	approveResult := createResult.ApproveResults[0]
 	printApproveResult(approveResult)
 
diff --git a/cmd/openrun/apply_cmd.go b/cmd/openrun/apply_cmd.go
@@ -18,6 +18,7 @@ import (
 func initApplyCommand(commonFlags []cli.Flag, clientConfig *types.ClientConfig) *cli.Command {
 	flags := make([]cli.Flag, 0, len(commonFlags)+2)
 	flags = append(flags, commonFlags...)
+	flags = append(flags, newBoolFlag("dev", "d", "Create apps in development mode", false))
 	flags = append(flags, newStringFlag("branch", "b", "The branch to checkout if using git source", "main"))
 	flags = append(flags, newStringFlag("commit", "c", "The commit SHA to checkout if using git source. This takes precedence over branch", ""))
 	flags = append(flags, newStringFlag("git-auth", "g", "The name of the git_auth entry in server config to use", ""))
@@ -74,6 +75,7 @@ Examples:
 			values.Add("promote", strconv.FormatBool(cCtx.Bool("promote")))
 			values.Add("clobber", strconv.FormatBool(cCtx.Bool("clobber")))
 			values.Add("forceReload", strconv.FormatBool(cCtx.Bool("force-reload")))
+			values.Add("dev", strconv.FormatBool(cCtx.Bool("dev")))
 
 			client := system.NewHttpClient(clientConfig.ServerUri, clientConfig.AdminUser, clientConfig.Client.AdminPassword, clientConfig.Client.SkipCertCheck)
 			var applyResponse types.AppApplyResponse
diff --git a/internal/app/apptype/builtins.go b/internal/app/apptype/builtins.go
@@ -467,6 +467,18 @@ func CreateConfigBuiltin(nodeConfig types.NodeConfig, allowedEnv []string) func(
 				return defaultVal, nil
 			}
 			return starlark.String(branchStr), nil
+		} else if key == "_dev" {
+			// _dev is a special input that returns the current dev status
+			dev := thread.Local(types.TL_DEV)
+			if dev == nil {
+				return defaultVal, nil
+			}
+
+			devBool, ok := dev.(bool)
+			if !ok {
+				return nil, fmt.Errorf("dev is not a boolean %v", dev)
+			}
+			return starlark.Bool(devBool), nil
 		}
 
 		localVal, ok := nodeConfig[string(key)]
diff --git a/internal/metadata/metadata.go b/internal/metadata/metadata.go
@@ -12,12 +12,12 @@ import (
 	"strings"
 	"time"
 
-	"github.com/openrundev/openrun/internal/system"
-	"github.com/openrundev/openrun/internal/types"
 	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgconn"
 	_ "github.com/jackc/pgx/v5/stdlib"
 	"github.com/jackc/pgxlisten"
+	"github.com/openrundev/openrun/internal/system"
+	"github.com/openrundev/openrun/internal/types"
 	_ "modernc.org/sqlite"
 )
 
@@ -460,6 +460,14 @@ func (m *Metadata) GetLinkedApps(ctx context.Context, tx types.Transaction, main
 	return apps, nil
 }
 
+func (m *Metadata) UpdateSourceUrl(ctx context.Context, tx types.Transaction, app *types.AppEntry) error {
+	_, err := tx.ExecContext(ctx, system.RebindQuery(m.dbType, `UPDATE apps set source_url = ? where path = ? and domain = ?`), app.SourceUrl, app.Path, app.Domain)
+	if err != nil {
+		return fmt.Errorf("error updating app source url: %w", err)
+	}
+	return nil
+}
+
 func (m *Metadata) UpdateAppMetadata(ctx context.Context, tx types.Transaction, app *types.AppEntry) error {
 	metadataJson, err := json.Marshal(app.Metadata)
 	if err != nil {
diff --git a/internal/server/app_apis.go b/internal/server/app_apis.go
@@ -165,11 +165,7 @@ func (s *Server) CreateAppTx(ctx context.Context, currentTx types.Transaction, a
 
 func (s *Server) createApp(ctx context.Context, tx types.Transaction,
 	appEntry *types.AppEntry, approve, dryRun bool, branch, commit, gitAuth string, applyInfo *types.CreateAppRequest, repoCache *RepoCache) (*types.AppCreateResponse, error) {
-	if system.IsGit(appEntry.SourceUrl) {
-		if appEntry.IsDev {
-			return nil, fmt.Errorf("cannot create dev mode app from git source. For dev mode, manually checkout the git repo and create app from the local path")
-		}
-	} else {
+	if !system.IsGit(appEntry.SourceUrl) {
 		if appEntry.SourceUrl != types.NO_SOURCE {
 			// Make sure the source path is absolute
 			var err error
@@ -258,6 +254,11 @@ func (s *Server) createApp(ctx context.Context, tx types.Transaction,
 		return nil, fmt.Errorf("App %s audit failed: %s", workEntry.Id, err)
 	}
 
+	// Persist the source url
+	if err := s.db.UpdateSourceUrl(ctx, tx, workEntry); err != nil {
+		return nil, err
+	}
+
 	// Persist the metadata so that any git info is saved
 	if err := s.db.UpdateAppMetadata(ctx, tx, workEntry); err != nil {
 		return nil, err
@@ -293,6 +294,8 @@ func (s *Server) createApp(ctx context.Context, tx types.Transaction,
 		HttpsUrl:       s.getAppHttpsUrl(appEntry),
 		DryRun:         dryRun,
 		ApproveResults: results,
+		OrigSourceUrl:  appEntry.Settings.OrigSourceUrl,
+		SourceUrl:      appEntry.SourceUrl,
 	}
 
 	return ret, nil
@@ -756,11 +759,22 @@ func (s *Server) loadSourceFromGit(ctx context.Context, tx types.Transaction, ap
 	gitAuth = cmp.Or(gitAuth, appEntry.Settings.GitAuthName)
 	branch = cmp.Or(branch, appEntry.Metadata.VersionMetadata.GitBranch, "main")
 
-	repo, folder, message, hash, err := repoCache.CheckoutRepo(appEntry.SourceUrl, branch, commit, gitAuth)
+	repo, folder, message, hash, err := repoCache.CheckoutRepo(appEntry.SourceUrl, branch, commit, gitAuth, appEntry.IsDev)
 	if err != nil {
 		return err
 	}
 
+	if system.IsGit(appEntry.SourceUrl) && appEntry.IsDev {
+		// Dev app from git, we need to point the app to the local checkout location
+		sourcePath := repo
+		if folder != "" {
+			sourcePath = path.Join(repo, folder)
+		}
+		// App metadata points to the local checkout location
+		appEntry.Settings.OrigSourceUrl = appEntry.SourceUrl
+		appEntry.SourceUrl = sourcePath
+	}
+
 	// Update the git info into the appEntry, the caller needs to persist it into the app metadata
 	// This function will persist it into the app_version metadata
 	appEntry.Metadata.VersionMetadata.GitCommit = hash
diff --git a/internal/server/apply.go b/internal/server/apply.go
@@ -29,7 +29,7 @@ const (
 	APP = "app"
 )
 
-func (s *Server) loadApplyInfo(fileName string, data []byte, branch string) ([]*types.CreateAppRequest, error) {
+func (s *Server) loadApplyInfo(fileName string, data []byte, branch string, dev bool) ([]*types.CreateAppRequest, error) {
 	appDefs := make([]*starlarkstruct.Struct, 0)
 
 	createAppBuiltin := func(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
@@ -82,6 +82,7 @@ func (s *Server) loadApplyInfo(fileName string, data []byte, branch string) ([]*
 	}
 
 	thread.SetLocal(types.TL_BRANCH, branch)
+	thread.SetLocal(types.TL_DEV, dev)
 
 	options := syntax.FileOptions{}
 	_, err := starlark.ExecFileOptions(&options, thread, fileName, data, builtins)
@@ -203,7 +204,7 @@ func (s *Server) setupSource(applyPath, branch, commit, gitAuth string, repoCach
 	}
 
 	branch = cmp.Or(branch, "main")
-	repo, applyFile, _, _, err := repoCache.CheckoutRepo(applyPath, branch, commit, gitAuth)
+	repo, applyFile, _, _, err := repoCache.CheckoutRepo(applyPath, branch, commit, gitAuth, false)
 	if err != nil {
 		return "", "", err
 	}
@@ -218,7 +219,8 @@ func (s *Server) setupSource(applyPath, branch, commit, gitAuth string, repoCach
 }
 
 func (s *Server) Apply(ctx context.Context, inputTx types.Transaction, applyPath string, appPathGlob string, approve, dryRun, promote bool,
-	reload types.AppReloadOption, branch, commit, gitAuth string, clobber, forceReload bool, lastRunCommitId string, repoCache *RepoCache) (*types.AppApplyResponse, []types.AppPathDomain, error) {
+	reload types.AppReloadOption, branch, commit, gitAuth string, clobber,
+	forceReload bool, lastRunCommitId string, repoCache *RepoCache, dev bool) (*types.AppApplyResponse, []types.AppPathDomain, error) {
 	var tx types.Transaction
 	var err error
 	if inputTx.Tx == nil {
@@ -289,7 +291,7 @@ func (s *Server) Apply(ctx context.Context, inputTx types.Transaction, applyPath
 			return nil, nil, fmt.Errorf("error reading file %s: %w", f, err)
 		}
 
-		fileConfig, err := s.loadApplyInfo(f, fileBytes, branch)
+		fileConfig, err := s.loadApplyInfo(f, fileBytes, branch, dev)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -365,6 +367,9 @@ func (s *Server) Apply(ctx context.Context, inputTx types.Transaction, applyPath
 	for _, newApp := range newApps {
 		s.Trace().Msgf("Applying create app %s", newApp)
 		applyInfo := applyConfig[newApp]
+		if dev {
+			applyInfo.IsDev = dev // Override the dev status from the apply command cli
+		}
 		res, err := s.CreateAppTx(ctx, tx, newApp.String(), approve, dryRun, applyInfo, repoCache)
 		if err != nil {
 			return nil, nil, err
diff --git a/internal/server/repo_cache.go b/internal/server/repo_cache.go
@@ -7,6 +7,8 @@ import (
 	"cmp"
 	"fmt"
 	"os"
+	"path"
+	"path/filepath"
 
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/config"
@@ -124,7 +126,7 @@ func latestCommitSHA(repoURL, branch string, auth transport.AuthMethod) (string,
 	return "", fmt.Errorf("branch %q not found", branch)
 }
 
-func (r *RepoCache) CheckoutRepo(sourceUrl, branch, commit, gitAuth string) (string, string, string, string, error) {
+func (r *RepoCache) CheckoutRepo(sourceUrl, branch, commit, gitAuth string, isDev bool) (string, string, string, string, error) {
 	gitAuth = cmp.Or(gitAuth, r.server.config.Security.DefaultGitAuth)
 	authEntry, err := r.server.loadGitKey(gitAuth)
 	if err != nil {
@@ -137,7 +139,8 @@ func (r *RepoCache) CheckoutRepo(sourceUrl, branch, commit, gitAuth string) (str
 		return "", "", "", "", err
 	}
 
-	dir, ok := r.cache[Repo{repo, branch, commit, gitAuth}]
+	repoKey := Repo{repo, branch, commit, gitAuth}
+	dir, ok := r.cache[repoKey]
 	if ok {
 		return dir.dir, folder, dir.commitMessage, dir.hash, nil
 	}
@@ -150,7 +153,9 @@ func (r *RepoCache) CheckoutRepo(sourceUrl, branch, commit, gitAuth string) (str
 		// No commit id specified, checkout specified branch
 		cloneOptions.ReferenceName = plumbing.NewBranchReferenceName(branch)
 		cloneOptions.SingleBranch = true
-		cloneOptions.Depth = 1
+		if !isDev {
+			cloneOptions.Depth = 1
+		}
 	}
 
 	if gitAuth != "" {
@@ -162,9 +167,19 @@ func (r *RepoCache) CheckoutRepo(sourceUrl, branch, commit, gitAuth string) (str
 		cloneOptions.Auth = auth
 	}
 
-	targetPath, err := os.MkdirTemp(r.rootDir, "repo_")
-	if err != nil {
-		return "", "", "", "", err
+	var targetPath string
+	if isDev {
+		// We don't have a previous dev checkout for this repo, create a new one
+		repoName := filepath.Base(repo)
+		targetPath = getUnusedRepoPath(os.ExpandEnv("$OPENRUN_HOME/app_src/"), repoName)
+		if err := os.MkdirAll(targetPath, 0744); err != nil {
+			return "", "", "", "", err
+		}
+	} else {
+		targetPath, err = os.MkdirTemp(r.rootDir, "repo_")
+		if err != nil {
+			return "", "", "", "", err
+		}
 	}
 
 	// Configure the repo to Clone
@@ -215,6 +230,20 @@ func (r *RepoCache) CheckoutRepo(sourceUrl, branch, commit, gitAuth string) (str
 	return targetPath, folder, newCommit.Message, newCommit.Hash.String(), nil
 }
 
+func getUnusedRepoPath(targetDir, repoName string) string {
+	if _, err := os.Stat(path.Join(targetDir, repoName)); os.IsNotExist(err) {
+		return path.Join(targetDir, repoName)
+	}
+	count := 2
+	for {
+		unusedName := fmt.Sprintf("%s%d", repoName, count)
+		if _, err := os.Stat(path.Join(targetDir, unusedName)); os.IsNotExist(err) {
+			return path.Join(targetDir, unusedName)
+		}
+		count++
+	}
+}
+
 func (r *RepoCache) Cleanup() {
 	if r.rootDir != "" {
 		os.RemoveAll(r.rootDir)
diff --git a/internal/server/router.go b/internal/server/router.go
@@ -18,11 +18,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/go-chi/chi"
+	"github.com/go-chi/chi/middleware"
 	"github.com/openrundev/openrun/internal/app"
 	"github.com/openrundev/openrun/internal/system"
 	"github.com/openrundev/openrun/internal/types"
-	"github.com/go-chi/chi"
-	"github.com/go-chi/chi/middleware"
 )
 
 const (
@@ -986,10 +986,15 @@ func (h *Handler) apply(r *http.Request) (any, error) {
 	}
 	updateOperationInContext(r, genOperationName("apply", promote, approve))
 
+	dev, err := parseBoolArg(r.URL.Query().Get("dev"), false)
+	if err != nil {
+		return nil, err
+	}
+
 	ret, _, err := h.server.Apply(r.Context(), types.Transaction{}, applyPath, appPathGlob, approve, dryRun, promote,
 		types.AppReloadOption(r.URL.Query().Get("reload")),
 		r.URL.Query().Get("branch"), r.URL.Query().Get("commit"), r.URL.Query().Get("gitAuth"),
-		clobber, forceReload, "", nil)
+		clobber, forceReload, "", nil, dev)
 	if err != nil {
 		return nil, types.CreateRequestError(err.Error(), http.StatusInternalServerError)
 	}
diff --git a/internal/server/sync.go b/internal/server/sync.go
@@ -242,7 +242,7 @@ func (s *Server) runSyncJob(ctx context.Context, inputTx types.Transaction, entr
 	}
 
 	applyInfo, updatedApps, applyErr := s.Apply(ctx, tx, entry.Path, "all", entry.Metadata.Approve, dryRun, entry.Metadata.Promote, types.AppReloadOption(entry.Metadata.Reload),
-		entry.Metadata.GitBranch, "", entry.Metadata.GitAuth, entry.Metadata.Clobber, entry.Metadata.ForceReload, lastRunCommitId, repoCache)
+		entry.Metadata.GitBranch, "", entry.Metadata.GitAuth, entry.Metadata.Clobber, entry.Metadata.ForceReload, lastRunCommitId, repoCache, false)
 
 	status := types.SyncJobStatus{
 		LastExecutionTime: time.Now(),
diff --git a/internal/types/api.go b/internal/types/api.go
@@ -108,6 +108,8 @@ type AppCreateResponse struct {
 	HttpUrl        string          `json:"http_url"`
 	HttpsUrl       string          `json:"https_url"`
 	ApproveResults []ApproveResult `json:"approve_results"`
+	OrigSourceUrl  string          `json:"orig_source_url"`
+	SourceUrl      string          `json:"source_url"`
 }
 
 type AppDeleteResponse struct {
diff --git a/internal/types/types.go b/internal/types/types.go
@@ -51,6 +51,7 @@ const (
 	TL_AUDIT_DETAIL             = "TL_audit_detail"
 	TL_CONTAINER_MANAGER        = "TL_container_manager"
 	TL_BRANCH                   = "TL_branch"
+	TL_DEV                      = "TL_dev"
 )
 
 const (
@@ -412,6 +413,7 @@ type AppSettings struct {
 	StageWriteAccess   bool          `json:"stage_write_access"`
 	PreviewWriteAccess bool          `json:"preview_write_access"`
 	WebhookTokens      WebhookTokens `json:"webhook_tokens"`
+	OrigSourceUrl      string        `json:"orig_source_url"` // the original source url of the app, used for git create in dev mode
 }
 
 type WebhookTokens struct {
diff --git a/tests/apply_files/apply_git.ace b/tests/apply_files/apply_git.ace
@@ -1,3 +1,6 @@
-app("/applytest/test1", "github.com/openrundev/apps/system/disk_usage")
-app("/applytest/test2", "github.com/openrundev/apps/system/memory_usage")
-app("/applytest/test3", "github.com/openrundev/apps/system/list_files")
+def path(p):
+    return "/dev/" + p if config("_dev", False) else p
+
+app(path("/applytest/test1"), "github.com/openrundev/apps/system/disk_usage")
+app(path("/applytest/test2"), "github.com/openrundev/apps/system/memory_usage")
+app(path("/applytest/test3"), "github.com/openrundev/apps/system/list_files")
diff --git a/tests/apply_files/apply_git2.ace b/tests/apply_files/apply_git2.ace
@@ -1,3 +1,6 @@
-app("/applytest/test1", "github.com/openrundev/apps/system/disk_usage", auth="system")
-app("/applytest/test2", "github.com/openrundev/apps/system/memory_usage", git_auth="abc")
-app("/applytest/test3", "github.com/openrundev/apps/system/list_files", spec="python-flask")
+def path(p):
+    return "/dev/" + p if config("_dev", False) else p
+
+app(path("/applytest/test1"), "github.com/openrundev/apps/system/disk_usage", auth="system")
+app(path("/applytest/test2"), "github.com/openrundev/apps/system/memory_usage", git_auth="abc")
+app(path("/applytest/test3"), "github.com/openrundev/apps/system/list_files", spec="python-flask")
diff --git a/tests/commander/test_apply_git.yaml b/tests/commander/test_apply_git.yaml
@@ -7,6 +7,10 @@ tests:
   apply_git0010: ## create all apps
     command: ../openrun apply ./apply_files/apply_git.ace all
     stdout: "3 app(s) created, 0 app(s) updated, 0 app(s) reloaded, 0 app(s) skipped, 0 app(s) approved, 0 app(s) promoted."
+  apply_git0011: ## create all apps in dev mode
+    command: ../openrun apply --dev ./apply_files/apply_git.ace all
+    stdout: "app_src"
+    exit-code: 0
   apply_git0020:
     command: ../openrun apply --approve --promote --reload=matched ./apply_files/apply_git.ace all
     stdout: "0 app(s) created, 0 app(s) updated, 0 app(s) reloaded, 3 app(s) skipped, 0 app(s) approved, 0 app(s) promoted."
diff --git a/tests/commander/test_load_app.yaml b/tests/commander/test_load_app.yaml
@@ -117,11 +117,20 @@ tests:
     command: ../openrun app create --approve github.com/openrundev/openrun/examples/disk_usage /ghload1
     stdout: "App created. Permissions have been approved"
     exit-code: 0
+  load0211: # Create app in dev mode from git
+    command: ../openrun app create --dev --approve github.com/openrundev/openrun/examples/disk_usage /ghloaddev1
+    stdout: "app_src"
+    exit-code: 0
   load0220: # test calling app
     command: curl -su "admin:qwerty" localhost:25222/ghload1/
     stderr:
       line-count: 0
     stdout: "Disk Usage"
+  load0221: # test calling dev app
+    command: curl -su "admin:qwerty" localhost:25222/ghloaddev1/
+    stderr:
+      line-count: 0
+    stdout: "Disk Usage"
   load0230: # Create app granting approval
     command: ../openrun app create --approve https://github.com/openrundev/openrun/examples/disk_usage /ghload2
     stdout: "App created. Permissions have been approved"

Original file line number	Diff line number	Diff line change
`@@ -229,6 +229,9 @@ func printCreateResult(cCtx *cli.Context, createResult types.AppCreateResponse)`
`229`	`229`	`if createResult.HttpsUrl != "" {`
`230`	`230`	`fmt.Printf("HTTPS Url: %s\n", createResult.HttpsUrl)`
`231`	`231`	`}`
	`232`	`+ if createResult.OrigSourceUrl != "" {`
	`233`	`+ fmt.Printf(" Source: %s (created from %s)\n", createResult.SourceUrl, createResult.OrigSourceUrl)`
	`234`	`+ }`
`232`	`235`	`approveResult := createResult.ApproveResults[0]`
`233`	`236`	`printApproveResult(approveResult)`
`234`	`237`
Original file line number	Diff line number	Diff line change
`@@ -242,7 +242,7 @@ func (s *Server) runSyncJob(ctx context.Context, inputTx types.Transaction, entr`
`242`	`242`	`}`
`243`	`243`
`244`	`244`	`applyInfo, updatedApps, applyErr := s.Apply(ctx, tx, entry.Path, "all", entry.Metadata.Approve, dryRun, entry.Metadata.Promote, types.AppReloadOption(entry.Metadata.Reload),`
`245`		`- entry.Metadata.GitBranch, "", entry.Metadata.GitAuth, entry.Metadata.Clobber, entry.Metadata.ForceReload, lastRunCommitId, repoCache)`
	`245`	`+ entry.Metadata.GitBranch, "", entry.Metadata.GitAuth, entry.Metadata.Clobber, entry.Metadata.ForceReload, lastRunCommitId, repoCache, false)`
`246`	`246`
`247`	`247`	`status := types.SyncJobStatus{`
`248`	`248`	`LastExecutionTime: time.Now(),`
Original file line number	Diff line number	Diff line change
`@@ -108,6 +108,8 @@ type AppCreateResponse struct {`
`108`	`108`	HttpUrl string `json:"http_url"`
`109`	`109`	HttpsUrl string `json:"https_url"`
`110`	`110`	ApproveResults []ApproveResult `json:"approve_results"`
	`111`	+ OrigSourceUrl string `json:"orig_source_url"`
	`112`	+ SourceUrl string `json:"source_url"`
`111`	`113`	`}`
`112`	`114`
`113`	`115`	`type AppDeleteResponse struct {`