From 9fe7204c8f699d8dc2b621f4c4e224afc38bc36e Mon Sep 17 00:00:00 2001 From: Kayra Date: Thu, 4 Sep 2025 13:11:29 +0300 Subject: [PATCH 1/2] feat(db): add database migration commands --- .github/workflows/rock-test.yaml | 2 +- cmd/migrate.go | 159 ++++++++++++++++++ cmd/start.go | 4 + go.mod | 11 +- go.sum | 46 +++-- internal/config/config.go | 1 + internal/config/types.go | 2 + internal/db/db_init.go | 31 ++-- internal/db/db_init_test.go | 17 ++ .../db/migrations/00001_initial_schema.sql | 71 ++++++++ internal/db/migrations/migration.go | 6 + internal/db/sql_stmts.go | 66 -------- internal/db/types.go | 1 + internal/testutils/db_test_utils.go | 18 +- 14 files changed, 337 insertions(+), 98 deletions(-) create mode 100644 cmd/migrate.go create mode 100644 internal/db/migrations/00001_initial_schema.sql create mode 100644 internal/db/migrations/migration.go diff --git a/.github/workflows/rock-test.yaml b/.github/workflows/rock-test.yaml index 07505d9c..d23944a9 100644 --- a/.github/workflows/rock-test.yaml +++ b/.github/workflows/rock-test.yaml @@ -35,7 +35,7 @@ jobs: sudo rockcraft.skopeo --insecure-policy copy oci-archive:rock-path/$ROCK_FILE_NAME docker-daemon:notary:latest - name: Run the image run: | - docker run -d -p 3000:3000 --name notary notary:latest + docker run -d -p 3000:3000 --name notary --entrypoint "notary start -m --config /etc/notary/config/config.yaml" notary:latest - name: Load config run: | docker exec notary /usr/bin/pebble mkdir -p /etc/notary/config diff --git a/cmd/migrate.go b/cmd/migrate.go new file mode 100644 index 00000000..e3def59f --- /dev/null +++ b/cmd/migrate.go @@ -0,0 +1,159 @@ +package cmd + +import ( + "database/sql" + "fmt" + "log" + "strconv" + + "github.com/canonical/notary/internal/db/migrations" + "github.com/pressly/goose/v3" + "github.com/spf13/cobra" +) + +var dsn string + +// migrateCmd represents the migrate commands. Without a specific command, it will only display help. +var migrateCmd = &cobra.Command{ + Use: "migrate", + Short: "Manage database migrations", + Long: `Manage the database migrations on the configured database. + +Applying migrations and removing migrations modifies the given database to work with Notary. +Read the help messages of the subcommands for more information. +`, + RunE: func(cmd *cobra.Command, args []string) error { + return cmd.Help() + }, +} + +// migrateUpCmd represents the migrate up command. +var migrateUpCmd = &cobra.Command{ + Use: "up", + Short: "Apply database migrations", + Long: `Apply database migrations in order. + +Use with no argument to apply all migrations. Use with a version number to apply migrations up to that version.`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + version, err := prepareGoose(args) + if err != nil { + return err + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + return err + } + if version == 0 { + err = goose.UpContext(cmd.Context(), db, ".", goose.WithNoColor(true)) + } else { + err = goose.UpToContext(cmd.Context(), db, ".", version, goose.WithNoColor(true)) + } + if err != nil { + return err + } + if err := db.Close(); err != nil { + return err + } + return nil + }, +} + +// migrateDownCmd represents the migrate down command. +var migrateDownCmd = &cobra.Command{ + Use: "down", + Short: "Remove database migrations", + Long: `Remove database migrations in order. + + +Use with no argument to apply all migrations. +Use with a version number to remove migrations up to that version. +0 will remove all migrations, which effectively drops all tables.`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + version, err := prepareGoose(args) + if err != nil { + return err + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + return err + } + defer func() { + if err := db.Close(); err != nil { + panic(err) + } + }() + if version == 0 { + err = goose.DownContext(cmd.Context(), db, ".", goose.WithNoColor(true)) + } else { + err = goose.DownToContext(cmd.Context(), db, ".", version, goose.WithNoColor(true)) + } + if err != nil { + return err + } + return nil + }, +} + +// migrateStatusCmd represents the migrate status command. +var migrateStatusCmd = &cobra.Command{ + Use: "status", + Short: "Show database migrations", + Long: `Show the status of database migrations. + +Will display the current migration version and the status of each migration.`, + RunE: func(cmd *cobra.Command, args []string) error { + goose.SetBaseFS(migrations.EmbedMigrations) + err := goose.SetDialect("sqlite") + if err != nil { + return err + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + log.Fatal(err) + } + return goose.StatusContext(cmd.Context(), db, ".", goose.WithNoColor(true)) + }, +} + +func init() { + migrateUpCmd.Flags().StringVarP(&dsn, "database-path", "d", "./notary.db", "A DSN for connecting to the database. Also accepts a path to a file, and will assume that the database is SQLite.") + migrateDownCmd.Flags().StringVarP(&dsn, "database-path", "d", "./notary.db", "A DSN for connecting to the database. Also accepts a path to a file, and will assume that the database is SQLite.") + migrateStatusCmd.Flags().StringVarP(&dsn, "database-path", "d", "./notary.db", "A DSN for connecting to the database. Also accepts a path to a file, and will assume that the database is SQLite.") + + if err := migrateUpCmd.MarkFlagRequired("database-path"); err != nil { + log.Fatalf("Error marking database-path flag as required: %v", err) + } + if err := migrateDownCmd.MarkFlagRequired("database-path"); err != nil { + log.Fatalf("Error marking database-path flag as required: %v", err) + } + if err := migrateStatusCmd.MarkFlagRequired("database-path"); err != nil { + log.Fatalf("Error marking database-path flag as required: %v", err) + } + + migrateCmd.AddCommand(migrateUpCmd) + migrateCmd.AddCommand(migrateDownCmd) + migrateCmd.AddCommand(migrateStatusCmd) + + rootCmd.AddCommand(migrateCmd) + +} + +func prepareGoose(args []string) (int64, error) { + version := int64(0) + if len(args) == 1 { + versionArg, err := strconv.ParseInt(args[0], 10, 64) + if err != nil { + return 0, fmt.Errorf("version must be a number") + } + version = versionArg + } + goose.SetBaseFS(migrations.EmbedMigrations) + err := goose.SetDialect("sqlite") + if err != nil { + return 0, err + } + + return version, nil +} diff --git a/cmd/start.go b/cmd/start.go index a693ca2d..51da7fda 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -33,6 +33,7 @@ https://canonical-notary.readthedocs-hosted.com/en/latest/reference/config_file/ // Initialize the database connection db, err := db.NewDatabase(&db.DatabaseOpts{ DatabasePath: appContext.DBPath, + ApplyMigrations: appContext.ApplyMigrations, Backend: appContext.EncryptionBackend, Logger: appContext.Logger, }) @@ -78,7 +79,10 @@ https://canonical-notary.readthedocs-hosted.com/en/latest/reference/config_file/ func init() { rootCmd.AddCommand(startCmd) + startCmd.Flags().StringVarP(&configFilePath, "config", "c", "", "path to the configuration file") + startCmd.Flags().BoolP("migrate-database", "m", false, "automatically apply database migrations if needed (use with caution)") + err := startCmd.MarkFlagRequired("config") if err != nil { log.Fatalf("couldn't mark flag required: %s", err) diff --git a/go.mod b/go.mod index faef63f0..f91518b2 100644 --- a/go.mod +++ b/go.mod @@ -7,12 +7,13 @@ require ( github.com/google/go-cmp v0.7.0 github.com/hashicorp/vault-client-go v0.4.3 github.com/mattn/go-sqlite3 v1.14.28 + github.com/pressly/goose/v3 v3.25.0 github.com/prometheus/client_golang v1.22.0 github.com/spf13/cobra v1.9.1 github.com/spf13/pflag v1.0.7 github.com/spf13/viper v1.20.1 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.39.0 + golang.org/x/crypto v0.40.0 ) require ( @@ -25,18 +26,20 @@ require ( github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mfridman/interpolate v0.0.2 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sagikazarmark/locafero v0.7.0 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.12.0 // indirect github.com/spf13/cast v1.7.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/text v0.26.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.8.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) @@ -50,6 +53,6 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.64.0 // indirect github.com/prometheus/procfs v0.16.1 // indirect - golang.org/x/sys v0.33.0 // indirect + golang.org/x/sys v0.34.0 // indirect google.golang.org/protobuf v1.36.6 // indirect ) diff --git a/go.sum b/go.sum index ba850160..3dc7a2bf 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -19,6 +21,8 @@ github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeD github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= @@ -47,16 +51,22 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= +github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pressly/goose/v3 v3.25.0 h1:6WeYhMWGRCzpyd89SpODFnCBCKz41KrVbRT58nVjGng= +github.com/pressly/goose/v3 v3.25.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -65,13 +75,17 @@ github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQP github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= @@ -85,8 +99,8 @@ github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8= +github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -95,12 +109,16 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= @@ -110,3 +128,11 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ= +modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.38.2 h1:Aclu7+tgjgcQVShZqim41Bbw9Cho0y/7WzYptXqkEek= +modernc.org/sqlite v1.38.2/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= diff --git a/internal/config/config.go b/internal/config/config.go index e820e2ed..8cbac6e6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -50,6 +50,7 @@ func CreateAppContext(cmdFlags *pflag.FlagSet, configFilePath string) (*NotaryAp appContext.Port = cfg.GetInt("port") appContext.ExternalHostname = cfg.GetString("external_hostname") appContext.DBPath = cfg.GetString("db_path") + appContext.ApplyMigrations = cfg.GetBool("migrate-database") appContext.PebbleNotificationsEnabled = cfg.GetBool("pebble_notifications") appContext.TLSCertificate = cert diff --git a/internal/config/types.go b/internal/config/types.go index 63a02e31..e2718ae3 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -105,6 +105,8 @@ type NotaryAppContext struct { // Path to store the database DBPath string + // Whether to apply database migrations automatically on startup if the database is outdated + ApplyMigrations bool // Send pebble notifications if enabled. Read more at github.com/canonical/pebble PebbleNotificationsEnabled bool diff --git a/internal/db/db_init.go b/internal/db/db_init.go index f319208e..d1354688 100644 --- a/internal/db/db_init.go +++ b/internal/db/db_init.go @@ -8,8 +8,10 @@ import ( "fmt" "strings" + "github.com/canonical/notary/internal/db/migrations" "github.com/canonical/sqlair" _ "github.com/mattn/go-sqlite3" + "github.com/pressly/goose/v3" ) // Close closes the connection to the repository cleanly. @@ -36,26 +38,23 @@ func NewDatabase(dbOpts *DatabaseOpts) (*Database, error) { if _, err := sqlConnection.Exec("PRAGMA foreign_keys = ON;"); err != nil { return nil, err } - if _, err := sqlConnection.Exec(queryCreateCertificateRequestsTable); err != nil { - return nil, err - } - if _, err := sqlConnection.Exec(queryCreateCertificatesTable); err != nil { - return nil, err - } - if _, err := sqlConnection.Exec(queryCreateUsersTable); err != nil { - return nil, err - } - if _, err := sqlConnection.Exec(queryCreatePrivateKeysTable); err != nil { - return nil, err - } - if _, err := sqlConnection.Exec(queryCreateCertificateAuthoritiesTable); err != nil { + err = goose.SetDialect("sqlite") + if err != nil { return nil, err } - if _, err := sqlConnection.Exec(queryCreateEncryptionKeysTable); err != nil { + version, err := goose.EnsureDBVersion(sqlConnection) + if err != nil { return nil, err } - if _, err := sqlConnection.Exec(queryCreateJWTSecretTable); err != nil { - return nil, err + if version < 1 { + if dbOpts.ApplyMigrations { + goose.SetBaseFS(migrations.EmbedMigrations) + if err := goose.Up(sqlConnection, ".", goose.WithNoColor(true)); err != nil { + return nil, fmt.Errorf("failed to apply migrations: %w", err) + } + } else { + return nil, errors.New("database migrations not applied. please migrate database using `notary migrate up`") + } } db := new(Database) db.stmts = PrepareStatements() diff --git a/internal/db/db_init_test.go b/internal/db/db_init_test.go index fecaf60b..50b4a11f 100644 --- a/internal/db/db_init_test.go +++ b/internal/db/db_init_test.go @@ -1,18 +1,35 @@ package db_test import ( + "database/sql" "log" "path/filepath" "testing" "github.com/canonical/notary/internal/db" + "github.com/canonical/notary/internal/db/migrations" eb "github.com/canonical/notary/internal/encryption_backend" + "github.com/pressly/goose/v3" "go.uber.org/zap" ) func TestConnect(t *testing.T) { logger, _ := zap.NewDevelopment() tempDir := t.TempDir() + + sqlConnection, err := sql.Open("sqlite3", filepath.Join(tempDir, "db.sqlite3")) + if err != nil { + t.Fatalf("Couldn't create temporary database: %s", err) + } + goose.SetBaseFS(migrations.EmbedMigrations) + err = goose.SetDialect("sqlite") + if err != nil { + t.Fatalf("Couldn't set goose dialect: %s", err) + } + err = goose.Up(sqlConnection, ".", goose.WithNoColor(true)) + if err != nil { + t.Fatalf("Couldn't apply database migrations: %s", err) + } db, err := db.NewDatabase(&db.DatabaseOpts{ DatabasePath: filepath.Join(tempDir, "db.sqlite3"), Backend: &eb.NoEncryptionBackend{}, diff --git a/internal/db/migrations/00001_initial_schema.sql b/internal/db/migrations/00001_initial_schema.sql new file mode 100644 index 00000000..e920aaad --- /dev/null +++ b/internal/db/migrations/00001_initial_schema.sql @@ -0,0 +1,71 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS certificate_requests +( + csr_id INTEGER PRIMARY KEY AUTOINCREMENT, + csr TEXT NOT NULL UNIQUE, + status TEXT DEFAULT 'Outstanding', + certificate_id INTEGER, + user_id INTEGER, + + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL, + + CHECK (status IN ('Outstanding', 'Rejected', 'Revoked', 'Active')), + CHECK (NOT (certificate_id == NULL AND status == 'Active' )), + CHECK (NOT (certificate_id != NULL AND status == 'Outstanding')), + CHECK (NOT (certificate_id != NULL AND status == 'Rejected')), + CHECK (NOT (certificate_id != NULL AND status == 'Revoked')) +); +CREATE TABLE IF NOT EXISTS certificates +( + certificate_id INTEGER PRIMARY KEY AUTOINCREMENT, + certificate TEXT NOT NULL UNIQUE, + issuer_id INTEGER +); +CREATE TABLE IF NOT EXISTS certificate_authorities +( + certificate_authority_id INTEGER PRIMARY KEY AUTOINCREMENT, + crl TEXT, + enabled INTEGER DEFAULT 0, + private_key_id INTEGER, + certificate_id INTEGER, + csr_id INTEGER NOT NULL UNIQUE +); +CREATE TABLE IF NOT EXISTS private_keys +( + private_key_id INTEGER PRIMARY KEY AUTOINCREMENT, + private_key TEXT NOT NULL UNIQUE +); +CREATE TABLE IF NOT EXISTS users +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + email TEXT NOT NULL UNIQUE, + hashed_password TEXT NOT NULL, + role_id INTEGER NOT NULL, + + CHECK (trim(email) != ''), + CHECK (trim(hashed_password) != '') +); +CREATE TABLE IF NOT EXISTS encryption_keys +( + encryption_key_id INTEGER PRIMARY KEY AUTOINCREMENT, + encryption_key TEXT NOT NULL UNIQUE +); +CREATE TABLE IF NOT EXISTS jwt_secret +( + id INTEGER PRIMARY KEY CHECK (id = 1), + encrypted_secret TEXT NOT NULL +); +-- +goose StatementEnd + + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS "certificate_requests"; +DROP TABLE IF EXISTS "certificate_authorities"; +DROP TABLE IF EXISTS "certificates"; +DROP TABLE IF EXISTS "private_keys"; +DROP TABLE IF EXISTS "users"; +DROP TABLE IF EXISTS "encryption_keys"; +DROP TABLE IF EXISTS "jwt_secret"; +-- +goose StatementEnd diff --git a/internal/db/migrations/migration.go b/internal/db/migrations/migration.go new file mode 100644 index 00000000..27853891 --- /dev/null +++ b/internal/db/migrations/migration.go @@ -0,0 +1,6 @@ +package migrations + +import "embed" + +//go:embed *.sql +var EmbedMigrations embed.FS diff --git a/internal/db/sql_stmts.go b/internal/db/sql_stmts.go index 8ed3e54c..af480a43 100644 --- a/internal/db/sql_stmts.go +++ b/internal/db/sql_stmts.go @@ -4,72 +4,6 @@ import ( "github.com/canonical/sqlair" ) -const ( - // Table definition SQL Strings - queryCreateCertificateRequestsTable = ` - CREATE TABLE IF NOT EXISTS certificate_requests ( - csr_id INTEGER PRIMARY KEY AUTOINCREMENT, - - csr TEXT NOT NULL UNIQUE, - certificate_id INTEGER, - user_id INTEGER, - status TEXT DEFAULT 'Outstanding', - - FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL, - - CHECK (status IN ('Outstanding', 'Rejected', 'Revoked', 'Active')), - CHECK (NOT (certificate_id == NULL AND status == 'Active' )), - CHECK (NOT (certificate_id != NULL AND status == 'Outstanding')) - CHECK (NOT (certificate_id != NULL AND status == 'Rejected')) - CHECK (NOT (certificate_id != NULL AND status == 'Revoked')) - )` - queryCreateCertificatesTable = ` - CREATE TABLE IF NOT EXISTS certificates ( - certificate_id INTEGER PRIMARY KEY AUTOINCREMENT, - issuer_id INTEGER, - - certificate TEXT NOT NULL UNIQUE - )` - queryCreateCertificateAuthoritiesTable = ` - CREATE TABLE IF NOT EXISTS certificate_authorities ( - certificate_authority_id INTEGER PRIMARY KEY AUTOINCREMENT, - - crl TEXT, - enabled INTEGER DEFAULT 0, - - private_key_id INTEGER, - certificate_id INTEGER, - csr_id INTEGER NOT NULL UNIQUE - )` - queryCreatePrivateKeysTable = ` - CREATE TABLE IF NOT EXISTS private_keys ( - private_key_id INTEGER PRIMARY KEY AUTOINCREMENT, - - private_key TEXT NOT NULL UNIQUE - )` - queryCreateUsersTable = ` - CREATE TABLE IF NOT EXISTS users ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - - email TEXT NOT NULL UNIQUE, - hashed_password TEXT NOT NULL, - role_id INTEGER NOT NULL, - - CHECK (trim(email) != ''), - CHECK (trim(hashed_password) != '') - )` - queryCreateEncryptionKeysTable = ` - CREATE TABLE IF NOT EXISTS encryption_keys ( - encryption_key_id INTEGER PRIMARY KEY AUTOINCREMENT, - encryption_key TEXT NOT NULL UNIQUE - )` - queryCreateJWTSecretTable = ` - CREATE TABLE IF NOT EXISTS jwt_secret ( - id INTEGER PRIMARY KEY CHECK (id = 1), -- Ensures only one row - encrypted_secret TEXT NOT NULL - )` -) - const ( // // // // // // // // // // // // // // Certificate Request SQL Strings // diff --git a/internal/db/types.go b/internal/db/types.go index e195d065..2b8339fb 100644 --- a/internal/db/types.go +++ b/internal/db/types.go @@ -8,6 +8,7 @@ import ( type DatabaseOpts struct { DatabasePath string + ApplyMigrations bool Backend encryption_backend.EncryptionBackend Logger *zap.Logger } diff --git a/internal/testutils/db_test_utils.go b/internal/testutils/db_test_utils.go index 58914cdf..0ac44a71 100644 --- a/internal/testutils/db_test_utils.go +++ b/internal/testutils/db_test_utils.go @@ -1,19 +1,35 @@ package testutils import ( + "database/sql" "path/filepath" "testing" "github.com/canonical/notary/internal/config" "github.com/canonical/notary/internal/db" + "github.com/canonical/notary/internal/db/migrations" "github.com/canonical/notary/internal/encryption_backend" + "github.com/pressly/goose/v3" "go.uber.org/zap" ) func MustPrepareEmptyDB(t *testing.T) *db.Database { t.Helper() - tempDir := t.TempDir() + tempDir := t.TempDir() + sqlConnection, err := sql.Open("sqlite3", filepath.Join(tempDir, "db.sqlite3")) + if err != nil { + t.Fatalf("Couldn't create temporary database: %s", err) + } + goose.SetBaseFS(migrations.EmbedMigrations) + err = goose.SetDialect("sqlite") + if err != nil { + t.Fatalf("Couldn't set goose dialect: %s", err) + } + err = goose.Up(sqlConnection, ".", goose.WithNoColor(true)) + if err != nil { + t.Fatalf("Couldn't apply database migrations: %s", err) + } database, err := db.NewDatabase(&db.DatabaseOpts{ DatabasePath: filepath.Join(tempDir, "db.sqlite3"), Backend: NoneEncryptionBackend, From f06930c84ce65923679e7bc70a818f9d58fdf649 Mon Sep 17 00:00:00 2001 From: Kayra Date: Tue, 9 Sep 2025 12:25:34 +0300 Subject: [PATCH 2/2] chore(workflow): update workflow to migrate the database --- .github/workflows/rock-test.yaml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/rock-test.yaml b/.github/workflows/rock-test.yaml index d23944a9..6ba276f0 100644 --- a/.github/workflows/rock-test.yaml +++ b/.github/workflows/rock-test.yaml @@ -29,21 +29,16 @@ jobs: run: | printf 'key_path: "/etc/notary/config/key.pem"\ncert_path: "/etc/notary/config/cert.pem"\ndb_path: "/var/lib/notary/database/notary.db"\nport: 3000\npebble_notifications: true\nlogging:\n system:\n level: "debug"\n output: "stdout"\nencryption_backend:\n type: "none"' > config.yaml openssl req -newkey rsa:2048 -nodes -keyout key.pem -x509 -days 1 -out cert.pem -subj "/CN=githubaction.example" + + mkdir config data + mv key.pem cert.pem config.yaml config/ - name: Import the image to Docker registry run: | ROCK_FILE_NAME=$(ls ./rock-path) sudo rockcraft.skopeo --insecure-policy copy oci-archive:rock-path/$ROCK_FILE_NAME docker-daemon:notary:latest - name: Run the image run: | - docker run -d -p 3000:3000 --name notary --entrypoint "notary start -m --config /etc/notary/config/config.yaml" notary:latest - - name: Load config - run: | - docker exec notary /usr/bin/pebble mkdir -p /etc/notary/config - docker exec notary /usr/bin/pebble mkdir -p /var/lib/notary/database - docker cp key.pem notary:/etc/notary/config/key.pem - docker cp cert.pem notary:/etc/notary/config/cert.pem - docker cp config.yaml notary:/etc/notary/config/config.yaml - docker restart notary + docker run -d -p 3000:3000 -v ./config:/etc/notary/config -v ./data:/var/lib/notary/database --name notary notary:latest --args notary -m --config /etc/notary/config/config.yaml - name: Check if Notary frontend is loaded run: | sleep 30 @@ -74,4 +69,3 @@ jobs: docker exec notary /usr/bin/pebble notices docker exec notary /usr/bin/pebble notices | grep canonical\\.com - docker exec notary /usr/bin/pebble notice 3