Skip to content

fix: prevent fallback to legacy store #954

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 86 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
86 commits
Select commit Hold shift + click to select a range
3fcb23f
feat: handle missing compatibility layer with v2.1
gfyrag Jan 2, 2025
1f4043b
fix: release workflow
gfyrag Jan 2, 2025
fe38474
fix: wrong encoding on a migration
gfyrag Jan 15, 2025
b0c5ea5
fix: transactions commit with special chars in metadata
gfyrag Jan 20, 2025
95c8317
fix: GET /_info breaking change (#666)
gfyrag Jan 27, 2025
e05ab37
chore: add openapi tag to satisfy global sdk (#670)
gfyrag Feb 7, 2025
a10356b
fix: migration using azure postgres with no superuser provileges
gfyrag Feb 12, 2025
8913cdb
fix: next migration
gfyrag Feb 12, 2025
aa20c11
fix: next migration (#689)
gfyrag Feb 17, 2025
19e664a
fix: incompatibility between v1 and v2 api format when using numbers …
gfyrag Feb 20, 2025
03de826
fix: potential inconsistent ledger creation
gfyrag Feb 20, 2025
3d27fa0
fix: missing filter on list logs (#708)
gfyrag Feb 25, 2025
56b70f9
fix: make ledger creations atomic
gfyrag Feb 25, 2025
0dbe402
fix(ledger) tx reversion handles empty middle accounts (#711)
Feb 26, 2025
a3b891c
fix: transaction list missing reference filter
gfyrag Feb 27, 2025
92da437
fix: failing queries (GET /accounts?expand=volumes) when migrations a…
gfyrag Mar 4, 2025
0f358f0
fix: migrations with pg 13
gfyrag Mar 5, 2025
19e894b
fix: regression regarding v2.1
gfyrag Mar 5, 2025
63541a0
fix: metadata override
gfyrag Mar 5, 2025
a01d9a7
feat: don't check all buckets on all health checks
gfyrag Mar 10, 2025
15920d2
chore: dirty
gfyrag Mar 10, 2025
aa6f6a9
fix: parallelize bucket migration rollback detection (#755)
flemzord Mar 10, 2025
96738a8
fix: Revert "feat: don't check all buckets on all health checks"
gfyrag Mar 11, 2025
1e809a0
fix: version api field
gfyrag Mar 7, 2025
f668845
feat: prevent potential deadlocks
gfyrag Mar 14, 2025
39bd88a
feat: upgrade skeapeasy and use streaming file system for logs import…
gfyrag Mar 13, 2025
2939c05
fix: import regression
gfyrag Mar 14, 2025
0d0d266
feat: hide textual errors on internal server errors
gfyrag Mar 17, 2025
535aac9
feat: add version as otlp resource attributes
gfyrag Mar 18, 2025
640f89a
fix: invalid unicode conversion while computing hash
gfyrag Mar 19, 2025
e1d2c19
fix: local test server in benchmarks
gfyrag Mar 19, 2025
d8358d1
feat: remove one UPDATE sql query in case of metadata usage
gfyrag Mar 20, 2025
a40fe4f
feat: improve pulumi
gfyrag Mar 21, 2025
606077a
feat: add multi ledger generator on pulumi program
gfyrag Mar 21, 2025
76fd5e9
fix: remove not used pulumi properties
gfyrag Mar 21, 2025
8ae43ee
fix: missing env var
gfyrag Mar 24, 2025
05e20ef
feat: add capability to retain rds component on deletion
gfyrag Mar 24, 2025
df6d0d7
fix: debug flag on migrate command
gfyrag Mar 24, 2025
7b43a96
feat: make tracing of logs import more granular
gfyrag Mar 24, 2025
1bce077
feat: add additional annotations on storage service
gfyrag Mar 25, 2025
e0884d5
fix: invalid post commit volumes format
gfyrag Mar 27, 2025
72a9137
fix: use full qualified name when using function aggregate_objects
gfyrag Mar 28, 2025
be86b48
fix: invalid pcv
gfyrag Mar 28, 2025
dcf8bdc
chore: upgrade go libs
gfyrag Mar 28, 2025
924ad40
fix: accounts metadata not overriden
gfyrag Apr 1, 2025
3961a31
feat: add otel env vars to migration job
gfyrag Apr 1, 2025
1c969c8
feat: add traces all along the migration path
gfyrag Apr 1, 2025
98a989f
fix(openapi): bulk response definition
gfyrag Apr 8, 2025
6231b80
fix: inconsistent accounts list
gfyrag Apr 8, 2025
3128034
fix: invalid migration
gfyrag Apr 9, 2025
63c3090
fix: missing metadata on reverts
gfyrag Apr 9, 2025
e4edd2d
fix: incorrect metadata on reversing transactions
gfyrag Apr 9, 2025
36eb52d
fix: backport get balances from main
gfyrag Apr 10, 2025
026f1ee
chore: simplify
gfyrag Apr 10, 2025
bb02940
feat: simplify
gfyrag Apr 10, 2025
c39e4d4
fix: invalid revert events (#868)
gfyrag Apr 11, 2025
49e7da6
fix: potential invalid balance during upgrade
gfyrag Apr 11, 2025
55f5c7c
fix: Migration 29
gfyrag Apr 14, 2025
08fe709
fix: updated_at not set
gfyrag Apr 18, 2025
fe2e3a5
fix: missing denormalized data in logs
gfyrag Apr 24, 2025
e54d9de
fix: incorrect join with specific combination of filters
gfyrag Apr 24, 2025
eb07b24
fix: potential non atomic ledger creation
gfyrag Apr 25, 2025
c0c32f6
fix: invalid date format
gfyrag Apr 25, 2025
3add656
fix: too big trace on imports
gfyrag Apr 28, 2025
cebc566
feat: remove sql transaction on imports
gfyrag Apr 29, 2025
5a9711b
test: add import test with error
gfyrag Apr 29, 2025
5802ae9
fix: from coredabbit
gfyrag Apr 29, 2025
c119df8
fix: missing decorator around the ledger controller
gfyrag Apr 29, 2025
0752f12
fix: useless comment
gfyrag Apr 29, 2025
fa2ef47
feat: add some tracing
gfyrag Apr 30, 2025
2b77643
feat: backport json streaming from 2.3
gfyrag Apr 29, 2025
7be55a8
fix: missing default options when connecting to default postgres
gfyrag May 6, 2025
5eac9b4
fix: missing yaml tag
gfyrag May 7, 2025
65e221a
fix: reverse query param on transactions list
gfyrag May 13, 2025
15b67f1
fix: pageSize not effective when a cursor is used
gfyrag May 13, 2025
d5d0160
fix: build with go 1.24.3
gfyrag May 13, 2025
55d59af
fix: return 400 instead of 500 for idempotency key reuse with differe…
flemzord May 16, 2025
1828952
fix: add sql transaction by log when importing
gfyrag May 19, 2025
fa461c4
fix: memento format
gfyrag May 19, 2025
da1c21f
feat: extract compute_hash function
gfyrag May 19, 2025
7b80907
feat(pulumi): expose rds cluster id
gfyrag May 19, 2025
d432240
fix: image-pull-policy ignored
gfyrag May 22, 2025
03723c8
fix: configure sequences after import
gfyrag May 20, 2025
8d9affd
fix: pagination with custom column (transaction list)
gfyrag May 21, 2025
285f189
fix: invalid state assigned
gfyrag May 22, 2025
20e5e6b
fix: prevent fallback to legacy store when migrating
gfyrag Jun 2, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
247 changes: 247 additions & 0 deletions .coderabbit.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,247 @@
language: en-US
tone_instructions: ''
early_access: true
enable_free_tier: true
reviews:
profile: chill
request_changes_workflow: false
high_level_summary: false
high_level_summary_placeholder: '@coderabbitai summary'
high_level_summary_in_walkthrough: false
auto_title_placeholder: '@coderabbitai'
auto_title_instructions: ''
review_status: true
commit_status: true
fail_commit_status: false
collapse_walkthrough: false
changed_files_summary: true
sequence_diagrams: true
assess_linked_issues: true
related_issues: true
related_prs: true
suggested_labels: true
auto_apply_labels: false
suggested_reviewers: true
auto_assign_reviewers: false
poem: true
labeling_instructions: []
path_filters:
- '!dist/**'
- '!**/*.app'
- '!**/*.bin'
- '!**/*.bz2'
- '!**/*.class'
- '!**/*.db'
- '!**/*.csv'
- '!**/*.tsv'
- '!**/*.dat'
- '!**/*.dll'
- '!**/*.dylib'
- '!**/*.egg'
- '!**/*.glif'
- '!**/*.gz'
- '!**/*.xz'
- '!**/*.zip'
- '!**/*.7z'
- '!**/*.rar'
- '!**/*.zst'
- '!**/*.ico'
- '!**/*.jar'
- '!**/*.tar'
- '!**/*.war'
- '!**/*.lo'
- '!**/*.log'
- '!**/*.mp3'
- '!**/*.wav'
- '!**/*.wma'
- '!**/*.mp4'
- '!**/*.avi'
- '!**/*.mkv'
- '!**/*.wmv'
- '!**/*.m4a'
- '!**/*.m4v'
- '!**/*.3gp'
- '!**/*.3g2'
- '!**/*.rm'
- '!**/*.mov'
- '!**/*.flv'
- '!**/*.iso'
- '!**/*.swf'
- '!**/*.flac'
- '!**/*.nar'
- '!**/*.o'
- '!**/*.ogg'
- '!**/*.otf'
- '!**/*.p'
- '!**/*.pdf'
- '!**/*.doc'
- '!**/*.docx'
- '!**/*.xls'
- '!**/*.xlsx'
- '!**/*.map'
- '!**/*.out'
- '!**/*.ppt'
- '!**/*.pptx'
- '!**/*.pkl'
- '!**/*.pickle'
- '!**/*.pyc'
- '!**/*.pyd'
- '!**/*.pyo'
- '!**/*.pub'
- '!**/*.pem'
- '!**/*.rkt'
- '!**/*.so'
- '!**/*.ss'
- '!**/*.eot'
- '!**/*.exe'
- '!**/*.pb.go'
- '!**/*.pb.gw.go'
- '!**/*.lock'
- '!**/*.ttf'
- '!**/*.yaml'
- '!**/*.yml'
- '!**/*.xml'
- '!**/*.cfg'
- '!**/*.toml'
- '!**/*.ini'
- '!**/*.mod'
- '!**/*.sum'
- '!**/*.work'
- '!**/*.json'
- '!**/*.mmd'
- '!**/*.svg'
- '!**/*.jpeg'
- '!**/*.jpg'
- '!**/*.png'
- '!**/*.gif'
- '!**/*.bmp'
- '!**/*.tiff'
- '!**/*.webm'
- '!**/*.woff'
- '!**/*.woff2'
- '!**/*.dot'
- '!**/*.md5sum'
- '!**/*.wasm'
- '!**/*.snap'
- '!**/*.parquet'
- '!**/gen/**'
- '!**/_gen/**'
- '!**/generated/**'
- '!**/@generated/**'
- '!**/vendor/**'
- '!**/*.min.js'
- '!**/*.min.js.map'
- '!**/*.min.js.css'
- '!**/*.tfstate'
- '!**/*.tfstate.backup'
path_instructions: []
abort_on_close: true
disable_cache: false
auto_review:
enabled: true
auto_incremental_review: true
ignore_title_keywords: []
labels: []
drafts: false
base_branches:
- main
- release/.*
finishing_touches:
docstrings:
enabled: true
tools:
ast-grep:
rule_dirs: []
util_dirs: []
essential_rules: true
packages: []
shellcheck:
enabled: true
ruff:
enabled: true
markdownlint:
enabled: true
github-checks:
enabled: true
timeout_ms: 90000
languagetool:
enabled: true
enabled_rules: []
disabled_rules: []
enabled_categories: []
disabled_categories: []
enabled_only: false
level: default
biome:
enabled: true
hadolint:
enabled: true
swiftlint:
enabled: true
phpstan:
enabled: true
level: default
golangci-lint:
enabled: true
yamllint:
enabled: true
gitleaks:
enabled: true
checkov:
enabled: true
detekt:
enabled: true
eslint:
enabled: true
rubocop:
enabled: true
buf:
enabled: true
regal:
enabled: true
actionlint:
enabled: true
pmd:
enabled: true
cppcheck:
enabled: true
semgrep:
enabled: true
circleci:
enabled: true
sqlfluff:
enabled: true
prismaLint:
enabled: true
oxc:
enabled: true
shopifyThemeCheck:
enabled: true
chat:
auto_reply: true
create_issues: true
integrations:
jira:
usage: auto
linear:
usage: auto
knowledge_base:
opt_out: false
web_search:
enabled: true
learnings:
scope: auto
issues:
scope: auto
jira:
usage: auto
project_keys: []
linear:
usage: auto
team_keys: []
pull_requests:
scope: auto
code_generation:
docstrings:
language: en-US
path_instructions: []
2 changes: 1 addition & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ on:
push:
branches:
- main
- releases/*
- release/*
pull_request:
types: [ assigned, opened, synchronize, reopened, labeled ]

Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ dist
vendor
worktrees
dumps
Pulumi.*.yaml
11 changes: 8 additions & 3 deletions Earthfile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
VERSION --wildcard-builds 0.8
PROJECT FormanceHQ/ledger

IMPORT github.com/formancehq/earthly:tags/v0.19.0 AS core
IMPORT github.com/formancehq/earthly:tags/v0.19.1 AS core

FROM core+base-image

Expand Down Expand Up @@ -181,9 +181,14 @@ release:

generate-client:
FROM node:20-alpine
RUN apk update && apk add yq jq
RUN apk update && apk add yq jq curl
WORKDIR /src
COPY (core+sources-speakeasy/speakeasy) /bin/speakeasy
ARG TARGETARCH
ARG SPEAKEASY_VERSION=v1.517.3
RUN curl -fsSL https://github.com/speakeasy-api/speakeasy/releases/download/${SPEAKEASY_VERSION}/speakeasy_linux_$TARGETARCH.zip -o /tmp/speakeasy_linux_$TARGETARCH.zip
RUN unzip /tmp/speakeasy_linux_$TARGETARCH.zip speakeasy
RUN chmod +x speakeasy
RUN mv speakeasy /bin/speakeasy
COPY (+openapi/openapi.yaml) openapi.yaml
RUN cat ./openapi.yaml | yq e -o json > openapi.json
COPY (core+sources/out --LOCATION=openapi-overlay.json) openapi-overlay.json
Expand Down
63 changes: 30 additions & 33 deletions cmd/buckets_upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@ package cmd
import (
"github.com/formancehq/go-libs/v2/bun/bunconnect"
"github.com/formancehq/go-libs/v2/logging"
"github.com/formancehq/go-libs/v2/otlp"
"github.com/formancehq/go-libs/v2/otlp/otlptraces"
"github.com/formancehq/go-libs/v2/service"
"github.com/formancehq/ledger/internal/storage/bucket"
"github.com/formancehq/ledger/internal/storage"
"github.com/formancehq/ledger/internal/storage/driver"
"github.com/formancehq/ledger/internal/storage/ledger"
systemstore "github.com/formancehq/ledger/internal/storage/system"
"github.com/spf13/cobra"
"github.com/uptrace/bun"
"go.uber.org/fx"
)

func NewBucketUpgrade() *cobra.Command {
Expand All @@ -18,22 +18,13 @@ func NewBucketUpgrade() *cobra.Command {
Args: cobra.ExactArgs(1),
SilenceUsage: true,
RunE: func(cmd *cobra.Command, args []string) error {
logger := logging.NewDefaultLogger(cmd.OutOrStdout(), service.IsDebug(cmd), false, false)
cmd.SetContext(logging.ContextWithLogger(cmd.Context(), logger))
return withStorageDriver(cmd, func(driver *driver.Driver) error {
if args[0] == "*" {
return driver.UpgradeAllBuckets(cmd.Context())
}

driver, db, err := getDriver(cmd)
if err != nil {
return err
}
defer func() {
_ = db.Close()
}()

if args[0] == "*" {
return driver.UpgradeAllBuckets(cmd.Context())
}

return driver.UpgradeBucket(cmd.Context(), args[0])
return driver.UpgradeBucket(cmd.Context(), args[0])
})
},
}

Expand All @@ -43,26 +34,32 @@ func NewBucketUpgrade() *cobra.Command {
return cmd
}

func getDriver(cmd *cobra.Command) (*driver.Driver, *bun.DB, error) {
func withStorageDriver(cmd *cobra.Command, fn func(driver *driver.Driver) error) error {

connectionOptions, err := bunconnect.ConnectionOptionsFromFlags(cmd)
if err != nil {
return nil, nil, err
}
logger := logging.NewDefaultLogger(cmd.OutOrStdout(), service.IsDebug(cmd), false, false)

db, err := bunconnect.OpenSQLDB(cmd.Context(), *connectionOptions)
connectionOptions, err := bunconnect.ConnectionOptionsFromFlags(cmd)
if err != nil {
return nil, nil, err
return err
}

driver := driver.New(
ledger.NewFactory(db),
systemstore.New(db),
bucket.NewDefaultFactory(db),
var d *driver.Driver
app := fx.New(
fx.NopLogger,
otlp.FXModuleFromFlags(cmd, otlp.WithServiceVersion(Version)),
otlptraces.FXModuleFromFlags(cmd),
bunconnect.Module(*connectionOptions, service.IsDebug(cmd)),
storage.NewFXModule(false),
fx.Supply(fx.Annotate(logger, fx.As(new(logging.Logger)))),
fx.Populate(&d),
)
if err := driver.Initialize(cmd.Context()); err != nil {
return nil, nil, err
err = app.Start(cmd.Context())
if err != nil {
return err
}
defer func() {
_ = app.Stop(cmd.Context())
}()

return driver, db, nil
return fn(d)
}
Loading
Loading