From 682ed730d84b96a0c7873b948589bb1d01d10b66 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Tue, 17 Jun 2025 12:31:41 +0200 Subject: [PATCH 1/4] chore: move code --- internal/storage/common/resource.go | 229 +-------------------------- internal/storage/common/schema.go | 233 ++++++++++++++++++++++++++++ 2 files changed, 234 insertions(+), 228 deletions(-) create mode 100644 internal/storage/common/schema.go diff --git a/internal/storage/common/resource.go b/internal/storage/common/resource.go index 404a83c66..50d4b014f 100644 --- a/internal/storage/common/resource.go +++ b/internal/storage/common/resource.go @@ -405,231 +405,4 @@ type ( type PaginatedResource[ResourceType, OptionsType any, PaginationQueryType PaginatedQuery[OptionsType]] interface { Resource[ResourceType, OptionsType] Paginate(ctx context.Context, paginationOptions PaginationQueryType) (*bunpaginate.Cursor[ResourceType], error) -} - -const ( - OperatorMatch = "$match" - OperatorExists = "$exists" - OperatorLike = "$like" - OperatorLT = "$lt" - OperatorGT = "$gt" - OperatorLTE = "$lte" - OperatorGTE = "$gte" -) - -type FieldType interface { - Operators() []string - ValidateValue(value any) error - IsIndexable() bool -} - -type Field struct { - Aliases []string - Type FieldType -} - -func (f Field) WithAliases(aliases ...string) Field { - f.Aliases = append(f.Aliases, aliases...) - return f -} - -func (f Field) matchKey(name, key string) bool { - if key == name { - return true - } - for _, alias := range f.Aliases { - if key == alias { - return true - } - } - - return false -} - -func NewField(t FieldType) Field { - return Field{ - Aliases: []string{}, - Type: t, - } -} - -// NewStringField creates a new field with TypeString as its type. -func NewStringField() Field { - return NewField(NewTypeString()) -} - -// NewDateField creates a new field with TypeDate as its type. -func NewDateField() Field { - return NewField(NewTypeDate()) -} - -// NewMapField creates a new field with TypeMap as its type, using the provided underlying type. -func NewMapField(underlyingType FieldType) Field { - return NewField(NewTypeMap(underlyingType)) -} - -// NewNumericField creates a new field with TypeNumeric as its type. -func NewNumericField() Field { - return NewField(NewTypeNumeric()) -} - -// NewBooleanField creates a new field with TypeBoolean as its type. -func NewBooleanField() Field { - return NewField(NewTypeBoolean()) -} - -// NewStringMapField creates a new field with TypeMap as its type, using TypeString as the underlying type. -func NewStringMapField() Field { - return NewMapField(NewTypeString()) -} - -// NewNumericMapField creates a new field with TypeMap as its type, using TypeNumeric as the underlying type. -func NewNumericMapField() Field { - return NewMapField(NewTypeNumeric()) -} - -type TypeString struct{} - -func (t TypeString) IsIndexable() bool { - return false -} - -func (t TypeString) Operators() []string { - return []string{ - OperatorMatch, - OperatorLike, - } -} - -func (t TypeString) ValidateValue(value any) error { - _, ok := value.(string) - if !ok { - return fmt.Errorf("expected string value, got %T", value) - } - return nil -} - -var _ FieldType = (*TypeString)(nil) - -func NewTypeString() TypeString { - return TypeString{} -} - -type TypeDate struct{} - -func (t TypeDate) IsIndexable() bool { - return false -} - -func (t TypeDate) Operators() []string { - return []string{ - OperatorMatch, - OperatorLT, - OperatorGT, - OperatorLTE, - OperatorGTE, - } -} - -func (t TypeDate) ValidateValue(value any) error { - switch value := value.(type) { - case string: - _, err := time.ParseTime(value) - if err != nil { - return fmt.Errorf("invalid date value: %w", err) - } - case time.Time, *time.Time: - default: - return fmt.Errorf("expected string, time.Time, or *time.Time value, got %T", value) - } - return nil -} - -func NewTypeDate() TypeDate { - return TypeDate{} -} - -var _ FieldType = (*TypeDate)(nil) - -type TypeMap struct { - underlyingType FieldType -} - -func (t TypeMap) IsIndexable() bool { - return true -} - -func (t TypeMap) Operators() []string { - return append(t.underlyingType.Operators(), OperatorMatch, OperatorExists) -} - -func (t TypeMap) ValidateValue(value any) error { - return t.underlyingType.ValidateValue(value) -} - -func NewTypeMap(underlyingType FieldType) TypeMap { - return TypeMap{ - underlyingType: underlyingType, - } -} - -var _ FieldType = (*TypeMap)(nil) - -type TypeNumeric struct{} - -func (t TypeNumeric) IsIndexable() bool { - return false -} - -func (t TypeNumeric) Operators() []string { - return []string{ - OperatorMatch, - OperatorLT, - OperatorGT, - OperatorLTE, - OperatorGTE, - } -} - -func (t TypeNumeric) ValidateValue(value any) error { - switch value.(type) { - case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float64, float32, - *int, *int8, *int16, *int32, *int64, *uint, *uint8, *uint16, *uint32, *uint64, *float64, *float32: - return nil - default: - return fmt.Errorf("expected numeric value, got %T", value) - } -} - -func NewTypeNumeric() TypeNumeric { - return TypeNumeric{} -} - -var _ FieldType = (*TypeNumeric)(nil) - -type TypeBoolean struct{} - -func (t TypeBoolean) IsIndexable() bool { - return false -} - -func (t TypeBoolean) Operators() []string { - return []string{ - OperatorMatch, - } -} - -func (t TypeBoolean) ValidateValue(value any) error { - _, ok := value.(bool) - if !ok { - return fmt.Errorf("expected boolean value, got %T", value) - } - - return nil -} - -func NewTypeBoolean() TypeBoolean { - return TypeBoolean{} -} - -var _ FieldType = (*TypeBoolean)(nil) +} \ No newline at end of file diff --git a/internal/storage/common/schema.go b/internal/storage/common/schema.go new file mode 100644 index 000000000..563fd4308 --- /dev/null +++ b/internal/storage/common/schema.go @@ -0,0 +1,233 @@ +package common + +import ( + "fmt" + "github.com/formancehq/go-libs/v3/time" +) + +const ( + OperatorMatch = "$match" + OperatorExists = "$exists" + OperatorLike = "$like" + OperatorLT = "$lt" + OperatorGT = "$gt" + OperatorLTE = "$lte" + OperatorGTE = "$gte" +) + +type FieldType interface { + Operators() []string + ValidateValue(value any) error + IsIndexable() bool +} + +type Field struct { + Aliases []string + Type FieldType +} + +func (f Field) WithAliases(aliases ...string) Field { + f.Aliases = append(f.Aliases, aliases...) + return f +} + +func (f Field) matchKey(name, key string) bool { + if key == name { + return true + } + for _, alias := range f.Aliases { + if key == alias { + return true + } + } + + return false +} + +func NewField(t FieldType) Field { + return Field{ + Aliases: []string{}, + Type: t, + } +} + +// NewStringField creates a new field with TypeString as its type. +func NewStringField() Field { + return NewField(NewTypeString()) +} + +// NewDateField creates a new field with TypeDate as its type. +func NewDateField() Field { + return NewField(NewTypeDate()) +} + +// NewMapField creates a new field with TypeMap as its type, using the provided underlying type. +func NewMapField(underlyingType FieldType) Field { + return NewField(NewTypeMap(underlyingType)) +} + +// NewNumericField creates a new field with TypeNumeric as its type. +func NewNumericField() Field { + return NewField(NewTypeNumeric()) +} + +// NewBooleanField creates a new field with TypeBoolean as its type. +func NewBooleanField() Field { + return NewField(NewTypeBoolean()) +} + +// NewStringMapField creates a new field with TypeMap as its type, using TypeString as the underlying type. +func NewStringMapField() Field { + return NewMapField(NewTypeString()) +} + +// NewNumericMapField creates a new field with TypeMap as its type, using TypeNumeric as the underlying type. +func NewNumericMapField() Field { + return NewMapField(NewTypeNumeric()) +} + +type TypeString struct{} + +func (t TypeString) IsIndexable() bool { + return false +} + +func (t TypeString) Operators() []string { + return []string{ + OperatorMatch, + OperatorLike, + } +} + +func (t TypeString) ValidateValue(value any) error { + _, ok := value.(string) + if !ok { + return fmt.Errorf("expected string value, got %T", value) + } + return nil +} + +var _ FieldType = (*TypeString)(nil) + +func NewTypeString() TypeString { + return TypeString{} +} + +type TypeDate struct{} + +func (t TypeDate) IsIndexable() bool { + return false +} + +func (t TypeDate) Operators() []string { + return []string{ + OperatorMatch, + OperatorLT, + OperatorGT, + OperatorLTE, + OperatorGTE, + } +} + +func (t TypeDate) ValidateValue(value any) error { + switch value := value.(type) { + case string: + _, err := time.ParseTime(value) + if err != nil { + return fmt.Errorf("invalid date value: %w", err) + } + case time.Time, *time.Time: + default: + return fmt.Errorf("expected string, time.Time, or *time.Time value, got %T", value) + } + return nil +} + +func NewTypeDate() TypeDate { + return TypeDate{} +} + +var _ FieldType = (*TypeDate)(nil) + +type TypeMap struct { + underlyingType FieldType +} + +func (t TypeMap) IsIndexable() bool { + return true +} + +func (t TypeMap) Operators() []string { + return append(t.underlyingType.Operators(), OperatorMatch, OperatorExists) +} + +func (t TypeMap) ValidateValue(value any) error { + return t.underlyingType.ValidateValue(value) +} + +func NewTypeMap(underlyingType FieldType) TypeMap { + return TypeMap{ + underlyingType: underlyingType, + } +} + +var _ FieldType = (*TypeMap)(nil) + +type TypeNumeric struct{} + +func (t TypeNumeric) IsIndexable() bool { + return false +} + +func (t TypeNumeric) Operators() []string { + return []string{ + OperatorMatch, + OperatorLT, + OperatorGT, + OperatorLTE, + OperatorGTE, + } +} + +func (t TypeNumeric) ValidateValue(value any) error { + switch value.(type) { + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float64, float32, + *int, *int8, *int16, *int32, *int64, *uint, *uint8, *uint16, *uint32, *uint64, *float64, *float32: + return nil + default: + return fmt.Errorf("expected numeric value, got %T", value) + } +} + +func NewTypeNumeric() TypeNumeric { + return TypeNumeric{} +} + +var _ FieldType = (*TypeNumeric)(nil) + +type TypeBoolean struct{} + +func (t TypeBoolean) IsIndexable() bool { + return false +} + +func (t TypeBoolean) Operators() []string { + return []string{ + OperatorMatch, + } +} + +func (t TypeBoolean) ValidateValue(value any) error { + _, ok := value.(bool) + if !ok { + return fmt.Errorf("expected boolean value, got %T", value) + } + + return nil +} + +func NewTypeBoolean() TypeBoolean { + return TypeBoolean{} +} + +var _ FieldType = (*TypeBoolean)(nil) From 16461e2c832d87e9bf81518a0e6eab3dc871fcfb Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Thu, 19 Jun 2025 18:30:22 +0200 Subject: [PATCH 2/4] refacto: pagination system --- .../bulking/mocks_ledger_controller_test.go | 8 +- .../common/mocks_ledger_controller_test.go | 8 +- .../common/mocks_system_controller_test.go | 2 +- internal/api/v1/controllers_accounts_list.go | 10 +- .../api/v1/controllers_accounts_list_test.go | 39 +++-- internal/api/v1/controllers_balances_list.go | 11 +- internal/api/v1/controllers_config.go | 7 +- internal/api/v1/controllers_logs_list.go | 10 +- internal/api/v1/controllers_logs_list_test.go | 20 ++- .../api/v1/controllers_transactions_list.go | 10 +- .../v1/controllers_transactions_list_test.go | 36 +++-- .../api/v1/mocks_ledger_controller_test.go | 8 +- .../api/v1/mocks_system_controller_test.go | 2 +- internal/api/v1/utils.go | 77 +++++----- internal/api/v2/common.go | 104 ++++++-------- internal/api/v2/controllers_accounts_list.go | 4 +- .../api/v2/controllers_accounts_list_test.go | 55 +++++-- internal/api/v2/controllers_ledgers_list.go | 4 +- .../api/v2/controllers_ledgers_list_test.go | 7 +- internal/api/v2/controllers_logs_list.go | 4 +- internal/api/v2/controllers_logs_list_test.go | 30 ++-- .../api/v2/controllers_transactions_list.go | 4 +- .../v2/controllers_transactions_list_test.go | 58 ++++---- internal/api/v2/controllers_volumes.go | 62 +++++--- internal/api/v2/controllers_volumes_test.go | 31 ++-- .../api/v2/mocks_ledger_controller_test.go | 8 +- .../api/v2/mocks_system_controller_test.go | 2 +- internal/controller/ledger/controller.go | 8 +- .../controller/ledger/controller_default.go | 33 ++--- .../ledger/controller_default_test.go | 32 ++--- .../ledger/controller_generated_test.go | 8 +- .../ledger/controller_with_traces.go | 8 +- internal/controller/ledger/mocks_test.go | 136 +++++++----------- internal/controller/ledger/stats_test.go | 4 +- internal/controller/ledger/store.go | 12 +- .../controller/ledger/store_generated_test.go | 16 +-- internal/controller/system/controller.go | 4 +- internal/controller/system/store.go | 2 +- internal/storage/bucket/migrations_test.go | 10 +- internal/storage/common/cursor.go | 101 +++++++++++++ internal/storage/common/paginator.go | 6 +- internal/storage/common/paginator_column.go | 105 +++++++------- internal/storage/common/paginator_offset.go | 51 ++++--- internal/storage/common/resource.go | 123 +++++++++++----- internal/storage/common/schema.go | 21 +++ internal/storage/driver/driver.go | 2 +- .../storage/driver/system_generated_test.go | 4 +- internal/storage/ledger/accounts.go | 2 +- internal/storage/ledger/accounts_test.go | 90 +++++++++--- internal/storage/ledger/logs_test.go | 20 ++- internal/storage/ledger/store.go | 38 ++--- internal/storage/ledger/transactions_test.go | 112 +++++++++++---- internal/storage/ledger/volumes_test.go | 122 ++++++++++++---- internal/storage/system/resource_ledgers.go | 1 + internal/storage/system/store.go | 12 +- internal/worker/async_block.go | 6 +- test/e2e/api_logs_list_test.go | 2 +- 57 files changed, 1061 insertions(+), 651 deletions(-) create mode 100644 internal/storage/common/cursor.go diff --git a/internal/api/bulking/mocks_ledger_controller_test.go b/internal/api/bulking/mocks_ledger_controller_test.go index b4e923b86..905dee11b 100644 --- a/internal/api/bulking/mocks_ledger_controller_test.go +++ b/internal/api/bulking/mocks_ledger_controller_test.go @@ -241,7 +241,7 @@ func (mr *LedgerControllerMockRecorder) GetTransaction(ctx, query any) *gomock.C } // GetVolumesWithBalances mocks base method. -func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q common.OffsetPaginatedQuery[ledger0.GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { +func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q common.PaginatedQuery[ledger0.GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetVolumesWithBalances", ctx, q) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount]) @@ -285,7 +285,7 @@ func (mr *LedgerControllerMockRecorder) IsDatabaseUpToDate(ctx any) *gomock.Call } // ListAccounts mocks base method. -func (m *LedgerController) ListAccounts(ctx context.Context, query common.OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { +func (m *LedgerController) ListAccounts(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListAccounts", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Account]) @@ -300,7 +300,7 @@ func (mr *LedgerControllerMockRecorder) ListAccounts(ctx, query any) *gomock.Cal } // ListLogs mocks base method. -func (m *LedgerController) ListLogs(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { +func (m *LedgerController) ListLogs(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListLogs", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Log]) @@ -315,7 +315,7 @@ func (mr *LedgerControllerMockRecorder) ListLogs(ctx, query any) *gomock.Call { } // ListTransactions mocks base method. -func (m *LedgerController) ListTransactions(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { +func (m *LedgerController) ListTransactions(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListTransactions", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Transaction]) diff --git a/internal/api/common/mocks_ledger_controller_test.go b/internal/api/common/mocks_ledger_controller_test.go index 15cfa1949..64be8969d 100644 --- a/internal/api/common/mocks_ledger_controller_test.go +++ b/internal/api/common/mocks_ledger_controller_test.go @@ -241,7 +241,7 @@ func (mr *LedgerControllerMockRecorder) GetTransaction(ctx, query any) *gomock.C } // GetVolumesWithBalances mocks base method. -func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q common.OffsetPaginatedQuery[ledger0.GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { +func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q common.PaginatedQuery[ledger0.GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetVolumesWithBalances", ctx, q) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount]) @@ -285,7 +285,7 @@ func (mr *LedgerControllerMockRecorder) IsDatabaseUpToDate(ctx any) *gomock.Call } // ListAccounts mocks base method. -func (m *LedgerController) ListAccounts(ctx context.Context, query common.OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { +func (m *LedgerController) ListAccounts(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListAccounts", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Account]) @@ -300,7 +300,7 @@ func (mr *LedgerControllerMockRecorder) ListAccounts(ctx, query any) *gomock.Cal } // ListLogs mocks base method. -func (m *LedgerController) ListLogs(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { +func (m *LedgerController) ListLogs(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListLogs", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Log]) @@ -315,7 +315,7 @@ func (mr *LedgerControllerMockRecorder) ListLogs(ctx, query any) *gomock.Call { } // ListTransactions mocks base method. -func (m *LedgerController) ListTransactions(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { +func (m *LedgerController) ListTransactions(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListTransactions", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Transaction]) diff --git a/internal/api/common/mocks_system_controller_test.go b/internal/api/common/mocks_system_controller_test.go index e8ded33b6..b9c44462f 100644 --- a/internal/api/common/mocks_system_controller_test.go +++ b/internal/api/common/mocks_system_controller_test.go @@ -101,7 +101,7 @@ func (mr *SystemControllerMockRecorder) GetLedgerController(ctx, name any) *gomo } // ListLedgers mocks base method. -func (m *SystemController) ListLedgers(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) { +func (m *SystemController) ListLedgers(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListLedgers", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Ledger]) diff --git a/internal/api/v1/controllers_accounts_list.go b/internal/api/v1/controllers_accounts_list.go index 4d70eb1cd..ab5cbb306 100644 --- a/internal/api/v1/controllers_accounts_list.go +++ b/internal/api/v1/controllers_accounts_list.go @@ -1,6 +1,8 @@ package v1 import ( + "github.com/formancehq/go-libs/v3/bun/bunpaginate" + storagecommon "github.com/formancehq/ledger/internal/storage/common" "net/http" "errors" @@ -12,19 +14,21 @@ import ( func listAccounts(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - rq, err := getOffsetPaginatedQuery[any](r) + rqBuilder, err := buildAccountsFilterQuery(r) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - rq.Options.Builder, err = buildAccountsFilterQuery(r) + rq, err := getPaginatedQuery(r, "address", bunpaginate.OrderAsc, func(resourceQuery *storagecommon.ResourceQuery[any]) { + resourceQuery.Builder = rqBuilder + }) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - cursor, err := l.ListAccounts(r.Context(), *rq) + cursor, err := l.ListAccounts(r.Context(), rq) if err != nil { switch { case errors.Is(err, ledgercontroller.ErrMissingFeature{}): diff --git a/internal/api/v1/controllers_accounts_list_test.go b/internal/api/v1/controllers_accounts_list_test.go index c60242d11..0b09d660a 100644 --- a/internal/api/v1/controllers_accounts_list_test.go +++ b/internal/api/v1/controllers_accounts_list_test.go @@ -1,6 +1,7 @@ package v1 import ( + "github.com/formancehq/go-libs/v3/pointer" "github.com/formancehq/ledger/internal/api/common" storagecommon "github.com/formancehq/ledger/internal/storage/common" "net/http" @@ -26,7 +27,7 @@ func TestAccountsList(t *testing.T) { type testCase struct { name string queryParams url.Values - expectQuery storagecommon.OffsetPaginatedQuery[any] + expectQuery storagecommon.PaginatedQuery[any] expectStatusCode int expectedErrorCode string expectBackendCall bool @@ -37,8 +38,10 @@ func TestAccountsList(t *testing.T) { { name: "nominal", expectBackendCall: true, - expectQuery: storagecommon.OffsetPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { @@ -47,11 +50,13 @@ func TestAccountsList(t *testing.T) { "metadata[roles]": []string{"admin"}, }, expectBackendCall: true, - expectQuery: storagecommon.OffsetPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, Options: storagecommon.ResourceQuery[any]{ Builder: query.Match("metadata[roles]", "admin"), }, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { @@ -60,20 +65,30 @@ func TestAccountsList(t *testing.T) { "address": []string{"foo"}, }, expectBackendCall: true, - expectQuery: storagecommon.OffsetPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, Options: storagecommon.ResourceQuery[any]{ Builder: query.Match("address", "foo"), }, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { name: "using empty cursor", queryParams: url.Values{ - "cursor": []string{bunpaginate.EncodeCursor(storagecommon.OffsetPaginatedQuery[any]{})}, + "cursor": []string{bunpaginate.EncodeCursor(storagecommon.ColumnPaginatedQuery[any]{ + InitialPaginatedQuery: storagecommon.InitialPaginatedQuery[any]{ + PageSize: bunpaginate.QueryDefaultPageSize, + }, + })}, }, expectBackendCall: true, - expectQuery: storagecommon.OffsetPaginatedQuery[any]{}, + expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + InitialPaginatedQuery: storagecommon.InitialPaginatedQuery[any]{ + PageSize: DefaultPageSize, + }, + }, }, { name: "using invalid cursor", @@ -97,8 +112,10 @@ func TestAccountsList(t *testing.T) { "pageSize": []string{"1000000"}, }, expectBackendCall: true, - expectQuery: storagecommon.OffsetPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: MaxPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { @@ -108,11 +125,13 @@ func TestAccountsList(t *testing.T) { "balanceOperator": []string{"e"}, }, expectBackendCall: true, - expectQuery: storagecommon.OffsetPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, Options: storagecommon.ResourceQuery[any]{ Builder: query.Match("balance", int64(100)), }, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { @@ -121,8 +140,10 @@ func TestAccountsList(t *testing.T) { expectedErrorCode: common.ErrValidation, returnErr: ledgercontroller.ErrMissingFeature{}, expectBackendCall: true, - expectQuery: storagecommon.OffsetPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, } diff --git a/internal/api/v1/controllers_balances_list.go b/internal/api/v1/controllers_balances_list.go index 491d9a587..f66450d6a 100644 --- a/internal/api/v1/controllers_balances_list.go +++ b/internal/api/v1/controllers_balances_list.go @@ -1,6 +1,7 @@ package v1 import ( + storagecommon "github.com/formancehq/ledger/internal/storage/common" "math/big" "net/http" @@ -12,20 +13,22 @@ import ( func getBalances(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - rq, err := getOffsetPaginatedQuery[any](r) + filter, err := buildAccountsFilterQuery(r) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - rq.Options.Builder, err = buildAccountsFilterQuery(r) + rq, err := getPaginatedQuery[any](r, "address", bunpaginate.OrderAsc, func(resourceQuery *storagecommon.ResourceQuery[any]) { + resourceQuery.Expand = append(resourceQuery.Expand, "volumes") + resourceQuery.Builder = filter + }) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - rq.Options.Expand = []string{"volumes"} - cursor, err := l.ListAccounts(r.Context(), *rq) + cursor, err := l.ListAccounts(r.Context(), rq) if err != nil { common.HandleCommonErrors(w, r, err) return diff --git a/internal/api/v1/controllers_config.go b/internal/api/v1/controllers_config.go index 89cf49c42..63db7554c 100644 --- a/internal/api/v1/controllers_config.go +++ b/internal/api/v1/controllers_config.go @@ -1,7 +1,6 @@ package v1 import ( - "context" _ "embed" "github.com/formancehq/ledger/internal/api/common" storagecommon "github.com/formancehq/ledger/internal/storage/common" @@ -37,10 +36,8 @@ func GetInfo(systemController system.Controller, version string) func(w http.Res return func(w http.ResponseWriter, r *http.Request) { ledgerNames := make([]string, 0) - if err := bunpaginate.Iterate(r.Context(), ledgercontroller.NewListLedgersQuery(100), - func(ctx context.Context, q storagecommon.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) { - return systemController.ListLedgers(ctx, q) - }, + if err := storagecommon.Iterate(r.Context(), ledgercontroller.NewListLedgersQuery(100), + systemController.ListLedgers, func(cursor *bunpaginate.Cursor[ledger.Ledger]) error { ledgerNames = append(ledgerNames, collectionutils.Map(cursor.Data, func(from ledger.Ledger) string { return from.Name diff --git a/internal/api/v1/controllers_logs_list.go b/internal/api/v1/controllers_logs_list.go index f4085ec7d..0f165bb49 100644 --- a/internal/api/v1/controllers_logs_list.go +++ b/internal/api/v1/controllers_logs_list.go @@ -1,6 +1,7 @@ package v1 import ( + storagecommon "github.com/formancehq/ledger/internal/storage/common" "net/http" "github.com/formancehq/go-libs/v3/api" @@ -35,15 +36,16 @@ func buildGetLogsQuery(r *http.Request) query.Builder { func getLogs(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - paginatedQuery, err := getColumnPaginatedQuery[any](r, "id", bunpaginate.OrderDesc) + + paginatedQuery, err := getPaginatedQuery[any](r, "id", bunpaginate.OrderDesc, func(resourceQuery *storagecommon.ResourceQuery[any]) { + resourceQuery.Builder = buildGetLogsQuery(r) + }) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - paginatedQuery.Options.Builder = buildGetLogsQuery(r) - - cursor, err := l.ListLogs(r.Context(), *paginatedQuery) + cursor, err := l.ListLogs(r.Context(), paginatedQuery) if err != nil { common.HandleCommonErrors(w, r, err) return diff --git a/internal/api/v1/controllers_logs_list_test.go b/internal/api/v1/controllers_logs_list_test.go index ee2b737f7..ffa1f28d6 100644 --- a/internal/api/v1/controllers_logs_list_test.go +++ b/internal/api/v1/controllers_logs_list_test.go @@ -27,7 +27,7 @@ func TestGetLogs(t *testing.T) { type testCase struct { name string queryParams url.Values - expectQuery storagecommon.ColumnPaginatedQuery[any] + expectQuery storagecommon.PaginatedQuery[any] expectStatusCode int expectedErrorCode string } @@ -36,7 +36,7 @@ func TestGetLogs(t *testing.T) { testCases := []testCase{ { name: "nominal", - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -47,7 +47,7 @@ func TestGetLogs(t *testing.T) { queryParams: url.Values{ "start_time": []string{now.Format(time.DateFormat)}, }, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -61,7 +61,7 @@ func TestGetLogs(t *testing.T) { queryParams: url.Values{ "end_time": []string{now.Format(time.DateFormat)}, }, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -73,9 +73,17 @@ func TestGetLogs(t *testing.T) { { name: "using empty cursor", queryParams: url.Values{ - "cursor": []string{bunpaginate.EncodeCursor(storagecommon.ColumnPaginatedQuery[any]{})}, + "cursor": []string{bunpaginate.EncodeCursor(storagecommon.ColumnPaginatedQuery[any]{ + InitialPaginatedQuery: storagecommon.InitialPaginatedQuery[any]{ + PageSize: DefaultPageSize, + }, + })}, + }, + expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + InitialPaginatedQuery: storagecommon.InitialPaginatedQuery[any]{ + PageSize: DefaultPageSize, + }, }, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{}, }, { name: "using invalid cursor", diff --git a/internal/api/v1/controllers_transactions_list.go b/internal/api/v1/controllers_transactions_list.go index 1936fdc01..cc0aef45d 100644 --- a/internal/api/v1/controllers_transactions_list.go +++ b/internal/api/v1/controllers_transactions_list.go @@ -1,6 +1,7 @@ package v1 import ( + storagecommon "github.com/formancehq/ledger/internal/storage/common" "net/http" "github.com/formancehq/go-libs/v3/api" @@ -11,15 +12,16 @@ import ( func listTransactions(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - paginatedQuery, err := getColumnPaginatedQuery[any](r, "id", bunpaginate.OrderDesc) + paginatedQuery, err := getPaginatedQuery[any](r, "id", bunpaginate.OrderDesc, func(resourceQuery *storagecommon.ResourceQuery[any]) { + resourceQuery.Expand = append(resourceQuery.Expand, "volumes") + resourceQuery.Builder = buildGetTransactionsQuery(r) + }) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - paginatedQuery.Options.Builder = buildGetTransactionsQuery(r) - paginatedQuery.Options.Expand = []string{"volumes"} - cursor, err := l.ListTransactions(r.Context(), *paginatedQuery) + cursor, err := l.ListTransactions(r.Context(), paginatedQuery) if err != nil { common.HandleCommonErrors(w, r, err) return diff --git a/internal/api/v1/controllers_transactions_list_test.go b/internal/api/v1/controllers_transactions_list_test.go index 9b4287a35..62d08e47c 100644 --- a/internal/api/v1/controllers_transactions_list_test.go +++ b/internal/api/v1/controllers_transactions_list_test.go @@ -27,7 +27,7 @@ func TestTransactionsList(t *testing.T) { type testCase struct { name string queryParams url.Values - expectQuery storagecommon.ColumnPaginatedQuery[any] + expectQuery storagecommon.PaginatedQuery[any] expectStatusCode int expectedErrorCode string } @@ -36,7 +36,7 @@ func TestTransactionsList(t *testing.T) { testCases := []testCase{ { name: "nominal", - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Column: "id", @@ -50,7 +50,7 @@ func TestTransactionsList(t *testing.T) { queryParams: url.Values{ "metadata[roles]": []string{"admin"}, }, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Column: "id", @@ -65,7 +65,7 @@ func TestTransactionsList(t *testing.T) { queryParams: url.Values{ "start_time": []string{now.Format(time.DateFormat)}, }, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Column: "id", @@ -80,7 +80,7 @@ func TestTransactionsList(t *testing.T) { queryParams: url.Values{ "end_time": []string{now.Format(time.DateFormat)}, }, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Column: "id", @@ -95,7 +95,7 @@ func TestTransactionsList(t *testing.T) { queryParams: url.Values{ "account": []string{"xxx"}, }, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Column: "id", @@ -110,7 +110,7 @@ func TestTransactionsList(t *testing.T) { queryParams: url.Values{ "reference": []string{"xxx"}, }, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Column: "id", @@ -125,7 +125,7 @@ func TestTransactionsList(t *testing.T) { queryParams: url.Values{ "destination": []string{"xxx"}, }, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Column: "id", @@ -140,7 +140,7 @@ func TestTransactionsList(t *testing.T) { queryParams: url.Values{ "source": []string{"xxx"}, }, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: DefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Column: "id", @@ -153,11 +153,21 @@ func TestTransactionsList(t *testing.T) { { name: "using empty cursor", queryParams: url.Values{ - "cursor": []string{bunpaginate.EncodeCursor(storagecommon.ColumnPaginatedQuery[any]{})}, + "cursor": []string{bunpaginate.EncodeCursor(storagecommon.ColumnPaginatedQuery[any]{ + InitialPaginatedQuery: storagecommon.InitialPaginatedQuery[any]{ + Options: storagecommon.ResourceQuery[any]{ + Expand: []string{"volumes"}, + }, + PageSize: DefaultPageSize, + }, + })}, }, expectQuery: storagecommon.ColumnPaginatedQuery[any]{ - Options: storagecommon.ResourceQuery[any]{ - Expand: []string{"volumes"}, + InitialPaginatedQuery: storagecommon.InitialPaginatedQuery[any]{ + Options: storagecommon.ResourceQuery[any]{ + Expand: []string{"volumes"}, + }, + PageSize: DefaultPageSize, }, }, }, @@ -182,7 +192,7 @@ func TestTransactionsList(t *testing.T) { queryParams: url.Values{ "pageSize": []string{"1000000"}, }, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: MaxPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Column: "id", diff --git a/internal/api/v1/mocks_ledger_controller_test.go b/internal/api/v1/mocks_ledger_controller_test.go index a93a8f8aa..6891ef94b 100644 --- a/internal/api/v1/mocks_ledger_controller_test.go +++ b/internal/api/v1/mocks_ledger_controller_test.go @@ -241,7 +241,7 @@ func (mr *LedgerControllerMockRecorder) GetTransaction(ctx, query any) *gomock.C } // GetVolumesWithBalances mocks base method. -func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q common.OffsetPaginatedQuery[ledger0.GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { +func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q common.PaginatedQuery[ledger0.GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetVolumesWithBalances", ctx, q) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount]) @@ -285,7 +285,7 @@ func (mr *LedgerControllerMockRecorder) IsDatabaseUpToDate(ctx any) *gomock.Call } // ListAccounts mocks base method. -func (m *LedgerController) ListAccounts(ctx context.Context, query common.OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { +func (m *LedgerController) ListAccounts(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListAccounts", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Account]) @@ -300,7 +300,7 @@ func (mr *LedgerControllerMockRecorder) ListAccounts(ctx, query any) *gomock.Cal } // ListLogs mocks base method. -func (m *LedgerController) ListLogs(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { +func (m *LedgerController) ListLogs(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListLogs", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Log]) @@ -315,7 +315,7 @@ func (mr *LedgerControllerMockRecorder) ListLogs(ctx, query any) *gomock.Call { } // ListTransactions mocks base method. -func (m *LedgerController) ListTransactions(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { +func (m *LedgerController) ListTransactions(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListTransactions", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Transaction]) diff --git a/internal/api/v1/mocks_system_controller_test.go b/internal/api/v1/mocks_system_controller_test.go index cc772b5ee..b1f6b3cb6 100644 --- a/internal/api/v1/mocks_system_controller_test.go +++ b/internal/api/v1/mocks_system_controller_test.go @@ -101,7 +101,7 @@ func (mr *SystemControllerMockRecorder) GetLedgerController(ctx, name any) *gomo } // ListLedgers mocks base method. -func (m *SystemController) ListLedgers(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) { +func (m *SystemController) ListLedgers(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListLedgers", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Ledger]) diff --git a/internal/api/v1/utils.go b/internal/api/v1/utils.go index 455d12402..fbae388c7 100644 --- a/internal/api/v1/utils.go +++ b/internal/api/v1/utils.go @@ -8,8 +8,6 @@ import ( ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" "github.com/formancehq/go-libs/v3/bun/bunpaginate" - - "github.com/formancehq/go-libs/v3/pointer" ) func getCommandParameters[INPUT any](r *http.Request, input INPUT) ledgercontroller.Parameters[INPUT] { @@ -25,44 +23,51 @@ func getCommandParameters[INPUT any](r *http.Request, input INPUT) ledgercontrol } } -func getOffsetPaginatedQuery[v any](r *http.Request, modifiers ...func(*v) error) (*storagecommon.OffsetPaginatedQuery[v], error) { - return bunpaginate.Extract[storagecommon.OffsetPaginatedQuery[v]](r, func() (*storagecommon.OffsetPaginatedQuery[v], error) { - rq, err := getResourceQuery[v](r, modifiers...) - if err != nil { - return nil, err - } +func getPaginatedQuery[Options any]( + r *http.Request, + defaultColumn string, + defaultOrder bunpaginate.Order, + modifiers ...func(resourceQuery *storagecommon.ResourceQuery[Options]), +) (storagecommon.PaginatedQuery[Options], error) { - pageSize, err := bunpaginate.GetPageSize(r, bunpaginate.WithMaxPageSize(MaxPageSize), bunpaginate.WithDefaultPageSize(DefaultPageSize)) - if err != nil { - return nil, err - } + return storagecommon.Extract[Options]( + r, + func() (*storagecommon.InitialPaginatedQuery[Options], error) { + rq, err := getResourceQuery[Options](r) + if err != nil { + return nil, err + } - return &storagecommon.OffsetPaginatedQuery[v]{ - PageSize: pageSize, - Options: *rq, - }, nil - }) -} + for _, modifier := range modifiers { + modifier(rq) + } -func getColumnPaginatedQuery[v any](r *http.Request, column string, order bunpaginate.Order, modifiers ...func(*v) error) (*storagecommon.ColumnPaginatedQuery[v], error) { - return bunpaginate.Extract[storagecommon.ColumnPaginatedQuery[v]](r, func() (*storagecommon.ColumnPaginatedQuery[v], error) { - rq, err := getResourceQuery[v](r, modifiers...) - if err != nil { - return nil, err - } - - pageSize, err := bunpaginate.GetPageSize(r, bunpaginate.WithMaxPageSize(MaxPageSize), bunpaginate.WithDefaultPageSize(DefaultPageSize)) - if err != nil { - return nil, err - } + pageSize, err := bunpaginate.GetPageSize( + r, + bunpaginate.WithMaxPageSize(MaxPageSize), + bunpaginate.WithDefaultPageSize(DefaultPageSize), + ) + if err != nil { + return nil, err + } - return &storagecommon.ColumnPaginatedQuery[v]{ - PageSize: pageSize, - Column: column, - Order: pointer.For(order), - Options: *rq, - }, nil - }) + return &storagecommon.InitialPaginatedQuery[Options]{ + Column: defaultColumn, + Order: &defaultOrder, + PageSize: pageSize, + Options: *rq, + }, nil + }, + func(query *storagecommon.InitialPaginatedQuery[Options]) error { + var err error + query.PageSize, err = bunpaginate.GetPageSize( + r, + bunpaginate.WithMaxPageSize(MaxPageSize), + bunpaginate.WithDefaultPageSize(query.PageSize), + ) + return err + }, + ) } func getResourceQuery[v any](r *http.Request, modifiers ...func(*v) error) (*storagecommon.ResourceQuery[v], error) { diff --git a/internal/api/v2/common.go b/internal/api/v2/common.go index 2e3a22f36..c48bcae1f 100644 --- a/internal/api/v2/common.go +++ b/internal/api/v2/common.go @@ -11,7 +11,6 @@ import ( "github.com/formancehq/go-libs/v3/bun/bunpaginate" "github.com/formancehq/go-libs/v3/time" - "github.com/formancehq/go-libs/v3/pointer" "github.com/formancehq/go-libs/v3/query" ) @@ -62,64 +61,51 @@ func getExpand(r *http.Request) []string { ) } -func getOffsetPaginatedQuery[v any](r *http.Request, paginationConfig common.PaginationConfig, modifiers ...func(*v) error) (*storagecommon.OffsetPaginatedQuery[v], error) { - ret, err := bunpaginate.Extract[storagecommon.OffsetPaginatedQuery[v]](r, func() (*storagecommon.OffsetPaginatedQuery[v], error) { - rq, err := getResourceQuery[v](r, modifiers...) - if err != nil { - return nil, err - } - - return &storagecommon.OffsetPaginatedQuery[v]{ - Options: *rq, - }, nil - }) - if err != nil { - return nil, err - } - - if ret.PageSize == 0 || r.URL.Query().Get(bunpaginate.QueryKeyPageSize) != "" { - ret.PageSize, err = bunpaginate.GetPageSize( - r, - bunpaginate.WithMaxPageSize(paginationConfig.MaxPageSize), - bunpaginate.WithDefaultPageSize(paginationConfig.DefaultPageSize), - ) - if err != nil { - return nil, err - } - } - - return ret, nil -} - -func getColumnPaginatedQuery[v any](r *http.Request, paginationConfig common.PaginationConfig, defaultPaginationColumn string, order bunpaginate.Order, modifiers ...func(*v) error) (*storagecommon.ColumnPaginatedQuery[v], error) { - ret, err := bunpaginate.Extract[storagecommon.ColumnPaginatedQuery[v]](r, func() (*storagecommon.ColumnPaginatedQuery[v], error) { - rq, err := getResourceQuery[v](r, modifiers...) - if err != nil { - return nil, err - } - - return &storagecommon.ColumnPaginatedQuery[v]{ - Column: defaultPaginationColumn, - Order: pointer.For(order), - Options: *rq, - }, nil - }) - if err != nil { - return nil, err - } - - if ret.PageSize == 0 || r.URL.Query().Get(bunpaginate.QueryKeyPageSize) != "" { - ret.PageSize, err = bunpaginate.GetPageSize( - r, - bunpaginate.WithMaxPageSize(paginationConfig.MaxPageSize), - bunpaginate.WithDefaultPageSize(paginationConfig.DefaultPageSize), - ) - if err != nil { - return nil, err - } - } - - return ret, nil +func getPaginatedQuery[Options any]( + r *http.Request, + paginationConfig common.PaginationConfig, + defaultColumn string, + defaultOrder bunpaginate.Order, + modifiers ...func(resourceQuery *storagecommon.ResourceQuery[Options]), +) (storagecommon.PaginatedQuery[Options], error) { + return storagecommon.Extract[Options]( + r, + func() (*storagecommon.InitialPaginatedQuery[Options], error) { + rq, err := getResourceQuery[Options](r) + if err != nil { + return nil, err + } + + for _, modifier := range modifiers { + modifier(rq) + } + + pageSize, err := bunpaginate.GetPageSize( + r, + bunpaginate.WithMaxPageSize(paginationConfig.MaxPageSize), + bunpaginate.WithDefaultPageSize(paginationConfig.DefaultPageSize), + ) + if err != nil { + return nil, err + } + + return &storagecommon.InitialPaginatedQuery[Options]{ + Column: defaultColumn, + Order: &defaultOrder, + PageSize: pageSize, + Options: *rq, + }, nil + }, + func(query *storagecommon.InitialPaginatedQuery[Options]) error { + var err error + query.PageSize, err = bunpaginate.GetPageSize( + r, + bunpaginate.WithMaxPageSize(paginationConfig.MaxPageSize), + bunpaginate.WithDefaultPageSize(query.PageSize), + ) + return err + }, + ) } func getResourceQuery[v any](r *http.Request, modifiers ...func(*v) error) (*storagecommon.ResourceQuery[v], error) { diff --git a/internal/api/v2/controllers_accounts_list.go b/internal/api/v2/controllers_accounts_list.go index 77de5006e..5d9956dc0 100644 --- a/internal/api/v2/controllers_accounts_list.go +++ b/internal/api/v2/controllers_accounts_list.go @@ -15,13 +15,13 @@ func listAccounts(paginationConfig common.PaginationConfig) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - query, err := getOffsetPaginatedQuery[any](r, paginationConfig) + query, err := getPaginatedQuery[any](r, paginationConfig, "address", bunpaginate.OrderAsc) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - cursor, err := l.ListAccounts(r.Context(), *query) + cursor, err := l.ListAccounts(r.Context(), query) if err != nil { switch { case errors.Is(err, storagecommon.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): diff --git a/internal/api/v2/controllers_accounts_list_test.go b/internal/api/v2/controllers_accounts_list_test.go index c257b911b..64635915a 100644 --- a/internal/api/v2/controllers_accounts_list_test.go +++ b/internal/api/v2/controllers_accounts_list_test.go @@ -2,6 +2,7 @@ package v2 import ( "bytes" + "github.com/formancehq/go-libs/v3/pointer" "github.com/formancehq/ledger/internal/api/common" storagecommon "github.com/formancehq/ledger/internal/storage/common" "net/http" @@ -29,7 +30,7 @@ func TestAccountsList(t *testing.T) { name string queryParams url.Values body string - expectQuery storagecommon.OffsetPaginatedQuery[any] + expectQuery storagecommon.PaginatedQuery[any] expectStatusCode int expectedErrorCode string expectBackendCall bool @@ -40,12 +41,14 @@ func TestAccountsList(t *testing.T) { testCases := []testCase{ { name: "nominal", - expectQuery: storagecommon.OffsetPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Options: storagecommon.ResourceQuery[any]{ PIT: &before, Expand: make([]string, 0), }, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, expectBackendCall: true, }, @@ -53,26 +56,30 @@ func TestAccountsList(t *testing.T) { name: "using metadata", body: `{"$match": { "metadata[roles]": "admin" }}`, expectBackendCall: true, - expectQuery: storagecommon.OffsetPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Options: storagecommon.ResourceQuery[any]{ PIT: &before, Builder: query.Match("metadata[roles]", "admin"), Expand: make([]string, 0), }, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { name: "using address", body: `{"$match": { "address": "foo" }}`, expectBackendCall: true, - expectQuery: storagecommon.OffsetPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Options: storagecommon.ResourceQuery[any]{ PIT: &before, Builder: query.Match("address", "foo"), Expand: make([]string, 0), }, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { @@ -80,13 +87,21 @@ func TestAccountsList(t *testing.T) { expectBackendCall: true, queryParams: url.Values{ "cursor": []string{bunpaginate.EncodeCursor(storagecommon.OffsetPaginatedQuery[any]{ - PageSize: bunpaginate.QueryDefaultPageSize, - Options: storagecommon.ResourceQuery[any]{}, + InitialPaginatedQuery: storagecommon.InitialPaginatedQuery[any]{ + PageSize: bunpaginate.QueryDefaultPageSize, + Options: storagecommon.ResourceQuery[any]{}, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), + }, })}, }, expectQuery: storagecommon.OffsetPaginatedQuery[any]{ - PageSize: bunpaginate.QueryDefaultPageSize, - Options: storagecommon.ResourceQuery[any]{}, + InitialPaginatedQuery: storagecommon.InitialPaginatedQuery[any]{ + PageSize: bunpaginate.QueryDefaultPageSize, + Options: storagecommon.ResourceQuery[any]{}, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), + }, }, }, { @@ -111,38 +126,44 @@ func TestAccountsList(t *testing.T) { queryParams: url.Values{ "pageSize": []string{"1000000"}, }, - expectQuery: storagecommon.OffsetPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.MaxPageSize, Options: storagecommon.ResourceQuery[any]{ PIT: &before, Expand: make([]string, 0), }, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { name: "using balance filter", expectBackendCall: true, body: `{"$lt": { "balance[USD/2]": 100 }}`, - expectQuery: storagecommon.OffsetPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Options: storagecommon.ResourceQuery[any]{ PIT: &before, Builder: query.Lt("balance[USD/2]", float64(100)), Expand: make([]string, 0), }, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { name: "using exists filter", expectBackendCall: true, body: `{"$exists": { "metadata": "foo" }}`, - expectQuery: storagecommon.OffsetPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Options: storagecommon.ResourceQuery[any]{ PIT: &before, Builder: query.Exists("metadata", "foo"), Expand: make([]string, 0), }, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { @@ -157,12 +178,14 @@ func TestAccountsList(t *testing.T) { expectedErrorCode: common.ErrValidation, expectBackendCall: true, returnErr: storagecommon.ErrInvalidQuery{}, - expectQuery: storagecommon.OffsetPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Options: storagecommon.ResourceQuery[any]{ PIT: &before, Expand: make([]string, 0), }, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { @@ -171,12 +194,14 @@ func TestAccountsList(t *testing.T) { expectedErrorCode: common.ErrValidation, expectBackendCall: true, returnErr: ledgercontroller.ErrMissingFeature{}, - expectQuery: storagecommon.OffsetPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Options: storagecommon.ResourceQuery[any]{ PIT: &before, Expand: make([]string, 0), }, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { @@ -185,12 +210,14 @@ func TestAccountsList(t *testing.T) { expectedErrorCode: api.ErrorInternal, expectBackendCall: true, returnErr: errors.New("undefined error"), - expectQuery: storagecommon.OffsetPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Options: storagecommon.ResourceQuery[any]{ PIT: &before, Expand: make([]string, 0), }, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, } diff --git a/internal/api/v2/controllers_ledgers_list.go b/internal/api/v2/controllers_ledgers_list.go index 2179e913d..d7cd782fc 100644 --- a/internal/api/v2/controllers_ledgers_list.go +++ b/internal/api/v2/controllers_ledgers_list.go @@ -15,13 +15,13 @@ import ( func listLedgers(b system.Controller, paginationConfig common.PaginationConfig) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - rq, err := getColumnPaginatedQuery[any](r, paginationConfig, "id", bunpaginate.OrderAsc) + rq, err := getPaginatedQuery[any](r, paginationConfig, "id", bunpaginate.OrderAsc) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - ledgers, err := b.ListLedgers(r.Context(), *rq) + ledgers, err := b.ListLedgers(r.Context(), rq) if err != nil { switch { case errors.Is(err, storagecommon.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): diff --git a/internal/api/v2/controllers_ledgers_list_test.go b/internal/api/v2/controllers_ledgers_list_test.go index 449a4974e..0188cb324 100644 --- a/internal/api/v2/controllers_ledgers_list_test.go +++ b/internal/api/v2/controllers_ledgers_list_test.go @@ -2,6 +2,7 @@ package v2 import ( "encoding/json" + "github.com/formancehq/go-libs/v3/pointer" "github.com/formancehq/ledger/internal/api/common" storagecommon "github.com/formancehq/ledger/internal/storage/common" "net/http" @@ -21,14 +22,14 @@ import ( "go.uber.org/mock/gomock" ) -func TestListLedgers(t *testing.T) { +func TestLedgersList(t *testing.T) { t.Parallel() ctx := logging.TestingContext() type testCase struct { name string - expectQuery storagecommon.ColumnPaginatedQuery[any] + expectQuery storagecommon.PaginatedQuery[any] queryParams url.Values returnData []ledger.Ledger returnErr error @@ -40,7 +41,7 @@ func TestListLedgers(t *testing.T) { for _, tc := range []testCase{ { name: "nominal", - expectQuery: ledgercontroller.NewListLedgersQuery(15), + expectQuery: pointer.For(ledgercontroller.NewListLedgersQuery(15)), returnData: []ledger.Ledger{ ledger.MustNewWithDefault(uuid.NewString()), ledger.MustNewWithDefault(uuid.NewString()), diff --git a/internal/api/v2/controllers_logs_list.go b/internal/api/v2/controllers_logs_list.go index 58174ffdf..6a999e5aa 100644 --- a/internal/api/v2/controllers_logs_list.go +++ b/internal/api/v2/controllers_logs_list.go @@ -14,13 +14,13 @@ func listLogs(paginationConfig common.PaginationConfig) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - rq, err := getColumnPaginatedQuery[any](r, paginationConfig, "id", bunpaginate.OrderDesc) + rq, err := getPaginatedQuery[any](r, paginationConfig, "id", bunpaginate.OrderDesc) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - cursor, err := l.ListLogs(r.Context(), *rq) + cursor, err := l.ListLogs(r.Context(), rq) if err != nil { switch { case errors.Is(err, storagecommon.ErrInvalidQuery{}): diff --git a/internal/api/v2/controllers_logs_list_test.go b/internal/api/v2/controllers_logs_list_test.go index 1c3f2ea4a..7f2388765 100644 --- a/internal/api/v2/controllers_logs_list_test.go +++ b/internal/api/v2/controllers_logs_list_test.go @@ -23,14 +23,14 @@ import ( "go.uber.org/mock/gomock" ) -func TestGetLogs(t *testing.T) { +func TestLogsList(t *testing.T) { t.Parallel() type testCase struct { name string queryParams url.Values body string - expectQuery storagecommon.ColumnPaginatedQuery[any] + expectQuery storagecommon.PaginatedQuery[any] expectStatusCode int expectedErrorCode string expectBackendCall bool @@ -41,7 +41,7 @@ func TestGetLogs(t *testing.T) { testCases := []testCase{ { name: "nominal", - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -54,7 +54,7 @@ func TestGetLogs(t *testing.T) { { name: "using start time", body: fmt.Sprintf(`{"$gte": {"date": "%s"}}`, now.Format(time.DateFormat)), - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -68,7 +68,7 @@ func TestGetLogs(t *testing.T) { { name: "using end time", body: fmt.Sprintf(`{"$lt": {"date": "%s"}}`, now.Format(time.DateFormat)), - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -83,15 +83,19 @@ func TestGetLogs(t *testing.T) { name: "using empty cursor", queryParams: url.Values{ "cursor": []string{bunpaginate.EncodeCursor(storagecommon.ColumnPaginatedQuery[any]{ - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + InitialPaginatedQuery: storagecommon.InitialPaginatedQuery[any]{ + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + }, })}, }, expectQuery: storagecommon.ColumnPaginatedQuery[any]{ - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + InitialPaginatedQuery: storagecommon.InitialPaginatedQuery[any]{ + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + }, }, expectBackendCall: true, }, @@ -120,7 +124,7 @@ func TestGetLogs(t *testing.T) { { name: "with invalid query", expectStatusCode: http.StatusBadRequest, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -135,7 +139,7 @@ func TestGetLogs(t *testing.T) { { name: "with unexpected error", expectStatusCode: http.StatusInternalServerError, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), diff --git a/internal/api/v2/controllers_transactions_list.go b/internal/api/v2/controllers_transactions_list.go index 6ac20f0cc..c788dd843 100644 --- a/internal/api/v2/controllers_transactions_list.go +++ b/internal/api/v2/controllers_transactions_list.go @@ -25,13 +25,13 @@ func listTransactions(paginationConfig common.PaginationConfig) http.HandlerFunc order = bunpaginate.OrderAsc } - rq, err := getColumnPaginatedQuery[any](r, paginationConfig, paginationColumn, order) + rq, err := getPaginatedQuery[any](r, paginationConfig, paginationColumn, order) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - cursor, err := l.ListTransactions(r.Context(), *rq) + cursor, err := l.ListTransactions(r.Context(), rq) if err != nil { switch { case errors.Is(err, storagecommon.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): diff --git a/internal/api/v2/controllers_transactions_list_test.go b/internal/api/v2/controllers_transactions_list_test.go index 5b047ce4a..f64ac68d3 100644 --- a/internal/api/v2/controllers_transactions_list_test.go +++ b/internal/api/v2/controllers_transactions_list_test.go @@ -29,7 +29,7 @@ func TestTransactionsList(t *testing.T) { name string queryParams url.Values body string - expectQuery storagecommon.ColumnPaginatedQuery[any] + expectQuery storagecommon.PaginatedQuery[any] expectStatusCode int expectedErrorCode string } @@ -38,7 +38,7 @@ func TestTransactionsList(t *testing.T) { testCases := []testCase{ { name: "nominal", - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -51,7 +51,7 @@ func TestTransactionsList(t *testing.T) { { name: "using metadata", body: `{"$match": {"metadata[roles]": "admin"}}`, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -65,7 +65,7 @@ func TestTransactionsList(t *testing.T) { { name: "using startTime", body: fmt.Sprintf(`{"$gte": {"start_time": "%s"}}`, now.Format(time.DateFormat)), - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -79,7 +79,7 @@ func TestTransactionsList(t *testing.T) { { name: "using endTime", body: fmt.Sprintf(`{"$lte": {"end_time": "%s"}}`, now.Format(time.DateFormat)), - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -93,7 +93,7 @@ func TestTransactionsList(t *testing.T) { { name: "using account", body: `{"$match": {"account": "xxx"}}`, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -107,7 +107,7 @@ func TestTransactionsList(t *testing.T) { { name: "using reference", body: `{"$match": {"reference": "xxx"}}`, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -121,7 +121,7 @@ func TestTransactionsList(t *testing.T) { { name: "using destination", body: `{"$match": {"destination": "xxx"}}`, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -135,7 +135,7 @@ func TestTransactionsList(t *testing.T) { { name: "using source", body: `{"$match": {"source": "xxx"}}`, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -149,10 +149,16 @@ func TestTransactionsList(t *testing.T) { { name: "using empty cursor", queryParams: url.Values{ - "cursor": []string{bunpaginate.EncodeCursor(storagecommon.ColumnPaginatedQuery[any]{})}, + "cursor": []string{bunpaginate.EncodeCursor(storagecommon.ColumnPaginatedQuery[any]{ + InitialPaginatedQuery: storagecommon.InitialPaginatedQuery[any]{ + PageSize: bunpaginate.QueryDefaultPageSize, + }, + })}, }, expectQuery: storagecommon.ColumnPaginatedQuery[any]{ - PageSize: bunpaginate.QueryDefaultPageSize, + InitialPaginatedQuery: storagecommon.InitialPaginatedQuery[any]{ + PageSize: bunpaginate.QueryDefaultPageSize, + }, }, }, { @@ -176,7 +182,7 @@ func TestTransactionsList(t *testing.T) { queryParams: url.Values{ "pageSize": []string{"1000000"}, }, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.MaxPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -191,28 +197,32 @@ func TestTransactionsList(t *testing.T) { queryParams: url.Values{ "cursor": []string{func() string { return bunpaginate.EncodeCursor(storagecommon.ColumnPaginatedQuery[any]{ - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), - Options: storagecommon.ResourceQuery[any]{ - PIT: &now, + InitialPaginatedQuery: storagecommon.InitialPaginatedQuery[any]{ + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: storagecommon.ResourceQuery[any]{ + PIT: &now, + }, }, }) }()}, }, expectQuery: storagecommon.ColumnPaginatedQuery[any]{ - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), - Options: storagecommon.ResourceQuery[any]{ - PIT: &now, + InitialPaginatedQuery: storagecommon.InitialPaginatedQuery[any]{ + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: storagecommon.ResourceQuery[any]{ + PIT: &now, + }, }, }, }, { name: "using $exists metadata filter", body: `{"$exists": {"metadata": "foo"}}`, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), @@ -226,7 +236,7 @@ func TestTransactionsList(t *testing.T) { { name: "paginate using effective order", queryParams: map[string][]string{"order": {"effective"}}, - expectQuery: storagecommon.ColumnPaginatedQuery[any]{ + expectQuery: storagecommon.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Column: "timestamp", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), diff --git a/internal/api/v2/controllers_volumes.go b/internal/api/v2/controllers_volumes.go index f2bc7e235..c872bb9ff 100644 --- a/internal/api/v2/controllers_volumes.go +++ b/internal/api/v2/controllers_volumes.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/formancehq/go-libs/v3/api" "github.com/formancehq/go-libs/v3/bun/bunpaginate" + "github.com/formancehq/go-libs/v3/time" ledger "github.com/formancehq/ledger/internal" "github.com/formancehq/ledger/internal/api/common" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" @@ -13,31 +14,28 @@ import ( ) func readVolumes(paginationConfig common.PaginationConfig) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - rq, err := getOffsetPaginatedQuery[ledgercontroller.GetVolumesOptions](r, paginationConfig, func(opts *ledgercontroller.GetVolumesOptions) error { - groupBy := r.URL.Query().Get("groupBy") - if groupBy != "" { - v, err := strconv.ParseInt(groupBy, 10, 64) - if err != nil { - return err - } - opts.GroupLvl = int(v) + var groupBy int + if queryGroupBy := r.URL.Query().Get("groupBy"); queryGroupBy != "" { + v, err := strconv.ParseInt(queryGroupBy, 10, 64) + if err != nil { + api.BadRequest(w, common.ErrValidation, err) + return } - - opts.UseInsertionDate = api.QueryParamBool(r, "insertionDate") - - return nil - }) - if err != nil { - api.BadRequest(w, common.ErrValidation, err) - return + groupBy = int(v) } + // Kept for compatibility with old version of the ledger + // the parameters used should bt pit and oot now + var ( + pit *time.Time + oot *time.Time + err error + ) if r.URL.Query().Get("endTime") != "" { - rq.Options.PIT, err = getDate(r, "endTime") + pit, err = getDate(r, "endTime") if err != nil { api.BadRequest(w, common.ErrValidation, err) return @@ -45,14 +43,38 @@ func readVolumes(paginationConfig common.PaginationConfig) http.HandlerFunc { } if r.URL.Query().Get("startTime") != "" { - rq.Options.OOT, err = getDate(r, "startTime") + oot, err = getDate(r, "startTime") if err != nil { api.BadRequest(w, common.ErrValidation, err) return } } - cursor, err := l.GetVolumesWithBalances(r.Context(), *rq) + rq, err := getPaginatedQuery[ledgercontroller.GetVolumesOptions]( + r, + paginationConfig, + "account", + bunpaginate.OrderAsc, + func(rq *storagecommon.ResourceQuery[ledgercontroller.GetVolumesOptions]) { + if groupBy > 0 { + rq.Opts.GroupLvl = groupBy + } + if pit != nil { + rq.PIT = pit + } + if oot != nil { + rq.OOT = oot + } + + rq.Opts.UseInsertionDate = api.QueryParamBool(r, "insertionDate") + }, + ) + if err != nil { + api.BadRequest(w, common.ErrValidation, err) + return + } + + cursor, err := l.GetVolumesWithBalances(r.Context(), rq) if err != nil { switch { case errors.Is(err, storagecommon.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): diff --git a/internal/api/v2/controllers_volumes_test.go b/internal/api/v2/controllers_volumes_test.go index db6cd0503..56f900a16 100644 --- a/internal/api/v2/controllers_volumes_test.go +++ b/internal/api/v2/controllers_volumes_test.go @@ -2,6 +2,7 @@ package v2 import ( "bytes" + "github.com/formancehq/go-libs/v3/pointer" "github.com/formancehq/ledger/internal/api/common" storagecommon "github.com/formancehq/ledger/internal/storage/common" "math/big" @@ -24,14 +25,14 @@ import ( "go.uber.org/mock/gomock" ) -func TestGetVolumes(t *testing.T) { +func TestVolumesList(t *testing.T) { t.Parallel() type testCase struct { name string queryParams url.Values body string - expectQuery storagecommon.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions] + expectQuery storagecommon.PaginatedQuery[ledgercontroller.GetVolumesOptions] expectStatusCode int expectedErrorCode string } @@ -40,36 +41,42 @@ func TestGetVolumes(t *testing.T) { testCases := []testCase{ { name: "basic", - expectQuery: storagecommon.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + expectQuery: storagecommon.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ PageSize: bunpaginate.QueryDefaultPageSize, Options: storagecommon.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &before, Expand: make([]string, 0), }, + Column: "account", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { name: "using metadata", body: `{"$match": { "metadata[roles]": "admin" }}`, - expectQuery: storagecommon.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + expectQuery: storagecommon.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ PageSize: bunpaginate.QueryDefaultPageSize, Options: storagecommon.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &before, Builder: query.Match("metadata[roles]", "admin"), Expand: make([]string, 0), }, + Column: "account", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { name: "using account", body: `{"$match": { "account": "foo" }}`, - expectQuery: storagecommon.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + expectQuery: storagecommon.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ PageSize: bunpaginate.QueryDefaultPageSize, Options: storagecommon.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &before, Builder: query.Match("account", "foo"), Expand: make([]string, 0), }, + Column: "account", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { @@ -84,7 +91,7 @@ func TestGetVolumes(t *testing.T) { "pit": []string{before.Format(time.RFC3339Nano)}, "groupBy": []string{"3"}, }, - expectQuery: storagecommon.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + expectQuery: storagecommon.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ PageSize: bunpaginate.QueryDefaultPageSize, Options: storagecommon.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &before, @@ -93,30 +100,36 @@ func TestGetVolumes(t *testing.T) { GroupLvl: 3, }, }, + Column: "account", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { - name: "using Exists metadata filter", + name: "using exists metadata filter", body: `{"$exists": { "metadata": "foo" }}`, - expectQuery: storagecommon.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + expectQuery: storagecommon.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ PageSize: bunpaginate.QueryDefaultPageSize, Options: storagecommon.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &before, Builder: query.Exists("metadata", "foo"), Expand: make([]string, 0), }, + Column: "account", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, { name: "using balance filter", body: `{"$gte": { "balance[EUR]": 50 }}`, - expectQuery: storagecommon.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + expectQuery: storagecommon.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ PageSize: bunpaginate.QueryDefaultPageSize, Options: storagecommon.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &before, Builder: query.Gte("balance[EUR]", float64(50)), Expand: make([]string, 0), }, + Column: "account", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }, }, } diff --git a/internal/api/v2/mocks_ledger_controller_test.go b/internal/api/v2/mocks_ledger_controller_test.go index d45b1434f..2daba3908 100644 --- a/internal/api/v2/mocks_ledger_controller_test.go +++ b/internal/api/v2/mocks_ledger_controller_test.go @@ -241,7 +241,7 @@ func (mr *LedgerControllerMockRecorder) GetTransaction(ctx, query any) *gomock.C } // GetVolumesWithBalances mocks base method. -func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q common.OffsetPaginatedQuery[ledger0.GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { +func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q common.PaginatedQuery[ledger0.GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetVolumesWithBalances", ctx, q) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount]) @@ -285,7 +285,7 @@ func (mr *LedgerControllerMockRecorder) IsDatabaseUpToDate(ctx any) *gomock.Call } // ListAccounts mocks base method. -func (m *LedgerController) ListAccounts(ctx context.Context, query common.OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { +func (m *LedgerController) ListAccounts(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListAccounts", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Account]) @@ -300,7 +300,7 @@ func (mr *LedgerControllerMockRecorder) ListAccounts(ctx, query any) *gomock.Cal } // ListLogs mocks base method. -func (m *LedgerController) ListLogs(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { +func (m *LedgerController) ListLogs(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListLogs", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Log]) @@ -315,7 +315,7 @@ func (mr *LedgerControllerMockRecorder) ListLogs(ctx, query any) *gomock.Call { } // ListTransactions mocks base method. -func (m *LedgerController) ListTransactions(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { +func (m *LedgerController) ListTransactions(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListTransactions", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Transaction]) diff --git a/internal/api/v2/mocks_system_controller_test.go b/internal/api/v2/mocks_system_controller_test.go index e5e596f0c..d9c538f08 100644 --- a/internal/api/v2/mocks_system_controller_test.go +++ b/internal/api/v2/mocks_system_controller_test.go @@ -101,7 +101,7 @@ func (mr *SystemControllerMockRecorder) GetLedgerController(ctx, name any) *gomo } // ListLedgers mocks base method. -func (m *SystemController) ListLedgers(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) { +func (m *SystemController) ListLedgers(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListLedgers", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Ledger]) diff --git a/internal/controller/ledger/controller.go b/internal/controller/ledger/controller.go index e11edfbb0..87ac954cc 100644 --- a/internal/controller/ledger/controller.go +++ b/internal/controller/ledger/controller.go @@ -29,13 +29,13 @@ type Controller interface { GetStats(ctx context.Context) (Stats, error) GetAccount(ctx context.Context, query common.ResourceQuery[any]) (*ledger.Account, error) - ListAccounts(ctx context.Context, query common.OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) + ListAccounts(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) CountAccounts(ctx context.Context, query common.ResourceQuery[any]) (int, error) - ListLogs(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) + ListLogs(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) CountTransactions(ctx context.Context, query common.ResourceQuery[any]) (int, error) - ListTransactions(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) + ListTransactions(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) GetTransaction(ctx context.Context, query common.ResourceQuery[any]) (*ledger.Transaction, error) - GetVolumesWithBalances(ctx context.Context, q common.OffsetPaginatedQuery[GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) + GetVolumesWithBalances(ctx context.Context, q common.PaginatedQuery[GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) GetAggregatedBalances(ctx context.Context, q common.ResourceQuery[GetAggregatedVolumesOptions]) (ledger.BalancesByAssets, error) // CreateTransaction accept a numscript script and returns a transaction diff --git a/internal/controller/ledger/controller_default.go b/internal/controller/ledger/controller_default.go index 19f414498..525534d45 100644 --- a/internal/controller/ledger/controller_default.go +++ b/internal/controller/ledger/controller_default.go @@ -7,7 +7,7 @@ import ( "math/big" "reflect" - "github.com/formancehq/ledger/internal/storage/common" + storagecommon "github.com/formancehq/ledger/internal/storage/common" "github.com/formancehq/go-libs/v3/pointer" "github.com/formancehq/go-libs/v3/time" @@ -142,31 +142,31 @@ func (ctrl *DefaultController) GetMigrationsInfo(ctx context.Context) ([]migrati return ctrl.store.GetMigrationsInfo(ctx) } -func (ctrl *DefaultController) ListTransactions(ctx context.Context, q common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { +func (ctrl *DefaultController) ListTransactions(ctx context.Context, q storagecommon.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { return ctrl.store.Transactions().Paginate(ctx, q) } -func (ctrl *DefaultController) CountTransactions(ctx context.Context, q common.ResourceQuery[any]) (int, error) { +func (ctrl *DefaultController) CountTransactions(ctx context.Context, q storagecommon.ResourceQuery[any]) (int, error) { return ctrl.store.Transactions().Count(ctx, q) } -func (ctrl *DefaultController) GetTransaction(ctx context.Context, q common.ResourceQuery[any]) (*ledger.Transaction, error) { +func (ctrl *DefaultController) GetTransaction(ctx context.Context, q storagecommon.ResourceQuery[any]) (*ledger.Transaction, error) { return ctrl.store.Transactions().GetOne(ctx, q) } -func (ctrl *DefaultController) CountAccounts(ctx context.Context, q common.ResourceQuery[any]) (int, error) { +func (ctrl *DefaultController) CountAccounts(ctx context.Context, q storagecommon.ResourceQuery[any]) (int, error) { return ctrl.store.Accounts().Count(ctx, q) } -func (ctrl *DefaultController) ListAccounts(ctx context.Context, q common.OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { +func (ctrl *DefaultController) ListAccounts(ctx context.Context, q storagecommon.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { return ctrl.store.Accounts().Paginate(ctx, q) } -func (ctrl *DefaultController) GetAccount(ctx context.Context, q common.ResourceQuery[any]) (*ledger.Account, error) { +func (ctrl *DefaultController) GetAccount(ctx context.Context, q storagecommon.ResourceQuery[any]) (*ledger.Account, error) { return ctrl.store.Accounts().GetOne(ctx, q) } -func (ctrl *DefaultController) GetAggregatedBalances(ctx context.Context, q common.ResourceQuery[GetAggregatedVolumesOptions]) (ledger.BalancesByAssets, error) { +func (ctrl *DefaultController) GetAggregatedBalances(ctx context.Context, q storagecommon.ResourceQuery[GetAggregatedVolumesOptions]) (ledger.BalancesByAssets, error) { ret, err := ctrl.store.AggregatedBalances().GetOne(ctx, q) if err != nil { return nil, err @@ -174,11 +174,11 @@ func (ctrl *DefaultController) GetAggregatedBalances(ctx context.Context, q comm return ret.Aggregated.Balances(), nil } -func (ctrl *DefaultController) ListLogs(ctx context.Context, q common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { +func (ctrl *DefaultController) ListLogs(ctx context.Context, q storagecommon.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { return ctrl.store.Logs().Paginate(ctx, q) } -func (ctrl *DefaultController) GetVolumesWithBalances(ctx context.Context, q common.OffsetPaginatedQuery[GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { +func (ctrl *DefaultController) GetVolumesWithBalances(ctx context.Context, q storagecommon.PaginatedQuery[GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { return ctrl.store.Volumes().Paginate(ctx, q) } @@ -187,8 +187,10 @@ func (ctrl *DefaultController) Import(ctx context.Context, stream chan ledger.Lo var lastLogID *uint64 // We can import only if the ledger is empty. - logs, err := ctrl.store.Logs().Paginate(ctx, common.ColumnPaginatedQuery[any]{ + logs, err := ctrl.store.Logs().Paginate(ctx, storagecommon.InitialPaginatedQuery[any]{ PageSize: 1, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }) if err != nil { return fmt.Errorf("error listing logs: %w", err) @@ -314,15 +316,14 @@ func (ctrl *DefaultController) importLog(ctx context.Context, store Store, log l } func (ctrl *DefaultController) Export(ctx context.Context, w ExportWriter) error { - return bunpaginate.Iterate( + return storagecommon.Iterate( ctx, - common.ColumnPaginatedQuery[any]{ + storagecommon.InitialPaginatedQuery[any]{ PageSize: 100, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), + Column: "id", }, - func(ctx context.Context, q common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { - return ctrl.store.Logs().Paginate(ctx, q) - }, + ctrl.store.Logs().Paginate, func(cursor *bunpaginate.Cursor[ledger.Log]) error { for _, data := range cursor.Data { if err := w.Write(ctx, data); err != nil { diff --git a/internal/controller/ledger/controller_default_test.go b/internal/controller/ledger/controller_default_test.go index c86f60333..08d368a31 100644 --- a/internal/controller/ledger/controller_default_test.go +++ b/internal/controller/ledger/controller_default_test.go @@ -231,12 +231,12 @@ func TestListTransactions(t *testing.T) { interpreterParser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() - transactions := NewMockPaginatedResource[ledger.Transaction, any, common.ColumnPaginatedQuery[any]](ctrl) + transactions := NewMockPaginatedResource[ledger.Transaction, any](ctrl) cursor := &bunpaginate.Cursor[ledger.Transaction]{} store.EXPECT().Transactions().Return(transactions) transactions.EXPECT(). - Paginate(gomock.Any(), common.ColumnPaginatedQuery[any]{ + Paginate(gomock.Any(), common.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Column: "id", @@ -244,7 +244,7 @@ func TestListTransactions(t *testing.T) { Return(cursor, nil) l := NewDefaultController(ledger.Ledger{}, store, parser, machineParser, interpreterParser) - ret, err := l.ListTransactions(ctx, common.ColumnPaginatedQuery[any]{ + ret, err := l.ListTransactions(ctx, common.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Column: "id", @@ -262,7 +262,7 @@ func TestCountAccounts(t *testing.T) { machineParser := NewMockNumscriptParser(ctrl) interpreterParser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() - accounts := NewMockPaginatedResource[ledger.Account, any, common.OffsetPaginatedQuery[any]](ctrl) + accounts := NewMockPaginatedResource[ledger.Account, any](ctrl) store.EXPECT().Accounts().Return(accounts) accounts.EXPECT().Count(gomock.Any(), common.ResourceQuery[any]{}).Return(1, nil) @@ -282,7 +282,7 @@ func TestGetTransaction(t *testing.T) { machineParser := NewMockNumscriptParser(ctrl) interpreterParser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() - transactions := NewMockPaginatedResource[ledger.Transaction, any, common.ColumnPaginatedQuery[any]](ctrl) + transactions := NewMockPaginatedResource[ledger.Transaction, any](ctrl) tx := ledger.Transaction{} store.EXPECT().Transactions().Return(transactions) @@ -307,7 +307,7 @@ func TestGetAccount(t *testing.T) { machineParser := NewMockNumscriptParser(ctrl) interpreterParser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() - accounts := NewMockPaginatedResource[ledger.Account, any, common.OffsetPaginatedQuery[any]](ctrl) + accounts := NewMockPaginatedResource[ledger.Account, any](ctrl) account := ledger.Account{} store.EXPECT().Accounts().Return(accounts) @@ -332,7 +332,7 @@ func TestCountTransactions(t *testing.T) { machineParser := NewMockNumscriptParser(ctrl) interpreterParser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() - transactions := NewMockPaginatedResource[ledger.Transaction, any, common.ColumnPaginatedQuery[any]](ctrl) + transactions := NewMockPaginatedResource[ledger.Transaction, any](ctrl) store.EXPECT().Transactions().Return(transactions) transactions.EXPECT().Count(gomock.Any(), common.ResourceQuery[any]{}).Return(1, nil) @@ -352,17 +352,17 @@ func TestListAccounts(t *testing.T) { machineParser := NewMockNumscriptParser(ctrl) interpreterParser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() - accounts := NewMockPaginatedResource[ledger.Account, any, common.OffsetPaginatedQuery[any]](ctrl) + accounts := NewMockPaginatedResource[ledger.Account, any](ctrl) cursor := &bunpaginate.Cursor[ledger.Account]{} store.EXPECT().Accounts().Return(accounts) - accounts.EXPECT().Paginate(gomock.Any(), common.OffsetPaginatedQuery[any]{ + accounts.EXPECT().Paginate(gomock.Any(), common.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }).Return(cursor, nil) l := NewDefaultController(ledger.Ledger{}, store, parser, machineParser, interpreterParser) - ret, err := l.ListAccounts(ctx, common.OffsetPaginatedQuery[any]{ + ret, err := l.ListAccounts(ctx, common.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) @@ -400,18 +400,18 @@ func TestListLogs(t *testing.T) { machineParser := NewMockNumscriptParser(ctrl) interpreterParser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() - logs := NewMockPaginatedResource[ledger.Log, any, common.ColumnPaginatedQuery[any]](ctrl) + logs := NewMockPaginatedResource[ledger.Log, any](ctrl) cursor := &bunpaginate.Cursor[ledger.Log]{} store.EXPECT().Logs().Return(logs) - logs.EXPECT().Paginate(gomock.Any(), common.ColumnPaginatedQuery[any]{ + logs.EXPECT().Paginate(gomock.Any(), common.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Column: "id", }).Return(cursor, nil) l := NewDefaultController(ledger.Ledger{}, store, parser, machineParser, interpreterParser) - ret, err := l.ListLogs(ctx, common.ColumnPaginatedQuery[any]{ + ret, err := l.ListLogs(ctx, common.InitialPaginatedQuery[any]{ PageSize: bunpaginate.QueryDefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), Column: "id", @@ -429,17 +429,17 @@ func TestGetVolumesWithBalances(t *testing.T) { machineParser := NewMockNumscriptParser(ctrl) interpreterParser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() - volumes := NewMockPaginatedResource[ledger.VolumesWithBalanceByAssetByAccount, GetVolumesOptions, common.OffsetPaginatedQuery[GetVolumesOptions]](ctrl) + volumes := NewMockPaginatedResource[ledger.VolumesWithBalanceByAssetByAccount, GetVolumesOptions](ctrl) balancesByAssets := &bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount]{} store.EXPECT().Volumes().Return(volumes) - volumes.EXPECT().Paginate(gomock.Any(), common.OffsetPaginatedQuery[GetVolumesOptions]{ + volumes.EXPECT().Paginate(gomock.Any(), common.InitialPaginatedQuery[GetVolumesOptions]{ PageSize: bunpaginate.QueryDefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }).Return(balancesByAssets, nil) l := NewDefaultController(ledger.Ledger{}, store, parser, machineParser, interpreterParser) - ret, err := l.GetVolumesWithBalances(ctx, common.OffsetPaginatedQuery[GetVolumesOptions]{ + ret, err := l.GetVolumesWithBalances(ctx, common.InitialPaginatedQuery[GetVolumesOptions]{ PageSize: bunpaginate.QueryDefaultPageSize, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) diff --git a/internal/controller/ledger/controller_generated_test.go b/internal/controller/ledger/controller_generated_test.go index 823b27998..17d042d27 100644 --- a/internal/controller/ledger/controller_generated_test.go +++ b/internal/controller/ledger/controller_generated_test.go @@ -240,7 +240,7 @@ func (mr *MockControllerMockRecorder) GetTransaction(ctx, query any) *gomock.Cal } // GetVolumesWithBalances mocks base method. -func (m *MockController) GetVolumesWithBalances(ctx context.Context, q common.OffsetPaginatedQuery[GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { +func (m *MockController) GetVolumesWithBalances(ctx context.Context, q common.PaginatedQuery[GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetVolumesWithBalances", ctx, q) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount]) @@ -284,7 +284,7 @@ func (mr *MockControllerMockRecorder) IsDatabaseUpToDate(ctx any) *gomock.Call { } // ListAccounts mocks base method. -func (m *MockController) ListAccounts(ctx context.Context, query common.OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { +func (m *MockController) ListAccounts(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListAccounts", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Account]) @@ -299,7 +299,7 @@ func (mr *MockControllerMockRecorder) ListAccounts(ctx, query any) *gomock.Call } // ListLogs mocks base method. -func (m *MockController) ListLogs(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { +func (m *MockController) ListLogs(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListLogs", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Log]) @@ -314,7 +314,7 @@ func (mr *MockControllerMockRecorder) ListLogs(ctx, query any) *gomock.Call { } // ListTransactions mocks base method. -func (m *MockController) ListTransactions(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { +func (m *MockController) ListTransactions(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListTransactions", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Transaction]) diff --git a/internal/controller/ledger/controller_with_traces.go b/internal/controller/ledger/controller_with_traces.go index 42d2b6549..07cafe585 100644 --- a/internal/controller/ledger/controller_with_traces.go +++ b/internal/controller/ledger/controller_with_traces.go @@ -211,7 +211,7 @@ func (c *ControllerWithTraces) GetMigrationsInfo(ctx context.Context) ([]migrati ) } -func (c *ControllerWithTraces) ListTransactions(ctx context.Context, q common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { +func (c *ControllerWithTraces) ListTransactions(ctx context.Context, q common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { return tracing.TraceWithMetric( ctx, "ListTransactions", @@ -259,7 +259,7 @@ func (c *ControllerWithTraces) CountAccounts(ctx context.Context, a common.Resou ) } -func (c *ControllerWithTraces) ListAccounts(ctx context.Context, a common.OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { +func (c *ControllerWithTraces) ListAccounts(ctx context.Context, a common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { return tracing.TraceWithMetric( ctx, "ListAccounts", @@ -295,7 +295,7 @@ func (c *ControllerWithTraces) GetAggregatedBalances(ctx context.Context, q comm ) } -func (c *ControllerWithTraces) ListLogs(ctx context.Context, q common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { +func (c *ControllerWithTraces) ListLogs(ctx context.Context, q common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { return tracing.TraceWithMetric( ctx, "ListLogs", @@ -343,7 +343,7 @@ func (c *ControllerWithTraces) IsDatabaseUpToDate(ctx context.Context) (bool, er ) } -func (c *ControllerWithTraces) GetVolumesWithBalances(ctx context.Context, q common.OffsetPaginatedQuery[GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { +func (c *ControllerWithTraces) GetVolumesWithBalances(ctx context.Context, q common.PaginatedQuery[GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { return tracing.TraceWithMetric( ctx, "GetVolumesWithBalances", diff --git a/internal/controller/ledger/mocks_test.go b/internal/controller/ledger/mocks_test.go index 348a878fc..52222cc7b 100644 --- a/internal/controller/ledger/mocks_test.go +++ b/internal/controller/ledger/mocks_test.go @@ -209,32 +209,68 @@ func (mr *MockResourceMockRecorder[ResourceType, OptionsType]) GetOne(ctx, query return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOne", reflect.TypeOf((*MockResource[ResourceType, OptionsType])(nil).GetOne), ctx, query) } +// MockPaginatedQuery is a mock of PaginatedQuery interface. +type MockPaginatedQuery[OptionsType any] struct { + ctrl *gomock.Controller + recorder *MockPaginatedQueryMockRecorder[OptionsType] + isgomock struct{} +} + +// MockPaginatedQueryMockRecorder is the mock recorder for MockPaginatedQuery. +type MockPaginatedQueryMockRecorder[OptionsType any] struct { + mock *MockPaginatedQuery[OptionsType] +} + +// NewMockPaginatedQuery creates a new mock instance. +func NewMockPaginatedQuery[OptionsType any](ctrl *gomock.Controller) *MockPaginatedQuery[OptionsType] { + mock := &MockPaginatedQuery[OptionsType]{ctrl: ctrl} + mock.recorder = &MockPaginatedQueryMockRecorder[OptionsType]{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPaginatedQuery[OptionsType]) EXPECT() *MockPaginatedQueryMockRecorder[OptionsType] { + return m.recorder +} + +// isPaginatedQuery mocks base method. +func (m *MockPaginatedQuery[OptionsType]) isPaginatedQuery() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "isPaginatedQuery") +} + +// isPaginatedQuery indicates an expected call of isPaginatedQuery. +func (mr *MockPaginatedQueryMockRecorder[OptionsType]) isPaginatedQuery() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "isPaginatedQuery", reflect.TypeOf((*MockPaginatedQuery[OptionsType])(nil).isPaginatedQuery)) +} + // MockPaginatedResource is a mock of PaginatedResource interface. -type MockPaginatedResource[ResourceType any, OptionsType any, PaginationQueryType common.PaginatedQuery[OptionsType]] struct { +type MockPaginatedResource[ResourceType any, OptionsType any] struct { ctrl *gomock.Controller - recorder *MockPaginatedResourceMockRecorder[ResourceType, OptionsType, PaginationQueryType] + recorder *MockPaginatedResourceMockRecorder[ResourceType, OptionsType] isgomock struct{} } // MockPaginatedResourceMockRecorder is the mock recorder for MockPaginatedResource. -type MockPaginatedResourceMockRecorder[ResourceType any, OptionsType any, PaginationQueryType common.PaginatedQuery[OptionsType]] struct { - mock *MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType] +type MockPaginatedResourceMockRecorder[ResourceType any, OptionsType any] struct { + mock *MockPaginatedResource[ResourceType, OptionsType] } // NewMockPaginatedResource creates a new mock instance. -func NewMockPaginatedResource[ResourceType any, OptionsType any, PaginationQueryType common.PaginatedQuery[OptionsType]](ctrl *gomock.Controller) *MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType] { - mock := &MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType]{ctrl: ctrl} - mock.recorder = &MockPaginatedResourceMockRecorder[ResourceType, OptionsType, PaginationQueryType]{mock} +func NewMockPaginatedResource[ResourceType any, OptionsType any](ctrl *gomock.Controller) *MockPaginatedResource[ResourceType, OptionsType] { + mock := &MockPaginatedResource[ResourceType, OptionsType]{ctrl: ctrl} + mock.recorder = &MockPaginatedResourceMockRecorder[ResourceType, OptionsType]{mock} return mock } // EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType]) EXPECT() *MockPaginatedResourceMockRecorder[ResourceType, OptionsType, PaginationQueryType] { +func (m *MockPaginatedResource[ResourceType, OptionsType]) EXPECT() *MockPaginatedResourceMockRecorder[ResourceType, OptionsType] { return m.recorder } // Count mocks base method. -func (m *MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType]) Count(ctx context.Context, query common.ResourceQuery[OptionsType]) (int, error) { +func (m *MockPaginatedResource[ResourceType, OptionsType]) Count(ctx context.Context, query common.ResourceQuery[OptionsType]) (int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Count", ctx, query) ret0, _ := ret[0].(int) @@ -243,13 +279,13 @@ func (m *MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType]) } // Count indicates an expected call of Count. -func (mr *MockPaginatedResourceMockRecorder[ResourceType, OptionsType, PaginationQueryType]) Count(ctx, query any) *gomock.Call { +func (mr *MockPaginatedResourceMockRecorder[ResourceType, OptionsType]) Count(ctx, query any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Count", reflect.TypeOf((*MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType])(nil).Count), ctx, query) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Count", reflect.TypeOf((*MockPaginatedResource[ResourceType, OptionsType])(nil).Count), ctx, query) } // GetOne mocks base method. -func (m *MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType]) GetOne(ctx context.Context, query common.ResourceQuery[OptionsType]) (*ResourceType, error) { +func (m *MockPaginatedResource[ResourceType, OptionsType]) GetOne(ctx context.Context, query common.ResourceQuery[OptionsType]) (*ResourceType, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetOne", ctx, query) ret0, _ := ret[0].(*ResourceType) @@ -258,13 +294,13 @@ func (m *MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType]) } // GetOne indicates an expected call of GetOne. -func (mr *MockPaginatedResourceMockRecorder[ResourceType, OptionsType, PaginationQueryType]) GetOne(ctx, query any) *gomock.Call { +func (mr *MockPaginatedResourceMockRecorder[ResourceType, OptionsType]) GetOne(ctx, query any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOne", reflect.TypeOf((*MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType])(nil).GetOne), ctx, query) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOne", reflect.TypeOf((*MockPaginatedResource[ResourceType, OptionsType])(nil).GetOne), ctx, query) } // Paginate mocks base method. -func (m *MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType]) Paginate(ctx context.Context, paginationOptions PaginationQueryType) (*bunpaginate.Cursor[ResourceType], error) { +func (m *MockPaginatedResource[ResourceType, OptionsType]) Paginate(ctx context.Context, paginationOptions common.PaginatedQuery[OptionsType]) (*bunpaginate.Cursor[ResourceType], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Paginate", ctx, paginationOptions) ret0, _ := ret[0].(*bunpaginate.Cursor[ResourceType]) @@ -273,73 +309,7 @@ func (m *MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType]) } // Paginate indicates an expected call of Paginate. -func (mr *MockPaginatedResourceMockRecorder[ResourceType, OptionsType, PaginationQueryType]) Paginate(ctx, paginationOptions any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Paginate", reflect.TypeOf((*MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType])(nil).Paginate), ctx, paginationOptions) -} - -// MockFieldType is a mock of FieldType interface. -type MockFieldType struct { - ctrl *gomock.Controller - recorder *MockFieldTypeMockRecorder - isgomock struct{} -} - -// MockFieldTypeMockRecorder is the mock recorder for MockFieldType. -type MockFieldTypeMockRecorder struct { - mock *MockFieldType -} - -// NewMockFieldType creates a new mock instance. -func NewMockFieldType(ctrl *gomock.Controller) *MockFieldType { - mock := &MockFieldType{ctrl: ctrl} - mock.recorder = &MockFieldTypeMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockFieldType) EXPECT() *MockFieldTypeMockRecorder { - return m.recorder -} - -// IsIndexable mocks base method. -func (m *MockFieldType) IsIndexable() bool { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsIndexable") - ret0, _ := ret[0].(bool) - return ret0 -} - -// IsIndexable indicates an expected call of IsIndexable. -func (mr *MockFieldTypeMockRecorder) IsIndexable() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsIndexable", reflect.TypeOf((*MockFieldType)(nil).IsIndexable)) -} - -// Operators mocks base method. -func (m *MockFieldType) Operators() []string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Operators") - ret0, _ := ret[0].([]string) - return ret0 -} - -// Operators indicates an expected call of Operators. -func (mr *MockFieldTypeMockRecorder) Operators() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Operators", reflect.TypeOf((*MockFieldType)(nil).Operators)) -} - -// ValidateValue mocks base method. -func (m *MockFieldType) ValidateValue(value any) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ValidateValue", value) - ret0, _ := ret[0].(error) - return ret0 -} - -// ValidateValue indicates an expected call of ValidateValue. -func (mr *MockFieldTypeMockRecorder) ValidateValue(value any) *gomock.Call { +func (mr *MockPaginatedResourceMockRecorder[ResourceType, OptionsType]) Paginate(ctx, paginationOptions any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateValue", reflect.TypeOf((*MockFieldType)(nil).ValidateValue), value) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Paginate", reflect.TypeOf((*MockPaginatedResource[ResourceType, OptionsType])(nil).Paginate), ctx, paginationOptions) } diff --git a/internal/controller/ledger/stats_test.go b/internal/controller/ledger/stats_test.go index 1df5824bd..70cd7ae4a 100644 --- a/internal/controller/ledger/stats_test.go +++ b/internal/controller/ledger/stats_test.go @@ -19,8 +19,8 @@ func TestStats(t *testing.T) { parser := NewMockNumscriptParser(ctrl) machineParser := NewMockNumscriptParser(ctrl) interpreterParser := NewMockNumscriptParser(ctrl) - transactions := NewMockPaginatedResource[ledger.Transaction, any, common.ColumnPaginatedQuery[any]](ctrl) - accounts := NewMockPaginatedResource[ledger.Account, any, common.OffsetPaginatedQuery[any]](ctrl) + transactions := NewMockPaginatedResource[ledger.Transaction, any](ctrl) + accounts := NewMockPaginatedResource[ledger.Account, any](ctrl) store.EXPECT().Transactions().Return(transactions) transactions.EXPECT().Count(ctx, common.ResourceQuery[any]{}).Return(10, nil) diff --git a/internal/controller/ledger/store.go b/internal/controller/ledger/store.go index c7a061c05..de9900039 100644 --- a/internal/controller/ledger/store.go +++ b/internal/controller/ledger/store.go @@ -58,11 +58,11 @@ type Store interface { IsUpToDate(ctx context.Context) (bool, error) GetMigrationsInfo(ctx context.Context) ([]migrations.Info, error) - Accounts() common.PaginatedResource[ledger.Account, any, common.OffsetPaginatedQuery[any]] - Logs() common.PaginatedResource[ledger.Log, any, common.ColumnPaginatedQuery[any]] - Transactions() common.PaginatedResource[ledger.Transaction, any, common.ColumnPaginatedQuery[any]] + Accounts() common.PaginatedResource[ledger.Account, any] + Logs() common.PaginatedResource[ledger.Log, any] + Transactions() common.PaginatedResource[ledger.Transaction, any] AggregatedBalances() common.Resource[ledger.AggregatedVolumes, GetAggregatedVolumesOptions] - Volumes() common.PaginatedResource[ledger.VolumesWithBalanceByAssetByAccount, GetVolumesOptions, common.OffsetPaginatedQuery[GetVolumesOptions]] + Volumes() common.PaginatedResource[ledger.VolumesWithBalanceByAssetByAccount, GetVolumesOptions] } type vmStoreAdapter struct { @@ -87,8 +87,8 @@ func newVmStoreAdapter(tx Store) *vmStoreAdapter { } } -func NewListLedgersQuery(pageSize uint64) common.ColumnPaginatedQuery[any] { - return common.ColumnPaginatedQuery[any]{ +func NewListLedgersQuery(pageSize uint64) common.InitialPaginatedQuery[any] { + return common.InitialPaginatedQuery[any]{ PageSize: pageSize, Column: "id", Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), diff --git a/internal/controller/ledger/store_generated_test.go b/internal/controller/ledger/store_generated_test.go index 46e05ffa0..f8508d2eb 100644 --- a/internal/controller/ledger/store_generated_test.go +++ b/internal/controller/ledger/store_generated_test.go @@ -46,10 +46,10 @@ func (m *MockStore) EXPECT() *MockStoreMockRecorder { } // Accounts mocks base method. -func (m *MockStore) Accounts() common.PaginatedResource[ledger.Account, any, common.OffsetPaginatedQuery[any]] { +func (m *MockStore) Accounts() common.PaginatedResource[ledger.Account, any] { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Accounts") - ret0, _ := ret[0].(common.PaginatedResource[ledger.Account, any, common.OffsetPaginatedQuery[any]]) + ret0, _ := ret[0].(common.PaginatedResource[ledger.Account, any]) return ret0 } @@ -238,10 +238,10 @@ func (mr *MockStoreMockRecorder) LockLedger(ctx any) *gomock.Call { } // Logs mocks base method. -func (m *MockStore) Logs() common.PaginatedResource[ledger.Log, any, common.ColumnPaginatedQuery[any]] { +func (m *MockStore) Logs() common.PaginatedResource[ledger.Log, any] { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Logs") - ret0, _ := ret[0].(common.PaginatedResource[ledger.Log, any, common.ColumnPaginatedQuery[any]]) + ret0, _ := ret[0].(common.PaginatedResource[ledger.Log, any]) return ret0 } @@ -297,10 +297,10 @@ func (mr *MockStoreMockRecorder) Rollback() *gomock.Call { } // Transactions mocks base method. -func (m *MockStore) Transactions() common.PaginatedResource[ledger.Transaction, any, common.ColumnPaginatedQuery[any]] { +func (m *MockStore) Transactions() common.PaginatedResource[ledger.Transaction, any] { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Transactions") - ret0, _ := ret[0].(common.PaginatedResource[ledger.Transaction, any, common.ColumnPaginatedQuery[any]]) + ret0, _ := ret[0].(common.PaginatedResource[ledger.Transaction, any]) return ret0 } @@ -360,10 +360,10 @@ func (mr *MockStoreMockRecorder) UpsertAccounts(ctx any, accounts ...any) *gomoc } // Volumes mocks base method. -func (m *MockStore) Volumes() common.PaginatedResource[ledger.VolumesWithBalanceByAssetByAccount, GetVolumesOptions, common.OffsetPaginatedQuery[GetVolumesOptions]] { +func (m *MockStore) Volumes() common.PaginatedResource[ledger.VolumesWithBalanceByAssetByAccount, GetVolumesOptions] { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Volumes") - ret0, _ := ret[0].(common.PaginatedResource[ledger.VolumesWithBalanceByAssetByAccount, GetVolumesOptions, common.OffsetPaginatedQuery[GetVolumesOptions]]) + ret0, _ := ret[0].(common.PaginatedResource[ledger.VolumesWithBalanceByAssetByAccount, GetVolumesOptions]) return ret0 } diff --git a/internal/controller/system/controller.go b/internal/controller/system/controller.go index 169e587b6..a5aa45c72 100644 --- a/internal/controller/system/controller.go +++ b/internal/controller/system/controller.go @@ -24,7 +24,7 @@ import ( type Controller interface { GetLedgerController(ctx context.Context, name string) (ledgercontroller.Controller, error) GetLedger(ctx context.Context, name string) (*ledger.Ledger, error) - ListLedgers(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) + ListLedgers(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) // CreateLedger can return following errors: // * ErrLedgerAlreadyExists // * ledger.ErrInvalidLedgerName @@ -131,7 +131,7 @@ func (ctrl *DefaultController) GetLedger(ctx context.Context, name string) (*led }) } -func (ctrl *DefaultController) ListLedgers(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) { +func (ctrl *DefaultController) ListLedgers(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) { return tracing.Trace(ctx, ctrl.tracerProvider.Tracer("system"), "ListLedgers", func(ctx context.Context) (*bunpaginate.Cursor[ledger.Ledger], error) { return ctrl.store.ListLedgers(ctx, query) }) diff --git a/internal/controller/system/store.go b/internal/controller/system/store.go index f3cd1181f..b167a65cf 100644 --- a/internal/controller/system/store.go +++ b/internal/controller/system/store.go @@ -13,7 +13,7 @@ import ( type Store interface { GetLedger(ctx context.Context, name string) (*ledger.Ledger, error) - ListLedgers(ctx context.Context, query common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) + ListLedgers(ctx context.Context, query common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) UpdateLedgerMetadata(ctx context.Context, name string, m metadata.Metadata) error DeleteLedgerMetadata(ctx context.Context, param string, key string) error OpenLedger(context.Context, string) (ledgercontroller.Store, *ledger.Ledger, error) diff --git a/internal/storage/bucket/migrations_test.go b/internal/storage/bucket/migrations_test.go index 51841e073..4aeac8d1b 100644 --- a/internal/storage/bucket/migrations_test.go +++ b/internal/storage/bucket/migrations_test.go @@ -3,7 +3,6 @@ package bucket_test import ( - "context" "errors" "fmt" "github.com/formancehq/go-libs/v3/bun/bunconnect" @@ -91,15 +90,14 @@ func TestMigrations(t *testing.T) { for i := 0; i < 5; i++ { store := ledgerstore.New(db, bucket.NewDefault(noop.Tracer{}, bucketName), ledgers[i]) - require.NoError(t, bunpaginate.Iterate( + require.NoError(t, common.Iterate( ctx, - common.ColumnPaginatedQuery[any]{ + common.InitialPaginatedQuery[any]{ PageSize: 100, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), + Column: "id", }, - func(ctx context.Context, q common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { - return store.Logs().Paginate(ctx, q) - }, + store.Logs().Paginate, func(cursor *bunpaginate.Cursor[ledger.Log]) error { return nil }, diff --git a/internal/storage/common/cursor.go b/internal/storage/common/cursor.go new file mode 100644 index 000000000..eabb40419 --- /dev/null +++ b/internal/storage/common/cursor.go @@ -0,0 +1,101 @@ +package common + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "github.com/formancehq/go-libs/v3/bun/bunpaginate" + "net/http" + "reflect" +) + +// todo: backport in go libs +func Extract[OF any]( + r *http.Request, + defaulter func() (*InitialPaginatedQuery[OF], error), + modifiers ...func(query *InitialPaginatedQuery[OF]) error, +) (PaginatedQuery[OF], error) { + if r.URL.Query().Get(bunpaginate.QueryKeyCursor) != "" { + return unmarshalCursor[OF](r.URL.Query().Get(bunpaginate.QueryKeyCursor), modifiers...) + } else { + initialQuery, err := defaulter() + if err != nil { + return nil, fmt.Errorf("extracting paginated query: %w", err) + } + return *initialQuery, nil + } +} + +func unmarshalCursor[OF any](v string, modifiers ...func(query *InitialPaginatedQuery[OF]) error) (PaginatedQuery[OF], error) { + res, err := base64.RawURLEncoding.DecodeString(v) + if err != nil { + return nil, err + } + + // todo: we should better rely on schema to determine the type of cursor + type aux struct { + Offset *uint64 `json:"offset"` + } + x := aux{} + if err := json.Unmarshal(res, &x); err != nil { + return nil, fmt.Errorf("invalid cursor: %w", err) + } + + var q PaginatedQuery[OF] + if x.Offset != nil { // Offset defined, this is an offset cursor + q = &OffsetPaginatedQuery[OF]{} + } else { + q = &ColumnPaginatedQuery[OF]{} + } + + if err := json.Unmarshal(res, &q); err != nil { + return nil, err + } + + var root *InitialPaginatedQuery[OF] + if x.Offset != nil { // Offset defined, this is an offset cursor + root = &q.(*OffsetPaginatedQuery[OF]).InitialPaginatedQuery + } else { + root = &q.(*ColumnPaginatedQuery[OF]).InitialPaginatedQuery + } + + for _, modifier := range modifiers { + if err := modifier(root); err != nil { + return nil, err + } + } + + return reflect.ValueOf(q).Elem().Interface().(PaginatedQuery[OF]), nil +} + +func Iterate[OF any, Options any]( + ctx context.Context, + initialQuery InitialPaginatedQuery[Options], + iterator func(ctx context.Context, q PaginatedQuery[Options]) (*bunpaginate.Cursor[OF], error), + cb func(cursor *bunpaginate.Cursor[OF]) error, +) error { + + var query PaginatedQuery[OF] = initialQuery + for { + cursor, err := iterator(ctx, query) + if err != nil { + return err + } + + if err := cb(cursor); err != nil { + return err + } + + if !cursor.HasMore { + break + } + + query, err = unmarshalCursor[OF](cursor.Next) + if err != nil { + return fmt.Errorf("paginating next request: %w", err) + } + } + + return nil +} diff --git a/internal/storage/common/paginator.go b/internal/storage/common/paginator.go index c8896c439..a869ea1c8 100644 --- a/internal/storage/common/paginator.go +++ b/internal/storage/common/paginator.go @@ -5,7 +5,7 @@ import ( "github.com/uptrace/bun" ) -type Paginator[ResourceType any, PaginationOptions any] interface { - Paginate(selectQuery *bun.SelectQuery, opts PaginationOptions) (*bun.SelectQuery, error) - BuildCursor(ret []ResourceType, opts PaginationOptions) (*bunpaginate.Cursor[ResourceType], error) +type Paginator[ResourceType any] interface { + Paginate(selectQuery *bun.SelectQuery) (*bun.SelectQuery, error) + BuildCursor(ret []ResourceType) (*bunpaginate.Cursor[ResourceType], error) } diff --git a/internal/storage/common/paginator_column.go b/internal/storage/common/paginator_column.go index 8dcf135be..cae90332d 100644 --- a/internal/storage/common/paginator_column.go +++ b/internal/storage/common/paginator_column.go @@ -5,51 +5,41 @@ import ( "github.com/formancehq/go-libs/v3/bun/bunpaginate" "github.com/formancehq/go-libs/v3/time" "github.com/uptrace/bun" - "github.com/uptrace/bun/schema" "math/big" "reflect" "strings" libtime "time" ) -type ColumnPaginator[ResourceType, OptionsType any] struct { - DefaultPaginationColumn string - DefaultOrder bunpaginate.Order - Table *schema.Table +type columnPaginator[ResourceType, OptionsType any] struct { + fieldType FieldType + fieldName string + query ColumnPaginatedQuery[OptionsType] } //nolint:unused -func (o ColumnPaginator[ResourceType, OptionsType]) Paginate(sb *bun.SelectQuery, query ColumnPaginatedQuery[OptionsType]) (*bun.SelectQuery, error) { +func (o columnPaginator[ResourceType, OptionsType]) Paginate(sb *bun.SelectQuery) (*bun.SelectQuery, error) { - paginationColumn := o.DefaultPaginationColumn - if query.Column != "" { - paginationColumn = query.Column - } - - originalOrder := o.DefaultOrder - if query.Order != nil { - originalOrder = *query.Order - } + paginationColumn := o.fieldName + originalOrder := *o.query.Order - pageSize := query.PageSize + pageSize := o.query.PageSize if pageSize == 0 { pageSize = bunpaginate.QueryDefaultPageSize } sb = sb.Limit(int(pageSize) + 1) // Fetch one additional item to find the next token + order := originalOrder - if query.Reverse { - order = originalOrder.Reverse() + if o.query.Reverse { + order = order.Reverse() } orderExpression := fmt.Sprintf("%s %s", paginationColumn, order) sb = sb.ColumnExpr("row_number() OVER (ORDER BY " + orderExpression + ")") - if query.PaginationID != nil { - paginationID := convertPaginationIDToSQLType( - o.Table.FieldMap[paginationColumn].DiscoveredSQLType, - query.PaginationID, - ) - if query.Reverse { + if o.query.PaginationID != nil { + paginationID := convertPaginationIDToSQLType(o.fieldType, o.query.PaginationID) + if o.query.Reverse { switch originalOrder { case bunpaginate.OrderAsc: sb = sb.Where(fmt.Sprintf("%s < ?", paginationColumn), paginationID) @@ -69,32 +59,17 @@ func (o ColumnPaginator[ResourceType, OptionsType]) Paginate(sb *bun.SelectQuery return sb, nil } -func convertPaginationIDToSQLType(sqlType string, id *big.Int) any { - switch sqlType { - case "timestamp without time zone", "timestamp": - return libtime.UnixMicro(id.Int64()) - default: - return id - } -} - //nolint:unused -func (o ColumnPaginator[ResourceType, OptionsType]) BuildCursor(ret []ResourceType, query ColumnPaginatedQuery[OptionsType]) (*bunpaginate.Cursor[ResourceType], error) { +func (o columnPaginator[ResourceType, OptionsType]) BuildCursor(ret []ResourceType) (*bunpaginate.Cursor[ResourceType], error) { - paginationColumn := query.Column - if paginationColumn == "" { - paginationColumn = o.DefaultPaginationColumn - } + paginationColumn := o.query.Column - pageSize := query.PageSize + pageSize := o.query.PageSize if pageSize == 0 { pageSize = bunpaginate.QueryDefaultPageSize } - order := o.DefaultOrder - if query.Order != nil { - order = *query.Order - } + order := *o.query.Order var v ResourceType fields := findPaginationFieldPath(v, paginationColumn) @@ -104,8 +79,8 @@ func (o ColumnPaginator[ResourceType, OptionsType]) BuildCursor(ret []ResourceTy ) for _, t := range ret { paginationID := findPaginationField(t, fields...) - if query.Bottom == nil { - query.Bottom = paginationID + if o.query.Bottom == nil { + o.query.Bottom = paginationID } paginationIDs = append(paginationIDs, paginationID) } @@ -114,7 +89,7 @@ func (o ColumnPaginator[ResourceType, OptionsType]) BuildCursor(ret []ResourceTy if hasMore { ret = ret[:len(ret)-1] } - if query.Reverse { + if o.query.Reverse { for i := 0; i < len(ret)/2; i++ { ret[i], ret[len(ret)-i-1] = ret[len(ret)-i-1], ret[i] } @@ -122,25 +97,26 @@ func (o ColumnPaginator[ResourceType, OptionsType]) BuildCursor(ret []ResourceTy var previous, next *ColumnPaginatedQuery[OptionsType] - if query.Reverse { - cp := query + if o.query.Reverse { + cp := o.query cp.Reverse = false next = &cp if hasMore { - cp := query + cp := o.query cp.PaginationID = paginationIDs[len(paginationIDs)-2] previous = &cp } } else { if hasMore { - cp := query + cp := o.query cp.PaginationID = paginationIDs[len(paginationIDs)-1] next = &cp } - if query.PaginationID != nil { - if (order == bunpaginate.OrderAsc && query.PaginationID.Cmp(query.Bottom) > 0) || (order == bunpaginate.OrderDesc && query.PaginationID.Cmp(query.Bottom) < 0) { - cp := query + if o.query.PaginationID != nil { + if (order == bunpaginate.OrderAsc && o.query.PaginationID.Cmp(o.query.Bottom) > 0) || + (order == bunpaginate.OrderDesc && o.query.PaginationID.Cmp(o.query.Bottom) < 0) { + cp := o.query cp.Reverse = true previous = &cp } @@ -156,7 +132,7 @@ func (o ColumnPaginator[ResourceType, OptionsType]) BuildCursor(ret []ResourceTy }, nil } -var _ Paginator[any, ColumnPaginatedQuery[any]] = &ColumnPaginator[any, any]{} +var _ Paginator[any] = &columnPaginator[any, any]{} //nolint:unused func findPaginationFieldPath(v any, paginationColumn string) []reflect.StructField { @@ -268,3 +244,24 @@ func encodeCursor[OptionsType any, PaginatedQueryType PaginatedQuery[OptionsType } return bunpaginate.EncodeCursor(v) } + +func newColumnPaginator[ResourceType, OptionsType any]( + query ColumnPaginatedQuery[OptionsType], + fieldName string, + fieldType FieldType, +) columnPaginator[ResourceType, OptionsType] { + return columnPaginator[ResourceType, OptionsType]{ + query: query, + fieldName: fieldName, + fieldType: fieldType, + } +} + +func convertPaginationIDToSQLType(fieldType FieldType, id *big.Int) any { + switch fieldType.(type) { + case TypeDate: + return libtime.UnixMicro(id.Int64()) + default: + return id + } +} diff --git a/internal/storage/common/paginator_offset.go b/internal/storage/common/paginator_offset.go index a574c5b09..09bb4a4d7 100644 --- a/internal/storage/common/paginator_offset.go +++ b/internal/storage/common/paginator_offset.go @@ -8,48 +8,41 @@ import ( ) type OffsetPaginator[ResourceType, OptionsType any] struct { - DefaultPaginationColumn string - DefaultOrder bunpaginate.Order + query OffsetPaginatedQuery[OptionsType] } //nolint:unused -func (o OffsetPaginator[ResourceType, OptionsType]) Paginate(sb *bun.SelectQuery, query OffsetPaginatedQuery[OptionsType]) (*bun.SelectQuery, error) { +func (o OffsetPaginator[ResourceType, OptionsType]) Paginate(sb *bun.SelectQuery) (*bun.SelectQuery, error) { - paginationColumn := o.DefaultPaginationColumn - if query.Column != "" { - paginationColumn = query.Column - } - originalOrder := o.DefaultOrder - if query.Order != nil { - originalOrder = *query.Order - } + paginationColumn := o.query.Column + originalOrder := *o.query.Order orderExpression := fmt.Sprintf("%s %s", paginationColumn, originalOrder) sb = sb.ColumnExpr("row_number() OVER (ORDER BY " + orderExpression + ")") - if query.Offset > math.MaxInt32 { + if o.query.Offset > math.MaxInt32 { return nil, fmt.Errorf("offset value exceeds maximum allowed value") } - if query.Offset > 0 { - sb = sb.Offset(int(query.Offset)) + if o.query.Offset > 0 { + sb = sb.Offset(int(o.query.Offset)) } - if query.PageSize > 0 { - sb = sb.Limit(int(query.PageSize) + 1) + if o.query.PageSize > 0 { + sb = sb.Limit(int(o.query.PageSize) + 1) } return sb, nil } //nolint:unused -func (o OffsetPaginator[ResourceType, OptionsType]) BuildCursor(ret []ResourceType, query OffsetPaginatedQuery[OptionsType]) (*bunpaginate.Cursor[ResourceType], error) { +func (o OffsetPaginator[ResourceType, OptionsType]) BuildCursor(ret []ResourceType) (*bunpaginate.Cursor[ResourceType], error) { var previous, next *OffsetPaginatedQuery[OptionsType] // Page with transactions before - if query.Offset > 0 { - cp := query - offset := int(query.Offset) - int(query.PageSize) + if o.query.Offset > 0 { + cp := o.query + offset := int(o.query.Offset) - int(o.query.PageSize) if offset < 0 { offset = 0 } @@ -58,19 +51,19 @@ func (o OffsetPaginator[ResourceType, OptionsType]) BuildCursor(ret []ResourceTy } // Page with transactions after - if query.PageSize != 0 && len(ret) > int(query.PageSize) { - cp := query + if o.query.PageSize != 0 && len(ret) > int(o.query.PageSize) { + cp := o.query // Check for potential overflow - if query.Offset > math.MaxUint64-query.PageSize { + if o.query.Offset > math.MaxUint64-o.query.PageSize { return nil, fmt.Errorf("offset overflow") } - cp.Offset = query.Offset + query.PageSize + cp.Offset = o.query.Offset + o.query.PageSize next = &cp ret = ret[:len(ret)-1] } return &bunpaginate.Cursor[ResourceType]{ - PageSize: int(query.PageSize), + PageSize: int(o.query.PageSize), HasMore: next != nil, Previous: encodeCursor[OptionsType, OffsetPaginatedQuery[OptionsType]](previous), Next: encodeCursor[OptionsType, OffsetPaginatedQuery[OptionsType]](next), @@ -78,4 +71,10 @@ func (o OffsetPaginator[ResourceType, OptionsType]) BuildCursor(ret []ResourceTy }, nil } -var _ Paginator[any, OffsetPaginatedQuery[any]] = &OffsetPaginator[any, any]{} +var _ Paginator[any] = &OffsetPaginator[any, any]{} + +func newOffsetPaginator[ResourceType, OptionsType any]( + query OffsetPaginatedQuery[OptionsType], +) OffsetPaginator[ResourceType, OptionsType] { + return OffsetPaginator[ResourceType, OptionsType]{query: query} +} \ No newline at end of file diff --git a/internal/storage/common/resource.go b/internal/storage/common/resource.go index 50d4b014f..aaaef9a7a 100644 --- a/internal/storage/common/resource.go +++ b/internal/storage/common/resource.go @@ -60,6 +60,16 @@ type EntitySchema struct { Fields map[string]Field } +func (s EntitySchema) GetFieldByNameOrAlias(name string) (string, *Field) { + for fieldName, field := range s.Fields { + if fieldName == name || slices.Contains(field.Aliases, name) { + return fieldName, &field + } + } + + return "", nil +} + type RepositoryHandlerBuildContext[Opts any] struct { ResourceQuery[Opts] filters map[string]any @@ -244,21 +254,51 @@ func NewResourceRepository[ResourceType, OptionsType any]( } } -type PaginatedResourceRepository[ResourceType, OptionsType any, PaginationQueryType PaginatedQuery[OptionsType]] struct { +type PaginatedResourceRepository[ResourceType, OptionsType any] struct { *ResourceRepository[ResourceType, OptionsType] - paginator Paginator[ResourceType, PaginationQueryType] } -func (r *PaginatedResourceRepository[ResourceType, OptionsType, PaginationQueryType]) Paginate( +func (r *PaginatedResourceRepository[ResourceType, OptionsType]) Paginate( ctx context.Context, - paginationOptions PaginationQueryType, + paginationQuery PaginatedQuery[OptionsType], ) (*bunpaginate.Cursor[ResourceType], error) { - var resourceQuery ResourceQuery[OptionsType] - switch v := any(paginationOptions).(type) { + switch v := any(paginationQuery).(type) { case OffsetPaginatedQuery[OptionsType]: + case ColumnPaginatedQuery[OptionsType]: + case InitialPaginatedQuery[OptionsType]: + _, field := r.resourceHandler.Schema().GetFieldByNameOrAlias(v.Column) + if field == nil { + return nil, fmt.Errorf("invalid property '%s' for pagination", v.Column) + } + + if field.Type.IsPaginated() { + paginationQuery = ColumnPaginatedQuery[OptionsType]{ + InitialPaginatedQuery: v, + } + } else { + paginationQuery = OffsetPaginatedQuery[OptionsType]{ + InitialPaginatedQuery: v, + } + } + default: + panic(fmt.Errorf("should not happen, got type when waiting for OffsetPaginatedQuery, ColumnPaginatedQuery, or InitialResourceQuery: %T", paginationQuery)) + } + + var ( + paginator Paginator[ResourceType] + resourceQuery ResourceQuery[OptionsType] + ) + switch v := any(paginationQuery).(type) { + case OffsetPaginatedQuery[OptionsType]: + paginator = newOffsetPaginator[ResourceType, OptionsType](v) resourceQuery = v.Options case ColumnPaginatedQuery[OptionsType]: + fieldName, field := r.resourceHandler.Schema().GetFieldByNameOrAlias(v.Column) + if field == nil { + return nil, fmt.Errorf("invalid property '%s' for pagination", v.Column) + } + paginator = newColumnPaginator[ResourceType, OptionsType](v, fieldName, field.Type) resourceQuery = v.Options default: panic("should not happen") @@ -269,7 +309,7 @@ func (r *PaginatedResourceRepository[ResourceType, OptionsType, PaginationQueryT return nil, fmt.Errorf("building filtered dataset: %w", err) } - finalQuery, err = r.paginator.Paginate(finalQuery, paginationOptions) + finalQuery, err = paginator.Paginate(finalQuery) if err != nil { return nil, fmt.Errorf("paginating request: %w", err) } @@ -286,30 +326,28 @@ func (r *PaginatedResourceRepository[ResourceType, OptionsType, PaginationQueryT return nil, fmt.Errorf("scanning results: %w", err) } - return r.paginator.BuildCursor(ret, paginationOptions) + return paginator.BuildCursor(ret) } -func NewPaginatedResourceRepository[ResourceType, OptionsType any, PaginationQueryType PaginatedQuery[OptionsType]]( +func NewPaginatedResourceRepository[ResourceType, OptionsType any]( handler RepositoryHandler[OptionsType], - paginator Paginator[ResourceType, PaginationQueryType], -) *PaginatedResourceRepository[ResourceType, OptionsType, PaginationQueryType] { - return &PaginatedResourceRepository[ResourceType, OptionsType, PaginationQueryType]{ +) *PaginatedResourceRepository[ResourceType, OptionsType] { + return &PaginatedResourceRepository[ResourceType, OptionsType]{ ResourceRepository: NewResourceRepository[ResourceType, OptionsType](handler), - paginator: paginator, } } type PaginatedResourceRepositoryMapper[ToResourceType any, OriginalResourceType interface { ToCore() ToResourceType -}, OptionsType any, PaginationQueryType PaginatedQuery[OptionsType]] struct { - *PaginatedResourceRepository[OriginalResourceType, OptionsType, PaginationQueryType] +}, OptionsType any] struct { + *PaginatedResourceRepository[OriginalResourceType, OptionsType] } -func (m PaginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, OptionsType, PaginationQueryType]) Paginate( +func (m PaginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, OptionsType]) Paginate( ctx context.Context, - paginationOptions PaginationQueryType, + paginatedQuery PaginatedQuery[OptionsType], ) (*bunpaginate.Cursor[ToResourceType], error) { - cursor, err := m.PaginatedResourceRepository.Paginate(ctx, paginationOptions) + cursor, err := m.PaginatedResourceRepository.Paginate(ctx, paginatedQuery) if err != nil { return nil, err } @@ -317,7 +355,7 @@ func (m PaginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, return bunpaginate.MapCursor(cursor, OriginalResourceType.ToCore), nil } -func (m PaginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, OptionsType, PaginationQueryType]) GetOne( +func (m PaginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, OptionsType]) GetOne( ctx context.Context, query ResourceQuery[OptionsType], ) (*ToResourceType, error) { @@ -331,12 +369,9 @@ func (m PaginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, func NewPaginatedResourceRepositoryMapper[ToResourceType any, OriginalResourceType interface { ToCore() ToResourceType -}, OptionsType any, PaginationQueryType PaginatedQuery[OptionsType]]( - handler RepositoryHandler[OptionsType], - paginator Paginator[OriginalResourceType, PaginationQueryType], -) *PaginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, OptionsType, PaginationQueryType] { - return &PaginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, OptionsType, PaginationQueryType]{ - PaginatedResourceRepository: NewPaginatedResourceRepository(handler, paginator), +}, OptionsType any](handler RepositoryHandler[OptionsType]) *PaginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, OptionsType] { + return &PaginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, OptionsType]{ + PaginatedResourceRepository: NewPaginatedResourceRepository[OriginalResourceType, OptionsType](handler), } } @@ -380,29 +415,43 @@ type Resource[ResourceType, OptionsType any] interface { } type ( - OffsetPaginatedQuery[OptionsType any] struct { + CursorMetadata struct { + Previous string + Next string + PageSize int + HasMore bool + } + InitialPaginatedQuery[OptionsType any] struct { Column string `json:"column"` - Offset uint64 `json:"offset"` Order *bunpaginate.Order `json:"order"` PageSize uint64 `json:"pageSize"` Options ResourceQuery[OptionsType] `json:"filters"` } + OffsetPaginatedQuery[OptionsType any] struct { + InitialPaginatedQuery[OptionsType] + Offset uint64 `json:"offset"` + } ColumnPaginatedQuery[OptionsType any] struct { - PageSize uint64 `json:"pageSize"` + InitialPaginatedQuery[OptionsType] Bottom *big.Int `json:"bottom"` - Column string `json:"column"` PaginationID *big.Int `json:"paginationID"` - // todo: backport in go-libs - Order *bunpaginate.Order `json:"order"` - Options ResourceQuery[OptionsType] `json:"filters"` - Reverse bool `json:"reverse"` + Reverse bool `json:"reverse"` } PaginatedQuery[OptionsType any] interface { - OffsetPaginatedQuery[OptionsType] | ColumnPaginatedQuery[OptionsType] + // Marker + isPaginatedQuery() } ) -type PaginatedResource[ResourceType, OptionsType any, PaginationQueryType PaginatedQuery[OptionsType]] interface { +func (i InitialPaginatedQuery[OptionsType]) isPaginatedQuery() {} + +var _ PaginatedQuery[any] = (*InitialPaginatedQuery[any])(nil) + +var _ PaginatedQuery[any] = (*OffsetPaginatedQuery[any])(nil) + +var _ PaginatedQuery[any] = (*ColumnPaginatedQuery[any])(nil) + +type PaginatedResource[ResourceType, OptionsType any] interface { Resource[ResourceType, OptionsType] - Paginate(ctx context.Context, paginationOptions PaginationQueryType) (*bunpaginate.Cursor[ResourceType], error) -} \ No newline at end of file + Paginate(ctx context.Context, paginationOptions PaginatedQuery[OptionsType]) (*bunpaginate.Cursor[ResourceType], error) +} diff --git a/internal/storage/common/schema.go b/internal/storage/common/schema.go index 563fd4308..1a3c08f4d 100644 --- a/internal/storage/common/schema.go +++ b/internal/storage/common/schema.go @@ -19,6 +19,7 @@ type FieldType interface { Operators() []string ValidateValue(value any) error IsIndexable() bool + IsPaginated() bool } type Field struct { @@ -88,6 +89,10 @@ func NewNumericMapField() Field { type TypeString struct{} +func (t TypeString) IsPaginated() bool { + return false +} + func (t TypeString) IsIndexable() bool { return false } @@ -115,6 +120,10 @@ func NewTypeString() TypeString { type TypeDate struct{} +func (t TypeDate) IsPaginated() bool { + return true +} + func (t TypeDate) IsIndexable() bool { return false } @@ -153,6 +162,10 @@ type TypeMap struct { underlyingType FieldType } +func (t TypeMap) IsPaginated() bool { + return false +} + func (t TypeMap) IsIndexable() bool { return true } @@ -175,6 +188,10 @@ var _ FieldType = (*TypeMap)(nil) type TypeNumeric struct{} +func (t TypeNumeric) IsPaginated() bool { + return true +} + func (t TypeNumeric) IsIndexable() bool { return false } @@ -207,6 +224,10 @@ var _ FieldType = (*TypeNumeric)(nil) type TypeBoolean struct{} +func (t TypeBoolean) IsPaginated() bool { + return false +} + func (t TypeBoolean) IsIndexable() bool { return false } diff --git a/internal/storage/driver/driver.go b/internal/storage/driver/driver.go index 549dd561e..e39f42949 100644 --- a/internal/storage/driver/driver.go +++ b/internal/storage/driver/driver.go @@ -180,7 +180,7 @@ func (d *Driver) DeleteLedgerMetadata(ctx context.Context, name string, key stri return d.systemStoreFactory.Create(d.db).DeleteLedgerMetadata(ctx, name, key) } -func (d *Driver) ListLedgers(ctx context.Context, q common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) { +func (d *Driver) ListLedgers(ctx context.Context, q common.PaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) { return d.systemStoreFactory.Create(d.db).Ledgers().Paginate(ctx, q) } diff --git a/internal/storage/driver/system_generated_test.go b/internal/storage/driver/system_generated_test.go index 0e24c5ebf..f4c626b81 100644 --- a/internal/storage/driver/system_generated_test.go +++ b/internal/storage/driver/system_generated_test.go @@ -134,10 +134,10 @@ func (mr *SystemStoreMockRecorder) IsUpToDate(ctx any) *gomock.Call { } // Ledgers mocks base method. -func (m *SystemStore) Ledgers() common.PaginatedResource[ledger.Ledger, any, common.ColumnPaginatedQuery[any]] { +func (m *SystemStore) Ledgers() common.PaginatedResource[ledger.Ledger, any] { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Ledgers") - ret0, _ := ret[0].(common.PaginatedResource[ledger.Ledger, any, common.ColumnPaginatedQuery[any]]) + ret0, _ := ret[0].(common.PaginatedResource[ledger.Ledger, any]) return ret0 } diff --git a/internal/storage/ledger/accounts.go b/internal/storage/ledger/accounts.go index ab9bc3d65..692c9bc75 100644 --- a/internal/storage/ledger/accounts.go +++ b/internal/storage/ledger/accounts.go @@ -137,4 +137,4 @@ func (store *Store) UpsertAccounts(ctx context.Context, accounts ...*ledger.Acco return nil }), )) -} +} \ No newline at end of file diff --git a/internal/storage/ledger/accounts_test.go b/internal/storage/ledger/accounts_test.go index 321778057..f8deb198f 100644 --- a/internal/storage/ledger/accounts_test.go +++ b/internal/storage/ledger/accounts_test.go @@ -4,6 +4,7 @@ package ledger_test import ( "context" + "github.com/formancehq/go-libs/v3/bun/bunpaginate" "github.com/formancehq/ledger/internal/storage/common" "math/big" "testing" @@ -77,17 +78,23 @@ func TestAccountsList(t *testing.T) { t.Run("list all", func(t *testing.T) { t.Parallel() - accounts, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{}) + accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), + }) require.NoError(t, err) require.Len(t, accounts.Data, 7) }) t.Run("list using metadata", func(t *testing.T) { t.Parallel() - accounts, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Match("metadata[category]", "1"), }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) @@ -95,10 +102,13 @@ func TestAccountsList(t *testing.T) { t.Run("list before date", func(t *testing.T) { t.Parallel() - accounts, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ PIT: &now, }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 2) @@ -107,11 +117,14 @@ func TestAccountsList(t *testing.T) { t.Run("list with volumes", func(t *testing.T) { t.Parallel() - accounts, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Match("address", "account:1"), Expand: []string{"volumes"}, }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) @@ -123,12 +136,15 @@ func TestAccountsList(t *testing.T) { t.Run("list with volumes using PIT", func(t *testing.T) { t.Parallel() - accounts, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Match("address", "account:1"), PIT: &now, Expand: []string{"volumes"}, }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) @@ -140,11 +156,14 @@ func TestAccountsList(t *testing.T) { t.Run("list with effective volumes", func(t *testing.T) { t.Parallel() - accounts, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Match("address", "account:1"), Expand: []string{"effectiveVolumes"}, }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) @@ -155,12 +174,15 @@ func TestAccountsList(t *testing.T) { t.Run("list with effective volumes using PIT", func(t *testing.T) { t.Parallel() - accounts, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Match("address", "account:1"), PIT: &now, Expand: []string{"effectiveVolumes"}, }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) @@ -171,51 +193,66 @@ func TestAccountsList(t *testing.T) { t.Run("list using filter on address", func(t *testing.T) { t.Parallel() - accounts, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Match("address", "account:"), }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 3) }) t.Run("list using filter on address and unbounded length", func(t *testing.T) { t.Parallel() - accounts, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Match("address", "account:..."), }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 3) }) t.Run("list using filter on multiple address", func(t *testing.T) { t.Parallel() - accounts, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Or( query.Match("address", "account:1"), query.Match("address", "orders:"), ), }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 3) }) t.Run("list using filter on balances", func(t *testing.T) { t.Parallel() - accounts, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Lt("balance[USD]", 0), }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) // world - accounts, err = store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + accounts, err = store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Gt("balance[USD]", 0), }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 2) @@ -224,22 +261,28 @@ func TestAccountsList(t *testing.T) { }) t.Run("list using filter on balances[USD] and PIT", func(t *testing.T) { t.Parallel() - accounts, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Lt("balance[USD]", 0), PIT: &now, }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) // world }) t.Run("list using filter on balances and PIT", func(t *testing.T) { t.Parallel() - accounts, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Lt("balance", 0), PIT: &now, }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) // world @@ -247,18 +290,23 @@ func TestAccountsList(t *testing.T) { t.Run("list using filter on exists metadata", func(t *testing.T) { t.Parallel() - accounts, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Exists("metadata", "foo"), }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 2) - accounts, err = store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + accounts, err = store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Exists("metadata", "category"), }, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 3) @@ -266,10 +314,13 @@ func TestAccountsList(t *testing.T) { t.Run("list using filter invalid field", func(t *testing.T) { t.Parallel() - _, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + _, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Lt("invalid", 0), }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.Error(t, err) require.True(t, errors.Is(err, common.ErrInvalidQuery{})) @@ -278,10 +329,13 @@ func TestAccountsList(t *testing.T) { t.Run("filter on first_usage", func(t *testing.T) { t.Parallel() - ret, err := store.Accounts().Paginate(ctx, common.OffsetPaginatedQuery[any]{ + ret, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ Options: common.ResourceQuery[any]{ Builder: query.Lte("first_usage", now), }, + PageSize: bunpaginate.QueryDefaultPageSize, + Column: "address", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, ret.Data, 2) diff --git a/internal/storage/ledger/logs_test.go b/internal/storage/ledger/logs_test.go index 9ae487194..56b3c3c3e 100644 --- a/internal/storage/ledger/logs_test.go +++ b/internal/storage/ledger/logs_test.go @@ -130,9 +130,10 @@ func TestLogsInsert(t *testing.T) { err := errGroup.Wait() require.NoError(t, err) - logs, err := store.Logs().Paginate(ctx, common.ColumnPaginatedQuery[any]{ + logs, err := store.Logs().Paginate(ctx, common.InitialPaginatedQuery[any]{ PageSize: countLogs, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), + Column: "id", }) require.NoError(t, err) @@ -220,14 +221,19 @@ func TestLogsList(t *testing.T) { require.NoError(t, err) } - cursor, err := store.Logs().Paginate(context.Background(), common.ColumnPaginatedQuery[any]{}) + cursor, err := store.Logs().Paginate(context.Background(), common.InitialPaginatedQuery[any]{ + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + }) require.NoError(t, err) require.Equal(t, bunpaginate.QueryDefaultPageSize, cursor.PageSize) require.Equal(t, 3, len(cursor.Data)) require.EqualValues(t, 3, *cursor.Data[0].ID) - cursor, err = store.Logs().Paginate(context.Background(), common.ColumnPaginatedQuery[any]{ + cursor, err = store.Logs().Paginate(context.Background(), common.InitialPaginatedQuery[any]{ + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), PageSize: 1, }) require.NoError(t, err) @@ -235,7 +241,7 @@ func TestLogsList(t *testing.T) { require.Equal(t, 1, cursor.PageSize) require.EqualValues(t, 3, *cursor.Data[0].ID) - cursor, err = store.Logs().Paginate(context.Background(), common.ColumnPaginatedQuery[any]{ + cursor, err = store.Logs().Paginate(context.Background(), common.InitialPaginatedQuery[any]{ PageSize: 10, Options: common.ResourceQuery[any]{ Builder: query.And( @@ -243,6 +249,8 @@ func TestLogsList(t *testing.T) { query.Lt("date", now.Add(-time.Hour)), ), }, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Equal(t, 10, cursor.PageSize) @@ -250,11 +258,13 @@ func TestLogsList(t *testing.T) { require.Len(t, cursor.Data, 1) require.EqualValues(t, 2, *cursor.Data[0].ID) - cursor, err = store.Logs().Paginate(context.Background(), common.ColumnPaginatedQuery[any]{ + cursor, err = store.Logs().Paginate(context.Background(), common.InitialPaginatedQuery[any]{ PageSize: 10, Options: common.ResourceQuery[any]{ Builder: query.Lt("id", 3), }, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }) require.NoError(t, err) require.Equal(t, 2, len(cursor.Data)) diff --git a/internal/storage/ledger/store.go b/internal/storage/ledger/store.go index 2d6f917cc..de94fa8be 100644 --- a/internal/storage/ledger/store.go +++ b/internal/storage/ledger/store.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "fmt" - "github.com/formancehq/go-libs/v3/bun/bunpaginate" "github.com/formancehq/go-libs/v3/migrations" "github.com/formancehq/go-libs/v3/platform/postgres" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" @@ -46,12 +45,11 @@ type Store struct { func (store *Store) Volumes() common.PaginatedResource[ ledger.VolumesWithBalanceByAssetByAccount, - ledgercontroller.GetVolumesOptions, - common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]] { - return common.NewPaginatedResourceRepository(&volumesResourceHandler{store: store}, common.OffsetPaginator[ledger.VolumesWithBalanceByAssetByAccount, ledgercontroller.GetVolumesOptions]{ - DefaultPaginationColumn: "account", - DefaultOrder: bunpaginate.OrderAsc, - }) + ledgercontroller.GetVolumesOptions] { + return common.NewPaginatedResourceRepository[ + ledger.VolumesWithBalanceByAssetByAccount, + ledgercontroller.GetVolumesOptions, + ](&volumesResourceHandler{store: store}) } func (store *Store) AggregatedVolumes() common.Resource[ledger.AggregatedVolumes, ledgercontroller.GetAggregatedVolumesOptions] { @@ -62,37 +60,23 @@ func (store *Store) AggregatedVolumes() common.Resource[ledger.AggregatedVolumes func (store *Store) Transactions() common.PaginatedResource[ ledger.Transaction, - any, - common.ColumnPaginatedQuery[any]] { - return common.NewPaginatedResourceRepository(&transactionsResourceHandler{store: store}, common.ColumnPaginator[ledger.Transaction, any]{ - DefaultPaginationColumn: "id", - DefaultOrder: bunpaginate.OrderDesc, - Table: store.db.Dialect().Tables().ByName("transactions"), - }) + any] { + return common.NewPaginatedResourceRepository[ledger.Transaction, any](&transactionsResourceHandler{store: store}) } func (store *Store) Logs() common.PaginatedResource[ ledger.Log, - any, - common.ColumnPaginatedQuery[any]] { - return common.NewPaginatedResourceRepositoryMapper[ledger.Log, Log, any, common.ColumnPaginatedQuery[any]](&logsResourceHandler{ + any] { + return common.NewPaginatedResourceRepositoryMapper[ledger.Log, Log, any](&logsResourceHandler{ store: store, - }, common.ColumnPaginator[Log, any]{ - DefaultPaginationColumn: "id", - DefaultOrder: bunpaginate.OrderDesc, - Table: store.db.Dialect().Tables().ByName("logs"), }) } func (store *Store) Accounts() common.PaginatedResource[ ledger.Account, - any, - common.OffsetPaginatedQuery[any]] { - return common.NewPaginatedResourceRepository(&accountsResourceHandler{ + any] { + return common.NewPaginatedResourceRepository[ledger.Account, any](&accountsResourceHandler{ store: store, - }, common.OffsetPaginator[ledger.Account, any]{ - DefaultPaginationColumn: "address", - DefaultOrder: bunpaginate.OrderAsc, }) } diff --git a/internal/storage/ledger/transactions_test.go b/internal/storage/ledger/transactions_test.go index 27b1ceda5..f687eed1c 100644 --- a/internal/storage/ledger/transactions_test.go +++ b/internal/storage/ledger/transactions_test.go @@ -7,6 +7,7 @@ import ( "database/sql" "fmt" "github.com/alitto/pond" + "github.com/formancehq/go-libs/v3/bun/bunpaginate" "github.com/formancehq/ledger/internal/storage/common" "math/big" "slices" @@ -452,11 +453,13 @@ func TestTransactionsCommit(t *testing.T) { require.NoError(t, err) } - cursor, err := store.Transactions().Paginate(ctx, common.ColumnPaginatedQuery[any]{ + cursor, err := store.Transactions().Paginate(ctx, common.InitialPaginatedQuery[any]{ PageSize: countTx, Options: common.ResourceQuery[any]{ Expand: []string{"volumes"}, }, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Column: "id", }) require.NoError(t, err) require.Len(t, cursor.Data, countTx) @@ -746,15 +749,24 @@ func TestTransactionsList(t *testing.T) { } testCases := []testCase{ { - name: "nominal", - query: common.ColumnPaginatedQuery[any]{}, + name: "nominal", + query: common.ColumnPaginatedQuery[any]{ + InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + }, + }, expected: []ledger.Transaction{tx5, tx4, tx3, tx2, tx1}, }, { name: "address filter", query: common.ColumnPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("account", "bob"), + InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("account", "bob"), + }, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }, }, expected: []ledger.Transaction{tx2}, @@ -762,8 +774,12 @@ func TestTransactionsList(t *testing.T) { { name: "address filter using segments matching two addresses by individual segments", query: common.ColumnPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("account", "users:amazon"), + InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("account", "users:amazon"), + }, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }, }, expected: []ledger.Transaction{}, @@ -771,8 +787,12 @@ func TestTransactionsList(t *testing.T) { { name: "address filter using segment", query: common.ColumnPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("account", "users:"), + InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("account", "users:"), + }, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }, }, expected: []ledger.Transaction{tx5, tx4, tx3}, @@ -780,8 +800,12 @@ func TestTransactionsList(t *testing.T) { { name: "address filter using segment and unbounded segment list", query: common.ColumnPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("account", "users:..."), + InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("account", "users:..."), + }, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }, }, expected: []ledger.Transaction{tx5, tx4, tx3}, @@ -789,8 +813,12 @@ func TestTransactionsList(t *testing.T) { { name: "filter using metadata", query: common.ColumnPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("metadata[category]", "2"), + InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("metadata[category]", "2"), + }, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }, }, expected: []ledger.Transaction{tx2}, @@ -798,8 +826,12 @@ func TestTransactionsList(t *testing.T) { { name: "using point in time", query: common.ColumnPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - PIT: pointer.For(now.Add(-time.Hour)), + InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + PIT: pointer.For(now.Add(-time.Hour)), + }, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }, }, expected: []ledger.Transaction{tx3BeforeRevert, tx2, tx1}, @@ -807,8 +839,12 @@ func TestTransactionsList(t *testing.T) { { name: "filter using invalid key", query: common.ColumnPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("invalid", "2"), + InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("invalid", "2"), + }, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }, }, expectError: common.ErrInvalidQuery{}, @@ -816,8 +852,12 @@ func TestTransactionsList(t *testing.T) { { name: "reverted transactions", query: common.ColumnPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("reverted", true), + InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("reverted", true), + }, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }, }, expected: []ledger.Transaction{tx3}, @@ -825,8 +865,12 @@ func TestTransactionsList(t *testing.T) { { name: "filter using exists metadata", query: common.ColumnPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Exists("metadata", "category"), + InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Exists("metadata", "category"), + }, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }, }, expected: []ledger.Transaction{tx3, tx2, tx1}, @@ -834,9 +878,13 @@ func TestTransactionsList(t *testing.T) { { name: "filter using metadata and pit", query: common.ColumnPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("metadata[category]", "2"), - PIT: pointer.For(tx3.Timestamp), + InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("metadata[category]", "2"), + PIT: pointer.For(tx3.Timestamp), + }, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }, }, expected: []ledger.Transaction{tx2}, @@ -844,8 +892,12 @@ func TestTransactionsList(t *testing.T) { { name: "filter using not exists metadata", query: common.ColumnPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Not(query.Exists("metadata", "category")), + InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Not(query.Exists("metadata", "category")), + }, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }, }, expected: []ledger.Transaction{tx5, tx4}, @@ -853,8 +905,12 @@ func TestTransactionsList(t *testing.T) { { name: "filter using timestamp", query: common.ColumnPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("timestamp", tx5.Timestamp.Format(time.RFC3339Nano)), + InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("timestamp", tx5.Timestamp.Format(time.RFC3339Nano)), + }, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }, }, expected: []ledger.Transaction{tx5, tx4}, diff --git a/internal/storage/ledger/volumes_test.go b/internal/storage/ledger/volumes_test.go index 429628266..e60a5e3ef 100644 --- a/internal/storage/ledger/volumes_test.go +++ b/internal/storage/ledger/volumes_test.go @@ -4,6 +4,7 @@ package ledger_test import ( "database/sql" + "github.com/formancehq/go-libs/v3/bun/bunpaginate" "github.com/formancehq/go-libs/v3/pointer" "github.com/formancehq/ledger/internal/storage/common" "math/big" @@ -106,10 +107,12 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with first account usage filter", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Builder: query.Lt("first_usage", now.Add(-3*time.Minute)), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 2) @@ -135,11 +138,13 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with first account usage filter and PIT", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Builder: query.Lt("first_usage", now.Add(-3*time.Minute)), PIT: pointer.For(now.Add(-3 * time.Minute)), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 2) @@ -165,12 +170,14 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ UseInsertionDate: true, }, }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -178,20 +185,25 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for effective date", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{}) + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), + }) require.NoError(t, err) require.Len(t, volumes.Data, 4) }) t.Run("Get all volumes with balance for insertion date with previous pit", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ UseInsertionDate: true, }, PIT: &previousPIT, }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -209,13 +221,15 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with futur pit", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ UseInsertionDate: true, }, PIT: &futurPIT, }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -223,13 +237,15 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with previous oot", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ UseInsertionDate: true, }, OOT: &previousOOT, }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -237,13 +253,15 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with future oot", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ UseInsertionDate: true, }, OOT: &futurOOT, }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -261,10 +279,12 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for effective date with previous pit", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &previousPIT, }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -282,10 +302,12 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for effective date with futur pit", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &futurPIT, }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -293,10 +315,12 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for effective date with previous oot", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ OOT: &previousOOT, }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -304,10 +328,12 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for effective date with futur oot", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ OOT: &futurOOT, }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -325,7 +351,7 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with future PIT and now OOT", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ UseInsertionDate: true, @@ -333,6 +359,8 @@ func TestVolumesList(t *testing.T) { PIT: &futurPIT, OOT: &now, }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -350,7 +378,7 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with previous OOT and now PIT", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ UseInsertionDate: true, @@ -358,6 +386,8 @@ func TestVolumesList(t *testing.T) { PIT: &now, OOT: &previousOOT, }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -375,11 +405,13 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for effective date with future PIT and now OOT", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &futurPIT, OOT: &now, }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -397,11 +429,13 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with previous OOT and now PIT", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &now, OOT: &previousOOT, }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -421,12 +455,14 @@ func TestVolumesList(t *testing.T) { t.Parallel() volumes, err := store.Volumes().Paginate(ctx, - common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &now, OOT: &previousOOT, Builder: query.Match("account", "account:1"), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }, ) require.NoError(t, err) @@ -447,10 +483,12 @@ func TestVolumesList(t *testing.T) { t.Parallel() volumes, err := store.Volumes().Paginate(ctx, - common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Builder: query.Match("metadata[foo]", "bar"), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }, ) require.NoError(t, err) @@ -461,10 +499,12 @@ func TestVolumesList(t *testing.T) { t.Parallel() volumes, err := store.Volumes().Paginate(ctx, - common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Builder: query.Exists("metadata", "category"), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }, ) require.NoError(t, err) @@ -475,10 +515,12 @@ func TestVolumesList(t *testing.T) { t.Parallel() volumes, err := store.Volumes().Paginate(ctx, - common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Builder: query.Exists("metadata", "foo"), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }, ) require.NoError(t, err) @@ -546,13 +588,15 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 0", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ UseInsertionDate: true, }, Builder: query.Match("account", "account::"), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 7) @@ -560,7 +604,7 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 1", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ UseInsertionDate: true, @@ -568,6 +612,8 @@ func TestVolumesAggregate(t *testing.T) { }, Builder: query.Match("account", "account::"), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 2) @@ -575,7 +621,7 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 2", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ UseInsertionDate: true, @@ -583,6 +629,8 @@ func TestVolumesAggregate(t *testing.T) { }, Builder: query.Match("account", "account::"), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -590,7 +638,7 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 3", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ UseInsertionDate: true, @@ -598,6 +646,8 @@ func TestVolumesAggregate(t *testing.T) { }, Builder: query.Match("account", "account::"), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 7) @@ -606,7 +656,7 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 1 && PIT && OOT && effectiveDate", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ GroupLvl: 1, @@ -615,6 +665,8 @@ func TestVolumesAggregate(t *testing.T) { OOT: &oot, Builder: query.Match("account", "account::"), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 2) @@ -641,7 +693,7 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 1 && PIT && OOT && effectiveDate && Balance Filter 1", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ GroupLvl: 1, @@ -650,6 +702,8 @@ func TestVolumesAggregate(t *testing.T) { OOT: &oot, Builder: query.And(query.Match("account", "account::"), query.Gte("balance[EUR]", 50)), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 1) @@ -666,7 +720,7 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 1 && Balance Filter 2", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ GroupLvl: 2, @@ -676,6 +730,8 @@ func TestVolumesAggregate(t *testing.T) { query.Match("account", "account:1:"), query.Lte("balance[USD]", 0)), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 3) @@ -711,7 +767,7 @@ func TestVolumesAggregate(t *testing.T) { t.Parallel() volumes, err := store.Volumes().Paginate(ctx, - common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ GroupLvl: 1, @@ -721,6 +777,8 @@ func TestVolumesAggregate(t *testing.T) { query.Match("metadata[foo]", "bar"), ), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -731,7 +789,7 @@ func TestVolumesAggregate(t *testing.T) { t.Parallel() volumes, err := store.Volumes().Paginate(ctx, - common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ GroupLvl: 1, @@ -742,6 +800,8 @@ func TestVolumesAggregate(t *testing.T) { query.Match("metadata[foo]", "bar"), ), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -752,13 +812,15 @@ func TestVolumesAggregate(t *testing.T) { t.Parallel() volumes, err := store.Volumes().Paginate(ctx, - common.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Opts: ledgercontroller.GetVolumesOptions{ GroupLvl: 1, }, Builder: query.Match("metadata[foo]", "bar"), }, + Column: "account", + Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }, ) require.NoError(t, err) diff --git a/internal/storage/system/resource_ledgers.go b/internal/storage/system/resource_ledgers.go index a7f6d25a4..1228386f8 100644 --- a/internal/storage/system/resource_ledgers.go +++ b/internal/storage/system/resource_ledgers.go @@ -23,6 +23,7 @@ func (h ledgersResourceHandler) Schema() common.EntitySchema { "features": common.NewStringMapField(), "metadata": common.NewStringMapField(), "name": common.NewStringField(), + "id": common.NewNumericField(), }, } } diff --git a/internal/storage/system/store.go b/internal/storage/system/store.go index 298d9595a..bb6cd9b8e 100644 --- a/internal/storage/system/store.go +++ b/internal/storage/system/store.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "github.com/formancehq/go-libs/v3/bun/bunpaginate" "github.com/formancehq/go-libs/v3/metadata" "github.com/formancehq/go-libs/v3/migrations" "github.com/formancehq/go-libs/v3/platform/postgres" @@ -21,7 +20,7 @@ type Store interface { CreateLedger(ctx context.Context, l *ledger.Ledger) error DeleteLedgerMetadata(ctx context.Context, name string, key string) error UpdateLedgerMetadata(ctx context.Context, name string, m metadata.Metadata) error - Ledgers() common.PaginatedResource[ledger.Ledger, any, common.ColumnPaginatedQuery[any]] + Ledgers() common.PaginatedResource[ledger.Ledger, any] GetLedger(ctx context.Context, name string) (*ledger.Ledger, error) GetDistinctBuckets(ctx context.Context) ([]string, error) @@ -97,13 +96,8 @@ func (d *DefaultStore) DeleteLedgerMetadata(ctx context.Context, name string, ke func (d *DefaultStore) Ledgers() common.PaginatedResource[ ledger.Ledger, - any, - common.ColumnPaginatedQuery[any]] { - return common.NewPaginatedResourceRepository(&ledgersResourceHandler{store: d}, common.ColumnPaginator[ledger.Ledger, any]{ - DefaultPaginationColumn: "id", - DefaultOrder: bunpaginate.OrderAsc, - Table: d.db.Dialect().Tables().ByName("_system.ledgers"), - }) + any] { + return common.NewPaginatedResourceRepository[ledger.Ledger, any](&ledgersResourceHandler{store: d}) } func (d *DefaultStore) GetLedger(ctx context.Context, name string) (*ledger.Ledger, error) { diff --git a/internal/worker/async_block.go b/internal/worker/async_block.go index 450c17c84..c5a5a4ee9 100644 --- a/internal/worker/async_block.go +++ b/internal/worker/async_block.go @@ -80,12 +80,10 @@ func (r *AsyncBlockRunner) run(ctx context.Context) error { initialQuery := ledgercontroller.NewListLedgersQuery(10) initialQuery.Options.Builder = query.Match(fmt.Sprintf("features[%s]", features.FeatureHashLogs), "ASYNC") systemStore := systemstore.New(r.db) - return bunpaginate.Iterate( + return common.Iterate( ctx, initialQuery, - func(ctx context.Context, q common.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Ledger], error) { - return systemStore.Ledgers().Paginate(ctx, q) - }, + systemStore.Ledgers().Paginate, func(cursor *bunpaginate.Cursor[ledger.Ledger]) error { for _, l := range cursor.Data { if err := r.processLedger(ctx, l); err != nil { diff --git a/test/e2e/api_logs_list_test.go b/test/e2e/api_logs_list_test.go index d8fbc0629..ebf451334 100644 --- a/test/e2e/api_logs_list_test.go +++ b/test/e2e/api_logs_list_test.go @@ -257,7 +257,7 @@ var _ = Context("Ledger logs list API tests", func() { AfterEach(func() { expectedLogs = nil }) - When(fmt.Sprintf("listing accounts using page size of %d", pageSize), func() { + When(fmt.Sprintf("listing logs using page size of %d", pageSize), func() { var ( rsp *operations.V2ListLogsResponse err error From 16362d4c753df6c2665b2122913ead2465ea1f9a Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Thu, 19 Jun 2025 19:15:37 +0200 Subject: [PATCH 3/4] feat: add sane default for each resource handler --- go.mod | 2 +- internal/api/v1/controllers_accounts_list.go | 19 +-- internal/api/v1/controllers_balances_list.go | 21 +-- internal/api/v1/controllers_config.go | 5 +- internal/api/v1/controllers_logs_list.go | 13 +- .../api/v1/controllers_transactions_list.go | 14 +- internal/api/v1/utils.go | 6 +- internal/api/v2/common.go | 8 +- internal/api/v2/controllers_ledgers_list.go | 2 + .../api/v2/controllers_ledgers_list_test.go | 19 +-- .../controller/ledger/controller_default.go | 3 - internal/controller/ledger/store.go | 13 -- internal/storage/bucket/migrations_test.go | 1 - internal/storage/common/cursor.go | 7 + internal/storage/common/pagination.go | 37 +++++ internal/storage/common/paginator_column.go | 8 -- internal/storage/common/resource.go | 65 ++++----- internal/storage/driver/driver_test.go | 9 +- internal/storage/ledger/accounts_test.go | 56 +------- internal/storage/ledger/logs_test.go | 11 +- internal/storage/ledger/store.go | 9 +- internal/storage/ledger/transactions_test.go | 136 ++++++------------ internal/storage/ledger/volumes_test.go | 64 +-------- internal/storage/system/store.go | 3 +- internal/storage/system/store_test.go | 9 +- internal/worker/async_block.go | 12 +- 26 files changed, 200 insertions(+), 352 deletions(-) create mode 100644 internal/storage/common/pagination.go diff --git a/go.mod b/go.mod index 051664a6c..56e6cbc14 100644 --- a/go.mod +++ b/go.mod @@ -104,7 +104,7 @@ require ( github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/containerd/continuity v0.4.5 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/dlclark/regexp2 v1.11.5 // indirect github.com/dnwe/otelsarama v0.0.0-20240308230250-9388d9d40bc0 // indirect github.com/docker/cli v27.4.1+incompatible // indirect diff --git a/internal/api/v1/controllers_accounts_list.go b/internal/api/v1/controllers_accounts_list.go index ab5cbb306..039e86182 100644 --- a/internal/api/v1/controllers_accounts_list.go +++ b/internal/api/v1/controllers_accounts_list.go @@ -14,15 +14,16 @@ import ( func listAccounts(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - rqBuilder, err := buildAccountsFilterQuery(r) - if err != nil { - api.BadRequest(w, common.ErrValidation, err) - return - } - - rq, err := getPaginatedQuery(r, "address", bunpaginate.OrderAsc, func(resourceQuery *storagecommon.ResourceQuery[any]) { - resourceQuery.Builder = rqBuilder - }) + rq, err := getPaginatedQuery( + r, + "address", + bunpaginate.OrderAsc, + func(resourceQuery *storagecommon.ResourceQuery[any]) error { + var err error + resourceQuery.Builder, err = buildAccountsFilterQuery(r) + return err + }, + ) if err != nil { api.BadRequest(w, common.ErrValidation, err) return diff --git a/internal/api/v1/controllers_balances_list.go b/internal/api/v1/controllers_balances_list.go index f66450d6a..ed1416378 100644 --- a/internal/api/v1/controllers_balances_list.go +++ b/internal/api/v1/controllers_balances_list.go @@ -13,16 +13,17 @@ import ( func getBalances(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - filter, err := buildAccountsFilterQuery(r) - if err != nil { - api.BadRequest(w, common.ErrValidation, err) - return - } - - rq, err := getPaginatedQuery[any](r, "address", bunpaginate.OrderAsc, func(resourceQuery *storagecommon.ResourceQuery[any]) { - resourceQuery.Expand = append(resourceQuery.Expand, "volumes") - resourceQuery.Builder = filter - }) + rq, err := getPaginatedQuery[any]( + r, + "address", + bunpaginate.OrderAsc, + func(resourceQuery *storagecommon.ResourceQuery[any]) error { + var err error + resourceQuery.Expand = append(resourceQuery.Expand, "volumes") + resourceQuery.Builder, err = buildAccountsFilterQuery(r) + return err + }, + ) if err != nil { api.BadRequest(w, common.ErrValidation, err) return diff --git a/internal/api/v1/controllers_config.go b/internal/api/v1/controllers_config.go index 63db7554c..73617196d 100644 --- a/internal/api/v1/controllers_config.go +++ b/internal/api/v1/controllers_config.go @@ -6,7 +6,6 @@ import ( storagecommon "github.com/formancehq/ledger/internal/storage/common" "net/http" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" "github.com/formancehq/ledger/internal/controller/system" "github.com/formancehq/go-libs/v3/bun/bunpaginate" @@ -36,7 +35,9 @@ func GetInfo(systemController system.Controller, version string) func(w http.Res return func(w http.ResponseWriter, r *http.Request) { ledgerNames := make([]string, 0) - if err := storagecommon.Iterate(r.Context(), ledgercontroller.NewListLedgersQuery(100), + if err := storagecommon.Iterate(r.Context(), storagecommon.InitialPaginatedQuery[any]{ + PageSize: 100, + }, systemController.ListLedgers, func(cursor *bunpaginate.Cursor[ledger.Ledger]) error { ledgerNames = append(ledgerNames, collectionutils.Map(cursor.Data, func(from ledger.Ledger) string { diff --git a/internal/api/v1/controllers_logs_list.go b/internal/api/v1/controllers_logs_list.go index 0f165bb49..fc7c3427d 100644 --- a/internal/api/v1/controllers_logs_list.go +++ b/internal/api/v1/controllers_logs_list.go @@ -36,10 +36,15 @@ func buildGetLogsQuery(r *http.Request) query.Builder { func getLogs(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - - paginatedQuery, err := getPaginatedQuery[any](r, "id", bunpaginate.OrderDesc, func(resourceQuery *storagecommon.ResourceQuery[any]) { - resourceQuery.Builder = buildGetLogsQuery(r) - }) + paginatedQuery, err := getPaginatedQuery[any]( + r, + "id", + bunpaginate.OrderDesc, + func(resourceQuery *storagecommon.ResourceQuery[any]) error { + resourceQuery.Builder = buildGetLogsQuery(r) + return nil + }, + ) if err != nil { api.BadRequest(w, common.ErrValidation, err) return diff --git a/internal/api/v1/controllers_transactions_list.go b/internal/api/v1/controllers_transactions_list.go index cc0aef45d..01a82719e 100644 --- a/internal/api/v1/controllers_transactions_list.go +++ b/internal/api/v1/controllers_transactions_list.go @@ -12,10 +12,16 @@ import ( func listTransactions(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - paginatedQuery, err := getPaginatedQuery[any](r, "id", bunpaginate.OrderDesc, func(resourceQuery *storagecommon.ResourceQuery[any]) { - resourceQuery.Expand = append(resourceQuery.Expand, "volumes") - resourceQuery.Builder = buildGetTransactionsQuery(r) - }) + paginatedQuery, err := getPaginatedQuery[any]( + r, + "id", + bunpaginate.OrderDesc, + func(resourceQuery *storagecommon.ResourceQuery[any]) error { + resourceQuery.Expand = append(resourceQuery.Expand, "volumes") + resourceQuery.Builder = buildGetTransactionsQuery(r) + return nil + }, + ) if err != nil { api.BadRequest(w, common.ErrValidation, err) return diff --git a/internal/api/v1/utils.go b/internal/api/v1/utils.go index fbae388c7..d4bfb4470 100644 --- a/internal/api/v1/utils.go +++ b/internal/api/v1/utils.go @@ -27,7 +27,7 @@ func getPaginatedQuery[Options any]( r *http.Request, defaultColumn string, defaultOrder bunpaginate.Order, - modifiers ...func(resourceQuery *storagecommon.ResourceQuery[Options]), + modifiers ...func(resourceQuery *storagecommon.ResourceQuery[Options]) error, ) (storagecommon.PaginatedQuery[Options], error) { return storagecommon.Extract[Options]( @@ -39,7 +39,9 @@ func getPaginatedQuery[Options any]( } for _, modifier := range modifiers { - modifier(rq) + if err := modifier(rq); err != nil { + return nil, err + } } pageSize, err := bunpaginate.GetPageSize( diff --git a/internal/api/v2/common.go b/internal/api/v2/common.go index c48bcae1f..3e6b9e20b 100644 --- a/internal/api/v2/common.go +++ b/internal/api/v2/common.go @@ -64,8 +64,8 @@ func getExpand(r *http.Request) []string { func getPaginatedQuery[Options any]( r *http.Request, paginationConfig common.PaginationConfig, - defaultColumn string, - defaultOrder bunpaginate.Order, + column string, + order bunpaginate.Order, modifiers ...func(resourceQuery *storagecommon.ResourceQuery[Options]), ) (storagecommon.PaginatedQuery[Options], error) { return storagecommon.Extract[Options]( @@ -90,8 +90,8 @@ func getPaginatedQuery[Options any]( } return &storagecommon.InitialPaginatedQuery[Options]{ - Column: defaultColumn, - Order: &defaultOrder, + Column: column, + Order: &order, PageSize: pageSize, Options: *rq, }, nil diff --git a/internal/api/v2/controllers_ledgers_list.go b/internal/api/v2/controllers_ledgers_list.go index d7cd782fc..1b6cdb8ff 100644 --- a/internal/api/v2/controllers_ledgers_list.go +++ b/internal/api/v2/controllers_ledgers_list.go @@ -1,6 +1,7 @@ package v2 import ( + "github.com/davecgh/go-spew/spew" "github.com/formancehq/ledger/internal/api/common" storagecommon "github.com/formancehq/ledger/internal/storage/common" "net/http" @@ -21,6 +22,7 @@ func listLedgers(b system.Controller, paginationConfig common.PaginationConfig) return } + spew.Dump(rq) ledgers, err := b.ListLedgers(r.Context(), rq) if err != nil { switch { diff --git a/internal/api/v2/controllers_ledgers_list_test.go b/internal/api/v2/controllers_ledgers_list_test.go index 0188cb324..993426e75 100644 --- a/internal/api/v2/controllers_ledgers_list_test.go +++ b/internal/api/v2/controllers_ledgers_list_test.go @@ -29,7 +29,6 @@ func TestLedgersList(t *testing.T) { type testCase struct { name string - expectQuery storagecommon.PaginatedQuery[any] queryParams url.Values returnData []ledger.Ledger returnErr error @@ -40,8 +39,7 @@ func TestLedgersList(t *testing.T) { for _, tc := range []testCase{ { - name: "nominal", - expectQuery: pointer.For(ledgercontroller.NewListLedgersQuery(15)), + name: "nominal", returnData: []ledger.Ledger{ ledger.MustNewWithDefault(uuid.NewString()), ledger.MustNewWithDefault(uuid.NewString()), @@ -49,8 +47,7 @@ func TestLedgersList(t *testing.T) { expectBackendCall: true, }, { - name: "invalid page size", - expectQuery: ledgercontroller.NewListLedgersQuery(15), + name: "invalid page size", queryParams: url.Values{ "pageSize": {"-1"}, }, @@ -60,7 +57,6 @@ func TestLedgersList(t *testing.T) { }, { name: "error from backend", - expectQuery: ledgercontroller.NewListLedgersQuery(15), expectedStatusCode: http.StatusInternalServerError, expectedErrorCode: api.ErrorInternal, expectBackendCall: true, @@ -72,7 +68,6 @@ func TestLedgersList(t *testing.T) { expectedErrorCode: common.ErrValidation, expectBackendCall: true, returnErr: storagecommon.ErrInvalidQuery{}, - expectQuery: ledgercontroller.NewListLedgersQuery(bunpaginate.QueryDefaultPageSize), }, { name: "with missing feature", @@ -80,7 +75,6 @@ func TestLedgersList(t *testing.T) { expectedErrorCode: common.ErrValidation, expectBackendCall: true, returnErr: ledgercontroller.ErrMissingFeature{}, - expectQuery: ledgercontroller.NewListLedgersQuery(bunpaginate.QueryDefaultPageSize), }, } { t.Run(tc.name, func(t *testing.T) { @@ -90,7 +84,14 @@ func TestLedgersList(t *testing.T) { if tc.expectBackendCall { systemController.EXPECT(). - ListLedgers(gomock.Any(), ledgercontroller.NewListLedgersQuery(15)). + ListLedgers(gomock.Any(), storagecommon.InitialPaginatedQuery[any]{ + PageSize: 15, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), + Options: storagecommon.ResourceQuery[any]{ + Expand: []string{}, + }, + }). Return(&bunpaginate.Cursor[ledger.Ledger]{ Data: tc.returnData, }, tc.returnErr) diff --git a/internal/controller/ledger/controller_default.go b/internal/controller/ledger/controller_default.go index 525534d45..b39fb1642 100644 --- a/internal/controller/ledger/controller_default.go +++ b/internal/controller/ledger/controller_default.go @@ -189,8 +189,6 @@ func (ctrl *DefaultController) Import(ctx context.Context, stream chan ledger.Lo // We can import only if the ledger is empty. logs, err := ctrl.store.Logs().Paginate(ctx, storagecommon.InitialPaginatedQuery[any]{ PageSize: 1, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }) if err != nil { return fmt.Errorf("error listing logs: %w", err) @@ -321,7 +319,6 @@ func (ctrl *DefaultController) Export(ctx context.Context, w ExportWriter) error storagecommon.InitialPaginatedQuery[any]{ PageSize: 100, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), - Column: "id", }, ctrl.store.Logs().Paginate, func(cursor *bunpaginate.Cursor[ledger.Log]) error { diff --git a/internal/controller/ledger/store.go b/internal/controller/ledger/store.go index de9900039..65372f2c4 100644 --- a/internal/controller/ledger/store.go +++ b/internal/controller/ledger/store.go @@ -3,8 +3,6 @@ package ledger import ( "context" "database/sql" - "github.com/formancehq/go-libs/v3/bun/bunpaginate" - "github.com/formancehq/go-libs/v3/pointer" "github.com/formancehq/ledger/internal/storage/common" "math/big" @@ -87,17 +85,6 @@ func newVmStoreAdapter(tx Store) *vmStoreAdapter { } } -func NewListLedgersQuery(pageSize uint64) common.InitialPaginatedQuery[any] { - return common.InitialPaginatedQuery[any]{ - PageSize: pageSize, - Column: "id", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), - Options: common.ResourceQuery[any]{ - Expand: make([]string, 0), - }, - } -} - // numscript rewrite implementation var _ numscript.Store = (*numscriptRewriteAdapter)(nil) diff --git a/internal/storage/bucket/migrations_test.go b/internal/storage/bucket/migrations_test.go index 4aeac8d1b..a382f8aba 100644 --- a/internal/storage/bucket/migrations_test.go +++ b/internal/storage/bucket/migrations_test.go @@ -95,7 +95,6 @@ func TestMigrations(t *testing.T) { common.InitialPaginatedQuery[any]{ PageSize: 100, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), - Column: "id", }, store.Logs().Paginate, func(cursor *bunpaginate.Cursor[ledger.Log]) error { diff --git a/internal/storage/common/cursor.go b/internal/storage/common/cursor.go index eabb40419..481eb744c 100644 --- a/internal/storage/common/cursor.go +++ b/internal/storage/common/cursor.go @@ -99,3 +99,10 @@ func Iterate[OF any, Options any]( return nil } + +func encodeCursor[OptionsType any, PaginatedQueryType PaginatedQuery[OptionsType]](v *PaginatedQueryType) string { + if v == nil { + return "" + } + return bunpaginate.EncodeCursor(v) +} diff --git a/internal/storage/common/pagination.go b/internal/storage/common/pagination.go new file mode 100644 index 000000000..4434bcb8d --- /dev/null +++ b/internal/storage/common/pagination.go @@ -0,0 +1,37 @@ +package common + +import ( + "github.com/formancehq/go-libs/v3/bun/bunpaginate" + "math/big" +) + +type ( + InitialPaginatedQuery[OptionsType any] struct { + Column string `json:"column"` + Order *bunpaginate.Order `json:"order"` + PageSize uint64 `json:"pageSize"` + Options ResourceQuery[OptionsType] `json:"filters"` + } + OffsetPaginatedQuery[OptionsType any] struct { + InitialPaginatedQuery[OptionsType] + Offset uint64 `json:"offset"` + } + ColumnPaginatedQuery[OptionsType any] struct { + InitialPaginatedQuery[OptionsType] + Bottom *big.Int `json:"bottom"` + PaginationID *big.Int `json:"paginationID"` + Reverse bool `json:"reverse"` + } + PaginatedQuery[OptionsType any] interface { + // Marker + isPaginatedQuery() + } +) + +func (i InitialPaginatedQuery[OptionsType]) isPaginatedQuery() {} + +var _ PaginatedQuery[any] = (*InitialPaginatedQuery[any])(nil) + +var _ PaginatedQuery[any] = (*OffsetPaginatedQuery[any])(nil) + +var _ PaginatedQuery[any] = (*ColumnPaginatedQuery[any])(nil) diff --git a/internal/storage/common/paginator_column.go b/internal/storage/common/paginator_column.go index cae90332d..ab310fd23 100644 --- a/internal/storage/common/paginator_column.go +++ b/internal/storage/common/paginator_column.go @@ -237,14 +237,6 @@ func findPaginationField(v any, fields ...reflect.StructField) *big.Int { return findPaginationField(v, fields[1:]...) } -//nolint:unused -func encodeCursor[OptionsType any, PaginatedQueryType PaginatedQuery[OptionsType]](v *PaginatedQueryType) string { - if v == nil { - return "" - } - return bunpaginate.EncodeCursor(v) -} - func newColumnPaginator[ResourceType, OptionsType any]( query ColumnPaginatedQuery[OptionsType], fieldName string, diff --git a/internal/storage/common/resource.go b/internal/storage/common/resource.go index aaaef9a7a..a64d3da5a 100644 --- a/internal/storage/common/resource.go +++ b/internal/storage/common/resource.go @@ -10,7 +10,6 @@ import ( "github.com/formancehq/go-libs/v3/query" "github.com/formancehq/go-libs/v3/time" "github.com/uptrace/bun" - "math/big" "slices" "strings" ) @@ -255,6 +254,8 @@ func NewResourceRepository[ResourceType, OptionsType any]( } type PaginatedResourceRepository[ResourceType, OptionsType any] struct { + defaultPaginationColumn string + defaultOrder bunpaginate.Order *ResourceRepository[ResourceType, OptionsType] } @@ -267,6 +268,17 @@ func (r *PaginatedResourceRepository[ResourceType, OptionsType]) Paginate( case OffsetPaginatedQuery[OptionsType]: case ColumnPaginatedQuery[OptionsType]: case InitialPaginatedQuery[OptionsType]: + + if v.Column == "" { + v.Column = r.defaultPaginationColumn + } + if v.Order == nil { + v.Order = pointer.For(r.defaultOrder) + } + if v.PageSize == 0 { + v.PageSize = bunpaginate.QueryDefaultPageSize + } + _, field := r.resourceHandler.Schema().GetFieldByNameOrAlias(v.Column) if field == nil { return nil, fmt.Errorf("invalid property '%s' for pagination", v.Column) @@ -331,9 +343,13 @@ func (r *PaginatedResourceRepository[ResourceType, OptionsType]) Paginate( func NewPaginatedResourceRepository[ResourceType, OptionsType any]( handler RepositoryHandler[OptionsType], + defaultPaginationColumn string, + defaultOrder bunpaginate.Order, ) *PaginatedResourceRepository[ResourceType, OptionsType] { return &PaginatedResourceRepository[ResourceType, OptionsType]{ - ResourceRepository: NewResourceRepository[ResourceType, OptionsType](handler), + ResourceRepository: NewResourceRepository[ResourceType, OptionsType](handler), + defaultPaginationColumn: defaultPaginationColumn, + defaultOrder: defaultOrder, } } @@ -369,9 +385,13 @@ func (m PaginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, func NewPaginatedResourceRepositoryMapper[ToResourceType any, OriginalResourceType interface { ToCore() ToResourceType -}, OptionsType any](handler RepositoryHandler[OptionsType]) *PaginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, OptionsType] { +}, OptionsType any]( + handler RepositoryHandler[OptionsType], + defaultPaginationColumn string, + defaultOrder bunpaginate.Order, +) *PaginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, OptionsType] { return &PaginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, OptionsType]{ - PaginatedResourceRepository: NewPaginatedResourceRepository[OriginalResourceType, OptionsType](handler), + PaginatedResourceRepository: NewPaginatedResourceRepository[OriginalResourceType, OptionsType](handler, defaultPaginationColumn, defaultOrder), } } @@ -414,43 +434,6 @@ type Resource[ResourceType, OptionsType any] interface { Count(ctx context.Context, query ResourceQuery[OptionsType]) (int, error) } -type ( - CursorMetadata struct { - Previous string - Next string - PageSize int - HasMore bool - } - InitialPaginatedQuery[OptionsType any] struct { - Column string `json:"column"` - Order *bunpaginate.Order `json:"order"` - PageSize uint64 `json:"pageSize"` - Options ResourceQuery[OptionsType] `json:"filters"` - } - OffsetPaginatedQuery[OptionsType any] struct { - InitialPaginatedQuery[OptionsType] - Offset uint64 `json:"offset"` - } - ColumnPaginatedQuery[OptionsType any] struct { - InitialPaginatedQuery[OptionsType] - Bottom *big.Int `json:"bottom"` - PaginationID *big.Int `json:"paginationID"` - Reverse bool `json:"reverse"` - } - PaginatedQuery[OptionsType any] interface { - // Marker - isPaginatedQuery() - } -) - -func (i InitialPaginatedQuery[OptionsType]) isPaginatedQuery() {} - -var _ PaginatedQuery[any] = (*InitialPaginatedQuery[any])(nil) - -var _ PaginatedQuery[any] = (*OffsetPaginatedQuery[any])(nil) - -var _ PaginatedQuery[any] = (*ColumnPaginatedQuery[any])(nil) - type PaginatedResource[ResourceType, OptionsType any] interface { Resource[ResourceType, OptionsType] Paginate(ctx context.Context, paginationOptions PaginatedQuery[OptionsType]) (*bunpaginate.Cursor[ResourceType], error) diff --git a/internal/storage/driver/driver_test.go b/internal/storage/driver/driver_test.go index b3ff78bec..5d9927b6b 100644 --- a/internal/storage/driver/driver_test.go +++ b/internal/storage/driver/driver_test.go @@ -8,8 +8,8 @@ import ( "github.com/formancehq/go-libs/v3/metadata" "github.com/formancehq/go-libs/v3/query" ledger "github.com/formancehq/ledger/internal" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" "github.com/formancehq/ledger/internal/storage/bucket" + storagecommon "github.com/formancehq/ledger/internal/storage/common" "github.com/formancehq/ledger/internal/storage/driver" ledgerstore "github.com/formancehq/ledger/internal/storage/ledger" "github.com/formancehq/ledger/internal/storage/system" @@ -102,8 +102,11 @@ func TestLedgersList(t *testing.T) { _, err = d.CreateLedger(ctx, l2) require.NoError(t, err) - q := ledgercontroller.NewListLedgersQuery(15) - q.Options.Builder = query.Match("bucket", bucket) + q := storagecommon.InitialPaginatedQuery[any]{ + Options: storagecommon.ResourceQuery[any]{ + Builder: query.Match("bucket", bucket), + }, + } cursor, err := d.ListLedgers(ctx, q) require.NoError(t, err) diff --git a/internal/storage/ledger/accounts_test.go b/internal/storage/ledger/accounts_test.go index f8deb198f..4c7db2796 100644 --- a/internal/storage/ledger/accounts_test.go +++ b/internal/storage/ledger/accounts_test.go @@ -4,7 +4,6 @@ package ledger_test import ( "context" - "github.com/formancehq/go-libs/v3/bun/bunpaginate" "github.com/formancehq/ledger/internal/storage/common" "math/big" "testing" @@ -78,10 +77,7 @@ func TestAccountsList(t *testing.T) { t.Run("list all", func(t *testing.T) { t.Parallel() - accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{ - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), - }) + accounts, err := store.Accounts().Paginate(ctx, common.InitialPaginatedQuery[any]{}) require.NoError(t, err) require.Len(t, accounts.Data, 7) }) @@ -92,9 +88,6 @@ func TestAccountsList(t *testing.T) { Options: common.ResourceQuery[any]{ Builder: query.Match("metadata[category]", "1"), }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) @@ -106,9 +99,6 @@ func TestAccountsList(t *testing.T) { Options: common.ResourceQuery[any]{ PIT: &now, }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 2) @@ -122,9 +112,6 @@ func TestAccountsList(t *testing.T) { Builder: query.Match("address", "account:1"), Expand: []string{"volumes"}, }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) @@ -142,9 +129,6 @@ func TestAccountsList(t *testing.T) { PIT: &now, Expand: []string{"volumes"}, }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) @@ -161,9 +145,6 @@ func TestAccountsList(t *testing.T) { Builder: query.Match("address", "account:1"), Expand: []string{"effectiveVolumes"}, }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) @@ -180,9 +161,6 @@ func TestAccountsList(t *testing.T) { PIT: &now, Expand: []string{"effectiveVolumes"}, }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) @@ -197,9 +175,6 @@ func TestAccountsList(t *testing.T) { Options: common.ResourceQuery[any]{ Builder: query.Match("address", "account:"), }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 3) @@ -210,9 +185,6 @@ func TestAccountsList(t *testing.T) { Options: common.ResourceQuery[any]{ Builder: query.Match("address", "account:..."), }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 3) @@ -226,9 +198,6 @@ func TestAccountsList(t *testing.T) { query.Match("address", "orders:"), ), }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 3) @@ -239,9 +208,6 @@ func TestAccountsList(t *testing.T) { Options: common.ResourceQuery[any]{ Builder: query.Lt("balance[USD]", 0), }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) // world @@ -250,9 +216,6 @@ func TestAccountsList(t *testing.T) { Options: common.ResourceQuery[any]{ Builder: query.Gt("balance[USD]", 0), }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 2) @@ -266,9 +229,6 @@ func TestAccountsList(t *testing.T) { Builder: query.Lt("balance[USD]", 0), PIT: &now, }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) // world @@ -280,9 +240,6 @@ func TestAccountsList(t *testing.T) { Builder: query.Lt("balance", 0), PIT: &now, }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 1) // world @@ -294,9 +251,6 @@ func TestAccountsList(t *testing.T) { Options: common.ResourceQuery[any]{ Builder: query.Exists("metadata", "foo"), }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 2) @@ -305,8 +259,6 @@ func TestAccountsList(t *testing.T) { Options: common.ResourceQuery[any]{ Builder: query.Exists("metadata", "category"), }, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, accounts.Data, 3) @@ -318,9 +270,6 @@ func TestAccountsList(t *testing.T) { Options: common.ResourceQuery[any]{ Builder: query.Lt("invalid", 0), }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.Error(t, err) require.True(t, errors.Is(err, common.ErrInvalidQuery{})) @@ -333,9 +282,6 @@ func TestAccountsList(t *testing.T) { Options: common.ResourceQuery[any]{ Builder: query.Lte("first_usage", now), }, - PageSize: bunpaginate.QueryDefaultPageSize, - Column: "address", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, ret.Data, 2) diff --git a/internal/storage/ledger/logs_test.go b/internal/storage/ledger/logs_test.go index 56b3c3c3e..ed7b109aa 100644 --- a/internal/storage/ledger/logs_test.go +++ b/internal/storage/ledger/logs_test.go @@ -133,7 +133,6 @@ func TestLogsInsert(t *testing.T) { logs, err := store.Logs().Paginate(ctx, common.InitialPaginatedQuery[any]{ PageSize: countLogs, Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), - Column: "id", }) require.NoError(t, err) @@ -221,10 +220,7 @@ func TestLogsList(t *testing.T) { require.NoError(t, err) } - cursor, err := store.Logs().Paginate(context.Background(), common.InitialPaginatedQuery[any]{ - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), - }) + cursor, err := store.Logs().Paginate(context.Background(), common.InitialPaginatedQuery[any]{}) require.NoError(t, err) require.Equal(t, bunpaginate.QueryDefaultPageSize, cursor.PageSize) @@ -232,8 +228,6 @@ func TestLogsList(t *testing.T) { require.EqualValues(t, 3, *cursor.Data[0].ID) cursor, err = store.Logs().Paginate(context.Background(), common.InitialPaginatedQuery[any]{ - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), PageSize: 1, }) require.NoError(t, err) @@ -249,7 +243,6 @@ func TestLogsList(t *testing.T) { query.Lt("date", now.Add(-time.Hour)), ), }, - Column: "id", Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -263,8 +256,6 @@ func TestLogsList(t *testing.T) { Options: common.ResourceQuery[any]{ Builder: query.Lt("id", 3), }, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }) require.NoError(t, err) require.Equal(t, 2, len(cursor.Data)) diff --git a/internal/storage/ledger/store.go b/internal/storage/ledger/store.go index de94fa8be..1ecb88d80 100644 --- a/internal/storage/ledger/store.go +++ b/internal/storage/ledger/store.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "github.com/formancehq/go-libs/v3/bun/bunpaginate" "github.com/formancehq/go-libs/v3/migrations" "github.com/formancehq/go-libs/v3/platform/postgres" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" @@ -49,7 +50,7 @@ func (store *Store) Volumes() common.PaginatedResource[ return common.NewPaginatedResourceRepository[ ledger.VolumesWithBalanceByAssetByAccount, ledgercontroller.GetVolumesOptions, - ](&volumesResourceHandler{store: store}) + ](&volumesResourceHandler{store: store}, "account", bunpaginate.OrderAsc) } func (store *Store) AggregatedVolumes() common.Resource[ledger.AggregatedVolumes, ledgercontroller.GetAggregatedVolumesOptions] { @@ -61,7 +62,7 @@ func (store *Store) AggregatedVolumes() common.Resource[ledger.AggregatedVolumes func (store *Store) Transactions() common.PaginatedResource[ ledger.Transaction, any] { - return common.NewPaginatedResourceRepository[ledger.Transaction, any](&transactionsResourceHandler{store: store}) + return common.NewPaginatedResourceRepository[ledger.Transaction, any](&transactionsResourceHandler{store: store}, "id", bunpaginate.OrderDesc) } func (store *Store) Logs() common.PaginatedResource[ @@ -69,7 +70,7 @@ func (store *Store) Logs() common.PaginatedResource[ any] { return common.NewPaginatedResourceRepositoryMapper[ledger.Log, Log, any](&logsResourceHandler{ store: store, - }) + }, "id", bunpaginate.OrderDesc) } func (store *Store) Accounts() common.PaginatedResource[ @@ -77,7 +78,7 @@ func (store *Store) Accounts() common.PaginatedResource[ any] { return common.NewPaginatedResourceRepository[ledger.Account, any](&accountsResourceHandler{ store: store, - }) + }, "address", bunpaginate.OrderAsc) } func (store *Store) BeginTX(ctx context.Context, options *sql.TxOptions) (*Store, *bun.Tx, error) { diff --git a/internal/storage/ledger/transactions_test.go b/internal/storage/ledger/transactions_test.go index f687eed1c..e4801fa59 100644 --- a/internal/storage/ledger/transactions_test.go +++ b/internal/storage/ledger/transactions_test.go @@ -7,7 +7,6 @@ import ( "database/sql" "fmt" "github.com/alitto/pond" - "github.com/formancehq/go-libs/v3/bun/bunpaginate" "github.com/formancehq/ledger/internal/storage/common" "math/big" "slices" @@ -458,8 +457,6 @@ func TestTransactionsCommit(t *testing.T) { Options: common.ResourceQuery[any]{ Expand: []string{"volumes"}, }, - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), - Column: "id", }) require.NoError(t, err) require.Len(t, cursor.Data, countTx) @@ -743,174 +740,121 @@ func TestTransactionsList(t *testing.T) { type testCase struct { name string - query common.ColumnPaginatedQuery[any] + query common.InitialPaginatedQuery[any] expected []ledger.Transaction expectError error } testCases := []testCase{ { - name: "nominal", - query: common.ColumnPaginatedQuery[any]{ - InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), - }, - }, + name: "nominal", + query: common.InitialPaginatedQuery[any]{}, expected: []ledger.Transaction{tx5, tx4, tx3, tx2, tx1}, }, { name: "address filter", - query: common.ColumnPaginatedQuery[any]{ - InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("account", "bob"), - }, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + query: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("account", "bob"), }, }, expected: []ledger.Transaction{tx2}, }, { name: "address filter using segments matching two addresses by individual segments", - query: common.ColumnPaginatedQuery[any]{ - InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("account", "users:amazon"), - }, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + query: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("account", "users:amazon"), }, }, expected: []ledger.Transaction{}, }, { name: "address filter using segment", - query: common.ColumnPaginatedQuery[any]{ - InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("account", "users:"), - }, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + query: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("account", "users:"), }, }, expected: []ledger.Transaction{tx5, tx4, tx3}, }, { name: "address filter using segment and unbounded segment list", - query: common.ColumnPaginatedQuery[any]{ - InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("account", "users:..."), - }, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + query: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("account", "users:..."), }, }, expected: []ledger.Transaction{tx5, tx4, tx3}, }, { name: "filter using metadata", - query: common.ColumnPaginatedQuery[any]{ - InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("metadata[category]", "2"), - }, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + query: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("metadata[category]", "2"), }, }, expected: []ledger.Transaction{tx2}, }, { name: "using point in time", - query: common.ColumnPaginatedQuery[any]{ - InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - PIT: pointer.For(now.Add(-time.Hour)), - }, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + query: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + PIT: pointer.For(now.Add(-time.Hour)), }, }, expected: []ledger.Transaction{tx3BeforeRevert, tx2, tx1}, }, { name: "filter using invalid key", - query: common.ColumnPaginatedQuery[any]{ - InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("invalid", "2"), - }, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + query: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("invalid", "2"), }, }, expectError: common.ErrInvalidQuery{}, }, { name: "reverted transactions", - query: common.ColumnPaginatedQuery[any]{ - InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("reverted", true), - }, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + query: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("reverted", true), }, }, expected: []ledger.Transaction{tx3}, }, { name: "filter using exists metadata", - query: common.ColumnPaginatedQuery[any]{ - InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Exists("metadata", "category"), - }, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + query: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Exists("metadata", "category"), }, }, expected: []ledger.Transaction{tx3, tx2, tx1}, }, { name: "filter using metadata and pit", - query: common.ColumnPaginatedQuery[any]{ - InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("metadata[category]", "2"), - PIT: pointer.For(tx3.Timestamp), - }, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + query: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("metadata[category]", "2"), + PIT: pointer.For(tx3.Timestamp), }, }, expected: []ledger.Transaction{tx2}, }, { name: "filter using not exists metadata", - query: common.ColumnPaginatedQuery[any]{ - InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Not(query.Exists("metadata", "category")), - }, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + query: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Not(query.Exists("metadata", "category")), }, }, expected: []ledger.Transaction{tx5, tx4}, }, { name: "filter using timestamp", - query: common.ColumnPaginatedQuery[any]{ - InitialPaginatedQuery: common.InitialPaginatedQuery[any]{ - Options: common.ResourceQuery[any]{ - Builder: query.Match("timestamp", tx5.Timestamp.Format(time.RFC3339Nano)), - }, - Column: "id", - Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + query: common.InitialPaginatedQuery[any]{ + Options: common.ResourceQuery[any]{ + Builder: query.Match("timestamp", tx5.Timestamp.Format(time.RFC3339Nano)), }, }, expected: []ledger.Transaction{tx5, tx4}, diff --git a/internal/storage/ledger/volumes_test.go b/internal/storage/ledger/volumes_test.go index e60a5e3ef..795d6687a 100644 --- a/internal/storage/ledger/volumes_test.go +++ b/internal/storage/ledger/volumes_test.go @@ -4,7 +4,6 @@ package ledger_test import ( "database/sql" - "github.com/formancehq/go-libs/v3/bun/bunpaginate" "github.com/formancehq/go-libs/v3/pointer" "github.com/formancehq/ledger/internal/storage/common" "math/big" @@ -111,8 +110,6 @@ func TestVolumesList(t *testing.T) { Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Builder: query.Lt("first_usage", now.Add(-3*time.Minute)), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 2) @@ -143,8 +140,6 @@ func TestVolumesList(t *testing.T) { Builder: query.Lt("first_usage", now.Add(-3*time.Minute)), PIT: pointer.For(now.Add(-3 * time.Minute)), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 2) @@ -176,8 +171,6 @@ func TestVolumesList(t *testing.T) { UseInsertionDate: true, }, }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -185,10 +178,7 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for effective date", func(t *testing.T) { t.Parallel() - volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{ - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), - }) + volumes, err := store.Volumes().Paginate(ctx, common.InitialPaginatedQuery[ledgercontroller.GetVolumesOptions]{}) require.NoError(t, err) require.Len(t, volumes.Data, 4) }) @@ -202,8 +192,6 @@ func TestVolumesList(t *testing.T) { }, PIT: &previousPIT, }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -228,8 +216,6 @@ func TestVolumesList(t *testing.T) { }, PIT: &futurPIT, }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -244,8 +230,6 @@ func TestVolumesList(t *testing.T) { }, OOT: &previousOOT, }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -260,8 +244,6 @@ func TestVolumesList(t *testing.T) { }, OOT: &futurOOT, }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -283,8 +265,6 @@ func TestVolumesList(t *testing.T) { Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &previousPIT, }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -306,8 +286,6 @@ func TestVolumesList(t *testing.T) { Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ PIT: &futurPIT, }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -319,8 +297,6 @@ func TestVolumesList(t *testing.T) { Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ OOT: &previousOOT, }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -332,8 +308,6 @@ func TestVolumesList(t *testing.T) { Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ OOT: &futurOOT, }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -359,8 +333,6 @@ func TestVolumesList(t *testing.T) { PIT: &futurPIT, OOT: &now, }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -386,8 +358,6 @@ func TestVolumesList(t *testing.T) { PIT: &now, OOT: &previousOOT, }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -410,8 +380,6 @@ func TestVolumesList(t *testing.T) { PIT: &futurPIT, OOT: &now, }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -434,8 +402,6 @@ func TestVolumesList(t *testing.T) { PIT: &now, OOT: &previousOOT, }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -461,8 +427,6 @@ func TestVolumesList(t *testing.T) { OOT: &previousOOT, Builder: query.Match("account", "account:1"), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }, ) require.NoError(t, err) @@ -487,8 +451,6 @@ func TestVolumesList(t *testing.T) { Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Builder: query.Match("metadata[foo]", "bar"), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }, ) require.NoError(t, err) @@ -503,8 +465,6 @@ func TestVolumesList(t *testing.T) { Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Builder: query.Exists("metadata", "category"), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }, ) require.NoError(t, err) @@ -519,8 +479,6 @@ func TestVolumesList(t *testing.T) { Options: common.ResourceQuery[ledgercontroller.GetVolumesOptions]{ Builder: query.Exists("metadata", "foo"), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }, ) require.NoError(t, err) @@ -595,8 +553,6 @@ func TestVolumesAggregate(t *testing.T) { }, Builder: query.Match("account", "account::"), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 7) @@ -612,8 +568,6 @@ func TestVolumesAggregate(t *testing.T) { }, Builder: query.Match("account", "account::"), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 2) @@ -629,8 +583,6 @@ func TestVolumesAggregate(t *testing.T) { }, Builder: query.Match("account", "account::"), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -646,8 +598,6 @@ func TestVolumesAggregate(t *testing.T) { }, Builder: query.Match("account", "account::"), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 7) @@ -665,8 +615,6 @@ func TestVolumesAggregate(t *testing.T) { OOT: &oot, Builder: query.Match("account", "account::"), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 2) @@ -702,8 +650,6 @@ func TestVolumesAggregate(t *testing.T) { OOT: &oot, Builder: query.And(query.Match("account", "account::"), query.Gte("balance[EUR]", 50)), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 1) @@ -730,8 +676,6 @@ func TestVolumesAggregate(t *testing.T) { query.Match("account", "account:1:"), query.Lte("balance[USD]", 0)), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) require.Len(t, volumes.Data, 3) @@ -777,8 +721,6 @@ func TestVolumesAggregate(t *testing.T) { query.Match("metadata[foo]", "bar"), ), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -800,8 +742,6 @@ func TestVolumesAggregate(t *testing.T) { query.Match("metadata[foo]", "bar"), ), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }) require.NoError(t, err) @@ -819,8 +759,6 @@ func TestVolumesAggregate(t *testing.T) { }, Builder: query.Match("metadata[foo]", "bar"), }, - Column: "account", - Order: (*bunpaginate.Order)(pointer.For(bunpaginate.OrderAsc)), }, ) require.NoError(t, err) diff --git a/internal/storage/system/store.go b/internal/storage/system/store.go index bb6cd9b8e..efdff686b 100644 --- a/internal/storage/system/store.go +++ b/internal/storage/system/store.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/formancehq/go-libs/v3/bun/bunpaginate" "github.com/formancehq/go-libs/v3/metadata" "github.com/formancehq/go-libs/v3/migrations" "github.com/formancehq/go-libs/v3/platform/postgres" @@ -97,7 +98,7 @@ func (d *DefaultStore) DeleteLedgerMetadata(ctx context.Context, name string, ke func (d *DefaultStore) Ledgers() common.PaginatedResource[ ledger.Ledger, any] { - return common.NewPaginatedResourceRepository[ledger.Ledger, any](&ledgersResourceHandler{store: d}) + return common.NewPaginatedResourceRepository[ledger.Ledger, any](&ledgersResourceHandler{store: d}, "id", bunpaginate.OrderAsc) } func (d *DefaultStore) GetLedger(ctx context.Context, name string) (*ledger.Ledger, error) { diff --git a/internal/storage/system/store_test.go b/internal/storage/system/store_test.go index c3778abae..866c67778 100644 --- a/internal/storage/system/store_test.go +++ b/internal/storage/system/store_test.go @@ -11,8 +11,7 @@ import ( "github.com/formancehq/go-libs/v3/metadata" "github.com/formancehq/go-libs/v3/testing/docker" ledger "github.com/formancehq/ledger/internal" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" - "github.com/formancehq/ledger/internal/storage/common" + storagecommon "github.com/formancehq/ledger/internal/storage/common" "github.com/google/uuid" "github.com/uptrace/bun" "golang.org/x/sync/errgroup" @@ -92,13 +91,15 @@ func TestLedgersList(t *testing.T) { ledgers = append(ledgers, l) } - cursor, err := store.Ledgers().Paginate(ctx, ledgercontroller.NewListLedgersQuery(pageSize)) + cursor, err := store.Ledgers().Paginate(ctx, storagecommon.InitialPaginatedQuery[any]{ + PageSize: pageSize, + }) require.NoError(t, err) require.Len(t, cursor.Data, int(pageSize)) require.Equal(t, ledgers[:pageSize], cursor.Data) for i := pageSize; i < count; i += pageSize { - query := common.ColumnPaginatedQuery[any]{} + query := storagecommon.ColumnPaginatedQuery[any]{} require.NoError(t, bunpaginate.UnmarshalCursor(cursor.Next, &query)) cursor, err = store.Ledgers().Paginate(ctx, query) diff --git a/internal/worker/async_block.go b/internal/worker/async_block.go index c5a5a4ee9..c7d6b4bc6 100644 --- a/internal/worker/async_block.go +++ b/internal/worker/async_block.go @@ -7,8 +7,7 @@ import ( "github.com/formancehq/go-libs/v3/logging" "github.com/formancehq/go-libs/v3/query" "github.com/formancehq/ledger/internal" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" - "github.com/formancehq/ledger/internal/storage/common" + storagecommon "github.com/formancehq/ledger/internal/storage/common" systemstore "github.com/formancehq/ledger/internal/storage/system" "github.com/formancehq/ledger/pkg/features" "github.com/robfig/cron/v3" @@ -77,10 +76,13 @@ func (r *AsyncBlockRunner) run(ctx context.Context) error { ctx, span := r.tracer.Start(ctx, "Run") defer span.End() - initialQuery := ledgercontroller.NewListLedgersQuery(10) - initialQuery.Options.Builder = query.Match(fmt.Sprintf("features[%s]", features.FeatureHashLogs), "ASYNC") + initialQuery := storagecommon.InitialPaginatedQuery[any]{ + Options: storagecommon.ResourceQuery[any]{ + Builder: query.Match(fmt.Sprintf("features[%s]", features.FeatureHashLogs), "ASYNC"), + }, + } systemStore := systemstore.New(r.db) - return common.Iterate( + return storagecommon.Iterate( ctx, initialQuery, systemStore.Ledgers().Paginate, From 858712a5f64959e6c343a1e9c98c5a4c0842fe38 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Fri, 20 Jun 2025 09:18:26 +0200 Subject: [PATCH 4/4] chore: add some integration tests regarding api v1 --- go.mod | 2 +- internal/api/v2/controllers_ledgers_list.go | 2 - internal/controller/ledger/mocks_test.go | 36 --- internal/storage/common/paginator_offset.go | 3 +- openapi.yaml | 2 +- openapi/v1.yaml | 2 +- pkg/client/.speakeasy/gen.lock | 2 +- pkg/client/v1.go | 2 +- test/e2e/v1_api_accounts_list_test.go | 305 ++++++++++++++++++ test/e2e/v1_api_logs_list_test.go | 328 ++++++++++++++++++++ test/e2e/v1_api_transactions_list_test.go | 203 ++++++++++++ 11 files changed, 842 insertions(+), 45 deletions(-) create mode 100644 test/e2e/v1_api_accounts_list_test.go create mode 100644 test/e2e/v1_api_logs_list_test.go create mode 100644 test/e2e/v1_api_transactions_list_test.go diff --git a/go.mod b/go.mod index 56e6cbc14..051664a6c 100644 --- a/go.mod +++ b/go.mod @@ -104,7 +104,7 @@ require ( github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/containerd/continuity v0.4.5 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/dnwe/otelsarama v0.0.0-20240308230250-9388d9d40bc0 // indirect github.com/docker/cli v27.4.1+incompatible // indirect diff --git a/internal/api/v2/controllers_ledgers_list.go b/internal/api/v2/controllers_ledgers_list.go index 1b6cdb8ff..d7cd782fc 100644 --- a/internal/api/v2/controllers_ledgers_list.go +++ b/internal/api/v2/controllers_ledgers_list.go @@ -1,7 +1,6 @@ package v2 import ( - "github.com/davecgh/go-spew/spew" "github.com/formancehq/ledger/internal/api/common" storagecommon "github.com/formancehq/ledger/internal/storage/common" "net/http" @@ -22,7 +21,6 @@ func listLedgers(b system.Controller, paginationConfig common.PaginationConfig) return } - spew.Dump(rq) ledgers, err := b.ListLedgers(r.Context(), rq) if err != nil { switch { diff --git a/internal/controller/ledger/mocks_test.go b/internal/controller/ledger/mocks_test.go index 52222cc7b..50dbd4178 100644 --- a/internal/controller/ledger/mocks_test.go +++ b/internal/controller/ledger/mocks_test.go @@ -209,42 +209,6 @@ func (mr *MockResourceMockRecorder[ResourceType, OptionsType]) GetOne(ctx, query return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOne", reflect.TypeOf((*MockResource[ResourceType, OptionsType])(nil).GetOne), ctx, query) } -// MockPaginatedQuery is a mock of PaginatedQuery interface. -type MockPaginatedQuery[OptionsType any] struct { - ctrl *gomock.Controller - recorder *MockPaginatedQueryMockRecorder[OptionsType] - isgomock struct{} -} - -// MockPaginatedQueryMockRecorder is the mock recorder for MockPaginatedQuery. -type MockPaginatedQueryMockRecorder[OptionsType any] struct { - mock *MockPaginatedQuery[OptionsType] -} - -// NewMockPaginatedQuery creates a new mock instance. -func NewMockPaginatedQuery[OptionsType any](ctrl *gomock.Controller) *MockPaginatedQuery[OptionsType] { - mock := &MockPaginatedQuery[OptionsType]{ctrl: ctrl} - mock.recorder = &MockPaginatedQueryMockRecorder[OptionsType]{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockPaginatedQuery[OptionsType]) EXPECT() *MockPaginatedQueryMockRecorder[OptionsType] { - return m.recorder -} - -// isPaginatedQuery mocks base method. -func (m *MockPaginatedQuery[OptionsType]) isPaginatedQuery() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "isPaginatedQuery") -} - -// isPaginatedQuery indicates an expected call of isPaginatedQuery. -func (mr *MockPaginatedQueryMockRecorder[OptionsType]) isPaginatedQuery() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "isPaginatedQuery", reflect.TypeOf((*MockPaginatedQuery[OptionsType])(nil).isPaginatedQuery)) -} - // MockPaginatedResource is a mock of PaginatedResource interface. type MockPaginatedResource[ResourceType any, OptionsType any] struct { ctrl *gomock.Controller diff --git a/internal/storage/common/paginator_offset.go b/internal/storage/common/paginator_offset.go index 09bb4a4d7..e83b26f7c 100644 --- a/internal/storage/common/paginator_offset.go +++ b/internal/storage/common/paginator_offset.go @@ -26,7 +26,6 @@ func (o OffsetPaginator[ResourceType, OptionsType]) Paginate(sb *bun.SelectQuery if o.query.Offset > 0 { sb = sb.Offset(int(o.query.Offset)) } - if o.query.PageSize > 0 { sb = sb.Limit(int(o.query.PageSize) + 1) } @@ -77,4 +76,4 @@ func newOffsetPaginator[ResourceType, OptionsType any]( query OffsetPaginatedQuery[OptionsType], ) OffsetPaginator[ResourceType, OptionsType] { return OffsetPaginator[ResourceType, OptionsType]{query: query} -} \ No newline at end of file +} diff --git a/openapi.yaml b/openapi.yaml index f99927bc2..4a71d929d 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -88,7 +88,7 @@ paths: additionalProperties: true example: metadata[key]=value1&metadata[a.nested.key]=value2 responses: - '200': + '204': description: OK headers: Count: diff --git a/openapi/v1.yaml b/openapi/v1.yaml index e78b54e0d..23cdba81b 100644 --- a/openapi/v1.yaml +++ b/openapi/v1.yaml @@ -90,7 +90,7 @@ paths: additionalProperties: true example: metadata[key]=value1&metadata[a.nested.key]=value2 responses: - '200': + '204': description: OK headers: Count: diff --git a/pkg/client/.speakeasy/gen.lock b/pkg/client/.speakeasy/gen.lock index 71882f76f..244bae39c 100644 --- a/pkg/client/.speakeasy/gen.lock +++ b/pkg/client/.speakeasy/gen.lock @@ -1,7 +1,7 @@ lockVersion: 2.0.0 id: a9ac79e1-e429-4ee3-96c4-ec973f19bec3 management: - docChecksum: 97c4248e367a820c2225d188361a800c + docChecksum: cf53669174b68bdafc357d6575035712 docVersion: v2 speakeasyVersion: 1.563.0 generationVersion: 2.629.1 diff --git a/pkg/client/v1.go b/pkg/client/v1.go index 535e7a3c2..4f34fb086 100644 --- a/pkg/client/v1.go +++ b/pkg/client/v1.go @@ -611,7 +611,7 @@ func (s *V1) CountAccounts(ctx context.Context, request operations.CountAccounts } switch { - case httpRes.StatusCode == 200: + case httpRes.StatusCode == 204: res.Headers = httpRes.Header default: diff --git a/test/e2e/v1_api_accounts_list_test.go b/test/e2e/v1_api_accounts_list_test.go new file mode 100644 index 000000000..1a89c2a12 --- /dev/null +++ b/test/e2e/v1_api_accounts_list_test.go @@ -0,0 +1,305 @@ +//go:build it + +package test_suite + +import ( + "fmt" + "github.com/formancehq/go-libs/v3/logging" + . "github.com/formancehq/go-libs/v3/testing/deferred/ginkgo" + "github.com/formancehq/go-libs/v3/testing/platform/natstesting" + "github.com/formancehq/go-libs/v3/testing/platform/pgtesting" + "github.com/formancehq/go-libs/v3/testing/testservice" + "github.com/formancehq/ledger/pkg/client/models/components" + "github.com/formancehq/ledger/pkg/client/models/operations" + . "github.com/formancehq/ledger/pkg/testserver" + "github.com/formancehq/ledger/pkg/testserver/ginkgo" + "math/big" + "sort" + "time" + + "github.com/formancehq/go-libs/v3/pointer" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Context("Ledger accounts list API tests", func() { + var ( + db = UseTemplatedDatabase() + ctx = logging.TestingContext() + ) + + testServer := ginkgo.DeferTestServer( + DeferMap(db, (*pgtesting.Database).ConnectionOptions), + testservice.WithInstruments( + testservice.NatsInstrumentation(DeferMap(natsServer, (*natstesting.NatsServer).ClientURL)), + testservice.DebugInstrumentation(debug), + testservice.OutputInstrumentation(GinkgoWriter), + ), + testservice.WithLogger(GinkgoT()), + ) + + When("counting and listing accounts", func() { + var ( + metadata1 = map[string]any{ + "clientType": "gold", + } + + metadata2 = map[string]any{ + "clientType": "silver", + } + + timestamp = time.Now().Round(time.Second).UTC() + bigInt, _ = big.NewInt(0).SetString("999999999999999999999999999999999999999999999999999999999999999999999999999999999999999", 10) + ) + BeforeEach(func(specContext SpecContext) { + _, err := Wait(specContext, DeferClient(testServer)).Ledger.V1.AddMetadataToAccount( + ctx, + operations.AddMetadataToAccountRequest{ + RequestBody: metadata1, + Address: "foo:foo", + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + + _, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.AddMetadataToAccount( + ctx, + operations.AddMetadataToAccountRequest{ + RequestBody: metadata2, + Address: "foo:bar", + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + + _, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.CreateTransaction( + ctx, + operations.CreateTransactionRequest{ + PostTransaction: components.PostTransaction{ + Metadata: map[string]any{}, + Postings: []components.Posting{{ + Amount: bigInt, + Asset: "USD", + Source: "world", + Destination: "foo:foo", + }}, + Timestamp: ×tamp, + }, + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + It("should be countable on api", func(specContext SpecContext) { + response, err := Wait(specContext, DeferClient(testServer)).Ledger.V1.CountAccounts( + ctx, + operations.CountAccountsRequest{ + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + Expect(response.Headers["Count"]).To(Equal([]string{"3"})) + }) + It("should be listed on api", func(specContext SpecContext) { + response, err := Wait(specContext, DeferClient(testServer)).Ledger.V1.ListAccounts( + ctx, + operations.ListAccountsRequest{ + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + + accountsCursorResponse := response.AccountsCursorResponse.Cursor.Data + Expect(accountsCursorResponse).To(HaveLen(3)) + Expect(accountsCursorResponse[0]).To(Equal(components.Account{ + Address: "foo:bar", + Metadata: metadata2, + })) + Expect(accountsCursorResponse[1]).To(Equal(components.Account{ + Address: "foo:foo", + Metadata: metadata1, + })) + Expect(accountsCursorResponse[2]).To(Equal(components.Account{ + Address: "world", + Metadata: map[string]any{}, + })) + }) + It("should be listed on api using address filters", func(specContext SpecContext) { + response, err := Wait(specContext, DeferClient(testServer)).Ledger.V1.ListAccounts( + ctx, + operations.ListAccountsRequest{ + Ledger: "default", + Address: pointer.For("foo:"), + }, + ) + Expect(err).ToNot(HaveOccurred()) + + accountsCursorResponse := response.AccountsCursorResponse.Cursor.Data + Expect(accountsCursorResponse).To(HaveLen(2)) + Expect(accountsCursorResponse[0]).To(Equal(components.Account{ + Address: "foo:bar", + Metadata: metadata2, + })) + Expect(accountsCursorResponse[1]).To(Equal(components.Account{ + Address: "foo:foo", + Metadata: metadata1, + })) + + response, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.ListAccounts( + ctx, + operations.ListAccountsRequest{ + Ledger: "default", + Address: pointer.For(":foo"), + }, + ) + Expect(err).ToNot(HaveOccurred()) + + accountsCursorResponse = response.AccountsCursorResponse.Cursor.Data + Expect(accountsCursorResponse).To(HaveLen(1)) + Expect(accountsCursorResponse[0]).To(Equal(components.Account{ + Address: "foo:foo", + Metadata: metadata1, + })) + }) + It("should be listed on api using metadata filters", func(specContext SpecContext) { + response, err := Wait(specContext, DeferClient(testServer)).Ledger.V1.ListAccounts( + ctx, + operations.ListAccountsRequest{ + Ledger: "default", + Metadata: map[string]any{ + "clientType": "gold", + }, + }, + ) + Expect(err).ToNot(HaveOccurred()) + + accountsCursorResponse := response.AccountsCursorResponse.Cursor.Data + Expect(accountsCursorResponse).To(HaveLen(1)) + Expect(accountsCursorResponse[0]).To(Equal(components.Account{ + Address: "foo:foo", + Metadata: metadata1, + })) + }) + }) + + When("counting and listing accounts empty", func() { + It("should be countable on api even if empty", func(specContext SpecContext) { + response, err := Wait(specContext, DeferClient(testServer)).Ledger.V1.CountAccounts( + ctx, + operations.CountAccountsRequest{ + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + Expect(response.Headers["Count"]).To(Equal([]string{"0"})) + }) + It("should be listed on api even if empty", func(specContext SpecContext) { + response, err := Wait(specContext, DeferClient(testServer)).Ledger.V1.ListAccounts( + ctx, + operations.ListAccountsRequest{ + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + Expect(response.AccountsCursorResponse.Cursor.Data).To(HaveLen(0)) + }) + }) + + const ( + pageSize = int64(10) + accountCounts = 2 * pageSize + ) + When("creating accounts", func() { + var ( + accounts []components.Account + ) + BeforeEach(func(specContext SpecContext) { + for i := 0; i < int(accountCounts); i++ { + m := map[string]any{ + "id": fmt.Sprintf("%d", i), + } + + _, err := Wait(specContext, DeferClient(testServer)).Ledger.V1.AddMetadataToAccount( + ctx, + operations.AddMetadataToAccountRequest{ + RequestBody: m, + Address: fmt.Sprintf("foo:%d", i), + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + + accounts = append(accounts, components.Account{ + Address: fmt.Sprintf("foo:%d", i), + Metadata: m, + }) + + sort.Slice(accounts, func(i, j int) bool { + return accounts[i].Address < accounts[j].Address + }) + } + }) + AfterEach(func() { + accounts = nil + }) + When(fmt.Sprintf("listing accounts using page size of %d", pageSize), func() { + var ( + response *operations.ListAccountsResponse + err error + ) + BeforeEach(func(specContext SpecContext) { + response, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.ListAccounts( + ctx, + operations.ListAccountsRequest{ + Ledger: "default", + PageSize: pointer.For(pageSize), + }, + ) + Expect(err).ToNot(HaveOccurred()) + + Expect(response.AccountsCursorResponse.Cursor.HasMore).To(BeTrue()) + Expect(response.AccountsCursorResponse.Cursor.Previous).To(BeNil()) + Expect(response.AccountsCursorResponse.Cursor.Next).NotTo(BeNil()) + }) + It("should return the first page", func() { + Expect(response.AccountsCursorResponse.Cursor.PageSize).To(Equal(pageSize)) + Expect(response.AccountsCursorResponse.Cursor.Data).To(Equal(accounts[:pageSize])) + }) + When("following next cursor", func() { + BeforeEach(func(specContext SpecContext) { + response, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.ListAccounts( + ctx, + operations.ListAccountsRequest{ + Cursor: response.AccountsCursorResponse.Cursor.Next, + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + It("should return next page", func() { + Expect(response.AccountsCursorResponse.Cursor.PageSize).To(Equal(pageSize)) + Expect(response.AccountsCursorResponse.Cursor.Data).To(Equal(accounts[pageSize : 2*pageSize])) + Expect(response.AccountsCursorResponse.Cursor.Next).To(BeNil()) + }) + When("following previous cursor", func() { + BeforeEach(func(specContext SpecContext) { + response, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.ListAccounts( + ctx, + operations.ListAccountsRequest{ + Ledger: "default", + Cursor: response.AccountsCursorResponse.Cursor.Previous, + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + It("should return first page", func() { + Expect(response.AccountsCursorResponse.Cursor.PageSize).To(Equal(pageSize)) + Expect(response.AccountsCursorResponse.Cursor.Data).To(Equal(accounts[:pageSize])) + Expect(response.AccountsCursorResponse.Cursor.Previous).To(BeNil()) + }) + }) + }) + }) + }) +}) diff --git a/test/e2e/v1_api_logs_list_test.go b/test/e2e/v1_api_logs_list_test.go new file mode 100644 index 000000000..d84e58ba2 --- /dev/null +++ b/test/e2e/v1_api_logs_list_test.go @@ -0,0 +1,328 @@ +//go:build it + +package test_suite + +import ( + "fmt" + "github.com/formancehq/go-libs/v3/logging" + "github.com/formancehq/go-libs/v3/pointer" + . "github.com/formancehq/go-libs/v3/testing/deferred/ginkgo" + "github.com/formancehq/go-libs/v3/testing/platform/pgtesting" + "github.com/formancehq/go-libs/v3/testing/testservice" + "github.com/formancehq/ledger/pkg/client/models/components" + "github.com/formancehq/ledger/pkg/client/models/operations" + . "github.com/formancehq/ledger/pkg/testserver" + "github.com/formancehq/ledger/pkg/testserver/ginkgo" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "math/big" + "sort" + "time" +) + +var _ = Context("Ledger logs list API tests", func() { + var ( + db = UseTemplatedDatabase() + ctx = logging.TestingContext() + ) + + testServer := ginkgo.DeferTestServer( + DeferMap(db, (*pgtesting.Database).ConnectionOptions), + testservice.WithInstruments( + testservice.DebugInstrumentation(debug), + testservice.OutputInstrumentation(GinkgoWriter), + ), + testservice.WithLogger(GinkgoT()), + ) + + BeforeEach(func(specContext SpecContext) { + _, err := Wait(specContext, DeferClient(testServer)).Ledger.V2.CreateLedger(ctx, operations.V2CreateLedgerRequest{ + Ledger: "default", + }) + Expect(err).To(BeNil()) + + _, err = Wait(specContext, DeferClient(testServer)).Ledger.V2.CreateLedger(ctx, operations.V2CreateLedgerRequest{ + Ledger: "another", + }) + Expect(err).To(BeNil()) + }) + When("listing logs", func() { + var ( + timestamp1 = time.Date(2023, 4, 11, 10, 0, 0, 0, time.UTC) + timestamp2 = time.Date(2023, 4, 12, 10, 0, 0, 0, time.UTC) + + m1 = map[string]any{ + "clientType": "silver", + } + m2 = map[string]any{ + "clientType": "gold", + } + ) + BeforeEach(func(specContext SpecContext) { + _, err := Wait(specContext, DeferClient(testServer)).Ledger.V1.CreateTransaction( + ctx, + operations.CreateTransactionRequest{ + PostTransaction: components.PostTransaction{ + Metadata: map[string]any{}, + Postings: []components.Posting{{ + Amount: big.NewInt(100), + Asset: "USD", + Source: "world", + Destination: "foo:foo", + }}, + Timestamp: ×tamp1, + }, + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + + _, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.CreateTransaction( + ctx, + operations.CreateTransactionRequest{ + PostTransaction: components.PostTransaction{ + Metadata: map[string]any{}, + Postings: []components.Posting{{ + Amount: big.NewInt(100), + Asset: "USD", + Source: "world", + Destination: "foo:foo", + }}, + Timestamp: ×tamp1, + }, + Ledger: "another", + }, + ) + Expect(err).ToNot(HaveOccurred()) + + _, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.CreateTransaction( + ctx, + operations.CreateTransactionRequest{ + PostTransaction: components.PostTransaction{ + Metadata: m1, + Postings: []components.Posting{{ + Amount: big.NewInt(200), + Asset: "USD", + Source: "world", + Destination: "foo:bar", + }}, + Timestamp: ×tamp2, + }, + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + + _, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.AddMetadataToAccount( + ctx, + operations.AddMetadataToAccountRequest{ + RequestBody: m2, + Address: "foo:baz", + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + It("should be listed on api with ListLogs", func(specContext SpecContext) { + response, err := Wait(specContext, DeferClient(testServer)).Ledger.V1.ListLogs( + ctx, + operations.ListLogsRequest{ + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + + Expect(response.LogsCursorResponse.Cursor.Data).To(HaveLen(3)) + + for _, data := range response.LogsCursorResponse.Cursor.Data { + Expect(data.Hash).NotTo(BeEmpty()) + } + + // Cannot check the date and the hash since they are changing at + // every run + Expect(response.LogsCursorResponse.Cursor.Data[0].ID).To(Equal(int64(3))) + Expect(response.LogsCursorResponse.Cursor.Data[0].Type).To(Equal(components.TypeSetMetadata)) + Expect(response.LogsCursorResponse.Cursor.Data[0].Data).To(Equal(map[string]any{ + "targetType": "ACCOUNT", + "metadata": map[string]any{ + "clientType": "gold", + }, + "targetId": "foo:baz", + })) + + Expect(response.LogsCursorResponse.Cursor.Data[1].ID).To(Equal(int64(2))) + Expect(response.LogsCursorResponse.Cursor.Data[1].Type).To(Equal(components.TypeNewTransaction)) + // Cannot check date and txid inside Data since they are changing at + // every run + Expect(response.LogsCursorResponse.Cursor.Data[1].Data["accountMetadata"]).To(Equal(map[string]any{})) + Expect(response.LogsCursorResponse.Cursor.Data[1].Data["transaction"]).To(BeAssignableToTypeOf(map[string]any{})) + transaction := response.LogsCursorResponse.Cursor.Data[1].Data["transaction"].(map[string]any) + Expect(transaction["metadata"]).To(Equal(map[string]any{ + "clientType": "silver", + })) + Expect(transaction["timestamp"]).To(Equal("2023-04-12T10:00:00Z")) + Expect(transaction["postings"]).To(Equal([]any{ + map[string]any{ + "amount": float64(200), + "asset": "USD", + "source": "world", + "destination": "foo:bar", + }, + })) + + Expect(response.LogsCursorResponse.Cursor.Data[2].ID).To(Equal(int64(1))) + Expect(response.LogsCursorResponse.Cursor.Data[2].Type).To(Equal(components.TypeNewTransaction)) + Expect(response.LogsCursorResponse.Cursor.Data[2].Data["accountMetadata"]).To(Equal(map[string]any{})) + Expect(response.LogsCursorResponse.Cursor.Data[2].Data["transaction"]).To(BeAssignableToTypeOf(map[string]any{})) + transaction = response.LogsCursorResponse.Cursor.Data[2].Data["transaction"].(map[string]any) + Expect(transaction["metadata"]).To(Equal(map[string]any{})) + Expect(transaction["timestamp"]).To(Equal("2023-04-11T10:00:00Z")) + Expect(transaction["postings"]).To(Equal([]any{ + map[string]any{ + "amount": float64(100), + "asset": "USD", + "source": "world", + "destination": "foo:foo", + }, + })) + }) + }) + + type expectedLog struct { + id *big.Int + typ components.Type + postings []any + } + + var ( + compareLogs = func(log components.Log, expected expectedLog) { + Expect(log.ID).To(Equal(expected.id.Int64())) + Expect(log.Type).To(Equal(expected.typ)) + Expect(log.Data["accountMetadata"]).To(Equal(map[string]any{})) + Expect(log.Data["transaction"]).To(BeAssignableToTypeOf(map[string]any{})) + transaction := log.Data["transaction"].(map[string]any) + Expect(transaction["metadata"]).To(Equal(map[string]any{})) + Expect(transaction["postings"]).To(Equal(expected.postings)) + } + ) + + const ( + pageSize = int64(10) + accountCounts = 2 * pageSize + ) + When("creating logs with transactions", func() { + var ( + expectedLogs []expectedLog + ) + BeforeEach(func(specContext SpecContext) { + for i := int64(0); i < accountCounts; i++ { + now := time.Now().Round(time.Millisecond).UTC() + + _, err := Wait(specContext, DeferClient(testServer)).Ledger.V1.CreateTransaction( + ctx, + operations.CreateTransactionRequest{ + PostTransaction: components.PostTransaction{ + Metadata: map[string]any{}, + Postings: []components.Posting{{ + Amount: big.NewInt(100), + Asset: "USD", + Source: "world", + Destination: fmt.Sprintf("foo:%d", i), + }}, + Timestamp: &now, + }, + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + + expectedLogs = append(expectedLogs, expectedLog{ + id: big.NewInt(i + 1), + typ: components.TypeNewTransaction, + postings: []any{ + map[string]any{ + "amount": float64(100), + "asset": "USD", + "source": "world", + "destination": fmt.Sprintf("foo:%d", i), + }, + }, + }) + } + + sort.Slice(expectedLogs, func(i, j int) bool { + return expectedLogs[i].id.Cmp(expectedLogs[j].id) > 0 + }) + }) + AfterEach(func() { + expectedLogs = nil + }) + When(fmt.Sprintf("listing logs using page size of %d", pageSize), func() { + var ( + rsp *operations.ListLogsResponse + err error + ) + BeforeEach(func(specContext SpecContext) { + rsp, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.ListLogs( + ctx, + operations.ListLogsRequest{ + Ledger: "default", + PageSize: pointer.For(pageSize), + }, + ) + Expect(err).ToNot(HaveOccurred()) + + Expect(rsp.LogsCursorResponse.Cursor.HasMore).To(BeTrue()) + Expect(rsp.LogsCursorResponse.Cursor.Previous).To(BeNil()) + Expect(rsp.LogsCursorResponse.Cursor.Next).NotTo(BeNil()) + }) + It("should return the first page", func() { + Expect(rsp.LogsCursorResponse.Cursor.PageSize).To(Equal(pageSize)) + Expect(len(rsp.LogsCursorResponse.Cursor.Data)).To(Equal(len(expectedLogs[:pageSize]))) + for i := range rsp.LogsCursorResponse.Cursor.Data { + compareLogs(rsp.LogsCursorResponse.Cursor.Data[i], expectedLogs[i]) + } + }) + When("following next cursor", func() { + BeforeEach(func(specContext SpecContext) { + rsp, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.ListLogs( + ctx, + operations.ListLogsRequest{ + Cursor: rsp.LogsCursorResponse.Cursor.Next, + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + It("should return next page", func() { + Expect(rsp.LogsCursorResponse.Cursor.PageSize).To(Equal(pageSize)) + Expect(len(rsp.LogsCursorResponse.Cursor.Data)).To(Equal(len(expectedLogs[pageSize : 2*pageSize]))) + for i := range rsp.LogsCursorResponse.Cursor.Data { + compareLogs(rsp.LogsCursorResponse.Cursor.Data[i], expectedLogs[int64(i)+pageSize]) + } + Expect(rsp.LogsCursorResponse.Cursor.Next).To(BeNil()) + }) + When("following previous cursor", func() { + BeforeEach(func(specContext SpecContext) { + rsp, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.ListLogs( + ctx, + operations.ListLogsRequest{ + Cursor: rsp.LogsCursorResponse.Cursor.Previous, + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + It("should return first page", func() { + Expect(rsp.LogsCursorResponse.Cursor.PageSize).To(Equal(pageSize)) + Expect(len(rsp.LogsCursorResponse.Cursor.Data)).To(Equal(len(expectedLogs[:pageSize]))) + for i := range rsp.LogsCursorResponse.Cursor.Data { + compareLogs(rsp.LogsCursorResponse.Cursor.Data[i], expectedLogs[i]) + } + Expect(rsp.LogsCursorResponse.Cursor.Previous).To(BeNil()) + }) + }) + }) + }) + }) +}) diff --git a/test/e2e/v1_api_transactions_list_test.go b/test/e2e/v1_api_transactions_list_test.go new file mode 100644 index 000000000..9185218fb --- /dev/null +++ b/test/e2e/v1_api_transactions_list_test.go @@ -0,0 +1,203 @@ +//go:build it + +package test_suite + +import ( + "fmt" + "github.com/formancehq/go-libs/v3/logging" + . "github.com/formancehq/go-libs/v3/testing/deferred/ginkgo" + "github.com/formancehq/go-libs/v3/testing/platform/pgtesting" + "github.com/formancehq/go-libs/v3/testing/testservice" + "github.com/formancehq/ledger/pkg/client/models/components" + "github.com/formancehq/ledger/pkg/client/models/operations" + . "github.com/formancehq/ledger/pkg/testserver" + "github.com/formancehq/ledger/pkg/testserver/ginkgo" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "math/big" + "time" + + "github.com/formancehq/go-libs/v3/pointer" +) + +var _ = Context("Ledger transactions list API tests", func() { + var ( + db = UseTemplatedDatabase() + ctx = logging.TestingContext() + ) + + testServer := ginkgo.DeferTestServer( + DeferMap(db, (*pgtesting.Database).ConnectionOptions), + testservice.WithInstruments( + testservice.DebugInstrumentation(debug), + testservice.OutputInstrumentation(GinkgoWriter), + ), + testservice.WithLogger(GinkgoT()), + ) + + JustBeforeEach(func(specContext SpecContext) { + _, err := Wait(specContext, DeferClient(testServer)).Ledger.V2.CreateLedger(ctx, operations.V2CreateLedgerRequest{ + Ledger: "default", + }) + Expect(err).To(BeNil()) + }) + const ( + pageSize = int64(10) + txCount = 2 * pageSize + ) + When(fmt.Sprintf("creating %d transactions", txCount), func() { + var ( + timestamp = time.Now() + transactions []components.Transaction + ) + JustBeforeEach(func(specContext SpecContext) { + for i := 0; i < int(txCount); i++ { + offset := time.Duration(int(txCount)-i) * time.Minute + // 1 transaction of 2 is backdated to test pagination using effective date + if offset%2 == 0 { + offset += 1 + } else { + offset -= 1 + } + txTimestamp := timestamp.Add(-offset) + + response, err := Wait(specContext, DeferClient(testServer)).Ledger.V1.CreateTransaction( + ctx, + operations.CreateTransactionRequest{ + PostTransaction: components.PostTransaction{ + Metadata: map[string]any{}, + Postings: []components.Posting{ + { + Amount: big.NewInt(100), + Asset: "USD", + Source: "world", + Destination: fmt.Sprintf("account:%d", i), + }, + { + Amount: big.NewInt(100), + Asset: "EUR", + Source: "world", + Destination: fmt.Sprintf("account:%d", i), + }, + }, + Timestamp: pointer.For(txTimestamp), + Reference: pointer.For(fmt.Sprintf("ref-%d", i)), + }, + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + + transactions = append([]components.Transaction{ + response.TransactionsResponse.Data[0], + }, transactions...) + } + }) + AfterEach(func() { + transactions = nil + }) + When("listing transactions using a page size of 5", func() { + var ( + rsp *operations.ListTransactionsResponse + err error + ) + JustBeforeEach(func(specContext SpecContext) { + rsp, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.ListTransactions( + ctx, + operations.ListTransactionsRequest{ + Ledger: "default", + PageSize: pointer.For(int64(5)), + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + When("using next page with a page size of 10", func() { + JustBeforeEach(func(specContext SpecContext) { + rsp, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.ListTransactions( + ctx, + operations.ListTransactionsRequest{ + Ledger: "default", + Cursor: rsp.TransactionsCursorResponse.Cursor.Next, + PageSize: pointer.For(int64(10)), + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + It("Should return 10 elements", func() { + Expect(rsp.TransactionsCursorResponse.Cursor.Data).To(HaveLen(10)) + }) + }) + }) + When(fmt.Sprintf("listing transactions using page size of %d", pageSize), func() { + var ( + rsp *operations.ListTransactionsResponse + req operations.ListTransactionsRequest + err error + ) + BeforeEach(func() { + req = operations.ListTransactionsRequest{ + Ledger: "default", + PageSize: pointer.For(pageSize), + EndTime: pointer.For(time.Now()), + Source: pointer.For("world"), + } + }) + JustBeforeEach(func(specContext SpecContext) { + rsp, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.ListTransactions(ctx, req) + Expect(err).ToNot(HaveOccurred()) + }) + Context("with a filter on reference", func() { + BeforeEach(func() { + req.Reference = pointer.For("ref-0") + }) + It("Should be ok, and returns transactions with reference 'ref-0'", func() { + Expect(rsp.TransactionsCursorResponse.Cursor.Data).To(HaveLen(1)) + Expect(rsp.TransactionsCursorResponse.Cursor.Data[0]).To(Equal(transactions[txCount-1])) + }) + }) + It("Should be ok", func() { + Expect(rsp.TransactionsCursorResponse.Cursor.PageSize).To(Equal(pageSize)) + Expect(rsp.TransactionsCursorResponse.Cursor.Data).To(Equal(transactions[:pageSize])) + }) + When("following next cursor", func() { + JustBeforeEach(func(specContext SpecContext) { + Expect(rsp.TransactionsCursorResponse.Cursor.HasMore).To(BeTrue()) + Expect(rsp.TransactionsCursorResponse.Cursor.Previous).To(BeNil()) + Expect(rsp.TransactionsCursorResponse.Cursor.Next).NotTo(BeNil()) + + rsp, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.ListTransactions( + ctx, + operations.ListTransactionsRequest{ + Cursor: rsp.TransactionsCursorResponse.Cursor.Next, + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + It("should return next page", func() { + Expect(rsp.TransactionsCursorResponse.Cursor.PageSize).To(Equal(pageSize)) + Expect(rsp.TransactionsCursorResponse.Cursor.Data).To(Equal(transactions[pageSize : 2*pageSize])) + Expect(rsp.TransactionsCursorResponse.Cursor.Next).To(BeNil()) + }) + When("following previous cursor", func() { + JustBeforeEach(func(specContext SpecContext) { + var err error + rsp, err = Wait(specContext, DeferClient(testServer)).Ledger.V1.ListTransactions( + ctx, + operations.ListTransactionsRequest{ + Cursor: rsp.TransactionsCursorResponse.Cursor.Previous, + Ledger: "default", + }, + ) + Expect(err).ToNot(HaveOccurred()) + }) + It("should return first page", func() { + Expect(rsp.TransactionsCursorResponse.Cursor.PageSize).To(Equal(pageSize)) + Expect(rsp.TransactionsCursorResponse.Cursor.Data).To(Equal(transactions[:pageSize])) + Expect(rsp.TransactionsCursorResponse.Cursor.Previous).To(BeNil()) + }) + }) + }) + }) + }) +})