From 24add24998666197a030a42f7945b44ebc43da14 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 15:46:22 -0400 Subject: [PATCH 01/30] [CT-1365] Safely Handle Exceptions to TWAP Suborders (backport #3039) (#3041) Co-authored-by: anmolagrawal345 --- protocol/x/clob/keeper/twap_order_state.go | 29 +++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/protocol/x/clob/keeper/twap_order_state.go b/protocol/x/clob/keeper/twap_order_state.go index ab1bb2d5c44..ebb904766d0 100644 --- a/protocol/x/clob/keeper/twap_order_state.go +++ b/protocol/x/clob/keeper/twap_order_state.go @@ -1,11 +1,13 @@ package keeper import ( + "fmt" "math/big" errorsmod "cosmossdk.io/errors" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/dydxprotocol/v4-chain/protocol/lib" + "github.com/dydxprotocol/v4-chain/protocol/lib/abci" "github.com/dydxprotocol/v4-chain/protocol/x/clob/types" ) @@ -206,7 +208,7 @@ func (k Keeper) GenerateAndPlaceTriggeredTwapSuborders(ctx sdk.Context) { ) // place triggered suborder - err := k.HandleMsgPlaceOrder(ctx, &types.MsgPlaceOrder{Order: *op.suborderToPlace}, true) + err := k.safeHandleMsgPlaceOrder(ctx, &types.MsgPlaceOrder{Order: *op.suborderToPlace}, true) if err != nil { // TODO: (anmol) emit indexer event (TWAP error) k.DeleteTWAPOrderPlacement(ctx, op.twapOrderPlacement.Order.GetOrderId()) @@ -398,3 +400,28 @@ func (k Keeper) GenerateSuborder( return &order, true } + +// safeHandleMsgPlaceOrder safely calls HandleMsgPlaceOrder with panic recovery. +// This is used in end blockers where panics should be caught and logged rather than +// causing the entire block to fail. +func (k Keeper) safeHandleMsgPlaceOrder( + ctx sdk.Context, + msg *types.MsgPlaceOrder, + isStateful bool, +) (err error) { + if err = abci.RunCached(ctx, func(ctx sdk.Context) error { + return k.HandleMsgPlaceOrder(ctx, msg, isStateful) + }); err != nil { + k.Logger(ctx).Error( + "failed to handle TWAP suborder placement via HandleMsgPlaceOrder (panic recovered or error)", + "cause", err, + "orderId", msg.GetOrder().OrderId, + "isStateful", isStateful, + "stack", fmt.Sprintf("%+v", err), + ) + + return err + } + + return nil +} From e69ea3df98bb6ec285483e602b6edf5d8715359a Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 13 Aug 2025 12:51:05 -0400 Subject: [PATCH 02/30] [CT-1361] Add Support for TWAP Events in the Indexer (backport #2815) (#3042) Co-authored-by: anmolagrawal345 --- ...d_twap_fields_to_orders_and_fills_table.ts | 71 +++ .../postgres/src/lib/protocol-translations.ts | 42 +- .../postgres/src/models/order-model.ts | 16 + .../postgres/src/types/db-model-types.ts | 4 + .../packages/postgres/src/types/fill-types.ts | 2 + .../postgres/src/types/order-types.ts | 12 + .../postgres/src/types/trade-types.ts | 4 + .../src/types/websocket-message-types.ts | 4 + .../packages/v4-proto-parser/src/constants.ts | 2 + .../dydxprotocol/indexer/events/events.ts | 69 +- .../dydxprotocol/indexer/protocol/v1/clob.ts | 165 ++++- .../comlink/public/api-documentation.md | 36 ++ indexer/services/comlink/public/swagger.json | 22 +- .../request-helpers/request-transformer.ts | 3 + .../order-fills/liquidation-handler.test.ts | 159 ++++- .../order-fills/order-handler.test.ts | 487 +++++++++++++- ...onditional-order-placement-handler.test.ts | 6 + .../stateful-order-placement-handler.test.ts | 131 ++-- .../stateful-order-removal-handler.test.ts | 125 +++- .../ender/__tests__/helpers/constants.ts | 18 + .../helpers/indexer-proto-helpers.ts | 45 +- .../ender/__tests__/scripts/scripts.test.ts | 71 ++- .../stateful-order-validator.test.ts | 13 +- .../stateful-order-placement-handler.ts | 8 + .../helpers/postgres/postgres-functions.ts | 3 +- .../handlers/dydx_stateful_order_handler.sql | 53 +- .../scripts/helpers/dydx_get_order_status.sql | 4 +- ...ydx_liquidation_fill_handler_per_order.sql | 84 ++- .../dydx_order_fill_handler_per_order.sql | 93 ++- .../src/scripts/helpers/dydx_order_flags.sql | 21 + ..._protocol_condition_type_to_order_type.sql | 17 - .../dydx_protocol_convert_to_order_type.sql | 25 + .../helpers/dydx_uuid_from_order_id.sql | 9 + .../validators/stateful-order-validator.ts | 75 ++- .../dydxprotocol/indexer/events/events.proto | 6 + .../indexer/protocol/v1/clob.proto | 36 +- protocol/indexer/events/events.pb.go | 599 +++++++++++++----- protocol/indexer/events/stateful_order.go | 14 + protocol/indexer/protocol/v1/types/clob.pb.go | 437 +++++++++++-- protocol/indexer/protocol/v1/v1_mappers.go | 14 + .../x/clob/keeper/msg_server_place_order.go | 16 +- .../keeper/msg_server_place_order_test.go | 12 +- protocol/x/clob/keeper/twap_order_state.go | 30 +- 43 files changed, 2607 insertions(+), 456 deletions(-) create mode 100644 indexer/packages/postgres/src/db/migrations/migration_files/20250423144330_add_twap_fields_to_orders_and_fills_table.ts create mode 100644 indexer/services/ender/src/scripts/helpers/dydx_order_flags.sql delete mode 100644 indexer/services/ender/src/scripts/helpers/dydx_protocol_condition_type_to_order_type.sql create mode 100644 indexer/services/ender/src/scripts/helpers/dydx_protocol_convert_to_order_type.sql diff --git a/indexer/packages/postgres/src/db/migrations/migration_files/20250423144330_add_twap_fields_to_orders_and_fills_table.ts b/indexer/packages/postgres/src/db/migrations/migration_files/20250423144330_add_twap_fields_to_orders_and_fills_table.ts new file mode 100644 index 00000000000..33c69aab65c --- /dev/null +++ b/indexer/packages/postgres/src/db/migrations/migration_files/20250423144330_add_twap_fields_to_orders_and_fills_table.ts @@ -0,0 +1,71 @@ +import * as Knex from 'knex'; + +import { formatAlterTableEnumSql } from '../helpers'; + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable('orders', (table) => { + table.string('duration').nullable(); + table.string('interval').nullable(); + table.string('priceTolerance').nullable(); + }); + + await knex.raw(formatAlterTableEnumSql( + 'orders', + 'type', + [ + 'LIMIT', + 'MARKET', + 'STOP_LIMIT', + 'STOP_MARKET', + 'TRAILING_STOP', + 'TAKE_PROFIT', + 'TAKE_PROFIT_MARKET', + 'LIQUIDATED', + 'LIQUIDATION', + 'HARD_TRADE', + 'FAILED_HARD_TRADE', + 'TRANSFER_PLACEHOLDER', + 'TWAP', + 'TWAP_SUBORDER', + ], + )); + + await knex.raw(formatAlterTableEnumSql( + 'fills', + 'type', + ['LIMIT', 'LIQUIDATED', 'LIQUIDATION', 'DELEVERAGED', 'OFFSETTING', 'TWAP_SUBORDER'], + )); +} + +export async function down(knex: Knex): Promise { + await knex.schema.alterTable('orders', (table) => { + table.dropColumn('duration'); + table.dropColumn('interval'); + table.dropColumn('priceTolerance'); + }); + + await knex.raw(formatAlterTableEnumSql( + 'orders', + 'type', + [ + 'LIMIT', + 'MARKET', + 'STOP_LIMIT', + 'STOP_MARKET', + 'TRAILING_STOP', + 'TAKE_PROFIT', + 'TAKE_PROFIT_MARKET', + 'LIQUIDATED', + 'LIQUIDATION', + 'HARD_TRADE', + 'FAILED_HARD_TRADE', + 'TRANSFER_PLACEHOLDER', + ], + )); + + await knex.raw(formatAlterTableEnumSql( + 'fills', + 'type', + ['LIMIT', 'LIQUIDATED', 'LIQUIDATION', 'DELEVERAGED', 'OFFSETTING'], + )); +} diff --git a/indexer/packages/postgres/src/lib/protocol-translations.ts b/indexer/packages/postgres/src/lib/protocol-translations.ts index ba98992faf6..01395573512 100644 --- a/indexer/packages/postgres/src/lib/protocol-translations.ts +++ b/indexer/packages/postgres/src/lib/protocol-translations.ts @@ -1,10 +1,17 @@ -import { bytesToBigInt } from '@dydxprotocol-indexer/v4-proto-parser'; import { + bytesToBigInt, + ORDER_FLAG_CONDITIONAL, + ORDER_FLAG_LONG_TERM, + ORDER_FLAG_SHORT_TERM, + ORDER_FLAG_TWAP, + ORDER_FLAG_TWAP_SUBORDER, +} from '@dydxprotocol-indexer/v4-proto-parser'; +import { + ClobPairStatus, IndexerOrder, + IndexerOrder_ConditionType, IndexerOrder_Side, IndexerOrder_TimeInForce, - IndexerOrder_ConditionType, - ClobPairStatus, } from '@dydxprotocol-indexer/v4-protos'; import Big from 'big.js'; import { DateTime } from 'luxon'; @@ -64,6 +71,9 @@ const ORDER_TYPE_TO_CONDITION_TYPE_MAP: Record>> 3) { + case 1: + message.order = IndexerOrder.decode(reader, reader.uint32()); + break; + + default: + reader.skipType(tag & 7); + break; + } + } + + return message; + }, + + fromPartial(object: DeepPartial): StatefulOrderEventV1_TwapOrderPlacementV1 { + const message = createBaseStatefulOrderEventV1_TwapOrderPlacementV1(); + message.order = object.order !== undefined && object.order !== null ? IndexerOrder.fromPartial(object.order) : undefined; + return message; + } + +}; + function createBaseAssetCreateEventV1(): AssetCreateEventV1 { return { id: 0, diff --git a/indexer/packages/v4-protos/src/codegen/dydxprotocol/indexer/protocol/v1/clob.ts b/indexer/packages/v4-protos/src/codegen/dydxprotocol/indexer/protocol/v1/clob.ts index 1f1589a82ab..cf248e8bf5c 100644 --- a/indexer/packages/v4-protos/src/codegen/dydxprotocol/indexer/protocol/v1/clob.ts +++ b/indexer/packages/v4-protos/src/codegen/dydxprotocol/indexer/protocol/v1/clob.ts @@ -452,15 +452,14 @@ export interface IndexerOrderId { clientId: number; /** * order_flags represent order flags for the order. This field is invalid if - * it's greater than 127 (larger than one byte). Each bit in the first byte - * represents a different flag. Currently only two flags are supported. + * it's greater than 257. Each bit represents a different flag. * - * Starting from the bit after the most MSB (note that the MSB is used in - * proto varint encoding, and therefore cannot be used): Bit 1 is set if this - * order is a Long-Term order (0x40, or 64 as a uint8). Bit 2 is set if this - * order is a Conditional order (0x20, or 32 as a uint8). - * - * If neither bit is set, the order is assumed to be a Short-Term order. + * The following are the valid orderId flags: + * ShortTerm = uint32(0) + * Conditional = uint32(32) + * LongTerm = uint32(64) + * Twap = uint32(128) + * TwapSuborder = uint32(256) (for internal use only) * * If both bits are set or bits other than the 2nd and 3rd are set, the order * ID is invalid. @@ -491,15 +490,14 @@ export interface IndexerOrderIdSDKType { client_id: number; /** * order_flags represent order flags for the order. This field is invalid if - * it's greater than 127 (larger than one byte). Each bit in the first byte - * represents a different flag. Currently only two flags are supported. - * - * Starting from the bit after the most MSB (note that the MSB is used in - * proto varint encoding, and therefore cannot be used): Bit 1 is set if this - * order is a Long-Term order (0x40, or 64 as a uint8). Bit 2 is set if this - * order is a Conditional order (0x20, or 32 as a uint8). + * it's greater than 257. Each bit represents a different flag. * - * If neither bit is set, the order is assumed to be a Short-Term order. + * The following are the valid orderId flags: + * ShortTerm = uint32(0) + * Conditional = uint32(32) + * LongTerm = uint32(64) + * Twap = uint32(128) + * TwapSuborder = uint32(256) (for internal use only) * * If both bits are set or bits other than the 2nd and 3rd are set, the order * ID is invalid. @@ -588,6 +586,12 @@ export interface IndexerOrder { */ orderRouterAddress: string; + /** + * twap_parameters represent the configuration for a TWAP order. This must be + * set for twap orders and will be ignored for all other order types. + */ + + twapParameters?: TwapParameters; } /** * IndexerOrderV1 represents a single order belonging to a `Subaccount` @@ -667,6 +671,58 @@ export interface IndexerOrderSDKType { */ order_router_address: string; + /** + * twap_parameters represent the configuration for a TWAP order. This must be + * set for twap orders and will be ignored for all other order types. + */ + + twap_parameters?: TwapParametersSDKType; +} +/** TwapParameters represents the necessary configuration for a TWAP order. */ + +export interface TwapParameters { + /** + * Duration of the TWAP order execution in seconds. Must be in the range + * [300 (5 minutes), 86400 (24 hours)]. + */ + duration: number; + /** + * Interval in seconds for each suborder to execute. Must be a + * whole number, a factor of the duration, and in the range + * [30 (30 seconds), 3600 (1 hour)]. + */ + + interval: number; + /** + * Price tolerance in ppm for each suborder. This will be applied to + * the oracle price each time a suborder is triggered. Must be + * be in the range [0, 1_000_000). + */ + + priceTolerance: number; +} +/** TwapParameters represents the necessary configuration for a TWAP order. */ + +export interface TwapParametersSDKType { + /** + * Duration of the TWAP order execution in seconds. Must be in the range + * [300 (5 minutes), 86400 (24 hours)]. + */ + duration: number; + /** + * Interval in seconds for each suborder to execute. Must be a + * whole number, a factor of the duration, and in the range + * [30 (30 seconds), 3600 (1 hour)]. + */ + + interval: number; + /** + * Price tolerance in ppm for each suborder. This will be applied to + * the oracle price each time a suborder is triggered. Must be + * be in the range [0, 1_000_000). + */ + + price_tolerance: number; } /** * BuilderCodeParameters represents the metadata for the partner or builder of @@ -784,7 +840,8 @@ function createBaseIndexerOrder(): IndexerOrder { conditionType: 0, conditionalOrderTriggerSubticks: Long.UZERO, builderCodeParams: undefined, - orderRouterAddress: "" + orderRouterAddress: "", + twapParameters: undefined }; } @@ -842,6 +899,10 @@ export const IndexerOrder = { writer.uint32(106).string(message.orderRouterAddress); } + if (message.twapParameters !== undefined) { + TwapParameters.encode(message.twapParameters, writer.uint32(114).fork()).ldelim(); + } + return writer; }, @@ -906,6 +967,10 @@ export const IndexerOrder = { message.orderRouterAddress = reader.string(); break; + case 14: + message.twapParameters = TwapParameters.decode(reader, reader.uint32()); + break; + default: reader.skipType(tag & 7); break; @@ -930,6 +995,72 @@ export const IndexerOrder = { message.conditionalOrderTriggerSubticks = object.conditionalOrderTriggerSubticks !== undefined && object.conditionalOrderTriggerSubticks !== null ? Long.fromValue(object.conditionalOrderTriggerSubticks) : Long.UZERO; message.builderCodeParams = object.builderCodeParams !== undefined && object.builderCodeParams !== null ? BuilderCodeParameters.fromPartial(object.builderCodeParams) : undefined; message.orderRouterAddress = object.orderRouterAddress ?? ""; + message.twapParameters = object.twapParameters !== undefined && object.twapParameters !== null ? TwapParameters.fromPartial(object.twapParameters) : undefined; + return message; + } + +}; + +function createBaseTwapParameters(): TwapParameters { + return { + duration: 0, + interval: 0, + priceTolerance: 0 + }; +} + +export const TwapParameters = { + encode(message: TwapParameters, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + if (message.duration !== 0) { + writer.uint32(8).uint32(message.duration); + } + + if (message.interval !== 0) { + writer.uint32(16).uint32(message.interval); + } + + if (message.priceTolerance !== 0) { + writer.uint32(24).uint32(message.priceTolerance); + } + + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): TwapParameters { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseTwapParameters(); + + while (reader.pos < end) { + const tag = reader.uint32(); + + switch (tag >>> 3) { + case 1: + message.duration = reader.uint32(); + break; + + case 2: + message.interval = reader.uint32(); + break; + + case 3: + message.priceTolerance = reader.uint32(); + break; + + default: + reader.skipType(tag & 7); + break; + } + } + + return message; + }, + + fromPartial(object: DeepPartial): TwapParameters { + const message = createBaseTwapParameters(); + message.duration = object.duration ?? 0; + message.interval = object.interval ?? 0; + message.priceTolerance = object.priceTolerance ?? 0; return message; } diff --git a/indexer/services/comlink/public/api-documentation.md b/indexer/services/comlink/public/api-documentation.md index b0b21813abd..b4e9810f3da 100644 --- a/indexer/services/comlink/public/api-documentation.md +++ b/indexer/services/comlink/public/api-documentation.md @@ -2299,6 +2299,8 @@ fetch(`${baseURL}/orders?address=string&subaccountNumber=0.1`, |type|TRAILING_STOP| |type|TAKE_PROFIT| |type|TAKE_PROFIT_MARKET| +|type|TWAP| +|type|TWAP_SUBORDER| > Example responses @@ -2326,6 +2328,9 @@ fetch(`${baseURL}/orders?address=string&subaccountNumber=0.1`, "builderAddress": "string", "feePpm": "string", "orderRouterAddress": "string", + "duration": "string", + "interval": "string", + "priceTolerance": "string", "timeInForce": "GTT", "status": "OPEN", "postOnly": true, @@ -2369,6 +2374,9 @@ Status Code **200** |» builderAddress|string|false|none|none| |» feePpm|string|false|none|none| |» orderRouterAddress|string|false|none|none| +|» duration|string|false|none|none| +|» interval|string|false|none|none| +|» priceTolerance|string|false|none|none| |» timeInForce|[APITimeInForce](#schemaapitimeinforce)|true|none|none| |» status|any|true|none|none| @@ -2407,6 +2415,8 @@ Status Code **200** |type|TRAILING_STOP| |type|TAKE_PROFIT| |type|TAKE_PROFIT_MARKET| +|type|TWAP| +|type|TWAP_SUBORDER| |timeInForce|GTT| |timeInForce|FOK| |timeInForce|IOC| @@ -2415,6 +2425,7 @@ Status Code **200** |*anonymous*|CANCELED| |*anonymous*|BEST_EFFORT_CANCELED| |*anonymous*|UNTRIGGERED| +|*anonymous*|ERROR| |*anonymous*|BEST_EFFORT_OPENED|