release to master

migrations/1759466478081_burnchain-rewards-reorg.js

@@ -0,0 +1,20 @@
+/* eslint-disable camelcase */
+
+exports.shorthands = undefined;
+
+exports.up = pgm => {
+  pgm.sql(`
+    UPDATE burnchain_rewards
+    SET canonical = COALESCE(
+      (
+        SELECT canonical
+        FROM blocks
+        WHERE blocks.burn_block_hash = burnchain_rewards.burn_block_hash
+        LIMIT 1
+      ),
+      false
+    )
+  `);
+};
+
+exports.down = pgm => {};

src/datastore/helpers.ts

@@ -1363,6 +1363,10 @@ export function newReOrgUpdatedEntities(): ReOrgUpdatedEntities {
   };
 }
 
+export function removeNullBytes(str: string): string {
+  return str.replace(/\x00/g, '');
+}
+
 /**
  * Priority queue for parallel Postgres write query execution. This helps performance because it
  * parallelizes the work postgres.js has to do when serializing JS types to PG types.

src/datastore/pg-store.ts

@@ -2051,19 +2051,25 @@ export class PgStore extends BasePgStore {
       >`
         WITH ordered_pox_events AS (
           SELECT
-            stacker, pox_addr, amount_ustx, unlock_burn_height::integer, tx_id,
+            stacker, pox_addr, amount_ustx, unlock_burn_height::integer, tx_id, name,
             block_height, microblock_sequence, tx_index, event_index
           FROM ${sql(args.poxTable)}
           WHERE
-            canonical = true AND microblock_canonical = true AND
-            name = ${SyntheticPoxEventName.DelegateStx} AND delegate_to = ${args.delegator} AND
-            block_height <= ${args.blockHeight} AND block_height > ${args.afterBlockHeight} AND
-            (unlock_burn_height > ${args.burnBlockHeight} OR unlock_burn_height IS NULL)
+            canonical = true
+            AND microblock_canonical = true
+            AND delegate_to = ${args.delegator}
+            AND name IN (
+              ${SyntheticPoxEventName.DelegateStx},
+              ${SyntheticPoxEventName.RevokeDelegateStx}
+            )
+            AND block_height <= ${args.blockHeight}
+            AND block_height > ${args.afterBlockHeight}
+            AND (unlock_burn_height > ${args.burnBlockHeight} OR unlock_burn_height IS NULL)
           ORDER BY stacker, block_height DESC, microblock_sequence DESC, tx_index DESC, event_index DESC
         ),
         distinct_rows AS (
           SELECT DISTINCT ON (stacker)
-            stacker, pox_addr, amount_ustx, unlock_burn_height, tx_id,
+            stacker, pox_addr, amount_ustx, unlock_burn_height, tx_id, name,
             block_height, microblock_sequence, tx_index, event_index
           FROM ordered_pox_events
           ORDER BY stacker, block_height DESC, microblock_sequence DESC, tx_index DESC, event_index DESC
@@ -2072,6 +2078,7 @@ export class PgStore extends BasePgStore {
           stacker, pox_addr, amount_ustx, unlock_burn_height, block_height::integer, tx_id,
           COUNT(*) OVER()::integer AS total_rows
         FROM distinct_rows
+        WHERE name = ${SyntheticPoxEventName.DelegateStx}
         ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC, event_index DESC
         LIMIT ${args.limit}
         OFFSET ${args.offset}

src/datastore/pg-write-store.ts

@@ -78,6 +78,7 @@ import {
   validateZonefileHash,
   newReOrgUpdatedEntities,
   PgWriteQueue,
+  removeNullBytes,
 } from './helpers';
 import { PgNotifier } from './pg-notifier';
 import { MIGRATIONS_DIR, PgStore } from './pg-store';
@@ -203,6 +204,13 @@ export class PgWriteStore extends PgStore {
   }
 
   async storeRawEventRequest(eventPath: string, payload: any): Promise<void> {
+    if (eventPath === '/new_block' && typeof payload === 'object') {
+      for (const tx of payload.transactions) {
+        if ('vm_error' in tx && tx.vm_error) {
+          tx.vm_error = removeNullBytes(tx.vm_error);
+        }
+      }
+    }
     await this.sqlWriteTransaction(async sql => {
       const insertResult = await sql<
         {
@@ -1930,35 +1938,8 @@ export class PgWriteStore extends PgStore {
     };
   }
 
-  async updateBurnchainRewards({
-    burnchainBlockHash,
-    burnchainBlockHeight,
-    rewards,
-  }: {
-    burnchainBlockHash: string;
-    burnchainBlockHeight: number;
-    rewards: DbBurnchainReward[];
-  }): Promise<void> {
+  async updateBurnchainRewards({ rewards }: { rewards: DbBurnchainReward[] }): Promise<void> {
     return await this.sqlWriteTransaction(async sql => {
-      const existingRewards = await sql<
-        {
-          reward_recipient: string;
-          reward_amount: string;
-        }[]
-      >`
-        UPDATE burnchain_rewards
-        SET canonical = false
-        WHERE canonical = true AND
-          (burn_block_hash = ${burnchainBlockHash}
-            OR burn_block_height >= ${burnchainBlockHeight})
-      `;
-
-      if (existingRewards.count > 0) {
-        logger.warn(
-          `Invalidated ${existingRewards.count} burnchain rewards after fork detected at burnchain block ${burnchainBlockHash}`
-        );
-      }
-
       for (const reward of rewards) {
         const values: BurnchainRewardInsertValues = {
           canonical: true,
@@ -2062,7 +2043,9 @@ export class PgWriteStore extends PgStore {
       token_transfer_memo: tx.token_transfer_memo ?? null,
       smart_contract_clarity_version: tx.smart_contract_clarity_version ?? null,
       smart_contract_contract_id: tx.smart_contract_contract_id ?? null,
-      smart_contract_source_code: tx.smart_contract_source_code ?? null,
+      smart_contract_source_code: tx.smart_contract_source_code
+        ? removeNullBytes(tx.smart_contract_source_code)
+        : null,
       contract_call_contract_id: tx.contract_call_contract_id ?? null,
       contract_call_function_name: tx.contract_call_function_name ?? null,
       contract_call_function_args: tx.contract_call_function_args ?? null,
@@ -2085,7 +2068,7 @@ export class PgWriteStore extends PgStore {
       execution_cost_runtime: tx.execution_cost_runtime,
       execution_cost_write_count: tx.execution_cost_write_count,
       execution_cost_write_length: tx.execution_cost_write_length,
-      vm_error: tx.vm_error ?? null,
+      vm_error: tx.vm_error ? removeNullBytes(tx.vm_error) : null,
     }));
 
     let count = 0;
@@ -3615,6 +3598,11 @@ export class PgWriteStore extends PgStore {
     if (orphanedBlockResult.length > 0) {
       const orphanedBlocks = orphanedBlockResult.map(b => parseBlockQueryResult(b));
       for (const orphanedBlock of orphanedBlocks) {
+        await sql`
+          UPDATE burnchain_rewards
+          SET canonical = false
+          WHERE canonical = true AND burn_block_hash = ${orphanedBlock.burn_block_hash}
+        `;
         const microCanonicalUpdateResult = await this.updateMicroCanonical(sql, {
           isCanonical: false,
           blockHeight: orphanedBlock.block_height,

src/datastore/redis-notifier.ts

@@ -54,7 +54,7 @@ export class RedisNotifier {
       },
     };
     logger.info(message, 'RedisNotifier broadcasting index progress message');
-    await this.redis.rpush(this.queue, JSON.stringify(message));
+    await this.redis.xadd(this.queue, '*', 'data', JSON.stringify(message));
   }
 
   async close() {

src/event-stream/event-server.ts

@@ -121,8 +121,6 @@ async function handleBurnBlockMessage(
     return slotHolder;
   });
   await db.updateBurnchainRewards({
-    burnchainBlockHash: burnBlockMsg.burn_block_hash,
-    burnchainBlockHeight: burnBlockMsg.burn_block_height,
     rewards: rewards,
   });
   await db.updateBurnchainRewardSlotHolders({

tests/2.5/pox-4-delegate-revoked-stacking.test.ts

@@ -24,7 +24,6 @@ import {
   fetchGet,
   readOnlyFnCall,
   standByForPoxCycle,
-  standByForPoxCycleEnd,
   standByForTx,
   standByForTxSuccess,
   testEnv,
@@ -207,6 +206,19 @@ describe('PoX-4 - Delegate Revoked Stacking', () => {
     );
     const delegatedAmount = BigInt(getDelegationInfo.data['amount-ustx'].value);
     expect(delegatedAmount).toBe(DELEGATE_HALF_AMOUNT);
+
+    // validate pool delegations
+    const stackersRes: any = await fetchGet(`/extended/v1/pox4/${POOL.stxAddr}/delegations`);
+    expect(stackersRes).toBeDefined();
+    expect(stackersRes.total).toBe(1);
+    expect(stackersRes.results).toHaveLength(1);
+    expect(stackersRes.results[0]).toEqual({
+      amount_ustx: DELEGATE_HALF_AMOUNT.toString(),
+      pox_addr: STACKER.btcTestnetAddr,
+      stacker: STACKER.stxAddr,
+      tx_id: delegateStxDbTx.tx_id,
+      block_height: delegateStxDbTx.block_height,
+    });
   });
 
   test('Perform delegate-stack-stx', async () => {
@@ -311,6 +323,12 @@ describe('PoX-4 - Delegate Revoked Stacking', () => {
     const coreBalanceInfo = await testEnv.client.getAccount(STACKER.stxAddr);
     expect(BigInt(coreBalanceInfo.locked)).toBe(DELEGATE_HALF_AMOUNT);
     expect(coreBalanceInfo.unlock_height).toBeGreaterThan(0);
+
+    // validate pool delegation no longer exists
+    const stackersRes: any = await fetchGet(`/extended/v1/pox4/${POOL.stxAddr}/delegations`);
+    expect(stackersRes).toBeDefined();
+    expect(stackersRes.total).toBe(0);
+    expect(stackersRes.results).toHaveLength(0);
   });
 
   test('Try to perform delegate-stack-stx - while revoked', async () => {

tests/api/burnchain.test.ts

@@ -201,13 +201,9 @@ describe('burnchain tests', () => {
       reward_index: 2,
     };
     await db.updateBurnchainRewards({
-      burnchainBlockHash: reward1.burn_block_hash,
-      burnchainBlockHeight: reward1.burn_block_height,
       rewards: [reward1, reward2],
     });
     await db.updateBurnchainRewards({
-      burnchainBlockHash: reward3.burn_block_hash,
-      burnchainBlockHeight: reward3.burn_block_height,
       rewards: [reward3, reward4, reward5],
     });
 
@@ -282,18 +278,12 @@ describe('burnchain tests', () => {
       reward_index: 0,
     };
     await db.updateBurnchainRewards({
-      burnchainBlockHash: reward1.burn_block_hash,
-      burnchainBlockHeight: reward1.burn_block_height,
       rewards: [reward1],
     });
     await db.updateBurnchainRewards({
-      burnchainBlockHash: reward2.burn_block_hash,
-      burnchainBlockHeight: reward2.burn_block_height,
       rewards: [reward2],
     });
     await db.updateBurnchainRewards({
-      burnchainBlockHash: reward3.burn_block_hash,
-      burnchainBlockHeight: reward3.burn_block_height,
       rewards: [reward3],
     });
     const rewardResult = await supertest(api.server).get(
@@ -320,8 +310,6 @@ describe('burnchain tests', () => {
       reward_index: 0,
     };
     await db.updateBurnchainRewards({
-      burnchainBlockHash: reward1.burn_block_hash,
-      burnchainBlockHeight: reward1.burn_block_height,
       rewards: [reward1],
     });
     const rewardResult = await supertest(api.server).get(`/extended/v1/burnchain/rewards/${addr1}`);
@@ -360,8 +348,6 @@ describe('burnchain tests', () => {
       reward_index: 0,
     };
     await db.updateBurnchainRewards({
-      burnchainBlockHash: reward1.burn_block_hash,
-      burnchainBlockHeight: reward1.burn_block_height,
       rewards: [reward1],
     });
     const rewardResult = await supertest(api.server).get(
@@ -402,8 +388,6 @@ describe('burnchain tests', () => {
       reward_index: 0,
     };
     await db.updateBurnchainRewards({
-      burnchainBlockHash: reward1.burn_block_hash,
-      burnchainBlockHeight: reward1.burn_block_height,
       rewards: [reward1],
     });
     const rewardResult = await supertest(api.server).get(
@@ -444,8 +428,6 @@ describe('burnchain tests', () => {
       reward_index: 0,
     };
     await db.updateBurnchainRewards({
-      burnchainBlockHash: reward1.burn_block_hash,
-      burnchainBlockHeight: reward1.burn_block_height,
       rewards: [reward1],
     });
     const rewardResult = await supertest(api.server).get(