feat(tesseract): Introduce named multistage time shifts for use with calendars (#9773)

* extend cube validator to support named timeshifts

* Initial support for named time shifts

* Resolve conficts

* fix Evaluator to support named time shifts

* update measure symbol to support named time shifts

* cargo clippy

* update snapshots

* add tests for named time shifts

Author: KSDaemon

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -28,15 +28,17 @@ export type DimensionDefinition = {
 };
 
 export type TimeShiftDefinition = {
-  timeDimension?: (...args: Array<unknown>) => ToString,
-  interval: string,
-  type: 'next' | 'prior',
+  timeDimension?: (...args: Array<unknown>) => ToString;
+  name?: string;
+  interval?: string;
+  type?: 'next' | 'prior';
 };
 
 export type TimeShiftDefinitionReference = {
-  timeDimension?: string,
-  interval: string,
-  type: 'next' | 'prior',
+  timeDimension?: string;
+  name?: string;
+  interval?: string;
+  type?: 'next' | 'prior';
 };
 
 export type MeasureDefinition = {
@@ -393,6 +395,7 @@ export class CubeEvaluator extends CubeSymbols {
         }
         if (member.timeShift) {
           member.timeShiftReferences = member.timeShift.map((s): TimeShiftDefinitionReference => ({
+            name: s.name,
             interval: s.interval,
             type: s.type,
             ...(typeof s.timeDimension === 'function'

packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts

@@ -576,9 +576,12 @@ const timeShiftItemRequired = Joi.object({
 
 const timeShiftItemOptional = Joi.object({
   timeDimension: Joi.func(), // not required
-  interval: regexTimeInterval.required(),
-  type: Joi.string().valid('next', 'prior').required(),
-});
+  interval: regexTimeInterval,
+  name: identifier,
+  type: Joi.string().valid('next', 'prior'),
+})
+  .xor('name', 'interval')
+  .and('interval', 'type');
 
 const MeasuresSchema = Joi.object().pattern(identifierRegex, Joi.alternatives().conditional(Joi.ref('.multiStage'), [
   {
@@ -623,6 +626,16 @@ const MeasuresSchema = Joi.object().pattern(identifierRegex, Joi.alternatives().
   ]
 ));
 
+const CalendarTimeShiftItem = Joi.object({
+  name: identifier,
+  interval: regexTimeInterval,
+  type: Joi.string().valid('next', 'prior'),
+  sql: Joi.func().required(),
+})
+  .or('name', 'interval')
+  .with('interval', 'type')
+  .with('type', 'interval');
+
 const DimensionsSchema = Joi.object().pattern(identifierRegex, Joi.alternatives().try(
   inherit(BaseDimensionWithoutSubQuery, {
     case: Joi.object().keys({
@@ -667,11 +680,7 @@ const DimensionsSchema = Joi.object().pattern(identifierRegex, Joi.alternatives(
   inherit(BaseDimensionWithoutSubQuery, {
     type: Joi.any().valid('time').required(),
     sql: Joi.func().required(),
-    timeShift: Joi.array().items(Joi.object({
-      interval: regexTimeInterval.required(),
-      type: Joi.string().valid('next', 'prior').required(),
-      sql: Joi.func().required(),
-    })),
+    timeShift: Joi.array().items(CalendarTimeShiftItem),
   })
 ));
 

packages/cubejs-schema-compiler/test/integration/postgres/calendars.test.ts

@@ -73,6 +73,20 @@ cubes:
           - interval: 1 year
             type: prior
 
+      - name: count_shifted_y_named
+        type: number
+        multi_stage: true
+        sql: "{count}"
+        time_shift:
+          - name: one_year
+
+      - name: count_shifted_y1d_named
+        type: number
+        multi_stage: true
+        sql: "{count}"
+        time_shift:
+          - name: one_year_and_one_day
+
       - name: count_shifted_calendar_m
         type: number
         multi_stage: true
@@ -244,6 +258,12 @@ cubes:
             type: prior
             sql: "{CUBE}.retail_date_prev_year"
 
+          - name: one_year
+            sql: "{CUBE}.retail_date_prev_year"
+
+          - name: one_year_and_one_day
+            sql: "({CUBE}.retail_date_prev_year + interval '1 day')"
+
       ##### Retail Dates ####
       - name: retail_date
         sql: retail_date
@@ -282,6 +302,12 @@ cubes:
             type: prior
             sql: "{CUBE}.retail_date_prev_year"
 
+          - name: one_year
+            sql: "{CUBE}.retail_date_prev_year"
+
+          - name: one_year_and_one_day
+            sql: "({CUBE}.retail_date_prev_year + interval '1 day')"
+
       - name: retail_year
         sql: "{CUBE}.retail_year_name"
         type: string
@@ -576,6 +602,22 @@ cubes:
         },
       ]));
 
+      it('Count shifted by retail year (custom named shift + custom granularity)', async () => runQueryTest({
+        measures: ['calendar_orders.count', 'calendar_orders.count_shifted_y_named'],
+        timeDimensions: [{
+          dimension: 'custom_calendar.retail_date',
+          granularity: 'year',
+          dateRange: ['2025-02-02', '2026-02-01']
+        }],
+        order: [{ id: 'custom_calendar.retail_date' }]
+      }, [
+        {
+          calendar_orders__count: '37',
+          calendar_orders__count_shifted_y_named: '39',
+          custom_calendar__retail_date_year: '2025-02-02T00:00:00.000Z',
+        },
+      ]));
+
       it('Count shifted by retail month (custom shift + common granularity)', async () => runQueryTest({
         measures: ['calendar_orders.count', 'calendar_orders.count_shifted_calendar_m'],
         timeDimensions: [{
@@ -677,6 +719,23 @@ cubes:
           custom_calendar__retail_date_week: '2025-04-06T00:00:00.000Z',
         },
       ]));
+
+      it('Count shifted by retail year and another custom calendar year (2 custom named shifts + custom granularity)', async () => runQueryTest({
+        measures: ['calendar_orders.count', 'calendar_orders.count_shifted_y_named', 'calendar_orders.count_shifted_y1d_named'],
+        timeDimensions: [{
+          dimension: 'custom_calendar.retail_date',
+          granularity: 'year',
+          dateRange: ['2025-02-02', '2026-02-01']
+        }],
+        order: [{ id: 'custom_calendar.retail_date' }]
+      }, [
+        {
+          calendar_orders__count: '37',
+          calendar_orders__count_shifted_y_named: '39',
+          calendar_orders__count_shifted_y1d_named: '39',
+          custom_calendar__retail_date_year: '2025-02-02T00:00:00.000Z',
+        },
+      ]));
     });
 
     describe('PK dimension time-shifts', () => {
@@ -696,6 +755,22 @@ cubes:
         },
       ]));
 
+      it.skip('Count shifted by retail year (custom named shift + custom granularity)1', async () => runQueryTest({
+        measures: ['calendar_orders.count', 'calendar_orders.count_shifted_y_named'],
+        timeDimensions: [{
+          dimension: 'custom_calendar.date_val',
+          granularity: 'year',
+          dateRange: ['2025-02-02', '2026-02-01']
+        }],
+        order: [{ id: 'custom_calendar.date_val' }]
+      }, [
+        {
+          calendar_orders__count: '37',
+          calendar_orders__count_shifted_y_named: '39',
+          custom_calendar__date_val_year: '2025-02-02T00:00:00.000Z',
+        },
+      ]));
+
       it.skip('Count shifted by retail month (custom shift + common granularity)', async () => runQueryTest({
         measures: ['calendar_orders.count', 'calendar_orders.count_shifted_calendar_m'],
         timeDimensions: [{
@@ -797,6 +872,23 @@ cubes:
           custom_calendar__date_val_week: '2025-04-06T00:00:00.000Z',
         },
       ]));
+
+      it.skip('Count shifted by retail year and another custom calendar year (2 custom named shifts + custom granularity)', async () => runQueryTest({
+        measures: ['calendar_orders.count', 'calendar_orders.count_shifted_y_named', 'calendar_orders.count_shifted_y1d_named'],
+        timeDimensions: [{
+          dimension: 'custom_calendar.date_val',
+          granularity: 'year',
+          dateRange: ['2025-02-02', '2026-02-01']
+        }],
+        order: [{ id: 'custom_calendar.date_val' }]
+      }, [
+        {
+          calendar_orders__count: '37',
+          calendar_orders__count_shifted_y_named: '39',
+          calendar_orders__count_shifted_y1d_named: '39',
+          custom_calendar__date_val_year: '2025-02-02T00:00:00.000Z',
+        },
+      ]));
     });
   });
 });

packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap

@@ -87,9 +87,8 @@ Object {
           "type": "prior",
         },
         Object {
-          "interval": "1 year",
+          "name": "retail_date_prev_year",
           "sql": [Function],
-          "type": "prior",
         },
         Object {
           "interval": "2 year",
@@ -273,9 +272,8 @@ Object {
           "type": "prior",
         },
         Object {
-          "interval": "1 year",
+          "name": "retail_date_prev_year",
           "sql": [Function],
-          "type": "prior",
         },
         Object {
           "interval": "2 year",
@@ -1894,6 +1892,7 @@ Object {
   "timeShiftReferences": Array [
     Object {
       "interval": "1 year",
+      "name": undefined,
       "timeDimension": "createdAt",
       "type": "prior",
     },

packages/cubejs-schema-compiler/test/unit/fixtures/calendar_orders.yml

@@ -46,6 +46,13 @@ cubes:
             interval: 1 year
             type: prior
 
+      - name: count_shifted_named
+        type: count
+        multi_stage: true
+        sql: "{count}"
+        time_shift:
+          - name: retail_date_prev_year
+
       - name: completed_count
         type: count
         filters:

packages/cubejs-schema-compiler/test/unit/fixtures/custom_calendar.js

@@ -41,8 +41,7 @@ cube(`custom_calendar_js`, {
               sql: `{CUBE.retail_date_prev_month}`,
           },
           {
-              interval: '1 year',
-              type: 'prior',
+              name: 'retail_date_prev_year',
               sql: `{CUBE.retail_date_prev_year}`,
           },
           {

packages/cubejs-schema-compiler/test/unit/fixtures/custom_calendar.yml

@@ -45,8 +45,7 @@ cubes:
             type: prior
             sql: "{CUBE.retail_date_prev_month}"
 
-          - interval: 1 year
-            type: prior
+          - name: retail_date_prev_year
             sql: "{CUBE.retail_date_prev_year}"
 
           # All needed intervals should be defined explicitly

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/measure_definition.rs

@@ -15,7 +15,8 @@ use std::rc::Rc;
 
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
 pub struct TimeShiftReference {
-    pub interval: String,
+    pub interval: Option<String>,
+    pub name: Option<String>,
     #[serde(rename = "type")]
     pub shift_type: Option<String>,
     #[serde(rename = "timeDimension")]

rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/timeshift_definition.rs

@@ -11,9 +11,10 @@ use std::rc::Rc;
 
 #[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq, Hash)]
 pub struct TimeShiftDefinitionStatic {
-    pub interval: String,
+    pub interval: Option<String>,
     #[serde(rename = "type")]
-    pub timeshift_type: String,
+    pub timeshift_type: Option<String>,
+    pub name: Option<String>,
 }
 
 #[nativebridge::native_bridge(TimeShiftDefinitionStatic)]

rust/cubesqlplanner/cubesqlplanner/src/logical_plan/multistage/common.rs

@@ -58,7 +58,13 @@ impl PrettyPrint for MultiStageAppliedState {
                 &format!(
                     "- {}: {}",
                     time_shift.dimension.full_name(),
-                    time_shift.interval.to_sql()
+                    if let Some(interval) = &time_shift.interval {
+                        interval.to_sql()
+                    } else if let Some(name) = &time_shift.name {
+                        format!("{} (named)", name.to_string())
+                    } else {
+                        "None".to_string()
+                    }
                 ),
                 &details_state,
             );