feat(schema-compiler): Allow to specify td with granularity in REST API query order section (#9630)

* feat(schema-compiler): Allow to specify td with granularity in REST API query order section

This also fix ordering sql building to use the lowest granularity if there are multiple granularities for the same td provided in the query

* add tests

* implement getFieldAlias() in baseQuery

* update orderHashToString() in QuestQuery

* update orderHashToString() in ClickHouseQuery

* update getFieldIndex in HiveQuery

* refactor AWSElasticSearchQuery & ElasticSearchQuery

* update getFieldAlias() in ElasticSearchQuery

* update getFieldIndex() in DatabricksQuery

* fix defaultOder() issue

* lint fix

* refactor findMinGranularityDimension()

* chore(schema-compiler): move QueryBuilder and QueryCache to ts

Author: KSDaemon

packages/cubejs-api-gateway/src/query.js

@@ -56,7 +56,8 @@ const evaluatedPatchMeasureExpression = parsedPatchMeasureExpression.keys({
 });
 
 const id = Joi.string().regex(/^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$/);
-const idOrMemberExpressionName = Joi.string().regex(/^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$|^[a-zA-Z0-9_]+$/);
+// It might be member name, td+granularity or member expression
+const idOrMemberExpressionName = Joi.string().regex(/^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$|^[a-zA-Z0-9_]+$|^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+$/);
 const dimensionWithTime = Joi.string().regex(/^[a-zA-Z0-9_]+\.[a-zA-Z0-9_]+(\.[a-zA-Z0-9_]+)?$/);
 const parsedMemberExpression = Joi.object().keys({
   expression: Joi.alternatives(

packages/cubejs-backend-shared/src/time.ts

@@ -12,6 +12,67 @@ export type TimeSeriesOptions = {
 };
 type ParsedInterval = Partial<Record<unitOfTime.DurationConstructor, number>>;
 
+const GRANULARITY_LEVELS: Record<string, number> = {
+  second: 1,
+  minute: 2,
+  hour: 3,
+  day: 4,
+  week: 5,
+  month: 6,
+  quarter: 7,
+  year: 8,
+  MAX: 1000,
+};
+
+export type DimensionToCompareGranularity = {
+  dimension: string;
+  expressionName?: string;
+  granularityObj?: {
+    minGranularity(): string;
+  }
+};
+
+/**
+ * Actually dimensions type is (BaseDimension|BaseTimeDimension)[], but can not ref due to cyclic dependencies refs.
+ */
+export function findMinGranularityDimension(id: string, dimensions: DimensionToCompareGranularity[]): { index: number, dimension: DimensionToCompareGranularity | undefined } | null {
+  const equalIgnoreCase = (a: any, b: any): boolean => (
+    typeof a === 'string' && typeof b === 'string' && a.toUpperCase() === b.toUpperCase()
+  );
+
+  let minGranularity = GRANULARITY_LEVELS.MAX;
+  let index = -1;
+  let minGranularityIndex = -1;
+  let field;
+  let minGranularityField;
+
+  dimensions.forEach((d, i) => {
+    if (equalIgnoreCase(d.dimension, id) || equalIgnoreCase(d.expressionName, id)) {
+      field = d;
+      index = i;
+
+      if ('granularityObj' in d && d.granularityObj) {
+        const gr = GRANULARITY_LEVELS[d.granularityObj?.minGranularity()];
+        if (gr < minGranularity) {
+          minGranularityIndex = i;
+          minGranularityField = d;
+          minGranularity = gr;
+        }
+      }
+    }
+  });
+
+  if (minGranularityIndex > -1) {
+    return { index: minGranularityIndex, dimension: minGranularityField };
+  }
+
+  if (index > -1) {
+    return { index, dimension: field };
+  }
+
+  return null;
+}
+
 export const TIME_SERIES: Record<string, (range: DateRange, timestampPrecision: number) => QueryDateRange[]> = {
   day: (range: DateRange, digits) => Array.from(range.snapTo('day').by('day'))
     .map(d => [d.format(`YYYY-MM-DDT00:00:00.${'0'.repeat(digits)}`), d.format(`YYYY-MM-DDT23:59:59.${'9'.repeat(digits)}`)]),

packages/cubejs-databricks-jdbc-driver/src/DatabricksQuery.ts

@@ -144,42 +144,19 @@ export class DatabricksQuery extends BaseQuery {
     return `\`${name}\``;
   }
 
-  public getFieldIndex(id: string) {
-    const dimension = this.dimensionsForSelect().find((d: any) => d.dimension === id);
-    if (dimension) {
-      return super.getFieldIndex(id);
+  public override getFieldIndex(id: string): string | number | null {
+    const idx = super.getFieldIndex(id);
+    if (idx !== null) {
+      return idx;
     }
+
     return this.escapeColumnName(this.aliasName(id, false));
   }
 
   public unixTimestampSql() {
     return 'unix_timestamp()';
   }
 
-  public orderHashToString(hash: any) {
-    if (!hash || !hash.id) {
-      return null;
-    }
-
-    const fieldIndex = this.getFieldIndex(hash.id);
-    if (fieldIndex === null) {
-      return null;
-    }
-
-    const dimensionsForSelect = this.dimensionsForSelect();
-    const dimensionColumns = R.flatten(
-      dimensionsForSelect.map((s: any) => s.selectColumns() && s.aliasName())
-    )
-      .filter(s => !!s);
-
-    if (dimensionColumns.length) {
-      const direction = hash.desc ? 'DESC' : 'ASC';
-      return `${fieldIndex} ${direction}`;
-    }
-
-    return null;
-  }
-
   public defaultRefreshKeyRenewalThreshold() {
     return 120;
   }

packages/cubejs-questdb-driver/src/QuestQuery.ts

@@ -1,4 +1,8 @@
-import { BaseFilter, BaseQuery, ParamAllocator } from '@cubejs-backend/schema-compiler';
+import {
+  BaseFilter,
+  BaseQuery,
+  ParamAllocator
+} from '@cubejs-backend/schema-compiler';
 
 const GRANULARITY_TO_INTERVAL: Record<string, string> = {
   second: 's',
@@ -109,7 +113,7 @@ export class QuestQuery extends BaseQuery {
     }
   }
 
-  public orderHashToString(hash: any): string | null {
+  public orderHashToString(hash: { id: string, desc: boolean }): string | null {
     // QuestDB has partial support for order by index column, so map these to the alias names.
     // So, instead of:
     // SELECT col_a as "a", col_b as "b" FROM tab ORDER BY 2 ASC
@@ -131,32 +135,6 @@ export class QuestQuery extends BaseQuery {
     return `${fieldAlias} ${direction}`;
   }
 
-  private getFieldAlias(id: string): string | null {
-    const equalIgnoreCase = (a: any, b: any) => (
-      typeof a === 'string' && typeof b === 'string' && a.toUpperCase() === b.toUpperCase()
-    );
-
-    let field;
-
-    field = this.dimensionsForSelect().find(
-      (d: any) => equalIgnoreCase(d.dimension, id),
-    );
-
-    if (field) {
-      return field.aliasName();
-    }
-
-    field = this.measures.find(
-      (d: any) => equalIgnoreCase(d.measure, id) || equalIgnoreCase(d.expressionName, id),
-    );
-
-    if (field) {
-      return field.aliasName();
-    }
-
-    return null;
-  }
-
   public groupByClause(): string {
     // QuestDB doesn't support group by index column, so map these to the alias names.
     // So, instead of:

packages/cubejs-schema-compiler/src/adapter/AWSElasticSearchQuery.ts

@@ -1,6 +1,5 @@
-import R from 'ramda';
 import { BaseFilter } from './BaseFilter';
-import { BaseQuery } from './BaseQuery';
+import { ElasticSearchQuery } from './ElasticSearchQuery';
 
 const GRANULARITY_TO_INTERVAL = {
   day: (date) => `DATE_FORMAT(${date}, 'yyyy-MM-dd 00:00:00.000')`,
@@ -21,83 +20,24 @@ class AWSElasticSearchQueryFilter extends BaseFilter {
   }
 }
 
-export class AWSElasticSearchQuery extends BaseQuery {
-  public newFilter(filter) {
+export class AWSElasticSearchQuery extends ElasticSearchQuery {
+  public override newFilter(filter) {
     return new AWSElasticSearchQueryFilter(this, filter);
   }
 
-  public convertTz(field) {
-    return `${field}`; // TODO
-  }
-
-  public timeStampCast(value) {
-    return `${value}`;
-  }
-
-  public dateTimeCast(value) {
-    return `${value}`; // TODO
-  }
-
-  public subtractInterval(date, interval) {
+  public override subtractInterval(date, interval) {
     return `DATE_SUB(${date}, INTERVAL ${interval})`;
   }
 
-  public addInterval(date, interval) {
+  public override addInterval(date, interval) {
     return `DATE_ADD(${date}, INTERVAL ${interval})`;
   }
 
-  public timeGroupedColumn(granularity, dimension) {
+  public override timeGroupedColumn(granularity, dimension) {
     return GRANULARITY_TO_INTERVAL[granularity](dimension);
   }
 
-  public groupByClause() {
-    if (this.ungrouped) {
-      return '';
-    }
-    const dimensionsForSelect = this.dimensionsForSelect();
-    const dimensionColumns = R.flatten(dimensionsForSelect.map(s => s.selectColumns() && s.dimensionSql()))
-      .filter(s => !!s);
-    return dimensionColumns.length ? ` GROUP BY ${dimensionColumns.join(', ')}` : '';
-  }
-
-  public orderHashToString(hash) {
-    if (!hash || !hash.id) {
-      return null;
-    }
-
-    const fieldAlias = this.getFieldAlias(hash.id);
-
-    if (fieldAlias === null) {
-      return null;
-    }
-
-    const direction = hash.desc ? 'DESC' : 'ASC';
-    return `${fieldAlias} ${direction}`;
-  }
-
-  public getFieldAlias(id) {
-    const equalIgnoreCase = (a, b) => (
-      typeof a === 'string' && typeof b === 'string' && a.toUpperCase() === b.toUpperCase()
-    );
-
-    let field;
-
-    field = this.dimensionsForSelect().find(d => equalIgnoreCase(d.dimension, id));
-
-    if (field) {
-      return field.dimensionSql();
-    }
-
-    field = this.measures.find(d => equalIgnoreCase(d.measure, id) || equalIgnoreCase(d.expressionName, id));
-
-    if (field) {
-      return field.aliasName(); // TODO isn't supported
-    }
-
-    return null;
-  }
-
-  public escapeColumnName(name) {
-    return `${name}`; // TODO
+  public override unixTimestampSql() {
+    return 'EXTRACT(EPOCH FROM NOW())';
   }
 }