feat(schema-compiler,api-gateway): Nested folders support (#9659)

* extend validation schema with nested folders

* support nested folders in CubeEvaluator

* support nested folders in meta

* add tests

* fix open api spec

* Fix TCubeFolder type

fix/add TCubeNestedFolder type in client

* Temporary marked folder members as strings

* Revert "Temporary marked folder members as strings"

This reverts commit f7101f3.

* update meta transformer with flat and nested folders

* add CUBEJS_NESTED_FOLDERS_DELIMITER env and related flow

* regenerate cube rust openapi client

* filter uniq folder members

* update tests

* update snapshots

* docs: Nested folders

---------

Co-authored-by: Igor Lukanin <igor@cube.dev>

Author: KSDaemon

docs/pages/product/apis-integrations.mdx

@@ -45,10 +45,11 @@ for an unofficial, community-maintained [client library for Python](https://gith
 Support for data modeling features differ across APIs, integrations, and [visualization
 tools][ref-viz-tools]. Some of the features with partial support are listed below:
 
-| Feature | ✅ Supported in | ❌ Not supported in |
-| --- | --- | --- |
-| [Hierarchies][ref-hierarchies] | [Microsoft Power BI][ref-powerbi] via the [DAX API][ref-dax-api]<br/>[Cube Cloud for Excel][ref-cube-cloud-for-excel]<br/>[Cube Cloud for Sheets][ref-cube-cloud-for-sheets]<br/>[Tableau][ref-tableau] via [Semantic Layer Sync][ref-sls] | All other tools |
-| [Folders][ref-folders] | [Microsoft Power BI][ref-powerbi] via the [DAX API][ref-dax-api]<br/>[Cube Cloud for Excel][ref-cube-cloud-for-excel]<br/>[Cube Cloud for Sheets][ref-cube-cloud-for-sheets]<br/>[Tableau][ref-tableau] via [Semantic Layer Sync][ref-sls]<br/>[Apache Superset][ref-superset] via [Semantic Layer Sync][ref-sls]<br/>[Preset][ref-preset] via [Semantic Layer Sync][ref-sls] | All other tools |
+| Feature | ✅ Supported in |
+| --- | --- |
+| [Hierarchies][ref-hierarchies] | [Microsoft Power BI][ref-powerbi] via the [DAX API][ref-dax-api]<br/>[Cube Cloud for Excel][ref-cube-cloud-for-excel]<br/>[Cube Cloud for Sheets][ref-cube-cloud-for-sheets]<br/>[Tableau][ref-tableau] via [Semantic Layer Sync][ref-sls]<br/><br/>Also, supported in [Playground][ref-playground] |
+| Flat [folders][ref-folders] | [Microsoft Power BI][ref-powerbi] via the [DAX API][ref-dax-api]<br/>[Cube Cloud for Excel][ref-cube-cloud-for-excel]<br/>[Cube Cloud for Sheets][ref-cube-cloud-for-sheets]<br/>[Tableau][ref-tableau] via [Semantic Layer Sync][ref-sls]<br/>[Apache Superset][ref-superset] via [Semantic Layer Sync][ref-sls]<br/>[Preset][ref-preset] via [Semantic Layer Sync][ref-sls]<br/><br/>Also, supported in [Playground][ref-playground] |
+| Nested [folders][ref-folders] | Currently, not supported in any tool |
 
 ### Authentication methods
 
@@ -96,3 +97,4 @@ API][ref-orchestration-api].
 [ref-auth-ntlm]: /product/auth/methods/ntlm
 [ref-superset]: /product/configuration/visualization-tools/superset
 [ref-preset]: /product/configuration/visualization-tools/superset
+[ref-playground]: /product/workspace/playground

docs/pages/product/apis-integrations/rest-api/reference.mdx

@@ -254,7 +254,7 @@ Response
   - `dimensions` - Array of dimensions in this cube/view
   - `hierarchies` - Array of hierarchies in this cube
   - `segments` - Array of segments in this cube/view
-  - `folders` - Array of folders in this view
+  - `folders` and `nestedFolders` - Arrays of flat and nested [folder][ref-folders] structures in this view, respectively
   - `connectedComponent` - An integer representing a join relationship. If the same value is returned for two cubes, then there is
     at least one join path between them.
 
@@ -638,4 +638,5 @@ Keep-Alive: timeout=5
 [ref-query-wpp]: /product/apis-integrations/queries#query-with-post-processing
 [ref-query-wpd]: /product/apis-integrations/queries#query-with-pushdown
 [ref-sql-api]: /product/apis-integrations/sql-api
-[ref-orchestration-api]: /product/apis-integrations/orchestration-api
+[ref-orchestration-api]: /product/apis-integrations/orchestration-api
+[ref-folders]: /product/data-modeling/reference/view#folders

docs/pages/product/configuration/reference/environment-variables.mdx

@@ -1297,6 +1297,21 @@ If `true`, the DAX API will expose time dimensions as calendar hierarchies.
 | --------------- | ---------------------- | --------------------- |
 | `true`, `false` | `true` | `true` |
 
+## `CUBEJS_NESTED_FOLDERS_DELIMITER`
+
+Specifies the delimiter used to flatten the names of nested [folder][ref-folders] in
+views when [visualization tools][ref-dataviz-tools] do not support nested folder
+structures. When set, nested folders will be presented at the root level with path-like
+names using the specified delimiter.
+
+| Possible Values | Default in Development | Default in Production |
+| --------------- | ---------------------- | --------------------- |
+| A valid string delimiter (e.g., ` / `) | N/A | N/A |
+
+For example, if set to `/`, a nested folder structure like the `Customer Information`
+folder with the `Personal Details` subfolder will be flattened to `Customer
+Information / Personal Details` at the root level.
+
 ## `CUBEJS_TELEMETRY`
 
 If `true`, then send telemetry to Cube.
@@ -1781,3 +1796,5 @@ The port for a Cube deployment to listen to API connections on.
 [ref-schema-ref-preagg-allownonstrict]: /product/data-modeling/reference/pre-aggregations#allow_non_strict_date_range_match
 [link-tesseract]: https://cube.dev/blog/introducing-next-generation-data-modeling-engine
 [ref-multi-stage-calculations]: /product/data-modeling/concepts/multi-stage-calculations
+[ref-folders]: /product/data-modeling/reference/view#folders
+[ref-dataviz-tools]: /product/configuration/visualization-tools

docs/pages/product/data-modeling/concepts.mdx

@@ -131,8 +131,7 @@ paths.
 <Diagram src="https://ucarecdn.com/bfc3e04a-b690-40bc-a6f8-14a9175fb4fd/" />
 
 Views do **not** define their own members. Instead, they reference cubes by
-specific join paths and include their members. Optionally, you can also group
-members of a view into [folders][ref-ref-folders].
+specific join paths and include their members.
 
 In the example below, we create the `orders` view which includes select members
 from `base_orders`, `products`, and `users` cubes:
@@ -212,6 +211,16 @@ See the reference documentaton for the full list of view [parameters][ref-views]
 
 </ReferenceBox>
 
+### Folders
+
+Optionally, members of a view can be organized into [folders][ref-ref-folders].
+Each folder would contain a subset of members of the view.
+
+Cube supports both flat and nested folder structures, which can be used with various
+[visualization tools][ref-viz-tools]. If a specific tool does not support nested folders,
+they will be exposed to such a tool as an equivalent flat structure. Check [APIs &
+Integrations][ref-apis-support] for details on the nested folders support.
+
 ## Dimensions
 
 _Dimensions_ represent the properties of a **single** data point in the cube.
@@ -841,4 +850,6 @@ See the reference documentaton for the full list of pre-aggregation
 [ref-avg-and-percentile-recipe]: /product/data-modeling/recipes/percentiles
 [ref-period-over-period-recipe]: /product/data-modeling/recipes/period-over-period
 [ref-custom-calendar-recipe]: /product/data-modeling/recipes/custom-calendar
-[ref-cube-with-dbt]: /product/data-modeling/recipes/dbt
+[ref-cube-with-dbt]: /product/data-modeling/recipes/dbt
+[ref-apis-support]: /product/apis-integrations#data-modeling
+[ref-viz-tools]: /product/configuration/visualization-tools

docs/pages/product/data-modeling/reference/view.mdx

@@ -441,6 +441,120 @@ views:
 
 </CodeTabs>
 
+Nested folders are also supported. The `includes` parameter can contain not only
+references to view members but also other folders:
+
+<CodeTabs>
+
+```javascript
+view(`customers`, {
+  cubes: [
+    {
+      join_path: `users`,
+      includes: `*`
+    },
+    {
+      join_path: `users.orders`,
+      prefix: true,
+      includes: [
+        `status`,
+        `price`,
+        `count`
+      ]
+    }
+  ],
+
+  folders: [
+    {
+      name: `Customer Information`,
+      includes: [
+        {
+          name: `Personal Details`,
+          includes: [
+            `name`,
+            `gender`
+          ]
+        },
+        {
+          name: `Location`,
+          includes: [
+            `address`,
+            `postal_code`,
+            `city`
+          ]
+        }
+      ]
+    },
+    {
+      name: `Order Analytics`,
+      includes: [
+        `orders_status`,
+        `orders_price`,
+        {
+          name: `Metrics`,
+          includes: [
+            `orders_count`,
+            `orders_average_value`
+          ]
+        }
+      ]
+    }
+  ]
+})
+```
+
+```yaml
+views:
+  - name: customers
+
+    cubes:
+      - join_path: users
+        includes: "*"
+
+      - join_path: users.orders
+        prefix: true
+        includes:
+          - status
+          - price
+          - count
+
+    folders:
+      - name: Customer Information
+        includes:
+          - name: Personal Details
+            includes:
+              - name
+              - gender
+
+          - name: Location
+            includes:
+              - address
+              - postal_code
+              - city
+
+      - name: Order Analytics
+        includes:
+          - orders_status
+          - orders_price
+          
+          - name: Metrics
+            includes:
+              - orders_count
+              - orders_average_value
+```
+
+</CodeTabs>
+
+You can still define nested folders in the data model even if some of your [visualization
+tools][ref-viz-tools] do not support them. Check [APIs & Integrations][ref-apis-support]
+for details on the nested folders support.
+
+For tools that do not support nested folders, the nested structure will be flattened:
+by default, the members of nested folders are merged into folders at the root level.
+You can also set the `CUBEJS_NESTED_FOLDERS_DELIMITER` environment variable to preserve
+nested folders and give them path-like names, e.g., `Customer Information / Personal
+Details`.
+
 ### `access_policy`
 
 The `access_policy` parameter is used to configure [data access policies][ref-ref-dap].

packages/cubejs-api-gateway/openspec.yml

@@ -175,6 +175,21 @@ components:
           type: array
           items:
             type: "string"
+    V1CubeMetaNestedFolder:
+      type: "object"
+      required:
+        - name
+        - members
+      properties:
+        name:
+          type: "string"
+        members:
+          type: array
+          items:
+            type: "string"
+            oneOf:
+              - type: string
+              - $ref: "#/components/schemas/V1CubeMetaNestedFolder"
     V1CubeMetaHierarchy:
       type: "object"
       required:
@@ -231,6 +246,10 @@ components:
           type: "array"
           items:
             $ref: "#/components/schemas/V1CubeMetaFolder"
+        nestedFolders:
+          type: "array"
+          items:
+            $ref: "#/components/schemas/V1CubeMetaNestedFolder"
         hierarchies:
           type: "array"
           items:

packages/cubejs-backend-shared/src/env.ts

@@ -240,6 +240,9 @@ const variables: Record<string, (...args: any) => any> = {
   transpilationNative: () => get('CUBEJS_TRANSPILATION_NATIVE')
     .default('false')
     .asBoolStrict(),
+  nestedFoldersDelimiter: () => get('CUBEJS_NESTED_FOLDERS_DELIMITER')
+    .default('')
+    .asString(),
 
   /** ****************************************************************
    * Common db options                                               *

packages/cubejs-client-core/src/types.ts

@@ -417,6 +417,11 @@ export type TCubeFolder = {
   members: string[];
 };
 
+export type TCubeNestedFolder = {
+  name: string;
+  members: (string | TCubeNestedFolder)[];
+};
+
 export type TCubeHierarchy = {
   name: string;
   title?: string;
@@ -451,6 +456,7 @@ export type Cube = {
   dimensions: TCubeDimension[];
   segments: TCubeSegment[];
   folders: TCubeFolder[];
+  nestedFolders: TCubeNestedFolder[];
   hierarchies: TCubeHierarchy[];
   connectedComponent?: number;
   type?: 'view' | 'cube';

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -261,34 +261,51 @@ export class CubeEvaluator extends CubeSymbols {
 
   private prepareFolders(cube: any, errorReporter: ErrorReporter) {
     const folders = cube.rawFolders();
-    if (folders.length) {
-      cube.folders = folders.map(it => {
-        const includedMembers = this.allMembersOrList(cube, it.includes);
-        const includes = includedMembers.map(memberName => {
-          if (memberName.includes('.')) {
-            errorReporter.error(
-              `Paths aren't allowed in the 'folders' but '${memberName}' has been provided for ${cube.name}`
-            );
-          }
+    if (!folders.length) return;
 
-          const member = cube.includedMembers.find(m => m.name === memberName);
-          if (!member) {
-            errorReporter.error(
-              `Member '${memberName}' included in folder '${it.name}' not found`
-            );
-            return null;
-          }
+    const checkMember = (memberName: string, folderName: string) => {
+      if (memberName.includes('.')) {
+        errorReporter.error(
+          `Paths aren't allowed in the 'folders' but '${memberName}' has been provided for ${cube.name}`
+        );
+      }
+
+      const member = cube.includedMembers.find(m => m.name === memberName);
+      if (!member) {
+        errorReporter.error(
+          `Member '${memberName}' included in folder '${folderName}' not found`
+        );
+        return null;
+      }
 
-          return member;
-        })
-          .filter(Boolean);
+      return member;
+    };
+
+    const processFolder = (folder: any): any => {
+      let includedMembers: string[];
+      let includes: any[] = [];
+
+      if (folder.includes === '*') {
+        includedMembers = this.allMembersOrList(cube, folder.includes);
+        includes = includedMembers.map(m => checkMember(m, folder.name)).filter(Boolean);
+      } else if (Array.isArray(folder.includes)) {
+        includes = folder.includes.map(item => {
+          if (typeof item === 'object' && item !== null) {
+            return processFolder(item);
+          }
 
-        return ({
-          ...it,
-          includes
+          return checkMember(item, folder.name);
         });
-      });
-    }
+      }
+
+      return {
+        ...folder,
+        type: 'folder',
+        includes: includes.filter(Boolean)
+      };
+    };
+
+    cube.folders = folders.map(processFolder);
   }
 
   private prepareHierarchies(cube: any, errorReporter: ErrorReporter): void {