diff --git a/.github/actions/init-ci/action.yaml b/.github/actions/init-ci/action.yaml index 917e3f87b6aea..4634c32066412 100644 --- a/.github/actions/init-ci/action.yaml +++ b/.github/actions/init-ci/action.yaml @@ -7,11 +7,11 @@ runs: using: "composite" steps: - name: Use Node.js ${{ inputs.node }} - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: ${{ inputs.node }} - name: Cache node modules - uses: actions/cache@v2 + uses: actions/cache@v3 with: # npm cache files are stored in `~/.npm` on Linux/macOS path: ~/.npm diff --git a/.github/actions/integration/snowflake.sh b/.github/actions/integration/snowflake.sh new file mode 100755 index 0000000000000..20423a5477404 --- /dev/null +++ b/.github/actions/integration/snowflake.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -eo pipefail + +# Debug log for test containers +export DEBUG=testcontainers + +echo "::group::Snowflake [cloud]" +export CUBEJS_DB_NAME=DEMO_DB +export CUBEJS_DB_SNOWFLAKE_ACCOUNT=qna80818 +export CUBEJS_DB_SNOWFLAKE_REGION=us-east-1 +export CUBEJS_DB_SNOWFLAKE_WAREHOUSE=COMPUTE_WH + +yarn lerna run --concurrency 1 --stream --no-prefix smoke:snowflake + +echo "::endgroup::" diff --git a/.github/actions/smoke.sh b/.github/actions/smoke.sh new file mode 100755 index 0000000000000..b79f638116829 --- /dev/null +++ b/.github/actions/smoke.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -eo pipefail + +# Debug log for test containers +export DEBUG=testcontainers + +echo "::group::Postgres" +yarn lerna run --concurrency 1 --stream --no-prefix smoke:postgres +echo "::endgroup::" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8aa9d365c33e6..f89a7a798cae0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,9 +4,3 @@ updates: directory: "/packages" schedule: interval: "weekly" - - package-ecosystem: "npm" - directory: "/examples" - schedule: - interval: "monthly" - allow: - - dependency-type: "production" diff --git a/.github/label-actions.yml b/.github/label-actions.yml index dd2caab7b2537..33ba840b80e66 100644 --- a/.github/label-actions.yml +++ b/.github/label-actions.yml @@ -1,5 +1,5 @@ "help wanted": comment: | If you are interested in working on this issue, please leave a comment below and we will be happy to assign the issue to you. - If this is the first time you are contributing a Pull Request to Cube.js, please check our [contribution guidelines](https://github.com/cube-js/cube.js/blob/master/CONTRIBUTING.md). + If this is the first time you are contributing a Pull Request to Cube.js, please check our [contribution guidelines](https://github.com/cube-js/cube/blob/master/CONTRIBUTING.md). You can also post any questions while contributing in the #contributors channel in the [Cube.js Slack](https://slack.cube.dev/). diff --git a/.github/workflows/push-cross-images.yml b/.github/workflows/push-cross-images.yml new file mode 100644 index 0000000000000..3fd747293e5e1 --- /dev/null +++ b/.github/workflows/push-cross-images.yml @@ -0,0 +1,54 @@ +name: Cross Images + +on: + push: + paths: + - '.github/workflows/push-cross-images.yml' + - 'rust/cubestore/cross/**' + branches: + - 'master' + pull_request: + paths: + - '.github/workflows/push-cross-images.yml' + - 'rust/cubestore/cross/**' + +jobs: + docker-dev: + name: Build cross image for ${{ matrix.target }} target + runs-on: ubuntu-20.04 + timeout-minutes: 60 + strategy: + matrix: + target: + - x86_64-unknown-linux-gnu + - x86_64-unknown-linux-musl + # - aarch64-unknown-linux-gnu + fail-fast: false + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Login to DockerHub + if: ${{ github.ref == 'refs/heads/master' }} + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + version: v0.9.1 + driver-opts: network=host + - name: Load .cross file + uses: xom9ikk/dotenv@v2 + with: + path: rust/cubestore/cross/ + - name: Push to Docker Hub + uses: docker/build-push-action@v3 + with: + context: ./ + file: ./rust/cubestore/cross/${{ matrix.target }}.Dockerfile + platforms: linux/amd64 + push: ${{ github.ref == 'refs/heads/master' }} + tags: cubejs/rust-cross:${{ matrix.target }},cubejs/rust-cross:${{ matrix.target }}-${{ env.CROSS_VERSION }} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 24b5edfe867a3..1587e0e3fa04a 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -26,7 +26,7 @@ jobs: strategy: matrix: - node-version: [12.x, 14.x, 16.x] + node-version: [ 14.x, 16.x, 18.x ] fail-fast: false steps: @@ -35,7 +35,7 @@ jobs: env: OUT: ${{ needs['latest-tag-sha'].outputs.sha }} - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: # pulls all commits (needed for codecov) fetch-depth: 2 @@ -46,25 +46,22 @@ jobs: override: true components: rustfmt - name: Install Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: ${{ matrix.node-version }} - name: Get yarn cache directory path id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: Restore lerna - uses: actions/cache@v2 - with: - path: | - ${{ steps.yarn-cache-dir-path.outputs.dir }} - node_modules - rust/cubestore/node_modules - packages/*/node_modules - key: ${{ runner.os }}-workspace-main-${{ matrix.node-version }}-${{ hashFiles('**/yarn.lock') }} + run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT + shell: bash + - name: Restore yarn cache + uses: actions/cache@v3 + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | - ${{ runner.os }}-workspace-main-${{ matrix.node-version }}- + ${{ runner.os }}-yarn- - name: Set Yarn version - run: yarn policies set-version v1.22.5 + run: yarn policies set-version v1.22.19 - name: Yarn install uses: nick-invision/retry@v2 env: @@ -79,14 +76,14 @@ jobs: run: yarn tsc - name: Build client run: yarn build - - name: Lerna test - run: yarn lerna run --concurrency 1 --stream --no-prefix unit - - uses: codecov/codecov-action@v1 - if: (matrix.node-version == '16.x') - with: - files: ./packages/*/coverage/clover.xml - flags: cube-backend - verbose: true # optional (default = false) + # - name: Lerna test + # run: yarn lerna run --concurrency 1 --stream --no-prefix unit + # - uses: codecov/codecov-action@v1 + # if: (matrix.node-version == '16.x') + # with: + # files: ./packages/*/coverage/clover.xml + # flags: cube-backend + # verbose: true # optional (default = false) lint: runs-on: ubuntu-20.04 @@ -96,33 +93,30 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Rust uses: actions-rs/toolchain@v1 with: toolchain: nightly-2022-03-08 override: true components: rustfmt - - name: Install Node.js 14.x - uses: actions/setup-node@v1 + - name: Install Node.js 16.x + uses: actions/setup-node@v3 with: - node-version: 14.x + node-version: 16.x - name: Get yarn cache directory path id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: Restore lerna - uses: actions/cache@v2 - with: - path: | - ${{ steps.yarn-cache-dir-path.outputs.dir }} - node_modules - rust/cubestore/node_modules - packages/*/node_modules - key: ${{ runner.os }}-workspace-main-14.x-${{ hashFiles('**/yarn.lock') }} + run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT + shell: bash + - name: Restore yarn cache + uses: actions/cache@v3 + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | - ${{ runner.os }}-workspace-main-14.x- + ${{ runner.os }}-yarn- - name: Set Yarn version - run: yarn policies set-version v1.22.5 + run: yarn policies set-version v1.22.19 - name: Yarn install uses: nick-invision/retry@v2 env: @@ -146,33 +140,30 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Rust uses: actions-rs/toolchain@v1 with: toolchain: nightly-2022-03-08 override: true components: rustfmt - - name: Install Node.js 14.x - uses: actions/setup-node@v1 + - name: Install Node.js 16.x + uses: actions/setup-node@v3 with: - node-version: 14.x + node-version: 16.x - name: Get yarn cache directory path id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: Restore lerna - uses: actions/cache@v2 - with: - path: | - ${{ steps.yarn-cache-dir-path.outputs.dir }} - node_modules - rust/cubestore/node_modules - packages/*/node_modules - key: ${{ runner.os }}-workspace-main-14.x-${{ hashFiles('**/yarn.lock') }} + run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT + shell: bash + - name: Restore yarn cache + uses: actions/cache@v3 + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | - ${{ runner.os }}-workspace-main-14.x- + ${{ runner.os }}-yarn- - name: Set Yarn version - run: yarn policies set-version v1.22.5 + run: yarn policies set-version v1.22.19 - name: Yarn install uses: nick-invision/retry@v2 env: @@ -189,6 +180,8 @@ jobs: run: yarn build - name: Build other packages run: yarn lerna run --concurrency 1 build + env: + NODE_OPTIONS: --max_old_space_size=4096 integration-redis: needs: [unit, lint, latest-tag-sha] @@ -209,12 +202,12 @@ jobs: strategy: matrix: - node-version: [14.x] + node-version: [16.x] fail-fast: false steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Rust uses: actions-rs/toolchain@v1 with: @@ -222,25 +215,22 @@ jobs: override: true components: rustfmt - name: Install Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: ${{ matrix.node-version }} - name: Get yarn cache directory path id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: Restore lerna - uses: actions/cache@v2 - with: - path: | - ${{ steps.yarn-cache-dir-path.outputs.dir }} - node_modules - rust/cubestore/node_modules - packages/*/node_modules - key: ${{ runner.os }}-workspace-main-${{ matrix.node-version }}-${{ hashFiles('**/yarn.lock') }} + run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT + shell: bash + - name: Restore yarn cache + uses: actions/cache@v3 + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | - ${{ runner.os }}-workspace-main-${{ matrix.node-version }}- + ${{ runner.os }}-yarn- - name: Set Yarn version - run: yarn policies set-version v1.22.5 + run: yarn policies set-version v1.22.19 - name: Yarn install uses: nick-invision/retry@v2 env: @@ -282,6 +272,71 @@ jobs: CUBEJS_REDIS_USE_IOREDIS: true CUBEJS_REDIS_SENTINEL: "redis+sentinel://localhost:5000,localhost:5001,localhost:5002/mymaster/0" + integration-cubestore: + needs: [latest-tag-sha] + runs-on: ubuntu-20.04 + timeout-minutes: 60 + if: (needs['latest-tag-sha'].outputs.sha != github.sha) + + strategy: + matrix: + node-version: [16.x] + fail-fast: false + + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly-2022-06-22 + override: true + components: rustfmt + - name: Install Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT + shell: bash + - name: Restore yarn cache + uses: actions/cache@v3 + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + - name: Set Yarn version + run: yarn policies set-version v1.22.19 + - name: Yarn install + uses: nick-invision/retry@v2 + env: + CUBESTORE_SKIP_POST_INSTALL: true + with: + max_attempts: 3 + retry_on: error + retry_wait_seconds: 15 + timeout_minutes: 20 + command: yarn install --frozen-lockfile + - name: Lerna tsc + run: yarn tsc + - uses: Swatinem/rust-cache@v1 + with: + working-directory: ./rust/cubestore + key: ubuntu-20.04 + - name: Build cubestore + uses: actions-rs/cargo@v1 + with: + command: build + args: --manifest-path rust/cubestore/Cargo.toml -j 4 + - name: Run Cube Store in background + run: RUNNER_TRACKING_ID="" && ./rust/cubestore/target/debug/cubestored & + - name: Run Cubestore Integration + timeout-minutes: 10 + run: | + yarn lerna run --concurrency 1 --stream --no-prefix integration:cubestore + integration: needs: [unit, lint, latest-tag-sha] runs-on: ubuntu-20.04 @@ -290,13 +345,13 @@ jobs: strategy: matrix: - node-version: [14.x] + node-version: [14.x, 16.x] db: ['postgres'] fail-fast: false steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Rust uses: actions-rs/toolchain@v1 with: @@ -304,25 +359,22 @@ jobs: override: true components: rustfmt - name: Install Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: ${{ matrix.node-version }} - name: Get yarn cache directory path id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: Restore lerna - uses: actions/cache@v2 - with: - path: | - ${{ steps.yarn-cache-dir-path.outputs.dir }} - node_modules - rust/node_modules - packages/*/node_modules - key: ${{ runner.os }}-workspace-main-${{ matrix.node-version }}-${{ hashFiles('**/yarn.lock') }} + run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT + shell: bash + - name: Restore yarn cache + uses: actions/cache@v3 + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | - ${{ runner.os }}-workspace-main-${{ matrix.node-version }}- + ${{ runner.os }}-yarn- - name: Set Yarn version - run: yarn policies set-version v1.22.5 + run: yarn policies set-version v1.22.19 - name: Yarn install uses: nick-invision/retry@v2 env: @@ -347,12 +399,12 @@ jobs: strategy: matrix: - node-version: [ 14.x ] + node-version: [ 14.x, 16.x ] fail-fast: false steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Rust uses: actions-rs/toolchain@v1 with: @@ -360,25 +412,22 @@ jobs: override: true components: rustfmt - name: Install Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: ${{ matrix.node-version }} - name: Get yarn cache directory path id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: Restore lerna - uses: actions/cache@v2 - with: - path: | - ${{ steps.yarn-cache-dir-path.outputs.dir }} - node_modules - rust/node_modules - packages/*/node_modules - key: ${{ runner.os }}-workspace-main-${{ matrix.node-version }}-${{ hashFiles('**/yarn.lock') }} + run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT + shell: bash + - name: Restore yarn cache + uses: actions/cache@v3 + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | - ${{ runner.os }}-workspace-main-${{ matrix.node-version }}- + ${{ runner.os }}-yarn- - name: Set Yarn version - run: yarn policies set-version v1.22.5 + run: yarn policies set-version v1.22.19 - name: Yarn install uses: nick-invision/retry@v2 env: @@ -389,6 +438,8 @@ jobs: retry_wait_seconds: 15 timeout_minutes: 20 command: yarn install --frozen-lockfile + - name: Install instant client for Oracle + uses: GoodManWEN/oracle-client-action@main - name: Build client run: yarn build - name: Lerna tsc @@ -402,18 +453,13 @@ jobs: with: command: build args: --manifest-path rust/cubestore/Cargo.toml -j 4 - - name: Run Integration smoke tests - timeout-minutes: 30 + - name: Copy Cube Store run: | mkdir -p rust/cubestore/downloaded/latest/bin cp rust/cubestore/target/debug/cubestored rust/cubestore/downloaded/latest/bin/cubestored - cd packages/cubejs-testing/ - yarn run smoke:postgres - yarn run smoke:crate - yarn run smoke:materialize - yarn run smoke:questdb - yarn run smoke:multidb - yarn run smoke:lambda + - name: Run Integration smoke tests + timeout-minutes: 30 + run: ./.github/actions/smoke.sh docker-image-latest-set-tag: # At least git should be completed pushed up until this moment @@ -424,7 +470,7 @@ jobs: tag: ${{ steps.get-tag.outputs.tag }} steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - id: get-tag run: echo "::set-output name=tag::$(git tag --contains $GITHUB_SHA)" env: @@ -435,7 +481,7 @@ jobs: outputs: sha: ${{ steps.get-tag.outputs.sha }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - id: git-log @@ -451,4 +497,4 @@ jobs: - id: get-tag-out run: echo "$OUT" env: - OUT: ${{ steps.get-tag.outputs.sha }} \ No newline at end of file + OUT: ${{ steps.get-tag.outputs.sha }} diff --git a/.github/workflows/rust-cubesql.yml b/.github/workflows/rust-cubesql.yml index 345c51deae9b2..66acdd9956b8c 100644 --- a/.github/workflows/rust-cubesql.yml +++ b/.github/workflows/rust-cubesql.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Rust uses: actions-rs/toolchain@v1 with: @@ -38,220 +38,3 @@ jobs: # CubeSQL is not ready for Clippy #- name: Clippy CubeSQL # run: cd rust/cubesql && cargo clippy -- -D warnings - - unit: - runs-on: ubuntu-20.04 - timeout-minutes: 40 - name: Unit (Rewrite Engine) - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - # pulls all commits (needed for codecov) - fetch-depth: 2 - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly-2022-03-08 - override: true - components: rustfmt - - uses: Swatinem/rust-cache@v1 - with: - working-directory: ./rust/cubesql - # default key - key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu-16 - sharedKey: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu-16 - - name: Install tarpaulin@0.20.1 - uses: baptiste0928/cargo-install@v1 - with: - crate: cargo-tarpaulin - version: "0.20.1" - - name: Unit tests (Rewrite Engine) - env: - CUBESQL_TESTING_CUBE_TOKEN: ${{ secrets.CUBESQL_TESTING_CUBE_TOKEN }} - CUBESQL_TESTING_CUBE_URL: ${{ secrets.CUBESQL_TESTING_CUBE_URL }} - CUBESQL_REWRITE_ENGINE: true - run: cd rust/cubesql && cargo tarpaulin --workspace --no-fail-fast --avoid-cfg-tarpaulin --out Xml - - name: Upload code coverage - uses: codecov/codecov-action@v2 - with: - files: ./rust/cubesql/cobertura.xml - verbose: true - flags: cubesql - fail_ci_if_error: true - - unit_legacy: - runs-on: ubuntu-20.04 - timeout-minutes: 40 - name: Unit (Legacy) - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - # pulls all commits (needed for codecov) - fetch-depth: 2 - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly-2022-03-08 - override: true - components: rustfmt - - uses: Swatinem/rust-cache@v1 - with: - working-directory: ./rust/cubesql - # default key - key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu-16 - sharedKey: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu-16 - - name: Unit tests (Legacy Engine) - env: - CUBESQL_TESTING_CUBE_TOKEN: ${{ secrets.CUBESQL_TESTING_CUBE_TOKEN }} - CUBESQL_TESTING_CUBE_URL: ${{ secrets.CUBESQL_TESTING_CUBE_URL }} - run: cd rust/cubesql && cargo test - - native_linux: - needs: [lint,unit,unit_legacy] - runs-on: ubuntu-20.04 - timeout-minutes: 60 - name: Build Linux GNU ${{ matrix.node-version }}.x ${{ matrix.target }} - strategy: - matrix: - node-version: [12, 14, 16, 17] - target: ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"] - fail-fast: false - container: - image: cubejs/rust-cross:${{ matrix.target }}-10052021 - - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly-2022-03-08 - override: true - components: rustfmt - target: ${{ matrix.target }} - - uses: Swatinem/rust-cache@v1 - with: - working-directory: ./rust/cubesql - key: cubesql-${{ runner.OS }}-${{ matrix.target }}-${{ matrix.node-version }} - sharedKey: cubesql-${{ runner.OS }}-${{ matrix.target }}-${{ matrix.node-version }} - - name: Install Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - name: Install Yarn - run: npm install -g yarn - - name: Set Yarn version - run: yarn policies set-version v1.22.5 - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: Restore lerna - uses: actions/cache@v2 - with: - path: | - ${{ steps.yarn-cache-dir-path.outputs.dir }} - node_modules - rust/cubesql/node_modules - packages/*/node_modules - key: ${{ runner.os }}-workspace-main-${{ matrix.node-version }}-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-workspace-main-${{ matrix.node-version }}- - - name: Yarn install - uses: nick-invision/retry@v2 - env: - CUBESTORE_SKIP_POST_INSTALL: true - with: - max_attempts: 3 - retry_on: error - retry_wait_seconds: 15 - timeout_minutes: 20 - command: yarn install --frozen-lockfile - - name: Build native - env: - CARGO_BUILD_TARGET: ${{ matrix.target }} - run: cd packages/cubejs-backend-native && yarn run native:build - - name: Test native (GNU only) - if: (matrix.target == 'x86_64-unknown-linux-gnu') - env: - CUBEJS_NATIVE_INTERNAL_DEBUG: true - run: cd packages/cubejs-backend-native && yarn run test:unit - - name: Run E2E Smoke testing over whole Cube (GNU only) - if: (matrix.target == 'x86_64-unknown-linux-gnu') - env: - CUBEJS_NATIVE_INTERNAL_DEBUG: true - run: yarn tsc && cd packages/cubejs-testing && yarn smoke:cubesql - - native: - needs: [lint,unit,unit_legacy] - runs-on: ${{ matrix.os-version }} - timeout-minutes: 60 - name: Build ${{ matrix.os-version }} ${{ matrix.node-version }} - - strategy: - matrix: - node-version: [12.x, 14.x, 16.x, 17.x] - os-version: [windows-2019, macos-11] - fail-fast: false - - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly-2022-03-08 - override: true - components: rustfmt - - name: Install Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - name: Install Yarn - run: npm install -g yarn - - name: Set Yarn version - run: yarn policies set-version v1.22.5 - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: Restore lerna (excluding Windows) - uses: actions/cache@v2 - if: ${{ !startsWith(matrix.os-version, 'windows') }} - with: - path: | - ${{ steps.yarn-cache-dir-path.outputs.dir }} - node_modules - rust/cubesql/node_modules - packages/*/node_modules - key: ${{ runner.os }}-workspace-main-${{ matrix.node-version }}-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-workspace-main-${{ matrix.node-version }}- - # Windows doesn't support links + cache action doesn't support exclude - # cache will include rust cache for native, which causes 3GB archive - # Right now, GH provides 10 gb for each repository with automatic retention. - - name: Restore lerna (Windows only) - uses: actions/cache@v2 - if: ${{ startsWith(matrix.os-version, 'windows') }} - with: - path: | - ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-workspace-main-${{ matrix.node-version }}-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-workspace-main-${{ matrix.node-version }}- - - name: Yarn install - uses: nick-invision/retry@v2 - env: - CUBESTORE_SKIP_POST_INSTALL: true - with: - max_attempts: 3 - retry_on: error - retry_wait_seconds: 15 - timeout_minutes: 20 - command: yarn install --frozen-lockfile - - name: Build native - env: - CUBEJS_NATIVE_INTERNAL_DEBUG: true - run: cd packages/cubejs-backend-native && yarn run native:build && yarn run test:unit diff --git a/.github/workflows/rust-cubestore.yml b/.github/workflows/rust-cubestore.yml index 22a888aef074a..791a4637940fd 100644 --- a/.github/workflows/rust-cubestore.yml +++ b/.github/workflows/rust-cubestore.yml @@ -19,7 +19,7 @@ jobs: RUST: ${{ matrix.rust }} steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Rust uses: actions-rs/toolchain@v1 with: @@ -62,28 +62,24 @@ jobs: target: x86_64-unknown-linux-gnu platforms: linux/amd64 build-args: WITH_AVX2=1 - - os: self-hosted - target: aarch64-unknown-linux-gnu - platforms: linux/arm64 - build-args: WITH_AVX2=0 timeout-minutes: 60 if: github.ref != 'refs/heads/master' steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 with: - version: v0.6.3 + version: v0.9.1 - name: Cache Docker layers - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-${{ matrix.target }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-${{ matrix.target }}-buildx- - name: Build only - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: ./rust/cubestore/ file: ./rust/cubestore/Dockerfile @@ -104,7 +100,6 @@ jobs: target: - x86_64-unknown-linux-gnu - x86_64-unknown-linux-musl - - x86_64-pc-windows-msvc - x86_64-apple-darwin - aarch64-unknown-linux-gnu include: @@ -113,7 +108,7 @@ jobs: executable_name: cubestored cross: true strip: true - compress: true + compress: false - os: ubuntu-20.04 target: x86_64-unknown-linux-musl executable_name: cubestored @@ -121,14 +116,7 @@ jobs: strip: true # cubestored: CantPackException: bad DT_HASH nbucket=0x344 len=0x1890 compress: false - - os: windows-2019 - target: x86_64-pc-windows-msvc - executable_name: cubestored.exe - cross: false - strip: true - # cubestored.exe: CantPackException: superfluous data between sections - compress: false - - os: macos-latest + - os: macos-11 target: x86_64-apple-darwin executable_name: cubestored cross: false @@ -140,11 +128,15 @@ jobs: cross: true # Unable to recognise the format of the input file `rust/cubestore/target/aarch64-unknown-linux-gnu/release/cubestored' strip: false - # UPX is broken, issue https://github.com/cube-js/cube.js/issues/4474 + # UPX is broken, issue https://github.com/cube-js/cube/issues/4474 compress: false fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Disable rustup update (issue workaround for Windows) + run: rustup set auto-self-update disable + if: contains(runner.os, 'windows') + shell: bash - name: Setup Rust toolchain uses: actions-rs/toolchain@v1 with: @@ -160,6 +152,8 @@ jobs: - run: source .github/actions/${{ matrix.before_script }}.sh if: ${{ matrix.before_script }} shell: bash + - uses: ilammy/msvc-dev-cmd@v1 + if: ${{ startsWith(matrix.os, 'windows') }} - name: Install OpenSSL for Windows if: ${{ startsWith(matrix.os, 'windows') }} run: vcpkg integrate install; vcpkg install openssl:x64-windows @@ -175,6 +169,10 @@ jobs: OPENSSL_LIB_DIR: 'C:/vcpkg/packages/openssl_x64-windows/lib' OPENSSL_INCLUDE_DIR: 'C:/vcpkg/packages/openssl_x64-windows/include' LIBCLANG_PATH: 'C:\Program Files\LLVM\bin' + # Hotfix before https://github.com/actions/runner-images/pull/7125 will be released/rolled on the productions servers + - name: Hotfix for macOS (pkg-config) + if: contains(runner.os, 'macos') + run: brew install pkg-config - name: Build with Cross if: ${{ matrix.cross }} run: | @@ -186,11 +184,6 @@ jobs: if: ${{ !matrix.cross }} run: | cd rust/cubestore && cargo build --release --target=${{ matrix.target }} - - name: Brew update & install upx (workaround to use upx from github, instead of bintray) - if: ${{ matrix.os == 'macos-latest' }} - run: | - brew update - brew install upx - name: Compress binaries uses: svenstaro/upx-action@v2 if: ${{ matrix.compress }} diff --git a/.nvmrc b/.nvmrc index d8ff873d1ac50..fac0b0a8394db 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -12.22.7 +16.20.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 592d41eec3af6..d43c0a4bfac23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,1891 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [0.33.11](https://github.com/cube-js/cube/compare/v0.33.10...v0.33.11) (2023-05-22) + + +### Bug Fixes + +* **docker:** Don't load native on alpine (not supported), fix [#6510](https://github.com/cube-js/cube/issues/6510) ([#6636](https://github.com/cube-js/cube/issues/6636)) ([a01d2f9](https://github.com/cube-js/cube/commit/a01d2f953973eee79b6135f7883ecff56bb71486)) + + +### Features + +* **cubestore:** Optimization of inmemory chunk operations for streaming ([#6354](https://github.com/cube-js/cube/issues/6354)) ([146798a](https://github.com/cube-js/cube/commit/146798affff184cab489628a1a40611b7fb5879a)) +* **docker:** Install DuckDB as built-in driver ([b1e77bd](https://github.com/cube-js/cube/commit/b1e77bdee1d9b35a46c51387d8d16565d332754b)) +* **duckdb-driver:** Upgrade DuckDB to 0.8 ([7d91f9f](https://github.com/cube-js/cube/commit/7d91f9f2ec187f8032a768cb35e46bbd557e8125)) +* Improve LangChain support ([15e51bc](https://github.com/cube-js/cube/commit/15e51bcb19ce22c38b71f4685484295fc637e44c)) + + + + + +## [0.33.10](https://github.com/cube-js/cube/compare/v0.33.9...v0.33.10) (2023-05-19) + + +### Bug Fixes + +* remove noisy Warning: Attribute `filter.dimension` is deprecated. Please use 'member' instead of 'dimension' ([546227a](https://github.com/cube-js/cube/commit/546227a96c5c41078c82a75d7e4697dfd505e181)) + + + + + +## [0.33.9](https://github.com/cube-js/cube/compare/v0.33.8...v0.33.9) (2023-05-18) + + +### Bug Fixes + +* **databricks-jdbc-driver:** Error: [Databricks][JDBC](11220) Parameters cannot be used with normal Statement objects, use PreparedStatements instead. ([328bb79](https://github.com/cube-js/cube/commit/328bb79853833420a287fc9e31dda72a9886999d)), closes [#6627](https://github.com/cube-js/cube/issues/6627) +* Invalid query format: \"filters[0]\" does not match any of the allowed types if queryRewrite is using non string values ([ac57b4c](https://github.com/cube-js/cube/commit/ac57b4c15cb39392faa75a01c042fdd5ecb19af9)) + + + + + +## [0.33.8](https://github.com/cube-js/cube/compare/v0.33.7...v0.33.8) (2023-05-17) + + +### Bug Fixes + +* **athena-driver:** Fix partitioned pre-aggregations and column values with `,` through export bucket ([#6596](https://github.com/cube-js/cube/issues/6596)) ([1214cab](https://github.com/cube-js/cube/commit/1214cabf69f9e6216c516d05acadfe7e6178cccf)) +* **graphql:** exclude empty cubes, revert equals/notEquals type ([#6619](https://github.com/cube-js/cube/issues/6619)) ([fab1b6a](https://github.com/cube-js/cube/commit/fab1b6abe198b1d83b4484bae66108782e62887c)) + + +### Features + +* **bigquery-driver:** Specify timeout for job on testConnection ([#6588](https://github.com/cube-js/cube/issues/6588)) ([d724f09](https://github.com/cube-js/cube/commit/d724f09113e4b4cb8c703fc02e9d9e13c3e05fb7)) +* **cubestore-driver:** Queue - protect possible race condition (query def) [#6616](https://github.com/cube-js/cube/issues/6616)) ([60294de](https://github.com/cube-js/cube/commit/60294defff88ee96d95d1d34b1fe17fccdb17b71)) + + + + + +## [0.33.7](https://github.com/cube-js/cube/compare/v0.33.6...v0.33.7) (2023-05-16) + + +### Bug Fixes + +* Limit override isn't respected in queryRewrite ([#6605](https://github.com/cube-js/cube/issues/6605)) ([172c39b](https://github.com/cube-js/cube/commit/172c39b6fb2609f003a364efb63d8091e39607a0)) + + +### Features + +* **cubestore-driver:** Queue - protect race conditions ([#6598](https://github.com/cube-js/cube/issues/6598)) ([d503598](https://github.com/cube-js/cube/commit/d50359845d388ccadc8d5a7233ed7317849558f4)) +* **cubestore:** Queue - allow to GET by processingId ([#6608](https://github.com/cube-js/cube/issues/6608)) ([f0c1156](https://github.com/cube-js/cube/commit/f0c11565426b5604bae665fc52d5b416f4f055d2)) +* **cubestore:** Queue - delete results via scheduler ([#6515](https://github.com/cube-js/cube/issues/6515)) ([8e1ab20](https://github.com/cube-js/cube/commit/8e1ab2044a92f9e71918085a31716a784cd5bb9f)) + + + + + +## [0.33.6](https://github.com/cube-js/cube/compare/v0.33.5...v0.33.6) (2023-05-13) + + +### Bug Fixes + +* **cubesql:** Improve NULL comparison with int/bool ([de1be39](https://github.com/cube-js/cube/commit/de1be39d07ceec5d87d6c7aef8adb65fbe246ee3)) +* **cubestore:** Optimize unique constraint check on insert ([c227ae9](https://github.com/cube-js/cube/commit/c227ae9f72cfd66fd47e60fc567c2fde88ebc932)) +* **cubestore:** Schedule compaction without blocking rw loop ([f9bbbac](https://github.com/cube-js/cube/commit/f9bbbac278898d02d50e6ff5911658d5e8caf7e3)) +* LIMIT is not enforced ([#6586](https://github.com/cube-js/cube/issues/6586)) ([8ca5234](https://github.com/cube-js/cube/commit/8ca52342944b9767f2c34591a9241bf31cf78c71)) +* **snowflake-driver:** Bind variable ? not set for partitioned pre-aggregations ([#6594](https://github.com/cube-js/cube/issues/6594)) ([0819075](https://github.com/cube-js/cube/commit/081907568d97fa79f56edf1898b2845affb925cf)) + + +### Features + +* **cubestore:** Queue - support cancel, ack, result_blocking by processingId ([#6551](https://github.com/cube-js/cube/issues/6551)) ([2e82839](https://github.com/cube-js/cube/commit/2e82839586f18d8b9ba385ab6f8e5641113ccf0c)) +* **cubestore:** Support custom CSV delimiters ([#6597](https://github.com/cube-js/cube/issues/6597)) ([f8c2c88](https://github.com/cube-js/cube/commit/f8c2c888150e9436c90de9ecd45f540bb0314855)) + + + + + +## [0.33.5](https://github.com/cube-js/cube/compare/v0.33.4...v0.33.5) (2023-05-11) + + +### Bug Fixes + +* **cubejs-playground:** sticky bar styles ([#6558](https://github.com/cube-js/cube/issues/6558)) ([c4864c5](https://github.com/cube-js/cube/commit/c4864c508ffd42bebbc8e1fb8d98523fa1d252b7)) +* fix includes all for views ([#6559](https://github.com/cube-js/cube/issues/6559)) ([f62044a](https://github.com/cube-js/cube/commit/f62044aa7b60f6a2eeafc60e5649629aadc287a2)) +* typo in docs ([8c36022](https://github.com/cube-js/cube/commit/8c3602297299af44bca1427646c95e50ddf4b991)) +* yml formatter escapse double quotes ([341e115](https://github.com/cube-js/cube/commit/341e115ef7fd6475f3d947fb0aa878addee86eb9)) + + +### Features + +* **cubestore:** Support converting Int16/32, UInt16/32 for DataFrame ([#6569](https://github.com/cube-js/cube/issues/6569)) ([5e0fd4f](https://github.com/cube-js/cube/commit/5e0fd4fd73717d50c1a0684e840d987fb95df495)) + + + + + +## [0.33.4](https://github.com/cube-js/cube/compare/v0.33.3...v0.33.4) (2023-05-07) + + +### Bug Fixes + +* **docs:** fix views example in style guide ([1296ddb](https://github.com/cube-js/cube/commit/1296ddb06d5e0ae151c8dc7ec7f0ad77e429333f)) +* join_path for JS schemas and * includes with prefix ([#6555](https://github.com/cube-js/cube/issues/6555)) ([c76bbef](https://github.com/cube-js/cube/commit/c76bbef36e5b6594bcb79174ceb672d8e2b870ff)) + + +### Features + +* **cubestore:** Projection in kafka streaming queries ([#6479](https://github.com/cube-js/cube/issues/6479)) ([e0495a9](https://github.com/cube-js/cube/commit/e0495a9f46dd8a78fce069b31382fdeec37da8d0)) + + + + + +## [0.33.3](https://github.com/cube-js/cube/compare/v0.33.2...v0.33.3) (2023-05-05) + + +### Bug Fixes + +* **databricks-driver:** Increase default JDBC acquire timeout to 2 minutes as it can take more time to reconnect ([dde9bc4](https://github.com/cube-js/cube/commit/dde9bc401d3629e5284e3d160ba41023316b7e8c)) +* **databricks-driver:** Remove package.json dependency from DatabricksDriver to avoid package.json not found issues ([#6521](https://github.com/cube-js/cube/issues/6521)) Thanks [@carlagouveia](https://github.com/carlagouveia) ! ([b3ac7a1](https://github.com/cube-js/cube/commit/b3ac7a10b74988af42bd6f289a271a71a92dee5e)) +* **graphql:** Pre-aggregations aren't used for GraphQL queries with date range ([#6549](https://github.com/cube-js/cube/issues/6549)) ([a343f4d](https://github.com/cube-js/cube/commit/a343f4d2f0bdf62120e99fb3252f6460f4a85e8e)) +* **snowflake-driver:** Float exported as decimal for pre-aggregations ([#6544](https://github.com/cube-js/cube/issues/6544)) ([7c8b8de](https://github.com/cube-js/cube/commit/7c8b8ded5f7bb16248989bd56b0913180321314d)) + + +### Features + +* **docker:** Security upgrade node from 16.19.1 to 16.20 ([#6547](https://github.com/cube-js/cube/issues/6547)) ([5523935](https://github.com/cube-js/cube/commit/55239350d8b629d55ec8ee61fd2c4a6f00057f88)) + + + + + +## [0.33.2](https://github.com/cube-js/cube/compare/v0.33.1...v0.33.2) (2023-05-04) + + +### Bug Fixes + +* **cubejs-playground:** remove less file import ([#6537](https://github.com/cube-js/cube/issues/6537)) ([2a4de4b](https://github.com/cube-js/cube/commit/2a4de4b85746ed8c74c57b2cd18c6f69f2e65bd4)) +* **cubestore:** Queue - protect possible race condition on GET_RESULT_BLOCKING ([#6535](https://github.com/cube-js/cube/issues/6535)) ([d962c05](https://github.com/cube-js/cube/commit/d962c058f703a5f5156c36ceb8cae074b8d15d02)) + + +### Features + +* **cubestore-driver:** Queue - track orphaned set result ([f97fbdc](https://github.com/cube-js/cube/commit/f97fbdcbe1e8c98d568f0614be8074e32d22343f)) +* **cubestore:** Queue - allow to do merge_extra, heartbeat by processingId ([#6534](https://github.com/cube-js/cube/issues/6534)) ([bf8df29](https://github.com/cube-js/cube/commit/bf8df29c261625f588faf65984047026d760020a)) +* **cubestore:** Queue - expose processingId in queue_add ([#6540](https://github.com/cube-js/cube/issues/6540)) ([24b4798](https://github.com/cube-js/cube/commit/24b4798fb5e26b2be0f04f3e1af76c1b8516a9b8)) +* **cubestore:** Queue - expose processingId in queue_retrieve ([#6530](https://github.com/cube-js/cube/issues/6530)) ([43913ce](https://github.com/cube-js/cube/commit/43913ce459e1b60dd04f3a23aed1c617de0e0422)) +* **cubestore:** Queue - return success marker on ack (set result) ([e8814bf](https://github.com/cube-js/cube/commit/e8814bf5ebd18431f0d4a61db7fee2a519a5bdd8)) + + + + + +## [0.33.1](https://github.com/cube-js/cube/compare/v0.33.0...v0.33.1) (2023-05-03) + + +### Bug Fixes + +* **firebolt-driver:** Automatically cast boolean parameters ([#6531](https://github.com/cube-js/cube/issues/6531)) ([90136dc](https://github.com/cube-js/cube/commit/90136dc565653a65a6682591206f4cea98b6bd97)) +* **snowflake-driver:** Revert read-only mode until useOriginalSqlPreAggregations flow is fixed for read only ([#6532](https://github.com/cube-js/cube/issues/6532)) ([20651a9](https://github.com/cube-js/cube/commit/20651a94f2b4e9919e2b5e7761bb725ca07efc54)) + + + + + +# [0.33.0](https://github.com/cube-js/cube/compare/v0.32.31...v0.33.0) (2023-05-02) + +**Note:** Version bump only for package cubejs + + + + + +## [0.32.31](https://github.com/cube-js/cube/compare/v0.32.30...v0.32.31) (2023-05-02) + + +### Bug Fixes + +* **bigquery-driver:** Revert read-only implementation as `LIMIT 1` queries generate too much usage ([#6523](https://github.com/cube-js/cube/issues/6523)) ([f023484](https://github.com/cube-js/cube/commit/f02348408103f7eeb9d6cd8bea959549a78fe943)) +* **cubesql:** Streaming error logging ([e27a525](https://github.com/cube-js/cube/commit/e27a5252380a55682f450bf057767074394179a2)) + + +### Features + +* **cubesql:** psqlodbc driver support ([fcfc7ea](https://github.com/cube-js/cube/commit/fcfc7ea9f8bfd027aa549d139b9ed12b773594ec)) +* Support snake case security_context in COMPILE_CONTEXT ([#6519](https://github.com/cube-js/cube/issues/6519)) ([feb3fe3](https://github.com/cube-js/cube/commit/feb3fe389462fc4cf3947f8ecf2146c3fe102b9e)) + + + + + +## [0.32.30](https://github.com/cube-js/cube/compare/v0.32.29...v0.32.30) (2023-04-28) + + +### Bug Fixes + +* Always log authentication errors for debugging purpose ([2c309b7](https://github.com/cube-js/cube/commit/2c309b7d8122c19397a649e51d4b2a032361f8ee)) +* **cubejs-playground:** SL-58 CC-1686 text and icon fixes ([#6490](https://github.com/cube-js/cube/issues/6490)) ([1fa96f0](https://github.com/cube-js/cube/commit/1fa96f04b7068887c0541f2fcfba51400b8ddc3c)) +* **cubesql:** Resolve Grafana introspection issues ([db32377](https://github.com/cube-js/cube/commit/db32377f6e6d45c8c16b12ee7e51fdf1e9687fc9)) +* **cubestore:** Error in Sparse HLL entry parsing ([#6498](https://github.com/cube-js/cube/issues/6498)) ([9719cdc](https://github.com/cube-js/cube/commit/9719cdcc39461948e85ea7f2c4bdae377a8dba83)) +* model style guide, views folder ([#6493](https://github.com/cube-js/cube/issues/6493)) ([1fa7f99](https://github.com/cube-js/cube/commit/1fa7f994551e05fd2ef417bd2ee7f472881bc2bc)) +* **snowflake-driver:** Int is exported to pre-aggregations as decimal… ([#6513](https://github.com/cube-js/cube/issues/6513)) ([3710b11](https://github.com/cube-js/cube/commit/3710b113160d4b0f53b40d6b31ae9c901aa51571)) + + +### Features + +* **bigquery-driver:** CI, read-only, streaming and unloading ([#6495](https://github.com/cube-js/cube/issues/6495)) ([4c07431](https://github.com/cube-js/cube/commit/4c07431033df7554ffb6f9d5f64eca156267b3e3)) +* **gateway:** GraphQL snake case support ([#6480](https://github.com/cube-js/cube/issues/6480)) ([45a46d0](https://github.com/cube-js/cube/commit/45a46d0556c07842830bf55447f4f575fc158ef6)) +* **playground:** cube type tag, public cubes ([#6482](https://github.com/cube-js/cube/issues/6482)) ([cede7a7](https://github.com/cube-js/cube/commit/cede7a71f7d2e8d9dc221669b6b1714ee146d8ea)) + + + + + +## [0.32.29](https://github.com/cube-js/cube/compare/v0.32.28...v0.32.29) (2023-04-25) + + +### Bug Fixes + +* **api-gateway:** passing defaultApiScope env to the contextToApiScopesFn ([#6485](https://github.com/cube-js/cube/issues/6485)) ([b2da950](https://github.com/cube-js/cube/commit/b2da950509e48bdfcd5d6f282f81a00b3794d559)) +* **schema-compiler:** yml backtick escaping ([#6478](https://github.com/cube-js/cube/issues/6478)) ([39a9ac0](https://github.com/cube-js/cube/commit/39a9ac0e9a641efeeca0e99d286962aa007551cd)) +* **server-core:** Display refresh scheduler interval error as warning ([#6473](https://github.com/cube-js/cube/issues/6473)) ([771f661](https://github.com/cube-js/cube/commit/771f661cadfe9c19bcaad9fe232f7f4901f0778b)) + + +### Features + +* **athena-driver:** read-only unload ([#6469](https://github.com/cube-js/cube/issues/6469)) ([d3fee7c](https://github.com/cube-js/cube/commit/d3fee7cbefe5c415573c4d2507b7e61a48f0c91a)) +* **cubestore:** Split partitions by filesize ([#6435](https://github.com/cube-js/cube/issues/6435)) ([5854add](https://github.com/cube-js/cube/commit/5854add7a65c39e05ba71c2a558032671797ec26)) +* **cubestore:** Traffic tracing for streaming ([#6379](https://github.com/cube-js/cube/issues/6379)) ([37a33e6](https://github.com/cube-js/cube/commit/37a33e65234740a8028c340334510facbea896d7)) +* Make configurable cubestore metrics address ([#6437](https://github.com/cube-js/cube/issues/6437)) Thanks [@fbalicchia](https://github.com/fbalicchia) ! ([16d0bcb](https://github.com/cube-js/cube/commit/16d0bcb1489c512f91c21970e21769c9882bfedc)) +* **schema-compiler:** Support public/shown for segments ([#6491](https://github.com/cube-js/cube/issues/6491)) ([df6160c](https://github.com/cube-js/cube/commit/df6160cc62f8ab2fccbfb85f2916040aec218fc3)) +* semanticLayerSync configs option ([#6483](https://github.com/cube-js/cube/issues/6483)) ([d3b8ed7](https://github.com/cube-js/cube/commit/d3b8ed7401f98429eca42227ef4cfe1f29f90d8e)) +* **snowflake-driver:** streaming export, read-only unload ([#6452](https://github.com/cube-js/cube/issues/6452)) ([67565b9](https://github.com/cube-js/cube/commit/67565b975c16f93070de0346056c6a3865bc9fd8)) + + + + + +## [0.32.28](https://github.com/cube-js/cube/compare/v0.32.27...v0.32.28) (2023-04-19) + + +### Bug Fixes + +* **mssql-driver:** release stream connection ([#6453](https://github.com/cube-js/cube/issues/6453)) ([7113fa1](https://github.com/cube-js/cube/commit/7113fa1247f3761c9fde5f8d1af8de6036350e72)) +* **schema-compiler:** Incorrect camelize for meta ([#6444](https://github.com/cube-js/cube/issues/6444)) ([c82844e](https://github.com/cube-js/cube/commit/c82844e130b31722fa338022b740ac7285c01190)) +* segment not found for path if rollup with segment is used within `rollupJoin` ([6a3ca80](https://github.com/cube-js/cube/commit/6a3ca8032151d530a30bab88c024c451d4141c01)) + + +### Features + +* **cubesql:** Support new Thoughtspot introspection ([a04c83a](https://github.com/cube-js/cube/commit/a04c83a070ddbd2924528ce5e7ecf10c7b4f235c)) +* **cubesql:** Support psql's `\list` command ([0b30def](https://github.com/cube-js/cube/commit/0b30def71aa5bb53dbc59ddb6a4b63bd59eda95a)) +* **playground:** use snake case for data models ([398993a](https://github.com/cube-js/cube/commit/398993ad9772a52ad377542937614148c597ca72)) +* using snake case for model files, model templates updates ([#6447](https://github.com/cube-js/cube/issues/6447)) ([37e3180](https://github.com/cube-js/cube/commit/37e3180d4218b342ee0c2fe0ef90464aa6633b54)) + + + + + +## [0.32.27](https://github.com/cube-js/cube/compare/v0.32.26...v0.32.27) (2023-04-14) + + +### Bug Fixes + +* Do not use in memory cache for more than 5 minutes to avoid any long term discrepancies with upstream cache ([aea2daf](https://github.com/cube-js/cube/commit/aea2daf38ddde61fd3f00a886b7679bef060948f)) +* **ksql-driver:** Drop table with the topic to avoid orphaned topics ([b113841](https://github.com/cube-js/cube/commit/b113841ad52a52b4ea65192277819fce50725f02)) +* Limit pre-aggregations API memory usage by introducing partition loading concurrency ([f565371](https://github.com/cube-js/cube/commit/f565371fe9d05267bbacd37369dbc25780e4da19)) + + + + + +## [0.32.26](https://github.com/cube-js/cube/compare/v0.32.25...v0.32.26) (2023-04-13) + + +### Bug Fixes + +* **cli:** Validate command - allow node require ([#6416](https://github.com/cube-js/cube/issues/6416)) ([ead63d8](https://github.com/cube-js/cube/commit/ead63d823253e8cd42e9353a09d4481e5e889d0b)) +* Ensure expired in memory cache value can't be used even in case renewalThreshold hasn't been met ([64f5878](https://github.com/cube-js/cube/commit/64f58789b6eab44d25f8d4305b3e8b299f4735bb)) +* re-export FileRepository ([#6425](https://github.com/cube-js/cube/issues/6425)) ([52c9252](https://github.com/cube-js/cube/commit/52c92529c5c13994b81b7af616fdf329cb50a0f6)) + + +### Features + +* **schema-compiler:** JS - support snake_case for any properties ([#6396](https://github.com/cube-js/cube/issues/6396)) ([f1b4c0b](https://github.com/cube-js/cube/commit/f1b4c0bfb2e8b8ad20853af6232388c6d764a780)) + + + + + +## [0.32.25](https://github.com/cube-js/cube/compare/v0.32.24...v0.32.25) (2023-04-12) + + +### Bug Fixes + +* **schema-compiler:** Yaml - crash on fully commented file ([#6413](https://github.com/cube-js/cube/issues/6413)) ([29d2eae](https://github.com/cube-js/cube/commit/29d2eaed8c2d8fe7575be732bc21acbf8dadafbb)) +* **server-core:** turn off the jobs API renewQuery flag ([#6414](https://github.com/cube-js/cube/issues/6414)) ([81dddb1](https://github.com/cube-js/cube/commit/81dddb1ecb6d5e5d97c6fe3dd277ff3b5f651b51)) + + +### Features + +* **api-gateway, server-core:** Orchestration API telemetry ([#6419](https://github.com/cube-js/cube/issues/6419)) ([af0854d](https://github.com/cube-js/cube/commit/af0854d0d8cdc05eab2df090a2d0314a3d27fc23)) +* **api-gateway, server-core:** Renamed Permissions to ApiScopes ([#6397](https://github.com/cube-js/cube/issues/6397)) ([800a96b](https://github.com/cube-js/cube/commit/800a96b68e5aa9bbeb2c52edaab0698832a18619)) +* **duckdb-driver:** Support countDistinctApprox ([#6409](https://github.com/cube-js/cube/issues/6409)) ([6b1fadb](https://github.com/cube-js/cube/commit/6b1fadb0954ce349a55b88a158d236c2ca31ecfc)) + + + + + +## [0.32.24](https://github.com/cube-js/cube/compare/v0.32.23...v0.32.24) (2023-04-10) + + +### Reverts + +* Revert "feat(docker): Install DuckDB as built-in driver" ([99ea6c1](https://github.com/cube-js/cube/commit/99ea6c15333b3d0c19011cb58ed8a8d10e6b0d3c)) + + + + + +## [0.32.23](https://github.com/cube-js/cube/compare/v0.32.22...v0.32.23) (2023-04-10) + + +### Bug Fixes + +* Fix typo in driver name, DuckDB ([c53a31c](https://github.com/cube-js/cube/commit/c53a31c37b6b70fa3c5e2fc0da97eb958e9fec5d)) + + +### Features + +* **docker:** Install DuckDB as built-in driver ([41a0ca1](https://github.com/cube-js/cube/commit/41a0ca11047ed907db354c3f6e49c3e097a584b9)) + + + + + +## [0.32.22](https://github.com/cube-js/cube/compare/v0.32.21...v0.32.22) (2023-04-10) + + +### Bug Fixes + +* Correct the syntax error in the ClickHouse cast method ([#6154](https://github.com/cube-js/cube/issues/6154)) Thanks [@fxx-real](https://github.com/fxx-real)! ([b5ccf18](https://github.com/cube-js/cube/commit/b5ccf18cd56ee9cc9d8c1c9bb5ffd519c00d0f3c)) + + +### Features + +* **ducksdb-driver:** Initial support ([#6403](https://github.com/cube-js/cube/issues/6403)) ([00e41bf](https://github.com/cube-js/cube/commit/00e41bfe8084e9dc7831ff14e2302199af2a8fe3)) +* **prestodb-driver:** Add schema filter to informationSchemaQuery ([#6382](https://github.com/cube-js/cube/issues/6382)) Thanks [@maatumo](https://github.com/maatumo) ! ([cbf4620](https://github.com/cube-js/cube/commit/cbf4620bd8b0f5bfa0f02006dee007102258744a)) +* **schema-compiler:** Yaml - report error on misconfiguration (object instead of array) ([#6407](https://github.com/cube-js/cube/issues/6407)) ([29e3778](https://github.com/cube-js/cube/commit/29e37780c9c4f320c868880266dc73f55840987f)) + + + + + +## [0.32.21](https://github.com/cube-js/cube/compare/v0.32.20...v0.32.21) (2023-04-06) + + +### Bug Fixes + +* **cubestore:** Query Cache - correct size calculation (complex TableValue) ([#6400](https://github.com/cube-js/cube/issues/6400)) ([b0e3c88](https://github.com/cube-js/cube/commit/b0e3c88e33fdcbb9555faf791b01df4eed884686)) +* **databricks-jdbc:** startsWith, endsWith, contains filters ([#6401](https://github.com/cube-js/cube/issues/6401)) ([7eeea60](https://github.com/cube-js/cube/commit/7eeea60353f332bd83ae91c10efa47e2371beea3)) + + +### Features + +* **cubestore:** Introduce system.query_cache table ([#6399](https://github.com/cube-js/cube/issues/6399)) ([a5e4bd3](https://github.com/cube-js/cube/commit/a5e4bd393c05518e041ca52d386b1bace672be22)) + + + + + +## [0.32.20](https://github.com/cube-js/cube/compare/v0.32.19...v0.32.20) (2023-04-05) + + +### Features + +* **cubestore:** Query cache - limits by time_to_idle and max_capacity ([#6132](https://github.com/cube-js/cube/issues/6132)) ([edd220f](https://github.com/cube-js/cube/commit/edd220f0e76d43b05d482ff457c710d9ab4c2393)) +* **cubestore:** Separate configuration for count of metastore and cachestore snapshots ([#6388](https://github.com/cube-js/cube/issues/6388)) ([df73851](https://github.com/cube-js/cube/commit/df738513e06a3c4199255ae58ab9f52db1a4f2ef)) +* **questdb-driver:** Add SSL support ([#6395](https://github.com/cube-js/cube/issues/6395)) ([e9a4458](https://github.com/cube-js/cube/commit/e9a445807638f01c3762430339a84f5fe8518a50)) + + + + + +## [0.32.19](https://github.com/cube-js/cube/compare/v0.32.18...v0.32.19) (2023-04-03) + + +### Features + +* **cubesql:** Support `date_trunc = literal date` filter ([#6376](https://github.com/cube-js/cube/issues/6376)) ([0ef53cb](https://github.com/cube-js/cube/commit/0ef53cb978e8995185a731985944b08f1f24949e)) +* **cubestore:** Increase default TRANSPORT_MAX_FRAME_SIZE to 32Mb ([#6381](https://github.com/cube-js/cube/issues/6381)) ([73f7ea9](https://github.com/cube-js/cube/commit/73f7ea9a79882cdcdeff9a7bc916077f86489852)) +* **schema-compiler:** Support new aliases for join.relationship ([#6367](https://github.com/cube-js/cube/issues/6367)) ([34a18d8](https://github.com/cube-js/cube/commit/34a18d8936d6e3e27d0dcba2456dbac3f03b0d76)) +* Support quarter(s) in dateRange for timeDimensions ([#6385](https://github.com/cube-js/cube/issues/6385)) ([5a08721](https://github.com/cube-js/cube/commit/5a087216dfd95be29d414ec83f4494de8e3a1fb2)) + + + + + +## [0.32.18](https://github.com/cube-js/cube/compare/v0.32.17...v0.32.18) (2023-04-02) + + +### Bug Fixes + +* **athena-driver:** Correct handling for NULL values ([#6171](https://github.com/cube-js/cube/issues/6171)) ([a1f4cd4](https://github.com/cube-js/cube/commit/a1f4cd498e84dde91d736cc87825ae3a8095f9bf)) +* **playground:** SQL API default port 15432 ([416a9de](https://github.com/cube-js/cube/commit/416a9de677cb9a1a48f426a74e755b78c18e0952)) + + +### Features + +* Cube based includes and meta exposure ([#6380](https://github.com/cube-js/cube/issues/6380)) ([ead4ac0](https://github.com/cube-js/cube/commit/ead4ac0b3b78c074f29fcdec3f86282bce42c45a)) +* **cubestore:** Allow to configure max message_size/frame_size for transport ([#6369](https://github.com/cube-js/cube/issues/6369)) ([be7d40b](https://github.com/cube-js/cube/commit/be7d40bcf3be29b2fc0edfe2983162c4b0cafbce)) +* **schema-compiler:** Introduce public field (alias for shown) ([#6361](https://github.com/cube-js/cube/issues/6361)) ([6d0c66a](https://github.com/cube-js/cube/commit/6d0c66aad43d51dd23cc3b0cbd6ba3454581afd6)) + + + + + +## [0.32.17](https://github.com/cube-js/cube/compare/v0.32.16...v0.32.17) (2023-03-29) + + +### Bug Fixes + +* **deps:** Upgrade minimist, @xmldom/xmldom, refs [#6340](https://github.com/cube-js/cube/issues/6340) ([#6364](https://github.com/cube-js/cube/issues/6364)) ([04a1c9a](https://github.com/cube-js/cube/commit/04a1c9a04ca2eaae841024eb797e7ee549e98445)) +* **deps:** Upgrade moment, moment-timezone, refs [#6340](https://github.com/cube-js/cube/issues/6340) ([#6365](https://github.com/cube-js/cube/issues/6365)) ([d967cf3](https://github.com/cube-js/cube/commit/d967cf3fcf5d3f92ee32e47925fd1090f381a9c5)) + + +### Features + +* **schema-compiler:** Introduce support for sqlTable (syntactic sugar) ([#6360](https://github.com/cube-js/cube/issues/6360)) ([c73b368](https://github.com/cube-js/cube/commit/c73b368ca20735c63d01d2364b1c2f26d3b24cc5)) +* **sqlite-driver:** Upgrade sqlite3 from 5.0.11 to 5.1.6, refs [#6340](https://github.com/cube-js/cube/issues/6340) ([#6287](https://github.com/cube-js/cube/issues/6287)) ([4ec1da3](https://github.com/cube-js/cube/commit/4ec1da361afa9146b4ac955addd89120cfe23e2d)) + + +### Reverts + +* Revert "docs(schema): replace camel-case properties with snake-case" ([4731f4a](https://github.com/cube-js/cube/commit/4731f4ae16c1568f6a0956bc89f79f9a81f53e06)) +* Revert "docs: use snake-case property names" ([eb97883](https://github.com/cube-js/cube/commit/eb97883d75985bea7abce5bf3cc434473297cc96)) + + + + + +## [0.32.16](https://github.com/cube-js/cube/compare/v0.32.15...v0.32.16) (2023-03-27) + +**Note:** Version bump only for package cubejs + + + + + +## [0.32.15](https://github.com/cube-js/cube/compare/v0.32.14...v0.32.15) (2023-03-24) + + +### Bug Fixes + +* **cubesql:** Allow any aggregation for number measure as it can be a wildcard ([48f8828](https://github.com/cube-js/cube/commit/48f882867b020ca4a0d058ff147e61ef9bea9555)) +* **cubestore-driver:** Fix informationSchemaQuery ([#6338](https://github.com/cube-js/cube/issues/6338)) ([387020d](https://github.com/cube-js/cube/commit/387020d13ae3e82bac7b55699880e16f79940016)) +* **cubestore:** RocksStore - specify TTL for WAL to protect logs uploading ([#6345](https://github.com/cube-js/cube/issues/6345)) ([3a9cba4](https://github.com/cube-js/cube/commit/3a9cba478387dd4712b9ea3c566db159a32302d5)) + + +### Features + +* **cubestore:** Use XXH3 checksum alg for RocksDB ([#6339](https://github.com/cube-js/cube/issues/6339)) ([866f965](https://github.com/cube-js/cube/commit/866f965044536dac72c74160f9da6abc923e3dfb)) +* **playground:** BI connection guides, frontend integration snippets ([#6341](https://github.com/cube-js/cube/issues/6341)) ([8fdeb1b](https://github.com/cube-js/cube/commit/8fdeb1b882d959f9e8aec0ae963e5a0b1995e517)) + + + + + +## [0.32.14](https://github.com/cube-js/cube/compare/v0.32.13...v0.32.14) (2023-03-23) + + +### Bug Fixes + +* **cubestore:** Create table failed: Internal: Directory not empty (os error 39) for local dir storage ([103cabe](https://github.com/cube-js/cube/commit/103cabec6e4d6c59ad1890a3dbe42f7d092eeb6e)) +* Use last rollup table types to avoid type guessing for unionWithSourceData lambda queries ([#6337](https://github.com/cube-js/cube/issues/6337)) ([15badfe](https://github.com/cube-js/cube/commit/15badfeb2e1829e81dd914db3bbfd33f105e36ea)) + + + + + +## [0.32.13](https://github.com/cube-js/cube/compare/v0.32.12...v0.32.13) (2023-03-22) + + +### Bug Fixes + +* **cubestore-driver:** Queue - correct APM events ([#6251](https://github.com/cube-js/cube/issues/6251)) ([f4fdfc0](https://github.com/cube-js/cube/commit/f4fdfc03278e64c709e91e7df6cf1a9c936a041c)) +* **cubestore:** A single-character boolean value in csv is parsed incorrectly ([#6325](https://github.com/cube-js/cube/issues/6325)) ([ce8f3e2](https://github.com/cube-js/cube/commit/ce8f3e21ec675181ec47174428f14c8ddb7f3bf1)) + + + + + +## [0.32.12](https://github.com/cube-js/cube/compare/v0.32.11...v0.32.12) (2023-03-22) + + +### Bug Fixes + +* **cubesql:** Support quicksight AVG Rebase window exprs: Physical plan does not support logical expression SUM(x) PARTITION BY ([#6328](https://github.com/cube-js/cube/issues/6328)) ([5a5d7e4](https://github.com/cube-js/cube/commit/5a5d7e497f05c69541e04df0a464c85eb9a5f506)) +* **cubestore:** If job has been scheduled to non-existent node it'd hang around forever ([#6242](https://github.com/cube-js/cube/issues/6242)) ([69ef0b6](https://github.com/cube-js/cube/commit/69ef0b697bbe87a00687137bff318bb0cfc88015)) +* **cubestore:** Retying download from external locations ([#6321](https://github.com/cube-js/cube/issues/6321)) ([50841d9](https://github.com/cube-js/cube/commit/50841d92b7e84d3feec7372119eb540327a4d233)) + + +### Features + +* **cubestore-driver:** Support timestamp(3) as a column type ([#6324](https://github.com/cube-js/cube/issues/6324)) ([71dbca4](https://github.com/cube-js/cube/commit/71dbca4ed1fb706d36af322d2659435465804825)) +* **server-core, query-orchestrator:** sql-runner - supp… ([#6142](https://github.com/cube-js/cube/issues/6142)) ([32c603d](https://github.com/cube-js/cube/commit/32c603d50f462bf3cf9ce852dc9dcf23fc271cf3)) + + + + + +## [0.32.11](https://github.com/cube-js/cube.js/compare/v0.32.10...v0.32.11) (2023-03-21) + + +### Bug Fixes + +* **cubesql:** Ignore timestamps which can't be represented as nanoseconds instead of failing ([e393b06](https://github.com/cube-js/cube.js/commit/e393b0601fb663b03158ea03143a30eb0086ebbf)) +* **cubesql:** Quicksight AVG measures support ([#6323](https://github.com/cube-js/cube.js/issues/6323)) ([ada0afd](https://github.com/cube-js/cube.js/commit/ada0afd17b42a54fbecac69b849abf40158991c1)) + + + + + +## [0.32.10](https://github.com/cube-js/cube.js/compare/v0.32.9...v0.32.10) (2023-03-20) + + +### Bug Fixes + +* Refresh Scheduler - empty securityContext in driverFactory ([#6316](https://github.com/cube-js/cube.js/issues/6316)) ([dd4c9d5](https://github.com/cube-js/cube.js/commit/dd4c9d5a508de03b076850daed78afa18e99eb4f)) + + + + + +## [0.32.9](https://github.com/cube-js/cube.js/compare/v0.32.8...v0.32.9) (2023-03-18) + + +### Bug Fixes + +* **cubesql:** Unexpected response from Cube, Field "count" doesn't exist in row ([6bdc91d](https://github.com/cube-js/cube.js/commit/6bdc91d3eaf51dcb25e7321d03b485147511f049)) + + + + + +## [0.32.8](https://github.com/cube-js/cube.js/compare/v0.32.7...v0.32.8) (2023-03-17) + + +### Bug Fixes + +* **cubesql:** Catch error on TcpStream.peer_addr() ([#6300](https://github.com/cube-js/cube.js/issues/6300)) ([d74a1f0](https://github.com/cube-js/cube.js/commit/d74a1f059ce7f1baa9dad5f216e013a3f0f1bc45)) +* **cubesql:** Use writable streams with plain objects instead of JSON.stringify pipe for streaming capability ([#6306](https://github.com/cube-js/cube.js/issues/6306)) ([a9b19fa](https://github.com/cube-js/cube.js/commit/a9b19fa1a1a9c2f0710c8058ed797a4b7a48ed7e)) +* **cubestore:** Panic: index out of bounds: the len is 0 but the index is 0 ([#6311](https://github.com/cube-js/cube.js/issues/6311)) ([41dbbc0](https://github.com/cube-js/cube.js/commit/41dbbc03365f6067e85a09e8d5cafbfc6fe39990)) +* **query-orchestrator:** queues reconciliation logic ([#6286](https://github.com/cube-js/cube.js/issues/6286)) ([b45def4](https://github.com/cube-js/cube.js/commit/b45def46d99710e4fcf43cbc84b1b38341b859f6)) + + +### Features + +* **cubestore:** RocksStore - panic protection for broken rw loop ([#6293](https://github.com/cube-js/cube.js/issues/6293)) ([1f2db32](https://github.com/cube-js/cube.js/commit/1f2db32c289c119468f4cbe522fa40bab1a00068)) + + + + + +## [0.32.7](https://github.com/cube-js/cube.js/compare/v0.32.6...v0.32.7) (2023-03-14) + + +### Bug Fixes + +* Cross data source `rollupLambda` uses incorrect refreshKey ([#6288](https://github.com/cube-js/cube.js/issues/6288)) ([f87be98](https://github.com/cube-js/cube.js/commit/f87be98c68bcdcb65bd317b845571ad186586bd1)) + + + + + +## [0.32.6](https://github.com/cube-js/cube.js/compare/v0.32.5...v0.32.6) (2023-03-14) + + +### Bug Fixes + +* **firebolt-driver:** numeric null value ([#6284](https://github.com/cube-js/cube.js/issues/6284)) ([b5cc5b5](https://github.com/cube-js/cube.js/commit/b5cc5b56a16defe3b6412bad384ae5e07eeaeace)) + + +### Features + +* **schema-compiler:** quarter granularity ([#6285](https://github.com/cube-js/cube.js/issues/6285)) ([22f6102](https://github.com/cube-js/cube.js/commit/22f6102e137940b6882d55f560d826feceaf470d)) + + + + + +## [0.32.5](https://github.com/cube-js/cube.js/compare/v0.32.4...v0.32.5) (2023-03-13) + + +### Bug Fixes + +* Allow to specify cubestore as cacheAndQueueDriver from config, fix [#6279](https://github.com/cube-js/cube.js/issues/6279) ([#6280](https://github.com/cube-js/cube.js/issues/6280)) ([6b48c65](https://github.com/cube-js/cube.js/commit/6b48c65bf5dfe042b507171afb726328fa16e75c)) +* **api-gateway:** permissions cache removed ([#6276](https://github.com/cube-js/cube.js/issues/6276)) ([03be6d2](https://github.com/cube-js/cube.js/commit/03be6d29afbf13a053d5c2036ee4eae53a500892)) +* **cubestore:** Setup format_version = 5 (previusly used with RocksDB 6.20.3) ([#6283](https://github.com/cube-js/cube.js/issues/6283)) ([5126d00](https://github.com/cube-js/cube.js/commit/5126d00c64331c99d6aadc3ab7ccbd1c49dc1732)) +* **hive-driver:** Follow DriverInterface for query method, fix [#6281](https://github.com/cube-js/cube.js/issues/6281) ([#6282](https://github.com/cube-js/cube.js/issues/6282)) ([e81aba9](https://github.com/cube-js/cube.js/commit/e81aba9fd3dcf8b08092b9c7302067b230d1bbdc)) + + + + + +## [0.32.4](https://github.com/cube-js/cube.js/compare/v0.32.3...v0.32.4) (2023-03-10) + + +### Bug Fixes + +* **api-gateway:** contextToPermissions default permissions parameter ([#6268](https://github.com/cube-js/cube.js/issues/6268)) ([b234afc](https://github.com/cube-js/cube.js/commit/b234afc9877e4d80aa05528b32d9234b2730f0c5)) +* **api-gateway:** permissions cache removed ([#6271](https://github.com/cube-js/cube.js/issues/6271)) ([cd32266](https://github.com/cube-js/cube.js/commit/cd322665e5973dd8551bb039b3d0ad66a3a16d7f)) +* **api-gateway:** permissions error message ([#6269](https://github.com/cube-js/cube.js/issues/6269)) ([53d71a2](https://github.com/cube-js/cube.js/commit/53d71a2167a9d262520f4be3fbc1a458e8cc704c)) +* **api-gateway:** permissions unit test fix ([#6270](https://github.com/cube-js/cube.js/issues/6270)) ([15df7c1](https://github.com/cube-js/cube.js/commit/15df7c101dc664084f475764743694993a5cacdb)) +* Catch exception from extendContext (unhandledRejection) ([#6256](https://github.com/cube-js/cube.js/issues/6256)) ([9fa0bd9](https://github.com/cube-js/cube.js/commit/9fa0bd95c49bf4885224b43c42cad0d2327d026d)) +* **cubesql:** Failed printing to stdout: Resource temporarily unavailable ([#6272](https://github.com/cube-js/cube.js/issues/6272)) ([d8dd8a6](https://github.com/cube-js/cube.js/commit/d8dd8a60dba1eb3092dd252ccf98fa6e1de43dd4)) + + +### Features + +* **cubestore/docker:** Upgrade CLang to 15 ([979246b](https://github.com/cube-js/cube.js/commit/979246bdc8498d874aee755a4858c3008761ed54)) +* **cubestore/docker:** Upgrade OS to bullseye ([c512586](https://github.com/cube-js/cube.js/commit/c51258643bb24279ebe71e1279d0078efe32fce9)) +* **cubestore:** Support quarter intervals ([#6266](https://github.com/cube-js/cube.js/issues/6266)) ([35cc5c8](https://github.com/cube-js/cube.js/commit/35cc5c83e270e507e36e0602e707c8ed3f64872f)) +* **cubestore:** Upgrade RocksDB to 7.9.2 (0.20) ([#6221](https://github.com/cube-js/cube.js/issues/6221)) ([92bbef3](https://github.com/cube-js/cube.js/commit/92bbef34bd8fe34d9c8126bcf93c0535adcb289d)) + + + + + +## [0.32.3](https://github.com/cube-js/cube.js/compare/v0.32.2...v0.32.3) (2023-03-07) + + +### Features + +* **cubestore:** Streaming optimizations ([#6228](https://github.com/cube-js/cube.js/issues/6228)) ([114a847](https://github.com/cube-js/cube.js/commit/114a84778a84ff050f4d08eafdafa21efbf46646)) +* **server-core, api-gateway:** Permissions API ([#6240](https://github.com/cube-js/cube.js/issues/6240)) ([aad6aa3](https://github.com/cube-js/cube.js/commit/aad6aa3e327503487d8c9b4b85ec047d3fc0843e)) + + + + + +## [0.32.2](https://github.com/cube-js/cube.js/compare/v0.32.1...v0.32.2) (2023-03-07) + + +### Bug Fixes + +* **cubestore-driver:** Correct pending count and active keys in queue logs ([#6250](https://github.com/cube-js/cube.js/issues/6250)) ([3607c67](https://github.com/cube-js/cube.js/commit/3607c67ea0fa137c5f97e9d3b6fc67805ce85f68)) +* **redshift-driver:** fixes column order ([#6068](https://github.com/cube-js/cube.js/issues/6068)) Thanks [@rdwoodring](https://github.com/rdwoodring)! ([3bba803](https://github.com/cube-js/cube.js/commit/3bba803289c15be8cc6c9fe8476dc4ea8f46f40b)) +* Replace deprecated `@hapi/joi` with `joi` ([#6223](https://github.com/cube-js/cube.js/issues/6223)) Thanks [@hehex9](https://github.com/hehex9) ! ([ccbcc50](https://github.com/cube-js/cube.js/commit/ccbcc501dc91ef68ca49ddced79316425ae8f215)) + + +### Features + +* connection validation and logging ([#6233](https://github.com/cube-js/cube.js/issues/6233)) ([6dc48f8](https://github.com/cube-js/cube.js/commit/6dc48f8dc8045234dfa9fe8922534c5204e6e569)) +* **cubestore:** QUEUE - support extended flag for retrieve ([#6248](https://github.com/cube-js/cube.js/issues/6248)) ([9f23924](https://github.com/cube-js/cube.js/commit/9f23924a306c8141aa0cc8d990aeffdd5f3b4135)) +* Enable drop pre-aggregations without touch by default. **NOTE:** This change may adversely affect deployments which has incorrectly configured Refresh Worker instance. ([291977b](https://github.com/cube-js/cube.js/commit/291977b58ffb09e5886b6b1dbd46f9acd16a32ec)) +* Support rollupLambda across different cubes and data sources ([#6245](https://github.com/cube-js/cube.js/issues/6245)) ([fa284a4](https://github.com/cube-js/cube.js/commit/fa284a4e04ac0f80e9e06fb1dc48b5c07204c5a4)) + + + + + +## [0.32.1](https://github.com/cube-js/cube.js/compare/v0.32.0...v0.32.1) (2023-03-03) + + +### Bug Fixes + +* **cubesql:** Replace stream buffering with async implementation ([#6127](https://github.com/cube-js/cube.js/issues/6127)) ([5186d30](https://github.com/cube-js/cube.js/commit/5186d308cedf103b08c8a8140de84984839c710a)) + + + + + +# [0.32.0](https://github.com/cube-js/cube.js/compare/v0.31.69...v0.32.0) (2023-03-02) + + +### Features + +* **docker:** Remove gcc/g++/cmake/python from final images ([bb0a0e7](https://github.com/cube-js/cube.js/commit/bb0a0e7eac6044141e446f802d858529b4ff9782)) +* **docker:** Upgrade Node.js to 16.x ([8fe0e04](https://github.com/cube-js/cube.js/commit/8fe0e048e8d485b8775105be29d96f27cb4adf40)) + +### Breaking changes + +* Use cubestore driver for queue & cache ([a54aef](https://github.com/cube-js/cube.js/commit/5d88fed03a3e6599f9b77610f8657161f9a54aef)) +* Remove absolute import for @cubejs-backend/server-core ([361848](https://github.com/cube-js/cube.js/commit/6ee42184da3b36c793ec43284964cd885c361848)) +* Remove absolute import for @cubejs-backend/schema-compiler ([f3669](https://github.com/cube-js/cube.js/commit/d028937a92e6bf5d718ffd89b1676f90976f3669)) +* Remove absolute import for @cubejs-backend/query-orchestrator ([578c9](https://github.com/cube-js/cube.js/commit/45bd678a547ac71fc7d69471c4c72e49238578c9)) +* Remove support for Node.js 12, 15 ([78d76f1](https://github.com/cube-js/cube.js/commit/6568057fa6e6cc7e2fa02a7826d07fe3778d76f1)) + +## [0.31.69](https://github.com/cube-js/cube.js/compare/v0.31.68...v0.31.69) (2023-03-01) + + +### Bug Fixes + +* **cubestore:** Wrong triggering of table deactivation in some cases ([#6119](https://github.com/cube-js/cube.js/issues/6119)) ([523c0c2](https://github.com/cube-js/cube.js/commit/523c0c2cbcd981a114ab348f83e719526a6a3a0a)) +* Throw error message on misconfiguration with Cube Store as queue/cache driver. ([#6199](https://github.com/cube-js/cube.js/issues/6199)) ([ae06ef7](https://github.com/cube-js/cube.js/commit/ae06ef74c15fed996cc2d745d6ecdd5e03bb86f3)) + + + + + +## [0.31.68](https://github.com/cube-js/cube.js/compare/v0.31.67...v0.31.68) (2023-02-28) + + +### Bug Fixes + +* **cubestore:** Metastore logs are uploaded without the corresponding snapshot ([#6222](https://github.com/cube-js/cube.js/issues/6222)) ([cfa4b47](https://github.com/cube-js/cube.js/commit/cfa4b4780f0b1549544bf26e2fb843d69ccdc981)) +* **jdbc-driver, databricks-jdbc-driver:** clearTimeout on validate catch, temp table name ([#6220](https://github.com/cube-js/cube.js/issues/6220)) ([c2c7118](https://github.com/cube-js/cube.js/commit/c2c711815a32fa597fa70937a27a14cdc86cad1f)) + + + + + +## [0.31.67](https://github.com/cube-js/cube.js/compare/v0.31.66...v0.31.67) (2023-02-27) + + +### Bug Fixes + +* Reconcile streaming queries in case of cluster execution ([#6204](https://github.com/cube-js/cube.js/issues/6204)) ([dcf7866](https://github.com/cube-js/cube.js/commit/dcf78660bb07bb033665ac698e8b45af3565845b)) +* **schema-compiler:** skip empty YAML files ([b068f7f](https://github.com/cube-js/cube.js/commit/b068f7f4f44c8bede3d45aac8df4bc8b4f38912e)) + + +### Features + +* **cube-cli:** Schema validation command ([#6208](https://github.com/cube-js/cube.js/issues/6208)) ([1fc6490](https://github.com/cube-js/cube.js/commit/1fc64906e5e628437fb58d35feea8ab3aa5bfe06)) + + + + + +## [0.31.66](https://github.com/cube-js/cube.js/compare/v0.31.65...v0.31.66) (2023-02-27) + + +### Bug Fixes + +* databricks pre-aggregation errors ([#6207](https://github.com/cube-js/cube.js/issues/6207)) ([e7297f7](https://github.com/cube-js/cube.js/commit/e7297f735b64d3ce80ff4715c7b40ccee73cdb40)) + + +### Features + +* **cubestore:** Streaming ingestion optimizations ([#6198](https://github.com/cube-js/cube.js/issues/6198)) ([fb896ba](https://github.com/cube-js/cube.js/commit/fb896ba10bb2cbeacd14c7192a4a9b2d1e40abef)) +* **docker:** Use Debian (bullseye) for dev images ([#6202](https://github.com/cube-js/cube.js/issues/6202)) ([b4c922a](https://github.com/cube-js/cube.js/commit/b4c922a898e81df0bf3b4116c8ac51072ecc2382)) + + + + + +## [0.31.65](https://github.com/cube-js/cube.js/compare/v0.31.64...v0.31.65) (2023-02-23) + + +### Bug Fixes + +* **oracle-driver:** Release connection after query execution ([#5469](https://github.com/cube-js/cube.js/issues/5469)) ([ff1af78](https://github.com/cube-js/cube.js/commit/ff1af789c6326e25b3a341c4bae1829da95c36c1)) + + + + + +## [0.31.64](https://github.com/cube-js/cube.js/compare/v0.31.63...v0.31.64) (2023-02-21) + + +### Bug Fixes + +* **cubejs-playground:** error overflow ([#6170](https://github.com/cube-js/cube.js/issues/6170)) ([b5f68bb](https://github.com/cube-js/cube.js/commit/b5f68bbc8c0091089bb2c43bcbcf44a9da8a7182)) +* **jdbc-driver:** ResourceRequest timeout ([#6186](https://github.com/cube-js/cube.js/issues/6186)) ([b58c44e](https://github.com/cube-js/cube.js/commit/b58c44e116930f6dfaa4107e9c8ca14b472bfd77)) + + +### Features + +* **cubesql:** Remove unexpected clone (reduce memory consumption) ([#6185](https://github.com/cube-js/cube.js/issues/6185)) ([904556b](https://github.com/cube-js/cube.js/commit/904556b83e724e6b55e65afd9dbd077bb6c9ea99)) +* **cubestore:** Allow tinyint type (map as Int) ([#6174](https://github.com/cube-js/cube.js/issues/6174)) ([47571e5](https://github.com/cube-js/cube.js/commit/47571e566b7991f3bac902fcecce9c3ac3328a10)) + + + + + +## [0.31.63](https://github.com/cube-js/cube.js/compare/v0.31.62...v0.31.63) (2023-02-20) + + +### Bug Fixes + +* **cubesql:** `CAST(column AS DATE)` to DateTrunc day ([8f6fbe2](https://github.com/cube-js/cube.js/commit/8f6fbe274fa659870f9a736f4ac0c8e8406c64d0)) +* **cubestore:** Support realiasing for information schema tables ([#6155](https://github.com/cube-js/cube.js/issues/6155)) ([423a3ec](https://github.com/cube-js/cube.js/commit/423a3ec33953e816ca94fb33ed9f4ec2084e640c)) +* **databricks-jdbc-driver:** export table drops twice in Databricks ([#6177](https://github.com/cube-js/cube.js/issues/6177)) ([a62d9f0](https://github.com/cube-js/cube.js/commit/a62d9f0c23943224170fddb19f3a7c784920aa48)) +* **playground:** closing the last query tab ([aaa75f4](https://github.com/cube-js/cube.js/commit/aaa75f4d0d2cda73ca9070495ef788ef704e7e17)) + + +### Features + +* **cubestore:** Use separate scheduler for Cache Store ([#6160](https://github.com/cube-js/cube.js/issues/6160)) ([a17f59b](https://github.com/cube-js/cube.js/commit/a17f59b673f6a6336d52c4eafffbc237c1ca39e7)) +* graphql api variables ([#6153](https://github.com/cube-js/cube.js/issues/6153)) ([5f0f705](https://github.com/cube-js/cube.js/commit/5f0f7053022f437e61d23739b9acfb364fb06a16)) + + + + + +## [0.31.62](https://github.com/cube-js/cube.js/compare/v0.31.61...v0.31.62) (2023-02-13) + + +### Bug Fixes + +* streaming ([#6156](https://github.com/cube-js/cube.js/issues/6156)) ([abcbc1e](https://github.com/cube-js/cube.js/commit/abcbc1ed8f496ffa322053cc2dac9a7cc5b38dcd)) + + +### Features + +* **cubestore:** Introduce information_schema.columns ([#6152](https://github.com/cube-js/cube.js/issues/6152)) ([c70b155](https://github.com/cube-js/cube.js/commit/c70b155780f3eba491f093cfbb56bd1c297eae34)) + + + + + +## [0.31.61](https://github.com/cube-js/cube.js/compare/v0.31.60...v0.31.61) (2023-02-10) + +**Note:** Version bump only for package cubejs + + + + + +## [0.31.60](https://github.com/cube-js/cube.js/compare/v0.31.59...v0.31.60) (2023-02-10) + + +### Bug Fixes + +* GraphQL API date range filter ([#6138](https://github.com/cube-js/cube.js/issues/6138)) ([dc2ea8c](https://github.com/cube-js/cube.js/commit/dc2ea8cb430a6e013cfa32f42294ee6a7c4206b6)) +* **shared:** getProxySettings - use global configuration ([#6137](https://github.com/cube-js/cube.js/issues/6137)) ([e2ba5c1](https://github.com/cube-js/cube.js/commit/e2ba5c1962f2c57e79ce405c22209ab6890f400d)) + + +### Features + +* **cubesql:** Redesign member pushdown to support more advanced join… ([#6122](https://github.com/cube-js/cube.js/issues/6122)) ([3bb85e4](https://github.com/cube-js/cube.js/commit/3bb85e492056d73c28b3d006a95e0f9765e6e026)) +* **cubestore:** Disable MetastoreEvents for queue & cache ([#6126](https://github.com/cube-js/cube.js/issues/6126)) ([0fd20cb](https://github.com/cube-js/cube.js/commit/0fd20cb409703945fecc3620833b8d5cc683ca0e)) +* **cubestore:** Limit pushdown ([#5604](https://github.com/cube-js/cube.js/issues/5604)) ([f593ffe](https://github.com/cube-js/cube.js/commit/f593ffe686b9dfd40af9719226b583fc43f43648)) +* **cubestore:** Support SYS DROP QUERY CACHE + metrics ([#6131](https://github.com/cube-js/cube.js/issues/6131)) ([e8d2670](https://github.com/cube-js/cube.js/commit/e8d2670848656fa130bd76f2f98bca6b282143d3)) + + + + + +## [0.31.59](https://github.com/cube-js/cube.js/compare/v0.31.58...v0.31.59) (2023-02-06) + + +### Bug Fixes + +* **server-core:** config validation if custom ContextAcceptor is ([#6120](https://github.com/cube-js/cube.js/issues/6120)) ([6524a84](https://github.com/cube-js/cube.js/commit/6524a848e3459cf7de59fecd2d38f7f88a5fdc6f)) + + +### Features + +* **api-gateway, server-core:** added endpoint to fetch db schema ([#5852](https://github.com/cube-js/cube.js/issues/5852)) ([8ffcfd2](https://github.com/cube-js/cube.js/commit/8ffcfd22f75b745c90023c54bad19f6c5c9e87e5)) + + + + + +## [0.31.58](https://github.com/cube-js/cube.js/compare/v0.31.57...v0.31.58) (2023-02-02) + + +### Bug Fixes + +* **jdbc-driver:** streamQuery connection release ([#6108](https://github.com/cube-js/cube.js/issues/6108)) ([3426e6e](https://github.com/cube-js/cube.js/commit/3426e6e46a12aba2e85671cf22dcf4dd005d785c)) + + +### Features + +* **cubesql:** Improve catching of panic's reason ([#6107](https://github.com/cube-js/cube.js/issues/6107)) ([c8cf300](https://github.com/cube-js/cube.js/commit/c8cf3007b5bcb4f0362e5e3721eccadf69bcea62)) +* **cubestore:** Queue - support custom orphaned timeout ([#6090](https://github.com/cube-js/cube.js/issues/6090)) ([d6702ab](https://github.com/cube-js/cube.js/commit/d6702ab1d7c38111799edf9227e3aa53c51cc237)) +* **native:** Correct error handling for neon::channel.send ([#6106](https://github.com/cube-js/cube.js/issues/6106)) ([f71255a](https://github.com/cube-js/cube.js/commit/f71255abdae1d933101e0bc4002fd83373278067)) +* **snowflake-sdk:** Security upgrade snowflake-sdk from 1.6.14 to 1.6.18 ([#6097](https://github.com/cube-js/cube.js/issues/6097)) ([1f10c8a](https://github.com/cube-js/cube.js/commit/1f10c8a7cf088328e4e9a30d74e34338709af9b1)) + + + + + +## [0.31.57](https://github.com/cube-js/cube.js/compare/v0.31.56...v0.31.57) (2023-02-02) + + +### Features + +* **cubejs-api-gateway:** Add scope check to sql runner ([#6053](https://github.com/cube-js/cube.js/issues/6053)) ([d79d3aa](https://github.com/cube-js/cube.js/commit/d79d3aae32ce185614a65912f1c8229128792c48)) +* **cubejs-server-core:** add dbType to dataSources ([#6089](https://github.com/cube-js/cube.js/issues/6089)) ([5c84467](https://github.com/cube-js/cube.js/commit/5c844677d841a65cb7eb3eadd36e69ed59881fd6)) +* **cubestore:** Cache/Queue - implement migrations (truncating) ([#6083](https://github.com/cube-js/cube.js/issues/6083)) ([e8daf5e](https://github.com/cube-js/cube.js/commit/e8daf5e70e947c04675e2acc6654deb9484e2497)) +* **cubestore:** Queue - correct sort over priority (+created) ([#6094](https://github.com/cube-js/cube.js/issues/6094)) ([bd489cd](https://github.com/cube-js/cube.js/commit/bd489cd7981dd94766f28ae4621fe993252d63c1)) + + + + + +## [0.31.56](https://github.com/cube-js/cube.js/compare/v0.31.55...v0.31.56) (2023-01-31) + + +### Bug Fixes + +* (dimensions.type.ownedByCube = true) is not allowed if dynamic schemas outside of schema folder are used ([88b09bd](https://github.com/cube-js/cube.js/commit/88b09bd14e13a05599c9f433063832a17664ed92)) +* **cubesql:** Allow Thoughtspot `EXTRACT YEAR AS date` ([22d0ad9](https://github.com/cube-js/cube.js/commit/22d0ad967380b4ece695b567e77a216a16b3bf17)) +* **cubestore-driver:** Correct cancellation handling ([#6087](https://github.com/cube-js/cube.js/issues/6087)) ([03089ce](https://github.com/cube-js/cube.js/commit/03089cebb5a932eb69abb637ef8b7998beac115c)) +* **playground:** filter nullish refs ([9732af6](https://github.com/cube-js/cube.js/commit/9732af6bfe2b3dad78c0b9e669ba1c6b8b4815dd)) + + +### Features + +* **cubestore:** Max disk space limit ([#6084](https://github.com/cube-js/cube.js/issues/6084)) ([cc6003b](https://github.com/cube-js/cube.js/commit/cc6003b6a6df9744a6cf4aab833230fdca0a4d11)) +* **cubestore:** Max disk space limit per worker ([#6085](https://github.com/cube-js/cube.js/issues/6085)) ([ed2ca79](https://github.com/cube-js/cube.js/commit/ed2ca798b38d5496d4a92a4fe0227e6a25361037)) + + + + + +## [0.31.55](https://github.com/cube-js/cube.js/compare/v0.31.54...v0.31.55) (2023-01-26) + + +### Bug Fixes + +* **cubesql:** Correct Thoughtspot day in quarter offset ([d62079e](https://github.com/cube-js/cube.js/commit/d62079eaadaaa81d9b1e45580b27d7597192263e)) + + +### Features + +* **cubestore-driver:** Queue - support persistent flag/stream handling ([#6046](https://github.com/cube-js/cube.js/issues/6046)) ([5b12ec8](https://github.com/cube-js/cube.js/commit/5b12ec874f33d30a02df55f792f1fc1ce0a30bf4)) +* new logo with background ([#6062](https://github.com/cube-js/cube.js/issues/6062)) ([7c3601b](https://github.com/cube-js/cube.js/commit/7c3601b259a05fe3c8318e2aa62f3553f45de5a3)) + + + + + +## [0.31.54](https://github.com/cube-js/cube.js/compare/v0.31.53...v0.31.54) (2023-01-25) + + +### Bug Fixes + +* **cubestore:** Support macOS 11 for standalone binary ([#6060](https://github.com/cube-js/cube.js/issues/6060)) ([6d2dff7](https://github.com/cube-js/cube.js/commit/6d2dff76021b2ee86ab6856f1f5c7864a9da658f)) + + +### Features + +* **cubestore:** Filter for data from kafka streams ([#6054](https://github.com/cube-js/cube.js/issues/6054)) ([3b1b1ab](https://github.com/cube-js/cube.js/commit/3b1b1ab052c96d1de635da0eb10fdea5701c0442)) +* **cubestore:** Queue - improve orphaned detection + optimizations ([#6051](https://github.com/cube-js/cube.js/issues/6051)) ([70e68ae](https://github.com/cube-js/cube.js/commit/70e68ae35ed7bc66b8306512ae68c7919e45e2cc)) + + + + + +## [0.31.53](https://github.com/cube-js/cube.js/compare/v0.31.52...v0.31.53) (2023-01-24) + + +### Bug Fixes + +* Last lambda partition is incorrectly filtered in case of middle partition is filtered out ([656edcf](https://github.com/cube-js/cube.js/commit/656edcf24f8a784cf3b5425e7d501260d48a9dc1)) + + +### Features + +* **cubestore:** Correct metrics for queue/cache (ignore long commands) ([#6047](https://github.com/cube-js/cube.js/issues/6047)) ([1d5a1f6](https://github.com/cube-js/cube.js/commit/1d5a1f679b3b1ed8a2b132acb63640405443dc43)) + + + + + +## [0.31.52](https://github.com/cube-js/cube.js/compare/v0.31.51...v0.31.52) (2023-01-23) + + +### Bug Fixes + +* **query-orchestrator:** streams cluster ([#6048](https://github.com/cube-js/cube.js/issues/6048)) ([c5b6702](https://github.com/cube-js/cube.js/commit/c5b6702f69788cbcbf959608e5e86abfa4bd5385)) + + +### Features + +* **api-gateway, server-core:** Added dataSources method ([#5789](https://github.com/cube-js/cube.js/issues/5789)) ([128d017](https://github.com/cube-js/cube.js/commit/128d017e2c8eb75d9439a3d09c6739bc0c552938)) +* **cubestore:** Support multiple parallel QUEUE RESULT_BLOCKING ([#6038](https://github.com/cube-js/cube.js/issues/6038)) ([d8be78a](https://github.com/cube-js/cube.js/commit/d8be78a95f8fae466615063433a5ae53ab1c1bd6)) +* **cubestore:** Upgrade warp to 0.3.3 (use crates.io instead of git) ([#6043](https://github.com/cube-js/cube.js/issues/6043)) ([d2307a8](https://github.com/cube-js/cube.js/commit/d2307a8ce3449d8085810463e97d33de4b340db4)) +* new dark scheme at README ([#6044](https://github.com/cube-js/cube.js/issues/6044)) ([a44ebbb](https://github.com/cube-js/cube.js/commit/a44ebbbe8c0cbd2b94db6dfd5f7b9f30fa18bd34)) + + + + + +## [0.31.51](https://github.com/cube-js/cube.js/compare/v0.31.50...v0.31.51) (2023-01-21) + + +### Bug Fixes + +* Incorrect partition filter for real time partitions ([6520279](https://github.com/cube-js/cube.js/commit/6520279096048f3cd19b38e5c431dd5a5c29323a)) + + + + + +## [0.31.50](https://github.com/cube-js/cube.js/compare/v0.31.49...v0.31.50) (2023-01-21) + + +### Bug Fixes + +* Lambda pre-aggregations aren't served by external refresh instance ([bd45720](https://github.com/cube-js/cube.js/commit/bd4572083a9560cddb7ef23e2eccc9c9c806378a)) +* Real time partitions seal just after created ([882a7df](https://github.com/cube-js/cube.js/commit/882a7df2d1184e754c41d7f5daea9f43f3ac53f8)) + + + + + +## [0.31.49](https://github.com/cube-js/cube.js/compare/v0.31.48...v0.31.49) (2023-01-20) + + +### Bug Fixes + +* Content version gets updated on build range change ([0adc634](https://github.com/cube-js/cube.js/commit/0adc63493d51c74a68004c09fb239f1234dffe4a)) +* **cubestore-driver:** Unexpected token u in JSON at position 0 ([#6037](https://github.com/cube-js/cube.js/issues/6037)) ([1d00521](https://github.com/cube-js/cube.js/commit/1d005214c2b18f092ede5c9187798af151793cf8)) + + + + + +## [0.31.48](https://github.com/cube-js/cube.js/compare/v0.31.47...v0.31.48) (2023-01-20) + + +### Features + +* **cubesql:** Postgres protocol - stream support ([#6025](https://github.com/cube-js/cube.js/issues/6025)) ([d5786df](https://github.com/cube-js/cube.js/commit/d5786df63a1f48dec2697a8bb5e8c017c1b13ae4)) +* **cubesql:** Streams - cancel query and drop conection handling ([8c585f2](https://github.com/cube-js/cube.js/commit/8c585f24003c768300a31e0ed6774a3a724e54fa)) +* **cubestore:** Collect metrics for queue/cache commands ([#6032](https://github.com/cube-js/cube.js/issues/6032)) ([9ac7d0c](https://github.com/cube-js/cube.js/commit/9ac7d0ce67571c6945b7afe4095c995a1190da19)) +* **cubestore:** Support negative priority for queue ([#6031](https://github.com/cube-js/cube.js/issues/6031)) ([9924a1a](https://github.com/cube-js/cube.js/commit/9924a1a900f83fe246ede0b3a19e0c6ea7d5efc4)) +* streaming desync ([#6034](https://github.com/cube-js/cube.js/issues/6034)) ([a4c8b09](https://github.com/cube-js/cube.js/commit/a4c8b09a680d2857d28a42817d29fa567dcf63b2)) + + + + + +## [0.31.47](https://github.com/cube-js/cube.js/compare/v0.31.46...v0.31.47) (2023-01-18) + + +### Bug Fixes + +* Re-use external connection for CubeStore in queue ([#6028](https://github.com/cube-js/cube.js/issues/6028)) ([ff724f7](https://github.com/cube-js/cube.js/commit/ff724f734039d059b762c9200688c7b1afcb0f76)) + + +### Features + +* **cubestore:** Correct queue add handling ([#6022](https://github.com/cube-js/cube.js/issues/6022)) ([0f4a431](https://github.com/cube-js/cube.js/commit/0f4a431bc6c2b1324bbfd53dd1204874120a5082)) + + + + + +## [0.31.46](https://github.com/cube-js/cube.js/compare/v0.31.45...v0.31.46) (2023-01-18) + + +### Bug Fixes + +* **athena-driver:** Help user to understand CUBEJS_AWS_S3_OUTPUT_LOCATION parameter is missing ([#5991](https://github.com/cube-js/cube.js/issues/5991)) Thanks [@fbalicchia](https://github.com/fbalicchia) ! ([eedd12a](https://github.com/cube-js/cube.js/commit/eedd12a18645d2d6faa68a43963a6cb98560b5a6)) +* Do not fail if partitions are not ready for specified date range interval but there's at least one ready for pre-aggregation ([#6026](https://github.com/cube-js/cube.js/issues/6026)) ([2d39fe4](https://github.com/cube-js/cube.js/commit/2d39fe48b31b544f4aefa00c87e084102d134b2e)) + + +### Features + +* **query-orchestrator:** Introduce CubeStoreQueueDriver ([#6014](https://github.com/cube-js/cube.js/issues/6014)) ([f4744bf](https://github.com/cube-js/cube.js/commit/f4744bfb218a8a8cb28effe28237867157d01074)) + + + + + +## [0.31.45](https://github.com/cube-js/cube.js/compare/v0.31.44...v0.31.45) (2023-01-16) + + +### Bug Fixes + +* **cubestore:** Panic on combination of `Union All` and single select in root `Union All` ([#6012](https://github.com/cube-js/cube.js/issues/6012)) ([0d1a3d8](https://github.com/cube-js/cube.js/commit/0d1a3d8e2732d47f193aa5cf018e8ca5c6d7cf31)) +* Do not update structure version on build range end update ([#6015](https://github.com/cube-js/cube.js/issues/6015)) ([7891b6c](https://github.com/cube-js/cube.js/commit/7891b6c7d3033cfdf4705402ce54a156f24f29e3)) + + +### Features + +* **cubestore:** Initial queue support ([#5541](https://github.com/cube-js/cube.js/issues/5541)) ([7109039](https://github.com/cube-js/cube.js/commit/7109039beaf5819fa1179751e98a3431a04b3cac)) + + + + + +## [0.31.44](https://github.com/cube-js/cube.js/compare/v0.31.43...v0.31.44) (2023-01-16) + + +### Bug Fixes + +* Rollups are sorted by name in `rollups` params of pre-aggregations ([#6011](https://github.com/cube-js/cube.js/issues/6011)) ([235ec70](https://github.com/cube-js/cube.js/commit/235ec70c296846f02e337acc4be90b4863556d15)) + + + + + +## [0.31.43](https://github.com/cube-js/cube.js/compare/v0.31.42...v0.31.43) (2023-01-16) + + +### Bug Fixes + +* **cubestore:** Panic when nested Union All ([#6010](https://github.com/cube-js/cube.js/issues/6010)) ([331242e](https://github.com/cube-js/cube.js/commit/331242e263435513544ece0ee7b174d14562f05b)) +* fix a mistake in the websocket context acceptance mechanism ([#6006](https://github.com/cube-js/cube.js/issues/6006)) ([1d8a9bd](https://github.com/cube-js/cube.js/commit/1d8a9bd06804681ef74f6e4e70357cefb05f9c29)) + + +### Features + +* Pre-aggregations API for `rollupLambda` support ([#6009](https://github.com/cube-js/cube.js/issues/6009)) ([b90b3f2](https://github.com/cube-js/cube.js/commit/b90b3f2b5c991a0046220d7821ce74fc11ddbb85)) + + + + + +## [0.31.42](https://github.com/cube-js/cube.js/compare/v0.31.41...v0.31.42) (2023-01-15) + + +### Bug Fixes + +* **cubestore:** Large memory consumption while waiting for the queue cache ([#6004](https://github.com/cube-js/cube.js/issues/6004)) ([8c029bb](https://github.com/cube-js/cube.js/commit/8c029bb96cf4c451f061cef2d83b5d5dc3a8637c)) +* **cubestore:** Large memory consumption while waiting for the queue cache: drop context ([#6005](https://github.com/cube-js/cube.js/issues/6005)) ([b39920f](https://github.com/cube-js/cube.js/commit/b39920f6f5f2b10b2ea598712bd5f5d1b02a3308)) + + +### Features + +* **api-gateway:** added result filter for sql-runner ([#5985](https://github.com/cube-js/cube.js/issues/5985)) ([89f1b92](https://github.com/cube-js/cube.js/commit/89f1b922e1e3d8a02fbdaf2bda2c2cb77c0bdc78)) +* Multiple rollups in `rollupLambda` support ([#6008](https://github.com/cube-js/cube.js/issues/6008)) ([84fff0d](https://github.com/cube-js/cube.js/commit/84fff0d7745ccf657bb8eca0c5cc426c82ffd516)) +* **native:** Channel.resolve/reject without lock ([#6001](https://github.com/cube-js/cube.js/issues/6001)) ([9133a3b](https://github.com/cube-js/cube.js/commit/9133a3bf885ca17f14ac18041a1692201d4c1cfb)) + + + + + +## [0.31.41](https://github.com/cube-js/cube.js/compare/v0.31.40...v0.31.41) (2023-01-13) + + +### Bug Fixes + +* **cubestore:** Metastore get_tables_with_path cache is not work correctly ([#6000](https://github.com/cube-js/cube.js/issues/6000)) ([75699c8](https://github.com/cube-js/cube.js/commit/75699c85f3b428d3e1967642b2c440be77e7b095)) + + +### Features + +* streaming capabilities ([#5995](https://github.com/cube-js/cube.js/issues/5995)) ([d336c4e](https://github.com/cube-js/cube.js/commit/d336c4eaa3547422484bb003df19dfd4c7be5f96)) + + + + + +## [0.31.40](https://github.com/cube-js/cube.js/compare/v0.31.39...v0.31.40) (2023-01-12) + + +### Features + +* **cubesql:** Support `date_trunc` over column filter ([c9e71e6](https://github.com/cube-js/cube.js/commit/c9e71e6a13f41e2af388b7f91043e4118ba91f40)) +* **docker:** Alpine - migrate to 3.15 (3.14 was removed) ([#5997](https://github.com/cube-js/cube.js/issues/5997)) ([cf9078a](https://github.com/cube-js/cube.js/commit/cf9078a2c29d31d78b8c1e100ce254495128e1a0)) +* Hooks for the context rejection mechanism ([#5901](https://github.com/cube-js/cube.js/issues/5901)) ([b6f0506](https://github.com/cube-js/cube.js/commit/b6f0506e1b5821a16857d6695a40c6b957887941)) + + + + + +## [0.31.39](https://github.com/cube-js/cube.js/compare/v0.31.38...v0.31.39) (2023-01-12) + + +### Bug Fixes + +* **cubesql:** Query cancellation for simple query protocol ([#5987](https://github.com/cube-js/cube.js/issues/5987)) ([aae758f](https://github.com/cube-js/cube.js/commit/aae758f83d45a2572caddfc5f85663e059406c78)) + + +### Features + +* Re-use connection to Cube Store from externalDriver ([#5993](https://github.com/cube-js/cube.js/issues/5993)) ([69a35ed](https://github.com/cube-js/cube.js/commit/69a35edb74e9b5e09f74b13a91afe7076fa344cb)) + + + + + +## [0.31.38](https://github.com/cube-js/cube.js/compare/v0.31.37...v0.31.38) (2023-01-11) + + +### Bug Fixes + +* **athena-driver:** Add catalog config ([#5835](https://github.com/cube-js/cube.js/issues/5835)) Thanks [@tkislan](https://github.com/tkislan) ! ([c33a5c5](https://github.com/cube-js/cube.js/commit/c33a5c596622c5a2b67987da6cfd3f8bef6acebe)) +* **athena-driver:** TypeError: table.join is not a function ([#5988](https://github.com/cube-js/cube.js/issues/5988)) ([4e56a04](https://github.com/cube-js/cube.js/commit/4e56a0402dbb13d757e073fb38a547522c0936d1)), closes [#5143](https://github.com/cube-js/cube.js/issues/5143) +* Delete `CUBEJS_DROP_PRE_AGG_WITHOUT_TOUCH` pre-aggregations only after refresh end to avoid cold start removals ([#5982](https://github.com/cube-js/cube.js/issues/5982)) ([58ad02f](https://github.com/cube-js/cube.js/commit/58ad02f24c5cc5940d6c668a0586fd66ff843795)) + + +### Features + +* **cubesql:** Improve memory usage in writting for pg-wire ([#4870](https://github.com/cube-js/cube.js/issues/4870)) ([401fbcf](https://github.com/cube-js/cube.js/commit/401fbcfa1e11a36d65555f7848280f5e60801808)) +* **docker:** Upgrade Node.js to 14.21.1 ([#5970](https://github.com/cube-js/cube.js/issues/5970)) ([0394ed2](https://github.com/cube-js/cube.js/commit/0394ed2b38827e345a4a43ab35b129dfd845057b)) + + + + + +## [0.31.37](https://github.com/cube-js/cube.js/compare/v0.31.36...v0.31.37) (2023-01-09) + + +### Bug Fixes + +* Remove missed `cacheFn` console.log ([ec9e56b](https://github.com/cube-js/cube.js/commit/ec9e56b79e5acf0399c634e3e50b3457175404b6)) + + +### Features + +* **cubejs-api-gateway:** added endpoint to run sql query directly ([#5786](https://github.com/cube-js/cube.js/issues/5786)) ([61d5798](https://github.com/cube-js/cube.js/commit/61d579834800c839defd3c2991aae65a42318027)) +* **cubestore:** Support lazy initialization for CacheStore ([#5933](https://github.com/cube-js/cube.js/issues/5933)) ([37b4a95](https://github.com/cube-js/cube.js/commit/37b4a953e6c3bc535520203b23fef1632bf04aee)) + + + + + +## [0.31.36](https://github.com/cube-js/cube.js/compare/v0.31.35...v0.31.36) (2023-01-08) + + +### Bug Fixes + +* graphql non capital cube name issue ([#5680](https://github.com/cube-js/cube.js/issues/5680)) Thanks @MattFanto! ([47956ea](https://github.com/cube-js/cube.js/commit/47956ea695443547a13c4f05eed382a750324f11)), closes [#5643](https://github.com/cube-js/cube.js/issues/5643) +* Potential OOM due to recursion while recovering Cube Store websocket ([8a0fd1f](https://github.com/cube-js/cube.js/commit/8a0fd1fe1901b80b369bd9fe5feff0da4f054ed5)) + + +### Features + +* **cubestore:** Set metastore current snapshot system command ([#5940](https://github.com/cube-js/cube.js/issues/5940)) ([f590a20](https://github.com/cube-js/cube.js/commit/f590a20a5ff3d81e500f54f03eeb31698fbf8a42)) + + + + + +## [0.31.35](https://github.com/cube-js/cube.js/compare/v0.31.34...v0.31.35) (2023-01-07) + + +### Bug Fixes + +* **client-core:** Added type CubesMap for cubeMap in Meta ([#5897](https://github.com/cube-js/cube.js/issues/5897)) ([92d1ccb](https://github.com/cube-js/cube.js/commit/92d1ccb9166e3a608424a1e11457f1746119e5f2)) +* **cubestore:** Get query execution results even after web socket disconnect ([#5931](https://github.com/cube-js/cube.js/issues/5931)) ([c6ccc1a](https://github.com/cube-js/cube.js/commit/c6ccc1a1c37d3f0c2fb5b4cdeacafdcad39321b3)) +* **cubestore:** Maintain minimum count of metastore snapshots ([#5925](https://github.com/cube-js/cube.js/issues/5925)) ([b303aa6](https://github.com/cube-js/cube.js/commit/b303aa68b7040d22334673c436808564a505e097)) +* Reduce memory footprint for pre-aggregations with many partitions by caching partition SQL ([5f72d8f](https://github.com/cube-js/cube.js/commit/5f72d8f99b588e579527ed3c8f550bf7949fab4e)) + + +### Features + +* **cubejs-cli:** add type-checking for `cube.js` files to newly-generated projects ([ba31d4f](https://github.com/cube-js/cube.js/commit/ba31d4fac969faa2a0bd35c15863595c66f36ea0)) +* **cubesql:** Support `NULLIF` in projection ([129fc58](https://github.com/cube-js/cube.js/commit/129fc580579062be73d362cfa829e3af82f37ad0)) +* **cubesql:** Support Thoughtspot starts/ends LIKE exprs ([e6798cc](https://github.com/cube-js/cube.js/commit/e6798cca8f9de33badf34b9cd64c41a2a7e6ce88)) + + + + + +## [0.31.34](https://github.com/cube-js/cube.js/compare/v0.31.33...v0.31.34) (2023-01-05) + + +### Bug Fixes + +* **client-core:** move @babel/runtime to dependencies ([#5917](https://github.com/cube-js/cube.js/issues/5917)) ([67221af](https://github.com/cube-js/cube.js/commit/67221afa040bb71381369306b3cc9b5067094589)) +* Reuse queries cache in refresh worker pre-aggregation iterator to reduce memory usage in high concurrency environment ([4ae21fa](https://github.com/cube-js/cube.js/commit/4ae21fa54d08b25420694caec5ffd6658c8e96f6)) + + + + + +## [0.31.33](https://github.com/cube-js/cube.js/compare/v0.31.32...v0.31.33) (2023-01-03) + + +### Bug Fixes + +* **@cubejs-client/core:** add missing Series.shortTitle typing ([#5860](https://github.com/cube-js/cube.js/issues/5860)) ([5dd78b9](https://github.com/cube-js/cube.js/commit/5dd78b9544965304757c72d8f305741f78bfc935)) + + +### Features + +* **cubesql:** Support Node.js 18 ([#5900](https://github.com/cube-js/cube.js/issues/5900)) ([839fa75](https://github.com/cube-js/cube.js/commit/839fa752229d23c25f97df66b845fe6d8ebfd341)) +* Introduce CubeStoreCacheDriver ([#5511](https://github.com/cube-js/cube.js/issues/5511)) ([fb39776](https://github.com/cube-js/cube.js/commit/fb3977631adb126183d5d2775c70821c6c099897)) + + + + + +## [0.31.32](https://github.com/cube-js/cube.js/compare/v0.31.31...v0.31.32) (2022-12-28) + + +### Features + +* **cubesql:** Allow postprocessing with JOIN below Cube query limit ([56f5399](https://github.com/cube-js/cube.js/commit/56f5399fb37dbb7b388951b1db0be21dda2a94d4)) +* **cubesql:** Support `LEFT`, `RIGHT` in projection ([282ad3a](https://github.com/cube-js/cube.js/commit/282ad3ab3106d81a1361f94a499cfc7dc716f3e6)) +* **cubestore:** Direct kafka download support for ksql streams and t… ([#5880](https://github.com/cube-js/cube.js/issues/5880)) ([fcb5c5e](https://github.com/cube-js/cube.js/commit/fcb5c5e7825df32249a8021580b582821761c7b1)) +* **cubestore:** Installer - download binary without GH API ([#5885](https://github.com/cube-js/cube.js/issues/5885)) ([51a0f39](https://github.com/cube-js/cube.js/commit/51a0f3951caf62fdf8e88d80faf16d11a2b9158b)) + + + + + +## [0.31.31](https://github.com/cube-js/cube.js/compare/v0.31.30...v0.31.31) (2022-12-23) + + +### Bug Fixes + +* **cubesql:** Improve Thoughtspot `WHERE IN` support ([6212efe](https://github.com/cube-js/cube.js/commit/6212efe428e504cd8c06797dfaa2b81783b80777)) +* **cubesql:** Support Thoughtspot DATEADD queries ([58b5669](https://github.com/cube-js/cube.js/commit/58b566903685ac3d14e78d61dc38c38c43aa5c3c)) + + +### Features + +* **cubestore-driver:** Introduce `CUBEJS_CUBESTORE_NO_HEART_BEAT_TIMEOUT` env ([e67e800](https://github.com/cube-js/cube.js/commit/e67e800119e4efe0456eef347a573fa9feaf10e7)) +* **cubestore:** Initial support for KV cache ([#5861](https://github.com/cube-js/cube.js/issues/5861)) ([498d4b5](https://github.com/cube-js/cube.js/commit/498d4b5c5dae2ddf1ff1336566580cff1a656a92)) +* **playground:** allow for query renaming ([#5862](https://github.com/cube-js/cube.js/issues/5862)) ([681a2d8](https://github.com/cube-js/cube.js/commit/681a2d804880b17254e37a132008086c7c3fe81f)) + + + + + +## [0.31.30](https://github.com/cube-js/cube.js/compare/v0.31.29...v0.31.30) (2022-12-22) + + +### Bug Fixes + +* **cubesql:** Improve Thoughtspot compatibility ([4d6511a](https://github.com/cube-js/cube.js/commit/4d6511a3d18ea877f06775c1aae154b5665feda0)) +* **cubestore-driver:** Increase retry timeout to avoid endless loop connection retries ([8bd19fa](https://github.com/cube-js/cube.js/commit/8bd19fafb36da6d97000da20417e98f23d92a424)) +* Pre-aggregation table is not found after it was successfully created for `CUBEJS_DROP_PRE_AGG_WITHOUT_TOUCH` strategy ([#5858](https://github.com/cube-js/cube.js/issues/5858)) ([f602fee](https://github.com/cube-js/cube.js/commit/f602feea9c2a25a0de8604e1297484f71ac8437a)) + + +### Features + +* **@cubejs-client/core:** expose shortTitle in seriesNames ([#5836](https://github.com/cube-js/cube.js/issues/5836)) Thanks [@euljr](https://github.com/euljr) ! ([1058d5a](https://github.com/cube-js/cube.js/commit/1058d5afd2784302b23215d73c679d35ceb785a8)) + + + + + +## [0.31.29](https://github.com/cube-js/cube.js/compare/v0.31.28...v0.31.29) (2022-12-18) + + +### Bug Fixes + +* Error: WebSocket is not open: readyState 2 (CLOSING) ([#5846](https://github.com/cube-js/cube.js/issues/5846)) ([a5be099](https://github.com/cube-js/cube.js/commit/a5be099f1d339ceb17c89439d6195c5718c726bb)) + + + + + +## [0.31.28](https://github.com/cube-js/cube.js/compare/v0.31.27...v0.31.28) (2022-12-16) + + +### Features + +* Support `string`, `time` and `boolean` measures ([#5842](https://github.com/cube-js/cube.js/issues/5842)) ([4543ede](https://github.com/cube-js/cube.js/commit/4543edefe5b2432c90bb8530bc6a3c24c5548de3)) + + + + + +## [0.31.27](https://github.com/cube-js/cube.js/compare/v0.31.26...v0.31.27) (2022-12-16) + + +### Bug Fixes + +* **api-gateway:** pre-aggregations/jobs auth middleware ([#5840](https://github.com/cube-js/cube.js/issues/5840)) ([b527cd9](https://github.com/cube-js/cube.js/commit/b527cd9fab348b404b6ed83fbbdb963c23928b07)) + + + + + +## [0.31.26](https://github.com/cube-js/cube.js/compare/v0.31.25...v0.31.26) (2022-12-13) + + +### Bug Fixes + +* **cubestore:** Temporary disable compression to check if it fixes running on Mac OS X Ventura ([8e89427](https://github.com/cube-js/cube.js/commit/8e8942776f28a4a35a5dc3ad22360691da43682f)), closes [#5712](https://github.com/cube-js/cube.js/issues/5712) +* **trino-driver:** Timezone issue fix ([#5731](https://github.com/cube-js/cube.js/issues/5731)) Thanks [@yuraborue](https://github.com/yuraborue) ! ([70df903](https://github.com/cube-js/cube.js/commit/70df9034f4246be06956cbf1dc69de709d4d3df8)) + + +### Features + +* **cubestore:** Introduce pre-aggregation table touch and allow to drop tables without touch using `CUBEJS_DROP_PRE_AGG_WITHOUT_TOUCH` env ([#5794](https://github.com/cube-js/cube.js/issues/5794)) ([ad6c1e8](https://github.com/cube-js/cube.js/commit/ad6c1e8d09c228c28bb957755339a1146f54d6c9)) +* persistent queue ([#5793](https://github.com/cube-js/cube.js/issues/5793)) ([02a8e02](https://github.com/cube-js/cube.js/commit/02a8e027c1f414c50cb49f257ce68c01425400ec)) + + + + + +## [0.31.25](https://github.com/cube-js/cube.js/compare/v0.31.24...v0.31.25) (2022-12-10) + + +### Bug Fixes + +* **cubesql:** normalize column names in filter node ([#5788](https://github.com/cube-js/cube.js/issues/5788)) ([28aa008](https://github.com/cube-js/cube.js/commit/28aa008d8060173b2af2052577afdc26cc32c36d)) + + + + + +## [0.31.24](https://github.com/cube-js/cube.js/compare/v0.31.23...v0.31.24) (2022-12-09) + + +### Bug Fixes + +* **cubesql:** Support `CAST` in `HAVING` clause ([17ba3e2](https://github.com/cube-js/cube.js/commit/17ba3e212fb801fbffec99ef043b443f6f2a698f)) + + +### Features + +* **api-gateway, server-core:** Added dataSources list to extended meta ([#5743](https://github.com/cube-js/cube.js/issues/5743)) ([2c5db32](https://github.com/cube-js/cube.js/commit/2c5db32f2ded074ebe5e83668eee8c024101240b)) + + +### Reverts + +* Revert "feat(api-gateway, server-core): Added dataSources list to extended meta (#5743)" (#5785) ([3c61467](https://github.com/cube-js/cube.js/commit/3c614674fed6ca17df08bbba8c835ef110167570)), closes [#5743](https://github.com/cube-js/cube.js/issues/5743) [#5785](https://github.com/cube-js/cube.js/issues/5785) +* Revert "chore(cubejs-api-gateway): added endpoint to run sql query directly (#5723)" (#5784) ([f1140de](https://github.com/cube-js/cube.js/commit/f1140de508e359970ac82b50bae1c4bf152f6041)), closes [#5723](https://github.com/cube-js/cube.js/issues/5723) [#5784](https://github.com/cube-js/cube.js/issues/5784) + + + + + +## [0.31.23](https://github.com/cube-js/cube.js/compare/v0.31.22...v0.31.23) (2022-12-09) + + +### Bug Fixes + +* **jdbc-driver:** set default encoding ([#5773](https://github.com/cube-js/cube.js/issues/5773)) ([734167a](https://github.com/cube-js/cube.js/commit/734167a4fd555d15f722945fe5d2957663a144df)) + + +### Features + +* query limits ([#5763](https://github.com/cube-js/cube.js/issues/5763)) ([4ec172b](https://github.com/cube-js/cube.js/commit/4ec172b3dd27193d145afcdb8d9bf7ef1bd7505d)) +* **cubesql:** Support `CASE` statements in cube projection ([e7ae68c](https://github.com/cube-js/cube.js/commit/e7ae68c1afcb1152c0248f61ee355f0b30cc9b73)) +* introducing query persistent flag ([#5744](https://github.com/cube-js/cube.js/issues/5744)) ([699e772](https://github.com/cube-js/cube.js/commit/699e772be0e2e1b6eef4e59ff8e3857d1166bcef)) + + + + + +## [0.31.22](https://github.com/cube-js/cube.js/compare/v0.31.21...v0.31.22) (2022-12-07) + + +### Bug Fixes + +* **cubesql:** Metabase - auto-generated charts for cubes containing string dimensions ([#5728](https://github.com/cube-js/cube.js/issues/5728)) ([72be686](https://github.com/cube-js/cube.js/commit/72be68671faaa4c938374f95cb8cb81578ef4fdb)) +* **cubestore:** Added long running job fetches burst network ([385c4a6](https://github.com/cube-js/cube.js/commit/385c4a6f92e2da209804276d85f1dec1d404ed8e)) +* **cubestore:** Increase default stale stream timeout to allow replays to catch up with large kafka streams ([43cf0b1](https://github.com/cube-js/cube.js/commit/43cf0b1c700c5fc15ed907c694916569e5647bf3)) +* **cubestore:** Row with id 1 is not found for SchemaRocksTable -- fix log replay order during metastore loading from dump ([b74c072](https://github.com/cube-js/cube.js/commit/b74c0721fb8d62d9e52570a3d5b883604b00648c)) +* **playground:** reload the page in case chunks got stale ([#5646](https://github.com/cube-js/cube.js/issues/5646)) ([2215595](https://github.com/cube-js/cube.js/commit/2215595369ac0ceab89c95760ecea6e9fd9cc1af)) +* return failed query request ids ([#5729](https://github.com/cube-js/cube.js/issues/5729)) ([22cd580](https://github.com/cube-js/cube.js/commit/22cd580c412a64ce92dd1410f13af19709c27b9d)) + + +### Features + +* **playground:** yaml rollup support ([#5727](https://github.com/cube-js/cube.js/issues/5727)) ([7a267f9](https://github.com/cube-js/cube.js/commit/7a267f99def74fe56f1505c7d56d42ea0c24f842)) + + + + + +## [0.31.21](https://github.com/cube-js/cube.js/compare/v0.31.20...v0.31.21) (2022-12-06) + + +### Bug Fixes + +* **cubestore:** `Error in processing loop: Row with id is not found for JobRocksTable` for streaming jobs ([f2de503](https://github.com/cube-js/cube.js/commit/f2de503365c4dece0b01a2f8817b231ab0fced4e)) +* **cubestore:** Avoid streaming jobs waiting for regular jobs to complete ([#5725](https://github.com/cube-js/cube.js/issues/5725)) ([89d5bf4](https://github.com/cube-js/cube.js/commit/89d5bf4eda6d2b8ba41febe121322aba5605d107)) +* **cubestore:** Chunk all streaming input into bigger chunks based on time spent in write chunk operations and remove sequence gap checks as those fail for tables ([1f2d9bf](https://github.com/cube-js/cube.js/commit/1f2d9bf6f1df8e8d961300d4c632240bc5290eb3)) +* **cubestore:** Replay streams with `earliest` offset even for `latest` setting after failure ([7ca71f5](https://github.com/cube-js/cube.js/commit/7ca71f5969f9550151afcd646864a4d5eaeca3ab)) +* **cubestore:** ReplayHandle reconcile fails due to merge ReplayHandles with Chunks ([#5713](https://github.com/cube-js/cube.js/issues/5713)) ([ac213a7](https://github.com/cube-js/cube.js/commit/ac213a7bb648fbe4a6f645227640d7085a0b578e)) +* **cubestore:** Streaming jobs stuck as stale ([1cf3432](https://github.com/cube-js/cube.js/commit/1cf3432c23a20ec682a40fd0f86192b98b0a6fc0)) +* **databricks-jdbc:** Databricks pre-aggregations schema with UC ([#5726](https://github.com/cube-js/cube.js/issues/5726)) ([6281471](https://github.com/cube-js/cube.js/commit/6281471ad981372a938449114cd51e5cf1326884)) +* **playground:** clear search value ([#5720](https://github.com/cube-js/cube.js/issues/5720)) ([5b177f2](https://github.com/cube-js/cube.js/commit/5b177f2531789928a75f1e0c47c8f25bdd7c0251)) +* More explanatory `No pre-aggregation partitions were built yet` message ([#5702](https://github.com/cube-js/cube.js/issues/5702)) ([ec39baa](https://github.com/cube-js/cube.js/commit/ec39baa80b9980e4b72be31c53f6704884b8ac5c)) + + + + + +## [0.31.20](https://github.com/cube-js/cube.js/compare/v0.31.19...v0.31.20) (2022-12-02) + + +### Bug Fixes + +* **clickhouse-driver:** Make ClickHouse driver `readOnly` by default ([e6a85d1](https://github.com/cube-js/cube.js/commit/e6a85d1856ee0a7d6b2b4f3e59f6e8edaed609a2)), closes [#5479](https://github.com/cube-js/cube.js/issues/5479) +* **clickhouse-driver:** ParserError("Expected ',' or ')' after column definition, found: (") when Nullable(Float64) is present ([81b2247](https://github.com/cube-js/cube.js/commit/81b224747f1423f263f384546f182b3d47f22a3d)) +* **cubesql:** Fix escape symbols in `LIKE` expressions ([5f3cd50](https://github.com/cube-js/cube.js/commit/5f3cd50ea311900adc27ba2b30c72a05a3453a1d)) +* **cubestore:** Orphaned replay handles after table drop ([0e4b876](https://github.com/cube-js/cube.js/commit/0e4b876556d8d88e6fb3e4270d6a7852acb0fd00)) +* **cubestore:** Sort by seq column to reduce Unexpected sequence increase gap ([b5f06d0](https://github.com/cube-js/cube.js/commit/b5f06d0c10217f6e193e6729adba30e2d9af2b92)) +* **oracle-driver:** Make oracle driver `readOnly` by default so pre-aggregations can be used ([5efccba](https://github.com/cube-js/cube.js/commit/5efccbaa8e15d4c85f59f0465bca62a919ace78b)) + + +### Features + +* **cubesql:** Support Thoughtspot include filter search ([745fe5d](https://github.com/cube-js/cube.js/commit/745fe5d2806b4c6c9e76d6061aa038892ec7438f)) +* **cubesql:** Support ThoughtSpot search filters ([ee0fde4](https://github.com/cube-js/cube.js/commit/ee0fde4798894c619f63cfd87cfc118c7ff1fc78)) + + + + + +## [0.31.19](https://github.com/cube-js/cube.js/compare/v0.31.18...v0.31.19) (2022-11-29) + + +### Bug Fixes + +* Incorrect filter pushdown when filtering two left-joined tables ([#5685](https://github.com/cube-js/cube.js/issues/5685)) ([c775869](https://github.com/cube-js/cube.js/commit/c77586961078ec67395af39e3f233025833d4f6e)), closes [#3777](https://github.com/cube-js/cube.js/issues/3777) +* packages/cubejs-client-core/package.json to reduce vulnerabilities ([#5352](https://github.com/cube-js/cube.js/issues/5352)) ([76e8e43](https://github.com/cube-js/cube.js/commit/76e8e43c05d857ce4338d0d184a987e4edae735c)) +* packages/cubejs-client-vue/package.json to reduce vulnerabilities ([#5361](https://github.com/cube-js/cube.js/issues/5361)) ([effc694](https://github.com/cube-js/cube.js/commit/effc694b29da181899ff74f7f0107eeaa9aaec76)) +* packages/cubejs-query-orchestrator/package.json to reduce vulnerabilities ([#5347](https://github.com/cube-js/cube.js/issues/5347)) ([3442ddf](https://github.com/cube-js/cube.js/commit/3442ddf319caeae8c5506f4cec0fde1ebcf9d147)) +* packages/cubejs-schema-compiler/package.json to reduce vulnerabilities ([#5359](https://github.com/cube-js/cube.js/issues/5359)) ([cd81554](https://github.com/cube-js/cube.js/commit/cd81554079533eae2c35ce142b26ad862b6e250d)) +* packages/cubejs-schema-compiler/package.json to reduce vulnerabilities ([#5408](https://github.com/cube-js/cube.js/issues/5408)) ([6b7a95f](https://github.com/cube-js/cube.js/commit/6b7a95f96dacb42a0a50fe77afcc11216f10c29a)) +* packages/cubejs-templates/package.json to reduce vulnerabilities ([#5342](https://github.com/cube-js/cube.js/issues/5342)) ([825605a](https://github.com/cube-js/cube.js/commit/825605aa8d85dca29b6d3d7a230e361c1374746b)) + + +### Features + +* catalog support for the Databricks driver ([#5666](https://github.com/cube-js/cube.js/issues/5666)) ([de5ba9a](https://github.com/cube-js/cube.js/commit/de5ba9a247543b432ea82b4371ddb052f1c91227)) + + + + + +## [0.31.18](https://github.com/cube-js/cube.js/compare/v0.31.17...v0.31.18) (2022-11-28) + + +### Bug Fixes + +* **cubesql:** Prevent infinite limit push down ([f26d40a](https://github.com/cube-js/cube.js/commit/f26d40a3b91a2651811cfa622424c6ed2e44dfef)) +* **cubesql:** Push down projection to CubeScan with literals ([207616d](https://github.com/cube-js/cube.js/commit/207616d8e51649a78b854b02fc83611472aea715)) + + +### Features + +* **cubesql:** Sigma Computing date filters ([404e3f4](https://github.com/cube-js/cube.js/commit/404e3f42ac02be8c0c9eaacb6cf81bb516616001)) +* yaml schema generation ([#5676](https://github.com/cube-js/cube.js/issues/5676)) ([fc0b810](https://github.com/cube-js/cube.js/commit/fc0b81016e5c66156d06f6834187d3323e9e8ca8)) + + + + + +## [0.31.17](https://github.com/cube-js/cube.js/compare/v0.31.16...v0.31.17) (2022-11-23) + + +### Features + +* **cubestore:** Introduce CacheStore ([#5607](https://github.com/cube-js/cube.js/issues/5607)) ([36dee61](https://github.com/cube-js/cube.js/commit/36dee61de05b544edb2b413afa4b9582f81a5eec)) + + + + + +## [0.31.16](https://github.com/cube-js/cube.js/compare/v0.31.15...v0.31.16) (2022-11-23) + + +### Features + +* **ksql-driver:** Support offset earliest, replays and per partition streaming ([#5663](https://github.com/cube-js/cube.js/issues/5663)) ([3a79d02](https://github.com/cube-js/cube.js/commit/3a79d02c9794b5e437a52f9f0be72eb82a92805a)) + + + + + +## [0.31.15](https://github.com/cube-js/cube.js/compare/v0.31.14...v0.31.15) (2022-11-17) + + +### Bug Fixes + +* **@cubejs-backend/snowflake-driver:** Make `CUBEJS_DB_SNOWFLAKE_CLIENT_SESSION_KEEP_ALIVE=true` by default ([be12c40](https://github.com/cube-js/cube.js/commit/be12c40ca7acda11409774f5aa407741fdfde871)) +* `extends` YAML support ([982885e](https://github.com/cube-js/cube.js/commit/982885e8e41b351e27919551688f50f7e5af3a5a)) +* **client-react:** check meta changes ([4c44551](https://github.com/cube-js/cube.js/commit/4c44551b880bd4ff34d443241c1c0c28cae0d5f8)) +* **druid-driver:** Respect day light saving ([#5613](https://github.com/cube-js/cube.js/issues/5613)) ([388c992](https://github.com/cube-js/cube.js/commit/388c992ca7a2d3730249fafb2a53b3accff21451)) +* packages/cubejs-client-core/package.json to reduce vulnerabilities ([#5415](https://github.com/cube-js/cube.js/issues/5415)) ([fb2de68](https://github.com/cube-js/cube.js/commit/fb2de682670bd28b2879d0460fac990d9b653dce)) +* packages/cubejs-client-react/package.json to reduce vulnerabilities ([#5390](https://github.com/cube-js/cube.js/issues/5390)) ([0ab9c30](https://github.com/cube-js/cube.js/commit/0ab9c30692c70d3776a8429197915090fef61d4f)) +* packages/cubejs-databricks-jdbc-driver/package.json to reduce vulnerabilities ([#5413](https://github.com/cube-js/cube.js/issues/5413)) ([6a891f0](https://github.com/cube-js/cube.js/commit/6a891f0bc34dcaa8c955cd0ac20121d4d074e228)) +* packages/cubejs-databricks-jdbc-driver/package.json to reduce vulnerabilities ([#5429](https://github.com/cube-js/cube.js/issues/5429)) ([a45c9a8](https://github.com/cube-js/cube.js/commit/a45c9a828b38d13da9a4194fdfc23e11674aa7cd)) +* packages/cubejs-query-orchestrator/package.json to reduce vulnerabilities ([#5409](https://github.com/cube-js/cube.js/issues/5409)) ([5e9fe68](https://github.com/cube-js/cube.js/commit/5e9fe68b40164ca12e9cecb0aefda31c06b57f28)) +* packages/cubejs-templates/package.json to reduce vulnerabilities ([#5403](https://github.com/cube-js/cube.js/issues/5403)) ([c9706cb](https://github.com/cube-js/cube.js/commit/c9706cbfcd6480dbd58ca18ab064ded185f710d3)) +* **server-core:** Force flush events if their count is greater than the agent frame size ([#5602](https://github.com/cube-js/cube.js/issues/5602)) ([17d1d98](https://github.com/cube-js/cube.js/commit/17d1d989bd03459adae1e6c6714843fc82d99163)) + + +### Features + +* **databricks-jdbc:** jdbc (jar) driver update ([#5610](https://github.com/cube-js/cube.js/issues/5610)) ([aacd8cd](https://github.com/cube-js/cube.js/commit/aacd8cd356429e4da21749b92eb457c03a1a3f76)) +* **databricks-jdbc:** jdbc (jar) driver update ([#5612](https://github.com/cube-js/cube.js/issues/5612)) ([372ed71](https://github.com/cube-js/cube.js/commit/372ed71c6edd61d862d62cb0522fbc47c0f997b2)) +* **docs:** add new component to display code snippets side-by-side ([146328c](https://github.com/cube-js/cube.js/commit/146328c7e7eab41c7cfe4e67d26af58eeca9c09c)) +* Replace YAML parser to provide more meaningful parse errors ([9984066](https://github.com/cube-js/cube.js/commit/99840665ee31aa8f14cf9c83b19d4a4cbc3a978a)) +* Support snake case in YAML relationship field ([f20fe6b](https://github.com/cube-js/cube.js/commit/f20fe6baa43142589a5a85be5af1daefb56acd8b)) + + +### Reverts + +* Revert "feat(databricks-jdbc): jdbc (jar) driver update (#5610)" (#5611) ([23ed416](https://github.com/cube-js/cube.js/commit/23ed416d9540afc4fe027f7e3c6917af387f06f7)), closes [#5610](https://github.com/cube-js/cube.js/issues/5610) [#5611](https://github.com/cube-js/cube.js/issues/5611) + + + + + +## [0.31.14](https://github.com/cube-js/cube.js/compare/v0.31.13...v0.31.14) (2022-11-14) + + +### Bug Fixes + +* **cubesql:** Allow referencing CTEs in UNION ([7f5bc83](https://github.com/cube-js/cube.js/commit/7f5bc8316d6119ffe1703bc1bb41e54586f9d19b)) +* **cubesql:** Keep CubeScan literal values relation ([6d3856a](https://github.com/cube-js/cube.js/commit/6d3856acdfeea8d93866a1f36513016a5a04b2e8)) +* **playground:** schemaVersion updates ([5c4880f](https://github.com/cube-js/cube.js/commit/5c4880f1fa30189798c0b0ea42df67c7fd0aea75)) + + +### Features + +* **@cubejs-backend/mssql-driver:** Make MSSQL `readOnly` by default ([#5584](https://github.com/cube-js/cube.js/issues/5584)) ([ddf0369](https://github.com/cube-js/cube.js/commit/ddf036992aebc61fdd99d2a67753c63528bba9db)) +* **cubesql:** Join Cubes ([#5585](https://github.com/cube-js/cube.js/issues/5585)) ([c687e42](https://github.com/cube-js/cube.js/commit/c687e42f9280f611152f7c154fdf136e6d9ce402)) +* **playground:** ability to rerun queries ([#5597](https://github.com/cube-js/cube.js/issues/5597)) ([6ef8ce9](https://github.com/cube-js/cube.js/commit/6ef8ce91740086dc0b10ea11d7133f8be1e2ef0a)) + + + + + +## [0.31.13](https://github.com/cube-js/cube.js/compare/v0.31.12...v0.31.13) (2022-11-08) + + +### Bug Fixes + +* Make Trino driver CommonJS compatible ([#5581](https://github.com/cube-js/cube.js/issues/5581)) ([ca8fd4e](https://github.com/cube-js/cube.js/commit/ca8fd4e42f4d8f87667507920f02cbb1a7072763)) +* **cubestore:** Merge for streaming union ([#5554](https://github.com/cube-js/cube.js/issues/5554)) ([310649a](https://github.com/cube-js/cube.js/commit/310649af9a280484481dc09063178c07f3f36131)) + + +### Features + +* export bucket CVS files escape symbol support ([#5570](https://github.com/cube-js/cube.js/issues/5570)) ([09ceffb](https://github.com/cube-js/cube.js/commit/09ceffbefc75417555f8ff90f6277bd9c419d751)) +* notStartsWith/notEndsWith filters support ([#5579](https://github.com/cube-js/cube.js/issues/5579)) ([8765833](https://github.com/cube-js/cube.js/commit/87658333df0194db07c3ce0ae6f94a292f8bd592)) +* YAML snake case and `.yaml` extension support ([#5578](https://github.com/cube-js/cube.js/issues/5578)) ([c8af286](https://github.com/cube-js/cube.js/commit/c8af2864af2dcc532abdc3629cf58893d874c190)) + + + + + +## [0.31.12](https://github.com/cube-js/cube.js/compare/v0.31.11...v0.31.12) (2022-11-05) + + +### Bug Fixes + +* Cannot read property 'apply' of undefined on missed dimension sql ([#5559](https://github.com/cube-js/cube.js/issues/5559)) ([6f85096](https://github.com/cube-js/cube.js/commit/6f850967ba834bbdd87284fa957859d66d19344a)) +* YAML filter support ([fb6fade](https://github.com/cube-js/cube.js/commit/fb6fade4bfa5b0fb2e3b6895e378aef97ea26f95)) +* **cubestore:** Fix partition pruning ([#5548](https://github.com/cube-js/cube.js/issues/5548)) ([8bc4aee](https://github.com/cube-js/cube.js/commit/8bc4aeeff5502d2e17763bf503ebe396227b48ae)) +* No column found in case non equals filter query incorrectly matched against rollup with no dimensions ([#5552](https://github.com/cube-js/cube.js/issues/5552)) ([73b3203](https://github.com/cube-js/cube.js/commit/73b3203925bf9d8221001f730bb23272dd4e47e6)) +* TypeError: Cannot read property 'dimension' of undefined for rolling window rollup without time dimension ([#5553](https://github.com/cube-js/cube.js/issues/5553)) ([03c3b6f](https://github.com/cube-js/cube.js/commit/03c3b6f4197ff8e6a77fc6bb7c08e4730cbfde66)) + + +### Features + +* Trino driver ([e58c392](https://github.com/cube-js/cube.js/commit/e58c3924781b65f5631ee241b39a0bee1366273d)) +* **cubesql:** Support Skyvia date granularities ([df69d93](https://github.com/cube-js/cube.js/commit/df69d93e3f0c016d4767e0509ca523b60bc74099)) + + + + + +## [0.31.11](https://github.com/cube-js/cube.js/compare/v0.31.10...v0.31.11) (2022-11-02) + + +### Bug Fixes + +* **@cubejs-backend/prestodb-driver:** Replace double escaping in contain filter ([#5529](https://github.com/cube-js/cube.js/issues/5529)) ([7870705](https://github.com/cube-js/cube.js/commit/7870705b04697faf8cb994c0794bc86437a9e3cf)), closes [#5528](https://github.com/cube-js/cube.js/issues/5528) +* **cubestore:** Fix streaming index ([#5550](https://github.com/cube-js/cube.js/issues/5550)) ([061305e](https://github.com/cube-js/cube.js/commit/061305e7a7549ae2d79e5391afd79f3d2af7d628)) + + +### Features + +* **cubestore:** Sealing partition ([#5523](https://github.com/cube-js/cube.js/issues/5523)) ([70ee72c](https://github.com/cube-js/cube.js/commit/70ee72cca5b9a77bf14994f05b0e77148089362c)) + + + + + +## [0.31.10](https://github.com/cube-js/cube.js/compare/v0.31.9...v0.31.10) (2022-11-01) + + +### Bug Fixes + +* Revert back strict shown checks behavior for consistency reasons ([#5551](https://github.com/cube-js/cube.js/issues/5551)) ([c3ee4e6](https://github.com/cube-js/cube.js/commit/c3ee4e6911d07f7d5cb8977f563555157f7b5f2b)), closes [#5542](https://github.com/cube-js/cube.js/issues/5542) + + + + + +## [0.31.9](https://github.com/cube-js/cube.js/compare/v0.31.8...v0.31.9) (2022-11-01) + + +### Bug Fixes + +* **@cubejs-client/core:** `startsWith` and `endsWith` to filterOperatorsForMember ([#5544](https://github.com/cube-js/cube.js/issues/5544)) ([583de4a](https://github.com/cube-js/cube.js/commit/583de4a58c841542f3138c5ce836dbfedd19d4de)) +* You requested hidden member in case of rolling window measure is being used in a view ([#5546](https://github.com/cube-js/cube.js/issues/5546)) ([c00ea43](https://github.com/cube-js/cube.js/commit/c00ea432021a02638361b278f8eb24801a7a867b)) + + + + + +## [0.31.8](https://github.com/cube-js/cube.js/compare/v0.31.7...v0.31.8) (2022-10-30) + + +### Bug Fixes + +* **cubesql:** Count measure type changed from u64 to i64 ([#5535](https://github.com/cube-js/cube.js/issues/5535)) ([f568851](https://github.com/cube-js/cube.js/commit/f568851948cff16ddc53a974d46a77a8698dbdf1)) + + +### Features + +* YAML support ([#5539](https://github.com/cube-js/cube.js/issues/5539)) ([29c19db](https://github.com/cube-js/cube.js/commit/29c19db9315ef7ad40350150f74f9519d6ff4a98)) +* **cubesql:** Support `BOOL_AND`, `BOOL_OR` aggregate functions ([#5533](https://github.com/cube-js/cube.js/issues/5533)) ([a2e6e38](https://github.com/cube-js/cube.js/commit/a2e6e386557bfbf43b2a4907c1fa3aef07ea90f2)) +* **cubesql:** Support Sigma Computing number filters ([f2f2abd](https://github.com/cube-js/cube.js/commit/f2f2abdbdafdd4669e1bd223b5b2f50a24c42b86)) +* **cubesql:** Thoughspot - count distinct with year and month ([#5450](https://github.com/cube-js/cube.js/issues/5450)) ([d44baad](https://github.com/cube-js/cube.js/commit/d44baad34dab8dbf70aa7c9b011dfe17f93b1375)) + + + + + +## [0.31.7](https://github.com/cube-js/cube.js/compare/v0.31.6...v0.31.7) (2022-10-27) + + +### Bug Fixes + +* Remove empty hidden cubes from meta ([#5531](https://github.com/cube-js/cube.js/issues/5531)) ([f93e851](https://github.com/cube-js/cube.js/commit/f93e8515598fd90708606220dd5d96b4e2396fd6)) + + +### Features + +* **cubesql:** Support `is null`, `is not null` in SELECT ([d499c47](https://github.com/cube-js/cube.js/commit/d499c47c6cec4ac7b72d26418598bfbe515c6c62)) +* **cubesql:** Support Sigma Computing string filters ([d18b971](https://github.com/cube-js/cube.js/commit/d18b9712c4eae1ced2b8f94681e1676d854e99a5)) + + + + + +## [0.31.6](https://github.com/cube-js/cube.js/compare/v0.31.5...v0.31.6) (2022-10-20) + + +### Bug Fixes + +* **docker:** Correct lock for deps with yarn.lock ([#5509](https://github.com/cube-js/cube.js/issues/5509)) ([97ca524](https://github.com/cube-js/cube.js/commit/97ca524556178a1210c796ee5bac42d476c7c3df)) + + +### Features + +* **snowflake-driver:** Upgrade snowflake-sdk ([#5508](https://github.com/cube-js/cube.js/issues/5508)) ([2863a93](https://github.com/cube-js/cube.js/commit/2863a93508c8ddf7a4a78a45386b79c2ae5f5ed6)) + + + + + +## [0.31.5](https://github.com/cube-js/cube.js/compare/v0.31.4...v0.31.5) (2022-10-20) + + +### Bug Fixes + +* **cubesql:** Use real database name in Postgres meta layer ([031a90f](https://github.com/cube-js/cube.js/commit/031a90fac068ce35f18823a034f65b486153c726)) + + +### Features + +* **cubesql:** Allow `EXTRACT` field to be parsed as a string ([cad1e0b](https://github.com/cube-js/cube.js/commit/cad1e0b4452233866f87d3856823ef81ca117444)) +* **cubestore:** CC-1133 - add headers to telemetry request ([#5473](https://github.com/cube-js/cube.js/issues/5473)) ([26fa817](https://github.com/cube-js/cube.js/commit/26fa8173961adfc756cee0a9bb626219f33c35b2)) +* pre-aggregations build jobs API endpoint ([#5251](https://github.com/cube-js/cube.js/issues/5251)) ([4aee3ef](https://github.com/cube-js/cube.js/commit/4aee3efba9c14b4698c81707d92af11c98978c81)) + + + + + +## [0.31.4](https://github.com/cube-js/cube.js/compare/v0.31.3...v0.31.4) (2022-10-13) + + +### Bug Fixes + +* **cubestore:** Extension planner for LogicalAlias error ([#5459](https://github.com/cube-js/cube.js/issues/5459)) ([8153459](https://github.com/cube-js/cube.js/commit/8153459c70ddf9acef7e886a6e44971c1aa82660)) +* **druid-driver:** query timezone fix ([#5372](https://github.com/cube-js/cube.js/issues/5372)) ([ccad5fc](https://github.com/cube-js/cube.js/commit/ccad5fc58e23c55283c8e3014908cb51fa5b5fbc)) + + +### Features + +* `shown` flag for cubes and views ([#5455](https://github.com/cube-js/cube.js/issues/5455)) ([e7ef446](https://github.com/cube-js/cube.js/commit/e7ef4467adfc581e07c14b315a43c0eb4d1e8c11)) + + + + + +## [0.31.3](https://github.com/cube-js/cube.js/compare/v0.31.2...v0.31.3) (2022-10-08) + + +### Bug Fixes + +* drivers imports alignment ([#5448](https://github.com/cube-js/cube.js/issues/5448)) ([ab12426](https://github.com/cube-js/cube.js/commit/ab1242650ba0368b855176b9c6ca2d73073acf0e)) + + + + + +## [0.31.2](https://github.com/cube-js/cube.js/compare/v0.31.1...v0.31.2) (2022-10-08) + + +### Bug Fixes + +* Added connection test completed successfully log ([#5444](https://github.com/cube-js/cube.js/issues/5444)) ([a8e1180](https://github.com/cube-js/cube.js/commit/a8e1180d4c05e8de94b116f46f40db531d91e0eb)) +* Temp tables dropped for ksql after successfully created ([#5445](https://github.com/cube-js/cube.js/issues/5445)) ([7726c82](https://github.com/cube-js/cube.js/commit/7726c82fc833a6b14eeeb498a31df62ba9818336)) +* **cubesql:** Handle Panic on simple query executiony ([#5394](https://github.com/cube-js/cube.js/issues/5394)) ([84dc442](https://github.com/cube-js/cube.js/commit/84dc442eb1c42bc3c7b7b03fe365c7c0a948e328)) + + +### Features + +* **mssql-driver:** add column type mapping for a bit type ([#5442](https://github.com/cube-js/cube.js/issues/5442)) ([0ed7ba4](https://github.com/cube-js/cube.js/commit/0ed7ba4b483fff8653602f89f43bd1b03b458859)) +* Includes and Excludes directives for Cube Views ([#5437](https://github.com/cube-js/cube.js/issues/5437)) ([7c35604](https://github.com/cube-js/cube.js/commit/7c356049b56a0ea58a4ad8f2628ffaff2ac307d4)) +* **cubesql:** Support boolean decoding in pg-wire ([#5436](https://github.com/cube-js/cube.js/issues/5436)) ([4fd2ee6](https://github.com/cube-js/cube.js/commit/4fd2ee6cd238161f889896739a00f09e6dc11651)) + + + + + +## [0.31.1](https://github.com/cube-js/cube.js/compare/v0.31.0...v0.31.1) (2022-10-04) + + +### Bug Fixes + +* Cube not found for path due to FILTER_PARAMS are used in members ([#5417](https://github.com/cube-js/cube.js/issues/5417)) ([bfe76bf](https://github.com/cube-js/cube.js/commit/bfe76bfbf3bd5f51a408088f8f5fefb35e17e22f)) +* **extensions:** fix SELECT handling in the Funnels extensions ([#5397](https://github.com/cube-js/cube.js/issues/5397)) ([041f591](https://github.com/cube-js/cube.js/commit/041f591482cc75d840310a9b7592158c1b33fa37)) +* **playground:** bar chart pivot config ([36b9ec2](https://github.com/cube-js/cube.js/commit/36b9ec2f3de5ba3b6b047f3669f8e162321d854e)) + + +### Features + +* **console-ui:** show error stack trace ([7ab15c6](https://github.com/cube-js/cube.js/commit/7ab15c6fbd46696d36c95f197711284d316c9f70)) + + + + + +# [0.31.0](https://github.com/cube-js/cube.js/compare/v0.30.75...v0.31.0) (2022-10-03) + + +* feat!: Cube Views implementation (#5278) ([9937356](https://github.com/cube-js/cube.js/commit/99373563b610d1f15dfc44fafac0c329c1dc9a0d)), closes [#5278](https://github.com/cube-js/cube.js/issues/5278) + + +### Bug Fixes + +* **cubesql:** Allow derived tables to have a dot in column name ([#5391](https://github.com/cube-js/cube.js/issues/5391)) ([f83009c](https://github.com/cube-js/cube.js/commit/f83009cf193a6313296dddffa45bffa08ec01725)) +* **cubesql:** cast strings to timestamp ([#5331](https://github.com/cube-js/cube.js/issues/5331)) ([a706258](https://github.com/cube-js/cube.js/commit/a706258f85faa3f99150127a2c78f885e99e3aaf)) +* **cubesql:** Metabase - substring __user ([#5328](https://github.com/cube-js/cube.js/issues/5328)) ([a25c8bf](https://github.com/cube-js/cube.js/commit/a25c8bf3ddad9c589918b91f05df440eb31a2ad4)) +* **cubesql:** udf format_type prepared statement fix ([#5260](https://github.com/cube-js/cube.js/issues/5260)) ([307ed1b](https://github.com/cube-js/cube.js/commit/307ed1b6cc9b242e76d48241cb871b36f571f91e)) +* **cubesql:** WHERE Lower / Upper in list with multiply items ([#5376](https://github.com/cube-js/cube.js/issues/5376)) ([2269b2b](https://github.com/cube-js/cube.js/commit/2269b2bb41293107f8e8fca118218c56bf3eca53)) +* **cubestore:** Aggregate function MERGE not allowed for column type bytes ([#5166](https://github.com/cube-js/cube.js/issues/5166)) ([7626ed5](https://github.com/cube-js/cube.js/commit/7626ed5fa308854ba625f0956c29b21c84484b67)) +* **cubestore:** Fix error: Internal: channel closed on the next request after cubestore cloud process got OOM ([#5238](https://github.com/cube-js/cube.js/issues/5238)) ([cb81fdb](https://github.com/cube-js/cube.js/commit/cb81fdb79c5b768892831437659e1591773d8e15)) +* **postgres-driver:** release method should not throw ([#5395](https://github.com/cube-js/cube.js/issues/5395)) ([9423f46](https://github.com/cube-js/cube.js/commit/9423f46141eb73eaac24d9b16c449ff2dbc2918a)) +* **query-orchestrator:** pre-aggs build range queries cache key alignment ([#5377](https://github.com/cube-js/cube.js/issues/5377)) ([5896c4a](https://github.com/cube-js/cube.js/commit/5896c4aa4d5a469a892ca9f0e758dc5c6ef6c350)) +* **schema-compiler:** throw an error for the empty pre-aggs ([#5392](https://github.com/cube-js/cube.js/issues/5392)) ([4afd604](https://github.com/cube-js/cube.js/commit/4afd6041dae8175fb8d292e9ee0db15969239c81)) + + +### Features + +* **cubesql:** Holistics - in dates list filter ([#5333](https://github.com/cube-js/cube.js/issues/5333)) ([94b6509](https://github.com/cube-js/cube.js/commit/94b650928a81be9ea203e50612ea194d9558b298)) +* **cubesql:** Support joins with distinct ([#5340](https://github.com/cube-js/cube.js/issues/5340)) ([da4304f](https://github.com/cube-js/cube.js/commit/da4304fef51e33d9c29627d9da92925569943083)) +* multiple data source ([#5326](https://github.com/cube-js/cube.js/issues/5326)) ([334af8c](https://github.com/cube-js/cube.js/commit/334af8c56cd02ae551844e9d1e9ab5e107fb1555)) +* **cubesql:** Add `float8`, `bool` casts ([b345ade](https://github.com/cube-js/cube.js/commit/b345ade898d6a0ec14e320d66129e985244cddb4)) +* **cubesql:** Allow `char_length` function to be used with cubes ([e99344f](https://github.com/cube-js/cube.js/commit/e99344f4e056ef6698f5d92c9e8b79801871a199)) +* **cubesql:** Allow filter by exact year (Tableau) ([#5367](https://github.com/cube-js/cube.js/issues/5367)) ([c31e59d](https://github.com/cube-js/cube.js/commit/c31e59d4763e0dd45e96b8e39eb9bcf914370eae)) +* **cubesql:** Holistics - support range of charts ([#5325](https://github.com/cube-js/cube.js/issues/5325)) ([d16b4c2](https://github.com/cube-js/cube.js/commit/d16b4c2dc0a582d8e28e48a1e5fae3ff2fe7b0de)) +* **cubesql:** Support `date_trunc` over column filter with `<=` ([b30d239](https://github.com/cube-js/cube.js/commit/b30d239ae4e00d8d547f0aa65b324f1f0d3af3f1)) +* **query-orchestrator:** introduce unload without temp table ([#5324](https://github.com/cube-js/cube.js/issues/5324)) ([3dcbd2e](https://github.com/cube-js/cube.js/commit/3dcbd2ed1d214d56bfde2183538fce3ec7d65595)) +* **testing:** databricks test suite ([#5311](https://github.com/cube-js/cube.js/issues/5311)) ([b77f33b](https://github.com/cube-js/cube.js/commit/b77f33ba9d804d8ca8746fe99d6050ebe26b4528)) + + +### BREAKING CHANGES + +* The logic of how cubes are included in joins has been changed. There are multiple member reference constructs that are now forbidden and should be rewritten. You can't reference foreign cubes anymore to define members inside other cubes: `${ForeignCube}.foo`. `foo` member should be defined in `ForeignCube` and referenced as `${ForeignCube.foo}`. You also can't mix references and members without `CUBE` self-reference. For example `${ForeignCube.foo} + bar` is invalid and `${ForeignCube.foo} + ${CUBE}.bar` should be used instead. If not fixed, it'll lead to missing tables in the `FROM` clause. + + + + + ## [0.30.75](https://github.com/cube-js/cube.js/compare/v0.30.74...v0.30.75) (2022-09-22) diff --git a/CODEOWNERS b/CODEOWNERS index 8e80c6cd84d37..d4b73a3fe7aa6 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -24,8 +24,8 @@ /packages/cubejs-schema-compiler/ @cube-js/schema-compiler /packages/*-schema-extension/ @cube-js/schema-compiler +# Doc reviewers +/docs/ @cube-js/doc-reviewers + # Developer Relations and Community -*.md @cube-js/developer-relations-and-community -/docs/ @cube-js/developer-relations-and-community /examples/ @cube-js/developer-relations-and-community -/guides/ @cube-js/developer-relations-and-community diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 36411b53c335c..3305773b2b2e8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,20 +18,20 @@ Please review the following sections before proposing code changes. ### Developer Certificate of Origin (DCO) -By contributing to Cube Dev, Inc., You accept and agree to the terms and conditions in the [Developer Certificate of Origin](https://github.com/cube-js/cube.js/blob/master/DCO.md) for Your present and future Contributions submitted to Cube Dev, Inc. Your contribution includes any submissions to the [Cube.js repository](https://github.com/cube-js) when you click on such buttons as `Propose changes` or `Create pull request`. Except for the licenses granted herein, You reserve all right, title, and interest in and to Your Contributions. +By contributing to Cube Dev, Inc., You accept and agree to the terms and conditions in the [Developer Certificate of Origin](https://github.com/cube-js/cube/blob/master/DCO.md) for Your present and future Contributions submitted to Cube Dev, Inc. Your contribution includes any submissions to the [Cube.js repository](https://github.com/cube-js) when you click on such buttons as `Propose changes` or `Create pull request`. Except for the licenses granted herein, You reserve all right, title, and interest in and to Your Contributions. ### Our quarterly roadmap -We publish our open source roadmap every quarter and discuss them during our [monthly community calls](https://cube.dev/community-call/). You can find our roadmap under [projects in our Cube.js repository](https://github.com/cube-js/cube.js/projects?query=is%3Aopen+sort%3Aupdated-desc). +We publish our open source roadmap every quarter and discuss them during our [monthly community calls](https://cube.dev/community-call/). You can find our roadmap under [projects in our Cube.js repository](https://github.com/cube-js/cube/projects?query=is%3Aopen+sort%3Aupdated-desc). ## Step-by-step guide to contributing -1. Find [issues](https://github.com/cube-js/cube.js/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc) where we need help. Search for issues with either [`good first issue`](https://github.com/cube-js/cube.js/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22+) and/or [`help wanted`](https://github.com/cube-js/cube.js/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22help+wanted%22) labels. +1. Find [issues](https://github.com/cube-js/cube/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc) where we need help. Search for issues with either [`good first issue`](https://github.com/cube-js/cube/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22+) and/or [`help wanted`](https://github.com/cube-js/cube/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22help+wanted%22) labels. 2. Follow the directions in the [Getting Started guide](https://cube.dev/docs/getting-started) to get Cube.js up and running (incl. the [Developer Playground](https://cube.dev/docs/dev-tools/dev-playground)). -3. Clone the [Cube.js repo](https://github.com/cube-js/cube.js). +3. Clone the [Cube.js repo](https://github.com/cube-js/cube). 4. Submit your Pull Request. -5. Testing: Please include test(s) for your code contribution. See some of the test examples for [drivers](https://github.com/cube-js/cube.js/pull/1333/commits/56dadccd62ac4eaceafe650d2853406f5d3d9d43) and [backend](https://github.com/cube-js/cube.js/tree/master/packages/cubejs-backend-shared/test). -6. Documentation: When new features are added or there are changes to existing features that require updates to documentation, we encourage you to add/update any missing documentation in the [`/docs` folder](https://github.com/cube-js/cube.js/tree/master/docs). To update an existing documentation page, you can simply click on the `Edit this page` button on the top right corner of the documentation page. +5. Testing: Please include test(s) for your code contribution. See some of the test examples for [drivers](https://github.com/cube-js/cube/pull/1333/commits/56dadccd62ac4eaceafe650d2853406f5d3d9d43) and [backend](https://github.com/cube-js/cube/tree/master/packages/cubejs-backend-shared/test). +6. Documentation: When new features are added or there are changes to existing features that require updates to documentation, we encourage you to add/update any missing documentation in the [`/docs` folder](https://github.com/cube-js/cube/tree/master/docs). To update an existing documentation page, you can simply click on the `Edit this page` button on the top right corner of the documentation page. 7. Relevant team(s) will be pinged automatically for a review based on information in the `CODEOWNERS` file. ## Development Workflow @@ -82,7 +82,7 @@ $ cd packages/cubejs-client-core && yarn && yarn link && cd ../.. && cd packages If you are going to develop a JDBC driver, you need to [install Java with JDK][link-java-guide]. [link-java-guide]: -https://github.com/cube-js/cube.js/blob/master/packages/cubejs-jdbc-driver/README.md#java-installation +https://github.com/cube-js/cube/blob/master/packages/cubejs-jdbc-driver/README.md#java-installation #### Development @@ -91,7 +91,7 @@ Cube.js is written in a mixture of plain JavaScript and TypeScript. TypeScript i > Attention: Cube.js uses TypeScript configured in incremental mode, which uses cache to speed up compilation, > but in some cases, you can run into a problem with a not recompiled file. To fix it, we recommend running `$ yarn clean` and `$ yarn tsc`. -1. Clone the Cube.js repository, `git clone https://github.com/cube-js/cube.js`. +1. Clone the Cube.js repository, `git clone https://github.com/cube-js/cube`. 2. Run `yarn install` in the root directory. 3. Run `yarn build` in the root directory to build the frontend dependent packages. 4. Run `yarn build` in `packages/cubejs-playground` to build the frontend. @@ -111,6 +111,18 @@ Cube.js is written in a mixture of plain JavaScript and TypeScript. TypeScript i 3. Create a new configuration, using `./node_modules/.bin/cubejs-server` for Node Parameters and the directory of your test project for Working directory. 4. Run/Debug dev cube.js servers using the new configuration. +## Contributing Database Drivers + +To enhance the adoption of community-contributed drivers, we decided to split the database driver contribution process into multiple stages. + +1. Each driver which is planned to be contributed to the main Cube repository should be published first as an npm package. Please see [Publishing Driver npm package](#publishing-driver-npm-package) on how to do that. +2. This NPM package should be contributed to the list of [Third-party community drivers](https://cube.dev/docs/config/databases#third-party-community-drivers). +3. Please make sure each npm package has a README with instructions on how to install it to the official docker image and how to connect it to the database. +4. Posting a backlink to an open-source repository would be a good idea here so people can provide feedback on it by posting issues. +5. Before creating PR for the main repository, please make sure it's tested with the standard Cube E2E testing suite. An example of an E2E testing suite can be found here: https://github.com/cube-js/cube/blob/master/packages/cubejs-testing/test/driver-postgres.test.ts +6. If you're creating PR for the main repo, please be prepared to become a maintainer for this driver and dedicate some time to it. There're no specific time requirements. As a rule of thumb, you should expect to spend time on a weekly basis. +7. Due to limited resources Core team will review and merge driver PRs based on popularity and development activity. + ### Implementing a Driver 1. Copy existing driver package structure and name it in `@cubejs-backend/-driver` format. @@ -124,7 +136,7 @@ The rest will be done by `BaseDriver` class. 6. If db requires connection pooling prefer use `generic-pool` implementation with settings similar to other db packages. 7. Make sure your driver has `release()` method in case DB expects graceful shutdowns for connections. 8. Please use yarn to add any dependencies and run `$ yarn` within the package before committing to ensure right `yarn.lock` is in place. -9. Add this driver dependency to [cubejs-server-core/core/DriverDependencies.js](https://github.com/cube-js/cube.js/blob/master/packages/cubejs-server-core/core/DriverDependencies.js#L1). +9. Add this driver dependency to [cubejs-server-core/core/DriverDependencies.js](https://github.com/cube-js/cube/blob/master/packages/cubejs-server-core/core/DriverDependencies.js#L1). ### Implementing a JDBC Driver @@ -156,6 +168,8 @@ If driver class contains `static dialectClass()` method it'll be used to lookup Cube.js looks up `cubejs-{dbType}-driver` package among installed modules to fullfil driver dependency if there's no corresponding default driver for the specified database type. For example one can publish `cubejs-foo-driver` npm package to fullfil driver dependency for the `foo` database type. +## Other Packages + ### Testing Schema Compiler In order to run tests in `cubejs-schema-compiler` package you need to have running [Docker](https://docs.docker.com/install/) on your machine. diff --git a/DCO.md b/DCO.md index 03c381fcaf5ed..0572a3fde2cd6 100644 --- a/DCO.md +++ b/DCO.md @@ -10,7 +10,7 @@ contribute and that they understand that the contribution is under the terms of the [Cube.js project licenses][link-licenses]. [link-licenses]: - https://github.com/cube-js/cube.js/blob/master/README.md#license + https://github.com/cube-js/cube/blob/master/README.md#license # Developer's Certificate of Origin @@ -23,7 +23,7 @@ the licenses granted herein, You reserve all right, title, and interest in and to Your Contributions. [link-dco]: https://developercertificate.org/ -[link-cubejs-repo]: https://github.com/cube-js/cube.js +[link-cubejs-repo]: https://github.com/cube-js/cube ``` Developer Certificate of Origin diff --git a/DEPRECATION.md b/DEPRECATION.md index 23cde834b21b7..ca5f6d88716b2 100644 --- a/DEPRECATION.md +++ b/DEPRECATION.md @@ -34,25 +34,27 @@ features: should not rely on this feature. | Status | Feature | Deprecated | Remove | -| ---------- | --------------------------------------------------------------------------------------------------------------------------------- | ---------- | --------- | +|------------|-----------------------------------------------------------------------------------------------------------------------------------|------------|-----------| | Removed | [Node.js 8](#nodejs-8) | v0.22.4 | v0.26.0 | | Deprecated | [`hearBeatInterval`](#hearbeatinterval) | v0.23.8 | June 2021 | | Removed | [`CUBEJS_ENABLE_TLS`](#cubejs_enable_tls) | v0.23.11 | v0.26.0 | | Deprecated | [Embedding Cube.js within Express](#embedding-cubejs-within-express) | v0.24.0 | June 2021 | -| Deprecated | [Absolute import for `@cubejs-backend/query-orchestrator`](#absolute-import-for-@cubejs-backendquery-orchestrator) | v0.24.2 | v0.28.0 | +| Removed | [Absolute import for `@cubejs-backend/query-orchestrator`](#absolute-import-for-@cubejs-backendquery-orchestrator) | v0.24.2 | v0.32.0 | | Removed | [`contextToDataSourceId`](#contexttodatasourceid) | v0.25.0 | v0.25.0 | -| Deprecated | [Absolute import for `@cubejs-backend/server-core`](#absolute-import-for-@cubejs-backendserver-core) | v0.25.4 | v0.30.0 | -| Deprecated | [Absolute import for `@cubejs-backend/schema-compiler`](#absolute-import-for-@cubejs-backendschema-compiler) | v0.25.21 | v0.32.0 | +| Removed | [Absolute import for `@cubejs-backend/server-core`](#absolute-import-for-@cubejs-backendserver-core) | v0.25.4 | v0.32.0 | +| Removed | [Absolute import for `@cubejs-backend/schema-compiler`](#absolute-import-for-@cubejs-backendschema-compiler) | v0.25.21 | v0.32.0 | | Deprecated | [`checkAuthMiddleware`](#checkauthmiddleware) | v0.26.0 | | | Removed | [Node.js 10](#nodejs-10) | v0.26.0 | v0.29.0 | -| Removed | [Node.js 15](#nodejs-15) | v0.26.0 | v0.29.0 | +| Removed | [Node.js 15](#nodejs-15) | v0.26.0 | v0.32.0 | | Deprecated | [`USER_CONTEXT`](#user_context) | v0.26.0 | | | Deprecated | [`authInfo`](#authinfo) | v0.26.0 | | | Deprecated | [Prefix Redis environment variables with `CUBEJS_`](#prefix-redis-environment-variables-with-cubejs_) | v0.27.0 | | -| Deprecated | [Node.js 12](#nodejs-12) | v0.29.0 | | +| Removed | [Node.js 12](#nodejs-12) | v0.29.0 | v0.32.0 | | Deprecated | [`CUBEJS_EXTERNAL_DEFAULT` and `CUBEJS_SCHEDULED_REFRESH_DEFAULT`](#cubejs_external_default-and-cubejs_scheduled_refresh_default) | v0.30.0 | | | Deprecated | [Using external databases for pre-aggregations](#using-external-databases-for-pre-aggregations) | v0.30.0 | | | Deprecated | [`dbType`](#dbtype) | v0.30.30 | | +| Deprecated | [Serverless Deployments](#serverless-deployments) | v0.31.64 | | +| Deprecated | [Node.js 14](#nodejs-14) | v0.32.0 | | ### Node.js 8 @@ -100,7 +102,7 @@ microservices, if necessary). ### Absolute import for `@cubejs-backend/query-orchestrator` -**Deprecated in Release: v0.24.2** +**Removed in Release: v0.32.0** Absolute imports are highly dependent on a path, and all API becomes public. We now provide a public API from the package directly. @@ -133,7 +135,7 @@ removing the `contextToDataSourceId` property completely. ### Absolute import for `@cubejs-backend/server-core` -**Deprecated in Release: v0.25.4** +**Removed in Release: v0.32.0** Absolute imports are highly dependent on a path, and all API becomes public. We now provide a public API from the package directly. @@ -152,7 +154,7 @@ const { CubejsServerCore } = require('@cubejs-backend/server-core'); ### Absolute import for `@cubejs-backend/schema-compiler` -**Deprecated in Release: v0.25.21** +**Removed in Release: v0.32.0** Absolute imports are highly dependent on a path, and all API becomes public. We now provide a public API from the package directly. @@ -293,13 +295,11 @@ more updates. Please upgrade to Node.js 14 or higher. ### Node.js 12 -**Deprecated in Release: v0.29.0** +**Removed in Release: v0.32.0** Node.js 12 reached [End of Life on May 19, 2021][link-nodejs-eol]. This means no more updates. Please upgrade to Node.js 14 or higher. -[link-nodejs-eol]: https://github.com/nodejs/Release#end-of-life-releases - ### Using non-Cube Store databases as external database **Deprecated in Release: v0.29.0** @@ -327,10 +327,27 @@ recommend [using Cube Store as a solution][ref-caching-in-prod]. [ref-caching-in-prod]: https://cube.dev/docs/caching/running-in-production -### dbType +### `dbType` **Deprecated in Release: v0.30.30** Using `dbType` is now deprecated, and we recommend using [`driverFactory`][self-driver-factory] to return a `DriverConfig` object instead. + +### Serverless Deployments + +**Deprecated in Release: v0.31.64** + +Using Serverless deployments with the `@cubejs-backend/serverless` package is +now deprecated; we **strongly** recommend using Docker-based deployments +instead. + +### Node.js 14 + +**Deprecated in Release: v0.32.0** + +Node.js 14 reached [End of Life on April 30, 2023][link-nodejs-eol]. This means no +more updates. Please upgrade to Node.js 16 or higher. + +[link-nodejs-eol]: https://github.com/nodejs/Release#end-of-life-releases diff --git a/README.md b/README.md index 0ed53f27378a4..dcfc5908c5306 100644 --- a/README.md +++ b/README.md @@ -27,19 +27,19 @@ CD part will take place from codefresh pipeline and will only publish specific p see examples in platform-analytics package.json file

- Cube — Headless Business Intelligence + Cube — Semantic Layer for Data Applications

-[Website](https://cube.dev?ref=github-readme) • [Getting Started](https://cube.dev/docs/getting-started?ref=github-readme) • [Docs](https://cube.dev/docs?ref=github-readme) • [Examples](https://cube.dev/docs/examples?ref=github-readme) • [Blog](https://cube.dev/blog?ref=github-readme) • [Slack](https://slack.cube.dev?ref=github-readme) • [Discourse](https://forum.cube.dev/) • [Twitter](https://twitter.com/thecubejs) +[Website](https://cube.dev?ref=github-readme) • [Getting Started](https://cube.dev/docs/getting-started?ref=github-readme) • [Docs](https://cube.dev/docs?ref=github-readme) • [Examples](https://cube.dev/docs/examples?ref=github-readme) • [Blog](https://cube.dev/blog?ref=github-readme) • [Slack](https://slack.cube.dev?ref=github-readme) • [Twitter](https://twitter.com/the_cube_dev) [![npm version](https://badge.fury.io/js/%40cubejs-backend%2Fserver.svg)](https://badge.fury.io/js/%40cubejs-backend%2Fserver) -[![GitHub Actions](https://github.com/cube-js/cube.js/workflows/Build/badge.svg)](https://github.com/cube-js/cube.js/actions?query=workflow%3ABuild+branch%3Amaster) +[![GitHub Actions](https://github.com/cube-js/cube/workflows/Build/badge.svg)](https://github.com/cube-js/cube/actions?query=workflow%3ABuild+branch%3Amaster) [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fcube-js%2Fcube.js.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fcube-js%2Fcube.js?ref=badge_shield) -__Cube is the headless business intelligence platform.__ It helps data engineers and application developers access data from modern data stores, organize it into consistent definitions, and deliver it to every application. +__Cube is the semantic layer for building data applications.__ It helps data engineers and application developers access data from modern data stores, organize it into consistent definitions, and deliver it to every application. @@ -72,7 +72,7 @@ Cube has the necessary infrastructure and features to implement efficient data m Get started now -For a step-by-step guide on Cube Cloud, [see the docs](https://cube.dev/docs/cloud/getting-started/create?ref=github-readme). +For a step-by-step guide on Cube Cloud, [see the docs](https://cube.dev/docs/getting-started/cloud/overview?ref=github-readme). ### Docker @@ -82,6 +82,7 @@ Once Docker is installed, in a new folder for your project, run the following co ```bash docker run -p 4000:4000 \ + -p 15432:15432 \ -v ${PWD}:/cube/conf \ -e CUBEJS_DEV_MODE=true \ cubejs/cube @@ -110,20 +111,20 @@ You are also welcome to join our **monthly community calls** where we discuss co ### Our quarterly roadmap -We publish our open source roadmap every quarter and discuss them during our [monthly community calls](https://cube.dev/community-call/). You can find our roadmap under [projects in our Cube.js repository](https://github.com/cube-js/cube.js/projects?query=is%3Aopen+sort%3Aupdated-desc). +We publish our open source roadmap every quarter and discuss them during our [monthly community calls](https://cube.dev/community-call/). You can find our roadmap under [projects in our Cube.js repository](https://github.com/cube-js/cube/projects?query=is%3Aopen+sort%3Aupdated-desc). ### Contributing There are many ways you can contribute to Cube! Here are a few possibilities: -* Star this repo and follow us on [Twitter](https://twitter.com/thecubejs). +* Star this repo and follow us on [Twitter](https://twitter.com/the_cube_dev). * Add Cube to your stack on [Stackshare](https://stackshare.io/cube-js). * Upvote issues with 👍 reaction so we know what's the demand for particular issue to prioritize it within road map. * Create issues every time you feel something is missing or goes wrong. * Ask questions on [Stack Overflow with cube.js tag](https://stackoverflow.com/questions/tagged/cube.js) if others can have these questions as well. -* Provide pull requests for all open issues and especially for those with [help wanted](https://github.com/cube-js/cube.js/issues?q=is%3Aissue+is%3Aopen+label%3A"help+wanted") and [good first issue](https://github.com/cube-js/cube.js/issues?q=is%3Aissue+is%3Aopen+label%3A"good+first+issue") labels. +* Provide pull requests for all open issues and especially for those with [help wanted](https://github.com/cube-js/cube/issues?q=is%3Aissue+is%3Aopen+label%3A"help+wanted") and [good first issue](https://github.com/cube-js/cube/issues?q=is%3Aissue+is%3Aopen+label%3A"good+first+issue") labels. -All sort of contributions are **welcome and extremely helpful** 🙌 Please refer to [the contribution guide](https://github.com/cube-js/cube.js/blob/master/CONTRIBUTING.md) for more information. +All sort of contributions are **welcome and extremely helpful** 🙌 Please refer to [the contribution guide](https://github.com/cube-js/cube/blob/master/CONTRIBUTING.md) for more information. ## License diff --git a/codefresh.yml b/codefresh.yml index 1f4d45965ca33..7eefff2080866 100644 --- a/codefresh.yml +++ b/codefresh.yml @@ -19,7 +19,7 @@ steps: install_dependencies: title: 'Installing dependencies' working_directory: ${{clone}} - image: node:12.22.9 + image: node:14.21.1 commands: - ls - yarn install --frozen-lockfile diff --git a/docs-gen/tasks/generate.js b/docs-gen/tasks/generate.js index 50cfdedb23ad5..a1d6707ff639e 100644 --- a/docs-gen/tasks/generate.js +++ b/docs-gen/tasks/generate.js @@ -2,7 +2,7 @@ const TypeDoc = require('typedoc'); const path = require('path'); const fs = require('fs-extra'); -const outputDir = '../docs/content/Frontend-Integrations'; +const outputDir = '../docs/content/Reference/Frontend'; const app = new TypeDoc.Application(); diff --git a/docs-gen/template/publish.js b/docs-gen/template/publish.js index 3f7fbd6018c49..7867f25151428 100644 --- a/docs-gen/template/publish.js +++ b/docs-gen/template/publish.js @@ -104,8 +104,8 @@ const tagValue = (doclet, tagOriginalTitle) => { const generateModuleSection = (doclet) => `--- title: '${doclet.name}' permalink: ${tagValue(doclet, 'permalink')} -category: ${tagValue(doclet, 'category')} -subCategory: Reference +category: Reference +subCategory: ${tagValue(doclet, 'subCategory')} menuOrder: ${tagValue(doclet, 'menuOrder')} --- diff --git a/docs-gen/tsconfig.json b/docs-gen/tsconfig.json index e03f1949993cd..a43517fdc041b 100644 --- a/docs-gen/tsconfig.json +++ b/docs-gen/tsconfig.json @@ -16,7 +16,8 @@ "paths": { "react": ["node_modules/@types/react"], "@cubejs-client/core": ["../packages/cubejs-client-core"] - } + }, + "skipLibCheck": true }, "exclude": ["./dist", "./test", "**/*.spec.ts"] } diff --git a/docs/.env.development b/docs/.env.development index d51ff4e196d79..cc187749255f6 100644 --- a/docs/.env.development +++ b/docs/.env.development @@ -1,2 +1,3 @@ -ALGOLIA_API_KEY='665bbb3a2c36759517a8b75ddba8601f' +ALGOLIA_APP_ID='PBBQOO27UD' +ALGOLIA_API_KEY='2998a34fa3a0f52837692ecdaa03f44b' ALGOLIA_INDEX_NAME='cubejs' diff --git a/docs/.env.production b/docs/.env.production index b90e6dad7fc92..7e76144e5090e 100644 --- a/docs/.env.production +++ b/docs/.env.production @@ -1,3 +1,4 @@ -ALGOLIA_API_KEY='665bbb3a2c36759517a8b75ddba8601f' +ALGOLIA_APP_ID='PBBQOO27UD' +ALGOLIA_API_KEY='2998a34fa3a0f52837692ecdaa03f44b' ALGOLIA_INDEX_NAME='cubejs' PATH_PREFIX='/docs' diff --git a/docs/.markdownlint.json b/docs/.markdownlint.json index 6dcc594fe3bf4..24d02034b461c 100644 --- a/docs/.markdownlint.json +++ b/docs/.markdownlint.json @@ -4,8 +4,19 @@ "no-inline-html": { "allowed_elements": [ "a", + "br", + "code", + "div", "img", - "p" + "iframe", + "p", + "pre", + "sup", + "CodeTabs", + "Grid", + "InfoBox", + "LoomVideo", + "WarningBox" ] }, "no-trailing-punctuation": "" diff --git a/docs/.prettierrc b/docs/.prettierrc index e3d733c19b2e5..3c800898e375d 100644 --- a/docs/.prettierrc +++ b/docs/.prettierrc @@ -3,7 +3,7 @@ "tabWidth": 2, "useTabs": false, "semi": true, - "singleQuote": true, + "singleQuote": false, "arrowParens": "always", "trailingComma": "es5", "bracketSpacing": true, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 016797f96302f..f190c3e8d3049 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -3,7 +3,7 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. -## [0.26.53](https://github.com/cube-js/cube.js/compare/v0.26.52...v0.26.53) (2021-03-11) +## [0.26.53](https://github.com/cube-js/cube/compare/v0.26.52...v0.26.53) (2021-03-11) **Note:** Version bump only for package @cubejs-docs/site @@ -11,18 +11,18 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.49](https://github.com/cube-js/cube.js/compare/v0.26.48...v0.26.49) (2021-03-05) +## [0.26.49](https://github.com/cube-js/cube/compare/v0.26.48...v0.26.49) (2021-03-05) ### Features -* **elasticsearch-driver:** Support for elastic.co & improve docs ([#2240](https://github.com/cube-js/cube.js/issues/2240)) ([d8557f6](https://github.com/cube-js/cube.js/commit/d8557f6487ea98c19c055cc94b94b284dd273835)) +* **elasticsearch-driver:** Support for elastic.co & improve docs ([#2240](https://github.com/cube-js/cube/issues/2240)) ([d8557f6](https://github.com/cube-js/cube/commit/d8557f6487ea98c19c055cc94b94b284dd273835)) -## [0.26.46](https://github.com/cube-js/cube.js/compare/v0.26.45...v0.26.46) (2021-03-04) +## [0.26.46](https://github.com/cube-js/cube/compare/v0.26.45...v0.26.46) (2021-03-04) **Note:** Version bump only for package @cubejs-docs/site @@ -30,7 +30,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.45](https://github.com/cube-js/cube.js/compare/v0.26.44...v0.26.45) (2021-03-04) +## [0.26.45](https://github.com/cube-js/cube/compare/v0.26.44...v0.26.45) (2021-03-04) **Note:** Version bump only for package @cubejs-docs/site @@ -38,7 +38,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.44](https://github.com/cube-js/cube.js/compare/v0.26.43...v0.26.44) (2021-03-02) +## [0.26.44](https://github.com/cube-js/cube/compare/v0.26.43...v0.26.44) (2021-03-02) **Note:** Version bump only for package @cubejs-docs/site @@ -46,7 +46,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.40](https://github.com/cube-js/cube.js/compare/v0.26.39...v0.26.40) (2021-03-01) +## [0.26.40](https://github.com/cube-js/cube/compare/v0.26.39...v0.26.40) (2021-03-01) **Note:** Version bump only for package @cubejs-docs/site @@ -54,7 +54,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.38](https://github.com/cube-js/cube.js/compare/v0.26.37...v0.26.38) (2021-02-26) +## [0.26.38](https://github.com/cube-js/cube/compare/v0.26.37...v0.26.38) (2021-02-26) **Note:** Version bump only for package @cubejs-docs/site @@ -62,7 +62,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.36](https://github.com/cube-js/cube.js/compare/v0.26.35...v0.26.36) (2021-02-25) +## [0.26.36](https://github.com/cube-js/cube/compare/v0.26.35...v0.26.36) (2021-02-25) **Note:** Version bump only for package @cubejs-docs/site @@ -70,7 +70,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.34](https://github.com/cube-js/cube.js/compare/v0.26.33...v0.26.34) (2021-02-25) +## [0.26.34](https://github.com/cube-js/cube/compare/v0.26.33...v0.26.34) (2021-02-25) **Note:** Version bump only for package @cubejs-docs/site @@ -78,7 +78,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.31](https://github.com/cube-js/cube.js/compare/v0.26.30...v0.26.31) (2021-02-23) +## [0.26.31](https://github.com/cube-js/cube/compare/v0.26.30...v0.26.31) (2021-02-23) **Note:** Version bump only for package @cubejs-docs/site @@ -86,7 +86,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.30](https://github.com/cube-js/cube.js/compare/v0.26.29...v0.26.30) (2021-02-22) +## [0.26.30](https://github.com/cube-js/cube/compare/v0.26.29...v0.26.30) (2021-02-22) **Note:** Version bump only for package @cubejs-docs/site @@ -94,7 +94,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.29](https://github.com/cube-js/cube.js/compare/v0.26.28...v0.26.29) (2021-02-22) +## [0.26.29](https://github.com/cube-js/cube/compare/v0.26.28...v0.26.29) (2021-02-22) **Note:** Version bump only for package @cubejs-docs/site @@ -102,7 +102,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.20](https://github.com/cube-js/cube.js/compare/v0.26.19...v0.26.20) (2021-02-19) +## [0.26.20](https://github.com/cube-js/cube/compare/v0.26.19...v0.26.20) (2021-02-19) **Note:** Version bump only for package @cubejs-docs/site @@ -110,29 +110,29 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.16](https://github.com/cube-js/cube.js/compare/v0.26.15...v0.26.16) (2021-02-18) +## [0.26.16](https://github.com/cube-js/cube/compare/v0.26.15...v0.26.16) (2021-02-18) ### Features -* **druid-driver:** Support CUBEJS_DB_SSL ([d8124d0](https://github.com/cube-js/cube.js/commit/d8124d0a91c926ce0e1ffd21d6d057c164b01e79)) +* **druid-driver:** Support CUBEJS_DB_SSL ([d8124d0](https://github.com/cube-js/cube/commit/d8124d0a91c926ce0e1ffd21d6d057c164b01e79)) -## [0.26.15](https://github.com/cube-js/cube.js/compare/v0.26.14...v0.26.15) (2021-02-16) +## [0.26.15](https://github.com/cube-js/cube/compare/v0.26.14...v0.26.15) (2021-02-16) ### Features -* **clickhouse-driver:** HTTPS and readOnly support ([3d60ead](https://github.com/cube-js/cube.js/commit/3d60ead920635eb85c76b51a5c301e2f1fb08cb6)) +* **clickhouse-driver:** HTTPS and readOnly support ([3d60ead](https://github.com/cube-js/cube/commit/3d60ead920635eb85c76b51a5c301e2f1fb08cb6)) -## [0.26.14](https://github.com/cube-js/cube.js/compare/v0.26.13...v0.26.14) (2021-02-15) +## [0.26.14](https://github.com/cube-js/cube/compare/v0.26.13...v0.26.14) (2021-02-15) **Note:** Version bump only for package @cubejs-docs/site @@ -140,7 +140,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.13](https://github.com/cube-js/cube.js/compare/v0.26.12...v0.26.13) (2021-02-12) +## [0.26.13](https://github.com/cube-js/cube/compare/v0.26.12...v0.26.13) (2021-02-12) **Note:** Version bump only for package @cubejs-docs/site @@ -148,7 +148,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.10](https://github.com/cube-js/cube.js/compare/v0.26.9...v0.26.10) (2021-02-09) +## [0.26.10](https://github.com/cube-js/cube/compare/v0.26.9...v0.26.10) (2021-02-09) **Note:** Version bump only for package @cubejs-docs/site @@ -156,19 +156,19 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.7](https://github.com/cube-js/cube.js/compare/v0.26.6...v0.26.7) (2021-02-09) +## [0.26.7](https://github.com/cube-js/cube/compare/v0.26.6...v0.26.7) (2021-02-09) ### Features -* Support for Redis Sentinel + IORedis driver. fix [#1769](https://github.com/cube-js/cube.js/issues/1769) ([a5e7972](https://github.com/cube-js/cube.js/commit/a5e7972485fa97faaf9965b9794b0cf48256f484)) -* Use REDIS_URL for IORedis options (with santinels) ([988bfe5](https://github.com/cube-js/cube.js/commit/988bfe5526be3506fe7b773d247ad89b3287fad4)) +* Support for Redis Sentinel + IORedis driver. fix [#1769](https://github.com/cube-js/cube/issues/1769) ([a5e7972](https://github.com/cube-js/cube/commit/a5e7972485fa97faaf9965b9794b0cf48256f484)) +* Use REDIS_URL for IORedis options (with santinels) ([988bfe5](https://github.com/cube-js/cube/commit/988bfe5526be3506fe7b773d247ad89b3287fad4)) -## [0.26.6](https://github.com/cube-js/cube.js/compare/v0.26.5...v0.26.6) (2021-02-08) +## [0.26.6](https://github.com/cube-js/cube/compare/v0.26.5...v0.26.6) (2021-02-08) **Note:** Version bump only for package @cubejs-docs/site @@ -176,7 +176,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.4](https://github.com/cube-js/cube.js/compare/v0.26.3...v0.26.4) (2021-02-02) +## [0.26.4](https://github.com/cube-js/cube/compare/v0.26.3...v0.26.4) (2021-02-02) **Note:** Version bump only for package @cubejs-docs/site @@ -184,7 +184,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.26.2](https://github.com/cube-js/cube.js/compare/v0.26.1...v0.26.2) (2021-02-01) +## [0.26.2](https://github.com/cube-js/cube/compare/v0.26.1...v0.26.2) (2021-02-01) **Note:** Version bump only for package @cubejs-docs/site @@ -192,18 +192,18 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -# [0.26.0](https://github.com/cube-js/cube.js/compare/v0.25.33...v0.26.0) (2021-02-01) +# [0.26.0](https://github.com/cube-js/cube/compare/v0.25.33...v0.26.0) (2021-02-01) ### Features -* Storing userContext inside payload.u is deprecated, moved to root ([559bd87](https://github.com/cube-js/cube.js/commit/559bd8757d9754ab486eed88d1fdb0c280b82dc9)) +* Storing userContext inside payload.u is deprecated, moved to root ([559bd87](https://github.com/cube-js/cube/commit/559bd8757d9754ab486eed88d1fdb0c280b82dc9)) -## [0.25.32](https://github.com/cube-js/cube.js/compare/v0.25.31...v0.25.32) (2021-01-29) +## [0.25.32](https://github.com/cube-js/cube/compare/v0.25.31...v0.25.32) (2021-01-29) **Note:** Version bump only for package @cubejs-docs/site @@ -211,7 +211,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.25.31](https://github.com/cube-js/cube.js/compare/v0.25.30...v0.25.31) (2021-01-28) +## [0.25.31](https://github.com/cube-js/cube/compare/v0.25.30...v0.25.31) (2021-01-28) **Note:** Version bump only for package @cubejs-docs/site @@ -219,7 +219,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.25.28](https://github.com/cube-js/cube.js/compare/v0.25.27...v0.25.28) (2021-01-25) +## [0.25.28](https://github.com/cube-js/cube/compare/v0.25.27...v0.25.28) (2021-01-25) **Note:** Version bump only for package @cubejs-docs/site @@ -227,7 +227,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.25.25](https://github.com/cube-js/cube.js/compare/v0.25.24...v0.25.25) (2021-01-24) +## [0.25.25](https://github.com/cube-js/cube/compare/v0.25.24...v0.25.25) (2021-01-24) **Note:** Version bump only for package @cubejs-docs/site @@ -235,7 +235,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.25.23](https://github.com/cube-js/cube.js/compare/v0.25.22...v0.25.23) (2021-01-22) +## [0.25.23](https://github.com/cube-js/cube/compare/v0.25.22...v0.25.23) (2021-01-22) **Note:** Version bump only for package @cubejs-docs/site @@ -243,7 +243,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.25.22](https://github.com/cube-js/cube.js/compare/v0.25.21...v0.25.22) (2021-01-21) +## [0.25.22](https://github.com/cube-js/cube/compare/v0.25.21...v0.25.22) (2021-01-21) **Note:** Version bump only for package @cubejs-docs/site @@ -251,7 +251,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.25.21](https://github.com/cube-js/cube.js/compare/v0.25.20...v0.25.21) (2021-01-19) +## [0.25.21](https://github.com/cube-js/cube/compare/v0.25.20...v0.25.21) (2021-01-19) **Note:** Version bump only for package @cubejs-docs/site @@ -259,7 +259,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.25.18](https://github.com/cube-js/cube.js/compare/v0.25.17...v0.25.18) (2021-01-14) +## [0.25.18](https://github.com/cube-js/cube/compare/v0.25.17...v0.25.18) (2021-01-14) **Note:** Version bump only for package @cubejs-docs/site @@ -267,7 +267,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.25.14](https://github.com/cube-js/cube.js/compare/v0.25.13...v0.25.14) (2021-01-11) +## [0.25.14](https://github.com/cube-js/cube/compare/v0.25.13...v0.25.14) (2021-01-11) **Note:** Version bump only for package @cubejs-docs/site @@ -275,7 +275,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.25.11](https://github.com/cube-js/cube.js/compare/v0.25.10...v0.25.11) (2021-01-04) +## [0.25.11](https://github.com/cube-js/cube/compare/v0.25.10...v0.25.11) (2021-01-04) **Note:** Version bump only for package @cubejs-docs/site @@ -283,41 +283,41 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.25.4](https://github.com/cube-js/cube.js/compare/v0.25.3...v0.25.4) (2020-12-30) +## [0.25.4](https://github.com/cube-js/cube/compare/v0.25.3...v0.25.4) (2020-12-30) ### Features -* **server-core:** Compatibility shim, for legacy imports ([2116799](https://github.com/cube-js/cube.js/commit/21167995045d7a5c0d1056dc034b14ec18205277)) -* **server-core:** Introduce CUBEJS_PRE_AGGREGATIONS_SCHEMA, use dev_preaggregations/prod_preaggregations by default ([e5bdf3d](https://github.com/cube-js/cube.js/commit/e5bdf3dfbd28d5e1c1e775c554c275304a0941f3)) +* **server-core:** Compatibility shim, for legacy imports ([2116799](https://github.com/cube-js/cube/commit/21167995045d7a5c0d1056dc034b14ec18205277)) +* **server-core:** Introduce CUBEJS_PRE_AGGREGATIONS_SCHEMA, use dev_preaggregations/prod_preaggregations by default ([e5bdf3d](https://github.com/cube-js/cube/commit/e5bdf3dfbd28d5e1c1e775c554c275304a0941f3)) -## [0.25.2](https://github.com/cube-js/cube.js/compare/v0.25.1...v0.25.2) (2020-12-27) +## [0.25.2](https://github.com/cube-js/cube/compare/v0.25.1...v0.25.2) (2020-12-27) ### Features -* Ability to set timeouts for polling in BigQuery/Athena ([#1675](https://github.com/cube-js/cube.js/issues/1675)) ([dc944b1](https://github.com/cube-js/cube.js/commit/dc944b1aaacc69dd74a9d9d31ceaf43e16d37ccd)), closes [#1672](https://github.com/cube-js/cube.js/issues/1672) +* Ability to set timeouts for polling in BigQuery/Athena ([#1675](https://github.com/cube-js/cube/issues/1675)) ([dc944b1](https://github.com/cube-js/cube/commit/dc944b1aaacc69dd74a9d9d31ceaf43e16d37ccd)), closes [#1672](https://github.com/cube-js/cube/issues/1672) -## [0.25.1](https://github.com/cube-js/cube.js/compare/v0.25.0...v0.25.1) (2020-12-24) +## [0.25.1](https://github.com/cube-js/cube/compare/v0.25.0...v0.25.1) (2020-12-24) ### Features -* **elasticsearch-driver:** Support CUBEJS_DB_ELASTIC_QUERY_FORMAT, Thanks [@dylman79](https://github.com/dylman79) ([a7460f5](https://github.com/cube-js/cube.js/commit/a7460f5d45dc7e9d96b65f6cc36df810a5b9312e)) +* **elasticsearch-driver:** Support CUBEJS_DB_ELASTIC_QUERY_FORMAT, Thanks [@dylman79](https://github.com/dylman79) ([a7460f5](https://github.com/cube-js/cube/commit/a7460f5d45dc7e9d96b65f6cc36df810a5b9312e)) -# [0.25.0](https://github.com/cube-js/cube.js/compare/v0.24.15...v0.25.0) (2020-12-21) +# [0.25.0](https://github.com/cube-js/cube/compare/v0.24.15...v0.25.0) (2020-12-21) **Note:** Version bump only for package @cubejs-docs/site @@ -325,18 +325,18 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.24.14](https://github.com/cube-js/cube.js/compare/v0.24.13...v0.24.14) (2020-12-19) +## [0.24.14](https://github.com/cube-js/cube/compare/v0.24.13...v0.24.14) (2020-12-19) ### Features -* Add HTTP Post to cubejs client core ([#1608](https://github.com/cube-js/cube.js/issues/1608)). Thanks to [@mnifakram](https://github.com/mnifakram)! ([1ebd6a0](https://github.com/cube-js/cube.js/commit/1ebd6a04ac97b31c6a51ef63bb1d4c040e524190)) +* Add HTTP Post to cubejs client core ([#1608](https://github.com/cube-js/cube/issues/1608)). Thanks to [@mnifakram](https://github.com/mnifakram)! ([1ebd6a0](https://github.com/cube-js/cube/commit/1ebd6a04ac97b31c6a51ef63bb1d4c040e524190)) -## [0.24.13](https://github.com/cube-js/cube.js/compare/v0.24.12...v0.24.13) (2020-12-18) +## [0.24.13](https://github.com/cube-js/cube/compare/v0.24.12...v0.24.13) (2020-12-18) **Note:** Version bump only for package @cubejs-docs/site @@ -344,7 +344,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.24.11](https://github.com/cube-js/cube.js/compare/v0.24.10...v0.24.11) (2020-12-17) +## [0.24.11](https://github.com/cube-js/cube/compare/v0.24.10...v0.24.11) (2020-12-17) **Note:** Version bump only for package @cubejs-docs/site @@ -352,7 +352,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline -## [0.24.9](https://github.com/cube-js/cube.js/compare/v0.24.8...v0.24.9) (2020-12-16) +## [0.24.9](https://github.com/cube-js/cube/compare/v0.24.8...v0.24.9) (2020-12-16) **Note:** Version bump only for package @cubejs-docs/site @@ -365,23 +365,23 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. -## [0.24.8](https://github.com/cube-js/cube.js/compare/v0.24.7...v0.24.8) (2020-12-15) +## [0.24.8](https://github.com/cube-js/cube/compare/v0.24.7...v0.24.8) (2020-12-15) ### Features - **@cubejs-client/core:** Added pivotConfig option to alias series with a - prefix ([#1594](https://github.com/cube-js/cube.js/issues/1594)). Thanks to + prefix ([#1594](https://github.com/cube-js/cube/issues/1594)). Thanks to @MattGson! - ([a3342f7](https://github.com/cube-js/cube.js/commit/a3342f7fd0389ce3ad0bc62686c0e787de25f411)) + ([a3342f7](https://github.com/cube-js/cube/commit/a3342f7fd0389ce3ad0bc62686c0e787de25f411)) -## [0.24.6](https://github.com/cube-js/cube.js/compare/v0.24.5...v0.24.6) (2020-12-13) +## [0.24.6](https://github.com/cube-js/cube/compare/v0.24.5...v0.24.6) (2020-12-13) **Note:** Version bump only for package @cubejs-docs/site -## [0.24.5](https://github.com/cube-js/cube.js/compare/v0.24.4...v0.24.5) (2020-12-09) +## [0.24.5](https://github.com/cube-js/cube/compare/v0.24.4...v0.24.5) (2020-12-09) **Note:** Version bump only for package @cubejs-docs/site -## [0.24.4](https://github.com/cube-js/cube.js/compare/v0.24.3...v0.24.4) (2020-12-07) +## [0.24.4](https://github.com/cube-js/cube/compare/v0.24.3...v0.24.4) (2020-12-07) **Note:** Version bump only for package @cubejs-docs/site diff --git a/docs/README.md b/docs/README.md index 9abe3c4890c1b..b37b9700fcc42 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,7 +9,7 @@ scan the `docs/content/` folder and generate a static HTML site. [link-gatsby]: https://www.gatsbyjs.com/ [link-docs-live]: https://cube.dev/docs -[link-docs-content]: https://github.com/cube-js/cube.js/tree/master/docs/content +[link-docs-content]: https://github.com/cube-js/cube/tree/master/docs/content ## Development diff --git a/docs/content/API-Reference/GraphQL-API-1.png b/docs/content/API-Reference/GraphQL-API-1.png deleted file mode 100644 index 39436d60fe0af..0000000000000 Binary files a/docs/content/API-Reference/GraphQL-API-1.png and /dev/null differ diff --git a/docs/content/API-Reference/GraphQL-API-2.png b/docs/content/API-Reference/GraphQL-API-2.png deleted file mode 100644 index 3b3a8d2ac31be..0000000000000 Binary files a/docs/content/API-Reference/GraphQL-API-2.png and /dev/null differ diff --git a/docs/content/API-Reference/GraphQL-API.mdx b/docs/content/API-Reference/GraphQL-API.mdx deleted file mode 100644 index 2c8961427cc4b..0000000000000 --- a/docs/content/API-Reference/GraphQL-API.mdx +++ /dev/null @@ -1,305 +0,0 @@ ---- -title: GraphQL API -permalink: /backend/graphql -category: API Reference -menuOrder: 4 ---- - - - -The Cube GraphQL API is currently in Preview, and there may be changes in a -future version. - - - -## Getting started - - - -The Cube GraphQL API does not currently support using variables from a GraphQL -client. Instead inline variables directly in the query. - - - -First, ensure you're running Cube v0.28.58 or later. Then start the project -locally in development mode, and navigate to -`http://localhost:4000/cubejs-api/graphql` in your browser. You should see the -GraphiQL interface: - -
- -
- -If you click the 'Docs' button in the top-right, you can explore the -introspected schema: - -
- -
- -As an example, let's use the `Orders` cube from the example eCommerce database: - -```js -cube(`Orders`, { - sql: `SELECT * FROM public.orders`, - - measures: { - count: { - type: `count`, - }, - }, - - dimensions: { - status: { - sql: `status`, - type: `string`, - }, - - createdAt: { - sql: `created_at`, - type: `time`, - }, - }, -}); -``` - -A GraphQL query to return the number of orders by status would look something -like this: - -```graphql -{ - cube { - orders { - count - status - createdAt { - day - } - } - } -} -``` - -The equivalent query to the REST API endpoint would look like this: - -```json -{ - "measures": ["Orders.count"], - "dimensions": ["Orders.status", "Orders.createdAt"], - "timeDimensions": [ - { - "dimension": "Orders.createdAt", - "granularity": "day" - } - ] -} -``` - -### <--{"id" : "Getting started"}--> Modifying time dimension granularity - -The granularity for a time dimension can easily be changed by specifying it in -the query: - -```graphql -{ - cube { - orders { - createdAt { - month - } - } - } -} -``` - -[Any supported granularity][ref-schema-ref-preagg-granularity] can be used. If -you prefer to not specify a granularity, then use `value`: - -```graphql -{ - cube { - orders { - createdAt { - value - } - } - } -} -``` - -[ref-schema-ref-preagg-granularity]: - /schema/reference/pre-aggregations#granularity - -### <--{"id" : "Getting started"}--> Specifying filters and ranges - -Filters can be set on the load query or on a specific cube. Specifying the -filter on the load query applies it to all cubes in the query. Filters can be -added to the query as follows: - -``` -query { - cube(limit: 100, offset: 50, timezone: "America/Los_Angeles") { - orders(orderBy: { createdAt: asc, count: desc }, where: {status: {equals: "completed"}}) { - count - status - createdAt - } - } -} -``` - -Some other differences between the JSON query filters and the GraphQL filters to -note: - -- `number` values are used for number types instead of strings -- The `notSet` filter is replaced by `{ set: false }` -- New `in` and `notIn` filters to check for multiple values -- `AND` and `OR` fields for boolean operators - -The GraphQL API supports `@skip` and `@include` directives too: - -``` -query GetOrders($byStatus: Boolean) { - cube(limit: 100, offset: 50, timezone: "America/Los_Angeles") { - orders(orderBy: { createdAt: asc, count: desc }, where: {status: {equals: "completed"}}) { - count - status @include(if: $byStatus) - createdAt - } - } -} -``` - -### <--{"id" : "Getting started"}--> Querying multiple cubes - -Using the same `Orders` cube as before, let's try and get the numbers of -products for each order status too. We can do this by adding the `Products` cube -to our query as follows: - -```graphql -{ - cube { - orders { - status - count - createdAt { - month - } - } - products { - count - } - } -} -``` - -## Reference - -### <--{"id" : "Reference"}--> cube - -``` -query { - cube [([cubeQueryArgs])] { - [([cubeArgs])] { - - } - } -} -``` - -| Key | Schema | Description | -| --------------- | ----------------------------------- | ------------------------------------------ | -| `cubeQueryArgs` | [`CubeQueryArgs`](#cube-query-args) | Options that apply to the entire query | -| `cubeArgs` | [`CubeArgs`](#cube-args) | Options that apply only to a specific cube | - -### <--{"id" : "Reference"}--> CubeQueryArgs - -| Key | Schema | Description | -| ------------ | ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | -| `where` | [`RootWhereInput`](#root-where-input) | Represents a SQL `WHERE` clause | -| `limit` | `Int` | A row limit for your query. The default value is `10000`. The maximum allowed limit is `50000` | -| `offset` | `Int` | The number of initial rows to be skipped for your query. The default value is `0` | -| `timezone` | `String` | The timezone to use for the query. The default value is `UTC` | -| `renewQuery` | `Boolean` | If `renewQuery` is set to `true`, Cube.js will renew all `refreshKey` for queries and query results in the foreground. The default value is `false` | - -### <--{"id" : "Reference"}--> RootWhereInput - -| Key | Schema | Description | -| ------------ | ---------------------------------------- | ----------- | -| `AND` | [`[RootWhereInput!]`](#root-where-input) | | -| `OR` | [`[RootWhereInput!]`](#root-where-input) | | -| `` | [`CubeWhereInput`](#cube-where-input) | | - -### <--{"id" : "Reference"}--> CubeArgs - -| Key | Schema | Description | -| --------- | ------------------------------------- | ----------- | -| `where` | [`CubeWhereInput`](#cube-where-input) | | -| `orderBy` | `CubeOrderByInput` | | - -### <--{"id" : "Reference"}--> CubeWhereInput - -| Key | Schema | Description | -| -------------- | ---------------------------------------- | ----------- | -| `AND` | [`[RootWhereInput!]`](#root-where-input) | | -| `OR` | [`[RootWhereInput!]`](#root-where-input) | | -| `` | [`Filter`](#filter) | | - -### <--{"id" : "Reference"}--> CubeOrderByInput - -| Key | Schema | Description | -| -------------- | ---------------------- | ----------- | -| `` | [`OrderBy`](#order-by) | | - -### <--{"id" : "Reference"}--> Filter - -[`DateTimeFilter`](#date-time-filter) | [`FloatFilter`](#float-time-filter) | -[`StringFilter`](#string-time-filter) - -### <--{"id" : "Reference"}--> DateTimeFilter - -| Key | Schema | Description | -| ---------------- | ---------- | ----------- | -| `equals` | `String` | | -| `notEquals` | `String` | | -| `in` | `[String]` | | -| `notIn` | `[String]` | | -| `inDateRange` | `[String]` | | -| `notInDateRange` | `[String]` | | -| `beforeDate` | `String` | | -| `afterDate` | `String` | | -| `set` | `Boolean` | | - -### <--{"id" : "Reference"}--> FloatFilter - -| Key | Schema | Description | -| ----------- | --------- | ----------- | -| `equals` | `Float` | | -| `notEquals` | `Float` | | -| `in` | `[Float]` | | -| `notIn` | `[Float]` | | -| `set` | `Boolean` | | - -### <--{"id" : "Reference"}--> StringFilter - -| Key | Schema | Description | -| ------------- | ---------- | ----------- | -| `equals` | `String` | | -| `notEquals` | `String` | | -| `in` | `[String]` | | -| `notIn` | `[String]` | | -| `contains` | `String` | | -| `notContains` | `String` | | -| `set` | `Boolean` | | - -### <--{"id" : "Reference"}--> OrderBy - -`asc` | `desc` diff --git a/docs/content/API-Reference/Query-Format.md b/docs/content/API-Reference/Query-Format.md deleted file mode 100644 index 964e192365600..0000000000000 --- a/docs/content/API-Reference/Query-Format.md +++ /dev/null @@ -1,560 +0,0 @@ ---- -title: Query Format -permalink: /query-format -category: API Reference -menuOrder: 2 ---- - -Cube Queries are plain JavaScript objects, describing an analytics query. The -basic elements of a query (query members) are `measures`, `dimensions`, and -`segments`. - -The query member format name is `CUBE_NAME.MEMBER_NAME`, for example the -dimension `email` in the Cube `Users` would have the name `Users.email`. - -In the case of dimension of type `time` granularity could be optionally added to -the name, in the following format `CUBE_NAME.TIME_DIMENSION_NAME.GRANULARITY`, -ex: `Stories.time.month`. - -Supported granularities: `second`, `minute`, `hour`, `day`, `week`, `month`, -`quarter` and `year`. - -The Cube.js client also accepts an array of queries. By default it will be -treated as a Data Blending query type. - -## Query Properties - -A Query has the following properties: - -- `measures`: An array of measures. -- `dimensions`: An array of dimensions. -- `filters`: An array of objects, describing filters. Learn about - [filters format](#filters-format). -- `timeDimensions`: A convenient way to specify a time dimension with a filter. - It is an array of objects in [timeDimension format.](#time-dimensions-format) -- `segments`: An array of segments. A segment is a named filter, created in the - Data Schema. -- `limit`: A row limit for your query. The default value is `10000`. The maximum - allowed limit is `50000`. -- `offset`: The number of initial rows to be skipped for your query. The default - value is `0`. -- `order`: An object, where the keys are measures or dimensions to order by and - their corresponding values are either `asc` or `desc`. The order of the fields - to order on is based on the order of the keys in the object. -- `timezone`: All time based calculations performed within Cube.js are - timezone-aware. This property is applied to all time dimensions during - aggregation and filtering. It isn't applied to the time dimension referenced - in a `dimensions` query property unless granularity or date filter is - specified. Using this property you can set your desired timezone in - [TZ Database Name](https://en.wikipedia.org/wiki/Tz_database) format, e.g.: - `America/Los_Angeles`. The default value is `UTC`. -- `renewQuery`: If `renewQuery` is set to `true`, Cube.js will renew all - [`refreshKey`][ref-schema-ref-preaggs-refreshkey] for queries and query - results in the foreground. However, if the - [`refreshKey`][ref-schema-ref-preaggs-refreshkey] (or - [`refreshKey.every`][ref-schema-ref-preaggs-refreshkey-every]) doesn't - indicate that there's a need for an update this setting has no effect. The - default value is `false`. - > **NOTE**: Cube.js provides only eventual consistency guarantee. Using a - > small [`refreshKey.every`][ref-schema-ref-preaggs-refreshkey-every] value - > together with `renewQuery` to achieve immediate consistency can lead to - > endless refresh loops and overall system instability. -- `ungrouped`: If `ungrouped` is set to `true` no `GROUP BY` statement will be - added to the query. Instead, the raw results after filtering and joining will - be returned without grouping. By default `ungrouped` queries require a primary - key as a dimension of every cube involved in the query for security purposes. - In case of `ungrouped` query measures will be rendered as underlying `sql` of - measures without aggregation and time dimensions will be truncated as usual - however not grouped by. - -```js -{ - measures: ['Stories.count'], - dimensions: ['Stories.category'], - filters: [{ - member: 'Stories.isDraft', - operator: 'equals', - values: ['No'] - }], - timeDimensions: [{ - dimension: 'Stories.time', - dateRange: ['2015-01-01', '2015-12-31'], - granularity: 'month' - }], - limit: 100, - offset: 50, - order: { - 'Stories.time': 'asc', - 'Stories.count': 'desc' - }, - timezone: 'America/Los_Angeles' -} -``` - -### <--{"id" : "Query Properties"}--> Default order - -If the `order` property is not specified in the query, Cube.js sorts results by -default using the following rules: - -- The first time dimension with granularity, ascending. If no time dimension - with granularity exists... -- The first measure, descending. If no measure exists... -- The first dimension, ascending. - -### <--{"id" : "Query Properties"}--> Alternative order format - -Also you can control the ordering of the `order` specification, Cube.js support -alternative order format - array of tuples: - -```js -{ - ..., - order: [ - ['Stories.time', 'asc'], - ['Stories.count', 'asc'] - ] - }, - ... -} -``` - -## Filters Format - -A filter is a Javascript object with the following properties: - -- `member`: Dimension or measure to be used in the filter, for example: - `Stories.isDraft`. See below on difference between filtering dimensions vs - filtering measures. -- `operator`: An operator to be used in the filter. Only some operators are - available for measures. For dimensions the available operators depend on the - type of the dimension. Please see the reference below for the full list of - available operators. -- `values`: An array of values for the filter. Values must be of type String. If - you need to pass a date, pass it as a string in `YYYY-MM-DD` format. - -#### Filtering Dimensions vs Filtering Measures - -Filters are applied differently to dimensions and measures. - -When you filter on a dimension, you are restricting the raw data before any -calculations are made. When you filter on a measure, you are restricting the -results after the measure has been calculated. - -## Filters Operators - -Only some operators are available for measures. For dimensions, the available -operators depend on the -[type of the dimension](/schema/reference/types-and-formats#types). - -### <--{"id" : "Filters Operators"}--> equals - -Use it when you need an exact match. It supports multiple values. - -- Applied to measures. -- Dimension types: `string`, `number`, `time`. - -```js -{ - member: "Users.country", - operator: "equals", - values: ["US", "Germany", "Israel"] -} -``` - -### <--{"id" : "Filters Operators"}--> notEquals - -The opposite operator of `equals`. It supports multiple values. - -- Applied to measures. -- Dimension types: `string`, `number`, `time`. - -```js -{ - member: "Users.country", - operator: "notEquals", - values: ["France"] -} -``` - -### <--{"id" : "Filters Operators"}--> contains - -The `contains` filter acts as a wildcard case insensitive `LIKE` operator. In -the majority of SQL backends it uses `ILIKE` operator with values being -surrounded by `%`. It supports multiple values. - -- Dimension types: `string`. - -```js -{ - member: "Posts.title", - operator: "contains", - values: ["serverless", "aws"] -} -``` - -### <--{"id" : "Filters Operators"}--> notContains - -The opposite operator of `contains`. It supports multiple values. - -- Dimension types: `string`. - -```js -{ - member: "Posts.title", - operator: "notContains", - values: ["ruby"] -} -``` - -### <--{"id" : "Filters Operators"}--> startsWith - -The `startsWith` filter acts as a case insensitive `LIKE` operator with a wildcard at the beginning. In -the majority of SQL backends, it uses the `ILIKE` operator with `%` at the start of each value. It supports multiple values. - -- Dimension types: `string`. - -```js -{ - member: "Posts.title", - operator: "startsWith", - values: ["ruby"] -} -``` - -### <--{"id" : "Filters Operators"}--> endsWith - -The `endsWith` filter acts as a case insensitive `LIKE` operator with a wildcard at the end. In -the majority of SQL backends, it uses the `ILIKE` operator with `%` at the end of each value. It supports multiple values. - -- Dimension types: `string`. - -```js -{ - member: "Posts.title", - operator: "endsWith", - values: ["ruby"] -} -``` - -### <--{"id" : "Filters Operators"}--> gt - -The `gt` operator means **greater than** and is used with measures or dimensions -of type `number`. - -- Applied to measures. -- Dimension types: `number`. - -```js -{ - member: "Posts.upvotesCount", - operator: "gt", - values: ["100"] -} -``` - -### <--{"id" : "Filters Operators"}--> gte - -The `gte` operator means **greater than or equal to** and is used with measures -or dimensions of type `number`. - -- Applied to measures. -- Dimension types: `number`. - -```js -{ - member: "Posts.upvotesCount", - operator: "gte", - values: ["100"] -} -``` - -### <--{"id" : "Filters Operators"}--> lt - -The `lt` operator means **less than** and is used with measures or dimensions of -type `number`. - -- Applied to measures. -- Dimension types: `number`. - -```js -{ - member: "Posts.upvotesCount", - operator: "lt", - values: ["10"] -} -``` - -### <--{"id" : "Filters Operators"}--> lte - -The `lte` operator means **less than or equal to** and is used with measures or -dimensions of type `number`. - -- Applied to measures. -- Dimension types: `number`. - -```js -{ - member: "Posts.upvotesCount", - operator: "lte", - values: ["10"] -} -``` - -### <--{"id" : "Filters Operators"}--> set - -Operator `set` checks whether the value of the member **is not** `NULL`. You -don't need to pass `values` for this operator. - -- Applied to measures. -- Dimension types: `number`, `string`, `time`. - -```js -{ - member: "Posts.authorName", - operator: "set" -} -``` - -### <--{"id" : "Filters Operators"}--> notSet - -An opposite to the `set` operator. It checks whether the value of the member -**is** `NULL`. You don't need to pass `values` for this operator. - -- Applied to measures. -- Dimension types: `number`, `string`, `time`. - -```js -{ - member: "Posts.authorName", - operator: "notSet" -} -``` - -### <--{"id" : "Filters Operators"}--> inDateRange - - - -From a pre-aggregation standpoint, `inDateRange` filter is applied as a generic filter. -All pre-aggregation granularity matching rules aren't applied in this case. -It feels like pre-aggregation isn't matched. -However, pre-aggregation is just missing the filtered time dimension in [dimensions][ref-schema-ref-preaggs-dimensions] list. -If you want date range filter to match [timeDimension][ref-schema-ref-preaggs-time-dimension] please use [timeDimensions](#time-dimensions-format) `dateRange` instead. - - - -The operator `inDateRange` is used to filter a time dimension into a specific -date range. The values must be an array of dates with the following format -'YYYY-MM-DD'. If only one date specified the filter would be set exactly to this -date. - -There is a convient way to use date filters with grouping - -[learn more about timeDimensions query property here](#time-dimensions-format) - -- Dimension types: `time`. - -```js -{ - member: "Posts.time", - operator: "inDateRange", - values: ['2015-01-01', '2015-12-31'] -} -``` - -### <--{"id" : "Filters Operators"}--> notInDateRange - - - -From a pre-aggregation standpoint, `notInDateRange` filter is applied as a generic filter. -All pre-aggregation granularity matching rules aren't applied in this case. -It feels like pre-aggregation isn't matched. -However, pre-aggregation is just missing the filtered time dimension in [dimensions][ref-schema-ref-preaggs-dimensions] list. - - - -An opposite operator to `inDateRange`, use it when you want to exclude specific -dates. The values format is the same as for `inDateRange`. - -- Dimension types: `time`. - -```js -{ - member: "Posts.time", - operator: "notInDateRange", - values: ['2015-01-01', '2015-12-31'] -} -``` - -### <--{"id" : "Filters Operators"}--> beforeDate - - - -From a pre-aggregation standpoint, `beforeDate` filter is applied as a generic filter. -All pre-aggregation granularity matching rules aren't applied in this case. -It feels like pre-aggregation isn't matched. -However, pre-aggregation is just missing the filtered time dimension in [dimensions][ref-schema-ref-preaggs-dimensions] list. - - - -Use it when you want to retreive all results before some specific date. The -values should be an array of one element in `YYYY-MM-DD` format. - -- Dimension types: `time`. - -```js -{ - member: "Posts.time", - operator: "beforeDate", - values: ['2015-01-01'] -} -``` - -### <--{"id" : "Filters Operators"}--> afterDate - - - -From a pre-aggregation standpoint, `afterDate` filter is applied as a generic filter. -All pre-aggregation granularity matching rules aren't applied in this case. -It feels like pre-aggregation isn't matched. -However, pre-aggregation is just missing the filtered time dimension in [dimensions][ref-schema-ref-preaggs-dimensions] list. - - - -The same as `beforeDate`, but is used to get all results after a specific date. - -- Dimension types: `time`. - -```js -{ - member: "Posts.time", - operator: "afterDate", - values: ['2015-01-01'] -} -``` - -## Boolean logical operators - -Filters can contain `or` and `and` logical operators. Logical operators have -only one of the following properties: - -- `or` An array with one or more filters or other logical operators -- `and` An array with one or more filters or other logical operators - -```js -{ - or: [ - { - member: 'visitors.source', - operator: 'equals', - values: ['some'], - }, - { - and: [ - { - member: 'visitors.source', - operator: 'equals', - values: ['other'], - }, - { - member: 'visitor_checkins.cardsCount', - operator: 'equals', - values: ['0'], - }, - ], - }, - ]; -} -``` - -> **Note:** You can not put dimensions and measures filters in the same logical -> operator. - -## Time Dimensions Format - -Since grouping and filtering by a time dimension is quite a common case, Cube.js -provides a convenient shortcut to pass a dimension and a filter as a -`timeDimension` property. - -- `dimension`: Time dimension name. -- `dateRange`: An array of dates with the following format `YYYY-MM-DD` or in - `YYYY-MM-DDTHH:mm:ss.SSS` format. Values should always be local and in query - `timezone`. Dates in `YYYY-MM-DD` format are also accepted. Such dates are - padded to the start and end of the day if used in start and end of date range - interval accordingly. If only one date is specified it's equivalent to passing - two of the same dates as a date range. You can also pass a string instead of - array with relative date range, for example: `last quarter`, `last 360 days`, - or `next 2 months`. -- `compareDateRange`: An array of date ranges to compare a measure change over - previous period -- `granularity`: A granularity for a time dimension. It supports the following - values `second`, `minute`, `hour`, `day`, `week`, `month`, `year`. If you pass - `null` to the granularity, Cube.js will only perform filtering by a specified - time dimension, without grouping. - -```js -{ - measures: ['Stories.count'], - timeDimensions: [{ - dimension: 'Stories.time', - dateRange: ['2015-01-01', '2015-12-31'], - granularity: 'month' - }] -} -``` - -You can use compare date range queries when you want to see, for example, how a -metric performed over a period in the past and how it performs now. You can pass -two or more date ranges where each of them is in the same format as a -`dateRange` - -```js -// ... -const resultSet = cubejsApi.load({ - measures: ['Stories.count'], - timeDimensions: [ - { - dimension: 'Stories.time', - compareDateRange: ['this week', ['2020-05-21', '2020-05-28']], - granularity: 'month', - }, - ], -}); -// ... -``` - -You can also set a relative `dateRange`, e.g. `today`, `yesterday`, `tomorrow`, -`last year`, `next month`, or `last 6 months`. - -```js -{ - measures: ['Stories.count'], - timeDimensions: [{ - dimension: 'Stories.time', - dateRange: 'last week', - granularity: 'day' - }] -} -``` - -Be aware that e.g. `Last 7 days` or `Next 2 weeks` do not include the current -date. If you need the current date also you can use `from N days ago to now` or -`from now to N days from now`. - -```js -{ - measures: ['Stories.count'], - timeDimensions: [{ - dimension: 'Stories.time', - dateRange: 'from 6 days ago to now', - granularity: 'day' - }] -} -``` - -[ref-schema-ref-preaggs-refreshkey]: - /schema/reference/pre-aggregations#parameters-refresh-key -[ref-schema-ref-preaggs-refreshkey-every]: - /schema/reference/pre-aggregations#parameters-refresh-key-every -[ref-schema-ref-preaggs-dimensions]: - /schema/reference/pre-aggregations#parameters-dimensions -[ref-schema-ref-preaggs-time-dimension]: - /schema/reference/pre-aggregations#parameters-time-dimension diff --git a/docs/content/API-Reference/REST-API.mdx b/docs/content/API-Reference/REST-API.mdx deleted file mode 100644 index 9272a0a0df2db..0000000000000 --- a/docs/content/API-Reference/REST-API.mdx +++ /dev/null @@ -1,434 +0,0 @@ ---- -title: REST API -permalink: /rest-api -category: API Reference -menuOrder: 1 ---- - -## Prerequisites - -### <--{"id" : "Prerequisites"}--> Base path - -REST API is used to communicate with Cube.js backend. All requests are prefixed -with **basePath** described in -[Backend Server Core](@cubejs-backend-server-core). By default it's -`/cubejs-api`. - -### <--{"id" : "Prerequisites"}--> Authentication - -Cube.js uses API tokens to authorize requests and also for passing additional -security context, which can be used in the -[`queryRewrite`][ref-config-queryrewrite] property in your [`cube.js` -configuration file][ref-config-js]. - -[ref-config-queryrewrite]: /config#query-rewrite -[ref-config-js]: /config - -The API Token is passed via the Authorization Header. The token itself is a -[JSON Web Token](https://jwt.io), the [Security section](security) describes how -to generate it. - -In the development environment the token is not required for authorization, but -you can still use it to pass a security context. - -### <--{"id" : "Prerequisites"}--> Example request - -```bash -curl -H "Authorization: EXAMPLE-API-TOKEN" https://example.com/cubejs-api/v1/sql -``` - -### <--{"id" : "Prerequisites"}--> Continue wait - -If the request takes too long to be processed, Cube.js Backend responds with -`{ "error": "Continue wait" }` and 200 status code. This is how the long polling -mechanism in Cube is implemented. Clients should continuously retry the same -query in a loop until they get a successful result. Subsequent calls to the Cube -endpoints are idempotent and don't lead to scheduling new database queries if -not required by `refreshKey`. Also, receiving `Continue wait` doesn't mean the -database query has been canceled, and it's actually still being processed by the -Cube. Database queries that weren't started and are no longer waited by the -client's long polling loop will be marked as orphaned and removed from the -querying queue. - -Possible reasons of **Continue wait**: - -- The query requested is heavy, and it takes some time for the database to - process it. Clients should wait for its completion, continuously sending the - same REST API request. [continueWaitTimeout](config#queue-options) can be - adjusted in order to change the time Cube waits before returning - `Continue wait` message. -- There are many queries requested and Cube.js backend queues them to save - database from overloading. - -### <--{"id" : "Prerequisites"}--> Error Handling - -Cube.js REST API has basic errors and HTTP Error codes for all requests. - -| Status | Error response | Description | -| ------ | ------------------------------ | ---------------------------------------------------------------------------------------------------- | -| 400 | Error message | General error. It may be a database error, timeout, or other issue. Check error message for details. | -| 403 | Authorization header isn't set | You didn't provide an auth token. Provide a valid API Token or disable authorization. | -| 403 | Invalid token | The auth token provided is not valid. It may be expired or have invalid signature. | -| 500 | Error message | Cube.js internal server error. Check error message for details. | - -### <--{"id" : "Prerequisites"}--> Request Span Annotation - -For monitoring tools such as Cube Cloud proper request span annotation should be -provided in `x-request-id` header of a request. Each request id should consist -of two parts: `spanId` and `requestSequenceId` which define `x-request-id` as -whole: `${spanId}-span-${requestSequenceId}`. Values of `x-request-id` header -should be unique for each separate request. `spanId` should define user -interaction span such us `Continue wait` retry cycle and it's value shouldn't -change during one single interaction. - -### <--{"id" : "Prerequisites"}--> Pagination - -Cube supports paginated requests for the `/v1/load` endpoint by including -[`limit` and `offset` parameters][ref-api-ref-query-format] in the query. For -example, the following query will retrieve rows 101-200 from the `Orders` cube: - -```json -{ - "dimensions": ["Orders.status"], - "measures": ["Orders.count"], - "timeDimensions": [ - { - "dimension": "Orders.createdAt", - "dateRange": "last year", - "granularity": "day" - } - ], - "limit": 100, - "offset": 100 -} -``` - -[ref-api-ref-query-format]: /query-format#query-properties - -## API Reference - -### <--{"id" : "API Reference"}--> /v1/load - -Get the data for a query. - -| Parameter | Description | -| --------- | ------------------------------------------ | -| query | URL encoded Cube.js [Query](/query-format) | - -Response - -- `query` - The query passed via params. It can be an array of queries and in - such case it will be treated as a [Data Blending](/recipes/data-blending) - query. -- `data` - Formatted dataset of query results. -- `annotation` - Metadata for query. Contains descriptions for all query items. - - `title` - Human readable title from data schema. - - `shortTitle` - Short title for visualization usage (ex. chart overlay) - - `type` - Data type - -Example request: - -```bash -# Request with http method GET -curl \ - -H "Authorization: EXAMPLE-API-TOKEN" \ - -G \ - --data-urlencode 'query={"measures":["Users.count"]}' \ - http://localhost:4000/cubejs-api/v1/load - -# Request with http method POST -# Use POST to fix problem with query length limits -curl \ - -X POST \ - -H "Content-Type: application/json" \ - -H "Authorization: EXAMPLE-API-TOKEN" \ - --data '{"query": {"measures":["Users.count"]}}' \ - http://localhost:4000/cubejs-api/v1/load -``` - -Example response: - -```javascript -{ - query:{ - measures:[ - "Users.count" - ], - filters:[], - timezone:"UTC", - dimensions:[], - timeDimensions:[] - }, - data:[ - { - "Users.count":"700" - } - ], - annotation:{ - measures:{ - "Users.count":{ - title:"Users Count", - shortTitle:"Count", - type:"number" - } - }, - dimensions:{}, - segments:{}, - timeDimensions:{} - } -} -``` - - - -Currently all fetched numericals are returned in the same format as driver -returns it without any additional processing. Most of drivers return numerical -values as strings instead of javascript integer or float to ensure there's no -loss of significance. Client code should take care of parsing such numerical -values. - - - -### <--{"id" : "API Reference"}--> /v1/sql - -Get the SQL Code generated by Cube.js to be executed in the database. - -| Parameter | Description | -| --------- | ----------------------------------------- | -| query | URLencoded Cube.js [Query](/query-format) | - -Response - -- `sql` - JSON Object with the following properties - - `sql` - Formatted SQL query with parameters - - `order` - Order fields and direction used in SQL query - - `cacheKeyQueries` - Key names and TTL of Cube.js data cache - - `preAggregations` - SQL queries used to build pre-aggregation tables - -Example request: - -```bash -curl \ - -H "Authorization: EXAMPLE-API-TOKEN" \ - -G \ - --data-urlencode 'query={"measures":["Users.count"], - "timeDimensions":[{"dimension": "Users.createdAt","granularity":"day","dateRange":["2019-03-01","2019-03-31"]}]}' \ - http://localhost:4000/cubejs-api/v1/sql -``` - -Example response: - -```javascript -{ - sql:{ - sql:[ - "SELECT\n date_trunc('day', (users.created_at::timestamptz AT TIME ZONE 'UTC')) \"users.created_at_date\", count(users.id) \"users.count\"\n FROM\n public.users AS users\n WHERE (users.created_at >= $1::timestamptz AND users.created_at <= $2::timestamptz) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000", - [ - "2019-03-01T00:00:00Z", - "2019-03-31T23:59:59Z" - ] - ], - timeDimensionAlias:"users.created_at_date", - timeDimensionField:"Users.createdAt", - order:[ - { - id:"Users.createdAt", - desc:false - } - ], - cacheKeyQueries:{ - queries:[ - [ - "select max(users.created_at) from public.users AS users", - [] - ] - ], - renewalThreshold:21600 - }, - preAggregations:[] - } -} -``` - -### <--{"id" : "API Reference"}--> /v1/meta - -Get meta-information for cubes defined in data schema - -Response - -- `cubes` - Array of cubes - - `name` - Codename of the cube - - `title` - Human-readable cube name - - `measures` - Array of measures defined within this cube - - `dimensions` - Array of dimensions defined within this cube - - `segments` - Array of segments defined within this cube - - `connectedComponent` - if it has the same value for two cubes, then there is - at least one join path between them. - -Example request: - -```bash -curl \ - -H "Authorization: EXAMPLE-API-TOKEN" \ - -G \ - http://localhost:4000/cubejs-api/v1/meta -``` - -Example response: - -```javascript -{ - cubes: [ - { - name: 'Users', - title: 'Users', - connectedComponent: 1, - measures: [ - { - name: 'Users.count', - title: 'Users Count', - shortTitle: 'Count', - aliasName: 'users.count', - type: 'number', - aggType: 'count', - drillMembers: ['Users.id', 'Users.city', 'Users.createdAt'], - }, - ], - dimensions: [ - { - name: 'Users.city', - title: 'Users City', - type: 'string', - aliasName: 'users.city', - shortTitle: 'City', - suggestFilterValues: true, - }, - ], - segments: [], - }, - ]; -} -``` - -### <--{"id" : "API Reference"}--> /v1/run-scheduled-refresh - -Trigger scheduled refresh run to refresh pre-aggregations. Use it in serverless -deployments. - - - -Single call to this API may be not enough to refresh everything is pending. This -call just populates queue with refresh workload and should be continuously -called until refresh jobs have completed; otherwise refresh jobs will be marked -as orphaned, and they will be removed from the queue. - - - -Learn more about -[scheduled refresh here](/caching/pre-aggregations/getting-started#keeping-pre-aggregations-up-to-date). - -| Parameter | Description | -| --------------- | ---------------------------------------------------------------------------- | -| queryingOptions | Optional URL encoded Cube.js [Query](/query-format) options such as timezone | - -Empty object response if scheduled successfully. - -Example request: - -```bash -curl \ - -H "Authorization: EXAMPLE-API-TOKEN" \ - -G \ - --data-urlencode 'queryingOptions={"timezone":"UTC"}' \ - http://localhost:4000/cubejs-api/v1/run-scheduled-refresh -``` - -### <--{"id" : "API Reference"}--> /readyz - -Returns the ready state of the deployment. - -**Single-tenant:** Ensures the orchestration layer is operational and tests the -connection to the default `dataSource`. - -**Multi-tenant:** Tests connections per-tenant. If no connections exist, it will -report as successful. - -Example request: - -```bash -curl -i http://localhost:4000/readyz -``` - -Successful example response: - -```bash -HTTP/1.1 200 OK -X-Powered-By: Express -Access-Control-Allow-Origin: * -Content-Type: application/json; charset=utf-8 -Content-Length: 19 -ETag: W/"13-MyluqxoYxC0tUxBeZCnbaWYVLhg" -Date: Mon, 18 Jan 2021 15:39:57 GMT -Connection: keep-alive -Keep-Alive: timeout=5 - -{"health":"HEALTH"} -``` - -Failure example response: - -```bash -HTTP/1.1 500 Internal Server Error -X-Powered-By: Express -Access-Control-Allow-Origin: * -Content-Type: application/json; charset=utf-8 -Content-Length: 19 -ETag: W/"13-MyluqxoYxC0tUxBeZCnbaWYVLhg" -Date: Mon, 18 Jan 2021 15:39:57 GMT -Connection: keep-alive -Keep-Alive: timeout=5 - -{"health":"DOWN"} -``` - -### <--{"id" : "API Reference"}--> /livez - -Returns the liveness state of the deployment. This is confirmed by testing any -existing connections to `dataSource`. If no connections exist, it will report as -successful. - -```bash -curl -i http://localhost:4000/livez -``` - -Successful example response: - -```bash -HTTP/1.1 200 OK -X-Powered-By: Express -Access-Control-Allow-Origin: * -Content-Type: application/json; charset=utf-8 -Content-Length: 19 -ETag: W/"13-MyluqxoYxC0tUxBeZCnbaWYVLhg" -Date: Mon, 18 Jan 2021 15:39:57 GMT -Connection: keep-alive -Keep-Alive: timeout=5 - -{"health":"HEALTH"} -``` - -Failure example response: - -```bash -HTTP/1.1 500 Internal Server Error -X-Powered-By: Express -Access-Control-Allow-Origin: * -Content-Type: application/json; charset=utf-8 -Content-Length: 19 -ETag: W/"13-MyluqxoYxC0tUxBeZCnbaWYVLhg" -Date: Mon, 18 Jan 2021 15:39:57 GMT -Connection: keep-alive -Keep-Alive: timeout=5 - -{"health":"DOWN"} -``` diff --git a/docs/content/API-Reference/SQL-API.mdx b/docs/content/API-Reference/SQL-API.mdx deleted file mode 100644 index a367718f37cbb..0000000000000 --- a/docs/content/API-Reference/SQL-API.mdx +++ /dev/null @@ -1,679 +0,0 @@ ---- -title: SQL API -permalink: /backend/sql -category: API Reference -menuOrder: 3 ---- - - - - This is documentation of Cube’s SQL API for the Postgres protocol. - The MySQL protocol used in the first versions of the Cube SQL API is no longer being developed and will be phased out soon. - Please consider migrating to the Postgres protocol. - - - -The Cube SQL API allows querying Cube via Postgres-compatible SQL. It enables the use -of BI applications, Python notebooks, reverse ETL tools, and other downstream tools on top of -Cube. - -### <--{"id" : "SQL API"}--> Supported Tools - -Cube SQL API has been tested with -- psql CLI -- Apache Superset -- Tableau Cloud -- Tableau Desktop with JDBC driver -- Power BI -- Metabase -- Google Data Studio -- Excel through Devart plugin -- Deepnote -- Hex -- Observable -- Streamlit -- Jupyter notebook -- Hightouch - -Please see [this GitHub issue](https://github.com/cube-js/cube.js/issues/3906) for the tools roadmap and to suggest and vote for tools of your interest. - -### <--{"id" : "SQL API"}--> Cube Cloud - -The first step to get started with the SQL API in Cube Cloud is to create a -deployment. You can follow this -[step-by-step guide on creating deployment within Cube Cloud](/cloud/getting-started/create). - -Once the deployment is ready, click **How to connect** link on the Overview page. -It will open a modal with instructions on different ways to connect to Cube. -Navigate to the SQL API tab and enable the SQL API. - -Once it is enabled, you should see a screen like the one below with your -connection credentials. - -
- -
- -Since the Cube SQL API is Postgres-compatible, please make sure to select **Postgres** as -a database type when connecting from BI tools. - -### <--{"id" : "SQL API"}--> Self-hosted Cube - -To enable the SQL API, we first need to add a new environment variable: - -```dotenv -CUBEJS_PG_SQL_PORT=5432 -``` - -If you're running Cube with Docker, remember to add a port mapping to the Cube -service for `CUBEJS_PG_SQL_PORT`. Docker compose example: - -```yaml -services: - cube_api: - ... - ports: - - 5432:5432 # Cube SQL API -``` - -Or running docker from command line: - -```bash -docker run -p 4000:4000 -p 5432:5432 \ - -v ${PWD}:/cube/conf \ - -e CUBEJS_DEV_MODE=true \ - -e CUBEJS_PG_SQL_PORT=5432 \ - cubejs/cube -``` - -Then, set Cube SQL credentials auth: - -``` -CUBEJS_SQL_USER=myusername -CUBEJS_SQL_PASSWORD=mypassword -``` - -Now, you can start your Cube instance and connect via the `psql` client with -provided credentials: - -```bash -psql -h 127.0.0.1 --port 5432 -U myusername --password -``` - -## Querying Fundamentals - -Under the hood, SQL API uses [Apache Datafusion](https://arrow.apache.org/datafusion/) as its SQL execution engine. -It's responsible for query planning and execution. -As the conversion process from SQL to Cube Query is ambiguous, additional step is done before the query is executed. -This step is called rewriting. -During this step, the query plan is being rewritten in a way a maximum number of Cube Queries can be detected within the given query plan. -Overall, rewriting is a seamless process. -There're some practical considerations that you should keep in mind while querying, though. - -### <--{"id" : "Querying Fundamentals"}--> Cube is a table - -In order to simplify interaction with data tools, every cube is represented as a table. -Measures, dimensions, and segments in this table are columns. -However, not all queries are possible on these tables. -In order to be valid, query against cube table should be: - -1. valid SQL statement and be compilable to the initial query plan by Datafusion; -2. one of the supported query types to the cube so it can be converted to Cube Query. - -Let's discuss some of the supported query types. - -### <--{"id" : "Querying Fundamentals"}--> Aggregated vs Non-aggregated queries - -There're two types of queries supported against cube tables: aggregated and non-aggregated. -Aggregated are those with `GROUP BY` statement, and non-aggregated are those without it. -Cube Queries issued to your database will always be aggregated, and it doesn't matter if you provide `GROUP BY` in a query or not. - -Whenever you use a non-aggregated query you need to provide only column names in SQL: - -``` -SELECT status, count FROM Orders -``` - -The same aggregated query should always aggregate measure columns using a corresponding aggregating function or special `MEASURE()` function: - - - - In cases where measure columns are not aggregated `Projection references non-aggregate values` error will be thrown. - It means there're columns that are neither in `GROUP BY` or aggregated. - It's a standard SQL `GROUP BY` operation consistency check enforced by SQL API as well. - - - -``` -SELECT status, SUM(count) FROM Orders GROUP BY 1 -SELECT status, MEASURE(count) FROM Orders GROUP BY 1 -``` - -### <--{"id" : "Querying Fundamentals"}--> Filtering - -Cube supports most of simple equality operators like `=`, `<>`, `<`, `<=`, `>`, `>=` as well as `IN` and `LIKE` operators. -Cube tries to push down all filters into Cube Query. -In some cases, SQL filters aren't available in Cube and can be done in a post-processing phase. -Time dimension filters will be converted to time dimension date ranges whenever it's possible. - -### <--{"id" : "Querying Fundamentals"}--> Ordering - -Cube tries to push down all `ORDER BY` statements into Cube Query. -If it can't be done ordering part would be done in a post-processing phase. -In case there're more than 50k rows in the result set, incorrect results can be received in this case. -Please use `EXPLAIN` in order to check if it's the case. - -### <--{"id" : "Querying Fundamentals"}--> Limit - -Limit push down is supported by Cube however, a limit over 50k can't be overridden. -In future versions, paging and streaming would be used to avoid this limitation. - -## Examples - -Consider the following schema. - -```js -cube(`Orders`, { - sql: `SELECT * FROM public.orders`, - - measures: { - count: { - type: `count`, - }, - }, - - dimensions: { - status: { - sql: `status`, - type: `string`, - }, - - created: { - sql: `created_at`, - type: `time`, - }, - }, -}); -``` - -It would be represented as table in SQL API with `count`, `status`, `created` -columns. - -To get the count of orders grouped by status we can run the following query. - -``` -cube=> SELECT count, status FROM Orders; - count | status --------+------------ - 15513 | completed - 14652 | processing - 13829 | shipped -(3 rows) -``` - -Cube will automatically apply the `GROUP BY` clause in case it is missing in the -query. We can also provide the `GROUP BY` statement to control how results are -grouped. In the following example we group orders by created month and also by -status within every month. - -``` -cube=> SELECT MEASURE(count), status, DATE_TRUNC('month', createdAt) date FROM Orders GROUP BY date, status ORDER BY date asc; - measure(Orders.count) | status | date ------------------------+------------+---------------------------- - 31 | shipped | 2016-01-01 00:00:00.000000 - 28 | completed | 2016-01-01 00:00:00.000000 - 28 | processing | 2016-01-01 00:00:00.000000 - 28 | shipped | 2016-02-01 00:00:00.000000 - 18 | processing | 2016-02-01 00:00:00.000000 - 28 | completed | 2016-02-01 00:00:00.000000 - 54 | processing | 2016-03-01 00:00:00.000000 - 57 | completed | 2016-03-01 00:00:00.000000 - 56 | shipped | 2016-03-01 00:00:00.000000 - 54 | shipped | 2016-04-01 00:00:00.000000 - 60 | completed | 2016-04-01 00:00:00.000000 - 43 | processing | 2016-04-01 00:00:00.000000 - 55 | shipped | 2016-05-01 00:00:00.000000 -``` - -### <--{"id" : "Examples"}--> Querying Dimensions - -Querying dimensions is straightforward, simply add any required fields to the -`SELECT` clause. - -```sql -cube=> SELECT status FROM Orders; - status ------------- - completed - processing - shipped -(3 rows) -``` - -### <--{"id" : "Examples"}--> Querying Measures - -Measures can similarly be queried through Cube SQL. - -Because measures are already aggregated in Cube there is no need to apply -aggregate functions to them in SQL API if you don't have a `GROUP BY` statement in query. - -``` -cube=> SELECT count FROM Orders; - count -------- - 43994 -(1 row) -``` - -Some of the BI systems or SQL constraints may require you to apply aggregate functions. To support -this Cube allows aggregate functions on measures as long as they match the type -of the measure. - -`count` measure in our example is of type `count`, It means we can apply -`COUNT()` aggregate function to it. The below query is similiar to the above -one. - -``` -cube=> SELECT COUNT(count) FROM Orders; - COUNT(Orders.count) ---------------------- - 43994 -(1 row) -``` - -There's also universal aggregate function `MEASURE()` that matches any measure type. - -``` -cube=> SELECT MEASURE(count) FROM Orders; - measure(Orders.count) ------------------------ - 43994 -(1 row) -``` - -Let's look at more measures types: - -```javascript -cube('Orders', { - ..., - - measures: { - count: { - type: `count`, - }, - distinctCount: { - sql: `id`, - type: `countDistinct`, - }, - approxDistinctCount: { - sql: `id`, - type: `countDistinctApprox`, - }, - minValue: { - sql: `min_value`, - type: `min` - }, - maxValue: { - sql: `max_value`, - type: `max` - }, - }, -}) -``` - -As we can see, we have a mix of measure types in the above schema. To query -them, we could use the following SQL statements: - -```sql ---- Both the following statements are equivalent -SELECT count FROM Orders -SELECT COUNT(*) FROM Orders - ---- Count distinct, and count distinct approx ---- Both the following statements are equivalent -SELECT distinctCount FROM Orders -SELECT COUNT(DISTINCT distinctCount) FROM Orders - ---- Both the following statements are equivalent -SELECT approxDistinctCount FROM Orders -SELECT COUNT(DISTINCT approxDistinctCount) FROM Orders - ---- Both the following statements are equivalent -SELECT minValue FROM Orders -SELECT MIN(minValue) FROM Orders - ---- Both the following statements are equivalent -SELECT maxValue FROM Orders -SELECT MAX(maxValue) FROM Orders -``` - -### <--{"id" : "Examples"}--> Querying Segments - -Any segments defined in a schema can also be used in Cube SQL queries. Looking -at the schema below, we have one segment `isCompleted`: - -```javascript -cube('Orders', { - ..., - - segments: { - isCompleted: { - sql: `${CUBE}.status = 'completed'`, - }, - }, -}); -``` - -Segments must be used as `boolean` types in Cube SQL queries: - -```sql -WHERE isCompleted = true -``` - -## Custom Authentication - -Cube can be configured with dynamic username & password verification system by -setting a [`checkSqlAuth()`][ref-config-check-sql-auth] function in the -`cube.js` configuration file. This function should verify username and return -object with password and security context. - -If password returned from this function matches provided in connection string -user will be authenticated with provided security context. - -```javascript -module.exports = { - checkSqlAuth: async (req, username) => { - if (username === 'fooUser') { - return { - password: 'mypassword', - securityContext: {}, - }; - } - - throw new Error('Incorrect user name or password'); - }, -}; -``` - -## Security Context (Row-Level Security) - -Cube's SQL API can also use the Security Context for -[Dynamic Schema Creation][ref-dynamic-schemas] or [`queryRewrite`][ref-config-queryrewrite] property in your [`cube.js` -configuration file][ref-config-js]. - -By default, the SQL API uses the current user's Security Context, but this behaviour can be modified so that certain users are allowed to switch. To do this, we must first define which user is allowed to change Security Context: - -First, you need to define what user is allowed to change security context: - -``` -CUBEJS_SQL_SUPER_USER=admin -``` - -If it's not enough for your case, you define your logic for check with `canSwitchSqlUser` property in your [`cube.js` -configuration file][ref-config-js]. - -You can change security context for specific query via virtual filter on: - -```sql -SELECT * FROM Orders WHERE __user = 'anotheruser'; -``` - -## Joins - -SQL API currently does not support `INNER JOIN`, `LEFT JOIN`, `RIGHT JOIN` and `FULL OUTER JOIN`. -We plan to support these types of joins in future releases. - -SQL API supports `CROSS JOIN` between cubes. -When `CROSS JOIN` is being processed by Cube, it generates proper joining conditions for the underlying data backend. - -For example, the following query joins `Orders` and `Products` tables under the hood by `product_id` in `Orders` and `id` in `Products` exactly the same way as the REST API query does: - -```sql -cube=> SELECT * FROM Orders CROSS JOIN Products LIMIT 5; - count | avgValue | totalValue | number | value | status | createdAt | completedAt | __user | count | name | description | createdAt | __user --------+----------+------------+--------+-------+-----------+----------------------------+----------------------------+--------+-------+---------------------------+------------------------------------+----------------------------+-------- - 1 | 20 | 20 | 40 | 20 | completed | 2020-10-26 00:00:00.000000 | 2020-11-07 00:00:00.000000 | | 1 | Incredible Fresh Chicken | Electronics Generic Fresh Computer | 2020-06-29 00:00:00.000000 | - 1 | 20 | 20 | 14 | 20 | completed | 2021-02-07 00:00:00.000000 | 2021-03-02 00:00:00.000000 | | 1 | Unbranded Wooden Mouse | Outdoors Incredible Rubber Car | 2019-07-16 00:00:00.000000 | - 1 | 20 | 20 | 23 | 20 | completed | 2022-07-23 00:00:00.000000 | 2022-08-11 00:00:00.000000 | | 1 | Handcrafted Plastic Chair | Electronics Sleek Rubber Tuna | 2021-02-27 00:00:00.000000 | - 1 | 20 | 20 | 86 | 20 | completed | 2023-04-19 00:00:00.000000 | 2023-04-25 00:00:00.000000 | | 1 | Practical Metal Chicken | Toys Awesome Frozen Chips | 2020-07-24 00:00:00.000000 | - 1 | 20 | 20 | 27 | 20 | completed | 2019-06-27 00:00:00.000000 | 2019-07-21 00:00:00.000000 | | 1 | Sleek Rubber Chair | Computers Refined Cotton Shirt | 2021-09-26 00:00:00.000000 | -(5 rows) -``` - -In the resulting query plan, you won't see any joins as you can't see those for REST API queries either: - -```sql -cube=> EXPLAIN SELECT * FROM Orders CROSS JOIN Products LIMIT 5; - plan_type | plan ----------------+----------------------------- - logical_plan | CubeScan: request={ + - | "measures": [ + - | "Orders.count", + - | "Orders.avgValue", + - | "Orders.totalValue", + - | "Orders.number", + - | "Products.count" + - | ], + - | "dimensions": [ + - | "Orders.value", + - | "Orders.status", + - | "Orders.createdAt", + - | "Orders.completedAt", + - | "Products.name", + - | "Products.description",+ - | "Products.createdAt" + - | ], + - | "segments": [], + - | "limit": 5 + - | } - physical_plan | CubeScanExecutionPlan + - | -(2 rows) -``` - -This feature allows you to `CROSS JOIN` cubes even with transitive joins only. - -Typically in tools that allow defining custom SQL datasets, you'd use joined tables as a dataset SQL. -For example: - -```sql -SELECT o.count as count, p.name as product_name, p.description as product_description FROM Orders o CROSS JOIN Products p -``` -Please note we use aliasing to avoid name clashing between cube members in a resulting data set. -In this case, wrapped SQL will be properly processed by Cube, pushing down all operations to Cube query: - -```sql -cube=> SELECT product_name, SUM(count) FROM ( - SELECT o.count as count, p.name as product_name, p.description as product_description FROM Orders o CROSS JOIN Products p -) joined -GROUP BY 1 -ORDER BY 2 DESC -LIMIT 5; - product_name | SUM(joined.count) ---------------------------+------------------- - Tasty Plastic Mouse | 121 - Intelligent Cotton Ball | 119 - Ergonomic Steel Tuna | 116 - Intelligent Rubber Pants | 116 - Generic Wooden Gloves | 116 -(5 rows) -``` - -We can see this by introspecting explain plan for this query: - -```sql -cube=> EXPLAIN SELECT product_name, SUM(count) FROM ( - SELECT o.count as count, p.name as product_name, p.description as product_description FROM Orders o CROSS JOIN Products p -) joined -GROUP BY 1 -ORDER BY 2 DESC -LIMIT 5; - plan_type | plan ----------------+----------------------- - logical_plan | CubeScan: request={ + - | "measures": [ + - | "Orders.count" + - | ], + - | "dimensions": [ + - | "Products.name" + - | ], + - | "segments": [], + - | "order": [ + - | [ + - | "Orders.count",+ - | "desc" + - | ] + - | ], + - | "limit": 5 + - | } - physical_plan | CubeScanExecutionPlan+ - | -(2 rows) -``` - -Please note even if `product_description` is in the inner selection, it isn't evaluated in the final query as it isn't used in any way. - -As an alternative to achieve joins it is also possible to define proxy dimension or measure inside the -Cube. - -```js -cube(`Orders`, { - sql: `SELECT * FROM public.orders`, - - joins: { - Users: { - relationship: `belongsTo`, - sql: `${CUBE}.user_id = ${Users}.id`, - }, - }, - - measures: { - count: { - type: `count`, - }, - }, - - dimensions: { - id: { - sql: `id`, - type: `number`, - primaryKey: true, - }, - - // this is proxy dimension - user_city: { - sql: `${Users.city}`, - type: `string`, - }, - }, -}); - -cube(`Users`, { - sql: `SELECT * FROM public.users`, - - measures: {}, - - dimensions: { - id: { - sql: `id`, - type: `number`, - primaryKey: true, - }, - - city: { - sql: `city`, - type: `string`, - }, - }, -}); -``` - -Now, it is possible to get orders count by users city with the following query. - -``` -cube=> SELECT count, user_city FROM Orders; - count | user_city --------+--------------- - 9524 | New York - 9408 | San Francisco - 6360 | Mountain View - 6262 | Seattle - 4393 | Los Angeles - 3183 | Chicago - 3060 | Austin - 1804 | Palo Alto -(8 rows) -``` - -## Limitations - -### <--{"id" : "Limitations"}--> Projection - -`SELECT` statements only support the following projections: - -**`*` for all dimensions:** - -```sql -SELECT * FROM Orders; -``` - -**A valid expression for a dimension or measure:** - -```sql -SELECT COUNT(*) FROM Orders; -``` - -**A valid expression as an alias:** - -```sql -SELECT COUNT(*) AS order_count FROM Orders; -``` - -### <--{"id" : "Limitations"}--> Selection - -Cube SQL supports most conditional checks for the `WHERE` clause. - -**Comparison operators:** - -```sql -WHERE price > 50 -WHERE price >= 50 AND <= 100 -``` - -**Boolean logic:** - -```sql -WHERE isPaid = true - AND isCompleted = false - OR isReviewed = false -``` - -**`IN` operator:**: - -```sql -WHERE status IN ('completed', 'shipped') -WHERE status NOT IN ('processing') -``` - -**`IS NULL`:** - -```sql -WHERE completedAt IS NULL -WHERE completedAt IS NOT NULL -``` - -**`LIKE`:** - -```sql -WHERE name LIKE 'joe' -WHERE name NOT LIKE 'bloggs' -``` - -[ref-config-check-sql-auth]: /config#check-sql-auth -[ref-config-queryrewrite]: /config#query-rewrite -[ref-config-js]: /config -[ref-dynamic-schemas]: /schema/dynamic-schema-creation diff --git a/docs/content/Auth/Overview.mdx b/docs/content/Auth/Overview.mdx index 99f28cbd8f944..e1b2c40d3b65d 100644 --- a/docs/content/Auth/Overview.mdx +++ b/docs/content/Auth/Overview.mdx @@ -6,29 +6,28 @@ category: Authentication & Authorization menuOrder: 1 --- -In Cube.js, authorization (or access control) is based on the **security -context**. The diagram below shows how it works during the request processing in -Cube.js: +In Cube, authorization (or access control) is based on the **security context**. +The diagram below shows how it works during the request processing in Cube:
-Authentication is handled outside of Cube.js. A typical use case would be: +Authentication is handled outside of Cube. A typical use case would be: -1. A web server serves an HTML page containing the Cube.js client, which needs - to communicate securely with the Cube.js API. +1. A web server serves an HTML page containing the Cube client, which needs to + communicate securely with the Cube API. 2. The web server should generate a JWT with an expiry to achieve this. The server could include the token in the HTML it serves or provide the token to the frontend via an XHR request, which is then stored it in local storage or a cookie. 3. The JavaScript client is initialized using this token, and includes it in - calls to the Cube.js API. -4. The token is received by Cube.js, and verified using any available JWKS (if + calls to the Cube API. +4. The token is received by Cube, and verified using any available JWKS (if configured) 5. Once decoded, the token claims are injected into the [security context][ref-sec-ctx]. @@ -42,9 +41,9 @@ can still use it to [pass a security context][ref-sec-ctx]. ## Generating JSON Web Tokens (JWT) -Authentication tokens are generated based on your API secret. Cube.js CLI -generates an API Secret when a project is scaffolded and saves this value in the -`.env` file as `CUBEJS_API_SECRET`. +Authentication tokens are generated based on your API secret. Cube CLI generates +an API Secret when a project is scaffolded and saves this value in the `.env` +file as `CUBEJS_API_SECRET`. You can generate two types of tokens: @@ -118,29 +117,20 @@ const cubejsApi = cubejs( ``` You can optionally store this token in local storage or in a cookie, so that you -can then use it to query the Cube.js API. +can then use it to query the Cube API. ## Using JSON Web Key Sets (JWKS) -Cube.js has out-of-the-box support for the following identity providers: - -- [Auth0][ref-jwt-auth0] -- [AWS Cognito][ref-jwt-aws-cognito] - -If you don't see your identity provider listed, please let us know with a post -under the [Ideas category on our Discourse forum][link-discourse-ideas]. +Looking for a guide on how to connect a specific identity provider? Check out +our recipes for using [Auth0][ref-recipe-auth0] or [AWS Cognito][ref-recipe-cognito] with Cube. -[link-discourse-ideas]: https://forum.cube.dev/c/ideas/12 -[ref-jwt-auth0]: /security/jwt/auth0 -[ref-jwt-aws-cognito]: /security/jwt/aws-cognito - ### <--{"id" : "Using JSON Web Key Sets (JWKS)"}--> Configuration -As mentioned previously, Cube.js supports verifying JWTs using industry-standard +As mentioned previously, Cube supports verifying JWTs using industry-standard JWKS. The JWKS can be provided either from a URL, or as a JSON object conforming to [JWK specification RFC 7517 Section 4][link-jwk-ref], encoded as a string. @@ -166,8 +156,8 @@ CUBEJS_JWT_KEY='' -When using a URL to fetch the JWKS, Cube.js will automatically cache the -response, re-use it and update if a key rotation has occurred. +When using a URL to fetch the JWKS, Cube will automatically cache the response, +re-use it and update if a key rotation has occurred. @@ -189,8 +179,8 @@ CUBEJS_JWK_URL='' ### <--{"id" : "Using JSON Web Key Sets (JWKS)"}--> Verifying claims -Cube.js can also verify the audience, subject and issuer claims in JWTs. -Similarly to JWK configuration, these can also be configured in the `cube.js` +Cube can also verify the audience, subject and issuer claims in JWTs. Similarly +to JWK configuration, these can also be configured in the `cube.js` configuration file: ```javascript @@ -213,7 +203,7 @@ CUBEJS_JWT_SUBJECT='' ### <--{"id" : "Using JSON Web Key Sets (JWKS)"}--> Custom claims namespace -Cube.js can also extract claims defined in custom namespaces. Simply specify the +Cube can also extract claims defined in custom namespaces. Simply specify the namespace in your `cube.js` configuration file: ```javascript @@ -226,10 +216,10 @@ module.exports = { ### <--{"id" : "Using JSON Web Key Sets (JWKS)"}--> Caching -Cube.js caches JWKS by default when +Cube caches JWKS by default when [`CUBEJS_JWK_URL` or `jwt.jwkUrl` is specified](#configuration). -- If the response contains a `Cache-Control` header, then Cube.js uses it to +- If the response contains a `Cache-Control` header, then Cube uses it to determine cache expiry. - The keys inside the JWKS are checked for expiry values and used for cache expiry. @@ -238,16 +228,16 @@ Cube.js caches JWKS by default when ## Custom authentication -Cube.js also allows you to provide your own JWT verification logic by setting a +Cube also allows you to provide your own JWT verification logic by setting a [`checkAuth()`][ref-config-check-auth] function in the `cube.js` configuration file. This function is expected to verify a JWT and assigns its' claims to the security context. -Previous versions of Cube.js allowed setting a `checkAuthMiddleware()` -parameter, which is now deprecated. We advise [migrating to a newer version of -Cube.js][ref-config-migrate-cubejs]. +Previous versions of Cube allowed setting a `checkAuthMiddleware()` parameter, +which is now deprecated. We advise [migrating to a newer version of +Cube][ref-config-migrate-cubejs]. @@ -274,4 +264,7 @@ module.exports = { [ref-config-check-auth]: /config#check-auth [ref-config-migrate-cubejs]: /configuration/overview#migration-from-express-to-docker-template +[ref-recipe-auth0]: /recipes/authn-with-auth0 +[ref-recipe-cognito]: /recipes/authn-with-aws-cognito [ref-sec-ctx]: /security/context +[link-slack]: https://slack.cube.dev/ diff --git a/docs/content/Auth/Security-Context.mdx b/docs/content/Auth/Security-Context.mdx index 7100199b3e41e..eafd54a9ea9f2 100644 --- a/docs/content/Auth/Security-Context.mdx +++ b/docs/content/Auth/Security-Context.mdx @@ -1,27 +1,27 @@ --- -title: Security Context +title: Security context permalink: /security/context category: Authentication & Authorization menuOrder: 2 --- Your authentication server issues JWTs to your client application, which, when -sent as part of the request, are verified and decoded by Cube.js to get security +sent as part of the request, are verified and decoded by Cube to get security context claims to evaluate access control rules. Inbound JWTs are decoded and verified using industry-standard [JSON Web Key Sets (JWKS)][link-auth0-jwks]. -For access control or authorization, Cube.js allows you to define granular -access control rules for every cube in your data schema. Cube.js uses both the -request and security context claims in the JWT token to generate a SQL query, -which includes row-level constraints from the access control rules. +For access control or authorization, Cube allows you to define granular access +control rules for every cube in your data model. Cube uses both the request and +security context claims in the JWT token to generate a SQL query, which includes +row-level constraints from the access control rules. -JWTs sent to Cube.js should be passed in the `Authorization: ` header to +JWTs sent to Cube should be passed in the `Authorization: ` header to authenticate requests. JWTs can also be used to pass additional information about the user, known as a **security context**. A security context is a verified set of claims about the -current user that the Cube.js server can use to ensure that users only have -access to the data that they are authorized to access. +current user that the Cube server can use to ensure that users only have access +to the data that they are authorized to access. It will be accessible as the [`securityContext`][ref-config-sec-ctx] property inside: @@ -38,11 +38,11 @@ with filters. For example, let's take the following query: ```json { - "dimensions": ["Orders.status"], - "measures": ["Orders.count", "Orders.total"], + "dimensions": ["orders.status"], + "measures": ["orders.count", "orders.total"], "timeDimensions": [ { - "dimension": "Orders.createdAt", + "dimension": "orders.createdAt", "dateRange": ["2015-01-01", "2015-12-31"], "granularity": "month" } @@ -63,10 +63,10 @@ be injected into the security context: -Cube.js expects the context to be an object. If you don't provide an object as -the JWT payload, you will receive the following error: +Cube expects the context to be an object. If you don't provide an object as the +JWT payload, you will receive the following error: -``` +```bash Cannot create proxy with a non-object as target or handler ``` @@ -84,7 +84,7 @@ module.exports = { } query.filters.push({ - member: 'Orders.userId', + member: 'orders.user_id', operator: 'equals', values: [securityContext.user_id], }); @@ -105,18 +105,18 @@ const cubejsToken = jwt.sign({ user_id: 42 }, CUBEJS_API_SECRET, { }); ``` -Using this token, we authorize our request to the Cube.js API by passing it in -the Authorization HTTP header. +Using this token, we authorize our request to the Cube API by passing it in the +Authorization HTTP header. -```bash +```bash{outputLines: 2-5} curl \ - -H "Authorization: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1Ijp7ImlkIjo0Mn0sImlhdCI6MTU1NjAyNTM1MiwiZXhwIjoxNTU4NjE3MzUyfQ._8QBL6nip6SkIrFzZzGq2nSF8URhl5BSSSGZYp7IJZ4" \ - -G \ - --data-urlencode 'query={"measures":["Orders.count"]}' \ - http://localhost:4000/cubejs-api/v1/load + -H "Authorization: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1Ijp7ImlkIjo0Mn0sImlhdCI6MTU1NjAyNTM1MiwiZXhwIjoxNTU4NjE3MzUyfQ._8QBL6nip6SkIrFzZzGq2nSF8URhl5BSSSGZYp7IJZ4" \ + -G \ + --data-urlencode 'query={"measures":["orders.count"]}' \ + http://localhost:4000/cubejs-api/v1/load ``` -And Cube.js will generate the following SQL: +And Cube will generate the following SQL: ```sql SELECT @@ -132,11 +132,11 @@ LIMIT 10000 In the example below `user_id`, `company_id`, `sub` and `iat` will be injected into the security context and will be accessible in both the [Security Context][ref-schema-sec-ctx] and [`COMPILE_CONTEXT`][ref-cubes-compile-ctx] -global variable in the Cube.js Data Schema. +global variable in the Cube data model. -`COMPILE_CONTEXT` is used by Cube.js at schema compilation time, which allows +`COMPILE_CONTEXT` is used by Cube at data model compilation time, which allows changing the underlying dataset completely; the Security Context is only used at query execution time, which simply filters the dataset with a `WHERE` clause. @@ -151,27 +151,37 @@ query execution time, which simply filters the dataset with a `WHERE` clause. } ``` -With the same JWT payload as before, we can modify schemas before they are -compiled. The following schema will ensure users only see results for their +With the same JWT payload as before, we can modify models before they are +compiled. The following cube will ensure users only see results for their `company_id` in a multi-tenant deployment: -```javascript -const { - securityContext: { company_id }, -} = COMPILE_CONTEXT; + + +```yaml +cubes: + - name: orders + sql_table: "{COMPILE_CONTEXT.security_context.company_id}.orders" + + measures: + - name: count + type: count +``` -cube(`Orders`, { - sql: `SELECT * FROM ${company_id}.orders`, +```javascript +cube(`orders`, { + sql_table: `${COMPILE_CONTEXT.security_context.company_id}.orders`, measures: { count: { - type: `count`, - }, - }, -}); + type: `count` + } + } +}) ``` -### <--{"id" : "Using COMPILE_CONTEXT"}--> Usage with Pre-Aggregations + + +### <--{"id" : "Using COMPILE_CONTEXT"}--> Usage with pre-aggregations To generate pre-aggregations that rely on `COMPILE_CONTEXT`, [configure `scheduledRefreshContexts` in your `cube.js` configuration @@ -186,7 +196,7 @@ build one from a JSON object. [link-auth0-jwks]: https://auth0.com/docs/tokens/json-web-tokens/json-web-key-sets -[link-multitenancy]: /multitenancy-setup +[link-multitenancy]: /config/multitenancy [ref-config-queryrewrite]: /config#query-rewrite [ref-config-sched-refresh]: /config#scheduled-refresh-contexts [ref-config-sec-ctx]: /config#security-context diff --git a/docs/content/Auth/auth0-01-new-app-01.png b/docs/content/Auth/auth0-01-new-app-01.png deleted file mode 100755 index 5d837d595ffdd..0000000000000 Binary files a/docs/content/Auth/auth0-01-new-app-01.png and /dev/null differ diff --git a/docs/content/Auth/auth0-01-new-app-02.png b/docs/content/Auth/auth0-01-new-app-02.png deleted file mode 100755 index 0f13cff192f46..0000000000000 Binary files a/docs/content/Auth/auth0-01-new-app-02.png and /dev/null differ diff --git a/docs/content/Auth/auth0-02-new-api-01.png b/docs/content/Auth/auth0-02-new-api-01.png deleted file mode 100755 index a06c2936e510e..0000000000000 Binary files a/docs/content/Auth/auth0-02-new-api-01.png and /dev/null differ diff --git a/docs/content/Auth/auth0-02-new-api-02.png b/docs/content/Auth/auth0-02-new-api-02.png deleted file mode 100755 index f31d30a50e859..0000000000000 Binary files a/docs/content/Auth/auth0-02-new-api-02.png and /dev/null differ diff --git a/docs/content/Auth/auth0-03-get-jwt-01.png b/docs/content/Auth/auth0-03-get-jwt-01.png deleted file mode 100755 index a316d1ac6a2a7..0000000000000 Binary files a/docs/content/Auth/auth0-03-get-jwt-01.png and /dev/null differ diff --git a/docs/content/Auth/auth0-03-get-jwt-02.png b/docs/content/Auth/auth0-03-get-jwt-02.png deleted file mode 100755 index 5124316b5a401..0000000000000 Binary files a/docs/content/Auth/auth0-03-get-jwt-02.png and /dev/null differ diff --git a/docs/content/Auth/auth0-03-get-jwt-03.png b/docs/content/Auth/auth0-03-get-jwt-03.png deleted file mode 100755 index 2dbfd338d740a..0000000000000 Binary files a/docs/content/Auth/auth0-03-get-jwt-03.png and /dev/null differ diff --git a/docs/content/Auth/auth0-04-dev-playground-01.png b/docs/content/Auth/auth0-04-dev-playground-01.png deleted file mode 100755 index a48289946a1b3..0000000000000 Binary files a/docs/content/Auth/auth0-04-dev-playground-01.png and /dev/null differ diff --git a/docs/content/Auth/auth0-04-dev-playground-02.png b/docs/content/Auth/auth0-04-dev-playground-02.png deleted file mode 100755 index ac57f87173623..0000000000000 Binary files a/docs/content/Auth/auth0-04-dev-playground-02.png and /dev/null differ diff --git a/docs/content/Auth/cognito-01-create-lambda-01.png b/docs/content/Auth/cognito-01-create-lambda-01.png deleted file mode 100755 index e7571999d4cf6..0000000000000 Binary files a/docs/content/Auth/cognito-01-create-lambda-01.png and /dev/null differ diff --git a/docs/content/Auth/cognito-01-create-lambda-02.png b/docs/content/Auth/cognito-01-create-lambda-02.png deleted file mode 100755 index a312dca54e274..0000000000000 Binary files a/docs/content/Auth/cognito-01-create-lambda-02.png and /dev/null differ diff --git a/docs/content/Auth/cognito-get-jwt-02.png b/docs/content/Auth/cognito-get-jwt-02.png deleted file mode 100755 index a8a954c6c1520..0000000000000 Binary files a/docs/content/Auth/cognito-get-jwt-02.png and /dev/null differ diff --git a/docs/content/Auth/cognito-get-jwt-03.png b/docs/content/Auth/cognito-get-jwt-03.png deleted file mode 100755 index a6ff2f346b41a..0000000000000 Binary files a/docs/content/Auth/cognito-get-jwt-03.png and /dev/null differ diff --git a/docs/content/Auth/cognito-get-jwt-04.png b/docs/content/Auth/cognito-get-jwt-04.png deleted file mode 100755 index da83d61b73a15..0000000000000 Binary files a/docs/content/Auth/cognito-get-jwt-04.png and /dev/null differ diff --git a/docs/content/Auth/cognito-get-jwt-05.png b/docs/content/Auth/cognito-get-jwt-05.png deleted file mode 100755 index 27bc99aaf22e9..0000000000000 Binary files a/docs/content/Auth/cognito-get-jwt-05.png and /dev/null differ diff --git a/docs/content/Caching/Getting-Started-Pre-Aggregations.md b/docs/content/Caching/Getting-Started-Pre-Aggregations.md deleted file mode 100644 index a730430ac7a56..0000000000000 --- a/docs/content/Caching/Getting-Started-Pre-Aggregations.md +++ /dev/null @@ -1,496 +0,0 @@ ---- -title: Getting Started with Pre-Aggregations -permalink: /caching/pre-aggregations/getting-started -category: Caching -menuOrder: 2 ---- - -Often at the beginning of an analytical application's lifecycle - when there is -a smaller dataset that queries execute over - the application works well and -delivers responses within acceptable thresholds. However, as the size of the -dataset grows, the time-to-response from a user's perspective can often suffer -quite heavily. This is true of both application and purpose-built data -warehousing solutions. - -This leaves us with a chicken-and-egg problem; application databases can deliver -low-latency responses with small-to-large datasets, but struggle with massive -analytical datasets; data warehousing solutions _usually_ make no guarantees -except to deliver a response, which means latency can vary wildly on a -query-to-query basis. - -| Database Type | Low Latency? | Massive Datasets? | -| ------------------------------ | ------------ | ----------------- | -| Application (Postgres/MySQL) | ✅ | ❌ | -| Analytical (BigQuery/Redshift) | ❌ | ✅ | - -Cube.js provides a solution to this problem: pre-aggregations. In layman's -terms, a pre-aggregation is a condensed version of the source data. It specifies -attributes from the source, which Cube.js uses to condense (or crunch) the data. -This simple yet powerful optimization can reduce the size of the dataset by -several orders of magnitude, and ensures subsequent queries can be served by the -same condensed dataset if any matching attributes are found. - -[Pre-aggregations are defined within each cube's data -schema][ref-schema-preaggs], and cubes can have as many pre-aggregations as they -require. The pre-aggregated data [can be stored either alongside the source data -in the same database, in an external database][ref-caching-preaggs-storage] that -is supported by Cube.js, [or in Cube Store, a dedicated pre-aggregation storage -layer][ref-caching-preaggs-cubestore]. - -## Pre-Aggregations without Time Dimension - -To illustrate pre-aggregations with an example, let's use a sample e-commerce -database. We have a schema representing all our `Orders`: - -```javascript -cube(`Orders`, { - sql: `SELECT * FROM public.orders`, - - measures: { - count: { - type: `count`, - drillMembers: [id, createdAt], - }, - }, - - dimensions: { - status: { - sql: `status`, - type: `string`, - }, - - id: { - sql: `id`, - type: `number`, - primaryKey: true, - }, - - completedAt: { - sql: `completed_at`, - type: `time`, - }, - }, -}); -``` - -Some sample data from this table might look like: - -| **id** | **status** | **completed_at** | -| ------ | ---------- | ----------------------- | -| 1 | completed | 2021-02-15T12:21:11.290 | -| 2 | completed | 2021-02-25T18:15:12.369 | -| 3 | shipped | 2021-03-15T20:40:57.404 | -| 4 | processing | 2021-03-13T10:30:21.360 | -| 5 | completed | 2021-03-10T18:25:32.109 | - -Our first requirement is to populate a dropdown in our front-end application -which shows all possible statuses. The Cube.js query to retrieve this -information might look something like: - -```json -{ - "dimensions": ["Orders.status"] -} -``` - -```javascript -cube(`Orders`, { - // Same content as before, but including the following: - preAggregations: { - orderStatuses: { - dimensions: [status], - }, - }, -}); -``` - -## Pre-Aggregations with Time Dimension - -Using the same schema as before, we are now finding that users frequently query -for the number of orders completed per day, and that this query is performing -poorly. This query might look something like: - -```json -{ - "measures": ["Orders.count"], - "timeDimensions": ["Orders.completedAt"] -} -``` - -In order to improve the performance of this query, we can add another -pre-aggregation definition to the `Orders` schema: - -```javascript -cube(`Orders`, { - // Same content as before, but including the following: - preAggregations: { - ordersByCompletedAt: { - measures: [count], - timeDimension: completedAt, - granularity: `month`, - }, - }, -}); -``` - -Note that we have added a `granularity` property with a value of `month` to this -definition. This allows Cube.js to aggregate the dataset to a single entry for -each month. - -The next time the API receives the same JSON query, Cube.js will build (if it -doesn't already exist) the pre-aggregated dataset, store it in the source -database server and use that dataset for any subsequent queries. A sample of the -data in this pre-aggregated dataset might look like: - -| **completed_at** | **count** | -| ----------------------- | --------- | -| 2021-02-01T00:00:00.000 | 2 | -| 2021-03-01T00:00:00.000 | 3 | - -## Keeping pre-aggregations up-to-date - -Pre-aggregations can become out-of-date or out-of-sync if the original dataset -changes. [Cube.js uses a refresh key to check the freshness of the -data][ref-caching-preaggs-refresh]; if a change in the refresh key is detected, -the pre-aggregations are rebuilt. These refreshes are performed in the -background as a scheduled process, unless configured otherwise. - -## Ensuring pre-aggregations are targeted by queries - -Cube.js selects the best available pre-aggregation based on the incoming queries -it receives via the API. The process for selection is summarized below: - -1. Are all measures of type `count`, `sum`, `min`, `max` or - `countDistinctApprox`? - -2. If yes, then check if - - - The pre-aggregation contains all dimensions, filter dimensions and leaf - measures from the query - - The measures aren't multiplied ([via a `hasMany` - relation][ref-schema-joins-hasmany]) - -3. If no, then check if - - - The query's time dimension granularity is set - - All query filter dimensions are included in query dimensions - - The pre-aggregation defines the **exact** set of dimensions and measures - used in the query - -You can find a complete flowchart [here][self-select-pre-agg]. - -### <--{"id" : "Ensuring pre-aggregations are targeted by queries"}--> Additivity - -So far, we've described pre-aggregations as aggregated versions of your existing -data. However, there are some rules that apply when Cube.js uses the -pre-aggregation. The **additivity** of fields specified in both the query and in -the pre-aggregation determines this. - -So what is additivity? Let's add another cube called `LineItems` to the previous -example to demonstrate. The `LineItems` **belong to** the `Orders` cube, and are -[joined][ref-schema-joins] as such: - -```javascript -cube(`LineItems`, { - sql: `SELECT * FROM public.line_items`, - - joins: { - Orders: { - sql: `${CUBE}.order_id = ${Orders}.id`, - relationship: `belongsTo`, - }, - }, - - measures: { - count: { - type: `count`, - drillMembers: [id, createdAt], - }, - }, - - dimensions: { - id: { - sql: `id`, - type: `number`, - primaryKey: true, - }, - - createdAt: { - sql: `created_at`, - type: `time`, - }, - }, -}); -``` - -Some sample data from the `line_items` table might look like: - -| **id** | **product_id** | **order_id** | **quantity** | **price** | **profit_margin** | **created_at** | -| ------ | -------------- | ------------ | ------------ | --------- | ----------------- | -------------------------- | -| 1 | 31 | 1 | 1 | 275 | 1 | 2021-01-20 00:00:00.000000 | -| 2 | 49 | 2 | 6 | 248 | 0.1 | 2021-01-20 00:00:00.000000 | -| 3 | 89 | 3 | 6 | 197 | 0.35 | 2021-01-21 00:00:00.000000 | -| 4 | 71 | 4 | 8 | 223 | 0.15 | 2021-01-21 00:00:00.000000 | -| 5 | 64 | 5 | 5 | 75 | 0.75 | 2021-01-22 00:00:00.000000 | -| 6 | 62 | 6 | 8 | 75 | 0.65 | 2021-01-22 00:00:00.000000 | - -Looking at the raw data, we can see that if the data were to be aggregated by -`created_at`, then we could simply add together the `quantity` and `price` -fields and still get a correct result: - -| **created_at** | **quantity** | **price** | -| -------------------------- | ------------ | --------- | -| 2021-01-20 00:00:00.000000 | 7 | 523 | -| 2021-01-21 00:00:00.000000 | 14 | 420 | -| 2021-01-22 00:00:00.000000 | 13 | 150 | - -This means that `quantity` and `price` are both **additive measures**, and we -can represent them in the `LineItems` schema as follows: - -```javascript -cube(`LineItems`, { - ..., - measures: { - ..., - quantity: { - sql: `quantity`, - type: `sum`, - }, - price: { - type: `sum`, - sql: `price`, - format: `currency`, - }, - }, - ..., -}); -``` - -Because neither `quantity` and `price` reference any other measures in our -`LineItems` cube, we can also say that they are **additive leaf measures**. Any -query requesting only these two measures can be called a **leaf measure -additive** query. Additive leaf measures can only be of the following -[types][ref-schema-types-measure]: `count`, `sum`, `min`, `max` or -`countDistinctApprox`. - -[ref-schema-types-measure]: /types-and-formats#measures-types - -### <--{"id" : "Ensuring pre-aggregations are targeted by queries"}--> Non-Additivity - -Using the same sample data for `line_items`, there's a `profit_margin` field -which is different for each row. However, despite the value being numerical, it -doesn't actually make sense to add up this value. Let's look at the rows for -`2021-01-20` in the sample data: - -| **id** | **product_id** | **order_id** | **quantity** | **price** | **profit_margin** | **created_at** | -| ------ | -------------- | ------------ | ------------ | --------- | ----------------- | -------------------------- | -| 1 | 31 | 1 | 1 | 275 | 1 | 2021-01-20 00:00:00.000000 | -| 2 | 49 | 2 | 6 | 248 | 0.1 | 2021-01-20 00:00:00.000000 | - -And now let's try and aggregate them: - -| **created_at** | **quantity** | **price** | **profit_margin** | -| -------------------------- | ------------ | --------- | ----------------- | -| 2021-01-20 00:00:00.000000 | 7 | 523 | 1.1 | - -Using the source data, we'll manually calculate the profit margin and see if it -matches the above. We'll use the following formula: - -$$ -x + (x * y) = z -$$ - -Where `x` is the original cost of the item, `y` is the profit margin and `z` is -the price the item was sold for. Let's use the formula to find the original cost -for both items sold on `2021-01-20`. For the row with `id = 1`: - -$$ -x + (x * 1) = 275\\ -2x = 275\\ -x = 275 / 2\\ -x = 137.5 -$$ - -And for the row where `id = 2`: - -$$ -x + (x * 0.1) = 248\\ -1.1x = 248\\ -x = 248 / 1.1\\ -x = 225.454545454545455 -$$ - -Which means the total cost for both items was: - -$$ -225.454545454545455 + 137.5\\ -362.954545454545455 -$$ - -Now that we have the cost of each item, let's use the same formula in reverse to -see if applying a profit margin of `1.1` will give us the same total price -(`523`) as calculated earlier: - -$$ -362.954545454545455 + (362.954545454545455 * 1.1) = z\\ -762.204545454545455 = z\\ -z = 762.204545454545455 -$$ - -We can clearly see that `523` **does not** equal `762.204545454545455`, and we -cannot treat the `profit_margin` column the same as we would any other additive -measure. Armed with the above knowledge, we can add the `profit_margin` field to -our schema **as a [dimension][ref-schema-dims]**: - -```javascript -cube(`LineItems`, { - ..., - dimensions: { - ..., - profitMargin: { - sql: `profit_margin`, - type: `number`, - format: 'percentage', - }, - }, - ..., -}); -``` - -Another approach might be to calculate the profit margin dynamically, and -instead saving the "cost" price. Because the cost price is an additive measure, -we are able to store it in a pre-aggregation: - -```javascript -cube(`LineItems`, { - ..., - measures: { - ..., - cost: { - sql: `${CUBE.price} / (1 + ${CUBE.profitMargin})`, - type: `sum`, - }, - }, - ..., -}); -``` - -Another example of a non-additive measure would be a distinct count of -`product_id`. If we took the distinct count of products sold over a month, and -then tried to sum the distinct count of products for each individual day and -compared them, we would not get the same results. We can add the measure like -this: - -```javascript -cube(`LineItems`, { - ..., - measures: { - ..., - countDistinctProducts: { - sql: `product_id`, - type: `countDistinct`, - }, - }, - ..., -}); -``` - -However the above cannot be used in for a pre-aggregation. We can instead change -the `type` to `countDistinctApprox`, and then use the measure in a -pre-aggregation definition: - -```javascript -cube(`LineItems`, { - ..., - measures: { - ..., - countDistinctProducts: { - sql: `product_id`, - type: `countDistinctApprox`, - }, - }, - preAggregations: { - myRollup: { - ..., - measures: [ CUBE.countDistinctProducts ], - } - }, - ..., -}); -``` - -### <--{"id" : "Ensuring pre-aggregations are targeted by queries"}--> Selecting the pre-aggregation - -To recap what we've learnt so far: - -- **Additive measures** are measures whose values can be added together - -- **Multiplied measures** are measures that define `hasMany` relations - -- **Leaf measures** are measures that do not reference any other measures in - their definition - -- **Calculated measures** are measures that reference other dimensions and - measures in their definition - -- A query is **leaf measure additive** if all of its leaf measures are one of: - `count`, `sum`, `min`, `max` or `countDistinctApprox` - -Cube looks for matching pre-aggregations in the order they are defined in a -cube's schema file. Each defined pre-aggregation is then tested for a match -based on the criteria in the flowchart below: - -
- Pre-Aggregation Selection Flowchart -
- -Some extra considerations for pre-aggregation selection: - -- The query's time dimension and granularity must match the pre-aggregation. - -- The query's time dimension and granularity together act as a dimension. If the - date range isn't aligned with granularity, a common granularity is used. This - common granularity is selected using the [greatest common divisor][wiki-gcd] - across both the query and pre-aggregation. For example, the common granularity - between `hour` and `day` is `hour` because both `hour` and `day` can be - divided by `hour`. - -- The query's granularity's date range must match the start date and end date - from the time dimensions. For example, when using a granularity of `month`, - the values should be the start and end days of the month i.e. - `['2020-01-01T00:00:00.000', '2020-01-31T23:59:59.999']`; when the granularity - is `day`, the values should be the start and end hours of the day i.e. - `['2020-01-01T00:00:00.000', '2020-01-01T23:59:59.999']`. Date ranges are - inclusive, and the minimum granularity is `second`. - -- The order in which pre-aggregations are defined in schemas matter; the first - matching pre-aggregation for a query is the one that is used. Both the - measures and dimensions of any cubes specified in the query are checked to - find a matching `rollup`. - -- `rollup` pre-aggregations **always** have priority over `originalSql`. Thus, - if you have both `originalSql` and `rollup` defined, Cube.js will try to match - `rollup` pre-aggregations before trying to match `originalSql`. You can - instruct Cube.js to use the original SQL pre-aggregations by using - [`useOriginalSqlPreAggregations`][ref-schema-preaggs-origsql]. - -[ref-caching-preaggs-cubestore]: - /caching/using-pre-aggregations#pre-aggregations-storage -[ref-caching-preaggs-refresh]: /caching/using-pre-aggregations#refresh-strategy -[ref-caching-preaggs-storage]: - /caching/using-pre-aggregations#pre-aggregations-storage -[ref-schema-dims]: /schema/reference/dimensions -[ref-schema-joins]: /schema/reference/joins -[ref-schema-joins-hasmany]: /schema/reference/joins#relationship -[ref-schema-preaggs]: /schema/reference/pre-aggregations -[ref-schema-preaggs-origsql]: - /schema/reference/pre-aggregations#type-originalsql -[self-select-pre-agg]: #selecting-the-pre-aggregation -[wiki-gcd]: https://en.wikipedia.org/wiki/Greatest_common_divisor diff --git a/docs/content/Caching/Getting-Started-Pre-Aggregations.mdx b/docs/content/Caching/Getting-Started-Pre-Aggregations.mdx new file mode 100644 index 0000000000000..2b77155084010 --- /dev/null +++ b/docs/content/Caching/Getting-Started-Pre-Aggregations.mdx @@ -0,0 +1,711 @@ +--- +title: Getting started with pre-aggregations +permalink: /caching/pre-aggregations/getting-started +category: Caching +menuOrder: 2 +--- + +Often at the beginning of an analytical application's lifecycle - when there is +a smaller dataset that queries execute over - the application works well and +delivers responses within acceptable thresholds. However, as the size of the +dataset grows, the time-to-response from a user's perspective can often suffer +quite heavily. This is true of both application and purpose-built data +warehousing solutions. + +This leaves us with a chicken-and-egg problem; application databases can deliver +low-latency responses with small-to-large datasets, but struggle with massive +analytical datasets; data warehousing solutions _usually_ make no guarantees +except to deliver a response, which means latency can vary wildly on a +query-to-query basis. + +| Database Type | Low Latency? | Massive Datasets? | +| ------------------------------ | ------------ | ----------------- | +| Application (Postgres/MySQL) | ✅ | ❌ | +| Analytical (BigQuery/Redshift) | ❌ | ✅ | + +Cube provides a solution to this problem: pre-aggregations. In layman's terms, a +pre-aggregation is a condensed version of the source data. It specifies +attributes from the source, which Cube uses to condense (or crunch) the data. +This simple yet powerful optimization can reduce the size of the dataset by +several orders of magnitude, and ensures subsequent queries can be served by the +same condensed dataset if any matching attributes are found. + +[Pre-aggregations are defined within each cube's data +schema][ref-schema-preaggs], and cubes can have as many pre-aggregations as they +require. The pre-aggregated data [can be stored either alongside the source data +in the same database, in an external database][ref-caching-preaggs-storage] that +is supported by Cube, [or in Cube Store, a dedicated pre-aggregation storage +layer][ref-caching-preaggs-cubestore]. + +## Pre-Aggregations without Time Dimension + +To illustrate pre-aggregations with an example, let's use a sample e-commerce +database. We have a data model representing all our `orders`: + + + +```yaml +cubes: + - name: orders + sql_table: orders + + measures: + - name: count + type: count + + dimensions: + - name: id + sql: id + type: number + primary_key: true + + - name: status + sql: status + type: string + + - name: completed_at + sql: completed_at + type: time +``` + +```javascript +cube(`orders`, { + sql_table: `orders`, + + measures: { + count: { + type: `count`, + }, + }, + + dimensions: { + id: { + sql: `id`, + type: `number`, + primary_key: true, + }, + + status: { + sql: `status`, + type: `string`, + }, + + completed_at: { + sql: `completed_at`, + type: `time`, + }, + }, +}); +``` + + + +Some sample data from this table might look like: + +| **id** | **status** | **completed_at** | +| ------ | ---------- | ----------------------- | +| 1 | completed | 2021-02-15T12:21:11.290 | +| 2 | completed | 2021-02-25T18:15:12.369 | +| 3 | shipped | 2021-03-15T20:40:57.404 | +| 4 | processing | 2021-03-13T10:30:21.360 | +| 5 | completed | 2021-03-10T18:25:32.109 | + +Our first requirement is to populate a dropdown in our front-end application +which shows all possible statuses. The Cube query to retrieve this information +might look something like: + +```json +{ + "dimensions": ["orders.status"] +} +``` + +In that case, we can add the following pre-aggregation to the `orders` cube: + + + +```yaml +cubes: + - name: orders + # ... + + pre_aggregations: + - name: order_statuses + dimensions: + - status +``` + +```javascript +cube(`orders`, { + // ... + + pre_aggregations: { + order_statuses: { + dimensions: [status], + }, + }, +}); +``` + + + +## Pre-Aggregations with Time Dimension + +Using the same data model as before, we are now finding that users frequently +query for the number of orders completed per day, and that this query is +performing poorly. This query might look something like: + +```json +{ + "measures": ["orders.count"], + "timeDimensions": ["orders.completed_at"] +} +``` + +In order to improve the performance of this query, we can add another +pre-aggregation definition to the `orders` cube: + + + +```yaml +cubes: + - name: orders + # ... + + pre_aggregations: + - name: orders_by_completed_at + measures: + - count + time_dimension: completed_at + granularity: month +``` + +```javascript +cube(`orders`, { + // ... + + pre_aggregations: { + orders_by_completed_at: { + measures: [count], + time_dimension: completed_at, + granularity: `month`, + }, + }, +}); +``` + + + +Note that we have added a `granularity` property with a value of `month` to this +definition. This allows Cube to aggregate the dataset to a single entry for each +month. + +The next time the API receives the same JSON query, Cube will build (if it +doesn't already exist) the pre-aggregated dataset, store it in the source +database server and use that dataset for any subsequent queries. A sample of the +data in this pre-aggregated dataset might look like: + +| **completed_at** | **count** | +| ----------------------- | --------- | +| 2021-02-01T00:00:00.000 | 2 | +| 2021-03-01T00:00:00.000 | 3 | + +## Keeping pre-aggregations up-to-date + +Pre-aggregations can become out-of-date or out-of-sync if the original dataset +changes. [Cube uses a refresh key to check the freshness of the +data][ref-caching-preaggs-refresh]; if a change in the refresh key is detected, +the pre-aggregations are rebuilt. These refreshes are performed in the +background as a scheduled process, unless configured otherwise. + +## Ensuring pre-aggregations are targeted by queries + +Cube selects the best available pre-aggregation based on the incoming queries it +receives via the API. The process for selection is summarized below: + +1. Are all measures of type `count`, `sum`, `min`, `max` or + `count_distinct_approx`? + +2. If yes, then check if + + - The pre-aggregation contains all dimensions, filter dimensions and leaf + measures from the query + - The measures aren't multiplied ([via a `one_to_many` + relationship][ref-schema-joins-rel]) + +3. If no, then check if + + - The query's time dimension granularity is set + - All query filter dimensions are included in query dimensions + - The pre-aggregation defines the **exact** set of dimensions and measures + used in the query + +You can find a complete flowchart [here][self-select-pre-agg]. + +### <--{"id" : "Ensuring pre-aggregations are targeted by queries"}--> Additivity + +So far, we've described pre-aggregations as aggregated versions of your existing +data. However, there are some rules that apply when Cube uses the +pre-aggregation. The **additivity** of fields specified in both the query and in +the pre-aggregation determines this. + +So what is additivity? Let's add another cube called `line_items` to the previous +example to demonstrate. Many `line_items` can belong to any order from the `orders` cube, and are +[joined][ref-schema-joins] as such: + + + +```yaml +cubes: + - name: line_items + sql_table: line_items + + joins: + - name: orders + sql: "{CUBE}.order_id = {orders.id}" + relationship: many_to_one + + measures: + - name: count + type: count + + dimensions: + - name: id + sql: id + type: number + primary_key: true + + - name: created_at + sql: created_at + type: time +``` + +```javascript +cube(`line_items`, { + sql_table: `line_items`, + + joins: { + orders: { + sql: `${CUBE}.order_id = ${orders.id}`, + relationship: `many_to_one`, + }, + }, + + measures: { + count: { + type: `count`, + }, + }, + + dimensions: { + id: { + sql: `id`, + type: `number`, + primary_key: true, + }, + + created_at: { + sql: `created_at`, + type: `time`, + }, + }, +}); +``` + + + +Some sample data from the `line_items` table might look like: + +| **id** | **product_id** | **order_id** | **quantity** | **price** | **profit_margin** | **created_at** | +| ------ | -------------- | ------------ | ------------ | --------- | ----------------- | -------------------------- | +| 1 | 31 | 1 | 1 | 275 | 1 | 2021-01-20 00:00:00.000000 | +| 2 | 49 | 2 | 6 | 248 | 0.1 | 2021-01-20 00:00:00.000000 | +| 3 | 89 | 3 | 6 | 197 | 0.35 | 2021-01-21 00:00:00.000000 | +| 4 | 71 | 4 | 8 | 223 | 0.15 | 2021-01-21 00:00:00.000000 | +| 5 | 64 | 5 | 5 | 75 | 0.75 | 2021-01-22 00:00:00.000000 | +| 6 | 62 | 6 | 8 | 75 | 0.65 | 2021-01-22 00:00:00.000000 | + +Looking at the raw data, we can see that if the data were to be aggregated by +`created_at`, then we could simply add together the `quantity` and `price` +fields and still get a correct result: + +| **created_at** | **quantity** | **price** | +| -------------------------- | ------------ | --------- | +| 2021-01-20 00:00:00.000000 | 7 | 523 | +| 2021-01-21 00:00:00.000000 | 14 | 420 | +| 2021-01-22 00:00:00.000000 | 13 | 150 | + +This means that `quantity` and `price` are both **additive measures**, and we +can represent them in the `line_items` cube as follows: + + + +```yaml +cubes: + - name: line_items + # ... + + measures: + # ... + + - name: quantity + sql: quantity + type: sum + + - name: price + type: sum + sql: price + format: currency + + # ... +``` + +```javascript +cube(`line_items`, { + // ... + + measures: { + // ... + + quantity: { + sql: `quantity`, + type: `sum`, + }, + + price: { + type: `sum`, + sql: `price`, + format: `currency`, + }, + }, + + // ... +}); +``` + + + +Because neither `quantity` and `price` reference any other measures in our +`line_items` cube, we can also say that they are **additive leaf measures**. Any +query requesting only these two measures can be called a **leaf measure +additive** query. Additive leaf measures can only be of the following +[types][ref-schema-types-measure]: `count`, `sum`, `min`, `max` or +`count_distinct_approx`. + +[ref-schema-types-measure]: /types-and-formats#measures-types + +### <--{"id" : "Ensuring pre-aggregations are targeted by queries"}--> Non-Additivity + +Using the same sample data for `line_items`, there's a `profit_margin` field +which is different for each row. However, despite the value being numerical, it +doesn't actually make sense to add up this value. Let's look at the rows for +`2021-01-20` in the sample data: + +| **id** | **product_id** | **order_id** | **quantity** | **price** | **profit_margin** | **created_at** | +| ------ | -------------- | ------------ | ------------ | --------- | ----------------- | -------------------------- | +| 1 | 31 | 1 | 1 | 275 | 1 | 2021-01-20 00:00:00.000000 | +| 2 | 49 | 2 | 6 | 248 | 0.1 | 2021-01-20 00:00:00.000000 | + +And now let's try and aggregate them: + +| **created_at** | **quantity** | **price** | **profit_margin** | +| -------------------------- | ------------ | --------- | ----------------- | +| 2021-01-20 00:00:00.000000 | 7 | 523 | 1.1 | + +Using the source data, we'll manually calculate the profit margin and see if it +matches the above. We'll use the following formula: + +$$ +x + (x * y) = z +$$ + +Where `x` is the original cost of the item, `y` is the profit margin and `z` is +the price the item was sold for. Let's use the formula to find the original cost +for both items sold on `2021-01-20`. For the row with `id = 1`: + +$$ +x + (x * 1) = 275\\ +2x = 275\\ +x = 275 / 2\\ +x = 137.5 +$$ + +And for the row where `id = 2`: + +$$ +x + (x * 0.1) = 248\\ +1.1x = 248\\ +x = 248 / 1.1\\ +x = 225.454545454545455 +$$ + +Which means the total cost for both items was: + +$$ +225.454545454545455 + 137.5\\ +362.954545454545455 +$$ + +Now that we have the cost of each item, let's use the same formula in reverse to +see if applying a profit margin of `1.1` will give us the same total price +(`523`) as calculated earlier: + +$$ +362.954545454545455 + (362.954545454545455 * 1.1) = z\\ +762.204545454545455 = z\\ +z = 762.204545454545455 +$$ + +We can clearly see that `523` **does not** equal `762.204545454545455`, and we +cannot treat the `profit_margin` column the same as we would any other additive +measure. Armed with the above knowledge, we can add the `profit_margin` field to +our cube **as a [dimension][ref-schema-dims]**: + + + +```yaml +cubes: + - name: line_items + # ... + + dimensions: + # ... + + - name: profit_margin + sql: profit_margin + type: number + format: percent + + # ... +``` + +```javascript +cube(`line_items`, { + // ... + + dimensions: { + // ... + + profit_margin: { + sql: `profit_margin`, + type: `number`, + format: 'percentage', + }, + }, + + // ... +}); +``` + + + +Another approach might be to calculate the profit margin dynamically, and +instead saving the "cost" price. Because the cost price is an additive measure, +we are able to store it in a pre-aggregation: + + + +```yaml +cubes: + - name: line_items + # ... + + measures: + # ... + + - name: cost + sql: "{CUBE.price} / (1 + {CUBE.profit_margin}" + type: sum + + # ... +``` + +```javascript +cube(`line_items`, { + // ... + + measures: { + // ... + + cost: { + sql: `${CUBE.price} / (1 + ${CUBE.profit_margin})`, + type: `sum`, + }, + }, + + // ... +}); +``` + + + +Another example of a non-additive measure would be a distinct count of +`product_id`. If we took the distinct count of products sold over a month, and +then tried to sum the distinct count of products for each individual day and +compared them, we would not get the same results. We can add the measure like +this: + + + +```yaml +cubes: + - name: line_items + # ... + + measures: + # ... + + - name: count_distinct_products + sql: product_id + type: count_distinct + + # ... +``` + +```javascript +cube(`line_items`, { + // ... + + measures: { + // ... + + count_distinct_products: { + sql: `product_id`, + type: `count_distinct`, + }, + }, + + // ... +}); +``` + + + +However the above cannot be used in for a pre-aggregation. We can instead change +the `type` to `count_distinct_approx`, and then use the measure in a +pre-aggregation definition: + + + +```yaml +cubes: + - name: line_items + # ... + + measures: + # ... + + - name: count_distinct_products + sql: product_id + type: count_distinct_approx + + pre_aggregations: + - name: my_rollup + # ... + + measures: + - count_distinct_products + + # ... +``` + +```javascript +cube(`line_items`, { + // ... + + measures: { + // ... + + count_distinct_products: { + sql: `product_id`, + type: `count_distinct_approx`, + }, + }, + + pre_aggregations: { + my_rollup: { + // ... + + measures: [ count_distinct_products ], + } + }, + + // ... +}); +``` + + + +### <--{"id" : "Ensuring pre-aggregations are targeted by queries"}--> Selecting the pre-aggregation + +To recap what we've learnt so far: + +- **Additive measures** are measures whose values can be added together + +- **Multiplied measures** are measures that define `one_to_many` relationships + +- **Leaf measures** are measures that do not reference any other measures in + their definition + +- **Calculated measures** are measures that reference other dimensions and + measures in their definition + +- A query is **leaf measure additive** if all of its leaf measures are one of: + `count`, `sum`, `min`, `max` or `count_distinct_approx` + +Cube looks for matching pre-aggregations in the order they are defined in a +cube's data model file. Each defined pre-aggregation is then tested for a match +based on the criteria in the flowchart below: + +
+ Pre-Aggregation Selection Flowchart +
+ +Some extra considerations for pre-aggregation selection: + +- The query's time dimension and granularity must match the pre-aggregation. + +- The query's time dimension and granularity together act as a dimension. If the + date range isn't aligned with granularity, a common granularity is used. This + common granularity is selected using the [greatest common divisor][wiki-gcd] + across both the query and pre-aggregation. For example, the common granularity + between `hour` and `day` is `hour` because both `hour` and `day` can be + divided by `hour`. + +- The query's granularity's date range must match the start date and end date + from the time dimensions. For example, when using a granularity of `month`, + the values should be the start and end days of the month i.e. + `['2020-01-01T00:00:00.000', '2020-01-31T23:59:59.999']`; when the granularity + is `day`, the values should be the start and end hours of the day i.e. + `['2020-01-01T00:00:00.000', '2020-01-01T23:59:59.999']`. Date ranges are + inclusive, and the minimum granularity is `second`. + +- The order in which pre-aggregations are defined in models matter; the first + matching pre-aggregation for a query is the one that is used. Both the + measures and dimensions of any cubes specified in the query are checked to + find a matching `rollup`. + +- `rollup` pre-aggregations **always** have priority over `original_sql`. Thus, + if you have both `original_sql` and `rollup` defined, Cube will try to match + `rollup` pre-aggregations before trying to match `original_sql`. You can + instruct Cube to use the original SQL pre-aggregations by using + [`use_original_sql_pre_aggregations`][ref-schema-preaggs-origsql]. + +[ref-caching-preaggs-cubestore]: + /caching/using-pre-aggregations#pre-aggregations-storage +[ref-caching-preaggs-refresh]: /caching/using-pre-aggregations#refresh-strategy +[ref-caching-preaggs-storage]: + /caching/using-pre-aggregations#pre-aggregations-storage +[ref-schema-dims]: /schema/reference/dimensions +[ref-schema-joins]: /schema/reference/joins +[ref-schema-joins-rel]: /schema/reference/joins#relationship +[ref-schema-preaggs]: /schema/reference/pre-aggregations +[ref-schema-preaggs-origsql]: + /schema/reference/pre-aggregations#type-originalsql +[self-select-pre-agg]: #selecting-the-pre-aggregation +[wiki-gcd]: https://en.wikipedia.org/wiki/Greatest_common_divisor diff --git a/docs/content/Caching/Lambda-Pre-Aggregations.mdx b/docs/content/Caching/Lambda-Pre-Aggregations.mdx index 3a7b2c7a3c98b..758bb1d07b3fc 100644 --- a/docs/content/Caching/Lambda-Pre-Aggregations.mdx +++ b/docs/content/Caching/Lambda-Pre-Aggregations.mdx @@ -1,107 +1,226 @@ --- -title: Lambda Pre-Aggregations +title: Lambda pre-aggregations permalink: /caching/pre-aggregations/lambda-pre-aggregations category: Caching menuOrder: 4 --- -Lambda pre-aggregations follow the [Lambda architecture](https://en.wikipedia.org/wiki/Lambda_architecture) design to union real-time and batch data. Cube acts as a serving layer and uses pre-aggregations as a batch layer and source data or other pre-aggregations, usually [streaming][streaming-pre-agg], as a speed layer. +Lambda pre-aggregations follow the +[Lambda architecture](https://en.wikipedia.org/wiki/Lambda_architecture) design +to union real-time and batch data. Cube acts as a serving layer and uses +pre-aggregations as a batch layer and source data or other pre-aggregations, +usually [streaming][streaming-pre-agg], as a speed layer. Due to this design, +lambda pre-aggregations **only** work with data that is newer than the existing +batched pre-aggregations. -Lambda pre-aggregations only work with Cube Store. - -Additionally, we’re going to remove support for external storages, other than Cube Store, later this year. [Cube Store will replace Redis](https://cube.dev/blog/replacing-redis-with-cube-store) and, therefore will be a required component to run Cube even without pre-aggregations. +Lambda pre-aggregations only work with Cube Store. ## Use cases -Below we are looking at the most common examples of using lambda pre-aggregations. +Below we are looking at the most common examples of using lambda +pre-aggregations. ### Batch and source data -Batch data is coming from pre-aggregation and real-time data is coming from the data source. +Batch data is coming from pre-aggregation and real-time data is coming from the +data source.
Lambda pre-aggregation batch and source diagram
-First, you need to create pre-aggregations that will contain your batch data. In the following example, we call it **batch.** Please note, it must have `timeDimension`, and Cube will use it to union batch data with source data. - -You control the batch part of your data with `buildRangeStart` and `buildRangeEnd` properties of pre-aggregation to determine specific window for your batched data. - -Next, you need to create a lambda pre-aggregation. To do that, create pre-aggregation with type `rollupLambda`, specify rollups you would like to use with `rollups` property, and finally set `unionWithSourceData: true` to use source data as a real-time layer. - -Please make sure that the lambda pre-aggregation definition comes first when defining your pre-aggregations. - +First, you need to create pre-aggregations that will contain your batch data. In +the following example, we call it `batch`. Please note, it must have a +`time_dimension` and `partition_granularity` specified. Cube will use these +properties to union batch data with freshly-retrieved source data. + +You may also control the batch part of your data with the `build_range_start` and +`build_range_end` properties of a pre-aggregation to determine a specific window +for your batched data. + +Next, you need to create a lambda pre-aggregation. To do that, create +pre-aggregation with type `rollup_lambda`, specify rollups you would like to use +with `rollups` property, and finally set `union_with_source_data: true` to use +source data as a real-time layer. + +Please make sure that the lambda pre-aggregation definition comes first when +defining your pre-aggregations. + + + +```yaml +cubes: + - name: users + # ... + + pre_aggregations: + - name: lambda + type: rollup_lambda + union_with_source_data: true + rollups: + - CUBE.batch + + - name: batch + measures: + - users.count + dimensions: + - users.name + time_dimension: users.created_at + granularity: day + partition_granularity: day + build_range_start: + sql: SELECT '2020-01-01' + build_range_end: + sql: SELECT '2022-05-30' +``` -```js -lambda: { - type: `rollupLambda`, - unionWithSourceData: true, - rollups: [Users.batch] -}, -batch: { - measures: [Users.count], - dimensions: [Users.name], - timeDimension: Users.createdAt, - granularity: `day`, - buildRangeStart: { - sql: `SELECT '2020-01-01'` +```javascript +cube('users', { + // ... + + pre_aggregations: { + lambda: { + type: `rollup_lambda`, + union_with_source_data: true, + rollups: [CUBE.batch] + }, + + batch: { + measures: [users.count], + dimensions: [users.name], + time_dimension: users.created_at, + granularity: `day`, + partition_granularity: `day`, + build_range_start: { + sql: `SELECT '2020-01-01'` + }, + build_range_end: { + sql: `SELECT '2022-05-30'` + }, + }, }, - buildRangeEnd: { - sql: `SELECT '2022-05-30'` - } -} +}) ``` + + ### Batch and streaming data -In this scenario, batch data is comes from one pre-aggregation and real-time data comes from a [streaming pre-aggregation][streaming-pre-agg]. +In this scenario, batch data is comes from one pre-aggregation and real-time +data comes from a [streaming pre-aggregation][streaming-pre-agg].
Lambda pre-aggregation batch and streaming diagram
+You can use lambda pre-aggregations to combine data from multiple +pre-aggregations, where one pre-aggregation can have batch data and another +streaming. + + + +```yaml +cubes: + - name: streaming_users + # This cube uses a streaming SQL data source such as ksqlDB + # ... + + pre_aggregations: + - name: streaming + type: rollup + measures: + - CUBE.count + dimensions: + - CUBE.name + time_dimension: CUBE.created_at + granularity: day, + partition_granularity: day + + - name: users + # This cube uses a data source such as ClickHouse or BigQuery + # ... + + pre_aggregations: + - name: batch_streaming_lambda + type: rollup_lambda + rollups: + - users.batch + - streaming_users.streaming + + - name: batch + type: rollup + measures: + - users.count + dimensions: + - users.name + time_dimension: users.created_at + granularity: day + partition_granularity: day + build_range_start: + sql: SELECT '2020-01-01' + build_range_end: + sql: SELECT '2022-05-30' +``` -You can use lambda pre-aggregations to combine data from multiple pre-aggregation, where one pre-aggregation can have batch data and another streaming. - -```js -batchStreamingLambda: { - type: `rollupLambda`, - rollups: [Users.batch, streaming] -}, -batch: { - type: `rollup`, - measures: [Users.count], - dimensions: [Users.name], - timeDimension: Users.createdAt, - granularity: `day`, - buildRangeStart: { - sql: `SELECT '2020-01-01'` +```javascript +// This cube uses a streaming SQL data source such as ksqlDB +cube('streaming_users', { + // ... + + pre_aggregations: { + streaming: { + type: `rollup`, + measures: [CUBE.count], + dimensions: [CUBE.name], + time_dimension: CUBE.created_at, + granularity: `day`, + partition_granularity: `day`, + }, }, - buildRangeEnd: { - sql: `SELECT '2022-05-30'` - } -}, -streaming: { - type: `rollup`, - measures: [StreamingUsers.count], - dimensions: [StreamingUsers.name], - timeDimension: StreamingUsers.createdAt, - granularity: `day` -} +}); + +// This cube uses a data source such as ClickHouse or BigQuery +cube('users', { + // ... + + pre_aggregations: { + batch_streaming_lambda: { + type: `rollup_lambda`, + rollups: [users.batch, streaming_users.streaming] + }, + + batch: { + type: `rollup`, + measures: [users.count], + dimensions: [users.name], + time_dimension: users.created_at, + granularity: `day`, + partition_granularity: `day`, + build_range_start: { + sql: `SELECT '2020-01-01'` + }, + build_range_end: { + sql: `SELECT '2022-05-30'` + }, + }, + }, +}); ``` -[streaming-pre-agg]: /caching/using-pre-aggregations#streaming-pre-aggregations \ No newline at end of file + + +[streaming-pre-agg]: /caching/using-pre-aggregations#streaming-pre-aggregations diff --git a/docs/content/Caching/Overview.mdx b/docs/content/Caching/Overview.mdx index f077928e508cb..56462f9b73169 100644 --- a/docs/content/Caching/Overview.mdx +++ b/docs/content/Caching/Overview.mdx @@ -6,14 +6,19 @@ category: Caching menuOrder: 1 --- -Cube.js provides a two-level caching system. The first level is **in-memory** -cache and is active by default. Currently, [Redis][link-redis] is being used for -in-memory cache and queue management for [clustered Cube -deployments][ref-production-checklist]. Cube Store will [replace -Redis][replace-redis] in late -2022. - -Cube.js [in-memory cache](#in-memory-cache) acts as a buffer for your database +
+ Request vs Cube caching layers +
+ +Cube provides a two-level caching system. The first level is **in-memory** cache +and is active by default. + +Cube's [in-memory cache](#in-memory-cache) acts as a buffer for your database when there's a burst of requests hitting the same data from multiple concurrent users while [pre-aggregations](#pre-aggregations) are designed to provide the right balance between time to insight and querying performance. @@ -29,124 +34,188 @@ unless it is necessary. To speed up query performance, consider using ## Pre-Aggregations -Pre-aggregations is a layer of the aggregated data built and refreshed by -Cube.js. It can dramatically improve the query performance and provide a higher +Pre-aggregations is a layer of the aggregated data built and refreshed by Cube. +It can dramatically improve the query performance and provide a higher concurrency. -To start building pre-aggregations, Cube.js requires write access to the -[pre-aggregations schema][ref-config-preagg-schema] in the source database. -Cube.js first builds pre-aggregations as tables in the source database and then -exports them into the pre-aggregations storage. +To start building pre-aggregations, depending on your data source, Cube may +require write access to the [pre-aggregations schema][ref-config-preagg-schema] +in the source database. In this case, Cube first builds pre-aggregations as +tables in the source database and then exports them into the pre-aggregations +storage. Please refer to the documentation for your specific driver to learn +more about read-only support and pre-aggregation build strategies. -Pre-aggregations are defined in the data schema. You can learn more about -defining pre-aggregations in [schema reference][ref-schema-ref-preaggs]. +Pre-aggregations are defined in the data model. You can learn more about +defining pre-aggregations in [data modeling reference][ref-schema-ref-preaggs]. + + + +```yaml +cubes: + - name: orders + sql_table: orders + + measures: + - name: total_amount + sql: amount + type: sum + + dimensions: + - name: created_at + sql: created_at + type: time + + pre_aggregations: + - name: amount_by_created + measures: + - total_amount + time_dimension: created_at + granularity: month +``` + +```javascript +cube(`orders`, { + sql_table: `orders`, -```js -cube(`Orders`, { measures: { - totalAmount: { + total_amount: { sql: `amount`, type: `sum`, }, }, dimensions: { - createdAt: { + created_at: { sql: `created_at`, type: `time`, }, }, - preAggregations: { - amountByCreated: { - measures: [totalAmount], - timeDimension: createdAt, + pre_aggregations: { + amount_by_created: { + measures: [total_amount], + time_dimension: created_at, granularity: `month`, }, }, }); ``` + + ## In-memory Cache -Cube.js caches the results of executed queries using in-memory cache. The cache -key is a generated SQL statement with any existing query-dependent -pre-aggregations. +Cube caches the results of executed queries using in-memory cache. The cache key +is a generated SQL statement with any existing query-dependent pre-aggregations. -Upon receiving an incoming request, Cube.js first checks the cache using this -key. If nothing is found in the cache, the query is executed in the database and -the result set is returned as well as updating the cache. +Upon receiving an incoming request, Cube first checks the cache using this key. +If nothing is found in the cache, the query is executed in the database and the +result set is returned as well as updating the cache. -If an existing value is present in the cache and the `refreshKey` value for the +If an existing value is present in the cache and the `refresh_key` value for the query hasn't changed, the cached value will be returned. Otherwise, an SQL query will be executed against either the pre-aggregations storage or the source database to populate the cache with the results and return them. ### <--{"id" : "In-memory Cache"}--> Refresh Keys -Cube.js takes great care to prevent unnecessary queries from hitting your -database. The first stage caching system caches query results, but Cube.js needs a way to know if the data powering that query result has changed. If the underlying data isn't any -different, the cached result is valid and can be returned skipping an expensive -query, but if there is a difference, the query needs to be re-run and its result -cached. +Cube takes great care to prevent unnecessary queries from hitting your database. +The first stage caching system caches query results, but Cube needs a way to +know if the data powering that query result has changed. If the underlying data +isn't any different, the cached result is valid and can be returned skipping an +expensive query, but if there is a difference, the query needs to be re-run and +its result cached. -To aid with this, Cube.js defines a `refreshKey` for each cube. [Refresh -keys][ref-schema-ref-cube-refresh-key] are evaluated by Cube.js to assess if the +To aid with this, Cube defines a `refresh_key` for each cube. [Refresh +keys][ref-schema-ref-cube-refresh-key] are evaluated by Cube to assess if the data needs to be refreshed. -```js -cube(`Orders`, { - // This refreshKey tells Cube.js to refresh data every 5 minutes - refreshKey: { +The following `refresh_key` tells Cube to refresh data every 5 minutes: + + + +```yaml +cubes: + - name: orders + # ... + + refresh_key: + every: 5 minute +``` + +```javascript +cube(`orders`, { + refresh_key: { every: `5 minute`, }, +}); +``` + + + +With the following `refresh_key`, Cube will only refresh the data if the value of +`MAX(created_at)` changes. By default, Cube will check this `refresh_key` +every 10 seconds: + + - // With this refreshKey Cube.js will only refresh the data if - // the value of previous MAX(created_at) changed - // By default Cube.js will check this refreshKey every 10 seconds - refreshKey: { +```yaml +cubes: + - name: orders + # ... + + refresh_key: + sql: SELECT MAX(created_at) FROM orders +``` + +```javascript +cube(`orders`, { + // ... + + refresh_key: { sql: `SELECT MAX(created_at) FROM orders`, }, }); ``` -By default, Cube.js will check and invalidate the cache in the background when -in [development mode][ref-development-mode]. In production environments, we + + +By default, Cube will check and invalidate the cache in the background when in +[development mode][ref-development-mode]. In production environments, we recommend [running a Refresh Worker as a separate instance][ref-production-checklist-refresh]. -We recommend enabling background cache invalidation in a separate Cube.js worker +We recommend enabling background cache invalidation in a separate Cube worker for production deployments. Please consult the [Production Checklist][ref-production-checklist] for more information. -If background refresh is disabled, Cube.js will refresh the cache during query +If background refresh is disabled, Cube will refresh the cache during query execution. Since this could lead to delays in responding to end-users, we recommend always enabling background refresh. ### <--{"id" : "In-memory Cache"}--> Default Refresh Keys -The default values for `refreshKey` are +The default values for `refresh_key` are -- `every: '2 minute'` for BigQuery, Athena, Snowflake, and Presto. -- `every: '10 second'` for all other databases. +- `every: 2 minute` for BigQuery, Athena, Snowflake, and Presto. +- `every: 10 second` for all other databases. -+You can use a custom SQL query to check if a refresh is required by changing -the [`refreshKey`][ref-schema-ref-cube-refresh-key] property in a cube's Data -Schema. Often, a `MAX(updated_at_timestamp)` for OLTP data is a viable option, -or examining a metadata table for whatever system is managing the data to see -when it last ran. +You can use a custom SQL query to check if a refresh is required by changing +the [`refresh_key`][ref-schema-ref-cube-refresh-key] property in a cube. Often, a +`MAX(updated_at_timestamp)` for OLTP data is a viable option, or examining a +metadata table for whatever system is managing the data to see when it last ran. ### <--{"id" : "In-memory Cache"}--> Disabling the cache -There's no straightforward way to disable caching in Cube.js. The reason is that -Cube.js not only stores cached values but also uses the cache as a point of +There's no straightforward way to disable caching in Cube. The reason is that +Cube not only stores cached values but also uses the cache as a point of synchronization and coordination between nodes in a cluster. For the sake of -design simplicity, Cube.js doesn't distinguish client invocations, and all calls -to the data load API are idempotent. This provides excellent reliability and +design simplicity, Cube doesn't distinguish client invocations, and all calls to +the data load API are idempotent. This provides excellent reliability and scalability but has some drawbacks. One of those load data calls can't be traced to specific clients, and as a consequence, there's no guaranteed way for a client to initiate a new data loading query or know if the current invocation @@ -154,9 +223,9 @@ wasn't initiated earlier by another client. Only Refresh Key freshness guarantees are provided in this case. For situations like real-time analytics or responding to live user changes to -underlying data, the `refreshKey` query cache can prevent fresh data from +underlying data, the `refresh_key` query cache can prevent fresh data from showing up immediately. For these situations, the cache can effectively be -disabled by setting the [`refreshKey.every`][ref-schema-ref-cube-refresh-key] +disabled by setting the [`refresh_key.every`][ref-schema-ref-cube-refresh-key] parameter to something very low, like `1 second`. ## Inspecting Queries @@ -167,7 +236,7 @@ Cloud][link-cube-cloud]. [Developer Playground][ref-dev-playground] can be used to inspect a single query. To do that, click the "cache" button after executing the query. It will -show you the information about the `refreshKey` for the query and whether the +show you the information about the `refresh_key` for the query and whether the query uses any pre-aggregations. To inspect multiple queries or list existing pre-aggregations, you can use [Cube Cloud][link-cube-cloud]. @@ -186,7 +255,6 @@ every pre-aggregation's details: the list of queries it serves and all its versions. [link-cube-cloud]: https://cube.dev/cloud -[link-redis]: https://redis.io [ref-config-preagg-schema]: /config#pre-aggregations-schema [ref-dev-playground]: /dev-tools/dev-playground [ref-development-mode]: /configuration/overview#development-mode @@ -195,4 +263,3 @@ versions. /deployment/production-checklist#set-up-refresh-worker [ref-schema-ref-cube-refresh-key]: /schema/reference/cube#refresh-key [ref-schema-ref-preaggs]: /schema/reference/pre-aggregations -[replace-redis]: https://cube.dev/blog/replacing-redis-with-cube-store diff --git a/docs/content/Caching/Running-in-Production.mdx b/docs/content/Caching/Running-in-Production.mdx index 7b4e374c0583a..bcb0d6702743a 100644 --- a/docs/content/Caching/Running-in-Production.mdx +++ b/docs/content/Caching/Running-in-Production.mdx @@ -1,29 +1,18 @@ --- -title: Running in Production +title: Running in production permalink: /caching/running-in-production category: Caching -menuOrder: 4 +menuOrder: 5 --- -Cube.js makes use of two different kinds of cache: +Cube makes use of two different kinds of cache: -- Redis, for in-memory storage of query results -- Cube Store for storing pre-aggregations +- In-memory storage of query results +- Pre-aggregations -In development, Cube.js uses in-memory storage on the server. In production, we -**strongly** recommend running Redis as a separate service. - - - -Cube Store [will replace -Redis][replace-redis] for in-memory cache and queue management in late -2022. - - - -Cube Store is enabled by default when running Cube.js in development mode. In +Cube Store is enabled by default when running Cube in development mode. In production, Cube Store **must** run as a separate process. The easiest way to do -this is to use the official Docker images for Cube.js and Cube Store. +this is to use the official Docker images for Cube and Cube Store. @@ -34,30 +23,30 @@ to run the following commands. You can run Cube Store with Docker with the following command: -```bash +```bash{promptUser: user} docker run -p 3030:3030 cubejs/cubestore ``` Cube Store can further be configured via environment variables. To see a -complete reference, please consult the [Cube Store section of the Environment -Variables reference][ref-config-env]. +complete reference, please consult the `CUBESTORE_*` environment variables in +the [Environment Variables reference][ref-config-env]. -Next, run Cube.js and tell it to connect to Cube Store running on `localhost` -(on the default port `3030`): +Next, run Cube and tell it to connect to Cube Store running on `localhost` (on +the default port `3030`): -```bash +```bash{outputLines: 2-4} docker run -p 4000:4000 \ -e CUBEJS_CUBESTORE_HOST=localhost \ -v ${PWD}:/cube/conf \ cubejs/cube ``` -In the command above, we're specifying `CUBEJS_CUBESTORE_HOST` to let Cube.js -know where Cube Store is running. +In the command above, we're specifying `CUBEJS_CUBESTORE_HOST` to let Cube know +where Cube Store is running. You can also use Docker Compose to achieve the same: @@ -82,12 +71,24 @@ services: links: - cubestore volumes: - - ./schema:/cube/conf/schema + - ./model:/cube/conf/model ``` ## Architecture -Deep dive on Cube Store architecture can be found in [this presentation](https://docs.google.com/presentation/d/1oQ-koloag0UcL-bUHOpBXK4txpqiGl41rxhgDVrw7gw/). +
+ Cube Store cluster with Cube +
+ +A Cube Store cluster consists of at least one Router and one or more Worker +instances. Cube sends queries to the Cube Store Router, which then distributes +the queries to the Cube Store Workers. The Workers then execute the queries and +return the results to the Router, which in turn returns the results to Cube. ## Scaling @@ -107,10 +108,24 @@ much concurrency as you require. In cluster mode, Cube Store runs two kinds of nodes: -- a single **router** node handles incoming client connections, manages database - metadata and serves simple queries. +- one or more **router** nodes handle incoming client connections, manage + database metadata and serve simple queries. - multiple **worker** nodes which execute SQL queries +Cube Store querying performance is optimal when the count of partitions in a +single query is less than or equal to the worker count. For example, you have a +200 million rows table that is partitioned by day, which is ten daily Cube +partitions or 100 Cube Store partitions in total. The query sent by the user +contains filters, and the resulting scan requires reading 16 Cube Store +partitions in total. Optimal query performance, in this case, can be achieved +with 16 or more workers. You can use `EXPLAIN` and `EXPLAIN ANALYZE` SQL +commands to see how many partitions would be used in a specific Cube Store +query. + +Resources required for the main node and workers can vary depending on the +configuration. With default settings, you should expect to allocate at least 4 +CPUs and up to 8GB per main or worker node. + The configuration required for each node can be found in the table below. More information about these variables can be found [in the Environment Variables reference][ref-config-env]. @@ -123,8 +138,10 @@ reference][ref-config-env]. | `CUBESTORE_WORKER_PORT` | - | Yes | | `CUBESTORE_META_ADDR` | - | Yes | -`CUBESTORE_WORKERS` and `CUBESTORE_META_ADDR` variables should be set with stable addresses, which should not change. -You can use stable DNS names and put load balancers in front of your worker and router instances to fulfill stable name requirements in environments where stable IP addresses can't be guaranteed. +`CUBESTORE_WORKERS` and `CUBESTORE_META_ADDR` variables should be set with +stable addresses, which should not change. You can use stable DNS names and put +load balancers in front of your worker and router instances to fulfill stable +name requirements in environments where stable IP addresses can't be guaranteed. @@ -133,7 +150,8 @@ recommend using [partitioned pre-aggregations][ref-caching-partitioning]. -A sample Docker Compose stack for the single machine setting this up might look like: +A sample Docker Compose stack for the single machine setting this up might look +like: ```yaml version: '2.2' @@ -188,11 +206,13 @@ services: ## Replication and High Availability -The open-source version of Cube Store doesn't support replicating any of its nodes. -The router node and every worker node should always have only one instance copy if served behind the load balancer or service address. -Replication will lead to undefined behavior of the cluster, including connection errors and data loss. -If any cluster node is down, it'll lead to a complete cluster outage. -If Cube Store replication and high availability are required, please consider using Cube Cloud. +The open-source version of Cube Store doesn't support replicating any of its +nodes. The router node and every worker node should always have only one +instance copy if served behind the load balancer or service address. Replication +will lead to undefined behavior of the cluster, including connection errors and +data loss. If any cluster node is down, it'll lead to a complete cluster outage. +If Cube Store replication and high availability are required, please consider +using Cube Cloud. ## Storage @@ -204,14 +224,17 @@ Cube Store can only use one type of remote storage at runtime. Cube Store makes use of a separate storage layer for storing metadata as well as for persisting pre-aggregations as Parquet files. Cube Store [can be configured -to use either AWS S3 or Google Cloud Storage][ref-config-env-cloud-storage]. -If desired, local path on the server can also be used in case all Cube Store cluster nodes are co-located on a single machine. +to use either AWS S3 or Google Cloud Storage][ref-config-env]. If desired, local +path on the server can also be used in case all Cube Store cluster nodes are +co-located on a single machine. -Cube Store requires strong consistency guarantees from underlying distributed storage. -AWS S3, Google Cloud Storage, and Azure Blob Storage (Cube Cloud only) are the only known implementations that provide strong consistency. -Using other implementations in production is discouraged and can lead to consistency and data corruption errors. +Cube Store requires strong consistency guarantees from underlying distributed +storage. AWS S3, Google Cloud Storage, and Azure Blob Storage (Cube Cloud only) +are the only known implementations that provide strong consistency. Using other +implementations in production is discouraged and can lead to consistency and +data corruption errors. @@ -247,9 +270,12 @@ services: ### Local Storage -Separately from remote storage, Cube Store requires local scratch space to warm up partitions by downloading Parquet files before querying them. -By default, this directory should be mounted to `.cubestore/data` dir inside contained and can be configured by [CUBESTORE_DATA_DIR][ref-config-env] environment variable. -It is advised to use local SSDs for this scratch space to maximize querying performance. +Separately from remote storage, Cube Store requires local scratch space to warm +up partitions by downloading Parquet files before querying them. By default, +this directory should be mounted to `.cubestore/data` dir inside contained and +can be configured by [CUBESTORE_DATA_DIR][ref-config-env] environment variable. +It is advised to use local SSDs for this scratch space to maximize querying +performance. ### <--{"id" : "Storage"}--> AWS @@ -271,10 +297,8 @@ default. Cube Store currently does not have any in-built authentication mechanisms. For this reason, we recommend running your Cube Store cluster on a network that only -allows requests from the Cube.js deployment. +allows requests from the Cube deployment. [link-wsl2]: https://docs.microsoft.com/en-us/windows/wsl/install-win10 [ref-caching-partitioning]: /caching/using-pre-aggregations#partitioning -[ref-config-env]: /reference/environment-variables#cube-store -[ref-config-env-cloud-storage]: /reference/environment-variables#cloud-storage -[replace-redis]: https://cube.dev/blog/replacing-redis-with-cube-store +[ref-config-env]: /reference/environment-variables diff --git a/docs/content/Caching/Using-Pre-Aggregations.mdx b/docs/content/Caching/Using-Pre-Aggregations.mdx index 21054a36b5f96..b8880e41693c2 100644 --- a/docs/content/Caching/Using-Pre-Aggregations.mdx +++ b/docs/content/Caching/Using-Pre-Aggregations.mdx @@ -1,36 +1,54 @@ --- -title: Using Pre-Aggregations +title: Using pre-aggregations permalink: /caching/using-pre-aggregations category: Caching menuOrder: 3 --- -Pre-aggregations is a powerful way to speed up your Cube.js queries. There are -many configuration options to consider. Please make sure to also check [the -Pre-Aggregations reference in the data schema section][ref-schema-ref-preaggs]. +Pre-aggregations is a powerful way to speed up your Cube queries. There are many +configuration options to consider. Please make sure to also check [the +Pre-Aggregations reference in the data modeling +section][ref-schema-ref-preaggs]. ## Refresh Strategy Refresh strategy can be customized by setting the -[`refreshKey`][ref-schema-ref-preaggs-refresh-key] property for the +[`refresh_key`][ref-schema-ref-preaggs-refresh-key] property for the pre-aggregation. -The default value of [`refreshKey`][ref-schema-ref-preaggs-refresh-key] is -`every: '1 hour'`. It can be redefined either by overriding the default value of +The default value of [`refresh_key`][ref-schema-ref-preaggs-refresh-key] is +`every: 1 hour`. It can be redefined either by overriding the default value of the [`every` property][ref-schema-ref-preaggs-refresh-key-every]: -```javascript -cube(`Orders`, { + + +```yaml +cubes: + - name: orders + # ... + + pre_aggregations: + - name: amount_by_created + type: rollup + measures: + - amount + time_dimension: created_at + granularity: month + refresh_key: + every: 12 hour +``` - ..., +```javascript +cube(`orders`, { + // ... - preAggregations: { - amountByCreated: { + pre_aggregations: { + amount_by_created: { type: `rollup`, measures: [amount], - timeDimension: createdAt, + time_dimension: created_at, granularity: `month`, - refreshKey: { + refresh_key: { every: `12 hour`, }, }, @@ -38,20 +56,39 @@ cube(`Orders`, { }); ``` + + Or by providing a [`sql` property][ref-schema-ref-preaggs-refresh-key-sql] -instead, and leaving `every` unchanged from its' default value: +instead, and leaving `every` unchanged from its default value: + + + +```yaml +cubes: + - name: orders + # ... + + pre_aggregations: + - name: amount_by_created + measures: + - amount + time_dimension: created_at + granularity: month + refresh_key: + # every will default to `10 seconds` here + sql: SELECT MAX(created_at) FROM orders +``` ```javascript -cube(`Orders`, { +cube(`orders`, { + // ... - ..., - - preAggregations: { - amountByCreated: { + pre_aggregations: { + amount_by_created: { measures: [amount], - timeDimension: createdAt, + time_dimension: created_at, granularity: `month`, - refreshKey: { + refresh_key: { // every will default to `10 seconds` here sql: `SELECT MAX(created_at) FROM orders`, }, @@ -60,19 +97,38 @@ cube(`Orders`, { }); ``` + + Or both `every` and `sql` can be defined together: -```javascript -cube(`Orders`, { + + +```yaml +cubes: + - name: orders + # ... + + pre_aggregations: + - name: amount_by_created + measures: + - amount + time_dimension: created_at + granularity: month + refresh_key: + every: 12 hour + sql: SELECT MAX(created_at) FROM orders +``` - ..., +```javascript +cube(`orders`, { + // ... - preAggregations: { - amountByCreated: { + pre_aggregations: { + amount_by_created: { measures: [amount], - timeDimension: createdAt, + time_dimension: created_at, granularity: `month`, - refreshKey: { + refresh_key: { every: `12 hour`, sql: `SELECT MAX(created_at) FROM orders`, }, @@ -81,21 +137,23 @@ cube(`Orders`, { }); ``` -When `every` and `sql` are used together, Cube.js will run the query from the -`sql` property on an interval defined by the `every` property. If the query -returns new results, then the pre-aggregation will be refreshed. + + +When `every` and `sql` are used together, Cube will run the query from the `sql` +property on an interval defined by the `every` property. If the query returns +new results, then the pre-aggregation will be refreshed. ## Rollup Only Mode -To make Cube.js _only_ serve requests from pre-aggregations, the -[`CUBEJS_ROLLUP_ONLY` environment variable][ref-config-env-general] can be set -to `true` on an API instance. This will prevent serving data on API requests +To make Cube _only_ serve requests from pre-aggregations, the +[`CUBEJS_ROLLUP_ONLY`][ref-config-env-rolluponly] environment variable can be +set to `true` on an API instance. This will prevent serving data on API requests from the source database. When using this configuration in a single node deployment (where the API -instance and [Refresh Worker ][ref-deploy-refresh-wrkr] are configured on the +instance and [Refresh Worker][ref-deploy-refresh-wrkr] are configured on the same host), requests made to the API that cannot be satisfied by a rollup throw an error. Scheduled refreshes will continue to work in the background. @@ -113,58 +171,221 @@ like [BigQuery](/config/databases/google-bigquery) or [AWS Athena](/config/databases/aws-athena). Any `rollup` pre-aggregation can be partitioned by time using the -`partitionGranularity` property in [a pre-aggregation +`partition_granularity` property in [a pre-aggregation definition][ref-schema-ref-preaggs]. In the example below, the -`partitionGranularity` is set to `month`, which means Cube will generate +`partition_granularity` is set to `month`, which means Cube will generate separate tables for each month's worth of data. Once built, it will continue to refresh on a daily basis the last 3 months of data. -```typescript -cube(`Orders`, { - sql: `select * from orders`, + + +```yaml +cubes: + - name: orders + # ... + + pre_aggregations: + - name: category_and_date + measures: + - count + - revenue + dimensions: + - category + time_dimension: created_at + granularity: day + partition_granularity: month + refresh_key: + every: 1 day + incremental: true + update_window: 3 months +``` - ..., +```javascript +cube(`orders`, { + // ... preAggregations: { - categoryAndDate: { - measures: [Orders.count, revenue], + category_and_date: { + measures: [count, revenue], dimensions: [category], - timeDimension: createdAt, + time_dimension: created_at, granularity: `day`, - partitionGranularity: `month`, - refreshKey: { + partition_granularity: `month`, + refresh_key: { every: `1 day`, incremental: true, - updateWindow: `3 months` + update_window: `3 months` } }, }, }); ``` + + ## Using Indexes -For larger pre-aggregations, performance can be significantly improved by adding -an [index][ref-schema-ref-preaggs-index]. This is done by defining the `indexes` -property for a pre-aggregation: +### When to use indexes? -```typescript -cube(`Orders`, { - ..., +Indexes are great when you filter large amounts of data across one or several +dimension columns. You can read more about them +[here][ref-schema-ref-preaggs-index]. + +### Best Practices + +To maximize performance, you can introduce an index per type of query so the set +of dimensions used in the query overlap as much as possible with the ones +defined in the index. Measures are traditionally only used in indexes if you +plan to filter a measured value and the cardinality of the possible values of +the measure is low. + +The order in which columns are specified in the index is **very** important; +suboptimal ordering can lead to diminished performance. To improve the +performance of an index the main thing to consider is the order of the columns +defined in it. + +The rule of thumb for index column order is: +- Single value filters come first +- `GROUP BY` columns come second +- Everything else used in the query comes afterward + +**Example:** + +Suppose you have a pre-aggregation that has millions of rows with the following +structure: + +| timestamp | product_name | product_category | zip_code | order_total | +| ------------------- | ------------- | ---------------- | -------- | ----------- | +| 2023-01-01 10:00:00 | Plastic Chair | Furniture | 88523 | 2000 | +| 2023-01-01 10:00:00 | Keyboard | Electronics | 88523 | 1000 | +| 2023-01-01 10:00:00 | Mouse | Electronics | 88523 | 800 | +| 2023-01-01 11:00:00 | Plastic Chair | Furniture | 88524 | 3000 | +| 2023-01-01 11:00:00 | Keyboard | Electronics | 88524 | 2000 | + +The pre-aggregation code would look as follows: + + + +```javascript +cube('orders', { + // ... + + pre_aggregations: { + main: { + measures: [order_total], + dimensions: [product_name, product_category, zip_code], + time_dimension: timestamp, + granularity: `hour`, + partition_granularity: `day`, + allow_non_strict_date_range_match: true, + refresh_key: { + every: `1 hour`, + incremental: true, + update_window: `1 day`, + }, + build_range_start: { + sql: `SELECT DATE_SUB(NOW(), 365)`, + }, + build_range_end: { + sql: `SELECT NOW()`, + }, + }, + }, +}); +``` + +```yaml +cubes: + - name: orders + # ... + + pre_aggregations: + - name: main + measures: + - order_total + dimensions: + - product_name + - product_category + - zip_code + time_dimension: timestamp + granularity: hour + partition_granularity: day + allow_non_strict_date_range_match: true + refresh_key: + every: 1 hour + incremental: true + update_window: 1 day + build_range_start: + sql: SELECT DATE_SUB(NOW(), 365) + build_range_end: + sql: SELECT NOW() +``` + + + +You run the following query on a regular basis, with the only difference between +queries being the filter values: + +```JSON +{ + "measures": [ + "orders.order_total" + ], + "timeDimensions": [ + { + "dimension": "orders.timestamp", + "granularity": "hour", + "dateRange": [ + "2022-12-14T06:00:00.000", + "2023-01-13T06:00:00.000" + ] + } + ], + "order": { + "orders.timestamp": "asc" + }, + "filters": [ + { + "member": "orders.product_category", + "operator": "equals", + "values": [ + "Electronics" + ] + }, + { + "member": "orders.product_name", + "operator": "equals", + "values": [ + "Keyboard", + "Mouse" + ] + } + ], + "dimensions": [ + "orders.zip_code" + ], + "limit": 10000 +} +``` + +After running this on a dataset with millions of records you find that it's +taking a long time to run, so you decide to add an index to target this specific +query. Taking into account the best practices mentioned previously you should +define an index as follows: + + + +```javascript +cube('orders', { + // ... + + pre_aggregations: { + main: { + // ... - preAggregations: { - status: { - measures: [CUBE.count], - dimensions: [CUBE.status, Products.name, ProductCategories.name], - timeDimension: CUBE.createdAt, - granularity: `day`, indexes: { - statusIndex: { - columns: [CUBE.status, Products.name], - }, - aggregatedIndex: { - columns: [CUBE.status], - type: `aggregate`, + category_productname_zipcode_index: { + columns: [product_category, zip_code, product_name], }, }, }, @@ -172,35 +393,97 @@ cube(`Orders`, { }); ``` -For example, if the pre-aggregated data looks like: +```yaml +cubes: + - name: orders + # ... + + pre_aggregations: + - name: main + # ... + + indexes: + - name: category_productname_zipcode_index + columns: + - product_category + - zip_code + - product_name +``` + + + +Then the data within `category_productname_zipcode_index` would look like: + +| product_category | product_name | zip_code | timestamp | order_total | +| ---------------- | ------------- | -------- | ------------------- | ----------- | +| Furniture | Plastic Chair | 88523 | 2023-01-01 10:00:00 | 2000 | +| Electronics | Keyboard | 88523 | 2023-01-01 10:00:00 | 1000 | +| Electronics | Mouse | 88523 | 2023-01-01 10:00:00 | 800 | +| Furniture | Plastic Chair | 88524 | 2023-01-01 11:00:00 | 3000 | +| Electronics | Keyboard | 88524 | 2023-01-01 11:00:00 | 2000 | + +`product_category` column comes first as it's a single value filter. +Then `zip_code` as it's `GROUP BY` column. +`product_name` comes last as it's a multiple value filter. + +It might sound counter-intuitive to have `GROUP BY` columns before filter ones, however Cube Store always performs scans on sorted data, and if `GROUP BY` matches index ordering, merge sort-based algorithms are used for querying, which are usually much faster than hash-based group by in case of index ordering doesn't match the query. If in doubt, always use `EXPLAIN` and `EXPLAIN ANALYZE` in Cube Store to figure out the final query plan. + +### Aggregated indexes + +Aggregated indexes can be defined as well. You can read more about them +[here][ref-schema-ref-preaggs-index]. -| created_at | status | products\_\_name | product_categories\_\_name | count | -| ---------- | ---------- | ---------------- | -------------------------- | ----- | -| 2022-08-01 | completed | Plastic Chair | Furniture | 4 | -| 2022-08-01 | processing | Plastic Chair | Furniture | 11 | -| 2022-08-01 | shipped | Plastic Chair | Furniture | 2 | -| 2022-08-01 | completed | Keyboard | Electronics | 1 | -| 2022-08-01 | processing | Keyboard | Electronics | 3 | -| 2022-08-01 | shipped | Keyboard | Electronics | 7 | +Example: -Then the data within `statusIndex` would look like: + -| status | products\_\_name | created_at | product_categories\_\_name | count | -| ---------- | ---------------- | ---------- | -------------------------- | ----- | -| completed | Plastic Chair | 2022-08-01 | Furniture | 4 | -| processing | Plastic Chair | 2022-08-01 | Furniture | 11 | -| shipped | Plastic Chair | 2022-08-01 | Furniture | 2 | -| completed | Keyboard | 2022-08-01 | Electronics | 1 | -| processing | Keyboard | 2022-08-01 | Electronics | 3 | -| shipped | Keyboard | 2022-08-01 | Electronics | 7 | +```javascript +cube('orders', { + // ... -And the data within `aggregatedIndex` would look like: + pre_aggregations: { + main: { + // ... -| status | count | -| ---------- | ----- | -| completed | 5 | -| processing | 14 | -| shipped | 9 | + indexes: { + // ... + + zip_code_index: { + columns: [zip_code], + type: `aggregate`, + }, + }, + }, + }, +}); +``` + +```yaml +cubes: + - name: orders + # ... + + pre_aggregations: + - name: main + # ... + + indexes: + # ... + + - name: zip_code_index + columns: + - zip_code + type: aggregate +``` + + + +And the data for `zip_code_index` would look like the following: + +| zip_code | order_total | +| -------- | ----------- | +| 88523 | 3800 | +| 88524 | 5000 | ## Inspecting Pre-Aggregations @@ -208,10 +491,19 @@ Cube Store partially supports the MySQL protocol. This allows you to execute simple queries using a familiar SQL syntax. You can connect using the MySQL CLI client, for example: -```bash +```bash{promptUser: user} mysql -h --user=cubestore -pcubestore ``` + + +Only Linux and Mac OS versions of MySQL client are supported as of right now. +You can install one on ubuntu using `apt-get install default-mysql-client` +command or `brew install mysql-client` on Mac OS. Windows versions of the MySQL +client aren't supported. + + + To check which pre-aggregations are managed by Cube Store, you could run the following query: @@ -283,8 +575,8 @@ Alternatively, you can store pre-aggregations **internally** in the source database. To store a pre-aggregation internally, set `external: false` in pre-aggregation definition. -Please note, that [originalSql][ref-original-sql] pre-aggregations are stored -**internally** by default. It is not recommended to store `originalSql` +Please note, that [original_sql][ref-original-sql] pre-aggregations are stored +**internally** by default. It is not recommended to store `original_sql` pre-aggregations in Cube Store. ## Joins between pre-aggregations @@ -301,57 +593,103 @@ When making a query that joins data from two different cubes, Cube can use pre-aggregations instead of running the base SQL queries. To get started, first ensure both cubes have valid pre-aggregations: -```javascript -// Orders -cube(`Orders`, { - sql: `SELECT * FROM public.orders`, + + +```yaml +cubes: + - name: orders + # ... + + pre_aggregations: + - name: orders_rollup + measures: + - CUBE.count + dimensions: + - CUBE.user_id + - CUBE.status + time_dimension: CUBE.created_at + granularity: day + + joins: + - name: users + sql: "{CUBE.user_id} = ${users.id}" + relationship: many_to_one + + - name: users + # ... + + pre_aggregations: + - name: users_rollup + dimensions: + - CUBE.id + - CUBE.name +``` - ..., +```javascript +cube(`orders`, { + // ... - preAggregations: { - ordersRollup: { + pre_aggregations: { + orders_rollup: { measures: [CUBE.count], - dimensions: [CUBE.userId, CUBE.status], - timeDimension: CUBE.createdAt, + dimensions: [CUBE.user_id, CUBE.status], + time_dimension: CUBE.created_at, granularity: `day`, }, }, joins: { - Users: { - sql: `${CUBE.userId} = ${Users.id}`, - relationship: `belongsTo` + users: { + sql: `${CUBE.user_id} = ${users.id}`, + relationship: `many_to_one` }, }, }); -// Users -cube(`Users`, { - sql: `SELECT * FROM public.users`, +cube(`users`, { + // ... - ..., - - preAggregations: { - usersRollup: { + pre_aggregations: { + users_rollup: { dimensions: [CUBE.id, CUBE.name], }, }, }); ``` -Before we continue, let's add an index to the `ordersRollup` pre-aggregation so -that the `rollupJoin` pre-aggregation can work correctly: + + +Before we continue, let's add an index to the `orders_rollup` pre-aggregation so +that the `rollup_join` pre-aggregation can work correctly: + + + +```yaml +cubes: + - name: orders + # ... + + pre_aggregations: + - name: orders_rollup + # ... + + indexes: + - name: user_index + columns: + - CUBE.user_id +``` ```javascript -cube(`Orders`, { - ..., +cube(`orders`, { + // ... + + pre_aggregations: { + orders_rollup: { + // ... - preAggregations: { - ordersRollup: { - ..., indexes: { - userIndex: { - columns: [CUBE.userId], + user_index: { + columns: [CUBE.user_id], }, }, }, @@ -359,41 +697,70 @@ cube(`Orders`, { }); ``` -Now we can add a new pre-aggregation of type `rollupJoin` to the `Orders` cube: + + +Now we can add a new pre-aggregation of type `rollup_join` to the `orders` cube: + + + +```yaml +cubes: + - name: orders + # ... + + pre_aggregations: + # ... + + - name: orders_with_users_rollup + type: rollup_join + measures: + - CUBE.count + dimensions: + - users.name + time_dimension: CUBE.created_at + granularity: day + rollups: + - users.users_rollup + - CUBE.orders_rollup +``` ```javascript -cube(`Orders`, { - ..., +cube(`orders`, { + // ... - preAggregations: { - ordersWithUsersRollup: { - type: `rollupJoin`, + pre_aggregations: { + // ... + + orders_with_users_rollup: { + type: `rollup_join`, measures: [CUBE.count], - dimensions: [Users.name], - timeDimension: CUBE.createdAt, + dimensions: [users.name], + time_dimension: CUBE.created_at, granularity: `day`, - rollups: [Users.usersRollup, CUBE.ordersRollup], + rollups: [users.users_rollup, CUBE.orders_rollup], }, }, }); ``` + + With all of the above set up, making a query such as the following will now use -`Orders.ordersRollup` and `Users.usersRollup`, avoiding a database request: +`orders.orders_rollup` and `users.users_rollup`, avoiding a database request: ```json { - "dimensions": ["Users.name"], + "dimensions": ["users.name"], "timeDimensions": [ { - "dimension": "Orders.createdAt", + "dimension": "orders.created_at", "dateRange": "This month" } ], "order": { - "Orders.count": "desc" + "orders.count": "desc" }, - "measures": ["Orders.count"] + "measures": ["orders.count"] } ``` @@ -427,7 +794,7 @@ or [Export Bucket][self-export-bucket] strategies instead.
Internal vs External vs External with Cube Store diagram @@ -435,21 +802,20 @@ or [Export Bucket][self-export-bucket] strategies instead. ### <--{"id" : "Pre-Aggregation Build Strategies"}--> Batching -Batching is a more performant strategy where Cube.js sends compressed CSVs for -Cube Store to ingest. +Batching is a more performant strategy where Cube sends compressed CSVs for Cube +Store to ingest.
Internal vs External vs External with Cube Store diagram
-The performance scales to the amount of memory available on the Cube.js -instance. Batching is automatically enabled for any databases that can support -it. +The performance scales to the amount of memory available on the Cube instance. +Batching is automatically enabled for any databases that can support it. ### <--{"id" : "Pre-Aggregation Build Strategies"}--> Export bucket @@ -468,7 +834,7 @@ Cube Store in parallel:
Internal vs External vs External with Cube Store diagram @@ -483,30 +849,34 @@ refer to the database-specific documentation for more details: - [Snowflake][ref-connect-db-snowflake] When using cloud storage, it is important to correctly configure any data -retention policies to clean up the data in the export bucket as Cube.js does not +retention policies to clean up the data in the export bucket as Cube does not currently manage this. For most use-cases, 1 day is sufficient. ## Streaming pre-aggregations -Streaming pre-aggregations are different from traditional pre-aggregations in the way they are being updated. Traditional pre-aggregations follow the “pull” model — Cube **pulls updates** from the data source based on some cadence and/or condition. Streaming pre-aggregations follow the “push” model — Cube **subscribes to the updates** from the data source and always keeps pre-aggregation up to date. - -You don’t need to define `refreshKey` for streaming pre-aggregations. Whether pre-aggregation is streaming or not is defined by the data source. - -Currently, Cube supports only one streaming data source - [ksqlDB](/config/databases/ksqldb). All pre-aggregations where data source is ksqlDB are streaming. +Streaming pre-aggregations are different from traditional pre-aggregations in +the way they are being updated. Traditional pre-aggregations follow the “pull” +model — Cube **pulls updates** from the data source based on some cadence and/or +condition. Streaming pre-aggregations follow the “push” model — Cube +**subscribes to the updates** from the data source and always keeps +pre-aggregation up to date. -We’re working on supporting streaming pre-aggregations for the following data sources - +You don’t need to define `refresh_key` for streaming pre-aggregations. Whether +pre-aggregation is streaming or not is defined by the data source. -- Materialize -- Flink SQL -- Spark Streaming +Currently, Cube supports only one streaming data source - +[ksqlDB](/config/databases/ksqldb). All pre-aggregations where data source is +ksqlDB are streaming. -Please [let us know](https://cube.dev/contact) if you are interested in early access to any of these drivers or would like Cube to support any other SQL streaming engine. +We are working on supporting more data sources for streaming pre-aggregations, +please [let us know](https://cube.dev/contact) if you are interested in early +access to any of these drivers or would like Cube to support any other SQL +streaming engine. [ref-caching-in-mem-default-refresh-key]: /caching#default-refresh-keys [ref-config-db]: /config/databases [ref-config-driverfactory]: /config#driver-factory -[ref-config-env]: /reference/environment-variables#cube-store -[ref-config-env-general]: /config#general +[ref-config-env-rolluponly]: /reference/environment-variables#cubejs-rollup-only [ref-config-extdriverfactory]: /config#external-driver-factory [ref-connect-db-athena]: /config/databases/aws-athena [ref-connect-db-redshift]: /config/databases/aws-redshift diff --git a/docs/content/Caching/build-batching.png b/docs/content/Caching/build-batching.png deleted file mode 100755 index d1e28b3b88c95..0000000000000 Binary files a/docs/content/Caching/build-batching.png and /dev/null differ diff --git a/docs/content/Caching/build-export-bucket.png b/docs/content/Caching/build-export-bucket.png deleted file mode 100755 index 7da2425938d20..0000000000000 Binary files a/docs/content/Caching/build-export-bucket.png and /dev/null differ diff --git a/docs/content/Caching/build-regular.png b/docs/content/Caching/build-regular.png deleted file mode 100644 index af4c3a2ef3c67..0000000000000 Binary files a/docs/content/Caching/build-regular.png and /dev/null differ diff --git a/docs/content/Caching/lambda-batch-source.png b/docs/content/Caching/lambda-batch-source.png deleted file mode 100644 index 6259534a4f0dc..0000000000000 Binary files a/docs/content/Caching/lambda-batch-source.png and /dev/null differ diff --git a/docs/content/Caching/lambda-batch-streaming.png b/docs/content/Caching/lambda-batch-streaming.png deleted file mode 100644 index d973cf35b9d00..0000000000000 Binary files a/docs/content/Caching/lambda-batch-streaming.png and /dev/null differ diff --git a/docs/content/Caching/pre-agg-selection-flow.png b/docs/content/Caching/pre-agg-selection-flow.png deleted file mode 100644 index 27fc1b4085a2e..0000000000000 Binary files a/docs/content/Caching/pre-agg-selection-flow.png and /dev/null differ diff --git a/docs/content/Caching/pre-aggregations.png b/docs/content/Caching/pre-aggregations.png deleted file mode 100644 index f05fe49281b47..0000000000000 Binary files a/docs/content/Caching/pre-aggregations.png and /dev/null differ diff --git a/docs/content/Configuration/Advanced/Multiple-Data-Sources.mdx b/docs/content/Configuration/Advanced/Multiple-Data-Sources.mdx new file mode 100644 index 0000000000000..b7956480fc01d --- /dev/null +++ b/docs/content/Configuration/Advanced/Multiple-Data-Sources.mdx @@ -0,0 +1,112 @@ +--- +title: Multiple data sources +permalink: /config/multiple-data-sources +category: Configuration +subCategory: Advanced +menuOrder: 2 +--- + +Cube supports connecting to multiple data sources, so that different +[cubes](/schema/reference/cube) reference data from different databases. + +Usually, data sources are configured **statically** (see below). However, Cube +can also lookup data sources **dynamically** which is useful in complex +scenarios involving [multitenancy][ref-config-multitenancy]. + +## Environment variables + +Declare the list of data sources using the `CUBEJS_DATASOURCES` environment +variable, then use +[decorated environment variables](#decorated-environment-variables) to configure +each data source: + +```dotenv +CUBEJS_DATASOURCES=default,datasource1 +CUBEJS_DB_TYPE=postgres +CUBEJS_DB_NAME=ecom +CUBEJS_DB_HOST=localhost +CUBEJS_DS_DATASOURCE1_DB_TYPE=postgres +CUBEJS_DS_DATASOURCE1_DB_NAME=ecom +CUBEJS_DS_DATASOURCE1_DB_HOST=remotehost +``` + + + +Cube expects that the `default` data source is **always** defined. Ensure that +`CUBEJS_DB_*` environment variables are set **or** that the `default` data +source is defined using [`driverFactory`] [ref-config-ref-driverfactory] in your +`cube.js` file. + + + +### Decorated environment variables + +Cube allows database-specific environment variables to be decorated with a data +source name: + +```dotenv +CUBEJS_[DS__] +``` + +For example, using the `datasource1` data source, `CUBEJS_DB_TYPE` could be +decorated as: + +```dotenv +CUBEJS_DS_DATASOURCE1_DB_TYPE=postgres +``` + +For more information on environment variables that support decoration, check the +[environment variables reference][ref-config-ref-env] or [database-specific +pages][ref-config-db]. + +## Data model + +Use the [`data_source`](/schema/reference/cube#parameters-data-source) property +to set a data source for each cube: + + + +```yaml +cubes: + - name: orders + # ... + + data_source: default + + - name: orders_from_other_data_source + # ... + + data_source: other_data_source +``` + +```javascript +cube(`orders`, { + // ... + + data_source: `default`, +}); + +cube(`orders_from_other_data_source`, { + // ... + + data_source: `other_data_source`, +}); +``` + + + +[ref-config-ref-env]: /reference/environment-variables +[ref-config-ref-driverfactory]: /config#options-reference-driver-factory +[ref-config-db]: /config/databases +[ref-config-multitenancy]: + /config/multitenancy#multitenancy-multitenancy-vs-multiple-data-sources + +## Cube Cloud + +Follow these steps to connect to multiple data sources in Cube Cloud: + +- Set up the `default` database connection when creating a new deployment. +- Ensure you have the correct [multitenancy](/config/multitenancy) configuration + in your `cube.js` file. +- Configure the corresponding environment variables in Settings → + Environment variables. diff --git a/docs/content/Configuration/Advanced/Multitenancy.mdx b/docs/content/Configuration/Advanced/Multitenancy.mdx new file mode 100644 index 0000000000000..d7943a37cbffc --- /dev/null +++ b/docs/content/Configuration/Advanced/Multitenancy.mdx @@ -0,0 +1,391 @@ +--- +title: Multitenancy +permalink: /config/multitenancy +category: Configuration +subCategory: Advanced +menuOrder: 3 +--- + +Cube supports multitenancy out of the box, both on database and data model +levels. Multiple drivers are also supported, meaning that you can have one +customer’s data in MongoDB and others in Postgres with one Cube instance. + +There are 6 [configuration options][ref-config-opts] you can leverage to make +your multitenancy setup. You can use all of them or just a couple, depending on +your specific case. The options are: + +- `contextToAppId` +- `contextToOrchestratorId` +- `driverFactory` +- `repositoryFactory` +- `preAggregationsSchema` +- `queryRewrite` + +All of the above options are functions, which you provide to Cube in the +[`cube.js` configuration file][ref-config]. The functions accept one argument - +a context object, which has a [`securityContext`][ref-config-security-ctx] +property where you can provide all the necessary data to identify a user e.g., +organization, app, etc. By default, the +[`securityContext`][ref-config-security-ctx] is defined by [Cube API +Token][ref-security]. + +There are several multitenancy setup scenarios that can be achieved by using +combinations of these configuration options. + +### <--{"id" : "Multitenancy"}--> Multitenancy vs Multiple Data Sources + +In cases where your Cube data model is spread across multiple different data +sources, consider using the [`data_source` cube property][ref-cube-datasource] +instead of multitenancy. Multitenancy is designed for cases where you need to +serve different datasets for multiple users, or tenants which aren't related to +each other. + +On the other hand, multitenancy can be used for scenarios where users need to +access the same data but from different databases. The multitenancy and multiple +data sources features aren't mutually exclusive and can be used together. + + + +A `default` data source **must** exist and be configured. It is used to resolve +target query data source for now. This behavior **will** be changed in future +releases. + + + +A simple configuration with two data sources might look like: + +**cube.js:** + +```javascript +module.exports = { + driverFactory: ({ dataSource } = {}) => { + if (dataSource === 'db1') { + return { + type: 'postgres', + database: process.env.DB1_NAME, + host: process.env.DB1_HOST, + user: process.env.DB1_USER, + password: process.env.DB1_PASS, + port: process.env.DB1_PORT, + }; + } else { + return { + type: 'postgres', + database: process.env.DB2_NAME, + host: process.env.DB2_HOST, + user: process.env.DB2_USER, + password: process.env.DB2_PASS, + port: process.env.DB2_PORT, + }; + } + }, +}; +``` + +A more advanced example that uses multiple [data sources][ref-config-db] could +look like: + +**cube.js:** + +```javascript +module.exports = { + driverFactory: ({ dataSource } = {}) => { + if (dataSource === 'web') { + return { + type: 'athena', + database: dataSource, + + // ... + }; + } else if (dataSource === 'googleAnalytics') { + return { + type: 'bigquery', + + // ... + }; + } else if (dataSource === 'financials') { + return { + type: 'postgres', + database: 'financials', + host: 'financials-db.acme.com', + user: process.env.FINANCIALS_DB_USER, + password: process.env.FINANCIALS_DB_PASS, + }; + } else { + return { + type: 'postgres', + + // ... + }; + } + }, +}; +``` + +More information can be found on the [Multiple Data Sources +page][ref-config-multi-data-src]. + +### <--{"id" : "Multitenancy"}--> queryRewrite vs Multitenant Compile Context + +As a rule of thumb, the [`queryRewrite`][ref-config-query-rewrite] should be +used in scenarios when you want to define row-level security within the same +database for different users of such database. For example, to separate access +of two e-commerce administrators who work on different product categories within +the same e-commerce store, you could configure your project as follows. + +Use the following `cube.js` configuration file: + +```javascript +module.exports = { + queryRewrite: (query, { securityContext }) => { + if (securityContext.categoryId) { + query.filters.push({ + member: 'products.category_id', + operator: 'equals', + values: [securityContext.categoryId], + }); + } + return query; + }, +}; +``` + +Also, you can use a data model like this: + + + +```yaml +cubes: + - name: products + sql_table: products +``` + +```javascript +cube(`products`, { + sql_table: `products`, +}); +``` + + + +On the other hand, multi-tenant [`COMPILE_CONTEXT`][ref-cube-security-ctx] +should be used when users need access to different databases. For example, if +you provide SaaS ecommerce hosting and each of your customers have a separate +database, then each e-commerce store should be modeled as a separate tenant. + + + +```yaml +cubes: + - name: products + sql_table: "{COMPILE_CONTEXT.security_context.userId}.products" +``` + +```javascript +cube(`products`, { + sql_table: `${COMPILE_CONTEXT.security_context.userId}.products` +}); +``` + + + +### <--{"id" : "Multitenancy"}--> Running in Production + +Each unique id generated by `contextToAppId` or `contextToOrchestratorId` will +generate a dedicated set of resources, including data model compile cache, SQL +compile cache, query queues, in-memory result caching, etc. Depending on your +data model complexity and usage patterns, those resources can have a pretty +sizable memory footprint ranging from single-digit MBs on the lower end and +dozens of MBs on the higher end. So you should make sure Node VM has enough +memory reserved for that. + +There're multiple strategies in terms of memory resource utilization here. The +first one is to bucket your actual tenants into variable-size buckets with +assigned `contextToAppId` or `contextToOrchestratorId` by some bucketing rule. +For example, you can bucket your biggest tenants in separate buckets and all the +smaller ones into a single bucket. This way, you'll end up with a very small +count of buckets that will easily fit a single node. + +Another strategy is to split all your tenants between different Cube nodes and +route traffic between them so that each Cube API node serves only its own set of +tenants and never serves traffic for another node. In that case, memory usage is +limited by the number of tenants served by each node. Cube Cloud utilizes +precisely this approach for scaling. Please note that in this case, you should +also split refresh workers and assign appropriate `scheduledRefreshContexts` to +them. + +## Same DB Instance with per Tenant Row Level Security + +Per tenant row-level security can be achieved by configuring +[`queryRewrite`][ref-config-query-rewrite], which adds a tenant identifier +filter to the original query. It uses the +[`securityContext`][ref-config-security-ctx] to determine which tenant is +requesting data. This way, every tenant starts to see their own data. However, +resources such as query queue and pre-aggregations are shared between all +tenants. + +**cube.js:** + +```javascript +module.exports = { + queryRewrite: (query, { securityContext }) => { + const user = securityContext; + if (user.id) { + query.filters.push({ + member: 'users.id', + operator: 'equals', + values: [user.id], + }); + } + return query; + }, +}; +``` + +## Multiple DB Instances with Same Data Model + +Let's consider an example where we store data for different users in different +databases, but on the same Postgres host. The database name format is +`my_app__`, so `my_app_1_2` is a valid database name. + +To make it work with Cube, first we need to pass the `appId` and `userId` as +context to every query. We should first ensure our JWTs contain those properties +so we can access them through the [security context][ref-config-security-ctx]. + +```javascript +const jwt = require('jsonwebtoken'); +const CUBE_API_SECRET = 'secret'; + +const cubejsToken = jwt.sign({ appId: '1', userId: '2' }, CUBE_API_SECRET, { + expiresIn: '30d', +}); +``` + +Now, we can access them through the [`securityContext`][ref-config-security-ctx] +property inside the context object. Let's use +[`contextToAppId`][ref-config-ctx-to-appid] and +[`contextToOrchestratorId`][ref-config-ctx-to-orch-id] to create a dynamic Cube +App ID and Orchestrator ID for every combination of `appId` and `userId`, as +well as defining [`driverFactory`][ref-config-driverfactory] to dynamically +select the database, based on the `appId` and `userId`: + + + +The App ID (the result of [`contextToAppId`][ref-config-ctx-to-appid]) is used +as a caching key for various in-memory structures like data model compilation +results, connection pool. The Orchestrator ID (the result of +[`contextToOrchestratorId`][ref-config-ctx-to-orch-id]) is used as a caching key +for database connections, execution queues and pre-aggregation table caches. Not +declaring these properties will result in unexpected caching issues such as the +data model or data of one tenant being used for another. + + + +**cube.js:** + +```javascript +module.exports = { + contextToAppId: ({ securityContext }) => + `CUBEJS_APP_${securityContext.appId}_${securityContext.userId}`, + contextToOrchestratorId: ({ securityContext }) => + `CUBEJS_APP_${securityContext.appId}_${securityContext.userId}`, + driverFactory: ({ securityContext }) => ({ + type: 'postgres', + database: `my_app_${securityContext.appId}_${securityContext.userId}`, + }), +}; +``` + +## Same DB Instance with per Tenant Pre-Aggregations + +To support per-tenant pre-aggregation of data within the same database instance, +you should configure the [`preAggregationsSchema`][ref-config-preagg-schema] +option in your `cube.js` configuration file. You should use also +[`securityContext`][ref-config-security-ctx] to determine which tenant is +requesting data. + +**cube.js:** + +```javascript +module.exports = { + contextToAppId: ({ securityContext }) => + `CUBEJS_APP_${securityContext.userId}`, + preAggregationsSchema: ({ securityContext }) => + `pre_aggregations_${securityContext.userId}`, +}; +``` + +## Multiple Data Models and Drivers + +What if for application with ID 3, the data is stored not in Postgres, but in +MongoDB? + +We can instruct Cube to connect to MongoDB in that case, instead of Postgres. To +do this, we'll use the [`driverFactory`][ref-config-driverfactory] option to +dynamically set database type. We will also need to modify our +[`securityContext`][ref-config-security-ctx] to determine which tenant is +requesting data. Finally, we want to have separate data models for every +application. We can use the [`repositoryFactory`][ref-config-repofactory] option +to dynamically set a repository with data model files depending on the `appId`: + +**cube.js:** + +```javascript +const { FileRepository } = require('@cubejs-backend/server-core'); + +module.exports = { + contextToAppId: ({ securityContext }) => + `CUBEJS_APP_${securityContext.appId}_${securityContext.userId}`, + contextToOrchestratorId: ({ securityContext }) => + `CUBEJS_APP_${securityContext.appId}_${securityContext.userId}`, + driverFactory: ({ securityContext }) => { + if (securityContext.appId === 3) { + return { + type: 'mongobi', + database: `my_app_${securityContext.appId}_${securityContext.userId}`, + port: 3307, + }; + } else { + return { + type: 'postgres', + database: `my_app_${securityContext.appId}_${securityContext.userId}`, + }; + } + }, + repositoryFactory: ({ securityContext }) => + new FileRepository(`model/${securityContext.appId}`), +}; +``` + +## Scheduled Refreshes for Pre-Aggregations + +If you need scheduled refreshes for your pre-aggregations in a multi-tenant +deployment, ensure you have configured +[`scheduledRefreshContexts`][ref-config-refresh-ctx] correctly. You may also +need to configure [`scheduledRefreshTimeZones`][ref-config-refresh-tz]. + + + +Leaving [`scheduledRefreshContexts`][ref-config-refresh-ctx] unconfigured will +lead to issues where the security context will be `undefined`. This is because +there is no way for Cube to know how to generate a context without the required +input. + + + +[ref-config]: /config +[ref-config-opts]: /config#options-reference +[ref-config-db]: /config/databases +[ref-config-driverfactory]: /config#driver-factory +[ref-config-repofactory]: /config#repository-factory +[ref-config-preagg-schema]: /config#pre-aggregations-schema +[ref-config-ctx-to-appid]: /config#context-to-app-id +[ref-config-ctx-to-orch-id]: /config#context-to-orchestrator-id +[ref-config-multi-data-src]: /config/multiple-data-sources +[ref-config-query-rewrite]: /config#query-rewrite +[ref-config-refresh-ctx]: /config#scheduled-refresh-contexts +[ref-config-refresh-tz]: /config#scheduled-refresh-time-zones +[ref-config-security-ctx]: /config#security-context +[ref-security]: /security +[ref-cube-datasource]: /schema/reference/cube#data-source +[ref-cube-security-ctx]: /schema/reference/cube#security-context diff --git a/docs/content/Configuration/Config.mdx b/docs/content/Configuration/Config.mdx deleted file mode 100644 index ddf25e1181589..0000000000000 --- a/docs/content/Configuration/Config.mdx +++ /dev/null @@ -1,842 +0,0 @@ ---- -title: Config -permalink: /config -category: Configuration -subCategory: Reference -menuOrder: 3 ---- - -Cube.js can be configured both via environment variables and by providing -configuration options in the `cube.js` file. - -Example of setting a custom logger in the `cube.js` file. - -```javascript -module.exports = { - logger: (msg, params) => { - console.log(`${msg}: ${JSON.stringify(params)}`); - }, -}; -``` - -## Options Reference - -You can provide the following configuration options to Cube.js. - -```typescript -interface CubejsConfiguration { - dbType: string | ((context: RequestContext) => string); - schemaPath: string; - basePath: string; - webSocketsBasePath: string; - logger: (msg: string, params: object) => any; - driverFactory: ( - context: DriverContext - ) => DriverConfig | BaseDriver | Promise | Promise; - contextToAppId: (context: RequestContext) => string; - contextToOrchestratorId: (context: RequestContext) => string; - repositoryFactory: (context: RequestContext) => SchemaFileRepository; - checkAuth: (req: ExpressRequest, authorization: string) => any; - checkSqlAuth: (req: SQLRequest, user: string | null) => any; - canSwitchSqlUser: ( - current: string | null, - user: string - ) => Promise | bool; - queryRewrite: (query: object, context: RequestContext) => object; - preAggregationsSchema: string | ((context: RequestContext) => string); - schemaVersion: (context: RequestContext) => string; - scheduledRefreshTimer: boolean | number; - scheduledRefreshTimeZones: string[]; - scheduledRefreshContexts: () => Promise; - extendContext: (req: ExpressRequest) => any; - compilerCacheSize: number; - maxCompilerCacheKeepAlive: number; - updateCompilerCacheKeepAlive: boolean; - allowUngroupedWithoutPrimaryKey: boolean; - telemetry: boolean; - http: { - cors: { - methods: string | string[]; - origin: string; - allowedHeaders: string | string[]; - exposedHeaders: string | string[]; - credentials: boolean; - maxAge: number; - preflightContinue: boolean; - optionsSuccessStatus: number; - }; - }; - jwt: { - jwkUrl?: ((payload: any) => string) | string; - key?: string; - algorithms?: string[]; - issuer?: string[]; - audience?: string; - subject?: string; - claimsNamespace?: string; - }; - cacheAndQueueDriver: 'memory' | 'redis'; - orchestratorOptions: - | OrchestratorOptions - | ((context: RequestContext) => OrchestratorOptions); - allowJsDuplicatePropsInSchema: boolean; - initApp: (app: ExpressApplication) => void; - processSubscriptionsInterval: number; -} - -interface OrchestratorOptions { - redisPrefix: string; - queryCacheOptions: { - refreshKeyRenewalThreshold: number; - backgroundRenew: boolean; - queueOptions: QueueOptions; - }; - preAggregationsOptions: { - externalRefresh: boolean; - maxPartitions: number; - queueOptions: QueueOptions; - }; -} - -interface QueueOptions { - concurrency: number; - continueWaitTimeout: number; - executionTimeout: number; - orphanedTimeout: number; - heartBeatInterval: number; -} - -interface RequestContext { - securityContext: object; - requestId: string; -} - -interface DriverContext extends RequestContext { - dataSource: string; -} - -interface SchemaFileRepository { - dataSchemaFiles(): Promise; -} - -interface FileContent { - fileName: string; - content: string; -} -``` - -### <--{"id" : "Options Reference"}--> dbType - - - -Since v0.30.30, using `dbType` is discouraged. Instead of using `dbType`, -consider defining [`driverFactory`][self-driver-factory] to return a -`DriverConfig` object instead. - - - -Either `String` or `Function` could be passed. Providing a `Function` allows to -dynamically select a database type depending on the user's context. It is -usually used in [Multitenancy Setup][ref-multitenancy]. - -If no option is passed, Cube.js will lookup for environment variable -`CUBEJS_DB_TYPE` to resolve `dbType`. - -Called only once per [`appId`][self-opts-ctx-to-appid]. - -### <--{"id" : "Options Reference"}--> schemaPath - -Path to schema files. The default value is `/schema`. - -### <--{"id" : "Options Reference"}--> basePath - -[REST API](/rest-api) base path. The default value is `/cubejs-api`. - -### <--{"id" : "Options Reference"}--> webSocketsBasePath - -The base path for the websockets server. By default, the WebSockets server will -run on the root path. - -### <--{"id" : "Options Reference"}--> logger - -A function to setup a custom logger. It accepts the following arguments: - -- `message`: Cube.js Backend event message -- `params`: Parameters of the call - -```javascript -module.exports = { - logger: (msg, params) => { - console.log(`${msg}: ${JSON.stringify(params)}`); - }, -}; -``` - -### <--{"id" : "Options Reference"}--> driverFactory - -Set a custom database driver. The function accepts context object as an argument -to allow dynamically loading database drivers, which is usually used in -[Multitenancy Applications][ref-multitenancy]. - -Called once per [`dataSource`][ref-schema-ref-datasource]. Can return a -`Promise` which resolves to a `DriverConfig`. `DriverConfig` consists of a -`type` field corresponding to database type and options passed to a driver -constructor. - -```javascript -module.exports = { - driverFactory: ({ dataSource }) => ({ - type: 'postgres', - database: dataSource, - }), -}; -``` - -Drivers can also be instantiated directly in case custom driver implementations -are used. - -```javascript -const PostgresDriver = require('@cubejs-backend/postgres-driver'); - -module.exports = { - driverFactory: ({ dataSource }) => - new PostgresDriver({ database: dataSource }), - dbType: ({ dataSource }) => 'postgres', -}; -``` - -### <--{"id" : "Options Reference"}--> contextToAppId - -It is a [Multitenancy Setup][ref-multitenancy] option. - -`contextToAppId` is a function to determine an App ID which is used as caching -key for various in-memory structures like schema compilation results, connection -pool, etc. - -Called on each request. - -```javascript -module.exports = { - contextToAppId: ({ securityContext }) => - `CUBEJS_APP_${securityContext.user_id}`, -}; -``` - -### <--{"id" : "Options Reference"}--> contextToOrchestratorId - - - -In versions of Cube.js prior to v0.29, each tenant would have an individual -instance of the Query Orchestrator. - - - -`contextToOrchestratorId` is a function used to determine a caching key for the -Query Orchestrator instance. The Query Orchestrator holds database connections, -execution queues, pre-aggregation table caches. By default, the same instance is -used for **all** tenants; override this property in situations where each tenant -requires their own Query Orchestrator. - -Called on each request. - -```javascript -module.exports = { - contextToAppId: ({ securityContext }) => - `CUBEJS_APP_${securityContext.tenantId}`, - contextToOrchestratorId: ({ securityContext }) => - `CUBEJS_APP_${securityContext.tenantId}`, -}; -``` - -### <--{"id" : "Options Reference"}--> repositoryFactory - -This option allows to customize the repository for Cube.js data schema files. It -is a function, which accepts a context object and can dynamically select -repositories with schema files based on -[`SchemaFileRepository`][self-schemafilerepo] contract. Learn more about it in -[Multitenancy guide][ref-multitenancy]. - -Called only once per [`appId`][self-opts-ctx-to-appid]. - -```javascript -const FileRepository = require('@cubejs-backend/server-core/core/FileRepository'); - -// using built-in SchemaFileRepository implementation and supplying the path to schema files -module.exports = { - repositoryFactory: ({ securityContext }) => - new FileRepository(`schema/${securityContext.appId}`), -}; - -// supplying your own SchemaFileRepository implementation to return array of files -module.exports = { - repositoryFactory: ({ securityContext }) => { - return { - dataSchemaFiles: async () => - await Promise.resolve([ - { fileName: 'file.js', content: 'contents of file' }, - ]), - }; - }, -}; -``` - -### <--{"id" : "Options Reference"}--> checkAuth - -Used in both REST and WebSockets API. Can be an `async` function. Default -implementation parses [JSON Web Tokens (JWT)][link-jwt] in `Authorization` -header and sets payload to `req.securityContext` if it's verified. More -information on how to generate these tokens is [here][ref-sec-ctx]. - -You can set `req.securityContext = userContextObj` inside the middleware if you -want to customize [`SECURITY_CONTEXT`][ref-schema-cube-ref-ctx-sec-ctx]. - -Called on each request. - -Also, you can use empty `checkAuth` function to disable built-in security. See -an example below. - -```javascript -module.exports = { - checkAuth: (req, auth) => {}, -}; -``` - -### <--{"id" : "Options Reference"}--> checkSqlAuth - -Used in [SQL API][ref-sql-api], and can be an `async` function. Default -implementation verify username & password from environment variables: -`CUBEJS_SQL_USER`, `CUBEJS_SQL_PASSWORD`, but in [development -mode][ref-development-mode] it ignores validation. - -Called on each request from Cube SQL API. - -For example, you can use `checkSqlAuth` to validate username and password. - -```javascript -module.exports = { - checkSqlAuth: (req, username) => { - if (username === 'fooUser') { - return { - password: 'mypassword', - securityContext: {}, - }; - } - - throw new Error('Incorrect user name or password'); - }, -}; -``` - -### <--{"id" : "Options Reference"}--> canSwitchSqlUser - -Used in [SQL API][ref-sql-api], and can be an `async` function. Default -implementation depends on `CUBEJS_SQL_SUPERUSER` and return `true` when it's -equal to session's user. - -Called on each change request from Cube SQL API. - -For example, you can use `canSwitchSqlUser` to define your custom logic: - -```javascript -module.exports = { - canSwitchSqlUser: async (current, username) => { - if (current === 'admin') { - return true; - } - - if (current === 'service') { - return username !== 'admin'; - } - - return false; - }, -}; -``` - -### <--{"id" : "Options Reference"}--> queryRewrite - - - -In previous versions of Cube.js, this was called `queryTransformer`. - - - -This is a security hook to check your query just before it gets processed. You -can use this very generic API to implement any type of custom security checks -your app needs and transform input query accordingly. - -Called on each request. - -For example, you can use `queryRewrite` to add row level security filter where -needed. - -```javascript -module.exports = { - queryRewrite: (query, { securityContext }) => { - if (securityContext.filterByRegion) { - query.filters.push({ - member: 'Regions.id', - operator: 'equals', - values: [securityContext.regionId], - }); - } - return query; - }, -}; -``` - -### <--{"id" : "Options Reference"}--> preAggregationsSchema - -Schema name to use for storing pre-aggregations. For some drivers like MySQL -it's name for pre-aggregation database as there's no database schema concept -there. Either `String` or `Function` could be passed. Providing a `Function` -allows to dynamically set the pre-aggregation schema name depending on the -user's context. - -Defaults to `dev_pre_aggregations` in [development mode][ref-development-mode] -and `prod_pre_aggregations` in production. - -Can be also set via environment variable `CUBEJS_PRE_AGGREGATIONS_SCHEMA`. - - - -We **strongly** recommend using different pre-aggregation schemas in development -and production environments to avoid pre-aggregation tables clashes. - - - -Called once per [`appId`][self-opts-ctx-to-appid]. - -```javascript -// Static usage -module.exports = { - preAggregationsSchema: `my_pre_aggregations`, -}; - -// Dynamic usage -module.exports = { - preAggregationsSchema: ({ securityContext }) => - `pre_aggregations_${securityContext.tenantId}`, -}; -``` - -### <--{"id" : "Options Reference"}--> schemaVersion - -Schema version can be used to tell Cube.js schema should be recompiled in case -schema code depends on dynamic definitions fetched from some external database -or API. This method is called on each request however `RequestContext` parameter -is reused per application ID as determined by -[`contextToAppId`][self-opts-ctx-to-appid]. If the returned string is different, -the schema will be recompiled. It can be used in both multi-tenant and single -tenant environments. - -```javascript -const tenantIdToDbVersion = {}; - -module.exports = { - schemaVersion: ({ securityContext }) => - tenantIdToDbVersion[securityContext.tenantId], -}; -``` - -### <--{"id" : "Options Reference"}--> scheduledRefreshTimer - - - -This is merely a refresh worker heart beat. It doesn't affect freshness of -pre-aggregations or refresh keys. Setting this value to `30s` doesn't mean -pre-aggregations would be refreshed every 30 seconds but rather checked for -freshness every 30 seconds. Please consult the [`refreshKey` -documentation][ref-pre-aggregations-refresh-key] on how to set refresh intervals -for pre-aggregations. - - - -Cube.js enables background refresh by default using the `CUBEJS_REFRESH_WORKER` -environment variable. - -```javascript -module.exports = { - scheduledRefreshTimer: 60, -}; -``` - -Learn more about [scheduled refreshes here][ref-caching-up-to-date]. - -Best practice is to run `scheduledRefreshTimer` in a separate worker Cube.js -instance. For Serverless deployments, [REST API][ref-rest-api-sched-refresh] -should be used instead. - -You may also need to configure -[`scheduledRefreshTimeZones`][self-opts-sched-refresh-tz] and -[`scheduledRefreshContexts`][self-opts-sched-refresh-ctxs]. - -### <--{"id" : "Options Reference"}--> scheduledRefreshTimeZones - -All time-based calculations performed within Cube.js are timezone-aware. Using -this property you can specify multiple timezones in [TZ Database -Name][link-wiki-tz] format e.g. `America/Los_Angeles`. The default value is -`UTC`. - -```javascript -module.exports = { - // You can define one or multiple timezones based on your requirements - scheduledRefreshTimeZones: ['America/Vancouver', 'America/Toronto'], -}; -``` - -This configuration option can be also set using the -`CUBEJS_SCHEDULED_REFRESH_TIMEZONES` environment variable. You can set a -comma-separated list of timezones to refresh in -`CUBEJS_SCHEDULED_REFRESH_TIMEZONES` environment variable. For example: - -```bash -CUBEJS_SCHEDULED_REFRESH_TIMEZONES=America/Los_Angeles,UTC -``` - -### <--{"id" : "Options Reference"}--> scheduledRefreshContexts - -When trying to configure scheduled refreshes for pre-aggregations that use the -`securityContext` inside `contextToAppId` or `contextToOrchestratorId`, you must -also set up `scheduledRefreshContexts`. This will allow Cube.js to generate the -necessary security contexts prior to running the scheduled refreshes. - - - -Leaving `scheduledRefreshContexts` unconfigured will lead to issues where the -security context will be `undefined`. This is because there is no way for -Cube.js to know how to generate a context without the required input. - - - -```javascript -module.exports = { - // scheduledRefreshContexts should return an array of `securityContext`s - scheduledRefreshContexts: async () => [ - { - securityContext: { - myappid: 'demoappid', - bucket: 'demo', - }, - }, - { - securityContext: { - myappid: 'demoappid2', - bucket: 'demo2', - }, - }, - ], -}; -``` - -### <--{"id" : "Options Reference"}--> extendContext - -Option to extend the `RequestContext` with custom values. This method is called -on each request. Can be async. - -The function should return an object which gets appended to the -[`RequestContext`][self-opts-req-ctx]. Make sure to register your value using -[`contextToAppId`][self-opts-ctx-to-appid] to use cache context for all possible -values that your extendContext object key can have. - - - -`extendContext` is applied only to requests that go through API. It isn't -applied to refresh worker execution. If you're looking for a way to provide -global environment variables for your schema please see [Execution environment -docs][ref-exec-environment-globals]. - - - -```javascript -module.exports = { - contextToAppId: (context) => `CUBEJS_APP_${context.activeOrganization}`, - extendContext: (req) => { - return { activeOrganization: req.headers.activeOrganization }; - }, -}; -``` - -You can use the custom value from extend context in your data schema like this: - -```javascript -const { activeOrganization } = COMPILE_CONTEXT; - -cube(`Users`, { - sql: `SELECT * FROM users where organization_id=${activeOrganization}`, -}); -``` - -### <--{"id" : "Options Reference"}--> compilerCacheSize - -Maximum number of compiled schemas to persist with in-memory cache. Defaults to -250, but optimum value will depend on deployed environment. When the max is -reached, will start dropping the least recently used schemas from the cache. - -### <--{"id" : "Options Reference"}--> maxCompilerCacheKeepAlive - -Maximum length of time in ms to keep compiled schemas in memory. Default keeps -schemas in memory indefinitely. - -### <--{"id" : "Options Reference"}--> updateCompilerCacheKeepAlive - -Providing `updateCompilerCacheKeepAlive: true` keeps frequently used schemas in -memory by reseting their `maxCompilerCacheKeepAlive` every time they are -accessed. - -### <--{"id" : "Options Reference"}--> allowUngroupedWithoutPrimaryKey - -Providing `allowUngroupedWithoutPrimaryKey: true` disables primary key inclusion -check for `ungrouped` queries. - -### <--{"id" : "Options Reference"}--> telemetry - -Cube.js collects high-level anonymous usage statistics for servers started in -development mode. It doesn't track any credentials, schema contents or queries -issued. This statistics is used solely for the purpose of constant cube.js -improvement. - -You can opt out of it any time by setting `telemetry` option to `false` or, -alternatively, by setting `CUBEJS_TELEMETRY` environment variable to `false`. - -```javascript -module.exports = { - telemetry: false, -}; -``` - -### <--{"id" : "Options Reference"}--> http - -#### cors - -CORS settings for the Cube.js REST API can be configured by providing an object -with options [from here][link-express-cors-opts]: - -```javascript -module.exports = { - http: { - cors: { - origin: '*', - methods: 'GET,HEAD,PUT,PATCH,POST,DELETE', - preflightContinue: false, - optionsSuccessStatus: 204, - maxAge: 86400, - credentials: true, - }, - }, -}; -``` - -### <--{"id" : "Options Reference"}--> jwt - -#### jwkUrl - -The URL from which JSON Web Key Sets (JWKS) can be retrieved. Can also be set -using `CUBEJS_JWK_URL`. - -#### key - -A JSON string that represents a cryptographic key. Similar to `API_SECRET`. Can -also be set using `CUBEJS_JWT_KEY`. - -#### algorithms - -[Any supported algorithm for decoding JWTs][gh-jsonwebtoken-algs]. Can also be -set using `CUBEJS_JWT_ALGS`. - -#### issuer - -An issuer value which will be used to enforce the [`iss` claim from inbound -JWTs][link-jwt-ref-iss]. Can also be set using `CUBEJS_JWT_ISSUER`. - -#### audience - -An audience value which will be used to enforce the [`aud` claim from inbound -JWTs][link-jwt-ref-aud]. Can also be set using `CUBEJS_JWT_AUDIENCE`. - -#### subject - -A subject value which will be used to enforce the [`sub` claim from inbound -JWTs][link-jwt-ref-sub]. Can also be set using `CUBEJS_JWT_SUBJECT`. - -#### claimsNamespace - -A namespace within the decoded JWT under which any custom claims can be found. -Can also be set using `CUBEJS_JWT_CLAIMS_NAMESPACE`. - -### <--{"id" : "Options Reference"}--> cacheAndQueueDriver - -The cache and queue driver to use for the Cube.js deployment. Defaults to -`memory` in development, `redis` in production. - -### <--{"id" : "Options Reference"}--> orchestratorOptions - - - -We **strongly** recommend leaving these options set to the defaults. Changing -these values can result in application instability and/or downtime. - - - -You can pass this object to set advanced options for Cube.js Query Orchestrator. - -| Option | Description | Default Value | -| -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | -| redisPrefix | Prefix to be set an all Redis keys | `STANDALONE` | -| rollupOnlyMode | When enabled, an error will be thrown if a query can't be served from a pre-aggregation (rollup) | `false` | -| queryCacheOptions | Query cache options for DB queries | `{}` | -| queryCacheOptions.refreshKeyRenewalThreshold | Time in seconds to cache the result of [`refreshKey`][ref-schema-cube-ref-refresh-key] check | `defined by DB dialect` | -| queryCacheOptions.backgroundRenew | Controls whether to wait in foreground for refreshed query data if `refreshKey` value has been changed. Refresh key queries or pre-aggregations are never awaited in foreground and always processed in background unless cache is empty. If `true` it immediately returns values from cache if available without [refreshKey][ref-schema-cube-ref-refresh-key] check to renew in foreground. | `false` | -| queryCacheOptions.queueOptions | Query queue options for DB queries | `{}` | -| preAggregationsOptions | Query cache options for pre-aggregations | `{}` | -| preAggregationsOptions.maxPartitions | The maximum number of partitions each pre-aggregation in a cube can use. | `10000` | -| preAggregationsOptions.queueOptions | Query queue options for pre-aggregations | `{}` | -| preAggregationsOptions.externalRefresh | When running a separate instance of Cube.js to refresh pre-aggregations in the background, this option can be set on the API instance to prevent it from trying to check for rollup data being current - it won't try to create or refresh them when this option is `true` | `false` | - -To set options for `queryCache` and `preAggregations`, set an object with key -queueOptions. `queryCacheOptions` are used while querying database tables, while -`preAggregationsOptions` settings are used to query pre-aggregated tables. - -```javascript -const queueOptions = { - concurrency: 3, -}; - -module.exports = { - orchestratorOptions: { - queryCacheOptions: { - refreshKeyRenewalThreshold: 30, - backgroundRenew: true, - queueOptions, - }, - preAggregationsOptions: { queueOptions }, - }, -}; -``` - -### <--{"id" : "Options Reference"}--> allowJsDuplicatePropsInSchema - -Boolean to enable or disable a check duplicate property names in all objects of -a schema. The default value is `false`, and it is means the compiler would use -the additional transpiler for check duplicates. - -### <--{"id" : "Options Reference"}--> initApp - - - -This configuration option is likely to change in future versions of Cube - - - -This function allows you to extend the Cube API server with custom Express -middleware. This is especially useful for adding monitoring and observability -solutions. - -```typescript -const myCustomMiddleware = (req, res, next) => { - req.foo = 'bar'; - next(); -}; - -module.exports = { - initApp: (app) => { - app.use(myCustomMiddleware); - }, -}; -``` - -### <--{"id" : "Options Reference"}--> processSubscriptionsInterval - -This property controls how often Websocket client subscriptions are refreshed. -Defaults to `5000`. - -## QueueOptions - -Timeout and interval options' values are in seconds. - -| Option | Description | Default Value | -| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | -| concurrency | Maximum number of queries to be processed simultaneosly. For drivers with connection pool `CUBEJS_DB_MAX_POOL` should be adjusted accordingly. | `2` | -| continueWaitTimeout | Long polling interval | `5` | -| executionTimeout | Total timeout of single query | `600` | -| orphanedTimeout | Query will be marked for cancellation if not requested during this period. | `120` | -| heartBeatInterval | Worker heartbeat interval. If `4*heartBeatInterval` time passes without reporting, the query gets cancelled. | `30` | - -## RequestContext - -`RequestContext` object is filled by context data on a HTTP request level. - -### <--{"id" : "RequestContext"}--> securityContext - -Defined as `req.securityContext` which should be set by -[`checkAuth`][self-opts-checkauth]. Default implementation of -[`checkAuth`][self-opts-checkauth] uses [JWT Security Token][ref-sec] payload -and sets it to `req.securityContext`. - -## SchemaFileRepository - - - -The default implementation of the `SchemaFileRepository` contract is defined by -the [`FileRepository`][gh-cube-filerepo] class. When using -[`FileRepository`][gh-cube-filerepo], all schema files must be within the same -directory. - - - -[gh-cube-filerepo]: - https://github.com/cube-js/cube.js/blob/master/packages/cubejs-server-core/src/core/FileRepository.ts - -The `SchemaFileRepository` contract defines an async `dataSchemaFiles` function -which returns the files to compile for a schema. Returned by -[repositoryFactory][self-repofactory]. The [`FileRepository`][gh-cube-filerepo] -implementation of the `SchemaFileRepository` contract accepts a -[`schemaPath`][self-schemapath] in its constructor. - -```javascript -class ApiFileRepository { - async dataSchemaFiles() { - const fileContents = await callExternalApiForFileContents(); - return [{ fileName: 'apiFile', content: fileContents }]; - } -} - -module.exports = { - repositoryFactory: ({ securityContext }) => new ApiFileRepository(), -}; -``` - -[gh-jsonwebtoken-algs]: - https://github.com/auth0/node-jsonwebtoken#algorithms-supported -[link-express-cors-opts]: - https://expressjs.com/en/resources/middleware/cors.html#configuration-options -[link-jwt]: https://jwt.io/ -[link-jwt-ref-iss]: https://tools.ietf.org/html/rfc7519#section-4.1.1 -[link-jwt-ref-sub]: https://tools.ietf.org/html/rfc7519#section-4.1.2 -[link-jwt-ref-aud]: https://tools.ietf.org/html/rfc7519#section-4.1.3 -[link-wiki-tz]: https://en.wikipedia.org/wiki/Tz_database -[ref-caching-up-to-date]: /caching#keeping-cache-up-to-date -[ref-development-mode]: /overview#development-mode -[ref-multitenancy]: /multitenancy-setup -[ref-rest-api]: /rest-api -[ref-sql-api]: /backend/sql -[ref-rest-api-sched-refresh]: /rest-api#v-1-run-scheduled-refresh -[ref-pre-aggregations-refresh-key]: - /schema/reference/pre-aggregations#refresh-key -[ref-schema-cube-ref-refresh-key]: /schema/reference/cube#refresh-key -[ref-schema-cube-ref-ctx-sec-ctx]: /schema/reference/cube#security-context -[ref-schema-ref-preaggs-rollup]: /schema/reference/pre-aggregations#type-rollup -[ref-sec]: /security -[ref-sec-ctx]: /security/context -[self-opts-req-ctx]: #request-context -[self-opts-checkauth]: #check-auth -[self-opts-ctx-to-appid]: #context-to-app-id -[self-driver-factory]: #driver-factory -[ref-schema-ref-datasource]: /schema/reference/cube#data-source -[self-opts-sched-refresh-ctxs]: #scheduled-refresh-contexts -[self-opts-sched-refresh-tz]: #scheduled-refresh-time-zones -[self-repofactory]: #repositoryFactory -[self-schemafilerepo]: #schema-file-repository -[self-schemapath]: #schemaPath -[ref-exec-environment-globals]: - /schema/reference/execution-environment#node-js-globals-process-env-console-log-and-others diff --git a/docs/content/Configuration/Connecting-to-Downstream-Tools.mdx b/docs/content/Configuration/Connecting-to-Downstream-Tools.mdx index 10c14b7301b0a..870883108369a 100644 --- a/docs/content/Configuration/Connecting-to-Downstream-Tools.mdx +++ b/docs/content/Configuration/Connecting-to-Downstream-Tools.mdx @@ -1,5 +1,5 @@ --- -title: Connecting to Visualization Tools +title: Connecting to visualization tools permalink: /config/downstream category: Configuration menuOrder: 3 @@ -8,8 +8,9 @@ menuOrder: 3 Choose a tool to get started with below. - If you'd like to connect to a tool which is not yet supported or present on this page, - please file an issue on GitHub. + If you'd like to connect to a tool which is not yet supported or present on + this page, please{' '} + file an issue on GitHub. ## BI & data exploration tools @@ -38,10 +39,16 @@ Choose a tool to get started with below. imageUrl="https://raw.githubusercontent.com/cube-js/cube.js/master/docs/static/icons/metabase.svg" title="Metabase" /> + + -You can learn more about SQL API on the [reference page](/backend/sql), including how to connect -to other BIs or visualization tools not listed here. +You can learn more about SQL API on the [reference page](/backend/sql), +including how to connect to other BIs or visualization tools not listed here. ## Notebooks @@ -73,6 +80,19 @@ to other BIs or visualization tools not listed here. /> +## Custom data experiences + +The following tools deliver data insights closer to end users, e.g., by +providing a conversational interface for semantic layer: + + + + + ## Low-code tools & internal tool builders @@ -98,7 +118,8 @@ to other BIs or visualization tools not listed here. /> -You can also find relevant step-by-step guides in the blog. +You can also find relevant +step-by-step guides in the blog. ## Frontend integrations @@ -124,7 +145,9 @@ Cube provides integration libraries for popular front-end frameworks: ## APIs references -All integrations above are powered by the following APIs. If you're a data engineer, please explore the SQL API. If you're an application developer, check out REST and GraphQL APIs. +All integrations above are powered by the following APIs. If you're a data +engineer, please explore the SQL API. If you're an application developer, check +out REST and GraphQL APIs. If you'd like to connect to a data store which is not yet listed on this page, - please file an issue on GitHub. - You can also contribute a driver, see this step-by-step guide. + please file an issue on + GitHub. You can also contribute a driver, see this{' '} + + step-by-step guide + + . ## Data Warehouses @@ -62,13 +69,18 @@ Choose a data store to get started with below. url="databases/hive-sparksql" imageUrl="https://raw.githubusercontent.com/cube-js/cube.js/master/packages/cubejs-playground/src/img/db/hive.svg" title="Hive/SparkSQL" - /> + /> - + /> + + ## Transactional Databases @@ -92,12 +104,12 @@ Choose a data store to get started with below. url="databases/oracle" imageUrl="https://raw.githubusercontent.com/cube-js/cube.js/master/packages/cubejs-playground/src/img/db/oracle.svg" title="Oracle" - /> + /> + /> ## Time Series Databases @@ -126,10 +138,11 @@ Choose a data store to get started with below. ## NoSQL & Other Data Sources + -## Multiple Databases - -Cube.js supports connection to multiple databases out-of-the-box. Please refer -to the [multitenancy guide][link-multitenancy] to learn more. - -[link-multitenancy]: /multitenancy-setup - ## Driver Support -Most of the drivers for data sources are supported either directly by the Cube team -or by their vendors. The rest are community-supported and will be highlighted -as such in their respective pages. - -You're more than welcome to contribute new drivers as well as new features and patches -to [existing drivers](https://github.com/cube-js/cube.js/tree/master/packages). -Please check the [contribution guidelines](https://github.com/cube-js/cube.js/blob/master/CONTRIBUTING.md#implementing-driver) -and join the `#contributors` channel in our [Slack community](https://slack.cube.dev). \ No newline at end of file +Most of the drivers for data sources are supported either directly by the Cube +team or by their vendors. The rest are community-supported and will be +highlighted as such in their respective pages. + +## Third-party Drivers + +- [CosmosDB driver](https://www.npmjs.com/package/cosmosdb-cubejs-driver) +- [SAP Hana driver](https://www.npmjs.com/package/cubejs-hana-driver) +- [ArangoDB driver](https://www.npmjs.com/package/arangodb-cubejs-driver) +- [OpenDistro Elastic driver](https://www.npmjs.com/package/opendistro-cubejs-driver) +- [Mydremio driver](https://www.npmjs.com/package/mydremio-cubejs-driver) +- [Trino driver](https://www.npmjs.com/package/trino-cubejs-driver) +- [Cratedb driver](https://www.npmjs.com/package/cratedb-cubejs-driver) +- [Dremio ODBC driver](https://www.npmjs.com/package/dremio-odbc-cubejs-driver) + +You're more than welcome to contribute new drivers as well as new features and +patches to +[existing drivers](https://github.com/cube-js/cube/tree/master/packages). Please +check the +[contribution guidelines](https://github.com/cube-js/cube/blob/master/CONTRIBUTING.md#contributing-database-drivers) +and join the `#contributors` channel in our +[Slack community](https://slack.cube.dev). + +[ref-config-multi-data-src]: /config/multiple-data-sources diff --git a/docs/content/Configuration/Databases/AWS-Athena.mdx b/docs/content/Configuration/Databases/AWS-Athena.mdx index 9570e8269870d..babc50d631c64 100644 --- a/docs/content/Configuration/Databases/AWS-Athena.mdx +++ b/docs/content/Configuration/Databases/AWS-Athena.mdx @@ -14,34 +14,70 @@ permalink: /config/databases/aws-athena ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=athena CUBEJS_AWS_KEY=AKIA************ CUBEJS_AWS_SECRET=**************************************** CUBEJS_AWS_REGION=us-east-1 CUBEJS_AWS_S3_OUTPUT_LOCATION=s3://my-athena-output-bucket CUBEJS_AWS_ATHENA_WORKGROUP=primary +CUBEJS_AWS_ATHENA_CATALOG=AwsDataCatalog ``` +### <--{"id" : "Setup"}--> Cube Cloud + + + +In some cases you'll need to allow connections from your Cube Cloud deployment IP +address to your database. You can copy the IP address from either the Database +Setup step in deployment creation, or from Settings → +Configuration in your deployment. + + + +In Cube Cloud, select **AWS Athena** when creating a new deployment and fill in +the required fields: + + + +Cube Cloud also supports connecting to data sources within private VPCs. If you already have VPCs enabled in +your account, check out the [VPC documentation][ref-cloud-conf-vpc] to learn how +to get started. + + + +VPC connectivity is available in Cube Cloud on [Premium](https://cube.dev/pricing) tier. [Contact us][cube-contact] for details. + + + +[cube-pricing]: https://cube.dev/pricing/ +[cube-contact]: https://cube.dev/contact/ +[ref-cloud-conf-vpc]: /cloud/configuration/connecting-with-a-vpc + ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| ------------------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------ | :------: | -| `CUBEJS_AWS_KEY` | The AWS Access Key ID to use for database connections | A valid AWS Access Key ID | ✅ | -| `CUBEJS_AWS_SECRET` | The AWS Secret Access Key to use for database connections | A valid AWS Secret Access Key | ✅ | -| `CUBEJS_AWS_REGION` | The AWS region of the Cube.js deployment | [A valid AWS region][aws-docs-regions] | ✅ | -| `CUBEJS_AWS_S3_OUTPUT_LOCATION` | The S3 path to store query results made by the Cube.js deployment | A valid S3 path | ❌ | -| `CUBEJS_AWS_ATHENA_WORKGROUP` | The name of the workgroup in which the query is being started | [A valid Athena Workgroup][aws-athena-workgroup] | ❌ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `5` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| ------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_AWS_KEY` | The AWS Access Key ID to use for database connections | A valid AWS Access Key ID | ✅ | ✅ | +| `CUBEJS_AWS_SECRET` | The AWS Secret Access Key to use for database connections | A valid AWS Secret Access Key | ✅ | ✅ | +| `CUBEJS_AWS_REGION` | The AWS region of the Cube deployment | [A valid AWS region][aws-docs-regions] | ✅ | ✅ | +| `CUBEJS_AWS_S3_OUTPUT_LOCATION` | The S3 path to store query results made by the Cube deployment | A valid S3 path | ❌ | ✅ | +| `CUBEJS_AWS_ATHENA_WORKGROUP` | The name of the workgroup in which the query is being started | [A valid Athena Workgroup][aws-athena-workgroup] | ❌ | ✅ | +| `CUBEJS_AWS_ATHENA_CATALOG` | The name of the catalog to use by default | [A valid Athena Catalog name][awsdatacatalog] | ❌ | ✅ | +| `CUBEJS_DB_SCHEMA` | The name of the schema to use as `information_schema` filter. Reduces count of tables loaded during schema generation. | A valid schema name | ❌ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `5` | A valid number | ❌ | ❌ | ## Pre-Aggregation Feature Support -### countDistinctApprox +### count_distinct_approx Measures of type -[`countDistinctApprox`][ref-schema-ref-types-formats-countdistinctapprox] can be +[`count_distinct_approx`][ref-schema-ref-types-formats-countdistinctapprox] can be used in pre-aggregations when using AWS Athena as a source database. To learn more about AWS Athena's support for approximate aggregate functions, [click here][aws-athena-docs-approx-agg-fns]. @@ -79,7 +115,7 @@ AWS Athena **only** supports using AWS S3 for export buckets. For [improved pre-aggregation performance with large datasets][ref-caching-large-preaggs], enable export bucket functionality by -configuring Cube.js with the following environment variables: +configuring Cube with the following environment variables: @@ -98,12 +134,14 @@ CUBEJS_DB_EXPORT_BUCKET_AWS_REGION= ## SSL -Cube.js does not require any additional configuration to enable SSL as AWS -Athena connections are made over HTTPS. +Cube does not require any additional configuration to enable SSL as AWS Athena +connections are made over HTTPS. [aws-athena]: https://aws.amazon.com/athena [aws-athena-workgroup]: https://docs.aws.amazon.com/athena/latest/ug/workgroups-benefits.html +[awsdatacatalog]: + https://docs.aws.amazon.com/athena/latest/ug/understanding-tables-databases-and-the-data-catalog.html [aws-s3]: https://aws.amazon.com/s3/ [aws-docs-athena-access]: https://docs.aws.amazon.com/athena/latest/ug/security-iam-athena.html @@ -116,6 +154,8 @@ Athena connections are made over HTTPS. [ref-caching-large-preaggs]: /caching/using-pre-aggregations#export-bucket [ref-caching-using-preaggs-build-strats]: /caching/using-pre-aggregations#pre-aggregation-build-strategies +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables [ref-schema-ref-types-formats-countdistinctapprox]: /schema/reference/types-and-formats#count-distinct-approx [self-preaggs-batching]: #batching diff --git a/docs/content/Configuration/Databases/AWS-Redshift.mdx b/docs/content/Configuration/Databases/AWS-Redshift.mdx index 9bf144c1f783e..35187bdb4a6e2 100644 --- a/docs/content/Configuration/Databases/AWS-Redshift.mdx +++ b/docs/content/Configuration/Databases/AWS-Redshift.mdx @@ -7,8 +7,8 @@ permalink: /config/databases/aws-redshift -If the cluster is configured within a [VPC][aws-vpc], then Cube.js **must** have -a network route to the cluster. +If the cluster is configured within a [VPC][aws-vpc], then Cube **must** have a +network route to the cluster. @@ -23,9 +23,9 @@ a network route to the cluster. ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=redshift CUBEJS_DB_HOST=my-redshift-cluster.cfbs3dkw1io8.eu-west-1.redshift.amazonaws.com CUBEJS_DB_NAME=my_redshift_database @@ -33,25 +33,58 @@ CUBEJS_DB_USER=redshift_user CUBEJS_DB_PASS=********** ``` +### <--{"id" : "Setup"}--> Cube Cloud + + + +In some cases you'll need to allow connections from your Cube Cloud deployment IP +address to your database. You can copy the IP address from either the Database +Setup step in deployment creation, or from Settings → +Configuration in your deployment. + + + +The following fields are required when creating an AWS Redshift connection: + + + +Cube Cloud also supports connecting to data sources within private VPCs. If you already have VPCs enabled in +your account, check out the [VPC documentation][ref-cloud-conf-vpc] to learn how +to get started. + + + +VPC connectivity is available in Cube Cloud on [Premium](https://cube.dev/pricing) tier. [Contact us][cube-contact] for details. + + + +[cube-pricing]: https://cube.dev/pricing/ +[cube-contact]: https://cube.dev/contact/ +[ref-cloud-conf-vpc]: /cloud/configuration/connecting-with-a-vpc + ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | -| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | -| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | -| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | -| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube.js | `true`, `false` | ❌ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `4` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `16` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| -------------------------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | ✅ | +| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | ✅ | +| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube | `true`, `false` | ❌ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `4` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `16` | A valid number | ❌ | ✅ | +| `CUBEJS_DB_EXPORT_BUCKET_REDSHIFT_ARN` | | | ❌ | ✅ | ## Pre-Aggregation Feature Support -### countDistinctApprox +### count_distinct_approx Measures of type -[`countDistinctApprox`][ref-schema-ref-types-formats-countdistinctapprox] can +[`count_distinct_approx`][ref-schema-ref-types-formats-countdistinctapprox] can not be used in pre-aggregations when using AWS Redshift as a source database. ## Pre-Aggregation Build Strategies @@ -87,7 +120,7 @@ AWS Redshift **only** supports using AWS S3 for export buckets. For [improved pre-aggregation performance with large datasets][ref-caching-large-preaggs], enable export bucket functionality by -configuring Cube.js with the following environment variables: +configuring Cube with the following environment variables: @@ -106,7 +139,7 @@ CUBEJS_DB_EXPORT_BUCKET_AWS_REGION= ## SSL -To enable SSL-encrypted connections between Cube.js and AWS Redshift, set the +To enable SSL-encrypted connections between Cube and AWS Redshift, set the `CUBEJS_DB_SSL` environment variable to `true`. For more information on how to configure custom certificates, please check out [Enable SSL Connections to the Database][ref-recipe-enable-ssl]. @@ -120,6 +153,8 @@ Database][ref-recipe-enable-ssl]. [ref-caching-large-preaggs]: /caching/using-pre-aggregations#export-bucket [ref-caching-using-preaggs-build-strats]: /caching/using-pre-aggregations#pre-aggregation-build-strategies +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables [ref-recipe-enable-ssl]: /recipes/enable-ssl-connections-to-database [ref-schema-ref-types-formats-countdistinctapprox]: /schema/reference/types-and-formats#count-distinct-approx diff --git a/docs/content/Configuration/Databases/ClickHouse.mdx b/docs/content/Configuration/Databases/ClickHouse.mdx index 960ad1266f239..ab1d595e0377c 100644 --- a/docs/content/Configuration/Databases/ClickHouse.mdx +++ b/docs/content/Configuration/Databases/ClickHouse.mdx @@ -13,9 +13,9 @@ permalink: /config/databases/clickhouse ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=clickhouse CUBEJS_DB_HOST=my.clickhouse.host CUBEJS_DB_NAME=my_clickhouse_database @@ -25,23 +25,23 @@ CUBEJS_DB_PASS=********** ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| ------------------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | -| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | -| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | -| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | -| `CUBEJS_DB_CLICKHOUSE_READONLY` | Whether the ClickHouse user has read-only access or not | `true`, `false` | ❌ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `5` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `20` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| ------------------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | ✅ | +| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | ✅ | +| `CUBEJS_DB_CLICKHOUSE_READONLY` | Whether the ClickHouse user has read-only access or not | `true`, `false` | ❌ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `5` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `20` | A valid number | ❌ | ✅ | ## Pre-Aggregation Feature Support -### countDistinctApprox +### count_distinct_approx Measures of type -[`countDistinctApprox`][ref-schema-ref-types-formats-countdistinctapprox] can +[`count_distinct_approx`][ref-schema-ref-types-formats-countdistinctapprox] can not be used in pre-aggregations when using ClickHouse as a source database. ## Pre-Aggregation Build Strategies @@ -71,7 +71,7 @@ ClickHouse does not support export buckets. ## SSL -To enable SSL-encrypted connections between Cube.js and ClickHouse, set the +To enable SSL-encrypted connections between Cube and ClickHouse, set the `CUBEJS_DB_SSL` environment variable to `true`. For more information on how to configure custom certificates, please check out [Enable SSL Connections to the Database][ref-recipe-enable-ssl]. @@ -89,6 +89,8 @@ You can connect to a ClickHouse database when your user's permissions are https://clickhouse.tech/docs/en/operations/settings/permissions-for-queries/#settings_readonly [ref-caching-using-preaggs-build-strats]: /caching/using-pre-aggregations#pre-aggregation-build-strategies +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables [ref-recipe-enable-ssl]: /recipes/enable-ssl-connections-to-database [ref-schema-ref-types-formats-countdistinctapprox]: /schema/reference/types-and-formats#count-distinct-approx diff --git a/docs/content/Configuration/Databases/Databricks-JDBC.mdx b/docs/content/Configuration/Databases/Databricks-JDBC.mdx index 2f87e69b9917a..da932538b27db 100644 --- a/docs/content/Configuration/Databases/Databricks-JDBC.mdx +++ b/docs/content/Configuration/Databases/Databricks-JDBC.mdx @@ -11,7 +11,7 @@ permalink: /config/databases/databricks/jdbc ## Setup -### Manual +### <--{"id" : "Setup"}--> Environment Variables Add the following to a `.env` file in your Cube project: @@ -20,29 +20,50 @@ CUBEJS_DB_TYPE=databricks-jdbc # CUBEJS_DB_NAME is optional CUBEJS_DB_NAME=default # You can find this inside the cluster's configuration -CUBEJS_DB_DATABRICKS_URL=jdbc:spark://dbc-XXXXXXX-XXXX.cloud.databricks.com:443/default;transportMode=http;ssl=1;httpPath=sql/protocolv1/o/XXXXX/XXXXX;AuthMech=3;UID=token +CUBEJS_DB_DATABRICKS_URL=jdbc:databricks://dbc-XXXXXXX-XXXX.cloud.databricks.com:443/default;transportMode=http;ssl=1;httpPath=sql/protocolv1/o/XXXXX/XXXXX;AuthMech=3;UID=token # You can specify the personal access token separately from `CUBEJS_DB_DATABRICKS_URL` by doing this: CUBEJS_DB_DATABRICKS_TOKEN=XXXXX +# This accepts the Databricks usage policy and must be set to `true` to use the Databricks JDBC driver +CUBEJS_DB_DATABRICKS_ACCEPT_POLICY=true +``` + +### <--{"id" : "Setup"}--> Docker + +Create a `.env` file [as above](#setup-environment-variables), then extend the +`cubejs/cube:jdk` Docker image tag to build a Cube image with the JDBC driver: + +```dockerfile +FROM cubejs/cube:jdk + +COPY . . +RUN npm install +``` + +You can then build and run the image using the following commands: + +```bash{promptUser: user} +docker build -t cubejs-jdk . +docker run -it -p 4000:4000 --env-file=.env cubejs-jdk ``` ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| ------------------------------------ | ----------------------------------------------------------------------------------------------- | --------------------- | :------: | -| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | -| `CUBEJS_DB_DATABRICKS_URL` | The URL for a JDBC connection | A valid JDBC URL | ✅ | -| `CUBEJS_DB_DATABRICKS_ACCEPT_POLICY` | Whether or not to accept the license terms for the Databricks JDBC driver | `true`, `false` | ✅ | -| `CUBEJS_DB_DATABRICKS_TOKEN` | The [personal access token][databricks-docs-pat] used to authenticate the Databricks connection | A valid token | ✅ | -| `CUBEJS_DB_EXPORT_BUCKET_MOUNT_DIR` | The path for the [Databricks DBFS mount][databricks-docs-dbfs] | A valid mount path | ❌ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| ------------------------------------ | ----------------------------------------------------------------------------------------------- | --------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | ✅ | +| `CUBEJS_DB_DATABRICKS_URL` | The URL for a JDBC connection | A valid JDBC URL | ✅ | ✅ | +| `CUBEJS_DB_DATABRICKS_ACCEPT_POLICY` | Whether or not to accept the license terms for the Databricks JDBC driver | `true`, `false` | ✅ | ✅ | +| `CUBEJS_DB_DATABRICKS_TOKEN` | The [personal access token][databricks-docs-pat] used to authenticate the Databricks connection | A valid token | ✅ | ✅ | +| `CUBEJS_DB_EXPORT_BUCKET_MOUNT_DIR` | The path for the [Databricks DBFS mount][databricks-docs-dbfs] | A valid mount path | ❌ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | ✅ | ## Pre-Aggregation Feature Support -### countDistinctApprox +### count_distinct_approx Measures of type -[`countDistinctApprox`][ref-schema-ref-types-formats-countdistinctapprox] can +[`count_distinct_approx`][ref-schema-ref-types-formats-countdistinctapprox] can not be used in pre-aggregations when using Databricks as a data source. ## Pre-Aggregation Build Strategies @@ -134,10 +155,12 @@ bucket][self-preaggs-export-bucket] **must be** configured. [databricks-docs-pat]: https://docs.databricks.com/dev-tools/api/latest/authentication.html#token-management [gh-cubejs-jdbc-install]: - https://github.com/cube-js/cube.js/blob/master/packages/cubejs-jdbc-driver/README.md#java-installation + https://github.com/cube-js/cube/blob/master/packages/cubejs-jdbc-driver/README.md#java-installation [ref-caching-large-preaggs]: /caching/using-pre-aggregations#export-bucket [ref-caching-using-preaggs-build-strats]: /caching/using-pre-aggregations#pre-aggregation-build-strategies +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables [ref-schema-ref-types-formats-countdistinctapprox]: /schema/reference/types-and-formats#count-distinct-approx [self-preaggs-simple]: #simple diff --git a/docs/content/Configuration/Databases/Druid.mdx b/docs/content/Configuration/Databases/Druid.mdx index 4fe8e639da9a7..dde5671c7e6aa 100644 --- a/docs/content/Configuration/Databases/Druid.mdx +++ b/docs/content/Configuration/Databases/Druid.mdx @@ -18,9 +18,9 @@ permalink: /config/databases/druid ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=druid CUBEJS_DB_URL=https://my.druid.host:8082 CUBEJS_DB_USER=druid @@ -29,17 +29,19 @@ CUBEJS_DB_PASS=********** ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| -------------------- | ----------------------------------------------------------------------------------- | ------------------------------ | :------: | -| `CUBEJS_DB_URL` | The URL for a database | A valid database URL for Druid | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| -------------------- | ----------------------------------------------------------------------------------- | ------------------------------ | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_URL` | The URL for a database | A valid database URL for Druid | ✅ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | ✅ | ## SSL -Cube.js does not require any additional configuration to enable SSL as Druid +Cube does not require any additional configuration to enable SSL as Druid connections are made over HTTPS. [druid]: https://druid.apache.org/ +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables diff --git a/docs/content/Configuration/Databases/Elasticsearch.mdx b/docs/content/Configuration/Databases/Elasticsearch.mdx index 91798c11dbb96..da1b5bec02286 100644 --- a/docs/content/Configuration/Databases/Elasticsearch.mdx +++ b/docs/content/Configuration/Databases/Elasticsearch.mdx @@ -26,18 +26,18 @@ If you're not using [Elastic Cloud][elastic-cloud], you **must** specify ### <--{"id" : "Setup"}--> Manual For a self-hosted Elasticsearch instance, add the following to a `.env` file in -your Cube.js project: +your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=elasticsearch CUBEJS_DB_URL=https://my.elasticsearch.host:9200 CUBEJS_DB_ELASTIC_QUERY_FORMAT=json ``` For an Elasticsearch instanced hosted by [Elastic.co][elastic-co], add the -following to a `.env` file in your Cube.js project: +following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=elasticsearch CUBEJS_DB_URL=https://my.elasticsearch.host:9200 CUBEJS_DB_ELASTIC_APIKEY_ID=VuaCfGcBCdbkQm-e5aOx @@ -46,19 +46,19 @@ CUBEJS_DB_ELASTIC_APIKEY_KEY=ui2lp2axTNmsyakw9tvNnw ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| -------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------- | :------: | -| `CUBEJS_DB_URL` | The URL for a database | A valid database URL for Elasticsearch | ✅ | -| `CUBEJS_DB_ELASTIC_QUERY_FORMAT` | By default, queries return data in JDBC format, but you can also return data in standard Elasticsearch JDBC, JSON, CSV, YAML or raw formats (only JSON and JDBC are currently supported) | `json`, `jdbc` | ❌ | -| `CUBEJS_DB_ELASTIC_OPENDISTRO` | If `true`, then use the Open Distro for Elasticsearch | `true`, `false` | ❌ | -| `CUBEJS_DB_ELASTIC_APIKEY_ID` | [ID of the API key from elastic.co][elastic-docs-api-keys] | A valid Elastic.co API key ID | ❌ | -| `CUBEJS_DB_ELASTIC_APIKEY_KEY` | [Value of the API key from elastic.co][elastic-docs-api-keys] | A valid Elastic.co API key value | ❌ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| -------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_URL` | The URL for a database | A valid database URL for Elasticsearch | ✅ | ✅ | +| `CUBEJS_DB_ELASTIC_QUERY_FORMAT` | By default, queries return data in JDBC format, but you can also return data in standard Elasticsearch JDBC, JSON, CSV, YAML or raw formats (only JSON and JDBC are currently supported) | `json`, `jdbc` | ❌ | ✅ | +| `CUBEJS_DB_ELASTIC_OPENDISTRO` | If `true`, then use the Open Distro for Elasticsearch | `true`, `false` | ❌ | ✅ | +| `CUBEJS_DB_ELASTIC_APIKEY_ID` | [ID of the API key from elastic.co][elastic-docs-api-keys] | A valid Elastic.co API key ID | ❌ | ✅ | +| `CUBEJS_DB_ELASTIC_APIKEY_KEY` | [Value of the API key from elastic.co][elastic-docs-api-keys] | A valid Elastic.co API key value | ❌ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | ✅ | ## SSL -Cube.js does not require any additional configuration to enable SSL as +Cube does not require any additional configuration to enable SSL as Elasticsearch connections are made over HTTPS. [elastic-co]: https://elastic.co/ @@ -66,3 +66,5 @@ Elasticsearch connections are made over HTTPS. [elasticsearch]: https://www.elastic.co/elasticsearch/ [elastic-docs-api-keys]: https://www.elastic.co/guide/en/kibana/master/api-keys.html#create-api-key +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables diff --git a/docs/content/Configuration/Databases/Firebolt.mdx b/docs/content/Configuration/Databases/Firebolt.mdx index 0163e7e20687f..f179695bac748 100644 --- a/docs/content/Configuration/Databases/Firebolt.mdx +++ b/docs/content/Configuration/Databases/Firebolt.mdx @@ -3,6 +3,13 @@ title: Firebolt permalink: /config/databases/firebolt --- + + +The driver for Firebolt is supported by its vendor. Please report any issues to +their [Help Center][firebolt-help]. + + + ## Prerequisites - The username/password for your [Firebolt][firebolt] account @@ -13,7 +20,7 @@ permalink: /config/databases/firebolt Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_NAME=firebolt_database CUBEJS_DB_USER=firebolt_user@customer.com CUBEJS_DB_PASS=********** @@ -23,15 +30,18 @@ CUBEJS_FIREBOLT_ENGINE_NAME=engine_name ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| ------------------------------ | ----------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | :------: | -| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | -| `CUBEJS_FIREBOLT_ACCOUNT` | Account name | An account name | - | -| `CUBEJS_FIREBOLT_ENGINE_NAME` | Engine name to connect to | A valid engine name | ✅ | -| `CUBEJS_FIREBOLT_API_ENDPOINT` | Firebolt API endpoint. Used for authentication | `api.dev.firebolt.io`, `api.staging.firebolt.io`, `api.app.firebolt.io` | - | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `5` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `20` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| ------------------------------ | ----------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | ✅ | +| `CUBEJS_FIREBOLT_ACCOUNT` | Account name | An account name | - | ✅ | +| `CUBEJS_FIREBOLT_ENGINE_NAME` | Engine name to connect to | A valid engine name | ✅ | ✅ | +| `CUBEJS_FIREBOLT_API_ENDPOINT` | Firebolt API endpoint. Used for authentication | `api.dev.firebolt.io`, `api.staging.firebolt.io`, `api.app.firebolt.io` | - | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `5` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `20` | A valid number | ❌ | ✅ | [firebolt]: https://www.firebolt.io/ +[firebolt-help]: https://help.firebolt.io/ +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables diff --git a/docs/content/Configuration/Databases/Google-BigQuery.mdx b/docs/content/Configuration/Databases/Google-BigQuery.mdx index 49254202f9cf8..09d0c337dc83e 100644 --- a/docs/content/Configuration/Databases/Google-BigQuery.mdx +++ b/docs/content/Configuration/Databases/Google-BigQuery.mdx @@ -7,27 +7,27 @@ permalink: /config/databases/google-bigquery -In order to connect Google BigQuery to Cube.js, you need to provide service -account credentials. Cube.js requires the service account to have **BigQuery -Data Viewer** and **BigQuery Job User** roles enabled. You can learn more about -acquiring Google BigQuery credentials [here][bq-docs-getting-started]. +In order to connect Google BigQuery to Cube, you need to provide service account +credentials. Cube requires the service account to have **BigQuery Data Viewer** +and **BigQuery Job User** roles enabled. You can learn more about acquiring +Google BigQuery credentials [here][bq-docs-getting-started]. -- [The Google Cloud Project ID][google-cloud-docs-projects] for the +- The [Google Cloud Project ID][google-cloud-docs-projects] for the [BigQuery][bq] project -- [A set of Google Cloud service credentials][google-support-create-svc-account] - [which allow access][bq-docs-getting-started] to the [BigQuery][bq] project -- [The Google Cloud region][bq-docs-regional-locations] for the [BigQuery][bq] +- A set of [Google Cloud service credentials][google-support-create-svc-account] + which [allow access][bq-docs-getting-started] to the [BigQuery][bq] project +- The [Google Cloud region][bq-docs-regional-locations] for the [BigQuery][bq] project ## Setup ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=bigquery CUBEJS_DB_BQ_PROJECT_ID=my-bigquery-project-12345 CUBEJS_DB_BQ_KEY_FILE=/path/to/my/keyfile.json @@ -40,25 +40,57 @@ You could also encode the key file using Base64 and set the result to CUBEJS_DB_BQ_CREDENTIALS=$(cat /path/to/my/keyfile.json | base64) ``` +### <--{"id" : "Setup"}--> Cube Cloud + + + +In some cases you'll need to allow connections from your Cube Cloud deployment IP +address to your database. You can copy the IP address from either the Database +Setup step in deployment creation, or from Settings → +Configuration in your deployment. + + + +The following fields are required when creating a BigQuery connection: + + + +Cube Cloud also supports connecting to data sources within private VPCs. If you already have VPCs enabled in +your account, check out the [VPC documentation][ref-cloud-conf-vpc] to learn how +to get started. + + + +VPC connectivity is available in Cube Cloud on [Premium](https://cube.dev/pricing) tier. [Contact us][cube-contact] for details. + + + +[cube-pricing]: https://cube.dev/pricing/ +[cube-contact]: https://cube.dev/contact/ +[ref-cloud-conf-vpc]: /cloud/configuration/connecting-with-a-vpc + ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| ------------------------------ | ------------------------------------------------------------------------------------ | ----------------------------------------------------------------------- | :------: | -| `CUBEJS_DB_BQ_PROJECT_ID` | The Google BigQuery project ID to connect to | A valid Google BigQuery Project ID | ✅ | -| `CUBEJS_DB_BQ_KEY_FILE` | The path to a JSON key file for connecting to Google BigQuery | A valid Google BigQuery JSON key file | ✅ | -| `CUBEJS_DB_BQ_CREDENTIALS` | A Base64 encoded JSON key file for connecting to Google BigQuery | A valid Google BigQuery JSON key file encoded as a Base64 string | ❌ | -| `CUBEJS_DB_BQ_LOCATION` | The Google BigQuery dataset location to connect to | [A valid Google BigQuery regional location][bq-docs-regional-locations] | ❌ | -| `CUBEJS_DB_EXPORT_BUCKET` | The name of a bucket in cloud storage | A valid bucket name from cloud storage | ❌ | -| `CUBEJS_DB_EXPORT_BUCKET_TYPE` | The cloud provider where the bucket is hosted | `gcp` | ❌ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `10` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `40` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_BQ_PROJECT_ID` | The Google BigQuery project ID to connect to | A valid Google BigQuery Project ID | ✅ | ✅ | +| `CUBEJS_DB_BQ_KEY_FILE` | The path to a JSON key file for connecting to Google BigQuery | A valid Google BigQuery JSON key file | ✅ | ✅ | +| `CUBEJS_DB_BQ_CREDENTIALS` | A Base64 encoded JSON key file for connecting to Google BigQuery | A valid Google BigQuery JSON key file encoded as a Base64 string | ❌ | ✅ | +| `CUBEJS_DB_BQ_LOCATION` | The Google BigQuery dataset location to connect to. Required if used with pre-aggregations outside of US. If not set then BQ driver will fail with `Dataset was not found in location US` error | [A valid Google BigQuery regional location][bq-docs-regional-locations] | ⚠️ | ✅ | +| `CUBEJS_DB_EXPORT_BUCKET` | The name of a bucket in cloud storage | A valid bucket name from cloud storage | ❌ | ✅ | +| `CUBEJS_DB_EXPORT_BUCKET_TYPE` | The cloud provider where the bucket is hosted | `gcp` | ❌ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `10` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `40` | A valid number | ❌ | ✅ | ## Pre-Aggregation Feature Support -### countDistinctApprox +### count_distinct_approx Measures of type -[`countDistinctApprox`][ref-schema-ref-types-formats-countdistinctapprox] can be +[`count_distinct_approx`][ref-schema-ref-types-formats-countdistinctapprox] can be used in pre-aggregations when using Google BigQuery as a source database. To learn more about Google BigQuery's support for approximate aggregate functions, [click here][bq-docs-approx-agg-fns]. @@ -96,7 +128,7 @@ BigQuery only supports using Google Cloud Storage for export buckets. For [improved pre-aggregation performance with large datasets][ref-caching-large-preaggs], enable export bucket functionality by -configuring Cube.js with the following environment variables: +configuring Cube with the following environment variables: @@ -112,7 +144,7 @@ CUBEJS_DB_EXPORT_BUCKET_TYPE=gcp ## SSL -Cube.js does not require any additional configuration to enable SSL as Google +Cube does not require any additional configuration to enable SSL as Google BigQuery connections are made over HTTPS. [bq]: https://cloud.google.com/bigquery @@ -131,7 +163,8 @@ BigQuery connections are made over HTTPS. [ref-caching-large-preaggs]: /caching/using-pre-aggregations#export-bucket [ref-caching-using-preaggs-build-strats]: /caching/using-pre-aggregations#pre-aggregation-build-strategies -[ref-env-var]: /reference/environment-variables#database-connection +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables [ref-schema-ref-types-formats-countdistinctapprox]: /schema/reference/types-and-formats#count-distinct-approx [self-preaggs-batching]: #batching diff --git a/docs/content/Configuration/Databases/Hive.mdx b/docs/content/Configuration/Databases/Hive.mdx index 86e4b450ede72..435e9f544345a 100644 --- a/docs/content/Configuration/Databases/Hive.mdx +++ b/docs/content/Configuration/Databases/Hive.mdx @@ -18,9 +18,9 @@ permalink: /config/databases/hive-sparksql ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=hive CUBEJS_DB_HOST=my.hive.host CUBEJS_DB_NAME=my_hive_database @@ -30,14 +30,20 @@ CUBEJS_DB_PASS=********** ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | -| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | -| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | -| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| --------------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | ✅ | +| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | ✅ | +| `CUBEJS_DB_HIVE_TYPE` | | | ❌ | ✅ | +| `CUBEJS_DB_HIVE_VER` | | | ❌ | ✅ | +| `CUBEJS_DB_HIVE_THRIFT_VER` | | | ❌ | ✅ | +| `CUBEJS_DB_HIVE_CDH_VER` | | | ❌ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | ✅ | [hive]: https://hive.apache.org/ +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables diff --git a/docs/content/Configuration/Databases/MS-SQL.mdx b/docs/content/Configuration/Databases/MS-SQL.mdx index dfd5a395e04a9..9a494560a8b56 100644 --- a/docs/content/Configuration/Databases/MS-SQL.mdx +++ b/docs/content/Configuration/Databases/MS-SQL.mdx @@ -13,9 +13,9 @@ permalink: /config/databases/mssql ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=mssql CUBEJS_DB_HOST=my.mssql.host CUBEJS_DB_NAME=my_mssql_database @@ -25,24 +25,24 @@ CUBEJS_DB_PASS=********** ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| -------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------- | :------: | -| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | -| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | -| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | -| `CUBEJS_DB_DOMAIN` | A domain name within the database to connect to | A valid domain name within a MSSQL database | ❌ | -| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube.js | `true`, `false` | ❌ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| -------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | ✅ | +| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | ✅ | +| `CUBEJS_DB_DOMAIN` | A domain name within the database to connect to | A valid domain name within a MSSQL database | ❌ | ✅ | +| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube | `true`, `false` | ❌ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | ✅ | ## Pre-Aggregation Feature Support -### countDistinctApprox +### count_distinct_approx Measures of type -[`countDistinctApprox`][ref-schema-ref-types-formats-countdistinctapprox] can +[`count_distinct_approx`][ref-schema-ref-types-formats-countdistinctapprox] can not be used in pre-aggregations when using MSSQL as a source database. ## Pre-Aggregation Build Strategies @@ -78,7 +78,7 @@ MSSQL does not support export buckets. ## SSL -To enable SSL-encrypted connections between Cube.js and MS SQL, set the +To enable SSL-encrypted connections between Cube and MS SQL, set the `CUBEJS_DB_SSL` environment variable to `true`. For more information on how to configure custom certificates, please check out [Enable SSL Connections to the Database][ref-recipe-enable-ssl]. @@ -102,6 +102,8 @@ module.exports = { [mssql]: https://www.microsoft.com/en-gb/sql-server/sql-server-2019 [ref-caching-using-preaggs-build-strats]: /caching/using-pre-aggregations#pre-aggregation-build-strategies +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables [ref-recipe-enable-ssl]: /recipes/enable-ssl-connections-to-database [ref-schema-ref-types-formats-countdistinctapprox]: /schema/reference/types-and-formats#count-distinct-approx diff --git a/docs/content/Configuration/Databases/Materialize.mdx b/docs/content/Configuration/Databases/Materialize.mdx index a8b97a036a92d..ce495edb55ddf 100644 --- a/docs/content/Configuration/Databases/Materialize.mdx +++ b/docs/content/Configuration/Databases/Materialize.mdx @@ -3,6 +3,13 @@ title: Materialize permalink: /config/databases/materialize --- + + +The driver for Materialize is supported by its vendor. Please report any issues +to their [Slack][materialize-slack]. + + + ## Prerequisites - The hostname for the [Materialize][materialize] database server @@ -11,9 +18,9 @@ permalink: /config/databases/materialize ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=materialize CUBEJS_DB_HOST=my.materialize.host CUBEJS_DB_PORT=6875 @@ -24,21 +31,24 @@ CUBEJS_DB_PASS=materialize ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | -| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | -| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ✅ | -| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ✅ | ✅ | +| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | ✅ | ## SSL -To enable SSL-encrypted connections between Cube.js and Materialize, set the +To enable SSL-encrypted connections between Cube and Materialize, set the `CUBEJS_DB_SSL` environment variable to `true`. For more information on how to configure custom certificates, please check out [Enable SSL Connections to the Database][ref-recipe-enable-ssl]. [materialize]: https://materialize.com/docs/ +[materialize-slack]: https://materialize.com/s/chat +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables diff --git a/docs/content/Configuration/Databases/MongoDB.mdx b/docs/content/Configuration/Databases/MongoDB.mdx index d1050222ab88b..04c4fc22bd156 100644 --- a/docs/content/Configuration/Databases/MongoDB.mdx +++ b/docs/content/Configuration/Databases/MongoDB.mdx @@ -13,7 +13,7 @@ permalink: /config/databases/mongodb -To use Cube.js with MongoDB you need to install the [MongoDB Connector for +To use Cube with MongoDB you need to install the [MongoDB Connector for BI][mongobi-download]. [Learn more about setup for MongoDB here][cube-blog-mongodb]. @@ -27,15 +27,19 @@ here][cube-blog-mongodb]. ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=mongobi -CUBEJS_DB_HOST=my.mongobi.host # mongo BI connector host, if using from local, should be either `localhost` or `127.0.0.1` -CUBEJS_DB_PORT=3307 # default port of mongo BI connector service +# The MongoBI connector host. If using on local machine, it should be either `localhost` or `127.0.0.1`: +CUBEJS_DB_HOST=my.mongobi.host +# The default port of the MongoBI connector service +CUBEJS_DB_PORT=3307 CUBEJS_DB_NAME=my_mongodb_database CUBEJS_DB_USER=mongodb_server_user CUBEJS_DB_PASS=mongodb_server_password +# MongoBI requires SSL connections, so set the following to `true`: +CUBEJS_DB_SSL=true ``` If you are connecting to a local MongoBI Connector, which is pointing to a local @@ -46,23 +50,23 @@ file][mongobi-with-remote-db]. ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | -| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | -| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | -| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | -| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube.js | `true`, `false` | ❌ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | ✅ | +| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | ✅ | +| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube | `true`, `false` | ✅ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | ✅ | ## Pre-Aggregation Feature Support -### countDistinctApprox +### count_distinct_approx Measures of type -[`countDistinctApprox`][ref-schema-ref-types-formats-countdistinctapprox] can +[`count_distinct_approx`][ref-schema-ref-types-formats-countdistinctapprox] can not be used in pre-aggregations when using MongoDB as a source database. ## Pre-Aggregation Build Strategies @@ -92,18 +96,11 @@ MongoDB does not support export buckets. ## SSL -To enable SSL-encrypted connections between Cube.js and MongoDB, set the +To enable SSL-encrypted connections between Cube and MongoDB, set the `CUBEJS_DB_SSL` environment variable to `true`. For more information on how to configure custom certificates, please check out [Enable SSL Connections to the Database][ref-recipe-enable-ssl]. -## Additional Configuration - -### <--{"id" : "Additional Configuratio"}--> MongoDB Atlas - -Use `CUBEJS_DB_SSL=true` to enable SSL as MongoDB Atlas requires it. All other -SSL-related environment variables can be left unset. - [mongodb]: https://www.mongodb.com/ [mongobi-with-remote-db]: https://docs.mongodb.com/bi-connector/current/reference/mongosqld/#example-configuration-file @@ -114,6 +111,8 @@ SSL-related environment variables can be left unset. https://nodejs.org/docs/latest/api/tls.html#tls_modifying_the_default_tls_cipher_suite [ref-caching-using-preaggs-build-strats]: /caching/using-pre-aggregations#pre-aggregation-build-strategies +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables [ref-recipe-enable-ssl]: /recipes/enable-ssl-connections-to-database [ref-schema-ref-types-formats-countdistinctapprox]: /schema/reference/types-and-formats#count-distinct-approx diff --git a/docs/content/Configuration/Databases/MySQL.mdx b/docs/content/Configuration/Databases/MySQL.mdx index c9ee1a15153d9..bb8973632199e 100644 --- a/docs/content/Configuration/Databases/MySQL.mdx +++ b/docs/content/Configuration/Databases/MySQL.mdx @@ -13,9 +13,9 @@ permalink: /config/databases/mysql ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=mysql CUBEJS_DB_HOST=my.mysql.host CUBEJS_DB_NAME=my_mysql_database @@ -25,23 +25,23 @@ CUBEJS_DB_PASS=********** ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | -| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | -| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | -| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | -| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube.js | `true`, `false` | ❌ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | ✅ | +| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | ✅ | +| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube | `true`, `false` | ❌ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | ✅ | ## Pre-Aggregation Feature Support -### countDistinctApprox +### count_distinct_approx Measures of type -[`countDistinctApprox`][ref-schema-ref-types-formats-countdistinctapprox] can +[`count_distinct_approx`][ref-schema-ref-types-formats-countdistinctapprox] can not be used in pre-aggregations when using MySQL as a source database. ## Pre-Aggregation Build Strategies @@ -71,7 +71,7 @@ MySQL does not support export buckets. ## SSL -To enable SSL-encrypted connections between Cube.js and MySQL, set the +To enable SSL-encrypted connections between Cube and MySQL, set the `CUBEJS_DB_SSL` environment variable to `true`. For more information on how to configure custom certificates, please check out [Enable SSL Connections to the Database][ref-recipe-enable-ssl]. @@ -90,6 +90,8 @@ SSL section][self-ssl] above for more details. [mysql]: https://www.mysql.com/ [ref-caching-using-preaggs-build-strats]: /caching/using-pre-aggregations#pre-aggregation-build-strategies +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables [ref-recipe-enable-ssl]: /recipes/enable-ssl-connections-to-database [ref-schema-ref-types-formats-countdistinctapprox]: /schema/reference/types-and-formats#count-distinct-approx diff --git a/docs/content/Configuration/Databases/Oracle.mdx b/docs/content/Configuration/Databases/Oracle.mdx index 07253c9434cfc..105af4ce51326 100644 --- a/docs/content/Configuration/Databases/Oracle.mdx +++ b/docs/content/Configuration/Databases/Oracle.mdx @@ -19,9 +19,9 @@ permalink: /config/databases/oracle ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=oracle CUBEJS_DB_HOST=my.oracle.host CUBEJS_DB_NAME=my_oracle_database @@ -31,23 +31,25 @@ CUBEJS_DB_PASS=********** ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | -| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | -| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | -| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | -| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube.js | `true`, `false` | ❌ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | ✅ | +| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | ✅ | +| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube | `true`, `false` | ❌ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | ✅ | ## SSL -To enable SSL-encrypted connections between Cube.js and Oracle, set the +To enable SSL-encrypted connections between Cube and Oracle, set the `CUBEJS_DB_SSL` environment variable to `true`. For more information on how to configure custom certificates, please check out [Enable SSL Connections to the Database][ref-recipe-enable-ssl]. [oracle]: https://www.oracle.com/uk/index.html +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables [ref-recipe-enable-ssl]: /recipes/enable-ssl-connections-to-database diff --git a/docs/content/Configuration/Databases/Postgres.mdx b/docs/content/Configuration/Databases/Postgres.mdx index 023363445a48f..8488478912d43 100644 --- a/docs/content/Configuration/Databases/Postgres.mdx +++ b/docs/content/Configuration/Databases/Postgres.mdx @@ -14,9 +14,9 @@ permalink: /config/databases/postgres ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=postgres CUBEJS_DB_HOST=my.postgres.host CUBEJS_DB_NAME=my_postgres_database @@ -26,23 +26,23 @@ CUBEJS_DB_PASS=********** ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | -| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | -| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | -| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | -| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube.js | `true`, `false` | ❌ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | ✅ | +| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | ✅ | +| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube | `true`, `false` | ❌ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | ✅ | ## Pre-Aggregation Feature Support -### countDistinctApprox +### count_distinct_approx Measures of type -[`countDistinctApprox`][ref-schema-ref-types-formats-countdistinctapprox] can +[`count_distinct_approx`][ref-schema-ref-types-formats-countdistinctapprox] can not be used in pre-aggregations when using Postgres as a source database. ## Pre-Aggregation Build Strategies @@ -72,7 +72,7 @@ Postgres does not support export buckets. ## SSL -To enable SSL-encrypted connections between Cube.js and Postgres, set the +To enable SSL-encrypted connections between Cube and Postgres, set the `CUBEJS_DB_SSL` environment variable to `true`. For more information on how to configure custom certificates, please check out [Enable SSL Connections to the Database][ref-recipe-enable-ssl]. @@ -99,10 +99,10 @@ Unless you're using a Private or Shield Heroku Postgres database, Heroku Postgres does not currently support verifiable certificates. [Here is the description of the issue from Heroku][heroku-postgres-issue]. -As a workaround, you can set `rejectUnauthorized` option to `false` in the -Cube.js Postgres driver: +As a workaround, you can set `rejectUnauthorized` option to `false` in the Cube +Postgres driver: -```js +```javascript const PostgresDriver = require('@cubejs-backend/postgres-driver'); module.exports = { driverFactory: () => @@ -124,6 +124,8 @@ module.exports = { [postgres]: https://www.postgresql.org/ [ref-caching-using-preaggs-build-strats]: /caching/using-pre-aggregations#pre-aggregation-build-strategies +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables [ref-recipe-enable-ssl]: /recipes/enable-ssl-connections-to-database [ref-schema-ref-types-formats-countdistinctapprox]: /schema/reference/types-and-formats#count-distinct-approx diff --git a/docs/content/Configuration/Databases/Presto.mdx b/docs/content/Configuration/Databases/Presto.mdx index b5bbb89c2f336..3669885d576a1 100644 --- a/docs/content/Configuration/Databases/Presto.mdx +++ b/docs/content/Configuration/Databases/Presto.mdx @@ -17,36 +17,35 @@ redirect_from: Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=prestodb CUBEJS_DB_HOST=my.presto.host -CUBEJS_DB_NAME=my_presto_database CUBEJS_DB_USER=presto_user CUBEJS_DB_PASS=********** -CUBEJS_DB_CATALOG=my_presto_catalog +CUBEJS_DB_PRESTO_CATALOG=my_presto_catalog CUBEJS_DB_SCHEMA=my_presto_schema ``` ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| -------------------- | ----------------------------------------------------------------------------------- | --------------------------------------------- | :------: | -| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | -| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | -| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | -| `CUBEJS_DB_CATALOG` | The catalog within the database to connect to | A valid catalog name within a Presto database | ✅ | -| `CUBEJS_DB_SCHEMA` | The schema within the database to connect to | A valid schema name within a Presto database | ✅ | -| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube | `true`, `false` | ❌ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| -------------------------- | ----------------------------------------------------------------------------------- | --------------------------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | ✅ | +| `CUBEJS_DB_PRESTO_CATALOG` | The catalog within Presto to connect to | A valid catalog name within a Presto database | ✅ | ✅ | +| `CUBEJS_DB_SCHEMA` | The schema within the database to connect to | A valid schema name within a Presto database | ✅ | ✅ | +| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube | `true`, `false` | ❌ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | ✅ | ## Pre-Aggregation Feature Support -### countDistinctApprox +### count_distinct_approx Measures of type -[`countDistinctApprox`][ref-schema-ref-types-formats-countdistinctapprox] can be +[`count_distinct_approx`][ref-schema-ref-types-formats-countdistinctapprox] can be used in pre-aggregations when using Presto as a source database. To learn more about Presto support for approximate aggregate functions, [click here][presto-docs-approx-agg-fns]. @@ -61,6 +60,8 @@ Database][ref-recipe-enable-ssl]. [presto]: https://prestodb.io/ [presto-docs-approx-agg-fns]: https://prestodb.io/docs/current/functions/aggregate.html +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables [ref-recipe-enable-ssl]: /recipes/enable-ssl-connections-to-database [ref-schema-ref-types-formats-countdistinctapprox]: /schema/reference/types-and-formats#count-distinct-approx diff --git a/docs/content/Configuration/Databases/QuestDB.mdx b/docs/content/Configuration/Databases/QuestDB.mdx index 75568212ebbc3..02ebc7f7b087d 100644 --- a/docs/content/Configuration/Databases/QuestDB.mdx +++ b/docs/content/Configuration/Databases/QuestDB.mdx @@ -3,6 +3,13 @@ title: QuestDB permalink: /config/databases/questdb --- + + +The driver for QuestDB is supported by its vendor. Please report any issues to +their [Slack][questdb-slack]. + + + ## Prerequisites - The hostname for the [QuestDB][questdb] database server @@ -11,9 +18,9 @@ permalink: /config/databases/questdb ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=questdb CUBEJS_DB_HOST=my.questdb.host CUBEJS_DB_PORT=8812 @@ -24,14 +31,18 @@ CUBEJS_DB_PASS=quest ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | -| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | -| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ✅ | -| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ✅ | ✅ | +| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | ✅ | +| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube | `true`, `false` | ❌ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | ✅ | [questdb]: https://questdb.io/ +[questdb-slack]: https://slack.questdb.io/ +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables diff --git a/docs/content/Configuration/Databases/SQLite.mdx b/docs/content/Configuration/Databases/SQLite.mdx index 9e859aa81f0fe..33e598f19bee9 100644 --- a/docs/content/Configuration/Databases/SQLite.mdx +++ b/docs/content/Configuration/Databases/SQLite.mdx @@ -15,21 +15,24 @@ permalink: /config/databases/sqlite ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=sqlite CUBEJS_DB_NAME=my_sqlite_database ``` ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| -------------------- | ----------------------------------------------------------------------------------- | --------------------- | :------: | -| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| -------------------- | ----------------------------------------------------------------------------------- | --------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | ✅ | ## SSL SQLite does not support SSL connections. + +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables diff --git a/docs/content/Configuration/Databases/Snowflake.mdx b/docs/content/Configuration/Databases/Snowflake.mdx index b9507dfcef630..4a77717d17bf4 100644 --- a/docs/content/Configuration/Databases/Snowflake.mdx +++ b/docs/content/Configuration/Databases/Snowflake.mdx @@ -5,7 +5,7 @@ permalink: /config/databases/snowflake ## Prerequisites -- [The account ID][dbt-docs-snowflake-account] for [Snowflake][snowflake] +- [The account ID][snowflake-docs-account-id] for [Snowflake][snowflake] - The warehouse name in the [Snowflake][snowflake] account - [The region][snowflake-docs-regions] for the [Snowflake][snowflake] warehouse - The username/password for the [Snowflake][snowflake] account @@ -14,42 +14,73 @@ permalink: /config/databases/snowflake ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=snowflake CUBEJS_DB_SNOWFLAKE_ACCOUNT=XXXXXXXXX.us-east-1 -CUBEJS_DB_SNOWFLAKE_REGION=us-east-1 CUBEJS_DB_SNOWFLAKE_WAREHOUSE=MY_SNOWFLAKE_WAREHOUSE CUBEJS_DB_NAME=my_snowflake_database CUBEJS_DB_USER=snowflake_user CUBEJS_DB_PASS=********** ``` +### <--{"id" : "Setup"}--> Cube Cloud + + + +In some cases you'll need to allow connections from your Cube Cloud deployment IP +address to your database. You can copy the IP address from either the Database +Setup step in deployment creation, or from Settings → +Configuration in your deployment. + + + +The following fields are required when creating a Snowflake connection: + + + +Cube Cloud also supports connecting to data sources within private VPCs. If you already have VPCs enabled in +your account, check out the [VPC documentation][ref-cloud-conf-vpc] to learn how +to get started. + + + +VPC connectivity is available in Cube Cloud on [Premium](https://cube.dev/pricing) tier. [Contact us][cube-contact] for details. + + + +[cube-pricing]: https://cube.dev/pricing/ +[cube-contact]: https://cube.dev/contact/ +[ref-cloud-conf-vpc]: /cloud/configuration/connecting-with-a-vpc + ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| ----------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------- | :------: | -| `CUBEJS_DB_SNOWFLAKE_ACCOUNT` | The Snowflake account ID to use when connecting to the database | [A valid Snowflake account ID][dbt-docs-snowflake-account] | ✅ | -| `CUBEJS_DB_SNOWFLAKE_REGION` | The Snowflake region to use when connecting to the database | [A valid Snowflake region][snowflake-docs-regions] | ✅ | -| `CUBEJS_DB_SNOWFLAKE_WAREHOUSE` | The Snowflake warehouse to use when connecting to the database | A valid Snowflake warehouse for the account | ✅ | -| `CUBEJS_DB_SNOWFLAKE_ROLE` | The Snowflake role to use when connecting to the database | A valid Snowflake role for the account | ❌ | -| `CUBEJS_DB_SNOWFLAKE_CLIENT_SESSION_KEEP_ALIVE` | If `true`, [keep the Snowflake connection alive indefinitely][snowflake-docs-connection-options] | `true`, `false` | ❌ | -| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | -| `CUBEJS_DB_SNOWFLAKE_AUTHENTICATOR` | The type of authenticator to use with Snowflake. Use `SNOWFLAKE` with username/password, or `SNOWFLAKE_JWT` with key pairs. Defaults to `SNOWFLAKE` | `SNOWFLAKE`, `SNOWFLAKE_JWT` | ❌ | -| `CUBEJS_DB_SNOWFLAKE_PRIVATE_KEY_PATH` | The path to the private RSA key folder | A valid path to the private RSA key | ❌ | -| `CUBEJS_DB_SNOWFLAKE_PRIVATE_KEY_PASS` | The password for the private RSA key. Only required for encrypted keys | A valid password for the encrypted private RSA key | ❌ | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `5` | A valid number | ❌ | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `20` | A valid number | ❌ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| ----------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_SNOWFLAKE_ACCOUNT` | The Snowflake account identifier to use when connecting to the database | [A valid Snowflake account ID][snowflake-docs-account-id] | ✅ | ✅ | +| `CUBEJS_DB_SNOWFLAKE_REGION` | The Snowflake region to use when connecting to the database | [A valid Snowflake region][snowflake-docs-regions] | ❌ | ✅ | +| `CUBEJS_DB_SNOWFLAKE_WAREHOUSE` | The Snowflake warehouse to use when connecting to the database | [A valid Snowflake warehouse][snowflake-docs-warehouse] in the account | ✅ | ✅ | +| `CUBEJS_DB_SNOWFLAKE_ROLE` | The Snowflake role to use when connecting to the database | [A valid Snowflake role][snowflake-docs-roles] in the account | ❌ | ✅ | +| `CUBEJS_DB_SNOWFLAKE_CLIENT_SESSION_KEEP_ALIVE` | If `true`, [keep the Snowflake connection alive indefinitely][snowflake-docs-connection-options] | `true`, `false` | ❌ | ✅ | +| `CUBEJS_DB_NAME` | The name of the database to connect to | A valid database name | ✅ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | ✅ | +| `CUBEJS_DB_SNOWFLAKE_AUTHENTICATOR` | The type of authenticator to use with Snowflake. Use `SNOWFLAKE` with username/password, or `SNOWFLAKE_JWT` with key pairs. Defaults to `SNOWFLAKE` | `SNOWFLAKE`, `SNOWFLAKE_JWT` | ❌ | ✅ | +| `CUBEJS_DB_SNOWFLAKE_PRIVATE_KEY_PATH` | The path to the private RSA key folder | A valid path to the private RSA key | ❌ | ✅ | +| `CUBEJS_DB_SNOWFLAKE_PRIVATE_KEY_PASS` | The password for the private RSA key. Only required for encrypted keys | A valid password for the encrypted private RSA key | ❌ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `5` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `20` | A valid number | ❌ | ✅ | ## Pre-Aggregation Feature Support -### countDistinctApprox +### count_distinct_approx Measures of type -[`countDistinctApprox`][ref-schema-ref-types-formats-countdistinctapprox] can be +[`count_distinct_approx`][ref-schema-ref-types-formats-countdistinctapprox] can be used in pre-aggregations when using Snowflake as a source database. To learn more about Snowflake's support for approximate aggregate functions, [click here][snowflake-docs-approx-agg-fns]. @@ -106,11 +137,11 @@ role to your BigQuery credentials (`CUBEJS_DB_EXPORT_GCS_CREDENTIALS`). -Before configuring Cube.js, an [integration must be created and configured in +Before configuring Cube, an [integration must be created and configured in Snowflake][snowflake-docs-gcs-integration]. Take note of the integration name -(`gcs_int` from the example link) as you'll need it to configure Cube.js. +(`gcs_int` from the example link) as you'll need it to configure Cube. -Once the Snowflake integration is set up, configure Cube.js using the following: +Once the Snowflake integration is set up, configure Cube using the following: ```dotenv CUBEJS_DB_EXPORT_BUCKET=snowflake-export-bucket @@ -121,25 +152,28 @@ CUBEJS_DB_EXPORT_INTEGRATION=gcs_int ## SSL -Cube.js does not require any additional configuration to enable SSL as Snowflake +Cube does not require any additional configuration to enable SSL as Snowflake connections are made over HTTPS. [aws-s3]: https://aws.amazon.com/s3/ -[dbt-docs-snowflake-account]: - https://docs.getdbt.com/reference/warehouse-profiles/snowflake-profile#account [google-cloud-storage]: https://cloud.google.com/storage [ref-caching-using-preaggs-build-strats]: /caching/using-pre-aggregations#pre-aggregation-build-strategies -[ref-env-var]: /reference/environment-variables#database-connection [ref-schema-ref-types-formats-countdistinctapprox]: /schema/reference/types-and-formats#count-distinct-approx [self-preaggs-batching]: #batching [snowflake]: https://www.snowflake.com/ +[snowflake-docs-account-id]: + https://docs.snowflake.com/en/user-guide/admin-account-identifier.html [snowflake-docs-connection-options]: https://docs.snowflake.com/en/user-guide/nodejs-driver-use.html#additional-connection-options [snowflake-docs-gcs-integration]: https://docs.snowflake.com/en/user-guide/data-load-gcs-config.html [snowflake-docs-regions]: https://docs.snowflake.com/en/user-guide/intro-regions.html +[snowflake-docs-roles]: + https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles [snowflake-docs-approx-agg-fns]: https://docs.snowflake.com/en/sql-reference/functions/approx_count_distinct.html +[snowflake-docs-warehouse]: + https://docs.snowflake.com/en/user-guide/warehouses.html diff --git a/docs/content/Configuration/Databases/Trino.mdx b/docs/content/Configuration/Databases/Trino.mdx new file mode 100644 index 0000000000000..5b6bf4275ed60 --- /dev/null +++ b/docs/content/Configuration/Databases/Trino.mdx @@ -0,0 +1,65 @@ +--- +title: Trino +permalink: /config/databases/trino +--- + +## Prerequisites + +- The hostname for the [Trino][trino] database server +- The username/password for the [Trino][trino] database server +- The name of the database to use within the [Trino][trino] database server + +## Setup + +### <--{"id" : "Setup"}--> Manual + +Add the following to a `.env` file in your Cube project: + +```dotenv +CUBEJS_DB_TYPE=trino +CUBEJS_DB_HOST=my.trino.host +CUBEJS_DB_USER=trino_user +CUBEJS_DB_PASS=********** +CUBEJS_DB_PRESTO_CATALOG=my_trino_catalog +CUBEJS_DB_SCHEMA=my_trino_schema +``` + +## Environment Variables + +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| -------------------------- | ----------------------------------------------------------------------------------- | --------------------------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_HOST` | The host URL for a database | A valid database host URL | ✅ | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ❌ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the database | A valid database username | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the database | A valid database password | ✅ | ✅ | +| `CUBEJS_DB_PRESTO_CATALOG` | The catalog within Presto to connect to | A valid catalog name within a Presto database | ✅ | ✅ | +| `CUBEJS_DB_SCHEMA` | The schema within the database to connect to | A valid schema name within a Presto database | ✅ | ✅ | +| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube | `true`, `false` | ❌ | ✅ | +| `CUBEJS_CONCURRENCY` | The number of concurrent connections each queue has to the database. Default is `2` | A valid number | ❌ | ❌ | +| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Default is `8` | A valid number | ❌ | ✅ | + +## Pre-Aggregation Feature Support + +### count_distinct_approx + +Measures of type +[`count_distinct_approx`][ref-schema-ref-types-formats-countdistinctapprox] can be +used in pre-aggregations when using Trino as a source database. To learn more +about Trino support for approximate aggregate functions, [click +here][trino-docs-approx-agg-fns]. + +## SSL + +To enable SSL-encrypted connections between Cube and Trino, set the +`CUBEJS_DB_SSL` environment variable to `true`. For more information on how to +configure custom certificates, please check out [Enable SSL Connections to the +Database][ref-recipe-enable-ssl]. + +[trino]: https://trino.io/ +[trino-docs-approx-agg-fns]: + https://trino.io/docs/current/functions/aggregate.html#approximate-aggregate-functions +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables +[ref-recipe-enable-ssl]: /recipes/enable-ssl-connections-to-database +[ref-schema-ref-types-formats-countdistinctapprox]: + /schema/reference/types-and-formats#count-distinct-approx diff --git a/docs/content/Configuration/Databases/ksqlDB.mdx b/docs/content/Configuration/Databases/ksqlDB.mdx index 420c7f5f21c69..62e4b88657f6a 100644 --- a/docs/content/Configuration/Databases/ksqlDB.mdx +++ b/docs/content/Configuration/Databases/ksqlDB.mdx @@ -4,7 +4,9 @@ permalink: /config/databases/ksqldb --- -ksqlDB driver is in preview. Please contact us if you need help running it in production. + ksqlDB driver is in preview. Please{' '} + contact us if you need help running it + in production. ## Prerequisites @@ -12,15 +14,16 @@ ksqlDB driver is in preview. Please contact u - Hostname for the ksqlDB server - Username and password to connect to ksqlDB server -If you are using Confluent Cloud, you need to generate API key and use **key as username** and **secret as password**. +If you are using Confluent Cloud, you need to generate API key and use **key as +username** and **secret as password**. ## Setup ### <--{"id" : "Setup"}--> Manual -Add the following to a `.env` file in your Cube.js project: +Add the following to a `.env` file in your Cube project: -```bash +```dotenv CUBEJS_DB_TYPE=ksql CUBEJS_DB_URL=https://xxxxxx-xxxxx.us-west4.gcp.confluent.cloud:443 CUBEJS_DB_USER=username @@ -29,12 +32,16 @@ CUBEJS_DB_PASS=password ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| -------------------- | ----------------------------------------------------------------------------------- | ------------------------- | :------: | -| `CUBEJS_DB_URL` | The host URL for ksqlDB with port | A valid database host URL | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the ksqlDB. API key for Confluent Cloud. | A valid port number | ✅ | -| `CUBEJS_DB_PASS` | The password used to connect to the ksqlDB. API secret for Confluent Cloud. | A valid database name | ✅ | +| Environment Variable | Description | Possible Values | Required | [Supports multiple data sources?][ref-config-multiple-ds-decorating-env] | +| -------------------- | --------------------------------------------------------------------------- | ------------------------- | :------: | :----------------------------------------------------------------------: | +| `CUBEJS_DB_URL` | The host URL for ksqlDB with port | A valid database host URL | ✅ | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the ksqlDB. API key for Confluent Cloud. | A valid port number | ✅ | ✅ | +| `CUBEJS_DB_PASS` | The password used to connect to the ksqlDB. API secret for Confluent Cloud. | A valid database name | ✅ | ✅ | ## Pre-Aggregations Support -ksqlDB supports only [streaming pre-aggregations](/caching/using-pre-aggregations#streaming-pre-aggregations). +ksqlDB supports only +[streaming pre-aggregations](/caching/using-pre-aggregations#streaming-pre-aggregations). + +[ref-config-multiple-ds-decorating-env]: + /config/multiple-data-sources#configuring-data-sources-with-environment-variables-decorated-environment-variables diff --git a/docs/content/Configuration/Downstream/Appsmith.mdx b/docs/content/Configuration/Downstream/Appsmith.mdx index 2fed5b5769b2e..66aea2ae8054b 100644 --- a/docs/content/Configuration/Downstream/Appsmith.mdx +++ b/docs/content/Configuration/Downstream/Appsmith.mdx @@ -3,7 +3,9 @@ title: Connecting to Appsmith permalink: /config/downstream/appsmith --- -[Appsmith](https://www.appsmith.com) is an open-source framework for building internal tools. You can connect a Cube deployment to Appsmith using Cube’s [REST API](https://cube.dev/docs/rest-api). +[Appsmith](https://www.appsmith.com) is an open-source framework for building +internal tools. You can connect a Cube deployment to Appsmith using Cube’s +[REST API](https://cube.dev/docs/rest-api). Here's a short video guide on how to connect Appsmith to Cube. @@ -11,74 +13,61 @@ Here's a short video guide on how to connect Appsmith to Cube. ## Use REST API in Cube -> Don't have a Cube project yet? [Learn how to get started here](https://cube.dev/docs/cloud/getting-started). +> Don't have a Cube project yet? +> [Learn how to get started here](https://cube.dev/docs/getting-started/cloud/overview). ### <--{"id" : "Use REST API in Cube"}--> Cube Cloud -Click the “How to connect” link on the Overview page, navigate to the REST API tab. You should see the screen below with your connection credentials (the REST API URL and the authorization token): +Click the “How to connect” link on the Overview page, navigate to the REST API +tab. You should see the screen below with your connection credentials (the REST +API URL and the authorization token): -
- -
+ ### <--{"id" : "Use REST API in Cube"}--> Self-hosted Cube -For a Cube instance publicly available at a specific `HOST`, the REST API URL would be `HOST/cubejs-api/v1`. Please refer to the [REST API page](https://cube.dev/docs/rest-api) for details. +For a Cube instance publicly available at a specific `HOST`, the REST API URL +would be `HOST/cubejs-api/v1`. Please refer to the +[REST API page](https://cube.dev/docs/rest-api) for details. -You will also need to generate a JSON Web Token that would be used to authenticate requests to Cube. +You will also need to generate a JSON Web Token that would be used to +authenticate requests to Cube. -Please check the [Security page](https://cube.dev/docs/security#generating-json-web-tokens-jwt) to learn how to generate a token. We suggest generating a long-lived JWT that won't expire soon. +Please check the +[Security page](https://cube.dev/docs/security#generating-json-web-tokens-jwt) +to learn how to generate a token. We suggest generating a long-lived JWT that +won't expire soon. ## Create a new Data Source in Appsmith -Copy and paste the REST API URL and the Authorization token to create a new data source in Appsmith. +Copy and paste the REST API URL and the Authorization token to create a new data +source in Appsmith. -
- -
+ -## Create a POST request in Appsmith +## Create a POST request in Appsmith -Get your Cube query in the JSON [query format](https://cube.dev/docs/query-format) ready. You can copy it from Cube’s Playground or compose manually: +Get your Cube query in the JSON +[query format](https://cube.dev/docs/query-format) ready. You can copy it from +Cube’s Playground or compose manually: -
- -
+ -Create a POST request, paste the JSON query in the **Body**. Make sure to add a `query` parameter for your JSON query. +Create a POST request, paste the JSON query in the **Body**. Make sure to add a +`query` parameter for your JSON query. -Because you saved the data source as `HOST/cubejs-api/v1`, don't forget to add a `/load` endpoint to the end of the data source API. +Because you saved the data source as `HOST/cubejs-api/v1`, don't forget to add a +`/load` endpoint to the end of the data source API. Next, hit Run. -
- -
+ ## Display the Data in Appsmith -You have many options to display the data in Appsmith. For instance, you can display the data in a table widget. Also, you can create a chart widget and map the values to *x* and *y* coordinates accordingly, give a *title* and *names* to the *axis*. +You have many options to display the data in Appsmith. For instance, you can +display the data in a table widget. Also, you can create a chart widget and map +the values to _x_ and _y_ coordinates accordingly, give a _title_ and _names_ to +the _axis_. -
- -
+ diff --git a/docs/content/Configuration/Downstream/Bubble.mdx b/docs/content/Configuration/Downstream/Bubble.mdx index eeab8a5b6ac1e..c13652301e759 100644 --- a/docs/content/Configuration/Downstream/Bubble.mdx +++ b/docs/content/Configuration/Downstream/Bubble.mdx @@ -3,8 +3,9 @@ title: Connecting to Bubble permalink: /config/downstream/bubble --- -[Bubble](https://bubble.io/) is one of the oldest and most comprehensive low-code app-building platforms on the market. -You can connect a Cube deployment to Bubble using Cube’s [REST API](https://cube.dev/docs/rest-api). +[Bubble](https://bubble.io/) is one of the oldest and most comprehensive +low-code app-building platforms on the market. You can connect a Cube deployment +to Bubble using Cube’s [REST API](https://cube.dev/docs/rest-api). Here's a short video guide on how to connect Bubble to Cube. @@ -12,27 +13,30 @@ Here's a short video guide on how to connect Bubble to Cube. ## Use REST API in Cube -> Don't have a Cube project yet? [Learn how to get started here](https://cube.dev/docs/cloud/getting-started). +> Don't have a Cube project yet? +> [Learn how to get started here](https://cube.dev/docs/getting-started/cloud/overview). ### <--{"id" : "Use REST API in Cube"}--> Cube Cloud -Click the "How to connect" link on the Overview page, navigate to the REST API tab. You will see your connection credentials, the REST API URL and the authorization token: +Click the "How to connect" link on the Overview page, navigate to the REST API +tab. You will see your connection credentials, the REST API URL and the +authorization token: -
- -
+ ### <--{"id" : "Use REST API in Cube"}--> Self-hosted Cube -For a Cube instance publicly available at a specific `HOST`, the REST API URL would be `HOST/cubejs-api/v1`. Please refer to the [REST API page](https://cube.dev/docs/rest-api) for details. +For a Cube instance publicly available at a specific `HOST`, the REST API URL +would be `HOST/cubejs-api/v1`. Please refer to the +[REST API page](https://cube.dev/docs/rest-api) for details. -You will also need to generate a JSON Web Token that would be used to authenticate requests to Cube. +You will also need to generate a JSON Web Token that would be used to +authenticate requests to Cube. -Please check the [Security page](https://cube.dev/docs/security#generating-json-web-tokens-jwt) to learn how to generate a token. We suggest generating a long-lived JWT that won't expire soon. +Please check the +[Security page](https://cube.dev/docs/security#generating-json-web-tokens-jwt) +to learn how to generate a token. We suggest generating a long-lived JWT that +won't expire soon. ## Add an API connector plugin to your Bubble app @@ -42,93 +46,58 @@ Navigate to the plugins. Select and install the API connector plugin. Click "Add another API". -
- -
- -Next, set the "Authentication" as "Private key in header", and copy and paste Cube's Authorization token. -Additionally, make sure to add a shared header for all calls by setting `content-type` to `application/json`. - -
- -
+ + +Next, set the "Authentication" as "Private key in header", and copy and paste +Cube's Authorization token. Additionally, make sure to add a shared header for +all calls by setting `content-type` to `application/json`. + + ## Create a POST request in Bubble Click "Add another call". -Copy and paste the Cube REST API, including the `/load` path, in the API field. Make sure to select "Use as: Action", and use the `JSON` "data type" and "body type". - -
- -
+Copy and paste the Cube REST API, including the `/load` path, in the API field. +Make sure to select "Use as: Action", and use the `JSON` "data type" and "body +type". -Get your Cube query in the JSON [query format](https://cube.dev/docs/query-format) ready. You can copy it from Cube’s Playground or compose manually: + -
- -
+Get your Cube query in the JSON +[query format](https://cube.dev/docs/query-format) ready. You can copy it from +Cube’s Playground or compose manually. -Paste the JSON query in the **Body**. Make sure to add a `query` parameter for your JSON query. +Paste the JSON query in the **Body**. Make sure to add a `query` parameter for +your JSON query. -
- -
+ Next, hit "Initialize / Reinitialize call". ## Create a workflow in Bubble -First, add a "Page is loaded" event. Add an action that uses the API connector plugin. Next, add another action to set the state of an element. Select the index element, and create a new custom state. In this example I named the custom state `OrdersByMonth` and selected the `data` field of the API response. Make sure to toggle the "This state is a list" checkbox as well. While setting the state's value you will need to select "Result of step 1" and the `data` field. +First, add a "Page is loaded" event. Add an action that uses the API connector +plugin. Next, add another action to set the state of an element. Select the +index element, and create a new custom state. In this example I named the custom +state `OrdersByMonth` and selected the `data` field of the API response. Make +sure to toggle the "This state is a list" checkbox as well. While setting the +state's value you will need to select "Result of step 1" and the `data` field. -
- -
+ ## Display the data in Bubble -Bubble has an amazing feature where you can drag and drop UI components into the design dashboard. +Bubble has an amazing feature where you can drag and drop UI components into the +design dashboard. -Once you added a chart, you configure the data points. More specifically, the series categories and series data. For the series categories and series data you need to select the `index` and `OrdersByMonth` state followed by the data point you want to use in order to populate the chart. +Once you added a chart, you configure the data points. More specifically, the +series categories and series data. For the series categories and series data you +need to select the `index` and `OrdersByMonth` state followed by the data point +you want to use in order to populate the chart. -
- -
+ Once you open the "Preview" you'll see your nicely rendered charts. -
- -
+ diff --git a/docs/content/Configuration/Downstream/Budibase.mdx b/docs/content/Configuration/Downstream/Budibase.mdx index f9490cb02670b..9cf43e2f8afde 100644 --- a/docs/content/Configuration/Downstream/Budibase.mdx +++ b/docs/content/Configuration/Downstream/Budibase.mdx @@ -3,7 +3,10 @@ title: Connecting to Budibase permalink: /config/downstream/budibase --- -Budibase is a low-code app-building platform that helps developers create responsive web applications with access methods to databases, APIs, and much more. You can connect a Cube deployment to Budibase using Cube's [REST API](https://cube.dev/docs/rest-api). +Budibase is a low-code app-building platform that helps developers create +responsive web applications with access methods to databases, APIs, and much +more. You can connect a Cube deployment to Budibase using Cube's +[REST API](https://cube.dev/docs/rest-api). Here's a short video guide on how to connect Budibase to Cube. @@ -11,83 +14,59 @@ Here's a short video guide on how to connect Budibase to Cube. ## Use REST API in Cube -> Don't have a Cube project yet? [Learn how to get started here](https://cube.dev/docs/cloud/getting-started). +> Don't have a Cube project yet? +> [Learn how to get started here](https://cube.dev/docs/getting-started/cloud/overview). ### <--{"id" : "Use REST API in Cube"}--> Cube Cloud -Click the "How to connect" link on the Overview page, navigate to the REST API tab. You will see your connection credentials, the REST API URL and the authorization token: +Click the "How to connect" link on the Overview page, navigate to the REST API +tab. You will see your connection credentials, the REST API URL and the +authorization token: -
- -
+ ### <--{"id" : "Use REST API in Cube"}--> Self-hosted Cube -For a Cube instance publicly available at a specific `HOST`, the REST API URL would be `HOST/cubejs-api/v1`. Please refer to the [REST API page](https://cube.dev/docs/rest-api) for details. +For a Cube instance publicly available at a specific `HOST`, the REST API URL +would be `HOST/cubejs-api/v1`. Please refer to the +[REST API page](https://cube.dev/docs/rest-api) for details. -You will also need to generate a JSON Web Token that would be used to authenticate requests to Cube. +You will also need to generate a JSON Web Token that would be used to +authenticate requests to Cube. -Please check the [Security page](https://cube.dev/docs/security#generating-json-web-tokens-jwt) to learn how to generate a token. We suggest generating a long-lived JWT that won't expire soon. +Please check the +[Security page](https://cube.dev/docs/security#generating-json-web-tokens-jwt) +to learn how to generate a token. We suggest generating a long-lived JWT that +won't expire soon. ## Create a new data source in Budibase Create a new "REST API" data source in Budibase. -
- -
+ -Next, add a "Authorization" header for the REST API. Copy and paste Cube's Authorization token. +Next, add a "Authorization" header for the REST API. Copy and paste Cube's +Authorization token. -
- -
+ ## Add a query in Budibase Click "+ Add Query". -Copy and paste the Cube REST API, including the `/load` path, in the API field. Make sure to select "POST" as the request. +Copy and paste the Cube REST API, including the `/load` path, in the API field. +Make sure to select "POST" as the request. -
- -
+ -Get your Cube query in the JSON [query format](https://cube.dev/docs/query-format) ready. You can copy it from Cube's Playground or compose manually: +Get your Cube query in the JSON +[query format](https://cube.dev/docs/query-format) ready. You can copy it from +Cube's Playground or compose manually. -
- -
+Paste the JSON query in the **Body** as "raw(JSON)". Make sure to add a `query` +parameter for your JSON query. -Paste the JSON query in the **Body** as "raw(JSON)". Make sure to add a `query` parameter for your JSON query. - -
- -
+ Edit the **Transform** to return the data of the request. @@ -95,52 +74,32 @@ Edit the **Transform** to return the data of the request. return data.data ``` -
- -
+ You can also give the query a name. In this sample it's called `OrdersByMonth`. -Lastly, click the "save" button and hit "send" to test the API and get a response back. +Lastly, click the "save" button and hit "send" to test the API and get a +response back. ## Create a data provider in Budibase Move to the **Design** section and open the **Components**. -First, add a Data Provider. Select the query from above. In this sample it's called `OrdersByMonth`. +First, add a Data Provider. Select the query from above. In this sample it's +called `OrdersByMonth`. -
- -
+ ## Display the data in Budibase -Add a chart into the data provider. Next, configure the data provider for the chart. Make sure it's set to the data provider you just created. +Add a chart into the data provider. Next, configure the data provider for the +chart. Make sure it's set to the data provider you just created. -Then, you set the "label column" and "data columns". In this sample you can set `Orders.createdAt` as the label column and `Orders.count` as the data columns. +Then, you set the "label column" and "data columns". In this sample you can set +`Orders.createdAt` as the label column and `Orders.count` as the data columns. -
- -
+ You'll see a nicely rendered chart show up. -
- -
+ diff --git a/docs/content/Configuration/Downstream/Deepnote.mdx b/docs/content/Configuration/Downstream/Deepnote.mdx index e98a194b21495..505bd41dee585 100644 --- a/docs/content/Configuration/Downstream/Deepnote.mdx +++ b/docs/content/Configuration/Downstream/Deepnote.mdx @@ -3,7 +3,9 @@ title: Connecting to Deepnote permalink: /config/downstream/deepnote --- -You can connect to Cube from Deepnote, a new kind of data notebook that's built for collaboration and is Jupyter compatible, using the [Cube SQL API][ref-sql-api]. +You can connect to Cube from Deepnote, a new kind of data notebook that's built +for collaboration and is Jupyter compatible, using the [Cube SQL +API][ref-sql-api]. Here's a short video guide on how to connect Deepnote to Cube. @@ -20,47 +22,37 @@ here][ref-getting-started]. ### <--{"id" : "Enable Cube SQL API"}--> Cube Cloud -Click **Deploy SQL API** and then the **How to connect your BI tool** link on the Overview page of your Cube deployment. -Navigate to the **BIs and Visualization Tools** tab. You should see the screen like the one below with +Click **Deploy SQL API** and then the **How to connect your BI tool** link on +the Overview page of your Cube deployment. Navigate to the **BIs and +Visualization Tools** tab. You should see the screen like the one below with your connection credentials: -
- -
+ ### <--{"id" : "Enable Cube SQL API"}--> Self-hosted Cube You need to set the following environment variables to enable the Cube SQL API. -These credentials will be required to connect to Cube from Deepnote -later. +These credentials will be required to connect to Cube from Deepnote later. ```dotenv CUBEJS_PG_SQL_PORT=5432 CUBEJS_SQL_USER=myusername CUBEJS_SQL_PASSWORD=mypassword ``` + ## Connecting from Deepnote Deepnote connects to Cube as to a Postgres database. -
- -
+ ## Querying data -Your cubes will be exposed as tables, where both your measures and dimensions are columns. +Your cubes will be exposed as tables, where both your measures and dimensions +are columns. -You can write SQL in Deepnote that will be executed in Cube. Learn more about Cube SQL -syntax on the [reference page][ref-sql-api]. +You can write SQL in Deepnote that will be executed in Cube. Learn more about +Cube SQL syntax on the [reference page][ref-sql-api].
-[ref-getting-started]: /cloud/getting-started +[ref-getting-started]: /getting-started/cloud/overview [ref-sql-api]: /backend/sql diff --git a/docs/content/Configuration/Downstream/Delphi.mdx b/docs/content/Configuration/Downstream/Delphi.mdx new file mode 100644 index 0000000000000..fce82e0d3d942 --- /dev/null +++ b/docs/content/Configuration/Downstream/Delphi.mdx @@ -0,0 +1,70 @@ +--- +title: Connecting to Delphi +permalink: /config/downstream/delphi +--- + +[Delphi][delphi-website] is a conversational interface for the semantic layer, +powered by the large language model (LLM) technology by OpenAI. + +It provides a Slack bot that takes questions in natural language, translates +them into queries to Cube, and delivers the results back to Slack. You can learn +more about Delphi from the [blog post][delphi-blog-post] or see it in action in +this video: + + + +To start using Delphi, you need to install its bot (application) to your Slack +workspace, configure it, add to necessary channels, and start asking questions. + +## Installing Delphi to Slack + +Go to the [Slack application][delphi-slack-install] page; this will start the +installation. + + + +You may need to be a Slack workspace administrator to install a new application. + + + +## Configuring the connection to Cube + +In Slack, navigate to the Delphi app on the left sidebar. + +In the Home tab, choose Cube as the connection type: + + + +Then, enter your credentials: + +- Delphi Client ID and API Key +- Cube [REST API][ref-rest-api] URL and a + [JSON Web Token](http://localhost:8000/security#generating-json-web-tokens-jwt) + + + + + +If you don’t have Delphi Client ID and API Key yet, get them by +[email][delphi-email] from the Delphi team. + + + +## Asking questions + +Send Delphi a direct message or add it to relevant channels. + +Start with `@Delphi` and ask anything about your Cube [data +model][ref-data-model]. + + + +[delphi-website]: https://www.delphihq.com/ +[delphi-blog-post]: + https://cube.dev/blog/conversational-interface-for-semantic-layer +[delphi-slack-install]: https://delphi-prod.onrender.com/slack/install +[delphi-email]: + mailto:michael@delphihq.com?subject=Delphi%20credentials%20to%20use%20with%20Cube +[ref-rest-api]: /http-api/rest +[ref-jwt]: /security#generating-json-web-tokens-jwt +[ref-data-model]: /schema/getting-started diff --git a/docs/content/Configuration/Downstream/Hex.mdx b/docs/content/Configuration/Downstream/Hex.mdx index 1a3ef5868bbf3..e3a0debc62bc0 100644 --- a/docs/content/Configuration/Downstream/Hex.mdx +++ b/docs/content/Configuration/Downstream/Hex.mdx @@ -3,8 +3,8 @@ title: Connecting to Hex permalink: /config/downstream/hex --- -You can connect to Cube from Hex, a collaborative data platform, using the [Cube SQL -API][ref-sql-api]. +You can connect to Cube from Hex, a collaborative data platform, using the [Cube +SQL API][ref-sql-api]. Here's a short video guide on how to connect Hex to Cube. @@ -21,43 +21,38 @@ here][ref-getting-started]. ### <--{"id" : "Enable Cube SQL API"}--> Cube Cloud -Click **How to connect your BI tool** link on the Overview page, navigate to the SQL API tab -and enable it. Once enabled, you should see the screen like the one below with -your connection credentials: +Click **How to connect your BI tool** link on the Overview page, navigate to the +SQL API tab and enable it. Once enabled, you should see the screen like the one +below with your connection credentials: -
- -
+ ### <--{"id" : "Enable Cube SQL API"}--> Self-hosted Cube You need to set the following environment variables to enable the Cube SQL API. -These credentials will be required to connect to Cube from Hex -later. +These credentials will be required to connect to Cube from Hex later. ```dotenv CUBEJS_PG_SQL_PORT=5432 CUBEJS_SQL_USER=myusername CUBEJS_SQL_PASSWORD=mypassword ``` + ## Connecting from Hex Hex connects to Cube as to a Postgres database. ## Querying data -Your cubes will be exposed as tables, where both your measures and dimensions are columns. +Your cubes will be exposed as tables, where both your measures and dimensions +are columns. -You can write SQL in Hex that will be executed in Cube. Learn more about Cube SQL -syntax on the [reference page][ref-sql-api]. +You can write SQL in Hex that will be executed in Cube. Learn more about Cube +SQL syntax on the [reference page][ref-sql-api].
@@ -67,11 +62,11 @@ You can also create a visualization of the executed SQL query.
-[ref-getting-started]: /cloud/getting-started +[ref-getting-started]: /getting-started/cloud/overview [ref-sql-api]: /backend/sql diff --git a/docs/content/Configuration/Downstream/Jupyter.mdx b/docs/content/Configuration/Downstream/Jupyter.mdx index 9d3b3cc1d979f..9debfac1ea0ac 100644 --- a/docs/content/Configuration/Downstream/Jupyter.mdx +++ b/docs/content/Configuration/Downstream/Jupyter.mdx @@ -3,7 +3,9 @@ title: Connecting to Jupyter permalink: /config/downstream/jupyter --- -You can connect to Cube from Jupyter using the [Cube SQL API][ref-sql-api]. The Jupyter Notebook is a web application for creating and sharing computational documents. +You can connect to Cube from Jupyter using the [Cube SQL API][ref-sql-api]. The +Jupyter Notebook is a web application for creating and sharing computational +documents. Here's a short video guide on how to connect Jupyter to Cube. @@ -13,42 +15,38 @@ Here's a short video guide on how to connect Jupyter to Cube. -Don't have a Cube project yet? [Learn how to get started here][ref-getting-started]. +Don't have a Cube project yet? [Learn how to get started +here][ref-getting-started]. ### <--{"id" : "Enable Cube SQL API"}--> Cube Cloud -Click **Deploy SQL API** and then the **How to connect your BI tool** link on the Overview page of your Cube deployment. -Navigate to the **BIs and Visualization Tools** tab. You should see the screen like the one below with +Click **Deploy SQL API** and then the **How to connect your BI tool** link on +the Overview page of your Cube deployment. Navigate to the **BIs and +Visualization Tools** tab. You should see the screen like the one below with your connection credentials: -
- -
+ ### <--{"id" : "Enable Cube SQL API"}--> Self-hosted Cube You need to set the following environment variables to enable the Cube SQL API. -These credentials will be required to connect to Cube from Jupyter -later. +These credentials will be required to connect to Cube from Jupyter later. -```bash +```dotenv CUBEJS_PG_SQL_PORT=5432 CUBEJS_SQL_USER=myusername CUBEJS_SQL_PASSWORD=mypassword ``` + ## Connecting from Jupyter Jupyter connects to Cube as to a Postgres database. Make sure to install the `sqlalchemy` and `pandas` modules. -```bash +```bash{promptUser: user} pip install sqlalchemy pip install pandas ``` @@ -78,10 +76,11 @@ connection = engine.connect() ## Querying data -Your cubes will be exposed as tables, where both your measures and dimensions are columns. +Your cubes will be exposed as tables, where both your measures and dimensions +are columns. -You can write SQL in Jupyter that will be executed in Cube. Learn more about Cube SQL -syntax on the [reference page][ref-sql-api]. +You can write SQL in Jupyter that will be executed in Cube. Learn more about +Cube SQL syntax on the [reference page][ref-sql-api]. ```python # ... @@ -110,5 +109,5 @@ You can also create a visualization of the executed SQL query. />
-[ref-getting-started]: /cloud/getting-started +[ref-getting-started]: /getting-started/cloud/overview [ref-sql-api]: /backend/sql diff --git a/docs/content/Configuration/Downstream/Metabase.mdx b/docs/content/Configuration/Downstream/Metabase.mdx index 846069b36e2b6..2b5de3b40b775 100644 --- a/docs/content/Configuration/Downstream/Metabase.mdx +++ b/docs/content/Configuration/Downstream/Metabase.mdx @@ -3,7 +3,9 @@ title: Connecting to Metabase permalink: /config/downstream/metabase --- -You can connect to Cube from Metabase by using the [Cube SQL API][ref-sql-api]. Metabase is the easy, open-source way to help everyone in your company work with data like an analyst. +You can connect to Cube from Metabase by using the [Cube SQL API][ref-sql-api]. +Metabase is the easy, open-source way to help everyone in your company work with +data like an analyst. Here's a short video guide on how to connect Metabase to Cube. @@ -13,35 +15,31 @@ Here's a short video guide on how to connect Metabase to Cube. -Don't have a Cube project yet? [Learn how to get started here][ref-getting-started]. +Don't have a Cube project yet? [Learn how to get started +here][ref-getting-started]. ### <--{"id" : "Enable Cube SQL API"}--> Cube Cloud -Click **Deploy SQL API** and then the **How to connect your BI tool** link on the Overview page of your Cube deployment. -Navigate to the **BIs and Visualization Tools** tab. You should see the screen like the one below with +Click **Deploy SQL API** and then the **How to connect your BI tool** link on +the Overview page of your Cube deployment. Navigate to the **BIs and +Visualization Tools** tab. You should see the screen like the one below with your connection credentials: -
- -
+ ### <--{"id" : "Enable Cube SQL API"}--> Self-hosted Cube You need to set the following environment variables to enable the Cube SQL API. -These credentials will be required to connect to Cube from Metabase -later. +These credentials will be required to connect to Cube from Metabase later. ```dotenv CUBEJS_PG_SQL_PORT=5432 CUBEJS_SQL_USER=myusername CUBEJS_SQL_PASSWORD=mypassword ``` + ## Connecting from Metabase Metabase connects to Cube as to a Postgres database. @@ -56,7 +54,8 @@ Metabase connects to Cube as to a Postgres database. ## Querying data -Your cubes will be exposed as tables, where both your measures and dimensions are columns. +Your cubes will be exposed as tables, where both your measures and dimensions +are columns.
-You can write SQL in Metabase that will be executed in Cube. Learn more about Cube SQL -syntax on the [reference page][ref-sql-api]. You can also create a visualization of the executed SQL query. +You can write SQL in Metabase that will be executed in Cube. Learn more about +Cube SQL syntax on the [reference page][ref-sql-api]. You can also create a +visualization of the executed SQL query.
-If you prefer using the UI interface to "Ask a question", you can do that as well. +If you prefer using the UI interface to "Ask a question", you can do that as +well.
-[ref-getting-started]: /cloud/getting-started +[ref-getting-started]: /getting-started/cloud/overview [ref-sql-api]: /backend/sql diff --git a/docs/content/Configuration/Downstream/Observable.mdx b/docs/content/Configuration/Downstream/Observable.mdx index 9d588a7521355..4c5bfe70f399e 100644 --- a/docs/content/Configuration/Downstream/Observable.mdx +++ b/docs/content/Configuration/Downstream/Observable.mdx @@ -3,7 +3,10 @@ title: Connecting to Observable permalink: /config/downstream/observable --- -You can connect to Cube from Observable, a new kind of collaborative data notebook that's built to uncover new insights, answer more questions, and make better decisions, using the [Cube SQL API][ref-sql-api] or [Cube REST API][ref-rest-api]. +You can connect to Cube from Observable, a new kind of collaborative data +notebook that's built to uncover new insights, answer more questions, and make +better decisions, using the [Cube SQL API][ref-sql-api] or [Cube REST +API][ref-rest-api]. Here's a short video guide on how to connect Observable to Cube. @@ -13,30 +16,24 @@ Here's a short video guide on how to connect Observable to Cube. -Don't have a Cube project yet? [Learn how to get started here][ref-getting-started]. +Don't have a Cube project yet? [Learn how to get started +here][ref-getting-started]. ### <--{"id" : "Use SQL API in Cube"}--> Cube Cloud - -Click **Deploy SQL API** and then the **How to connect your BI tool** link on the Overview page of your Cube deployment. -Navigate to the **BIs and Visualization Tools** tab. You should see the screen like the one below with +Click **Deploy SQL API** and then the **How to connect your BI tool** link on +the Overview page of your Cube deployment. Navigate to the **BIs and +Visualization Tools** tab. You should see the screen like the one below with your connection credentials: -
- -
+ ### <--{"id" : "Use SQL API in Cube"}--> Self-hosted Cube You need to set the following environment variables to enable the Cube SQL API. -These credentials will be required to connect to Cube from Observable -later. +These credentials will be required to connect to Cube from Observable later. ```dotenv CUBEJS_PG_SQL_PORT=5432 @@ -48,19 +45,15 @@ CUBEJS_SQL_PASSWORD=mypassword Observable connects to Cube as to a Postgres database. -
- -
+ ### <--{"id" : "Connecting Cube SQL API from Observable"}--> Querying data with SQL API -Your cubes will be exposed as tables, where both your measures and dimensions are columns. +Your cubes will be exposed as tables, where both your measures and dimensions +are columns. -Make sure to add a database to your notebook, and select **Database query** when adding a new block. +Make sure to add a database to your notebook, and select **Database query** when +adding a new block.
-You can write SQL in Observable that will be executed in Cube. Learn more about Cube SQL -syntax on the [reference page][ref-sql-api]. +You can write SQL in Observable that will be executed in Cube. Learn more about +Cube SQL syntax on the [reference page][ref-sql-api].
Don't have a Cube project yet? [Learn how to get started here](https://cube.dev/docs/cloud/getting-started). +> Don't have a Cube project yet? +> [Learn how to get started here](https://cube.dev/docs/getting-started/cloud/overview). ### <--{"id" : "Use REST API in Cube"}--> Cube Cloud -Click the "How to connect" link on the Overview page, navigate to the REST API tab. You will see your connection credentials, the REST API URL and the authorization token: +Click the "How to connect" link on the Overview page, navigate to the REST API +tab. You will see your connection credentials, the REST API URL and the +authorization token: -
- -
+ ### <--{"id" : "Use REST API in Cube"}--> Self-hosted Cube -For a Cube instance publicly available at a specific `HOST`, the REST API URL would be `HOST/cubejs-api/v1`. Please refer to the [REST API page](https://cube.dev/docs/rest-api) for details. +For a Cube instance publicly available at a specific `HOST`, the REST API URL +would be `HOST/cubejs-api/v1`. Please refer to the +[REST API page](https://cube.dev/docs/rest-api) for details. -You will also need to generate a JSON Web Token that would be used to authenticate requests to Cube. +You will also need to generate a JSON Web Token that would be used to +authenticate requests to Cube. -Please check the [Security page](https://cube.dev/docs/security#generating-json-web-tokens-jwt) to learn how to generate a token. We suggest generating a long-lived JWT that won't expire soon. +Please check the +[Security page](https://cube.dev/docs/security#generating-json-web-tokens-jwt) +to learn how to generate a token. We suggest generating a long-lived JWT that +won't expire soon. ## Connecting Cube REST API from Observable @@ -131,62 +127,65 @@ First, add two generic **JavaScript** cells: />
-Next, copy Cube's REST API URL and the Authorization token and paste them into their respective cells. +Next, copy Cube's REST API URL and the Authorization token and paste them into +their respective cells. -```js -cubeRestApi = 'https://thirsty-raccoon.aws-eu-central-1.cubecloudapp.dev/cubejs-api/v1/load' +```javascript +cubeRestApi = + 'https://thirsty-raccoon.aws-eu-central-1.cubecloudapp.dev/cubejs-api/v1/load'; ``` -Because the Cube REST API has the format of `HOST/cubejs-api/v1`, don't forget to add the `/load` endpoint to the end of the data source API. +Because the Cube REST API has the format of `HOST/cubejs-api/v1`, don't forget +to add the `/load` endpoint to the end of the data source API. -```js -cubeRestApiJwtToken = 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE2NTgzMzM3OTZ9.gUOoDgo_RJka_ZANdwSw3v8GkM4ZzH9LjxrxKxkGAk0' +```javascript +cubeRestApiJwtToken = + 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE2NTgzMzM3OTZ9.gUOoDgo_RJka_ZANdwSw3v8GkM4ZzH9LjxrxKxkGAk0'; ``` -Also make sure to add the token next to the Bearer part of the Authorization header. - -Get your Cube query in the JSON [query format](https://cube.dev/docs/query-format) ready. You can copy it from Cube's Playground or compose manually: +Also make sure to add the token next to the Bearer part of the Authorization +header. -
- -
+Get your Cube query in the JSON +[query format](https://cube.dev/docs/query-format) ready. You can copy it from +Cube's Playground or compose manually. -Paste the JSON query in another JavaScript cell as an object literal and give it a name, I chose `jsonBody` for simplicity. Make sure to add a `query` parameter for your JSON query. +Paste the JSON query in another JavaScript cell as an object literal and give it +a name, I chose `jsonBody` for simplicity. Make sure to add a `query` parameter +for your JSON query. -```js -jsonQuery = ({ - "query": { - "measures": [ - "Orders.count" - ], - "timeDimensions": [ +```javascript +jsonQuery = { + query: { + measures: ['orders.count'], + timeDimensions: [ { - "dimension": "Orders.createdAt", - "granularity": "month" - } + dimension: 'orders.created_at', + granularity: 'month', + }, ], - "order": { - "Orders.createdAt": "asc" - } - } -}); + order: { + 'orders.created_at': 'asc', + }, + }, +}; ``` -Next, create another JavaScript cell with a POST request. Paste this POST request in the cell. Don't forget to put the `jsonBody` object inside the `JSON.stringify` call. +Next, create another JavaScript cell with a POST request. Paste this POST +request in the cell. Don't forget to put the `jsonBody` object inside the +`JSON.stringify` call. -```js +```javascript orders_over_time = fetch(cubeRestApi, { method: 'POST', headers: { - 'Authorization': cubeRestApiJwtToken, - 'Content-Type': 'application/json' + Authorization: cubeRestApiJwtToken, + 'Content-Type': 'application/json', }, body: JSON.stringify(jsonQuery), -}).then(response => response.json()).then(json => json.data) +}) + .then((response) => response.json()) + .then((json) => json.data); ``` Next, click the play button on the top right of the cell. @@ -209,6 +208,6 @@ You can also create a visualization of the executed REST API request. />
-[ref-getting-started]: /cloud/getting-started +[ref-getting-started]: /getting-started/cloud/overview [ref-sql-api]: /backend/sql [ref-rest-api]: /backend/rest-api diff --git a/docs/content/Configuration/Downstream/PowerBI.md b/docs/content/Configuration/Downstream/PowerBI.md deleted file mode 100644 index fe0973ffdd9f0..0000000000000 --- a/docs/content/Configuration/Downstream/PowerBI.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -title: Connecting from Power BI -permalink: /config/downstream/powerbi ---- - - - -Power BI support is in preview, not all features and requests may work at this -point. - - - -You can connect to Cube from Power BI, interactive data visualization software product developed by Microsoft, using the [Cube SQL -API][ref-sql-api]. - -## Enable Cube SQL API - - - -Don't have a Cube project yet? [Learn how to get started -here][ref-getting-started]. - - - -### <--{"id" : "Enable Cube SQL API"}--> Cube Cloud - -Click **How to connect your BI tool** link on the Overview page, navigate to the SQL API tab -and enable it. Once enabled, you should see the screen like the one below with -your connection credentials: - -
- -
- -### <--{"id" : "Enable Cube SQL API"}--> Self-hosted Cube - -You need to set the following environment variables to enable the Cube SQL API. -These credentials will be required to connect to Cube from PowerBI -later. - -```dotenv -CUBEJS_PG_SQL_PORT=5432 -CUBEJS_SQL_USER=myusername -CUBEJS_SQL_PASSWORD=mypassword -``` - -## Connecting from Power BI - -Power BI connects to Cube as to a Postgres database. - -## Querying data - -Your cubes will be exposed as tables, where both your measures and dimensions are columns. - - -[ref-getting-started]: /cloud/getting-started -[ref-sql-api]: /backend/sql diff --git a/docs/content/Configuration/Downstream/PowerBI.mdx b/docs/content/Configuration/Downstream/PowerBI.mdx new file mode 100644 index 0000000000000..87d5ed5d8f052 --- /dev/null +++ b/docs/content/Configuration/Downstream/PowerBI.mdx @@ -0,0 +1,54 @@ +--- +title: Connecting from Power BI +permalink: /config/downstream/powerbi +--- + + + +Power BI support is in preview, not all features and requests may work at this +point. + + + +You can connect to Cube from Power BI, interactive data visualization software +product developed by Microsoft, using the [Cube SQL API][ref-sql-api]. + +## Enable Cube SQL API + + + +Don't have a Cube project yet? [Learn how to get started +here][ref-getting-started]. + + + +### <--{"id" : "Enable Cube SQL API"}--> Cube Cloud + +Click **How to connect your BI tool** link on the Overview page, navigate to the +SQL API tab and enable it. Once enabled, you should see the screen like the one +below with your connection credentials: + + + +### <--{"id" : "Enable Cube SQL API"}--> Self-hosted Cube + +You need to set the following environment variables to enable the Cube SQL API. +These credentials will be required to connect to Cube from PowerBI later. + +```dotenv +CUBEJS_PG_SQL_PORT=5432 +CUBEJS_SQL_USER=myusername +CUBEJS_SQL_PASSWORD=mypassword +``` + +## Connecting from Power BI + +Power BI connects to Cube as to a Postgres database. + +## Querying data + +Your cubes will be exposed as tables, where both your measures and dimensions +are columns. + +[ref-getting-started]: /getting-started/cloud/overview +[ref-sql-api]: /backend/sql diff --git a/docs/content/Configuration/Downstream/Retool.mdx b/docs/content/Configuration/Downstream/Retool.mdx index 28936ff3f0498..fac1c0c8d197a 100644 --- a/docs/content/Configuration/Downstream/Retool.mdx +++ b/docs/content/Configuration/Downstream/Retool.mdx @@ -3,7 +3,10 @@ title: Connecting to Retool permalink: /config/downstream/retool --- -[Retool](https://retool.com/) is the fast way to build internal tools. It lets you visually design apps that interface with any database or API. You can connect a Cube deployment to Retool using Cube’s [REST API](https://cube.dev/docs/rest-api). +[Retool](https://retool.com/) is the fast way to build internal tools. It lets +you visually design apps that interface with any database or API. You can +connect a Cube deployment to Retool using Cube’s +[REST API](https://cube.dev/docs/rest-api). Here's a short video guide on how to connect Retool to Cube. @@ -11,81 +14,65 @@ Here's a short video guide on how to connect Retool to Cube. ## Use REST API in Cube -> Don't have a Cube project yet? [Learn how to get started here](https://cube.dev/docs/cloud/getting-started). +> Don't have a Cube project yet? +> [Learn how to get started here](https://cube.dev/docs/getting-started/cloud/overview). ### <--{"id" : "Use REST API in Cube"}--> Cube Cloud -Click the "How to connect" link on the Overview page, navigate to the REST API tab. You will see your connection credentials, the REST API URL and the authorization token: +Click the "How to connect" link on the Overview page, navigate to the REST API +tab. You will see your connection credentials, the REST API URL and the +authorization token: -
- -
+ ### <--{"id" : "Use REST API in Cube"}--> Self-hosted Cube -For a Cube instance publicly available at a specific `HOST`, the REST API URL would be `HOST/cubejs-api/v1`. Please refer to the [REST API page](https://cube.dev/docs/rest-api) for details. +For a Cube instance publicly available at a specific `HOST`, the REST API URL +would be `HOST/cubejs-api/v1`. Please refer to the +[REST API page](https://cube.dev/docs/rest-api) for details. -You will also need to generate a JSON Web Token that would be used to authenticate requests to Cube. +You will also need to generate a JSON Web Token that would be used to +authenticate requests to Cube. -Please check the [Security page](https://cube.dev/docs/security#generating-json-web-tokens-jwt) to learn how to generate a token. We suggest generating a long-lived JWT that won't expire soon. +Please check the +[Security page](https://cube.dev/docs/security#generating-json-web-tokens-jwt) +to learn how to generate a token. We suggest generating a long-lived JWT that +won't expire soon. ## Create a new resource in Retool Create a new "REST API" resource in Retool. -
- -
+ Next, copy and paste Cube's REST API URL and the Authorization token. -
- -
+ -## Create a POST request in Retool +## Create a POST request in Retool -Get your Cube query in the JSON [query format](https://cube.dev/docs/query-format) ready. You can copy it from Cube’s Playground or compose manually: +Get your Cube query in the JSON +[query format](https://cube.dev/docs/query-format) ready. You can copy it from +Cube’s Playground or compose manually. -
- -
+Create a POST request, paste the JSON query in the **Body**. Make sure to add a +`query` parameter for your JSON query. -Create a POST request, paste the JSON query in the **Body**. Make sure to add a `query` parameter for your JSON query. - -Because the Cube REST API has the format of `HOST/cubejs-api/v1`, don't forget to add the `/load` endpoint to the end of the data source API. +Because the Cube REST API has the format of `HOST/cubejs-api/v1`, don't forget +to add the `/load` endpoint to the end of the data source API. Next, hit Run. -
- -
+ ## Display the data in Retool -Retool has an amazing feature where you can drag and drop UI components into the dashboard. You can use this to add a tables, bar charts, and much more. +Retool has an amazing feature where you can drag and drop UI components into the +dashboard. You can use this to add a tables, bar charts, and much more. -Because the name of the Retool query in the example is `OrdersByMonth`, using the data binding curly brackets will populate the charts with data from the REST API. +Because the name of the Retool query in the example is `OrdersByMonth`, using +the data binding curly brackets will populate the charts with data from the REST +API. ``` {{ OrdersByMonth.data.data }} @@ -93,20 +80,8 @@ Because the name of the Retool query in the example is `OrdersByMonth`, using th Reference the name of the query in your Retool charts. -
- -
+ Get nicely rendered charts. -
- -
+ diff --git a/docs/content/Configuration/Downstream/Streamlit.mdx b/docs/content/Configuration/Downstream/Streamlit.mdx index 91037b77a4547..4907342d53d82 100644 --- a/docs/content/Configuration/Downstream/Streamlit.mdx +++ b/docs/content/Configuration/Downstream/Streamlit.mdx @@ -3,7 +3,8 @@ title: Connecting to Streamlit permalink: /config/downstream/streamlit --- -You can connect to Cube from Streamlit using the [Cube SQL API][ref-sql-api]. Streamlit turns data scripts into shareable web apps in minutes. +You can connect to Cube from Streamlit using the [Cube SQL API][ref-sql-api]. +Streamlit turns data scripts into shareable web apps in minutes. Here's a short video guide on how to connect Streamlit to Cube. @@ -13,42 +14,38 @@ Here's a short video guide on how to connect Streamlit to Cube. -Don't have a Cube project yet? [Learn how to get started here][ref-getting-started]. +Don't have a Cube project yet? [Learn how to get started +here][ref-getting-started]. ### <--{"id" : "Enable Cube SQL API"}--> Cube Cloud -Click **Deploy SQL API** and then the **How to connect your BI tool** link on the Overview page of your Cube deployment. -Navigate to the **BIs and Visualization Tools** tab. You should see the screen like the one below with +Click **Deploy SQL API** and then the **How to connect your BI tool** link on +the Overview page of your Cube deployment. Navigate to the **BIs and +Visualization Tools** tab. You should see the screen like the one below with your connection credentials: -
- -
+ ### <--{"id" : "Enable Cube SQL API"}--> Self-hosted Cube You need to set the following environment variables to enable the Cube SQL API. -These credentials will be required to connect to Cube from Streamlit -later. +These credentials will be required to connect to Cube from Streamlit later. -```bash +```dotenv CUBEJS_PG_SQL_PORT=5432 CUBEJS_SQL_USER=myusername CUBEJS_SQL_PASSWORD=mypassword ``` + ## Connecting from Jupyter Jupyter connects to Cube as to a Postgres database. Make sure to install the `streamlit`, `sqlalchemy` and `pandas` modules. -```bash +```bash{promptUser: user} pip install streamlit pip install sqlalchemy pip install pandas @@ -80,21 +77,23 @@ connection = engine.connect() ## Querying data -Your cubes will be exposed as tables, where both your measures and dimensions are columns. +Your cubes will be exposed as tables, where both your measures and dimensions +are columns. -You can write SQL in Streamlit that will be executed in Cube. Learn more about Cube SQL -syntax on the [reference page][ref-sql-api]. +You can write SQL in Streamlit that will be executed in Cube. Learn more about +Cube SQL syntax on the [reference page][ref-sql-api]. ```python # ... with streamlit.echo(): - query = "select sum(count) as orders_count, status from orders group by status;" + query = "SELECT sum(count) AS orders_count, status FROM orders GROUP BY status;" df = pandas.read_sql_query(query, connection) streamlit.dataframe(df) ``` -In your Streamlit notebook it'll look like this. You can create a visualization of the executed SQL query by using `streamlit.dataframe(df)`. +In your Streamlit notebook it'll look like this. You can create a visualization +of the executed SQL query by using `streamlit.dataframe(df)`.
-[ref-getting-started]: /cloud/getting-started +[ref-getting-started]: /getting-started/cloud/overview [ref-sql-api]: /backend/sql diff --git a/docs/content/Configuration/Downstream/Superset.mdx b/docs/content/Configuration/Downstream/Superset.mdx index 3b8df70ad3f16..fd6ae56f27e37 100644 --- a/docs/content/Configuration/Downstream/Superset.mdx +++ b/docs/content/Configuration/Downstream/Superset.mdx @@ -29,21 +29,14 @@ Click **How to connect your BI tool** link on the Overview page, navigate to the SQL API tab and enable it. Once enabled, you should see the screen like the one below with your connection credentials: -
- -
+ ### <--{"id" : "Enable Cube SQL API"}--> Self-hosted Cube You need to set the following environment variables to enable the Cube SQL API. -These credentials will be required to connect to Cube from Superset -later. +These credentials will be required to connect to Cube from Superset later. -```bash +```dotenv CUBEJS_PG_SQL_PORT=5432 CUBEJS_SQL_USER=myusername CUBEJS_SQL_PASSWORD=mypassword @@ -56,25 +49,42 @@ Apache Superset connects to Cube as to a Postgres database. In Apache Superset, go to **Data > Databases**, then click **+ Database** to add a new database: -
- Apache Superset: databases page -
+ ## Querying data Your cubes will be exposed as tables, where both your measures and dimensions are columns. -Let's use the following Cube data schema: +Let's use the following Cube data model: -```js -cube(`Orders`, { - sql: `SELECT * FROM public.orders`, + + +```yaml +cubes: + - name: orders + sql_table: orders + + measures: + - name: count + type: count + + dimensions: + - name: status + sql: status + type: string + + - name: created + sql: created_at + type: time +``` + +```javascript +cube(`orders`, { + sql_table: `orders`, measures: { count: { @@ -88,7 +98,7 @@ cube(`Orders`, { type: `string`, }, - created: { + created_at: { sql: `created_at`, type: `time`, }, @@ -96,48 +106,41 @@ cube(`Orders`, { }); ``` -Using the SQL API, `Orders` will be exposed as a table. In Superset, we can -create datasets based on tables. Let's create one from `Orders` table: + + +Using the SQL API, `orders` will be exposed as a table. In Superset, we can +create datasets based on tables. Let's create one from `orders` table: -
- Apache Superset: SQL Editor page with successful query -
+ Now, we can explore this dataset. Let's create a new chart of type line with -**Orders** dataset. - -
- Apache Superset: SQL Editor page with successful query -
- -We can select the `COUNT(*)` as a metric and `createdAt` as the time column with +"Orders" dataset. + + + +We can select the `COUNT(*)` as a metric and `created_at` as the time column with a time grain of `month`. The `COUNT(*)` aggregate function is being mapped to a measure of type [count](/schema/reference/types-and-formats#measures-types-count) in Cube's -**Orders** schema file. +**Orders** data model file. ## Additional Configuration ### <--{"id" : "Additional Configuration"}--> Pre-Aggregations To allow queries from Superset to match pre-aggregations in Cube, [the -`allowNonStrictDateRangeMatch` property][ref-schema-ref-preagg-allownonstrict] +`allow_non_strict_date_range_match` property][ref-schema-ref-preagg-allownonstrict] must be set to `true` in the pre-aggregation definition. This is because Superset uses loose date ranges when generating SQL queries. -[ref-cube-getting-started-docker]: https://cube.dev/docs/getting-started/docker -[ref-getting-started]: /cloud/getting-started +[ref-getting-started]: /getting-started/cloud/overview [ref-schema-ref-preagg-allownonstrict]: /schema/reference/pre-aggregations#allow-non-strict-date-range-match [superset]: https://superset.apache.org/ diff --git a/docs/content/Configuration/Downstream/Tableau.mdx b/docs/content/Configuration/Downstream/Tableau.mdx index 0ce5f74974d57..c6d80942f7499 100644 --- a/docs/content/Configuration/Downstream/Tableau.mdx +++ b/docs/content/Configuration/Downstream/Tableau.mdx @@ -3,36 +3,30 @@ title: Connecting from Tableau permalink: /config/downstream/tableau --- -You can connect to Cube from Tableau, a visual analytics platform, using the [Cube SQL -API][ref-sql-api]. +You can connect to Cube from Tableau, a visual analytics platform, using the +[Cube SQL API][ref-sql-api]. ## Enable Cube SQL API -Don't have a Cube project yet? [Learn how to get started here][ref-getting-started]. +Don't have a Cube project yet? [Learn how to get started +here][ref-getting-started]. ### <--{"id" : "Enable Cube SQL API"}--> Cube Cloud -Click **How to connect your BI tool** link on the Overview page, navigate to the SQL API tab -and enable it. Once enabled, you should see the screen like the one below with -your connection credentials: +Click **How to connect your BI tool** link on the Overview page, navigate to the +SQL API tab and enable it. Once enabled, you should see the screen like the one +below with your connection credentials: -
- -
+ ### <--{"id" : "Enable Cube SQL API"}--> Self-hosted Cube You need to set the following environment variables to enable the Cube SQL API. -These credentials will be required to connect to Cube from Tableau -later. +These credentials will be required to connect to Cube from Tableau later. ```dotenv CUBEJS_PG_SQL_PORT=5432 @@ -47,16 +41,20 @@ Tableau connects to Cube as to a Postgres database. In Tableau, select PostgreSQL connector and enter credentials from the above step. -![adding Cube to Tableau as a SQL data source][ref-connecting-from-tableau] + ## Querying data -Your cubes will be exposed as tables, where both your measures and dimensions are columns. +Your cubes will be exposed as tables, where both your measures and dimensions +are columns. -![Cube data in Tableau][ref-querying-from-tableau] + -[ref-getting-started]: /cloud/getting-started +[ref-getting-started]: /getting-started/cloud/overview [ref-sql-api]: /backend/sql - -[ref-connecting-from-tableau]: https://cubedev-blog-images.s3.us-east-2.amazonaws.com/dc025b24-674f-4f32-ac44-421d546ee676.GIF -[ref-querying-from-tableau]: https://cubedev-blog-images.s3.us-east-2.amazonaws.com/ea73a998-e2ce-4814-863e-425b4d35860c.gif diff --git a/docs/content/Configuration/Downstream/Thoughtspot.mdx b/docs/content/Configuration/Downstream/Thoughtspot.mdx new file mode 100644 index 0000000000000..a325f05492f96 --- /dev/null +++ b/docs/content/Configuration/Downstream/Thoughtspot.mdx @@ -0,0 +1,71 @@ +--- +title: Connecting from Thoughtspot +permalink: /config/downstream/thoughtspot +--- + +You can connect to Cube from [Thoughtspot][thoughtspot] using the [Cube SQL +API][ref-sql-api]. + +## Enable Cube SQL API + + + +Don't have a Cube project yet? [Learn how to get started +here][ref-getting-started]. + + + +### <--{"id" : "Enable Cube SQL API"}--> Cube Cloud + +Click **How to connect your BI tool** link on the Overview page, navigate to the +SQL API tab and enable it. Once enabled, you should see the screen like the one +below with your connection credentials: + + + +### <--{"id" : "Enable Cube SQL API"}--> Self-hosted Cube + +You need to set the following environment variables to enable the Cube SQL API. +These credentials will be required to connect to Cube from Tableau later. + +```dotenv +CUBEJS_PG_SQL_PORT=5432 +CUBEJS_SQL_USER=myusername +CUBEJS_SQL_PASSWORD=mypassword +``` + +## Connecting from Thoughtspot + +Thoughtspot connects to Cube as a Redshift database. + +In Thoughtspot, go to the Setup tab, then click **Connect now** to add a new +data source: + + + +Enter a name for the data source, choose **Amazon Redshift** as the data +warehouse and click **Continue** in the header: + + + +Enter credentials from the previous step and click **Continue**: + + + +Select columns from the desired cubes and click **Create Connection** on the +next screen: + + + +## Querying data + +Your cubes will be exposed as tables, where both your measures and dimensions +are columns. + +[ref-getting-started]: /getting-started/cloud/overview +[ref-sql-api]: /backend/sql +[ref-connecting-from-tableau]: + https://cubedev-blog-images.s3.us-east-2.amazonaws.com/dc025b24-674f-4f32-ac44-421d546ee676.GIF +[ref-querying-from-tableau]: + https://cubedev-blog-images.s3.us-east-2.amazonaws.com/ea73a998-e2ce-4814-863e-425b4d35860c.gif +[thoughtspot]: https://www.thoughtspot.com/ diff --git a/docs/content/Configuration/Downstream/apache-superset-1.png b/docs/content/Configuration/Downstream/apache-superset-1.png deleted file mode 100755 index 4433fdb021fe7..0000000000000 Binary files a/docs/content/Configuration/Downstream/apache-superset-1.png and /dev/null differ diff --git a/docs/content/Configuration/Downstream/apache-superset-2.png b/docs/content/Configuration/Downstream/apache-superset-2.png deleted file mode 100755 index 5d234e13c4565..0000000000000 Binary files a/docs/content/Configuration/Downstream/apache-superset-2.png and /dev/null differ diff --git a/docs/content/Configuration/Downstream/apache-superset-3.png b/docs/content/Configuration/Downstream/apache-superset-3.png deleted file mode 100644 index a28010fb4de5b..0000000000000 Binary files a/docs/content/Configuration/Downstream/apache-superset-3.png and /dev/null differ diff --git a/docs/content/Configuration/Downstream/apache-superset-4.png b/docs/content/Configuration/Downstream/apache-superset-4.png deleted file mode 100755 index c6ab3418e4669..0000000000000 Binary files a/docs/content/Configuration/Downstream/apache-superset-4.png and /dev/null differ diff --git a/docs/content/Configuration/Downstream/apache-superset-5.png b/docs/content/Configuration/Downstream/apache-superset-5.png deleted file mode 100755 index d629760277c37..0000000000000 Binary files a/docs/content/Configuration/Downstream/apache-superset-5.png and /dev/null differ diff --git a/docs/content/Configuration/Downstream/apache-superset-6.png b/docs/content/Configuration/Downstream/apache-superset-6.png deleted file mode 100755 index f8a71c4d16b46..0000000000000 Binary files a/docs/content/Configuration/Downstream/apache-superset-6.png and /dev/null differ diff --git a/docs/content/Configuration/Downstream/appsmith-1.png b/docs/content/Configuration/Downstream/appsmith-1.png deleted file mode 100644 index 80bfc02b440e3..0000000000000 Binary files a/docs/content/Configuration/Downstream/appsmith-1.png and /dev/null differ diff --git a/docs/content/Configuration/Downstream/appsmith-2.png b/docs/content/Configuration/Downstream/appsmith-2.png deleted file mode 100644 index 48936eff25646..0000000000000 Binary files a/docs/content/Configuration/Downstream/appsmith-2.png and /dev/null differ diff --git a/docs/content/Configuration/Downstream/appsmith-3.png b/docs/content/Configuration/Downstream/appsmith-3.png deleted file mode 100644 index 24b8e96700b5e..0000000000000 Binary files a/docs/content/Configuration/Downstream/appsmith-3.png and /dev/null differ diff --git a/docs/content/Configuration/Downstream/appsmith-4.png b/docs/content/Configuration/Downstream/appsmith-4.png deleted file mode 100644 index 1f7791203cc43..0000000000000 Binary files a/docs/content/Configuration/Downstream/appsmith-4.png and /dev/null differ diff --git a/docs/content/Configuration/Downstream/appsmith-5.png b/docs/content/Configuration/Downstream/appsmith-5.png deleted file mode 100644 index daa26b941ca68..0000000000000 Binary files a/docs/content/Configuration/Downstream/appsmith-5.png and /dev/null differ diff --git a/docs/content/Configuration/Downstream/hex-1.png b/docs/content/Configuration/Downstream/hex-1.png deleted file mode 100644 index fab5c6359e90f..0000000000000 Binary files a/docs/content/Configuration/Downstream/hex-1.png and /dev/null differ diff --git a/docs/content/Configuration/Downstream/hex-2.png b/docs/content/Configuration/Downstream/hex-2.png deleted file mode 100644 index 872bad2be781a..0000000000000 Binary files a/docs/content/Configuration/Downstream/hex-2.png and /dev/null differ diff --git a/docs/content/Configuration/Downstream/hex.png b/docs/content/Configuration/Downstream/hex.png deleted file mode 100644 index b12433f565ca6..0000000000000 Binary files a/docs/content/Configuration/Downstream/hex.png and /dev/null differ diff --git a/docs/content/Configuration/Downstream/streamlit.png b/docs/content/Configuration/Downstream/streamlit.png deleted file mode 100644 index bc68e563ef815..0000000000000 Binary files a/docs/content/Configuration/Downstream/streamlit.png and /dev/null differ diff --git a/docs/content/Configuration/Environment-Variables-Reference.md b/docs/content/Configuration/Environment-Variables-Reference.md deleted file mode 100644 index feba017fe89fc..0000000000000 --- a/docs/content/Configuration/Environment-Variables-Reference.md +++ /dev/null @@ -1,170 +0,0 @@ ---- -title: Environment Variables -permalink: /reference/environment-variables -category: Configuration -subCategory: Reference -menuOrder: 4 ---- - -Cube.js defines a number of environment variables that can be used to change -behavior. Some of these variables can also be set via [configuration -options][link-config]. - -[link-config]: /config - -## General - -| Environment variable | Description | Possible Values | -| -------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| `CUBEJS_CACHE_AND_QUEUE_DRIVER` | The cache and queue driver to use for the Cube.js deployment. Defaults to `redis` | `redis`, `memory` | -| `CUBEJS_DEV_MODE` | If `true`, enables development mode. Defaults to `false` | `true`, `false` | -| `CUBEJS_API_SECRET` | The secret key used to sign and verify JWTs. Generated on project scaffold | A valid string | -| `CUBEJS_APP` | An application ID used to uniquely identify the Cube.js deployment. Can be different for multitenant setups. Defaults to `cubejs` | A valid string | -| `CUBEJS_BUILD_PRE_AGGREGATIONS` | If `true`, this instance of Cube will build pre-aggregations, even if it is an API instance. Defaults to `false` | `true`, `false` | -| `CUBEJS_REFRESH_WORKER` | If `true`, this instance of Cube.js will **only** refresh pre-aggregations. Defaults to `false` | `true`, `false` | -| `CUBEJS_ROLLUP_ONLY` | If `true`, this instance of Cube.js will **only** query rollup pre-aggregations. Defaults to `false` | `true`, `false` | -| `CUBEJS_SCHEDULED_REFRESH_TIMEZONES` | A comma-separated [list of timezones to schedule refreshes for][ref-config-sched-refresh-timer]. | [A valid timezone from the tz database][link-tz-database] | -| `CUBEJS_SCHEDULED_REFRESH_CONCURRENCY` | How many pre-aggregations refresh worker will build in parallel. Please note changing this param doesn't change queue concurrency and it should be adjusted accordingly | A valid number of concurrent refresh processes | -| `CUBEJS_PRE_AGGREGATIONS_SCHEMA` | The [schema name][ref-config-preagg-schema-name] to use for storing pre-aggregations. Defaults to `dev_pre_aggregations`/`prod_pre_aggregations` for development/production mode | A valid string | -| `CUBEJS_SCHEMA_PATH` | The path where Cube.js loads schemas from. Defaults to `schema` | A valid folder containing Cube.js schemas | -| `CUBEJS_TELEMETRY` | If `true`, then send telemetry to CubeJS. Defaults to `true` | `true`, `false` | -| `CUBEJS_WEB_SOCKETS` | If `true`, then use WebSocket for data fetching. Defaults to `true` | `true`, `false` | -| `PORT` | The port for a Cube.js deployment to listen to API connections on. Defaults to `4000` | A valid port number | -| `CUBEJS_LOG_LEVEL` | The logging level for Cube.js. Defaults to `warn` | `error`, `info`, `trace`, `warn` | -| `DEBUG_LOG` | If `true`, enables debug logging. Defaults to `false` | `true`, `false` | -| `CUBEJS_CONCURRENCY` | The number of concurrent connections each query queue has to the database. Defaults are database-dependent, check database page for more details | A valid number | -| `CUBEJS_DB_MAX_POOL` | The maximum number of concurrent database connections to pool. Defaults are database-dependent, check database page for more details | A valid number | -| `CUBEJS_DB_QUERY_TIMEOUT` | The timeout value for any queries made to the database by Cube. The value can be a number in seconds or a duration string. Defaults to `10m` | `5000`, `1s`, `1m`, `1h` | -| `CUBEJS_REDIS_URL` | The host URL for a Redis server | A valid Redis host URL | -| `CUBEJS_REDIS_PASSWORD` | The password used to connect to the Redis server | A valid Redis password | -| `CUBEJS_REDIS_TLS` | If `true`, then the connection to the Redis server is protected by TLS authentication. Defaults to `false` | `true`, `false` | -| `CUBEJS_REDIS_POOL_MAX` | The maximum number of connections to keep active in the Redis connection pool for a single `appId` (tenant). Must be higher than `CUBEJS_REDIS_POOL_MIN`. Defaults to `1000` | A valid number of connections. | -| `CUBEJS_REDIS_POOL_MIN` | The minimum number of connections to keep active in the Redis connection pool for a single `appId` (tenant). Must be lower than `CUBEJS_REDIS_POOL_MAX`. Defaults to `2` | A valid number of connections | -| `CUBEJS_REDIS_USE_IOREDIS` | Use [`ioredis`][gh-ioredis] instead of[ `redis`][gh-node-redis]. Defaults to `false` | `true`, `false` | -| `CUBEJS_JWK_URL` | A valid URL to a JSON Web Key Sets (JWKS) | `https://.auth0.com/.well-known/jwks.json` | -| `CUBEJS_JWT_KEY` | The secret key used to sign and verify JWTs. Similar to `CUBEJS_API_SECRET` | A valid string | -| `CUBEJS_JWT_AUDIENCE` | An audience value which will be used to enforce the [`aud` claim from inbound JWTs][link-jwt-ref-aud] | `https://myapp.com` | -| `CUBEJS_JWT_ISSUER` | An issuer value which will be used to enforce the [`iss` claim from inbound JWTs][link-jwt-ref-iss] | `https://.auth0.com/` | -| `CUBEJS_JWT_SUBJECT` | A subject value which will be used to enforce the [`sub` claim from inbound JWTs][link-jwt-ref-sub] | `person@example.com` | -| `CUBEJS_JWT_ALGS` | [Any supported algorithm for decoding JWTs][gh-jsonwebtoken-algs] | `HS256`, `RS256` | -| `CUBEJS_JWT_CLAIMS_NAMESPACE` | A namespace within the decoded JWT under which any custom claims can be found | `https://myapp.com` | -| `CUBEJS_CUBESTORE_HOST` | The hostname of the Cube Store deployment | A valid hostname | -| `CUBEJS_CUBESTORE_PORT` | The port of the Cube Store deployment | A valid port number | -| `CUBEJS_MAX_PARTITIONS_PER_CUBE` | The maximum number of partitions each pre-aggregation in a cube can use. Defaults to `10000` | A valid number | -| `CUBEJS_TOPIC_NAME` | The name of the Amazon SNS or Google Cloud Pub/Sub topic (defaults to `-process` if undefined, and finally `cubejs-process`) | A valid topic name | -| `CUBEJS_GH_API_TOKEN` | A Github Personal Token to avoid Github API rate limit at downloading cubestore | It can be a personal access token, an OAuth token, an installation access token or a JSON Web Token for GitHub App authentication | - -[ref-config-sched-refresh-timer]: /config#scheduled-refresh-timer -[ref-config-preagg-schema-name]: /config#pre-aggregations-schema -[gh-ioredis]: https://github.com/luin/ioredis -[gh-node-redis]: https://github.com/NodeRedis/node-redis -[link-tz-database]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones -[link-jwt-ref-iss]: https://tools.ietf.org/html/rfc7519#section-4.1.1 -[link-jwt-ref-sub]: https://tools.ietf.org/html/rfc7519#section-4.1.2 -[link-jwt-ref-aud]: https://tools.ietf.org/html/rfc7519#section-4.1.3 -[gh-jsonwebtoken-algs]: - https://github.com/auth0/node-jsonwebtoken#algorithms-supported -[link-jwk-ref]: https://tools.ietf.org/html/rfc7517#section-4 -[link-preaggregations-storage]: - /caching/using-pre-aggregations#pre-aggregations-storage - -## Database Connection - -To see a complete list of environment variables for your specific database, -please use the [database connection guide][link-connecting-to-db]. - -[link-connecting-to-db]: /config/databases - -## Export Bucket - -| Environment variable | Description | Possible Values | -| -------------------------------------- | -------------------------------------------------------------------------------------------- | ---------------------------------------------------------------- | -| `CUBEJS_DB_EXPORT_BUCKET` | The name of a bucket in cloud storage | `exports-20210505` | -| `CUBEJS_DB_EXPORT_BUCKET_TYPE` | The cloud provider where the bucket is hosted | `gcp`, `s3` | -| `CUBEJS_DB_EXPORT_BUCKET_AWS_KEY` | The AWS Access Key ID to use for the export bucket | A valid AWS Access Key ID | -| `CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET` | The AWS Secret Access Key to use for the export bucket | A valid AWS Secret Access Key | -| `CUBEJS_DB_EXPORT_BUCKET_AWS_REGION` | The AWS region of the export bucket | [A valid AWS region][link-aws-regions] | -| `CUBEJS_DB_EXPORT_BUCKET_REDSHIFT_ARN` | ARN of iam_role with permission to write to the provided `bucket` | A valid ARN to an IAM Role associated to the target Redshift DB | -| `CUBEJS_DB_EXPORT_GCS_CREDENTIALS` | A Base64 encoded JSON key file for connecting to Google Cloud | A valid Google Cloud JSON key file encoded as a Base64 string | -| `CUBEJS_DB_EXPORT_INTEGRATION` | The name of the integration used in the database. Only required when using Snowflake and GCS | A valid string matching the name of the integration in Snowflake | - -## SQL API - -| Environment variable | Description | Possible Values | -| ----------------------- | ---------------------------------------------------------------------- | ------------------- | -| `CUBEJS_SQL_USER` | Required username to access SQL API | A valid string | -| `CUBEJS_SQL_PASSWORD` | Required password to access SQL API | A valid string | -| `CUBEJS_SQL_PORT` | The port to listen to MySQL compatibility connections on. | A valid port number | -| `CUBEJS_PG_SQL_PORT` | The port to listen to PostgreSQL compatibility connections on. | A valid port number | -| `CUBEJS_SQL_SUPER_USER` | A name of specific user who will be allowed to change security context | A valid string | - -## Cube Store - -| Environment variable | Description | Possible Values | -| ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------- | -| `CUBESTORE_BIND_ADDR` | The address/port pair for Cube Store's MySQL-compatible interface. Defaults to `0.0.0.0:3306` | A valid address/port pair | -| `CUBESTORE_DATA_DIR` | A path on the local filesystem to store a local replica of the data. Must be unique on each node and different from `CUBESTORE_REMOTE_DIR`. Defaults to `.cubestore/data` | A valid path on the local filesystem with read/write access | -| `CUBESTORE_HTTP_BIND_ADDR` | The address/port pair for Cube Store's HTTP interface. Defaults to `0.0.0.0:3030` | A valid address/port pair | -| `CUBESTORE_HTTP_PORT` | The port for Cube Store to listen to HTTP connections on. Ignored when `CUBESTORE_HTTP_BIND_ADDR` is set. Defaults to `3030` | A valid port number | -| `CUBESTORE_JOB_RUNNERS` | The number of parallel tasks that process non-interactive jobs like data insertion, compaction etc. Defaults to `4` | A valid number | -| `CUBESTORE_LOG_LEVEL` | The logging level for Cube Store. Defaults to `error` | `error`, `warn`, `info`, `debug`, `trace` | -| `CUBESTORE_META_ADDR` | The address/port pair for the **router** node in the cluster | A valid address/port pair | -| `CUBESTORE_META_PORT` | The port for the **router** node to listen for connections on. Ignored when `CUBESTORE_META_ADDR` is set. | A valid port number | -| `CUBESTORE_NO_UPLOAD` | If `true`, prevents uploading serialized pre-aggregations to cloud storage | `true`, `false` | -| `CUBESTORE_PORT` | The port for Cube Store to listen to connections on. Ignored when `CUBESTORE_BIND_ADDR` is set. Defaults to `3306` | A valid port number | -| `CUBESTORE_QUERY_TIMEOUT` | The timeout for SQL queries in seconds. Defaults to `120` | A number in seconds | -| `CUBESTORE_REMOTE_DIR` | A path on the local filesystem to store metadata and datasets from all nodes as if it were remote storage. Not required if using GCS/S3. Not recommended for production usage | A valid path on the local filesystem with read/write access | -| `CUBESTORE_SELECT_WORKERS` | The number of Cube Store sub-processes that handle `SELECT` queries. Defaults to `4` | A valid number | -| `CUBESTORE_SERVER_NAME` | The full name and port number of the Cube Store server. Must be unique for each instance in cluster mode. Defaults to `localhost` | A valid address/port pair | -| `CUBESTORE_WAL_SPLIT_THRESHOLD` | The maximum number of rows to keep in a single chunk of data right after insertion. Defaults to `262144` | A valid number | -| `CUBESTORE_WORKER_PORT` | The port for Cube Store workers to listen to connections on. When set, the node will start as a **worker** in the cluster | A valid port number | -| `CUBESTORE_WORKERS` | A comma-separated list of address/port pairs; for example `worker-1:3123,localhost:3124,123.124.125.128:3123` | A comma-separated list of address/port pairs | - -### <--{"id" : "Cube Store"}--> Cloud Storage - -| Environment variable | Description | Possible Values | -| ------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | --- | -| `CUBESTORE_AWS_ACCESS_KEY_ID` | The Access Key ID for AWS. Required when using AWS S3 | [A valid AWS Access Key ID][link-aws-creds] | -| `CUBESTORE_AWS_SECRET_ACCESS_KEY` | The Secret Access Key for AWS. Required when using AWS S3 | [A valid AWS Secret Access Key][link-aws-creds] | -| `CUBESTORE_AWS_CREDS_REFRESH_EVERY_MINS` | The number of minutes after which Cube Store should refresh AWS credentials. Required when using an AWS instance role. Default is `180` | A valid number in minutes | -| `CUBESTORE_S3_BUCKET` | The name of a bucket in AWS S3. Required when using AWS S3 | A valid bucket name in the AWS account | -| `CUBESTORE_S3_REGION` | The region of a bucket in AWS S3. Required when using AWS S3 | [A valid AWS region][link-aws-regions] | -| `CUBESTORE_S3_SUB_PATH` | The path in a AWS S3 bucket to store pre-aggregations. Optional | - | -| `CUBESTORE_GCP_CREDENTIALS` | A Base64 encoded JSON key file for connecting to Google Cloud. Required when using Google Cloud Storage | [A valid Google BigQuery JSON key file encoded as a Base64 string][link-gcp-creds-json] | -| `CUBESTORE_GCP_KEY_FILE` | The path to a JSON key file for connecting to Google Cloud. Required when using Google Cloud Storage | [A valid Google Cloud JSON key file][link-gcp-creds-json] | -| `CUBESTORE_GCS_BUCKET` | The name of a bucket in GCS. Required when using GCS | A valid bucket name in the Google Cloud account | -| `CUBESTORE_GCS_SUB_PATH` | The path in a GCS bucket to store pre-aggregations. Optional | - | -| `CUBESTORE_MINIO_ACCESS_KEY_ID` | The Access Key ID for minIO. Required when using minIO | A valid minIO Access Key ID | -| `CUBESTORE_MINIO_SECRET_ACCESS_KEY` | The Secret Access Key for minIO. Required when using minIO | A valid minIO Secret Access Key | | -| `CUBESTORE_MINIO_BUCKET` | The name of the bucket that you want to use minIO. Required when using minIO | A valid bucket name in the AWS account | -| `CUBESTORE_MINIO_REGION` | The region of a bucket in S3 that you want to use minIO. Optional when using minIO | A valid S3 region name, an empty string if not present | -| `CUBESTORE_MINIO_SERVER_ENDPOINT` | The minIO server endpoint. Required when using minIO | A valid minIO endpoint e.g. `http://localhost:9000` | -| `CUBESTORE_MINIO_CREDS_REFRESH_EVERY_MINS` | The number of minutes after which Cube Store should refresh minIO credentials. Default is `180` | A valid number in minutes | - -[link-aws-creds]: - https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -[link-aws-regions]: - https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions -[link-aws-athena-workgroup]: - https://docs.aws.amazon.com/athena/latest/ug/workgroups-benefits.html -[link-elastic-api-keys]: - https://www.elastic.co/guide/en/kibana/master/api-keys.html#create-api-key -[link-gcp-creds-json]: - https://cloud.google.com/iam/docs/creating-managing-service-account-keys -[link-cubejs-databases]: /connecting-to-the-database -[link-nodejs-tls-options]: - https://nodejs.org/docs/latest/api/tls.html#tls_tls_createsecurecontext_options -[link-nodejs-tls-connect-opts]: - https://nodejs.org/docs/latest/api/tls.html#tls_tls_connect_options_callback -[link-nodejs-tls-ciphers]: - https://nodejs.org/docs/latest/api/tls.html#tls_modifying_the_default_tls_cipher_suite -[link-hive-cdh-versions]: - https://docs.cloudera.com/documentation/enterprise/6/release-notes/topics/rg_cdh_6_download.html -[link-hive-thrift-versions]: https://github.com/apache/thrift/releases -[link-hive-versions]: https://hive.apache.org/downloads.html -[link-snowflake-account]: - https://docs.getdbt.com/reference/warehouse-profiles/snowflake-profile#account -[link-snowflake-regions]: - https://docs.snowflake.com/en/user-guide/intro-regions.html -[link-snowflake-connection-options]: - https://docs.snowflake.com/en/user-guide/nodejs-driver-use.html#additional-connection-options diff --git a/docs/content/Configuration/Extending-Cubejs.mdx b/docs/content/Configuration/Extending-Cubejs.mdx deleted file mode 100644 index 9cf9836a8c3b5..0000000000000 --- a/docs/content/Configuration/Extending-Cubejs.mdx +++ /dev/null @@ -1,122 +0,0 @@ ---- -title: Extending Cube -permalink: /configuration/extending-cubejs -category: Configuration -menuOrder: 5 ---- - -For some advanced use-cases like instrumentation, or custom authentication, Cube -can be extended by installing third-party Node.js packages and using them in the -[`cube.js` configuration file][ref-config]. - -## Examples - -### Sending Express metrics to Prometheus - -```typescript -const prometheusMiddleware = require('express-prometheus-middleware'); - -module.exports = { - initApp: (app) => { - app.use( - prometheusMiddleware({ - metricsPath: '/metrics', - collectDefaultMetrics: true, - requestDurationBuckets: [0.1, 0.5, 1, 1.5], - requestLengthBuckets: [512, 1024, 5120, 10240, 51200, 102400], - responseLengthBuckets: [512, 1024, 5120, 10240, 51200, 102400], - }) - ); - }, -}; -``` - -### Sending Cube logs to Loggly - -The example below shows how to use the Node.js Loggly client to collect and send -logs from Cube. - -First, you'd need to install the third-party library with [NPM][link-npm]. In -our example, we're going to use [winston-loggly-bulk][link-loggly-client] to -collect and send logs to Loggly. You can install it with the following command: - -```bash -$ npm install --save winston-loggly-bulk -``` - - - -When installing custom Node.js packages for Cube running in Docker container, -make sure you mount the project _including_ the `node_modules` subfolder: - -```bash -$ docker run -d \ - -v ~/my-cubejs-project:/cube/conf - cubejs/cube -``` - -If you need to use third-party Node.js packages with native extensions, you'll -need to [build your own Docker image][ref-deployment-docker-extend]. - - - -Now we can require and use `winston-loggly-bulk` library inside `cube.js`: - -```javascript -const winston = require('winston'); -const { loggly } = require('winston-loggly-bulk'); - -winston.add( - new loggly({ - token: 'LOGGLY-TOKEN', - subdomain: 'your-subdomain', - tags: ['winston-nodejs'], - json: true, - }) -); - -module.exports = { - logger: (msg, params) => { - console.log(`${msg}: ${json.stringify(params)}`); - winston.log('info', msg, params); - }, -}; -``` - -## Using CubejsServer and CubejsServerCore - -You can directly use `@cubejs-backend/server-core` or `@cubejs-backend/server` -Node.js packages to run Cube. - - - -We do not recommend embedding Cube into existing Express application to share -the runtime. Cube should be scaled very differently vs. other parts of Express, -and embedding Cube **is not** the right approach in the long term, especially -for larger deployments. - - - -You can create an `index.js` file with the following content. - -```javascript -const CubejsServer = require('@cubejs-backend/server'); - -const server = new CubejsServer(); - -server.listen().then(({ version, port }) => { - console.log(`🚀 Cube server (${version}) is listening on ${port}`); -}); -``` - -Then start Cube as a regular Node.js application: - -```bash -node index.js -``` - -[link-npm]: https://www.npmjs.com/ -[link-loggly-client]: https://github.com/loggly/winston-loggly-bulk -[ref-config]: /config -[ref-deployment-docker-extend]: - /deployment/platforms/docker#extend-the-docker-image diff --git a/docs/content/Configuration/Multitenancy-Setup.mdx b/docs/content/Configuration/Multitenancy-Setup.mdx deleted file mode 100644 index 1cf17661ba6b5..0000000000000 --- a/docs/content/Configuration/Multitenancy-Setup.mdx +++ /dev/null @@ -1,367 +0,0 @@ ---- -title: Multitenancy -permalink: /multitenancy-setup -category: Configuration -menuOrder: 4 ---- - -Cube supports multitenancy out of the box, both on database and data schema -levels. Multiple drivers are also supported, meaning that you can have one -customer’s data in MongoDB and others in Postgres with one Cube instance. - -There are 6 [configuration options][ref-config-opts] you can leverage to make -your multitenancy setup. You can use all of them or just a couple, depending on -your specific case. The options are: - -- `contextToAppId` -- `contextToOrchestratorId` -- `driverFactory` -- `repositoryFactory` -- `preAggregationsSchema` -- `queryRewrite` - -All of the above options are functions, which you provide to Cube in the -[`cube.js` configuration file][ref-config]. The functions accept one argument - -a context object, which has a [`securityContext`][ref-config-security-ctx] -property where you can provide all the necessary data to identify a user e.g., -organization, app, etc. By default, the -[`securityContext`][ref-config-security-ctx] is defined by [Cube API -Token][ref-security]. - -There are several multitenancy setup scenarios that can be achieved by using -combinations of these configuration options. - -### <--{"id" : "Multitenancy"}--> Multitenancy vs Multiple Data Sources - -In cases where your Cube schema is spread across multiple different databases -you may consider using the [`dataSource` cube property][ref-cube-datasource] -instead of multitenancy. Multitenancy is designed for cases where you need to -serve different datasets for multiple users, or tenants which aren't related to -each other. - -On the other hand, multiple data sources can be used for scenarios where users -need to access the same data but from different databases. The multitenancy and -multiple data sources features aren't mutually exclusive and can be used -together. - - - -A `default` data source **must** exist and be configured. It is used to resolve -target query data source for now. This behavior **will** be changed in future -releases. - - - -A simple configuration with two data sources might look like: - -**cube.js:** - -```javascript -module.exports = { - driverFactory: ({ dataSource } = {}) => { - if (dataSource === 'db1') { - return { - type: 'postgres', - database: process.env.DB1_NAME, - host: process.env.DB1_HOST, - user: process.env.DB1_USER, - password: process.env.DB1_PASS, - port: process.env.DB1_PORT, - }; - } else { - return { - type: 'postgres', - database: process.env.DB2_NAME, - host: process.env.DB2_HOST, - user: process.env.DB2_USER, - password: process.env.DB2_PASS, - port: process.env.DB2_PORT, - }; - } - }, -}; -``` - -A more advanced example that uses multiple [data sources][ref-config-db] could -look like: - -**cube.js:** - -```javascript -module.exports = { - driverFactory: ({ dataSource } = {}) => { - if (dataSource === 'web') { - return { - type: 'athena', - database: dataSource, - ..., - }; - } else if (dataSource === 'googleAnalytics') { - return { - type: 'bigquery', - ..., - }; - } else if (dataSource === 'financials') { - return { - type: 'postgres', - database: 'financials', - host: 'financials-db.acme.com', - user: process.env.FINANCIALS_DB_USER, - password: process.env.FINANCIALS_DB_PASS, - }; - } else { - return { - type: 'postgres', - ..., - }; - } - }, -}; -``` - -### <--{"id" : "Multitenancy"}--> queryRewrite vs Multitenant Compile Context - -As a rule of thumb, the [`queryRewrite`][ref-config-query-rewrite] should be -used in scenarios when you want to define row-level security within the same -database for different users of such database. For example, to separate access -of two e-commerce administrators who work on different product categories within -the same e-commerce store, you could configure your project as follows: - -```javascript -// cube.js configuration file -module.exports = { - queryRewrite: (query, { securityContext }) => { - if (securityContext.categoryId) { - query.filters.push({ - member: 'Products.categoryId', - operator: 'equals', - values: [securityContext.categoryId], - }); - } - return query; - }, -}; - -// Products cube -cube(`Products`, { - sql: `select * from products`, -}); -``` - -On the other hand, multi-tenant [`COMPILE_CONTEXT`][ref-cube-security-ctx] -should be used when users need access to different databases. For example, if -you provide SaaS ecommerce hosting and each of your customers have a separate -database, then each e-commerce store should be modeled as a separate tenant. - -```javascript -const { - securityContext: { userId }, -} = COMPILE_CONTEXT; - -cube(`Products`, { - sql: `select * from ${userId}.products`, -}); -``` - -## Same DB Instance with per Tenant Row Level Security - -Per tenant row-level security can be achieved by configuring -[`queryRewrite`][ref-config-query-rewrite], which adds a tenant identifier -filter to the original query. It uses the -[`securityContext`][ref-config-security-ctx] to determine which tenant is -requesting data. This way, every tenant starts to see their own data. However, -resources such as query queue and pre-aggregations are shared between all -tenants. - -**cube.js:** - -```javascript -module.exports = { - queryRewrite: (query, { securityContext }) => { - const user = securityContext; - if (user.id) { - query.filters.push({ - member: 'Users.id', - operator: 'equals', - values: [user.id], - }); - } - return query; - }, -}; -``` - -## Multiple DB Instances with Same Schema - -Let's consider an example where we store data for different users in different -databases, but on the same Postgres host. The database name format is -`my_app__`, so `my_app_1_2` is a valid database name. - -To make it work with Cube, first we need to pass the `appId` and `userId` as -context to every query. We should first ensure our JWTs contain those properties -so we can access them through the [security context][ref-config-security-ctx]. - -```javascript -const jwt = require('jsonwebtoken'); -const CUBE_API_SECRET = 'secret'; - -const cubejsToken = jwt.sign({ appId: '1', userId: '2' }, CUBE_API_SECRET, { - expiresIn: '30d', -}); -``` - -Now, we can access them through the [`securityContext`][ref-config-security-ctx] -property inside the context object. Let's use -[`contextToAppId`][ref-config-ctx-to-appid] and -[`contextToOrchestratorId`][ref-config-ctx-to-orch-id] to create a dynamic Cube -App ID and Orchestrator ID for every combination of `appId` and `userId`, as -well as defining [`driverFactory`][ref-config-driverfactory] to dynamically -select the database, based on the `appId` and `userId`: - - - -The App ID (the result of [`contextToAppId`][ref-config-ctx-to-appid]) is used -as a caching key for various in-memory structures like schema compilation -results, connection pool. The Orchestrator ID (the result of -[`contextToOrchestratorId`][ref-config-ctx-to-orch-id]) is used as a caching key -for database connections, execution queues and pre-aggregation table caches. Not -declaring these properties will result in unexpected caching issues such as -schema or data of one tenant being used for another. - - - -**cube.js:** - -```javascript -module.exports = { - contextToAppId: ({ securityContext }) => - `CUBEJS_APP_${securityContext.appId}_${securityContext.userId}`, - contextToOrchestratorId: ({ securityContext }) => - `CUBEJS_APP_${securityContext.appId}_${securityContext.userId}`, - driverFactory: ({ securityContext }) => ({ - type: 'postgres', - database: `my_app_${securityContext.appId}_${securityContext.userId}`, - }), -}; -``` - -## Same DB Instance with per Tenant Pre-Aggregations - -To support per-tenant pre-aggregation of data within the same database instance, -you should configure the [`preAggregationsSchema`][ref-config-preagg-schema] -option in your `cube.js` configuration file. You should use also -[`securityContext`][ref-config-security-ctx] to determine which tenant is -requesting data. - -**cube.js:** - -```javascript -module.exports = { - contextToAppId: ({ securityContext }) => - `CUBEJS_APP_${securityContext.userId}`, - preAggregationsSchema: ({ securityContext }) => - `pre_aggregations_${securityContext.userId}`, -}; -``` - -## Multiple Schema and Drivers - -What if for application with ID 3, the data is stored not in Postgres, but in -MongoDB? - -We can instruct Cube to connect to MongoDB in that case, instead of Postgres. To -do this, we'll use the [`driverFactory`][ref-config-driverfactory] option to -dynamically set database type. We will also need to modify our -[`securityContext`][ref-config-security-ctx] to determine which tenant is -requesting data. Finally, we want to have separate data schemas for every -application. We can use the [`repositoryFactory`][ref-config-repofactory] option -to dynamically set a repository with schema files depending on the `appId`: - -**cube.js:** - -```javascript -module.exports = { - contextToAppId: ({ securityContext }) => - `CUBEJS_APP_${securityContext.appId}_${securityContext.userId}`, - contextToOrchestratorId: ({ securityContext }) => - `CUBEJS_APP_${securityContext.appId}_${securityContext.userId}`, - driverFactory: ({ securityContext }) => { - if (securityContext.appId === 3) { - return { - type: 'mongobi', - database: `my_app_${securityContext.appId}_${securityContext.userId}`, - port: 3307, - }; - } else { - return { - type: 'postgres', - database: `my_app_${securityContext.appId}_${securityContext.userId}`, - }; - } - }, - repositoryFactory: ({ securityContext }) => - new FileRepository(`schema/${securityContext.appId}`), -}; -``` - -## Serverless Deployment - -If you are deploying Cube to AWS Lambda with the [Serverless -template][ref-deployment-sls], you need to use `AWSHandlers` from the -[`@cubejs-backend/serverless-aws`][npm-cubejs-sls-aws] package. - -Add the following code to your `cube.js` file for the serverless multitenancy -setup. - -**cube.js:** - -```javascript -const AWSHandlers = require('@cubejs-backend/serverless-aws'); - -module.exports = new AWSHandlers({ - contextToAppId: ({ securityContext }) => - `CUBEJS_APP_${securityContext.appId}`, - driverFactory: ({ securityContext }) => ({ - type: 'postgres', - database: `my_app_${securityContext.appId}`, - }), -}); -``` - -## Scheduled Refreshes for Pre-Aggregations - -If you need scheduled refreshes for your pre-aggregations in a multi-tenant -deployment, ensure you have configured -[`scheduledRefreshContexts`][ref-config-refresh-ctx] correctly. You may also -need to configure [`scheduledRefreshTimeZones`][ref-config-refresh-tz]. - - - -Leaving [`scheduledRefreshContexts`][ref-config-refresh-ctx] unconfigured will -lead to issues where the security context will be `undefined`. This is because -there is no way for Cube to know how to generate a context without the required -input. - - - -## Connecting to Redis - -When configured for multitenancy, Cube uses a separate connection pool for each -configured tenant. This means that the `CUBEJS_REDIS_POOL_MIN` and -`CUBEJS_REDIS_POOL_MAX` environment variables specify the minimum and maximum -number of Redis connections **per-tenant**. - -[npm-cubejs-sls-aws]https://www.npmjs.com/package/@cubejs-backend/serverless-aws -[ref-config] /config [ref-config-opts] /config#options-reference [ref-config-db] -/config/databases [ref-config-driverfactory] /config#driver-factory -[ref-config-repofactory] /config#repository-factory [ref-config-preagg-schema] -/config#pre-aggregations-schema [ref-config-ctx-to-appid] -/config#context-to-app-id [ref-config-ctx-to-orch-id] -/config#context-to-orchestrator-id [ref-config-query-rewrite] -/config#query-rewrite [ref-config-refresh-ctx] -/config#scheduled-refresh-contexts [ref-config-refresh-tz] -/config#scheduled-refresh-time-zones [ref-config-security-ctx] -/config#security-context [ref-deployment-sls] /deployment/serverless/aws -[ref-security] /security [ref-cube-datasource] -/schema/reference/cube#data-source [ref-cube-security-ctx] -/schema/reference/cube#security-context diff --git a/docs/content/Configuration/Overview.mdx b/docs/content/Configuration/Overview.mdx index 21717a04e7c69..28e42c5e835df 100644 --- a/docs/content/Configuration/Overview.mdx +++ b/docs/content/Configuration/Overview.mdx @@ -6,27 +6,32 @@ category: Configuration menuOrder: 1 --- -Cube.js is designed to work with different configuration sources. There are two -ways you can set configuration options; via [a configuration file][link-config], -commonly known as the `cube.js` file, and [environment -variables][link-env-vars]. +Cube can be configured in two ways. The first is through [options][link-config] +in a configuration file, commonly known as the `cube.js` file, and secondly, +through [environment variables][link-env-vars]. Values specified in `cube.js` +will take precedence over environment variables. When using Docker, ensure that the `cube.js` configuration file and your -`schema/` folder are mounted to `/cube/conf` within the Docker container. +`model/` [folder][ref-folder-structure] are mounted to `/cube/conf` within the +Docker container. -## Configuration Precedence +## Cube Cloud -In Cube.js, values specified in `cube.js` take precedence over environment -variables. +You can set environment variables in Settings → Configuration: + + ## Development Mode -Cube.js can be run in an insecure, development mode by setting the -`CUBEJS_DEV_MODE` environment variable to `true`. Putting Cube.js in development +Cube can be run in an insecure, development mode by setting the +`CUBEJS_DEV_MODE` environment variable to `true`. Putting Cube in development mode does the following: - Disables authentication checks @@ -35,29 +40,10 @@ mode does the following: pre-aggregations][link-scheduled-refresh] - Allows another log level to be set (`trace`) - Enables [Developer Playground][link-dev-playground] on `http://localhost:4000` -- Uses `memory` instead of `redis` as the default cache/queue engine +- Uses `memory` instead of `cubestore` as the default cache/queue engine - Logs incorrect/invalid configuration for `externalRefresh` /`waitForRenew` instead of throwing errors -## Configuring CORS - -The Cube.js REST API supports Cross-Origin Resource Sharing (CORS) for all API -requests. By default, the middleware allows requests from any origin (`*`). To -change the allowed domain, you can do the following: - -```javascript -module.exports = { - http: { - cors: { - origin: 'https://myapp.com', - }, - }, -}; -``` - -Please consult the Configuration Reference [for more -options][link-config-cors-opts]. - ## Concurrency and pooling @@ -76,104 +62,8 @@ using the `CUBEJS_DB_MAX_POOL` environment variable; if changing this from the default, you must ensure that the new value is greater than the number of concurrent connections used by Cube's query queues and refresh scheduler. -## Migrating from Express to Docker - -Since [`v0.23`][link-v-023-release], Cube.js CLI uses the `docker` template -instead of `express` as a default for project creation, and it is the -recommended route for production. To migrate you should move all Cube.js -dependencies in `package.json` to `devDependencies` and leave dependencies that -you use to configure Cube.js in `dependencies`. - -For example, your existing `package.json` might look something like: - -```json -{ - "name": "cubejs-app", - "version": "0.0.1", - "private": true, - "scripts": { - "dev": "node index.js" - }, - "dependencies": { - "@cubejs-backend/postgres-driver": "^0.20.0", - "@cubejs-backend/server": "^0.20.0" - } -} -``` - -It should become: - -```json -{ - "name": "cubejs-app", - "version": "0.0.1", - "private": true, - "scripts": { - "dev": "./node_modules/.bin/cubejs-server server" - }, - "dependencies": {}, - "devDependencies": { - "@cubejs-backend/postgres-driver": "^0.23.6", - "@cubejs-backend/server": "^0.23.7" - } -} -``` - -You should also rename your `index.js` file to `cube.js` and replace the -`CubejsServer.create()` call with export of configuration using -`module.exports`. - -For an `index.js` like the following: - -```javascript -const CubejsServer = require('@cubejs-backend/server'); - -const server = new CubejsServer({ - logger: (msg, params) => { - console.log(`${msg}: ${JSON.stringify(params)}`); - }, -}); - -server - .listen() - .then(({ version, port }) => { - console.log(`🚀 Cube.js server (${version}) is listening on ${port}`); - }) - .catch((e) => { - console.error('Fatal error during server start: '); - console.error(e.stack || e); - }); -``` - -It should be renamed to `cube.js` and its' contents should look like: - -```javascript -module.exports = { - logger: (msg, params) => { - console.log(`${msg}: ${JSON.stringify(params)}`); - }, -}; -``` - -Finally, add a `docker-compose.yml` file alongside the `cube.js` configuration -file: - -```yaml -version: '2.2' - -services: - cube: - image: cubejs/cube:latest - ports: - - 4000:4000 - env_file: .env - volumes: - - .:/cube/conf -``` - +[ref-folder-structure]: /data-modeling/syntax#folder-structure [link-config]: /config -[link-config-cors-opts]: /config#http [link-dev-playground]: /dev-tools/dev-playground [link-env-vars]: /reference/environment-variables [link-scheduled-refresh]: /schema/reference/pre-aggregations#scheduled-refresh -[link-v-023-release]: https://github.com/cube-js/cube.js/releases/tag/v0.23.0 diff --git a/docs/content/Configuration/Reference/@cubejs-backend-server-core.md b/docs/content/Configuration/Reference/@cubejs-backend-server-core.md deleted file mode 100644 index 58161a76740a2..0000000000000 --- a/docs/content/Configuration/Reference/@cubejs-backend-server-core.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: cubejs-backend-server-core -permalink: /config/reference/@cubejs-backend-server-core -category: Configuration -subCategory: Reference -menuOrder: 6 -redirect_from: - - /@cubejs-backend-server-core ---- - -This package provides wiring of all essential Cube.js components and is used by -[@cubejs-backend/server][ref-config-ref-backend-server]. - -## API Reference - -### <--{"id" : "API Reference"}--> CubejsServerCore.create(options) - -`CubejsServerCore.create` is an entry point for a Cube.js server application. It -creates an instance of `CubejsServerCore`, which could be embedded for example -into Express application. - -```javascript -const { CubejsServerCore } = require('@cubejs-backend/server-core'); -const express = require('express'); -const path = require('path'); - -const expressApp = express(); - -const dbType = 'mysql'; -const options = { - dbType, - logger: (msg, params) => { - console.log(`${msg}: ${JSON.stringify(params)}`); - }, - schemaPath: path.join('assets', 'schema'), -}; - -const core = CubejsServerCore.create(options); -core.initApp(expressApp); -``` - -`CubejsServerCore.create` method accepts an object with the [Cube.js -configuration options][ref-config]. - -### <--{"id" : "API Reference"}--> CubejsServerCore.version() - -`CubejsServerCore.version` is a method that returns the semantic package version -of `@cubejs-backend/server`. - -[ref-config-ref-backend-server]: /config/reference/@cubejs-backend-server -[ref-config]: /config diff --git a/docs/content/Configuration/Reference/@cubejs-backend-server.md b/docs/content/Configuration/Reference/@cubejs-backend-server.md deleted file mode 100644 index 8bdbfb59c8ee9..0000000000000 --- a/docs/content/Configuration/Reference/@cubejs-backend-server.md +++ /dev/null @@ -1,134 +0,0 @@ ---- -title: cubejs-backend-server -permalink: /config/reference/@cubejs-backend-server -category: Configuration -subCategory: Reference -menuOrder: 7 -redirect_from: - - /@cubejs-backend-server ---- - -`@cubejs-backend/server` is a web server for the -[@cubejs-backend/server-core][ref-config-ref-backend-server-core]. There are -also multiple options to run Cube.js Backend Server [in -production][ref-deployment]. - -## API Reference - -### <--{"id" : "API Reference"}--> CubejsServer.create(options) - -Creates an instance of `CubejsServer`. - -You can set server port using `PORT` environment variable. Default port is -`4000`. - -#### Example - -```javascript -const CubejsServer = require('@cubejs-backend/server'); - -const server = new CubejsServer(); - -server.listen().then(({ version, port }) => { - console.log(`🚀 Cube.js server (${version}) is listening on ${port}`); -}); -``` - -#### Options Reference - -The options for `CubejsServer` include the `CubejsServerCore` -[options][ref-config-ref-backend-server-core-opts] plus the following additional -ones specific to `CubejsServer`: - -```typescript -type CubejsServerOptions = { - webSockets?: boolean; - initApp?(app: express.Application): void | Promise; -}; -``` - -##### webSockets - -Boolean to enable or disable [web sockets][ref-frontend-real-time-ws] on the -backend. Can also be enabled using the `CUBEJS_WEB_SOCKETS` environment -variable. - -##### initApp - -A function to setup the instance of Express. It accepts the following argument: - -- `app`: the instance of Express - -This method is invoked prior to any routes having been added. Since routes can't -be overridden, this allows customization / overriding of the routes and other -aspects of the Express application early in its lifecycle. - -An example usage is customizing the base route `/` in production mode to return -a 404: - -`initApp.ts` - -```typescript -import type { Application, Request, Response } from 'express'; - -export function initApp(app: Application) { - app.get('/', (req: Request, res: Response) => { - res.sendStatus(404); - }); -} -``` - -`index.ts` - -```typescript -import { initApp } from './initApp'; - -const options = {}; - -// ... - -if (process.env.NODE_ENV === 'production') { - options.initApp = initApp; -} - -const server = new CubejsServer(options); -``` - -### <--{"id" : "API Reference"}--> CubejsServer.version() - -`CubejsServer.version` is a method that returns the semantic package version of -`@cubejs-backend/server`. - -```javascript -const CubejsServer = require('@cubejs-backend/server'); - -console.log(CubejsServer.version()); -``` - -### <--{"id" : "API Reference"}--> this.listen(options) - -Instantiates the Express.js App to listen to the specified `PORT`. Returns a -promise that resolves with the following members: - -- `port {number}` The port at which CubejsServer is listening for insecure - connections for redirection to HTTPS, as specified by the environment variable - `PORT`. Defaults to 4000. -- `app {Express.Application}` The express App powering CubejsServer -- `server {http.Server}` The `http` Server instance. If TLS is enabled, returns - a `https.Server` instance instead. -- `version {string}` The semantic package version of `@cubejs-backend/server` - -### <--{"id" : "API Reference"}--> this.testConnections() - -Tests all existing open connections in the application. - -### <--{"id" : "API Reference"}--> this.close() - -Shuts down the server and closes any open db connections. - -[ref-config-ref-backend-server-core]: - /config/reference/@cubejs-backend-server-core -[ref-config-ref-backend-server-core-opts]: - /config/reference/@cubejs-backend-server-core#options-reference -[ref-deployment]: /deployment/overview -[ref-frontend-real-time-ws]: /real-time-data-fetch#web-sockets diff --git a/docs/content/Configuration/VPC/Connecting-with-a-VPC-AWS.mdx b/docs/content/Configuration/VPC/Connecting-with-a-VPC-AWS.mdx new file mode 100644 index 0000000000000..96dcf0c0e616d --- /dev/null +++ b/docs/content/Configuration/VPC/Connecting-with-a-VPC-AWS.mdx @@ -0,0 +1,126 @@ +--- +title: Connecting with a VPC on AWS +permalink: /cloud/configuration/connecting-with-a-vpc/aws +--- + +To connect with a VPC on AWS, you need to collect the necessary information and +hand it over to your Cube Cloud representative. Next, you'll have to accept a +VPC peering request sent by Cube Cloud. Finally, you'll need to configure +security groups and route tables to ensure Cube Cloud can connect to your data +source. + +## Prerequisites + +To allow Cube Cloud to connect to a [VPC on AWS][aws-docs-vpc], the following +information is required: + +- **AWS Account ID:** The AWS account ID of the VPC owner. This can be found in + the top-right corner of [the AWS Console][aws-console]. +- **AWS Region:** [The AWS region][aws-docs-regions] that the VPC resides in. + Ensure that the region is available in [Supported Regions](#supported-regions) + to see if Cube Cloud VPC connectivity is available in your region. +- **AWS VPC ID:** The ID of the VPC that Cube Cloud will connect to, for + example, `vpc-0099aazz` +- **AWS VPC CIDR:** The [CIDR block][wiki-cidr-block] of the VPC that Cube Cloud + will connect to, for example, `10.0.0.0/16` + +After receiving the above information, a Customer Success Manager will provide +you with the **AWS account ID**, **region**, **VPC ID** and the [**CIDR +block**][wiki-cidr-block] used by Cube Cloud to connect to your VPC. + +## Setup + +### <--{"id" : "Setup"}--> VPC Peering Request + +After receiving the information above, Cube Cloud will send a [VPC peering +request][aws-docs-vpc-peering] that must be accepted. This can be done either +through the [AWS Web Console][aws-console] or through an infrastructure-as-code +tool. + +To [accept the VPC peering request][aws-docs-vpc-peering-accept] through the AWS +Web Console, follow the instructions below: + +1. Open the [Amazon VPC console](https://console.aws.amazon.com/vpc/). + + + + Ensure you have the necessary permissions to accept a VPC peering request. If + you are unsure, please contact your AWS administrator. + + + +2. Use the Region selector to choose the Region of the accepter VPC. + +3. In the navigation pane, choose Peering connections. + +4. Select the pending VPC peering connection (the status should be + `pending-acceptance`), then choose Actions, followed by  + ​Accept request. + + + + Ensure the peering request is from Cube Cloud by checking that the **AWS account + ID**, **region** and **VPC IDs** match those provided by your CSM. + + + +5. When prompted for confirmation, choose Accept request. + +6. Choose Modify my route tables now to add a route to the VPC route + table so that you can send and receive traffic across the peering connection. + + + +For more information about peering connection lifecycle statuses, check out the +[VPC peering connection lifecycle on AWS][aws-docs-vpc-peering-lifecycle]. + + + +### <--{"id" : "Setup"}--> Updating security groups + +The initial VPC setup will not allow traffic from Cube Cloud; this is because +[the security group][aws-docs-vpc-security-group] for the database will need to +allow access from the Cube Cloud CIDR block. + +This can be achieved by adding a new security group rule: + +| Protocol | Port Range | Source/Destination | +| -------- | ---------- | --------------------------------------------- | +| TCP | 3306 | The Cube Cloud CIDR block for the AWS region. | + +### <--{"id" : "Setup"}--> Update route tables + +The final step is to update route tables in your VPC to allow traffic from Cube +Cloud to reach your database. The Cube Cloud CIDR block must be added to the +route tables of all subnets that connect to the database. To do this, follow the +instructions on [the AWS documentation][aws-docs-vpc-peering-routing]. + +## Troubleshooting + +Database connection issues with misconfigured VPCs often manifest as connection +timeouts. If you are experiencing connection issues, please check the following: + +- Verify that + [all security groups allow traffic](#setup-updating-security-groups) from the + Cube Cloud provided CIDR block. +- Verify that + [a route exists to the Cube Cloud provided CIDR block](#setup-update-route-tables) + from the subnets that connect to the database. + +[aws-console]: https://console.aws.amazon.com/ +[aws-docs-regions]: + https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions +[aws-docs-vpc]: + https://docs.aws.amazon.com/vpc/latest/userguide/what-is-amazon-vpc.html +[aws-docs-vpc-peering-accept]: + https://docs.aws.amazon.com/vpc/latest/peering/create-vpc-peering-connection.html#different-account-different-region +[aws-docs-vpc-peering-lifecycle]: + https://docs.aws.amazon.com/vpc/latest/peering/vpc-peering-basics.html#vpc-peering-lifecycle +[aws-docs-vpc-peering-routing]: + https://docs.aws.amazon.com/vpc/latest/peering/vpc-peering-routing.html +[aws-docs-vpc-peering]: + https://docs.aws.amazon.com/vpc/latest/peering/what-is-vpc-peering.html +[aws-docs-vpc-security-group]: + https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html#SecurityGroupRules +[wiki-cidr-block]: + https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_blocks diff --git a/docs/content/Configuration/VPC/Connecting-with-a-VPC-Azure.mdx b/docs/content/Configuration/VPC/Connecting-with-a-VPC-Azure.mdx new file mode 100644 index 0000000000000..25ea3bfec589a --- /dev/null +++ b/docs/content/Configuration/VPC/Connecting-with-a-VPC-Azure.mdx @@ -0,0 +1,86 @@ +--- +title: Connecting with a VPC on Azure +permalink: /cloud/configuration/connecting-with-a-vpc/azure +--- + +## Prerequisites + +To allow Cube Cloud to connect to a Virtual Network on Azure, the following +information is required: + +- **Virtual Network Name:** This can be found in the Virtual Networks section of + the [Azure Portal][azure-console]. +- **Tenant ID:** This can be found under `Azure Active Directory` > `Properties` > + `Tenant ID` in the [Azure Portal][azure-console]. + +## Setup + +For cross-tenant peering in Azure, you are supposed to assign the peering role to +the service principal of the peering party. + +Using the steps outlined below, you would register Cube Cloud tenant at your organization +and grant peering access to Cube Cloud service principal. + +### Add Cube tenant to your organization + +First you should add the Cube Cloud tenant to your organization. To do this, open +the [Azure Portal][azure-console] and go to `Azure Active Directory` > `External +Identities` > `Cross-tenant access settings` > `Organizational Settings` > `Add +Organization`. + +For Tenant ID, enter `197e5263-87f4-4ce1-96c4-351b0c0c714a`. + +Make sure that `B2B Collaboration` > `Inbound Access` > `Applications` is set to `Allows access`. + +### Register Cube Cloud service principal at your organization + +To register the Cube Cloud service principal for your organization, follow these steps: + +1. Log in with an account that has permissions to register Enterprise applications. +2. Open a browser tab and go to the following URL, replacing "TenantID" with your tenant ID: +https://login.microsoftonline.com/TenantID/oauth2/authorize?client_id=0c5d0d4b-6cee-402e-9a08-e5b79f199481&response_type=code&redirect_uri=https%3A%2F%2Fwww.microsoft.com%2F +3. The Cube Cloud service principal has specific credentials. Check that the following details match exactly what you see on the dialog box that pops up: +- Client ID: `0c5d0d4b-6cee-402e-9a08-e5b79f199481` +- Name: `cube-dedicated-infra-sp` + +Once you have confirmed that all the information is correct, select "Consent on behalf of your organization" and click "Accept". + +### Grant peering permissions to Cube Cloud service principal on your `Virtual Network` + +As `peering role` you can use built-in `Network Contributor` or create custom role (e.g. `cube-peering-role`) with the following permissions: + +- `Microsoft.Network/virtualNetworks/virtualNetworkPeerings/write` +- `Microsoft.Network/virtualNetworks/peer/action` +- `Microsoft.ClassicNetwork/virtualNetworks/peer/action` +- `Microsoft.Network/virtualNetworks/virtualNetworkPeerings/read` +- `Microsoft.Network/virtualNetworks/virtualNetworkPeerings/delete` + +On the [Azure Portal][azure-console], go to `Virtual networks` > *Virtual +Network Name* > `Access Control (IAM)` > `Add` > `Add role assignment` and fill in the following details: +- Role = `Network Contributor` or `cube-peering-role` +- Members: `cube-dedicated-infra-sp` + +### Firewall + +Make sure that your firewall rules allow inbound and outbound traffic to IP/port your database is listening at. + +## Information required by Cube Cloud support + +When you are reaching out Cube Cloud support please provide following +information: + +- **Virtual Network ID:** You can find it at `Virtual Networks` > *Virtual + Network Name* > `Overview` > `JSON view` > `Resource ID` on [Azure + Portal][azure-console]. +- **Virtual Network Address Spaces:** You can find it at `Virtual Networks` > + *Virtual Network Name* > `Overview` > `JSON view` > `properties` > + `addressSpace` on [Azure Portal][azure-console]. +- **Tenant ID:** You can find it in `Azure Active Directory` > `Properties` > + `Tenant ID` section of [Azure Portal][azure-console]. + +## Supported Regions + +We support all general-purpose regions. Cube Store is currently located only in `US Central` +so pre-aggregations performance might depend on geographical proximity to it. + +[azure-console]: https://portal.azure.com diff --git a/docs/content/Cube-Cloud/Configuration/VPC/Connecting-with-a-VPC-GCP.mdx b/docs/content/Configuration/VPC/Connecting-with-a-VPC-GCP.mdx similarity index 100% rename from docs/content/Cube-Cloud/Configuration/VPC/Connecting-with-a-VPC-GCP.mdx rename to docs/content/Configuration/VPC/Connecting-with-a-VPC-GCP.mdx diff --git a/docs/content/Configuration/VPC/Connecting-with-a-VPC.mdx b/docs/content/Configuration/VPC/Connecting-with-a-VPC.mdx new file mode 100644 index 0000000000000..5bf2f88d220ee --- /dev/null +++ b/docs/content/Configuration/VPC/Connecting-with-a-VPC.mdx @@ -0,0 +1,39 @@ +--- +title: Connecting with a VPC +permalink: /cloud/configuration/connecting-with-a-vpc +category: Configuration +menuOrder: 4 +--- + +For improved stability and security, Cube Cloud supports connecting to one or +more VPCs (virtual private clouds) in your Azure, AWS, or GCP accounts. + + + +VPC connectivity is available in Cube Cloud on +[Enterprise](https://cube.dev/pricing) tier. +[Contact us](https://cube.dev/contact) for details. + + + +VPC connection improves stability through dedicated infrastructure for a +deployment and improves security by preventing your database traffic from being +routed through the public internet. + + + + + + diff --git a/docs/content/Cube-Cloud/Cloud-Runtime.mdx b/docs/content/Cube-Cloud/Cloud-Runtime.mdx deleted file mode 100644 index 600d9baa5d7b5..0000000000000 --- a/docs/content/Cube-Cloud/Cloud-Runtime.mdx +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: Cloud Runtime -permalink: /cloud/cloud-runtime -category: Cloud Runtime -menuOrder: 2 ---- - -Cloud Runtime extends Cube Core with several components that -substantially improve reliability, performance, and security: - -* __Performance:__ Cube Store in-memory caching, Cube Store autoscaling, autoscaling for multi-tenancy. - -* __Security:__ Cube Store encryption at rest, SQL API connection via SSL, secured access to Cube Playground, security audits and updates to Docker images. - -* __Reliability:__ Cube Store replication for high availability, distributed file storage failovers and consistency checking. - -* __Integrations:__ Optimized Databricks driver, optimized Elasticsearch driver, support for Azure blob storage in Cube Store, and integration with monitoring solutions. - - -Cloud Runtime powers Cube Cloud. Cube Cloud is available as [managed -service](https://cubecloud.dev/auth/signup) and -through "bring your own cloud" model. [Please contact -us](https://cube.dev/contact) if you'd like to install -Cube Cloud within your AWS, GCP, or Azure account. diff --git a/docs/content/Cube-Cloud/Configuration/Access Control.mdx b/docs/content/Cube-Cloud/Configuration/Access Control.mdx deleted file mode 100644 index 4ad4a11752eb3..0000000000000 --- a/docs/content/Cube-Cloud/Configuration/Access Control.mdx +++ /dev/null @@ -1,80 +0,0 @@ ---- -title: Access Control -permalink: /cloud/access-control/ -category: Configuration -menuOrder: 6 ---- - - - -Role-based access control is only available on the Enterprise plan. - - - -Cube Cloud allows account administrators to define roles with specific -permissions for resources and then apply those roles to users in their account. - -## List all roles - -To see a list of roles in your account, first go to the Team settings page by -clicking on your avatar in the top right corner, then clicking on the "Team" -button. - -On the Team settings page, click the "Roles" tab to see all the roles in your -account: - -
- Cube Cloud Team Roles tab -
- -## Create a role - -To create a new role, click the "Add Role" button. Enter a name and optional -description for the role, then click "Add Policy" and select either "Deployment" -or "Global" for this policy's scope. - -Deployment policies apply to deployment-level functionality, such as the -Playground and Schema Editor. Global policies apply to account-level -functionality, such as Alerts and Billing. Once the policy scope has been -selected, you can restrict which actions this role can perform by selecting -"Specific" and using the dropdown to select specific actions. - -
- Cube Cloud Team Roles tab -
- -When you are finished, click "Create Role" to create the role. - -## Assigning roles to users - -Roles are assigned to new users when inviting them: - -
- Cube Cloud Team Roles tab -
- -Existing users' roles can be modified from the "Members" tab on the Team page: - -
- Cube Cloud Team Roles tab -
diff --git a/docs/content/Cube-Cloud/Configuration/Connecting-to-Databases.mdx b/docs/content/Cube-Cloud/Configuration/Connecting-to-Databases.mdx deleted file mode 100644 index 05d615fa7d0b7..0000000000000 --- a/docs/content/Cube-Cloud/Configuration/Connecting-to-Databases.mdx +++ /dev/null @@ -1,194 +0,0 @@ ---- -title: Connecting to the Database -permalink: /cloud/configuration/connecting-to-the-database -category: Configuration -menuOrder: 1 -redirect_from: - - /cloud/configuration/connecting-to-databases ---- - -You can connect all Cube.js supported databases to your Cube Cloud deployment. - -
- Cube Cloud Supported Databases Screen -
- -Below you can find guides on how to use Cube Cloud with specific database -vendors. - -- [AWS Athena](#aws-athena) -- [AWS Redshift](#aws-redshift) -- [BigQuery](#big-query) -- [Snowflake](#snowflake) - -## Guides - -### <--{"id" : "Guides"}--> AWS Athena - -The following fields are required when creating an AWS Athena connection: - -| Field | Description | Examples | -| ------------------------- | ----------------------------------------------------------------- | ------------------------------------------ | -| **AWS Access Key ID** | The AWS Access Key ID to use for database connections | `AKIAXXXXXXXXXXXXXXXX` | -| **AWS Secret Access Key** | The AWS Secret Access Key to use for database connections | `asd+/Ead123456asc23ASD2Acsf23/1A3fAc56af` | -| **AWS Region** | The AWS region of the Cube.js deployment | `us-east-1` | -| **S3 Output Location** | The S3 path to store query results made by the Cube.js deployment | `s3://my-output-bucket/outputs/` | - -
- Cube Cloud AWS Athena Configuration Screen -
- -### <--{"id" : "Guides"}--> AWS Redshift - - - -Ensure that the database can be accessed over the public Internet. If you'd -prefer to keep the database on a private network, -[contact us for VPC peering solutions](#connecting-to-a-database-not-exposed-over-the-internet). - - - -The following fields are required when creating an AWS Redshift connection: - -| Field | Description | Examples | -| ------------ | -------------------------------------------- | -------------------------------------------------------------- | -| **Hostname** | The host URL for the AWS Redshift cluster | `examplecluster.abc123xyz789.us-west-2.redshift.amazonaws.com` | -| **Port** | The port for the AWS Redshift cluster | `5439` | -| **Database** | The name of the database to connect to | `public` | -| **Username** | The username used to connect to the database | `redshift` | -| **Password** | The password used to connect to the database | `MY_SUPER_SECRET_PASSWORD` | - -
- Cube Cloud AWS Redshift Configuration Screen -
- -### <--{"id" : "Guides"}--> BigQuery - -The following fields are required when creating a BigQuery connection: - -| Field | Description | Examples | -| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------- | -| **Service Account JSON file** | A JSON key file for connecting to Google BigQuery | A valid Google BigQuery JSON key file | -| **Project ID** | The Google BigQuery project ID to connect to | `my-bigquery-project` | -| **Encoded Key File** | A Base64 encoded JSON key file for connecting to Google BigQuery. Required if Service Account JSON file is not provided | A valid Google BigQuery JSON key file encoded as a Base64 string | - -
- Cube Cloud BigQuery Configuration Screen -
- -### <--{"id" : "Guides"}--> Snowflake - -The following fields are required when creating a Snowflake connection: - -| Field | Description | Examples | -| ------------- | --------------------------------------------------------------- | -------------------------- | -| **Username** | The username used to connect to the database | `cube` | -| **Password** | The password used to connect to the database | `MY_SUPER_SECRET_PASSWORD` | -| **Database** | The name of the database to connect to | `MY_SNOWFLAKE_DB` | -| **Account** | The Snowflake account ID to use when connecting to the database | `qna90001` | -| **Region** | The Snowflake region to use when connecting to the database | `us-east-1` | -| **Warehouse** | The Snowflake warehouse to use when connecting to the database | `MY_WAREHOUSE` | -| **Role** | The Snowflake role to use when connecting to the database | `PUBLIC` | - -
- Cube Cloud Snowflake Configuration Screen -
- -## Connecting to multiple databases - -If you are connecting to multiple databases, set up the first database -connection during the deployment creation wizard. First, make sure you have the -correct configuration in your `cube.js` file according to your -[multitenancy setup](/multitenancy-setup). Next, configure the corresponding -environment variables on the **Settings - Env Vars page**. - -## Connecting via SSL - -When setting up a new deployment, simply select the SSL checkbox when entering -database credentials: - -
- Cube Cloud Database Connection Screen -
- -### <--{"id" : "Connecting via SSL"}--> Custom SSL certificates - -To use custom SSL certificates between Cube Cloud and your database server, go -to the **Env vars** tab in **Settings**: - - - -Depending on how SSL is configured on your database server, you may need to -specify additional environment variables, please check the [Environment -Variables reference][ref-config-env-vars] for more information. - - - -
- Cube Cloud Database Connection Screen -
- -Add the following environment variables: - -| Environment Variable | Description | Example | -| -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------- | -| `CUBEJS_DB_SSL` | If `true`, enables SSL encryption for database connections from Cube.js | `true`, `false` | -| `CUBEJS_DB_SSL_CA` | The contents of a CA bundle in PEM format, or a path to one. For more information, check the `options.ca` property for TLS Secure Contexts [in the Node.js documentation][link-nodejs-tls-options] | A valid CA bundle or a path to one | -| `CUBEJS_DB_SSL_CERT` | The contents of an SSL certificate in PEM format, or a path to one. For more information, check the `options.cert` property for TLS Secure Contexts [in the Node.js documentation][link-nodejs-tls-options] | A valid SSL certificate or a path to one | -| `CUBEJS_DB_SSL_KEY` | The contents of a private key in PEM format, or a path to one. For more information, check the `options.key` property for TLS Secure Contexts [in the Node.js documentation][link-nodejs-tls-options] | A valid SSL private key or a path to one | - -## Allowing connections from Cube Cloud IP - -In some cases you'd need to allow connections from your Cube Cloud deployment IP -address to your database. You can copy the IP address from either the Database -Setup step in deployment creation, or from the Env Vars tab in your deployment -Settings page. - -## Connecting to a database not exposed over the internet - -[Contact us](mailto:support@cube.dev) for VPC peering and on-premise solutions. - -[link-aws-regions]: - https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions -[link-nodejs-tls-options]: - https://nodejs.org/docs/latest/api/tls.html#tls_tls_createsecurecontext_options -[link-snowflake-account]: - https://docs.getdbt.com/reference/warehouse-profiles/snowflake-profile#account -[link-snowflake-regions]: - https://docs.snowflake.com/en/user-guide/intro-regions.html -[ref-config-env-vars]: /reference/environment-variables#database-connection diff --git a/docs/content/Cube-Cloud/Configuration/Custom-Domains.mdx b/docs/content/Cube-Cloud/Configuration/Custom-Domains.mdx deleted file mode 100644 index 8af335b89f02c..0000000000000 --- a/docs/content/Cube-Cloud/Configuration/Custom-Domains.mdx +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Custom Domains -permalink: /cloud/configuration/custom-domains -category: Configuration -menuOrder: 5 ---- - - - -DNS changes can sometimes take up to 15 minutes to propagate, please wait at -least 15 minutes and/or try using another DNS provider to verify the `CNAME` -record correctly before raising a new Support ticket. - - - -To set up a custom domain, go to your deployment's settings page. Under the -**Custom Domain** section, type in your custom domain and click **Add**: - -
- -
- -After doing this, copy the provided domain and add the following `CNAME` records -to your DNS provider: - -| Name | Value | -| ------------- | ------------------------------------------- | -| `YOUR_DOMAIN` | The copied value from Cube Cloud's settings | - -Using the example subdomain from the screenshot, a `CNAME` record for `acme.dev` -would look like: - -| Name | Value | -| ---------- | ----------------------------------------------------- | -| `insights` | `colossal-crowville.gcp-us-central1.cubecloudapp.dev` | diff --git a/docs/content/Cube-Cloud/Configuration/Deployment-Env-Vars.md b/docs/content/Cube-Cloud/Configuration/Deployment-Env-Vars.md deleted file mode 100644 index 996c5343925a7..0000000000000 --- a/docs/content/Cube-Cloud/Configuration/Deployment-Env-Vars.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: Environment Variables -permalink: /cloud/configuration/env-vars -category: Configuration -menuOrder: 3 ---- - -You can set [Cube.js environment variables](/reference/environment-variables) -and other custom environment variables on the **Settings - Env Vars** page. - -
- Cube Cloud Environment Variables Screen -
diff --git a/docs/content/Cube-Cloud/Configuration/Deployment-Types.md b/docs/content/Cube-Cloud/Configuration/Deployment-Types.md deleted file mode 100644 index 2b45b02892e32..0000000000000 --- a/docs/content/Cube-Cloud/Configuration/Deployment-Types.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: Deployment Types -permalink: /cloud/configuration/deployment-types -category: Configuration -menuOrder: 2 ---- - -Cube Cloud deployment type determines the features, scalability, and -availability of your deployment. - -## Single Instance Deployments - -Single Instance deployments are designed for development use-cases only. -If not used for over an hour, it'll take additional time to process the first request. -Refresh Worker is active only when a Single Instance serves the workload. - -## Cluster Deployments - -Cluster deployments are designed for production workload. Cluster deployment -includes the following components: - -- Cube API instances -- Cube Refresh Workers -- Redis for Cache and Queue diff --git a/docs/content/Cube-Cloud/Configuration/VPC/Connecting-with-a-VPC-AWS.mdx b/docs/content/Cube-Cloud/Configuration/VPC/Connecting-with-a-VPC-AWS.mdx deleted file mode 100644 index 0d78ba81d2b5a..0000000000000 --- a/docs/content/Cube-Cloud/Configuration/VPC/Connecting-with-a-VPC-AWS.mdx +++ /dev/null @@ -1,71 +0,0 @@ ---- -title: Connecting with a VPC on AWS -permalink: /cloud/configuration/connecting-with-a-vpc/aws ---- - -## Prerequisites - -To allow Cube Cloud to connect to a VPC on AWS, the following information is -required: - -- **AWS Account ID:** The AWS account ID of the VPC owner. This can be found in - the top-right corner of [the AWS Console][aws-console]. -- **AWS Region:** [The AWS region][aws-docs-regions] that the VPC resides in. - Ensure that the region is available in [Supported Regions](#supported-regions) - to see if Cube Cloud VPC connectivity is available in your region. -- **AWS VPC ID:** The ID of the VPC that Cube Cloud will connect to, for example - `vpc-0099aazz` -- **AWS VPC CIDR:** The CIDR block of the VPC that Cube Cloud will connect to, - for example `10.0.0.0/16` - -## Setup - -### VPC Peering Request - -After receiving the information above, Cube Cloud will send a [VPC peering -request][aws-docs-vpc-peering] that must be accepted. This can be done either -through the [AWS Web Console][aws-console] or through an infrastructure-as-code -tool. - -To accept the VPC peering request through the AWS Web Console, follow [the -instructions here][aws-docs-vpc-peering-accept] with the following amendments: - -- On Step 4, verify the peering request is from Cube Cloud by checking that the - AWS account ID, region and VPC IDs match those provided by Support. -- On Step 5, ensure **Modify my route tables now** is selected so that the - necessary routes are created. -- After Step 5, the security group for any databases within the VPC - [may require updating](#updating-security-groups) to allow traffic from Cube - Cloud. - -### Updating security groups - -Often the initial VPC setup will not allow traffic from Cube Cloud; this is -usually because [the security group][aws-docs-vpc-security-group] for the -database will need to allow access from the Cube Cloud CIDR block. - -This can be achieved by adding a new security group rule: - -- **Protocol:** TCP -- **Port Range:** The database port, usually `3306` for MySQL or `5432` for - Postgres. -- **Source/Destination:** The Cube Cloud CIDR block for the AWS region. - -## Supported Regions - -- `ca-central-1` -- `eu-central-1` -- `eu-west-1` -- `us-east-1` -- `us-east-2` -- `us-west-2` - -[aws-console]: https://console.aws.amazon.com/ -[aws-docs-regions]: - https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions -[aws-docs-vpc-peering-accept]: - https://docs.aws.amazon.com/vpc/latest/peering/create-vpc-peering-connection.html#accept-vpc-peering-connection -[aws-docs-vpc-peering]: - https://docs.aws.amazon.com/vpc/latest/peering/what-is-vpc-peering.html -[aws-docs-vpc-security-group]: - https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html#SecurityGroupRules diff --git a/docs/content/Cube-Cloud/Configuration/VPC/Connecting-with-a-VPC-Azure.mdx b/docs/content/Cube-Cloud/Configuration/VPC/Connecting-with-a-VPC-Azure.mdx deleted file mode 100644 index c2b15a43a4a58..0000000000000 --- a/docs/content/Cube-Cloud/Configuration/VPC/Connecting-with-a-VPC-Azure.mdx +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Connecting with a VPC on Azure -permalink: /cloud/configuration/connecting-with-a-vpc/azure ---- - -## Prerequisites - -To allow Cube Cloud to connect to a Virtual Network on Azure, the following -information is required: - -- **Virtual Network Name:** This can be found in the Virtual Networks section of - the [Azure Portal][azure-console]. -- **Tenant ID:** This can be found under Azure Active Directory > Properties > - Tenant ID in the [Azure Portal][azure-console]. - -## Setup - -Unfortunately Azure cross-tenant peering suppose that you grant peering role to -the user id/service principal to the peering party Below the list of steps that -would register Cube Cloud tenant at your organization and grant peering access -to Cube Cloud service principal - -### Add Cube tenant to your organization - -First the Cube Cloud tenant must be added to your organization. To do this, -open the [Azure Portal][azure-console] and go to Azure Active Directory > External -Identities > Cross-tenant access settings > Organizational Settings > Add -Organization. - -For Tenant ID, enter `197e5263-87f4-4ce1-96c4-351b0c0c714a`. Make sure that -B2B Collaboration > Inbound Access > Applications is set (or inherited) so -that it `Allows access` - -### Register Cube Cloud service principal at your organization - -`Cube Cloud service principal` - -info: -Client ID: `0c5d0d4b-6cee-402e-9a08-e5b79f199481` -Name: `cube-dedicated-infra-sp` - -Using browser tab where account that has rights to register -`Enterprise applications` is logged in open the following url: -https://login.microsoftonline.com/**Tenant -ID\*\*/oauth2/authorize?client_id=0c5d0d4b-6cee-402e-9a08-e5b79f199481&response_type=code&redirect_uri=https%3A%2F%2Fwww.microsoft.com%2F -Select `Consent on behalf of your organization` And click `Accept` - -### Grant peering permissions to Cube Cloud service principal on your `Virtual Network` - -As `peering role` you can use built-in `Network contributor` or custom role that -has the following permissions: - -- Microsoft.Network/virtualNetworks/virtualNetworkPeerings/write -- Microsoft.Network/virtualNetworks/peer/action -- Microsoft.ClassicNetwork/virtualNetworks/peer/action -- Microsoft.Network/virtualNetworks/virtualNetworkPeerings/read -- Microsoft.Network/virtualNetworks/virtualNetworkPeerings/delete - -On the [Azure Portal][azure-console], go to `Virtual networks` > **Virtual -Network Name** > `Access Control (IAM)` > `Add` > `Add role assignment` Role = -`peering role` Members: `cube-dedicated-infra-sp` - -### Firewall - -Make sure that your firewall rules allow inbound and outbound traffic - -## Information required by Cube Cloud support - -When you are reaching out Cube Cloud support please provide following -information: - -- **Virtual Network ID:** You can find it at `Virtual Networks` > **Virtual - Network Name** > `Overview` > `JSON view` > `Resource ID` on [Azure - Portal][azure-console]. -- **Virtual Network Address Spaces:** You can find it at `Virtual Networks` > - **Virtual Network Name** > `Overview` > `JSON view` > `properties` > - `addressSpace` on [Azure Portal][azure-console]. -- **Tenant ID:** You can find it in `Azure Active Directory` > `Properties` > - `Tenant ID` section of [Azure Portal][azure-console]. - -## Supported Regions - -We support all general purpose regions - -[azure-console]: https://portal.azure.com diff --git a/docs/content/Cube-Cloud/Configuration/VPC/Connecting-with-a-VPC.mdx b/docs/content/Cube-Cloud/Configuration/VPC/Connecting-with-a-VPC.mdx deleted file mode 100644 index 905f6ff9c68f5..0000000000000 --- a/docs/content/Cube-Cloud/Configuration/VPC/Connecting-with-a-VPC.mdx +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Connecting with a VPC -permalink: /cloud/configuration/connecting-with-a-vpc -category: Configuration -menuOrder: 4 ---- - - - -To enable VPC support for your account, please [contact Sales][cube-contact]. - - - -For improved stability and security, Cube Cloud also supports connecting to one -or more VPCs in your AWS or Google Cloud accounts. A Cube Cloud VPC connection -improves stability through dedicated infrastructure for a deployment, and -improves security by preventing your database traffic from being routed through -the public Internet. - - - - - - - -[cube-contact]: https://cube.dev/contact/ diff --git a/docs/content/Cube-Cloud/Configuration/VPC/gcp.svg b/docs/content/Cube-Cloud/Configuration/VPC/gcp.svg deleted file mode 100644 index 81b7d245478a1..0000000000000 --- a/docs/content/Cube-Cloud/Configuration/VPC/gcp.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/docs/content/Cube-Cloud/Configuration/connect-db-athena.png b/docs/content/Cube-Cloud/Configuration/connect-db-athena.png deleted file mode 100644 index ebd82f58f634f..0000000000000 Binary files a/docs/content/Cube-Cloud/Configuration/connect-db-athena.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/Configuration/connect-db-bigquery.png b/docs/content/Cube-Cloud/Configuration/connect-db-bigquery.png deleted file mode 100755 index 72db6adec5681..0000000000000 Binary files a/docs/content/Cube-Cloud/Configuration/connect-db-bigquery.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/Configuration/connect-db-redshift.png b/docs/content/Cube-Cloud/Configuration/connect-db-redshift.png deleted file mode 100644 index 3d14114c3ef5b..0000000000000 Binary files a/docs/content/Cube-Cloud/Configuration/connect-db-redshift.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/Configuration/connect-db-snowflake.png b/docs/content/Cube-Cloud/Configuration/connect-db-snowflake.png deleted file mode 100644 index 7a75750c02376..0000000000000 Binary files a/docs/content/Cube-Cloud/Configuration/connect-db-snowflake.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/Configuration/ssl-custom.png b/docs/content/Cube-Cloud/Configuration/ssl-custom.png deleted file mode 100755 index d8b25b33d63fa..0000000000000 Binary files a/docs/content/Cube-Cloud/Configuration/ssl-custom.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/Configuration/ssl-wizard.png b/docs/content/Cube-Cloud/Configuration/ssl-wizard.png deleted file mode 100755 index 77b340c335aae..0000000000000 Binary files a/docs/content/Cube-Cloud/Configuration/ssl-wizard.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/Cube-Cloud-Hybrid.md b/docs/content/Cube-Cloud/Cube-Cloud-Hybrid.md deleted file mode 100644 index 2edb63f983054..0000000000000 --- a/docs/content/Cube-Cloud/Cube-Cloud-Hybrid.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -title: Cube Cloud Hybrid Installation -permalink: /cube-cloud-hybrid -category: Cube Cloud -menuOrder: 1 ---- - -Cube Cloud can be installed as a hybrid cloud option. -In this case Cube Cloud acts as a Kubernetes operator. -In this case it doesn't have access to your VPC or database and manages your deployment only by means of Kubernetes API. -Below are instructions for popular cloud providers. - -## AWS - -First step of Cube Cloud installation is to create AWS EKS cluster: - -1. Please see https://docs.aws.amazon.com/eks/latest/userguide/create-cluster.html on how to create cluster. -If you choose to use AWS Management Console UI path please ensure you use same credentials to authorize in AWS Management Console and your AWS CLI. -2. While creating cluster please ensure you use the same VPC and security groups that your database use. -3. Please refer to https://docs.aws.amazon.com/eks/latest/userguide/worker_node_IAM_role.html if you have problems with setting up worker node roles. -4. Install `kubectl` if you don't have one: https://kubernetes.io/docs/tasks/tools/install-kubectl/. -5. Setup `kubectl` using https://docs.aws.amazon.com/eks/latest/userguide/create-kubeconfig.html. -6. Go to Cube Cloud deployment Overview page and copy `kubectl apply -f ...` installation command. -7. Run just copied `kubectl apply -f ...` in your command line. -8. After all required Kubernetes services started up you should see your deployment status at the Cube Cloud deployment Overview page. diff --git a/docs/content/Cube-Cloud/Deploys.mdx b/docs/content/Cube-Cloud/Deploys.mdx deleted file mode 100644 index aa5e7106d41bd..0000000000000 --- a/docs/content/Cube-Cloud/Deploys.mdx +++ /dev/null @@ -1,107 +0,0 @@ ---- -title: Deploys -permalink: /cloud/deploys -category: Deploys -menuOrder: 1 ---- - -This guide covers features and tools you can use to deploy your Cube.js project -to Cube Cloud. - -## Deploy with Git - -Continuous deployment works by connecting a Git repository to a Cube Cloud -deployment and keeping the two in sync. - -First, go to **Settings > Build & Deploy** to make sure your deployment is -configured to deploy with Git. Then click **Generate Git credentials** to obtain -Git credentials: - -
- -
- -The instructions to set up Cube Cloud as a Git remote are also available on the -same screen: - -```bash -$ git config credential.helper store -$ git remote add cubecloud -$ git push cubecloud master -``` - -## Deploy with GitHub - -First, ensure your deployment is configured to deploy with Git. Then connect -your GitHub repository to your deployment by clicking the **Connect to GitHub** -button, and selecting your repository. - -
- -
- -Cube Cloud will automatically deploy from the specified production branch -(**master** by default). - -## Deploy with CLI - -You can use the CLI to set up continuous deployment for a Git repository. You -can also use the CLI to manually deploy changes without continuous deployment. - -### <--{"id" : "Deploy with CLI"}--> Manual Deploys - -You can deploy your Cube.js project manually. This method uploads data schema -and configuration files directly from your local project directory. - -You can obtain Cube Cloud deploy token from your deployment **Settings** page. - -```bash -$ npx cubejs-cli deploy --token TOKEN -``` - -### <--{"id" : "Deploy with CLI"}--> Continuous Deployment - -You can use Cube.js CLI with your continuous integration tool. - - - -You can use the `CUBE_CLOUD_DEPLOY_AUTH` environment variable to pass the Cube -Cloud deploy token to Cube.js CLI. - - - -Below is an example configuration for GitHub Actions: - -```bash -name: My Cube.js App -on: - push: - paths: - - '**' - branches: - - 'master' -jobs: - deploy: - name: Deploy My Cube.js App - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Use Node.js 14.x - uses: actions/setup-node@v1 - with: - node-version: 14.x - - name: Deploy to Cube Cloud - run: npx cubejs-cli deploy - env: - CUBE_CLOUD_DEPLOY_AUTH: ${{ secrets.CUBE_CLOUD_DEPLOY_AUTH }} -``` diff --git a/docs/content/Cube-Cloud/Developer-Tools/Alerts.mdx b/docs/content/Cube-Cloud/Developer-Tools/Alerts.mdx deleted file mode 100644 index 446752ea6a5bb..0000000000000 --- a/docs/content/Cube-Cloud/Developer-Tools/Alerts.mdx +++ /dev/null @@ -1,125 +0,0 @@ ---- -title: Alerts in Cube Cloud -menuTitle: Alerts -permalink: /cloud/dev-tools/alerts -category: Developer Tools -menuOrder: 3 ---- - -Cube Cloud lets you set up alerts so you can be notified when there are issues -with any of the deployments in your account. - -Alert conditions are checked **once per minute** for configured -[alert types](#alert-types) and [deployments](#deployments). If an alert -condition is triggered, an email notification will be sent to configured -[recipients](#recipients). - -Cube Cloud only sends an outage email **once** per configured alert; when the -issue resolves, a resolution email is also sent. - - - -This feature is not available on the free plan. -See [pricing plans](https://cube.dev/pricing/) for details. - - - -## Alert Types - -Cube Cloud has three different types of alerts: - -- **API outages.** These are triggered if the deployment's API becomes - unavailable by monitoring the [`/livez` endpoint][ref-rest-api-livez]. -- **Database timeouts.** These are triggered if either the APIs or - the database are taking longer than expected to respond. -- **Pre-aggregation build failures.** These are triggered if any pre-aggregation - fails to build. - -## How to Configure - - - -Only **admin** users can configure alerts in Cube Cloud. - - - -To set up alerts in Cube Cloud, click your username in the top-right for the -menu, then click **Alerts**: - -
- Finding the Alerts menu in Cube Cloud -
- -You should now see the Alerts page. The example screenshot below shows no alerts -configured for this deployment: - -
- The Cube Cloud account's Alerts Page -
- -Click the **New Alert** button to bring up a dialog where you can configure the -alert: - -
- Creating a new Alert on Cube Cloud -
- -### Deployments - -By default, alerts are configured for all deployments in your account. You can -configure alerts for specific deployments by clicking **Specific** instead of -**All** and then checking the specific deployments you want to monitor: - -
- Selecting deployments for an alert on Cube Cloud -
- -### Recipients - -When creating a new alert, by default, all users will receive email alerts for -any issues. You can also specify particular users who should receive alerts by -toggling the **Specific users** option and selecting them: - -
- Setting receipients for an alert on Cube Cloud -
- -Optionally, you can also add additional email addresses that should receive -email notifications by entering the email address in the **Custom email** -field: - -
- Setting an custom email for an alert on Cube Cloud -
- -[ref-rest-api-livez]: /rest-api#api-reference-livez diff --git a/docs/content/Cube-Cloud/Developer-Tools/Cube-IDE.mdx b/docs/content/Cube-Cloud/Developer-Tools/Cube-IDE.mdx deleted file mode 100644 index 95c8ed8475af1..0000000000000 --- a/docs/content/Cube-Cloud/Developer-Tools/Cube-IDE.mdx +++ /dev/null @@ -1,54 +0,0 @@ ---- -title: Cube IDE in Cube Cloud -menuTitle: Cube IDE -permalink: /cloud/dev-tools/cube-ide -category: Developer Tools -menuOrder: 1 -redirect_from: - - /cloud/cube-ide ---- - -With the Cube IDE, you can write and test and your Cube.js data schemas from -your browser. Cube Cloud can create branch-based development API instances to -quickly test changes in the data schema in your frontend applications before -pushing them into production. - - - -## Development Mode - -In development mode, you can safely make changes to your project without -affecting production deployment. Development mode uses a separate Git branch and -allows testing your changes in Playground or via a separate API endpoint -specific to this branch. This development API hot-reloads your schema changes, -allowing you to quickly test API changes from your applications. - -To enter development mode, navigate to the **Schema** page and click **Enter -Development Mode**. - -
- -
- -When development mode is active, a grey bar will be visible at the top of the -screen. It provides several useful controls and indicators: - -- The name of the current development Git branch -- The status of the development API. After any changes to the project, the API - will hot-reload, and the API status will indicate when it's ready. -- 'Copy API URL' will copy the API URL to the clipboard for the current - development branch. - -You can exit development mode by clicking **Exit** button in the grey banner. - -
- -
diff --git a/docs/content/Cube-Cloud/Developer-Tools/Developer-Playground.md b/docs/content/Cube-Cloud/Developer-Tools/Developer-Playground.md deleted file mode 100644 index 4d2c3ef8db412..0000000000000 --- a/docs/content/Cube-Cloud/Developer-Tools/Developer-Playground.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: Developer Playground in Cube Cloud -menuTitle: Developer Playground -permalink: /cloud/dev-tools/dev-playground -category: Developer Tools -menuOrder: 2 ---- - -The Cube Cloud Developer Playground is mostly similar to the one available in -Cube.js with the notable difference of being able to query the Development Mode -API. - -## Development Mode - -After making changes to the schema through the [Cube IDE][ref-devtools-ide], the -API will restart and its status will be visible in the Playground: - -
- Playground during API restart -
- -[ref-devtools-ide]: /cloud/dev-tools/cube-ide diff --git a/docs/content/Cube-Cloud/Developer-Tools/alerts-1.png b/docs/content/Cube-Cloud/Developer-Tools/alerts-1.png deleted file mode 100755 index d3f336fe9e64a..0000000000000 Binary files a/docs/content/Cube-Cloud/Developer-Tools/alerts-1.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/Developer-Tools/alerts-2.png b/docs/content/Cube-Cloud/Developer-Tools/alerts-2.png deleted file mode 100755 index 6b12c504a9331..0000000000000 Binary files a/docs/content/Cube-Cloud/Developer-Tools/alerts-2.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/Developer-Tools/alerts-3.png b/docs/content/Cube-Cloud/Developer-Tools/alerts-3.png deleted file mode 100755 index 5f3c5b5b305a3..0000000000000 Binary files a/docs/content/Cube-Cloud/Developer-Tools/alerts-3.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/Developer-Tools/alerts-4.png b/docs/content/Cube-Cloud/Developer-Tools/alerts-4.png deleted file mode 100755 index 61a492d47963f..0000000000000 Binary files a/docs/content/Cube-Cloud/Developer-Tools/alerts-4.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/Developer-Tools/alerts-5.png b/docs/content/Cube-Cloud/Developer-Tools/alerts-5.png deleted file mode 100755 index 66099701bff44..0000000000000 Binary files a/docs/content/Cube-Cloud/Developer-Tools/alerts-5.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/Developer-Tools/alerts-6.png b/docs/content/Cube-Cloud/Developer-Tools/alerts-6.png deleted file mode 100755 index 2b3eb80e53da3..0000000000000 Binary files a/docs/content/Cube-Cloud/Developer-Tools/alerts-6.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/Developer-Tools/playground-restart.png b/docs/content/Cube-Cloud/Developer-Tools/playground-restart.png deleted file mode 100644 index 4963f3e88cff5..0000000000000 Binary files a/docs/content/Cube-Cloud/Developer-Tools/playground-restart.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/Getting-Started/Create-new.mdx b/docs/content/Cube-Cloud/Getting-Started/Create-new.mdx deleted file mode 100644 index 0ca8915760713..0000000000000 --- a/docs/content/Cube-Cloud/Getting-Started/Create-new.mdx +++ /dev/null @@ -1,132 +0,0 @@ ---- -title: 'Getting Started with Cube Cloud: Create a project' -permalink: /cloud/getting-started/create ---- - -This guide walks you through setting up Cube Cloud and connecting your Cube -project to your database. - -## Step 1: Create an account - -Navigate to [cubecloud.dev](https://cubecloud.dev/), and create a new Cube Cloud -account. - -## Step 2: Create a new Deployment - -Click the `Create Deployment` button. This is the first step in the deployment -creation. Give it a name and select the cloud provider and region of your -choice. - -
- Cube Cloud Create Deployment Screen -
- -## Step 3: Set up the Cube project from scratch - -Next up, the second step in creating a Cube App from scratch in Cube Cloud is to -click the `+ create` button. - -
- Cube Cloud Upload Project Screen -
- -## Step 4: Connect your Database - -Enter your credentials to connect to your database. Check the [connecting to -databases][link-connecting-to-databases] guide for more details. - - - -Want to use a sample database instead? We also have a sample database where you -can try out Cube Cloud: - -
- -| Field | Value | -| -------- | ------------------ | -| Host | `demo-db.cube.dev` | -| Port | `5432` | -| Database | `ecom` | -| Username | `cube` | -| Password | `12345` | - -
- -
- Cube Cloud Setup Database Screen -
- -In the UI it'll look exactly like the image below. - -
- Cube Cloud Setup Database Screen -
- -If you run into issues here, make sure to allow the Cube Cloud IPs to access -your database. This means you need to enable these IPs in your firewall. If you -are using AWS, this would mean adding a security group with allowed IPs. - -## Step 5: Generate the Data Schema - -Step four in this case consists of generating a data schema. Start by selecting -the database tables to generate the data schema from, then hit `generate`. - -
- Cube Cloud Setup Database Screen -
- -Cube Cloud will generate the data schema and spin up your Cube deployment. With -this, you're done. You've created a Cube deployment, configured a database -connection, and generated a data schema! - -
- Cube Cloud Setup Database Screen -
- -You're ready for the last step, running queries in the Playground. - -## Step 6: Try out Cube Cloud - -Now you can navigate to Playground to try out your queries or connect your -application to Cube Cloud API. - -
- Cube Cloud Playground -
- -[link-connecting-to-databases]: /cloud/configuration/connecting-to-databases diff --git a/docs/content/Cube-Cloud/Getting-Started/Import-Bitbucket-repository-via-SSH.mdx b/docs/content/Cube-Cloud/Getting-Started/Import-Bitbucket-repository-via-SSH.mdx deleted file mode 100644 index a35d614e336be..0000000000000 --- a/docs/content/Cube-Cloud/Getting-Started/Import-Bitbucket-repository-via-SSH.mdx +++ /dev/null @@ -1,198 +0,0 @@ ---- -title: 'Getting Started with Cube Cloud: Import Bitbucket repository via SSH' -permalink: /cloud/getting-started/ssh/bitbucket ---- - -This guide walks you through creating a new deployment on Cube Cloud from a -[Bitbucket][bitbucket] repository via SSH. - -## Step 1: Create an account - -Navigate to [cubecloud.dev](https://cubecloud.dev/), and create a new Cube Cloud -account. - -## Step 2: Create a new Deployment - -Click the `Create Deployment` button. This is the first step in the deployment -creation. Give it a name and select the cloud provider and region of your -choice. - -
- Cube Cloud Create Deployment Screen -
- -## Step 3: Import Git repository - -Next up, the second step in creating a Cube App from scratch in Cube Cloud is to -click the `Import Git repository via SSH` button. - -
- Cube Cloud Upload Project Screen -
- -Now go to your Bitbucket repository and on the `Clone` dialog, switch to `SSH` -and copy the URL: - -
- Getting the repository's SSH URL from Bitbucket -
- -Back in Cube Cloud, paste the URL and click `Generate SSH key`: - -
- Getting SSH key from Cube Cloud -
- -Now copy the SSH key and go back to Bitbucket and into the repository's -settings. Click `Access keys` from the navigation, then click the `Add key` -button. Give the key a label (`Cube Cloud`, for example) and paste the SSH key -in the relevant field: - -
- Add Cube Cloud deploy key to Bitbucket -
- -Click `Add SSH key` to save, then go back to Cube Cloud and click `Connect`. -After a connection is successfully established, you should see the next screen: - -
- Getting webhook URL from Cube Cloud -
- -Copy the `Cube Cloud Git Webhook URL` and go to your Bitbucket repository's -settings and click `Webhooks` from the navigation. Click `Add webhook`, then add -a title (`Cube Cloud`, for example). Paste the URL into the correct field, -ensure the `Repository > Push` trigger is checked and click `Save`. - -
- Add Cube Cloud webhook to Bitbucket -
- -Back in Cube Cloud, click `Connect` to test the webhook. - -## Step 4: Connect your Database - -Enter your credentials to connect to your database. Check the [connecting to -databases][link-connecting-to-databases] guide for more details. - - - -Want to use a sample database instead? We also have a sample database where you -can try out Cube Cloud: - -
- -| Field | Value | -| -------- | ------------------ | -| Host | `demo-db.cube.dev` | -| Port | `5432` | -| Database | `ecom` | -| Username | `cube` | -| Password | `12345` | - -
- -
- Cube Cloud Setup Database Screen -
- -In the UI it'll look exactly like the image below. - -
- Cube Cloud Setup Database Screen -
- -If you run into issues here, make sure to allow the Cube Cloud IPs to access -your database. This means you need to enable these IPs in your firewall. If you -are using AWS, this would mean adding a security group with allowed IPs. - -## Step 5: Generate the Data Schema - -Step five in this case consists of generating a data schema. Start by selecting -the database tables to generate the data schema from, then hit `generate`. - -
- Cube Cloud Setup Database Screen -
- -Cube Cloud will generate the data schema and spin up your Cube deployment. With -this, you're done. You've created a Cube deployment, configured a database -connection, and generated a data schema! - -
- Cube Cloud Setup Database Screen -
- -You're ready for the last step, running queries in the Playground. - -## Step 6: Try out Cube Cloud - -Now you can navigate to Playground to try out your queries or connect your -application to the Cube Cloud API. - -
- Cube Cloud Playground -
- -[bitbucket]: https://bitbucket.org/ -[link-connecting-to-databases]: /cloud/configuration/connecting-to-databases diff --git a/docs/content/Cube-Cloud/Getting-Started/Import-Git-repository-via-SSH.mdx b/docs/content/Cube-Cloud/Getting-Started/Import-Git-repository-via-SSH.mdx deleted file mode 100644 index cb6b2c00bf67f..0000000000000 --- a/docs/content/Cube-Cloud/Getting-Started/Import-Git-repository-via-SSH.mdx +++ /dev/null @@ -1,160 +0,0 @@ ---- -title: 'Getting Started with Cube Cloud: Import Git repository via SSH' -permalink: /cloud/getting-started/ssh/git ---- - -This guide walks you through creating a new deployment on Cube Cloud from a Git -repository via SSH. - -## Step 1: Create an account - -Navigate to [cubecloud.dev](https://cubecloud.dev/), and create a new Cube Cloud -account. - -## Step 2: Create a new Deployment - -Click the `Create Deployment` button. This is the first step in the deployment -creation. Give it a name and select the cloud provider and region of your -choice. - -
- Cube Cloud Create Deployment Screen -
- -## Step 3: Import Git repository - -Next up, the second step in creating a Cube App from scratch in Cube Cloud is to -click the `Import Git repository via SSH` button. - -
- Cube Cloud Upload Project Screen -
- -Enter the SSH URL of your Git repository, and click `Generate SSH key`: - -
- Getting SSH key from Cube Cloud -
- -Copy the SSH key and add it to your hosted Git repository. Back in Cube Cloud, -click `Connect`. After a connection is successfully established, you should see -the next screen: - -
- Getting webhook URL from Cube Cloud -
- -Copy the `Cube Cloud Git Webhook URL` and add it your hosted Git repository's -webhooks. Ensure that the Git repository can push events which should trigger a -build on Cube Cloud. Back in Cube Cloud, click `Connect` to test the webhook. - -## Step 4: Connect your Database - -Enter your credentials to connect to your database. Check the [connecting to -databases][link-connecting-to-databases] guide for more details. - - - -Want to use a sample database instead? We also have a sample database where you -can try out Cube Cloud: - -
- -| Field | Value | -| -------- | ------------------ | -| Host | `demo-db.cube.dev` | -| Port | `5432` | -| Database | `ecom` | -| Username | `cube` | -| Password | `12345` | - -
- -
- Cube Cloud Setup Database Screen -
- -In the UI it'll look exactly like the image below. - -
- Cube Cloud Setup Database Screen -
- -If you run into issues here, make sure to allow the Cube Cloud IPs to access -your database. This means you need to enable these IPs in your firewall. If you -are using AWS, this would mean adding a security group with allowed IPs. - -## Step 5: Generate the Data Schema - -Step five in this case consists of generating a data schema. Start by selecting -the database tables to generate the data schema from, then hit `generate`. - -
- Cube Cloud Setup Database Screen -
- -Cube Cloud will generate the data schema and spin up your Cube deployment. With -this, you're done. You've created a Cube deployment, configured a database -connection, and generated a data schema! - -
- Cube Cloud Setup Database Screen -
- -You're ready for the last step, running queries in the Playground. - -## Step 6: Try out Cube Cloud - -Now you can navigate to Playground to try out your queries or connect your -application to the Cube Cloud API. - -
- Cube Cloud Playground -
- -[link-connecting-to-databases]: /cloud/configuration/connecting-to-databases diff --git a/docs/content/Cube-Cloud/Getting-Started/Import-GitLab-repository-via-SSH.mdx b/docs/content/Cube-Cloud/Getting-Started/Import-GitLab-repository-via-SSH.mdx deleted file mode 100644 index 11dac9191e976..0000000000000 --- a/docs/content/Cube-Cloud/Getting-Started/Import-GitLab-repository-via-SSH.mdx +++ /dev/null @@ -1,197 +0,0 @@ ---- -title: 'Getting Started with Cube Cloud: Import GitLab repository via SSH' -permalink: /cloud/getting-started/ssh/gitlab ---- - -This guide walks you through creating a new deployment on Cube Cloud from a -[GitLab][gitlab] repository via SSH. - -## Step 1: Create an account - -Navigate to [cubecloud.dev](https://cubecloud.dev/), and create a new Cube Cloud -account. - -## Step 2: Create a new Deployment - -Click the `Create Deployment` button. This is the first step in the deployment -creation. Give it a name and select the cloud provider and region of your -choice. - -
- Cube Cloud Create Deployment Screen -
- -## Step 3: Import Git repository - -Next up, the second step in creating a Cube App from scratch in Cube Cloud is to -click the `Import Git repository via SSH` button. - -
- Cube Cloud Upload Project Screen -
- -Now go to your GitLab repository and from the `Clone` dropdown menu, copy the -`Clone with SSH` URL: - -
- Getting the repository's SSH URL from GitLab -
- -Back in Cube Cloud, paste the URL and click `Generate SSH key`: - -
- Getting SSH key from Cube Cloud -
- -Now copy the SSH key and go back to GitLab and paste it into the repository's -settings. Find the `Deploy keys` section and click `Expand`. Give the key a -title (`Cube Cloud`, for example) and paste the SSH key in the relevant field: - -
- Add Cube Cloud deploy key to GitLab -
- -Ensure `Grant write permissions to this key` is checked, then click `Add key`. -Go back to Cube Cloud and click `Connect`. After a connection is successfully -established, you should see the next screen: - -
- Getting webhook URL from Cube Cloud -
- -Copy the `Cube Cloud Git Webhook URL` and go to your GitLab project's `Webhooks` -settings. Paste the URL into the correct field, ensure the `Push events` trigger -is checked and click `Add webhook`. - -
- Add Cube Cloud webhook to GitLab -
- -Back in Cube Cloud, click `Connect` to test the webhook. - -## Step 4: Connect your Database - -Enter your credentials to connect to your database. Check the [connecting to -databases][link-connecting-to-databases] guide for more details. - - - -Want to use a sample database instead? We also have a sample database where you -can try out Cube Cloud: - -
- -| Field | Value | -| -------- | ------------------ | -| Host | `demo-db.cube.dev` | -| Port | `5432` | -| Database | `ecom` | -| Username | `cube` | -| Password | `12345` | - -
- -
- Cube Cloud Setup Database Screen -
- -In the UI it'll look exactly like the image below. - -
- Cube Cloud Setup Database Screen -
- -If you run into issues here, make sure to allow the Cube Cloud IPs to access -your database. This means you need to enable these IPs in your firewall. If you -are using AWS, this would mean adding a security group with allowed IPs. - -## Step 5: Generate the Data Schema - -Step five in this case consists of generating a data schema. Start by selecting -the database tables to generate the data schema from, then hit `generate`. - -
- Generating schemas for a new Cube Cloud deployment -
- -Cube Cloud will generate the data schema and spin up your Cube deployment. With -this, you're done. You've created a Cube deployment, configured a database -connection, and generated a data schema! - -
- Cube Cloud Playground -
- -You're ready for the last step, running queries in the Playground. - -## Step 6: Try out Cube Cloud - -Now you can navigate to Playground to try out your queries or connect your -application to the Cube Cloud API. - -
- Cube Cloud Playground -
- -[gitlab]: https://gitlab.com/ -[link-connecting-to-databases]: /cloud/configuration/connecting-to-databases diff --git a/docs/content/Cube-Cloud/Getting-Started/Overview.mdx b/docs/content/Cube-Cloud/Getting-Started/Overview.mdx deleted file mode 100644 index 066231b5e3a7c..0000000000000 --- a/docs/content/Cube-Cloud/Getting-Started/Overview.mdx +++ /dev/null @@ -1,44 +0,0 @@ ---- -title: Getting Started with Cube Cloud -menuTitle: Getting Started -permalink: /cloud/getting-started -category: Cube Cloud Getting Started -menuOrder: 1 -redirect_from: - - /cloud/quickstart ---- - -## Console - - - - - - -## SSH - - - - - - diff --git a/docs/content/Cube-Cloud/Getting-Started/Upload-with-CLI.mdx b/docs/content/Cube-Cloud/Getting-Started/Upload-with-CLI.mdx deleted file mode 100644 index 99102b361578b..0000000000000 --- a/docs/content/Cube-Cloud/Getting-Started/Upload-with-CLI.mdx +++ /dev/null @@ -1,92 +0,0 @@ ---- -title: 'Getting Started with Cube Cloud: Upload with CLI' -permalink: /cloud/getting-started/cli ---- - -This guide walks you through setting up Cube Cloud and connecting to your -database. - - - -## Step 1: Create Deployment - -Click Create Deployment. Then give it a name and select the cloud provider and -region of your choice. - -
- Cube Cloud Create Deployment Screen -
- -## Step 2: Upload your Cube.js project - -The next step is to upload your existing Cube.js project to the Cube Cloud. - -You can do it by running the following command from terminal in your Cube.js -project directory. - -```bash -npx cubejs-cli deploy --token -``` - -
- Cube Cloud Upload Project Screen -
- -## Step 3: Connect your Database - -Enter your credentials to connect to your database. Check the [connecting to -databases][ref-cloud-connecting-to-databases] guide for more details. - - - -Want to use a sample database instead? We also have a sample database where you -can try out Cube Cloud: - -
- -| Field | Value | -| -------- | ------------------ | -| Host | `demo-db.cube.dev` | -| Port | `5432` | -| Database | `ecom` | -| Username | `cube` | -| Password | `12345` | - -
- -
- Cube Cloud Setup Database Screen -
- -## Step 4: Try out Cube Cloud - -Now you can navigate to [the Playground][ref-cloud-playground] to try out your -queries or connect your application to Cube Cloud API. - -
- Cube Cloud Playground -
- -[ref-cloud-connecting-to-databases]: - /cloud/configuration/connecting-to-databases -[ref-cloud-playground]: /cloud/dev-tools/dev-playground diff --git a/docs/content/Cube-Cloud/Index.mdx b/docs/content/Cube-Cloud/Index.mdx deleted file mode 100644 index 75d4acf2d83c7..0000000000000 --- a/docs/content/Cube-Cloud/Index.mdx +++ /dev/null @@ -1,44 +0,0 @@ ---- -title: Cube Cloud -permalink: /cloud -category: Overview -menuOrder: 1 ---- - -[Cube Cloud][link-cube-cloud] offers hosted Cube.js service with extra features -for reliability and performance. It includes all the core features of Cube.js, -while taking care of infrastructure concerns such as the number of instances, -memory, high-availability, pre-aggregations management, caching, scalability, -real-time monitoring and tracing. - -[Get Started with Cube Cloud](/cloud/quickstart) - -## Develop Cube.js projects - -You can use Cube Cloud IDE to develop and run Cube.js applications. By -connecting your GitHub account, you can keep your data schema under version -control. - - - -## Run and Scale Cube.js applications - -Cube Cloud provides on-demand scalable infrastructure and pre-aggregations -storage. Cube Cloud runs hundreds of [Cube Store][ref-cubestore] instances to -ingest and query pre-aggregations with low latency and high concurrency. It is -available to all users on the Standard plan and higher. - -## Live preview your feature branches - -Cube Cloud can spin up Cube.js API instances to test changes to the data schema -in feature branches. You can use branch-based development API URLs in your -frontend application to test changes before shipping them to production. - -## Inspect Cube.js queries - -You can trace and inspect your Cube.js queries to find performance flaws and -apply optimizations. Cube Cloud also provides tips and suggestions on what -pre-aggregation should be used. - -[link-cube-cloud]: https://cube.dev/cloud -[ref-cubestore]: /caching/using-pre-aggregations#pre-aggregations-storage diff --git a/docs/content/Cube-Cloud/Inspecting-Queries.md b/docs/content/Cube-Cloud/Inspecting-Queries.md deleted file mode 100644 index 877600b3a6683..0000000000000 --- a/docs/content/Cube-Cloud/Inspecting-Queries.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: Inspecting Queries -permalink: /cloud/inspecting-queries -category: Inspecting Queries -menuOrder: 1 ---- - -You can inspect your Cube.js queries in Cube Cloud. - -
- -
- -## Find slow queries - -You can use filters to find slow and not pre-aggregated queries. - -
- -
- -## Pre-aggregation suggestions - -Cube Cloud provides useful tips for pre-aggregation strategies and recommends -you the best pre-aggregation for the specific query. -Click on the any query on the **Queries** page and navigate to the **Pre-Aggregations** -tab inside the query details page. - -
- -
diff --git a/docs/content/Cube-Cloud/Release-Notes/12-2021.mdx b/docs/content/Cube-Cloud/Release-Notes/12-2021.mdx deleted file mode 100644 index b0c89ebedbccd..0000000000000 --- a/docs/content/Cube-Cloud/Release-Notes/12-2021.mdx +++ /dev/null @@ -1,115 +0,0 @@ ---- -title: Release Notes -permalink: /cloud/release-notes/12-2021 -category: Release Notes -menuOrder: 1 ---- - -Following Cube Cloud becoming generally available in October 2021, we've been -hard at work making improvements across the board. Let's have a quick run -through them: - -## Database Connections - -Cube Cloud now supports configuring your own specified SSL certificates when -connecting to databases. When creating a new deployment, check the 'SSL' box, -and paste the relevant SSL certificates: - -![SSL configuration options on Set Up a Database Connection screen|496x500](https://aws1.discourse-cdn.com/standard11/uploads/cube1/original/1X/336591951b312cdf38c5cd6a5883e0b1b86057a8.png) - -## Git integration - -We've made it really easy to quickly add more Git branches to your Cube Cloud -deployment and switch between them, just click the branch name in the status -bar: - -![Branch switcher|685x500](https://aws1.discourse-cdn.com/standard11/uploads/cube1/original/1X/68dd08d82e696646f8eb94c109cdeeec623272e9.png) - -Speaking of Git branches, you can now easily add and remove branches with the -same switcher; click 'Add Branch' and enter a name for the new branch in the -popup: - -![Create a new branch|690x431](https://aws1.discourse-cdn.com/standard11/uploads/cube1/original/1X/f3a2e6f328ea207599cc784f1051a10b1d47ffa8.png) - -These branches are shared; meaning everyone on your Cube Cloud team can see and -edit them. This makes them super useful for out-of-band experiments where you -can quickly test things in Cube Cloud without having to go through a CI/CD -process. - -To round things off, you can also delete unused branches too; first make sure -you are already on the branch, then open the switcher and click 'Remove Branch': - -![Delete a branch|685x500](https://aws1.discourse-cdn.com/standard11/uploads/cube1/original/1X/93b46895de0cf8c1dfac85640438c4d30a56e767.png) - -## GitHub integration - -When connecting your GitHub repo to Cube Cloud, you can now choose a default -branch for deployments: - -![Integrate with GitHub repository|690x440](https://aws1.discourse-cdn.com/standard11/uploads/cube1/original/1X/2673d0d9e21fcdbc1ae9ec5658badcdc17e4eb24.png) - -## Developer Mode - -We've seen a round of improvements to the Cube Cloud Developer Mode experience. -When you're in Developer Mode, the status bar is now purple so you always know -when it's active: - -![Development Mode now shows a purple status bar|690x431](https://aws1.discourse-cdn.com/standard11/uploads/cube1/original/1X/e7e2349db8994cecce4979466d270280a85214dd.png) - -If you've been editing a schema and navigate away, we warn you if there are any -unsaved changes: - -![Unsaved changes warning modal|690x431](https://aws1.discourse-cdn.com/standard11/uploads/cube1/original/1X/2f1f9c64ef6a6a6b66092bc38332f0c8bbced378.png) - -And last but not least, we've added a new 'Generate Schema' button to the Schema -page. Clicking this will let you re-generate schemas from your source database, -or alternatively add rollups to existing schemas: - -![Generate rollups or schema modal|690x428](https://aws1.discourse-cdn.com/standard11/uploads/cube1/original/1X/ae01322061fa45357275939c10927d5339cd723b.png) - -## Query Tracing - -When inspecting queries in the Queries tab, you can now also see whether or not -the query used a pre-aggregation: - -![Query showing pre-aggregation lightning mark|674x500](https://aws1.discourse-cdn.com/standard11/uploads/cube1/original/1X/c953bbe2684339bc36fafaa1efb39202e18f85b2.jpeg) - -If you click a query from the list, we now provide more relevant information, -like if the query built a pre-aggregation or not: - -![Query request details|690x428](https://aws1.discourse-cdn.com/standard11/uploads/cube1/original/1X/7160cd06f0888494f4409744ca472e0efb7a0449.png) - -## SQL API - -Following the release of the SQL API in Cube, we're happy to announce that you -can also use it in Cube Cloud. You can enable it by clicking 'Deploy SQL API' -from the Overview page, then click the 'How to connect your BI tool'. You should -then see something similar to the following: - -![SQL API details modal|690x428](https://aws1.discourse-cdn.com/standard11/uploads/cube1/original/1X/8501d40a2062967723b4de78af6e6b496f60be0b.png) - -You can read more about the Cube SQL API -[in the documentation](https://cube.dev/docs/backend/sql), and we even have -[a recipe that shows you how to connect the SQL API to Apache Superset](https://cube.dev/docs/recipes/using-apache-superset-with-cube-sql). - -## GraphQL API - -In parallel to the SQL API, we also are excited to announce a new GraphQL API. -You can copy its’ URL from the Overview page, just under the SQL API: - -![GraphQL API|690x310](https://aws1.discourse-cdn.com/standard11/uploads/cube1/original/1X/85dd23cf30bdd78b3ca5f4bd04ec925ab674c70b.png) - -More information can be found -[in the documentation](https://cube.dev/docs/backend/graphql); we hope you enjoy -using it as much as we are 🤓 - -## Other Notable Improvements - -- When a deployment is connected to a Git monorepo, only commits to the Cube - project folder will trigger a rebuild on Cube Cloud. -- When switching from `stable` track to `latest`, Cube Cloud will now - automatically restart the deployment. -- `COMPILE_CONTEXT` now works correctly in Developer Mode. -- Automatically switch to Developer Mode after adding a rollup using the Rollup Designer. - -That's all for this release, stay tuned for the release notes next time 😀 diff --git a/docs/content/Cube-Cloud/cube-ide.png b/docs/content/Cube-Cloud/cube-ide.png deleted file mode 100644 index 3e4bf50f58be5..0000000000000 Binary files a/docs/content/Cube-Cloud/cube-ide.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/deploy-with-git-creds.png b/docs/content/Cube-Cloud/deploy-with-git-creds.png deleted file mode 100644 index 0d7f78552c51a..0000000000000 Binary files a/docs/content/Cube-Cloud/deploy-with-git-creds.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/deploy-with-github.png b/docs/content/Cube-Cloud/deploy-with-github.png deleted file mode 100755 index a87fb69c8e63e..0000000000000 Binary files a/docs/content/Cube-Cloud/deploy-with-github.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/dev-mode-bar.png b/docs/content/Cube-Cloud/dev-mode-bar.png deleted file mode 100644 index aa783e9aed688..0000000000000 Binary files a/docs/content/Cube-Cloud/dev-mode-bar.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/enter-dev-mode.png b/docs/content/Cube-Cloud/enter-dev-mode.png deleted file mode 100644 index 25ced97c0dd7c..0000000000000 Binary files a/docs/content/Cube-Cloud/enter-dev-mode.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/getting-started-2.png b/docs/content/Cube-Cloud/getting-started-2.png deleted file mode 100644 index 3241e2cad3dd6..0000000000000 Binary files a/docs/content/Cube-Cloud/getting-started-2.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/getting-started-4.png b/docs/content/Cube-Cloud/getting-started-4.png deleted file mode 100644 index 18b91f9e87345..0000000000000 Binary files a/docs/content/Cube-Cloud/getting-started-4.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/inspect-queries-1.png b/docs/content/Cube-Cloud/inspect-queries-1.png deleted file mode 100644 index 277a2e7a90b64..0000000000000 Binary files a/docs/content/Cube-Cloud/inspect-queries-1.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/inspect-queries-2.png b/docs/content/Cube-Cloud/inspect-queries-2.png deleted file mode 100644 index 0165e59f2a894..0000000000000 Binary files a/docs/content/Cube-Cloud/inspect-queries-2.png and /dev/null differ diff --git a/docs/content/Cube-Cloud/inspect-queries.png b/docs/content/Cube-Cloud/inspect-queries.png deleted file mode 100644 index 75e45466eab08..0000000000000 Binary files a/docs/content/Cube-Cloud/inspect-queries.png and /dev/null differ diff --git a/docs/content/Cube.js-Introduction.md b/docs/content/Cube.js-Introduction.md deleted file mode 100644 index 0ca465f5c1b7c..0000000000000 --- a/docs/content/Cube.js-Introduction.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: Introduction -permalink: /introduction -category: Cube.js Introduction -redirect_from: - - /cubejs-introduction ---- - -**Cube is the headless business intelligence platform.** It helps data engineers and application developers access data from modern data stores, organize it into consistent definitions, and deliver it to every application. - - - -Cube was designed to work with all SQL-enabled data sources, including cloud data warehouses like Snowflake or Google BigQuery, query engines like Presto or Amazon Athena, and application databases like Postgres. Cube has a built-in caching engine to provide sub-second latency and high concurrency for API requests. - -With Cube, you can build a data model, manage access control and caching, and expose your data to every application via REST, GraphQL, and SQL APIs. -Cube is headless, API-first, and decoupled from visualizations. You can use any charting library to build custom UI, or connect existing dashboarding and reporting tools to Cube. - -## Why Cube? - -If you are building a data application—such as a business intelligence tool or a customer-facing analytics feature—you’ll probably face the following problems: - -1. __SQL code organization.__ Sooner or later, modeling even a dozen metrics with a dozen dimensions using pure SQL queries becomes a maintenance nightmare, which leads to building a modeling framework. -2. __Performance.__ Most of the time and effort in modern analytics software development is spent providing adequate time to insight. In a world where every company’s data is big data, writing just SQL queries to get insight isn’t enough anymore. -3. __Access Control.__ It is important to secure and govern access to data for all downstream data consuming applications. - -Cube has the necessary infrastructure and features to implement efficient data modeling, access control, and performance optimizations so that every application can access consistent data via REST, SQL, and GraphQL APIs. Achieve insights from raw data within minutes, and get an API with sub-second response times on up to a trillion data points. - -
- -
- -## Architecture - -**Cube acts as a data access layer**, translating API requests into SQL, managing caching, queuing, and database connection. - -The Cube accepts queries via REST, GraphQL or SQL interfaces. Based on the data model and an incoming query, Cube generates -a SQL query and executes it in your database. Cube fully manages query orchestration, database connections, as well as caching and access control layers. The result is then sent back to the client. - -
- -
- -[ref-query-format]: https://cube.dev/docs/query-format diff --git a/docs/content/Cube.js-Introduction.mdx b/docs/content/Cube.js-Introduction.mdx new file mode 100644 index 0000000000000..5d15c1b20da1a --- /dev/null +++ b/docs/content/Cube.js-Introduction.mdx @@ -0,0 +1,104 @@ +--- +title: Introduction +permalink: /introduction +category: Cube.js Introduction +redirect_from: + - /cubejs-introduction +--- + +**Cube is the Semantic Layer for building data apps.** It helps data engineers +and application developers access data from modern data stores, organize it into +consistent definitions, and deliver it to every application. + + + +Cube was designed to work with all SQL-enabled data sources, including cloud +data warehouses like Snowflake or Google BigQuery, query engines like Presto or +Amazon Athena, and application databases like Postgres. Cube has a built-in +caching engine to provide sub-second latency and high concurrency for API +requests. + +With Cube, you can build a data model, manage access control and caching, and +expose your data to every application via REST, GraphQL, and SQL APIs. Cube is +headless, API-first, and decoupled from visualizations. You can use any charting +library to build custom UI, or connect existing dashboarding and reporting tools +to Cube. + +## Why Cube? + +If you are building a data application—such as a business intelligence tool or a +customer-facing analytics feature—you’ll probably face the following problems: + +1. **SQL code organization.** Sooner or later, modeling even a dozen metrics + with a dozen dimensions using pure SQL queries becomes a maintenance + nightmare, which leads to building a modeling framework. +2. **Performance.** Most of the time and effort in modern analytics software + development is spent providing adequate time to insight. In a world where + every company’s data is big data, writing just SQL queries to get insight + isn’t enough anymore. +3. **Access Control.** It is important to secure and govern access to data for + all downstream data consuming applications. + +Cube has the necessary infrastructure and features to implement efficient data +modeling, access control, and performance optimizations so that every +application can access consistent data via REST, SQL, and GraphQL APIs. Achieve +insights from raw data within minutes, and get an API with sub-second response +times on up to a trillion data points. + +
+ +
+ +## Architecture + +**Cube acts as a data access layer**, translating API requests into SQL, +managing caching, queuing, and database connection. + +Cube accepts queries via REST, GraphQL or SQL interfaces. Based on the data +model and an incoming query, Cube generates a SQL query and executes it in your +database. Cube fully manages query orchestration, database connections, as well +as caching and access control layers. The result is then sent back to the +client. + +
+ +
+ +## Why Cube Cloud? + +[Cube Cloud][cube-cloud] offers a managed Cube service with extra features for +reliability and performance. It enhances Cube with features that substantially +improve reliability, performance, and security: + +- **Performance:** Cube Store in-memory caching, Cube Store autoscaling, + autoscaling for multi-tenancy. + +- **Security:** Cube Store encryption at rest, SQL API connection via SSL, + secured access to Cube Playground, security audits and updates to Docker + images. + +- **Reliability:** Cube Store replication for high availability, distributed + file storage failovers and consistency checking. + +- **Integrations:** Optimized Databricks driver, optimized Elasticsearch driver, + support for Azure blob storage in Cube Store, and integration with monitoring + solutions. + +Cube Cloud is available as [managed service](https://cubecloud.dev/auth/signup) +and through "bring your own cloud" model. +[Please contact us](https://cube.dev/contact) if you'd like to install Cube +Cloud within your AWS, GCP, or Azure account. + +[cube-cloud]: https://cube.dev/cloud diff --git a/docs/content/Deployment/Cloud/Auto-Suspension.mdx b/docs/content/Deployment/Cloud/Auto-Suspension.mdx new file mode 100644 index 0000000000000..0a566f2550a41 --- /dev/null +++ b/docs/content/Deployment/Cloud/Auto-Suspension.mdx @@ -0,0 +1,97 @@ +--- +title: Auto-suspension +permalink: /deployment/cloud/auto-suspension +category: Deployment +subCategory: Cube Cloud +menuOrder: 3 +--- + + + +Auto-suspension is available in Cube Cloud on +[Starter and above tiers](https://cube.dev/pricing). + + + +Cube Cloud can automatically suspend deployments when not in use to prevent +[resource consumption][ref-deployment-pricing] when infrastructure is not being +actively used, which helps manage spend and preventing unnecessary quota use. + +This is useful for deployments that are not used 24/7, such as staging +deployments. Auto-suspension will hibernate the deployment when **no** API +requests are received after a period of time, and automatically resume the +deployment when API requests start coming in again: + +
+ Cube Cloud auto-suspend flowchart +
+ +[Development Instances][ref-deployment-dev-instance] are auto-suspended +automatically when not in use for 10 minutes, whereas [Production +Clusters][ref-deployment-prod-cluster] and [Production +Multi-Clusters][ref-deployment-prod-multi-cluster] can auto-suspend after no API +requests were received within a configurable time period. While suspended, +[pre-aggregation][ref-caching-preaggs-gs] builds will also be paused to prevent +unnecessary resource consumption. + +## Configuration + +To configure auto-suspension settings, navigate to the Settings +screen in your deployment and click the Configuration tab, then +ensure Enable Auto-suspend is turned on: + + + +To configure how long Cube Cloud should wait before suspending the deployment, +adjust Auto-suspend threshold (minutes) to the desired value and +click Apply: + + + +The Cube API instances will temporarily become unavailable while they are +configured; this usually takes less than a minute. + +## Resuming a suspended deployment + +To resume a suspended deployment, send a query to Cube using the API or by +navigating to the deployment in Cube Cloud. + + + +Currently, Cube Cloud's auto-suspension feature cannot guarantee a 100% resume +rate on the first query or a specific time frame for resume. While in most +cases, deployment resumes within several seconds of the first query, there is +still a possibility that it may take longer to resume your deployment. This can +potentially lead to an error response code for the initial query. + + + +Deployments typically resume in under 30 seconds, but can take significantly +longer in certain situations depending on two major factors: + +- **Data model:** How many cubes and views are defined. +- **Query complexity:** How complicated the queries being sent to the API are + +Complex data models take more time to compile, and complex queries can cause +response times to be significantly longer than usual. + +[ref-caching-preaggs-gs]: /caching/pre-aggregations/getting-started +[ref-deployment-dev-instance]: + /cloud/configuration/deployment-types#development-instance +[ref-deployment-prod-cluster]: + /cloud/configuration/deployment-types#production-cluster +[ref-deployment-prod-multi-cluster]: + /cloud/configuration/deployment-types#production-multi-cluster +[ref-deployment-pricing]: /cloud/pricing +[ref-workspace-dev-api]: /cloud/workspace/development-api diff --git a/docs/content/Deployment/Cloud/Continuous-Deployment.mdx b/docs/content/Deployment/Cloud/Continuous-Deployment.mdx new file mode 100644 index 0000000000000..6cf3a054510e6 --- /dev/null +++ b/docs/content/Deployment/Cloud/Continuous-Deployment.mdx @@ -0,0 +1,115 @@ +--- +title: Continuous deployment +permalink: /cloud/deploys +category: Deployment +subCategory: Cube Cloud +menuOrder: 3 +--- + +This guide covers features and tools you can use to deploy your Cube project to +Cube Cloud. + +## Deploy with Git + +Continuous deployment works by connecting a Git repository to a Cube Cloud +deployment and keeping the two in sync. + +First, go to the Build & Deploy tab on the Settings screen +to make sure your deployment is configured to deploy with Git. Then +click Generate Git credentials to obtain Git credentials: + + + +The instructions to set up Cube Cloud as a Git remote are also available on the +same screen: + +```bash{promptUser: user} +git config credential.helper store +git remote add cubecloud +git push cubecloud master +``` + +## Deploy with GitHub + +First, ensure your deployment is configured to deploy with Git. Then connect +your GitHub repository to your deployment by clicking the Connect to +GitHub button, and selecting your repository. + +
+ +
+ +Cube Cloud will automatically deploy from the specified production branch +(`master` by default). + +## Deploy with CLI + + + +Enabling this option will cause the Data Model page to display the +last known state of a Git-based codebase (if available), instead of reflecting +the latest modifications made. It is important to note that the logic will still +be updated in both the API and the Playground. + + + +You can use [the CLI][ref-workspace-cli] to set up continuous deployment for a +Git repository. You can also use it to manually deploy changes without +continuous deployment. + +### <--{"id" : "Deploy with CLI"}--> Manual Deploys + +You can deploy your Cube project manually. This method uploads data models and +configuration files directly from your local project directory. + +You can obtain a Cube Cloud deploy token from your +deployment's Settings screen. + +```bash{promptUser: user} +npx cubejs-cli deploy --token TOKEN +``` + +### <--{"id" : "Deploy with CLI"}--> Continuous Deployment + +You can use Cube CLI with your continuous integration tool. + + + +You can use the `CUBE_CLOUD_DEPLOY_AUTH` environment variable to pass the Cube +Cloud deploy token to Cube CLI. + + + +Below is an example configuration for GitHub Actions: + +```yaml +name: My Cube App +on: + push: + paths: + - "**" + branches: + - "master" +jobs: + deploy: + name: Deploy My Cube App + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Use Node.js 14.x + uses: actions/setup-node@v1 + with: + node-version: 14.x + - name: Deploy to Cube Cloud + run: npx cubejs-cli deploy + env: + CUBE_CLOUD_DEPLOY_AUTH: ${{ secrets.CUBE_CLOUD_DEPLOY_AUTH }} +``` + +[ref-workspace-cli]: /using-the-cubejs-cli diff --git a/docs/content/Deployment/Cloud/Custom-Domains.mdx b/docs/content/Deployment/Cloud/Custom-Domains.mdx new file mode 100644 index 0000000000000..77705edb4c50e --- /dev/null +++ b/docs/content/Deployment/Cloud/Custom-Domains.mdx @@ -0,0 +1,45 @@ +--- +title: Custom domains +permalink: /cloud/configuration/custom-domains +category: Deployment +subCategory: Cube Cloud +menuOrder: 4 +--- + +By default, Cube Cloud deployments and their API endpoints use auto-generated +anonymized domain names, e.g., `emerald-llama.gcp-us-central1.cubecloudapp.dev`. +You can also assign a custom domain to your deployment. + + + +Custom domains are available in Cube Cloud on +[Enterprise Premier](https://cube.dev/pricing) tier. +[Contact us](https://cube.dev/contact) for details. + + + +To set up a custom domain, go to your deployment's settings page. Under +the Custom Domain section, type in your custom domain and +click Add: + + + +After doing this, copy the provided domain and add the following `CNAME` records +to your DNS provider: + +| Name | Value | +| ------------- | ------------------------------------------- | +| `YOUR_DOMAIN` | The copied value from Cube Cloud's settings | + +Using the example subdomain from the screenshot, a `CNAME` record for `acme.dev` +would look like: + +| Name | Value | +| ---------- | ----------------------------------------------------- | +| `insights` | `colossal-crowville.gcp-us-central1.cubecloudapp.dev` | + +DNS changes can sometimes take up to 15 minutes to propagate, please wait at +least 15 minutes and/or try using another DNS provider to verify the `CNAME` +record correctly before raising a new support ticket. + +[cube-contact]: https://cube.dev/contact diff --git a/docs/content/Deployment/Cloud/Deployment-Types.mdx b/docs/content/Deployment/Cloud/Deployment-Types.mdx new file mode 100644 index 0000000000000..6e18b2ebe8d0a --- /dev/null +++ b/docs/content/Deployment/Cloud/Deployment-Types.mdx @@ -0,0 +1,171 @@ +--- +title: Deployment types +permalink: /cloud/configuration/deployment-types +category: Deployment +subCategory: Cube Cloud +menuOrder: 5 +--- + +## Development Instance + + + +Development instance is available in Cube Cloud for free, no credit card +required. Your free trial is limited to 2 development instances and only 1,000 +queries per day. Upgrade to [any tier](https://cube.dev/pricing) to unlock all +features. + + + +
+ High-level architecture diagram of a Cube Cloud Development Instance +
+ +Development Instances are designed for development use-cases only. This makes it +easy to get started with Cube Cloud quickly, and also allows you to build and +query pre-aggregations on-demand. + +Development Instances do not provide high-availability nor do they guarantee +fast response times. Development Instances also hibernate after 60 minutes of +inactivity, which can cause the first request after hibernation to take +additional time to process. They also have a 10,000 queries per day limit, and +are limited to a maximum of 2 Cube Store Workers. We **strongly** advise not +using a development instance in a production environment, it is for testing and +learning about Cube only and will not deliver a production-level experience for +your users. + +You can try a Cube Cloud development instance by +[signing up for Cube Cloud](https://cubecloud.dev/auth/signup) to try it free +(no credit card required). + +## Production Cluster + + + +Production cluster is available in Cube Cloud on +[all tiers](https://cube.dev/pricing). + + + +
+ High-level architecture diagram of a Cube Cloud Production Cluster +
+ +Production Clusters are designed to support high-availability production +workloads. It consists of several key components, including starting with 2 Cube +API instances, 1 Cube Refresh Worker and 2 Cube Store Routers - all of which run +on dedicated infrastructure. The cluster can automatically scale to meet the +needs of your workload by adding more components as necessary; check the +[Limits section](#limits) below. + +## Production multi-cluster + +You can deploy a production multi-cluster in Cube Cloud, which allows for larger +amounts of data as well as load balancing for multi-tenant deployments with over +1,000 tenants. + + + +Production multi-cluster is available in Cube Cloud on +[Enterprise Premier](https://cube.dev/pricing) tier. +[Contact us](https://cube.dev/contact) for details. + + + +
+ High-level architecture diagram of a Cube Cloud Production Multi-Cluster +
+ +Cube Cloud routes traffic between clusters based on +[`contextToAppId()`][ref-conf-ref-ctx-to-app-id]. + +Each cluster is billed separately, and all clusters can use auto-scaling to +match demand. + +## Limits + +Cube Cloud also allows adding additional infrastructure to your deployment to +increase scalability and performance beyond what is available with each +Production Deployment. + +### <--{"id" : "Limits"}--> Cube Store Worker + +| | Free Tier | Starter | Premium | Enterprise | Enterprise Premier | +| ------------------------- | :-------: | :-----: | :-----: | :--------: | :------------------------------: | +| Maximum number of workers | 2 | 16 | 16 | 16 | [Talk to Sales][cube-contact-us] | + +Cube Store Workers are used to build and persist pre-aggregations. Each Worker +has a **maximum of 150GB** of storage; additional Cube Store workers can be +added to your deployment to both increase storage space and improve +pre-aggregation performance. A **minimum of 2** Cube Store Workers is required +for pre-aggregations; this can be adjusted. For a rough estimate, it will take +approximately 2 Cube Store Workers per 4 gigs of pre-aggregated data per day. + + + +Idle workers will automatically hibernate after 10 minutes of inactivity, and +will not consume CCUs until they are resumed. Workers are resumed automatically +when Cube receives a query that should be accelerated by a pre-aggregation, or +when a scheduled refresh is triggered. + + + +To change the number of Cube Store Workers in a deployment, go to the +deployment’s Settings screen, and open the Configuration +tab. From this screen, you can set the number of Cube Store Workers from the +dropdown: + + + +### <--{"id" : "Limits"}--> Cube API Instance + +| | Free Tier | Starter | Premium | Enterprise | Enterprise Premier | +| --------------- | :-------: | :-----: | :-------: | :--------: | :----------------: | +| Queries per day | 1,000 | 50,000 | Unlimited | Unlimited | Unlimited | + +With a Production Deployment, 2 Cube API Instances are included. That said, it +is very common to use more, and additional Cube API instances can be added to +your deployment to increase the throughput of your queries. A rough estimate is +that 1 Cube API Instance is needed for every 5-10 requests-per-second served. +Cube API Instances can also auto-scale as needed. + +To change how many Cube API instances are available in the Production Cluster, +go to the deployment’s Settings screen, and open +the Configuration tab. From this screen, you can set the minimum +and maximum number of Cube API instances for a deployment: + + + +## Switching between deployment types + +To switch a deployment's type, go to the deployment's Settings screen +and select from the available options: + + + +[cube-contact-us]: https://cube.dev/contact +[ref-conf-ref-ctx-to-app-id]: /config#options-reference-context-to-app-id diff --git a/docs/content/Deployment/Cloud/Overview.mdx b/docs/content/Deployment/Cloud/Overview.mdx new file mode 100644 index 0000000000000..30eb723706b2c --- /dev/null +++ b/docs/content/Deployment/Cloud/Overview.mdx @@ -0,0 +1,41 @@ +--- +title: Overview +permalink: /deployment/platforms/cube-cloud +category: Deployment +subCategory: Cube Cloud +menuOrder: 2 +--- + +This guide walks you through deploying Cube with [Cube Cloud][link-cube-cloud], +a purpose-built platform to run Cube applications in production. It is made by +the creators of Cube and incorporates all the best practices of running and +scaling Cube applications. + + + +Are you moving from a self-hosted installation to Cube Cloud? [Click +here][blog-migrate-to-cube-cloud] to learn more. + + + +
+ +
+ +## Prerequisites + +- A Cube Cloud account + +## Configuration + +Learn more about [deployment with Cube Cloud here][ref-cubecloud-getstart]. + +[blog-migrate-to-cube-cloud]: + https://cube.dev/blog/migrating-from-self-hosted-to-cube-cloud/ +[link-cube-cloud]: https://cubecloud.dev +[link-cube-cloud-waitlist]: https://cube.dev/cloud +[ref-cubecloud-getstart]: /getting-started/cloud/overview diff --git a/docs/content/Deployment/Cloud/Pricing.mdx b/docs/content/Deployment/Cloud/Pricing.mdx new file mode 100644 index 0000000000000..b8eba5c48b539 --- /dev/null +++ b/docs/content/Deployment/Cloud/Pricing.mdx @@ -0,0 +1,168 @@ +--- +title: Pricing +permalink: /cloud/pricing +category: Deployment +subCategory: Cube Cloud +menuOrder: 6 +--- + +Cube Cloud pricing is based on consumption of compute power which we measure +using Cube Compute Units. The units are priced based on which subscription plan +you have chosen. Each plan has different features and functionality that you +should review as you think about what is right for your business. + + + +A **Cube Compute Unit** is a way to measure compute power used to run Cube Cloud +infrastructure. The price of a compute unit is determined by the Cube Cloud plan +you're subscribed to. + + + +| Node Type | Compute Unit per hour | +| ------------------------- | :-------------------: | +| Cube Production Cluster | `4` | +| Cube Development Instance | `1` | +| Cube Store Worker | `1` | +| Cube API Instance | `1` | + +Cube Cloud has four subscription plans that you can review +[here](https://cube.dev/pricing). The subscription plan you choose will +determine the price per Cube Compute Unit (CCU). Each subscription plan +determines the features, scalability, availability, as well as the speed and +scope of support you may receive for your deployment. + +## Starter + +The Starter plan starts at a minimum of $99/month and targets low-scale +production that is not business-critical. + +It offers a [Production Cluster][ref-cloud-deployment-prod-cluster], the ability +to use third-party packages from the npm registry, AWS and GCP support in select +regions, pre-aggregations of upto 150GB in size, [alerts][ref-cloud-alerts], +auto-suspend controls and a 50,000 queries per day limit. + +### Support + +The Starter plan includes support via the community Slack channel and in-product +chat support with three questions per week. + +| Priority | Response Time | +| -------- | ---------------- | +| P0 | 8 business hours | +| P1 | 8 business hours | +| P2 | 8 business hours | +| P3 | 2 business days | + +## Premium + +The Premium plan starts at a minimum of $12,000/year, and is designed for basic +small-scale production deployments. + +It offers everything in the [Starter plan](#starter) as well as Okta support for +single sign-on, AWS and GCP support in all regions, and unlimited queries. + +### Support + +The Premium plan includes support via the community Slack channel and unlimited +in-product chat support on weekdays. Cube Cloud provides a 99.95% uptime SLA for +this plan. + +| Priority | Response Time | +| -------- | ---------------- | +| P0 | 60 minutes | +| P1 | 4 hours | +| P2 | 8 business hours | +| P3 | 2 business days | + +## Enterprise + +The Enterprise plan starts at a minimum of $18,000/year, and is suitable for +high-scale or mission-critical production deployments with more significant +security and compliance needs. + +It offers everything in the [Premium plan](#premium) as well as SAML/LDAP +support for single sign-on, Azure support for all regions, [VPC +peering][ref-cloud-vpc-peering], [logs/metrics export][ref-cloud-log-export], +and [role-based access control][ref-cloud-acl]. + +### Support + +The Enterprise plan offers improved support response times as compared to the +[Premium plan](#premium-support) with the addition of a dedicated CSM. Cube +Cloud provides a 99.99% uptime SLA for this plan. + +| Priority | Response Time | +| -------- | ---------------- | +| P0 | 30 minutes | +| P1 | 2 hours | +| P2 | 8 business hours | +| P3 | 2 business days | + +## Enterprise Premier + +The Enterprise Premium plan caters to high-scale, high-availability +mission-critical production deployments with security and compliance needs. + +It offers everything in the [Enterprise plan](#enterprise) as well as enabling +the use of [Production Multi-Clusters][ref-cloud-deployment-prod-multicluster], +unlimited pre-aggregation sizes, support for kSQL/Elasticsearch and [custom +domains][ref-cloud-custom-domains]. + +### Support + +The Enterprise Premier plan includes the same +[support plan as the Enterprise plan](#enterprise-support). Cube Cloud provides +a 99.995% uptime SLA for this plan. + +| Priority | Response Time | +| -------- | ---------------- | +| P0 | 30 minutes | +| P1 | 2 hours | +| P2 | 8 business hours | +| P3 | 2 business days | + +## Support Priority Definitions + +We prioritize support requests based on their severity, as follows: + +- **P0**: The platform is severely impacted or completely shut down. We will + assign specialists to work continuously to fix the issue, provide ongoing + updates, and start working on a temporary workaround or fix. + +- **P1**: The platform is functioning with limited capabilities or facing + critical issues preventing a production deployment. We will assign specialists + to fix the issue, provide ongoing updates, and start working on a temporary + workaround or fix. + +- **P2**: There are issues with workaround solutions or non-critical functions. + We will use resources during local business hours until the issue is resolved + or a workaround is in place. + +- **P3**: There is a need for clarification in the documentation, or a + suggestion for product enhancement. We will triage the request, provide + clarification when possible, and may include a resolution in a future update. + +## Payment Terms + +Future purchases and upgrades are subject to the pricing that is in effect at +the time of the order. Payments are non-refundable. If your usage of resources +exceeds the balance of CCUs in your account, services may degrade or be +suspended until new CCUs are purchased. You may upgrade your CCUs to a +higher-level subscription plan at any time by paying the difference in per-Cube +Compute Unit pricing, or by asking to convert the price paid for the remaining +CCUs into CCUs for the higher subscription plan at the CCU pricing for that plan +(resulting in a lower number of available CCUs but upgraded to the higher +subscription plan). Future purchases and upgrades are subject to the pricing +that is in effect at the time of the order. No credit is allowed for downgrading +CCUs to a lower subscription plan level. + +[ref-cloud-deployment-prod-cluster]: + /cloud/configuration/deployment-types#production-cluster +[ref-cloud-alerts]: /cloud/workspace/alerts +[ref-cloud-log-export]: /cloud/workspace/logs +[ref-cloud-acl]: /cloud/access-control/ +[ref-cloud-deployment-prod-multicluster]: + /cloud/configuration/deployment-types#production-multi-cluster +[ref-cloud-custom-domains]: /cloud/configuration/custom-domains +[ref-cloud-vpc-peering]: /cloud/configuration/connecting-with-a-vpc diff --git a/docs/content/Deployment/Core/Overview.mdx b/docs/content/Deployment/Core/Overview.mdx new file mode 100644 index 0000000000000..5b0d5e9e706c1 --- /dev/null +++ b/docs/content/Deployment/Core/Overview.mdx @@ -0,0 +1,352 @@ +--- +title: Overview +permalink: /deployment/platforms/docker +category: Deployment +subCategory: Cube Core +menuOrder: 11 +--- + +This guide walks you through deploying Cube with Docker. + + + +This is an example of a production-ready deployment, but real-world deployments +can vary significantly depending on desired performance and scale. + + + +## Prerequisites + +- [Docker Desktop][link-docker-app] + +## Configuration + +Create a Docker Compose stack by creating a `docker-compose.yml`. A +production-ready stack would at minimum consist of: + +- One or more Cube API instance +- A Cube Refresh Worker +- A Cube Store Router node +- One or more Cube Store Worker nodes + +An example stack using BigQuery as a data source is provided below: + + + +Using macOS or Windows? Use `CUBEJS_DB_HOST=host.docker.internal` instead of +`localhost` if your database is on the same machine. + + + +```yaml +version: '2.2' + +services: + cube_api: + restart: always + image: cubejs/cube:v%CURRENT_VERSION + ports: + - 4000:4000 + environment: + - CUBEJS_DB_TYPE=bigquery + - CUBEJS_DB_BQ_PROJECT_ID=cubejs-bq-cluster + - CUBEJS_DB_BQ_CREDENTIALS= + - CUBEJS_DB_EXPORT_BUCKET=cubestore + - CUBEJS_CUBESTORE_HOST=cubestore_router + - CUBEJS_API_SECRET=secret + volumes: + - .:/cube/conf + depends_on: + - cube_refresh_worker + - cubestore_router + - cubestore_worker_1 + - cubestore_worker_2 + + cube_refresh_worker: + restart: always + image: cubejs/cube:v%CURRENT_VERSION + environment: + - CUBEJS_DB_TYPE=bigquery + - CUBEJS_DB_BQ_PROJECT_ID=cubejs-bq-cluster + - CUBEJS_DB_BQ_CREDENTIALS= + - CUBEJS_DB_EXPORT_BUCKET=cubestore + - CUBEJS_CUBESTORE_HOST=cubestore_router + - CUBEJS_API_SECRET=secret + - CUBEJS_REFRESH_WORKER=true + volumes: + - .:/cube/conf + + cubestore_router: + restart: always + image: cubejs/cubestore:v%CURRENT_VERSION + environment: + - CUBESTORE_WORKERS=cubestore_worker_1:10001,cubestore_worker_2:10002 + - CUBESTORE_REMOTE_DIR=/cube/data + - CUBESTORE_META_PORT=9999 + - CUBESTORE_SERVER_NAME=cubestore_router:9999 + volumes: + - .cubestore:/cube/data + + cubestore_worker_1: + restart: always + image: cubejs/cubestore:v%CURRENT_VERSION + environment: + - CUBESTORE_WORKERS=cubestore_worker_1:10001,cubestore_worker_2:10002 + - CUBESTORE_SERVER_NAME=cubestore_worker_1:10001 + - CUBESTORE_WORKER_PORT=10001 + - CUBESTORE_REMOTE_DIR=/cube/data + - CUBESTORE_META_ADDR=cubestore_router:9999 + volumes: + - .cubestore:/cube/data + depends_on: + - cubestore_router + + cubestore_worker_2: + restart: always + image: cubejs/cubestore:v%CURRENT_VERSION + environment: + - CUBESTORE_WORKERS=cubestore_worker_1:10001,cubestore_worker_2:10002 + - CUBESTORE_SERVER_NAME=cubestore_worker_2:10002 + - CUBESTORE_WORKER_PORT=10002 + - CUBESTORE_REMOTE_DIR=/cube/data + - CUBESTORE_META_ADDR=cubestore_router:9999 + volumes: + - .cubestore:/cube/data + depends_on: + - cubestore_router +``` + +## Set up reverse proxy + +In production, the Cube API should be served over an HTTPS connection to +ensure security of the data in-transit. We recommend using a reverse proxy; as +an example, let's use [NGINX][link-nginx]. + + + +You can also use a reverse proxy to enable HTTP 2.0 and GZIP compression + + + +First we'll create a new server configuration file called `nginx/cube.conf`: + +```nginx +server { + listen 443 ssl; + server_name cube.my-domain.com; + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ecdh_curve secp384r1; + # Replace the ciphers with the appropriate values + ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384 OLD_TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 OLD_TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256"; + ssl_prefer_server_ciphers on; + ssl_certificate /etc/ssl/private/cert.pem; + ssl_certificate_key /etc/ssl/private/key.pem; + ssl_session_timeout 10m; + ssl_session_cache shared:SSL:10m; + ssl_session_tickets off; + ssl_stapling on; + ssl_stapling_verify on; + + location / { + proxy_pass http://cube:4000/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } +} +``` + +Then we'll add a new service to our Docker Compose stack: + +```yaml +services: + ... + nginx: + image: nginx + ports: + - 443:443 + volumes: + - ./nginx:/etc/nginx/conf.d + - ./ssl:/etc/ssl/private +``` + +Don't forget to create a `ssl` directory with the `cert.pem` and `key.pem` files +inside so the Nginx service can find them. + +For automatically provisioning SSL certificates with LetsEncrypt, [this blog +post][medium-letsencrypt-nginx] may be useful. + +## Security + +### <--{"id" : "Security"}--> Use JSON Web Tokens + +Cube can be configured to use industry-standard JSON Web Key Sets for +securing its API and limiting access to data. To do this, we'll define the +relevant options on our Cube API instance: + + + +If you're using [`queryRewrite`][ref-config-queryrewrite] for access control, +then you must also configure +[`scheduledRefreshContexts`][ref-config-sched-ref-ctx] so the refresh workers +can correctly create pre-aggregations. + + + +```yaml +services: + cube_api: + image: cubejs/cube:v%CURRENT_VERSION + ports: + - 4000:4000 + environment: + - CUBEJS_DB_TYPE=bigquery + - CUBEJS_DB_BQ_PROJECT_ID=cubejs-bq-cluster + - CUBEJS_DB_BQ_CREDENTIALS= + - CUBEJS_DB_EXPORT_BUCKET=cubestore + - CUBEJS_CUBESTORE_HOST=cubestore_router + - CUBEJS_API_SECRET=secret + - CUBEJS_JWK_URL=https://cognito-idp..amazonaws.com//.well-known/jwks.json + - CUBEJS_JWT_AUDIENCE= + - CUBEJS_JWT_ISSUER=https://cognito-idp..amazonaws.com/ + - CUBEJS_JWT_ALGS=RS256 + - CUBEJS_JWT_CLAIMS_NAMESPACE= + volumes: + - .:/cube/conf + depends_on: + - cubestore_worker_1 + - cubestore_worker_2 + - cube_refresh_worker +``` + +### <--{"id" : "Security"}--> Securing Cube Store + +All Cube Store nodes (both router and workers) should only be accessible to +Cube API instances and refresh workers. To do this with Docker Compose, we +simply need to make sure that none of the Cube Store services have any exposed + +## Monitoring + +All Cube logs can be found by through the Docker Compose CLI: + +```bash{outputLines: 2-9,11-18} +docker-compose ps + + Name Command State Ports +--------------------------------------------------------------------------------------------------------------------------------- +cluster_cube_1 docker-entrypoint.sh cubej ... Up 0.0.0.0:4000->4000/tcp,:::4000->4000/tcp +cluster_cubestore_router_1 ./cubestored Up 3030/tcp, 3306/tcp +cluster_cubestore_worker_1_1 ./cubestored Up 3306/tcp, 9001/tcp +cluster_cubestore_worker_2_1 ./cubestored Up 3306/tcp, 9001/tcp + +docker-compose logs + +cubestore_router_1 | 2021-06-02 15:03:20,915 INFO [cubestore::metastore] Creating metastore from scratch in /cube/.cubestore/data/metastore +cubestore_router_1 | 2021-06-02 15:03:20,950 INFO [cubestore::cluster] Meta store port open on 0.0.0.0:9999 +cubestore_router_1 | 2021-06-02 15:03:20,951 INFO [cubestore::mysql] MySQL port open on 0.0.0.0:3306 +cubestore_router_1 | 2021-06-02 15:03:20,952 INFO [cubestore::http] Http Server is listening on 0.0.0.0:3030 +cube_1 | 🚀 Cube API server (%CURRENT_VERSION) is listening on 4000 +cubestore_worker_2_1 | 2021-06-02 15:03:24,945 INFO [cubestore::cluster] Worker port open on 0.0.0.0:9001 +cubestore_worker_1_1 | 2021-06-02 15:03:24,830 INFO [cubestore::cluster] Worker port open on 0.0.0.0:9001 +``` + +## Update to the latest version + +Find the latest stable release version (currently `v%CURRENT_VERSION`) [from +Docker Hub][link-cubejs-docker]. Then update your `docker-compose.yml` to use +the tag: + +```yaml +version: '2.2' + +services: + cube_api: + image: cubejs/cube:v%CURRENT_VERSION + ports: + - 4000:4000 + environment: + - CUBEJS_DB_TYPE=bigquery + - CUBEJS_DB_BQ_PROJECT_ID=cubejs-bq-cluster + - CUBEJS_DB_BQ_CREDENTIALS= + - CUBEJS_DB_EXPORT_BUCKET=cubestore + - CUBEJS_CUBESTORE_HOST=cubestore_router + - CUBEJS_API_SECRET=secret + volumes: + - .:/cube/conf + depends_on: + - cubestore_router + - cube_refresh_worker +``` + +## Extend the Docker image + +If you need to use npm packages with native extensions inside [the `cube.js` +configuration file][ref-config-js], you'll need to build your own Docker image. +You can do this by first creating a `Dockerfile` and a corresponding +`.dockerignore`: + +```bash{promptUser: user} +touch Dockerfile +touch .dockerignore +``` + +Add this to the `Dockerfile`: + +```dockerfile +FROM cubejs/cube:latest + +COPY . . +RUN npm install +``` + +And this to the `.dockerignore`: + +```gitignore +node_modules +npm-debug.log +schema +cube.js +.env +``` + +Then start the build process by running the following command: + +```bash{promptUser: user} +docker build -t /cubejs-custom-build . +``` + +Finally, update your `docker-compose.yml` to use your newly-built image: + +```yaml +version: '2.2' + +services: + cube_api: + image: /cubejs-custom-build + ports: + - 4000:4000 + environment: + - CUBEJS_DB_TYPE=bigquery + - CUBEJS_DB_BQ_PROJECT_ID=cubejs-bq-cluster + - CUBEJS_DB_BQ_CREDENTIALS= + - CUBEJS_DB_EXPORT_BUCKET=cubestore + - CUBEJS_CUBESTORE_HOST=cubestore_router + - CUBEJS_API_SECRET=secret + volumes: + - .:/cube/conf + # Prevent dev dependencies leaking + - .empty:/cube/conf/node_modules/@cubejs-backend/ + depends_on: + - cubestore_router + - cube_refresh_worker +``` + +[medium-letsencrypt-nginx]: + https://pentacent.medium.com/nginx-and-lets-encrypt-with-docker-in-less-than-5-minutes-b4b8a60d3a71 +[link-cubejs-docker]: https://hub.docker.com/r/cubejs/cube +[link-docker-app]: https://www.docker.com/products/docker-app +[link-nginx]: https://www.nginx.com/ +[ref-config-js]: /config +[ref-config-queryrewrite]: /config#query-rewrite +[ref-config-sched-ref-ctx]: /config#scheduled-refresh-contexts diff --git a/docs/content/Deployment/Overview.mdx b/docs/content/Deployment/Overview.mdx index c09620c8d6a14..f257cd6c606ae 100644 --- a/docs/content/Deployment/Overview.mdx +++ b/docs/content/Deployment/Overview.mdx @@ -9,23 +9,26 @@ redirect_from: - /deployment/guide --- -This section contains a general overview of deploying Cube.js cluster in -production. You can find platform-specific guides for: +This section contains a general overview of deploying a Cube cluster in +production. You can also check platform-specific guides for [Cube +Cloud][ref-deploy-cubecloud] and [Docker][ref-deploy-docker]. -- [Cube Cloud][ref-deploy-cubecloud] -- [Docker][ref-deploy-docker] -- [Kubernetes][ref-deploy-k8s] - -If you are moving Cube.js to production, check out the [Production +If you are moving Cube to production, check out the [Production Checklist][ref-deploy-prod-list]. -As shown in the diagram below, a typical production Cube.js cluster consists of -one or multiple API instances, a Refresh Worker, Redis and a Cube Store cluster. +## Components + +As shown in the diagram below, a typical production deployment of Cube includes +the following components: + +- One or multiple API instances +- A Refresh Worker +- A Cube Store cluster
Deployment Overview @@ -37,167 +40,69 @@ Worker** builds and refreshes pre-aggregations in the background. **Cube Store** ingests pre-aggregations built by Refresh Worker and responds to queries from API instances. - **Redis** is used to manage the queue and query-level cache. Please note, we -are planning to [replace Redis with Cube Store][replace-redis] for in-memory cache and queue management in late -2022. - -API instances and Refresh Worker can be configured via [environment -variables][ref-config-env] or [cube.js configuration file][ref-config-js]. They -also need access to the data schema files. - -Cube Store cluster can be configured via environment variables. - -Below you can find an example Docker Compose configuration for a Cube.js -cluster: - -```yaml -version: '2.2' - -services: - cube_api: - image: cubejs/cube - ports: - - 4000:4000 - environment: - - CUBEJS_DB_TYPE=bigquery - - CUBEJS_DB_BQ_PROJECT_ID=cubejs-k8s-cluster - - CUBEJS_DB_BQ_CREDENTIALS= - - CUBEJS_DB_EXPORT_BUCKET=cubestore - - - CUBEJS_CUBESTORE_HOST=cubestore_router - - - CUBEJS_REDIS_URL=redis://redis:6379 - - - CUBEJS_API_SECRET=secret - volumes: - - .:/cube/conf - depends_on: - - cubestore_worker_1 - - cubestore_worker_2 - - cube_refresh_worker - - redis - - cube_refresh_worker: - image: cubejs/cube - environment: - - CUBEJS_DB_TYPE=bigquery - - CUBEJS_DB_BQ_PROJECT_ID=cubejs-k8s-cluster - - CUBEJS_DB_BQ_CREDENTIALS= - - CUBEJS_DB_EXPORT_BUCKET=cubestore - - - CUBEJS_CUBESTORE_HOST=cubestore_router - - - CUBEJS_REDIS_URL=redis://redis:6379 - - - CUBEJS_API_SECRET=secret - - - CUBEJS_REFRESH_WORKER=true - volumes: - - .:/cube/conf - - cubestore_router: - image: cubejs/cubestore:latest - environment: - - CUBESTORE_WORKERS=cubestore_worker_1:10001,cubestore_worker_2:10002 - - CUBESTORE_REMOTE_DIR=/cube/data - - CUBESTORE_META_PORT=9999 - - CUBESTORE_SERVER_NAME=cubestore_router:9999 - volumes: - - .cubestore:/cube/data - - cubestore_worker_1: - image: cubejs/cubestore:latest - environment: - - CUBESTORE_WORKERS=cubestore_worker_1:10001,cubestore_worker_2:10002 - - CUBESTORE_SERVER_NAME=cubestore_worker_1:10001 - - CUBESTORE_WORKER_PORT=10001 - - CUBESTORE_REMOTE_DIR=/cube/data - - CUBESTORE_META_ADDR=cubestore_router:9999 - volumes: - - .cubestore:/cube/data - depends_on: - - cubestore_router - - cubestore_worker_2: - image: cubejs/cubestore:latest - environment: - - CUBESTORE_WORKERS=cubestore_worker_1:10001,cubestore_worker_2:10002 - - CUBESTORE_SERVER_NAME=cubestore_worker_2:10002 - - CUBESTORE_WORKER_PORT=10002 - - CUBESTORE_REMOTE_DIR=/cube/data - - CUBESTORE_META_ADDR=cubestore_router:9999 - volumes: - - .cubestore:/cube/data - depends_on: - - cubestore_router +API instances and Refresh Workers can be configured via [environment +variables][ref-config-env] or the [`cube.js` configuration file][ref-config-js]. +They also need access to the data model files. Cube Store clusters can be +configured via environment variables. - redis: - image: bitnami/redis:latest - environment: - - ALLOW_EMPTY_PASSWORD=yes - logging: - driver: none -``` +You can find an example Docker Compose configuration for a Cube deployment in +the platform-specific guide for [Docker][ref-deploy-docker]. -## API Instance +## API instances -API instances process incoming API requests and query either Cube Store -for pre-aggregated data or connected data sources for raw data. It is possible to +API instances process incoming API requests and query either Cube Store for +pre-aggregated data or connected data sources for raw data. It is possible to horizontally scale API instances and use a load balancer to balance incoming requests between multiple API instances. The [Cube Docker image][dh-cubejs] is used for API Instance. -API instance needs to be configured via environment variables, cube.js file and -has access to the data schema files. +API instances can be configured via environment variables or the `cube.js` +configuration file, and **must** have access to the data model files (as +specified by [`schemaPath`][ref-conf-ref-schemapath]. ## Refresh Worker -A Refresh Worker updates pre-aggregations and the in-memory cache in the -background. They also keep the refresh keys up-to-date for all defined schemas -and pre-aggregations. +A Refresh Worker updates pre-aggregations and invalidates the in-memory cache in +the background. They also keep the refresh keys up-to-date for all data models +and pre-aggregations. Please note that the in-memory cache is just invalidated +but not populated by Refresh Worker. In-memory cache is populated lazily during +querying. On the other hand, pre-aggregations are eagerly populated and kept +up-to-date by Refresh Worker. -[Cube.js Docker image][dh-cubejs] can be used for creating Refresh Workers; to +The [Cube Docker image][dh-cubejs] can be used for creating Refresh Workers; to make the service act as a Refresh Worker, `CUBEJS_REFRESH_WORKER=true` should be set in the environment variables. ## Cube Store -Cube Store is the purpose-built pre-aggregations storage for Cube.js. +Cube Store is the purpose-built pre-aggregations storage for Cube. Cube Store uses a distributed query engine architecture. In every Cube Store cluster: -- a one or many router nodes handle incoming connections, manages database - metadata, builds query plans, and orchestrates their execution -- multiple worker nodes ingest warmed up data and execute queries in parallel +- a one or many [router nodes](#cube-store-cube-store-router) handle incoming + connections, manages database metadata, builds query plans, and orchestrates + their execution +- multiple [worker nodes](#cube-store-cube-store-worker) ingest warmed up data + and execute queries in parallel - a local or cloud-based blob storage keeps pre-aggregated data in columnar format -![](https://cubedev-blog-images.s3.us-east-2.amazonaws.com/db0e1aeb-3101-4280-b4a4-902e21bcd9a0.png) - -More information on Cube Store architecture can be found in [this presentation](https://docs.google.com/presentation/d/1oQ-koloag0UcL-bUHOpBXK4txpqiGl41rxhgDVrw7gw/). - -By default, Cube Store listens on the port `3030` for queries coming from -Cube.js. The port could be changed by setting `CUBESTORE_HTTP_PORT` environment -variable. In a case of using custom port, please make sure to change -`CUBEJS_CUBESTORE_PORT` environment variable for Cube.js API Instances and -Refresh Worker. - -### <--{"id" : "Cube Store"}--> Scaling - -Although Cube Store _can_ be run in single-instance mode, this is often -unsuitable for production deployments. For high concurrency and data throughput, -we **strongly** recommend running Cube Store as a cluster of multiple instances -instead. Because the storage layer is decoupled from the query processing -engine, you can horizontally scale your Cube Store cluster for as much -concurrency as you require. - -Cube Store has two "kinds" of nodes: +
+ Cube Store Router with two Cube Store Workers +
-- The **router** node handles incoming client connections, manages database - metadata and serves simple queries -- Multiple **worker** nodes which execute SQL queries received from Cube.js +By default, Cube Store listens on the port `3030` for queries coming from Cube. +The port could be changed by setting `CUBESTORE_HTTP_PORT` environment variable. +In a case of using custom port, please make sure to change +`CUBEJS_CUBESTORE_PORT` environment variable for Cube API Instances and Refresh +Worker. Both the router and worker use the [Cube Store Docker image][dh-cubestore]. The following environment variables should be used to manage the roles: @@ -210,6 +115,61 @@ following environment variables should be used to manage the roles: | `CUBESTORE_WORKER_PORT` | - | Yes | | `CUBESTORE_META_ADDR` | - | Yes | + + +Looking for a deeper dive on Cube Store architecture? Check out +[this presentation](https://docs.google.com/presentation/d/1oQ-koloag0UcL-bUHOpBXK4txpqiGl41rxhgDVrw7gw/) +by our CTO, [Pavel][gh-pavel]. + + + +### <--{"id" : "Cube Store"}--> Cube Store Router + +
+ Cube Store Router with two Cube Store Workers +
+ +The Router in a Cube Store cluster is responsible for receiving queries from +Cube, managing metadata for the Cube Store cluster, and query planning and +distribution for the Workers. It also [provides a MySQL-compatible +interface][ref-caching-inspect-sql] that can be used to query pre-aggregations +from Cube Store directly. Cube **only** communicates with the Router, and does +not interact with Workers directly. + +[ref-caching-inspect-sql]: + /caching/using-pre-aggregations#inspecting-pre-aggregations + +### <--{"id" : "Cube Store"}--> Cube Store Worker + +
+ Cube Store Router with two Cube Store Workers +
+ +Workers in a Cube Store cluster receive and execute subqueries from the Router, +and directly interact with the underlying distributed storage for insertions, +selections and pre-aggregation warmup. Workers **do not** interact with each +other directly, and instead rely on the Router to distribute queries and manage +any associated metadata. + +### <--{"id" : "Cube Store"}--> Scaling + +Although Cube Store _can_ be run in single-instance mode, this is often +unsuitable for production deployments. For high concurrency and data throughput, +we **strongly** recommend running Cube Store as a cluster of multiple instances +instead. Because the storage layer is decoupled from the query processing +engine, you can horizontally scale your Cube Store cluster for as much +concurrency as you require. + A sample Docker Compose stack setting Cube Store cluster up might look like: ```yaml @@ -256,7 +216,8 @@ services: Cube Store makes use of a separate storage layer for storing metadata as well as for persisting pre-aggregations as Parquet files. Cube Store can use both AWS S3 -and Google Cloud, or if desired, a local path on the server if all nodes of a cluster run on a single machine. +and Google Cloud, or if desired, a local path on the server if all nodes of a +cluster run on a single machine. A simplified example using AWS S3 might look like: @@ -290,71 +251,35 @@ services: ## Redis - - -Cube Store [will replace -Redis][replace-redis] for in-memory cache and queue management in late -2022. - - - -Cube.js uses Redis, an in-memory data structure store, for query caching and -queue. - -Set the `CUBEJS_REDIS_URL` environment variable to allow Cube.js API Instances -and Refresh Worker to connect to Redis. If your Redis instance also has a -password, please set it via the `CUBEJS_REDIS_PASSWORD` environment variable. -Set the `CUBEJS_REDIS_TLS` environment variable to true if you want to enable -SSL-secured connections. Ensure your Redis cluster allows at least 15 concurrent -connections. - -### <--{"id" : "Redis"}--> Redis Sentinel - -Cube.js supports Redis Sentinel via `CUBEJS_REDIS_USE_IOREDIS=true` environment -variable. Then set `CUBEJS_REDIS_URL` to the -`redis+sentinel://localhost:26379,otherhost:26479/mymaster/5` to allow Cube.js -to connect to the Redis Sentinel. - -Cube.js server instances used by same tenant environments should have same Redis -instances. Otherwise they will have different query queues which can lead to -incorrect pre-aggregation states and intermittent data access errors. - -### <--{"id" : "Redis"}--> Redis Pool - -If `CUBEJS_REDIS_URL` is provided Cube.js, will create a Redis connection pool -with a minimum of 2 and maximum of 1000 concurrent connections, by default. The -`CUBEJS_REDIS_POOL_MIN` and `CUBEJS_REDIS_POOL_MAX` environment variables can be -used to tweak pool size limits. To disable connection pooling, and instead -create connections on-demand, you can set `CUBEJS_REDIS_POOL_MAX` to 0. - -If your maximum concurrent connections limit is too low, you may see -`TimeoutError: ResourceRequest timed out` errors. As a rule of a thumb, you need -to have `Queue Size * Number of tenants` concurrent connections to ensure the -best performance possible. If you use clustered deployments, please make sure -you have enough connections for all Cube.js server instances. A lower number of -connections still can work, however Redis becomes a performance bottleneck in -this case. - -### <--{"id" : "Redis"}--> Running without Redis - -If you want to run Cube.js in production without Redis, you can use -`CUBEJS_CACHE_AND_QUEUE_DRIVER` environment variable to `memory`. +Earlier, [Redis][redis] was used in production deployments as storage for +in-memory cache and query queue. Since version v0.32.0, Cube Store is used for +that purpose. It is still possible to [configure][ref-config-redis] Cube to use +Redis; however, it is strongly not recommended. Please check the [blog +post][blog-details] for details. -Clustered deployments can't be run without Redis as it is used to -manage the query queue. +Redis support is deprecated and will be removed from Cube in the future. Upgrade +to v0.32.0 or later to use Cube Store instead of Redis. See the [migration +guide][blog-migration-guide]. [dh-cubejs]: https://hub.docker.com/r/cubejs/cube [dh-cubestore]: https://hub.docker.com/r/cubejs/cubestore +[gh-cube-examples-k8s]: + https://github.com/cube-js/cube/tree/master/examples/kubernetes +[gh-cube-examples-k8s-helm]: + https://github.com/cube-js/cube/tree/master/examples/helm-charts [ref-deploy-prod-list]: /deployment/production-checklist [ref-deploy-cubecloud]: /deployment/platforms/cube-cloud [ref-deploy-docker]: /deployment/platforms/docker -[ref-deploy-k8s]: /deployment/platforms/kubernetes -[ref-deploy-sls-aws]: /deployment/platforms/serverless/aws -[ref-deploy-sls-gcp]: /deployment/platforms/serverless/google-cloud [ref-config-env]: /reference/environment-variables [ref-config-js]: /config -[replace-redis]: https://cube.dev/blog/replacing-redis-with-cube-store +[ref-conf-ref-schemapath]: /config#options-reference-schema-path +[redis]: https://redis.io +[ref-config-redis]: /reference/environment-variables#cubejs-redis-password +[blog-details]: https://cube.dev/blog/how-you-win-by-using-cube-store-part-1 +[blog-migration-guide]: + https://cube.dev/blog/how-you-win-by-using-cube-store-part-1#how-to-migrate-to-cube-store +[gh-pavel]: https://github.com/paveltiunov diff --git a/docs/content/Deployment/Platform-Cube-Cloud.mdx b/docs/content/Deployment/Platform-Cube-Cloud.mdx deleted file mode 100644 index be962e7d7ac8b..0000000000000 --- a/docs/content/Deployment/Platform-Cube-Cloud.mdx +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: Deploy to Cube Cloud -menuTitle: Cube Cloud -permalink: /deployment/platforms/cube-cloud -category: Deployment -subCategory: Platforms -menuOrder: 1 ---- - -This guide walks you through deploying Cube with [Cube Cloud][link-cube-cloud], -a purpose-built platform to run Cube applications in production. It is made by -the creators of Cube and incorporates all the best practices of running and -scaling Cube applications. - - - -Are you moving from a self-hosted installation to Cube Cloud? [Click -here][blog-migrate-to-cube-cloud] to learn more. - - - -
- -
- -## Prerequisites - -- A Cube Cloud account - -## Configuration - -Learn more about [deployment with Cube Cloud here][ref-cubecloud-getstart]. - -[blog-migrate-to-cube-cloud]: - https://cube.dev/blog/migrating-from-self-hosted-to-cube-cloud/ -[link-cube-cloud]: https://cubecloud.dev -[link-cube-cloud-waitlist]: https://cube.dev/cloud -[ref-cubecloud-getstart]: /cloud/getting-started diff --git a/docs/content/Deployment/Platform-Docker.mdx b/docs/content/Deployment/Platform-Docker.mdx deleted file mode 100644 index 502618a64d9ec..0000000000000 --- a/docs/content/Deployment/Platform-Docker.mdx +++ /dev/null @@ -1,370 +0,0 @@ ---- -title: Deploy to Docker -menuTitle: Docker -permalink: /deployment/platforms/docker -category: Deployment -subCategory: Platforms -menuOrder: 2 ---- - -This guide walks you through deploying Cube.js with Docker. - - - -This is an example of a production-ready deployment, but real-world deployments -can vary significantly depending on desired performance and scale. - - - -## Prerequisites - -- [Docker Desktop][link-docker-app] - -## Configuration - -Create a Docker Compose stack by creating a `docker-compose.yml`. A -production-ready stack would at minimum consist of: - -- One or more Cube.js API instance -- A Cube.js Refresh Worker -- A Cube Store Router node -- One or more Cube Store Worker nodes -- A Redis instance - -An example stack using BigQuery as a data source is provided below: - - - -Using macOS or Windows? Use `CUBEJS_DB_HOST=host.docker.internal` instead of -`localhost` if your database is on the same machine. - - - -```yaml -version: '2.2' - -services: - cube_api: - restart: always - image: cubejs/cube:v%CURRENT_VERSION - ports: - - 4000:4000 - environment: - - CUBEJS_DB_TYPE=bigquery - - CUBEJS_DB_BQ_PROJECT_ID=cubejs-bq-cluster - - CUBEJS_DB_BQ_CREDENTIALS= - - CUBEJS_DB_EXPORT_BUCKET=cubestore - - CUBEJS_CUBESTORE_HOST=cubestore_router - - CUBEJS_REDIS_URL=redis://redis:6379 - - CUBEJS_API_SECRET=secret - volumes: - - .:/cube/conf - depends_on: - - cubestore_worker_1 - - cubestore_worker_2 - - cube_refresh_worker - - redis - - cube_refresh_worker: - restart: always - image: cubejs/cube:v%CURRENT_VERSION - environment: - - CUBEJS_DB_TYPE=bigquery - - CUBEJS_DB_BQ_PROJECT_ID=cubejs-bq-cluster - - CUBEJS_DB_BQ_CREDENTIALS= - - CUBEJS_DB_EXPORT_BUCKET=cubestore - - CUBEJS_CUBESTORE_HOST=cubestore_router - - CUBEJS_REDIS_URL=redis://redis:6379 - - CUBEJS_API_SECRET=secret - - CUBEJS_REFRESH_WORKER=true - volumes: - - .:/cube/conf - - cubestore_router: - restart: always - image: cubejs/cubestore:v%CURRENT_VERSION - environment: - - CUBESTORE_WORKERS=cubestore_worker_1:10001,cubestore_worker_2:10002 - - CUBESTORE_REMOTE_DIR=/cube/data - - CUBESTORE_META_PORT=9999 - - CUBESTORE_SERVER_NAME=cubestore_router:9999 - volumes: - - .cubestore:/cube/data - - cubestore_worker_1: - restart: always - image: cubejs/cubestore:v%CURRENT_VERSION - environment: - - CUBESTORE_WORKERS=cubestore_worker_1:10001,cubestore_worker_2:10002 - - CUBESTORE_SERVER_NAME=cubestore_worker_1:10001 - - CUBESTORE_WORKER_PORT=10001 - - CUBESTORE_REMOTE_DIR=/cube/data - - CUBESTORE_META_ADDR=cubestore_router:9999 - volumes: - - .cubestore:/cube/data - depends_on: - - cubestore_router - - cubestore_worker_2: - restart: always - image: cubejs/cubestore:v%CURRENT_VERSION - environment: - - CUBESTORE_WORKERS=cubestore_worker_1:10001,cubestore_worker_2:10002 - - CUBESTORE_SERVER_NAME=cubestore_worker_2:10002 - - CUBESTORE_WORKER_PORT=10002 - - CUBESTORE_REMOTE_DIR=/cube/data - - CUBESTORE_META_ADDR=cubestore_router:9999 - volumes: - - .cubestore:/cube/data - depends_on: - - cubestore_router - - redis: - image: bitnami/redis:latest - environment: - - ALLOW_EMPTY_PASSWORD=yes - logging: - driver: none -``` - -## Set up reverse proxy - -In production, the Cube.js API should be served over an HTTPS connection to -ensure security of the data in-transit. We recommend using a reverse proxy; as -an example, let's use [NGINX][link-nginx]. - - - -You can also use a reverse proxy to enable HTTP 2.0 and GZIP compression - - - -First we'll create a new server configuration file called `nginx/cube.conf`: - -``` -server { - listen 443 ssl; - server_name cube.my-domain.com; - - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_ecdh_curve secp384r1; - # Replace the ciphers with the appropriate values - ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384 OLD_TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 OLD_TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256"; - ssl_prefer_server_ciphers on; - ssl_certificate /etc/ssl/private/cert.pem; - ssl_certificate_key /etc/ssl/private/key.pem; - ssl_session_timeout 10m; - ssl_session_cache shared:SSL:10m; - ssl_session_tickets off; - ssl_stapling on; - ssl_stapling_verify on; - - location / { - proxy_pass http://cube:4000/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } -} -``` - -Then we'll add a new service to our Docker Compose stack: - -```yaml -services: - ... - nginx: - image: nginx - ports: - - 443:443 - volumes: - - ./nginx:/etc/nginx/conf.d - - ./ssl:/etc/ssl/private -``` - -Don't forget to create a `ssl` directory with the `cert.pem` and `key.pem` files -inside so the Nginx service can find them. - -For automatically provisioning SSL certificates with LetsEncrypt, [this blog -post][medium-letsencrypt-nginx] may be useful. - -## Security - -### <--{"id" : "Security"}--> Use JSON Web Tokens - -Cube.js can be configured to use industry-standard JSON Web Key Sets for -securing its API and limiting access to data. To do this, we'll define the -relevant options on our Cube.js API instance: - - - -If you're using [`queryRewrite`][ref-config-queryrewrite] for access control, -then you must also configure -[`scheduledRefreshContexts`][ref-config-sched-ref-ctx] so the refresh workers -can correctly create pre-aggregations. - - - -```yaml -services: - cube_api: - image: cubejs/cube:v%CURRENT_VERSION - ports: - - 4000:4000 - environment: - - CUBEJS_DB_TYPE=bigquery - - CUBEJS_DB_BQ_PROJECT_ID=cubejs-bq-cluster - - CUBEJS_DB_BQ_CREDENTIALS= - - CUBEJS_DB_EXPORT_BUCKET=cubestore - - CUBEJS_CUBESTORE_HOST=cubestore_router - - CUBEJS_REDIS_URL=redis://redis:6379 - - CUBEJS_API_SECRET=secret - - CUBEJS_JWK_URL=https://cognito-idp..amazonaws.com//.well-known/jwks.json - - CUBEJS_JWT_AUDIENCE= - - CUBEJS_JWT_ISSUER=https://cognito-idp..amazonaws.com/ - - CUBEJS_JWT_ALGS=RS256 - - CUBEJS_JWT_CLAIMS_NAMESPACE= - volumes: - - .:/cube/conf - depends_on: - - cubestore_worker_1 - - cubestore_worker_2 - - cube_refresh_worker - - redis -``` - -### <--{"id" : "Security"}--> Securing Cube Store - -All Cube Store nodes (both router and workers) should only be accessible to -Cube.js API instances and refresh workers. To do this with Docker Compose, we -simply need to make sure that none of the Cube Store services have any exposed -ports. - -## Monitoring - -All Cube.js logs can be found by through the Docker Compose CLI: - -```bash -$ docker-compose ps - - Name Command State Ports ---------------------------------------------------------------------------------------------------------------------------------- -cluster_cube_1 docker-entrypoint.sh cubej ... Up 0.0.0.0:4000->4000/tcp,:::4000->4000/tcp -cluster_cubestore_router_1 ./cubestored Up 3030/tcp, 3306/tcp -cluster_cubestore_worker_1_1 ./cubestored Up 3306/tcp, 9001/tcp -cluster_cubestore_worker_2_1 ./cubestored Up 3306/tcp, 9001/tcp - -$ docker-compose logs - -cubestore_router_1 | 2021-06-02 15:03:20,915 INFO [cubestore::metastore] Creating metastore from scratch in /cube/.cubestore/data/metastore -cubestore_router_1 | 2021-06-02 15:03:20,950 INFO [cubestore::cluster] Meta store port open on 0.0.0.0:9999 -cubestore_router_1 | 2021-06-02 15:03:20,951 INFO [cubestore::mysql] MySQL port open on 0.0.0.0:3306 -cubestore_router_1 | 2021-06-02 15:03:20,952 INFO [cubestore::http] Http Server is listening on 0.0.0.0:3030 -cube_1 | 🚀 Cube.js server (%CURRENT_VERSION) is listening on 4000 -cubestore_worker_2_1 | 2021-06-02 15:03:24,945 INFO [cubestore::cluster] Worker port open on 0.0.0.0:9001 -cubestore_worker_1_1 | 2021-06-02 15:03:24,830 INFO [cubestore::cluster] Worker port open on 0.0.0.0:9001 -``` - -## Update to the latest version - -Find the latest stable release version (currently `v%CURRENT_VERSION`) [from -Docker Hub][link-cubejs-docker]. Then update your `docker-compose.yml` to use -the tag: - -```yaml -version: '2.2' - -services: - cube_api: - image: cubejs/cube:v%CURRENT_VERSION - ports: - - 4000:4000 - environment: - - CUBEJS_DB_TYPE=bigquery - - CUBEJS_DB_BQ_PROJECT_ID=cubejs-bq-cluster - - CUBEJS_DB_BQ_CREDENTIALS= - - CUBEJS_DB_EXPORT_BUCKET=cubestore - - CUBEJS_CUBESTORE_HOST=cubestore_router - - CUBEJS_REDIS_URL=redis://redis:6379 - - CUBEJS_API_SECRET=secret - volumes: - - .:/cube/conf - depends_on: - - cubestore_router - - cube_refresh_worker - - redis -``` - -## Extend the Docker image - -If you need to use npm packages with native extensions inside [the `cube.js` -configuration file][ref-config-js], you'll need to build your own Docker image. -You can do this by first creating a `Dockerfile` and a corresponding -`.dockerignore`: - -```bash -touch Dockerfile -touch .dockerignore -``` - -Add this to the `Dockerfile`: - -```dockerfile -FROM cubejs/cube:latest - -COPY . . -RUN npm install -``` - -And this to the `.dockerignore`: - -```bash -node_modules -npm-debug.log -schema -cube.js -.env -``` - -Then start the build process by running the following command: - -```bash -docker build -t /cubejs-custom-build . -``` - -Finally, update your `docker-compose.yml` to use your newly-built image: - -```bash -version: '2.2' - -services: - cube_api: - image: /cubejs-custom-build - ports: - - 4000:4000 - environment: - - CUBEJS_DB_TYPE=bigquery - - CUBEJS_DB_BQ_PROJECT_ID=cubejs-bq-cluster - - CUBEJS_DB_BQ_CREDENTIALS= - - CUBEJS_DB_EXPORT_BUCKET=cubestore - - CUBEJS_CUBESTORE_HOST=cubestore_router - - CUBEJS_REDIS_URL=redis://redis:6379 - - CUBEJS_API_SECRET=secret - volumes: - - .:/cube/conf - # Prevent dev dependencies leaking - - .empty:/cube/conf/node_modules/@cubejs-backend/ - depends_on: - - cubestore_router - - cube_refresh_worker - - redis -``` - -[medium-letsencrypt-nginx]: - https://pentacent.medium.com/nginx-and-lets-encrypt-with-docker-in-less-than-5-minutes-b4b8a60d3a71 -[link-cubejs-docker]: https://hub.docker.com/r/cubejs/cube -[link-docker-app]: https://www.docker.com/products/docker-app -[link-nginx]: https://www.nginx.com/ -[ref-config-js]: /config -[ref-config-queryrewrite]: /config#query-rewrite -[ref-config-sched-ref-ctx]: /config#scheduled-refresh-contexts diff --git a/docs/content/Deployment/Platform-Kubernetes.mdx b/docs/content/Deployment/Platform-Kubernetes.mdx deleted file mode 100644 index e0d38ae8a8723..0000000000000 --- a/docs/content/Deployment/Platform-Kubernetes.mdx +++ /dev/null @@ -1,268 +0,0 @@ ---- -title: Deploy to Kubernetes -menuTitle: Kubernetes -permalink: /deployment/platforms/kubernetes -category: Deployment -subCategory: Platforms -menuOrder: 3 ---- - -This guide walks you through deploying Cube.js with [Kubernetes][k8s]. This -particular deployment makes use of a `hostPath` volume to mount schema files -into the containers. - - - -This is an example of a production-ready deployment, but real-world deployments -can vary significantly depending on desired performance and scale. For an -example of deploying with [Helm][helm-k8s] or plain [Kubernetes][k8s], check out -the respective examples [here][gh-cubejs-examples-k8s-helm] and -[here][gh-cube-examples-k8s]. - - - -## Prerequisites - -- [Kubernetes][k8s] - -## Configuration - -### <--{"id" : "Configuration"}--> Create Cube.js API instance and Refresh Worker - -To deploy Cube.js, we will use [`Deployment`][k8s-docs-deployment]s and -[Service][k8s-docs-service]s. We'll start by creating a Redis deployment in a -file called `redis-deployment.yaml`: - - - -And a corresponding service for the Redis deployment in a file named -`redis-service.yaml`: - - - -Next, create a file called `cube-api-deployment.yaml` with the following -contents: - - - -The exact set of `CUBEJS_DB_*` environment variables depends on your database; -please reference [Connecting to the Database page][ref-config-db] for specific -configuration instructions. - -We'll also create a corresponding service for this deployment in a file called -`cube-api-service.yaml`: - - - -Now that we've configured our Cube.js API, let's also set up a deployment for a -Refresh Worker in `cube-refresh-worker-deployment.yaml`: - - - -### <--{"id" : "Configuration"}--> Create Cube Store Router and Worker nodes - -With our Cube.js API and Refresh Worker set up, we can now begin setting up Cube -Store. We will make two [`StatefulSet`][k8s-docs-statefulset]s with -corresponding [Service][k8s-docs-service]s; one for the router node and one for -the worker nodes. - -Create a new file called `cubestore-router-statefulset.yaml`: - - - -And a corresponding service declaration in a file called -`cubestore-router-service.yaml`: - - - -With the router set up, let's move onto setting up our worker nodes. Let's -create a new file called `cubestore-workers-statefulset.yaml`: - - - -Next, create its corresponding service in `cubestore-workers-service.yaml`. By -specifying `clusterIP: None`, you create a [headless -service][k8s-docs-headless-svc]. For this use case, a headless service is the -better solution. - - - -## Set up reverse proxy - -In production, the Cube.js API should be served over an HTTPS connection to -ensure security of the data in-transit. We recommend using a reverse proxy; as -an example, let's assume we're running the Kubernetes cluster on a cloud -provider such as AWS or GCP. We'll also assume we already have the SSL -certificate and key available on our local filesystem. - - - -You can also use a reverse proxy to enable HTTP 2.0 and GZIP compression - - - -First, we'll run the following to create a secret in Kubernetes: - -```bash -kubectl create secret tls --key --cert -``` - -Next, we'll create a new ingress rule in a file called `ingress.yml`: - - - - - -For more advanced configuration, or for platforms where ingress is manually -managed, please refer to the Kubernetes documentation for [Ingress -Controllers][k8s-docs-ingress-controllers]. You can also find an [example of an -Ingress Controller here][gh-cubejs-examples-k8s-ingress-ctrl]. - - - -## Security - -### <--{"id" : "Security"}--> Use JSON Web Tokens - -Cube.js can be configured to use industry-standard JSON Web Key Sets for -securing its API and limiting access to data. To do this, we'll define the -relevant options on our Cube.js API instance and Refresh Worker deployments: - - - -If you're using [`queryRewrite`][ref-config-queryrewrite] for access control, -then you must also configure -[`scheduledRefreshContexts`][ref-config-sched-ref-ctx] so the refresh workers -can correctly create pre-aggregations. - - - -```yaml -apiVersion: apps/v1 -kind: Deployment -... -spec: - template: - ... - spec: - containers: - - env: - ... - - name: CUBEJS_JWK_URL - value: https://cognito-idp..amazonaws.com//.well-known/jwks.json - - name: CUBEJS_JWT_AUDIENCE - value: - - name: CUBEJS_JWT_ISSUER - value: https://cognito-idp..amazonaws.com/ - - name: CUBEJS_JWT_ALGS - value: RS256 - - name: CUBEJS_JWT_CLAIMS_NAMESPACE - value: -``` - -### <--{"id" : "Security"}--> Securing Cube Store - -All Cube Store nodes, both router and workers, should only be accessible to -Cube.js API instances and refresh workers. To do this with Kubernetes, we need -to make sure that none of the Cube Store services are exposed in our Ingress -configuration. The Cube Store services should only be accessible from other -services within the cluster. - -## Monitoring - -All Cube.js logs can be found through the Kubernetes CLI: - -```bash -kubectl logs -``` - -## Update to the latest version - -Specify the tag for the Docker image available from [Docker -Hub][dockerhub-cubejs] (currently `v%CURRENT_VERSION`). Then update your -`cube-api-deployment.yaml` and `cube-refresh-deployment.yaml` to use the new -tag: - -```yaml -apiVersion: apps/v1 -kind: Deployment -... -spec: - ... - template: - ... - spec: - containers: - - name: cube-api - image: cubejs/cube:v%CURRENT_VERSION -``` - -Similarly, for `cubestore-router-statefulset.yaml` and -`cubestore-workers-statefulset.yaml`: - -```yaml -apiVersion: apps/v1 -kind: StatefulSet -... -spec: - ... - template: - ... - spec: - containers: - - name: cube-api - image: cubejs/cubestore:v%CURRENT_VERSION -``` - -[dockerhub-cubejs]: https://hub.docker.com/r/cubejs/cube -[gh-cube-examples-k8s]: - https://github.com/cube-js/cube.js/tree/master/examples/kubernetes -[gh-cubejs-examples-k8s-helm]: - https://github.com/cube-js/cube.js/tree/master/examples/helm-charts -[gh-cubejs-examples-k8s-ingress-ctrl]: - https://github.com/cube-js/cube.js/blob/master/examples/kubernetes/prod/ingress-controller.yaml -[helm-k8s]: https://helm.sh/ -[k8s-docs-deployment]: - https://kubernetes.io/docs/concepts/workloads/controllers/deployment/ -[k8s-docs-headless-svc]: - https://kubernetes.io/docs/concepts/services-networking/service/#headless-services -[k8s-docs-ingress-controllers]: - https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/ -[k8s-docs-service]: - https://kubernetes.io/docs/concepts/services-networking/service/ -[k8s-docs-statefulset]: - https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/ -[k8s]: https://kubernetes.io/ -[ref-config-db]: /config/databases -[ref-config-queryrewrite]: /config#query-rewrite -[ref-config-sched-ref-ctx]: /config#options-reference-scheduled-refresh-contexts diff --git a/docs/content/Deployment/Platform-Serverless-AWS.mdx b/docs/content/Deployment/Platform-Serverless-AWS.mdx deleted file mode 100644 index f04a5a1273a6f..0000000000000 --- a/docs/content/Deployment/Platform-Serverless-AWS.mdx +++ /dev/null @@ -1,208 +0,0 @@ ---- -title: Deploy to Serverless Framework on AWS -menuTitle: Serverless Framework on AWS -permalink: /deployment/platforms/serverless/aws -category: Deployment -subCategory: Platforms -menuOrder: 4 ---- - -This guide walks you through deploying Cube.js with the [Serverless -Framework][link-sls] on [AWS][link-aws]. - - - -WARNING! - -We do not recommend deploying Cube with AWS Lambda, AWS Fargate and -Google Cloud Functions. This guide will be removed soon. Additionally, please -note that Cube Store [will replace Redis for in-memory caching and queue management in -late 2022](https://cube.dev/blog/replacing-redis-with-cube-store) and will be a -required component to run clustered Cube deployments. - - - -## Prerequisites - -- An AWS account -- An [Elasticache Redis][aws-redis] cluster URL for caching and queuing -- A separate Cube Store deployment for pre-aggregations -- Node.js 12+ -- Serverless Framework - -## Configuration - -Create a Serverless Framework project by creating a `serverless.yml`. A -production-ready stack would at minimum consist of: - -- A [Lambda function][aws-lambda] for a Cube.js API instance -- A [Lambda function][aws-lambda] for a Cube.js Refresh Worker - -The `serverless.yml` for an example project is provided below: - -```yaml -service: hello-cube-sls - -provider: - name: aws - runtime: nodejs12.x - iamRoleStatements: - - Effect: 'Allow' - Action: - - 'sns:*' - Resource: '*' - environment: - CUBEJS_DB_TYPE: - CUBEJS_DB_HOST: - CUBEJS_DB_NAME: - CUBEJS_DB_USER: - CUBEJS_DB_PASS: - CUBEJS_DB_PORT: - CUBEJS_REDIS_URL: - CUBEJS_API_SECRET: - CUBEJS_APP: '${self:service.name}-${self:provider.stage}' - NODE_ENV: production - AWS_ACCOUNT_ID: - Fn::Join: - - '' - - - Ref: 'AWS::AccountId' - -functions: - cubejs: - handler: index.api - timeout: 30 - events: - - http: - path: / - method: GET - - http: - path: /{proxy+} - method: ANY - cors: - origin: '*' - headers: - - Content-Type - - Authorization - - X-Request-Id - - X-Amz-Date - - X-Amz-Security-Token - - X-Api-Key - cubejsProcess: - handler: index.process - timeout: 630 - events: - - sns: '${self:service.name}-${self:provider.stage}-process' - -plugins: - - serverless-express -``` - -### <--{"id" : "Configuration"}--> Refresh Worker - - - -Running a refresh worker using Serverless requires a _slightly_ different setup -than Docker. You must continuously call the endpoint once every 60 seconds to -keep the pre-aggregation creation queries in the queue. Failing to do this will -prevent pre-aggregations from being built. - - - -To begin the scheduled refresh, first call the -[`/v1/run-scheduled-refresh`][ref-restapi-sched-refresh] endpoint. The endpoint -will return `{ "finished": false }` whilst the pre-aggregations are being built; -once they are successfully built, the response will change to: - -```json -{ - "finished": true -} -``` - -### <--{"id" : "Configuration"}--> Cube Store - -Unfortunately, Cube Store currently cannot be run using serverless platforms; we -recommend using [Cube Cloud][link-cube-cloud] which provides a similar -"serverless" experience instead. If you prefer self-hosting, you can use a PaaS -such as [AWS ECS][aws-ecs] or [AWS EKS][aws-eks]. More instructions can be found -in the [Running in Production page under Caching][ref-caching-prod]. - -## Security - -### <--{"id" : "Security"}--> Networking - -To run Cube.js within a VPC, add a `vpc` property to the `serverless.yml`: - -```yaml -provider: - ... - vpc: - securityGroupIds: - - sg-12345678901234567 # Add your DB and Redis security groups here - subnetIds: - # Add any subnets with access to your DB, Redis and the Internet - - subnet-12345678901234567 -``` - -### <--{"id" : "Security"}--> Use JSON Web Tokens - -Cube.js can be configured to use industry-standard JSON Web Key Sets for -securing its API and limiting access to data. To do this, we'll define the -relevant options on our Cube.js API instance: - - - -If you're using [`queryRewrite`][ref-config-queryrewrite] for access control, -then you must also configure -[`scheduledRefreshContexts`][ref-config-sched-ref-ctx] so the refresh workers -can correctly create pre-aggregations. - - - -```yaml -provider: - ... - environment: - ... - CUBEJS_JWK_URL: https://cognito-idp..amazonaws.com//.well-known/jwks.json - CUBEJS_JWT_AUDIENCE: - CUBEJS_JWT_ISSUER: https://cognito-idp..amazonaws.com/ - CUBEJS_JWT_ALGS: RS256 - CUBEJS_JWT_CLAIMS_NAMESPACE: -... -``` - -## Monitoring - -All Cube.js logs can be found in the [AWS CloudWatch][aws-cloudwatch] log group -for the Serverless project. - -## Update to the latest version - -Find the latest stable release version (currently `v%CURRENT_VERSION`) [from -npm][link-cubejs-sls-npm]. Then update your `package.json` to use the version: - -```json -{ - "dependencies": { - "@cubejs-backend/serverless-aws": "%CURRENT_VERSION", - "@cubejs-backend/serverless": "%CURRENT_VERSION" - } -} -``` - -[aws-cloudwatch]: https://aws.amazon.com/cloudwatch/ -[aws-ec2]: https://aws.amazon.com/ec2/ -[aws-ecs]: https://aws.amazon.com/ecs/ -[aws-eks]: https://aws.amazon.com/eks/ -[aws-lambda]: https://aws.amazon.com/lambda/ -[aws-redis]: https://aws.amazon.com/elasticache/redis/ -[link-aws]: https://aws.amazon.com/ -[link-sls]: https://www.serverless.com/ -[link-cube-cloud]: https://cubecloud.dev -[link-cubejs-sls-npm]: https://www.npmjs.com/package/@cubejs-backend/serverless -[link-docker-app]: https://www.docker.com/products/docker-app -[ref-caching-prod]: /caching/running-in-production -[ref-config-queryrewrite]: /config#query-rewrite -[ref-config-sched-ref-ctx]: /config#scheduled-refresh-contexts -[ref-restapi-sched-refresh]: /rest-api#v-1-run-scheduled-refresh diff --git a/docs/content/Deployment/Platform-Serverless-Google.mdx b/docs/content/Deployment/Platform-Serverless-Google.mdx deleted file mode 100644 index 490e953267f36..0000000000000 --- a/docs/content/Deployment/Platform-Serverless-Google.mdx +++ /dev/null @@ -1,209 +0,0 @@ ---- -title: Deploy to Serverless Framework on GCP -menuTitle: Serverless Framework on GCP -permalink: /deployment/platforms/serverless/google-cloud -category: Deployment -subCategory: Platforms -menuOrder: 5 ---- - -This guide walks you through deploying Cube.js with the [Serverless -Framework][link-sls] on [Google Cloud][link-google-cloud]. - - - -WARNING! - -We do not recommend deploying Cube with AWS Lambda, AWS Fargate and -Google Cloud Functions. This guide will be removed soon. Additionally, please -note that Cube Store [will replace Redis for in-memory caching and queue management in -late 2022](https://cube.dev/blog/replacing-redis-with-cube-store) and will be a -required component to run clustered Cube deployments. - - - -## Prerequisites - -- A Google Cloud account -- A [Memorystore][gcp-redis] cluster URL for caching and queuing -- A separate Cube Store deployment for pre-aggregations -- Node.js 12+ -- Serverless Framework - -## Configuration - -Create a Serverless Framework project by creating a `serverless.yml`. A -production-ready stack would at minimum consist of: - -- A [Cloud Function][gcp-cloud-funcs] for a Cube.js API instance -- A [Cloud Function][gcp-cloud-funcs] for a Cube.js Refresh Worker - -The `serverless.yml` for an example project is provided below: - -```yaml -service: hello-cube-sls # NOTE: Don't put the word "google" in here - -provider: - name: google - stage: dev - runtime: nodejs12 - region: us-central1 - project: - # The GCF credentials can be a little tricky to set up. Luckily we've documented this for you here: - # https://serverless.com/framework/docs/providers/google/guide/credentials/ - # - # the path to the credentials file needs to be absolute - credentials: - environment: - CUBEJS_DB_TYPE: postgres - CUBEJS_DB_HOST: - CUBEJS_DB_NAME: - CUBEJS_DB_USER: - CUBEJS_DB_PASS: - CUBEJS_DB_PORT: - CUBEJS_REDIS_URL: - CUBEJS_API_SECRET: - CUBEJS_APP: '${self:service.name}-${self:provider.stage}' - CUBEJS_SERVERLESS_PLATFORM: '${self:provider.name}' - -plugins: - - serverless-google-cloudfunctions - - serverless-express - -# needs more granular excluding in production as only the serverless provider npm -# package should be excluded (and not the whole node_modules directory) -package: - exclude: - - node_modules/** - - .gitignore - - .git/** - -functions: - cubejs: - handler: api - events: - - http: ANY - cubejsProcess: - handler: process - events: - - event: - eventType: providers/cloud.pubsub/eventTypes/topic.publish - resource: 'projects/${self:provider.project}/topics/${self:service.name}-${self:provider.stage}-process' -``` - -### <--{"id" : "Configuration"}--> Refresh Worker - - - -Running a refresh worker using Serverless requires a _slightly_ different setup -than Docker. You must continuously call the endpoint once every 60 seconds to -keep the pre-aggregation creation queries in the queue. Failing to do this will -prevent pre-aggregations from being built. - - - -To begin the scheduled refresh, first call the -[`/v1/run-scheduled-refresh`][ref-restapi-sched-refresh] endpoint. The endpoint -will return `{ "finished": false }` whilst the pre-aggregations are being built; -once they are successfully built, the response will change to: - -```json -{ - "finished": true -} -``` - -### <--{"id" : "Configuration"}--> Cube Store - -Unfortunately, Cube Store currently cannot be run using serverless platforms; we -recommend using [Cube Cloud][link-cube-cloud] which provides a similar -"serverless" experience instead. If you prefer self-hosting, you can use a PaaS -such as [Compute Engine][gcp-compute] or [Google Kubernetes Engine][gcp-k8s]. -More instructions can be found in the [Running in Production page under -Caching][ref-caching-prod]. - -## Security - -### <--{"id" : "Security"}--> Networking - -To run Cube.js within a VPC, add a `vpc` property to the function definition in -`serverless.yml`: - -```yaml -functions: - cubejs: - ... - vpc: 'projects//locations//connectors/' - cubejsProcess: - ... - vpc: 'projects//locations//connectors/' -``` - -### <--{"id" : "Security"}--> Use JSON Web Tokens - -Cube.js can be configured to use industry-standard JSON Web Key Sets for -securing its API and limiting access to data. To do this, we'll define the -relevant options on our Cube.js API instance: - - - -If you're using [`queryRewrite`][ref-config-queryrewrite] for access control, -then you must also configure -[`scheduledRefreshContexts`][ref-config-sched-ref-ctx] so the refresh workers -can correctly create pre-aggregations. - - - -```yaml -provider: - ... - environment: - ... - CUBEJS_JWK_URL: https://cognito-idp..amazonaws.com//.well-known/jwks.json - CUBEJS_JWT_AUDIENCE: - CUBEJS_JWT_ISSUER: https://cognito-idp..amazonaws.com/ - CUBEJS_JWT_ALGS: RS256 - CUBEJS_JWT_CLAIMS_NAMESPACE: -... -``` - -### <--{"id" : "Security"}--> Cube Store - -All Cube Store nodes (both router and workers) should only be accessible to -Cube.js API instances and refresh workers. - -## Monitoring - -All Cube.js logs can be found in the [Google Cloud Console for Cloud -Functions][gcp-cloud-funcs-logs]. - -[gcp-cloud-funcs-logs]: - https://console.cloud.google.com/project/_/logs?service=cloudfunctions.googleapis.com - -## Update to the latest version - -Find the latest stable release version (currently `v%CURRENT_VERSION`) [from -npm][link-cubejs-sls-npm]. Then update your `package.json` to use the version: - -```json -{ - "dependencies": { - "@cubejs-backend/serverless-google": "%CURRENT_VERSION", - "@cubejs-backend/serverless": "%CURRENT_VERSION" - } -} -``` - -[gcp-cloud-funcs]: https://cloud.google.com/functions/ -[gcp-compute]: https://cloud.google.com/compute/ -[gcp-k8s]: https://cloud.google.com/kubernetes-engine/ -[gcp-redis]: https://cloud.google.com/memorystore/ -[link-google-cloud]: https://cloud.google.com/ -[link-sls]: https://www.serverless.com/ -[link-cube-cloud]: https://cubecloud.dev -[link-cubejs-sls-npm]: https://www.npmjs.com/package/@cubejs-backend/serverless -[link-docker-app]: https://www.docker.com/products/docker-app -[ref-caching-prod]: /caching/running-in-production -[ref-config-queryrewrite]: /config#query-rewrite -[ref-config-sched-ref-ctx]: /config#scheduled-refresh-contexts -[ref-restapi-sched-refresh]: /rest-api#v-1-run-scheduled-refresh diff --git a/docs/content/Deployment/Production-Checklist.mdx b/docs/content/Deployment/Production-Checklist.mdx index e4adc88f82867..4c95d8ade4bd1 100644 --- a/docs/content/Deployment/Production-Checklist.mdx +++ b/docs/content/Deployment/Production-Checklist.mdx @@ -1,5 +1,5 @@ --- -title: Production Checklist +title: Production checklist permalink: /deployment/production-checklist category: Deployment menuOrder: 2 @@ -13,16 +13,16 @@ installation to [Cube Cloud][link-cube-cloud]. -This is a checklist for configuring and securing Cube.js for a production +This is a checklist for configuring and securing Cube for a production deployment. ## Disable Development Mode -When running Cube.js in production environments, make sure development mode is -disabled both on API Instances and Refresh Worker. Running Cube.js in -development mode in a production environment can lead to security -vulnerabilities. You can read more on the differences between [production and -development mode here][link-cubejs-dev-vs-prod]. +When running Cube in production environments, make sure development mode is +disabled both on API Instances and Refresh Worker. Running Cube in development +mode in a production environment can lead to security vulnerabilities. You can +read more on the differences between [production and development mode +here][link-cubejs-dev-vs-prod]. @@ -30,7 +30,7 @@ Development mode is disabled by default. -```bash +```dotenv # Set this to false or leave unset to disable development mode CUBEJS_DEV_MODE=false ``` @@ -38,125 +38,107 @@ CUBEJS_DEV_MODE=false ## Set up Refresh Worker To refresh in-memory cache and [pre-aggregations][ref-schema-ref-preaggs] in the -background, we recommend running a separate Cube.js Refresh Worker instance. -This allows your Cube.js API Instance to continue to serve requests with high +background, we recommend running a separate Cube Refresh Worker instance. This +allows your Cube API Instance to continue to serve requests with high availability. -```bash -# Set to true so a Cube.js instance acts as a refresh worker +```dotenv +# Set to true so a Cube instance acts as a refresh worker CUBEJS_REFRESH_WORKER=true ``` ## Set up Cube Store -Cube Store is the purpose-built pre-aggregations storage for Cube.js. Follow the -[instructions here][ref-caching-cubestore] to set it up. + + +While Cube can operate with in-memory cache and queue storage, there're multiple +parts of Cube which require Cube Store in production mode. Replicating Cube +instances without Cube Store can lead to source database degraded performance, +various race conditions and cached data inconsistencies. + + -Depending on your database, Cube.js may need to "stage" pre-aggregations inside -your database first before ingesting them into Cube Store. In this case, Cube.js +Cube Store manages in-memory cache, queue and pre-aggregations for Cube. Follow +the [instructions here][ref-caching-cubestore] to set it up. + +Depending on your database, Cube may need to "stage" pre-aggregations inside +your database first before ingesting them into Cube Store. In this case, Cube will require write access to the `prod_pre_aggregations` schema inside your database. The schema name can be modified by the -`CUBEJS_PRE_AGGREGATIONS_SCHEMA` environment variable; see the [Environment -Variables][ref-env-vars-general] page for more details. +[`CUBEJS_PRE_AGGREGATIONS_SCHEMA`][ref-conf-ref-env-preaggs-schema] environment +variable; see the [Environment Variables reference][ref-env-vars] for more +details. -You may consider enabling an export bucket which allows Cube.js to build large +You may consider enabling an export bucket which allows Cube to build large pre-aggregations in a much faster manner. It is currently supported for BigQuery, Redshift and Snowflake. Check [the relevant documentation for your configured database][ref-config-connect-db] to set it up. -## Set up Redis - - - -Cube Store [will replace -Redis][replace-redis] for in-memory cache and queue management in late -2022. - - - -Cube.js requires [Redis](https://redis.io/), an in-memory data structure store, -to run in production. - -It uses Redis for query caching and queue. Set the `CUBEJS_REDIS_URL` -environment variable to allow Cube.js to connect to Redis. If your Redis -instance also has a password, please set it via the `CUBEJS_REDIS_PASSWORD` -environment variable. Set the `CUBEJS_REDIS_TLS` environment variable to `true` -if you want to enable SSL-secured connections. Ensure your Redis cluster allows -at least 15 concurrent connections. - -### <--{"id" : "Set up Redis"}--> Redis Sentinel - -Redis provides functionality for high availability through -[`Redis Sentinel`][link-redis-sentinel]. - -For Redis Sentinel support, the npm package [`ioredis`][gh-ioredis] needs to be -used instead of[`redis`][gh-node-redis]. This is done by setting the -`CUBEJS_REDIS_USE_IOREDIS` environment variable to `true`. Then set -`CUBEJS_REDIS_URL` to the -`redis+sentinel://localhost:26379,otherhost:26479/mymaster/5` to allow Cube.js -to connect to the Redis Sentinel. - - - -Cube.js server instances used by same tenant environments should have same Redis -instances. Otherwise they will have different query queues which can lead to -incorrect pre-aggregation states and intermittent data access errors. +## Secure the deployment - +If you're using JWTs, you can configure Cube to correctly decode them and inject +their contents into the [Security Context][ref-sec-ctx]. Add your authentication +provider's configuration under [the `jwt` property of your `cube.js` +configuration file][ref-config-jwt], or if using environment variables, see +`CUBEJS_JWK_*`, `CUBEJS_JWT_*` in the [Environment Variables +reference][ref-env-vars]. -### <--{"id" : "Set up Redis"}--> Redis Pool +## Set up health checks -If `CUBEJS_REDIS_URL` is provided Cube.js, will create a Redis connection pool -with a minimum of 2 and maximum of 1000 concurrent connections, by default. The -`CUBEJS_REDIS_POOL_MIN` and `CUBEJS_REDIS_POOL_MAX` environment variables can be -used to tweak pool size limits. To disable connection pooling, and instead -create connections on-demand, you can set `CUBEJS_REDIS_POOL_MAX` to 0. +Cube provides [Kubernetes-API compatible][link-k8s-healthcheck-api] health check +(or probe) endpoints that indicate the status of the deployment. Configure your +monitoring service of choice to use the [`/readyz`][ref-api-readyz] and +[`/livez`][ref-api-livez] API endpoints so you can check on the Cube +deployment's health and be alerted to any issues. -If your maximum concurrent connections limit is too low, you may see -`TimeoutError: ResourceRequest timed out` errors. As a rule of a thumb, you need -to have `Queue Size * Number of tenants` concurrent connections to ensure the -best performance possible. If you use clustered deployments, please make sure -you have enough connections for all Cube.js server instances. A lower number of -connections still can work, however Redis becomes a performance bottleneck in -this case. +## Appropriate cluster sizing -### <--{"id" : "Set up Redis"}--> Running without Redis +There's no one-size-fits-all when it comes to sizing a Cube cluster and its +resources. Resources required by Cube significantly depend on the amount of +traffic Cube needs to serve and the amount of data it needs to process. The +following sizing estimates are based on default settings and are very generic, +which may not fit your Cube use case, so you should always tweak resources based +on consumption patterns you see. -If you want to run Cube.js in production without Redis, you can use -`CUBEJS_CACHE_AND_QUEUE_DRIVER` environment variable to `memory`. +### <--{"id" : "Appropriate cluster sizing"}--> Memory and CPU - +Each Cube cluster should contain at least 2 Cube API instances. Every Cube API +instance should have at least 3GB of RAM and 2 CPU cores allocated for it. -Serverless and clustered deployments can't be run without Redis as it is used to -manage the query queue. +Refresh workers tend to be much more CPU and memory intensive, so at least 6GB +of RAM is recommended. Please note that to take advantage of all available RAM, +the Node.js heap size should be adjusted accordingly by using the +[`--max-old-space-size` option][node-heap-size]: - +```sh +NODE_OPTIONS="--max-old-space-size=6144" +``` -## Secure the deployment +[node-heap-size]: + https://nodejs.org/api/cli.html#--max-old-space-sizesize-in-megabytes -If you're using JWTs, you can configure Cube.js to correctly decode them and -inject their contents into the [Security Context][ref-sec-ctx]. Add your -authentication provider's configuration under [the `jwt` property of your -`cube.js` configuration file][ref-config-jwt], or use [the corresponding -environment variables (`CUBEJS_JWK_URL`, -`CUBEJS_JWT_*`)][ref-config-env-vars-general]. +The Cube Store router node should have at least 6GB of RAM and 4 CPU cores +allocated for it. Every Cube Store worker node should have at least 8GB of RAM +and 4 CPU cores allocated for it. The Cube Store cluster should have at least +two worker nodes. -## Set up health checks +### <--{"id" : "Appropriate cluster sizing"}--> RPS and data volume -Cube.js provides [Kubernetes-API compatible][link-k8s-healthcheck-api] health -check (or probe) endpoints that indicate the status of the deployment. Configure -your monitoring service of choice to use the [`/readyz`][ref-api-readyz] and -[`/livez`][ref-api-livez] API endpoints so you can check on the Cube.js -deployment's health and be alerted to any issues. +Depending on data model size, every Core Cube API instance can serve 1 to 10 +requests per second. Every Core Cube Store router node can serve 50-100 queries +per second. As a rule of thumb, you should provision 1 Cube Store worker node +per one Cube Store partition or 1M of rows scanned in a query. For example if +your queries scan 16M of rows per query, you should have at least 16 Cube Store +worker nodes provisioned. `EXPLAIN ANALYZE` can be used to see partitions +involved in a Cube Store query. Cube Cloud ballpark performance numbers can +differ as it has different Cube runtime. [blog-migrate-to-cube-cloud]: https://cube.dev/blog/migrating-from-self-hosted-to-cube-cloud/ -[gh-ioredis]: https://github.com/luin/ioredis -[gh-node-redis]: https://github.com/NodeRedis/node-redis [link-caddy]: https://caddyserver.com/ [link-cube-cloud]: https://cubecloud.dev [link-cubejs-dev-vs-prod]: /configuration/overview#development-mode @@ -165,15 +147,16 @@ deployment's health and be alerted to any issues. [link-kong]: https://konghq.com/kong/ [link-nginx]: https://www.nginx.com/ [link-nginx-docs]: https://nginx.org/en/docs/http/configuring_https_servers.html -[link-redis-sentinel]: https://redis.io/topics/sentinel [ref-config-connect-db]: /connecting-to-the-database [ref-caching-cubestore]: /caching/running-in-production -[ref-env-vars-general]: /reference/environment-variables#general +[ref-conf-ref-env-cachequeue-driver]: + /reference/environment-variables#cubejs-cache-and-queue-driver +[ref-conf-ref-env-preaggs-schema]: + /reference/environment-variables#cubejs-pre-aggregations-schema +[ref-env-vars]: /reference/environment-variables [ref-schema-ref-preaggs]: /schema/reference/pre-aggregations [ref-api-scheduled-refresh]: /rest-api#v-1-run-scheduled-refresh [ref-sec-ctx]: /security/context [ref-config-jwt]: /config#jwt -[ref-config-env-vars-general]: /reference/environment-variables#general [ref-api-readyz]: /rest-api#readyz [ref-api-livez]: /rest-api#livez -[replace-redis]: https://cube.dev/blog/replacing-redis-with-cube-store diff --git a/docs/content/Deployment/deployment-overview.png b/docs/content/Deployment/deployment-overview.png deleted file mode 100644 index b807b9f64d3ba..0000000000000 Binary files a/docs/content/Deployment/deployment-overview.png and /dev/null differ diff --git a/docs/content/Deployment/how-cube-cloud-works.png b/docs/content/Deployment/how-cube-cloud-works.png deleted file mode 100644 index 3e21f0ecd8d4d..0000000000000 Binary files a/docs/content/Deployment/how-cube-cloud-works.png and /dev/null differ diff --git a/docs/content/Developer-Tools/CLI-Reference.mdx b/docs/content/Developer-Tools/CLI-Reference.mdx deleted file mode 100644 index e1a644edd1cc0..0000000000000 --- a/docs/content/Developer-Tools/CLI-Reference.mdx +++ /dev/null @@ -1,162 +0,0 @@ ---- -title: CLI Command Reference -permalink: /reference -category: Developer Tools -menuOrder: 2 ---- - -## create - -The `create` command generates barebones Cube.js app. - -### <--{"id" : "create"}--> Usage - -```bash -$ cubejs create APP-NAME -d DB-TYPE [-t TEMPLATE] -``` - -### <--{"id" : "create"}--> Flags - -| Parameter | Description | Values | -| --------------------------- | ----------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | -| `-d, --db-type ` | Preconfigure Cube.js app for selected database. | `postgres`, `mysql`, `athena`, `mongobi`, `bigquery`, `redshift`, `mssql`, `clickhouse`, `snowflake`, `presto`, `druid` | -| `-t, --template