Merge pull request #9177 from ZhiHanZ/open-sharing

feat: add prototype open sharing and add sharing stateful tests

Author: BohuTANG

.github/actions/artifact_download/action.yml

@@ -59,8 +59,12 @@ runs:
         aws s3 cp \
           s3://databend-ci/${{ inputs.profile }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/ \
           ${{ steps.info.outputs.path }}/ --recursive --exclude "*" --include "databend-*" --no-progress
+        aws s3 cp \
+          s3://databend-ci/${{ inputs.profile }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/ \
+          ${{ steps.info.outputs.path }}/ --recursive --exclude "*" --include "open-sharing" --no-progress || true
 
     - name: make artifacts excutable
       shell: bash
       run: |
         chmod +x ${{ steps.info.outputs.path }}/databend-*
+        chmod +x ${{ steps.info.outputs.path }}/open-sharing || true

.github/actions/artifact_upload/action.yml

@@ -46,3 +46,4 @@ runs:
         for line in $(ls ./target/${{ inputs.target }}/${{ inputs.profile }}/databend-*); do
           aws s3 cp $line s3://databend-ci/${{ inputs.profile }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/ --no-progress
         done
+        aws s3 cp ./target/${{ inputs.target }}/${{ inputs.profile }}/open-sharing s3://databend-ci/${{ inputs.profile }}/${{ inputs.sha }}/${{ inputs.target }}/${{ inputs.category }}/ --no-progress || true

Cargo.lock

@@ -30,6 +30,7 @@ members = [
     "src/query/catalog",
     "src/query/datablocks",
     "src/query/sharing",
+    "src/query/sharing-endpoint",
     "src/query/datavalues",
     "src/query/expression",
     "src/query/formats",
@@ -76,6 +77,7 @@ members = [
     "src/meta/protos",
     # databend-meta
     "src/meta/service",
+
 ]
 
 [workspace.dependencies]

Cargo.toml

@@ -8,5 +8,5 @@ SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
 cd "$SCRIPT_PATH/../.." || exit
 
 echo "Build(DEBUG) start..."
-cargo build --bin=databend-query --bin=databend-meta --bin=databend-metactl
+cargo build --bin=databend-query --bin=databend-meta --bin=databend-metactl --bin=open-sharing
 echo "All done..."

scripts/build/build-debug.sh

@@ -8,5 +8,5 @@ SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
 cd "$SCRIPT_PATH/../.." || exit
 
 echo "Build(RELEASE) start..."
-cargo build --bin=databend-query --bin=databend-meta --bin=databend-metactl --release
+cargo build --bin=databend-query --bin=databend-meta --bin=databend-metactl --bin=open-sharing --release
 echo "All done..."

scripts/build/build-release.sh

@@ -17,6 +17,7 @@ export STORAGE_S3_ENDPOINT_URL=http://127.0.0.1:9900
 export STORAGE_S3_ACCESS_KEY_ID=minioadmin
 export STORAGE_S3_SECRET_ACCESS_KEY=minioadmin
 export STORAGE_ALLOW_INSECURE=true
+export ALLOW_SHARING=true
 
 echo "Install dependence"
 python3 -m pip install --quiet mysql-connector-python
@@ -25,8 +26,19 @@ echo "calling test suite"
 echo "Starting standalone DatabendQuery(debug)"
 ./scripts/ci/deploy/databend-query-standalone.sh
 
+# only expected to get adopted in stateful tests
+if [[ "$ALLOW_SHARING" == "true" ]]; then
+	./scripts/ci/deploy/databend-query-sharing.sh
+fi
+
 SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
 cd "$SCRIPT_PATH/../../tests" || exit
 
 echo "Starting databend-test"
+
 ./databend-test $1 --mode 'standalone' --run-dir 1_stateful
+
+# only expected to get adopted in stateful tests
+if [[ "$ALLOW_SHARING" == "true" ]]; then
+	./databend-test $1 --mode 'standalone' --run-dir 3_stateful_sharing
+fi

scripts/ci/ci-run-stateful-tests-standalone-s3.sh

@@ -0,0 +1,112 @@
+# Usage:
+# databend-query -c databend_query_config_spec.toml
+
+[query]
+max_active_sessions = 256
+wait_timeout_mills = 5000
+
+# For flight rpc.
+flight_api_address = "0.0.0.0:59091"
+
+# Databend Query http address.
+# For admin RESET API.
+admin_api_address = "0.0.0.0:58080"
+
+# Databend Query metrics RESET API.
+metric_api_address = "0.0.0.0:57070"
+
+# Databend Query MySQL Handler.
+mysql_handler_host = "0.0.0.0"
+mysql_handler_port = 53307
+
+# Databend Query ClickHouse Handler.
+clickhouse_http_handler_host = "0.0.0.0"
+clickhouse_http_handler_port = 58124
+
+# Databend Query HTTP Handler.
+http_handler_host = "0.0.0.0"
+http_handler_port = 58000
+
+tenant_id = "shared_tenant"
+cluster_id = "test_cluster"
+
+table_engine_memory_enabled = true
+database_engine_github_enabled = true
+
+table_cache_enabled = true
+table_memory_cache_mb_size = 1024
+table_disk_cache_root = "_cache"
+table_disk_cache_mb_size = 10240
+table_cache_bloom_index_meta_count=3000
+table_cache_bloom_index_data_bytes=1073741824
+
+share_endpoint_address = "127.0.0.1:33003" # receive shared information from open sharing
+# [[query.users]]
+# name = "admin"
+# auth_type = "no_password"
+
+# [[query.users]]
+# name = "databend"
+# auth_type = "double_sha1_password"
+# # echo -n "databend" | sha1sum | cut -d' ' -f1 | xxd -r -p | sha1sum
+# auth_string = "3081f32caef285c232d066033c89a78d88a6d8a5"
+
+# [[query.users]]
+# name = "datafuselabs"
+# auth_type = "sha256_password"
+# #  echo -n "datafuselabs" | sha256sum
+# auth_string = "6db1a2f5da402b43c066fcadcbf78f04260b3236d9035e44dd463f21e29e6f3b"
+
+
+[log]
+
+[log.file]
+level = "ERROR"
+format = "text"
+dir = "./.databend/logs_1"
+
+[meta]
+endpoints = ["0.0.0.0:9191"]
+username = "root"
+password = "root"
+client_timeout_in_second = 60
+auto_sync_interval = 60
+
+# Storage config.
+[storage]
+# fs | s3 | azblob | obs | oss
+type = "fs"
+
+# Set a local folder to store your data.
+# Comment out this block if you're NOT using local file system as storage.
+[storage.fs]
+data_path = "./.databend/stateless_test_data"
+
+# To use S3-compatible object storage, uncomment this block and set your values.
+# [storage.s3]
+# bucket = "<your-bucket-name>"
+# endpoint_url = "<your-endpoint>"
+# access_key_id = "<your-key-id>"
+# secret_access_key = "<your-account-key>"
+# enable_virtual_host_style = false
+
+# To use Azure Blob storage, uncomment this block and set your values.
+# [storage.azblob]
+# endpoint_url = "https://<your-storage-account-name>.blob.core.windows.net"
+# container = "<your-azure-storage-container-name>"
+# account_name = "<your-storage-account-name>"
+# account_key = "<your-account-key>"
+
+# To use OBS object storage, uncomment this block and set your values.
+# [storage.obs]
+# bucket = "<your-bucket-name>"
+# endpoint_url = "<your-endpoint>"
+# access_key_id = "<your-key-id>"
+# secret_access_key = "<your-account-key>"
+
+# To use OSS object storage, uncomment this block and set your values.
+# [storage.oss]
+# bucket = "<your-bucket-name>"
+# endpoint_url = "<your-endpoint>"
+# access_key_id = "<your-key-id>"
+# access_key_secret = "<your-account-key>"

scripts/ci/deploy/config/databend-query-node-shared.toml

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright 2022 The Databend Authors.
+# SPDX-License-Identifier: Apache-2.0.
+set -e
+
+SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
+cd "$SCRIPT_PATH/../../.." || exit
+BUILD_PROFILE=${BUILD_PROFILE:-debug}
+
+echo "*************************************"
+echo "* Test on Databend openSharing endpoint    *"
+echo "* it will start a node from another tenant *"
+echo "* Please make sure that S3 backend         *"
+echo "* is ready, and configured properly.       *"
+echo "*************************************"
+echo "Start open-sharing..."
+export TENANT=test_tenant
+nohup target/${BUILD_PROFILE}/open-sharing &
+python3 scripts/ci/wait_tcp.py --timeout 5 --port 33003
+
+echo 'Start databend-query...'
+export STORAGE_S3_ROOT=shared
+
+nohup target/${BUILD_PROFILE}/databend-query -c scripts/ci/deploy/config/databend-query-node-shared.toml &
+
+python3 scripts/ci/wait_tcp.py --timeout 5 --port 53307

scripts/ci/deploy/databend-query-sharing.sh

@@ -10,6 +10,7 @@ BUILD_PROFILE=${BUILD_PROFILE:-debug}
 
 killall databend-query || true
 killall databend-meta || true
+killall open-sharing || true
 sleep 1
 
 for bin in databend-query databend-meta; do