add end to end tests on sharing

Author: ZhiHanZ

scripts/build/build-debug.sh

@@ -8,5 +8,5 @@ SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
 cd "$SCRIPT_PATH/../.." || exit
 
 echo "Build(DEBUG) start..."
-cargo build --bin=databend-query --bin=databend-meta --bin=databend-metactl
+cargo build --bin=databend-query --bin=databend-meta --bin=databend-metactl --bin=open-sharing
 echo "All done..."

scripts/build/build-release.sh

@@ -8,5 +8,5 @@ SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
 cd "$SCRIPT_PATH/../.." || exit
 
 echo "Build(RELEASE) start..."
-cargo build --bin=databend-query --bin=databend-meta --bin=databend-metactl --release
+cargo build --bin=databend-query --bin=databend-meta --bin=databend-metactl --bin=open-sharing --release
 echo "All done..."

scripts/ci/ci-run-stateful-tests-standalone-s3.sh

@@ -17,6 +17,7 @@ export STORAGE_S3_ENDPOINT_URL=http://127.0.0.1:9900
 export STORAGE_S3_ACCESS_KEY_ID=minioadmin
 export STORAGE_S3_SECRET_ACCESS_KEY=minioadmin
 export STORAGE_ALLOW_INSECURE=true
+export ALLOW_SHARING=true
 
 echo "Install dependence"
 python3 -m pip install --quiet mysql-connector-python
@@ -25,6 +26,8 @@ echo "calling test suite"
 echo "Starting standalone DatabendQuery(debug)"
 ./scripts/ci/deploy/databend-query-standalone.sh
 
+export ALLOW_SHARING=false
+
 SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
 cd "$SCRIPT_PATH/../../tests" || exit
 

scripts/ci/deploy/config/databend-query-node-shared.toml

@@ -0,0 +1,112 @@
+# Usage:
+# databend-query -c databend_query_config_spec.toml
+
+[query]
+max_active_sessions = 256
+wait_timeout_mills = 5000
+
+# For flight rpc.
+flight_api_address = "0.0.0.0:59091"
+
+# Databend Query http address.
+# For admin RESET API.
+admin_api_address = "0.0.0.0:58080"
+
+# Databend Query metrics RESET API.
+metric_api_address = "0.0.0.0:57070"
+
+# Databend Query MySQL Handler.
+mysql_handler_host = "0.0.0.0"
+mysql_handler_port = 53307
+
+# Databend Query ClickHouse Handler.
+clickhouse_http_handler_host = "0.0.0.0"
+clickhouse_http_handler_port = 58124
+
+# Databend Query HTTP Handler.
+http_handler_host = "0.0.0.0"
+http_handler_port = 58000
+
+tenant_id = "shared_tenant"
+cluster_id = "test_cluster"
+
+table_engine_memory_enabled = true
+database_engine_github_enabled = true
+
+table_cache_enabled = true
+table_memory_cache_mb_size = 1024
+table_disk_cache_root = "_cache"
+table_disk_cache_mb_size = 10240
+table_cache_bloom_index_meta_count=3000
+table_cache_bloom_index_data_bytes=1073741824
+
+share_endpoint_address = "127.0.0.1:33003" # receive shared information from open sharing
+# [[query.users]]
+# name = "admin"
+# auth_type = "no_password"
+
+# [[query.users]]
+# name = "databend"
+# auth_type = "double_sha1_password"
+# # echo -n "databend" | sha1sum | cut -d' ' -f1 | xxd -r -p | sha1sum
+# auth_string = "3081f32caef285c232d066033c89a78d88a6d8a5"
+
+# [[query.users]]
+# name = "datafuselabs"
+# auth_type = "sha256_password"
+# #  echo -n "datafuselabs" | sha256sum
+# auth_string = "6db1a2f5da402b43c066fcadcbf78f04260b3236d9035e44dd463f21e29e6f3b"
+
+
+[log]
+
+[log.file]
+level = "ERROR"
+format = "text"
+dir = "./.databend/logs_1"
+
+[meta]
+endpoints = ["0.0.0.0:9191"]
+username = "root"
+password = "root"
+client_timeout_in_second = 60
+auto_sync_interval = 60
+
+# Storage config.
+[storage]
+# fs | s3 | azblob | obs | oss
+type = "fs"
+
+# Set a local folder to store your data.
+# Comment out this block if you're NOT using local file system as storage.
+[storage.fs]
+data_path = "./.databend/stateless_test_data"
+
+# To use S3-compatible object storage, uncomment this block and set your values.
+# [storage.s3]
+# bucket = "<your-bucket-name>"
+# endpoint_url = "<your-endpoint>"
+# access_key_id = "<your-key-id>"
+# secret_access_key = "<your-account-key>"
+# enable_virtual_host_style = false
+
+# To use Azure Blob storage, uncomment this block and set your values.
+# [storage.azblob]
+# endpoint_url = "https://<your-storage-account-name>.blob.core.windows.net"
+# container = "<your-azure-storage-container-name>"
+# account_name = "<your-storage-account-name>"
+# account_key = "<your-account-key>"
+
+# To use OBS object storage, uncomment this block and set your values.
+# [storage.obs]
+# bucket = "<your-bucket-name>"
+# endpoint_url = "<your-endpoint>"
+# access_key_id = "<your-key-id>"
+# secret_access_key = "<your-account-key>"
+
+# To use OSS object storage, uncomment this block and set your values.
+# [storage.oss]
+# bucket = "<your-bucket-name>"
+# endpoint_url = "<your-endpoint>"
+# access_key_id = "<your-key-id>"
+# access_key_secret = "<your-account-key>"

scripts/ci/deploy/databend-query-standalone.sh

@@ -10,6 +10,7 @@ BUILD_PROFILE=${BUILD_PROFILE:-debug}
 
 killall databend-query || true
 killall databend-meta || true
+killall open-sharing || true
 sleep 1
 
 for bin in databend-query databend-meta; do
@@ -29,3 +30,24 @@ nohup target/${BUILD_PROFILE}/databend-query -c scripts/ci/deploy/config/databen
 
 echo "Waiting on databend-query 10 seconds..."
 python3 scripts/ci/wait_tcp.py --timeout 5 --port 3307
+
+# only expected to get adopted in stateful tests
+if [[ "$ALLOW_SHARING" == "true" ]]; then
+	echo "*************************************"
+	echo "* Test on Databend openSharing endpoint    *"
+	echo "* it will start a node from another tenant *"
+	echo "* Please make sure that S3 backend         *"
+	echo "* is ready, and configured properly.       *"
+	echo "*************************************"
+	echo "Start open-sharing..."
+	export TENANT=test_tenant
+	nohup target/${BUILD_PROFILE}/open-sharing &
+	python3 scripts/ci/wait_tcp.py --timeout 5 --port 33003
+
+	echo 'Start databend-query...'
+	export STORAGE_S3_ROOT=shared
+
+	nohup target/${BUILD_PROFILE}/databend-query -c scripts/ci/deploy/config/databend-query-node-shared.toml &
+
+	python3 scripts/ci/wait_tcp.py --timeout 5 --port 53307
+fi

src/binaries/Cargo.toml

@@ -51,14 +51,14 @@ anyhow = { workspace = true }
 clap = { workspace = true }
 limits-rs = "0.1.0"
 openraft = { workspace = true }
+poem = { version = "1", features = ["rustls", "multipart", "compression"] }
 sentry = "0.27.0"
 serde = { workspace = true }
 serde_json = { workspace = true }
 tokio-stream = "0.1.10"
 tonic = "0.8.1"
 tracing = "0.1.36"
 url = "2.3.1"
-poem = { version = "1", features = ["rustls", "multipart", "compression"] }
 
 [[bin]]
 name = "databend-meta"

src/binaries/opensharing/main.rs

@@ -12,19 +12,28 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use poem::{EndpointExt, Route, Server};
 use common_base::base::tokio;
 use poem::listener::TcpListener;
+use poem::EndpointExt;
+use poem::Route;
+use poem::Server;
+use sharing_endpoint::configs::Config;
 use sharing_endpoint::handlers::presign_files;
 use sharing_endpoint::middlewares::SharingAuth;
-use sharing_endpoint::configs::Config;
 use sharing_endpoint::services::SharingServices;
 #[tokio::main]
-async fn main() ->Result<(), std::io::Error> {
+async fn main() -> Result<(), std::io::Error> {
     let config = Config::load().expect("cfgs");
     println!("config: {:?}", config);
-    SharingServices::init(config).await.expect("failed to init sharing service");
-    let app = Route::new().at("/tenant/:tenant_id/:share_name/table/:table_name/presign", poem::post(presign_files)).with(SharingAuth);
+    SharingServices::init(config)
+        .await
+        .expect("failed to init sharing service");
+    let app = Route::new()
+        .at(
+            "/tenant/:tenant_id/:share_name/table/:table_name/presign",
+            poem::post(presign_files),
+        )
+        .with(SharingAuth);
 
     // TODO(zhihanz): remove the hard coded port into a config
     Server::new(TcpListener::bind("127.0.0.1:33003"))

src/query/config/src/lib.rs

@@ -32,8 +32,6 @@ pub use inner::CatalogHiveConfig;
 pub use inner::Config;
 pub use inner::QueryConfig;
 pub use inner::ThriftProtocol;
-
 pub use outer_v0::StorageConfig;
-
 pub use version::DATABEND_COMMIT_VERSION;
 pub use version::QUERY_SEMVER;

src/query/sharing-endpoint/Cargo.toml

@@ -10,22 +10,21 @@ edition = { workspace = true }
 doctest = false
 
 [dependencies]
-common-exception = { path = "../../common/exception" }
 common-base = { path = "../../common/base" }
-common-storage = { path = "../../common/storage" }
 common-config = { path = "../../query/config" }
-uuid = { version = "1.1.2", features = ["serde", "v4"] }
+common-exception = { path = "../../common/exception" }
+common-storage = { path = "../../common/storage" }
 once_cell = "1.15.0"
+uuid = { version = "1.1.2", features = ["serde", "v4"] }
 
 time = { version = "0.3", features = ["serde"] }
 
 anyhow = { workspace = true }
-serde = { workspace = true }
-serde_json = { workspace = true }
-thiserror = { version = "1" }
+base64 = "0.13.0"
+clap = { workspace = true }
 opendal = "0.22.2"
 poem = { version = "1", features = ["rustls", "multipart", "compression"] }
-clap = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
 serfig = "0.0.3"
-base64 = "0.13.0"
-
+thiserror = { version = "1" }