Merge branch 'main' into fix/ISSUE-7684

Author: zhang2014

.github/actions/test_stateful_cluster_linux/action.yml

@@ -0,0 +1,58 @@
+name: "Test Stateful Cluster Linux"
+description: "Running stateful tests in cluster mode"
+inputs:
+  profile:
+    description: "The profile for this test"
+    required: true
+    default: "debug"
+  target:
+    description: ""
+    required: true
+    default: "x86_64-unknown-linux-gnu"
+runs:
+  using: "composite"
+  steps:
+    - name: Download artifact
+      uses: ./.github/actions/artifact_download
+      with:
+        profile: ${{ inputs.profile }}
+        sha: ${{ github.sha }}
+        target: ${{ inputs.target }}
+
+    - name: Minio Setup for (ubuntu-latest only)
+      shell: bash
+      run: |
+        docker run -d -p 9900:9000 --name minio \
+                -e "MINIO_ACCESS_KEY=minioadmin" \
+                -e "MINIO_SECRET_KEY=minioadmin" \
+                -v /tmp/data:/data \
+                -v /tmp/config:/root/.minio \
+                minio/minio server /data
+
+        export AWS_ACCESS_KEY_ID=minioadmin
+        export AWS_SECRET_ACCESS_KEY=minioadmin
+        export AWS_EC2_METADATA_DISABLED=true
+
+        aws --endpoint-url http://127.0.0.1:9900/ s3 mb s3://testbucket
+        aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data s3://testbucket/admin/data  --recursive
+        aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data/ontime_200.csv s3://testbucket/admin/data/ontime_200_v1.csv
+        aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data/ontime_200.parquet s3://testbucket/admin/data/ontime_200_v1.parquet
+
+    - name: Run Stateful Tests with Cluster mode (ubuntu-latest only)
+      shell: bash
+      run: |
+        ./scripts/ci/ci-run-stateful-tests-cluster-s3.sh
+
+    - name: Upload .stdout files
+      if: failure()
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-stateful-cluster-linux-stdouts
+        path: |
+           ./tests/suites/1_stateful/*/*.stdout
+
+    - name: Upload failure
+      if: failure()
+      uses: ./.github/actions/artifact_failure
+      with:
+        name: test-stateful-cluster-linux

.github/mergify.yml

@@ -10,9 +10,9 @@ queue_rules:
       - check-success~=^build_(aarch64|x86_64)_musl$
       - check-success=test_unit
       - check-success=test_metactl
-      - check-success=test_stateless_standalone_linux
-      - check-success=test_stateless_cluster_linux
-      - check-success=test_stateful_standalone_linux
+      # state tests
+      - check-success~=^test_(stateless|stateful)_(standalone|cluster)_linux$
+      # sqllogic tests
       - check-success=test_sqllogic_base_standalone_linux
       - check-success=test_sqllogic_ydb_standalone_linux
       - check-success=test_sqllogic_base_cluster_linux

.github/workflows/dev-linux.yml

@@ -112,22 +112,6 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/test_meta_cluster
 
-  test_stateless_standalone_linux:
-    timeout-minutes: 10
-    runs-on: [self-hosted, X64, Linux, development]
-    needs: build_gnu
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/test_stateless_standalone_linux
-
-  test_stateless_cluster_linux:
-    timeout-minutes: 15
-    runs-on: [self-hosted, X64, Linux, development]
-    needs: build_gnu
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/test_stateless_cluster_linux
-
   test_sqllogic_standalone_linux:
     timeout-minutes: 30
     name: test_sqllogic_${{ matrix.dirs }}_standalone_linux
@@ -161,10 +145,26 @@ jobs:
         with:
           dirs: ${{ matrix.dirs }}
 
-  test_stateful_standalone_linux:
-    timeout-minutes: 10
+  test_stateless_standalone_linux:
+    timeout-minutes: 20
+    runs-on: [self-hosted, X64, Linux, development]
+    needs: build_gnu
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/test_stateless_standalone_linux
+
+  test_stateless_cluster_linux:
+    timeout-minutes: 20
     runs-on: [self-hosted, X64, Linux, development]
     needs: build_gnu
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/test_stateless_cluster_linux
+
+  test_stateful_standalone_linux:
+    timeout-minutes: 20
+    runs-on: [ self-hosted, X64, Linux, development ]
+    needs: build_gnu
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/test_stateful_standalone_linux

Cargo.lock

@@ -0,0 +1,30 @@
+#!/bin/bash
+# Copyright 2020-2021 The Databend Authors.
+# SPDX-License-Identifier: Apache-2.0.
+
+echo "*************************************"
+echo "* Setting STORAGE_TYPE to S3.       *"
+echo "*                                   *"
+echo "* Please make sure that S3 backend  *"
+echo "* is ready, and configured properly.*"
+echo "*************************************"
+export STORAGE_TYPE=s3
+export STORAGE_S3_BUCKET=testbucket
+export STORAGE_S3_ROOT=admin
+export STORAGE_S3_ENDPOINT_URL=http://127.0.0.1:9900
+export STORAGE_S3_ACCESS_KEY_ID=minioadmin
+export STORAGE_S3_SECRET_ACCESS_KEY=minioadmin
+export STORAGE_ALLOW_INSECURE=true
+
+echo "Install dependence"
+python3 -m pip install --quiet mysql-connector-python
+
+echo "calling test suite"
+echo "Starting Cluster databend-query"
+./scripts/ci/deploy/databend-query-cluster-3-nodes.sh
+
+SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
+cd "$SCRIPT_PATH/../../tests" || exit
+
+echo "Starting databend-test"
+./databend-test --mode 'cluster' --run-dir 1_stateful

scripts/ci/ci-run-stateful-tests-cluster-s3.sh

@@ -15,6 +15,7 @@ test = false
 common-base = { path = "../base" }
 
 # Crates.io dependencies
+ahash = "0.7.6"
 ordered-float = { git = "https://github.com/andylokandy/rust-ordered-float.git", branch = "as", features = ["serde"] }
 primitive-types = "0.11.1"
 

src/common/hashtable/Cargo.toml

@@ -30,4 +30,8 @@ impl<Key: HashTableKeyable, Grower: HashTableGrower, Allocator: AllocatorTrait +
             self.insert_key(value.get_key(), &mut inserted);
         }
     }
+
+    pub fn contains(&self, key: &Key) -> bool {
+        self.find_key(key).is_some()
+    }
 }

src/common/hashtable/src/hash_set.rs

@@ -126,7 +126,7 @@ impl HashTableKeyable for OrderedFloat<f32> {
 
     #[inline(always)]
     fn is_zero(&self) -> bool {
-        self.is_nan()
+        self.0 == 0.0
     }
 
     #[inline(always)]
@@ -144,7 +144,7 @@ impl HashTableKeyable for OrderedFloat<f64> {
 
     #[inline(always)]
     fn is_zero(&self) -> bool {
-        self.is_nan()
+        self.0 == 0.0
     }
 
     #[inline(always)]

src/common/hashtable/src/hash_table_key.rs

@@ -0,0 +1,79 @@
+// Copyright 2021 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::hash::Hasher;
+
+use ahash::AHasher;
+
+use super::HashTableKeyable;
+
+#[derive(Clone, Copy)]
+pub struct KeysRef {
+    pub length: usize,
+    pub address: usize,
+}
+
+impl KeysRef {
+    pub fn create(address: usize, length: usize) -> KeysRef {
+        KeysRef { length, address }
+    }
+}
+
+impl Eq for KeysRef {}
+
+impl PartialEq for KeysRef {
+    fn eq(&self, other: &Self) -> bool {
+        if self.length != other.length {
+            return false;
+        }
+
+        unsafe {
+            let self_value = std::slice::from_raw_parts(self.address as *const u8, self.length);
+            let other_value = std::slice::from_raw_parts(other.address as *const u8, other.length);
+            self_value == other_value
+        }
+    }
+}
+
+impl HashTableKeyable for KeysRef {
+    const BEFORE_EQ_HASH: bool = true;
+
+    fn is_zero(&self) -> bool {
+        self.length == 0
+    }
+
+    fn fast_hash(&self) -> u64 {
+        unsafe {
+            // TODO(Winter) We need more efficient hash algorithm
+            let value = std::slice::from_raw_parts(self.address as *const u8, self.length);
+
+            let mut hasher = AHasher::default();
+            hasher.write(value);
+            hasher.finish()
+        }
+    }
+
+    fn set_key(&mut self, new_value: &Self) {
+        self.length = new_value.length;
+        self.address = new_value.address;
+    }
+}
+
+impl std::hash::Hash for KeysRef {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        let self_value =
+            unsafe { std::slice::from_raw_parts(self.address as *const u8, self.length) };
+        self_value.hash(state);
+    }
+}

src/common/hashtable/src/keys_ref.rs

@@ -36,8 +36,11 @@ mod hash_table_entity;
 mod hash_table_grower;
 mod hash_table_iter;
 mod hash_table_key;
+mod keys_ref;
 mod two_level_hash_table;
 
+pub use keys_ref::KeysRef;
+
 #[cfg(not(target_os = "linux"))]
 type HashTableAllocator = common_base::mem_allocator::JEAllocator;
 #[cfg(target_os = "linux")]