Skip to content

Commit bd95e0e

Browse files
BohuTANGBohuTANG
authored
Merge pull request #7680 from Xuanwo/add-stateful-test-for-cluster
ci: Add stateful tests for cluster
2 parents 1b0d982 + 3939283 commit bd95e0e

File tree

4 files changed

+103
-22
lines changed

4 files changed

+103
-22
lines changed

‎.github/actions/test_stateful_cluster_linux/action.yml

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
name: "Test Stateful Cluster Linux"
2+
description: "Running stateful tests in cluster mode"
3+
inputs:
4+
profile:
5+
description: "The profile for this test"
6+
required: true
7+
default: "debug"
8+
target:
9+
description: ""
10+
required: true
11+
default: "x86_64-unknown-linux-gnu"
12+
runs:
13+
using: "composite"
14+
steps:
15+
- name: Download artifact
16+
uses: ./.github/actions/artifact_download
17+
with:
18+
profile: ${{ inputs.profile }}
19+
sha: ${{ github.sha }}
20+
target: ${{ inputs.target }}
21+
22+
- name: Minio Setup for (ubuntu-latest only)
23+
shell: bash
24+
run: |
25+
docker run -d -p 9900:9000 --name minio \
26+
-e "MINIO_ACCESS_KEY=minioadmin" \
27+
-e "MINIO_SECRET_KEY=minioadmin" \
28+
-v /tmp/data:/data \
29+
-v /tmp/config:/root/.minio \
30+
minio/minio server /data
31+
32+
export AWS_ACCESS_KEY_ID=minioadmin
33+
export AWS_SECRET_ACCESS_KEY=minioadmin
34+
export AWS_EC2_METADATA_DISABLED=true
35+
36+
aws --endpoint-url http://127.0.0.1:9900/ s3 mb s3://testbucket
37+
aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data s3://testbucket/admin/data --recursive
38+
aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data/ontime_200.csv s3://testbucket/admin/data/ontime_200_v1.csv
39+
aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data/ontime_200.parquet s3://testbucket/admin/data/ontime_200_v1.parquet
40+
41+
- name: Run Stateful Tests with Cluster mode (ubuntu-latest only)
42+
shell: bash
43+
run: |
44+
./scripts/ci/ci-run-stateful-tests-cluster-s3.sh
45+
46+
- name: Upload .stdout files
47+
if: failure()
48+
uses: actions/upload-artifact@v3
49+
with:
50+
name: test-stateful-cluster-linux-stdouts
51+
path: |
52+
./tests/suites/1_stateful/*/*.stdout
53+
54+
- name: Upload failure
55+
if: failure()
56+
uses: ./.github/actions/artifact_failure
57+
with:
58+
name: test-stateful-cluster-linux

‎.github/mergify.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ queue_rules:
1010
- check-success~=^build_(aarch64|x86_64)_musl$
1111
- check-success=test_unit
1212
- check-success=test_metactl
13-
- check-success=test_stateless_standalone_linux
14-
- check-success=test_stateless_cluster_linux
15-
- check-success=test_stateful_standalone_linux
13+
# state tests
14+
- check-success~=^test_(stateless|stateful)_(standalone|cluster)_linux$
15+
# sqllogic tests
1616
- check-success=test_sqllogic_base_standalone_linux
1717
- check-success=test_sqllogic_ydb_standalone_linux
1818
- check-success=test_sqllogic_base_cluster_linux

‎.github/workflows/dev-linux.yml

Lines changed: 12 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -112,22 +112,6 @@ jobs:
112112
- uses: actions/checkout@v3
113113
- uses: ./.github/actions/test_meta_cluster
114114

115-
test_stateless_standalone_linux:
116-
timeout-minutes: 10
117-
runs-on: [self-hosted, X64, Linux, development]
118-
needs: build_gnu
119-
steps:
120-
- uses: actions/checkout@v3
121-
- uses: ./.github/actions/test_stateless_standalone_linux
122-
123-
test_stateless_cluster_linux:
124-
timeout-minutes: 15
125-
runs-on: [self-hosted, X64, Linux, development]
126-
needs: build_gnu
127-
steps:
128-
- uses: actions/checkout@v3
129-
- uses: ./.github/actions/test_stateless_cluster_linux
130-
131115
test_sqllogic_standalone_linux:
132116
timeout-minutes: 30
133117
name: test_sqllogic_${{ matrix.dirs }}_standalone_linux
@@ -161,13 +145,22 @@ jobs:
161145
with:
162146
dirs: ${{ matrix.dirs }}
163147

164-
test_stateful_standalone_linux:
165-
timeout-minutes: 10
148+
test_state_linux:
149+
timeout-minutes: 20
150+
name: test_${{ matrix.state }}_${{ matrix.mode }}_linux
166151
runs-on: [self-hosted, X64, Linux, development]
167152
needs: build_gnu
153+
strategy:
154+
matrix:
155+
state:
156+
- "stateful"
157+
- "stateless"
158+
mode:
159+
- "standalone"
160+
- "cluster"
168161
steps:
169162
- uses: actions/checkout@v3
170-
- uses: ./.github/actions/test_stateful_standalone_linux
163+
- uses: ./.github/actions/test_${{ matrix.state }}_${{ matrix.mode }}_linux
171164

172165
test_fuzz_standalone_linux:
173166
timeout-minutes: 10

‎scripts/ci/ci-run-stateful-tests-cluster-s3.sh

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
# Copyright 2020-2021 The Databend Authors.
3+
# SPDX-License-Identifier: Apache-2.0.
4+
5+
echo "*************************************"
6+
echo "* Setting STORAGE_TYPE to S3. *"
7+
echo "* *"
8+
echo "* Please make sure that S3 backend *"
9+
echo "* is ready, and configured properly.*"
10+
echo "*************************************"
11+
export STORAGE_TYPE=s3
12+
export STORAGE_S3_BUCKET=testbucket
13+
export STORAGE_S3_ROOT=admin
14+
export STORAGE_S3_ENDPOINT_URL=http://127.0.0.1:9900
15+
export STORAGE_S3_ACCESS_KEY_ID=minioadmin
16+
export STORAGE_S3_SECRET_ACCESS_KEY=minioadmin
17+
export STORAGE_ALLOW_INSECURE=true
18+
19+
echo "Install dependence"
20+
python3 -m pip install --quiet mysql-connector-python
21+
22+
echo "calling test suite"
23+
echo "Starting Cluster databend-query"
24+
./scripts/ci/deploy/databend-query-cluster-3-nodes.sh
25+
26+
SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
27+
cd "$SCRIPT_PATH/../../tests" || exit
28+
29+
echo "Starting databend-test"
30+
./databend-test --mode 'cluster' --run-dir 1_stateful

0 commit comments

Comments
 (0)