ci: fix ttc test (#17807)

everpcpc · web-flow · commit f384932641eb · 2025-04-21T16:33:04.000+08:00
diff --git a/.github/actions/build_linux/action.yml b/.github/actions/build_linux/action.yml
@@ -90,7 +90,7 @@ runs:
     - name: Check Binary Info
       shell: bash
       run: |
-        grep ' Compiled in ' target/sccache.log
+        grep ' Compiled in ' target/sccache.log || true
         readelf -p .comment ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true
         ldd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true
 
diff --git a/.github/actions/pack_binaries/action.yml b/.github/actions/pack_binaries/action.yml
@@ -72,9 +72,9 @@ runs:
         name: sha256sums-${{ inputs.category }}-${{ inputs.target }}
         path: sha256-${{ steps.pack_binaries.outputs.pkg_name }}.txt
         retention-days: 1
-    - name: post binaries
-      uses: actions/upload-artifact@v4
-      with:
-        name: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
-        path: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
-        retention-days: 1
+    # - name: post binaries
+    #   uses: actions/upload-artifact@v4
+    #   with:
+    #     name: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
+    #     path: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
+    #     retention-days: 1
diff --git a/.github/actions/setup_build_tool/action.yml b/.github/actions/setup_build_tool/action.yml
@@ -49,7 +49,7 @@ runs:
         export SCCACHE_S3_KEY_PREFIX="cache/sccache/"
         export SCCACHE_S3_USE_SSL="true"
         export SCCACHE_ERROR_LOG=/workspace/target/sccache.log
-        export SCCACHE_LOG=debug
+        export SCCACHE_LOG=info
         EOF
             ;;
 
@@ -62,7 +62,7 @@ runs:
         export SCCACHE_GCS_BUCKET=databend-ci
         export SCCACHE_GCS_KEY_PREFIX="cache/sccache/"
         export SCCACHE_ERROR_LOG=/workspace/target/sccache.log
-        export SCCACHE_LOG=debug
+        export SCCACHE_LOG=info
         EOF
             ;;
 
@@ -101,6 +101,13 @@ runs:
       shell: bash
       run: df -h
 
+    - name: check system info
+      shell: bash
+      run: |
+        uname -a
+        grep "^model" /proc/cpuinfo
+        free -h
+
     # - uses: everpcpc/actions-cache@v1
     #   if: env.RUNNER_PROVIDER == 'aws'
     #   env:
diff --git a/.github/actions/setup_test/action.yml b/.github/actions/setup_test/action.yml
@@ -49,3 +49,16 @@ runs:
       shell: bash
       run: |
         echo "LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${{ env.LD_LIBRARY_PATH }}" >> $GITHUB_ENV
+
+    - name: check docker info
+      shell: bash
+      run: |
+        docker info
+        docker version
+        docker ps -a
+        docker images
+
+    - name: setup env
+      shell: bash
+      run: |
+        echo "RUST_LOG=info" >> $GITHUB_ENV
diff --git a/.github/actions/test_unit/action.yml b/.github/actions/test_unit/action.yml
@@ -12,7 +12,7 @@ runs:
     - shell: bash
       run: |
         cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps nextest run --no-fail-fast --hide-progress-bar
-        grep ' Compiled in ' target/sccache.log
+        grep ' Compiled in ' target/sccache.log || true
       env:
         RUST_TEST_THREADS: "8"
         RUST_LOG: ERROR
diff --git a/scripts/ci/deploy/config/databend-meta-node-1.toml b/scripts/ci/deploy/config/databend-meta-node-1.toml
@@ -1,20 +1,28 @@
 # Usage:
 # databend-meta -c databend-meta-node-1.toml
 
-log_dir                 = "./.databend/logs1"
 admin_api_address       = "0.0.0.0:28101"
 grpc_api_address        = "0.0.0.0:9191"
 # databend-query fetch this address to update its databend-meta endpoints list,
 # in case databend-meta cluster changes.
 grpc_api_advertise_host = "127.0.0.1"
 
+[log]
+[log.stderr]
+  on = false
+[log.file]
+  on = true
+  level = "INFO"
+  format = "json"
+  dir = "./.databend/logs1"
+
 [raft_config]
 id            = 1
 raft_dir      = "./.databend/meta1"
 raft_api_port = 28103
 
 # Assign raft_{listen|advertise}_host in test config.
-# This allows you to catch a bug in unit tests when something goes wrong in raft meta nodes communication. 
+# This allows you to catch a bug in unit tests when something goes wrong in raft meta nodes communication.
 raft_listen_host = "127.0.0.1"
 raft_advertise_host = "localhost"
 
diff --git a/scripts/ci/deploy/config/databend-meta-node-2.toml b/scripts/ci/deploy/config/databend-meta-node-2.toml
@@ -1,13 +1,22 @@
 # Usage:
 # databend-meta -c databend-meta-node-2.toml
 
-log_dir                 = "./.databend/logs2"
 admin_api_address       = "0.0.0.0:28201"
 grpc_api_address        = "0.0.0.0:28202"
 # databend-query fetch this address to update its databend-meta endpoints list,
 # in case databend-meta cluster changes.
 grpc_api_advertise_host = "127.0.0.1"
 
+
+[log]
+[log.stderr]
+  on = false
+[log.file]
+  on = true
+  level = "INFO"
+  format = "json"
+  dir = "./.databend/logs2"
+
 [raft_config]
 id            = 2
 raft_dir      = "./.databend/meta2"
diff --git a/scripts/ci/deploy/config/databend-meta-node-3.toml b/scripts/ci/deploy/config/databend-meta-node-3.toml
@@ -1,13 +1,21 @@
 # Usage:
 # databend-meta -c databend-meta-node-3.toml
 
-log_dir                 = "./.databend/logs3"
 admin_api_address       = "0.0.0.0:28301"
 grpc_api_address        = "0.0.0.0:28302"
 # databend-query fetch this address to update its databend-meta endpoints list,
 # in case databend-meta cluster changes.
 grpc_api_advertise_host = "127.0.0.1"
 
+[log]
+[log.stderr]
+  on = false
+[log.file]
+  on = true
+  level = "INFO"
+  format = "json"
+  dir = "./.databend/logs3"
+
 [raft_config]
 id            = 3
 raft_dir      = "./.databend/meta3"
diff --git a/scripts/ci/deploy/config/databend-meta-node-for-bendsave.toml b/scripts/ci/deploy/config/databend-meta-node-for-bendsave.toml
@@ -1,16 +1,24 @@
 # Usage:
-# databend-meta -c databend-meta-node-1.toml
+# databend-meta -c databend-meta-node-for-bendsave.toml
 
-log_dir                 = "./.databend/logs1"
 admin_api_address       = "0.0.0.0:28101"
 grpc_api_address        = "0.0.0.0:9191"
 # databend-query fetch this address to update its databend-meta endpoints list,
 # in case databend-meta cluster changes.
 grpc_api_advertise_host = "127.0.0.1"
 
+[log]
+[log.stderr]
+  on = false
+[log.file]
+  on = true
+  level = "INFO"
+  format = "json"
+  dir = "./.databend/logs-bendsave"
+
 [raft_config]
 id            = 1
-raft_dir      = "/tmp/databend/meta1"
+raft_dir      = "/tmp/databend/meta-bendsave"
 raft_api_port = 28103
 
 # Assign raft_{listen|advertise}_host in test config.
diff --git a/scripts/ci/deploy/databend-query-management-mode.sh b/scripts/ci/deploy/databend-query-management-mode.sh
@@ -23,7 +23,7 @@ done
 sleep 1
 
 echo 'Start databend-meta...'
-nohup target/${BUILD_PROFILE}/databend-meta --single --log-level=INFO &
+nohup target/${BUILD_PROFILE}/databend-meta -c scripts/ci/deploy/config/databend-meta-node-1.toml &
 echo "Waiting on databend-meta 10 seconds..."
 python3 scripts/ci/wait_tcp.py --timeout 30 --port 9191
 
diff --git a/scripts/ci/deploy/databend-query-standalone-for-bendsave.sh b/scripts/ci/deploy/databend-query-standalone-for-bendsave.sh
@@ -23,7 +23,7 @@ done
 sleep 1
 
 echo 'Start databend-meta...'
-nohup target/${BUILD_PROFILE}/databend-meta -c scripts/ci/deploy/config/databend-meta-node-for-bendsave.toml --single --log-level=INFO &
+nohup target/${BUILD_PROFILE}/databend-meta -c scripts/ci/deploy/config/databend-meta-node-for-bendsave.toml &
 echo "Waiting on databend-meta 10 seconds..."
 python3 scripts/ci/wait_tcp.py --timeout 30 --port 9191
 
diff --git a/scripts/ci/deploy/databend-query-standalone-hive.sh b/scripts/ci/deploy/databend-query-standalone-hive.sh
@@ -23,7 +23,7 @@ done
 sleep 1
 
 echo 'Start databend-meta...'
-nohup target/${BUILD_PROFILE}/databend-meta --single --log-level=INFO &
+nohup target/${BUILD_PROFILE}/databend-meta -c scripts/ci/deploy/config/databend-meta-node-1.toml &
 echo "Waiting on databend-meta 10 seconds..."
 python3 scripts/ci/wait_tcp.py --timeout 30 --port 9191
 
diff --git a/scripts/ci/deploy/databend-query-standalone-logging.sh b/scripts/ci/deploy/databend-query-standalone-logging.sh
@@ -23,7 +23,7 @@ done
 sleep 1
 
 echo 'Start databend-meta...'
-nohup target/${BUILD_PROFILE}/databend-meta --single --log-level=INFO &
+nohup target/${BUILD_PROFILE}/databend-meta -c scripts/ci/deploy/config/databend-meta-node-1.toml &
 echo "Waiting on databend-meta 10 seconds..."
 python3 scripts/ci/wait_tcp.py --timeout 30 --port 9191
 
diff --git a/scripts/ci/deploy/databend-query-standalone-native.sh b/scripts/ci/deploy/databend-query-standalone-native.sh
@@ -23,7 +23,7 @@ done
 sleep 1
 
 echo 'Start databend-meta...'
-nohup target/${BUILD_PROFILE}/databend-meta --single --log-level=INFO &
+nohup target/${BUILD_PROFILE}/databend-meta -c scripts/ci/deploy/config/databend-meta-node-1.toml &
 echo "Waiting on databend-meta 10 seconds..."
 python3 scripts/ci/wait_tcp.py --timeout 30 --port 9191
 
diff --git a/scripts/ci/deploy/databend-query-standalone.sh b/scripts/ci/deploy/databend-query-standalone.sh
@@ -23,7 +23,7 @@ done
 sleep 1
 
 echo 'Start databend-meta...'
-nohup target/${BUILD_PROFILE}/databend-meta --single --log-level=INFO &
+nohup target/${BUILD_PROFILE}/databend-meta -c scripts/ci/deploy/config/databend-meta-node-1.toml &
 echo "Waiting on databend-meta 10 seconds..."
 python3 scripts/ci/wait_tcp.py --timeout 30 --port 9191
 
diff --git a/src/common/tracing/src/config.rs b/src/common/tracing/src/config.rs
@@ -100,15 +100,8 @@ impl Display for StderrConfig {
     fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
         write!(
             f,
-            "enabled={}{}, level={}, format={}",
-            self.on,
-            if !self.on {
-                "(To enable: LOG_STDERR_ON=true or RUST_LOG=info)"
-            } else {
-                ""
-            },
-            self.level,
-            self.format,
+            "enabled={}, level={}, format={}",
+            self.on, self.level, self.format,
         )
     }
 }
diff --git a/src/meta/service/src/configs/outer_v0.rs b/src/meta/service/src/configs/outer_v0.rs
@@ -809,10 +809,8 @@ impl Default for StderrLogConfig {
 impl Into<InnerStderrLogConfig> for StderrLogConfig {
     fn into(self) -> InnerStderrLogConfig {
         InnerStderrLogConfig {
-            on: std::env::var("LOG_STDERR_ON").as_deref() == Ok("true")
-                || std::env::var("RUST_LOG").is_ok()
-                || self.stderr_on,
-            level: std::env::var("RUST_LOG").unwrap_or(self.stderr_level),
+            on: self.stderr_on,
+            level: self.stderr_level,
             format: self.stderr_format,
         }
     }
diff --git a/tests/sqllogictests/src/util.rs b/tests/sqllogictests/src/util.rs
@@ -19,6 +19,7 @@ use std::path::PathBuf;
 use std::time::Duration;
 use std::time::Instant;
 
+use bollard::container::ListContainersOptions;
 use bollard::container::RemoveContainerOptions;
 use bollard::Docker;
 use clap::Parser;
@@ -27,6 +28,7 @@ use serde::Deserialize;
 use serde::Serialize;
 use serde_json::Value;
 use testcontainers::core::client::docker_client_instance;
+use testcontainers::core::logs::consumer::logging_consumer::LoggingConsumer;
 use testcontainers::core::IntoContainerPort;
 use testcontainers::core::WaitFor;
 use testcontainers::runners::AsyncRunner;
@@ -44,7 +46,7 @@ use crate::error::DSqlLogicTestError;
 use crate::error::Result;
 
 const CONTAINER_RETRY_TIMES: usize = 3;
-const CONTAINER_STARTUP_TIMEOUT_SECONDS: u64 = 180;
+const CONTAINER_STARTUP_TIMEOUT_SECONDS: u64 = 60;
 const CONTAINER_TIMEOUT_SECONDS: u64 = 300;
 
 #[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
@@ -262,22 +264,24 @@ pub async fn run_ttc_container(
     let container_name = format!("databend-ttc-{}-{}", port, x);
     let start = Instant::now();
     println!("Starting container {container_name}");
+    let dsn = format!(
+        "databend://root:@127.0.0.1:{}?sslmode=disable",
+        http_server_port
+    );
 
     let mut i = 1;
     loop {
+        let log_consumer = LoggingConsumer::new();
+
         let container_res = GenericImage::new(image, tag)
             .with_exposed_port(port.tcp())
             .with_wait_for(WaitFor::message_on_stdout("Ready to accept connections"))
+            .with_startup_timeout(Duration::from_secs(CONTAINER_STARTUP_TIMEOUT_SECONDS))
             .with_network("host")
-            .with_env_var(
-                "DATABEND_DSN",
-                format!(
-                    "databend://root:@127.0.0.1:{}?sslmode=disable",
-                    http_server_port
-                ),
-            )
+            .with_env_var("DATABEND_DSN", &dsn)
             .with_env_var("TTC_PORT", format!("{port}"))
             .with_container_name(&container_name)
+            .with_log_consumer(log_consumer)
             .start()
             .await;
         let duration = start.elapsed().as_secs();
@@ -455,6 +459,25 @@ async fn run_mysql_server(docker: &Docker) -> Result<ContainerAsync<Mysql>> {
 
 // Stop the running container to avoid conflict
 async fn stop_container(docker: &Docker, container_name: &str) {
+    let opts = Some(ListContainersOptions::<String> {
+        all: true,
+        ..Default::default()
+    });
+    let containers = docker.list_containers(opts).await;
+    if let Ok(containers) = containers {
+        if !containers.is_empty() {
+            println!("==> list containers");
+            for container in containers {
+                if let Some(names) = container.names {
+                    println!(
+                        " -> container name: {:?}, status: {:?}",
+                        names, container.state
+                    );
+                }
+            }
+        }
+    }
+
     let container = docker.inspect_container(container_name, None).await;
     if let Ok(container) = container {
         println!(
diff --git a/tests/sqllogictests/suites/stage/formats/parquet/infer_schema.test b/tests/sqllogictests/suites/stage/formats/parquet/infer_schema.test
@@ -1,10 +1,10 @@
-query 
+query
 select * from infer_schema(location => '@data/parquet/tuple.parquet')
 ----
 id INT 0 0
 t TUPLE(A INT32, B STRING) 0 1
 
-query 
+query
 select * from infer_schema(location => '@data/parquet/complex.parquet')
 ----
 resourceType VARCHAR 1 0
@@ -39,13 +39,13 @@ yy__us_core_race VARCHAR 1 28
 yy__us_core_ethnicity VARCHAR 1 29
 yy__us_core_birthsex TUPLE(VALUECODE STRING,) 1 30
 
-query 
+query
 select * from infer_schema(location => '@data/parquet/variant.parquet')
 ----
 a INT 0 0
 b VARIANT 0 1
 
-query 
+query
 select * from infer_schema(location => '@data/parquet/', FILE_FORMAT => 'PARQUET',  pattern => 'tuple.*')
 ----
 id INT 0 0
@@ -55,7 +55,7 @@ statement ok
 drop CONNECTION IF EXISTS my_conn
 
 statement ok
-create CONNECTION my_conn STORAGE_TYPE = 's3' access_key_id='minioadmin' secret_access_key='minioadmin' endpoint_url='http://127.0.0.1:9900/'
+create CONNECTION my_conn STORAGE_TYPE = 's3' access_key_id='minioadmin' secret_access_key='minioadmin' endpoint_url='http://127.0.0.1:9900/' region='auto'
 
 query
 select * from INFER_SCHEMA(location => 's3://testbucket/data/parquet/tuple.parquet', connection_name => 'my_conn')
