tests: Use eatmydata to disable fsync

bin/ci-builder

@@ -339,10 +339,10 @@ case "$cmd" in
             args+=(--volume "$GIT_ROOT_DIR:$GIT_ROOT_DIR")
         fi
         rm -f "$cid_file"
-        docker run "${args[@]}" "materialize/ci-builder:$tag" "${docker_command[@]}"
+        docker run "${args[@]}" "materialize/ci-builder:$tag" eatmydata "${docker_command[@]}"
         ;;
     root-shell)
-        docker exec --interactive --tty --user 0:0 "$(<"$cid_file")" ci/builder/root-shell.sh
+        docker exec --interactive --tty --user 0:0 "$(<"$cid_file")" eatmydata ci/builder/root-shell.sh
         ;;
     *)
         printf "unknown command %q\n" "$cmd"

ci/builder/Dockerfile

@@ -40,6 +40,7 @@ RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-ge
     ca-certificates \
     curl \
     docker.io \
+    eatmydata \
     gdb \
     git \
     gnupg2 \

misc/images/materialized-base/Dockerfile

@@ -37,9 +37,9 @@ RUN groupadd --system --gid=999 materialize \
        fi \
     && rm -rf /var/lib/apt/lists/* \
     && apt-get clean \
-    && mkdir -p /mzdata /scratch /var/lib/postgresql/data /var/run/postgresql /var/lib/nginx /var/log/nginx \
+    && mkdir -p /mzdata /scratch /var/lib/postgresql/data /var/run/postgresql /var/lib/nginx /var/log/nginx /etc/postgresql/16/main \
     && touch /run/nginx.pid \
-    && chown -R materialize /mzdata /scratch /var/lib/postgresql/data /var/run/postgresql /var/log/postgresql /var/lib/postgresql/16/main /var/lib/nginx /var/log/nginx /run/nginx.pid
+    && chown -R materialize /mzdata /scratch /var/lib/postgresql/data /var/run/postgresql /var/log/postgresql /var/lib/postgresql/16/main /var/lib/nginx /var/log/nginx /run/nginx.pid /etc/postgresql/16/main
 
 COPY postgresql.conf pg_hba.conf /etc/postgresql/16/main/
 

misc/images/ubuntu-base/Dockerfile

@@ -19,3 +19,5 @@ ENV RUST_LIB_BACKTRACE=0
 RUN sed -i -e 's#http://archive\.ubuntu\.com#http://us-east-1.ec2.archive.ubuntu.com#' \
            -e 's#http://security\.ubuntu\.com#http://us-east-1.ec2.archive.ubuntu.com#' \
            -e 's#http://ports\.ubuntu\.com#http://us-east-1.ec2.ports.ubuntu.com#' /etc/apt/sources.list.d/ubuntu.sources
+
+RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends eatmydata

misc/python/materialize/mzcompose/services/clusterd.py

@@ -37,6 +37,7 @@ def __init__(
             "CLUSTERD_LOG_FILTER",
             f"CLUSTERD_GRPC_HOST={name}",
             "MZ_SOFT_ASSERTIONS=1",
+            "MZ_EAT_MY_DATA=1",
             *environment_extra,
         ]
 

misc/python/materialize/mzcompose/services/materialized.py

@@ -115,6 +115,7 @@ def __init__(
         environment = [
             "MZ_NO_TELEMETRY=1",
             "MZ_NO_BUILTIN_CONSOLE=1",
+            "MZ_EAT_MY_DATA=1",
             "MZ_TEST_ONLY_DUMMY_SEGMENT_CLIENT=true",
             f"MZ_SOFT_ASSERTIONS={int(soft_assertions)}",
             # The following settings can not be baked in the default image, as they

misc/python/materialize/mzcompose/services/minio.py

@@ -43,7 +43,12 @@ def __init__(
                 "image": image,
                 "ports": ports,
                 "allow_host_ports": allow_host_ports,
-                "environment": ["MINIO_STORAGE_CLASS_STANDARD=EC:0"],
+                "environment": [
+                    "MINIO_STORAGE_CLASS_STANDARD=EC:0",
+                    "MINIO_HEAL_DISABLE=on",
+                    "MINIO_DISK_WATERMARK_LOW=1",
+                    "MINIO_DISK_WATERMARK_HIGH=1",
+                ],
                 "healthcheck": {
                     "test": [
                         "CMD",

misc/python/materialize/mzcompose/services/postgres.py

@@ -26,7 +26,11 @@ def __init__(
         image: str | None = None,
         ports: list[str] = ["5432"],
         extra_command: list[str] = [],
-        environment: list[str] = ["POSTGRESDB=postgres", "POSTGRES_PASSWORD=postgres"],
+        environment: list[str] = [
+            "POSTGRESDB=postgres",
+            "POSTGRES_PASSWORD=postgres",
+            "LD_PRELOAD=libeatmydata.so",
+        ],
         volumes: list[str] = [],
         max_wal_senders: int = 100,
         max_replication_slots: int = 100,

misc/python/materialize/mzcompose/services/sql_logic_test.py

@@ -27,6 +27,7 @@ def __init__(
         if environment is None:
             environment = [
                 "MZ_SOFT_ASSERTIONS=1",
+                "LD_PRELOAD=libeatmydata.so",
             ]
         environment += [
             "MZ_SYSTEM_PARAMETER_DEFAULT="

misc/python/materialize/mzcompose/services/testdrive.py

@@ -88,6 +88,7 @@ def __init__(
         environment += [
             f"CLUSTER_REPLICA_SIZES={json.dumps(cluster_replica_size)}",
             "MZ_CI_LICENSE_KEY",
+            "LD_PRELOAD=libeatmydata.so",
         ]
 
         volumes = [

src/clusterd/ci/entrypoint.sh

@@ -32,4 +32,10 @@ if [[ "${KUBERNETES_SERVICE_HOST:-}" ]]; then
     export CLUSTERD_PROCESS=${CLUSTERD_PROCESS:-${HOSTNAME##*-}}
 fi
 
+if [ -z "${MZ_EAT_MY_DATA:-}" ]; then
+    unset LD_PRELOAD
+else
+    export LD_PRELOAD=libeatmydata.so
+fi
+
 exec clusterd "$@"

src/environmentd/ci/entrypoint.sh

@@ -11,6 +11,12 @@
 
 set -euo pipefail
 
+if [ -z "${MZ_EAT_MY_DATA:-}" ]; then
+    unset LD_PRELOAD
+else
+    export LD_PRELOAD=libeatmydata.so
+fi
+
 if environmentd "$@"; then
     echo "environmentd exited gracefully; sleeping forever" >&2
     sleep infinity

src/materialized/ci/entrypoint.sh

@@ -39,6 +39,14 @@ hosted offering we run these services scaled across many machines.
 ********************************* WARNING ********************************
 EOF
 
+if [ -z "${MZ_EAT_MY_DATA:-}" ]; then
+    unset LD_PRELOAD
+    echo > /etc/postgresql/16/main/environment
+else
+    export LD_PRELOAD="libeatmydata.so"
+    echo "LD_PRELOAD=libeatmydata.so" > /etc/postgresql/16/main/environment
+fi
+
 # Start PostgreSQL, unless suppressed.
 if [ -z "${MZ_NO_BUILTIN_POSTGRES:-}" ]; then
   (trap 'pg_ctlcluster 16 main stop --mode=fast --force' SIGTERM SIGINT

test/cluster/mzcompose.py

@@ -88,11 +88,13 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
     def process(name: str) -> None:
         # incident-70 and refresh-mv-restart are slow, run in separate CI step
         # concurrent-connections is too flaky
+        # TODO: Reenable test-memory-limiter when database-issues/9502 is fixed
         if name in (
             "default",
             "test-incident-70",
             "test-concurrent-connections",
             "test-refresh-mv-restart",
+            "test-memory-limiter",
         ):
             return
         with c.test_case(name):

test/postgres/Dockerfile

@@ -13,9 +13,10 @@ FROM postgres:17.4
 
 ENV POSTGRES_PASSWORD=postgres
 
-RUN apt update
-
-RUN apt install postgresql-17-cron
+RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends postgresql-17-cron eatmydata \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /usr/share/doc/* /usr/share/man/* /usr/share/info/* /usr/share/locale/* /var/cache/* /var/log/*
 
 COPY --chown=postgres --from=certs /secrets/* /share/secrets/
 COPY pg_hba.conf /share/conf/pg_hba.conf

test/rqg/Dockerfile

@@ -21,7 +21,8 @@ RUN apt-get update && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y -
     libparse-yapp-perl \
     postgresql-client \
     && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /usr/share/doc/* /usr/share/man/* /usr/share/info/* /usr/share/locale/* /var/cache/* /var/log/*
 
 RUN perl -MCPAN -e 'install DBIx::MyParsePP'
 
@@ -30,6 +31,6 @@ RUN git clone --single-branch https://github.com/MaterializeInc/RQG.git \
     && git checkout 4fe4e6f09f9599ce9f8ffd3f75d593b20518cc50 \
     && rm -rf .git
 
-ENTRYPOINT ["/usr/bin/perl"]
+ENTRYPOINT ["/usr/bin/eatmydata", "/usr/bin/perl"]
 
 WORKDIR RQG

test/sqlancer/Dockerfile

@@ -19,12 +19,13 @@ RUN apt-get update && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y -
     default-jdk \
     maven \
     && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /usr/share/doc/* /usr/share/man/* /usr/share/info/* /usr/share/locale/* /var/cache/* /var/log/*
 
 # Build SQLancer
 RUN git clone --depth=1 --single-branch https://github.com/sqlancer/sqlancer \
     && cd sqlancer \
     && rm -rf .git \
     && mvn package -DskipTests
 
-ENTRYPOINT ["/usr/bin/java", "-jar", "sqlancer/target/sqlancer-2.0.0.jar"]
+ENTRYPOINT ["/usr/bin/eatmydata", "/usr/bin/java", "-jar", "sqlancer/target/sqlancer-2.0.0.jar"]

test/sqllogictest/cardinality.slt

@@ -430,7 +430,7 @@ SHOW optimizer_oneshot_stats_timeout;
 COMPLETE 1
 
 statement ok
-SELECT mz_unsafe.mz_sleep(3)
+SELECT mz_unsafe.mz_sleep(5)
 
 query T multiline
 EXPLAIN OPTIMIZED PLAN WITH(join implementations, humanized expressions) AS VERBOSE TEXT FOR SELECT * FROM t JOIN t2 ON t.x = t2.x JOIN t3 ON t.x = t3.x JOIN t4 ON t.x = t4.x JOIN t5 ON t.x = t5.x JOIN t6 ON t.x = t6.x JOIN t7 ON t.x = t7.x JOIN t8 ON t.x = t8.x JOIN t9 ON t.x = t9.x JOIN t10 ON t.x = t10.x;

test/sqlsmith/Dockerfile

@@ -35,4 +35,4 @@ RUN git clone --single-branch --branch=master https://github.com/MaterializeInc/
     && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ . \
     && cmake --build . -j `nproc`
 
-ENTRYPOINT ["sqlsmith/sqlsmith"]
+ENTRYPOINT ["/usr/bin/eatmydata", "sqlsmith/sqlsmith"]

test/testdrive/github-6335.td

@@ -16,7 +16,7 @@
 # introspection sources that take a while to update.
 
 $ set-regex match=\d{13,20} replacement=<TIMESTAMP>
-$ set-sql-timeout duration=60s
+$ set-sql-timeout duration=120s
 
 # This test uses introspection queries that need to be targeted to a replica
 > SET cluster_replica = r1