Riskier: Try to save mzdata

Author: def-

misc/images/materialized-base/Dockerfile

@@ -29,20 +29,22 @@ RUN apt-get update \
         ca-certificates \
         curl \
         gettext-base \
+        haveged \
         nginx \
         postgresql \
-        postgresql-contrib \
         tini \
         ssh \
+        zstd \
     && if [ "$CI_SANITIZER" != "none" ]; then \
         TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get -qy install --no-install-recommends llvm; \
        fi \
     && rm -rf /var/lib/apt/lists/* \
     && apt-get clean \
-    && mkdir -p /mzdata /scratch /var/lib/postgresql/data /var/run/postgresql /var/lib/nginx /var/log/nginx \
+    && mkdir -p /mzdata /scratch /var/lib/postgresql/data /var/run/postgresql /var/lib/nginx /var/log/nginx /archive \
     && touch /run/nginx.pid \
-    && chown -R materialize /mzdata /scratch /var/lib/postgresql/data /var/run/postgresql /var/log/postgresql /var/lib/postgresql/16/main /var/lib/nginx /var/log/nginx /run/nginx.pid
+    && chown -R materialize /mzdata /scratch /var/lib/postgresql/data /var/run/postgresql /var/log/postgresql /var/lib/postgresql/16/main /var/lib/nginx /var/log/nginx /run/nginx.pid /archive
 
+COPY postgresql.conf pg_hba.conf /var/lib/postgresql/16/main/
 COPY postgresql.conf pg_hba.conf /etc/postgresql/16/main/
 
 COPY --from=console /usr/share/nginx/html /usr/share/nginx/html
@@ -57,12 +59,3 @@ RUN MZ_ENDPOINT=http://localhost:6876 \
     && rm /etc/nginx/templates/default.conf.template
 
 USER materialize
-
-RUN PGUSER=postgres pg_ctlcluster 16 main start \
-    && psql -U postgres -c "CREATE ROLE root WITH LOGIN PASSWORD 'root'" \
-    && psql -U postgres -c "CREATE DATABASE root OWNER root" \
-    && psql -U root -c "CREATE SCHEMA consensus" \
-    && psql -U root -c "CREATE SCHEMA storage" \
-    && psql -U root -c "CREATE SCHEMA adapter" \
-    && psql -U root -c "CREATE SCHEMA tsoracle" \
-    && pg_ctlcluster 16 main stop

src/materialized/ci/Dockerfile

@@ -15,6 +15,20 @@ COPY materialized entrypoint.sh /usr/local/bin/
 USER root
 RUN ln -s /usr/local/bin/materialized /usr/local/bin/environmentd \
   && ln -s /usr/local/bin/materialized /usr/local/bin/clusterd
+
 USER materialize
 
+RUN bash -c '\
+  set -euo pipefail; \
+  tini -- entrypoint.sh & \
+  tini_pid=$!; \
+  until curl -sf localhost:6878/api/readyz; do sleep 0.1; done; \
+  pg_pid=$(cat /var/run/postgresql/16-main.pid); \
+  kill -TERM "$pg_pid"; \
+  kill -TERM "$tini_pid"; \
+  wait "$tini_pid" || true; \
+  while kill -0 "$pg_pid" 2> /dev/null; do sleep 0.1; done; \
+  tar --zstd -cf /archive/mzdata.tar.zst -C / mzdata scratch; \
+'
+
 ENTRYPOINT ["tini", "--", "entrypoint.sh"]

src/materialized/ci/entrypoint.sh

@@ -42,7 +42,13 @@ EOF
 # Start PostgreSQL, unless suppressed.
 if [ -z "${MZ_NO_BUILTIN_POSTGRES:-}" ]; then
   rm -f /var/run/postgresql/.s.PGSQL.26257.lock
-  PGUSER=postgres pg_ctlcluster 16 main start &
+  /usr/lib/postgresql/16/bin/postgres -D /var/lib/postgresql/16/main &
+  until psql -U postgres -c "SELECT 1" > /dev/null; do
+    sleep 0.1;
+  done
+  psql -U postgres -c "CREATE ROLE root WITH LOGIN PASSWORD 'root'" || true
+  psql -U postgres -c "CREATE DATABASE root OWNER root" || true
+  psql -U root -c "CREATE SCHEMA IF NOT EXISTS consensus; CREATE SCHEMA IF NOT EXISTS storage; CREATE SCHEMA IF NOT EXISTS adapter; CREATE SCHEMA IF NOT EXISTS tsoracle"
 fi
 
 # Start nginx to serve the console.
@@ -207,6 +213,10 @@ if [ -z "${MZ_NO_TELEMETRY:-}" ]; then
     export MZ_SEGMENT_CLIENT_SIDE=${MZ_SEGMENT_CLIENT_SIDE:-true}
 fi
 
+if [ -z "$(find /mzdata -mindepth 1 -print -quit)" ]; then
+  tar --zstd -xf /archive/mzdata.tar.zst -C /
+fi
+
 if [ -n "${MZ_RESTART_ON_FAILURE:-}" ]; then
     for ((i = 0; i < ${MZ_RESTART_LIMIT:-9999999999}; i++)); do
         # Run `environmentd` inside of an `if` to avoid tripping `set -e`