ci: Fix cargo-test dependencies and test-certs upping

Author: def-

ci/mkpipeline.py

@@ -196,6 +196,14 @@ def fetch_hashes() -> None:
     add_version_to_preflight_tests(pipeline)
     trim_builds_prep_thread.join()
     trim_builds(pipeline, hash_check)
+    add_cargo_test_dependency(
+        pipeline,
+        args.pipeline,
+        args.coverage,
+        args.sanitizer,
+        args.bazel_remote_cache,
+        bazel_lto,
+    )
     remove_dependencies_on_prs(pipeline, args.pipeline, hash_check)
     remove_mz_specific_keys(pipeline)
 
@@ -856,6 +864,45 @@ def visit(step: PipelineStep) -> None:
     ]
 
 
+def add_cargo_test_dependency(
+    pipeline: Any,
+    pipeline_name: str,
+    coverage: bool,
+    sanitizer: Sanitizer,
+    bazel_remote_cache: str,
+    bazel_lto: bool,
+) -> None:
+    """Cargo Test normally doesn't have to wait for the build to complete, but it requires a few images (ubuntu-base, postgres), which are rarely changed. So only add a dependency when those images are not on Dockerhub yet."""
+    if pipeline_name not in ("test", "nightly"):
+        return
+    if ui.env_is_truthy("BUILDKITE_PULL_REQUEST") and pipeline_name == "test":
+        for step in steps(pipeline):
+            if step.get("id") == "cargo-test":
+                step["depends_on"] = "build-x86_64"
+        return
+
+    repo = mzbuild.Repository(
+        Path("."),
+        arch=Arch.X86_64,
+        coverage=coverage,
+        sanitizer=sanitizer,
+        bazel=True,
+        bazel_remote_cache=bazel_remote_cache,
+        bazel_lto=bazel_lto,
+    )
+    composition = Composition(repo, name="cargo-test")
+    deps = composition.dependencies
+    if deps.check():
+        # We already have the dependencies available, no need to add a build dependency
+        return
+
+    for step in steps(pipeline):
+        if step.get("id") == "cargo-test":
+            step["depends_on"] = "build-x86_64"
+        if step.get("id") == "miri-test":
+            step["depends_on"] = "build-aarch64"
+
+
 def remove_dependencies_on_prs(
     pipeline: Any,
     pipeline_name: str,

misc/python/materialize/mysql_util.py

@@ -40,6 +40,7 @@ def retrieve_ssl_context_for_mysql(c: Composition) -> MySqlSslContext:
 
 def retrieve_invalid_ssl_context_for_mysql(c: Composition) -> MySqlSslContext:
     # Use the TestCert service to obtain a wrong CA and client cert/key:
+    c.up({"name": "test-certs", "persistent": True})
     ssl_wrong_ca = c.run("test-certs", "cat", "/secrets/ca.crt", capture=True).stdout
     ssl_wrong_client_cert = c.run(
         "test-certs", "cat", "/secrets/certuser.crt", capture=True

test/pg-cdc-old-syntax/mzcompose.py

@@ -314,6 +314,7 @@ def workflow_cdc(c: Composition, parser: WorkflowArgumentParser) -> None:
     )
     print(f"Files: {sharded_files}")
 
+    c.up({"name": "test-certs", "persistent": True})
     ssl_ca = c.run("test-certs", "cat", "/secrets/ca.crt", capture=True).stdout
     ssl_cert = c.run("test-certs", "cat", "/secrets/certuser.crt", capture=True).stdout
     ssl_key = c.run("test-certs", "cat", "/secrets/certuser.key", capture=True).stdout
@@ -396,6 +397,7 @@ def workflow_migration(c: Composition, parser: WorkflowArgumentParser) -> None:
     )
     print(f"Files: {sharded_files}")
 
+    c.up({"name": "test-certs", "persistent": True})
     ssl_ca = c.run("test-certs", "cat", "/secrets/ca.crt", capture=True).stdout
     ssl_cert = c.run("test-certs", "cat", "/secrets/certuser.crt", capture=True).stdout
     ssl_key = c.run("test-certs", "cat", "/secrets/certuser.key", capture=True).stdout

test/pg-cdc/mzcompose.py

@@ -300,6 +300,7 @@ def workflow_cdc(c: Composition, parser: WorkflowArgumentParser) -> None:
     )
     print(f"Files: {sharded_files}")
 
+    c.up({"name": "test-certs", "persistent": True})
     ssl_ca = c.run("test-certs", "cat", "/secrets/ca.crt", capture=True).stdout
     ssl_cert = c.run("test-certs", "cat", "/secrets/certuser.crt", capture=True).stdout
     ssl_key = c.run("test-certs", "cat", "/secrets/certuser.key", capture=True).stdout