Add an env var that forces mzbuild to wait for an image to become available (MaterializeInc#3393)

We've had several load tests fail because they tried to use the most recent image, and it
just happened to not be ready yet.

Author: quodlibetor

misc/python/materialize/mzbuild.py

@@ -45,6 +45,7 @@
 import stat
 import subprocess
 import sys
+import time
 
 import yaml
 
@@ -451,11 +452,15 @@ def acquire(self) -> AcquiredFrom:
             acquired_from: How the image was acquired.
         """
         if self.image.publish:
-            try:
-                spawn.runv(["docker", "pull", self.spec()])
-                return AcquiredFrom.REGISTRY
-            except subprocess.CalledProcessError:
-                pass
+            while True:
+                try:
+                    spawn.runv(["docker", "pull", self.spec()])
+                    return AcquiredFrom.REGISTRY
+                except subprocess.CalledProcessError:
+                    if not ui.env_is_truthy("MZBUILD_WAIT_FOR_IMAGE"):
+                        break
+                    print(f"waiting for mzimage to become available", file=sys.stderr)
+                    time.sleep(10)
         self.build()
         return AcquiredFrom.LOCAL_BUILD
 

misc/python/materialize/ui.py

@@ -107,7 +107,7 @@ def shell_quote(args: Iterable[Any]) -> str:
 
 
 def env_is_truthy(env_var: str) -> bool:
-    """Return true if `env_var` is set and is not one of: 0, n, no"""
+    """Return true if `env_var` is set and is not one of: 0, '', no"""
     env = os.getenv(env_var)
     if env is not None:
         return env not in ("", "0", "no")