tc

Author: clee2000

.jenkins/sphinx_files.py

@@ -1,13 +1,31 @@
+import glob
+from pathlib import Path
+import shutil
 import subprocess
 import os
+from get_files_to_run import remove_other_files, compute_files_to_keep, calculate_shards, get_all_files
+from validate_tutorials_built import NOT_RUN
 
 def main() -> None:
-    files_to_run = os.environ["FILES_TO_RUN"]
+    all_files = get_all_files()
+    files_to_run = calculate_shards(all_files, num_shards=20)[int(os.environ.get("WORKER_ID", "1")) - 1]
+    files_to_run = [x for x in files_to_run if x not in [f"{f}.py" for f in NOT_RUN]]
+
+    os.mkdir("docs_to_zip", exist_ok=True)
+
     env = os.environ.copy()
-    for file in files_to_run.split(" "):
+    for file in files_to_run:
+        remove_other_files(all_files, compute_files_to_keep(file))
         print(f"Running {file}")
-        env["RUNTHIS"] = file
+        stem = Path(file).stem
+        env["RUNTHIS"] = stem
         subprocess.check_output(["make", "html"], env=env)
+        for file in glob.glob(f"docs/**/*", recursive=True):
+            if stem in file:
+                shutil.copy(file, f"docs_to_zip/{file}")
+        subprocess.check_output(["git", "reset", "--hard", "HEAD"])
+    os.remove("docs")
+    shutil.move("docs_to_zip", "docs")
 
 if __name__ == "__main__":
     main()