NOTICKET Add connectome establishment software, make Arm64 conditional

Author: matz-e

.gitlab-ci.yml

@@ -15,15 +15,18 @@ workflow:
       when: never
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     - if: $CI_PIPELINE_SOURCE == "web"
+
 stages:
   - unit tests
   - generate
   - build base containers
   - build cache
   - build spacktainer containers
   - cleanup
+
 variables:
   SPACK_BRANCH: develop
+
 generate base pipeline:
   image: ubuntu:22.04
   stage: generate
@@ -36,8 +39,7 @@ generate base pipeline:
       skopeo
     - pip install --upgrade pip setuptools
     - pip install -e ./job_creator
-    - jc create-jobs --singularity-version ${SINGULARITY_VERSION} --s3cmd-version
-      ${S3CMD_VERSION}
+    - jc create-jobs --singularity-version ${SINGULARITY_VERSION} --s3cmd-version ${S3CMD_VERSION}
   artifacts:
     when: always
     paths:
@@ -97,6 +99,9 @@ populate buildcache for amd64:
     strategy: depend
   extends: .rules
 populate buildcache for arm64:
+  rules:
+    - exists:
+      - container_definitions/arm64
   stage: build cache
   needs:
     - job: gather child artifacts
@@ -120,6 +125,9 @@ build spacktainers for amd64:
     strategy: depend
   extends: .rules
 build spacktainers for arm64:
+  rules:
+    - exists:
+      - container_definitions/arm64
   stage: build spacktainer containers
   needs:
     - job: gather child artifacts

container_definitions/arm64/cwl-registry.yaml renamed to container_definitions/_arm64/cwl-registry.yaml

@@ -0,0 +1,3 @@
+spack:
+  specs:
+    - py-connectome-manipulator

container_definitions/amd64/connectome-manipulator.yaml

@@ -0,0 +1,3 @@
+spack:
+  specs:
+    - py-functionalizer

container_definitions/amd64/functionalizer.yaml

@@ -0,0 +1,3 @@
+spack:
+  specs:
+    - touchdetector

container_definitions/amd64/touchdetector.yaml

@@ -1,6 +1,7 @@
 import glob
 import logging
 import os
+from pathlib import Path
 from pprint import pformat
 
 import click
@@ -103,7 +104,7 @@ def get_artifacts(parent_pipeline, private_token):
     jobs = session.get(jobs_url(base_url, project_id, run_pipeline["id"])).json()
     logger.debug(f"Jobs: {pformat(jobs)}")
     for architecture in [
-        os.path.basename(archdir) for archdir in glob.glob("container_definitions/*")
+        archdir.name for archdir in Path("container_definitions").glob("[!_]*")
     ]:
         logger.info(f"Architecture: {architecture}")
         process_job = next(

get_artifacts/src/get_artifacts/__main__.py

@@ -1,8 +1,7 @@
-import glob
 import json
 import logging
 import logging.config
-import os
+from pathlib import Path
 
 import requests
 import ruamel.yaml
@@ -70,7 +69,7 @@ def get_architectures():
     Retrieve the architectures that need to be built based on the container definitions
     """
     architectures = [
-        os.path.basename(archdir) for archdir in glob.glob("container_definitions/*")
+        archdir.name for archdir in Path("container_definitions").glob("[!_]*")
     ]
 
     return architectures