support custom TopCP images

Procfile

@@ -1,4 +1,5 @@
-minikube-mount: minikube mount $LOCAL_DIR:/mnt/servicex
+minikube-mount-app: minikube mount $LOCAL_DIR:/mnt/servicex
+minikube-mount-topcp: minikube mount $LOCAL_DIR/code_generator_TopCPToolkit:/mnt/topcp
 helm-install: sleep 5; cd $CHART_DIR && helm install -f $VALUES_FILE servicex . && tail -f /dev/null
 port-forward-app: sleep 30 && cd $LOCAL_DIR && bash local/port-forward.sh app
 port-forward-minio: sleep 20 && cd $LOCAL_DIR && bash local/port-forward.sh minio

code_generator_TopCPToolkit/Dockerfile

@@ -17,9 +17,6 @@ RUN poetry config virtualenvs.create false && \
 
 RUN pip install gunicorn
 
-COPY boot.sh ./
-RUN chmod 755 boot.sh
-
 COPY transformer_capabilities.json ./
 RUN chmod 644 transformer_capabilities.json
 
@@ -29,9 +26,11 @@ RUN chmod 777 -R servicex
 COPY app.conf .
 RUN chmod 755 app.conf
 
+RUN chmod 755 /home/servicex/servicex/boot.sh
+
 USER servicex
 
 ENV CODEGEN_CONFIG_FILE="/home/servicex/app.conf"
 
 EXPOSE 5000
-ENTRYPOINT ["./boot.sh"]
+ENTRYPOINT ["/home/servicex/servicex/boot.sh"]

code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py

@@ -36,6 +36,11 @@
         "ifTrue": ["--no-filter"],
         "ifFalse": None,
     },
+    "docker_image": {
+        "properType": str,
+        "properTypeString": "string",
+        "optional": True,
+    },
 }
 
 
@@ -52,16 +57,19 @@ def generate_files_from_query(query, query_file_path):
         "customConfig",
     ]
 
-    # ensure all keys are specified
+    # ensure all required keys are specified
     for key in options:
         if key not in jquery:
+            # Skip optional parameters
+            if options[key].get("optional", False):
+                continue
             raise ValueError(
                 key + " must be specified. May be type None or ",
                 options[key]["properTypeString"],
             )
 
     for key in jquery:
-        # ensure only aviable options are allowed
+        # ensure only available options are allowed
         if key not in options:
             raise KeyError(
                 key + " is not implemented. Available keys: " + str(options.keys())
@@ -78,7 +86,7 @@ def generate_files_from_query(query, query_file_path):
             )
 
         # check for reco.yaml, parton.yaml and particle.yaml files
-        if isinstance(jquery[key], str):
+        if isinstance(jquery[key], str) and "fileName" in options[key]:
             with open(
                 os.path.join(query_file_path, options[key]["fileName"]), "w"
             ) as file:

code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py

@@ -62,11 +62,12 @@ def generate_code(self, query, cache_path: str):
         capabilities_path = os.environ.get(
             "CAPABILITIES_PATH", "/home/servicex/transformer_capabilities.json"
         )
+
+        query_translate.generate_files_from_query(query, query_file_path)
+
         shutil.copyfile(
             capabilities_path,
             os.path.join(query_file_path, "transformer_capabilities.json"),
         )
 
-        query_translate.generate_files_from_query(query, query_file_path)
-
         return GeneratedFileResult(_hash, query_file_path)

code_generator_TopCPToolkit/servicex/boot.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Initialize reload flag
+RELOAD=""
+
+# Parse command line arguments
+for arg in "$@"
+do
+    if [ "$arg" = "--reload" ]; then
+        RELOAD="--reload"
+        break
+    fi
+done
+
+# Running the web server?
+action=${1:-web_service}
+if [ "$action" = "web_service" ] ; then
+    mkdir instance
+    exec gunicorn -b :5000 $RELOAD --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.TopCP_code_generator:create_app()"
+else
+    echo "Unknown action '$action'"
+fi

code_generator_TopCPToolkit/servicex/templates/transform_single_file.py

@@ -12,21 +12,23 @@ def transform_single_file(file_path: str, output_path: Path, output_format: str)
     # create input.txt file for event loop and insert file_path as only line
     with open("input.txt", "w") as f:
         f.write(file_path)
-    # move reco.yaml, parton.yaml and particle.yaml if they exit to CONFIG_LOC loacation
+
+    # move reco.yaml, parton.yaml and particle.yaml if they exist to CONFIG_LOC location
+    config_loc = os.environ.get("CONFIG_LOC", os.getcwd())
     if os.path.exists("/generated/reco.yaml"):
         shutil.copyfile(
             "/generated/reco.yaml",
-            os.path.join(os.environ.get("CONFIG_LOC"), "reco.yaml"),
+            os.path.join(config_loc, "reco.yaml"),
         )
     if os.path.exists("/generated/parton.yaml"):
         shutil.copyfile(
             "/generated/parton.yaml",
-            os.path.join(os.environ.get("CONFIG_LOC"), "parton.yaml"),
+            os.path.join(config_loc, "parton.yaml"),
         )
     if os.path.exists("/generated/particle.yaml"):
         shutil.copyfile(
             "/generated/particle.yaml",
-            os.path.join(os.environ.get("CONFIG_LOC"), "particle.yaml"),
+            os.path.join(config_loc, "particle.yaml"),
         )
 
     generated_transformer.runTop_el()

helm/servicex/templates/app/configmap.yaml

@@ -183,6 +183,11 @@ data:
     CODE_GEN_IMAGES = { {{ join "," $code_gen_images }} }
     {{- end }}
 
+    {{- if .Values.codeGen.topcp.enabled }}
+    # TopCP custom image configuration
+    TOPCP_ALLOWED_IMAGES = {{ .Values.codeGen.topcp.allowedImages | toJson }}
+    {{- end }}
+
     {{- $didFinders := list }}
     {{- if .Values.didFinder.CERNOpenData.enabled }}
       {{- $didFinders = append $didFinders "cernopendata" }}

helm/servicex/templates/codegen/deployment.yaml

@@ -19,6 +19,13 @@ spec:
       containers:
       - name: {{ $.Release.Name }}-code-gen-{{ $codeGenName }}
         image: {{ .image }}:{{ .tag }}
+        {{- if eq $.Values.app.environment "dev" }}
+        {{- if eq $codeGenName "topcp" }}
+        {{- if $.Values.app.reload }}
+        command: [ "./servicex/boot.sh", "web_service", "--reload" ]
+        {{- end }}
+        {{- end }}
+        {{- end }}
         env:
         - name: INSTANCE_NAME
           value: {{ $.Release.Name }}
@@ -35,5 +42,26 @@ spec:
         imagePullPolicy: {{ .pullPolicy }}
         ports:
           - containerPort: 5000
+        {{- if eq $codeGenName "topcp" }}
+        {{- if eq $.Values.app.environment "dev" }}
+        {{- if $.Values.app.mountLocal }}
+        volumeMounts:
+          - name: host-topcp-volume
+            mountPath: /home/servicex
+        {{- end }}
+        {{- end }}
+        {{- end }}
+
+      {{- if eq $codeGenName "topcp" }}
+      {{- if eq $.Values.app.environment "dev" }}
+      {{- if $.Values.app.mountLocal }}
+      volumes:
+        - name: host-topcp-volume
+          hostPath:
+            path: /mnt/topcp
+            type: DirectoryOrCreate
+      {{- end }}
+      {{- end }}
+      {{- end }}
 {{- end }}
 {{- end }}

helm/servicex/values.yaml

@@ -104,10 +104,12 @@ codeGen:
   topcp:
     enabled: true
     image: sslhep/servicex_code_gen_topcp
-    pullPolicy: Always
+    pullPolicy: IfNotPresent
     tag: develop
     defaultScienceContainerImage: sslhep/servicex_science_image_topcp
     defaultScienceContainerTag: 2.17.0-25.2.45
+    allowedImages:
+      - "sslhep/servicex_science_image_topcp:"
 
 didFinder:
   CERNOpenData:
@@ -175,7 +177,6 @@ postgresql:
     image:
       repository: bitnamilegacy/os-shell
       tag: 12-debian-12-r51
-
   global:
     postgresql:
       auth:
@@ -200,7 +201,6 @@ rabbitmq:
     image:
       repository: bitnamilegacy/os-shell
       tag: 12-debian-12-r51
-
   extraConfiguration: |-
     consumer_timeout = 3600000
 

servicex_app/servicex_app/code_gen_adapter.py

@@ -25,6 +25,8 @@
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+from typing import Optional
+
 import requests
 from requests_toolbelt.multipart import decoder
 
@@ -43,7 +45,11 @@ def post_request(self, post_url, post_obj):
         return result
 
     def generate_code_for_selection(
-        self, request_record: TransformRequest, namespace: str, user_codegen_name: str
+        self,
+        request_record: TransformRequest,
+        namespace: str,
+        user_codegen_name: str,
+        custom_image: Optional[str] = None,
     ) -> tuple[str, str, str, str]:
         """
         Generates the C++ code for a request's selection string.

servicex_app/servicex_app/resources/transformation/submit.py

@@ -26,6 +26,8 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 import uuid
+import json
+import os
 from datetime import datetime, timezone
 from typing import Optional, List
 
@@ -36,7 +38,29 @@
 from servicex_app.did_parser import DIDParser
 from servicex_app.models import TransformRequest, db, TransformStatus
 from servicex_app.resources.servicex_resource import ServiceXResource
-from werkzeug.exceptions import BadRequest
+from werkzeug.exceptions import BadRequest, HTTPException
+
+
+def validate_custom_docker_image(image_name: str) -> bool:
+    allowed_images_json = os.environ.get("TOPCP_ALLOWED_IMAGES")
+
+    if not allowed_images_json:
+        raise BadRequest("Custom Docker images are not allowed.")
+
+    try:
+        allowed_prefixes = json.loads(allowed_images_json)
+
+        if not isinstance(allowed_prefixes, list):
+            raise BadRequest("TopCP allowed images are improperly configured.")
+
+        for prefix in allowed_prefixes:
+            if image_name.startswith(prefix):
+                return True
+
+        raise BadRequest(f"Custom Docker image '{image_name}' not allowed.")
+
+    except json.JSONDecodeError:
+        raise BadRequest("TopCP allowed images are improperly configured.")
 
 
 class SubmitTransformationRequest(ServiceXResource):
@@ -99,7 +123,6 @@ def _initialize_dataset_manager(
         request_id: str,
         config: dict,
     ) -> DatasetManager:
-
         # did xor file_list
         if bool(did) == bool(file_list):
             raise BadRequest("Must provide did or file-list but not both")
@@ -151,7 +174,6 @@ def post(self):
             uuid.uuid4()
         )  # make sure we have a request id for all messages
         try:
-
             try:
                 args = self.parser.parse_args()
             except BadRequest as bad_request:
@@ -223,7 +245,20 @@ def post(self):
                 request_rec, namespace, user_codegen_name
             )
 
-            request_rec.image = codegen_transformer_image
+            custom_docker_image = None
+            if user_codegen_name == "topcp":
+                try:
+                    selection = json.loads(args["selection"])
+                    if "docker_image" in selection:
+                        custom_docker_image = selection["docker_image"]
+                        validate_custom_docker_image(custom_docker_image)
+                except json.decoder.JSONDecodeError:
+                    raise BadRequest("Malformed JSON submitted")
+
+            if custom_docker_image:
+                request_rec.image = custom_docker_image
+            else:
+                request_rec.image = codegen_transformer_image
 
             # Check to make sure the transformer docker image actually exists (if enabled)
             if config["TRANSFORMER_VALIDATE_DOCKER_IMAGE"]:
@@ -270,6 +305,8 @@ def post(self):
                 "Transformation request submitted!", extra={"requestId": request_id}
             )
             return {"request_id": str(request_id)}
+        except HTTPException:
+            raise
         except Exception as eek:
             current_app.logger.exception(
                 "Got exception while submitting transformation request",