Release 1.4.0

Author: christiam

.github/workflows/build-multi-pex.yml

@@ -39,7 +39,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.13"]
 
     steps:
 
@@ -83,6 +83,7 @@ jobs:
           tar axvf elastic-blast-3.9/elastic-blast3.9.tar.gz
           tar axvf elastic-blast-3.10/elastic-blast3.10.tar.gz
           tar axvf elastic-blast-3.11/elastic-blast3.11.tar.gz
+          tar axvf elastic-blast-3.13/elastic-blast3.13.tar.gz
           rm -fvr elastic-blast-*.tar.gz
           tar -czvf ~/elastic-blast.tar.gz elastic-blast elastic-blast*md5 elastic-blast3.*
       - name: 'Upload Artifact'

CITATION.cff

@@ -1,8 +1,8 @@
 cff-version: "1.2.0"
 message: "If you use this software, please cite it using these metadata."
 title: ElasticBLAST
-version: "1.3.1"
-date-released: 2024-07-24
+version: "1.4.0"
+date-released: 2025-03-17
 license: "NCBI Public Domain"
 repository-code: "https://github.com/ncbi/elastic-blast/"
 url: "https://blast.ncbi.nlm.nih.gov/doc/elastic-blast/"

Makefile

@@ -41,6 +41,14 @@ elastic-blast: ${PYTHON_SRC} ${YAML_TEMPLATES} ${VENV} validate-cf-templates
 	source ${VENV}/bin/activate && pex --python-shebang='/usr/bin/env python3' --disable-cache . -r requirements/base.txt --python=python${PYTHON_VERSION} -c $@ -o $@
 	-./$@ --version
 
+elastic-blast3.13: ${PYTHON_SRC} ${YAML_TEMPLATES} ${VENV} validate-cf-templates
+	source ${VENV}/bin/activate && pex --disable-cache . -r requirements/base.txt --python=python3.13 -c elastic-blast -o $@
+	-./$@ --version
+
+elastic-blast3.11: ${PYTHON_SRC} ${YAML_TEMPLATES} ${VENV} validate-cf-templates
+	source ${VENV}/bin/activate && pex --disable-cache . -r requirements/base.txt --python=python3.11 -c elastic-blast -o $@
+	-./$@ --version
+
 elastic-blast3.9: ${PYTHON_SRC} ${YAML_TEMPLATES} ${VENV} validate-cf-templates
 	source ${VENV}/bin/activate && pex --disable-cache . -r requirements/base.txt --python=python3.9 -c elastic-blast -o $@
 	-./$@ --version

docker-blast/Makefile

@@ -26,7 +26,8 @@
 
 SHELL=/bin/bash
 IMG?=ncbi/elb
-GCP_IMG?=gcr.io/ncbi-sandbox-blast/${IMG}
+GCP_PROJECT?=$(shell gcloud config get-value project 2>/dev/null)
+GCP_IMG?=gcr.io/${GCP_PROJECT}/${IMG}
 AWS_SERVER?=public.ecr.aws/i6v3i0i9
 AWS_IMG?=${AWS_SERVER}/elasticblast-elb
 AWS_REGION?=us-east-1

docker-janitor/Makefile

@@ -27,7 +27,7 @@ SHELL=/bin/bash
 .PHONY: all pre-check check clean build publish gcp-build gcp-check gcp-clean
 
 IMG?=ncbi/elasticblast-janitor
-VERSION?=0.3.2
+VERSION?=0.4.0
 ELB_VERSION?=$(shell git describe --tags --abbrev=0)
 GCP_PROJECT?=$(shell gcloud config get-value project 2>/dev/null)
 GCP_TEST_BUCKET?=gs://elasticblast-test/query-split-run-test

docker-job-submit/.gitignore

@@ -0,0 +1,6 @@
+templates/
+src/
+bin/
+requirements/
+setup.cfg_cloud
+setup.py

docker-job-submit/Dockerfile-build-from-local-sources.aws

@@ -30,8 +30,8 @@ COPY submit_jobs.py /usr/bin/
 
 RUN chmod +x /usr/bin/submit_jobs.py && \
     apk -U upgrade && \
-    apk add --no-cache bash python3 py3-pip py3-wheel && \
-    pip3 install --no-cache-dir --upgrade --break-system-packages pip && \
+    apk add --no-cache bash python3 py3-pip py3-wheel curl unzip && \
+    pip3 install --no-cache-dir --upgrade pip && \
     mkdir -p /var/elastic-blast && \
     rm -rf /var/cache/apk/*
 
@@ -43,6 +43,8 @@ COPY setup.cfg_cloud /var/elastic-blast/setup.cfg
 
 WORKDIR /var/elastic-blast
 
-RUN pip3 install . -r requirements/base.txt --break-system-packages
+RUN python3 -m venv /var/elastic-blast/.venv && \
+    source /var/elastic-blast/.venv && \
+    pip3 install . -r requirements/base.txt --no-cache-dir
 
 CMD ["/usr/bin/submit_jobs.py", "--help"]

docker-job-submit/Dockerfile.aws

@@ -27,8 +27,9 @@ COPY submit_jobs.py /usr/bin/
 RUN chmod +x /usr/bin/submit_jobs.py && \
     apk -U upgrade && \
     apk add --no-cache bash python3 py3-pip py3-wheel curl unzip && \
-    pip3 install --no-cache-dir --upgrade --break-system-packages  pip && \
-    pip3 install --no-cache-dir -r requirements.txt --break-system-packages && rm -rf /var/cache/apk/* requirements.txt
+    pip3 install --no-cache-dir --upgrade pip && \
+    python3 -m venv /.venv && source /.venv/bin/activate && \
+    pip3 install --no-cache-dir -r requirements.txt && rm -rf /var/cache/apk/* requirements.txt
 
 LABEL Description="NCBI ElasticBLAST Cloud Job Submission Module"
 LABEL Version=${version}

docker-job-submit/Makefile

@@ -28,7 +28,7 @@ SHELL=/bin/bash
 .PHONY: all pre-check check clean build publish gcp-build gcp-check gcp-clean
 
 IMG?=ncbi/elasticblast-job-submit
-VERSION?=4.0.3
+VERSION?=4.1.0
 ELB_VERSION?=$(shell git describe --tags --abbrev=0)
 GCP_PROJECT?=$(shell gcloud config get-value project 2>/dev/null)
 GCP_TEST_BUCKET?=gs://elasticblast-test/cloud-job-submission
@@ -64,6 +64,7 @@ check:
 gcp-build:
 	rsync -a ../src/elastic_blast/templates ${PWD}/
 	gcloud builds submit --config cloudbuild.yaml --substitutions _VERSION=${VERSION},_IMG=${IMG}
+	rm -fr templates
 
 .PHONY: aws-build
 aws-build:

docker-job-submit/cloud-job-submit.sh

@@ -73,9 +73,9 @@ while true; do
     sleep 30
 done
 
-s=`${KUBECTL} get jobs -l app=setup -o jsonpath='{.items[*].status.conditions[*].type}'`
+s=`${KUBECTL} get jobs -l app=setup -o jsonpath="{.items[*].status.conditions[?(@.type=='Complete')].type}"`
 if [[ "$s" != Complete*( Complete) ]]; then
-    echo "Setup job(s) failed:" `${KUBECTL} get jobs -l app=setup -o jsonpath='{.items[?(@.status.failed)].metadata.name}'` 
+    echo "Setup job(s) failed:" `${KUBECTL} get jobs -l app=setup -o json`
     copy_job_logs_to_results_bucket setup "${K8S_JOB_GET_BLASTDB}"
     copy_job_logs_to_results_bucket setup "${K8S_JOB_IMPORT_QUERY_BATCHES}"
     exit 1

docker-qs/README.md

@@ -11,4 +11,3 @@ various repositories.
 If you have `docker` available, run `make build` to build the image, and `make
 check` to test it locally.
 
-You will need credentials for NCBI-AWS-ELASTICBLAST-OPERATIONS to push to AWS ECR, see *To get NCBI AWS credentials in NCBI-AWS-ELASTICBLAST-OPERATIONS* in  [README-ncbi.md](../README-ncbi.md).

requirements/base.txt

@@ -1,10 +1,10 @@
-wheel==0.43.0
-setuptools==70.0.0
+wheel==0.41.2
+setuptools==68.2.0
 importlib-resources==6.1.1
 importlib-metadata==7.0.0
-pex==2.9.0
-boto3==1.34.141
-botocore==1.34.141
+pex==2.33.4
+boto3==1.37.12
+botocore==1.37.12
 awslimitchecker==12.0.0
-tenacity==8.5.0
+tenacity==9.0.0
 dataclasses-json==0.6.7

requirements/test.txt

@@ -1,14 +1,13 @@
 -r base.txt
 
-pytest==8.2.2
-pytest-cov==5.0.0
+pytest==8.3.5
+pytest-cov==6.0.0
 pytest-mock==3.14.0
-teamcity-messages==1.32
-mypy==1.10.1
-pylint==2.7.4
+teamcity-messages==1.33
+mypy==1.15.0
+pylint==3.3.5
 tox==4.4.12
-virtualenv==20.24.5
-yamllint==1.35.1
+yamllint==1.36.0
 moto==4.2.14
 docker==7.1.0
-cfn-lint==1.5.1
+cfn-lint==1.30.0

src/elastic_blast/base.py

@@ -29,7 +29,7 @@
 import re
 from dataclasses import dataclass, field, Field, fields, _MISSING_TYPE
 from enum import Enum, auto
-from typing import Dict, List, Union, Optional, NamedTuple
+from typing import Dict, List, Union, Optional, NamedTuple, get_origin, get_args
 
 @dataclass(frozen=True)
 class InstanceProperties:
@@ -317,10 +317,8 @@ def get_non_union_type(field: Field):
             If fieds.type is a Union, then the first type that is not None,
             otherwise field.type"""
         ftype = field.type
-        # FIXME: in python3.8 this can be done via typing.get_origin()
-        if getattr(ftype, '__origin__', None) is not None and \
-               ftype.__origin__ == Union:
-            ftype = [t for t in ftype.__args__ if t != type(None)][0]
+        if get_origin(ftype) is not None and get_origin(ftype) == Union:
+            ftype = [t for t in get_args(ftype) if t != type(None)][0]
         return ftype
 
 

src/elastic_blast/constants.py

@@ -156,7 +156,7 @@ class ElbExecutionMode(Enum):
 ELB_LOCAL_SSD_BLAST_JOB_TEMPLATE = 'resource:templates/blast-batch-job-local-ssd.yaml.template'
 GCS_DFLT_BUCKET = 'gs://blast-db'
 
-GCP_APIS = ['compute', 'serviceusage', 'container', 'storage-api', 'storage-component']
+GCP_APIS = ['compute', 'serviceusage', 'artifactregistry', 'container', 'storage-api', 'storage-component']
 # https://cloud.google.com/kubernetes-engine/docs/how-to/creating-managing-labels#requirements
 GCP_MAX_NUM_LABELS = 64
 # https://cloud.google.com/kubernetes-engine/docs/how-to/creating-managing-labels#requirements
@@ -212,8 +212,8 @@ def __str__(self):
 
 ELB_DOCKER_VERSION = '1.3.2'    # ElasticBLAST 1.3.0 uses BLAST+ 2.16.0
 ELB_QS_DOCKER_VERSION = '0.1.4'
-ELB_JANITOR_DOCKER_VERSION = '0.3.2'
-ELB_JOB_SUBMIT_DOCKER_VERSION = '4.0.3'
+ELB_JANITOR_DOCKER_VERSION = '0.4.0'
+ELB_JOB_SUBMIT_DOCKER_VERSION = '4.1.0'
 
 ELB_DOCKER_IMAGE_GCP = f'gcr.io/ncbi-sandbox-blast/ncbi/elb:{ELB_DOCKER_VERSION}'
 ELB_DOCKER_IMAGE_AWS = f'public.ecr.aws/ncbi-elasticblast/elasticblast-elb:{ELB_DOCKER_VERSION}'

src/elastic_blast/gcp.py

@@ -245,19 +245,17 @@ def _check_status(self, extended=False) -> Tuple[ElbStatus, Dict[str, int], Dict
         selector = 'app=blast'
         kubectl = f'kubectl --context={k8s_ctx}'
 
-        # if we need name of the job in the future add NAME:.metadata.name to custom-columns
-        # get status of jobs (pending/running, succeeded, failed)
-        cmd = f'{kubectl} get jobs -o custom-columns=STATUS:.status.conditions[0].type -l {selector}'.split()
+        cmd = f'{kubectl} get jobs -l {selector} -o custom-columns=STATUS:.status.conditions[*].type'.split()
         if self.dry_run:
             logging.debug(cmd)
         else:
             proc = safe_exec(cmd)
             for line in proc.stdout.decode().split('\n'):
                 if not line or line.startswith('STATUS'):
                     continue
-                if line.startswith('Complete'):
+                if 'Complete' in line:
                     counts['succeeded'] += 1
-                elif line.startswith('Failed'):
+                elif 'Failed' in line:
                     counts['failed'] += 1
                 else:
                     counts['pending'] += 1
@@ -303,7 +301,8 @@ def _job_status_by_app(self, app):
         selector = f'app={app}'
         k8s_ctx = self._get_gke_credentials()
         kubectl = f'kubectl --context={k8s_ctx}'
-        cmd = f'{kubectl} get jobs -o custom-columns=STATUS:.status.conditions[0].type -l {selector}'.split()
+        cmd = f'{kubectl} get jobs -l {selector} -o custom-columns=STATUS:.status.conditions[*].type'
+        cmd = cmd.split()
         if self.dry_run:
             logging.debug(cmd)
         else:
@@ -315,9 +314,9 @@ def _job_status_by_app(self, app):
             for line in proc.stdout.decode().split('\n'):
                 if not line or line.startswith('STATUS'):
                     continue
-                if line.startswith('Complete'):
+                if 'Complete' in line:
                     succeeded += 1
-                elif line.startswith('Failed'):
+                elif 'Failed' in line:
                     failed += 1
                 else:
                     pending += 1

src/elastic_blast/kubernetes.py

@@ -344,7 +344,10 @@ def _job_succeeded(k8s_ctx: str, k8s_job_file: pathlib.Path, dry_run: bool = Fal
 
     final_status = ''
     if 'conditions' in json_output['status'] and len(json_output['status']['conditions']) > 0:
-        final_status = json_output['status']['conditions'][0]['type']
+        final_status = 'Failed' # assume failure
+        for condition in json_output['status']['conditions']:
+            if 'type' in condition and condition['type'] in ('Complete', 'Failed'):
+                final_status = condition['type']
 
     if final_status == 'Complete' and 'succeeded' in json_output['status']:
         retval = json_output['status']['succeeded']

src/elastic_blast/util.py

@@ -41,7 +41,7 @@
 from .constants import DEPENDENCY_ERROR, AWS_MAX_TAG_LENGTH, GCP_MAX_LABEL_LENGTH
 from .constants import AWS_MAX_JOBNAME_LENGTH, CSP, ELB_GCS_PREFIX
 from .constants import ELB_DFLT_LOGLEVEL, ELB_DFLT_LOGFILE
-from .constants import INPUT_ERROR
+from .constants import INPUT_ERROR, UNKNOWN_ERROR
 from .constants import ELB_S3_PREFIX
 from .base import DBSource
 
@@ -244,11 +244,14 @@ def safe_exec(cmd: Union[List[str], str], env: Optional[Dict[str, str]] = None)
         msg = f'The command "{" ".join(e.cmd)}" returned with exit code {e.returncode}\n{e.stderr.decode()}\n{e.stdout.decode()}'
         if e.output is not None:
             '\n'.join([msg, f'{e.output.decode()}'])
-            raise SafeExecError(e.returncode, msg)
+            error_code = UNKNOWN_ERROR if e.returncode is None else e.returncode
+            raise SafeExecError(error_code, msg)
     except PermissionError as e:
-        raise SafeExecError(e.errno, str(e))
+        error_code = UNKNOWN_ERROR if e.errno is None else e.errno
+        raise SafeExecError(error_code, str(e))
     except FileNotFoundError as e:
-        raise SafeExecError(e.errno, e.strerror)
+        error_code = UNKNOWN_ERROR if e.errno is None else e.errno
+        raise SafeExecError(error_code, str(e))
     return p
 
 

tests/utils.py

@@ -366,7 +366,7 @@ def mocked_safe_exec(cmd: Union[List[str], str], env: Optional[Dict[str, str]] =
     elif cmd[0] == 'kubectl' and 'get pods -o custom-columns=STATUS' in ' '.join(cmd):
         return MockedCompletedProcess('\n'.join(['STATUS'] + ['Running' for i in K8S_JOB_STATUS if i == 'Running']))
 
-    elif cmd[0] == 'kubectl' and 'get jobs -o custom-columns=STATUS' in ' '.join(cmd):
+    elif cmd[0] == 'kubectl' and 'get jobs' in ' '.join(cmd):
         switcher = {'Failed': 'Failed',
                     'Succeeded': 'Complete',
                     'Running': '<none>',