Release 0.2.5

Author: boratyng

CITATION.cff

@@ -1,8 +1,8 @@
 cff-version: "1.1.0"
 message: "If you use this software, please cite it using these metadata."
 title: ElasticBLAST
-version: "0.2.4"
-date-released: 2022-03-04
+version: "0.2.5"
+date-released: 2022-03-24
 license: "NCBI Public Domain"
 repository-code: "https://github.com/ncbi/elastic-blast/"
 authors: 

bin/aws-create-elastic-blast-janitor-role.sh

@@ -5,7 +5,6 @@
 # Author: Christiam Camacho (camacho@ncbi.nlm.nih.gov)
 # Created: Mon Nov 29 17:29:31 EST 2021
 
-export PATH=/bin:/usr/local/bin:/usr/bin
 set -xeuo pipefail
 shopt -s nullglob
 

bin/aws-delete-elastic-blast-janitor-role.sh

@@ -5,7 +5,6 @@
 # Author: Christiam Camacho (camacho@ncbi.nlm.nih.gov)
 # Created: Tue Nov 30 08:07:53 EST 2021
 
-export PATH=/bin:/usr/local/bin:/usr/bin
 set -xuo pipefail
 shopt -s nullglob
 

bin/aws-describe-elastic-blast-janitor-role.sh

@@ -5,7 +5,6 @@
 # Author: Christiam Camacho (camacho@ncbi.nlm.nih.gov)
 # Created: Tue Nov 30 08:09:03 EST 2021
 
-export PATH=/bin:/usr/local/bin:/usr/bin
 set -xuo pipefail
 shopt -s nullglob
 

bin/gcp-setup-elastic-blast-janitor.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# bin/gcp-setup-elastic-blast-janitor.sh: Script to set up the ElasticBLAST
+# janitor permissions in GCP
+#
+# Author: Christiam Camacho (camacho@ncbi.nlm.nih.gov)
+# Created: Tue 08 Mar 2022 04:53:15 PM EST
+
+set -euo pipefail
+shopt -s nullglob
+
+command -v gcloud >&/dev/null || { echo "gcloud must be in your PATH for this script to work"; exit 1; }
+
+user=$(gcloud config get-value account)
+prj=$(gcloud config get-value project)
+
+usage() {
+    echo -e "$0 [-h] [-u USERNAME] [-p GCP_PROJECT_ID]\n"
+    echo -e "This script sets up the permissions to the ElasticBLAST janitor in GCP"
+    echo -e "Options:"
+    echo -e "\t-u USERNAME: GCP user, group or service account to configure (default: user:$user)"
+    echo -e "\t\tFor specific format, please see https://cloud.google.com/sdk/gcloud/reference/projects/add-iam-policy-binding#--member"
+    echo -e "\t-p GCP_PROJECT_ID: GCP project ID (default: ${prj})"
+    echo -e "\t\tDocumentation: https://cloud.google.com/sdk/gcloud/reference/projects/add-iam-policy-binding#PROJECT_ID"
+    echo -e "\t-h: Show this message"
+}
+
+while getopts "u:ph" OPT; do
+    case $OPT in 
+        u) user=${OPTARG}
+            ;;
+        p) prj=${OPTARG}
+            ;;
+        h) usage
+           exit 0
+            ;;
+    esac
+done
+
+gcloud projects add-iam-policy-binding ${prj} --member=user:${user} --role=roles/container.admin

docker-blast/Makefile

@@ -30,7 +30,7 @@ GCP_IMG?=gcr.io/ncbi-sandbox-blast/${IMG}
 AWS_SERVER?=public.ecr.aws/i6v3i0i9
 AWS_IMG?=${AWS_SERVER}/elasticblast-elb
 AWS_REGION?=us-east-1
-VERSION?=1.0.3
+VERSION?=1.1.0
 
 ifeq (, $(shell which vmtouch 2>/dev/null))
 NOVMTOUCH?=--no-vmtouch

requirements/base.txt

@@ -1,11 +1,11 @@
-wheel
-setuptools
-importlib-resources
-importlib-metadata
-pex
-boto3==1.21.10
-botocore==1.24.10
-awslimitchecker
-tenacity
-dataclasses_json
-types-pkg-resources
+wheel == 0.37.1
+setuptools == 56.0.0
+importlib-resources == 5.4.0
+importlib-metadata == 4.11.3
+pex == 2.1.73
+boto3 == 1.21.24
+botocore == 1.24.24
+awslimitchecker == 12.0.0
+tenacity == 8.0.1
+dataclasses-json == 0.5.7
+types-pkg-resources == 0.1.3

requirements/test.txt

@@ -1,13 +1,13 @@
 -r base.txt
 
-pytest
-pytest-cov
-pytest-mock
-teamcity-messages
-mypy
-pylint==2.7.4
-tox
-yamllint
-moto >= 2.2.19
-docker
-cfn-lint
+pytest == 7.1.1
+pytest-cov == 3.0.0
+pytest-mock == 3.7.0
+teamcity-messages == 1.31
+mypy == 0.941
+pylint == 2.7.4
+tox == 3.24.5
+yamllint == 1.26.3
+moto == 3.1.1
+docker == 5.0.3
+cfn-lint == 0.58.4

src/elastic_blast/aws.py

@@ -63,6 +63,7 @@
 from .constants import ELB_AWS_JANITOR_LAMBDA_DEPLOYMENT_BUCKET, ELB_AWS_JANITOR_LAMBDA_DEPLOYMENT_KEY
 from .constants import CFG_CLOUD_PROVIDER, CFG_CP_AWS_AUTO_SHUTDOWN_ROLE, CSP
 from .constants import AWS_JANITOR_ROLE_NAME
+from .constants import STATUS_MESSAGE_ERROR, STATUS_MESSAGE_VERBOSE
 from .filehelper import parse_bucket_name_key
 from .aws_traits import get_machine_properties, create_aws_config, get_availability_zones_for
 from .object_storage_utils import write_to_s3
@@ -933,20 +934,20 @@ def upload_query_length(self, query_length: int) -> None:
             logging.debug('dry-run: would have uploaded query length')
 
 
-    def check_status(self, extended=False) -> Tuple[ElbStatus, Dict[str, int], str]:
+    def check_status(self, extended=False) -> Tuple[ElbStatus, Dict[str, int], Dict[str, str]]:
         """ Check execution status of ElasticBLAST search
         Parameters:
             extended - do we need verbose information about jobs
         Returns:
             tuple of
                 status - cluster status, ElbStatus
                 counts - job counts for all job states
-                verbose_result - detailed info about jobs
+                verbose_result - a dictionary with entries: label, detailed info about jobs
         """
         try:
             retval = self._status_from_results()
             if retval != ElbStatus.UNKNOWN:
-                return retval, self.cached_counts, self.cached_failure_message
+                return retval, self.cached_counts, {STATUS_MESSAGE_ERROR: self.cached_failure_message}
 
             counts, details = self._check_status(extended)
             njobs = sum(counts.values())
@@ -995,12 +996,12 @@ def _load_job_ids_from_aws(self):
                     raise
 
     @handle_aws_error
-    def _check_status(self, extended) -> Tuple[Dict[str, int], str]:
+    def _check_status(self, extended) -> Tuple[Dict[str, int], Dict[str, str]]:
         """ Internal check_status, converts AWS exceptions to UserReportError  """
         counts : Dict[str, int] = defaultdict(int)
         if self.dry_run:
             logging.info('dry-run: would have checked status')
-            return counts, ''
+            return counts, {}
 
         if extended:
             return self._check_status_extended()
@@ -1024,9 +1025,9 @@ def _check_status(self, extended) -> Tuple[Dict[str, int], str]:
             'succeeded': counts['SUCCEEDED'],
             'failed': counts['FAILED'],
         }
-        return status, ''
+        return status, {}
 
-    def _check_status_extended(self) -> Tuple[Dict[str, int], str]:
+    def _check_status_extended(self) -> Tuple[Dict[str, int], Dict[str, str]]:
         """ Internal check_status_extended, not protected against exceptions in AWS """
         logging.debug(f'Retrieving jobs for queue {self.job_queue_name}')
         jobs = {}
@@ -1073,7 +1074,7 @@ def _check_status_extended(self) -> Tuple[Dict[str, int], str]:
             detailed_rep.append(f'{status.capitalize()} {jobs_in_status}')
             if jobs_in_status:
                 detailed_rep.append('\n'.join(detailed_info[status]))
-        return counts, '\n'.join(detailed_rep)
+        return counts, {STATUS_MESSAGE_VERBOSE: '\n'.join(detailed_rep)}
 
     def _remove_ancillary_data(self, bucket_prefix: str) -> None:
         """ Removes ancillary data from the end user's result bucket

src/elastic_blast/commands/status.py

@@ -29,7 +29,7 @@
 import logging
 from typing import Any, List
 
-from elastic_blast.constants import ElbCommand, ElbStatus
+from elastic_blast.constants import ElbCommand, ElbStatus, STATUS_MESSAGE_ERROR
 from elastic_blast.elasticblast_factory import ElasticBlastFactory
 from elastic_blast.elb_config import ElasticBlastConfig
 
@@ -53,7 +53,7 @@ def _status(args, cfg: ElasticBlastConfig, clean_up_stack: List[Any]) -> int:
     cfg.validate(ElbCommand.STATUS)
     returncode = 0
     try:
-        verbose_result = ''
+        verbose_result = {}
         elastic_blast = ElasticBlastFactory(cfg, False, clean_up_stack)
         while True:
             status, counts, verbose_result = elastic_blast.check_status(args.verbose)
@@ -90,8 +90,10 @@ def _status(args, cfg: ElasticBlastConfig, clean_up_stack: List[Any]) -> int:
                 print(f'Your ElasticBLAST search succeeded, results can be found in {cfg.cluster.results}')
             elif status == ElbStatus.FAILURE:
                 print(FAILURE_MESSAGE)
+                if not args.verbose and STATUS_MESSAGE_ERROR in verbose_result:
+                    print(verbose_result[STATUS_MESSAGE_ERROR])
             else:
                 print(result)
             if args.verbose and verbose_result:
-                print(verbose_result)
+                print('\n'.join(verbose_result.values()))
     return returncode

src/elastic_blast/constants.py

@@ -200,7 +200,7 @@ def __str__(self):
 ELB_DFLT_GCP_REGION = 'us-east4'
 ELB_DFLT_AWS_REGION = 'us-east-1'
 
-ELB_DOCKER_VERSION = '1.0.3'
+ELB_DOCKER_VERSION = '1.1.0'
 ELB_QS_DOCKER_VERSION = '0.1.2'
 ELB_JANITOR_DOCKER_VERSION = '0.2.0'
 ELB_JOB_SUBMIT_DOCKER_VERSION = '2.0.0'
@@ -327,3 +327,9 @@ def __str__(self):
 ELB_DFLT_NUM_BATCHES_FOR_TESTING = 100
 ELB_DFLT_LOGLEVEL = 'DEBUG'
 ELB_DFLT_LOGFILE = 'elastic-blast.log'
+
+# Labels for messages generated when checking search status
+# alwsays printed to the screen
+STATUS_MESSAGE_ERROR = 'error'
+# printed only when requested
+STATUS_MESSAGE_VERBOSE = 'verbose'

src/elastic_blast/elasticblast.py

@@ -76,15 +76,15 @@ def submit(self, query_batches: List[str], query_length, one_stage_cloud_query_s
                                               of executing a regular job """
 
     @abstractmethod
-    def check_status(self, extended=False) -> Tuple[ElbStatus, Dict[str, int], str]:
+    def check_status(self, extended=False) -> Tuple[ElbStatus, Dict[str, int], Dict[str, str]]:
         """ Check execution status of ElasticBLAST search
         Parameters:
             extended - do we need verbose information about jobs
         Returns:
             tuple of
                 status - cluster status, ElbStatus
                 counts - job counts for all job states
-                verbose_result - detailed info about jobs
+                verbose_result - a dictionary where each entry is label, detailed info about jobs
         """
 
     # Compatibility method, used now only in janitor.py

src/elastic_blast/gcp.py

@@ -59,6 +59,7 @@
 from .constants import GKE_CLUSTER_STATUS_PROVISIONING, GKE_CLUSTER_STATUS_RECONCILING
 from .constants import GKE_CLUSTER_STATUS_RUNNING, GKE_CLUSTER_STATUS_RUNNING_WITH_ERROR
 from .constants import GKE_CLUSTER_STATUS_STOPPING, GKE_CLUSTER_STATUS_ERROR
+from .constants import STATUS_MESSAGE_ERROR
 from .elb_config import ElasticBlastConfig
 from .elasticblast import ElasticBlast
 
@@ -176,41 +177,41 @@ def submit(self, query_batches: List[str], query_length, one_stage_cloud_query_s
         self.cleanup_stack.clear()
         self.cleanup_stack.append(lambda: kubernetes.collect_k8s_logs(self.cfg))
 
-    def check_status(self, extended=False) -> Tuple[ElbStatus, Dict[str, int], str]:
+    def check_status(self, extended=False) -> Tuple[ElbStatus, Dict[str, int], Dict[str, str]]:
         """ Check execution status of ElasticBLAST search
         Parameters:
             extended - do we need verbose information about jobs
         Returns:
             tuple of
                 status - cluster status, ElbStatus
                 counts - job counts for all job states
-                verbose_result - detailed info about jobs
+                verbose_result - a dictionary with enrties: label, detailed info about jobs
         """
         try:
             return self._check_status(extended)
         except SafeExecError as err:
             # cluster is not valid, return empty result
             msg = err.message.strip()
             logging.info(msg)
-            return ElbStatus.UNKNOWN, defaultdict(int), msg
+            return ElbStatus.UNKNOWN, defaultdict(int), {STATUS_MESSAGE_ERROR: msg} if msg else {}
 
-    def _check_status(self, extended=False) -> Tuple[ElbStatus, Dict[str, int], str]:
+    def _check_status(self, extended=False) -> Tuple[ElbStatus, Dict[str, int], Dict[str, str]]:
         # We cache only status from gone cluster - it can't change anymore
         if self.cached_status:
-            return self.cached_status, self.cached_counts, self.cached_failure_message
+            return self.cached_status, self.cached_counts, {STATUS_MESSAGE_ERROR: self.cached_failure_message} if self.cached_failure_message else {}
         counts: DefaultDict[str, int] = defaultdict(int)
         self._enable_gcp_apis()
         status = self._status_from_results()
         if status != ElbStatus.UNKNOWN:
-            return status, self.cached_counts, self.cached_failure_message
+            return status, self.cached_counts, {STATUS_MESSAGE_ERROR: self.cached_failure_message} if self.cached_failure_message else {}
 
         gke_status = check_cluster(self.cfg)
         if not gke_status:
-            return ElbStatus.UNKNOWN, {}, f'Cluster "{self.cfg.cluster.name}" was not found'
+            return ElbStatus.UNKNOWN, {}, {STATUS_MESSAGE_ERROR: f'Cluster "{self.cfg.cluster.name}" was not found'}
 
         logging.debug(f'GKE status: {gke_status}')
         if gke_status in [GKE_CLUSTER_STATUS_RECONCILING, GKE_CLUSTER_STATUS_PROVISIONING]:
-            return ElbStatus.SUBMITTING, {}, ''
+            return ElbStatus.SUBMITTING, {}, {}
 
         if gke_status == GKE_CLUSTER_STATUS_STOPPING:
             # TODO: This behavior is consistent with current tests, consider returning a value
@@ -272,7 +273,7 @@ def _check_status(self, extended=False) -> Tuple[ElbStatus, Dict[str, int], str]
                 if failed > 0:
                     status = ElbStatus.FAILURE
 
-        return status, counts, ''
+        return status, counts, {}
 
     def _job_status_by_app(self, app):
         """ get status of jobs (pending/running, succeeded, failed) by app """