Release 0.1.2

Author: Victor Joukov

.gitignore

@@ -39,3 +39,4 @@ creds.sh
 /*logs
 iam-policy.json
 clouseau*
+submit-and-wait-for-results.sh

Makefile

@@ -33,7 +33,7 @@ VENV?=.env
 BLAST_USAGE_REPORT=false
 
 # Cluster/GCP configuration
-ELB_CLUSTER_NAME?=elasticblast-${USER}
+ELB_CLUSTER_NAME?=`make -s results2clustername`
 ELB_NUM_NODES?=1
 # FIXME: should this be made the default? Allow enabling via env. var.? Something else? EB-297
 ELB_USE_PREEMPTIBLE?=
@@ -194,6 +194,8 @@ ELB_TEST_TIMEOUT_BLASTP_NOPAL?=960 # Based on EB-718: 20% more than 13h:20m (i.e
 
 ELB_TEST_TIMEOUT_BLASTN_16S_CHICKEN_GUT_METAGENOME?=10300 # Based on EB-736: 20% more than 143h (i.e.: 8,580 mins)
 
+ELB_TEST_TIMEOUT_BLASTP_NR_SMALL_DARK_MATTER?=45
+
 #############################################################################
 # Real world, performance tests
 
@@ -328,7 +330,7 @@ aws_regression_blastn_non_default_params: elastic-blast
 	-ELB_CLUSTER_NAME=${ELB_CLUSTER_NAME} \
 	ELB_RESULTS=${ELB_RESULTS} \
 		tests/tc-bash-runner.sh tests/integration-test.sh share/etc/elb-aws-blastn-non-default-params.ini ${ELB_TEST_TIMEOUT_MANE_VS_PDBNT}
-	test $$(zcat batch_000.out.gz | wc -l) -eq 5 
+	test $$(zcat batch_000-blastn-pdbnt.out.gz | wc -l) -eq 5 
 
 .PHONY: aws_regression_pdbnt_vs_mane_single_node_sync
 aws_regression_pdbnt_vs_mane_single_node_sync: elastic-blast
@@ -343,12 +345,17 @@ aws_regression_pdbnt_vs_mane_optimal_instance_type: elastic-blast
 	-ELB_RESULTS_BUCKET=${ELB_RESULTS_BUCKET} \
 		tests/tc-bash-runner.sh tests/integration-test.sh share/etc/elb-aws-spot-optimal-instance-type-blastn-pdbnt.ini ${ELB_TEST_TIMEOUT_MANE_VS_PDBNT_OPTIMAL_INSTANCE_TYPE}
 
-.PHONY: aws_regression_nt_vs_hepatitis_multi_node_sync
-aws_regression_nt_vs_hepatitis_multi_node_sync: elastic-blast
-	#-ELB_CLUSTER_NAME=${ELB_CLUSTER_NAME} \
-	#ELB_RESULTS=${ELB_RESULTS} \
-	#	tests/tc-bash-runner.sh tests/integration-test-synchronous.sh share/etc/elb-aws-blastn-nt-8-nodes.ini ${ELB_TEST_TIMEOUT_HEPATITIS_VS_NT}
-	true
+.PHONY: aws_regression_nt_vs_hepatitis_multi_node
+aws_regression_nt_vs_hepatitis_multi_node: elastic-blast
+	-ELB_CLUSTER_NAME=${ELB_CLUSTER_NAME} \
+	ELB_RESULTS=${ELB_RESULTS} \
+		tests/tc-bash-runner.sh tests/integration-test.sh share/etc/elb-aws-blastn-nt-8-nodes.ini ${ELB_TEST_TIMEOUT_HEPATITIS_VS_NT}
+
+.PHONY: aws_regression_blastp_nr_vs_small_dark_matter
+aws_regression_blastp_nr_vs_small_dark_matter: elastic-blast
+	-ELB_CLUSTER_NAME=${ELB_CLUSTER_NAME} \
+	ELB_RESULTS=${ELB_RESULTS} \
+		tests/tc-bash-runner.sh tests/integration-test.sh share/etc/elb-aws-blastp-nr-small-dark-matter.ini ${ELB_TEST_TIMEOUT_BLASTP_NR_SMALL_DARK_MATTER}
 
 .PHONY: aws_regression_blastp_pataa_vs_dark_matter_multi_node_sync
 aws_regression_blastp_pataa_vs_dark_matter_multi_node_sync: elastic-blast
@@ -362,9 +369,9 @@ aws_regression_blastn_taxid_filtering: elastic-blast
 	ELB_CLUSTER_NAME=${ELB_CLUSTER_NAME} \
 	ELB_RESULTS=${ELB_RESULTS} \
 		tests/tc-bash-runner.sh tests/integration-test.sh share/etc/elb-aws-blastn-taxidfiltering.ini ${ELB_TEST_TIMEOUT_MANE_VS_PDBNT}
-	test $$(zcat batch_000.out.gz | grep 246196 | wc -l) -gt 0
-	test $$(zcat batch_000.out.gz | grep 3562 | wc -l) -gt 0
-	test $$(zcat batch_000.out.gz | cut -f 13 | sort | uniq | wc -l) -eq 2 
+	test $$(zcat batch_000-blastn-pdbnt.out.gz | grep 246196 | wc -l) -gt 0
+	test $$(zcat batch_000-blastn-pdbnt.out.gz | grep 3562 | wc -l) -gt 0
+	test $$(zcat batch_000-blastn-pdbnt.out.gz | cut -f 13 | sort | uniq | wc -l) -eq 2 
 
 .PHONY: aws_regression_blastn_multi_file
 aws_regression_blastn_multi_file: elastic-blast
@@ -375,6 +382,14 @@ aws_regression_blastn_multi_file: elastic-blast
 	test $$(cat batch_003.fa | awk '/>/ {print substr($$L, 1, 11);}' | grep SRR5665119 | wc -l) -gt 0
 	test $$(cat batch_004.fa | awk '/>/ {print substr($$L, 1, 11);}' | grep RFQT0100 | wc -l) -gt 0
 
+.PHONY: aws_regression_failed_db_download
+aws_regression_failed_db_download: elastic-blast
+	ELB_CLUSTER_NAME=${ELB_CLUSTER_NAME} \
+	ELB_RESULTS=${ELB_RESULTS} \
+		tests/tc-bash-runner.sh tests/integration-test.sh share/etc/elb-aws-blastn-corrupted-db.ini ${ELB_TEST_TIMEOUT_MANE_VS_PDBNT} false
+	./elastic-blast run-summary --cfg share/etc/elb-aws-blastn-corrupted-db.ini -l batch-logs >/dev/null
+	test $$(aws batch describe-jobs --jobs $$(cat batch-logs | grep ^job | cut -f 2) | jq .jobs[0].attempts[].container.taskArn | wc -l) -gt 1
+
 .PHONY: regression_blastn_multi_file
 regression_blastn_multi_file: elastic-blast
 	ELB_CLUSTER_NAME=${ELB_CLUSTER_NAME} \
@@ -806,6 +821,19 @@ monitor:
 	-kubectl top pods --containers
 	kubectl top nodes
 
+# AWS Batch job management
+job_queue?=$(shell aws cloudformation describe-stacks --stack-name ${ELB_CLUSTER_NAME} --region ${ELB_AWS_REGION} --query "Stacks[0].Outputs[?OutputKey=='JobQueueName'].OutputValue" --output text)
+comp_env_name=$(shell aws cloudformation describe-stacks --stack-name ${ELB_CLUSTER_NAME} --region ${ELB_AWS_REGION} --query "Stacks[0].Outputs[?OutputKey=='ComputeEnvironmentName'].OutputValue" --output text)
+job_def=$(shell aws cloudformation describe-stacks --stack-name ${ELB_CLUSTER_NAME} --region ${ELB_AWS_REGION} --query "Stacks[0].Outputs[?OutputKey=='JobDefinitionName'].OutputValue" --output text)
+
+.PHONY: aws-monitor
+aws-monitor: AWS_PAGER=''
+aws-monitor:
+	for s in SUBMITTED PENDING RUNNABLE STARTING RUNNING SUCCEEDED FAILED; do \
+		echo "Checking $$s jobs"; \
+		aws batch list-jobs --region ${ELB_AWS_REGION} --job-queue ${job_queue} --job-status $$s; \
+	done
+
 progress:
 	for status in Pending Running Succeeded Failed; do \
 		echo -n "$$status "; \
@@ -835,6 +863,13 @@ list_resources: init
 	-gcloud compute disks list
 	-gcloud compute instances list
 
+# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-interruptions.html#finding-an-interrupted-Spot-Instance
+aws_list_interrupted_spot_instances: creds.sh
+	aws ec2 describe-instances \
+		--filters Name=instance-lifecycle,Values=spot Name=instance-state-name,Values=terminated,stopped Name=tag:billingcode,Values=elastic-blast Name=tag:Owner,Values=${USER} \
+			    --query "Reservations[*].Instances[*].InstanceId"
+
+
 aws_list_resources: export AWS_PAGER=
 aws_list_resources: creds.sh
 	-aws cloudformation describe-stacks --stack-name ${ELB_CLUSTER_NAME}
@@ -843,8 +878,9 @@ aws_list_resources: creds.sh
 	-aws batch describe-job-definitions --status ACTIVE --output json
 	-aws batch describe-compute-environments --output json
 
-aws_monitor: creds.sh
-	-source creds.sh && aws batch describe-jobs --jobs `aws s3 cp ${ELB_RESULTS}/metadata/job-ids - | jq -r .[] | tr '\n' ' ' `
+# 100 is the limit on number of arguments to --jobs
+aws_get_100_job_ids: creds.sh
+	-source creds.sh && aws batch describe-jobs --jobs `aws s3 cp ${ELB_RESULTS}/metadata/job-ids.json - | jq -r .[] | head -100 | tr '\n' ' '`
 
 ###############################################################################
 # AWS ElasticBLAST suport
@@ -903,4 +939,4 @@ scrub-code: clouseau_venv
 
 .PHONY: results2clustername
 results2clustername:
-	./share/tools/results2clustername.sh ${ELB_RESULTS}
+	@./share/tools/results2clustername.sh ${ELB_RESULTS}

README.md

@@ -1,12 +1,19 @@
 # ElasticBLAST
 
-The National Center for Biotechnology (NCBI), part of the National Library of
-Medicine, is making the source code for ElasticBLAST available on GitHub as an
+ElasticBLAST is a cloud-based tool to perform your BLAST searches faster and make you more effective.
+
+ElasticBLAST is ideal for users who have a large number (thousands or more) of queries to BLAST or who prefer to use cloud infrastructure for their searches.  It can run BLAST searches that cannot be done on [NCBI WebBLAST](https://blast.ncbi.nlm.nih.gov) and runs them more quickly than stand-alone [BLAST+](https://www.ncbi.nlm.nih.gov/books/NBK279690/).
+
+ElasticBLAST speeds up your work by distributing your BLAST+ searches across multiple cloud instances. The ability to scale resources in this way allows larger numbers of queries to be searched in a shorter time than you could with BLAST+ on a single host.
+
+The National Center for Biotechnology Information ([NCBI](https://www.ncbi.nlm.nih.gov)), part of the National Library of
+Medicine at the NIH, developed and maintains ElasticBLAST.
+
+The NCBI is making the source code for ElasticBLAST available on GitHub as an
 Open Distribution to allow the user community to easily obtain and examine
 that code.  GitHub also provides a means for users to report issues and
 suggest modifications through pull requests. 
 
-
 The NCBI will use internal source code control as the repository of record and
 push regular releases of the ElasticBLAST
 source code to GitHub.  The BLAST developers will work to ensure that

share/etc/elb-aws-blastn-nt-8-nodes.ini

@@ -6,16 +6,13 @@ aws-region = us-east-1
 [cluster]
 machine-type = m5.8xlarge
 num-nodes = 8
-num-cpus = 30
-disk-type = gp2
-# minimal size to trigger faster IO speed for gp2
-pd-size = 334G
-use-preemptible = true
+num-cpus = 16
+# Try more modern and larger disk size to prevent running out of burst credits
+disk-type = gp3
+pd-size = 1000G
 
 [blast]
 program = blastn
 db = nt
-mem-request = 64G
-mem-limit = 110G
+mem-limit = 60G
 queries = s3://elasticblast-test/queries/hepatitis.fsa.gz
-batch-len = 2000000

src/elb/base.py

@@ -98,7 +98,7 @@ def __new__(cls, value):
         str_value = str(value)
         number_re = re.compile(r'^\d+[kKmMgGtT]$|^\d+.\d+[kKmMgGtT]$')
         if not number_re.match(str_value):
-            raise ValueError('Memory request or limit must be specifed by a number followed by a unit, for example 100m')
+            raise ValueError('Memory request or limit must be specified by a number followed by a unit, for example 100m')
         if float(str_value[:-1]) <= 0:
             raise ValueError('Memory request or limit must be larger than zero')
         return super(cls, cls).__new__(cls, str_value)

src/elb/commands/run_summary.py

@@ -42,8 +42,11 @@
 from elb.constants import CLUSTER_ERROR, ELB_AWS_JOB_IDS, ELB_AWS_QUERY_LENGTH, ELB_METADATA_DIR, PERMISSIONS_ERROR
 from elb.constants import ELB_LOG_DIR, CSP, ElbCommand
 
+# Artificial exit codes to differentiate failure modes
+# of AWS job.
 JOB_EXIT_CODE_UNINITIALIZED = -1
 JOB_EXIT_CODE_FAILED_WITH_NO_ATTEMPT = 100000
+JOB_EXIT_CODE_FAILED_WITH_NO_EXIT_CODE = 100001
 
 
 @dataclass
@@ -458,11 +461,14 @@ def _read_job_logs_aws(cfg, write_logs):
                 if len(attempts) > 0:
                     attempt = attempts[-1]
                     container = attempt['container']
-                    job_exit_code = container['exitCode']
-                    created = job['createdAt'] / 1000
-                    started = job['startedAt'] / 1000
-                    stopped = job['stoppedAt'] / 1000
-                    parameters = job['parameters']
+                    if 'exitCode' in container:
+                        job_exit_code = container['exitCode']
+                        created = job['createdAt'] / 1000
+                        started = job['startedAt'] / 1000
+                        stopped = job['stoppedAt'] / 1000
+                        parameters = job['parameters']
+                    else:
+                        job_exit_code = JOB_EXIT_CODE_FAILED_WITH_NO_EXIT_CODE
                 else:
                     # Signal that job failed without attempts
                     job_exit_code = JOB_EXIT_CODE_FAILED_WITH_NO_ATTEMPT

src/elb/config.py

@@ -35,7 +35,7 @@
 import getpass
 from hashlib import md5
 from .util import check_positive_int, get_query_batch_size
-from .util import get_blastdb_mem_requirements, ElbSupportedPrograms
+from .util import ElbSupportedPrograms
 from .util import validate_gcp_string, validate_gke_cluster_name
 from .util import validate_aws_region
 from .constants import APP_STATE, CFG_BLAST, CFG_BLAST_BATCH_LEN, CFG_BLAST_DB, CFG_BLAST_DB_MEM_MARGIN, CFG_BLAST_DB_SRC, CFG_BLAST_MEM_LIMIT, CFG_BLAST_MEM_REQUEST, CFG_BLAST_OPTIONS, CFG_BLAST_PROGRAM, CFG_BLAST_QUERY, CFG_BLAST_RESULTS, CFG_CLOUD_PROVIDER, CFG_CLUSTER, CFG_CLUSTER_BID_PERCENTAGE, CFG_CLUSTER_DISK_TYPE, CFG_CLUSTER_DRY_RUN, CFG_CLUSTER_EXP_USE_LOCAL_SSD, CFG_CLUSTER_MACHINE_TYPE, CFG_CLUSTER_MAX_NODES, CFG_CLUSTER_MIN_NODES, CFG_CLUSTER_NAME, CFG_CLUSTER_NUM_CPUS, CFG_CLUSTER_NUM_NODES, CFG_CLUSTER_PD_SIZE, CFG_CLUSTER_PROVISIONED_IOPS, CFG_CLUSTER_RUN_LABEL, CFG_CLUSTER_USE_PREEMPTIBLE, CFG_CP_AWS_REGION, CFG_CP_GCP_NETWORK, CFG_CP_GCP_PROJECT, CFG_CP_GCP_REGION, CFG_CP_GCP_SUBNETWORK, CFG_CP_GCP_ZONE, CFG_CP_NAME, CFG_TIMEOUTS, CFG_TIMEOUT_BLAST_K8S_JOB, CFG_TIMEOUT_INIT_PV
@@ -49,9 +49,6 @@
 from .util import UserReportError
 from .filehelper import parse_bucket_name_key
 from typing import List
-from .gcp_traits import get_machine_properties as gcp_get_machine_properties
-from .aws_traits import get_machine_properties as aws_get_machine_properties
-from .aws_traits import create_aws_config
 
 
 def _set_sections(cfg: configparser.ConfigParser) -> None:

src/elb/constants.py

@@ -119,7 +119,7 @@ class ElbCommand(Enum):
 MOL_TYPE_NUCL = 'nucl'
 
 ELB_DFLT_AWS_REGION = 'us-east-1'
-ELB_DOCKER_IMAGE = 'ncbi/elb:0.0.20'
+ELB_DOCKER_IMAGE = 'ncbi/elb:0.0.24'
 ELB_DFLT_AWS_DISK_TYPE = 'gp2'
 # minimal size of gp2 disk which triggers fastest speed
 ELB_DFLT_AWS_PD_SIZE = '334G'

src/elb/templates/elastic-blast-cf.yaml

@@ -511,13 +511,44 @@ Resources:
       ComputeEnvironmentOrder:
       - ComputeEnvironment: !Ref ComputeEnvironment
         Order: 1
+      Tags:
+         Project: BLAST
+         billingcode: elastic-blast
+         Owner: !Ref Owner
+         Name: !Join [-, [elasticblast, !Ref Owner, !Ref RandomToken]]
 
   Ec2LaunchTemplate:
     Type: AWS::EC2::LaunchTemplate
     Properties:
       LaunchTemplateName: !Join [-, [elasticblast, !Ref Owner, !Ref RandomToken]]
       LaunchTemplateData:
         EbsOptimized: true
+        UserData:
+          Fn::Base64: !Sub |
+            MIME-Version: 1.0
+            Content-Type: multipart/mixed; boundary="==MYBOUNDARY=="
+
+            --==MYBOUNDARY==
+            Content-Type: text/x-shellscript; charset="us-ascii"
+
+            #!/bin/bash -xe
+
+            yum install -y unzip
+            curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+            unzip awscliv2.zip
+            ./aws/install
+            rm -f awscliv2.zip
+
+            # Set tags on attached volume
+            AWS_INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
+            ROOT_VOLUME_IDS=$(aws ec2 describe-instances --region ${AWS::Region} --instance-id $AWS_INSTANCE_ID --output text --query Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId)
+            aws ec2 create-tags --resources $ROOT_VOLUME_IDS --region ${AWS::Region} --tags Key=Name,Value=${AWS::StackName}
+            aws ec2 create-tags --resources $ROOT_VOLUME_IDS --region ${AWS::Region} --tags Key=Owner,Value=${Owner}
+            aws ec2 create-tags --resources $ROOT_VOLUME_IDS --region ${AWS::Region} --tags Key=billingcode,Value=elastic-blast
+            aws ec2 create-tags --resources $ROOT_VOLUME_IDS --region ${AWS::Region} --tags Key=Project,Value=BLAST
+            aws ec2 create-tags --resources $ROOT_VOLUME_IDS --region ${AWS::Region} --tags Key=Created-For-Instance-ID,Value=$AWS_INSTANCE_ID
+
+            --==MYBOUNDARY==
         BlockDeviceMappings:
         - DeviceName: /dev/xvda
           Ebs:
@@ -529,6 +560,15 @@ Resources:
               - ProvisionedIopsRequested
               - !Ref ProvisionedIops
               - !Ref AWS::NoValue
+      TagSpecifications:
+        - ResourceType: launch-template
+          Tags:
+            - Key: Project
+              Value: BLAST
+            - Key: Owner
+              Value: !Ref Owner
+            - Key: billingcode
+              Value: elastic-blast
 
   BlastSearchJobDefinition:
     Type: AWS::Batch::JobDefinition
@@ -582,9 +622,21 @@ Resources:
         # retry on infrastructure problems (including terminated spot instance)
         - OnStatusReason: Host EC2*
           Action: RETRY
-        # no retries for other reasons (like a non-zero exit code)
+        # retry on database-related errors
+        - OnExitCode: 2
+          Action: RETRY
+        # These may occur if burst credits on EBS are exhausted
+        - OnReason: DockerTimeoutError*
+          Action: RETRY
+        - OnReason: CannotInspectContainer*
+          Action: RETRY
         - OnReason: "*"
           Action: EXIT
+      Tags:
+         Project: BLAST
+         billingcode: elastic-blast
+         Owner: !Ref Owner
+         Name: !Join [-, [elasticblast, !Ref Owner, !Ref RandomToken]]
 
 
   # AWS advises to use region-specific IAM resource names. Otherwise using the
@@ -614,6 +666,15 @@ Resources:
           Action: sts:AssumeRole
       ManagedPolicyArns:
       - arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role
+      Tags:
+        - Key: Name
+          Value: !Join [-, [elasticblast, !Ref Owner, !Ref RandomToken]]
+        - Key: Project
+          Value: BLAST
+        - Key: Owner
+          Value: !Ref Owner
+        - Key: billingcode
+          Value: elastic-blast
 
   CreatedBatchServiceRole:
     Type: AWS::IAM::Role
@@ -631,6 +692,15 @@ Resources:
           Action: sts:AssumeRole
       ManagedPolicyArns:
       - arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole
+      Tags:
+        - Key: Name
+          Value: !Join [-, [elasticblast, !Ref Owner, !Ref RandomToken]]
+        - Key: Project
+          Value: BLAST
+        - Key: Owner
+          Value: !Ref Owner
+        - Key: billingcode
+          Value: elastic-blast
 
   CreatedBatchJobRole:
     Type: AWS::IAM::Role
@@ -647,6 +717,15 @@ Resources:
           Action: sts:AssumeRole
       ManagedPolicyArns:
       - arn:aws:iam::aws:policy/AmazonS3FullAccess
+      Tags:
+        - Key: Name
+          Value: !Join [-, [elasticblast, !Ref Owner, !Ref RandomToken]]
+        - Key: Project
+          Value: BLAST
+        - Key: Owner
+          Value: !Ref Owner
+        - Key: billingcode
+          Value: elastic-blast
 
   CreatedSpotFleetRole:
     Type: AWS::IAM::Role
@@ -664,6 +743,15 @@ Resources:
           Action: sts:AssumeRole
       ManagedPolicyArns:
       - arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetTaggingRole
+      Tags:
+        - Key: Name
+          Value: !Join [-, [elasticblast, !Ref Owner, !Ref RandomToken]]
+        - Key: Project
+          Value: BLAST
+        - Key: Owner
+          Value: !Ref Owner
+        - Key: billingcode
+          Value: elastic-blast
 
 Outputs:
   JobQueueName: