AN-246 Create prod pools with Batch API enabled

local-dev/render-config.sh

@@ -6,7 +6,7 @@ set -e
 
 VAULT_TOKEN=${1:-$(cat $HOME/.vault-token)}
 DSDE_TOOLBOX_DOCKER_IMAGE=broadinstitute/dsde-toolbox:dev
-VAULT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/toolsalpha/buffer/app-sa
+VAULT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/buffertest/buffer/app-sa
 VAULT_JANITOR_CLIENT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/tools/crl_janitor/client-sa
 SERVICE_ACCOUNT_OUTPUT_FILE_PATH="$(dirname $0)"/../src/test/resources/rendered/sa-account.json
 JANITOR_CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH="$(dirname $0)"/../src/test/resources/rendered/janitor-client-sa-account.json

local-dev/run_local.sh

@@ -13,7 +13,7 @@ export BUFFER_CRL_JANITOR_CLIENT_CREDENTIAL_FILE_PATH="$(dirname $0)"/../src/tes
 export BUFFER_CRL_JANITOR_TRACK_RESOURCE_PROJECT_ID=terra-kernel-k8s
 export BUFFER_CRL_JANITOR_TRACK_RESOURCE_TOPIC_ID=crljanitor-tools-pubsub-topic
 export BUFFER_CRL_TEST_RESOURCE_TIME_TO_LIVE=1h
-export BUFFER_POOL_CONFIG_PATH=config/toolsalpha
+export BUFFER_POOL_CONFIG_PATH=config/buffertest
 export SPRING_PROFILES_INCLUDE=human-readable-logging
 export TERRA_COMMON_STAIRWAY_FORCE_CLEAN_START=true
 export TERRA_COMMON_TRACING_STACKDRIVER_EXPORT_ENABLED=false

src/main/resources/config/prod/pool_schema.yml

@@ -4,6 +4,9 @@ poolConfigs:
   - poolId: "cwb_ws_prod_v7"
     size: 3000
     resourceConfigName: "cwb_ws_prod_v7"
+  - poolId: "cwb_ws_prod_v8"
+    size: 3000
+    resourceConfigName: "cwb_ws_prod_v8"
   - poolId: "datarepo_v1"
     size: 1000
     resourceConfigName: "datarepo_v1"
@@ -13,3 +16,6 @@ poolConfigs:
   - poolId: "vpc_sc_v11"
     size: 1000
     resourceConfigName: "vpc_sc_v11"
+  - poolId: "vpc_sc_v12"
+    size: 1000
+    resourceConfigName: "vpc_sc_v12"

src/main/resources/config/prod/resource-config/cwb_ws_prod_v8.yml

@@ -0,0 +1,33 @@
+# Community Workbench buffered workspace template
+---
+configName: "cwb_ws_prod_v8"
+gcpProjectConfig:
+  projectIdSchema:
+    prefix: "terra"
+    scheme: "RANDOM_CHAR"
+  # firecloud.org/prod/CommunityWorkbench
+  parentFolderId: "710468670182"
+  billingAccount: "0106B0-41CAA9-427C96"
+  enabledApis:
+    - "bigquery-json.googleapis.com"
+    - "batch.googleapis.com"
+    - "compute.googleapis.com"
+    - "container.googleapis.com"
+    - "containerregistry.googleapis.com"
+    - "cloudbilling.googleapis.com"
+    - "clouderrorreporting.googleapis.com"
+    - "cloudkms.googleapis.com"
+    - "cloudtrace.googleapis.com"
+    - "dataflow.googleapis.com"
+    - "dataproc.googleapis.com"
+    - "dns.googleapis.com"
+    - "lifesciences.googleapis.com"
+    - "logging.googleapis.com"
+    - "monitoring.googleapis.com"
+    - "storage-api.googleapis.com"
+    - "storage-component.googleapis.com"
+  network:
+    enableNetworkMonitoring: "false"
+    enablePrivateGoogleAccess: "true"
+  kubernetesEngine:
+    createGkeDefaultServiceAccount: "true"

src/main/resources/config/prod/resource-config/vpc_sc_v12.yml

@@ -0,0 +1,42 @@
+# Projects with VPC-SC configuration
+---
+configName: "vpc_sc_v12"
+gcpProjectConfig:
+  projectIdSchema:
+    prefix: "terra-vpc-sc"
+    scheme: "RANDOM_CHAR"
+  # firecloud.org/prod/for_vpc_sc_unclaimed
+  parentFolderId: "160283235721"
+  billingAccount: "0106B0-41CAA9-427C96"
+  enabledApis:
+    - "bigquery-json.googleapis.com"
+    - "batch.googleapis.com"
+    - "compute.googleapis.com"
+    - "container.googleapis.com"
+    - "cloudbilling.googleapis.com"
+    - "clouderrorreporting.googleapis.com"
+    - "cloudkms.googleapis.com"
+    - "cloudtrace.googleapis.com"
+    - "containerregistry.googleapis.com"
+    - "dataflow.googleapis.com"
+    - "dataproc.googleapis.com"
+    - "dns.googleapis.com"
+    - "lifesciences.googleapis.com"
+    - "logging.googleapis.com"
+    - "monitoring.googleapis.com"
+    - "serviceusage.googleapis.com"
+    - "storage-api.googleapis.com"
+    - "storage-component.googleapis.com"
+  network:
+    enableNetworkMonitoring: "true"
+    enablePrivateGoogleAccess: "true"
+    enableCloudRegistryPrivateGoogleAccess: "true"
+    enableArtifactRegistryPrivateGoogleAccess: "true"
+    blockBatchInternetAccess: "true"
+  kubernetesEngine:
+    createGkeDefaultServiceAccount: "true"
+  serviceUsage:
+    bigQuery:
+      overrideBigQueryDailyUsageQuota: true
+      bigQueryDailyUsageQuotaOverrideValueMebibytes: 38146972 # 40 TB
+  securityGroup: "high"

src/test/java/bio/terra/buffer/config/PoolSchemaTest.java

@@ -26,7 +26,7 @@ public class PoolSchemaTest {
   /** List of pool config folders for all environments, e.g. prod, staging, dev. */
   private static final List<String> POOL_CONFIG_FOLDERS =
       ImmutableList.of(
-          "alpha/", "buffertest/", "dev/", "prod/", "perf/", "staging/", "tools/", "toolsalpha/");
+          "alpha/", "buffertest/", "dev/", "prod/", "perf/", "staging/", "tools/");
 
   @Test
   public void testConfigValid() {

src/test/resources/application-integration.yml

@@ -7,4 +7,4 @@ buffer:
     janitor-track-resource-project-id: terra-kernel-k8s
     janitor-track-resource-topic-id: crljanitor-tools-pubsub-topic
   pool:
-    config-path: config/toolsalpha
+    config-path: config/buffertest