From 4e88684a321e8b29d9c37a7e4f42f9f782442a3c Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Fri, 22 Nov 2024 10:14:08 -0500 Subject: [PATCH 1/2] Create prod pools with Batch API enabled --- .../resources/config/prod/pool_schema.yml | 6 +++ .../prod/resource-config/cwb_ws_prod_v8.yml | 33 +++++++++++++++ .../prod/resource-config/vpc_sc_v12.yml | 42 +++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 src/main/resources/config/prod/resource-config/cwb_ws_prod_v8.yml create mode 100644 src/main/resources/config/prod/resource-config/vpc_sc_v12.yml diff --git a/src/main/resources/config/prod/pool_schema.yml b/src/main/resources/config/prod/pool_schema.yml index 741d14d0..23e54ae5 100644 --- a/src/main/resources/config/prod/pool_schema.yml +++ b/src/main/resources/config/prod/pool_schema.yml @@ -4,6 +4,9 @@ poolConfigs: - poolId: "cwb_ws_prod_v7" size: 3000 resourceConfigName: "cwb_ws_prod_v7" + - poolId: "cwb_ws_prod_v8" + size: 3000 + resourceConfigName: "cwb_ws_prod_v8" - poolId: "datarepo_v1" size: 1000 resourceConfigName: "datarepo_v1" @@ -13,3 +16,6 @@ poolConfigs: - poolId: "vpc_sc_v11" size: 1000 resourceConfigName: "vpc_sc_v11" + - poolId: "vpc_sc_v12" + size: 1000 + resourceConfigName: "vpc_sc_v12" diff --git a/src/main/resources/config/prod/resource-config/cwb_ws_prod_v8.yml b/src/main/resources/config/prod/resource-config/cwb_ws_prod_v8.yml new file mode 100644 index 00000000..2e15a174 --- /dev/null +++ b/src/main/resources/config/prod/resource-config/cwb_ws_prod_v8.yml @@ -0,0 +1,33 @@ +# Community Workbench buffered workspace template +--- +configName: "cwb_ws_prod_v8" +gcpProjectConfig: + projectIdSchema: + prefix: "terra" + scheme: "RANDOM_CHAR" + # firecloud.org/prod/CommunityWorkbench + parentFolderId: "710468670182" + billingAccount: "0106B0-41CAA9-427C96" + enabledApis: + - "bigquery-json.googleapis.com" + - "batch.googleapis.com" + - "compute.googleapis.com" + - "container.googleapis.com" + - "containerregistry.googleapis.com" + - "cloudbilling.googleapis.com" + - "clouderrorreporting.googleapis.com" + - "cloudkms.googleapis.com" + - "cloudtrace.googleapis.com" + - "dataflow.googleapis.com" + - "dataproc.googleapis.com" + - "dns.googleapis.com" + - "lifesciences.googleapis.com" + - "logging.googleapis.com" + - "monitoring.googleapis.com" + - "storage-api.googleapis.com" + - "storage-component.googleapis.com" + network: + enableNetworkMonitoring: "false" + enablePrivateGoogleAccess: "true" + kubernetesEngine: + createGkeDefaultServiceAccount: "true" diff --git a/src/main/resources/config/prod/resource-config/vpc_sc_v12.yml b/src/main/resources/config/prod/resource-config/vpc_sc_v12.yml new file mode 100644 index 00000000..4c536f76 --- /dev/null +++ b/src/main/resources/config/prod/resource-config/vpc_sc_v12.yml @@ -0,0 +1,42 @@ +# Projects with VPC-SC configuration +--- +configName: "vpc_sc_v12" +gcpProjectConfig: + projectIdSchema: + prefix: "terra-vpc-sc" + scheme: "RANDOM_CHAR" + # firecloud.org/prod/for_vpc_sc_unclaimed + parentFolderId: "160283235721" + billingAccount: "0106B0-41CAA9-427C96" + enabledApis: + - "bigquery-json.googleapis.com" + - "batch.googleapis.com" + - "compute.googleapis.com" + - "container.googleapis.com" + - "cloudbilling.googleapis.com" + - "clouderrorreporting.googleapis.com" + - "cloudkms.googleapis.com" + - "cloudtrace.googleapis.com" + - "containerregistry.googleapis.com" + - "dataflow.googleapis.com" + - "dataproc.googleapis.com" + - "dns.googleapis.com" + - "lifesciences.googleapis.com" + - "logging.googleapis.com" + - "monitoring.googleapis.com" + - "serviceusage.googleapis.com" + - "storage-api.googleapis.com" + - "storage-component.googleapis.com" + network: + enableNetworkMonitoring: "true" + enablePrivateGoogleAccess: "true" + enableCloudRegistryPrivateGoogleAccess: "true" + enableArtifactRegistryPrivateGoogleAccess: "true" + blockBatchInternetAccess: "true" + kubernetesEngine: + createGkeDefaultServiceAccount: "true" + serviceUsage: + bigQuery: + overrideBigQueryDailyUsageQuota: true + bigQueryDailyUsageQuotaOverrideValueMebibytes: 38146972 # 40 TB + securityGroup: "high" From 4eeeaafd023c760de1961507e050774e6adac4dc Mon Sep 17 00:00:00 2001 From: Janet Gainer-Dewar Date: Tue, 3 Dec 2024 10:13:20 -0500 Subject: [PATCH 2/2] Fix tests by switching from toolsalpha to buffertest --- local-dev/render-config.sh | 2 +- local-dev/run_local.sh | 2 +- src/test/java/bio/terra/buffer/config/PoolSchemaTest.java | 2 +- src/test/resources/application-integration.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/local-dev/render-config.sh b/local-dev/render-config.sh index e2cb0a73..59616741 100755 --- a/local-dev/render-config.sh +++ b/local-dev/render-config.sh @@ -6,7 +6,7 @@ set -e VAULT_TOKEN=${1:-$(cat $HOME/.vault-token)} DSDE_TOOLBOX_DOCKER_IMAGE=broadinstitute/dsde-toolbox:dev -VAULT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/toolsalpha/buffer/app-sa +VAULT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/buffertest/buffer/app-sa VAULT_JANITOR_CLIENT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/tools/crl_janitor/client-sa SERVICE_ACCOUNT_OUTPUT_FILE_PATH="$(dirname $0)"/../src/test/resources/rendered/sa-account.json JANITOR_CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH="$(dirname $0)"/../src/test/resources/rendered/janitor-client-sa-account.json diff --git a/local-dev/run_local.sh b/local-dev/run_local.sh index 061f20f4..a0e5497b 100755 --- a/local-dev/run_local.sh +++ b/local-dev/run_local.sh @@ -13,7 +13,7 @@ export BUFFER_CRL_JANITOR_CLIENT_CREDENTIAL_FILE_PATH="$(dirname $0)"/../src/tes export BUFFER_CRL_JANITOR_TRACK_RESOURCE_PROJECT_ID=terra-kernel-k8s export BUFFER_CRL_JANITOR_TRACK_RESOURCE_TOPIC_ID=crljanitor-tools-pubsub-topic export BUFFER_CRL_TEST_RESOURCE_TIME_TO_LIVE=1h -export BUFFER_POOL_CONFIG_PATH=config/toolsalpha +export BUFFER_POOL_CONFIG_PATH=config/buffertest export SPRING_PROFILES_INCLUDE=human-readable-logging export TERRA_COMMON_STAIRWAY_FORCE_CLEAN_START=true export TERRA_COMMON_TRACING_STACKDRIVER_EXPORT_ENABLED=false diff --git a/src/test/java/bio/terra/buffer/config/PoolSchemaTest.java b/src/test/java/bio/terra/buffer/config/PoolSchemaTest.java index 057c1e1e..844b1c14 100644 --- a/src/test/java/bio/terra/buffer/config/PoolSchemaTest.java +++ b/src/test/java/bio/terra/buffer/config/PoolSchemaTest.java @@ -26,7 +26,7 @@ public class PoolSchemaTest { /** List of pool config folders for all environments, e.g. prod, staging, dev. */ private static final List POOL_CONFIG_FOLDERS = ImmutableList.of( - "alpha/", "buffertest/", "dev/", "prod/", "perf/", "staging/", "tools/", "toolsalpha/"); + "alpha/", "buffertest/", "dev/", "prod/", "perf/", "staging/", "tools/"); @Test public void testConfigValid() { diff --git a/src/test/resources/application-integration.yml b/src/test/resources/application-integration.yml index 6eb412f2..8a4f559d 100644 --- a/src/test/resources/application-integration.yml +++ b/src/test/resources/application-integration.yml @@ -7,4 +7,4 @@ buffer: janitor-track-resource-project-id: terra-kernel-k8s janitor-track-resource-topic-id: crljanitor-tools-pubsub-topic pool: - config-path: config/toolsalpha + config-path: config/buffertest