From 18ae2b68cae73cd567426b5f9cd3d92da124c97f Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Tue, 18 Feb 2025 18:35:22 -0500 Subject: [PATCH 1/2] reduce pool sizes --- src/main/resources/config/buffertest/pool_schema.yml | 2 +- src/main/resources/config/dev/pool_schema.yml | 8 ++++---- src/main/resources/config/prod/pool_schema.yml | 2 +- src/main/resources/config/staging/pool_schema.yml | 2 +- src/main/resources/config/tools/pool_schema.yml | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/resources/config/buffertest/pool_schema.yml b/src/main/resources/config/buffertest/pool_schema.yml index 9eb2891c..2a11b1e8 100644 --- a/src/main/resources/config/buffertest/pool_schema.yml +++ b/src/main/resources/config/buffertest/pool_schema.yml @@ -2,5 +2,5 @@ --- poolConfigs: - poolId: "resource_buffer_test_v3" - size: 1500 + size: 0 resourceConfigName: "resource_buffer_test_v3" diff --git a/src/main/resources/config/dev/pool_schema.yml b/src/main/resources/config/dev/pool_schema.yml index fa07bcd6..63ea054e 100644 --- a/src/main/resources/config/dev/pool_schema.yml +++ b/src/main/resources/config/dev/pool_schema.yml @@ -2,17 +2,17 @@ --- poolConfigs: - poolId: "cwb_ws_dev_v8" - size: 300 + size: 100 resourceConfigName: "cwb_ws_resource_dev_v8" - poolId: "datarepo_fakeprod_v1" size: 50 resourceConfigName: "datarepo_fakeprod_v1" - poolId: "datarepo_v3" - size: 300 + size: 100 resourceConfigName: "datarepo_v3" - poolId: "vpc_sc_v13" - size: 300 + size: 100 resourceConfigName: "vpc_sc_v13" - poolId: "workspace_manager_v12" - size: 500 + size: 50 resourceConfigName: "workspace_manager_v12" diff --git a/src/main/resources/config/prod/pool_schema.yml b/src/main/resources/config/prod/pool_schema.yml index 90357061..fd53046a 100644 --- a/src/main/resources/config/prod/pool_schema.yml +++ b/src/main/resources/config/prod/pool_schema.yml @@ -2,7 +2,7 @@ --- poolConfigs: - poolId: "cwb_ws_prod_v8" - size: 3000 + size: 1000 resourceConfigName: "cwb_ws_prod_v8" - poolId: "datarepo_v1" size: 1000 diff --git a/src/main/resources/config/staging/pool_schema.yml b/src/main/resources/config/staging/pool_schema.yml index 62d13308..29c7b47f 100644 --- a/src/main/resources/config/staging/pool_schema.yml +++ b/src/main/resources/config/staging/pool_schema.yml @@ -5,7 +5,7 @@ poolConfigs: size: 100 resourceConfigName: "cwb_ws_resource_staging_v8" - poolId: "datarepo_v1" - size: 300 + size: 100 resourceConfigName: "datarepo_v1" - poolId: "vpc_sc_v7" size: 100 diff --git a/src/main/resources/config/tools/pool_schema.yml b/src/main/resources/config/tools/pool_schema.yml index 8e00e21e..b736fca4 100644 --- a/src/main/resources/config/tools/pool_schema.yml +++ b/src/main/resources/config/tools/pool_schema.yml @@ -3,7 +3,7 @@ --- poolConfigs: - poolId: "cwb_ws_quality_v11" - size: 1000 + size: 200 resourceConfigName: "cwb_ws_resource_quality_v11" - poolId: "datarepo_v1" size: 1000 @@ -12,5 +12,5 @@ poolConfigs: size: 100 resourceConfigName: "vpc_sc_qa-fiab_v6" - poolId: "workspace_manager_v12" - size: 500 + size: 100 resourceConfigName: "workspace_manager_v12" From a32ff58a0c0b6c97aea43bda6097a84d873a3661 Mon Sep 17 00:00:00 2001 From: Marcus Talbott Date: Wed, 19 Feb 2025 10:59:23 -0500 Subject: [PATCH 2/2] remove alpha and perf references --- .../src/main/resources/servers/perf.json | 14 ------- .../perf-buffer-client-sa.json | 5 --- buffer-clienttests/tools/render-config.sh | 8 +--- .../resources/config/alpha/pool_schema.yml | 12 ------ .../cwb_ws_resource_alpha_v7.yml | 32 --------------- .../alpha/resource-config/datarepo_v1.yml | 20 ---------- .../alpha/resource-config/vpc_sc_v6.yml | 34 ---------------- .../resources/config/perf/pool_schema.yml | 15 ------- .../perf/resource-config/cwb_ws_perf_v7.yml | 32 --------------- .../perf/resource-config/datarepo_v1.yml | 20 ---------- .../config/perf/resource-config/vpc_sc_v9.yml | 40 ------------------- .../resource-config/workspace_manager_v6.yml | 29 -------------- .../terra/buffer/config/PoolSchemaTest.java | 2 +- 13 files changed, 2 insertions(+), 261 deletions(-) delete mode 100644 buffer-clienttests/src/main/resources/servers/perf.json delete mode 100644 buffer-clienttests/src/main/resources/serviceaccounts/perf-buffer-client-sa.json delete mode 100644 src/main/resources/config/alpha/pool_schema.yml delete mode 100644 src/main/resources/config/alpha/resource-config/cwb_ws_resource_alpha_v7.yml delete mode 100644 src/main/resources/config/alpha/resource-config/datarepo_v1.yml delete mode 100644 src/main/resources/config/alpha/resource-config/vpc_sc_v6.yml delete mode 100644 src/main/resources/config/perf/pool_schema.yml delete mode 100644 src/main/resources/config/perf/resource-config/cwb_ws_perf_v7.yml delete mode 100644 src/main/resources/config/perf/resource-config/datarepo_v1.yml delete mode 100644 src/main/resources/config/perf/resource-config/vpc_sc_v9.yml delete mode 100644 src/main/resources/config/perf/resource-config/workspace_manager_v6.yml diff --git a/buffer-clienttests/src/main/resources/servers/perf.json b/buffer-clienttests/src/main/resources/servers/perf.json deleted file mode 100644 index b799017e..00000000 --- a/buffer-clienttests/src/main/resources/servers/perf.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "perf", - "description": "Perf environment in terra-perf cluster.", - - "bufferUri": "https://buffer.dsde-perf.broadinstitute.org", - "bufferClientServiceAccountFile": "perf-buffer-client-sa.json", - - "cluster": {}, - "deploymentScript": {}, - "testRunnerServiceAccountFile": "perf-buffer-client-sa.json", - - "skipDeployment": true, - "skipKubernetes": true -} diff --git a/buffer-clienttests/src/main/resources/serviceaccounts/perf-buffer-client-sa.json b/buffer-clienttests/src/main/resources/serviceaccounts/perf-buffer-client-sa.json deleted file mode 100644 index 7b864ae0..00000000 --- a/buffer-clienttests/src/main/resources/serviceaccounts/perf-buffer-client-sa.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "buffer-client-perf@broad-dsde-perf.iam.gserviceaccount.com", - "jsonKeyFilename": "perf-buffer-client-sa-key.json", - "jsonKeyDirectoryPath": "./rendered" -} diff --git a/buffer-clienttests/tools/render-config.sh b/buffer-clienttests/tools/render-config.sh index f052fabc..78b2b038 100755 --- a/buffer-clienttests/tools/render-config.sh +++ b/buffer-clienttests/tools/render-config.sh @@ -14,13 +14,7 @@ VAULT_TOKEN=${2:-$(cat $HOME/.vault-token)} DSDE_TOOLBOX_DOCKER_IMAGE=broadinstitute/dsde-toolbox:consul-0.20.0 VAULT_CLIENT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/${ENV}/buffer/client-sa CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH="$(dirname $0)"/../rendered/${ENV}-buffer-client-sa-key.json -# We uses different Vault for personal envs and perf/dev environments. We need a better solution once we start using -# preview environment for test. -if [ "$ENV" = "perf" ]; then - VAULT_CLIENT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/${ENV}/${ENV}/buffer/client-sa -else - VAULT_CLIENT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/${ENV}/buffer/client-sa -fi +VAULT_CLIENT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/${ENV}/buffer/client-sa mkdir -p ../rendered docker run --rm -e VAULT_TOKEN=$VAULT_TOKEN ${DSDE_TOOLBOX_DOCKER_IMAGE} \ diff --git a/src/main/resources/config/alpha/pool_schema.yml b/src/main/resources/config/alpha/pool_schema.yml deleted file mode 100644 index 00578f0f..00000000 --- a/src/main/resources/config/alpha/pool_schema.yml +++ /dev/null @@ -1,12 +0,0 @@ -# Resource Buffer Service Pools Schema in alpha environment ---- -poolConfigs: - - poolId: "cwb_ws_alpha_v7" - size: 100 - resourceConfigName: "cwb_ws_resource_alpha_v7" - - poolId: "datarepo_v1" - size: 300 - resourceConfigName: "datarepo_v1" - - poolId: "vpc_sc_v6" - size: 100 - resourceConfigName: "vpc_sc_v6" \ No newline at end of file diff --git a/src/main/resources/config/alpha/resource-config/cwb_ws_resource_alpha_v7.yml b/src/main/resources/config/alpha/resource-config/cwb_ws_resource_alpha_v7.yml deleted file mode 100644 index 51bbcbf9..00000000 --- a/src/main/resources/config/alpha/resource-config/cwb_ws_resource_alpha_v7.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Community Workbench buffered workspace template ---- -configName: "cwb_ws_resource_alpha_v7" -gcpProjectConfig: - projectIdSchema: - prefix: "terra-alpha" - scheme: "RANDOM_CHAR" - # test.firecloud.org/alpha/CommunityWorkbench - parentFolderId: "102618195587" - billingAccount: "01A82E-CA8A14-367457" - enabledApis: - - "bigquery-json.googleapis.com" - - "compute.googleapis.com" - - "container.googleapis.com" - - "containerregistry.googleapis.com" - - "cloudbilling.googleapis.com" - - "clouderrorreporting.googleapis.com" - - "cloudkms.googleapis.com" - - "cloudtrace.googleapis.com" - - "dataflow.googleapis.com" - - "dataproc.googleapis.com" - - "dns.googleapis.com" - - "lifesciences.googleapis.com" - - "logging.googleapis.com" - - "monitoring.googleapis.com" - - "storage-api.googleapis.com" - - "storage-component.googleapis.com" - network: - enableNetworkMonitoring: "false" - enablePrivateGoogleAccess: "true" - kubernetesEngine: - createGkeDefaultServiceAccount: "true" diff --git a/src/main/resources/config/alpha/resource-config/datarepo_v1.yml b/src/main/resources/config/alpha/resource-config/datarepo_v1.yml deleted file mode 100644 index b526b073..00000000 --- a/src/main/resources/config/alpha/resource-config/datarepo_v1.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Data Repo buffered workspace template ---- -configName: "datarepo_v1" -gcpProjectConfig: - projectIdSchema: - prefix: "datarepo-alpha" - scheme: "RANDOM_CHAR" - parentFolderId: "270278425081" #data.test-terra.bio/repos/jade-dev - billingAccount: "01A82E-CA8A14-367457" - enabledApis: - - "bigquery-json.googleapis.com" - - "firestore.googleapis.com" - - "firebaserules.googleapis.com" - - "storage-component.googleapis.com" - - "storage-api.googleapis.com" - - "cloudbilling.googleapis.com" - - "appengine.googleapis.com" - - "compute.googleapis.com" - network: - enableNetworkMonitoring: "false" diff --git a/src/main/resources/config/alpha/resource-config/vpc_sc_v6.yml b/src/main/resources/config/alpha/resource-config/vpc_sc_v6.yml deleted file mode 100644 index fa260cbb..00000000 --- a/src/main/resources/config/alpha/resource-config/vpc_sc_v6.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Community Workbench buffered workspace template ---- -configName: "vpc_sc_v6" -gcpProjectConfig: - projectIdSchema: - prefix: "terra-vpc-sc-alpha" - scheme: "RANDOM_CHAR" - # test.firecloud.org/alpha/for_vpc_sc_unclaimed - parentFolderId: "38455243798" - billingAccount: "01A82E-CA8A14-367457" - enabledApis: - - "bigquery-json.googleapis.com" - - "compute.googleapis.com" - - "container.googleapis.com" - - "containerregistry.googleapis.com" - - "cloudbilling.googleapis.com" - - "clouderrorreporting.googleapis.com" - - "cloudkms.googleapis.com" - - "cloudtrace.googleapis.com" - - "dataflow.googleapis.com" - - "dataproc.googleapis.com" - - "dns.googleapis.com" - - "lifesciences.googleapis.com" - - "logging.googleapis.com" - - "monitoring.googleapis.com" - - "storage-api.googleapis.com" - - "storage-component.googleapis.com" - network: - enableNetworkMonitoring: "true" - enablePrivateGoogleAccess: "true" - enableCloudRegistryPrivateGoogleAccess: "true" - blockBatchInternetAccess: "true" - kubernetesEngine: - createGkeDefaultServiceAccount: "true" diff --git a/src/main/resources/config/perf/pool_schema.yml b/src/main/resources/config/perf/pool_schema.yml deleted file mode 100644 index 9cb7d875..00000000 --- a/src/main/resources/config/perf/pool_schema.yml +++ /dev/null @@ -1,15 +0,0 @@ -# RBS Pools Schema for running buffer service performance test. ---- -poolConfigs: - - poolId: "cwb_ws_perf_v7" - size: 500 - resourceConfigName: "cwb_ws_perf_v7" - - poolId: "datarepo_v1" - size: 300 - resourceConfigName: "datarepo_v1" - - poolId: "vpc_sc_v9" - size: 500 - resourceConfigName: "vpc_sc_v9" - - poolId: "workspace_manager_v6" - size: 20 - resourceConfigName: "workspace_manager_v6" \ No newline at end of file diff --git a/src/main/resources/config/perf/resource-config/cwb_ws_perf_v7.yml b/src/main/resources/config/perf/resource-config/cwb_ws_perf_v7.yml deleted file mode 100644 index 45a202c0..00000000 --- a/src/main/resources/config/perf/resource-config/cwb_ws_perf_v7.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Community Workbench buffered workspace template ---- -configName: "cwb_ws_perf_v7" -gcpProjectConfig: - projectIdSchema: - prefix: "terra-perf" - scheme: "RANDOM_CHAR" - # test.firecloud.org/perf/CommunityWorkbench - parentFolderId: "246142551922" - billingAccount: "01A82E-CA8A14-367457" - enabledApis: - - "bigquery-json.googleapis.com" - - "compute.googleapis.com" - - "container.googleapis.com" - - "containerregistry.googleapis.com" - - "cloudbilling.googleapis.com" - - "clouderrorreporting.googleapis.com" - - "cloudkms.googleapis.com" - - "cloudtrace.googleapis.com" - - "dataflow.googleapis.com" - - "dataproc.googleapis.com" - - "dns.googleapis.com" - - "lifesciences.googleapis.com" - - "logging.googleapis.com" - - "monitoring.googleapis.com" - - "storage-api.googleapis.com" - - "storage-component.googleapis.com" - network: - enableNetworkMonitoring: "false" - enablePrivateGoogleAccess: "true" - kubernetesEngine: - createGkeDefaultServiceAccount: "true" diff --git a/src/main/resources/config/perf/resource-config/datarepo_v1.yml b/src/main/resources/config/perf/resource-config/datarepo_v1.yml deleted file mode 100644 index 89e0627a..00000000 --- a/src/main/resources/config/perf/resource-config/datarepo_v1.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Data Repo buffered workspace template ---- -configName: "datarepo_v1" -gcpProjectConfig: - projectIdSchema: - prefix: "datarepo-perf" - scheme: "RANDOM_CHAR" - parentFolderId: "270278425081" #data.test-terra.bio/repos/jade-dev - billingAccount: "01A82E-CA8A14-367457" - enabledApis: - - "bigquery-json.googleapis.com" - - "firestore.googleapis.com" - - "firebaserules.googleapis.com" - - "storage-component.googleapis.com" - - "storage-api.googleapis.com" - - "cloudbilling.googleapis.com" - - "appengine.googleapis.com" - - "compute.googleapis.com" - network: - enableNetworkMonitoring: "false" diff --git a/src/main/resources/config/perf/resource-config/vpc_sc_v9.yml b/src/main/resources/config/perf/resource-config/vpc_sc_v9.yml deleted file mode 100644 index dbcb1fbb..00000000 --- a/src/main/resources/config/perf/resource-config/vpc_sc_v9.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Projects with VPC-SC configuration ---- -configName: "vpc_sc_v9" -gcpProjectConfig: - projectIdSchema: - prefix: "terra-vpc-sc-perf" - scheme: "RANDOM_CHAR" - # test.firecloud.org/perf/for_vpc_sc_unclaimed - parentFolderId: "533417334224" - billingAccount: "01A82E-CA8A14-367457" - enabledApis: - - "bigquery-json.googleapis.com" - - "compute.googleapis.com" - - "container.googleapis.com" - - "cloudbilling.googleapis.com" - - "clouderrorreporting.googleapis.com" - - "cloudkms.googleapis.com" - - "cloudtrace.googleapis.com" - - "containerregistry.googleapis.com" - - "dataflow.googleapis.com" - - "dataproc.googleapis.com" - - "dns.googleapis.com" - - "lifesciences.googleapis.com" - - "logging.googleapis.com" - - "monitoring.googleapis.com" - - "serviceusage.googleapis.com" - - "storage-api.googleapis.com" - - "storage-component.googleapis.com" - network: - enableNetworkMonitoring: "true" - enablePrivateGoogleAccess: "true" - enableCloudRegistryPrivateGoogleAccess: "true" - blockBatchInternetAccess: "true" - kubernetesEngine: - createGkeDefaultServiceAccount: "true" - serviceUsage: - bigQuery: - overrideBigQueryDailyUsageQuota: true - bigQueryDailyUsageQuotaOverrideValueMebibytes: 38146972 # 40 TB - securityGroup: "high" diff --git a/src/main/resources/config/perf/resource-config/workspace_manager_v6.yml b/src/main/resources/config/perf/resource-config/workspace_manager_v6.yml deleted file mode 100644 index b38378d3..00000000 --- a/src/main/resources/config/perf/resource-config/workspace_manager_v6.yml +++ /dev/null @@ -1,29 +0,0 @@ -# Workspace Manager buffered workspace template ---- -configName: "workspace_manager_v6" -gcpProjectConfig: - projectIdSchema: - prefix: "terra-wsm-dev" - scheme: "RANDOM_CHAR" - parentFolderId: "199572063713" #test.firecloud.org/perf/buffer-perf/workspace_manager - billingAccount: "01A82E-CA8A14-367457" - enabledApis: - - "bigquery-json.googleapis.com" - - "compute.googleapis.com" - - "container.googleapis.com" - - "cloudbilling.googleapis.com" - - "clouderrorreporting.googleapis.com" - - "cloudkms.googleapis.com" - - "cloudtrace.googleapis.com" - - "containerregistry.googleapis.com" - - "dataflow.googleapis.com" - - "dataproc.googleapis.com" - - "dns.googleapis.com" - - "lifesciences.googleapis.com" - - "logging.googleapis.com" - - "monitoring.googleapis.com" - - "notebooks.googleapis.com" - - "storage-api.googleapis.com" - - "storage-component.googleapis.com" - network: - enableNetworkMonitoring: "true" diff --git a/src/test/java/bio/terra/buffer/config/PoolSchemaTest.java b/src/test/java/bio/terra/buffer/config/PoolSchemaTest.java index 844b1c14..e69a5714 100644 --- a/src/test/java/bio/terra/buffer/config/PoolSchemaTest.java +++ b/src/test/java/bio/terra/buffer/config/PoolSchemaTest.java @@ -26,7 +26,7 @@ public class PoolSchemaTest { /** List of pool config folders for all environments, e.g. prod, staging, dev. */ private static final List POOL_CONFIG_FOLDERS = ImmutableList.of( - "alpha/", "buffertest/", "dev/", "prod/", "perf/", "staging/", "tools/"); + "buffertest/", "dev/", "prod/", "staging/", "tools/"); @Test public void testConfigValid() {