From 364455b2650ecaaeece8477e725f20fa902c15b6 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 31 Dec 2024 15:00:01 +0300 Subject: [PATCH 01/13] feat: Use S3 node store with garage --- docker-compose.yml | 12 ++++++++++++ garage.toml | 29 +++++++++++++++++++++++++++++ install.sh | 1 + install/bootstrap-garage.sh | 23 +++++++++++++++++++++++ sentry/Dockerfile | 6 ++++-- sentry/sentry.conf.example.py | 19 +++++++++++++++++++ 6 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 garage.toml create mode 100644 install/bootstrap-garage.sh diff --git a/docker-compose.yml b/docker-compose.yml index 9d8ee5a9e4..d27a805f75 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,6 +34,8 @@ x-sentry-defaults: &sentry_defaults <<: *depends_on-default smtp: <<: *depends_on-default + garage: + <<: *depends_on-default snuba-api: <<: *depends_on-default symbolicator: @@ -139,6 +141,7 @@ services: kafka: <<: *restart_policy image: "confluentinc/cp-kafka:7.6.1" + user: root environment: # https://docs.confluent.io/platform/current/installation/docker/config-reference.html#cp-kakfa-example KAFKA_PROCESS_ROLES: "broker,controller" @@ -207,6 +210,14 @@ services: interval: 10s timeout: 10s retries: 30 + garage: + image: dxflrs/garage:v1.0.1 + volumes: + - type: bind + read_only: true + source: ./garage.toml + target: /etc/garage.toml + - "sentry-garage:/var/lib/garage/" snuba-api: <<: *snuba_defaults # Kafka consumer responsible for feeding events into Clickhouse @@ -540,3 +551,4 @@ volumes: sentry-kafka-log: sentry-smtp-log: sentry-clickhouse-log: + sentry-garage: diff --git a/garage.toml b/garage.toml new file mode 100644 index 0000000000..94f1554736 --- /dev/null +++ b/garage.toml @@ -0,0 +1,29 @@ +metadata_dir = "/var/lib/garage/meta" +data_dir = "/var/lib/garage/data" +db_engine = "sqlite" + +replication_factor = 1 +# idk why rcp_ is needed without replication but ok +rpc_bind_addr = "[::]:3901" +rpc_public_addr = "127.0.0.1:3901" +# This secret will not ever be used as we are in a single node setup +# so this static value is OK. Don't forget to change it if you ever +# decide to use garage in a multi-node setup. +rpc_secret = "28f2fc651d4f280cea95e166f9113fe589039f9711d6c4f461872c1c5aaf7dec" + +compression_level = 2 + +[s3_api] +s3_region = "garage" +api_bind_addr = "[::]:3900" +root_domain = ".s3.garage.localhost" + +[s3_web] +bind_addr = "[::]:3902" +root_domain = ".web.garage.localhost" +index = "index.html" + +[admin] +api_bind_addr = "0.0.0.0:3903" +metrics_token = "c6k30kKqovA5NoTfKxMVRmnKKVwd7jaF4OB2xshQVEA=" +admin_token = "c6k30kKqovA5NoTfKxMVRmnKKVwd7jaF4OB2xshQVEA=" diff --git a/install.sh b/install.sh index e9460d36ce..246842f076 100755 --- a/install.sh +++ b/install.sh @@ -32,6 +32,7 @@ source install/ensure-relay-credentials.sh source install/generate-secret-key.sh source install/update-docker-images.sh source install/build-docker-images.sh +source install/bootstrap-garage.sh source install/bootstrap-snuba.sh source install/upgrade-postgres.sh source install/set-up-and-migrate-database.sh diff --git a/install/bootstrap-garage.sh b/install/bootstrap-garage.sh new file mode 100644 index 0000000000..192be20a04 --- /dev/null +++ b/install/bootstrap-garage.sh @@ -0,0 +1,23 @@ +echo "${_group}Bootstrapping garage (node store)..." + +$dc up --wait garage +if [[ $($dc exec garage /garage bucket list | tail -1 | awk '{print $1}' != 'nodestore' )]]; then + node_id = $(exec garage /garage status | tail -1 | awk '{print $1}') + $dc exec garage /garage layout assign -z dc1 -c 100G "$node_id" + $dc exec garage /garage layout apply --version 1 + + $dc exec garage /garage bucket create nodestore + key_info=$($dc exec garage /garage key create nodestore-key | head -3 | tail -2) + key_id=$(echo "$key_info" | head -1 | awk '{print $1}') + key_secret=$(echo "$key_info" | tail -1 | awk '{print $1}') + + $dc exec garage /garage bucket allow --read --write --owner nodestore --key nodestore-key + + sed -i -e "s//$key_info/" $SENTRY_CONFIG_PY + sed -i -e "s//$key_secret/" $SENTRY_CONFIG_PY + echo "Set Garage keys for SENTRY_NODESTORE_OPTIONS in $SENTRY_CONFIG_PY" +else + echo "Node store already exists, skipping..." +fi + +echo "${_endgroup}" diff --git a/sentry/Dockerfile b/sentry/Dockerfile index 557046f143..40398a773f 100644 --- a/sentry/Dockerfile +++ b/sentry/Dockerfile @@ -1,13 +1,15 @@ ARG SENTRY_IMAGE FROM ${SENTRY_IMAGE} +RUN pip install https://github.com/stayallive/sentry-nodestore-s3/archive/main.zip + COPY . /usr/src/sentry RUN if [ -s /usr/src/sentry/enhance-image.sh ]; then \ /usr/src/sentry/enhance-image.sh; \ -fi + fi RUN if [ -s /usr/src/sentry/requirements.txt ]; then \ echo "sentry/requirements.txt is deprecated, use sentry/enhance-image.sh - see https://develop.sentry.dev/self-hosted/#enhance-sentry-image"; \ pip install -r /usr/src/sentry/requirements.txt; \ -fi + fi diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 98e7a6c0fb..7677573945 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -84,6 +84,25 @@ def get_internal_network(): # See https://develop.sentry.dev/self-hosted/experimental/errors-only/ SENTRY_SELF_HOSTED_ERRORS_ONLY = env("COMPOSE_PROFILES") != "feature-complete" +############## +# Node Store # +############## + +SENTRY_NODESTORE = "sentry_nodestore_s3.S3PassthroughDjangoNodeStorage" +SENTRY_NODESTORE_OPTIONS = { + "delete_through": True, + "write_through": False, + "read_through": True, + "compression": False, # we have compression enabled in Garage itself + "endpoint_url": "http://garage:3900", + "bucket_path": "nodestore", + "bucket_name": "nodestore", + "retry_attempts": 3, + "region_name": "garage", + "aws_access_key_id": "", + "aws_secret_access_key": "", +} + ######### # Redis # ######### From 803bef37334fc3a6bb544a90cb4dafd3c1b63b1f Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 31 Dec 2024 16:22:20 +0300 Subject: [PATCH 02/13] lol, fix bash --- install/bootstrap-garage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/bootstrap-garage.sh b/install/bootstrap-garage.sh index 192be20a04..2c868f94c0 100644 --- a/install/bootstrap-garage.sh +++ b/install/bootstrap-garage.sh @@ -1,7 +1,7 @@ echo "${_group}Bootstrapping garage (node store)..." $dc up --wait garage -if [[ $($dc exec garage /garage bucket list | tail -1 | awk '{print $1}' != 'nodestore' )]]; then +if [[ $($dc exec garage /garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; then node_id = $(exec garage /garage status | tail -1 | awk '{print $1}') $dc exec garage /garage layout assign -z dc1 -c 100G "$node_id" $dc exec garage /garage layout apply --version 1 From 58b301c3e2065af275166abc08c9b9e6ce3b035f Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 31 Dec 2024 16:26:57 +0300 Subject: [PATCH 03/13] moar bash --- install/bootstrap-garage.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/install/bootstrap-garage.sh b/install/bootstrap-garage.sh index 2c868f94c0..c736873117 100644 --- a/install/bootstrap-garage.sh +++ b/install/bootstrap-garage.sh @@ -1,17 +1,19 @@ echo "${_group}Bootstrapping garage (node store)..." $dc up --wait garage -if [[ $($dc exec garage /garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; then - node_id = $(exec garage /garage status | tail -1 | awk '{print $1}') - $dc exec garage /garage layout assign -z dc1 -c 100G "$node_id" - $dc exec garage /garage layout apply --version 1 +garage="$garage" - $dc exec garage /garage bucket create nodestore - key_info=$($dc exec garage /garage key create nodestore-key | head -3 | tail -2) +if [[ $($garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; then + node_id = $($garage status | tail -1 | awk '{print $1}') + $garage layout assign -z dc1 -c 100G "$node_id" + $garage layout apply --version 1 + + $garage bucket create nodestore + key_info=$($garage key create nodestore-key | head -3 | tail -2) key_id=$(echo "$key_info" | head -1 | awk '{print $1}') key_secret=$(echo "$key_info" | tail -1 | awk '{print $1}') - $dc exec garage /garage bucket allow --read --write --owner nodestore --key nodestore-key + $garage bucket allow --read --write --owner nodestore --key nodestore-key sed -i -e "s//$key_info/" $SENTRY_CONFIG_PY sed -i -e "s//$key_secret/" $SENTRY_CONFIG_PY From bd414d5b9da6812d5526922c707da61f7083ed9e Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 31 Dec 2024 16:30:20 +0300 Subject: [PATCH 04/13] lol --- install/bootstrap-garage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/bootstrap-garage.sh b/install/bootstrap-garage.sh index c736873117..5efc3dec36 100644 --- a/install/bootstrap-garage.sh +++ b/install/bootstrap-garage.sh @@ -1,7 +1,7 @@ echo "${_group}Bootstrapping garage (node store)..." $dc up --wait garage -garage="$garage" +garage="$dc exec garage /garage" if [[ $($garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; then node_id = $($garage status | tail -1 | awk '{print $1}') From a2ac10d3abb7f9076639b567be90b0066aa25c10 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 31 Dec 2024 16:34:28 +0300 Subject: [PATCH 05/13] hate bash --- install/bootstrap-garage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/bootstrap-garage.sh b/install/bootstrap-garage.sh index 5efc3dec36..a628b90139 100644 --- a/install/bootstrap-garage.sh +++ b/install/bootstrap-garage.sh @@ -4,7 +4,7 @@ $dc up --wait garage garage="$dc exec garage /garage" if [[ $($garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; then - node_id = $($garage status | tail -1 | awk '{print $1}') + node_id=$($garage status | tail -1 | awk '{print $1}') $garage layout assign -z dc1 -c 100G "$node_id" $garage layout apply --version 1 From 7b28dfb97a53fbbacdab7d45153b8f352af1a2f0 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 31 Dec 2024 16:42:03 +0300 Subject: [PATCH 06/13] fix moar bash --- install/bootstrap-garage.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 install/bootstrap-garage.sh diff --git a/install/bootstrap-garage.sh b/install/bootstrap-garage.sh old mode 100644 new mode 100755 index a628b90139..0f0543be65 --- a/install/bootstrap-garage.sh +++ b/install/bootstrap-garage.sh @@ -10,12 +10,12 @@ if [[ $($garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; the $garage bucket create nodestore key_info=$($garage key create nodestore-key | head -3 | tail -2) - key_id=$(echo "$key_info" | head -1 | awk '{print $1}') - key_secret=$(echo "$key_info" | tail -1 | awk '{print $1}') + key_id=$(echo "$key_info" | head -1 | awk '{print $3}') + key_secret=$(echo "$key_info" | tail -1 | awk '{print $3}') $garage bucket allow --read --write --owner nodestore --key nodestore-key - sed -i -e "s//$key_info/" $SENTRY_CONFIG_PY + sed -i -e "s//$key_id/" $SENTRY_CONFIG_PY sed -i -e "s//$key_secret/" $SENTRY_CONFIG_PY echo "Set Garage keys for SENTRY_NODESTORE_OPTIONS in $SENTRY_CONFIG_PY" else From 4a6d337b30a024c9c6e25b161248de86b97cdc7e Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 31 Dec 2024 15:23:02 +0000 Subject: [PATCH 07/13] Add healthcheck to garage service Co-authored-by: Reinaldy Rafli --- docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index d27a805f75..4d076cd306 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -218,6 +218,11 @@ services: source: ./garage.toml target: /etc/garage.toml - "sentry-garage:/var/lib/garage/" + healthcheck: + test: "wget -no-verbose --tries=1 --spider http://localhost:3903/health || exit 1" + interval: 20s + timeout: 20s + retries: 10 snuba-api: <<: *snuba_defaults # Kafka consumer responsible for feeding events into Clickhouse From 595ed68f82443d470ac468fe3539bf53bf4b96d2 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 31 Dec 2024 16:46:43 +0300 Subject: [PATCH 08/13] revert +x --- install/bootstrap-garage.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 install/bootstrap-garage.sh diff --git a/install/bootstrap-garage.sh b/install/bootstrap-garage.sh old mode 100755 new mode 100644 From 99576bfd7131f71b5dcb58d3bd097e931cb3aca4 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 31 Dec 2024 23:43:36 +0300 Subject: [PATCH 09/13] fix healthcheck, fix config --- docker-compose.yml | 8 ++++---- garage.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4d076cd306..96f6476b8b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -219,10 +219,10 @@ services: target: /etc/garage.toml - "sentry-garage:/var/lib/garage/" healthcheck: - test: "wget -no-verbose --tries=1 --spider http://localhost:3903/health || exit 1" - interval: 20s - timeout: 20s - retries: 10 + test: ["CMD", "/garage", "status"] + interval: 10s + timeout: 2s + retries: 2 snuba-api: <<: *snuba_defaults # Kafka consumer responsible for feeding events into Clickhouse diff --git a/garage.toml b/garage.toml index 94f1554736..6be0ca075b 100644 --- a/garage.toml +++ b/garage.toml @@ -5,7 +5,7 @@ db_engine = "sqlite" replication_factor = 1 # idk why rcp_ is needed without replication but ok rpc_bind_addr = "[::]:3901" -rpc_public_addr = "127.0.0.1:3901" +rpc_public_addr = "0.0.0.0:3901" # This secret will not ever be used as we are in a single node setup # so this static value is OK. Don't forget to change it if you ever # decide to use garage in a multi-node setup. From 2a55a4080bab943b84bf5c4b1d177eff441c4912 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 31 Dec 2024 23:47:23 +0300 Subject: [PATCH 10/13] add env var for garage size --- .env | 4 ++++ install/bootstrap-garage.sh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 5688114306..2239b6f6ac 100644 --- a/.env +++ b/.env @@ -22,3 +22,7 @@ HEALTHCHECK_RETRIES=10 POSTGRES_MAX_CONNECTIONS=100 # Set SETUP_JS_SDK_ASSETS to 1 to enable the setup of JS SDK assets # SETUP_JS_SDK_ASSETS=1 +# Storage size for node store in Garage +# If you need to change this after first install, you'd need to change the Garage layout +# See https://garagehq.deuxfleurs.fr/documentation/quick-start/#creating-a-cluster-layout +GARAGE_STORAGE_SIZE=100G diff --git a/install/bootstrap-garage.sh b/install/bootstrap-garage.sh index 0f0543be65..180e2886c7 100644 --- a/install/bootstrap-garage.sh +++ b/install/bootstrap-garage.sh @@ -5,7 +5,7 @@ garage="$dc exec garage /garage" if [[ $($garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; then node_id=$($garage status | tail -1 | awk '{print $1}') - $garage layout assign -z dc1 -c 100G "$node_id" + $garage layout assign -z local -c $GARAGE_STORAGE_SIZE "$node_id" $garage layout apply --version 1 $garage bucket create nodestore From a29cf0b78fac1103bd7a222d3bc06b782f8e8f57 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 1 Jan 2025 00:22:46 +0300 Subject: [PATCH 11/13] use better compression level --- garage.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/garage.toml b/garage.toml index 6be0ca075b..b2858d0c0c 100644 --- a/garage.toml +++ b/garage.toml @@ -11,7 +11,8 @@ rpc_public_addr = "0.0.0.0:3901" # decide to use garage in a multi-node setup. rpc_secret = "28f2fc651d4f280cea95e166f9113fe589039f9711d6c4f461872c1c5aaf7dec" -compression_level = 2 +# Setting 0 chooses the default which is 3 for now +compression_level = 0 [s3_api] s3_region = "garage" From a0ae9b124b9bc80905982464a447bb49c0a2fb8c Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 1 Jan 2025 23:07:18 +0300 Subject: [PATCH 12/13] simpler garage config --- garage.toml | 11 ----------- install/bootstrap-garage.sh | 2 ++ 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/garage.toml b/garage.toml index b2858d0c0c..d86df63e76 100644 --- a/garage.toml +++ b/garage.toml @@ -17,14 +17,3 @@ compression_level = 0 [s3_api] s3_region = "garage" api_bind_addr = "[::]:3900" -root_domain = ".s3.garage.localhost" - -[s3_web] -bind_addr = "[::]:3902" -root_domain = ".web.garage.localhost" -index = "index.html" - -[admin] -api_bind_addr = "0.0.0.0:3903" -metrics_token = "c6k30kKqovA5NoTfKxMVRmnKKVwd7jaF4OB2xshQVEA=" -admin_token = "c6k30kKqovA5NoTfKxMVRmnKKVwd7jaF4OB2xshQVEA=" diff --git a/install/bootstrap-garage.sh b/install/bootstrap-garage.sh index 180e2886c7..17058a8f17 100644 --- a/install/bootstrap-garage.sh +++ b/install/bootstrap-garage.sh @@ -10,6 +10,7 @@ if [[ $($garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; the $garage bucket create nodestore key_info=$($garage key create nodestore-key | head -3 | tail -2) + echo "$key_info" key_id=$(echo "$key_info" | head -1 | awk '{print $3}') key_secret=$(echo "$key_info" | tail -1 | awk '{print $3}') @@ -17,6 +18,7 @@ if [[ $($garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; the sed -i -e "s//$key_id/" $SENTRY_CONFIG_PY sed -i -e "s//$key_secret/" $SENTRY_CONFIG_PY + echo "Set Garage keys for SENTRY_NODESTORE_OPTIONS in $SENTRY_CONFIG_PY" else echo "Node store already exists, skipping..." From 8d7c1ff8419a13ce141d2944ced9d2097d1ada4c Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 2 Jan 2025 01:16:02 +0300 Subject: [PATCH 13/13] add migration support --- install/bootstrap-garage.sh | 20 +++++++++++++++----- sentry/sentry.conf.example.py | 4 ---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/install/bootstrap-garage.sh b/install/bootstrap-garage.sh index 17058a8f17..568675ed01 100644 --- a/install/bootstrap-garage.sh +++ b/install/bootstrap-garage.sh @@ -1,6 +1,6 @@ echo "${_group}Bootstrapping garage (node store)..." -$dc up --wait garage +$dc up --wait garage postgres garage="$dc exec garage /garage" if [[ $($garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; then @@ -8,6 +8,15 @@ if [[ $($garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; the $garage layout assign -z local -c $GARAGE_STORAGE_SIZE "$node_id" $garage layout apply --version 1 + # Only touch if no existing nodestore config is found + if ! grep -q "SENTRY_NODESTORE" $SENTRY_CONFIG_PY; then + nodestore_config=$(sed -n '/SENTRY_NODESTORE/,/[}]/{p}' sentry/sentry.conf.example.py) + if [[ $($dc exec postgres psql -qAt -U postgres -c "select exists (select * from nodestore_node limit 1)") = "f" ]]; then + nodestore_config=$(echo -e "$nodestore_config" | sed '$s/\}/ "read_through": True,\n "delete_through": True,\n\}/') + fi + echo "$nodestore_config" >>$SENTRY_CONFIG_PY + fi + $garage bucket create nodestore key_info=$($garage key create nodestore-key | head -3 | tail -2) echo "$key_info" @@ -16,10 +25,11 @@ if [[ $($garage bucket list | tail -1 | awk '{print $1}') != 'nodestore' ]]; the $garage bucket allow --read --write --owner nodestore --key nodestore-key - sed -i -e "s//$key_id/" $SENTRY_CONFIG_PY - sed -i -e "s//$key_secret/" $SENTRY_CONFIG_PY - - echo "Set Garage keys for SENTRY_NODESTORE_OPTIONS in $SENTRY_CONFIG_PY" + if grep -q "" $SENTRY_CONFIG_PY; then + sed -i -e "s//$key_id/" $SENTRY_CONFIG_PY + sed -i -e "s//$key_secret/" $SENTRY_CONFIG_PY + echo "Set Garage keys for SENTRY_NODESTORE_OPTIONS in $SENTRY_CONFIG_PY" + fi else echo "Node store already exists, skipping..." fi diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 7677573945..77d1007e93 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -90,14 +90,10 @@ def get_internal_network(): SENTRY_NODESTORE = "sentry_nodestore_s3.S3PassthroughDjangoNodeStorage" SENTRY_NODESTORE_OPTIONS = { - "delete_through": True, - "write_through": False, - "read_through": True, "compression": False, # we have compression enabled in Garage itself "endpoint_url": "http://garage:3900", "bucket_path": "nodestore", "bucket_name": "nodestore", - "retry_attempts": 3, "region_name": "garage", "aws_access_key_id": "", "aws_secret_access_key": "",