Merge pull request #33 from gatewayd-io/circuit-breaking-all-the-way

mostafa · web-flow · commit 5a278326ade7 · 2023-11-19T16:08:20.000+01:00
Circuit breaking all the way
diff --git a/.vale/Vocab/accept.txt b/.vale/Vocab/accept.txt
@@ -71,6 +71,10 @@ acceptancePolicy
 receiveDeadline
 receiveTimeout
 sendDeadline
+dialTimeout
+backoff
+backoffMultiplier
+disableBackoffCaps
 API
 DBA
 DBRE
@@ -82,3 +86,6 @@ maxBackups
 Postgres
 GatewayD
 gatewayd
+frontmatter
+startTimeout
+tls_connections
diff --git a/developing-plugins/grpc-api-reference.md b/developing-plugins/grpc-api-reference.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: gRPC API Reference
 description: GatewayD exposes a gRPC API that can be used to interact with the GatewayD plugin system. This API can be used by the GatewayD plugins and is available in the GatewayD SDK.
diff --git a/developing-plugins/index.md b/developing-plugins/index.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Developing Plugins
 nav_order: 4
diff --git a/developing-plugins/plugin-developers-guide.md b/developing-plugins/plugin-developers-guide.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Plugin Developers Guide
 description: Plugin developers' guide of GatewayD
diff --git a/developing-plugins/sdk-reference.md b/developing-plugins/sdk-reference.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: SDK Reference
 description: The GatewayD plugin SDK provides a number of interfaces, structs and methods to help you build your plugin.
diff --git a/developing-plugins/template-projects.md b/developing-plugins/template-projects.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Template Projects
 description: Template projects can be used to quickly get started with developing plugins.
diff --git a/getting-started/index.md b/getting-started/index.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Getting Started
 nav_order: 1
diff --git a/getting-started/installation.md b/getting-started/installation.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Installation
 description: How to install GatewayD and its plugins on different platforms and how to build it from source.
diff --git a/getting-started/running-gatewayd.md b/getting-started/running-gatewayd.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Running GatewayD
 description: How to run GatewayD and test it with psql
@@ -95,20 +95,20 @@ Running GatewayD will produce the following log output, which means that Gateway
 5. exposing an HTTP and a gRPC API on ports `18080` and `19090`.
 
 ```bash
-2023-04-08T02:01:04+02:00 INF configuring client automatic mTLS plugin=gatewayd-plugin-cache
-2023-04-08T02:01:04+02:00 INF Starting metrics server via HTTP over Unix domain socket endpoint=/metrics plugin=gatewayd-plugin-cache timestamp=2023-04-08T02:01:04.242+0200 unixDomainSocket=/tmp/gatewayd-plugin-cache.sock
-2023-04-08T02:01:04+02:00 INF configuring server automatic mTLS plugin=gatewayd-plugin-cache timestamp=2023-04-08T02:01:04.243+0200
-2023-04-08T02:01:04+02:00 INF Registering plugin hooks name=gatewayd-plugin-cache
-2023-04-08T02:01:04+02:00 INF Plugin is ready name=gatewayd-plugin-cache
-2023-04-08T02:01:04+02:00 INF Started the metrics merger scheduler metricsMergerPeriod=5s startDelay=1680912069
-2023-04-08T02:01:04+02:00 INF Starting plugin health check scheduler healthCheckPeriod=5s
-2023-04-08T02:01:04+02:00 INF Metrics are exposed address=http://localhost:9090/metrics
-2023-04-08T02:01:04+02:00 INF There are clients available in the pool count=10 name=default
-2023-04-08T02:01:04+02:00 INF Started the client health check scheduler healthCheckPeriod=1m0s startDelay=2023-04-08T02:02:04+02:00
-2023-04-08T02:01:04+02:00 INF GatewayD is listening address=0.0.0.0:15432
-2023-04-08T02:01:04+02:00 INF Started the HTTP API address=localhost:18080
-2023-04-08T02:01:04+02:00 INF Started the gRPC API address=localhost:19090 network=tcp
-2023-04-08T02:01:04+02:00 INF GatewayD is running pid=41568
+2023-11-15T22:50:29+01:00 INF configuring client automatic mTLS plugin=gatewayd-plugin-cache
+2023-11-15T22:50:29+01:00 INF Starting metrics server via HTTP over Unix domain socket endpoint=/metrics plugin=gatewayd-plugin-cache timestamp=2023-11-15T22:50:29.589+0100 unixDomainSocket=/tmp/gatewayd-plugin-cache.sock
+2023-11-15T22:50:29+01:00 INF configuring server automatic mTLS plugin=gatewayd-plugin-cache timestamp=2023-11-15T22:50:29.596+0100
+2023-11-15T22:50:29+01:00 INF Registering plugin hooks name=gatewayd-plugin-cache
+2023-11-15T22:50:29+01:00 INF Plugin is ready name=gatewayd-plugin-cache
+2023-11-15T22:50:29+01:00 INF Started the metrics merger scheduler metricsMergerPeriod=5s startDelay=2023-11-15T22:50:34+01:00
+2023-11-15T22:50:29+01:00 INF Starting plugin health check scheduler healthCheckPeriod=5s
+2023-11-15T22:50:29+01:00 INF Metrics are exposed address=http://localhost:9090/metrics readHeaderTimeout=10s timeout=10s
+2023-11-15T22:50:29+01:00 INF There are clients available in the pool count=10 name=default
+2023-11-15T22:50:29+01:00 INF Started the client health check scheduler healthCheckPeriod=1m0s startDelay=2023-11-15T22:51:29+01:00
+2023-11-15T22:50:29+01:00 INF GatewayD is listening address=0.0.0.0:15432
+2023-11-15T22:50:29+01:00 INF Started the HTTP API address=localhost:18080
+2023-11-15T22:50:29+01:00 INF Started the gRPC API address=localhost:19090 network=tcp
+2023-11-15T22:50:29+01:00 INF GatewayD is running pid=32371
 ```
 
 {: .note }
diff --git a/getting-started/welcome.md b/getting-started/welcome.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Welcome
 description: Introduction to GatewayD and its key features
diff --git a/miscellaneous/glossary.md b/miscellaneous/glossary.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Glossary
 description: Glossary of GatewayD terms
diff --git a/miscellaneous/index.md b/miscellaneous/index.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Miscellaneous
 nav_order: 6
diff --git a/miscellaneous/telemetry-and-usage-report.md b/miscellaneous/telemetry-and-usage-report.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Telemetry and Usage Report
 description: Telemetry and usage report of GatewayD
diff --git a/plugins/gatewayd-plugin-cache.md b/plugins/gatewayd-plugin-cache.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: gatewayd-plugin-cache
 description: GatewayD plugin for caching query results in Redis.
diff --git a/plugins/index.md b/plugins/index.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Plugins
 nav_order: 5
diff --git a/using-gatewayd/API.md b/using-gatewayd/API.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: API
 description: GatewayD exposes a gRPC API with an HTTP gateway for querying and managing the `gatewayd` process and its plugins.
diff --git a/using-gatewayd/CLI.md b/using-gatewayd/CLI.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: CLI
 description: GatewayD is a CLI application that runs on Windows, Linux-based distributions and macOS.
diff --git a/using-gatewayd/clients.md b/using-gatewayd/clients.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-19 14:25:14 +0100
 layout: default
 title: Clients
 description: Client object is a client that can connect to the database servers over TCP, UDP and Unix Domain Socket.
@@ -40,3 +40,53 @@ You have the option to set deadlines on send and receive calls to the database s
 ## Receive timeout
 
 Since setting receive deadline kills the connection, the `receiveTimeout` property is introduced to stop the receive function from blocking the connection and waiting forever. The current value is zero, which means that it behaves like before, but one can set it to a duration string value.
+
+## Dial timeout
+
+The dial timeout is the amount of time the client should wait before giving up on dialing the database. The default value is `60s`. Setting it to `0s` disables the dial timeout.
+
+## Retries and backoff
+
+The first attempt to connect to the database server is made when the client is created, which happens instantly when GatewayD starts. However, if the first attempt fails, the client will retry the connection. The default number of retries is `3`, which means that the client will retry the connection three times before giving up. Setting it to `0` disables the retry mechanism. The client can also back off before retrying the connection. The backoff duration is set to `1s` by default and `0s` disables the backoff mechanism. The backoff multiplier is set to `2` by default and `0` disables the backoff. The backoff multiplier is applied to the backoff duration. The backoff duration is capped at `60s` by default and the max retry is capped at `10` by default. Setting `disableBackoffCaps` to `true` disables the backoff and retry caps.
+
+{: .note }
+> The first attempt to connect to the database server is counted as a retry, hence the three retries are actually four attempts (one instant attempt and three retry attempts), and the backoff duration is applied to the second attempt and so on.
+
+The backoff duration is calculated by multiplying the backoff duration by the backoff multiplier raised to the power of the number of retries. For example, if the backoff duration is 1 second and the backoff multiplier is 2, the backoff duration will be 1 second, 2 seconds, 4 seconds, 8 seconds, etc. The backoff duration is capped at 1 minute and the backoff multiplier is capped at 10, so the backoff duration will be 1 minute after 6 retries. The backoff multiplier is capped at 10 to prevent the backoff duration from growing too quickly, unless the backoff caps are disabled. The following is the formula for calculating the backoff duration:
+
+```text
+backoff duration = backoff * (backoff multiplier ^ current retry number)
+```
+
+Considering the default values, the backoff duration will be calculated as follows:
+
+```text
+1 * 2 ^ 1 = 2 seconds
+1 * 2 ^ 2 = 4 seconds
+1 * 2 ^ 3 = 8 seconds
+```
+
+If the retries are set to `11`, the backoff duration from the fourth attempt will be calculated as follows, which is capped at 1 minute and 10 retries are made:
+
+```text
+1 * 2 ^ 4 = 16 seconds
+1 * 2 ^ 5 = 32 seconds
+1 * 2 ^ 6 = 1 minute
+1 * 2 ^ 7 = 1 minute (capped)
+1 * 2 ^ 8 = 1 minute (capped)
+1 * 2 ^ 9 = 1 minute (capped)
+1 * 2 ^ 10 = 1 minute (capped)
+```
+
+And if the backoff caps are disabled, the uncapped backoff duration for the fourth attempt will be calculated as follows:
+
+```text
+1 * 2 ^ 4 = 16 seconds
+1 * 2 ^ 5 = 32 seconds
+1 * 2 ^ 6 = 1 minute 4 seconds (uncapped)
+1 * 2 ^ 7 = 2 minutes 8 seconds (uncapped)
+1 * 2 ^ 8 = 4 minutes 16 seconds (uncapped)
+1 * 2 ^ 9 = 8 minutes 32 seconds (uncapped)
+1 * 2 ^ 10 = 17 minutes 4 seconds (uncapped)
+1 * 2 ^ 11 = 34 minutes 8 seconds (uncapped)
+```
diff --git a/using-gatewayd/configuration.md b/using-gatewayd/configuration.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-19 14:25:14 +0100
 layout: default
 title: Configuration
 description: GatewayD is fully configurable via various sources, including default values, YAML config files, environment variables, CLI flags and plugins.
@@ -82,6 +82,12 @@ clients:
     receiveDeadline: 0s # duration, 0ms/0s means no deadline
     receiveTimeout: 0s # duration, 0ms/0s means no timeout
     sendDeadline: 0s # duration, 0ms/0s means no deadline
+    dialTimeout: 60s # duration, 0ms/0s means no timeout
+    # Retry configuration
+    retries: 3 # 0 means no retry
+    backoff: 1s # duration
+    backoffMultiplier: 2.0 # 0 means no backoff
+    disableBackoffCaps: false
 
 pools:
   default:
@@ -130,6 +136,7 @@ metricsMergerPeriod: 5s
 healthCheckPeriod: 5s
 reloadOnCrash: True
 timeout: 30s
+startTimeout: 1m
 plugins:
   - name: gatewayd-plugin-cache
     enabled: True
diff --git a/using-gatewayd/connection-lifecycle.md b/using-gatewayd/connection-lifecycle.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Connection Lifecycle
 description: Connection Lifecycle of GatewayD
diff --git a/using-gatewayd/global-configuration/api.md b/using-gatewayd/global-configuration/api.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: API
 description: GatewayD gRPC API configuration
diff --git a/using-gatewayd/global-configuration/clients.md b/using-gatewayd/global-configuration/clients.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-19 14:25:14 +0100
 layout: default
 title: Clients
 description: GatewayD client configuration
@@ -24,6 +24,11 @@ GatewayD supports multiple client configurations. Each client in each configurat
 | receiveDeadline    | duration (string) | 0s             | Valid duration strings | The amount of time the client should wait before giving up on call to receive. `0s` disables receive deadline. This k |
 | receiveTimeout     | duration (string) | 0s             | Valid duration strings | The amount of time the client should wait before giving up on call to receive. `0s` disables receive timeout.         |
 | sendDeadline       | duration (string) | 0s             | Valid duration strings | The amount of time the client should wait before giving up on call to send. `0s` disables send deadline.              |
+| dialTimeout        | duration (string) | 60s            | Valid duration strings | The amount of time the client should wait before giving up on dialing the database. `0s` disables dial timeout.       |
+| retries            | number            | 3              | Positive integers      | The amount of times to retry a failed connection. `0` means no retry.                                                 |
+| backoff            | duration (string) | 1s             | Valid duration strings | The amount of time to wait before retrying a failed connection. `0s` means no backoff.                                |
+| backoffMultiplier  | number            | 2.0            | Positive integers      | The multiplier to apply to the backoff duration. `0` means no backoff.                                                |
+| disableBackoffCaps | boolean           | False          | True, False            | Whether to disable the backoff caps for backoff multiplier and backoff duration.                                      |
 
 ```yaml
 clients:
@@ -36,4 +41,10 @@ clients:
     receiveDeadline: 0s # duration, 0ms/0s means no deadline
     receiveTimeout: 0s # duration, 0ms/0s means no timeout
     sendDeadline: 0s # duration, 0ms/0s means no deadline
+    dialTimeout: 60s # duration, 0ms/0s means no timeout
+    # Retry configuration
+    retries: 3 # 0 means no retry
+    backoff: 1s # duration
+    backoffMultiplier: 2.0 # 0 means no backoff
+    disableBackoffCaps: false
 ```
diff --git a/using-gatewayd/global-configuration/index.md b/using-gatewayd/global-configuration/index.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Global Configuration
 nav_order: 2
diff --git a/using-gatewayd/global-configuration/loggers.md b/using-gatewayd/global-configuration/loggers.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Loggers
 description: GatewayD logger configuration
@@ -38,6 +38,9 @@ GatewayD supports multiple loggers, and each logger supports sending logs to mul
 | compress   | boolean | True          | True, False       | Whether to compress rotated backup files or not                                                     |
 | localTime  | boolean | False         | True, False       | Whether to use the local system time for formatting the timestamps in backup files. Default is UTC. |
 
+{: .note }
+> If maxBackups and maxAge are both 0, no old log files will be deleted.
+
 ### Rsyslog
 
 | Name           | Type   | Default value | Possible values                                       | Description                                                                                                                               |
diff --git a/using-gatewayd/global-configuration/metrics.md b/using-gatewayd/global-configuration/metrics.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Metrics
 description: GatewayD metrics configuration
@@ -29,6 +29,9 @@ scrape_configs:
 | address           | string            | localhost:9090 | Valid host:port       | The HTTP address and port to expose metrics on                                                                       |
 | path              | string            | /metrics       | Valid path values     | The endpoint to expose metrics on                                                                                    |
 | readHeaderTimeout | duration (string) | 10s            | Valid duration values | The maximum duration for reading the request headers. Setting this to a non-zero value will block Slowloris attacks. |
+| timeout           | duration (string) | 10s            | Valid duration values | The maximum duration before timing out. This applies to writes, reads, idle and handler timeouts separately.         |
+| certFile          | string            | ""             | Valid path values     | The path to the TLS certificate file                                                                                 |
+| keyFile           | string            | ""             | Valid path values     | The path to the TLS key file                                                                                         |
 
 ## Built-in Metrics
 
@@ -38,6 +41,7 @@ The following are built-in metrics emitted by GatewayD. The namespace of all the
 | ------------------------------------ | ------- | ------------------------------------------------------------------ |
 | client_connections                   | Gauge   | Number of client connections                                       |
 | server_connections                   | Gauge   | Number of server connections                                       |
+| tls_connections                      | Gauge   | Number of TLS connections                                          |
 | server_ticks_fired_total             | Counter | Total number of server ticks fired                                 |
 | bytes_received_from_client           | Summary | Number of bytes received from client                               |
 | bytes_sent_to_server                 | Summary | Number of bytes sent to server                                     |
diff --git a/using-gatewayd/global-configuration/pools.md b/using-gatewayd/global-configuration/pools.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Pools
 description: GatewayD pool configuration
diff --git a/using-gatewayd/global-configuration/proxies.md b/using-gatewayd/global-configuration/proxies.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Proxies
 description: GatewayD proxy configuration
diff --git a/using-gatewayd/global-configuration/servers.md b/using-gatewayd/global-configuration/servers.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Servers
 description: GatewayD server configuration
diff --git a/using-gatewayd/index.md b/using-gatewayd/index.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Using GatewayD
 nav_order: 2
diff --git a/using-gatewayd/observability.md b/using-gatewayd/observability.md
@@ -1,5 +1,5 @@
 ---
-last_modified_date: 2023-11-05 18:05:24 +0100
+last_modified_date: 2023-11-17 17:49:50 +0100
 layout: default
 title: Observability
 description: Observability is a first-class citizen of GatewayD. It generates logs, metrics and traces to make it easier to see what is going on inside.
diff --git a/using-gatewayd/plugins-configuration/general-configurations.md b/using-gatewayd/plugins-configuration/general-configurations.md
diff --git a/using-gatewayd/plugins-configuration/index.md b/using-gatewayd/plugins-configuration/index.md
diff --git a/using-gatewayd/plugins-configuration/plugins-configuration.md b/using-gatewayd/plugins-configuration/plugins-configuration.md
diff --git a/using-gatewayd/pools.md b/using-gatewayd/pools.md
diff --git a/using-gatewayd/protocols.md b/using-gatewayd/protocols.md
diff --git a/using-gatewayd/proxies.md b/using-gatewayd/proxies.md
diff --git a/using-gatewayd/servers.md b/using-gatewayd/servers.md
diff --git a/using-plugins/hook-registry.md b/using-plugins/hook-registry.md
diff --git a/using-plugins/hooks.md b/using-plugins/hooks.md
diff --git a/using-plugins/index.md b/using-plugins/index.md
diff --git a/using-plugins/plugin-registry.md b/using-plugins/plugin-registry.md
diff --git a/using-plugins/plugin-types.md b/using-plugins/plugin-types.md
diff --git a/using-plugins/plugins.md b/using-plugins/plugins.md
diff --git a/using-plugins/proposals.md b/using-plugins/proposals.md
