add metric to measure how long it takes to obtain a db connection

Author: pietroalbini

src/app.rs

@@ -62,6 +62,9 @@ impl App {
     pub fn new(config: config::Server, http_client: Option<Client>) -> App {
         use oauth2::{AuthUrl, ClientId, ClientSecret, TokenUrl};
 
+        let instance_metrics =
+            InstanceMetrics::new().expect("could not initialize instance metrics");
+
         let github = GitHubClient::new(http_client.clone(), config.gh_base_url.clone());
 
         let github_oauth = BasicClient::new(
@@ -116,7 +119,14 @@ impl App {
                 .connection_customizer(Box::new(primary_db_connection_config))
                 .thread_pool(thread_pool.clone());
 
-            DieselPool::new(&config.db.primary.url, primary_db_config).unwrap()
+            DieselPool::new(
+                &config.db.primary.url,
+                primary_db_config,
+                instance_metrics
+                    .database_time_to_obtain_connection
+                    .with_label_values(&["primary"]),
+            )
+            .unwrap()
         };
 
         let replica_database = if let Some(url) = config.db.replica.as_ref().map(|c| &c.url) {
@@ -135,7 +145,16 @@ impl App {
                     .connection_customizer(Box::new(replica_db_connection_config))
                     .thread_pool(thread_pool);
 
-                Some(DieselPool::new(url, replica_db_config).unwrap())
+                Some(
+                    DieselPool::new(
+                        url,
+                        replica_db_config,
+                        instance_metrics
+                            .database_time_to_obtain_connection
+                            .with_label_values(&["follower"]),
+                    )
+                    .unwrap(),
+                )
             }
         } else {
             None
@@ -150,8 +169,7 @@ impl App {
             downloads_counter: DownloadsCounter::new(),
             emails: Emails::from_environment(),
             service_metrics: ServiceMetrics::new().expect("could not initialize service metrics"),
-            instance_metrics: InstanceMetrics::new()
-                .expect("could not initialize instance metrics"),
+            instance_metrics,
             http_client,
         }
     }

src/db.rs

@@ -2,6 +2,7 @@ use conduit::RequestExt;
 use diesel::prelude::*;
 use diesel::r2d2::{self, ConnectionManager, CustomizeConnection};
 use parking_lot::{ReentrantMutex, ReentrantMutexGuard};
+use prometheus::Histogram;
 use std::sync::Arc;
 use std::{ops::Deref, time::Duration};
 use url::Url;
@@ -10,14 +11,18 @@ use crate::middleware::app::RequestApp;
 
 #[derive(Clone)]
 pub enum DieselPool {
-    Pool(r2d2::Pool<ConnectionManager<PgConnection>>),
+    Pool {
+        pool: r2d2::Pool<ConnectionManager<PgConnection>>,
+        time_to_obtain_connection_metric: Histogram,
+    },
     Test(Arc<ReentrantMutex<PgConnection>>),
 }
 
 impl DieselPool {
     pub(crate) fn new(
         url: &str,
         config: r2d2::Builder<ConnectionManager<PgConnection>>,
+        time_to_obtain_connection_metric: Histogram,
     ) -> Result<DieselPool, PoolError> {
         let manager = ConnectionManager::new(connection_url(url));
 
@@ -32,7 +37,10 @@ impl DieselPool {
         // serving errors for the first connections until the pool is initialized) and if we can't
         // establish any connection continue booting up the application. The database pool will
         // automatically be marked as unhealthy and the rest of the application will adapt.
-        let pool = DieselPool::Pool(config.build_unchecked(manager));
+        let pool = DieselPool::Pool {
+            pool: config.build_unchecked(manager),
+            time_to_obtain_connection_metric,
+        };
         match pool.wait_until_healthy(Duration::from_secs(5)) {
             Ok(()) => {}
             Err(PoolError::UnhealthyPool) => {}
@@ -52,22 +60,25 @@ impl DieselPool {
 
     pub fn get(&self) -> Result<DieselPooledConn<'_>, PoolError> {
         match self {
-            DieselPool::Pool(pool) => {
+            DieselPool::Pool {
+                pool,
+                time_to_obtain_connection_metric,
+            } => time_to_obtain_connection_metric.observe_closure_duration(|| {
                 if let Some(conn) = pool.try_get() {
                     Ok(DieselPooledConn::Pool(conn))
                 } else if !self.is_healthy() {
                     Err(PoolError::UnhealthyPool)
                 } else {
                     Ok(DieselPooledConn::Pool(pool.get().map_err(PoolError::R2D2)?))
                 }
-            }
+            }),
             DieselPool::Test(conn) => Ok(DieselPooledConn::Test(conn.lock())),
         }
     }
 
     pub fn state(&self) -> PoolState {
         match self {
-            DieselPool::Pool(pool) => {
+            DieselPool::Pool { pool, .. } => {
                 let state = pool.state();
                 PoolState {
                     connections: state.connections,
@@ -83,7 +94,7 @@ impl DieselPool {
 
     pub fn wait_until_healthy(&self, timeout: Duration) -> Result<(), PoolError> {
         match self {
-            DieselPool::Pool(pool) => match pool.get_timeout(timeout) {
+            DieselPool::Pool { pool, .. } => match pool.get_timeout(timeout) {
                 Ok(_) => Ok(()),
                 Err(_) if !self.is_healthy() => Err(PoolError::UnhealthyPool),
                 Err(err) => Err(PoolError::R2D2(err)),

src/metrics/instance.rs

@@ -29,6 +29,8 @@ metrics! {
         database_idle_conns: IntGaugeVec["pool"],
         /// Number of used database connections in the pool
         database_used_conns: IntGaugeVec["pool"],
+        /// Amount of time required to obtain a database connection
+        pub database_time_to_obtain_connection: HistogramVec["pool"],
 
         /// Number of requests processed by this instance
         pub requests_total: IntCounter,