Added basic support for redis sentinel based on https://github.com/Namoshek/laravel-redis-sentinel

.github/workflows/tests.yml

@@ -30,10 +30,16 @@ jobs:
           - 5432:5432
 
     env:
-      # The hostname used to communicate with the Redis service container
+      # The hostname used to communicate with the Redis/Sentinel service containers
       REDIS_HOST: redis
+      # The hostname for redis sentinel
+      REDIS_SENTINEL_HOST: redis-sentinel
       # The default Redis port
       REDIS_PORT: 6379
+      # The default Redis Sentinel port
+      REDIS_SENTINEL_PORT: 26379
+      # The default Redis Sentinel primary
+      REDIS_SENTINEL_SERVICE: myprimary
       # MySQL
       DB_DATABASE: test
       DB_USER: root
@@ -69,6 +75,20 @@ jobs:
         with:
           redis-version: ${{ matrix.redis-version }}
 
+      - name: Generate Redis Sentinel conf compatible with redis 5 assuming 127.0.0.1 (no resolve hostname)
+        run: |
+          REDIS_SENTINEL_IP=127.0.0.1
+          cat <<EOF > sentinel5.conf
+          port ${{ env.REDIS_SENTINEL_PORT }}
+          sentinel monitor ${{ env.REDIS_SENTINEL_SERVICE }} $REDIS_SENTINEL_IP ${{ env.REDIS_PORT }} 2
+          sentinel down-after-milliseconds ${{ env.REDIS_SENTINEL_SERVICE }} 10000
+          sentinel failover-timeout ${{ env.REDIS_SENTINEL_SERVICE }} 180000
+          sentinel parallel-syncs ${{ env.REDIS_SENTINEL_SERVICE }} 2
+          EOF
+
+      - name: Start Redis Sentinel
+        run: docker run -d --name ${{env.REDIS_SENTINEL_HOST}} -p 26379:26379 --link redis:redis -v $PWD/sentinel5.conf:/data/sentinel.conf redis:${{ matrix.redis-version }} redis-server sentinel.conf --sentinel
+
       - name: Execute tests (PDO with Sqlite)
         run: vendor/bin/phpunit
 

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "github-actions.workflows.pinned.workflows": [],
+    "githubPullRequests.ignoredPullRequestBranches": [
+        "main"
+    ]
+}

README.md

@@ -79,6 +79,20 @@ Change the Redis options (the example shows the defaults):
         'timeout' => 0.1, // in seconds
         'read_timeout' => '10', // in seconds
         'persistent_connections' => false
+        'sentinel' => [ // sentinel options
+            'enable' => false,   // if enabled uses sentinel to get the master before connecting to redis
+            'host' => '127.0.0.1',  //  phpredis sentinel address of the redis, default is the same as redis host if empty
+            'port' => 26379, //  phpredis sentinel port of the primary redis server, default 26379 if empty.
+            'service' => 'myprimary', //, phpredis sentinel primary name, default myprimary
+            'timeout' => 0, // phpredis sentinel connection timeout
+            'persistent' => null, // phpredis sentinel persistence parameter
+            'retry_interval' => 0, // phpredis sentinel retry interval
+            'read_timeout' => 0,  // phpredis sentinel read timeout
+            'reconnect' => 0, // retries after losing connection to redis asking for a new primary, if -1 will retry indefinetely
+            'username' => '', // phpredis sentinel auth username
+            'password' => '', // phpredis sentinel auth password
+            'ssl' => null,
+        ]
     ]
 );
 ```

docker-compose.yml

@@ -14,12 +14,21 @@ services:
       - redis
     environment:
       - REDIS_HOST=redis
+      - REDIS_SENTINEL_HOST=redis-sentinel
 
   redis:
     image: redis
     ports:
       - 6379:6379
 
+  redis-sentinel:
+    image: redis
+    volumes:
+      - ./sentinel.conf://etc/sentinel.conf
+    ports:
+      - 26379:26379
+    command: redis-sentinel /etc/sentinel.conf
+
   phpunit:
     build: php-fpm/
     volumes:
@@ -29,3 +38,4 @@ services:
       - nginx
     environment:
       - REDIS_HOST=redis
+      - REDIS_SENTINEL_HOST=redis-sentinel

examples/flush_adapter.php

@@ -8,8 +8,9 @@
 
 if ($adapterName === 'redis') {
     define('REDIS_HOST', $_SERVER['REDIS_HOST'] ?? '127.0.0.1');
+    define('REDIS_SENTINEL_HOST', $_SERVER['REDIS_SENTINEL_HOST'] ?? '127.0.0.1');
 
-    $adapter = new Prometheus\Storage\Redis(['host' => REDIS_HOST]);
+    $adapter = new Prometheus\Storage\Redis(['host' => REDIS_HOST, 'sentinel' => ['host' => REDIS_SENTINEL_HOST]]);
 } elseif ($adapterName === 'apc') {
     $adapter = new Prometheus\Storage\APC();
 } elseif ($adapterName === 'apcng') {

sentinel.conf

@@ -0,0 +1,7 @@
+bind 0.0.0.0
+port 26379
+sentinel monitor myprimary redis 6379 2
+sentinel resolve-hostnames yes
+sentinel down-after-milliseconds myprimary 10000
+sentinel failover-timeout myprimary 10000
+sentinel parallel-syncs myprimary 1

src/Prometheus/Storage/Redis.php

@@ -30,8 +30,38 @@ class Redis implements Adapter
         'persistent_connections' => false,
         'password' => null,
         'user' => null,
+        'sentinel' => [ // sentinel options
+            'enable' => false,   // if enabled uses sentinel to get the master before connecting to redis
+            'host' => '127.0.0.1',  //  phpredis sentinel address of the redis, default is the same as redis host if empty
+            'port' => 26379, //  phpredis sentinel port of the primary redis server, default 26379 if empty.
+            'service' => 'myprimary', //, phpredis sentinel primary name, default myprimary
+            'timeout' => 0, // phpredis sentinel connection timeout
+            'persistent' => null, // phpredis sentinel persistence parameter
+            'retry_interval' => 0, // phpredis sentinel retry interval
+            'read_timeout' => 0,  // phpredis sentinel read timeout
+            'reconnect' => 0, // retries after losing connection to redis asking for a new primary, if -1 will retry indefinetely
+            'username' => '', // phpredis sentinel auth username
+            'password' => '', // phpredis sentinel auth password
+            'ssl' => null,
+        ]
     ];
 
+    // The following array contains all exception message parts which are interpreted as a connection loss or
+    // another unavailability of Redis.
+    private const ERROR_MESSAGES_INDICATING_UNAVAILABILITY = [
+        'connection closed',
+        'connection refused',
+        'connection lost',
+        'failed while reconnecting',
+        'is loading the dataset in memory',
+        'php_network_getaddresses',
+        'read error on connection',
+        'socket',
+        'went away',
+        'loading',
+        'readonly',
+        "can't write against a read only replica",
+    ];
     /**
      * @var string
      */
@@ -47,6 +77,11 @@ class Redis implements Adapter
      */
     private $redis;
 
+    /**
+     * @var RedisSentinel
+     */
+    private $sentinel = null;
+
     /**
      * @var boolean
      */
@@ -58,17 +93,49 @@ class Redis implements Adapter
      */
     public function __construct(array $options = [])
     {
-        $this->options = array_merge(self::$defaultOptions, $options);
+        $this->options = [...self::$defaultOptions, ...$options];
+        $this->options['sentinel'] = [...self::$defaultOptions['sentinel'] ?? [], ...$options['sentinel'] ?? []];
         $this->redis = new \Redis();
+        if (boolval($this->options['sentinel']['enable'])) {
+            $options['sentinel']['host'] = $options['sentinel']['host'] ?? $options['host'];
+            $this->sentinel = new RedisSentinel($options['sentinel']);
+        }
+    }
+
+    /**
+     * Sentinels  discoverMaster
+     * @return void
+     */
+    public function updateSentinelPrimary(): void
+    {
+        $master = $this->sentinel->getMaster();
+
+        if (is_array($master)) {
+            $this->options['host'] = $master['ip'];
+            $this->options['port'] = $master['port'];
+        }
+    }
+
+    /**
+     * @return \RedisSentinel
+     */
+    public function getRedisSentinel(): \RedisSentinel
+    {
+        return $this->sentinel->getSentinel();
     }
 
     /**
      * @param \Redis $redis
+     * @param \RedisSentinel $redisSentinel
      * @return self
      * @throws StorageException
      */
-    public static function fromExistingConnection(\Redis $redis): self
+    public static function fromExistingConnection(\Redis $redis, ?\RedisSentinel $redisSentinel = null): self
     {
+        if (isset($redisSentinel)) {
+            RedisSentinel::fromExistingConnection($redisSentinel);
+        }
+
         if ($redis->isConnected() === false) {
             throw new StorageException('Connection to Redis server not established');
         }
@@ -97,8 +164,8 @@ public static function setPrefix(string $prefix): void
     }
 
     /**
-     * @deprecated use replacement method wipeStorage from Adapter interface
      * @throws StorageException
+     * @deprecated use replacement method wipeStorage from Adapter interface
      */
     public function flushRedis(): void
     {
@@ -127,7 +194,7 @@ public function wipeStorage(): void
             <<<LUA
 redis.replicate_commands()
 local cursor = "0"
-repeat 
+repeat
     local results = redis.call('SCAN', cursor, 'MATCH', ARGV[1])
     cursor = results[1]
     for _, key in ipairs(results[2]) do
@@ -187,6 +254,27 @@ function (array $metric): MetricFamilySamples {
         );
     }
 
+    /**
+     * Inspects the given exception and reconnects the client if the reported error indicates that the server
+     * went away or is in readonly mode, which may happen in case of a Redis Sentinel failover.
+     */
+    private function reconnectIfRedisIsUnavailableOrReadonly(\RedisException $exception): bool
+    {
+        // We convert the exception message to lower-case in order to perform case-insensitive comparison.
+        $exceptionMessage = strtolower($exception->getMessage());
+
+        // Because we also match only partial exception messages, we cannot use in_array() at this point.
+        foreach (self::ERROR_MESSAGES_INDICATING_UNAVAILABILITY as $errorMessage) {
+            if (str_contains($exceptionMessage, $errorMessage)) {
+                // Here we reconnect through Redis Sentinel if we lost connection to the server or if another unavailability occurred.
+                // We may actually reconnect to the same, broken server. But after a failover occured, we should be ok.
+                // It may take a moment until the Sentinel returns the new master, so this may be triggered multiple times.
+                return true;
+            }
+        }
+        return false;
+    }
+
     /**
      * @throws StorageException
      */
@@ -196,7 +284,30 @@ private function ensureOpenConnection(): void
             return;
         }
 
-        $this->connectToServer();
+        if ($this->sentinel !== null && boolval($this->options['sentinel']['enable'])) {
+            $reconnect = $this->options['sentinel']['reconnect'];
+            $retries = 0;
+            while ($retries <= $reconnect) {
+                try {
+                    $this->updateSentinelPrimary();
+                    $this->connectToServer();
+                    break;
+                } catch (\RedisException $e) {
+                    $retry = $this->reconnectIfRedisIsUnavailableOrReadonly($e);
+                    if (!$retry) {
+                        throw new StorageException(
+                            sprintf("Can't connect to Redis server. %s", $e->getMessage()),
+                            $e->getCode(),
+                            $e
+                        );
+                    }
+                }
+                $retries++;
+            }
+        } else {
+            $this->connectToServer();
+        }
+
         $authParams = [];
 
         if (isset($this->options['user']) && $this->options['user'] !== '') {
@@ -225,28 +336,27 @@ private function ensureOpenConnection(): void
      */
     private function connectToServer(): void
     {
-        try {
-            $connection_successful = false;
-            if ($this->options['persistent_connections'] !== false) {
-                $connection_successful = $this->redis->pconnect(
-                    $this->options['host'],
-                    (int) $this->options['port'],
-                    (float) $this->options['timeout']
-                );
-            } else {
+        $connection_successful = false;
+        if ($this->options['persistent_connections'] !== false) {
+            $connection_successful = $this->redis->pconnect(
+                $this->options['host'],
+                (int) $this->options['port'],
+                (float) $this->options['timeout']
+            );
+        } else {
+            try {
                 $connection_successful = $this->redis->connect($this->options['host'], (int) $this->options['port'], (float) $this->options['timeout']);
-            }
-            if (!$connection_successful) {
+            } catch (\RedisException $ex) {
                 throw new StorageException(
-                    sprintf("Can't connect to Redis server. %s", $this->redis->getLastError()),
-                    0
+                    sprintf("Can't connect to Redis server. %s", $ex->getMessage()),
+                    $ex->getCode()
                 );
             }
-        } catch (\RedisException $e) {
+        }
+        if (!$connection_successful) {
             throw new StorageException(
-                sprintf("Can't connect to Redis server. %s", $e->getMessage()),
-                $e->getCode(),
-                $e
+                sprintf("Can't connect to Redis server. %s", $this->redis->getLastError()),
+                0
             );
         }
     }