diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 78a3122f..3aa31e75 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,10 +30,16 @@ jobs: - 5432:5432 env: - # The hostname used to communicate with the Redis service container + # The hostname used to communicate with the Redis/Sentinel service containers REDIS_HOST: redis + # The hostname for redis sentinel + REDIS_SENTINEL_HOST: redis-sentinel # The default Redis port REDIS_PORT: 6379 + # The default Redis Sentinel port + REDIS_SENTINEL_PORT: 26379 + # The default Redis Sentinel primary + REDIS_SENTINEL_SERVICE: myprimary # MySQL DB_DATABASE: test DB_USER: root @@ -69,6 +75,20 @@ jobs: with: redis-version: ${{ matrix.redis-version }} + - name: Generate Redis Sentinel conf compatible with redis 5 assuming 127.0.0.1 (no resolve hostname) + run: | + REDIS_SENTINEL_IP=127.0.0.1 + cat < sentinel5.conf + port ${{ env.REDIS_SENTINEL_PORT }} + sentinel monitor ${{ env.REDIS_SENTINEL_SERVICE }} $REDIS_SENTINEL_IP ${{ env.REDIS_PORT }} 2 + sentinel down-after-milliseconds ${{ env.REDIS_SENTINEL_SERVICE }} 10000 + sentinel failover-timeout ${{ env.REDIS_SENTINEL_SERVICE }} 180000 + sentinel parallel-syncs ${{ env.REDIS_SENTINEL_SERVICE }} 2 + EOF + + - name: Start Redis Sentinel + run: docker run -d --name ${{env.REDIS_SENTINEL_HOST}} -p 26379:26379 --link redis:redis -v $PWD/sentinel5.conf:/data/sentinel.conf redis:${{ matrix.redis-version }} redis-server sentinel.conf --sentinel + - name: Execute tests (PDO with Sqlite) run: vendor/bin/phpunit diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..a04f8bd1 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "github-actions.workflows.pinned.workflows": [], + "githubPullRequests.ignoredPullRequestBranches": [ + "main" + ] +} \ No newline at end of file diff --git a/README.md b/README.md index 3866cfd4..eb4fc026 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,20 @@ Change the Redis options (the example shows the defaults): 'timeout' => 0.1, // in seconds 'read_timeout' => '10', // in seconds 'persistent_connections' => false + 'sentinel' => [ // sentinel options + 'enable' => false, // if enabled uses sentinel to get the master before connecting to redis + 'host' => '127.0.0.1', // phpredis sentinel address of the redis, default is the same as redis host if empty + 'port' => 26379, // phpredis sentinel port of the primary redis server, default 26379 if empty. + 'service' => 'myprimary', //, phpredis sentinel primary name, default myprimary + 'timeout' => 0, // phpredis sentinel connection timeout + 'persistent' => null, // phpredis sentinel persistence parameter + 'retry_interval' => 0, // phpredis sentinel retry interval + 'read_timeout' => 0, // phpredis sentinel read timeout + 'reconnect' => 0, // retries after losing connection to redis asking for a new primary, if -1 will retry indefinetely + 'username' => '', // phpredis sentinel auth username + 'password' => '', // phpredis sentinel auth password + 'ssl' => null, + ] ] ); ``` diff --git a/docker-compose.yml b/docker-compose.yml index c6a794b0..b448f69d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,12 +14,21 @@ services: - redis environment: - REDIS_HOST=redis + - REDIS_SENTINEL_HOST=redis-sentinel redis: image: redis ports: - 6379:6379 + redis-sentinel: + image: redis + volumes: + - ./sentinel.conf://etc/sentinel.conf + ports: + - 26379:26379 + command: redis-sentinel /etc/sentinel.conf + phpunit: build: php-fpm/ volumes: @@ -29,3 +38,4 @@ services: - nginx environment: - REDIS_HOST=redis + - REDIS_SENTINEL_HOST=redis-sentinel diff --git a/examples/flush_adapter.php b/examples/flush_adapter.php index 1c00eab7..f3defb4e 100644 --- a/examples/flush_adapter.php +++ b/examples/flush_adapter.php @@ -8,8 +8,9 @@ if ($adapterName === 'redis') { define('REDIS_HOST', $_SERVER['REDIS_HOST'] ?? '127.0.0.1'); + define('REDIS_SENTINEL_HOST', $_SERVER['REDIS_SENTINEL_HOST'] ?? '127.0.0.1'); - $adapter = new Prometheus\Storage\Redis(['host' => REDIS_HOST]); + $adapter = new Prometheus\Storage\Redis(['host' => REDIS_HOST, 'sentinel' => ['host' => REDIS_SENTINEL_HOST]]); } elseif ($adapterName === 'apc') { $adapter = new Prometheus\Storage\APC(); } elseif ($adapterName === 'apcng') { diff --git a/sentinel.conf b/sentinel.conf new file mode 100644 index 00000000..0197f209 --- /dev/null +++ b/sentinel.conf @@ -0,0 +1,7 @@ +bind 0.0.0.0 +port 26379 +sentinel monitor myprimary redis 6379 2 +sentinel resolve-hostnames yes +sentinel down-after-milliseconds myprimary 10000 +sentinel failover-timeout myprimary 10000 +sentinel parallel-syncs myprimary 1 \ No newline at end of file diff --git a/src/Prometheus/Storage/Redis.php b/src/Prometheus/Storage/Redis.php index b10e3f9c..96164cdf 100644 --- a/src/Prometheus/Storage/Redis.php +++ b/src/Prometheus/Storage/Redis.php @@ -30,8 +30,38 @@ class Redis implements Adapter 'persistent_connections' => false, 'password' => null, 'user' => null, + 'sentinel' => [ // sentinel options + 'enable' => false, // if enabled uses sentinel to get the master before connecting to redis + 'host' => '127.0.0.1', // phpredis sentinel address of the redis, default is the same as redis host if empty + 'port' => 26379, // phpredis sentinel port of the primary redis server, default 26379 if empty. + 'service' => 'myprimary', //, phpredis sentinel primary name, default myprimary + 'timeout' => 0, // phpredis sentinel connection timeout + 'persistent' => null, // phpredis sentinel persistence parameter + 'retry_interval' => 0, // phpredis sentinel retry interval + 'read_timeout' => 0, // phpredis sentinel read timeout + 'reconnect' => 0, // retries after losing connection to redis asking for a new primary, if -1 will retry indefinetely + 'username' => '', // phpredis sentinel auth username + 'password' => '', // phpredis sentinel auth password + 'ssl' => null, + ] ]; + // The following array contains all exception message parts which are interpreted as a connection loss or + // another unavailability of Redis. + private const ERROR_MESSAGES_INDICATING_UNAVAILABILITY = [ + 'connection closed', + 'connection refused', + 'connection lost', + 'failed while reconnecting', + 'is loading the dataset in memory', + 'php_network_getaddresses', + 'read error on connection', + 'socket', + 'went away', + 'loading', + 'readonly', + "can't write against a read only replica", + ]; /** * @var string */ @@ -47,6 +77,11 @@ class Redis implements Adapter */ private $redis; + /** + * @var RedisSentinel + */ + private $sentinel = null; + /** * @var boolean */ @@ -58,17 +93,49 @@ class Redis implements Adapter */ public function __construct(array $options = []) { - $this->options = array_merge(self::$defaultOptions, $options); + $this->options = [...self::$defaultOptions, ...$options]; + $this->options['sentinel'] = [...self::$defaultOptions['sentinel'] ?? [], ...$options['sentinel'] ?? []]; $this->redis = new \Redis(); + if (boolval($this->options['sentinel']['enable'])) { + $options['sentinel']['host'] = $options['sentinel']['host'] ?? $options['host']; + $this->sentinel = new RedisSentinel($options['sentinel']); + } + } + + /** + * Sentinels discoverMaster + * @return void + */ + public function updateSentinelPrimary(): void + { + $master = $this->sentinel->getMaster(); + + if (is_array($master)) { + $this->options['host'] = $master['ip']; + $this->options['port'] = $master['port']; + } + } + + /** + * @return \RedisSentinel + */ + public function getRedisSentinel(): \RedisSentinel + { + return $this->sentinel->getSentinel(); } /** * @param \Redis $redis + * @param \RedisSentinel $redisSentinel * @return self * @throws StorageException */ - public static function fromExistingConnection(\Redis $redis): self + public static function fromExistingConnection(\Redis $redis, ?\RedisSentinel $redisSentinel = null): self { + if (isset($redisSentinel)) { + RedisSentinel::fromExistingConnection($redisSentinel); + } + if ($redis->isConnected() === false) { throw new StorageException('Connection to Redis server not established'); } @@ -97,8 +164,8 @@ public static function setPrefix(string $prefix): void } /** - * @deprecated use replacement method wipeStorage from Adapter interface * @throws StorageException + * @deprecated use replacement method wipeStorage from Adapter interface */ public function flushRedis(): void { @@ -127,7 +194,7 @@ public function wipeStorage(): void <<getMessage()); + + // Because we also match only partial exception messages, we cannot use in_array() at this point. + foreach (self::ERROR_MESSAGES_INDICATING_UNAVAILABILITY as $errorMessage) { + if (str_contains($exceptionMessage, $errorMessage)) { + // Here we reconnect through Redis Sentinel if we lost connection to the server or if another unavailability occurred. + // We may actually reconnect to the same, broken server. But after a failover occured, we should be ok. + // It may take a moment until the Sentinel returns the new master, so this may be triggered multiple times. + return true; + } + } + return false; + } + /** * @throws StorageException */ @@ -196,7 +284,30 @@ private function ensureOpenConnection(): void return; } - $this->connectToServer(); + if ($this->sentinel !== null && boolval($this->options['sentinel']['enable'])) { + $reconnect = $this->options['sentinel']['reconnect']; + $retries = 0; + while ($retries <= $reconnect) { + try { + $this->updateSentinelPrimary(); + $this->connectToServer(); + break; + } catch (\RedisException $e) { + $retry = $this->reconnectIfRedisIsUnavailableOrReadonly($e); + if (!$retry) { + throw new StorageException( + sprintf("Can't connect to Redis server. %s", $e->getMessage()), + $e->getCode(), + $e + ); + } + } + $retries++; + } + } else { + $this->connectToServer(); + } + $authParams = []; if (isset($this->options['user']) && $this->options['user'] !== '') { @@ -225,28 +336,27 @@ private function ensureOpenConnection(): void */ private function connectToServer(): void { - try { - $connection_successful = false; - if ($this->options['persistent_connections'] !== false) { - $connection_successful = $this->redis->pconnect( - $this->options['host'], - (int) $this->options['port'], - (float) $this->options['timeout'] - ); - } else { + $connection_successful = false; + if ($this->options['persistent_connections'] !== false) { + $connection_successful = $this->redis->pconnect( + $this->options['host'], + (int) $this->options['port'], + (float) $this->options['timeout'] + ); + } else { + try { $connection_successful = $this->redis->connect($this->options['host'], (int) $this->options['port'], (float) $this->options['timeout']); - } - if (!$connection_successful) { + } catch (\RedisException $ex) { throw new StorageException( - sprintf("Can't connect to Redis server. %s", $this->redis->getLastError()), - 0 + sprintf("Can't connect to Redis server. %s", $ex->getMessage()), + $ex->getCode() ); } - } catch (\RedisException $e) { + } + if (!$connection_successful) { throw new StorageException( - sprintf("Can't connect to Redis server. %s", $e->getMessage()), - $e->getCode(), - $e + sprintf("Can't connect to Redis server. %s", $this->redis->getLastError()), + 0 ); } } diff --git a/src/Prometheus/Storage/RedisSentinel.php b/src/Prometheus/Storage/RedisSentinel.php new file mode 100644 index 00000000..b1a957b4 --- /dev/null +++ b/src/Prometheus/Storage/RedisSentinel.php @@ -0,0 +1,159 @@ + false, // if enabled uses sentinel to get the master before connecting to redis + 'host' => '127.0.0.1', // phpredis sentinel address of the redis + 'port' => 26379, // phpredis sentinel port of the primary redis server, default 26379 if empty. + 'service' => 'myprimary', //, phpredis sentinel primary name, default myprimary + 'timeout' => 0, // phpredis sentinel connection timeout + 'persistent' => null, // phpredis sentinel persistence parameter + 'retry_interval' => 0, // phpredis sentinel retry interval + 'read_timeout' => 0, // phpredis sentinel read timeout + 'reconnect' => 0, // retries after losing connection to redis asking for a new primary, if -1 will retry indefinetely + 'username' => '', // phpredis sentinel auth username + 'password' => '', // phpredis sentinel auth password + 'ssl' => null, + ]; + + /** + * @param \RedisSentinel $redisSentinel + * @return self + * @throws StorageException + */ + public static function fromExistingConnection(\RedisSentinel $redisSentinel): self + { + $sentinel = new self(); + $sentinel->sentinel = $redisSentinel; + $sentinel->getMaster(); + return $sentinel; + } + + /** + * Redis constructor. + * @param mixed[] $options + */ + public function __construct(array $options = []) + { + $this->options = [...self::$defaultOptions, ...$options]; + $this->sentinel = $this->initSentinel(); + } + + /** + * {@inheritdoc} + * @return mixed[]|bool + * @throws StorageException|\RedisException + */ + public function getMaster(): array|bool + { + $service = $this->options['service']; + + try { + $master = $this->sentinel->master($service); + } catch (\RedisException $e) { + throw new StorageException( + sprintf("Can't connect to RedisSentinel server. %s", $e->getMessage()), + $e->getCode(), + $e + ); + } + + if (! $this->isValidMaster($master)) { + throw new StorageException(sprintf("No master found for service '%s'.", $service)); + } + + return $master; + } + + /** + * Check whether master is valid or not. + * @param mixed[]|bool $master + * @return bool + */ + protected function isValidMaster(array|bool $master): bool + { + return is_array($master) && isset($master['ip']) && isset($master['port']); + } + + /** + * Connect to the configured Redis Sentinel instance. + * @return \RedisSentinel + * @throws StorageException + */ + private function initSentinel(): \RedisSentinel + { + $host = $this->options['host'] ?? ''; + $port = $this->options['port'] ?? 26379; + $timeout = $this->options['timeout'] ?? 0.2; + $persistent = $this->options['persistent'] ?? null; + $retryInterval = $this->options['retry_interval'] ?? 0; + $readTimeout = $this->options['read_timeout'] ?? 0; + $username = $this->options['username'] ?? ''; + $password = $this->options['password'] ?? ''; + $ssl = $this->options['ssl'] ?? null; + + if (strlen(trim($host)) === 0) { + throw new StorageException('No host has been specified for the Redis Sentinel connection.'); + } + + $auth = null; + if (strlen(trim($username)) !== 0 && strlen(trim($password)) !== 0) { + $auth = [$username, $password]; + } elseif (strlen(trim($password)) !== 0) { + $auth = $password; + } + + if (version_compare((string)phpversion('redis'), '6.0', '>=')) { + $options = [ + 'host' => $host, + 'port' => $port, + 'connectTimeout' => $timeout, + 'persistent' => $persistent, + 'retryInterval' => $retryInterval, + 'readTimeout' => $readTimeout, + ]; + + if ($auth !== null) { + $options['auth'] = $auth; + } + + if (version_compare((string)phpversion('redis'), '6.1', '>=') && $ssl !== null) { + $options['ssl'] = $ssl; + } + // @phpstan-ignore arguments.count, argument.type + return new \RedisSentinel($options); + } + + if ($auth !== null) { + /** + * @phpstan-ignore arguments.count + **/ + return new \RedisSentinel($host, $port, $timeout, $persistent, $retryInterval, $readTimeout, $auth); + } + return new \RedisSentinel($host, $port, $timeout, $persistent, $retryInterval, $readTimeout); + } + + public function getSentinel(): \RedisSentinel + { + return $this->sentinel; + } +} diff --git a/tests/Test/Prometheus/Storage/RedisSentinelTest.php b/tests/Test/Prometheus/Storage/RedisSentinelTest.php new file mode 100644 index 00000000..97a1af71 --- /dev/null +++ b/tests/Test/Prometheus/Storage/RedisSentinelTest.php @@ -0,0 +1,101 @@ +redisConnection = new \Redis(); + $this->redisConnection->connect(REDIS_HOST); + $this->redisConnection->flushAll(); + } + + /** + * @test + */ + public function itShouldThrowAnExceptionOnConnectionFailureWithRedisSentinelNotEnabled(): void + { + $redis = new Redis(['host' => '/dev/null', 'sentinel' => ['host' => '/dev/null']]); + + $this->expectException(StorageException::class); + $this->expectExceptionMessage("Can't connect to Redis server"); + + $redis->collect(); + $redis->wipeStorage(); + } + + /** + * @test + */ + public function itShouldThrowAnExceptionOnConnectionFailure(): void + { + $redis = new Redis(['host' => '/dev/null', 'sentinel' => ['host' => '/dev/null', 'enable' => true]]); + + $this->expectException(StorageException::class); + $this->expectExceptionMessage("Can't connect to RedisSentinel server"); + + $redis->collect(); + $redis->wipeStorage(); + } + + /** + * @test + */ + public function itShouldThrowExceptionWhenInjectedRedisIsNotConnected(): void + { + $connection = new \Redis(); + // @phpstan-ignore arguments.count + + $sentinel = version_compare((string)phpversion('redis'), '6.0', '>=') ? + new \RedisSentinel(['host' => '/dev/null']) : + new \RedisSentinel('/dev/null'); + + self::expectException(StorageException::class); + self::expectExceptionMessageMatches("/Can't connect to RedisSentinel server\\..*/"); + + Redis::fromExistingConnection($connection, $sentinel); + } + + /** + * @test + */ + public function itShouldThrowAnExceptionOnPrimaryFailure(): void + { + $redis = new Redis(['host' => '/dev/null', 'sentinel' => ['host' => '/dev/null', 'enable' => true, 'service' => 'dummy']]); + + $this->expectException(StorageException::class); + $this->expectExceptionMessage("Can't connect to RedisSentinel server"); + + $redis->collect(); + $redis->wipeStorage(); + } + + /** + * @test + */ + public function itShouldGetMaster(): void + { + $redis = new Redis(['host' => REDIS_HOST, + 'sentinel' => ['host' => REDIS_SENTINEL_HOST, 'enable' => true, 'service' => 'myprimary'] + ]); + + $redis->collect(); + $redis->wipeStorage(); + $this->expectNotToPerformAssertions(); + } +} diff --git a/tests/bootstrap.php b/tests/bootstrap.php index 12456d29..3523cbdb 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -17,3 +17,4 @@ $loader->add('Test\\Performance', __DIR__); define('REDIS_HOST', isset($_ENV['REDIS_HOST']) ? $_ENV['REDIS_HOST'] : '127.0.0.1'); +define('REDIS_SENTINEL_HOST', isset($_ENV['REDIS_SENTINEL_HOST']) ? $_ENV['REDIS_SENTINEL_HOST'] : '127.0.0.1');