Merge branch 'master' into fix-ssl-verification

Author: petyaslavova

.github/actions/run-tests/action.yml

@@ -14,6 +14,10 @@ inputs:
     description: 'hiredis version to test against'
     required: false
     default: '>3.0.0'
+  hiredis-branch:
+    description: 'hiredis branch to test against'
+    required: false
+    default: 'master'
   event-loop:
     description: 'Event loop to use'
     required: false
@@ -28,6 +32,14 @@ runs:
         python-version: ${{ inputs.python-version }}
         cache: 'pip'
 
+    - uses: actions/checkout@v4
+      if: ${{ inputs.parser-backend == 'hiredis' && inputs.hiredis-version == 'unstable' }}
+      with:
+        repository: redis/hiredis-py
+        submodules: true
+        path: hiredis-py
+        ref: ${{ inputs.hiredis-branch }}
+
     - name: Setup Test environment
       env:
         REDIS_VERSION: ${{ inputs.redis-version }}
@@ -40,15 +52,21 @@ runs:
         pip uninstall -y redis  # uninstall Redis package installed via redis-entraid
         pip install -e .[jwt]  # install the working copy
         if [ "${{inputs.parser-backend}}" == "hiredis" ]; then
-          pip install "hiredis${{inputs.hiredis-version}}"
-          echo "PARSER_BACKEND=$(echo "${{inputs.parser-backend}}_${{inputs.hiredis-version}}" | sed 's/[^a-zA-Z0-9]/_/g')" >> $GITHUB_ENV        
+          if [[ "${{inputs.hiredis-version}}" == "unstable" ]]; then
+            echo "Installing unstable version of hiredis from local directory"
+            pip install -e ./hiredis-py
+          else
+            pip install "hiredis${{inputs.hiredis-version}}"
+          fi          
+          echo "PARSER_BACKEND=$(echo "${{inputs.parser-backend}}_${{inputs.hiredis-version}}" | sed 's/[^a-zA-Z0-9]/_/g')" >> $GITHUB_ENV
         else
           echo "PARSER_BACKEND=${{inputs.parser-backend}}" >> $GITHUB_ENV
         fi
         echo "::endgroup::"
         
         echo "::group::Starting Redis servers"
         redis_major_version=$(echo "$REDIS_VERSION" | grep -oP '^\d+')
+        echo "REDIS_MAJOR_VERSION=${redis_major_version}" >> $GITHUB_ENV
         
         if (( redis_major_version < 8 )); then
           echo "Using redis-stack for module tests"
@@ -70,8 +88,7 @@ runs:
                   
           if (( redis_major_version < 7 )); then
             export REDIS_STACK_EXTRA_ARGS="--tls-auth-clients optional --save ''"
-            export REDIS_EXTRA_ARGS="--tls-auth-clients optional --save ''"
-            echo "REDIS_MAJOR_VERSION=${redis_major_version}" >> $GITHUB_ENV
+            export REDIS_EXTRA_ARGS="--tls-auth-clients optional --save ''"            
           fi
         
           invoke devenv --endpoints=all-stack
@@ -108,12 +125,10 @@ runs:
           fi
 
           echo "::endgroup::"
-    
-          if [ "$protocol" == "2" ] || [ "${{inputs.parser-backend}}" != 'hiredis' ]; then
-            echo "::group::RESP${protocol} cluster tests"
-            invoke cluster-tests $eventloop --protocol=${protocol}
-            echo "::endgroup::"
-          fi
+            
+          echo "::group::RESP${protocol} cluster tests"
+          invoke cluster-tests $eventloop --protocol=${protocol}
+          echo "::endgroup::"          
         }
 
         run_tests 2 "${{inputs.event-loop}}"

.github/wordlist.txt

@@ -2,6 +2,7 @@ APM
 ARGV
 BFCommands
 CacheImpl
+CAS
 CFCommands
 CMSCommands
 ClusterNode

.github/workflows/hiredis-py-integration.yaml

@@ -0,0 +1,66 @@
+name: Hiredis-py integration tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      redis-py-branch:
+        description: 'redis-py branch to run tests on'
+        required: true
+        default: 'master'
+      hiredis-branch:
+        description: 'hiredis-py branch to run tests on'
+        required: true
+        default: 'master'
+
+concurrency:
+  group: ${{ github.event.pull_request.number || github.ref }}-hiredis-integration
+  cancel-in-progress: true
+
+permissions:
+  contents: read  #  to fetch code (actions/checkout)
+
+env:
+  CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+  # this speeds up coverage with Python 3.12: https://github.com/nedbat/coveragepy/issues/1665
+  COVERAGE_CORE: sysmon
+  CURRENT_CLIENT_LIBS_TEST_STACK_IMAGE_TAG: 'rs-7.4.0-v2'
+  CURRENT_REDIS_VERSION: '7.4.2'
+
+jobs:
+  redis_version:
+    runs-on: ubuntu-latest
+    outputs:
+      CURRENT: ${{ env.CURRENT_REDIS_VERSION }}
+    steps:
+      - name: Compute outputs
+        run: |
+          echo "CURRENT=${{ env.CURRENT_REDIS_VERSION }}" >> $GITHUB_OUTPUT  
+
+  hiredis-tests:
+    runs-on: ubuntu-latest
+    needs: [redis_version]
+    timeout-minutes: 60
+    strategy:
+      max-parallel: 15
+      fail-fast: false
+      matrix:
+        redis-version: [ '${{ needs.redis_version.outputs.CURRENT }}' ]
+        python-version: [ '3.8', '3.13']
+        parser-backend: [ 'hiredis' ]
+        hiredis-version: [ 'unstable' ]
+        event-loop: [ 'asyncio' ]
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+    name: Redis ${{ matrix.redis-version }}; Python ${{ matrix.python-version }}; RESP Parser:${{matrix.parser-backend}} (${{ matrix.hiredis-version }}); EL:${{matrix.event-loop}}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.redis-py-branch }}
+      - name: Run tests
+        uses: ./.github/actions/run-tests
+        with:
+          python-version: ${{ matrix.python-version }}
+          parser-backend: ${{ matrix.parser-backend }}
+          redis-version: ${{ matrix.redis-version }}
+          hiredis-version: ${{ matrix.hiredis-version }}
+          hiredis-branch: ${{ inputs.hiredis-branch }}

.github/workflows/integration.yaml

@@ -74,7 +74,7 @@ jobs:
       max-parallel: 15
       fail-fast: false
       matrix:
-        redis-version: ['8.0-RC2-pre', '${{ needs.redis_version.outputs.CURRENT }}', '7.2.7', '6.2.17']
+        redis-version: ['8.0.1-pre', '${{ needs.redis_version.outputs.CURRENT }}', '7.2.7', '6.2.17']
         python-version: ['3.8', '3.13']
         parser-backend: ['plain']
         event-loop: ['asyncio']

.gitignore

@@ -9,6 +9,7 @@ vagrant/.vagrant
 .cache
 .eggs
 .idea
+.vscode
 .coverage
 env
 venv

CHANGES

@@ -1,3 +1,4 @@
+    * Support transactions in ClusterPipeline
     * Removing support for RedisGraph module. RedisGraph support is deprecated since Redis Stack 7.2 (https://redis.com/blog/redisgraph-eol/)
     * Fix lock.extend() typedef to accept float TTL extension
     * Update URL in the readme linking to Redis University
@@ -71,6 +72,7 @@
     * Close SSL sockets if the connection attempt fails, or if validations fail. (#3317)
     * Eliminate mutable default arguments in the `redis.commands.core.Script` class. (#3332)
     * Fix SSL verification with `ssl_cert_reqs="none"` and `ssl_check_hostname=True` by automatically setting `check_hostname=False` when `verify_mode=ssl.CERT_NONE` (#3635) 
+    * Allow newer versions of PyJWT as dependency. (#3630)
 
 * 4.1.3 (Feb 8, 2022)
   * Fix flushdb and flushall (#1926)

docs/advanced_features.rst

@@ -167,6 +167,7 @@ the server.
 
 .. code:: python
 
+   >>> rc = RedisCluster()
    >>> with rc.pipeline() as pipe:
    ...     pipe.set('foo', 'value1')
    ...     pipe.set('bar', 'value2')
@@ -177,20 +178,110 @@ the server.
    ...     pipe.set('foo1', 'bar1').get('foo1').execute()
    [True, b'bar1']
 
-Please note: - RedisCluster pipelines currently only support key-based
-commands. - The pipeline gets its ‘read_from_replicas’ value from the
-cluster’s parameter. Thus, if read from replications is enabled in the
-cluster instance, the pipeline will also direct read commands to
-replicas. - The ‘transaction’ option is NOT supported in cluster-mode.
-In non-cluster mode, the ‘transaction’ option is available when
-executing pipelines. This wraps the pipeline commands with MULTI/EXEC
-commands, and effectively turns the pipeline commands into a single
-transaction block. This means that all commands are executed
-sequentially without any interruptions from other clients. However, in
-cluster-mode this is not possible, because commands are partitioned
-according to their respective destination nodes. This means that we can
-not turn the pipeline commands into one transaction block, because in
-most cases they are split up into several smaller pipelines.
+Please note:
+
+-  RedisCluster pipelines currently only support key-based commands.
+-  The pipeline gets its ‘load_balancing_strategy’ value from the
+   cluster’s parameter. Thus, if read from replications is enabled in
+   the cluster instance, the pipeline will also direct read commands to
+   replicas.
+
+
+Transactions in clusters
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Transactions are supported in cluster-mode with one caveat: all keys of
+all commands issued on a transaction pipeline must reside on the
+same slot. This is similar to the limitation of multikey commands in
+cluster. The reason behind this is that the Redis engine does not offer
+a mechanism to block or exchange key data across nodes on the fly. A
+client may add some logic to abstract engine limitations when running
+on a cluster, such as the pipeline behavior explained on the previous
+block, but there is no simple way that a client can enforce atomicity
+across nodes on a distributed system.
+
+The compromise of limiting the transaction pipeline to same-slot keys
+is exactly that: a compromise. While this behavior is different from
+non-transactional cluster pipelines, it simplifies migration of clients
+from standalone to cluster under some circumstances. Note that application
+code that issues multi/exec commands on a standalone client without
+embedding them within a pipeline would eventually get ‘AttributeError’.
+With this approach, if the application uses ‘client.pipeline(transaction=True)’,
+then switching the client with a cluster-aware instance would simplify
+code changes (to some extent). This may be true for application code that
+makes use of hash keys, since its transactions may already be
+mapping all commands to the same slot.
+
+An alternative is some kind of two-step commit solution, where a slot
+validation is run before the actual commands are run. This could work
+with controlled node maintenance but does not cover single node failures.
+
+Given the cluster limitations for transactions, by default pipeline isn't in
+transactional mode. To enable transactional context set:
+
+.. code:: python
+
+   >>> p = rc.pipeline(transaction=True)
+
+After entering the transactional context you can add commands to a transactional
+context, by one of the following ways:
+
+.. code:: python
+
+   >>> p = rc.pipeline(transaction=True) # Chaining commands
+   >>> p.set("key", "value")
+   >>> p.get("key")
+   >>> response = p.execute()
+
+Or
+
+.. code:: python
+
+   >>> with rc.pipeline(transaction=True) as pipe: # Using context manager
+   ...     pipe.set("key", "value")
+   ...     pipe.get("key")
+   ...     response = pipe.execute()
+
+As you see there's no need to explicitly send `MULTI/EXEC` commands to control context start/end
+`ClusterPipeline` will take care of it.
+
+To ensure that different keys will be mapped to a same hash slot on the server side
+prepend your keys with the same hash tag, the technique that allows you to control
+keys distribution.
+More information `here <https://redis.io/docs/latest/operate/oss_and_stack/reference/cluster-spec/#hash-tags>`_
+
+.. code:: python
+
+   >>> with rc.pipeline(transaction=True) as pipe:
+   ...     pipe.set("{tag}foo", "bar")
+   ...     pipe.set("{tag}bar", "foo")
+   ...     pipe.get("{tag}foo")
+   ...     pipe.get("{tag}bar")
+   ...     response = pipe.execute()
+
+CAS Transactions
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you want to apply optimistic locking for certain keys, you have to execute
+`WATCH` command in transactional context. `WATCH` command follows the same limitations
+as any other multi key command - all keys should be mapped to the same hash slot.
+
+However, the difference between CAS transaction and normal one is that you have to
+explicitly call MULTI command to indicate the start of transactional context, WATCH
+command itself and any subsequent commands before MULTI will be immediately executed
+on the server side so you can apply optimistic locking and get necessary data before
+transaction execution.
+
+.. code:: python
+
+   >>> with rc.pipeline(transaction=True) as pipe:
+   ...     pipe.watch("mykey")       # Apply locking by immediately executing command
+   ...     val = pipe.get("mykey")   # Immediately retrieves value
+   ...     val = val + 1             # Increment value
+   ...     pipe.multi()              # Starting transaction context
+   ...     pipe.set("mykey", val)    # Command will be pipelined
+   ...     response = pipe.execute() # Returns OK or None if key was modified in the meantime
+
 
 Publish / Subscribe
 -------------------

pyproject.toml

@@ -41,7 +41,7 @@ ocsp = [
     "requests>=2.31.0",
 ]
 jwt = [
-    "PyJWT~=2.9.0",
+    "PyJWT>=2.9.0",
 ]
 
 [project.urls]

redis/__init__.py

@@ -16,11 +16,14 @@
     BusyLoadingError,
     ChildDeadlockedError,
     ConnectionError,
+    CrossSlotTransactionError,
     DataError,
+    InvalidPipelineStack,
     InvalidResponse,
     OutOfMemoryError,
     PubSubError,
     ReadOnlyError,
+    RedisClusterException,
     RedisError,
     ResponseError,
     TimeoutError,
@@ -56,15 +59,18 @@ def int_or_str(value):
     "ConnectionError",
     "ConnectionPool",
     "CredentialProvider",
+    "CrossSlotTransactionError",
     "DataError",
     "from_url",
     "default_backoff",
+    "InvalidPipelineStack",
     "InvalidResponse",
     "OutOfMemoryError",
     "PubSubError",
     "ReadOnlyError",
     "Redis",
     "RedisCluster",
+    "RedisClusterException",
     "RedisError",
     "ResponseError",
     "Sentinel",

redis/asyncio/cluster.py

@@ -1313,7 +1313,9 @@ async def initialize(self) -> None:
         startup_nodes_reachable = False
         fully_covered = False
         exception = None
-        for startup_node in self.startup_nodes.values():
+        # Convert to tuple to prevent RuntimeError if self.startup_nodes
+        # is modified during iteration
+        for startup_node in tuple(self.startup_nodes.values()):
             try:
                 # Make sure cluster mode is enabled on this node
                 try: