2
2
3
3
# python std lib
4
4
import sys
5
+ import logging
5
6
6
7
# rediscluster imports
7
8
from .client import RedisCluster
15
16
from redis .exceptions import ConnectionError , RedisError , TimeoutError
16
17
from redis ._compat import imap , unicode
17
18
19
+ from gevent import monkey ; monkey .patch_all ()
20
+ import gevent
21
+
22
+ log = logging .getLogger (__name__ )
18
23
19
24
ERRORS_ALLOW_RETRY = (ConnectionError , TimeoutError , MovedError , AskError , TryAgainError )
20
25
@@ -174,71 +179,127 @@ def send_cluster_commands(self, stack, raise_on_error=True, allow_redirections=T
174
179
# If it fails the configured number of times then raise exception back to caller of this method
175
180
raise ClusterDownError ("CLUSTERDOWN error. Unable to rebuild the cluster" )
176
181
177
- def _send_cluster_commands (self , stack , raise_on_error = True , allow_redirections = True ):
178
- """
179
- Send a bunch of cluster commands to the redis cluster.
182
+ def _execute_node_commands (self , n ):
183
+ n .write ()
180
184
181
- `allow_redirections` If the pipeline should follow `ASK` & `MOVED` responses
182
- automatically. If set to false it will raise RedisClusterException.
183
- """
184
- # the first time sending the commands we send all of the commands that were queued up.
185
- # if we have to run through it again, we only retry the commands that failed.
186
- attempt = sorted (stack , key = lambda x : x .position )
185
+ n .read ()
187
186
188
- # build a list of node objects based on node names we need to
187
+ def _get_commands_by_node ( self , cmds ):
189
188
nodes = {}
189
+ proxy_node_by_master = {}
190
+ connection_by_node = {}
190
191
191
- # as we move through each command that still needs to be processed,
192
- # we figure out the slot number that command maps to, then from the slot determine the node.
193
- for c in attempt :
192
+ for c in cmds :
194
193
# refer to our internal node -> slot table that tells us where a given
195
194
# command should route to.
196
195
slot = self ._determine_slot (* c .args )
197
- node = self .connection_pool .get_node_by_slot (slot )
198
196
199
- # little hack to make sure the node name is populated. probably could clean this up.
200
- self .connection_pool .nodes .set_node_name (node )
197
+ master_node = self .connection_pool .get_node_by_slot (slot )
198
+
199
+ # for the same master_node, it should always get the same proxy_node to group
200
+ # as many commands as possible per node
201
+ if master_node ['name' ] in proxy_node_by_master :
202
+ node = proxy_node_by_master [master_node ['name' ]]
203
+ else :
204
+ # TODO: should determine if using replicas by if command is read only
205
+ node = self .connection_pool .get_node_by_slot (slot , self .read_from_replicas )
206
+ proxy_node_by_master [master_node ['name' ]] = node
207
+
208
+ # little hack to make sure the node name is populated. probably could clean this up.
209
+ self .connection_pool .nodes .set_node_name (node )
201
210
202
- # now that we know the name of the node ( it's just a string in the form of host:port )
203
- # we can build a list of commands for each node.
204
211
node_name = node ['name' ]
205
212
if node_name not in nodes :
206
- nodes [node_name ] = NodeCommands (self .parse_response , self .connection_pool .get_connection_by_node (node ))
213
+ if node_name in connection_by_node :
214
+ connection = connection_by_node [node_name ]
215
+ else :
216
+ connection = self .connection_pool .get_connection_by_node (node )
217
+ connection_by_node [node_name ] = connection
218
+ nodes [node_name ] = NodeCommands (self .parse_response , connection )
207
219
208
220
nodes [node_name ].append (c )
209
221
210
- # send the commands in sequence.
211
- # we write to all the open sockets for each node first, before reading anything
212
- # this allows us to flush all the requests out across the network essentially in parallel
213
- # so that we can read them all in parallel as they come back.
214
- # we dont' multiplex on the sockets as they come available, but that shouldn't make too much difference.
215
- node_commands = nodes .values ()
216
- for n in node_commands :
217
- n .write ()
218
-
219
- for n in node_commands :
220
- n .read ()
221
-
222
- # release all of the redis connections we allocated earlier back into the connection pool.
223
- # we used to do this step as part of a try/finally block, but it is really dangerous to
224
- # release connections back into the pool if for some reason the socket has data still left in it
225
- # from a previous operation. The write and read operations already have try/catch around them for
226
- # all known types of errors including connection and socket level errors.
227
- # So if we hit an exception, something really bad happened and putting any of
228
- # these connections back into the pool is a very bad idea.
229
- # the socket might have unread buffer still sitting in it, and then the
230
- # next time we read from it we pass the buffered result back from a previous
231
- # command and every single request after to that connection will always get
232
- # a mismatched result. (not just theoretical, I saw this happen on production x.x).
233
- for n in nodes .values ():
234
- self .connection_pool .release (n .connection )
222
+ return nodes , connection_by_node
223
+
224
+ def _execute_single_command (self , cmd ):
225
+ try :
226
+ # send each command individually like we do in the main client.
227
+ cmd .result = super (ClusterPipeline , self ).execute_command (* cmd .args , ** cmd .options )
228
+ except RedisError as e :
229
+ cmd .result = e
230
+
231
+ def _send_cluster_commands (self , stack , raise_on_error = True , allow_redirections = True ):
232
+ """
233
+ Send a bunch of cluster commands to the redis cluster.
234
+
235
+ `allow_redirections` If the pipeline should follow `ASK` & `MOVED` responses
236
+ automatically. If set to false it will raise RedisClusterException.
237
+ """
238
+ # the first time sending the commands we send all of the commands that were queued up.
239
+ # if we have to run through it again, we only retry the commands that failed.
240
+ cmds = sorted (stack , key = lambda x : x .position )
241
+
242
+ max_redirects = 5
243
+ cur_attempt = 0
244
+
245
+ while cur_attempt < max_redirects :
246
+
247
+ # build a list of node objects based on node names we need to
248
+ nodes , connection_by_node = self ._get_commands_by_node (cmds )
249
+
250
+ # send the commands in sequence.
251
+ # we write to all the open sockets for each node first, before reading anything
252
+ # this allows us to flush all the requests out across the network essentially in parallel
253
+ # so that we can read them all in parallel as they come back.
254
+ # we dont' multiplex on the sockets as they come available, but that shouldn't make too much difference.
255
+
256
+ # duke-cliff: I think it would still be faster if we use gevent to make the command in parallel
257
+ # the io is non-blocking, but serialization/deserialization will still be blocking previously
258
+ node_commands = nodes .values ()
259
+ events = []
260
+ for n in node_commands :
261
+ events .append (gevent .spawn (self ._execute_node_commands , n ))
262
+
263
+ gevent .joinall (events )
264
+
265
+ # release all of the redis connections we allocated earlier back into the connection pool.
266
+ # we used to do this step as part of a try/finally block, but it is really dangerous to
267
+ # release connections back into the pool if for some reason the socket has data still left in it
268
+ # from a previous operation. The write and read operations already have try/catch around them for
269
+ # all known types of errors including connection and socket level errors.
270
+ # So if we hit an exception, something really bad happened and putting any of
271
+ # these connections back into the pool is a very bad idea.
272
+ # the socket might have unread buffer still sitting in it, and then the
273
+ # next time we read from it we pass the buffered result back from a previous
274
+ # command and every single request after to that connection will always get
275
+ # a mismatched result. (not just theoretical, I saw this happen on production x.x).
276
+ for conn in connection_by_node .values ():
277
+ self .connection_pool .release (conn )
278
+
279
+ # will regroup moved commands and retry using pipeline(stacked commands)
280
+ # this would increase the pipeline performance a lot
281
+ moved_cmds = []
282
+ for c in cmds :
283
+ if isinstance (c .result , MovedError ):
284
+ e = c .result
285
+ node = self .connection_pool .nodes .get_node (e .host , e .port , server_type = 'master' )
286
+ self .connection_pool .nodes .move_slot_to_node (e .slot_id , node )
287
+
288
+ moved_cmds .append (c )
289
+
290
+ if moved_cmds :
291
+ cur_attempt += 1
292
+ cmds = sorted (moved_cmds , key = lambda x : x .position )
293
+ continue
294
+
295
+ break
235
296
236
297
# if the response isn't an exception it is a valid response from the node
237
298
# we're all done with that command, YAY!
238
299
# if we have more commands to attempt, we've run into problems.
239
300
# collect all the commands we are allowed to retry.
240
301
# (MOVED, ASK, or connection errors or timeout errors)
241
- attempt = sorted ([c for c in attempt if isinstance (c .result , ERRORS_ALLOW_RETRY )], key = lambda x : x .position )
302
+ attempt = sorted ([c for c in stack if isinstance (c .result , ERRORS_ALLOW_RETRY )], key = lambda x : x .position )
242
303
if attempt and allow_redirections :
243
304
# RETRY MAGIC HAPPENS HERE!
244
305
# send these remaing comamnds one at a time using `execute_command`
@@ -255,13 +316,19 @@ def _send_cluster_commands(self, stack, raise_on_error=True, allow_redirections=
255
316
# If a lot of commands have failed, we'll be setting the
256
317
# flag to rebuild the slots table from scratch. So MOVED errors should
257
318
# correct themselves fairly quickly.
319
+
320
+ # with the previous redirect retries, I could barely see the slow mode happening again
321
+ log .info ("pipeline in slow mode to execute failed commands: {}" .format ([c .result for c in attempt ]))
322
+
258
323
self .connection_pool .nodes .increment_reinitialize_counter (len (attempt ))
324
+
325
+ # even in the slow mode, we could use gevent to make things faster
326
+ events = []
259
327
for c in attempt :
260
- try :
261
- # send each command individually like we do in the main client.
262
- c .result = super (ClusterPipeline , self ).execute_command (* c .args , ** c .options )
263
- except RedisError as e :
264
- c .result = e
328
+ events .append (gevent .spawn (self ._execute_single_command , c ))
329
+
330
+ gevent .joinall (events )
331
+
265
332
266
333
# turn the response back into a simple flat array that corresponds
267
334
# to the sequence of commands issued in the stack in pipeline.execute()
0 commit comments