File tree Expand file tree Collapse file tree 1 file changed +5
-7
lines changed
src/servicex_did_finder_lib Expand file tree Collapse file tree 1 file changed +5
-7
lines changed Original file line number Diff line number Diff line change 32
32
33
33
34
34
MAX_RETRIES = 3
35
-
36
-
37
- def chunks (lst , n ):
38
- """Yield successive n-sized chunks from lst."""
39
- for i in range (0 , len (lst ), n ):
40
- yield lst [i :i + n ]
35
+ CH_LEN = 30
41
36
42
37
43
38
class ServiceXAdapter :
@@ -98,7 +93,10 @@ def put_file_add(self, file_info):
98
93
f'message: { str (file_info )} - Ignoring error.' )
99
94
100
95
def put_file_add_bulk (self , file_list ):
101
- for chunk in chunks (file_list , 30 ):
96
+ # we first chunk up file_list as it can be very large in
97
+ # case there are a lot of replicas and a lot of files.
98
+ chunks = [file_list [i :i + CH_LEN ] for i in range (0 , len (file_list ), CH_LEN )]
99
+ for chunk in chunks :
102
100
success = False
103
101
attempts = 0
104
102
mesg = []
You can’t perform that action at this time.
0 commit comments