Skip to content

Commit 4dddef0

Browse files
anandswaminathandanielhochman
anandswaminathan
authored and
danielhochman
committed
\rate_limited_scan fix to handle consumed capacity (#235)
1 parent e5858ef commit 4dddef0

File tree

3 files changed

+20
-20
lines changed

3 files changed

+20
-20
lines changed

‎pynamodb/connection/base.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1019,7 +1019,7 @@ def rate_limited_scan(self,
10191019
if not limit:
10201020
return
10211021

1022-
latest_scan_consumed_capacity = data.get(CONSUMED_CAPACITY)
1022+
latest_scan_consumed_capacity = data.get(CONSUMED_CAPACITY).get(CAPACITY_UNITS)
10231023
last_evaluated_key = data.get(LAST_EVALUATED_KEY, None)
10241024
consecutive_provision_throughput_exceeded_ex = 0
10251025
except ScanError as e:

‎pynamodb/tests/test_base_connection.py

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1485,7 +1485,7 @@ def verify_scan_call_args(call_args,
14851485
self.assertEqual(call_args[1]['segment'], segment)
14861486

14871487
with patch(SCAN_METHOD_TO_PATCH) as req:
1488-
req.return_value = {'Items': [], 'ConsumedCapacity': 10 }
1488+
req.return_value = {'Items': [], 'ConsumedCapacity': {'TableName': table_name, 'CapacityUnits': 10.0}}
14891489
resp = conn.rate_limited_scan(
14901490
table_name
14911491
)
@@ -1494,7 +1494,7 @@ def verify_scan_call_args(call_args,
14941494
verify_scan_call_args(req.call_args, table_name)
14951495

14961496
with patch(SCAN_METHOD_TO_PATCH) as req:
1497-
req.return_value = {'Items': [], 'ConsumedCapacity': 10 }
1497+
req.return_value = {'Items': [], 'ConsumedCapacity': {'TableName': table_name, 'CapacityUnits': 10.0}}
14981498
resp = conn.rate_limited_scan(
14991499
table_name,
15001500
limit=10,
@@ -1511,7 +1511,7 @@ def verify_scan_call_args(call_args,
15111511
limit=10)
15121512

15131513
with patch(SCAN_METHOD_TO_PATCH) as req:
1514-
req.return_value = {'Items': [], 'ConsumedCapacity': 10 }
1514+
req.return_value = {'Items': [], 'ConsumedCapacity': {'TableName': table_name, 'CapacityUnits': 10.0}}
15151515
scan_filter = {
15161516
'ForumName': {
15171517
'AttributeValueList': [
@@ -1551,7 +1551,7 @@ def verify_scan_call_args(call_args,
15511551
conditional_operator='AND')
15521552

15531553
with patch(SCAN_METHOD_TO_PATCH) as req:
1554-
req.return_value = {'Items': [], 'ConsumedCapacity': 10 }
1554+
req.return_value = {'Items': [], 'ConsumedCapacity': {'TableName': table_name, 'CapacityUnits': 10.0}}
15551555
resp = conn.rate_limited_scan(
15561556
table_name,
15571557
page_size=5,
@@ -1565,7 +1565,7 @@ def verify_scan_call_args(call_args,
15651565
limit=5)
15661566

15671567
with patch(SCAN_METHOD_TO_PATCH) as req:
1568-
req.return_value = {'Items': [], 'ConsumedCapacity': 10 }
1568+
req.return_value = {'Items': [], 'ConsumedCapacity': {'TableName': table_name, 'CapacityUnits': 10.0}}
15691569
resp = conn.rate_limited_scan(
15701570
table_name,
15711571
limit=10,
@@ -1767,8 +1767,8 @@ def test_ratelimited_scan_with_pagination_ends(self, scan_mock, time_mock):
17671767
c = Connection()
17681768
time_mock.side_effect = [1, 10, 20, 30, 40]
17691769
scan_mock.side_effect = [
1770-
{'Items': ['Item-1'], 'ConsumedCapacity': 1, 'LastEvaluatedKey': 'XX' },
1771-
{'Items': ['Item-2'], 'ConsumedCapacity': 1 }
1770+
{'Items': ['Item-1'], 'ConsumedCapacity': {'TableName': 'Table_1', 'CapacityUnits': 1}, 'LastEvaluatedKey': 'XX' },
1771+
{'Items': ['Item-2'], 'ConsumedCapacity': {'TableName': 'Table_1', 'CapacityUnits': 1}}
17721772
]
17731773
resp = c.rate_limited_scan('Table_1')
17741774
values = list(resp)
@@ -1787,7 +1787,7 @@ def test_ratelimited_scan_retries_on_throttling(self, api_mock, sleep_mock, time
17871787

17881788
api_mock.side_effect = [
17891789
VerboseClientError(botocore_expected_format, 'operation_name', {}),
1790-
{'Items': ['Item-1', 'Item-2'], 'ConsumedCapacity': 40 }
1790+
{'Items': ['Item-1', 'Item-2'], 'ConsumedCapacity': {'TableName': 'Table_1', 'CapacityUnits': 40}}
17911791
]
17921792
resp = c.rate_limited_scan('Table_1')
17931793
values = list(resp)
@@ -1853,8 +1853,8 @@ def test_rate_limited_scan_retries_on_rate_unavailable(self, scan_mock, sleep_mo
18531853
sleep_mock.return_value = 1
18541854
time_mock.side_effect = [1, 4, 6, 12]
18551855
scan_mock.side_effect = [
1856-
{'Items': ['Item-1'], 'ConsumedCapacity': 80, 'LastEvaluatedKey': 'XX' },
1857-
{'Items': ['Item-2'], 'ConsumedCapacity': 41 }
1856+
{'Items': ['Item-1'], 'ConsumedCapacity': {'TableName': 'Table_1', 'CapacityUnits': 80}, 'LastEvaluatedKey': 'XX' },
1857+
{'Items': ['Item-2'], 'ConsumedCapacity': {'TableName': 'Table_1', 'CapacityUnits': 41}}
18581858
]
18591859
resp = c.rate_limited_scan('Table_1')
18601860
values = list(resp)
@@ -1873,8 +1873,8 @@ def test_rate_limited_scan_retries_on_rate_unavailable_within_s(self, scan_mock,
18731873
sleep_mock.return_value = 1
18741874
time_mock.side_effect = [1.0, 1.5, 4.0]
18751875
scan_mock.side_effect = [
1876-
{'Items': ['Item-1'], 'ConsumedCapacity': 10, 'LastEvaluatedKey': 'XX' },
1877-
{'Items': ['Item-2'], 'ConsumedCapacity': 11 }
1876+
{'Items': ['Item-1'], 'ConsumedCapacity': {'TableName': 'Table_1', 'CapacityUnits': 10}, 'LastEvaluatedKey': 'XX' },
1877+
{'Items': ['Item-2'], 'ConsumedCapacity': {'TableName': 'Table_1', 'CapacityUnits': 11}}
18781878
]
18791879
resp = c.rate_limited_scan('Table_1', read_capacity_to_consume_per_second=5)
18801880
values = list(resp)
@@ -1892,8 +1892,8 @@ def test_rate_limited_scan_retries_max_sleep(self, scan_mock, sleep_mock, time_m
18921892
sleep_mock.return_value = 1
18931893
time_mock.side_effect = [1.0, 1.5, 250, 350]
18941894
scan_mock.side_effect = [
1895-
{'Items': ['Item-1'], 'ConsumedCapacity': 1000, 'LastEvaluatedKey': 'XX' },
1896-
{'Items': ['Item-2'], 'ConsumedCapacity': 11 }
1895+
{'Items': ['Item-1'], 'ConsumedCapacity': {'TableName': 'Table_1', 'CapacityUnits': 1000}, 'LastEvaluatedKey': 'XX' },
1896+
{'Items': ['Item-2'], 'ConsumedCapacity': {'TableName': 'Table_1', 'CapacityUnits': 11}}
18971897
]
18981898
resp = c.rate_limited_scan(
18991899
'Table_1',
@@ -1915,8 +1915,8 @@ def test_rate_limited_scan_retries_min_sleep(self, scan_mock, sleep_mock, time_m
19151915
sleep_mock.return_value = 1
19161916
time_mock.side_effect = [1, 2, 3, 4]
19171917
scan_mock.side_effect = [
1918-
{'Items': ['Item-1'], 'ConsumedCapacity': 10, 'LastEvaluatedKey': 'XX' },
1919-
{'Items': ['Item-2'], 'ConsumedCapacity': 11 }
1918+
{'Items': ['Item-1'], 'ConsumedCapacity': {'TableName': 'Table_1', 'CapacityUnits': 10}, 'LastEvaluatedKey': 'XX' },
1919+
{'Items': ['Item-2'], 'ConsumedCapacity': {'TableName': 'Table_1', 'CapacityUnits': 11}}
19201920
]
19211921
resp = c.rate_limited_scan('Table_1', read_capacity_to_consume_per_second=8)
19221922
values = list(resp)
@@ -1934,8 +1934,8 @@ def test_rate_limited_scan_retries_timeout(self, scan_mock, sleep_mock, time_moc
19341934
sleep_mock.return_value = 1
19351935
time_mock.side_effect = [1, 20, 30, 40]
19361936
scan_mock.side_effect = [
1937-
{'Items': ['Item-1'], 'ConsumedCapacity': 1000, 'LastEvaluatedKey': 'XX' },
1938-
{'Items': ['Item-2'], 'ConsumedCapacity': 11 }
1937+
{'Items': ['Item-1'], 'ConsumedCapacity': {'TableName': 'Table_1', 'CapacityUnits': 1000}, 'LastEvaluatedKey': 'XX' },
1938+
{'Items': ['Item-2'], 'ConsumedCapacity': {'TableName': 'Table_1', 'CapacityUnits': 11}}
19391939
]
19401940
resp = c.rate_limited_scan(
19411941
'Table_1',

‎pynamodb/tests/test_model.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1973,7 +1973,7 @@ def test_rate_limited_scan(self):
19731973
item = copy.copy(GET_MODEL_ITEM_DATA.get(ITEM))
19741974
item['user_id'] = {STRING_SHORT: 'id-{0}'.format(idx)}
19751975
items.append(item)
1976-
req.return_value = {'Items': items}
1976+
req.return_value = {'Items': items, 'ConsumedCapacity': {'TableName': 'UserModel', 'CapacityUnits': 10}}
19771977
scan_result = UserModel.rate_limited_scan(
19781978
user_id__contains='tux',
19791979
zip_code__null=False,

0 commit comments

Comments
 (0)