|
30 | 30 | import requests
|
31 | 31 | import numpy
|
32 | 32 | import json
|
| 33 | +import time |
33 | 34 | import struct
|
34 | 35 | import ctypes
|
35 | 36 | import logging
|
@@ -134,14 +135,17 @@ def __getserv():
|
134 | 135 | #
|
135 | 136 | def __errserv(serv):
|
136 | 137 | global server_table, server_max_errors
|
| 138 | + wait_time = 0 |
137 | 139 | with server_lock:
|
138 | 140 | try:
|
139 | 141 | server_table[serv] += 1
|
140 | 142 | logger.critical(serv + " encountered consecutive error " + str(server_table[serv]))
|
141 | 143 | if server_table[serv] > server_max_errors:
|
142 | 144 | server_table.pop(serv, None)
|
| 145 | + wait_time = server_table[serv] |
143 | 146 | except Exception as e:
|
144 | 147 | logger.debug(serv + " already removed from table")
|
| 148 | + time.sleep(wait_time) |
145 | 149 |
|
146 | 150 | #
|
147 | 151 | # __clrserv
|
@@ -338,14 +342,18 @@ def source (api, parm={}, stream=False, callbacks={'eventrec': __logeventrec}):
|
338 | 342 | rsps = requests.get(url, data=rqst, timeout=request_timeout).json()
|
339 | 343 | else:
|
340 | 344 | data = requests.post(url, data=rqst, timeout=request_timeout, stream=True)
|
| 345 | + data.raise_for_status() |
341 | 346 | rsps = __parse(data, callbacks)
|
342 | 347 | __clrserv(serv)
|
343 | 348 | complete = True
|
344 | 349 | except requests.ConnectionError as e:
|
345 | 350 | logger.error("Failed to connect to endpoint {} ... retrying request".format(url))
|
346 | 351 | __errserv(serv)
|
347 | 352 | except requests.HTTPError as e:
|
348 |
| - logger.error("Invalid HTTP response from endpoint {} ... retrying request".format(url)) |
| 353 | + if e.response.status_code == 503: |
| 354 | + logger.error("Server experiencing heavy load, stalling on request to {} ... will retry".format(url)) |
| 355 | + else: |
| 356 | + logger.error("Invalid HTTP response from endpoint {} ... retrying request".format(url)) |
349 | 357 | __errserv(serv)
|
350 | 358 | except requests.Timeout as e:
|
351 | 359 | logger.error("Timed-out waiting for response from endpoint {} ... retrying request".format(url))
|
|
0 commit comments