Skip to content

Commit c47f0f0

Browse files
committed
worker scale factor reduced to 3; back off added to error handling; explicit handling of 503 errors
1 parent e60e838 commit c47f0f0

File tree

2 files changed

+10
-2
lines changed

2 files changed

+10
-2
lines changed

sliderule/icesat2.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
###############################################################################
4747

4848
# configuration
49-
SERVER_SCALE_FACTOR = 6
49+
SERVER_SCALE_FACTOR = 3
5050

5151
# create logger
5252
logger = logging.getLogger(__name__)

sliderule/sliderule.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import requests
3131
import numpy
3232
import json
33+
import time
3334
import struct
3435
import ctypes
3536
import logging
@@ -134,14 +135,17 @@ def __getserv():
134135
#
135136
def __errserv(serv):
136137
global server_table, server_max_errors
138+
wait_time = 0
137139
with server_lock:
138140
try:
139141
server_table[serv] += 1
140142
logger.critical(serv + " encountered consecutive error " + str(server_table[serv]))
141143
if server_table[serv] > server_max_errors:
142144
server_table.pop(serv, None)
145+
wait_time = server_table[serv]
143146
except Exception as e:
144147
logger.debug(serv + " already removed from table")
148+
time.sleep(wait_time)
145149

146150
#
147151
# __clrserv
@@ -338,14 +342,18 @@ def source (api, parm={}, stream=False, callbacks={'eventrec': __logeventrec}):
338342
rsps = requests.get(url, data=rqst, timeout=request_timeout).json()
339343
else:
340344
data = requests.post(url, data=rqst, timeout=request_timeout, stream=True)
345+
data.raise_for_status()
341346
rsps = __parse(data, callbacks)
342347
__clrserv(serv)
343348
complete = True
344349
except requests.ConnectionError as e:
345350
logger.error("Failed to connect to endpoint {} ... retrying request".format(url))
346351
__errserv(serv)
347352
except requests.HTTPError as e:
348-
logger.error("Invalid HTTP response from endpoint {} ... retrying request".format(url))
353+
if e.response.status_code == 503:
354+
logger.error("Server experiencing heavy load, stalling on request to {} ... will retry".format(url))
355+
else:
356+
logger.error("Invalid HTTP response from endpoint {} ... retrying request".format(url))
349357
__errserv(serv)
350358
except requests.Timeout as e:
351359
logger.error("Timed-out waiting for response from endpoint {} ... retrying request".format(url))

0 commit comments

Comments
 (0)