Don't retry 410, move event processing into reusable method

.github/workflows/pythonapp.yml

@@ -8,11 +8,11 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v1
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v1
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: 3.x
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

legistar/base.py

@@ -359,9 +359,10 @@ def pages(self, url, params=None, item_key=None):
             page_num += 1
 
     def accept_response(self, response, **kwargs):
-        '''
+        """
         This overrides a method that controls whether
         the scraper should retry on an error. We don't
-        want to retry if the API returns a 400
-        '''
-        return response.status_code < 401
+        want to retry if the API returns a 400, except for
+        410, which means the record no longer exists.
+        """
+        return response.status_code < 401 or response.status_code == 410

legistar/events.py

@@ -240,36 +240,44 @@ def api_events(self, since_datetime=None):
 
     def events(self, since_datetime=None):
         for api_event in self.api_events(since_datetime=since_datetime):
+            if event := self.event(api_event):
+                yield event
 
-            time_str = api_event['EventTime']
-            if not time_str:  # If we don't have an event time, skip it
-                continue
-
-            try:
-                # Start times are entered manually. Sometimes, they don't
-                # conform to this format. Log events with invalid start times,
-                # but don't interrupt the scrape for them.
-                start_time = time.strptime(time_str, self.time_string_format)
-            except ValueError:
-                event_url = '{0}/events/{1}'.format(self.BASE_URL, api_event['EventId'])
-                self.logger.error('API event has invalid start time "{0}": {1}'.format(time_str, event_url))
-                continue
+    def event(self, api_event):
+        time_str = api_event["EventTime"]
+        if not time_str:  # If we don't have an event time, skip it
+            return
+        try:
+            # Start times are entered manually. Sometimes, they don't
+            # conform to this format. Log  events with invalid start times,
+            # but don't interrupt the scrape for them.
+            start_time = time.strptime(time_str, self.time_string_format)
+        except ValueError:
+            event_url = "{0}/events/{1}".format(self.BASE_URL, api_event["EventId"])
+            self.logger.error(
+                'API event has invalid start time "{0}": {1}'.format(
+                    time_str, event_url
+                )
+            )
+            return
 
-            start = self.toTime(api_event['EventDate'])
-            api_event['start'] = start.replace(hour=start_time.tm_hour,
-                                               minute=start_time.tm_min)
+        start = self.toTime(api_event["EventDate"])
+        api_event["start"] = start.replace(
+            hour=start_time.tm_hour, minute=start_time.tm_min
+        )
 
-            api_event['status'] = self._event_status(api_event)
+        api_event["status"] = self._event_status(api_event)
 
-            web_event = self._get_web_event(api_event)
+        web_event = self._get_web_event(api_event)
 
-            if web_event:
-                yield api_event, web_event
+        if web_event:
+            return api_event, web_event
 
-            else:
-                event_url = '{0}/events/{1}'.format(self.BASE_URL, api_event['EventId'])
-                self.warning('API event could not be found in web interface: {0}'.format(event_url))
-                continue
+        else:
+            event_url = "{0}/events/{1}".format(self.BASE_URL, api_event["EventId"])
+            self.warning(
+                "API event could not be found in web interface: {0}".format(event_url)
+            )
 
     def agenda(self, event):
         agenda_url = (self.BASE_URL +