Merge pull request #60 from censys/logbook-bugfix

fix: fetch subsequent logbook pages correctly

Author: mbremerkamp

censys/asm/api.py

@@ -103,6 +103,7 @@ def _get_logbook_page(
         while not end_of_events:
             res = self._get(path, args=args)
             end_of_events = res["endOfEvents"]
+            args = {"cursor": res["nextCursor"]}
 
             for event in res["events"]:
                 yield event

censys/asm/events.py

@@ -40,7 +40,7 @@ def get_cursor(
 
     def get_events(self, cursor: Optional[str] = None) -> Generator[dict, None, None]:
         """
-        Requests a logbook cursor.
+        Requests logbook events from inception or from the provided cursor.
 
         Args:
             cursor (str, optional): Logbook cursor.

tests/asm/test_events.py

@@ -19,6 +19,7 @@
 EVENTS_RESOURCE_TYPE = "events"
 
 TEST_CURSOR = "eyJmaWx0ZXIiOnt9LCJzdGFydCI6MH0"
+TEST_NEXT_CURSOR = "eyJmaWx0ZXIiOnt9LCJzdGFydCI6MjA3MTJ9"
 TEST_START_DATE = "2020-10-29T19:26:34.371Z"
 TEST_START_ID = 20712
 
@@ -108,9 +109,12 @@ def test_get_all_events(self, mock):
         res = [event for event in events]
 
         self.assertEqual(RESOURCE_PAGING_RESULTS, res)
-        mock.assert_called_with(
+        mock.assert_any_call(
             EVENTS_URL, params={"cursor": None}, timeout=TEST_TIMEOUT
         )
+        mock.assert_any_call(
+            EVENTS_URL, params={"cursor": TEST_NEXT_CURSOR}, timeout=TEST_TIMEOUT
+        )
 
     @patch("censys.base.requests.Session.get")
     def test_get_events_with_cursor(self, mock):
@@ -119,9 +123,12 @@ def test_get_events_with_cursor(self, mock):
         res = [event for event in events]
 
         self.assertEqual(RESOURCE_PAGING_RESULTS, res)
-        mock.assert_called_with(
+        mock.assert_any_call(
             EVENTS_URL, params={"cursor": TEST_CURSOR}, timeout=TEST_TIMEOUT
         )
+        mock.assert_any_call(
+            EVENTS_URL, params={"cursor": TEST_NEXT_CURSOR}, timeout=TEST_TIMEOUT
+        )
 
 
 if __name__ == "__main__":

tests/asm/utils.py

@@ -40,7 +40,8 @@ def __init__(self, status_code, resource_type, page_number=1):
             resource_type: self.get_resource(),  # Return dummy resource generator
             "pageNumber": 1,
             "totalPages": 3,  # Default to 3 pages of resources
-            "cursor": "eyJmaWx0ZXIiOnt9LCJzdGFydCI6MjA3MTJ9",
+            "nextCursor": "eyJmaWx0ZXIiOnt9LCJzdGFydCI6MjA3MTJ9",
+            "cursor": "eyJmaWx0ZXIiOnt9LCJzdGFydCI6MH0",
             "endOfEvents": False,
         }