Merge pull request #125 from censys/adh/new-v2-endpoints

Add Support for new endpoints

Author: thehappydinoa

.flake8

@@ -2,7 +2,7 @@
 max-line-length = 88
 exclude = 
     docs,
-    venv,
+    .venv,
     # No need to traverse our git directory
     .git,
     # There's no value in checking cache directories

.github/workflows/pypi.yml

@@ -19,19 +19,13 @@ jobs:
         with:
           python-version: 3.7
 
-      - name: Test install
-        run: |
-          pip install -e .
-
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install setuptools wheel twine
+          python -m pip install --upgrade pip poetry
+          poetry install --no-dev
 
       - name: Build and publish
-        env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
         run: |
-          python setup.py sdist bdist_wheel --universal
-          twine upload dist/*
+          poetry build -n
+          poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }}
+          poetry publish

.github/workflows/python-ci.yml

@@ -20,31 +20,35 @@ jobs:
         with:
           python-version: "3.x"
 
-      - name: Cache pip
+      - name: Install poetry
+        run: |
+          python -m pip install --upgrade pip poetry
+          poetry config virtualenvs.in-project true
+
+      - name: Cache poetry
+        id: cached-poetry-dependencies
         uses: actions/cache@v2
         with:
           # This path is specific to Ubuntu
-          path: ~/.cache/pip
+          path: .venv
           # Look to see if there is a cache hit for the corresponding requirements file
-          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-            ${{ runner.os }}-
+          key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
 
       - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: |
-          python -m pip install --upgrade pip
-          pip install ".[dev]"
+          poetry install
 
       - name: Lint with flake8
+        if: startsWith(matrix.os, 'ubuntu')
         run: |
-          # The GitHub editor is 127 chars wide
-          flake8 --max-line-length=127
+          poetry run flake8 --max-line-length=127
 
       - name: Type checking with mypy
+        if: startsWith(matrix.os, 'ubuntu')
         run: |
-          mypy censys
+          poetry run mypy censys
 
       - name: Test with pytest
         run: |
-          pytest
+          poetry run pytest

.readthedocs.yml

@@ -11,13 +11,22 @@ sphinx:
 
 # Optionally build your docs in additional formats such as PDF
 formats:
-  - epub
   - pdf
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
-  version: 3.7
+  version: "3.8"
   install:
     - requirements: docs/requirements.txt
     - method: pip
       path: .
+
+# Set a custom search rank over pages matching a pattern
+search:
+  ranking:
+    censys.asm.html: 5
+    usage-asm.html: 5
+    censys.search.v2.html: 5
+    usage-v2.html: 5
+    censys.search.v1.html: -1
+    usage-v1.html: -1

MANIFEST.in

@@ -73,10 +73,12 @@ All contributions (no matter how small) are always welcome.
 
 ## Development
 
+This project uses [poetry](https://python-poetry.org/) for dependency management. Please ensure you have [installed the latest version](https://python-poetry.org/docs/#installation).
+
 ```bash
 git clone git@github.com:censys/censys-python.git
 cd censys-python/
-pip install -e ".[dev]"
+poetry install
 ```
 
 ## Testing

README.md

@@ -0,0 +1,5 @@
+"""Common types for the Censys Python SDK."""
+import datetime
+from typing import Union
+
+Datetime = Union[str, datetime.date, datetime.datetime]

censys/common/types.py

@@ -0,0 +1,19 @@
+"""Common utilities for the Censys Python SDK."""
+
+import datetime
+
+from .types import Datetime
+
+
+def format_rfc3339(time: Datetime) -> str:
+    """Formats a datetime object into an RFC3339 string.
+
+    Args:
+        time (Datetime): Datetime object to format.
+
+    Returns:
+        str: RFC3339 formatted string.
+    """
+    if isinstance(time, (datetime.date, datetime.datetime)):
+        time = time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+    return time

censys/common/utils.py

@@ -1,2 +1,7 @@
 """Censys Version."""
-__version__ = "2.0.3"
+try:
+    import importlib_metadata
+except ImportError:
+    import importlib.metadata as importlib_metadata  # type: ignore
+
+__version__: str = importlib_metadata.version("censys")

censys/common/version.py

@@ -1,8 +1,7 @@
 """Base for interacting with the Censys Search API."""
-import datetime
 import os
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Dict, Iterable, Iterator, List, Optional, Type, Union
+from typing import Dict, Iterable, Iterator, List, Optional, Type
 
 from requests.models import Response
 
@@ -13,6 +12,8 @@
     CensysExceptionMapper,
     CensysSearchException,
 )
+from censys.common.types import Datetime
+from censys.common.utils import format_rfc3339
 
 Fields = Optional[List[str]]
 
@@ -69,6 +70,7 @@ def __init__(
         self.view_path = f"/{self.INDEX_NAME}/"
         self.search_path = f"/{self.INDEX_NAME}/search"
         self.aggregate_path = f"/{self.INDEX_NAME}/aggregate"
+        self.metadata_path = f"/metadata/{self.INDEX_NAME}"
 
     def _get_exception_class(  # type: ignore
         self, res: Response
@@ -222,7 +224,7 @@ def search(
     def view(
         self,
         document_id: str,
-        at_time: Optional[Union[str, datetime.date, datetime.datetime]] = None,
+        at_time: Optional[Datetime] = None,
     ) -> dict:
         """View document from current index.
 
@@ -239,9 +241,7 @@ def view(
         """
         args = {}
         if at_time:
-            if isinstance(at_time, (datetime.date, datetime.datetime)):
-                at_time = at_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
-            args["at_time"] = at_time
+            args["at_time"] = format_rfc3339(at_time)
 
         return self._get(self.view_path + document_id, args)["result"]
 
@@ -263,3 +263,11 @@ def aggregate(
         """
         args = {"q": query, "field": field, "num_buckets": num_buckets}
         return self._get(self.aggregate_path, args)["result"]
+
+    def metadata(self) -> dict:
+        """Get current index metadata.
+
+        Returns:
+            dict: The result set returned.
+        """
+        return self._get(self.metadata_path)["result"]