[MRG] Add a repo provider for CKAN datasets

binderhub/app.py

@@ -56,6 +56,7 @@
 from .ratelimit import RateLimiter
 from .registry import DockerRegistry
 from .repoproviders import (
+    CKANProvider,
     DataverseProvider,
     FigshareProvider,
     GistRepoProvider,
@@ -586,6 +587,7 @@ def _default_build_namespace(self):
             "figshare": FigshareProvider,
             "hydroshare": HydroshareProvider,
             "dataverse": DataverseProvider,
+            "ckan": CKANProvider,
         },
         config=True,
         help="""

binderhub/event-schemas/launch.json

@@ -14,7 +14,8 @@
         "Zenodo",
         "Figshare",
         "Hydroshare",
-        "Dataverse"
+        "Dataverse",
+        "CKAN"
       ],
       "description": "Provider for the repository being launched"
     },

binderhub/main.py

@@ -22,6 +22,7 @@
     "figshare": "Figshare",
     "hydroshare": "Hydroshare",
     "dataverse": "Dataverse",
+    "ckan": "CKAN",
 }
 
 

binderhub/repoproviders.py

@@ -15,7 +15,7 @@
 import time
 import urllib.parse
 from datetime import datetime, timedelta, timezone
-from urllib.parse import urlparse
+from urllib.parse import parse_qs, urlparse
 
 import escapism
 from prometheus_client import Gauge
@@ -448,6 +448,87 @@ def get_build_slug(self):
         return f"hydroshare-{self.record_id}"
 
 
+class CKANProvider(RepoProvider):
+    """Provide contents of a CKAN dataset
+    Users must provide a spec consisting of the CKAN dataset URL.
+    """
+
+    name = Unicode("CKAN")
+
+    display_name = "CKAN dataset"
+
+    labels = {
+        "text": "CKAN dataset URL (https://demo.ckan.org/dataset/sample-dataset-1)",
+        "tag_text": "Git ref (branch, tag, or commit)",
+        "ref_prop_disabled": True,
+        "label_prop_disabled": True,
+    }
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.repo = urllib.parse.unquote(self.spec)
+
+    async def get_resolved_ref(self):
+        parsed_repo = urlparse(self.repo)
+
+        url_parts_1 = parsed_repo.path.split("/history/")
+        url_parts_2 = url_parts_1[0].split("/")
+        if url_parts_2[-2] == "dataset":
+            self.dataset_id = url_parts_2[-1]
+        else:
+            return None
+
+        api_url_path = "/api/3/action/"
+        api = parsed_repo._replace(
+            path="/".join(url_parts_2[:-2]) + api_url_path, query=""
+        ).geturl()
+
+        # handle the activites
+        activity_id = None
+        if parse_qs(parsed_repo.query).get("activity_id") is not None:
+            activity_id = parse_qs(parsed_repo.query).get("activity_id")[0]
+        if len(url_parts_1) == 2:
+            activity_id = url_parts_1[-1]
+        if activity_id:
+            fetch_url = (
+                f"{api}activity_data_show?" f"id={activity_id}&object_type=package"
+            )
+        else:
+            fetch_url = f"{api}package_show?id={self.dataset_id}"
+
+        client = AsyncHTTPClient()
+        try:
+            r = await client.fetch(fetch_url, user_agent="BinderHub")
+        except HTTPError:
+            return None
+
+        json_response = json.loads(r.body)
+        date = json_response["result"]["metadata_modified"]
+        parsed_date = datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.%f")
+        epoch = parsed_date.replace(tzinfo=timezone(timedelta(0))).timestamp()
+        # truncate the timestamp
+        dataset_version = str(int(epoch))
+
+        self.record_id = f"{self.dataset_id}.v{dataset_version}"
+
+        return self.record_id
+
+    async def get_resolved_spec(self):
+        if not hasattr(self, "record_id"):
+            await self.get_resolved_ref()
+        return self.repo
+
+    def get_repo_url(self):
+        return self.repo
+
+    async def get_resolved_ref_url(self):
+        resolved_spec = await self.get_resolved_spec()
+        return resolved_spec
+
+    def get_build_slug(self):
+        return f"ckan-{self.dataset_id}"
+
+
 class GitRepoProvider(RepoProvider):
     """Bare bones git repo provider.
 

binderhub/static/js/src/form.js

@@ -31,7 +31,8 @@ export function getBuildFormValues() {
     providerPrefix === "zenodo" ||
     providerPrefix === "figshare" ||
     providerPrefix === "dataverse" ||
-    providerPrefix === "hydroshare"
+    providerPrefix === "hydroshare" ||
+    providerPrefix === "ckan"
   ) {
     ref = "";
   }

binderhub/tests/test_repoproviders.py

@@ -6,6 +6,7 @@
 from tornado.ioloop import IOLoop
 
 from binderhub.repoproviders import (
+    CKANProvider,
     DataverseProvider,
     FigshareProvider,
     GistRepoProvider,
@@ -209,6 +210,34 @@ async def test_dataverse(
     assert spec == resolved_spec
 
 
+@pytest.mark.parametrize(
+    "spec,resolved_spec,resolved_ref,resolved_ref_url,build_slug",
+    [
+        [
+            "https://demo.ckan.org/dataset/sample-dataset-1",
+            "https://demo.ckan.org/dataset/sample-dataset-1",
+            "sample-dataset-1.v",
+            "https://demo.ckan.org/dataset/sample-dataset-1",
+            "ckan-sample-dataset-1",
+        ],
+    ],
+)
+async def test_ckan(spec, resolved_spec, resolved_ref, resolved_ref_url, build_slug):
+    provider = CKANProvider(spec=spec)
+
+    ref = await provider.get_resolved_ref()
+    assert resolved_ref in ref
+
+    slug = provider.get_build_slug()
+    assert slug == build_slug
+    repo_url = provider.get_repo_url()
+    assert repo_url == spec
+    ref_url = await provider.get_resolved_ref_url()
+    assert ref_url == resolved_ref_url
+    spec = await provider.get_resolved_spec()
+    assert spec == resolved_spec
+
+
 @pytest.mark.github_api
 @pytest.mark.parametrize(
     "repo,unresolved_ref,resolved_ref",

docs/source/developer/repoproviders.rst

@@ -36,6 +36,8 @@ Currently supported providers, their prefixes and specs are:
     +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+
     | Dataverse  | ``dataverse``      | ``<dataverse-DOI>``                                         | `Dataverse <https://dataverse.org/>`_ is open source research data repository software installed all over the world.                      |
     +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+
+    | CKAN       | ``ckan``           | ``<url-escaped-url>/<dataset-id>``                          | `CKAN <https://ckan.org/>`_ is an open source data management system.                                                                     |
+    +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+
     | Git        | ``git``            | ``<url-escaped-url>/<commit-sha>``                          | A generic repository provider for URLs that point directly to a git repository.                                                           |
     +------------+--------------------+-------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------+
 

docs/source/reference/repoproviders.rst

@@ -65,6 +65,11 @@ Module: :mod:`binderhub.repoproviders`
 .. autoconfigurable:: DataverseProvider
     :members:
 
+:class:`CKANProvider`
+---------------------------
+
+.. autoconfigurable:: CKANProvider
+    :members:
 
 :class:`GitRepoProvider`
 ---------------------------