From af9517618d27d574048329b0ba4f9f5ba30ed599 Mon Sep 17 00:00:00 2001 From: stevejpurves Date: Sun, 7 Jul 2024 17:44:37 -0400 Subject: [PATCH 1/4] Add meca content provider --- repo2docker/app.py | 1 + repo2docker/contentproviders/__init__.py | 1 + repo2docker/contentproviders/meca.py | 130 +++++++++++++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 repo2docker/contentproviders/meca.py diff --git a/repo2docker/app.py b/repo2docker/app.py index 41e831e2..85774938 100755 --- a/repo2docker/app.py +++ b/repo2docker/app.py @@ -155,6 +155,7 @@ def _default_log_level(self): contentproviders.CKAN, contentproviders.Mercurial, contentproviders.Git, + contentproviders.Meca, ], config=True, help=""" diff --git a/repo2docker/contentproviders/__init__.py b/repo2docker/contentproviders/__init__.py index a0a2e019..fef02e33 100755 --- a/repo2docker/contentproviders/__init__.py +++ b/repo2docker/contentproviders/__init__.py @@ -7,3 +7,4 @@ from .mercurial import Mercurial from .swhid import Swhid from .zenodo import Zenodo +from .meca import Meca diff --git a/repo2docker/contentproviders/meca.py b/repo2docker/contentproviders/meca.py new file mode 100644 index 00000000..515b8360 --- /dev/null +++ b/repo2docker/contentproviders/meca.py @@ -0,0 +1,130 @@ +from .base import ContentProvider +from requests import Session +import os +from hashlib import md5 +from os import path +import tempfile +import shutil +import xml.etree.ElementTree as ET +from zipfile import ZipFile, is_zipfile +from urllib.parse import urlparse, urlunparse + +def get_hashed_slug(url, changes_with_content): + """Return a unique slug that is invariant to query parameters in the url""" + parsed_url = urlparse(url) + stripped_url = urlunparse( + (parsed_url.scheme, parsed_url.netloc, parsed_url.path, "", "", "") + ) + + return "meca-" + md5(f"{stripped_url}-{changes_with_content}".encode()).hexdigest() + +def fetch_zipfile(session, url, dst_dir): + resp = session.get(url, headers={"accept": "application/zip"}, stream=True) + resp.raise_for_status() + + dst_filename = path.join(dst_dir, "meca.zip") + with open(dst_filename, "wb") as dst: + for chunk in resp.iter_content(chunk_size=128): + dst.write(chunk) + + return dst_filename + + +def handle_items(_, item): + print(item) + + +def extract_validate_and_identify_bundle(zip_filename, dst_dir): + if not os.path.exists(zip_filename): + raise RuntimeError("Download MECA bundle not found") + + if not is_zipfile(zip_filename): + raise RuntimeError("MECA bundle is not a zip file") + + with ZipFile(zip_filename, "r") as zip_ref: + zip_ref.extractall(dst_dir) + + try: + manifest = path.join(dst_dir, "manifest.xml") + if not os.path.exists(manifest): + raise RuntimeError("MECA bundle is missing manifest.xml") + article_source_dir = "bundle/" + + tree = ET.parse(manifest) + root = tree.getroot() + + bundle_instance = root.findall( + "{*}item[@item-type='article-source-directory']/{*}instance" + ) + for attr in bundle_instance[0].attrib: + if attr.endswith("href"): + article_source_dir = bundle_instance[0].get(attr) + + return True, path.join(dst_dir, article_source_dir) + except: + return False, dst_dir + + +class Meca(ContentProvider): + """A repo2docker content provider for MECA bundles""" + + def __init__(self): + super().__init__() + self.session = Session() + self.session.headers.update( + { + "user-agent": f"repo2docker MECA", + } + ) + + def detect(self, spec, ref=None, extra_args=None): + """`spec` contains a faux protocol of meca+http[s] for detection purposes + and we assume `spec` trusted as a reachable MECA bundle from an allowed origin + (binderhub RepoProvider class already checking for this). + + An other HEAD check in made here in order to get the content-length header + """ + parsed = urlparse(spec) + if not parsed.scheme.endswith("+meca"): + return None + parsed = parsed._replace(scheme=parsed.scheme[:-5]) + url = urlunparse(parsed) + + r = self.session.head(url) + changes_with_content = r.headers.get("ETag") or r.headers.get("Content-Length") + + self.hashed_slug = get_hashed_slug(url, changes_with_content) + + return {"url": url, "slug": self.hashed_slug} + + def fetch(self, spec, output_dir, yield_output=False): + hashed_slug = spec["slug"] + url = spec["url"] + + yield f"Creating temporary directory.\n" + with tempfile.TemporaryDirectory() as tmpdir: + yield f"Temporary directory created at {tmpdir}.\n" + + yield f"Fetching MECA Bundle {url}.\n" + zip_filename = fetch_zipfile(self.session, url, tmpdir) + + yield f"Extracting MECA Bundle {zip_filename}.\n" + is_meca, bundle_dir = extract_validate_and_identify_bundle( + zip_filename, tmpdir + ) + + if not is_meca: + yield f"This doesn't look like a meca bundle, extracting everything.\n" + + yield f"Copying MECA Bundle at {bundle_dir} to {output_dir}.\n" + files = os.listdir(bundle_dir) + for f in files: + shutil.move(os.path.join(bundle_dir, f), output_dir) + + yield f"Removing temporary directory.\n" + + yield f"MECA Bundle {hashed_slug} fetched and unpacked.\n" + + @property + def content_id(self): + return self.hashed_slug From 0e5d6457d7a303b653f93147882b644489f40d16 Mon Sep 17 00:00:00 2001 From: stevejpurves Date: Sun, 7 Jul 2024 17:44:37 -0400 Subject: [PATCH 2/4] pre-commit fixes --- repo2docker/contentproviders/__init__.py | 2 +- repo2docker/contentproviders/meca.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/repo2docker/contentproviders/__init__.py b/repo2docker/contentproviders/__init__.py index fef02e33..7dbc6c67 100755 --- a/repo2docker/contentproviders/__init__.py +++ b/repo2docker/contentproviders/__init__.py @@ -4,7 +4,7 @@ from .figshare import Figshare from .git import Git from .hydroshare import Hydroshare +from .meca import Meca from .mercurial import Mercurial from .swhid import Swhid from .zenodo import Zenodo -from .meca import Meca diff --git a/repo2docker/contentproviders/meca.py b/repo2docker/contentproviders/meca.py index 515b8360..e362a7e6 100644 --- a/repo2docker/contentproviders/meca.py +++ b/repo2docker/contentproviders/meca.py @@ -1,13 +1,16 @@ -from .base import ContentProvider -from requests import Session import os -from hashlib import md5 -from os import path -import tempfile import shutil +import tempfile import xml.etree.ElementTree as ET -from zipfile import ZipFile, is_zipfile +from hashlib import md5 +from os import path from urllib.parse import urlparse, urlunparse +from zipfile import ZipFile, is_zipfile + +from requests import Session + +from .base import ContentProvider + def get_hashed_slug(url, changes_with_content): """Return a unique slug that is invariant to query parameters in the url""" @@ -18,6 +21,7 @@ def get_hashed_slug(url, changes_with_content): return "meca-" + md5(f"{stripped_url}-{changes_with_content}".encode()).hexdigest() + def fetch_zipfile(session, url, dst_dir): resp = session.get(url, headers={"accept": "application/zip"}, stream=True) resp.raise_for_status() From 85ce3002de8f9481fe14d945902f468bd1f6ee7b Mon Sep 17 00:00:00 2001 From: Steve Purves Date: Sun, 7 Jul 2024 17:44:37 -0400 Subject: [PATCH 3/4] Apply suggestions from code review Co-authored-by: Samuel Gaist --- repo2docker/contentproviders/meca.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/repo2docker/contentproviders/meca.py b/repo2docker/contentproviders/meca.py index e362a7e6..df83fdc4 100644 --- a/repo2docker/contentproviders/meca.py +++ b/repo2docker/contentproviders/meca.py @@ -13,7 +13,7 @@ def get_hashed_slug(url, changes_with_content): - """Return a unique slug that is invariant to query parameters in the url""" + """Returns a unique slug that is invariant to query parameters in the url""" parsed_url = urlparse(url) stripped_url = urlunparse( (parsed_url.scheme, parsed_url.netloc, parsed_url.path, "", "", "") @@ -34,13 +34,9 @@ def fetch_zipfile(session, url, dst_dir): return dst_filename -def handle_items(_, item): - print(item) - - def extract_validate_and_identify_bundle(zip_filename, dst_dir): if not os.path.exists(zip_filename): - raise RuntimeError("Download MECA bundle not found") + raise RuntimeError("Downloaded MECA bundle not found") if not is_zipfile(zip_filename): raise RuntimeError("MECA bundle is not a zip file") @@ -84,7 +80,7 @@ def __init__(self): def detect(self, spec, ref=None, extra_args=None): """`spec` contains a faux protocol of meca+http[s] for detection purposes and we assume `spec` trusted as a reachable MECA bundle from an allowed origin - (binderhub RepoProvider class already checking for this). + (binderhub RepoProvider class is already checking for this). An other HEAD check in made here in order to get the content-length header """ @@ -94,8 +90,8 @@ def detect(self, spec, ref=None, extra_args=None): parsed = parsed._replace(scheme=parsed.scheme[:-5]) url = urlunparse(parsed) - r = self.session.head(url) - changes_with_content = r.headers.get("ETag") or r.headers.get("Content-Length") + headers = self.session.head(url).headers + changes_with_content = headers.get("ETag") or headers.get("Content-Length") self.hashed_slug = get_hashed_slug(url, changes_with_content) From 850751e02c78d31969c3b65564dcc4edda8f2cd6 Mon Sep 17 00:00:00 2001 From: stevejpurves Date: Sun, 7 Jul 2024 17:44:58 -0400 Subject: [PATCH 4/4] enable launch from local meca file --- repo2docker/app.py | 2 +- repo2docker/contentproviders/meca.py | 40 +++++++++++++++++++--------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/repo2docker/app.py b/repo2docker/app.py index 85774938..14a94924 100755 --- a/repo2docker/app.py +++ b/repo2docker/app.py @@ -153,9 +153,9 @@ def _default_log_level(self): contentproviders.Hydroshare, contentproviders.Swhid, contentproviders.CKAN, + contentproviders.Meca, contentproviders.Mercurial, contentproviders.Git, - contentproviders.Meca, ], config=True, help=""" diff --git a/repo2docker/contentproviders/meca.py b/repo2docker/contentproviders/meca.py index df83fdc4..60ff05e3 100644 --- a/repo2docker/contentproviders/meca.py +++ b/repo2docker/contentproviders/meca.py @@ -1,3 +1,4 @@ +import hashlib import os import shutil import tempfile @@ -78,35 +79,50 @@ def __init__(self): ) def detect(self, spec, ref=None, extra_args=None): - """`spec` contains a faux protocol of meca+http[s] for detection purposes + """`spec` contains a faux protocol of http[s]+meca for detection purposes and we assume `spec` trusted as a reachable MECA bundle from an allowed origin (binderhub RepoProvider class is already checking for this). An other HEAD check in made here in order to get the content-length header """ - parsed = urlparse(spec) - if not parsed.scheme.endswith("+meca"): - return None - parsed = parsed._replace(scheme=parsed.scheme[:-5]) - url = urlunparse(parsed) - - headers = self.session.head(url).headers - changes_with_content = headers.get("ETag") or headers.get("Content-Length") + is_local_file = False + if spec.endswith(".meca.zip") and os.path.isfile(spec): + url = os.path.abspath(spec) + is_local_file = True + with open(url, "rb") as f: + file_hash = hashlib.blake2b() + while chunk := f.read(8192): + file_hash.update(chunk) + changes_with_content = file_hash.hexdigest() + else: + parsed = urlparse(spec) + if not parsed.scheme.endswith("+meca"): + return None + parsed = parsed._replace(scheme=parsed.scheme[:-5]) + url = urlunparse(parsed) + + headers = self.session.head(url).headers + changes_with_content = headers.get("ETag") or headers.get("Content-Length") self.hashed_slug = get_hashed_slug(url, changes_with_content) - return {"url": url, "slug": self.hashed_slug} + return {"url": url, "slug": self.hashed_slug, "is_local_file": is_local_file} def fetch(self, spec, output_dir, yield_output=False): hashed_slug = spec["slug"] url = spec["url"] + is_local_file = spec["is_local_file"] yield f"Creating temporary directory.\n" with tempfile.TemporaryDirectory() as tmpdir: yield f"Temporary directory created at {tmpdir}.\n" - yield f"Fetching MECA Bundle {url}.\n" - zip_filename = fetch_zipfile(self.session, url, tmpdir) + if is_local_file: + yield f"Found MECA Bundle {url}.\n" + zip_filename = url + else: + yield f"Fetching MECA Bundle {url}.\n" + zip_filename = fetch_zipfile(self.session, url, tmpdir) yield f"Extracting MECA Bundle {zip_filename}.\n" is_meca, bundle_dir = extract_validate_and_identify_bundle(