From 212984be0b954656e9f88a70eb473e4571415ed5 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Wed, 15 May 2024 23:06:17 +0100 Subject: [PATCH 1/5] Yardify repository and tasks Relinting/formatting Update CONTRIBUTING instructions More yardification --- .gitattributes | 2 +- .pre-commit-config.yaml | 2 +- CONTRIBUTING.md | 23 ++++++++++--------- Procfile | 1 - README.md | 4 ++-- pyproject.toml | 15 +++++------- schemas | 2 +- tasks.py | 23 +++++++------------ {marda_registry => yard}/__init__.py | 0 {marda_registry => yard}/app.py | 5 ++-- .../data/extractors/galvani.yml | 0 .../data/extractors/pasta-converters.yml | 0 .../data/extractors/quantumespresso.yml | 0 .../data/extractors/rosettasciio.yml | 0 .../data/extractors/yadg.yml | 0 .../data/extractors/zeiss-tiff-meta.yml | 0 .../data/filetypes/agilent-ch.yml | 0 .../data/filetypes/agilent-dx.yml | 0 .../data/filetypes/biologic-mpr.yml | 0 .../data/filetypes/biologic-mpt.yml | 0 .../data/filetypes/doli-mvl.yml | 0 .../data/filetypes/fischer-scope-hap.yml | 0 .../data/filetypes/keysight-g200-mss.yml | 0 .../data/filetypes/kla-g200x-nmd.yml | 0 .../data/filetypes/neware-nda.yml | 0 .../data/filetypes/panalytical-xrdml.yml | 0 .../data/filetypes/phi-spe.yml | 0 .../data/filetypes/qe-cp-in.yml | 0 .../data/filetypes/qe-pw-in.yml | 0 .../data/filetypes/quantum-design-dat.yml | 0 .../data/filetypes/renishaw-wdf.yml | 0 .../data/filetypes/zeiss-tiff.yml | 0 .../data/lfs/agilent-ch/hplc.CH | 0 .../data/lfs/agilent-dx/hplc.dx | 0 .../data/lfs/biologic-mpr/ca.mpr | 0 .../data/lfs/biologic-mpr/gcpl.mpr | 0 .../data/lfs/biologic-mpr/peis.mpr | 0 .../data/lfs/biologic-mpt/cv.mpt | 0 .../data/lfs/biologic-mpt/gcpl.mpt | 0 .../data/lfs/biologic-mpt/lsv.mpt | 0 .../data/lfs/doli-mvl/Membrane2_03.mvl | 0 .../data/lfs/fischer-scope-hap/Al.hap | 0 .../data/lfs/keysight-g200-mss/FS.mss | 0 .../data/lfs/kla-g200x-nmd/FS.NMD | 0 .../data/lfs/panalytical-xrdml/xrd.xrdml | 0 .../data/lfs/phi-spe/xps.spe | 0 .../data/lfs/qe-cp-in/cp.in | 0 .../data/lfs/qe-pw-in/pw.in | 0 .../data/lfs/renishaw-wdf/raman_example.wdf | 0 .../data/lfs/zeiss-tiff/example.tif | 0 {marda_registry => yard}/data/meta.json | 0 {marda_registry => yard}/models/__init__.py | 0 .../static/background.svg | 0 {marda_registry => yard}/static/style.css | 0 {marda_registry => yard}/templates/base.html | 2 +- .../templates/extractor.html | 0 .../templates/extractors.html | 0 .../templates/filetype.html | 0 .../templates/filetypes.html | 0 {marda_registry => yard}/templates/index.html | 2 +- {marda_registry => yard}/utils.py | 0 61 files changed, 35 insertions(+), 46 deletions(-) delete mode 100644 Procfile rename {marda_registry => yard}/__init__.py (100%) rename {marda_registry => yard}/app.py (98%) rename {marda_registry => yard}/data/extractors/galvani.yml (100%) rename {marda_registry => yard}/data/extractors/pasta-converters.yml (100%) rename {marda_registry => yard}/data/extractors/quantumespresso.yml (100%) rename {marda_registry => yard}/data/extractors/rosettasciio.yml (100%) rename {marda_registry => yard}/data/extractors/yadg.yml (100%) rename {marda_registry => yard}/data/extractors/zeiss-tiff-meta.yml (100%) rename {marda_registry => yard}/data/filetypes/agilent-ch.yml (100%) rename {marda_registry => yard}/data/filetypes/agilent-dx.yml (100%) rename {marda_registry => yard}/data/filetypes/biologic-mpr.yml (100%) rename {marda_registry => yard}/data/filetypes/biologic-mpt.yml (100%) rename {marda_registry => yard}/data/filetypes/doli-mvl.yml (100%) rename {marda_registry => yard}/data/filetypes/fischer-scope-hap.yml (100%) rename {marda_registry => yard}/data/filetypes/keysight-g200-mss.yml (100%) rename {marda_registry => yard}/data/filetypes/kla-g200x-nmd.yml (100%) rename {marda_registry => yard}/data/filetypes/neware-nda.yml (100%) rename {marda_registry => yard}/data/filetypes/panalytical-xrdml.yml (100%) rename {marda_registry => yard}/data/filetypes/phi-spe.yml (100%) rename {marda_registry => yard}/data/filetypes/qe-cp-in.yml (100%) rename {marda_registry => yard}/data/filetypes/qe-pw-in.yml (100%) rename {marda_registry => yard}/data/filetypes/quantum-design-dat.yml (100%) rename {marda_registry => yard}/data/filetypes/renishaw-wdf.yml (100%) rename {marda_registry => yard}/data/filetypes/zeiss-tiff.yml (100%) rename {marda_registry => yard}/data/lfs/agilent-ch/hplc.CH (100%) rename {marda_registry => yard}/data/lfs/agilent-dx/hplc.dx (100%) rename {marda_registry => yard}/data/lfs/biologic-mpr/ca.mpr (100%) rename {marda_registry => yard}/data/lfs/biologic-mpr/gcpl.mpr (100%) rename {marda_registry => yard}/data/lfs/biologic-mpr/peis.mpr (100%) rename {marda_registry => yard}/data/lfs/biologic-mpt/cv.mpt (100%) rename {marda_registry => yard}/data/lfs/biologic-mpt/gcpl.mpt (100%) rename {marda_registry => yard}/data/lfs/biologic-mpt/lsv.mpt (100%) rename {marda_registry => yard}/data/lfs/doli-mvl/Membrane2_03.mvl (100%) rename {marda_registry => yard}/data/lfs/fischer-scope-hap/Al.hap (100%) rename {marda_registry => yard}/data/lfs/keysight-g200-mss/FS.mss (100%) rename {marda_registry => yard}/data/lfs/kla-g200x-nmd/FS.NMD (100%) rename {marda_registry => yard}/data/lfs/panalytical-xrdml/xrd.xrdml (100%) rename {marda_registry => yard}/data/lfs/phi-spe/xps.spe (100%) rename {marda_registry => yard}/data/lfs/qe-cp-in/cp.in (100%) rename {marda_registry => yard}/data/lfs/qe-pw-in/pw.in (100%) rename {marda_registry => yard}/data/lfs/renishaw-wdf/raman_example.wdf (100%) rename {marda_registry => yard}/data/lfs/zeiss-tiff/example.tif (100%) rename {marda_registry => yard}/data/meta.json (100%) rename {marda_registry => yard}/models/__init__.py (100%) rename {marda_registry => yard}/static/background.svg (100%) rename {marda_registry => yard}/static/style.css (100%) rename {marda_registry => yard}/templates/base.html (96%) rename {marda_registry => yard}/templates/extractor.html (100%) rename {marda_registry => yard}/templates/extractors.html (100%) rename {marda_registry => yard}/templates/filetype.html (100%) rename {marda_registry => yard}/templates/filetypes.html (100%) rename {marda_registry => yard}/templates/index.html (80%) rename {marda_registry => yard}/utils.py (100%) diff --git a/.gitattributes b/.gitattributes index afe057f..6912e57 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ -marda_registry/data/lfs/*/*.* filter=lfs diff=lfs merge=lfs -text +yard/data/lfs/*/*.* filter=lfs diff=lfs merge=lfs -text diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b7d323d..8fb850d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ --- -exclude: ^(marda_registry/data/) +exclude: ^(yard/data/) repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5d0a137..933cd73 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,14 +2,14 @@ -#
MaRDA Metadata Extractors: Registry
+#
Datatractor Yard
##
Contributing guidelines
Contributions to this registry are welcome. We pledge to follow the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/2/1/code_of_conduct/). -You can contribute `FileType` and `Extractor` entries to this registry by creating pull requests (PRs), which should include YAML files under `./data/filetypes` and `./data/extractors`, respectively. These YAML files must follow the schemas at [marda-alliance/metadata_extractors_schema](https://github.com/marda-alliance/metadata_extractors_schema) (see the [online documentation](https://marda-alliance.github.io/metadata_extractors_schema) for more details). +You can contribute `FileType` and `Extractor` entries to this registry by creating pull requests (PRs), which should include YAML files under `./data/filetypes` and `./data/extractors`, respectively. These YAML files must follow the schemas at [datatractor/schemas](https://github.com/datatractor/schema) (see the [online documentation](https://datatractor.github.io/schema) for more details). -An example of the process can be seen in this [pull request](https://github.com/marda-alliance/metadata_extractors_registry/pull/54). The workflow steps are as follows: +An example of the process can be seen in this [pull request](https://github.com/marda-alliance/metadata_extractors_registry/pull/54) (in the previous iteration of this repository). The workflow steps are as follows: ### Opening a pull request @@ -20,13 +20,13 @@ An example of the process can be seen in this [pull request](https://github.com/ - Please add the appropriate labels (`Extractor` and `FileType`). ### Example files -- If you are adding new `FileTypes` in this PR, add example files of those `FileTypes` using `git-lfs` into `marda_registry/data/lfs`. Do not commit example files into the repository directly. For example, to add a new file called `xmpl.tpe` (corresponding to the `FileType` with ID `example-type`) you would: +- If you are adding new `FileTypes` in this PR, add example files of those `FileTypes` using `git-lfs` into `yard/data/lfs`. Do not commit example files into the repository directly. For example, to add a new file called `xmpl.tpe` (corresponding to the `FileType` with ID `example-type`) you would: ``` bash - mkdir marda_registry/data/lfs/example-type # create the dir using FileType ID - cp xmpl.tpe marda_registry/data/lfs/example-type # copy the example file + mkdir yard/data/lfs/example-type # create the dir using FileType ID + cp xmpl.tpe yard/data/lfs/example-type # copy the example file git lfs install # setup git-lfs, only necessary if you haven't used git-lfs before - git add marda_registry/data/lfs/example-type/* # track files + git add yard/data/lfs/example-type/* # track files ... git lfs ls-files # check that your new example files are tracked before committing git commit @@ -40,14 +40,15 @@ An example of the process can be seen in this [pull request](https://github.com/ - `build`, which makes sure the new registry website can be built. Therefore, it is mainly the `lint` and `validate` actions you will have to pay attention to. In case anything is unclear or you cannot find the error, ping one of the [Registry Maintainers](./README.md#registry-maintainers). -- You can of course validate your definitions locally, by using the [![Schema](https://badgen.net/static/marda-alliance/metadata_extractors_schema/?icon=github)](https://github.com/marda-alliance/metadata_extractors_schema/) package, see the [Usage section](https://marda-alliance.github.io/metadata_extractors_schema/main/usage.html) of the documentation. +- You can of course validate your definitions locally, by using the [![Schema](https://badgen.net/static/datatractor/schema/?icon=github)](https://github.com/datatractor/schema/) package, see the [Usage section](https://datatractor.github.io/schema/main/usage.html) of the documentation. ### Testing of Extraction -- You should check that your `yml` definitions can be used to install the `Extractor` and and extract any example files from the supported `FileTypes` locally, by using our [![Schema](https://badgen.net/static/marda-alliance/metadata_extractors_api/?icon=github)](https://github.com/marda-alliance/metadata_extractors_api/) package, see the [Installation](https://github.com/marda-alliance/metadata_extractors_api#installation) and [Usage](https://github.com/marda-alliance/metadata_extractors_api#usage) sections. +- You should check that your `yml` definitions can be used to install the `Extractor` and and extract any example files from the supported `FileTypes` locally, by using our [![Schema](https://badgen.net/static/datatractor/beam/?icon=github)](https://github.com/datatractor/beam/) package, see the [Installation](https://github.com/datatractor/beam#installation) and [Usage](https://github.com/datatractor/beam#usage) sections. -> At the moment, the above CI is not checking whether the extraction of the example files actually works. See [this issue](https://github.com/marda-alliance/metadata_extractors_registry/issues/34) for further details. +> At the moment, the above CI is not checking whether the extraction of the example files actually works. See [this issue](https://github.com/datatractor/yard/issues/9) for further details. ### Approval and Deployment + - If you are a first-time contributor to the project, we will need to pre-approve your PR so that the CI can be run. Ping one of the [Registry Maintainers](./README.md#registry-maintainers). - Once the CI passes, and all issues that are raised during the review are addressed, your PR can be merged. -- Once your PR is merged into the Registry, your new definitions should be available at the [Registry Website](https://marda-registry.fly.dev/). Cheers! +- Once your PR is merged into the Registry, your new definitions should be available at the [Registry Website](https://yard.datatractor.org/). Cheers! diff --git a/Procfile b/Procfile deleted file mode 100644 index 9d55d8a..0000000 --- a/Procfile +++ /dev/null @@ -1 +0,0 @@ -web: uvicorn marda_registry.app:app --host 0.0.0.0 --port $PORT diff --git a/README.md b/README.md index b57bf81..dae0f11 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ -A place to develop and discuss the MaRDA Extractors WG registry. +A place to develop and discuss the Datatractor Yard (formerly the MaRDA Extractors WG registry). The idea is to collect various file formats used in materials science and chemistry, describe them with metadata, and provide links to software projects that can parse them. By providing this data in a web API, it hoped that users can discover new extractors more easily and metadata standards can be developed for the output of extractors to enable schemas to proliferate throughout the field. @@ -42,7 +42,7 @@ invoke regenerate-models From the repository root directory, launch the server with uvicorn: ``` -uvicorn marda_registry.app:app +uvicorn yard.app:app ``` then navigate to http://localhost:5000 to test. diff --git a/pyproject.toml b/pyproject.toml index 05dc43e..0c62640 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,19 +1,19 @@ [build-system] -requires = ["hatchling", "hatch-vcs >= 0.3.0"] +requires = ["hatchling", "versioningit"] build-backend = "hatchling.build" [tool.hatch.version] -path = "marda_registry/__init__.py" +source = "versioningit" [tool.hatch.metadata] # required to allow git deps in optional dependencies allow-direct-references = true [tool.hatch.build.targets.wheel] -packages = ["marda_registry"] +packages = ["yard"] [project] -name = "marda-extractors-registry" +name = "datatractor-yard" readme = "README.md" dynamic = ["version"] classifiers = [ @@ -44,7 +44,7 @@ dependencies = [ [project.optional-dependencies] test = [ "pytest", - "marda-extractors-api[formats] @ git+https://github.com/marda-alliance/metadata_extractors_api.git", + "marda-extractors-api[formats] @ git+https://github.com/datatractor/yard.git", ] dev = [ @@ -52,12 +52,9 @@ dev = [ ] [project.urls] -repository = "https://github.com/marda-alliance/metadata_extractors_registry" +repository = "https://github.com/datatractor/yard" [tool.ruff] -extend-exclude = [ - "providers", -] target-version = "py310" [tool.ruff.lint] diff --git a/schemas b/schemas index 11e051c..bc23e81 160000 --- a/schemas +++ b/schemas @@ -1 +1 @@ -Subproject commit 11e051c40b80256833ae559eb62c79778b59465d +Subproject commit bc23e81704b0fe70896cea0058f4ab802d468de5 diff --git a/tasks.py b/tasks.py index 5ff9064..6a73468 100644 --- a/tasks.py +++ b/tasks.py @@ -2,7 +2,7 @@ from invoke import task -MODEL_DIRECTORY = Path(__file__).parent / "marda_registry" / "models" +MODEL_DIRECTORY = Path(__file__).parent / "yard" / "models" @task @@ -35,8 +35,8 @@ def regenerate_models(_): def validate_entries(_): print("Validating entries") - from marda_registry.models import Extractor, FileType - from marda_registry.utils import load_registry_collection + from yard.models import Extractor, FileType + from yard.utils import load_registry_collection counts = {} errors = [] @@ -52,9 +52,7 @@ def validate_entries(_): if type_ is Extractor: filetype_ids = set( d.stem - for d in Path(__file__).parent.glob( - "./marda_registry/data/filetypes/*.yml" - ) + for d in Path(__file__).parent.glob("./yard/data/filetypes/*.yml") ) for extractor in entries: @@ -76,12 +74,8 @@ def check_for_yaml(_): print("Checking for erroneous .yaml files.") - extractors = list( - Path(__file__).parent.glob("./marda_registry/data/extractors/*.yaml") - ) - filetypes = list( - Path(__file__).parent.glob("./marda_registry/data/filetypes/*.yaml") - ) + extractors = list(Path(__file__).parent.glob("./yard/data/extractors/*.yaml")) + filetypes = list(Path(__file__).parent.glob("./yard/data/filetypes/*.yaml")) for e in extractors: print(f"Found {e} with bad file extension (should be .yml here)") @@ -100,12 +94,11 @@ def validate_lfs_examples(_): """Loop through the LFS dir and check that each directory has a corresponding filetype.""" filetype_ids = set( - d.stem - for d in Path(__file__).parent.glob("./marda_registry/data/filetypes/*.yaml") + d.stem for d in Path(__file__).parent.glob("./yard/data/filetypes/*.yaml") ) lfs_filetype_dirs = set( - d.name for d in Path(__file__).parent.glob("./marda_registry/data/lfs/*") + d.name for d in Path(__file__).parent.glob("./yard/data/lfs/*") ) errors = [] diff --git a/marda_registry/__init__.py b/yard/__init__.py similarity index 100% rename from marda_registry/__init__.py rename to yard/__init__.py diff --git a/marda_registry/app.py b/yard/app.py similarity index 98% rename from marda_registry/app.py rename to yard/app.py index 3465e40..8dd3e93 100644 --- a/marda_registry/app.py +++ b/yard/app.py @@ -10,15 +10,14 @@ from fastapi.responses import HTMLResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates +from yard import __api_version__ from pydantic import BaseModel -from marda_registry import __api_version__ - from .models import Extractor, FileType from .utils import load_registry_collection app = FastAPI( - title="MaRDA extractors registry API", + title="Datatractor Yard API", description=f"This server implements v{__api_version__} of the [MaRDA extractors WG](https://github.com/marda-alliance/metadata_extractors) registry API.", # noqa: E501 version=__api_version__, ) diff --git a/marda_registry/data/extractors/galvani.yml b/yard/data/extractors/galvani.yml similarity index 100% rename from marda_registry/data/extractors/galvani.yml rename to yard/data/extractors/galvani.yml diff --git a/marda_registry/data/extractors/pasta-converters.yml b/yard/data/extractors/pasta-converters.yml similarity index 100% rename from marda_registry/data/extractors/pasta-converters.yml rename to yard/data/extractors/pasta-converters.yml diff --git a/marda_registry/data/extractors/quantumespresso.yml b/yard/data/extractors/quantumespresso.yml similarity index 100% rename from marda_registry/data/extractors/quantumespresso.yml rename to yard/data/extractors/quantumespresso.yml diff --git a/marda_registry/data/extractors/rosettasciio.yml b/yard/data/extractors/rosettasciio.yml similarity index 100% rename from marda_registry/data/extractors/rosettasciio.yml rename to yard/data/extractors/rosettasciio.yml diff --git a/marda_registry/data/extractors/yadg.yml b/yard/data/extractors/yadg.yml similarity index 100% rename from marda_registry/data/extractors/yadg.yml rename to yard/data/extractors/yadg.yml diff --git a/marda_registry/data/extractors/zeiss-tiff-meta.yml b/yard/data/extractors/zeiss-tiff-meta.yml similarity index 100% rename from marda_registry/data/extractors/zeiss-tiff-meta.yml rename to yard/data/extractors/zeiss-tiff-meta.yml diff --git a/marda_registry/data/filetypes/agilent-ch.yml b/yard/data/filetypes/agilent-ch.yml similarity index 100% rename from marda_registry/data/filetypes/agilent-ch.yml rename to yard/data/filetypes/agilent-ch.yml diff --git a/marda_registry/data/filetypes/agilent-dx.yml b/yard/data/filetypes/agilent-dx.yml similarity index 100% rename from marda_registry/data/filetypes/agilent-dx.yml rename to yard/data/filetypes/agilent-dx.yml diff --git a/marda_registry/data/filetypes/biologic-mpr.yml b/yard/data/filetypes/biologic-mpr.yml similarity index 100% rename from marda_registry/data/filetypes/biologic-mpr.yml rename to yard/data/filetypes/biologic-mpr.yml diff --git a/marda_registry/data/filetypes/biologic-mpt.yml b/yard/data/filetypes/biologic-mpt.yml similarity index 100% rename from marda_registry/data/filetypes/biologic-mpt.yml rename to yard/data/filetypes/biologic-mpt.yml diff --git a/marda_registry/data/filetypes/doli-mvl.yml b/yard/data/filetypes/doli-mvl.yml similarity index 100% rename from marda_registry/data/filetypes/doli-mvl.yml rename to yard/data/filetypes/doli-mvl.yml diff --git a/marda_registry/data/filetypes/fischer-scope-hap.yml b/yard/data/filetypes/fischer-scope-hap.yml similarity index 100% rename from marda_registry/data/filetypes/fischer-scope-hap.yml rename to yard/data/filetypes/fischer-scope-hap.yml diff --git a/marda_registry/data/filetypes/keysight-g200-mss.yml b/yard/data/filetypes/keysight-g200-mss.yml similarity index 100% rename from marda_registry/data/filetypes/keysight-g200-mss.yml rename to yard/data/filetypes/keysight-g200-mss.yml diff --git a/marda_registry/data/filetypes/kla-g200x-nmd.yml b/yard/data/filetypes/kla-g200x-nmd.yml similarity index 100% rename from marda_registry/data/filetypes/kla-g200x-nmd.yml rename to yard/data/filetypes/kla-g200x-nmd.yml diff --git a/marda_registry/data/filetypes/neware-nda.yml b/yard/data/filetypes/neware-nda.yml similarity index 100% rename from marda_registry/data/filetypes/neware-nda.yml rename to yard/data/filetypes/neware-nda.yml diff --git a/marda_registry/data/filetypes/panalytical-xrdml.yml b/yard/data/filetypes/panalytical-xrdml.yml similarity index 100% rename from marda_registry/data/filetypes/panalytical-xrdml.yml rename to yard/data/filetypes/panalytical-xrdml.yml diff --git a/marda_registry/data/filetypes/phi-spe.yml b/yard/data/filetypes/phi-spe.yml similarity index 100% rename from marda_registry/data/filetypes/phi-spe.yml rename to yard/data/filetypes/phi-spe.yml diff --git a/marda_registry/data/filetypes/qe-cp-in.yml b/yard/data/filetypes/qe-cp-in.yml similarity index 100% rename from marda_registry/data/filetypes/qe-cp-in.yml rename to yard/data/filetypes/qe-cp-in.yml diff --git a/marda_registry/data/filetypes/qe-pw-in.yml b/yard/data/filetypes/qe-pw-in.yml similarity index 100% rename from marda_registry/data/filetypes/qe-pw-in.yml rename to yard/data/filetypes/qe-pw-in.yml diff --git a/marda_registry/data/filetypes/quantum-design-dat.yml b/yard/data/filetypes/quantum-design-dat.yml similarity index 100% rename from marda_registry/data/filetypes/quantum-design-dat.yml rename to yard/data/filetypes/quantum-design-dat.yml diff --git a/marda_registry/data/filetypes/renishaw-wdf.yml b/yard/data/filetypes/renishaw-wdf.yml similarity index 100% rename from marda_registry/data/filetypes/renishaw-wdf.yml rename to yard/data/filetypes/renishaw-wdf.yml diff --git a/marda_registry/data/filetypes/zeiss-tiff.yml b/yard/data/filetypes/zeiss-tiff.yml similarity index 100% rename from marda_registry/data/filetypes/zeiss-tiff.yml rename to yard/data/filetypes/zeiss-tiff.yml diff --git a/marda_registry/data/lfs/agilent-ch/hplc.CH b/yard/data/lfs/agilent-ch/hplc.CH similarity index 100% rename from marda_registry/data/lfs/agilent-ch/hplc.CH rename to yard/data/lfs/agilent-ch/hplc.CH diff --git a/marda_registry/data/lfs/agilent-dx/hplc.dx b/yard/data/lfs/agilent-dx/hplc.dx similarity index 100% rename from marda_registry/data/lfs/agilent-dx/hplc.dx rename to yard/data/lfs/agilent-dx/hplc.dx diff --git a/marda_registry/data/lfs/biologic-mpr/ca.mpr b/yard/data/lfs/biologic-mpr/ca.mpr similarity index 100% rename from marda_registry/data/lfs/biologic-mpr/ca.mpr rename to yard/data/lfs/biologic-mpr/ca.mpr diff --git a/marda_registry/data/lfs/biologic-mpr/gcpl.mpr b/yard/data/lfs/biologic-mpr/gcpl.mpr similarity index 100% rename from marda_registry/data/lfs/biologic-mpr/gcpl.mpr rename to yard/data/lfs/biologic-mpr/gcpl.mpr diff --git a/marda_registry/data/lfs/biologic-mpr/peis.mpr b/yard/data/lfs/biologic-mpr/peis.mpr similarity index 100% rename from marda_registry/data/lfs/biologic-mpr/peis.mpr rename to yard/data/lfs/biologic-mpr/peis.mpr diff --git a/marda_registry/data/lfs/biologic-mpt/cv.mpt b/yard/data/lfs/biologic-mpt/cv.mpt similarity index 100% rename from marda_registry/data/lfs/biologic-mpt/cv.mpt rename to yard/data/lfs/biologic-mpt/cv.mpt diff --git a/marda_registry/data/lfs/biologic-mpt/gcpl.mpt b/yard/data/lfs/biologic-mpt/gcpl.mpt similarity index 100% rename from marda_registry/data/lfs/biologic-mpt/gcpl.mpt rename to yard/data/lfs/biologic-mpt/gcpl.mpt diff --git a/marda_registry/data/lfs/biologic-mpt/lsv.mpt b/yard/data/lfs/biologic-mpt/lsv.mpt similarity index 100% rename from marda_registry/data/lfs/biologic-mpt/lsv.mpt rename to yard/data/lfs/biologic-mpt/lsv.mpt diff --git a/marda_registry/data/lfs/doli-mvl/Membrane2_03.mvl b/yard/data/lfs/doli-mvl/Membrane2_03.mvl similarity index 100% rename from marda_registry/data/lfs/doli-mvl/Membrane2_03.mvl rename to yard/data/lfs/doli-mvl/Membrane2_03.mvl diff --git a/marda_registry/data/lfs/fischer-scope-hap/Al.hap b/yard/data/lfs/fischer-scope-hap/Al.hap similarity index 100% rename from marda_registry/data/lfs/fischer-scope-hap/Al.hap rename to yard/data/lfs/fischer-scope-hap/Al.hap diff --git a/marda_registry/data/lfs/keysight-g200-mss/FS.mss b/yard/data/lfs/keysight-g200-mss/FS.mss similarity index 100% rename from marda_registry/data/lfs/keysight-g200-mss/FS.mss rename to yard/data/lfs/keysight-g200-mss/FS.mss diff --git a/marda_registry/data/lfs/kla-g200x-nmd/FS.NMD b/yard/data/lfs/kla-g200x-nmd/FS.NMD similarity index 100% rename from marda_registry/data/lfs/kla-g200x-nmd/FS.NMD rename to yard/data/lfs/kla-g200x-nmd/FS.NMD diff --git a/marda_registry/data/lfs/panalytical-xrdml/xrd.xrdml b/yard/data/lfs/panalytical-xrdml/xrd.xrdml similarity index 100% rename from marda_registry/data/lfs/panalytical-xrdml/xrd.xrdml rename to yard/data/lfs/panalytical-xrdml/xrd.xrdml diff --git a/marda_registry/data/lfs/phi-spe/xps.spe b/yard/data/lfs/phi-spe/xps.spe similarity index 100% rename from marda_registry/data/lfs/phi-spe/xps.spe rename to yard/data/lfs/phi-spe/xps.spe diff --git a/marda_registry/data/lfs/qe-cp-in/cp.in b/yard/data/lfs/qe-cp-in/cp.in similarity index 100% rename from marda_registry/data/lfs/qe-cp-in/cp.in rename to yard/data/lfs/qe-cp-in/cp.in diff --git a/marda_registry/data/lfs/qe-pw-in/pw.in b/yard/data/lfs/qe-pw-in/pw.in similarity index 100% rename from marda_registry/data/lfs/qe-pw-in/pw.in rename to yard/data/lfs/qe-pw-in/pw.in diff --git a/marda_registry/data/lfs/renishaw-wdf/raman_example.wdf b/yard/data/lfs/renishaw-wdf/raman_example.wdf similarity index 100% rename from marda_registry/data/lfs/renishaw-wdf/raman_example.wdf rename to yard/data/lfs/renishaw-wdf/raman_example.wdf diff --git a/marda_registry/data/lfs/zeiss-tiff/example.tif b/yard/data/lfs/zeiss-tiff/example.tif similarity index 100% rename from marda_registry/data/lfs/zeiss-tiff/example.tif rename to yard/data/lfs/zeiss-tiff/example.tif diff --git a/marda_registry/data/meta.json b/yard/data/meta.json similarity index 100% rename from marda_registry/data/meta.json rename to yard/data/meta.json diff --git a/marda_registry/models/__init__.py b/yard/models/__init__.py similarity index 100% rename from marda_registry/models/__init__.py rename to yard/models/__init__.py diff --git a/marda_registry/static/background.svg b/yard/static/background.svg similarity index 100% rename from marda_registry/static/background.svg rename to yard/static/background.svg diff --git a/marda_registry/static/style.css b/yard/static/style.css similarity index 100% rename from marda_registry/static/style.css rename to yard/static/style.css diff --git a/marda_registry/templates/base.html b/yard/templates/base.html similarity index 96% rename from marda_registry/templates/base.html rename to yard/templates/base.html index d00c406..4d61a12 100644 --- a/marda_registry/templates/base.html +++ b/yard/templates/base.html @@ -9,7 +9,7 @@ - {% block title %}{% endblock %} - MaRDA Registry + {% block title %}{% endblock %} - Datatractor Yard {% endblock %} diff --git a/marda_registry/templates/extractor.html b/yard/templates/extractor.html similarity index 100% rename from marda_registry/templates/extractor.html rename to yard/templates/extractor.html diff --git a/marda_registry/templates/extractors.html b/yard/templates/extractors.html similarity index 100% rename from marda_registry/templates/extractors.html rename to yard/templates/extractors.html diff --git a/marda_registry/templates/filetype.html b/yard/templates/filetype.html similarity index 100% rename from marda_registry/templates/filetype.html rename to yard/templates/filetype.html diff --git a/marda_registry/templates/filetypes.html b/yard/templates/filetypes.html similarity index 100% rename from marda_registry/templates/filetypes.html rename to yard/templates/filetypes.html diff --git a/marda_registry/templates/index.html b/yard/templates/index.html similarity index 80% rename from marda_registry/templates/index.html rename to yard/templates/index.html index 747b00a..106e321 100644 --- a/marda_registry/templates/index.html +++ b/yard/templates/index.html @@ -7,7 +7,7 @@

By providing this data in a web API, it hoped that users can discover new extractors more easily and metadata standards can be developed for the output of extractors to enable schemas to proliferate throughout the field.

This Registry accompanies the Datatractor Schemas ( repository) and API to enable programmatic use and discovery of extractors.

+ class="fa fa-github"> Registry accompanies the Datatractor Schemas ( repository) and API to enable programmatic use and discovery of extractors.

This project proof-of-concept project was devised by the MaRDA Metadata Extractors Working Group, and further discussions and instructions for how to get involved can be found at the discussions meta-repository.

diff --git a/marda_registry/utils.py b/yard/utils.py similarity index 100% rename from marda_registry/utils.py rename to yard/utils.py From 31f442c50535f042c2d488901c2313b5bb2db8e8 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Wed, 15 May 2024 23:07:13 +0100 Subject: [PATCH 2/5] Update the Docker build to pull the git tag for versioning --- Dockerfile | 8 +++++--- yard/__init__.py | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 19dc80f..5846d47 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ from python:3.11-slim-buster +run apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* + env PORT=8000 workdir /app @@ -9,11 +11,11 @@ copy CONTRIBUTING.md LICENSE /app/ copy pyproject.toml /app copy schemas /app/schemas -copy marda_registry /app/marda_registry +copy yard /app/yard copy README.md /app/ # Needed to grab the VCS version from git tags -run pip install . +run --mount=type=bind,target=/app/.git,source=.git pip install . copy tasks.py /app # Regenerate models from the current schemas @@ -22,7 +24,7 @@ run invoke regenerate-models # Validate all entries against the schema run invoke validate-entries -cmd uvicorn marda_registry.app:app --host 0.0.0.0 --port ${PORT} +cmd uvicorn yard.app:app --host 0.0.0.0 --port ${PORT} healthcheck --interval=5m --timeout=3s --start-period=10s \ cmd curl --fail http://localhost:${PORT}/api || exit 1 diff --git a/yard/__init__.py b/yard/__init__.py index 5e0c5ee..409f9ca 100644 --- a/yard/__init__.py +++ b/yard/__init__.py @@ -1,2 +1,4 @@ -__version__ = "0.3.0" +from importlib.metadata import version + +__version__ = version("datatractor-yard") __api_version__ = __version__ From e3d6ce2f1bf6e4609325d972cc1694244db92258 Mon Sep 17 00:00:00 2001 From: Matthew Evans <7916000+ml-evs@users.noreply.github.com> Date: Thu, 16 May 2024 09:47:30 +0100 Subject: [PATCH 3/5] Apply suggestions from code review Co-authored-by: Peter Kraus --- CONTRIBUTING.md | 2 +- yard/app.py | 5 +++-- yard/templates/index.html | 8 +++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 933cd73..c593c16 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,7 @@ An example of the process can be seen in this [pull request](https://github.com/ - You can of course validate your definitions locally, by using the [![Schema](https://badgen.net/static/datatractor/schema/?icon=github)](https://github.com/datatractor/schema/) package, see the [Usage section](https://datatractor.github.io/schema/main/usage.html) of the documentation. ### Testing of Extraction -- You should check that your `yml` definitions can be used to install the `Extractor` and and extract any example files from the supported `FileTypes` locally, by using our [![Schema](https://badgen.net/static/datatractor/beam/?icon=github)](https://github.com/datatractor/beam/) package, see the [Installation](https://github.com/datatractor/beam#installation) and [Usage](https://github.com/datatractor/beam#usage) sections. +- You should check that your `yml` definitions can be used to install the `Extractor` and and extract any example files from the supported `FileTypes` locally, by using the datatractor [![beam](https://badgen.net/static/datatractor/beam/?icon=github)](https://github.com/datatractor/beam/) package, see the [Installation](https://github.com/datatractor/beam#installation) and [Usage](https://github.com/datatractor/beam#usage) sections. > At the moment, the above CI is not checking whether the extraction of the example files actually works. See [this issue](https://github.com/datatractor/yard/issues/9) for further details. diff --git a/yard/app.py b/yard/app.py index 8dd3e93..0226e4a 100644 --- a/yard/app.py +++ b/yard/app.py @@ -10,15 +10,16 @@ from fastapi.responses import HTMLResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates -from yard import __api_version__ from pydantic import BaseModel +from yard import __api_version__ + from .models import Extractor, FileType from .utils import load_registry_collection app = FastAPI( title="Datatractor Yard API", - description=f"This server implements v{__api_version__} of the [MaRDA extractors WG](https://github.com/marda-alliance/metadata_extractors) registry API.", # noqa: E501 + description=f"This server implements v{__api_version__} of the [Datatractor Yard](https://github.com/datatractor/yard/) API.", # noqa: E501 version=__api_version__, ) diff --git a/yard/templates/index.html b/yard/templates/index.html index 106e321..8f4c284 100644 --- a/yard/templates/index.html +++ b/yard/templates/index.html @@ -5,12 +5,10 @@

The idea of this site is to collect various file formats used in materials science and chemistry, describe them with metadata, and provide links to software projects that can parse them.

By providing this data in a web API, it hoped that users can discover new extractors more easily and metadata standards can be developed for the output of extractors to enable schemas to proliferate throughout the field.

-

This Registry accompanies the Datatractor Schemas ( repository) and API to enable programmatic use and discovery of extractors.

-

This project proof-of-concept project was devised by the MaRDA Metadata Extractors Working Group, and further discussions and instructions for how to get involved can be found at the discussions meta-repository.

+

This registry uses the Datatractor Schemas ( repository) and enables programmatic use and discovery of extractors, as shown in our reference implementation.

+ +

The proof-of-concept of this project was devised as part of the MaRDA Metadata Extractors Working Group, and is continued as the Datatractor project. Further discussions and instructions for how to get involved can be found at the Datatractor discussions page.

The registry and the data within it is available under the terms of the MIT license.

From b32af61e06d88d408bfb3ef8e5f397beb115b858 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Thu, 16 May 2024 09:59:03 +0100 Subject: [PATCH 4/5] Refactor CSS colours for easier modification --- yard/static/style.css | 60 ++++++++++++++++++++++----------------- yard/templates/index.html | 4 +-- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/yard/static/style.css b/yard/static/style.css index 091d97f..f44bf93 100644 --- a/yard/static/style.css +++ b/yard/static/style.css @@ -10,6 +10,14 @@ h1 { text-underline-offset: 10px; } +:root { + --primary-color: #58D0BB; + --secondary-color: #FFAB40; + --tertiary-color: #EEFF41; + --body-color: #F8F8F8; + --background-color: #0B1B28; +} + html { margin: 0; padding: 0; @@ -25,7 +33,7 @@ body { height: 100%; margin: 0; padding: 0; - color: #F8F8F8; + color: var(--body-color); font-family: Iosevka Web; font-size: 12pt; } @@ -44,7 +52,7 @@ body { } .logo:hover { - filter: drop-shadow(0px 0px 20px #58D0BB) saturate(150%); + filter: drop-shadow(0px 0px 20px var(--primary-color)) saturate(150%); } .header-title { @@ -54,7 +62,7 @@ body { } .container { - background-color: #0B1B28E3; + background-color: var(--background-color); min-height: 100%; min-height: 100vh; max-width: 900px; @@ -77,35 +85,35 @@ body { } .filetype { - border: 2px solid #EEFF41; + border: 2px solid var(--tertiary-color); } .extractor { - border: 2px solid #FFAB40; + border: 2px solid var(--secondary-color); } .filetype:hover { - box-shadow: 0px 0px 6px 1px #EEFF41; + box-shadow: 0px 0px 6px 1px var(--tertiary-color); } .extractor:hover { - box-shadow: 0px 0px 6px 1px #FFAB40; + box-shadow: 0px 0px 6px 1px var(--secondary-color); } .ft { - color: #EEFF41; + color: var(--tertiary-color); } .ex { - color: #FFAB40; + color: var(--secondary-color); } .read-more { - color: #F8F8F8; + color: var(--body-color); } .ft-id, .ft-id > a { - color: #EEFF41; + color: var(--tertiary-color); border-radius: 5px; padding: 2px; font-family: monospace; @@ -113,7 +121,7 @@ body { } .ex-id, .ex-id > a { - color: #FFAB40; + color: var(--secondary-color); border-radius: 5px; padding: 2px; font-family: monospace; @@ -152,36 +160,36 @@ body { } a { - color: #F8F8F8; + color: var(--body-color); } p > a{ - color: #58D0BB; + color: var(--primary-color); text-decoration: none; } p > a:hover{ - color: #F8F8F8; - box-shadow: 0px 0px 6px 1px #58D0BB; + color: var(--body-color); + box-shadow: 0px 0px 6px 1px var(--primary-color); text-decoration: none; font-weight: bold; } .navbar a { display: block; - color: #58D0BB; + color: var(--primary-color); text-align: center; padding: 15px; text-decoration: none; - border: 2px solid #58D0BB; + border: 2px solid var(--primary-color); } .navbar a:hover { text-decoration: none; font-weight: bold; - color: #F8F8F8; - box-shadow: 0px 0px 6px 2px #58D0BB; + color: var(--body-color); + box-shadow: 0px 0px 6px 2px var(--primary-color); } ::view-transition-old(root), @@ -190,21 +198,21 @@ p > a:hover{ } .ex-link > a { - color: #FFAB40; - border: 2px solid #FFAB40; + color: var(--secondary-color); + border: 2px solid var(--secondary-color); } .ex-link > a:hover { - box-shadow: 0px 0px 6px 2px #FFAB40; + box-shadow: 0px 0px 6px 2px var(--secondary-color); } .ft-link > a:hover { - box-shadow: 0px 0px 6px 2px #EEFF41; + box-shadow: 0px 0px 6px 2px var(--tertiary-color); } .ft-link > a { - color: #EEFF41; - border: 2px solid #EEFF41; + color: var(--tertiary-color); + border: 2px solid var(--tertiary-color); } diff --git a/yard/templates/index.html b/yard/templates/index.html index 8f4c284..693ba17 100644 --- a/yard/templates/index.html +++ b/yard/templates/index.html @@ -6,9 +6,9 @@

The idea of this site is to collect various file formats used in materials science and chemistry, describe them with metadata, and provide links to software projects that can parse them.

By providing this data in a web API, it hoped that users can discover new extractors more easily and metadata standards can be developed for the output of extractors to enable schemas to proliferate throughout the field.

-

This registry uses the Datatractor Schemas ( repository) and enables programmatic use and discovery of extractors, as shown in our reference implementation.

+

This registry uses the Datatractor Schemas ( repository) and enables programmatic use and discovery of extractors, as shown in our reference implementation.

-

The proof-of-concept of this project was devised as part of the MaRDA Metadata Extractors Working Group, and is continued as the Datatractor project. Further discussions and instructions for how to get involved can be found at the Datatractor discussions page.

+

The proof-of-concept of this project was devised as part of the MaRDA Metadata Extractors Working Group, and is continued as the Datatractor project. Further discussions and instructions for how to get involved can be found at the Datatractor discussions page.

The registry and the data within it is available under the terms of the MIT license.

From dbc981510ebb414eac3cd7e459a9ba48a2074e66 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Thu, 16 May 2024 10:15:29 +0100 Subject: [PATCH 5/5] Rebranding of colours --- yard/static/background.svg | 34 ++++++++++++++++++++++------------ yard/static/style.css | 9 +++++---- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/yard/static/background.svg b/yard/static/background.svg index 88fc337..9c177d3 100644 --- a/yard/static/background.svg +++ b/yard/static/background.svg @@ -1,15 +1,25 @@ - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/yard/static/style.css b/yard/static/style.css index f44bf93..aba78bb 100644 --- a/yard/static/style.css +++ b/yard/static/style.css @@ -11,11 +11,12 @@ h1 { } :root { - --primary-color: #58D0BB; - --secondary-color: #FFAB40; - --tertiary-color: #EEFF41; + --primary-color: GhostWhite; + --secondary-color: LightCoral; + --tertiary-color: #58D0BB; --body-color: #F8F8F8; --background-color: #0B1B28; + --highlight-color: LemonChiffon; } html { @@ -164,7 +165,7 @@ a { } p > a{ - color: var(--primary-color); + color: var(--highlight-color); text-decoration: none; }