Assorted fixes for disorder and reduced formulae

Dockerfile

@@ -183,9 +183,14 @@ if [ -z "$CSD_ACTIVATION_KEY" ]; then
  exit 1
 fi
 
-gpg --batch --passphrase ${CSD_ACTIVATION_KEY} --decrypt /opt/csd-optimade/csd-optimade.jsonl.gz.gpg | gunzip > /opt/csd-optimade/csd-optimade.jsonl
+if [ "$CSD_OPTIMADE_INSERT" = "1" ] || [ "$CSD_OPTIMADE_INSERT" = "true" ]; then
+    # Run the API twice: once to wipe and reinsert the data then exit, the second to run the API
+    (gpg --batch --passphrase ${CSD_ACTIVATION_KEY} --decrypt /opt/csd-optimade/csd-optimade.jsonl.gz.gpg | gunzip > /opt/csd-optimade/csd-optimade.jsonl &&
+    exec uv run --no-sync csd-serve --port 5001 --exit-after-insert --drop-first /opt/csd-optimade/csd-optimade.jsonl) &
+fi
+
+exec uv run --no-sync csd-serve --no-insert /opt/csd-optimade/csd-optimade.jsonl
 
-exec uv run --no-sync csd-serve --drop-first /opt/csd-optimade/csd-optimade.jsonl
 EOF
 
 RUN chmod +x /entrypoint.sh

README.md

@@ -113,7 +113,7 @@ Buildx.
 Once configured, you can build the container with
 
 ```shell
-docker build --secret id=env,src=.env -t csd-optimade .
+docker build --secret id=env,src=.env --target csd-optimade-server -t csd-optimade-server .
 ```
 
 This will install the CSD inside the container, run the ingestion pipeline and
@@ -124,11 +124,45 @@ To launch the container (which will decrypt the file and start the OPTIMADE
 API locally):
 
 ```shell
-docker run --env-file .env -p 5000:5000 csd-optimade
+docker run --env-file .env -p 5000:5000 csd-optimade-server
 ```
 
-For development, you may prefer to use the bake definitions in
-`docker-bake.hcl` to build and tag the relevant build stages.
+If using a persistent database, future runs of the API can be controlled with
+the `CSD_OPTIMADE_INSERT` environment variable. If `true`, the configured database will be
+
+
+For development and deployment, you may prefer to use the bake definitions in
+`docker-bake.hcl` to build and tag the relevant build stages:
+
+```shell
+docker buildx bake csd-optimade-server
+docker run --env-file .env -p 5000:5000 ghcr.io/datalab-industries/csd-optimade-server
+```
+
+### Runtime configuration options
+
+As noted above, the `CSD_ACTIVATION_KEY` used to build the container must be provided at runtime.
+
+The API container can also be configured with all the `OPTIMAKE_` prefixed environment variables.
+
+The most important ones are listed here:
+
+- `OPTIMAKE_MONGO_URI`: to use a persistent MongoDB backend, you can provide a `MONGO_URI` via:
+
+  ```shell
+  OPTIMAKE_DATABSE_BACKEND=mongodb
+  OPTIMAKE_MONGO_URI=mongodb://mongodb_server:27017/optimade
+  ```
+
+- `OPTIMAKE_BASE_URL`: to set the base URL of the API (used to generate pagination links), you can provide a `BASE_URL` via:
+
+  ```shell
+  OPTIMAKE_BASE_URL=https://my-csd-deployment.com
+  ```
+
+Finally, if using a persistent database, future runs of the API can be controlled with the `CSD_OPTIMADE_INSERT` environment variable.
+If `true` (default), the configured database will be wiped and rebuilt from the JSONL file directly, and a separate process will run the API.
+If `false`, only the API will be started, with no database rebuild.
 
 ## Contributing and Getting Help
 

pyproject.toml

@@ -20,7 +20,7 @@ classifiers = [
 ]
 requires-python = ">= 3.11, < 3.12"
 dependencies = [
-    "optimade @ git+https://github.com/Materials-Consortia/optimade-python-tools.git@ml-evs/jsonl-relationships-links",
+    "optimade @ git+https://github.com/Materials-Consortia/optimade-python-tools.git",
     "optimade-maker @ git+https://github.com/materialscloud-org/optimade-maker.git@ml-evs/qol-server",
     "tqdm ~= 4.66",
     "pymongo >= 4, < 5",
@@ -80,6 +80,4 @@ testpaths = "tests"
 addopts = "-rs"
 filterwarnings = [
     "error",
-    "ignore:.*total_num_atoms.*:RuntimeWarning",
-    "ignore:.*unable to reduce formula.*:UserWarning"
 ]

src/csd_optimade/fields.py

@@ -1,3 +1,7 @@
+from optimade import __api_version__
+from optimade.models.baseinfo import BaseInfoAttributes, BaseInfoResource
+
+
 def generate_csd_provider_fields():
     return {
         "structures": [
@@ -92,8 +96,43 @@ def generate_csd_provider_fields():
             },
             {
                 "name": "_csd_z_prime",
-                "type": "integer",
+                "type": "float",
                 "description": "The number of formula units in the asymmetric unit.",
             },
+            {
+                "name": "_csd_remarks",
+                "type": "string",
+                "description": "Free-text remarks about the structure.",
+            },
         ]
     }
+
+
+def generate_csd_provider_info():
+    return {
+        "prefix": "csd",
+        "name": "Cambridge Structural Database",
+        "description": "A database of crystal structures curated by the Cambridge Crystallographic Data Centre.",
+        "homepage=": "https://www.ccdc.cam.ac.uk",
+    }
+
+
+def generate_license_link():
+    return "https://www.ccdc.cam.ac.uk/licence-agreement"
+
+
+def generate_csd_info_endpoint() -> dict[str, BaseInfoResource]:
+    return {
+        "data": BaseInfoResource(
+            attributes=BaseInfoAttributes(
+                api_version=__api_version__,
+                available_api_versions=[],
+                formats=["json"],
+                available_endpoints=["info", "structures", "references"],
+                entry_types_by_format={"json": ["info", "structures", "references"]},
+                is_index=False,
+                license={"href": generate_license_link()},
+                available_licenses=None,
+            )
+        )
+    }

src/csd_optimade/ingest.py

@@ -1,8 +1,17 @@
 from __future__ import annotations
 
+from optimade import __api_version__
+
+from csd_optimade.fields import (
+    generate_csd_info_endpoint,
+    generate_csd_provider_fields,
+    generate_csd_provider_info,
+)
+
 BAD_IDENTIFIERS = {
     "QIJZOB",  # hangs infinitely during mapping
     "VOHZIB",  # no 3D structure
+    "YIGKOP",
 }
 
 import glob
@@ -189,13 +198,38 @@ def cli():
     with open(tmp_jsonl_path) as tmp_jsonl:
         ids_by_type: dict[str, set] = {}
         with open(output_file, "w") as final_jsonl:
-            # Write headers
+            # Write headers and info endpoints
             final_jsonl.write(
-                json.dumps({"x-optimade": {"meta": {"api_version": "1.1.0"}}}) + "\n"
+                json.dumps({"x-optimade": {"meta": {"api_version": __api_version__}}})
+                + "\n"
+            )
+
+            info = generate_csd_info_endpoint()
+            provider = generate_csd_provider_info()
+            final_jsonl.write(
+                json.dumps(
+                    {
+                        "data": info["data"].model_dump(
+                            exclude_unset=True, exclude_none=False
+                        )
+                    }
+                )
+                + "\n"
             )
             final_jsonl.write(
                 _construct_entry_type_info(
-                    "structures", properties=[], provider_prefix=""
+                    "structures",
+                    properties=generate_csd_provider_fields()["structures"],
+                    provider_prefix=provider["prefix"],
+                ).model_dump_json()
+                + "\n"
+            )
+
+            final_jsonl.write(
+                _construct_entry_type_info(
+                    "references",
+                    properties=[],
+                    provider_prefix=provider["prefix"],
                 ).model_dump_json()
                 + "\n"
             )