Add 'all-keys' REST Query Parameter (#74)

* add AllKeys class, _get_projection(), & _limit_result_list()

* use _get_projection() & _limit_result_list(); type-hint, black-fmt, doc

* keys=None bug fix

* add nack count_files() kwargs

* count_files() bug fix

* add argbuilder.py: move build_files_query() & make build_keys()

* bug fix: kwargs not **kwargs

* test "all-keys" argument

* Revert "test "all-keys" argument"

This reverts commit 8fd2d28.

* test "all-keys" argument

* add build_limit()

* add build_limit() - 2

* add build_limit() - 3

* add build_start()

* comments: "id_" -> "_id"

* reuse logging messages for Exceptions

* body -> args

Author: ric-evans

.circleci/config.yml

@@ -24,7 +24,7 @@ jobs:
             - run: |
                 . env/bin/activate &&
                 resources/enable_profiling.py &&
-                pytest --tb=short &&
+                pytest -vvvvv --tb=short &&
                 resources/profile_queries.py
     deploy:
         docker:

file_catalog/argbuilder.py

@@ -0,0 +1,80 @@
+"""Builder utility functions for arg/kwargs dicts."""
+
+from typing import Any, Dict
+
+from tornado.escape import json_decode
+
+# local imports
+from file_catalog.mongo import AllKeys
+
+
+def build_limit(kwargs: Dict[str, Any], config: Dict[str, Any]) -> None:
+    """Build the `"limit"` argument."""
+    if "limit" in kwargs:
+        kwargs["limit"] = int(kwargs["limit"])
+        if kwargs["limit"] < 1:
+            raise Exception("limit is not positive")
+
+        # check with config
+        if kwargs["limit"] > config["FC_QUERY_FILE_LIST_LIMIT"]:
+            kwargs["limit"] = config["FC_QUERY_FILE_LIST_LIMIT"]
+    else:
+        # if no limit has been defined, set max limit
+        kwargs["limit"] = config["FC_QUERY_FILE_LIST_LIMIT"]
+
+
+def build_start(kwargs: Dict[str, Any]) -> None:
+    """Build the `"start"` argument."""
+    if "start" in kwargs:
+        kwargs["start"] = int(kwargs["start"])
+        if kwargs["start"] < 0:
+            raise Exception("start is negative")
+
+
+def build_files_query(kwargs: Dict[str, Any]) -> None:
+    """Build `"query"` dict with formatted/fully-named arguments.
+
+    Pop corresponding shortcut-keys from `kwargs`.
+    """
+    if "query" in kwargs:
+        # keep whatever was already in here, then add to it
+        if isinstance(kwargs["query"], (str, bytes)):
+            query = json_decode(kwargs.pop("query"))
+        else:
+            query = kwargs.pop("query")
+    else:
+        query = {}
+
+    if "locations.archive" not in query:
+        query["locations.archive"] = None
+
+    # shortcut query params
+    if "logical_name" in kwargs:
+        query["logical_name"] = kwargs.pop("logical_name")
+    if "run_number" in kwargs:
+        query["run.run_number"] = kwargs.pop("run_number")
+    if "dataset" in kwargs:
+        query["iceprod.dataset"] = kwargs.pop("dataset")
+    if "event_id" in kwargs:
+        e = kwargs.pop("event_id")
+        query["run.first_event"] = {"$lte": e}
+        query["run.last_event"] = {"$gte": e}
+    if "processing_level" in kwargs:
+        query["processing_level"] = kwargs.pop("processing_level")
+    if "season" in kwargs:
+        query["offline_processing_metadata.season"] = kwargs.pop("season")
+
+    kwargs["query"] = query
+
+
+def build_keys(kwargs: Dict[str, Any]) -> None:
+    """Build `"keys"` list, potentially using `"all-keys"`.
+
+    Pop `"all-keys"`.
+    """
+    use_all_keys = kwargs.pop("all-keys", None) in ["True", "true", 1]
+
+    if use_all_keys:
+        kwargs["keys"] = AllKeys()
+    elif "keys" in kwargs:
+        kwargs["keys"] = kwargs["keys"].split("|")