Merge pull request #53 from linkml/format-utils

format utils

Author: cmungall

.gitignore

@@ -137,3 +137,5 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+**/.claude/settings.local.json

poetry.lock

@@ -31,6 +31,7 @@ py2neo = { version="*", optional = true }
 networkx = { version="*", optional = true }
 #chromadb = { version="*", optional = true }
 pyarrow = { version="*", optional = true }
+pyreadr = { version="*", optional = true }
 h5py = { version="*", optional = true }
 scipy = { version="*", optional = true }
 scikit-learn = { version="*", optional = true }
@@ -49,6 +50,7 @@ fastapi = { version="*", optional = true }
 uvicorn = { version="*", optional = true }
 xmltodict = ">=0.13.0"
 jsonpatch = ">=1.33"
+jsonpath-ng = "*"
 python-dotenv = "^1.0.1"
 
 [tool.poetry.group.dev.dependencies]
@@ -90,6 +92,7 @@ neo4j = ["neo4j", "py2neo", "networkx"]
 #chromadb = ["chromadb"]
 h5py = ["h5py"]
 pyarrow = ["pyarrow"]
+pyreadr = ["pyreadr"]
 validation = ["linkml"]
 map = ["linkml_map"]
 renderer = ["linkml_renderer"]

pyproject.toml

@@ -25,6 +25,7 @@
 from linkml_store.utils.format_utils import Format, guess_format, load_objects, render_output, write_output
 from linkml_store.utils.object_utils import object_path_update
 from linkml_store.utils.pandas_utils import facet_summary_to_dataframe_unmelted
+from linkml_store.plotting.cli import plot_cli
 
 DEFAULT_LOCAL_CONF_PATH = Path("linkml.yaml")
 # global path is ~/.linkml.yaml in the user's home directory
@@ -205,9 +206,10 @@ def drop(ctx):
 @click.option("--format", "-f", type=format_choice, help="Input format")
 @click.option("--object", "-i", multiple=True, help="Input object as YAML")
 @click.option("--source-field", help="If provided, inject file path source as this field")
+@click.option("--glob-files/--no-glob-files", default=False, show_default=True, help="If true, use glob to find files")
 @json_select_query_option
 @click.pass_context
-def insert(ctx, files, replace, object, format, source_field, json_select_query):
+def insert(ctx, files, glob_files, replace, object, format, source_field, json_select_query):
     """Insert objects from files (JSON, YAML, TSV) into the specified collection.
 
     Using a configuration:
@@ -216,6 +218,11 @@ def insert(ctx, files, replace, object, format, source_field, json_select_query)
 
     Note: if you don't provide a schema this will be inferred, but it is
     usually better to provide an explicit schema
+
+    You can use --glob-files if the list of files is too long
+
+        linkml-store -C config.yaml -c genes insert "data/genes/*.json" --glob-files
+
     """
     settings = ctx.obj["settings"]
     collection = settings.collection
@@ -226,7 +233,15 @@ def insert(ctx, files, replace, object, format, source_field, json_select_query)
     load_objects_args = {}
     if json_select_query:
         load_objects_args["select_query"] = json_select_query
+    if glob_files:
+        import glob
+        new_files = []
+        for file_path in files:
+            new_files.extend(glob.glob(file_path))
+        logger.info(f"Found {len(new_files)} files matching glob pattern {files}")
+        files = new_files
     for file_path in files:
+        
         if format:
             objects = load_objects(file_path, format=format, **load_objects_args)
         else:
@@ -486,12 +501,14 @@ def fq(ctx, where, limit, columns, output_type, wide, output, **kwargs):
 
     Nested columns:
 
-        linkml-store -d phenopackets fq subject.timeAtLastEncounter.age
+        linkml-store -d phenopackets fq -S subject.timeAtLastEncounter.age
 
     Compound keys:
 
         linkml-store -d phenopackets fq subject.sex+subject.timeAtLastEncounter.age
 
+    (TODO: compound keys do not work on solr)
+
     """
     collection = ctx.obj["settings"].collection
     where_clause = yaml.safe_load(where) if where else None
@@ -948,5 +965,7 @@ def validate(ctx, output_type, output, collection_only, **kwargs):
         click.echo(output_data)
 
 
+cli.add_command(plot_cli, name="plot")
+
 if __name__ == "__main__":
     cli()

src/linkml_store/cli.py

@@ -44,6 +44,7 @@ class Format(Enum):
     DAT = "dat"
     MARKDOWN = "markdown"
     PKL = "pkl"
+    RDS = "rds"
     PYTHON = "python"
     PARQUET = "parquet"
     HDF5 = "hdf5"
@@ -195,6 +196,9 @@ def process_file(
         objs = xmltodict.parse(f.read())
     elif format == Format.PKL:
         objs = pd.read_pickle(f).to_dict(orient="records")
+    elif format == Format.RDS:
+        import pyreadr
+        objs = pyreadr.read_r(f)
     elif format == Format.XLSX:
         xls = pd.ExcelFile(f)
         objs = {sheet: clean_nested_structure(xls.parse(sheet).to_dict(orient="records")) for sheet in xls.sheet_names}
@@ -349,21 +353,25 @@ def load_objects(
             all_objects = process_file(f, format, expected_type, header_comment_token)
 
     logger.debug(f"Loaded {len(all_objects)} objects from {file_path}")
-    if select_query:
-        import jsonpath_ng as jp
-
-        path_expr = jp.parse(select_query)
-        new_objs = []
-        for obj in all_objects:
-            for match in path_expr.find(obj):
-                logging.debug(f"Match: {match.value}")
-                if isinstance(match.value, list):
-                    new_objs.extend(match.value)
-                else:
-                    new_objs.append(match.value)
-        all_objects = new_objs
+    all_objects = transform_objects(all_objects, select_query)
     return all_objects
 
+def transform_objects(all_objects: List[Dict[str, Any]], select_query: Optional[str]) -> List[Dict[str, Any]]:
+    if not select_query:
+        return all_objects
+    import jsonpath_ng as jp
+
+    path_expr = jp.parse(select_query)
+    new_objs = []
+    for obj in all_objects:
+        for match in path_expr.find(obj):
+            logging.debug(f"Match: {match.value}")
+            if isinstance(match.value, list):
+                new_objs.extend(match.value)
+            else:
+                new_objs.append(match.value)
+    all_objects = new_objs
+    return all_objects
 
 def write_output(
     data: Union[List[Dict[str, Any]], Dict[str, Any], pd.DataFrame],

src/linkml_store/utils/format_utils.py

@@ -6,6 +6,7 @@
 
 import pytest
 import yaml
+from linkml_store.utils.format_utils import Format, load_objects, render_output, transform_objects
 
 from linkml_store.utils.format_utils import Format, load_objects, render_output
 from tests.conftest import CSV_FILE, JSON_FILE, TEST_DATA, TSV_FILE, YAML_FILE
@@ -133,3 +134,34 @@ def test_load_objects_from_tgz():
             for loaded_obj in matching_objects:
                 assert loaded_obj["name"] == original_obj["name"]
                 assert int(loaded_obj["age"]) == int(original_obj["age"])  # Convert to int for comparison
+
+OBJS = [
+    {
+        "id": "P1",
+         "address": {
+             "street": "1 oak st",
+             "city": "Oakland",
+         },
+     },
+     {
+        "id": "P2",
+         "address": {
+             "street": "2 spruce st",
+             "city": "Spruceland",
+         },
+     },
+]
+
+@pytest.mark.parametrize(
+    "objects, select_expr, expected",
+    [
+        ([], None, []),
+        ([], "x", []),
+        (OBJS, None, OBJS),
+        (OBJS, "id", ["P1", "P2"]),
+        (OBJS, "address.city", ["Oakland", "Spruceland"]),
+]
+)
+def test_transform_objects(objects, select_expr, expected):
+    tr_objects = transform_objects(objects, select_expr)
+    assert tr_objects == expected