Merge pull request #1343 from cta-observatory/dl1dl2tool

DL1 to DL2 tool

Author: moralejo

docs/introduction.rst

@@ -78,7 +78,7 @@ Here is an example configuration file for this step.
 DL1 to DL2
 ----------
 
-Use ``lstchain.scripts.lstchain_dl1_to_dl2`` for real data and MC.
+Use ``lstchain.tools.lstchain_dl1_to_dl2`` for real data and MC.
 
 For more information, try ``--help`` or see the :doc:`lstchain_api/index`.
 

docs/lstchain_api/scripts/index.rst

@@ -14,7 +14,6 @@ The scripts to be executed from the command line are described below:
 * `lstchain_data_create_time_calibration_file`_
 * `lstchain_data_r0_to_dl1`_
 * `lstchain_dl1_muon_analysis`_
-* `lstchain_dl1_to_dl2`_
 * `lstchain_dl1ab`_
 * `lstchain_dump_config`_
 * `lstchain_find_pedestals`_
@@ -99,19 +98,6 @@ Usage
    :func: parser
    :prog: lstchain_dl1_muon_analysis
 
-.. _lstchain_dl1_to_dl2:
-
-lstchain_dl1_to_dl2
-+++++++++++++++++++
-
-.. automodule:: lstchain.scripts.lstchain_dl1_to_dl2
-
-Usage
------
-.. argparse::
-   :module: lstchain.scripts.lstchain_dl1_to_dl2
-   :func: parser
-   :prog: lstchain_dl1_to_dl2
 
 .. _lstchain_dl1ab:
 

docs/lstchain_api/tools/index.rst

@@ -24,3 +24,5 @@ Reference/API
    :no-inheritance-diagram:
 .. automodapi:: lstchain.tools.lstchain_fit_intensity_scan
    :no-inheritance-diagram:
+.. automodapi:: lstchain.tools.lstchain_dl1_to_dl2
+   :no-inheritance-diagram:

lstchain/io/provenance.py

@@ -0,0 +1,79 @@
+import json
+import h5py
+from ctapipe.core import Provenance
+import logging
+
+logger = logging.getLogger()
+
+def write_provenance(hdf5_file_path, stage_name):
+    """
+    Write JSON provenance information to an HDF5 file.
+    It uses the current activity's provenance information and should typically be called within a ctapipe Tool.
+
+    Parameters:
+    -----------
+    hdf5_file_path : str or Path
+        Path to the HDF5 file
+    stage_name : str
+        Name of the stage generating the provenance
+
+    Returns:
+    --------
+    None
+    """
+    try:
+        with h5py.File(hdf5_file_path, 'a') as h5file:
+            if 'provenance' not in h5file:
+                h5file.create_group('provenance')
+            
+            # Get the provenance dictionary from the current activity
+            provenance_data = Provenance().current_activity.provenance
+            # Dump the dictionary to a JSON string and write it to the HDF5 file
+            h5file['provenance'].create_dataset(stage_name, data=json.dumps(provenance_data, default=str))
+    
+    except Exception as e:
+        raise Exception(f"Error writing provenance: {e}")
+
+
+def read_provenance(hdf5_file_path, dataset_name):
+    """
+    Read JSON provenance from HDF5 file's dataset attributes.
+
+    Parameters:
+    -----------
+    hdf5_file_path : str
+        Path to the HDF5 file
+    dataset_name : s
+        Name of the dataset containing provenance
+
+    Returns:
+    --------
+    dict
+        Provenance information as JSON-decoded dictionary
+    """
+    logger.log(logging.INFO, f"reading provenance from {hdf5_file_path}")
+    with h5py.File(hdf5_file_path, 'r') as h5file:
+        if 'provenance' not in h5file:
+            raise ValueError("No provenance found in HDF5 file")
+        elif dataset_name not in h5file['provenance']:
+            raise ValueError(f"No provenance found for {dataset_name}")
+        else:
+            return json.loads(h5file['provenance'][dataset_name][()])
+
+
+def read_dl2_provenance(hdf5_file_path):
+    """
+    Read JSON provenance from HDF5 file's dataset attributes.
+    This function is a wrapper around read_provenance() that reads the provenance for the 'dl2' dataset.
+
+    Parameters:
+    -----------
+    hdf5_file_path : str
+        Path to the HDF5 file
+
+    Returns:
+    --------
+    dict
+        Provenance information as JSON-decoded dictionary
+    """
+    return read_provenance(hdf5_file_path, 'dl1_to_dl2')

lstchain/io/tests/test_provenance.py

@@ -0,0 +1,28 @@
+import pytest
+import h5py
+from lstchain.io.provenance import write_provenance, read_provenance
+from ctapipe.core import Provenance
+
+@pytest.fixture
+def hdf5_file(tmp_path):
+    test_file = tmp_path / 'test_provenance.h5'
+    yield test_file
+
+def test_write_provenance(hdf5_file):
+    stage_name = 'test_stage'
+    p = Provenance()
+    p.add_input_file('input file', role='test input file')
+    p.add_output_file('output file', role='test output file')
+    write_provenance(hdf5_file, stage_name)
+    with h5py.File(hdf5_file, 'r') as h5file:
+        assert 'provenance' in h5file
+        assert stage_name in h5file['provenance']
+
+
+def test_read_provenance(hdf5_file):
+    stage_name = 'test_stage'
+    write_provenance(hdf5_file, stage_name)
+    result = read_provenance(hdf5_file, stage_name)    
+    assert 'activity_name' in result
+    assert 'activity_uuid' in result
+

lstchain/scripts/tests/test_lstchain_scripts.py

@@ -17,6 +17,7 @@
 from ctapipe.io import read_table
 from ctapipe.io import EventSource
 from ctapipe.containers import EventType
+from lstchain.io.provenance import read_dl2_provenance
 
 
 from lstchain.io.config import get_srcdep_config, get_standard_config
@@ -365,6 +366,7 @@ def test_lstchain_merged_dl1_to_dl2(
         "lstchain_dl1_to_dl2",
         "-f",
         simulated_dl1_file_,
+        "-f",
         merged_simulated_dl1_file,
         "-p",
         rf_models["path"],
@@ -385,6 +387,12 @@ def test_lstchain_dl1_to_dl2(simulated_dl2_file):
     assert "reco_disp_dy" in dl2_df.columns
     assert "reco_src_x" in dl2_df.columns
     assert "reco_src_y" in dl2_df.columns
+    
+    prov = read_dl2_provenance(simulated_dl2_file)
+    assert "activity_name" in prov
+    assert "config" in prov
+    assert "path_models" in prov['config']['DL1ToDL2Tool']
+    assert prov['config']['DL1ToDL2Tool']['path_models'] is not None
 
 
 def test_lstchain_dl1_to_dl2_srcdep(simulated_srcdep_dl2_file):