Multiple xrootdfiles

.github/workflows/ci.yaml

@@ -51,7 +51,7 @@ jobs:
 
       - name: Test with pytest
         run: |
-          coverage run -m pytest --cov=./ --cov-report=xml
+          coverage run -m pytest tests --cov=./ --cov-report=xml
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3.1.4

examples/config_databinder_func_adl.yaml

@@ -6,7 +6,7 @@ General:
 
 Sample:
   - Name: ggH
-    XRootdFile: DEF_ggH_input
+    XRootDFiles: DEF_ggH_input
     NFiles: 5
     Query: DEF_ttH_nominal_query
 

examples/config_databinder_python.yaml

@@ -17,7 +17,7 @@ Sample:
     Function: DEF_function1
 
   - Name: ggH
-    XRootdFile: DEF_ggH_input
+    XRootDFiles: DEF_ggH_input
     Function: DEF_function2
 
 Definition:

examples/python_codegen.py

@@ -50,7 +50,7 @@ def run_query(input_filenames=None):
     Sample=[
         Sample(
             Name="Python Codegen",
-            XRootdFile="root://eospublic.cern.ch//eos/opendata/atlas/OutreachDatasets/2020-01-22/4lep/MC/mc_345060.ggH125_ZZ4lep.4lep.root",
+            XRootDFiles="root://eospublic.cern.ch//eos/opendata/atlas/OutreachDatasets/2020-01-22/4lep/MC/mc_345060.ggH125_ZZ4lep.4lep.root",
             Function=run_query
         )
     ]

examples/single_file_uproot.py

@@ -58,7 +58,7 @@
     Sample=[
         Sample(
             Name="mc_345060.ggH125_ZZ4lep.4lep",
-            XRootdFile="root://eospublic.cern.ch//eos/opendata/atlas/OutreachDatasets/2020-01-22/4lep/MC/mc_345060.ggH125_ZZ4lep.4lep.root", # NOQA E501
+            XRootDFiles="root://eospublic.cern.ch//eos/opendata/atlas/OutreachDatasets/2020-01-22/4lep/MC/mc_345060.ggH125_ZZ4lep.4lep.root", # NOQA E501
             Query=query
         )
     ]

examples/stress_test.py

@@ -0,0 +1,57 @@
+# Copyright (c) 2022, IRIS-HEP
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import asyncio
+
+from servicex import ServiceXSpec, General, Sample
+from servicex.func_adl.func_adl_dataset import FuncADLQuery
+from servicex.servicex_client import deliver
+
+print(f"Is it running {asyncio.get_event_loop().is_running()}")
+query = FuncADLQuery().Select(lambda e: {'el_pt': e['el_pt']})
+
+spec = ServiceXSpec(
+    General=General(
+        ServiceX="servicex-uc-af",
+        Codegen="uproot",
+        OutputFormat="parquet",
+        # Delivery="LocalCache"
+        Delivery="SignedURLs"
+    ),
+    Sample=[
+        Sample(
+            Name=f"bigger_uproot_{i}",
+            RucioDID="user.kchoi:user.kchoi.fcnc_tHq_ML.ttH.v8",
+            Tree="nominal",
+            Query=query
+        ) for i in range(20)
+    ]
+)
+
+result = deliver(spec)
+for k in result.keys():
+    print(f"key: {k} values: {len(result[k])}")

servicex/databinder_models.py

@@ -27,7 +27,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 from enum import Enum
 from typing import Union, Optional, Callable, List
-from pydantic import BaseModel, Field, AnyUrl, root_validator, constr, validator
+from pydantic import BaseModel, Field, root_validator, constr, validator
 
 from servicex.dataset_identifier import RucioDatasetIdentifier, FileListDataset
 from servicex.func_adl import func_adl_dataset
@@ -37,7 +37,7 @@ class Sample(BaseModel):
     Name: str
     Codegen: Optional[str]
     RucioDID: Optional[str]
-    XRootdFile: Optional[Union[str, AnyUrl]]
+    XRootDFiles: Optional[Union[str, List[str]]]
     NFiles: Optional[int] = Field(default=None)
     Function: Optional[Union[str, Callable]] = Field(default=None)
     Query: Optional[Union[str, func_adl_dataset.Query]] = Field(default=None)
@@ -51,8 +51,8 @@ class Config:
     def dataset_identifier(self):
         if self.RucioDID:
             return RucioDatasetIdentifier(self.RucioDID, num_files=self.NFiles or 0)
-        elif self.XRootdFile:
-            return FileListDataset(self.XRootdFile)
+        elif self.XRootDFiles:
+            return FileListDataset(self.XRootDFiles)
 
     @root_validator
     def validate_did_xor_file(cls, values):
@@ -61,10 +61,10 @@ def validate_did_xor_file(cls, values):
         :param values:
         :return:
         """
-        if values['XRootdFile'] and values['RucioDID']:
-            raise ValueError("Only specify one of XRootdFile or RucioDID, not both.")
-        if not values['XRootdFile'] and not values['RucioDID']:
-            raise ValueError("Must specify one of XRootdFile or RucioDID.")
+        if values['XRootDFiles'] and values['RucioDID']:
+            raise ValueError("Only specify one of XRootDFiles or RucioDID, not both.")
+        if not values['XRootDFiles'] and not values['RucioDID']:
+            raise ValueError("Must specify one of XRootDFiles or RucioDID.")
         return values
 
     @root_validator
@@ -92,8 +92,10 @@ class DeliveryEnum(str, Enum):
 
     ServiceX: str = Field(..., alias="ServiceX")
     Codegen: str
-    OutputFormat: Union[OutputFormatEnum, constr(regex='^(parquet|root-file)$')]  # NOQA F722
-    Delivery: Union[DeliveryEnum, constr(regex='^(LocalCache|SignedURLs)$')] # NOQA F722
+    OutputFormat: Union[OutputFormatEnum,
+        constr(regex='^(parquet|root-file)$')] = Field(default=OutputFormatEnum.root)  # NOQA F722
+
+    Delivery: Union[DeliveryEnum, constr(regex='^(LocalCache|SignedURLs)$')] = Field(default=DeliveryEnum.LocalCache) # NOQA F722
 
 
 class Definition(BaseModel):