Allow multiple endpoints in a .servicex file (#103)

- Add .servicex entries for doing multiple end-points
- Update the documentation

Fixes #101

This is a breaking change. What worked before will no longer work!

Author: gordonwatts

.vscode/settings.json

@@ -16,6 +16,7 @@
         "Reconstructor",
         "STDM",
         "SXPASS",
+        "SXTYPE",
         "SXUSER",
         "Servivce",
         "Topo",
@@ -39,6 +40,7 @@
         "leftfoot",
         "linq",
         "localds",
+        "mcrn",
         "minio",
         "miniouser",
         "mino",

README.md

@@ -38,14 +38,17 @@ If no endpoint is specified, then the library defaults to the developer endpoint
 Create a `.servicex` file, in the `yaml` format, in the appropriate place for your work that contains the following:
 
 ```yaml
-api_endpoint:
-  endpoint: <your-endpoint>
-  email: <api-email>
-  password: <api-password>
+api_endpoints:
+  - endpoint: <your-endpoint>
+    email: <api-email>
+    password: <api-password>
+    type: xaod
 ```
 
 All strings are expanded using python's [os.path.expand](https://docs.python.org/3/library/os.path.html#os.path.expandvars) method - so `$NAME` and `${NAME}` will work to expand existing environment variables.
 
+You can list multiple end points by repeating the block of 4 dictionary items, but using a different type. For example, `uproot`.
+
 Finally, you can create the objects `ServiceXAdaptor` and `MinioAdaptor` by hand in your code, passing them as arguments to `ServiceXDataset` and inject custom endpoints and credentials, avoiding the configuration system. This is probably only useful for advanced users.
 
 ## Usage
@@ -56,7 +59,7 @@ The following lines will return a `pandas.DataFrame` containing all the jet pT's
     from servicex import ServiceX
     query = "(call ResultTTree (call Select (call SelectMany (call EventDataset (list 'localds:bogus')) (lambda (list e) (call (attr e 'Jets') 'AntiKt4EMTopoJets'))) (lambda (list j) (/ (call (attr j 'pt')) 1000.0))) (list 'JetPt') 'analysis' 'junk.root')"
     dataset = "mc15_13TeV:mc15_13TeV.361106.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Zee.merge.DAOD_STDM3.e3601_s2576_s2132_r6630_r6264_p2363_tid05630052_00"
-    ds = ServiceXDataset(dataset)
+    ds = ServiceXDataset(dataset, 'xaod')
     r = ds.get_data_pandas_df(query)
     print(r)
 ```
@@ -140,6 +143,7 @@ Everything is based around the `ServiceXDataset` object. Below is the documentat
 
 ```python
   ServiceXDataset(dataset: str,
+                 backend_type: Optional[str] = None,
                  image: str = 'sslhep/servicex_func_adl_xaod_transformer:v0.4',
                  max_workers: int = 20,
                  servicex_adaptor: ServiceXAdaptor = None,
@@ -148,12 +152,17 @@ Everything is based around the `ServiceXDataset` object. Below is the documentat
                  status_callback_factory: Optional[StatusUpdateFactory] = _run_default_wrapper,
                  local_log: log_adaptor = None,
                  session_generator: Callable[[], Awaitable[aiohttp.ClientSession]] = None,
-                 config_adaptor: ConfigView = None)
+                 config_adaptor: ConfigView = None):
+  '''
       Create and configure a ServiceX object for a dataset.
 
       Arguments
 
           dataset                     Name of a dataset from which queries will be selected.
+          backend_type                The type of backend. Used only if we need to find an
+                                      end-point. If we do not have a `servicex_adaptor` then this
+                                      cannot be null. Possible types are `uproot`, `xaod`,
+                                      and anything that finds a match in the `.servicex` file.
           image                       Name of transformer image to use to transform the data
           max_workers                 Maximum number of transformers to run simultaneously on
                                       ServiceX.
@@ -183,6 +192,7 @@ Everything is based around the `ServiceXDataset` object. Below is the documentat
              takes `(total_files, transformed, downloaded, skipped)` as an argument. The
              `total_files` parameter may be `None` until the system knows how many files need to
              be processed (and some files can even be completed before that is known).
+  '''
  ```
 
 To get the data use one of the `get_data` method. They all have the same API, differing only by what they return.

scripts/run_test.py

@@ -12,6 +12,7 @@
 def run_query(endpoint: Optional[ServiceXAdaptor]) -> None:
     ds = ServiceXDataset(
         "mc16_13TeV:mc16_13TeV.361106.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Zee.deriv.DAOD_STDM3.e3601_e5984_s3126_r10201_r10210_p3975_tid20425969_00",  # NOQA
+        backend_type='xaod',
         max_workers=100,
         servicex_adaptor=endpoint)
 

servicex/servicex.py

@@ -37,6 +37,7 @@ class ServiceXDataset(ServiceXABC):
     '''
     def __init__(self,
                  dataset: str,
+                 backend_type: Optional[str] = None,
                  image: str = 'sslhep/servicex_func_adl_xaod_transformer:v1.0.0-rc.2',  # NOQA
                  max_workers: int = 20,
                  servicex_adaptor: ServiceXAdaptor = None,
@@ -52,6 +53,10 @@ def __init__(self,
         Arguments
 
             dataset                     Name of a dataset from which queries will be selected.
+            backend_type                The type of backend. Used only if we need to find an
+                                        end-point. If we do not have a `servicex_adaptor` then this
+                                        cannot be null. Possible types are `uproot`, `xaod`,
+                                        and anything that finds a match in the `.servicex` file.
             image                       Name of transformer image to use to transform the data
             max_workers                 Maximum number of transformers to run simultaneously on
                                         ServiceX.
@@ -86,6 +91,10 @@ def __init__(self,
                              status_callback_factory,
                              )
 
+        # Make sure the arguments are reasonable
+        if backend_type is None and servicex_adaptor is None:
+            raise ServiceXException('Specify backend_type or servicex_adaptor')
+
         # Get the local settings
         config = config_adaptor if config_adaptor is not None \
             else ConfigSettings('servicex', 'servicex')
@@ -96,7 +105,9 @@ def __init__(self,
             else cache_adaptor
 
         if not servicex_adaptor:
-            servicex_adaptor = servicex_adaptor_factory(config)
+            # Given servicex adaptor is none, this should be ok. Fixes type checkers
+            assert backend_type is not None
+            servicex_adaptor = servicex_adaptor_factory(config, backend_type)
         self._servicex_adaptor = servicex_adaptor
 
         if not minio_adaptor:

servicex/servicex_adaptor.py

@@ -20,17 +20,6 @@
 servicex_status_poll_time = 5.0
 
 
-def servicex_adaptor_factory(c: ConfigView):
-    # It is an error if this is not specified somewhere.
-    endpoint = c['api_endpoint']['endpoint'].as_str_expanded()
-
-    # We can default these to "None"
-    email = c['api_endpoint']['email'].as_str_expanded() if 'email' in c['api_endpoint'] else None
-    password = c['api_endpoint']['password'].as_str_expanded() if 'password' in c['api_endpoint'] \
-        else None
-    return ServiceXAdaptor(endpoint, email, password)
-
-
 # Low level routines for interacting with a ServiceX instance via the WebAPI
 class ServiceXAdaptor:
     def __init__(self, endpoint, email=None, password=None):
@@ -260,3 +249,35 @@ async def trap_servicex_failures(stream: AsyncIterator[TransformTuple]) \
                                               f'processed: {processed}).')
 
         yield p
+
+
+def servicex_adaptor_factory(c: ConfigView, backend_type: str) -> ServiceXAdaptor:
+    '''Given a configuration and the backend, find an appropriate configuration
+    for us to grab and create a `servicex_adaptor`.
+
+    Args:
+        c (ConfigView): The config information loaded form files.
+        backend_type (str): The backend type we need to match
+
+    Returns:
+        [ServiceXAdaptor]: A servicex adaptor.
+    '''
+    # Find a list of all endpoints.
+    # It is an error if this is not specified somewhere.
+    endpoints = c['api_endpoints']
+    seen_types = []
+    for ep in endpoints:
+        if ep['type'].as_str_expanded() == backend_type:
+            endpoint = ep['endpoint'].as_str_expanded()
+            email = ep['email'].as_str_expanded() if 'email' in ep else None
+            password = ep['password'].as_str_expanded() if 'password' in ep \
+                else None
+
+            # We can default these to "None"
+            return ServiceXAdaptor(endpoint, email, password)
+        else:
+            seen_types.append(ep['type'].as_str_expanded())
+
+    # If we are here, we found no matching type.
+    raise ServiceXException(f'Unable to find type {backend_type} '
+                            f'in configuration. Saw: {", ".join(seen_types)}')

tests/test_servicex.py

@@ -1,5 +1,6 @@
 import asyncio
 from pathlib import Path
+from servicex.servicex_adaptor import servicex_adaptor_factory
 from typing import Optional
 
 import pandas as pd
@@ -22,8 +23,23 @@ def clean_fname(fname: str):
                 .replace(':', '_')
 
 
-def test_default_ctor():
-    fe.ServiceXDataset('localds://dude')
+def test_default_ctor(mocker):
+    '''Test the default ctor. This requires that a .servicex file be present to work.
+    '''
+    factory = mocker.MagicMock(spec=servicex_adaptor_factory)
+    mocker.patch('servicex.servicex.servicex_adaptor_factory', factory)
+
+    fe.ServiceXDataset('localds://dude', "uproot-ftw")
+
+    factory.assert_called_once()
+    assert factory.call_args[0][1] == 'uproot-ftw'
+
+
+def test_default_ctor_no_type():
+    with pytest.raises(ServiceXException) as e:
+        fe.ServiceXDataset('localds://dude')
+
+    assert "type" in str(e.value)
 
 
 @pytest.mark.asyncio

tests/test_servicex_adaptor.py

@@ -501,30 +501,61 @@ def test_servicex_adaptor_settings():
     from confuse import Configuration
     c = Configuration('bogus', 'bogus')
     c.clear()
-    c['api_endpoint']['endpoint'] = 'http://my-left-foot.com:5000'
-    c['api_endpoint']['email'] = 'thegoodplace@example.com'
-    c['api_endpoint']['password'] = 'forkingshirtballs'
-
-    sx = servicex_adaptor_factory(c)
+    c['api_endpoints'] = [
+        {
+            'type': 'my-type',
+            'endpoint': 'http://my-left-foot.com:5000',
+            'email': 'thegoodplace@example.com',
+            'password': 'forkingshirtballs',
+        }
+    ]
+
+    sx = servicex_adaptor_factory(c, 'my-type')
     assert sx._endpoint == 'http://my-left-foot.com:5000'
     assert sx._email == 'thegoodplace@example.com'
     assert sx._password == 'forkingshirtballs'
 
 
+def test_servicex_adaptor_settings_wrong_type():
+    from confuse import Configuration
+    c = Configuration('bogus', 'bogus')
+    c.clear()
+    c['api_endpoints'] = [
+        {
+            'type': 'my-type',
+            'endpoint': 'http://my-left-foot.com:5000',
+            'email': 'thegoodplace@example.com',
+            'password': 'forkingshirtballs',
+        }
+    ]
+
+    with pytest.raises(ServiceXException) as e:
+        servicex_adaptor_factory(c, 'your-type')
+
+    assert 'Unable to find type' in str(e.value)
+    assert 'my-type' in str(e.value)
+
+
 def test_servicex_adaptor_settings_env():
     from confuse import Configuration
     c = Configuration('bogus', 'bogus')
     c.clear()
-    c['api_endpoint']['endpoint'] = '${ENDPOINT}:5000'
-    c['api_endpoint']['email'] = '${SXUSER}'
-    c['api_endpoint']['password'] = '${SXPASS}'
+    c['api_endpoints'] = [
+        {
+            'type': '${SXTYPE}',
+            'endpoint': '${ENDPOINT}:5000',
+            'email': '${SXUSER}',
+            'password': '${SXPASS}',
+        }
+    ]
 
     from os import environ
     environ['ENDPOINT'] = 'http://tachi.com'
     environ['SXUSER'] = 'Holden'
     environ['SXPASS'] = 'protomolecule'
+    environ['SXTYPE'] = 'mcrn'
 
-    sx = servicex_adaptor_factory(c)
+    sx = servicex_adaptor_factory(c, 'mcrn')
     assert sx._endpoint == 'http://tachi.com:5000'
     assert sx._email == 'Holden'
     assert sx._password == 'protomolecule'