Obey default data return formats (#260)

* Fix #259 - get_awkward now properly requests parquet files from an uproot background, and root files from a xaod backend.
* get_pandas does the same things
* Spell checker was run on multiple source files and spelling updated.
* A global variable in `servicex.py` limits you to what data formats you can request. It is now possible to alter that global variable if you want to add a new format without having to `hack` the package.

Author: gordonwatts

servicex/servicex.py

@@ -106,7 +106,7 @@ def url(self) -> str:
 
     @property
     def bucket(self) -> str:
-        """Returns the buck name - unique and constant accross transformations.
+        """Returns the buck name - unique and constant across transformations.
         Can be used to order the results
 
         Returns:
@@ -166,7 +166,7 @@ def data(self) -> Any:
 
 class ServiceXDataset(ServiceXABC):
     """
-    Used to access an instance of ServiceX at an end point on the internet. Support convieration
+    Used to access an instance of ServiceX at an end point on the internet. Support conversion
     by configuration object `config_adaptor` or by creating the adaptors defined in the `__init__`
     function.
     """
@@ -210,7 +210,7 @@ def __init__(
                                         Defaults to object-store, but could be used to save
                                         results to a posix volume
             servicex_adaptor            Object to control communication with the servicex instance
-                                        at a particular ip address with certian login credentials.
+                                        at a particular ip address with certain login credentials.
                                         Will be configured via the `config_adaptor` by default.
             minio_adaptor               Object to control communication with the minio servicex
                                         instance.
@@ -347,7 +347,7 @@ async def get_data_rootfiles_stream(
         access the data.
 
         Args:
-            selection_query (str): The `qastle` query for the data to retreive.
+            selection_query (str): The `qastle` query for the data to retrieve.
 
         Yields:
             AsyncIterator[StreamInfoPath]: As ServiceX completes the data, and it is downloaded
@@ -375,7 +375,7 @@ async def get_data_parquet_stream(
         access the data.
 
         Args:
-            selection_query (str): The `qastle` query for the data to retreive.
+            selection_query (str): The `qastle` query for the data to retrieve.
 
         Yields:
             AsyncIterator[StreamInfoPath]: As ServiceX completes the data, and it is downloaded
@@ -393,9 +393,13 @@ async def get_data_parquet_stream(
     async def get_data_pandas_df_async(
         self, selection_query: str, title: Optional[str] = None
     ):
+        data_format = self._return_types[0]
         return self._converter.combine_pandas(
             await self._data_return(
-                selection_query, lambda f: self._converter.convert_to_pandas(f), title
+                selection_query,
+                lambda f: self._converter.convert_to_pandas(f),
+                title,
+                data_format=data_format,
             )
         )
 
@@ -404,9 +408,13 @@ async def get_data_pandas_df_async(
     async def get_data_awkward_async(
         self, selection_query: str, title: Optional[str] = None
     ):
+        data_format = self._return_types[0]
         return self._converter.combine_awkward(
             await self._data_return(
-                selection_query, lambda f: self._converter.convert_to_awkward(f), title
+                selection_query,
+                lambda f: self._converter.convert_to_awkward(f),
+                title,
+                data_format=data_format,
             )
         )
 
@@ -417,7 +425,7 @@ async def get_data_awkward_stream(
         as a separate `awkward` array. The data is returned in a `StreamInfoData` object.
 
         Args:
-            selection_query (str): The `qastle` query for the data to retreive.
+            selection_query (str): The `qastle` query for the data to retrieve.
 
         Yields:
             AsyncIterator[StreamInfoData]: As ServiceX completes the data, and it is downloaded
@@ -437,7 +445,7 @@ async def get_data_pandas_stream(
         as a separate `pandas.DataFrame` array. The data is returned in a `StreamInfoData` object.
 
         Args:
-            selection_query (str): The `qastle` query for the data to retreive.
+            selection_query (str): The `qastle` query for the data to retrieve.
 
         Yields:
             AsyncIterator[StreamInfoData]: As ServiceX completes the data, and it is downloaded
@@ -539,7 +547,7 @@ async def _file_return(
         Given a query, return the list of files, in a unique order, that hold
         the data for the query.
 
-        For certian types of exceptions, the queries will be repeated. For example,
+        For certain types of exceptions, the queries will be repeated. For example,
         if `ServiceX` indicates that it was restarted in the middle of the query, then
         the query will be re-submitted.
 
@@ -670,7 +678,7 @@ async def _data_return(
         """Given a query, return the data, in a unique order, that hold
         the data for the query.
 
-        For certian types of exceptions, the queries will be repeated. For example,
+        For certain types of exceptions, the queries will be repeated. For example,
         if `ServiceX` indicates that it was restarted in the middle of the query, then
         the query will be re-submitted.
 
@@ -710,7 +718,7 @@ async def _stream_return(
         """Given a query, return the data, in the order it arrives back
         converted as appropriate.
 
-        For certian types of exceptions, the queries will be repeated. For example,
+        For certain types of exceptions, the queries will be repeated. For example,
         if `ServiceX` indicates that it was restarted in the middle of the query, then
         the query will be re-submitted.
 
@@ -741,7 +749,7 @@ async def _stream_local_files(
         that contain the results of the query. This is an async generator, and files
         are returned as they arrive.
 
-        For certian types of exceptions, the queries will be repeated. For example,
+        For certain types of exceptions, the queries will be repeated. For example,
         if `ServiceX` indicates that it was restarted in the middle of the query, then
         the query will be re-submitted.
 
@@ -779,7 +787,7 @@ async def _get_files(
         Return a list of files from servicex as they have been downloaded to this machine. The
         return type is an awaitable that will yield the path to the file.
 
-        For certian types of `ServiceX` failures we will automatically attempt a few retries:
+        For certain types of `ServiceX` failures we will automatically attempt a few retries:
 
             - When `ServiceX` forgets the query. This sometimes happens when a user submits a
               query, and then disconnects from the network, `ServiceX` is restarted, and then the
@@ -815,7 +823,7 @@ async def _get_files(
                 self._cache.lookup_query_status(request_id)
             )
 
-            # Look up the cache, and then fetch an iterator going thorugh the results
+            # Look up the cache, and then fetch an iterator going through the results
             # from either servicex or the cache, depending.
             try:
                 cached_files = self._cache.lookup_files(request_id)
@@ -1052,7 +1060,7 @@ def _build_json_query(
         json_query: Dict[str, Union[str, Iterable[str]]] = {
             "selection": selection_query,
             "result-destination": self._result_destination,
-            "result-format": "parquet" if data_format == "parquet" else "root-file",
+            "result-format": data_format,
             "chunk-size": "1000",
             "workers": str(self._max_workers),
         }

servicex/servicexabc.py

@@ -119,15 +119,15 @@ async def get_data_pandas_df_async(
     ) -> pd.DataFrame:
         """
         Fetch query data from ServiceX matching `selection_query` and return it as
-        a pandas dataframe. The data is uniquely ordered (the same query will always
+        a pandas DataFrame. The data is uniquely ordered (the same query will always
         return the same order).
 
         Arguments:
             selection_query     The `qastle` string specifying the data to be queried
             title               Title reported to the ServiceX backend for status reporting
 
         Returns:
-            df                  The pandas dataframe
+            df                  The pandas DataFrame
 
         Exceptions:
             xxx                 If the data is not the correct shape (e.g. a flat,

servicex/utils.py

@@ -350,7 +350,7 @@ def _init_tqdm(self):
     def _update_bar(
         self, bar: Optional[tqdm], total: Optional[int], num: int, failed: int
     ):
-        assert bar is not None, "Internal error - bar was not initalized"
+        assert bar is not None, "Internal error - bar was not initialized"
         if total is not None:
             if bar.total != total:
                 # There is a bug in the tqm library if running in a notebook
@@ -417,7 +417,7 @@ def clean_linq(q: str) -> str:
 
     Arguments
 
-        q           Strign containing the qastle code`
+        q           String containing the qastle code`
 
     Returns
 

tests/conftest.py

@@ -48,9 +48,9 @@ def __init__(
         self,
         mocker: MockFixture,
         request_id: str,
-        mock_transform_status: Mock = None,
-        mock_query: Mock = None,
-        mock_transform_query_status: Mock = None,
+        mock_transform_status: Optional[Mock] = None,
+        mock_query: Optional[Mock] = None,
+        mock_transform_query_status: Optional[Mock] = None,
     ):
         self.request_id = request_id
         self._endpoint = "http://localhost:5000"
@@ -139,11 +139,11 @@ def access_called_with(self) -> Optional[Tuple[str, str]]:
 
 def build_cache_mock(
     mocker,
-    query_cache_return: str = None,
+    query_cache_return: Optional[str] = None,
     files: Optional[List[Tuple[str, Path]]] = None,
     in_memory: Any = None,
     make_in_memory_work: bool = False,
-    data_file_return: str = None,
+    data_file_return: Optional[str] = None,
     query_status_lookup_return: Optional[Dict[str, str]] = None,
 ) -> Cache:
     c = mocker.MagicMock(spec=Cache)