Add a presigned URL to the mix

Working on getting minio direct access async #132

Author: gordonwatts

servicex/minio_adaptor.py

@@ -62,6 +62,22 @@ def __init__(self, minio_endpoint: str,
     def get_files(self, request_id):
         return [f.object_name for f in self._client.list_objects(request_id)]
 
+    def get_access_url(self, request_id: str, object_name: str) -> str:
+        '''Given a request ID and the file name in that request id, return a URL
+        that can be directly accessed to download the file.
+
+        Args:
+            request_id (str): The request id guid (that is the bucket in minio)
+            object_name (str): The file (the object in the minio bucket)
+
+        Raises:
+            NotImplementedError: [description]
+
+        Returns:
+            str: A url good for some amount of time to access the bucket.
+        '''
+        return self._client.presigned_get_object(request_id, object_name)
+
     async def download_file(self,
                             request_id: str,
                             bucket_fname: str,

servicex/servicex.py

@@ -251,6 +251,7 @@ async def _get_minio_buckets(self, selection_query: str, data_format: str) \
                     yield {
                         'bucket': request_id,
                         'file': r,
+                        'url': minio_adaptor.get_access_url(request_id, r),
                     }
 
                 # Cache the final status

tests/conftest.py

@@ -95,6 +95,7 @@ class MockMinioAdaptor(MinioAdaptor):
     def __init__(self, mocker: MockFixture, files: List[str] = []):
         self._files = files
         self.mock_download_file = mocker.Mock()
+        self._access_called_with = None
         pass
 
     async def download_file(self, request_id: str, minio_object_name: str, final_path: Path):
@@ -104,6 +105,14 @@ def get_files(self, request_id) -> List[str]:
         'Return files in the bucket'
         return self._files
 
+    def get_access_url(self, request_id: str, object_name: str) -> str:
+        self._access_called_with = (request_id, object_name)
+        return "http://the.url.com"
+
+    @property
+    def access_called_with(self) -> Optional[Tuple[str, str]]:
+        return self._access_called_with
+
 
 __g_inmem_value = None
 

tests/test_minio_adaptor.py

@@ -26,6 +26,7 @@ def good_minio_client(mocker):
         [make_minio_file('root:::dcache-atlas-xrootd-wan.desy.de:1094::pnfs:desy.de:atlas'
                          ':dq2:atlaslocalgroupdisk:rucio:mc15_13TeV:8a:f1:DAOD_STDM3.05630052'
                          '._000001.pool.root.198fbd841d0a28cb0d9dfa6340c890273-1.part.minio')]
+    minio_client.presigned_get_object.return_value = 'http://data.done'
 
     mocker.patch('servicex.minio_adaptor.Minio', return_value=minio_client)
 
@@ -130,6 +131,13 @@ async def test_download_already_there(mocker, good_minio_client):
     p_exists.assert_called_once()
 
 
+def test_access_url(good_minio_client):
+    'Make sure the presigned_get_object is properly called'
+    mn = MinioAdaptor('localhost:9000')
+    assert mn.get_access_url('123-456', 'file1') == 'http://data.done'
+    good_minio_client[1].presigned_get_object.assert_called_with('123-456', 'file1')
+
+
 def test_list_objects(good_minio_client):
     ma = MinioAdaptor('localhost:9000')
 

tests/test_servicex.py

@@ -412,8 +412,10 @@ async def test_good_run_root_files_from_minio(mocker):
     assert len(lst) == 1
     assert lst[0]['bucket'] == '123-456'
     assert lst[0]['file'] == 'one_minio_entry'
+    assert lst[0]['url'] == 'http://the.url.com'
 
     assert mock_servicex_adaptor.query_json['result-format'] == 'root-file'
+    assert mock_minio_adaptor.access_called_with == ('123-456', 'one_minio_entry')
 
 
 @pytest.mark.asyncio