|
34 | 34 | from servicex.minio_adapter import MinioAdapter
|
35 | 35 | from servicex.models import ResultFile
|
36 | 36 |
|
37 |
| -import asyncio |
38 |
| - |
39 | 37 | DOWNLOAD_PATCH_COUNTER = 0
|
40 | 38 |
|
41 |
| -_semaphore = asyncio.Semaphore(1) |
42 |
| - |
43 | 39 |
|
44 | 40 | def mock_downloader(**args):
|
45 | 41 | global DOWNLOAD_PATCH_COUNTER
|
@@ -70,16 +66,15 @@ def minio_adapter(moto_services, moto_patch_session) -> MinioAdapter:
|
70 | 66 |
|
71 | 67 | @fixture
|
72 | 68 | async def populate_bucket(request, minio_adapter):
|
73 |
| - async with _semaphore: |
74 |
| - async with minio_adapter.minio.client( |
75 |
| - "s3", endpoint_url=minio_adapter.endpoint_host |
76 |
| - ) as s3: |
77 |
| - await s3.create_bucket(Bucket=minio_adapter.bucket) |
78 |
| - await s3.put_object( |
79 |
| - Bucket=minio_adapter.bucket, Key=request.param, Body=b"\x01" * 10 |
80 |
| - ) |
81 |
| - yield |
82 |
| - await s3.delete_object(Bucket=minio_adapter.bucket, Key=request.param) |
| 69 | + async with minio_adapter.minio.client( |
| 70 | + "s3", endpoint_url=minio_adapter.endpoint_host |
| 71 | + ) as s3: |
| 72 | + await s3.create_bucket(Bucket=minio_adapter.bucket) |
| 73 | + await s3.put_object( |
| 74 | + Bucket=minio_adapter.bucket, Key=request.param, Body=b"\x01" * 10 |
| 75 | + ) |
| 76 | + yield |
| 77 | + await s3.delete_object(Bucket=minio_adapter.bucket, Key=request.param) |
83 | 78 |
|
84 | 79 |
|
85 | 80 | @fixture
|
@@ -180,33 +175,33 @@ async def test_download_short_filename_change(minio_adapter, populate_bucket):
|
180 | 175 |
|
181 | 176 | @pytest.mark.parametrize("populate_bucket", ["test.txt"], indirect=True)
|
182 | 177 | @pytest.mark.asyncio
|
| 178 | +async def test_download_file_retry(download_patch, minio_adapter, populate_bucket): |
| 179 | + result = await minio_adapter.download_file("test.txt", local_dir="/tmp/foo") |
| 180 | + assert str(result).endswith("test.txt") |
| 181 | + assert result.exists() |
| 182 | + assert len(download_patch.call_args_list) == 3 |
| 183 | + result.unlink() |
| 184 | + |
| 185 | + |
| 186 | +@pytest.mark.parametrize("populate_bucket", ["test2.txt"], indirect=True) |
| 187 | +@pytest.mark.asyncio |
183 | 188 | async def test_download_repeat(minio_adapter, populate_bucket):
|
184 | 189 | import asyncio
|
185 | 190 |
|
186 | 191 | print(await minio_adapter.list_bucket())
|
187 |
| - result = await minio_adapter.download_file("test.txt", local_dir="/tmp/foo") |
188 |
| - assert str(result).endswith("test.txt") |
| 192 | + result = await minio_adapter.download_file("test2.txt", local_dir="/tmp/foo") |
| 193 | + assert str(result).endswith("test2.txt") |
189 | 194 | assert result.exists()
|
190 | 195 | t0 = result.stat().st_mtime_ns
|
191 | 196 | await asyncio.sleep(4) # hopefully long enough for Windows/FAT32 ... ?
|
192 | 197 |
|
193 |
| - result2 = await minio_adapter.download_file("test.txt", local_dir="/tmp/foo") |
| 198 | + result2 = await minio_adapter.download_file("test2.txt", local_dir="/tmp/foo") |
194 | 199 | assert result2.exists()
|
195 | 200 | assert result2 == result
|
196 | 201 | assert t0 == result2.stat().st_mtime_ns
|
197 | 202 | result.unlink() # it should exist, from above ...
|
198 | 203 |
|
199 | 204 |
|
200 |
| -@pytest.mark.parametrize("populate_bucket", ["test.txt"], indirect=True) |
201 |
| -@pytest.mark.asyncio |
202 |
| -async def test_download_file_retry(download_patch, minio_adapter, populate_bucket): |
203 |
| - result = await minio_adapter.download_file("test.txt", local_dir="/tmp/foo") |
204 |
| - assert str(result).endswith("test.txt") |
205 |
| - assert result.exists() |
206 |
| - assert len(download_patch.call_args_list) == 3 |
207 |
| - result.unlink() |
208 |
| - |
209 |
| - |
210 | 205 | @pytest.mark.asyncio
|
211 | 206 | async def test_get_signed_url(minio_adapter, moto_services):
|
212 | 207 | result = await minio_adapter.get_signed_url("test.txt")
|
|
0 commit comments