add proper s3 object sourcing

Author: MattShirley

servicex_app/servicex_app/resources/internal/transformer_file_complete.py

@@ -89,8 +89,9 @@ def put(self, request_id):
         info = request.get_json()
         logger = current_app.logger
         log_extra = {
-                    'requestId': request_id,
-                    'file-id': info['file-id']
+            'requestId': request_id,
+            'file-id': info['file-id'],
+            's3-object-name': info['s3-object-name'],
         }
 
         logger.info("FileComplete", extra={**log_extra, 'metric': info})

servicex_app/servicex_app/resources/transformation/results.py

@@ -1,4 +1,3 @@
-import logging
 import datetime
 
 from flask_restful import reqparse
@@ -7,8 +6,6 @@
 from servicex_app.models import TransformationResult
 from servicex_app.resources.servicex_resource import ServiceXResource
 
-logger = logging.getLogger(__name__)
-
 
 class TransformationResults(ServiceXResource):
     @auth_required

transformer_sidecar/src/transformer_sidecar/servicex_adapter.py

@@ -104,10 +104,13 @@ def put_file_complete(self, rec: FileCompleteRecord):
                            fkwargs={"json": rec.to_json(), "timeout": (0.5, None)},
                            tries=MAX_RETRIES,
                            delay=RETRY_DELAY)
-                self.logger.info("Put file complete.", extra={'requestId': rec.request_id,
-                                                              "file-id": rec.file_id,
-                                                              "place": PLACE,
-                                                              "file_path": rec.file_path})
+                self.logger.info("Put file complete.", extra={
+                    'requestId': rec.request_id,
+                    "file-id": rec.file_id,
+                    "place": PLACE,
+                    "file_path": rec.file_path,
+                    "s3-object-name": rec.s3_object_name,
+                })
             except requests.exceptions.ConnectionError:
                 self.logger.exception("Connection Error in put_file_complete",
                                       extra={'requestId': rec.request_id,

transformer_sidecar/src/transformer_sidecar/transformer.py

@@ -163,7 +163,7 @@ def transform_file(
     transformer_stats = TransformerStats()
     try:
         # Loop through the replicas
-        for path, _file_path in zip(paths, _file_paths):
+        for _file_path in _file_paths:
             logger.info(
                 "trying to transform file",
                 extra={
@@ -215,19 +215,19 @@ def transform_file(
                 rec = FileCompleteRecord(
                     request_id=request_id,
                     file_path=_file_path,
-                    s3_object_name=path,
+                    s3_object_name="none",
                     file_id=file_id,
                     status="success",
                     total_time=time.time() - total_time,
                     total_events=transformer_stats.total_events,
                     total_bytes=os.path.getsize(output_path),
                 )
-
                 if object_store:
-                    upload_file(Path(transform_request["safeOutputFileName"]),
-                                servicex,
-                                rec
-                                )
+                    upload_file(
+                        Path(transform_request["safeOutputFileName"]),
+                        servicex,
+                        rec
+                    )
                 else:
                     servicex.put_file_complete(rec)
 
@@ -257,8 +257,8 @@ def transform_file(
             rec = FileCompleteRecord(
                 request_id=request_id,
                 file_path=_file_paths[0],
-                s3_object_name=paths[0],
                 file_id=file_id,
+                s3_object_name="none",
                 status="failure",
                 total_time=time.time() - total_time,
                 total_events=0,
@@ -294,7 +294,7 @@ def transform_file(
         rec = FileCompleteRecord(
             request_id=request_id,
             file_path=_file_paths[0],
-            s3_object_name=paths[0],
+            s3_object_name="none",
             file_id=file_id,
             status="failure",
             total_time=time.time() - total_time,
@@ -354,6 +354,8 @@ def upload_file(source_path: Path,
         file_to_upload = source_path
         object_name = source_path.name
 
+    rec.s3_object_name = object_name
+
     logger.info("Uploading file to object store.",
                 extra={'requestId': request_id,
                        "file-id": rec.file_id,