chore: patch imported log entries to handle dict to protobuf parsing (#12711)

* chore: patch imported log entries to handle dict to protobuf parsing problem

patch logs to replace /protoPayload/serviceData with /protoPayload/metadata.
handle the problem with python logging library that fails to parse dict to protobuf when dict has deprecated field serviceData.

* chore: fix linting errors

* chore: fix lint error and incorrect call of _patch_entry method

* chore: fix another lint error

* chore: minor update in README link

reflects adding new lines to the code.

---------

Co-authored-by: Maciej Strzelczyk <strzelczyk@google.com>

Author: minherz

logging/import-logs/README.md

@@ -129,4 +129,4 @@ After applying the changes, [build](#build) a custom container image and use it
 [retention]: https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#FIELDS.timestamp
 [current]: https://github.com/GoogleCloudPlatform/python-docs-samples/blob/e2709a218072c86ec1a9b9101db45057ebfdbff0/logging/import-logs/main.py
 [code1]: https://github.com/GoogleCloudPlatform/python-docs-samples/blob/86f12a752a4171e137adaa855c7247be9d5d39a2/logging/import-logs/main.py#L81-L83
-[code2]: https://github.com/GoogleCloudPlatform/python-docs-samples/blob/86f12a752a4171e137adaa855c7247be9d5d39a2/logging/import-logs/main.py#L186-L187
+[code2]: https://github.com/GoogleCloudPlatform/python-docs-samples/blob/86f12a752a4171e137adaa855c7247be9d5d39a2/logging/import-logs/main.py#L188-L189

logging/import-logs/main.py

@@ -173,7 +173,8 @@ def _patch_entry(log: dict, project_id: str) -> None:
     """Modify entry fields to allow importing entry to destination project.
 
     Save logName as a user label.
-    Replace logName with the fixed value "projects/PROJECT_ID/logs/imported_logs"
+    Replace logName with the fixed value "projects/PROJECT_ID/logs/imported_logs".
+    Rename the obsolete key "serviceData" with "metadata".
     """
     log_name = log.get("logName")
     labels = log.get("labels")
@@ -182,6 +183,13 @@ def _patch_entry(log: dict, project_id: str) -> None:
         labels = dict()
         log["labels"] = labels
     labels["original_logName"] = log_name
+    # TODO: remove after the following issue is fixed:
+    # https://github.com/googleapis/python-logging/issues/945
+    if "protoPayload" in log:
+        payload = log.get("protoPayload")
+        if "serviceData" in payload:
+            # the following line changes the place of metadata in the dictionary
+            payload["metadata"] = payload.pop("serviceData")
     # uncomment the following 2 lines if import range includes dates older than 29 days from now
     # labels["original_timestamp"] = log["timestamp"]
     # log["timestamp"] = None

logging/import-logs/main_test.py

@@ -369,3 +369,106 @@ def test_parse_date() -> None:
     assert (
         test_date_str == TEST_DATE_STR
     ), f"expected {TEST_DATE_STR}, got {test_date_str}"
+
+
+TEST_LOG_WITH_SERVICEDATA = {
+    "logName": "projects/someproject/logs/somelog",
+    "protoPayload": {
+        "@type": "type.googleapis.com/google.cloud.audit.AuditLog",
+        "authenticationInfo": {
+            "principalEmail": "service@gcp-sa-scc-notification.iam.gserviceaccount.com"
+        },
+        "authorizationInfo": [
+            {
+                "granted": True,
+                "permission": "bigquery.tables.update",
+                "resource": "projects/someproject/datasets/someds/tables/sometbl",
+                "resourceAttributes": {}
+            }
+        ],
+        "serviceData": {
+            '@type': 'type.googleapis.com/google.cloud.bigquery.logging.v1.AuditData',
+            'tableUpdateRequest': {
+                'resource': {
+                    'info': {},
+                    'schemaJson': '{}',
+                    'updateTime': '2024-08-20T15:01:48.399Z',
+                    'view': {}
+                }
+            }
+        },
+        "methodName": "google.cloud.bigquery.v2.TableService.PatchTable",
+        "requestMetadata": {
+            "callerIp": "private",
+            "destinationAttributes": {},
+            "requestAttributes": {}
+        },
+        "resourceName": "projects/someproject/datasets/someds/tables/sometbl",
+        "serviceName": "bigquery.googleapis.com",
+        "status": {}
+    },
+    "resource": {
+        "labels": {
+            "dataset_id": "someds",
+            "project_id": "someproject"
+        },
+        "type": "bigquery_dataset"
+    },
+    "severity": "NOTICE",
+}
+TEST_LOG_WITH_PATCHED_SERVICEDATA = {
+    "logName": f"projects/{TEST_PROJECT_ID}/logs/imported_logs",
+    "protoPayload": {
+        "@type": "type.googleapis.com/google.cloud.audit.AuditLog",
+        "authenticationInfo": {
+            "principalEmail": "service@gcp-sa-scc-notification.iam.gserviceaccount.com"
+        },
+        "authorizationInfo": [
+            {
+                "granted": True,
+                "permission": "bigquery.tables.update",
+                "resource": "projects/someproject/datasets/someds/tables/sometbl",
+                "resourceAttributes": {}
+            }
+        ],
+        # this field is renamed from 'serviceData'
+        "metadata": {
+            '@type': 'type.googleapis.com/google.cloud.bigquery.logging.v1.AuditData',
+            'tableUpdateRequest': {
+                'resource': {
+                    'info': {},
+                    'schemaJson': '{}',
+                    'updateTime': '2024-08-20T15:01:48.399Z',
+                    'view': {}
+                }
+            }
+        },
+        "methodName": "google.cloud.bigquery.v2.TableService.PatchTable",
+        "requestMetadata": {
+            "callerIp": "private",
+            "destinationAttributes": {},
+            "requestAttributes": {}
+        },
+        "resourceName": "projects/someproject/datasets/someds/tables/sometbl",
+        "serviceName": "bigquery.googleapis.com",
+        "status": {}
+    },
+    "resource": {
+        "labels": {
+            "dataset_id": "someds",
+            "project_id": "someproject"
+        },
+        "type": "bigquery_dataset"
+    },
+    "labels": {
+        "original_logName": "projects/someproject/logs/somelog",
+    },
+    "severity": "NOTICE",
+}
+
+
+def test_patch_serviceData_field() -> None:
+    log = dict(TEST_LOG_WITH_SERVICEDATA)
+    main._patch_entry(log, TEST_PROJECT_ID)
+
+    assert (log == TEST_LOG_WITH_PATCHED_SERVICEDATA)