From 566b9b6bfc1e3186c2bd81e78737996b78a96523 Mon Sep 17 00:00:00 2001 From: anupksingh Date: Tue, 29 Apr 2025 07:40:06 +0000 Subject: [PATCH] Fixes addressing two issues identified during DTS transfer runs in the customer environment: 1. handling of null XmlObject instances that previously caused failures. 2. schema mismatches where between declared schema types and actual values resulted in casting exceptions for INT types. --- .../source/batch/MapToRecordTransformer.java | 7 +++- .../batch/SoapRecordToMapTransformer.java | 4 ++ .../batch/SalesforceBulkRecordReaderTest.java | 42 +++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/MapToRecordTransformer.java b/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/MapToRecordTransformer.java index 1fa74af1..b48c0f23 100644 --- a/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/MapToRecordTransformer.java +++ b/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/MapToRecordTransformer.java @@ -22,6 +22,7 @@ import io.cdap.plugin.salesforce.SalesforceSchemaUtil; import io.cdap.plugin.salesforce.SalesforceTransformUtil; +import java.math.BigDecimal; import java.util.List; import java.util.Map; import java.util.Objects; @@ -78,7 +79,11 @@ private Object convertValue(String fieldName, Object value, Schema fieldSchema) case BOOLEAN: return Boolean.parseBoolean(castValue(value, fieldName, String.class)); case INT: - return Integer.parseInt(castValue(value, fieldName, String.class)); + // 'Number' field in 'Site' sObject is defined as INT in schema, + // however, the actual data is received as a Decimal (e.g., 0.0) + // Wrapping as BigDecimal to safely extract the int value and avoid cast exceptions. + String data = castValue(value, fieldName, String.class); + return new BigDecimal(data).intValue(); case LONG: return Long.parseLong(castValue(value, fieldName, String.class)); case FLOAT: diff --git a/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SoapRecordToMapTransformer.java b/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SoapRecordToMapTransformer.java index 24fc0ab5..e9fbc6fc 100644 --- a/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SoapRecordToMapTransformer.java +++ b/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SoapRecordToMapTransformer.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -53,6 +54,9 @@ public class SoapRecordToMapTransformer { Iterable subValues = () -> child.getChildren(SUB_QUERY_FIELDS_PARENT); List> subQueryValues = StreamSupport.stream(subValues.spliterator(), false) + // Since null XmlObject instances were encountered in the customer transfer runs, a + // validation check is now applied to exclude them during processing. + .filter(Objects::nonNull) .map(subValue -> transformRowToMap(subValue, childSObjectDescriptor)) .collect(Collectors.toList()); result.put(childSObjectDescriptor.getName(), subQueryValues); diff --git a/src/test/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceBulkRecordReaderTest.java b/src/test/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceBulkRecordReaderTest.java index cb3f8c40..3491c140 100644 --- a/src/test/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceBulkRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceBulkRecordReaderTest.java @@ -562,4 +562,46 @@ public void testCaseSenstiveMultipleResults() throws Exception { assertRecordReaderOutputRecords(new String[] {csvString1, csvString2}, schema, expectedRecords); } + @Test + public void testCaseIntTypeWithDecimalDataInResults() throws Exception { + String csvString1 = "\"Id\",\"IsDeleted\",\"ExpectedRevenue\",\"TotalRevenue\"\n" + + "\"0061i000003XNcBAAW\",\"false\",\"1500.0\",\"1234.56\"\n"; + String csvString2 = "\"Id\",\"IsDeleted\",\"ExpectedRevenue\",\"TotalRevenue\"\n" + + "\"0061i000003XNcCAAW\",\"false\",\"112500.0\",\"1234.56\"\n" + + "\"0061i000003XNcDAAW\",\"false\",\"220000.0\",\"1234.56\"\n"; + + Schema schema = Schema.recordOf("output", + Schema.Field.of("id", Schema.of(Schema.Type.STRING)), + Schema.Field.of("isDeleted", Schema.of(Schema.Type.BOOLEAN)), + Schema.Field.of("ExpectedRevenue", Schema.of(Schema.Type.INT)), + Schema.Field.of("TotalRevenue", Schema.of(Schema.Type.DOUBLE)) + ); + + List> expectedRecords = new ImmutableList.Builder>() + .add(new ImmutableMap.Builder() + .put("id", "0061i000003XNcBAAW") + .put("isDeleted", false) + .put("ExpectedRevenue", 1500) + .put("TotalRevenue", 1234.56) + .build() + ) + .add(new ImmutableMap.Builder() + .put("id", "0061i000003XNcCAAW") + .put("isDeleted", false) + .put("ExpectedRevenue", 112500) + .put("TotalRevenue", 1234.56) + .build() + ) + .add(new ImmutableMap.Builder() + .put("id", "0061i000003XNcDAAW") + .put("isDeleted", false) + .put("ExpectedRevenue", 220000) + .put("TotalRevenue", 1234.56) + .build() + ) + .build(); + + assertRecordReaderOutputRecords(new String[] {csvString1, csvString2}, schema, expectedRecords); + } + }