From 35001931c574570c9c0d6999456a69972174cbe2 Mon Sep 17 00:00:00 2001 From: hegdeshruti23 <189654939+hegdeshruti23@users.noreply.github.com> Date: Tue, 24 Dec 2024 21:18:41 +0530 Subject: [PATCH 1/2] BQ Plugin needs to support JSON and TIMESTAMP data types --- pkg/bigquery/driver/rows.go | 2 +- pkg/bigquery/driver/utils.go | 4 +++- pkg/bigquery/driver/utils_test.go | 17 +++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/pkg/bigquery/driver/rows.go b/pkg/bigquery/driver/rows.go index 3c893c5b..952417e9 100644 --- a/pkg/bigquery/driver/rows.go +++ b/pkg/bigquery/driver/rows.go @@ -78,7 +78,7 @@ func (r *rows) bigqueryTypeOf(columnType *string) (reflect.Type, error) { return reflect.TypeOf(time.Time{}), nil case "DATE", "TIME", "DATETIME": return reflect.TypeOf(""), nil - case "RECORD", "GEOGRAPHY": + case "RECORD", "GEOGRAPHY", "JSON": return reflect.TypeOf(""), nil default: return nil, fmt.Errorf("unknown column type `%s`", *columnType) diff --git a/pkg/bigquery/driver/utils.go b/pkg/bigquery/driver/utils.go index 9ef12275..152e04fe 100644 --- a/pkg/bigquery/driver/utils.go +++ b/pkg/bigquery/driver/utils.go @@ -6,7 +6,7 @@ import ( "fmt" "math/big" "strings" - + "time" "cloud.google.com/go/bigquery" "cloud.google.com/go/civil" ) @@ -48,6 +48,8 @@ func ConvertColumnValue(v bigquery.Value, fieldSchema *bigquery.FieldSchema) (dr return v.(bool), nil case "TIME": return bigquery.CivilTimeString(v.(civil.Time)), nil + case "TIMESTAMP": + return v.(time.Time), nil case "DATE": res := v.(civil.Date) if !res.IsValid() { diff --git a/pkg/bigquery/driver/utils_test.go b/pkg/bigquery/driver/utils_test.go index 9995237f..f1f4e2bb 100644 --- a/pkg/bigquery/driver/utils_test.go +++ b/pkg/bigquery/driver/utils_test.go @@ -5,6 +5,7 @@ import ( "fmt" "math/big" "testing" + "time" "cloud.google.com/go/bigquery" "cloud.google.com/go/civil" @@ -365,6 +366,22 @@ func Test_ConvertColumnValue(t *testing.T) { expectedType: "[]interface {}", expectedValue: "[null,{\"col1\":2,\"col2\":2.99999,\"col3\":\"text value 2\",\"col4\":\"02:02:02\",\"col5\":\"2019-02-02 01:01:01\"}]", }, + { + name: "TIMESTAMP", + value: bigquery.Value(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)), + columnType: "TIMESTAMP", + schema: &bigquery.FieldSchema{Type: "TIMESTAMP"}, + expectedType: "string", + expectedValue: "2009-11-10 23:00:00.000000 UTC", + }, + { + name: "TIMESTAMP repeated", + value: bigquery.Value([]bigquery.Value{time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC), time.Date(2019, time.November, 10, 23, 30, 0, 0, time.UTC)}), + columnType: "TIMESTAMP", + schema: &bigquery.FieldSchema{Type: "TIMESTAMP", Repeated: true}, + expectedType: "string", + expectedValue: "2009-11-10 23:00:00.000000 UTC,2019-11-10 23:30:00.000000 UTC", + }, } for _, tt := range tests { From 6ce9597c629af0d89f380337e91f7b75d5d6dd6d Mon Sep 17 00:00:00 2001 From: Shruti Hegde <189654939+hegdeshruti23@users.noreply.github.com> Date: Tue, 7 Jan 2025 13:59:40 +0530 Subject: [PATCH 2/2] Fix test cases --- pkg/bigquery/driver/utils_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/bigquery/driver/utils_test.go b/pkg/bigquery/driver/utils_test.go index f1f4e2bb..277a396e 100644 --- a/pkg/bigquery/driver/utils_test.go +++ b/pkg/bigquery/driver/utils_test.go @@ -371,16 +371,16 @@ func Test_ConvertColumnValue(t *testing.T) { value: bigquery.Value(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)), columnType: "TIMESTAMP", schema: &bigquery.FieldSchema{Type: "TIMESTAMP"}, - expectedType: "string", - expectedValue: "2009-11-10 23:00:00.000000 UTC", + expectedType: "time.Time", + expectedValue: "2009-11-10 23:00:00 +0000 UTC", }, { name: "TIMESTAMP repeated", value: bigquery.Value([]bigquery.Value{time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC), time.Date(2019, time.November, 10, 23, 30, 0, 0, time.UTC)}), columnType: "TIMESTAMP", schema: &bigquery.FieldSchema{Type: "TIMESTAMP", Repeated: true}, - expectedType: "string", - expectedValue: "2009-11-10 23:00:00.000000 UTC,2019-11-10 23:30:00.000000 UTC", + expectedType: "time.Time", + expectedValue: "2009-11-10 23:00:00 +0000 UTC,2019-11-10 23:30:00 +0000 UTC", }, }