Skip to content

Commit 1051f3f

Browse files
potato2003potato2003
authored
Merge pull request #116 from trocco-io/add-datetime
Add DATETIME support
2 parents 261733a + 416bc91 commit 1051f3f

File tree

4 files changed

+71
-5
lines changed

4 files changed

+71
-5
lines changed

‎README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -307,12 +307,12 @@ Column options are used to aid guessing BigQuery schema, or to define conversion
307307

308308
- **column_options**: advanced: an array of options for columns
309309
- **name**: column name
310-
- **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE`, and `RECORD`. See belows for supported conversion type.
310+
- **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, and `RECORD`. See belows for supported conversion type.
311311
- boolean: `BOOLEAN`, `STRING` (default: `BOOLEAN`)
312312
- long: `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `INTEGER`)
313313
- double: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `FLOAT`)
314-
- string: `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE`, `RECORD` (default: `STRING`)
315-
- timestamp: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE` (default: `TIMESTAMP`)
314+
- string: `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, `RECORD` (default: `STRING`)
315+
- timestamp: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE` (default: `TIMESTAMP`)
316316
- json: `STRING`, `RECORD` (default: `STRING`)
317317
- **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`)
318318
- **fields**: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column.

‎lib/embulk/output/bigquery/value_converter_factory.rb

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,20 @@ def string_converter
210210
TimeWithZone.set_zone_offset(Time.parse(val), zone_offset).strftime("%Y-%m-%d")
211211
end
212212
}
213+
when 'DATETIME'
214+
if @timestamp_format
215+
Proc.new {|val|
216+
next nil if val.nil?
217+
with_typecast_error(val) do |val|
218+
Time.strptime(val, @timestamp_format).strftime("%Y-%m-%d %H:%M:%S.%6N")
219+
end
220+
}
221+
else
222+
Proc.new {|val|
223+
next nil if val.nil?
224+
val # Users must care of BQ timestamp format
225+
}
226+
end
213227
when 'RECORD'
214228
Proc.new {|val|
215229
next nil if val.nil?
@@ -252,6 +266,11 @@ def timestamp_converter
252266
next nil if val.nil?
253267
val.localtime(zone_offset).strftime("%Y-%m-%d")
254268
}
269+
when 'DATETIME'
270+
Proc.new {|val|
271+
next nil if val.nil?
272+
val.localtime(zone_offset).strftime("%Y-%m-%d %H:%M:%S.%6N")
273+
}
255274
else
256275
raise NotSupportedType, "cannot take column type #{type} for timestamp column"
257276
end

‎test/test_helper.rb

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,8 @@ def test_fields_from_embulk_schema_with_column_options
6363
Column.new({index: 3, name: 'string', type: :string}),
6464
Column.new({index: 4, name: 'timestamp', type: :timestamp}),
6565
Column.new({index: 5, name: 'date', type: :timestamp}),
66-
Column.new({index: 6, name: 'json', type: :json}),
66+
Column.new({index: 6, name: 'datetime', type: :timestamp}),
67+
Column.new({index: 7, name: 'json', type: :json}),
6768
])
6869
task = {
6970
'column_options' => [
@@ -73,6 +74,7 @@ def test_fields_from_embulk_schema_with_column_options
7374
{'name' => 'string', 'type' => 'INTEGER'},
7475
{'name' => 'timestamp', 'type' => 'INTEGER'},
7576
{'name' => 'date', 'type' => 'DATE'},
77+
{'name' => 'datetime', 'type' => 'DATETIME'},
7678
{'name' => 'json', 'type' => 'RECORD', 'fields' => [
7779
{ 'name' => 'key1', 'type' => 'STRING' },
7880
]},
@@ -85,6 +87,7 @@ def test_fields_from_embulk_schema_with_column_options
8587
{name: 'string', type: 'INTEGER'},
8688
{name: 'timestamp', type: 'INTEGER'},
8789
{name: 'date', type: 'DATE'},
90+
{name: 'datetime', type: 'DATETIME'},
8891
{name: 'json', type: 'RECORD', fields: [
8992
{name: 'key1', type: 'STRING'},
9093
]},

‎test/test_value_converter_factory.rb

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,10 @@ def test_date
9494
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
9595
end
9696

97+
def test_datetime
98+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
99+
end
100+
97101
def test_record
98102
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
99103
end
@@ -138,6 +142,10 @@ def test_date
138142
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
139143
end
140144

145+
def test_datetime
146+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
147+
end
148+
141149
def test_record
142150
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
143151
end
@@ -178,6 +186,10 @@ def test_date
178186
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
179187
end
180188

189+
def test_datetime
190+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
191+
end
192+
181193
def test_record
182194
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
183195
end
@@ -236,6 +248,20 @@ def test_date
236248
assert_raise { converter.call('foo') }
237249
end
238250

251+
def test_datetime
252+
converter = ValueConverterFactory.new(
253+
SCHEMA_TYPE, 'DATETIME',
254+
timestamp_format: '%Y/%m/%d'
255+
).create_converter
256+
assert_equal nil, converter.call(nil)
257+
assert_equal "2016-02-26 00:00:00.000000", converter.call("2016/02/26")
258+
259+
# Users must care of BQ datetime format by themselves with no timestamp_format
260+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter
261+
assert_equal nil, converter.call(nil)
262+
assert_equal "2016-02-26 00:00:00", converter.call("2016-02-26 00:00:00")
263+
end
264+
239265
def test_record
240266
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
241267
assert_equal({'foo'=>'foo'}, converter.call(%Q[{"foo":"foo"}]))
@@ -294,7 +320,7 @@ def test_date
294320
timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00")
295321
expected = "2016-02-26"
296322
assert_equal expected, converter.call(timestamp)
297-
323+
298324
converter = ValueConverterFactory.new(
299325
SCHEMA_TYPE, 'DATE', timezone: 'Asia/Tokyo'
300326
).create_converter
@@ -306,6 +332,24 @@ def test_date
306332
assert_raise { converter.call('foo') }
307333
end
308334

335+
def test_datetime
336+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter
337+
assert_equal nil, converter.call(nil)
338+
timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00")
339+
expected = "2016-02-26 00:00:00.500000"
340+
assert_equal expected, converter.call(timestamp)
341+
342+
converter = ValueConverterFactory.new(
343+
SCHEMA_TYPE, 'DATETIME', timezone: 'Asia/Tokyo'
344+
).create_converter
345+
assert_equal nil, converter.call(nil)
346+
timestamp = Time.parse("2016-02-25 15:00:00.500000 +00:00")
347+
expected = "2016-02-26 00:00:00.500000"
348+
assert_equal expected, converter.call(timestamp)
349+
350+
assert_raise { converter.call('foo') }
351+
end
352+
309353
def test_record
310354
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
311355
end

0 commit comments

Comments
 (0)