Skip to content

Commit db1c5e9

Browse files
tksfjt1024tksfjt1024
committed
add DATE type converter
1 parent e28eb30 commit db1c5e9

File tree

4 files changed

+62
-5
lines changed

4 files changed

+62
-5
lines changed

‎README.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -307,17 +307,17 @@ Column options are used to aid guessing BigQuery schema, or to define conversion
307307

308308
- **column_options**: advanced: an array of options for columns
309309
- **name**: column name
310-
- **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, and `RECORD`. See belows for supported conversion type.
310+
- **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE`, and `RECORD`. See belows for supported conversion type.
311311
- boolean: `BOOLEAN`, `STRING` (default: `BOOLEAN`)
312312
- long: `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `INTEGER`)
313313
- double: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `FLOAT`)
314-
- string: `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `RECORD` (default: `STRING`)
315-
- timestamp: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `TIMESTAMP`)
314+
- string: `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE`, `RECORD` (default: `STRING`)
315+
- timestamp: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE` (default: `TIMESTAMP`)
316316
- json: `STRING`, `RECORD` (default: `STRING`)
317317
- **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`)
318318
- **fields**: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column.
319319
- **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is `default_timestamp_format`)
320-
- **timezone**: timezone to convert into/from `timestamp` (string, default is `default_timezone`).
320+
- **timezone**: timezone to convert into/from `timestamp`, `date` (string, default is `default_timezone`).
321321
- **default_timestamp_format**: default timestamp format for column_options (string, default is "%Y-%m-%d %H:%M:%S.%6N")
322322
- **default_timezone**: default timezone for column_options (string, default is "UTC")
323323

‎lib/embulk/output/bigquery/value_converter_factory.rb

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,13 @@ def string_converter
203203
val # Users must care of BQ timestamp format
204204
}
205205
end
206+
when 'DATE'
207+
Proc.new {|val|
208+
next nil if val.nil?
209+
with_typecast_error(val) do |val|
210+
TimeWithZone.set_zone_offset(Time.parse(val), zone_offset).strftime("%Y-%m-%d")
211+
end
212+
}
206213
when 'RECORD'
207214
Proc.new {|val|
208215
next nil if val.nil?
@@ -240,6 +247,11 @@ def timestamp_converter
240247
next nil if val.nil?
241248
val.strftime("%Y-%m-%d %H:%M:%S.%6N %:z")
242249
}
250+
when 'DATE'
251+
Proc.new {|val|
252+
next nil if val.nil?
253+
val.localtime(zone_offset).strftime("%Y-%m-%d")
254+
}
243255
else
244256
raise NotSupportedType, "cannot take column type #{type} for timestamp column"
245257
end

‎test/test_helper.rb

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@ def test_fields_from_embulk_schema_with_column_options
6262
Column.new({index: 2, name: 'double', type: :double}),
6363
Column.new({index: 3, name: 'string', type: :string}),
6464
Column.new({index: 4, name: 'timestamp', type: :timestamp}),
65-
Column.new({index: 5, name: 'json', type: :json}),
65+
Column.new({index: 5, name: 'date', type: :timestamp}),
66+
Column.new({index: 6, name: 'json', type: :json}),
6667
])
6768
task = {
6869
'column_options' => [
@@ -71,6 +72,7 @@ def test_fields_from_embulk_schema_with_column_options
7172
{'name' => 'double', 'type' => 'STRING'},
7273
{'name' => 'string', 'type' => 'INTEGER'},
7374
{'name' => 'timestamp', 'type' => 'INTEGER'},
75+
{'name' => 'date', 'type' => 'DATE'},
7476
{'name' => 'json', 'type' => 'RECORD', 'fields' => [
7577
{ 'name' => 'key1', 'type' => 'STRING' },
7678
]},
@@ -82,6 +84,7 @@ def test_fields_from_embulk_schema_with_column_options
8284
{name: 'double', type: 'STRING'},
8385
{name: 'string', type: 'INTEGER'},
8486
{name: 'timestamp', type: 'INTEGER'},
87+
{name: 'date', type: 'DATE'},
8588
{name: 'json', type: 'RECORD', fields: [
8689
{name: 'key1', type: 'STRING'},
8790
]},

‎test/test_value_converter_factory.rb

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,10 @@ def test_timestamp
9090
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter }
9191
end
9292

93+
def test_date
94+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
95+
end
96+
9397
def test_record
9498
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
9599
end
@@ -130,6 +134,10 @@ def test_timestamp
130134
assert_equal 1408452095, converter.call(1408452095)
131135
end
132136

137+
def test_date
138+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
139+
end
140+
133141
def test_record
134142
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
135143
end
@@ -166,6 +174,10 @@ def test_timestamp
166174
assert_equal 1408452095.188766, converter.call(1408452095.188766)
167175
end
168176

177+
def test_date
178+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
179+
end
180+
169181
def test_record
170182
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
171183
end
@@ -216,6 +228,14 @@ def test_timestamp
216228
assert_equal "2016-02-26 00:00:00", converter.call("2016-02-26 00:00:00")
217229
end
218230

231+
def test_date
232+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter
233+
assert_equal nil, converter.call(nil)
234+
assert_equal "2016-02-26", converter.call("2016-02-26")
235+
assert_equal "2016-02-26", converter.call("2016-02-26 00:00:00")
236+
assert_raise { converter.call('foo') }
237+
end
238+
219239
def test_record
220240
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
221241
assert_equal({'foo'=>'foo'}, converter.call(%Q[{"foo":"foo"}]))
@@ -268,6 +288,24 @@ def test_timestamp
268288
assert_equal expected, converter.call(Time.at(subject).utc)
269289
end
270290

291+
def test_date
292+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter
293+
assert_equal nil, converter.call(nil)
294+
timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00")
295+
expected = "2016-02-26"
296+
assert_equal expected, converter.call(timestamp)
297+
298+
converter = ValueConverterFactory.new(
299+
SCHEMA_TYPE, 'DATE', timezone: 'Asia/Tokyo'
300+
).create_converter
301+
assert_equal nil, converter.call(nil)
302+
timestamp = Time.parse("2016-02-25 15:00:00.500000 +00:00")
303+
expected = "2016-02-26"
304+
assert_equal expected, converter.call(timestamp)
305+
306+
assert_raise { converter.call('foo') }
307+
end
308+
271309
def test_record
272310
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
273311
end
@@ -298,6 +336,10 @@ def test_timestamp
298336
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter }
299337
end
300338

339+
def test_date
340+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
341+
end
342+
301343
def test_record
302344
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
303345
assert_equal nil, converter.call(nil)

0 commit comments

Comments
 (0)