Skip to content

Commit 33f4d00

Browse files
Merge pull request #142 from fagai/support_description
Support description of columns and tables
2 parents 9cb40d1 + dd311f5 commit 33f4d00

File tree

5 files changed

+12
-6
lines changed

5 files changed

+12
-6
lines changed

README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ OAuth flow for installed applications.
6060
| gcs_bucket | string | optional | nil | See [GCS Bucket](#gcs-bucket) |
6161
| auto_create_gcs_bucket | boolean | optional | false | See [GCS Bucket](#gcs-bucket) |
6262
| progress_log_interval | float | optional | nil (Disabled) | Progress log interval. The progress log is disabled by nil (default). NOTE: This option may be removed in a future because a filter plugin can achieve the same goal |
63+
| description | string | optional | nil | description of table |
6364

6465
Client or request options
6566

@@ -329,6 +330,7 @@ Column options are used to aid guessing BigQuery schema, or to define conversion
329330
- json: `STRING`, `RECORD` (default: `STRING`)
330331
- **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`)
331332
- **fields**: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column.
333+
- **description**: description (string, default is `None`).
332334
- **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is `default_timestamp_format`)
333335
- **timezone**: timezone to convert into/from `timestamp`, `date` (string, default is `default_timezone`).
334336
- **default_timestamp_format**: default timestamp format for column_options (string, default is "%Y-%m-%d %H:%M:%S.%6N")

lib/embulk/output/bigquery.rb

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ def self.configure(config, schema, task_count)
6363
'payload_column' => config.param('payload_column', :string, :default => nil),
6464
'payload_column_index' => config.param('payload_column_index', :integer, :default => nil),
6565

66+
'description' => config.param('description', :string, :default => nil),
67+
6668
'open_timeout_sec' => config.param('open_timeout_sec', :integer, :default => nil),
6769
'timeout_sec' => config.param('timeout_sec', :integer, :default => nil), # google-api-ruby-client < v0.11.0
6870
'send_timeout_sec' => config.param('send_timeout_sec', :integer, :default => nil), # google-api-ruby-client >= v0.11.0

lib/embulk/output/bigquery/bigquery_client.rb

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,7 @@ def create_table_if_not_exists(table, dataset: nil, options: nil)
420420
table_reference: {
421421
table_id: table,
422422
},
423+
description: @task['description'],
423424
schema: {
424425
fields: fields,
425426
}

lib/embulk/output/bigquery/helper.rb

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,11 @@ def self.fields_from_embulk_schema(task, schema)
4646
embulk_type = column[:type]
4747
column_option = column_options_map[column_name] || {}
4848
{}.tap do |field|
49-
field[:name] = column_name
50-
field[:type] = (column_option['type'] || bq_type_from_embulk_type(embulk_type)).upcase
51-
field[:mode] = column_option['mode'] if column_option['mode']
52-
field[:fields] = deep_symbolize_keys(column_option['fields']) if column_option['fields']
49+
field[:name] = column_name
50+
field[:type] = (column_option['type'] || bq_type_from_embulk_type(embulk_type)).upcase
51+
field[:mode] = column_option['mode'] if column_option['mode']
52+
field[:fields] = deep_symbolize_keys(column_option['fields']) if column_option['fields']
53+
field[:description] = column_option['description'] if column_option['description']
5354
end
5455
end
5556
end

test/test_helper.rb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ def test_fields_from_embulk_schema_with_column_options
6868
])
6969
task = {
7070
'column_options' => [
71-
{'name' => 'boolean', 'type' => 'STRING', 'mode' => 'REQUIRED'},
71+
{'name' => 'boolean', 'type' => 'STRING', 'mode' => 'REQUIRED', 'description' => 'hoge'},
7272
{'name' => 'long', 'type' => 'STRING'},
7373
{'name' => 'double', 'type' => 'STRING'},
7474
{'name' => 'string', 'type' => 'INTEGER'},
@@ -81,7 +81,7 @@ def test_fields_from_embulk_schema_with_column_options
8181
],
8282
}
8383
expected = [
84-
{name: 'boolean', type: 'STRING', mode: 'REQUIRED'},
84+
{name: 'boolean', type: 'STRING', mode: 'REQUIRED', description: 'hoge'},
8585
{name: 'long', type: 'STRING'},
8686
{name: 'double', type: 'STRING'},
8787
{name: 'string', type: 'INTEGER'},

0 commit comments

Comments
 (0)