From b20a8a0374bd2e260ed8ad254a49d7bd93ce7fee Mon Sep 17 00:00:00 2001 From: Hokuto Morita Date: Tue, 5 Jul 2022 07:42:28 +0000 Subject: [PATCH 01/13] =?UTF-8?q?[clean]=E8=A8=AD=E5=AE=9A=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + Gemfile | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 230adea..a4a49ec 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ vendor/ .tags your-project-000.json embulk.jar +config.yml diff --git a/Gemfile b/Gemfile index 0262182..8e8c4c3 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,10 @@ source 'https://rubygems.org/' gemspec -gem 'embulk' +# INFO: v0.9系のembulkを使用している場合は『gem 'embulk', '< 0.10'』と指定してください +# 参考: https://zenn.dev/hiroysato/articles/957b1b4f77d549 +# gem 'embulk' +gem 'embulk', '< 0.10' gem 'liquid', '= 4.0.0' # the version included in embulk.jar gem 'embulk-parser-none' gem 'embulk-parser-jsonl' From 2199b2ccd84c580a6ab70bc23a6035d0a163950b Mon Sep 17 00:00:00 2001 From: Hokuto Morita Date: Tue, 5 Jul 2022 07:42:47 +0000 Subject: [PATCH 02/13] =?UTF-8?q?[add]NUMERIC=E5=AF=BE=E5=BF=9C=E3=81=AE?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../output/bigquery/value_converter_factory.rb | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/embulk/output/bigquery/value_converter_factory.rb b/lib/embulk/output/bigquery/value_converter_factory.rb index fd6b7e2..638cb18 100644 --- a/lib/embulk/output/bigquery/value_converter_factory.rb +++ b/lib/embulk/output/bigquery/value_converter_factory.rb @@ -1,6 +1,7 @@ require 'time' require 'time_with_zone' require 'json' +require 'bigdecimal' require_relative 'helper' module Embulk @@ -14,6 +15,7 @@ class TypeCastError < StandardError; end DEFAULT_TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S.%6N" # BigQuery timestamp format DEFAULT_TIMEZONE = "UTC" + DEFAULT_SCALE = 9 # @param [Hash] task # @option task [String] default_timestamp_format @@ -29,6 +31,7 @@ def self.create_converters(task, schema) column_name = column[:name] embulk_type = column[:type] column_option = column_options_map[column_name] || {} + scale = column_option['scale'] || DEFAULT_SCALE self.new( embulk_type, column_option['type'], timestamp_format: column_option['timestamp_format'], @@ -36,6 +39,7 @@ def self.create_converters(task, schema) strict: column_option['strict'], default_timestamp_format: default_timestamp_format, default_timezone: default_timezone, + scale: scale ).create_converter end end @@ -46,7 +50,8 @@ def initialize( embulk_type, type = nil, timestamp_format: nil, timezone: nil, strict: nil, default_timestamp_format: DEFAULT_TIMESTAMP_FORMAT, - default_timezone: DEFAULT_TIMEZONE + default_timezone: DEFAULT_TIMEZONE, + scale: DEFAULT_SCALE ) @embulk_type = embulk_type @type = (type || Helper.bq_type_from_embulk_type(embulk_type)).upcase @@ -55,6 +60,7 @@ def initialize( @timezone = timezone || default_timezone @zone_offset = TimeWithZone.zone_offset(@timezone) @strict = strict.nil? ? true : strict + @scale = scale end def create_converter @@ -231,6 +237,13 @@ def string_converter JSON.parse(val) end } + when 'NUMERIC' + Proc.new {|val| + next nil if val.nil? + with_typecast_error(val) do |val| + BigDecimal(val).round(@scale, BigDecimal::ROUND_CEILING) + end + } else raise NotSupportedType, "cannot take column type #{type} for string column" end From e112293163f47b62f443c5b154c4674b14efd9cb Mon Sep 17 00:00:00 2001 From: Hokuto Morita Date: Tue, 5 Jul 2022 07:48:03 +0000 Subject: [PATCH 03/13] [add]README --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6571b7b..fe5e5ed 100644 --- a/README.md +++ b/README.md @@ -307,18 +307,20 @@ Column options are used to aid guessing BigQuery schema, or to define conversion - **column_options**: advanced: an array of options for columns - **name**: column name - - **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, and `RECORD`. See belows for supported conversion type. + - **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, `RECORD`, and `NUMERIC`. See belows for supported conversion type. - boolean: `BOOLEAN`, `STRING` (default: `BOOLEAN`) - long: `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `INTEGER`) - double: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `FLOAT`) - string: `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, `RECORD` (default: `STRING`) - timestamp: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE` (default: `TIMESTAMP`) - json: `STRING`, `RECORD` (default: `STRING`) + - numeric: `STRING` - **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`) - **fields**: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column. - **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is `default_timestamp_format`) - **timezone**: timezone to convert into/from `timestamp`, `date` (string, default is `default_timezone`). - **description**: description for the column. + - **scale**: optional, [scale](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types?hl=ja#decimal_types) for numeric column (long, default is 9). - **default_timestamp_format**: default timestamp format for column_options (string, default is "%Y-%m-%d %H:%M:%S.%6N") - **default_timezone**: default timezone for column_options (string, default is "UTC") From 8611ef98e28ee3a31a9b96c993b6b08bb931d890 Mon Sep 17 00:00:00 2001 From: Hokuto Morita Date: Mon, 11 Jul 2022 04:31:14 +0000 Subject: [PATCH 04/13] =?UTF-8?q?[clean]gem=E3=83=95=E3=82=A1=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E3=81=AE=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Gemfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 8e8c4c3..45c9954 100644 --- a/Gemfile +++ b/Gemfile @@ -1,10 +1,7 @@ source 'https://rubygems.org/' gemspec -# INFO: v0.9系のembulkを使用している場合は『gem 'embulk', '< 0.10'』と指定してください -# 参考: https://zenn.dev/hiroysato/articles/957b1b4f77d549 -# gem 'embulk' -gem 'embulk', '< 0.10' +gem 'embulk', '< 0.10' # INFO: v0.9系のembulkコマンドの場合エラーになるのでバージョン指定してください。参考: https://zenn.dev/hiroysato/articles/957b1b4f77d549 gem 'liquid', '= 4.0.0' # the version included in embulk.jar gem 'embulk-parser-none' gem 'embulk-parser-jsonl' From f61e41fd48e0cfcf891f70d264f4aee125d453b9 Mon Sep 17 00:00:00 2001 From: Hokuto Morita Date: Mon, 11 Jul 2022 04:32:51 +0000 Subject: [PATCH 05/13] =?UTF-8?q?[update]embulk=20gem=E3=81=AE=E3=83=90?= =?UTF-8?q?=E3=83=BC=E3=82=B8=E3=83=A7=E3=83=B3=E3=82=A2=E3=83=83=E3=83=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- embulk-output-bigquery.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embulk-output-bigquery.gemspec b/embulk-output-bigquery.gemspec index df11e69..3c13399 100644 --- a/embulk-output-bigquery.gemspec +++ b/embulk-output-bigquery.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |spec| spec.name = "embulk-output-bigquery" - spec.version = "0.6.13" + spec.version = "0.6.14" spec.authors = ["Satoshi Akama", "Naotoshi Seo"] spec.summary = "Google BigQuery output plugin for Embulk" spec.description = "Embulk plugin that insert records to Google BigQuery." From 2639d898e3289375c883fce6bc8c249af6250109 Mon Sep 17 00:00:00 2001 From: Hokuto Morita Date: Mon, 11 Jul 2022 05:56:44 +0000 Subject: [PATCH 06/13] [add]github-actions --- .github/workflows/gem-push.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/gem-push.yml diff --git a/.github/workflows/gem-push.yml b/.github/workflows/gem-push.yml new file mode 100644 index 0000000..443ade3 --- /dev/null +++ b/.github/workflows/gem-push.yml @@ -0,0 +1,25 @@ +name: Ruby Gem + +on: + workflow_dispatch: + push: + tags: + - 'v*' + +jobs: + build: + name: Build + Publish + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v2 + - name: Set up Ruby 2.7 + uses: ruby/setup-ruby@v1 + with: + ruby-version: 2.7 + - name: push gem + uses: trocco-io/push-gem-to-gpr-action@v1 + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" From b3b005e06832e7cd14116454f0b36ca22b955b56 Mon Sep 17 00:00:00 2001 From: Hokuto Morita Date: Mon, 11 Jul 2022 06:06:56 +0000 Subject: [PATCH 07/13] =?UTF-8?q?[clean]=E5=8B=95=E4=BD=9C=E7=A2=BA?= =?UTF-8?q?=E8=AA=8D=E7=94=A8=E3=81=AEgem=E3=83=90=E3=83=BC=E3=82=B8?= =?UTF-8?q?=E3=83=A7=E3=83=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- embulk-output-bigquery.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embulk-output-bigquery.gemspec b/embulk-output-bigquery.gemspec index 3c13399..28779c7 100644 --- a/embulk-output-bigquery.gemspec +++ b/embulk-output-bigquery.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |spec| spec.name = "embulk-output-bigquery" - spec.version = "0.6.14" + spec.version = "0.6.14.2639d898e3289.pre" spec.authors = ["Satoshi Akama", "Naotoshi Seo"] spec.summary = "Google BigQuery output plugin for Embulk" spec.description = "Embulk plugin that insert records to Google BigQuery." From 0bfe0b4e1cecc3fced4599b7eb1fd0c15b8f3033 Mon Sep 17 00:00:00 2001 From: Hokuto Morita Date: Mon, 11 Jul 2022 06:09:11 +0000 Subject: [PATCH 08/13] =?UTF-8?q?[clean]=E3=83=90=E3=83=BC=E3=82=B8?= =?UTF-8?q?=E3=83=A7=E3=83=B3=E3=82=92=E5=85=83=E3=81=AB=E6=88=BB=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- embulk-output-bigquery.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embulk-output-bigquery.gemspec b/embulk-output-bigquery.gemspec index 28779c7..3c13399 100644 --- a/embulk-output-bigquery.gemspec +++ b/embulk-output-bigquery.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |spec| spec.name = "embulk-output-bigquery" - spec.version = "0.6.14.2639d898e3289.pre" + spec.version = "0.6.14" spec.authors = ["Satoshi Akama", "Naotoshi Seo"] spec.summary = "Google BigQuery output plugin for Embulk" spec.description = "Embulk plugin that insert records to Google BigQuery." From bbf58eb108c3c291655d3ea40225c70bd7357f33 Mon Sep 17 00:00:00 2001 From: Hokuto Morita Date: Tue, 12 Jul 2022 04:31:37 +0000 Subject: [PATCH 09/13] =?UTF-8?q?[clean]Gemfile=E3=81=AE=E3=82=B3=E3=83=A1?= =?UTF-8?q?=E3=83=B3=E3=83=88=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 45c9954..cd18cd5 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ source 'https://rubygems.org/' gemspec -gem 'embulk', '< 0.10' # INFO: v0.9系のembulkコマンドの場合エラーになるのでバージョン指定してください。参考: https://zenn.dev/hiroysato/articles/957b1b4f77d549 +gem 'embulk', '< 0.10' # INFO: v0.9系のembulkコマンドを使う場合はエラーになるのでバージョン指定してください。参考: https://zenn.dev/hiroysato/articles/957b1b4f77d549 gem 'liquid', '= 4.0.0' # the version included in embulk.jar gem 'embulk-parser-none' gem 'embulk-parser-jsonl' From 822a312a79bb7dfc1723fe2d86514fc3c0e808a7 Mon Sep 17 00:00:00 2001 From: Hokuto Morita Date: Tue, 12 Jul 2022 04:32:19 +0000 Subject: [PATCH 10/13] =?UTF-8?q?[clean]gitignore=E3=81=AE=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index a4a49ec..230adea 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,3 @@ vendor/ .tags your-project-000.json embulk.jar -config.yml From edea85693cec0526477b003ddb4e40bb9fa5aaed Mon Sep 17 00:00:00 2001 From: Hokuto Morita Date: Wed, 13 Jul 2022 03:45:03 +0000 Subject: [PATCH 11/13] =?UTF-8?q?[clean]Gemfile=E3=81=AE=E3=82=B3=E3=83=A1?= =?UTF-8?q?=E3=83=B3=E3=83=88=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index cd18cd5..1b22031 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ source 'https://rubygems.org/' gemspec -gem 'embulk', '< 0.10' # INFO: v0.9系のembulkコマンドを使う場合はエラーになるのでバージョン指定してください。参考: https://zenn.dev/hiroysato/articles/957b1b4f77d549 +gem 'embulk', '< 0.10' # INFO: v0.9系のembulkを使用するためのバージョン指定、参考: https://zenn.dev/hiroysato/articles/957b1b4f77d549 gem 'liquid', '= 4.0.0' # the version included in embulk.jar gem 'embulk-parser-none' gem 'embulk-parser-jsonl' From ae9c662dee8425325c824aedad318d4d2bc789e3 Mon Sep 17 00:00:00 2001 From: Hokuto Morita Date: Wed, 13 Jul 2022 06:06:05 +0000 Subject: [PATCH 12/13] =?UTF-8?q?[clean]Gemfile=E3=81=AE=E3=82=B3=E3=83=A1?= =?UTF-8?q?=E3=83=B3=E3=83=88=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 1b22031..7328ea6 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ source 'https://rubygems.org/' gemspec -gem 'embulk', '< 0.10' # INFO: v0.9系のembulkを使用するためのバージョン指定、参考: https://zenn.dev/hiroysato/articles/957b1b4f77d549 +gem 'embulk', '< 0.10' gem 'liquid', '= 4.0.0' # the version included in embulk.jar gem 'embulk-parser-none' gem 'embulk-parser-jsonl' From dd44c7fbb6a96f26e70f0a1e6af810d5a186ab62 Mon Sep 17 00:00:00 2001 From: Hokuto Morita Date: Thu, 14 Jul 2022 04:58:33 +0000 Subject: [PATCH 13/13] =?UTF-8?q?[update]embulk=20gem=E3=81=AE=E3=83=90?= =?UTF-8?q?=E3=83=BC=E3=82=B8=E3=83=A7=E3=83=B3=E3=82=A2=E3=83=83=E3=83=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- embulk-output-bigquery.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embulk-output-bigquery.gemspec b/embulk-output-bigquery.gemspec index 3c13399..5469cbb 100644 --- a/embulk-output-bigquery.gemspec +++ b/embulk-output-bigquery.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |spec| spec.name = "embulk-output-bigquery" - spec.version = "0.6.14" + spec.version = "0.7.0" spec.authors = ["Satoshi Akama", "Naotoshi Seo"] spec.summary = "Google BigQuery output plugin for Embulk" spec.description = "Embulk plugin that insert records to Google BigQuery."