From 34de823ff55849057b94cc611a62c4a4e27bc1a8 Mon Sep 17 00:00:00 2001 From: p-eye Date: Mon, 22 Jul 2024 14:22:37 +0900 Subject: [PATCH 1/6] add TIME type conversion to string converter --- lib/embulk/output/bigquery/value_converter_factory.rb | 7 +++++++ test/test_value_converter_factory.rb | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/lib/embulk/output/bigquery/value_converter_factory.rb b/lib/embulk/output/bigquery/value_converter_factory.rb index ac0bae3..0e575b2 100644 --- a/lib/embulk/output/bigquery/value_converter_factory.rb +++ b/lib/embulk/output/bigquery/value_converter_factory.rb @@ -224,6 +224,13 @@ def string_converter val # Users must care of BQ timestamp format } end + when 'TIME' + Proc.new {|val| + next nil if val.nil? + with_typecast_error(val) do |val| + Time.parse(val).strftime("%H:%M:%S.%6N") + end + } when 'RECORD' Proc.new {|val| next nil if val.nil? diff --git a/test/test_value_converter_factory.rb b/test/test_value_converter_factory.rb index fcfa116..a5f84c1 100644 --- a/test/test_value_converter_factory.rb +++ b/test/test_value_converter_factory.rb @@ -262,6 +262,15 @@ def test_datetime assert_equal "2016-02-26 00:00:00", converter.call("2016-02-26 00:00:00") end + def test_time + converter = ValueConverterFactory.new( + SCHEMA_TYPE, 'TIME', + timestamp_format: '%H:%M:%S' + ).create_converter + assert_equal nil, converter.call(nil) + assert_equal "00:03:22.000000", converter.call("00:03:22") + end + def test_record converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter assert_equal({'foo'=>'foo'}, converter.call(%Q[{"foo":"foo"}])) From 8eaab04eef62d98e781d2aaba88af8ee0b18904c Mon Sep 17 00:00:00 2001 From: p-eye Date: Thu, 25 Jul 2024 08:48:42 +0900 Subject: [PATCH 2/6] fix: add more test cases --- test/test_value_converter_factory.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test_value_converter_factory.rb b/test/test_value_converter_factory.rb index a5f84c1..e84dae4 100644 --- a/test/test_value_converter_factory.rb +++ b/test/test_value_converter_factory.rb @@ -269,6 +269,10 @@ def test_time ).create_converter assert_equal nil, converter.call(nil) assert_equal "00:03:22.000000", converter.call("00:03:22") + assert_equal "15:22:00.000000", converter.call("3:22 PM") + assert_equal "03:22:00.000000", converter.call("3:22 AM") + assert_equal "15:22:00.000000", converter.call("15:22") + assert_equal "10:00:00.000000", converter.call("2024-07-24 10:00") end def test_record From 8fd7b4eb8fe19cf4323f97556431ede90f95ad65 Mon Sep 17 00:00:00 2001 From: p-eye Date: Thu, 25 Jul 2024 19:04:06 +0900 Subject: [PATCH 3/6] fix: timezone setting for TIME type --- .../output/bigquery/value_converter_factory.rb | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/embulk/output/bigquery/value_converter_factory.rb b/lib/embulk/output/bigquery/value_converter_factory.rb index 0e575b2..89fb6e5 100644 --- a/lib/embulk/output/bigquery/value_converter_factory.rb +++ b/lib/embulk/output/bigquery/value_converter_factory.rb @@ -225,12 +225,19 @@ def string_converter } end when 'TIME' - Proc.new {|val| - next nil if val.nil? - with_typecast_error(val) do |val| + if @timestamp_format + Proc.new {|val| + next nil if val.nil? + with_typecast_error(val) do |val| + Time.strptime(val, @timestamp_format).strftime("%H:%M:%S.%6N") + end + } + else + Proc.new {|val| + next nil if val.nil? Time.parse(val).strftime("%H:%M:%S.%6N") - end - } + } + end when 'RECORD' Proc.new {|val| next nil if val.nil? From 8e7df4a8ea638a8cc5a77e991e967d40e5c6e2a4 Mon Sep 17 00:00:00 2001 From: p-eye Date: Sat, 27 Jul 2024 09:47:30 +0900 Subject: [PATCH 4/6] fix: add time type to timestamp_converter --- .../bigquery/value_converter_factory.rb | 5 ++++ test/test_value_converter_factory.rb | 30 +++++++++++++++---- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/lib/embulk/output/bigquery/value_converter_factory.rb b/lib/embulk/output/bigquery/value_converter_factory.rb index 89fb6e5..0ad1319 100644 --- a/lib/embulk/output/bigquery/value_converter_factory.rb +++ b/lib/embulk/output/bigquery/value_converter_factory.rb @@ -285,6 +285,11 @@ def timestamp_converter next nil if val.nil? val.localtime(zone_offset).strftime("%Y-%m-%d %H:%M:%S.%6N") } + when 'TIME' + Proc.new {|val| + next nil if val.nil? + val.localtime(zone_offset).strftime("%H:%M:%S.%6N") + } else raise NotSupportedType, "cannot take column type #{type} for timestamp column" end diff --git a/test/test_value_converter_factory.rb b/test/test_value_converter_factory.rb index e84dae4..d8443bc 100644 --- a/test/test_value_converter_factory.rb +++ b/test/test_value_converter_factory.rb @@ -263,16 +263,16 @@ def test_datetime end def test_time - converter = ValueConverterFactory.new( - SCHEMA_TYPE, 'TIME', - timestamp_format: '%H:%M:%S' - ).create_converter + converter = ValueConverterFactory.new(SCHEMA_TYPE, 'TIME').create_converter assert_equal nil, converter.call(nil) assert_equal "00:03:22.000000", converter.call("00:03:22") assert_equal "15:22:00.000000", converter.call("3:22 PM") assert_equal "03:22:00.000000", converter.call("3:22 AM") - assert_equal "15:22:00.000000", converter.call("15:22") - assert_equal "10:00:00.000000", converter.call("2024-07-24 10:00") + + # Users must care of BQ datetime format by themselves with no timestamp_format + converter = ValueConverterFactory.new(SCHEMA_TYPE, 'TIME').create_converter + assert_equal nil, converter.call(nil) + assert_equal "00:00:00.000000", converter.call("2016-02-26 00:00:00") end def test_record @@ -363,6 +363,24 @@ def test_datetime assert_raise { converter.call('foo') } end + def test_time + converter = ValueConverterFactory.new(SCHEMA_TYPE, 'TIME').create_converter + assert_equal nil, converter.call(nil) + timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00") + expected = "00:00:00.500000" + assert_equal expected, converter.call(timestamp) + + converter = ValueConverterFactory.new( + SCHEMA_TYPE, 'TIME', timezone: 'Asia/Tokyo' + ).create_converter + assert_equal nil, converter.call(nil) + timestamp = Time.parse("2016-02-25 15:00:00.500000 +00:00") + expected = "00:00:00.500000" + assert_equal expected, converter.call(timestamp) + + assert_raise { converter.call('foo') } + end + def test_record assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter } end From 489b6b2a1650878983db77f3bdab46e411e0b227 Mon Sep 17 00:00:00 2001 From: p-eye Date: Wed, 14 Aug 2024 10:30:24 +0900 Subject: [PATCH 5/6] fix: use TimeWithZone for time type, but timezone doesn't affect any changes --- .../bigquery/value_converter_factory.rb | 20 +++++++------------ test/test_value_converter_factory.rb | 12 +++++++---- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/lib/embulk/output/bigquery/value_converter_factory.rb b/lib/embulk/output/bigquery/value_converter_factory.rb index 0ad1319..fda53e5 100644 --- a/lib/embulk/output/bigquery/value_converter_factory.rb +++ b/lib/embulk/output/bigquery/value_converter_factory.rb @@ -225,19 +225,13 @@ def string_converter } end when 'TIME' - if @timestamp_format - Proc.new {|val| - next nil if val.nil? - with_typecast_error(val) do |val| - Time.strptime(val, @timestamp_format).strftime("%H:%M:%S.%6N") - end - } - else - Proc.new {|val| - next nil if val.nil? - Time.parse(val).strftime("%H:%M:%S.%6N") - } - end + # TimeWithZone doesn't affect any change to the time value + Proc.new {|val| + next nil if val.nil? + with_typecast_error(val) do |val| + TimeWithZone.set_zone_offset(Time.parse(val), zone_offset).strftime("%H:%M:%S.%6N") + end + } when 'RECORD' Proc.new {|val| next nil if val.nil? diff --git a/test/test_value_converter_factory.rb b/test/test_value_converter_factory.rb index d8443bc..c49ef07 100644 --- a/test/test_value_converter_factory.rb +++ b/test/test_value_converter_factory.rb @@ -268,11 +268,15 @@ def test_time assert_equal "00:03:22.000000", converter.call("00:03:22") assert_equal "15:22:00.000000", converter.call("3:22 PM") assert_equal "03:22:00.000000", converter.call("3:22 AM") - - # Users must care of BQ datetime format by themselves with no timestamp_format - converter = ValueConverterFactory.new(SCHEMA_TYPE, 'TIME').create_converter - assert_equal nil, converter.call(nil) assert_equal "00:00:00.000000", converter.call("2016-02-26 00:00:00") + + # TimeWithZone doesn't affect any change to the time value + converter = ValueConverterFactory.new( + SCHEMA_TYPE, 'TIME', timezone: 'Asia/Tokyo' + ).create_converter + assert_equal "15:00:01.000000", converter.call("15:00:01") + + assert_raise { converter.call('foo') } end def test_record From 3c0744fda60ee2ad43b53dfb2596ea1237cf7403 Mon Sep 17 00:00:00 2001 From: p-eye Date: Sun, 18 Aug 2024 22:29:24 +0900 Subject: [PATCH 6/6] fix: remove spaces --- test/test_value_converter_factory.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_value_converter_factory.rb b/test/test_value_converter_factory.rb index c49ef07..89c31c1 100644 --- a/test/test_value_converter_factory.rb +++ b/test/test_value_converter_factory.rb @@ -275,7 +275,7 @@ def test_time SCHEMA_TYPE, 'TIME', timezone: 'Asia/Tokyo' ).create_converter assert_equal "15:00:01.000000", converter.call("15:00:01") - + assert_raise { converter.call('foo') } end