From 25bf96a88b33fa89af13f7224687a01266e8a4ce Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Wed, 15 May 2024 20:03:03 +0100 Subject: [PATCH] Update integration-test --- arrow-integration-test/src/lib.rs | 61 ++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/arrow-integration-test/src/lib.rs b/arrow-integration-test/src/lib.rs index 8ea788aa4dd3..66fa9f3320e0 100644 --- a/arrow-integration-test/src/lib.rs +++ b/arrow-integration-test/src/lib.rs @@ -21,7 +21,7 @@ //! //! This is not a canonical format, but provides a human-readable way of verifying language implementations -use arrow_buffer::{IntervalMonthDayNano, ScalarBuffer}; +use arrow_buffer::{IntervalDayTime, IntervalMonthDayNano, ScalarBuffer}; use hex::decode; use num::BigInt; use num::Signed; @@ -32,7 +32,6 @@ use std::sync::Arc; use arrow::array::*; use arrow::buffer::{Buffer, MutableBuffer}; -use arrow::compute; use arrow::datatypes::*; use arrow::error::{ArrowError, Result}; use arrow::util::bit_util; @@ -349,10 +348,7 @@ pub fn array_from_json( } Ok(Arc::new(b.finish())) } - DataType::Int32 - | DataType::Date32 - | DataType::Time32(_) - | DataType::Interval(IntervalUnit::YearMonth) => { + DataType::Int32 | DataType::Date32 | DataType::Time32(_) => { let mut b = Int32Builder::with_capacity(json_col.count); for (is_valid, value) in json_col .validity @@ -367,14 +363,29 @@ pub fn array_from_json( }; } let array = Arc::new(b.finish()) as ArrayRef; - compute::cast(&array, field.data_type()) + arrow::compute::cast(&array, field.data_type()) + } + DataType::Interval(IntervalUnit::YearMonth) => { + let mut b = IntervalYearMonthBuilder::with_capacity(json_col.count); + for (is_valid, value) in json_col + .validity + .as_ref() + .unwrap() + .iter() + .zip(json_col.data.unwrap()) + { + match is_valid { + 1 => b.append_value(value.as_i64().unwrap() as i32), + _ => b.append_null(), + }; + } + Ok(Arc::new(b.finish())) } DataType::Int64 | DataType::Date64 | DataType::Time64(_) | DataType::Timestamp(_, _) - | DataType::Duration(_) - | DataType::Interval(IntervalUnit::DayTime) => { + | DataType::Duration(_) => { let mut b = Int64Builder::with_capacity(json_col.count); for (is_valid, value) in json_col .validity @@ -387,6 +398,25 @@ pub fn array_from_json( 1 => b.append_value(match value { Value::Number(n) => n.as_i64().unwrap(), Value::String(s) => s.parse().expect("Unable to parse string as i64"), + _ => panic!("Unable to parse {value:?} as number"), + }), + _ => b.append_null(), + }; + } + let array = Arc::new(b.finish()) as ArrayRef; + arrow::compute::cast(&array, field.data_type()) + } + DataType::Interval(IntervalUnit::DayTime) => { + let mut b = IntervalDayTimeBuilder::with_capacity(json_col.count); + for (is_valid, value) in json_col + .validity + .as_ref() + .unwrap() + .iter() + .zip(json_col.data.unwrap()) + { + match is_valid { + 1 => b.append_value(match value { Value::Object(ref map) if map.contains_key("days") && map.contains_key("milliseconds") => { @@ -397,13 +427,9 @@ pub fn array_from_json( match (days, milliseconds) { (Value::Number(d), Value::Number(m)) => { - let mut bytes = [0_u8; 8]; - let m = (m.as_i64().unwrap() as i32).to_le_bytes(); - let d = (d.as_i64().unwrap() as i32).to_le_bytes(); - - let c = [d, m].concat(); - bytes.copy_from_slice(c.as_slice()); - i64::from_le_bytes(bytes) + let days = d.as_i64().unwrap() as _; + let millis = m.as_i64().unwrap() as _; + IntervalDayTime::new(days, millis) } _ => { panic!("Unable to parse {value:?} as interval daytime") @@ -418,8 +444,7 @@ pub fn array_from_json( _ => b.append_null(), }; } - let array = Arc::new(b.finish()) as ArrayRef; - compute::cast(&array, field.data_type()) + Ok(Arc::new(b.finish())) } DataType::UInt8 => { let mut b = UInt8Builder::with_capacity(json_col.count);