Merge pull request #2328 from PsiACE/bump-arrow2

[deps] bump arrow2 & try arrow-format

Author: databend-bot

Cargo.lock

@@ -8,17 +8,17 @@ edition = "2021"
 
 [features]
 default = ["arrow-default", "parquet-default"]
-arrow-default = ["arrow/compute", "arrow/regex", "arrow/merge_sort", "arrow/io_csv", "arrow/io_parquet", "arrow/io_json"]
-parquet-default = ["parquet2/stream"]
+arrow-default = ["arrow/compute", "arrow/regex", "arrow/merge_sort", "arrow/io_csv", "arrow/io_parquet", "arrow/io_json", "arrow/io_flight"]
+parquet-default = ["parquet2/stream", "parquet2/lz4"]
 simd = ["arrow/simd"]
 
 [dependencies] # In alphabetical order
 # Workspace dependencies
 
 # Github dependencies
-arrow = { package = "arrow2", git="https://github.com/datafuse-extras/arrow2", default-features = false, rev = "3f3d76c" }
-arrow-flight = { git="https://github.com/datafuse-extras/arrow2", rev = "3f3d76c" }
-parquet2 = { version = "0.5", optional = false, default_features = false, features = ["stream", "lz4"] }
+arrow = { package = "arrow2", git="https://github.com/datafuse-extras/arrow2", default-features = false, rev = "bd8f02b" }
+arrow-format = { version = "0.2.1", features = ["flight-data", "flight-service"]}
+parquet2 = { version = "0.6", default_features = false }
 # Crates.io dependencies
 
 [dev-dependencies]

common/arrow/Cargo.toml

@@ -13,5 +13,5 @@
 // limitations under the License.
 
 pub use arrow;
-pub use arrow_flight;
+pub use arrow_format;
 pub use parquet2 as parquet;

common/arrow/src/lib.rs

@@ -98,13 +98,9 @@ where T: DFPrimitiveType
     type Output = Result<DFPrimitiveArray<T>>;
 
     fn add(self, rhs: Self) -> Self::Output {
-        arithmetic_helper(
-            self,
-            rhs,
-            basic::add::add,
-            basic::add::add_scalar,
-            |lhs, rhs| lhs + rhs,
-        )
+        arithmetic_helper(self, rhs, basic::add, basic::add_scalar, |lhs, rhs| {
+            lhs + rhs
+        })
     }
 }
 
@@ -121,13 +117,9 @@ where
     type Output = Result<DFPrimitiveArray<T>>;
 
     fn sub(self, rhs: Self) -> Self::Output {
-        arithmetic_helper(
-            self,
-            rhs,
-            basic::sub::sub,
-            basic::sub::sub_scalar,
-            |lhs, rhs| lhs - rhs,
-        )
+        arithmetic_helper(self, rhs, basic::sub, basic::sub_scalar, |lhs, rhs| {
+            lhs - rhs
+        })
     }
 }
 
@@ -145,13 +137,9 @@ where
     type Output = Result<DFPrimitiveArray<T>>;
 
     fn mul(self, rhs: Self) -> Self::Output {
-        arithmetic_helper(
-            self,
-            rhs,
-            basic::mul::mul,
-            basic::mul::mul_scalar,
-            |lhs, rhs| lhs * rhs,
-        )
+        arithmetic_helper(self, rhs, basic::mul, basic::mul_scalar, |lhs, rhs| {
+            lhs * rhs
+        })
     }
 }
 
@@ -170,13 +158,9 @@ where
     type Output = Result<DFPrimitiveArray<T>>;
 
     fn div(self, rhs: Self) -> Self::Output {
-        arithmetic_helper(
-            self,
-            rhs,
-            basic::div::div,
-            basic::div::div_scalar,
-            |lhs, rhs| lhs / rhs,
-        )
+        arithmetic_helper(self, rhs, basic::div, basic::div_scalar, |lhs, rhs| {
+            lhs / rhs
+        })
     }
 }
 
@@ -244,13 +228,10 @@ where
             }
 
             _ => {
-                let array = arithmetic_helper(
-                    self,
-                    rhs,
-                    basic::rem::rem,
-                    basic::rem::rem_scalar,
-                    |lhs, rhs| lhs % rhs,
-                )?;
+                let array =
+                    arithmetic_helper(self, rhs, basic::rem, basic::rem_scalar, |lhs, rhs| {
+                        lhs % rhs
+                    })?;
                 Ok(array.into_series())
             }
         }
@@ -388,14 +369,14 @@ where
     fn pow_f32(&self, exp: f32) -> DFFloat32Array {
         let arr = self.cast_with_type(&DataType::Float32).expect("f32 array");
         let arr = arr.f32().unwrap();
-        DFFloat32Array::new(basic::pow::powf_scalar(&arr.array, exp))
+        DFFloat32Array::new(basic::powf_scalar(&arr.array, exp))
     }
 
     fn pow_f64(&self, exp: f64) -> DFFloat64Array {
         let arr = self.cast_with_type(&DataType::Float64).expect("f64 array");
         let arr = arr.f64().unwrap();
 
-        DFFloat64Array::new(basic::pow::powf_scalar(&arr.array, exp))
+        DFFloat64Array::new(basic::powf_scalar(&arr.array, exp))
     }
 }
 

common/datavalues/src/arrays/arithmetic.rs

@@ -83,7 +83,8 @@ where
         + NumCast
         + Zero
         + Into<DataValue>
-        + AsPrimitive<T::LargestType>,
+        + AsPrimitive<T::LargestType>
+        + std::iter::Sum,
 
     T::LargestType: Into<DataValue> + AddAssign + Default,
 

common/datavalues/src/arrays/ops/agg.rs

@@ -17,9 +17,9 @@ use std::sync::Arc;
 
 use async_trait::async_trait;
 use common_arrow::arrow::datatypes::Schema;
+use common_arrow::arrow::io::flight::serialize_schema;
 use common_arrow::arrow::io::ipc::write::common::IpcWriteOptions;
-use common_arrow::arrow_flight::utils::flight_data_from_arrow_schema;
-use common_arrow::arrow_flight::FlightData;
+use common_arrow::arrow_format::flight::data::FlightData;
 use common_exception::ErrorCode;
 use common_exception::Result;
 use common_meta_api::MetaApi;
@@ -118,7 +118,7 @@ impl MetaApi for MetaEmbedded {
         tracing::info!("create table: {:}: {:?}", &db_name, &table_name);
 
         let options = IpcWriteOptions::default();
-        let flight_data = flight_data_from_arrow_schema(&plan.schema.to_arrow(), &options);
+        let flight_data = serialize_schema(&plan.schema.to_arrow(), &options);
 
         let table = Table {
             table_id: 0,

common/meta/embedded/src/meta_api_impl.rs

@@ -16,7 +16,7 @@ use std::convert::TryInto;
 use std::io::Cursor;
 use std::sync::Arc;
 
-use common_arrow::arrow_flight::Action;
+use common_arrow::arrow_format::flight::data::Action;
 use common_exception::ErrorCode;
 use common_meta_types::CreateDatabaseReply;
 use common_meta_types::CreateTableReply;

common/meta/flight/src/flight_action.rs

@@ -15,10 +15,10 @@
 use std::convert::TryInto;
 use std::time::Duration;
 
-use common_arrow::arrow_flight::flight_service_client::FlightServiceClient;
-use common_arrow::arrow_flight::Action;
-use common_arrow::arrow_flight::BasicAuth;
-use common_arrow::arrow_flight::HandshakeRequest;
+use common_arrow::arrow_format::flight::data::Action;
+use common_arrow::arrow_format::flight::data::BasicAuth;
+use common_arrow::arrow_format::flight::data::HandshakeRequest;
+use common_arrow::arrow_format::flight::service::flight_service_client::FlightServiceClient;
 use common_exception::ErrorCode;
 use common_exception::Result;
 use common_flight_rpc::ConnectionFactory;

common/meta/flight/src/flight_client.rs

@@ -16,20 +16,20 @@ use std::pin::Pin;
 use std::thread::sleep;
 use std::time::Duration;
 
-use common_arrow::arrow_flight::flight_service_server::FlightService;
-use common_arrow::arrow_flight::flight_service_server::FlightServiceServer;
-use common_arrow::arrow_flight::Action;
-use common_arrow::arrow_flight::ActionType;
-use common_arrow::arrow_flight::Criteria;
-use common_arrow::arrow_flight::Empty;
-use common_arrow::arrow_flight::FlightData;
-use common_arrow::arrow_flight::FlightDescriptor;
-use common_arrow::arrow_flight::FlightInfo;
-use common_arrow::arrow_flight::HandshakeRequest;
-use common_arrow::arrow_flight::HandshakeResponse;
-use common_arrow::arrow_flight::PutResult;
-use common_arrow::arrow_flight::SchemaResult;
-use common_arrow::arrow_flight::Ticket;
+use common_arrow::arrow_format::flight::data::Action;
+use common_arrow::arrow_format::flight::data::ActionType;
+use common_arrow::arrow_format::flight::data::Criteria;
+use common_arrow::arrow_format::flight::data::Empty;
+use common_arrow::arrow_format::flight::data::FlightData;
+use common_arrow::arrow_format::flight::data::FlightDescriptor;
+use common_arrow::arrow_format::flight::data::FlightInfo;
+use common_arrow::arrow_format::flight::data::HandshakeRequest;
+use common_arrow::arrow_format::flight::data::HandshakeResponse;
+use common_arrow::arrow_format::flight::data::PutResult;
+use common_arrow::arrow_format::flight::data::SchemaResult;
+use common_arrow::arrow_format::flight::data::Ticket;
+use common_arrow::arrow_format::flight::service::flight_service_server::FlightService;
+use common_arrow::arrow_format::flight::service::flight_service_server::FlightServiceServer;
 use common_base::tokio;
 use futures::Stream;
 use rand::Rng;
@@ -108,7 +108,7 @@ impl FlightService for FlightServiceForTestImpl {
 
     type DoActionStream = Pin<
         Box<
-            dyn Stream<Item = Result<common_arrow::arrow_flight::Result, Status>>
+            dyn Stream<Item = Result<common_arrow::arrow_format::flight::data::Result, Status>>
                 + Send
                 + Sync
                 + 'static,

common/meta/flight/src/tests/flight_server.rs

@@ -23,7 +23,7 @@ use async_raft::raft::Entry;
 use async_raft::raft::EntryPayload;
 use async_raft::raft::MembershipConfig;
 use common_arrow::arrow::datatypes::Schema;
-use common_arrow::arrow_flight::FlightData;
+use common_arrow::arrow_format::flight::data::FlightData;
 use common_exception::prelude::ErrorCode;
 use common_exception::ToErrorCode;
 use common_meta_sled_store::get_sled_db;