Support Writing Arrow files (#8608)

* write arrow files

* update datafusion-cli lock

* fix toml formatting

* Update insert_to_external.slt

Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>

* add ticket tracking arrow options

* default to lz4 compression

* update datafusion-cli lock

* cargo update

---------

Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>

Cargo.toml

@@ -17,24 +17,7 @@
 
 [workspace]
 exclude = ["datafusion-cli"]
-members = [
-    "datafusion/common",
-    "datafusion/core",
-    "datafusion/expr",
-    "datafusion/execution",
-    "datafusion/optimizer",
-    "datafusion/physical-expr",
-    "datafusion/physical-plan",
-    "datafusion/proto",
-    "datafusion/proto/gen",
-    "datafusion/sql",
-    "datafusion/sqllogictest",
-    "datafusion/substrait",
-    "datafusion/wasmtest",
-    "datafusion-examples",
-    "docs",
-    "test-utils",
-    "benchmarks",
+members = ["datafusion/common", "datafusion/core", "datafusion/expr", "datafusion/execution", "datafusion/optimizer", "datafusion/physical-expr", "datafusion/physical-plan", "datafusion/proto", "datafusion/proto/gen", "datafusion/sql", "datafusion/sqllogictest", "datafusion/substrait", "datafusion/wasmtest", "datafusion-examples", "docs", "test-utils", "benchmarks",
 ]
 resolver = "2"
 
@@ -53,24 +36,26 @@ arrow = { version = "49.0.0", features = ["prettyprint"] }
 arrow-array = { version = "49.0.0", default-features = false, features = ["chrono-tz"] }
 arrow-buffer = { version = "49.0.0", default-features = false }
 arrow-flight = { version = "49.0.0", features = ["flight-sql-experimental"] }
+arrow-ipc = { version = "49.0.0", default-features = false, features=["lz4"] }
 arrow-ord = { version = "49.0.0", default-features = false }
 arrow-schema = { version = "49.0.0", default-features = false }
 async-trait = "0.1.73"
 bigdecimal = "0.4.1"
 bytes = "1.4"
+chrono = { version = "0.4.31", default-features = false }
 ctor = "0.2.0"
+dashmap = "5.4.0"
 datafusion = { path = "datafusion/core", version = "34.0.0" }
 datafusion-common = { path = "datafusion/common", version = "34.0.0" }
+datafusion-execution = { path = "datafusion/execution", version = "34.0.0" }
 datafusion-expr = { path = "datafusion/expr", version = "34.0.0" }
-datafusion-sql = { path = "datafusion/sql", version = "34.0.0" }
 datafusion-optimizer = { path = "datafusion/optimizer", version = "34.0.0" }
 datafusion-physical-expr = { path = "datafusion/physical-expr", version = "34.0.0" }
 datafusion-physical-plan = { path = "datafusion/physical-plan", version = "34.0.0" }
-datafusion-execution = { path = "datafusion/execution", version = "34.0.0" }
 datafusion-proto = { path = "datafusion/proto", version = "34.0.0" }
+datafusion-sql = { path = "datafusion/sql", version = "34.0.0" }
 datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "34.0.0" }
 datafusion-substrait = { path = "datafusion/substrait", version = "34.0.0" }
-dashmap = "5.4.0"
 doc-comment = "0.3"
 env_logger = "0.10"
 futures = "0.3"
@@ -88,7 +73,6 @@ serde_json = "1"
 sqlparser = { version = "0.40.0", features = ["visitor"] }
 tempfile = "3"
 thiserror = "1.0.44"
-chrono = { version = "0.4.31", default-features = false }
 url = "2.2"
 
 [profile.release]

datafusion/core/Cargo.toml

@@ -55,6 +55,7 @@ ahash = { version = "0.8", default-features = false, features = ["runtime-rng"]
 apache-avro = { version = "0.16", optional = true }
 arrow = { workspace = true }
 arrow-array = { workspace = true }
+arrow-ipc = { workspace = true }
 arrow-schema = { workspace = true }
 async-compression = { version = "0.4.0", features = ["bzip2", "gzip", "xz", "zstd", "futures-io", "tokio"], optional = true }
 async-trait = { workspace = true }