diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5f18846de95a..38c4f2fc6ea3 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -430,7 +430,7 @@ jobs: export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data cd datafusion # Force all hash values to collide - cargo test --features=force_hash_collisions + cargo test --all --features=force_hash_collisions env: CARGO_HOME: "/github/home/.cargo" CARGO_TARGET_DIR: "/github/home/target" diff --git a/Cargo.toml b/Cargo.toml index 9a060707c7ce..52a5ee1d10eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,21 +17,21 @@ [workspace] members = [ - "datafusion", - "datafusion-common", - "datafusion-expr", - "datafusion-jit", - "datafusion-physical-expr", + "datafusion/core", + "datafusion/common", + "datafusion/expr", + "datafusion/jit", + "datafusion/physical-expr", + "datafusion/proto", "datafusion-cli", "datafusion-examples", - "datafusion-proto", - "datafusion-storage", "benchmarks", "ballista/rust/client", "ballista/rust/core", "ballista/rust/executor", "ballista/rust/scheduler", "ballista-examples", + "data-access", ] [profile.release] diff --git a/ballista-examples/Cargo.toml b/ballista-examples/Cargo.toml index a053e073e7d2..fb956ea2cd9c 100644 --- a/ballista-examples/Cargo.toml +++ b/ballista-examples/Cargo.toml @@ -36,7 +36,7 @@ required-features = ["ballista/standalone"] [dependencies] ballista = { path = "../ballista/rust/client", version = "0.6.0" } -datafusion = { path = "../datafusion" } +datafusion = { path = "../datafusion/core" } futures = "0.3" num_cpus = "1.13.0" prost = "0.9" diff --git a/ballista/rust/client/Cargo.toml b/ballista/rust/client/Cargo.toml index 799a23e48f68..8644f3d740ea 100644 --- a/ballista/rust/client/Cargo.toml +++ b/ballista/rust/client/Cargo.toml @@ -31,7 +31,7 @@ ballista-core = { path = "../core", version = "0.6.0" } ballista-executor = { path = "../executor", version = "0.6.0", optional = true } ballista-scheduler = { path = "../scheduler", version = "0.6.0", optional = true } -datafusion = { path = "../../../datafusion", version = "7.0.0" } +datafusion = { path = "../../../datafusion/core", version = "7.0.0" } futures = "0.3" log = "0.4" parking_lot = "0.12" diff --git a/ballista/rust/core/Cargo.toml b/ballista/rust/core/Cargo.toml index b7125368e72a..617003dae00c 100644 --- a/ballista/rust/core/Cargo.toml +++ b/ballista/rust/core/Cargo.toml @@ -27,6 +27,8 @@ edition = "2018" build = "build.rs" [features] +# Used for testing ONLY: causes all values to hash to the same value (test for collisions) +force_hash_collisions = ["datafusion/force_hash_collisions"] simd = ["datafusion/simd"] [dependencies] @@ -36,8 +38,8 @@ arrow-flight = { version = "11" } async-trait = "0.1.41" chrono = { version = "0.4", default-features = false } clap = { version = "3", features = ["derive", "cargo"] } -datafusion = { path = "../../../datafusion", version = "7.0.0" } -datafusion-proto = { path = "../../../datafusion-proto", version = "7.0.0" } +datafusion = { path = "../../../datafusion/core", version = "7.0.0" } +datafusion-proto = { path = "../../../datafusion/proto", version = "7.0.0" } futures = "0.3" hashbrown = "0.12" log = "0.4" diff --git a/ballista/rust/core/src/execution_plans/shuffle_writer.rs b/ballista/rust/core/src/execution_plans/shuffle_writer.rs index 9058cc40b086..530f123d989f 100644 --- a/ballista/rust/core/src/execution_plans/shuffle_writer.rs +++ b/ballista/rust/core/src/execution_plans/shuffle_writer.rs @@ -452,6 +452,8 @@ mod tests { use tempfile::TempDir; #[tokio::test] + // number of rows in each partition is a function of the hash output, so don't test here + #[cfg(not(feature = "force_hash_collisions"))] async fn test() -> Result<()> { let session_ctx = SessionContext::new(); let task_ctx = session_ctx.task_ctx(); @@ -507,6 +509,8 @@ mod tests { } #[tokio::test] + // number of rows in each partition is a function of the hash output, so don't test here + #[cfg(not(feature = "force_hash_collisions"))] async fn test_partitioned() -> Result<()> { let session_ctx = SessionContext::new(); let task_ctx = session_ctx.task_ctx(); diff --git a/ballista/rust/core/src/serde/logical_plan/mod.rs b/ballista/rust/core/src/serde/logical_plan/mod.rs index 3e36c00eedf8..05f396608978 100644 --- a/ballista/rust/core/src/serde/logical_plan/mod.rs +++ b/ballista/rust/core/src/serde/logical_plan/mod.rs @@ -922,7 +922,7 @@ mod roundtrip_tests { use core::panic; use datafusion::{ arrow::datatypes::{DataType, Field, Schema}, - datafusion_storage::{ + datafusion_data_access::{ self, object_store::{ local::LocalFileSystem, FileMetaStream, ListEntryStream, ObjectReader, @@ -949,7 +949,7 @@ mod roundtrip_tests { async fn list_file( &self, _prefix: &str, - ) -> datafusion_storage::Result { + ) -> datafusion_data_access::Result { Err(io::Error::new( io::ErrorKind::Unsupported, "this is only a test object store".to_string(), @@ -960,7 +960,7 @@ mod roundtrip_tests { &self, _prefix: &str, _delimiter: Option, - ) -> datafusion_storage::Result { + ) -> datafusion_data_access::Result { Err(io::Error::new( io::ErrorKind::Unsupported, "this is only a test object store".to_string(), @@ -970,7 +970,7 @@ mod roundtrip_tests { fn file_reader( &self, _file: SizedFile, - ) -> datafusion_storage::Result> { + ) -> datafusion_data_access::Result> { Err(io::Error::new( io::ErrorKind::Unsupported, "this is only a test object store".to_string(), diff --git a/ballista/rust/core/src/serde/mod.rs b/ballista/rust/core/src/serde/mod.rs index 5f3d054e2034..2cd02b9f9f87 100644 --- a/ballista/rust/core/src/serde/mod.rs +++ b/ballista/rust/core/src/serde/mod.rs @@ -351,7 +351,7 @@ fn str_to_byte(s: &str) -> Result { mod tests { use async_trait::async_trait; use datafusion::arrow::datatypes::SchemaRef; - use datafusion::datafusion_storage::object_store::local::LocalFileSystem; + use datafusion::datafusion_data_access::object_store::local::LocalFileSystem; use datafusion::error::DataFusionError; use datafusion::execution::context::{QueryPlanner, SessionState, TaskContext}; use datafusion::execution::runtime_env::{RuntimeConfig, RuntimeEnv}; @@ -715,7 +715,7 @@ mod tests { let scan = LogicalPlanBuilder::scan_csv( store, - "../../../datafusion/tests/customer.csv", + "../../../datafusion/core/tests/customer.csv", CsvReadOptions::default(), None, 1, diff --git a/ballista/rust/core/src/serde/physical_plan/from_proto.rs b/ballista/rust/core/src/serde/physical_plan/from_proto.rs index 19fdcfa53506..13f98535e9d5 100644 --- a/ballista/rust/core/src/serde/physical_plan/from_proto.rs +++ b/ballista/rust/core/src/serde/physical_plan/from_proto.rs @@ -26,7 +26,7 @@ use crate::serde::{from_proto_binary_op, proto_error, protobuf}; use crate::{convert_box_required, convert_required}; use chrono::{TimeZone, Utc}; -use datafusion::datafusion_storage::{ +use datafusion::datafusion_data_access::{ object_store::local::LocalFileSystem, FileMeta, SizedFile, }; use datafusion::datasource::listing::PartitionedFile; diff --git a/ballista/rust/core/src/serde/physical_plan/mod.rs b/ballista/rust/core/src/serde/physical_plan/mod.rs index 39463c791be0..0361df677607 100644 --- a/ballista/rust/core/src/serde/physical_plan/mod.rs +++ b/ballista/rust/core/src/serde/physical_plan/mod.rs @@ -33,7 +33,7 @@ use crate::serde::{ use crate::{convert_box_required, convert_required, into_physical_plan, into_required}; use datafusion::arrow::compute::SortOptions; use datafusion::arrow::datatypes::SchemaRef; -use datafusion::datafusion_storage::object_store::local::LocalFileSystem; +use datafusion::datafusion_data_access::object_store::local::LocalFileSystem; use datafusion::datasource::listing::PartitionedFile; use datafusion::execution::runtime_env::RuntimeEnv; use datafusion::logical_plan::window_frames::WindowFrame; @@ -1020,7 +1020,7 @@ mod roundtrip_tests { compute::kernels::sort::SortOptions, datatypes::{DataType, Field, Schema}, }, - datafusion_storage::object_store::local::LocalFileSystem, + datafusion_data_access::object_store::local::LocalFileSystem, datasource::listing::PartitionedFile, logical_plan::{JoinType, Operator}, physical_plan::{ diff --git a/ballista/rust/executor/Cargo.toml b/ballista/rust/executor/Cargo.toml index e75bd3224fbf..69326352f43a 100644 --- a/ballista/rust/executor/Cargo.toml +++ b/ballista/rust/executor/Cargo.toml @@ -39,7 +39,7 @@ async-trait = "0.1.41" ballista-core = { path = "../core", version = "0.6.0" } chrono = { version = "0.4", default-features = false } configure_me = "0.4.0" -datafusion = { path = "../../../datafusion", version = "7.0.0" } +datafusion = { path = "../../../datafusion/core", version = "7.0.0" } env_logger = "0.9" futures = "0.3" hyper = "0.14.4" diff --git a/ballista/rust/scheduler/Cargo.toml b/ballista/rust/scheduler/Cargo.toml index c07397e412b0..25465adf53e5 100644 --- a/ballista/rust/scheduler/Cargo.toml +++ b/ballista/rust/scheduler/Cargo.toml @@ -40,7 +40,7 @@ async-trait = "0.1.41" ballista-core = { path = "../core", version = "0.6.0" } clap = { version = "3", features = ["derive", "cargo"] } configure_me = "0.4.0" -datafusion = { path = "../../../datafusion", version = "7.0.0" } +datafusion = { path = "../../../datafusion/core", version = "7.0.0" } env_logger = "0.9" etcd-client = { version = "0.8", optional = true } futures = "0.3" diff --git a/ballista/rust/scheduler/src/scheduler_server/grpc.rs b/ballista/rust/scheduler/src/scheduler_server/grpc.rs index eabcb503e9f0..27fe64c4caa5 100644 --- a/ballista/rust/scheduler/src/scheduler_server/grpc.rs +++ b/ballista/rust/scheduler/src/scheduler_server/grpc.rs @@ -33,7 +33,9 @@ use ballista_core::serde::scheduler::{ ExecutorData, ExecutorDataChange, ExecutorMetadata, }; use ballista_core::serde::{AsExecutionPlan, AsLogicalPlan}; -use datafusion::datafusion_storage::object_store::{local::LocalFileSystem, ObjectStore}; +use datafusion::datafusion_data_access::object_store::{ + local::LocalFileSystem, ObjectStore, +}; use datafusion::datasource::file_format::parquet::ParquetFormat; use datafusion::datasource::file_format::FileFormat; use futures::StreamExt; diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index ebab1899492a..f2dc122d6bef 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -33,7 +33,7 @@ snmalloc = ["snmalloc-rs"] [dependencies] ballista = { path = "../ballista/rust/client" } -datafusion = { path = "../datafusion" } +datafusion = { path = "../datafusion/core" } env_logger = "0.9" futures = "0.3" mimalloc = { version = "0.1", optional = true, default-features = false } diff --git a/benchmarks/src/bin/tpch.rs b/benchmarks/src/bin/tpch.rs index a2932ebdc271..d337857b042e 100644 --- a/benchmarks/src/bin/tpch.rs +++ b/benchmarks/src/bin/tpch.rs @@ -52,7 +52,7 @@ use datafusion::{ }; use datafusion::{ arrow::util::pretty, - datafusion_storage::object_store::local::LocalFileSystem, + datafusion_data_access::object_store::local::LocalFileSystem, datasource::listing::{ListingOptions, ListingTable, ListingTableConfig}, }; diff --git a/datafusion-storage/Cargo.toml b/data-access/Cargo.toml similarity index 88% rename from datafusion-storage/Cargo.toml rename to data-access/Cargo.toml index 44e9152ba9aa..aaa869f0ab86 100644 --- a/datafusion-storage/Cargo.toml +++ b/data-access/Cargo.toml @@ -16,9 +16,9 @@ # under the License. [package] -name = "datafusion-storage" -description = "Storage for DataFusion query engine" -version = "7.0.0" +name = "datafusion-data-access" +description = "General data access layer currently mainly based on the object store interfaces" +version = "1.0.0" homepage = "https://github.com/apache/arrow-datafusion" repository = "https://github.com/apache/arrow-datafusion" readme = "README.md" @@ -29,7 +29,7 @@ edition = "2021" rust-version = "1.59" [lib] -name = "datafusion_storage" +name = "datafusion_data_access" path = "src/lib.rs" [dependencies] diff --git a/datafusion-storage/README.md b/data-access/README.md similarity index 76% rename from datafusion-storage/README.md rename to data-access/README.md index 859223e29242..36fdb70951b6 100644 --- a/datafusion-storage/README.md +++ b/data-access/README.md @@ -17,8 +17,6 @@ under the License. --> -# DataFusion Storage +# Data Access Layer -This module contains an `async` API for the [DataFusion][df] to access data, either remotely or locally. - -[df]: https://crates.io/crates/datafusion +This module contains an `async` API for accessing data, either remotely or locally. Currently, it's based on the object store interfaces. In the future, this module may include interfaces for accessing databases, or streaming data. \ No newline at end of file diff --git a/datafusion-storage/src/lib.rs b/data-access/src/lib.rs similarity index 100% rename from datafusion-storage/src/lib.rs rename to data-access/src/lib.rs diff --git a/datafusion-storage/src/object_store/local.rs b/data-access/src/object_store/local.rs similarity index 100% rename from datafusion-storage/src/object_store/local.rs rename to data-access/src/object_store/local.rs diff --git a/datafusion-storage/src/object_store/mod.rs b/data-access/src/object_store/mod.rs similarity index 100% rename from datafusion-storage/src/object_store/mod.rs rename to data-access/src/object_store/mod.rs diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index c636f095f94b..03d394f61b88 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -31,7 +31,7 @@ rust-version = "1.59" arrow = { version = "11" } ballista = { path = "../ballista/rust/client", version = "0.6.0", optional = true } clap = { version = "3", features = ["derive", "cargo"] } -datafusion = { path = "../datafusion", version = "7.0.0" } +datafusion = { path = "../datafusion/core", version = "7.0.0" } dirs = "4.0.0" env_logger = "0.9" mimalloc = { version = "*", default-features = false } diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml index d0bc5ec52dfb..02d3abef5833 100644 --- a/datafusion-examples/Cargo.toml +++ b/datafusion-examples/Cargo.toml @@ -36,7 +36,7 @@ required-features = ["datafusion/avro"] [dev-dependencies] arrow-flight = { version = "11" } async-trait = "0.1.41" -datafusion = { path = "../datafusion" } +datafusion = { path = "../datafusion/core" } futures = "0.3" num_cpus = "1.13.0" prost = "0.9" diff --git a/datafusion-examples/examples/flight_server.rs b/datafusion-examples/examples/flight_server.rs index fcdec561ae38..fa277829f0df 100644 --- a/datafusion-examples/examples/flight_server.rs +++ b/datafusion-examples/examples/flight_server.rs @@ -19,7 +19,7 @@ use std::pin::Pin; use std::sync::Arc; use arrow_flight::SchemaAsIpc; -use datafusion::datafusion_storage::object_store::local::LocalFileSystem; +use datafusion::datafusion_data_access::object_store::local::LocalFileSystem; use datafusion::datasource::file_format::parquet::ParquetFormat; use datafusion::datasource::listing::ListingOptions; use futures::Stream; diff --git a/datafusion-common/Cargo.toml b/datafusion/common/Cargo.toml similarity index 100% rename from datafusion-common/Cargo.toml rename to datafusion/common/Cargo.toml diff --git a/datafusion-common/README.md b/datafusion/common/README.md similarity index 100% rename from datafusion-common/README.md rename to datafusion/common/README.md diff --git a/datafusion-common/src/column.rs b/datafusion/common/src/column.rs similarity index 100% rename from datafusion-common/src/column.rs rename to datafusion/common/src/column.rs diff --git a/datafusion-common/src/dfschema.rs b/datafusion/common/src/dfschema.rs similarity index 100% rename from datafusion-common/src/dfschema.rs rename to datafusion/common/src/dfschema.rs diff --git a/datafusion-common/src/error.rs b/datafusion/common/src/error.rs similarity index 100% rename from datafusion-common/src/error.rs rename to datafusion/common/src/error.rs diff --git a/datafusion-common/src/lib.rs b/datafusion/common/src/lib.rs similarity index 100% rename from datafusion-common/src/lib.rs rename to datafusion/common/src/lib.rs diff --git a/datafusion-common/src/pyarrow.rs b/datafusion/common/src/pyarrow.rs similarity index 100% rename from datafusion-common/src/pyarrow.rs rename to datafusion/common/src/pyarrow.rs diff --git a/datafusion-common/src/scalar.rs b/datafusion/common/src/scalar.rs similarity index 100% rename from datafusion-common/src/scalar.rs rename to datafusion/common/src/scalar.rs diff --git a/datafusion/Cargo.toml b/datafusion/core/Cargo.toml similarity index 89% rename from datafusion/Cargo.toml rename to datafusion/core/Cargo.toml index d8d4a115f30e..f17456e3dd88 100644 --- a/datafusion/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -59,11 +59,11 @@ arrow = { version = "11", features = ["prettyprint"] } async-trait = "0.1.41" avro-rs = { version = "0.13", features = ["snappy"], optional = true } chrono = { version = "0.4", default-features = false } -datafusion-common = { path = "../datafusion-common", version = "7.0.0", features = ["parquet"] } -datafusion-expr = { path = "../datafusion-expr", version = "7.0.0" } -datafusion-jit = { path = "../datafusion-jit", version = "7.0.0", optional = true } -datafusion-physical-expr = { path = "../datafusion-physical-expr", version = "7.0.0" } -datafusion-storage = { path = "../datafusion-storage", version = "7.0.0" } +datafusion-common = { path = "../common", version = "7.0.0", features = ["parquet"] } +datafusion-data-access = { path = "../../data-access", version = "1.0.0" } +datafusion-expr = { path = "../expr", version = "7.0.0" } +datafusion-jit = { path = "../jit", version = "7.0.0", optional = true } +datafusion-physical-expr = { path = "../physical-expr", version = "7.0.0" } futures = "0.3" hashbrown = { version = "0.12", features = ["raw"] } lazy_static = { version = "^1.4.0" } diff --git a/datafusion/benches/aggregate_query_sql.rs b/datafusion/core/benches/aggregate_query_sql.rs similarity index 100% rename from datafusion/benches/aggregate_query_sql.rs rename to datafusion/core/benches/aggregate_query_sql.rs diff --git a/datafusion/benches/data_utils/mod.rs b/datafusion/core/benches/data_utils/mod.rs similarity index 100% rename from datafusion/benches/data_utils/mod.rs rename to datafusion/core/benches/data_utils/mod.rs diff --git a/datafusion/benches/filter_query_sql.rs b/datafusion/core/benches/filter_query_sql.rs similarity index 100% rename from datafusion/benches/filter_query_sql.rs rename to datafusion/core/benches/filter_query_sql.rs diff --git a/datafusion/benches/jit.rs b/datafusion/core/benches/jit.rs similarity index 100% rename from datafusion/benches/jit.rs rename to datafusion/core/benches/jit.rs diff --git a/datafusion/benches/math_query_sql.rs b/datafusion/core/benches/math_query_sql.rs similarity index 100% rename from datafusion/benches/math_query_sql.rs rename to datafusion/core/benches/math_query_sql.rs diff --git a/datafusion/benches/parquet_query_sql.rs b/datafusion/core/benches/parquet_query_sql.rs similarity index 100% rename from datafusion/benches/parquet_query_sql.rs rename to datafusion/core/benches/parquet_query_sql.rs diff --git a/datafusion/benches/parquet_query_sql.sql b/datafusion/core/benches/parquet_query_sql.sql similarity index 100% rename from datafusion/benches/parquet_query_sql.sql rename to datafusion/core/benches/parquet_query_sql.sql diff --git a/datafusion/benches/physical_plan.rs b/datafusion/core/benches/physical_plan.rs similarity index 100% rename from datafusion/benches/physical_plan.rs rename to datafusion/core/benches/physical_plan.rs diff --git a/datafusion/benches/scalar.rs b/datafusion/core/benches/scalar.rs similarity index 100% rename from datafusion/benches/scalar.rs rename to datafusion/core/benches/scalar.rs diff --git a/datafusion/benches/sort_limit_query_sql.rs b/datafusion/core/benches/sort_limit_query_sql.rs similarity index 98% rename from datafusion/benches/sort_limit_query_sql.rs rename to datafusion/core/benches/sort_limit_query_sql.rs index bad723a1af44..faeff6bc9c69 100644 --- a/datafusion/benches/sort_limit_query_sql.rs +++ b/datafusion/core/benches/sort_limit_query_sql.rs @@ -18,7 +18,7 @@ #[macro_use] extern crate criterion; use criterion::Criterion; -use datafusion::datafusion_storage::object_store::local::LocalFileSystem; +use datafusion::datafusion_data_access::object_store::local::LocalFileSystem; use datafusion::datasource::file_format::csv::CsvFormat; use datafusion::datasource::listing::{ListingOptions, ListingTable, ListingTableConfig}; diff --git a/datafusion/benches/window_query_sql.rs b/datafusion/core/benches/window_query_sql.rs similarity index 100% rename from datafusion/benches/window_query_sql.rs rename to datafusion/core/benches/window_query_sql.rs diff --git a/datafusion/fuzz-utils/Cargo.toml b/datafusion/core/fuzz-utils/Cargo.toml similarity index 100% rename from datafusion/fuzz-utils/Cargo.toml rename to datafusion/core/fuzz-utils/Cargo.toml diff --git a/datafusion/fuzz-utils/src/lib.rs b/datafusion/core/fuzz-utils/src/lib.rs similarity index 100% rename from datafusion/fuzz-utils/src/lib.rs rename to datafusion/core/fuzz-utils/src/lib.rs diff --git a/datafusion/src/avro_to_arrow/arrow_array_reader.rs b/datafusion/core/src/avro_to_arrow/arrow_array_reader.rs similarity index 100% rename from datafusion/src/avro_to_arrow/arrow_array_reader.rs rename to datafusion/core/src/avro_to_arrow/arrow_array_reader.rs diff --git a/datafusion/src/avro_to_arrow/mod.rs b/datafusion/core/src/avro_to_arrow/mod.rs similarity index 100% rename from datafusion/src/avro_to_arrow/mod.rs rename to datafusion/core/src/avro_to_arrow/mod.rs diff --git a/datafusion/src/avro_to_arrow/reader.rs b/datafusion/core/src/avro_to_arrow/reader.rs similarity index 100% rename from datafusion/src/avro_to_arrow/reader.rs rename to datafusion/core/src/avro_to_arrow/reader.rs diff --git a/datafusion/src/avro_to_arrow/schema.rs b/datafusion/core/src/avro_to_arrow/schema.rs similarity index 100% rename from datafusion/src/avro_to_arrow/schema.rs rename to datafusion/core/src/avro_to_arrow/schema.rs diff --git a/datafusion/src/catalog/catalog.rs b/datafusion/core/src/catalog/catalog.rs similarity index 100% rename from datafusion/src/catalog/catalog.rs rename to datafusion/core/src/catalog/catalog.rs diff --git a/datafusion/src/catalog/information_schema.rs b/datafusion/core/src/catalog/information_schema.rs similarity index 100% rename from datafusion/src/catalog/information_schema.rs rename to datafusion/core/src/catalog/information_schema.rs diff --git a/datafusion/src/catalog/mod.rs b/datafusion/core/src/catalog/mod.rs similarity index 100% rename from datafusion/src/catalog/mod.rs rename to datafusion/core/src/catalog/mod.rs diff --git a/datafusion/src/catalog/schema.rs b/datafusion/core/src/catalog/schema.rs similarity index 98% rename from datafusion/src/catalog/schema.rs rename to datafusion/core/src/catalog/schema.rs index 62f37729c31b..9ef5c3d67895 100644 --- a/datafusion/src/catalog/schema.rs +++ b/datafusion/core/src/catalog/schema.rs @@ -27,7 +27,7 @@ use crate::datasource::listing::{ListingTable, ListingTableConfig}; use crate::datasource::object_store_registry::ObjectStoreRegistry; use crate::datasource::TableProvider; use crate::error::{DataFusionError, Result}; -use datafusion_storage::object_store::ObjectStore; +use datafusion_data_access::object_store::ObjectStore; /// Represents a schema, comprising a number of named tables. pub trait SchemaProvider: Sync + Send { @@ -251,7 +251,7 @@ mod tests { use crate::catalog::schema::{ MemorySchemaProvider, ObjectStoreSchemaProvider, SchemaProvider, }; - use crate::datafusion_storage::object_store::local::LocalFileSystem; + use crate::datafusion_data_access::object_store::local::LocalFileSystem; use crate::datasource::empty::EmptyTable; use crate::execution::context::SessionContext; diff --git a/datafusion/src/dataframe.rs b/datafusion/core/src/dataframe.rs similarity index 100% rename from datafusion/src/dataframe.rs rename to datafusion/core/src/dataframe.rs diff --git a/datafusion/src/datasource/datasource.rs b/datafusion/core/src/datasource/datasource.rs similarity index 100% rename from datafusion/src/datasource/datasource.rs rename to datafusion/core/src/datasource/datasource.rs diff --git a/datafusion/src/datasource/empty.rs b/datafusion/core/src/datasource/empty.rs similarity index 100% rename from datafusion/src/datasource/empty.rs rename to datafusion/core/src/datasource/empty.rs diff --git a/datafusion/src/datasource/file_format/avro.rs b/datafusion/core/src/datasource/file_format/avro.rs similarity index 98% rename from datafusion/src/datasource/file_format/avro.rs rename to datafusion/core/src/datasource/file_format/avro.rs index f54650f86988..59c7b2e15345 100644 --- a/datafusion/src/datasource/file_format/avro.rs +++ b/datafusion/core/src/datasource/file_format/avro.rs @@ -32,7 +32,7 @@ use crate::logical_plan::Expr; use crate::physical_plan::file_format::{AvroExec, FileScanConfig}; use crate::physical_plan::ExecutionPlan; use crate::physical_plan::Statistics; -use datafusion_storage::object_store::{ObjectReader, ObjectReaderStream}; +use datafusion_data_access::object_store::{ObjectReader, ObjectReaderStream}; /// The default file extension of avro files pub const DEFAULT_AVRO_EXTENSION: &str = ".avro"; @@ -75,7 +75,7 @@ impl FileFormat for AvroFormat { #[cfg(feature = "avro")] mod tests { use crate::{ - datafusion_storage::object_store::local::{ + datafusion_data_access::object_store::local::{ local_object_reader, local_object_reader_stream, LocalFileSystem, }, physical_plan::collect, @@ -394,7 +394,7 @@ mod tests { mod tests { use super::*; - use crate::datafusion_storage::object_store::local::local_object_reader_stream; + use crate::datafusion_data_access::object_store::local::local_object_reader_stream; use crate::error::DataFusionError; #[tokio::test] diff --git a/datafusion/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rs similarity index 98% rename from datafusion/src/datasource/file_format/csv.rs rename to datafusion/core/src/datasource/file_format/csv.rs index 955bc0f78ecc..29ca84a1262e 100644 --- a/datafusion/src/datasource/file_format/csv.rs +++ b/datafusion/core/src/datasource/file_format/csv.rs @@ -31,7 +31,7 @@ use crate::logical_plan::Expr; use crate::physical_plan::file_format::{CsvExec, FileScanConfig}; use crate::physical_plan::ExecutionPlan; use crate::physical_plan::Statistics; -use datafusion_storage::object_store::{ObjectReader, ObjectReaderStream}; +use datafusion_data_access::object_store::{ObjectReader, ObjectReaderStream}; /// The default file extension of csv files pub const DEFAULT_CSV_EXTENSION: &str = ".csv"; @@ -141,7 +141,7 @@ mod tests { use crate::datasource::listing::local_unpartitioned_file; use crate::prelude::{SessionConfig, SessionContext}; use crate::{ - datafusion_storage::object_store::local::{ + datafusion_data_access::object_store::local::{ local_object_reader, local_object_reader_stream, LocalFileSystem, }, datasource::file_format::FileScanConfig, diff --git a/datafusion/src/datasource/file_format/json.rs b/datafusion/core/src/datasource/file_format/json.rs similarity index 98% rename from datafusion/src/datasource/file_format/json.rs rename to datafusion/core/src/datasource/file_format/json.rs index 77d489895394..da51d62bb514 100644 --- a/datafusion/src/datasource/file_format/json.rs +++ b/datafusion/core/src/datasource/file_format/json.rs @@ -35,7 +35,7 @@ use crate::logical_plan::Expr; use crate::physical_plan::file_format::NdJsonExec; use crate::physical_plan::ExecutionPlan; use crate::physical_plan::Statistics; -use datafusion_storage::object_store::{ObjectReader, ObjectReaderStream}; +use datafusion_data_access::object_store::{ObjectReader, ObjectReaderStream}; /// The default file extension of json files pub const DEFAULT_JSON_EXTENSION: &str = ".json"; @@ -104,7 +104,7 @@ mod tests { use super::*; use crate::prelude::{SessionConfig, SessionContext}; use crate::{ - datafusion_storage::object_store::local::{ + datafusion_data_access::object_store::local::{ local_object_reader, local_object_reader_stream, LocalFileSystem, }, datasource::{file_format::FileScanConfig, listing::local_unpartitioned_file}, diff --git a/datafusion/src/datasource/file_format/mod.rs b/datafusion/core/src/datasource/file_format/mod.rs similarity index 97% rename from datafusion/src/datasource/file_format/mod.rs rename to datafusion/core/src/datasource/file_format/mod.rs index 611da32ee46e..062430a2dfaa 100644 --- a/datafusion/src/datasource/file_format/mod.rs +++ b/datafusion/core/src/datasource/file_format/mod.rs @@ -34,7 +34,7 @@ use crate::physical_plan::{ExecutionPlan, Statistics}; use async_trait::async_trait; -use datafusion_storage::object_store::{ObjectReader, ObjectReaderStream}; +use datafusion_data_access::object_store::{ObjectReader, ObjectReaderStream}; /// This trait abstracts all the file format specific implementations /// from the `TableProvider`. This helps code re-utilization across diff --git a/datafusion/src/datasource/file_format/parquet.rs b/datafusion/core/src/datasource/file_format/parquet.rs similarity index 99% rename from datafusion/src/datasource/file_format/parquet.rs rename to datafusion/core/src/datasource/file_format/parquet.rs index 374e360c2a96..b1272780c08a 100644 --- a/datafusion/src/datasource/file_format/parquet.rs +++ b/datafusion/core/src/datasource/file_format/parquet.rs @@ -49,7 +49,7 @@ use crate::physical_plan::expressions::{MaxAccumulator, MinAccumulator}; use crate::physical_plan::file_format::ParquetExec; use crate::physical_plan::ExecutionPlan; use crate::physical_plan::{Accumulator, Statistics}; -use datafusion_storage::object_store::{ObjectReader, ObjectReaderStream}; +use datafusion_data_access::object_store::{ObjectReader, ObjectReaderStream}; /// The default file exetension of parquet files pub const DEFAULT_PARQUET_EXTENSION: &str = ".parquet"; @@ -361,7 +361,7 @@ impl ChunkReader for ChunkObjectReader { mod tests { use crate::datasource::listing::local_unpartitioned_file; use crate::physical_plan::collect; - use datafusion_storage::object_store::local::{ + use datafusion_data_access::object_store::local::{ local_object_reader, local_object_reader_stream, LocalFileSystem, }; diff --git a/datafusion/src/datasource/listing/helpers.rs b/datafusion/core/src/datasource/listing/helpers.rs similarity index 99% rename from datafusion/src/datasource/listing/helpers.rs rename to datafusion/core/src/datasource/listing/helpers.rs index d4ae381bede1..a236bf1e91bb 100644 --- a/datafusion/src/datasource/listing/helpers.rs +++ b/datafusion/core/src/datasource/listing/helpers.rs @@ -46,7 +46,7 @@ use crate::{ }; use super::{PartitionedFile, PartitionedFileStream}; -use datafusion_storage::{object_store::ObjectStore, FileMeta, SizedFile}; +use datafusion_data_access::{object_store::ObjectStore, FileMeta, SizedFile}; const FILE_SIZE_COLUMN_NAME: &str = "_df_part_file_size_"; const FILE_PATH_COLUMN_NAME: &str = "_df_part_file_path_"; @@ -235,7 +235,7 @@ pub async fn pruned_partition_list( .chunks(1024) .map(|v| { v.into_iter() - .collect::>>() + .collect::>>() }) .map_err(DataFusionError::IoError) .map(move |metas| paths_to_batch(table_partition_cols, &stream_path, &metas?)) diff --git a/datafusion/src/datasource/listing/mod.rs b/datafusion/core/src/datasource/listing/mod.rs similarity index 96% rename from datafusion/src/datasource/listing/mod.rs rename to datafusion/core/src/datasource/listing/mod.rs index 2a2042efe842..200349b5efb0 100644 --- a/datafusion/src/datasource/listing/mod.rs +++ b/datafusion/core/src/datasource/listing/mod.rs @@ -22,7 +22,7 @@ mod helpers; mod table; use datafusion_common::ScalarValue; -use datafusion_storage::{object_store::local, FileMeta, Result, SizedFile}; +use datafusion_data_access::{object_store::local, FileMeta, Result, SizedFile}; use futures::Stream; use std::pin::Pin; diff --git a/datafusion/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs similarity index 99% rename from datafusion/src/datasource/listing/table.rs rename to datafusion/core/src/datasource/listing/table.rs index 42b836c361b7..a470e32c6693 100644 --- a/datafusion/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -43,7 +43,7 @@ use crate::datasource::{ }; use super::PartitionedFile; -use datafusion_storage::object_store::ObjectStore; +use datafusion_data_access::object_store::ObjectStore; use super::helpers::{expr_applicable_for_cols, pruned_partition_list, split_files}; @@ -398,7 +398,7 @@ impl ListingTable { mod tests { use crate::datasource::file_format::avro::DEFAULT_AVRO_EXTENSION; use crate::{ - datafusion_storage::object_store::local::LocalFileSystem, + datafusion_data_access::object_store::local::LocalFileSystem, datasource::file_format::{avro::AvroFormat, parquet::ParquetFormat}, logical_plan::{col, lit}, test::{columns, object_store::TestObjectStore}, diff --git a/datafusion/src/datasource/memory.rs b/datafusion/core/src/datasource/memory.rs similarity index 100% rename from datafusion/src/datasource/memory.rs rename to datafusion/core/src/datasource/memory.rs diff --git a/datafusion/src/datasource/mod.rs b/datafusion/core/src/datasource/mod.rs similarity index 100% rename from datafusion/src/datasource/mod.rs rename to datafusion/core/src/datasource/mod.rs diff --git a/datafusion/src/datasource/object_store_registry.rs b/datafusion/core/src/datasource/object_store_registry.rs similarity index 96% rename from datafusion/src/datasource/object_store_registry.rs rename to datafusion/core/src/datasource/object_store_registry.rs index e0504c003548..637fa1cf50db 100644 --- a/datafusion/src/datasource/object_store_registry.rs +++ b/datafusion/core/src/datasource/object_store_registry.rs @@ -20,8 +20,8 @@ //! and query data inside these systems. use datafusion_common::{DataFusionError, Result}; -use datafusion_storage::object_store::local::{LocalFileSystem, LOCAL_SCHEME}; -use datafusion_storage::object_store::ObjectStore; +use datafusion_data_access::object_store::local::{LocalFileSystem, LOCAL_SCHEME}; +use datafusion_data_access::object_store::ObjectStore; use parking_lot::RwLock; use std::collections::HashMap; use std::fmt; diff --git a/datafusion/src/error.rs b/datafusion/core/src/error.rs similarity index 100% rename from datafusion/src/error.rs rename to datafusion/core/src/error.rs diff --git a/datafusion/src/execution/context.rs b/datafusion/core/src/execution/context.rs similarity index 100% rename from datafusion/src/execution/context.rs rename to datafusion/core/src/execution/context.rs diff --git a/datafusion/src/execution/disk_manager.rs b/datafusion/core/src/execution/disk_manager.rs similarity index 100% rename from datafusion/src/execution/disk_manager.rs rename to datafusion/core/src/execution/disk_manager.rs diff --git a/datafusion/src/execution/memory_manager.rs b/datafusion/core/src/execution/memory_manager.rs similarity index 100% rename from datafusion/src/execution/memory_manager.rs rename to datafusion/core/src/execution/memory_manager.rs diff --git a/datafusion/src/execution/mod.rs b/datafusion/core/src/execution/mod.rs similarity index 100% rename from datafusion/src/execution/mod.rs rename to datafusion/core/src/execution/mod.rs diff --git a/datafusion/src/execution/options.rs b/datafusion/core/src/execution/options.rs similarity index 100% rename from datafusion/src/execution/options.rs rename to datafusion/core/src/execution/options.rs diff --git a/datafusion/src/execution/runtime_env.rs b/datafusion/core/src/execution/runtime_env.rs similarity index 98% rename from datafusion/src/execution/runtime_env.rs rename to datafusion/core/src/execution/runtime_env.rs index 70df02248bee..73bbc836ec92 100644 --- a/datafusion/src/execution/runtime_env.rs +++ b/datafusion/core/src/execution/runtime_env.rs @@ -28,7 +28,7 @@ use crate::{ use crate::datasource::object_store_registry::ObjectStoreRegistry; use datafusion_common::DataFusionError; -use datafusion_storage::object_store::ObjectStore; +use datafusion_data_access::object_store::ObjectStore; use std::fmt::{Debug, Formatter}; use std::path::PathBuf; use std::sync::Arc; diff --git a/datafusion/src/from_slice.rs b/datafusion/core/src/from_slice.rs similarity index 100% rename from datafusion/src/from_slice.rs rename to datafusion/core/src/from_slice.rs diff --git a/datafusion/src/lib.rs b/datafusion/core/src/lib.rs similarity index 98% rename from datafusion/src/lib.rs rename to datafusion/core/src/lib.rs index 2c5f9c570f2f..a952b69ae7dc 100644 --- a/datafusion/src/lib.rs +++ b/datafusion/core/src/lib.rs @@ -226,7 +226,7 @@ pub use arrow; pub use parquet; // re-export object store dependencies -pub use datafusion_storage; +pub use datafusion_data_access; #[cfg(feature = "row")] pub mod row; @@ -238,10 +238,10 @@ pub mod test; pub mod test_util; #[cfg(doctest)] -doc_comment::doctest!("../../README.md", readme_example_test); +doc_comment::doctest!("../../../README.md", readme_example_test); #[cfg(doctest)] doc_comment::doctest!( - "../../docs/source/user-guide/example-usage.md", + "../../../docs/source/user-guide/example-usage.md", user_guid_example_tests ); diff --git a/datafusion/src/logical_plan/builder.rs b/datafusion/core/src/logical_plan/builder.rs similarity index 99% rename from datafusion/src/logical_plan/builder.rs rename to datafusion/core/src/logical_plan/builder.rs index 86e9db535e09..23a16fc472c1 100644 --- a/datafusion/src/logical_plan/builder.rs +++ b/datafusion/core/src/logical_plan/builder.rs @@ -36,7 +36,7 @@ use arrow::{ datatypes::{DataType, Schema, SchemaRef}, record_batch::RecordBatch, }; -use datafusion_storage::object_store::ObjectStore; +use datafusion_data_access::object_store::ObjectStore; use std::convert::TryFrom; use std::iter; use std::{ diff --git a/datafusion/src/logical_plan/dfschema.rs b/datafusion/core/src/logical_plan/dfschema.rs similarity index 100% rename from datafusion/src/logical_plan/dfschema.rs rename to datafusion/core/src/logical_plan/dfschema.rs diff --git a/datafusion/src/logical_plan/display.rs b/datafusion/core/src/logical_plan/display.rs similarity index 100% rename from datafusion/src/logical_plan/display.rs rename to datafusion/core/src/logical_plan/display.rs diff --git a/datafusion/src/logical_plan/expr.rs b/datafusion/core/src/logical_plan/expr.rs similarity index 100% rename from datafusion/src/logical_plan/expr.rs rename to datafusion/core/src/logical_plan/expr.rs diff --git a/datafusion/src/logical_plan/expr_rewriter.rs b/datafusion/core/src/logical_plan/expr_rewriter.rs similarity index 100% rename from datafusion/src/logical_plan/expr_rewriter.rs rename to datafusion/core/src/logical_plan/expr_rewriter.rs diff --git a/datafusion/src/logical_plan/expr_schema.rs b/datafusion/core/src/logical_plan/expr_schema.rs similarity index 100% rename from datafusion/src/logical_plan/expr_schema.rs rename to datafusion/core/src/logical_plan/expr_schema.rs diff --git a/datafusion/src/logical_plan/expr_simplier.rs b/datafusion/core/src/logical_plan/expr_simplier.rs similarity index 100% rename from datafusion/src/logical_plan/expr_simplier.rs rename to datafusion/core/src/logical_plan/expr_simplier.rs diff --git a/datafusion/src/logical_plan/expr_visitor.rs b/datafusion/core/src/logical_plan/expr_visitor.rs similarity index 100% rename from datafusion/src/logical_plan/expr_visitor.rs rename to datafusion/core/src/logical_plan/expr_visitor.rs diff --git a/datafusion/src/logical_plan/extension.rs b/datafusion/core/src/logical_plan/extension.rs similarity index 100% rename from datafusion/src/logical_plan/extension.rs rename to datafusion/core/src/logical_plan/extension.rs diff --git a/datafusion/src/logical_plan/mod.rs b/datafusion/core/src/logical_plan/mod.rs similarity index 100% rename from datafusion/src/logical_plan/mod.rs rename to datafusion/core/src/logical_plan/mod.rs diff --git a/datafusion/src/logical_plan/operators.rs b/datafusion/core/src/logical_plan/operators.rs similarity index 100% rename from datafusion/src/logical_plan/operators.rs rename to datafusion/core/src/logical_plan/operators.rs diff --git a/datafusion/src/logical_plan/plan.rs b/datafusion/core/src/logical_plan/plan.rs similarity index 100% rename from datafusion/src/logical_plan/plan.rs rename to datafusion/core/src/logical_plan/plan.rs diff --git a/datafusion/src/logical_plan/registry.rs b/datafusion/core/src/logical_plan/registry.rs similarity index 100% rename from datafusion/src/logical_plan/registry.rs rename to datafusion/core/src/logical_plan/registry.rs diff --git a/datafusion/src/logical_plan/window_frames.rs b/datafusion/core/src/logical_plan/window_frames.rs similarity index 100% rename from datafusion/src/logical_plan/window_frames.rs rename to datafusion/core/src/logical_plan/window_frames.rs diff --git a/datafusion/src/optimizer/common_subexpr_eliminate.rs b/datafusion/core/src/optimizer/common_subexpr_eliminate.rs similarity index 100% rename from datafusion/src/optimizer/common_subexpr_eliminate.rs rename to datafusion/core/src/optimizer/common_subexpr_eliminate.rs diff --git a/datafusion/src/optimizer/eliminate_filter.rs b/datafusion/core/src/optimizer/eliminate_filter.rs similarity index 100% rename from datafusion/src/optimizer/eliminate_filter.rs rename to datafusion/core/src/optimizer/eliminate_filter.rs diff --git a/datafusion/src/optimizer/eliminate_limit.rs b/datafusion/core/src/optimizer/eliminate_limit.rs similarity index 100% rename from datafusion/src/optimizer/eliminate_limit.rs rename to datafusion/core/src/optimizer/eliminate_limit.rs diff --git a/datafusion/src/optimizer/filter_push_down.rs b/datafusion/core/src/optimizer/filter_push_down.rs similarity index 100% rename from datafusion/src/optimizer/filter_push_down.rs rename to datafusion/core/src/optimizer/filter_push_down.rs diff --git a/datafusion/src/optimizer/limit_push_down.rs b/datafusion/core/src/optimizer/limit_push_down.rs similarity index 100% rename from datafusion/src/optimizer/limit_push_down.rs rename to datafusion/core/src/optimizer/limit_push_down.rs diff --git a/datafusion/src/optimizer/mod.rs b/datafusion/core/src/optimizer/mod.rs similarity index 100% rename from datafusion/src/optimizer/mod.rs rename to datafusion/core/src/optimizer/mod.rs diff --git a/datafusion/src/optimizer/optimizer.rs b/datafusion/core/src/optimizer/optimizer.rs similarity index 100% rename from datafusion/src/optimizer/optimizer.rs rename to datafusion/core/src/optimizer/optimizer.rs diff --git a/datafusion/src/optimizer/projection_push_down.rs b/datafusion/core/src/optimizer/projection_push_down.rs similarity index 100% rename from datafusion/src/optimizer/projection_push_down.rs rename to datafusion/core/src/optimizer/projection_push_down.rs diff --git a/datafusion/src/optimizer/simplify_expressions.rs b/datafusion/core/src/optimizer/simplify_expressions.rs similarity index 100% rename from datafusion/src/optimizer/simplify_expressions.rs rename to datafusion/core/src/optimizer/simplify_expressions.rs diff --git a/datafusion/src/optimizer/single_distinct_to_groupby.rs b/datafusion/core/src/optimizer/single_distinct_to_groupby.rs similarity index 100% rename from datafusion/src/optimizer/single_distinct_to_groupby.rs rename to datafusion/core/src/optimizer/single_distinct_to_groupby.rs diff --git a/datafusion/src/optimizer/to_approx_perc.rs b/datafusion/core/src/optimizer/to_approx_perc.rs similarity index 100% rename from datafusion/src/optimizer/to_approx_perc.rs rename to datafusion/core/src/optimizer/to_approx_perc.rs diff --git a/datafusion/src/optimizer/utils.rs b/datafusion/core/src/optimizer/utils.rs similarity index 100% rename from datafusion/src/optimizer/utils.rs rename to datafusion/core/src/optimizer/utils.rs diff --git a/datafusion/src/physical_optimizer/aggregate_statistics.rs b/datafusion/core/src/physical_optimizer/aggregate_statistics.rs similarity index 100% rename from datafusion/src/physical_optimizer/aggregate_statistics.rs rename to datafusion/core/src/physical_optimizer/aggregate_statistics.rs diff --git a/datafusion/src/physical_optimizer/coalesce_batches.rs b/datafusion/core/src/physical_optimizer/coalesce_batches.rs similarity index 100% rename from datafusion/src/physical_optimizer/coalesce_batches.rs rename to datafusion/core/src/physical_optimizer/coalesce_batches.rs diff --git a/datafusion/src/physical_optimizer/hash_build_probe_order.rs b/datafusion/core/src/physical_optimizer/hash_build_probe_order.rs similarity index 100% rename from datafusion/src/physical_optimizer/hash_build_probe_order.rs rename to datafusion/core/src/physical_optimizer/hash_build_probe_order.rs diff --git a/datafusion/src/physical_optimizer/merge_exec.rs b/datafusion/core/src/physical_optimizer/merge_exec.rs similarity index 100% rename from datafusion/src/physical_optimizer/merge_exec.rs rename to datafusion/core/src/physical_optimizer/merge_exec.rs diff --git a/datafusion/src/physical_optimizer/mod.rs b/datafusion/core/src/physical_optimizer/mod.rs similarity index 100% rename from datafusion/src/physical_optimizer/mod.rs rename to datafusion/core/src/physical_optimizer/mod.rs diff --git a/datafusion/src/physical_optimizer/optimizer.rs b/datafusion/core/src/physical_optimizer/optimizer.rs similarity index 100% rename from datafusion/src/physical_optimizer/optimizer.rs rename to datafusion/core/src/physical_optimizer/optimizer.rs diff --git a/datafusion/src/physical_optimizer/pruning.rs b/datafusion/core/src/physical_optimizer/pruning.rs similarity index 100% rename from datafusion/src/physical_optimizer/pruning.rs rename to datafusion/core/src/physical_optimizer/pruning.rs diff --git a/datafusion/src/physical_optimizer/repartition.rs b/datafusion/core/src/physical_optimizer/repartition.rs similarity index 100% rename from datafusion/src/physical_optimizer/repartition.rs rename to datafusion/core/src/physical_optimizer/repartition.rs diff --git a/datafusion/src/physical_optimizer/utils.rs b/datafusion/core/src/physical_optimizer/utils.rs similarity index 100% rename from datafusion/src/physical_optimizer/utils.rs rename to datafusion/core/src/physical_optimizer/utils.rs diff --git a/datafusion/src/physical_plan/aggregate_rule.rs b/datafusion/core/src/physical_plan/aggregate_rule.rs similarity index 100% rename from datafusion/src/physical_plan/aggregate_rule.rs rename to datafusion/core/src/physical_plan/aggregate_rule.rs diff --git a/datafusion/src/physical_plan/aggregates.rs b/datafusion/core/src/physical_plan/aggregates.rs similarity index 100% rename from datafusion/src/physical_plan/aggregates.rs rename to datafusion/core/src/physical_plan/aggregates.rs diff --git a/datafusion/src/physical_plan/analyze.rs b/datafusion/core/src/physical_plan/analyze.rs similarity index 100% rename from datafusion/src/physical_plan/analyze.rs rename to datafusion/core/src/physical_plan/analyze.rs diff --git a/datafusion/src/physical_plan/coalesce_batches.rs b/datafusion/core/src/physical_plan/coalesce_batches.rs similarity index 100% rename from datafusion/src/physical_plan/coalesce_batches.rs rename to datafusion/core/src/physical_plan/coalesce_batches.rs diff --git a/datafusion/src/physical_plan/coalesce_partitions.rs b/datafusion/core/src/physical_plan/coalesce_partitions.rs similarity index 99% rename from datafusion/src/physical_plan/coalesce_partitions.rs rename to datafusion/core/src/physical_plan/coalesce_partitions.rs index d0a776a57d9d..b2832f73c9e8 100644 --- a/datafusion/src/physical_plan/coalesce_partitions.rs +++ b/datafusion/core/src/physical_plan/coalesce_partitions.rs @@ -221,7 +221,7 @@ mod tests { use futures::FutureExt; use super::*; - use crate::datafusion_storage::object_store::local::LocalFileSystem; + use crate::datafusion_data_access::object_store::local::LocalFileSystem; use crate::physical_plan::file_format::{CsvExec, FileScanConfig}; use crate::physical_plan::{collect, common}; use crate::prelude::SessionContext; diff --git a/datafusion/src/physical_plan/common.rs b/datafusion/core/src/physical_plan/common.rs similarity index 100% rename from datafusion/src/physical_plan/common.rs rename to datafusion/core/src/physical_plan/common.rs diff --git a/datafusion/src/physical_plan/cross_join.rs b/datafusion/core/src/physical_plan/cross_join.rs similarity index 100% rename from datafusion/src/physical_plan/cross_join.rs rename to datafusion/core/src/physical_plan/cross_join.rs diff --git a/datafusion/src/physical_plan/display.rs b/datafusion/core/src/physical_plan/display.rs similarity index 100% rename from datafusion/src/physical_plan/display.rs rename to datafusion/core/src/physical_plan/display.rs diff --git a/datafusion/src/physical_plan/empty.rs b/datafusion/core/src/physical_plan/empty.rs similarity index 100% rename from datafusion/src/physical_plan/empty.rs rename to datafusion/core/src/physical_plan/empty.rs diff --git a/datafusion/src/physical_plan/explain.rs b/datafusion/core/src/physical_plan/explain.rs similarity index 100% rename from datafusion/src/physical_plan/explain.rs rename to datafusion/core/src/physical_plan/explain.rs diff --git a/datafusion/src/physical_plan/file_format/avro.rs b/datafusion/core/src/physical_plan/file_format/avro.rs similarity index 100% rename from datafusion/src/physical_plan/file_format/avro.rs rename to datafusion/core/src/physical_plan/file_format/avro.rs diff --git a/datafusion/src/physical_plan/file_format/csv.rs b/datafusion/core/src/physical_plan/file_format/csv.rs similarity index 99% rename from datafusion/src/physical_plan/file_format/csv.rs rename to datafusion/core/src/physical_plan/file_format/csv.rs index 8a5356b80b67..c3bf7f9b201d 100644 --- a/datafusion/src/physical_plan/file_format/csv.rs +++ b/datafusion/core/src/physical_plan/file_format/csv.rs @@ -221,7 +221,7 @@ pub async fn plan_to_csv( #[cfg(test)] mod tests { use super::*; - use crate::datafusion_storage::object_store::local::LocalFileSystem; + use crate::datafusion_data_access::object_store::local::LocalFileSystem; use crate::datasource::listing::local_unpartitioned_file; use crate::prelude::*; use crate::test_util::aggr_test_schema_with_missing_col; diff --git a/datafusion/src/physical_plan/file_format/file_stream.rs b/datafusion/core/src/physical_plan/file_format/file_stream.rs similarity index 99% rename from datafusion/src/physical_plan/file_format/file_stream.rs rename to datafusion/core/src/physical_plan/file_format/file_stream.rs index 78222ee42045..97a95b568a61 100644 --- a/datafusion/src/physical_plan/file_format/file_stream.rs +++ b/datafusion/core/src/physical_plan/file_format/file_stream.rs @@ -28,7 +28,7 @@ use arrow::{ error::{ArrowError, Result as ArrowResult}, record_batch::RecordBatch, }; -use datafusion_storage::object_store::ObjectStore; +use datafusion_data_access::object_store::ObjectStore; use futures::Stream; use std::{ io::Read, diff --git a/datafusion/src/physical_plan/file_format/json.rs b/datafusion/core/src/physical_plan/file_format/json.rs similarity index 99% rename from datafusion/src/physical_plan/file_format/json.rs rename to datafusion/core/src/physical_plan/file_format/json.rs index afcdde4cfb84..a6e7840e7d33 100644 --- a/datafusion/src/physical_plan/file_format/json.rs +++ b/datafusion/core/src/physical_plan/file_format/json.rs @@ -193,7 +193,7 @@ mod tests { use arrow::datatypes::{Field, Schema}; use futures::StreamExt; - use crate::datafusion_storage::object_store::local::{ + use crate::datafusion_data_access::object_store::local::{ local_object_reader_stream, LocalFileSystem, }; use crate::datasource::{ diff --git a/datafusion/src/physical_plan/file_format/mod.rs b/datafusion/core/src/physical_plan/file_format/mod.rs similarity index 99% rename from datafusion/src/physical_plan/file_format/mod.rs rename to datafusion/core/src/physical_plan/file_format/mod.rs index 78d1dc250754..1e8f9e2d8d2c 100644 --- a/datafusion/src/physical_plan/file_format/mod.rs +++ b/datafusion/core/src/physical_plan/file_format/mod.rs @@ -44,7 +44,7 @@ use crate::{ scalar::ScalarValue, }; use arrow::array::{new_null_array, UInt16BufferBuilder}; -use datafusion_storage::object_store::ObjectStore; +use datafusion_data_access::object_store::ObjectStore; use lazy_static::lazy_static; use log::info; use std::{ diff --git a/datafusion/src/physical_plan/file_format/parquet.rs b/datafusion/core/src/physical_plan/file_format/parquet.rs similarity index 99% rename from datafusion/src/physical_plan/file_format/parquet.rs rename to datafusion/core/src/physical_plan/file_format/parquet.rs index 7d797477df08..6b784cd6aa92 100644 --- a/datafusion/src/physical_plan/file_format/parquet.rs +++ b/datafusion/core/src/physical_plan/file_format/parquet.rs @@ -40,8 +40,8 @@ use crate::{ scalar::ScalarValue, }; use datafusion_common::Column; +use datafusion_data_access::object_store::ObjectStore; use datafusion_expr::Expr; -use datafusion_storage::object_store::ObjectStore; use arrow::{ array::ArrayRef, @@ -580,7 +580,7 @@ pub async fn plan_to_parquet( mod tests { use crate::{ assert_batches_sorted_eq, assert_contains, - datafusion_storage::{ + datafusion_data_access::{ object_store::local::{local_object_reader_stream, LocalFileSystem}, FileMeta, SizedFile, }, diff --git a/datafusion/src/physical_plan/filter.rs b/datafusion/core/src/physical_plan/filter.rs similarity index 99% rename from datafusion/src/physical_plan/filter.rs rename to datafusion/core/src/physical_plan/filter.rs index 218fea5c5b0b..0822aa4ff5d9 100644 --- a/datafusion/src/physical_plan/filter.rs +++ b/datafusion/core/src/physical_plan/filter.rs @@ -241,7 +241,7 @@ impl RecordBatchStream for FilterExecStream { mod tests { use super::*; - use crate::datafusion_storage::object_store::local::LocalFileSystem; + use crate::datafusion_data_access::object_store::local::LocalFileSystem; use crate::physical_plan::collect; use crate::physical_plan::expressions::*; use crate::physical_plan::file_format::{CsvExec, FileScanConfig}; diff --git a/datafusion/src/physical_plan/functions.rs b/datafusion/core/src/physical_plan/functions.rs similarity index 100% rename from datafusion/src/physical_plan/functions.rs rename to datafusion/core/src/physical_plan/functions.rs diff --git a/datafusion/src/physical_plan/hash_aggregate.rs b/datafusion/core/src/physical_plan/hash_aggregate.rs similarity index 100% rename from datafusion/src/physical_plan/hash_aggregate.rs rename to datafusion/core/src/physical_plan/hash_aggregate.rs diff --git a/datafusion/src/physical_plan/hash_join.rs b/datafusion/core/src/physical_plan/hash_join.rs similarity index 100% rename from datafusion/src/physical_plan/hash_join.rs rename to datafusion/core/src/physical_plan/hash_join.rs diff --git a/datafusion/src/physical_plan/hash_utils.rs b/datafusion/core/src/physical_plan/hash_utils.rs similarity index 100% rename from datafusion/src/physical_plan/hash_utils.rs rename to datafusion/core/src/physical_plan/hash_utils.rs diff --git a/datafusion/src/physical_plan/join_utils.rs b/datafusion/core/src/physical_plan/join_utils.rs similarity index 100% rename from datafusion/src/physical_plan/join_utils.rs rename to datafusion/core/src/physical_plan/join_utils.rs diff --git a/datafusion/src/physical_plan/limit.rs b/datafusion/core/src/physical_plan/limit.rs similarity index 99% rename from datafusion/src/physical_plan/limit.rs rename to datafusion/core/src/physical_plan/limit.rs index 6b37a67c7806..5aa2ab6ff3b3 100644 --- a/datafusion/src/physical_plan/limit.rs +++ b/datafusion/core/src/physical_plan/limit.rs @@ -428,7 +428,7 @@ mod tests { use common::collect; use super::*; - use crate::datafusion_storage::object_store::local::LocalFileSystem; + use crate::datafusion_data_access::object_store::local::LocalFileSystem; use crate::physical_plan::coalesce_partitions::CoalescePartitionsExec; use crate::physical_plan::common; use crate::physical_plan::file_format::{CsvExec, FileScanConfig}; diff --git a/datafusion/src/physical_plan/memory.rs b/datafusion/core/src/physical_plan/memory.rs similarity index 100% rename from datafusion/src/physical_plan/memory.rs rename to datafusion/core/src/physical_plan/memory.rs diff --git a/datafusion/src/physical_plan/metrics/baseline.rs b/datafusion/core/src/physical_plan/metrics/baseline.rs similarity index 100% rename from datafusion/src/physical_plan/metrics/baseline.rs rename to datafusion/core/src/physical_plan/metrics/baseline.rs diff --git a/datafusion/src/physical_plan/metrics/builder.rs b/datafusion/core/src/physical_plan/metrics/builder.rs similarity index 100% rename from datafusion/src/physical_plan/metrics/builder.rs rename to datafusion/core/src/physical_plan/metrics/builder.rs diff --git a/datafusion/src/physical_plan/metrics/composite.rs b/datafusion/core/src/physical_plan/metrics/composite.rs similarity index 100% rename from datafusion/src/physical_plan/metrics/composite.rs rename to datafusion/core/src/physical_plan/metrics/composite.rs diff --git a/datafusion/src/physical_plan/metrics/mod.rs b/datafusion/core/src/physical_plan/metrics/mod.rs similarity index 100% rename from datafusion/src/physical_plan/metrics/mod.rs rename to datafusion/core/src/physical_plan/metrics/mod.rs diff --git a/datafusion/src/physical_plan/metrics/tracker.rs b/datafusion/core/src/physical_plan/metrics/tracker.rs similarity index 100% rename from datafusion/src/physical_plan/metrics/tracker.rs rename to datafusion/core/src/physical_plan/metrics/tracker.rs diff --git a/datafusion/src/physical_plan/metrics/value.rs b/datafusion/core/src/physical_plan/metrics/value.rs similarity index 100% rename from datafusion/src/physical_plan/metrics/value.rs rename to datafusion/core/src/physical_plan/metrics/value.rs diff --git a/datafusion/src/physical_plan/mod.rs b/datafusion/core/src/physical_plan/mod.rs similarity index 100% rename from datafusion/src/physical_plan/mod.rs rename to datafusion/core/src/physical_plan/mod.rs diff --git a/datafusion/src/physical_plan/planner.rs b/datafusion/core/src/physical_plan/planner.rs similarity index 99% rename from datafusion/src/physical_plan/planner.rs rename to datafusion/core/src/physical_plan/planner.rs index 36b984043b7e..7504dd4385e4 100644 --- a/datafusion/src/physical_plan/planner.rs +++ b/datafusion/core/src/physical_plan/planner.rs @@ -1444,7 +1444,7 @@ fn tuple_err(value: (Result, Result)) -> Result<(T, R)> { #[cfg(test)] mod tests { use super::*; - use crate::datafusion_storage::object_store::local::LocalFileSystem; + use crate::datafusion_data_access::object_store::local::LocalFileSystem; use crate::execution::context::TaskContext; use crate::execution::options::CsvReadOptions; use crate::execution::runtime_env::{RuntimeConfig, RuntimeEnv}; diff --git a/datafusion/src/physical_plan/projection.rs b/datafusion/core/src/physical_plan/projection.rs similarity index 99% rename from datafusion/src/physical_plan/projection.rs rename to datafusion/core/src/physical_plan/projection.rs index 425b73027761..4419b6f91276 100644 --- a/datafusion/src/physical_plan/projection.rs +++ b/datafusion/core/src/physical_plan/projection.rs @@ -300,7 +300,7 @@ impl RecordBatchStream for ProjectionStream { mod tests { use super::*; - use crate::datafusion_storage::object_store::local::LocalFileSystem; + use crate::datafusion_data_access::object_store::local::LocalFileSystem; use crate::physical_plan::expressions::{self, col}; use crate::physical_plan::file_format::{CsvExec, FileScanConfig}; use crate::prelude::SessionContext; diff --git a/datafusion/src/physical_plan/repartition.rs b/datafusion/core/src/physical_plan/repartition.rs similarity index 100% rename from datafusion/src/physical_plan/repartition.rs rename to datafusion/core/src/physical_plan/repartition.rs diff --git a/datafusion/src/physical_plan/sorts/mod.rs b/datafusion/core/src/physical_plan/sorts/mod.rs similarity index 100% rename from datafusion/src/physical_plan/sorts/mod.rs rename to datafusion/core/src/physical_plan/sorts/mod.rs diff --git a/datafusion/src/physical_plan/sorts/sort.rs b/datafusion/core/src/physical_plan/sorts/sort.rs similarity index 99% rename from datafusion/src/physical_plan/sorts/sort.rs rename to datafusion/core/src/physical_plan/sorts/sort.rs index 67b9085bb4a7..67a6e5fec244 100644 --- a/datafusion/src/physical_plan/sorts/sort.rs +++ b/datafusion/core/src/physical_plan/sorts/sort.rs @@ -595,7 +595,7 @@ async fn do_sort( #[cfg(test)] mod tests { use super::*; - use crate::datafusion_storage::object_store::local::LocalFileSystem; + use crate::datafusion_data_access::object_store::local::LocalFileSystem; use crate::execution::context::SessionConfig; use crate::execution::runtime_env::RuntimeConfig; use crate::physical_plan::coalesce_partitions::CoalescePartitionsExec; diff --git a/datafusion/src/physical_plan/sorts/sort_preserving_merge.rs b/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs similarity index 99% rename from datafusion/src/physical_plan/sorts/sort_preserving_merge.rs rename to datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs index 71e51571161d..3fb480de21bd 100644 --- a/datafusion/src/physical_plan/sorts/sort_preserving_merge.rs +++ b/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs @@ -605,7 +605,7 @@ impl RecordBatchStream for SortPreservingMergeStream { #[cfg(test)] mod tests { - use crate::datafusion_storage::object_store::local::LocalFileSystem; + use crate::datafusion_data_access::object_store::local::LocalFileSystem; use crate::from_slice::FromSlice; use crate::physical_plan::metrics::MetricValue; use crate::test::exec::{assert_strong_count_converges_to_zero, BlockingExec}; diff --git a/datafusion/src/physical_plan/stream.rs b/datafusion/core/src/physical_plan/stream.rs similarity index 100% rename from datafusion/src/physical_plan/stream.rs rename to datafusion/core/src/physical_plan/stream.rs diff --git a/datafusion/src/physical_plan/type_coercion.rs b/datafusion/core/src/physical_plan/type_coercion.rs similarity index 100% rename from datafusion/src/physical_plan/type_coercion.rs rename to datafusion/core/src/physical_plan/type_coercion.rs diff --git a/datafusion/src/physical_plan/udaf.rs b/datafusion/core/src/physical_plan/udaf.rs similarity index 100% rename from datafusion/src/physical_plan/udaf.rs rename to datafusion/core/src/physical_plan/udaf.rs diff --git a/datafusion/src/physical_plan/udf.rs b/datafusion/core/src/physical_plan/udf.rs similarity index 100% rename from datafusion/src/physical_plan/udf.rs rename to datafusion/core/src/physical_plan/udf.rs diff --git a/datafusion/src/physical_plan/union.rs b/datafusion/core/src/physical_plan/union.rs similarity index 99% rename from datafusion/src/physical_plan/union.rs rename to datafusion/core/src/physical_plan/union.rs index bf6fb7ce6623..f156a60fe346 100644 --- a/datafusion/src/physical_plan/union.rs +++ b/datafusion/core/src/physical_plan/union.rs @@ -239,7 +239,9 @@ fn stats_union(mut left: Statistics, right: Statistics) -> Statistics { #[cfg(test)] mod tests { use super::*; - use crate::datafusion_storage::object_store::{local::LocalFileSystem, ObjectStore}; + use crate::datafusion_data_access::object_store::{ + local::LocalFileSystem, ObjectStore, + }; use crate::{test, test_util}; use crate::prelude::SessionContext; diff --git a/datafusion/src/physical_plan/values.rs b/datafusion/core/src/physical_plan/values.rs similarity index 100% rename from datafusion/src/physical_plan/values.rs rename to datafusion/core/src/physical_plan/values.rs diff --git a/datafusion/src/physical_plan/window_functions.rs b/datafusion/core/src/physical_plan/window_functions.rs similarity index 100% rename from datafusion/src/physical_plan/window_functions.rs rename to datafusion/core/src/physical_plan/window_functions.rs diff --git a/datafusion/src/physical_plan/windows/mod.rs b/datafusion/core/src/physical_plan/windows/mod.rs similarity index 99% rename from datafusion/src/physical_plan/windows/mod.rs rename to datafusion/core/src/physical_plan/windows/mod.rs index f45b0560a6e6..34f7730f9de5 100644 --- a/datafusion/src/physical_plan/windows/mod.rs +++ b/datafusion/core/src/physical_plan/windows/mod.rs @@ -152,7 +152,7 @@ fn create_built_in_window_expr( #[cfg(test)] mod tests { use super::*; - use crate::datafusion_storage::object_store::local::LocalFileSystem; + use crate::datafusion_data_access::object_store::local::LocalFileSystem; use crate::physical_plan::aggregates::AggregateFunction; use crate::physical_plan::expressions::col; use crate::physical_plan::file_format::{CsvExec, FileScanConfig}; diff --git a/datafusion/src/physical_plan/windows/window_agg_exec.rs b/datafusion/core/src/physical_plan/windows/window_agg_exec.rs similarity index 100% rename from datafusion/src/physical_plan/windows/window_agg_exec.rs rename to datafusion/core/src/physical_plan/windows/window_agg_exec.rs diff --git a/datafusion/src/prelude.rs b/datafusion/core/src/prelude.rs similarity index 100% rename from datafusion/src/prelude.rs rename to datafusion/core/src/prelude.rs diff --git a/datafusion/src/row/mod.rs b/datafusion/core/src/row/mod.rs similarity index 100% rename from datafusion/src/row/mod.rs rename to datafusion/core/src/row/mod.rs diff --git a/datafusion/src/row/reader.rs b/datafusion/core/src/row/reader.rs similarity index 100% rename from datafusion/src/row/reader.rs rename to datafusion/core/src/row/reader.rs diff --git a/datafusion/src/row/writer.rs b/datafusion/core/src/row/writer.rs similarity index 100% rename from datafusion/src/row/writer.rs rename to datafusion/core/src/row/writer.rs diff --git a/datafusion/src/scalar.rs b/datafusion/core/src/scalar.rs similarity index 100% rename from datafusion/src/scalar.rs rename to datafusion/core/src/scalar.rs diff --git a/datafusion/src/sql/mod.rs b/datafusion/core/src/sql/mod.rs similarity index 100% rename from datafusion/src/sql/mod.rs rename to datafusion/core/src/sql/mod.rs diff --git a/datafusion/src/sql/parser.rs b/datafusion/core/src/sql/parser.rs similarity index 100% rename from datafusion/src/sql/parser.rs rename to datafusion/core/src/sql/parser.rs diff --git a/datafusion/src/sql/planner.rs b/datafusion/core/src/sql/planner.rs similarity index 100% rename from datafusion/src/sql/planner.rs rename to datafusion/core/src/sql/planner.rs diff --git a/datafusion/src/sql/utils.rs b/datafusion/core/src/sql/utils.rs similarity index 100% rename from datafusion/src/sql/utils.rs rename to datafusion/core/src/sql/utils.rs diff --git a/datafusion/src/test/exec.rs b/datafusion/core/src/test/exec.rs similarity index 100% rename from datafusion/src/test/exec.rs rename to datafusion/core/src/test/exec.rs diff --git a/datafusion/src/test/mod.rs b/datafusion/core/src/test/mod.rs similarity index 100% rename from datafusion/src/test/mod.rs rename to datafusion/core/src/test/mod.rs diff --git a/datafusion/src/test/object_store.rs b/datafusion/core/src/test/object_store.rs similarity index 98% rename from datafusion/src/test/object_store.rs rename to datafusion/core/src/test/object_store.rs index 61b4265c0dc1..cc9c02305a76 100644 --- a/datafusion/src/test/object_store.rs +++ b/datafusion/core/src/test/object_store.rs @@ -22,7 +22,7 @@ use std::{ sync::Arc, }; -use crate::datafusion_storage::{ +use crate::datafusion_data_access::{ object_store::{FileMetaStream, ListEntryStream, ObjectReader, ObjectStore}, FileMeta, Result, SizedFile, }; diff --git a/datafusion/src/test/user_defined.rs b/datafusion/core/src/test/user_defined.rs similarity index 100% rename from datafusion/src/test/user_defined.rs rename to datafusion/core/src/test/user_defined.rs diff --git a/datafusion/src/test/variable.rs b/datafusion/core/src/test/variable.rs similarity index 100% rename from datafusion/src/test/variable.rs rename to datafusion/core/src/test/variable.rs diff --git a/datafusion/src/test_util.rs b/datafusion/core/src/test_util.rs similarity index 98% rename from datafusion/src/test_util.rs rename to datafusion/core/src/test_util.rs index 8ee0298f72ce..1dd36316b797 100644 --- a/datafusion/src/test_util.rs +++ b/datafusion/core/src/test_util.rs @@ -156,7 +156,7 @@ macro_rules! assert_not_contains { /// assert!(std::path::PathBuf::from(csvdata).exists()); /// ``` pub fn arrow_test_data() -> String { - match get_data_dir("ARROW_TEST_DATA", "../testing/data") { + match get_data_dir("ARROW_TEST_DATA", "../../testing/data") { Ok(pb) => pb.display().to_string(), Err(err) => panic!("failed to get arrow data dir: {}", err), } @@ -178,7 +178,7 @@ pub fn arrow_test_data() -> String { /// assert!(std::path::PathBuf::from(filename).exists()); /// ``` pub fn parquet_test_data() -> String { - match get_data_dir("PARQUET_TEST_DATA", "../parquet-testing/data") { + match get_data_dir("PARQUET_TEST_DATA", "../../parquet-testing/data") { Ok(pb) => pb.display().to_string(), Err(err) => panic!("failed to get parquet data dir: {}", err), } diff --git a/datafusion/src/variable/mod.rs b/datafusion/core/src/variable/mod.rs similarity index 100% rename from datafusion/src/variable/mod.rs rename to datafusion/core/src/variable/mod.rs diff --git a/datafusion/tests/aggregate_simple.csv b/datafusion/core/tests/aggregate_simple.csv similarity index 100% rename from datafusion/tests/aggregate_simple.csv rename to datafusion/core/tests/aggregate_simple.csv diff --git a/datafusion/tests/custom_sources.rs b/datafusion/core/tests/custom_sources.rs similarity index 100% rename from datafusion/tests/custom_sources.rs rename to datafusion/core/tests/custom_sources.rs diff --git a/datafusion/tests/customer.csv b/datafusion/core/tests/customer.csv similarity index 100% rename from datafusion/tests/customer.csv rename to datafusion/core/tests/customer.csv diff --git a/datafusion/tests/dataframe.rs b/datafusion/core/tests/dataframe.rs similarity index 100% rename from datafusion/tests/dataframe.rs rename to datafusion/core/tests/dataframe.rs diff --git a/datafusion/tests/dataframe_functions.rs b/datafusion/core/tests/dataframe_functions.rs similarity index 100% rename from datafusion/tests/dataframe_functions.rs rename to datafusion/core/tests/dataframe_functions.rs diff --git a/datafusion/tests/example.csv b/datafusion/core/tests/example.csv similarity index 100% rename from datafusion/tests/example.csv rename to datafusion/core/tests/example.csv diff --git a/datafusion/tests/jsons/1.json b/datafusion/core/tests/jsons/1.json similarity index 100% rename from datafusion/tests/jsons/1.json rename to datafusion/core/tests/jsons/1.json diff --git a/datafusion/tests/jsons/2.json b/datafusion/core/tests/jsons/2.json similarity index 100% rename from datafusion/tests/jsons/2.json rename to datafusion/core/tests/jsons/2.json diff --git a/datafusion/tests/jsons/3.json b/datafusion/core/tests/jsons/3.json similarity index 100% rename from datafusion/tests/jsons/3.json rename to datafusion/core/tests/jsons/3.json diff --git a/datafusion/tests/jsons/schema_infer_limit.json b/datafusion/core/tests/jsons/schema_infer_limit.json similarity index 100% rename from datafusion/tests/jsons/schema_infer_limit.json rename to datafusion/core/tests/jsons/schema_infer_limit.json diff --git a/datafusion/tests/merge_fuzz.rs b/datafusion/core/tests/merge_fuzz.rs similarity index 100% rename from datafusion/tests/merge_fuzz.rs rename to datafusion/core/tests/merge_fuzz.rs diff --git a/datafusion/tests/order_spill_fuzz.rs b/datafusion/core/tests/order_spill_fuzz.rs similarity index 100% rename from datafusion/tests/order_spill_fuzz.rs rename to datafusion/core/tests/order_spill_fuzz.rs diff --git a/datafusion/tests/parquet_pruning.rs b/datafusion/core/tests/parquet_pruning.rs similarity index 100% rename from datafusion/tests/parquet_pruning.rs rename to datafusion/core/tests/parquet_pruning.rs diff --git a/datafusion/tests/path_partition.rs b/datafusion/core/tests/path_partition.rs similarity index 98% rename from datafusion/tests/path_partition.rs rename to datafusion/core/tests/path_partition.rs index 351b85d3d50e..80610baf892b 100644 --- a/datafusion/tests/path_partition.rs +++ b/datafusion/core/tests/path_partition.rs @@ -22,7 +22,7 @@ use std::{fs, io, sync::Arc}; use async_trait::async_trait; use datafusion::{ assert_batches_sorted_eq, - datafusion_storage::{ + datafusion_data_access::{ object_store::{ local::LocalFileSystem, FileMetaStream, ListEntryStream, ObjectReader, ObjectStore, @@ -355,7 +355,7 @@ impl ObjectStore for MirroringObjectStore { async fn list_file( &self, prefix: &str, - ) -> datafusion_storage::Result { + ) -> datafusion_data_access::Result { let prefix = prefix.to_owned(); let size = self.file_size; Ok(Box::pin( @@ -378,14 +378,14 @@ impl ObjectStore for MirroringObjectStore { &self, _prefix: &str, _delimiter: Option, - ) -> datafusion_storage::Result { + ) -> datafusion_data_access::Result { unimplemented!() } fn file_reader( &self, file: SizedFile, - ) -> datafusion_storage::Result> { + ) -> datafusion_data_access::Result> { assert_eq!( self.file_size, file.size, "Requested files should have the same size as the mirrored file" diff --git a/datafusion/tests/provider_filter_pushdown.rs b/datafusion/core/tests/provider_filter_pushdown.rs similarity index 100% rename from datafusion/tests/provider_filter_pushdown.rs rename to datafusion/core/tests/provider_filter_pushdown.rs diff --git a/datafusion/tests/simplification.rs b/datafusion/core/tests/simplification.rs similarity index 100% rename from datafusion/tests/simplification.rs rename to datafusion/core/tests/simplification.rs diff --git a/datafusion/tests/sql/aggregates.rs b/datafusion/core/tests/sql/aggregates.rs similarity index 100% rename from datafusion/tests/sql/aggregates.rs rename to datafusion/core/tests/sql/aggregates.rs diff --git a/datafusion/tests/sql/avro.rs b/datafusion/core/tests/sql/avro.rs similarity index 100% rename from datafusion/tests/sql/avro.rs rename to datafusion/core/tests/sql/avro.rs diff --git a/datafusion/tests/sql/create_drop.rs b/datafusion/core/tests/sql/create_drop.rs similarity index 100% rename from datafusion/tests/sql/create_drop.rs rename to datafusion/core/tests/sql/create_drop.rs diff --git a/datafusion/tests/sql/errors.rs b/datafusion/core/tests/sql/errors.rs similarity index 100% rename from datafusion/tests/sql/errors.rs rename to datafusion/core/tests/sql/errors.rs diff --git a/datafusion/tests/sql/explain.rs b/datafusion/core/tests/sql/explain.rs similarity index 100% rename from datafusion/tests/sql/explain.rs rename to datafusion/core/tests/sql/explain.rs diff --git a/datafusion/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rs similarity index 100% rename from datafusion/tests/sql/explain_analyze.rs rename to datafusion/core/tests/sql/explain_analyze.rs diff --git a/datafusion/tests/sql/expr.rs b/datafusion/core/tests/sql/expr.rs similarity index 100% rename from datafusion/tests/sql/expr.rs rename to datafusion/core/tests/sql/expr.rs diff --git a/datafusion/tests/sql/functions.rs b/datafusion/core/tests/sql/functions.rs similarity index 100% rename from datafusion/tests/sql/functions.rs rename to datafusion/core/tests/sql/functions.rs diff --git a/datafusion/tests/sql/group_by.rs b/datafusion/core/tests/sql/group_by.rs similarity index 100% rename from datafusion/tests/sql/group_by.rs rename to datafusion/core/tests/sql/group_by.rs diff --git a/datafusion/tests/sql/information_schema.rs b/datafusion/core/tests/sql/information_schema.rs similarity index 100% rename from datafusion/tests/sql/information_schema.rs rename to datafusion/core/tests/sql/information_schema.rs diff --git a/datafusion/tests/sql/intersection.rs b/datafusion/core/tests/sql/intersection.rs similarity index 100% rename from datafusion/tests/sql/intersection.rs rename to datafusion/core/tests/sql/intersection.rs diff --git a/datafusion/tests/sql/joins.rs b/datafusion/core/tests/sql/joins.rs similarity index 100% rename from datafusion/tests/sql/joins.rs rename to datafusion/core/tests/sql/joins.rs diff --git a/datafusion/tests/sql/json.rs b/datafusion/core/tests/sql/json.rs similarity index 100% rename from datafusion/tests/sql/json.rs rename to datafusion/core/tests/sql/json.rs diff --git a/datafusion/tests/sql/limit.rs b/datafusion/core/tests/sql/limit.rs similarity index 100% rename from datafusion/tests/sql/limit.rs rename to datafusion/core/tests/sql/limit.rs diff --git a/datafusion/tests/sql/mod.rs b/datafusion/core/tests/sql/mod.rs similarity index 100% rename from datafusion/tests/sql/mod.rs rename to datafusion/core/tests/sql/mod.rs diff --git a/datafusion/tests/sql/order.rs b/datafusion/core/tests/sql/order.rs similarity index 100% rename from datafusion/tests/sql/order.rs rename to datafusion/core/tests/sql/order.rs diff --git a/datafusion/tests/sql/parquet.rs b/datafusion/core/tests/sql/parquet.rs similarity index 100% rename from datafusion/tests/sql/parquet.rs rename to datafusion/core/tests/sql/parquet.rs diff --git a/datafusion/tests/sql/partitioned_csv.rs b/datafusion/core/tests/sql/partitioned_csv.rs similarity index 100% rename from datafusion/tests/sql/partitioned_csv.rs rename to datafusion/core/tests/sql/partitioned_csv.rs diff --git a/datafusion/tests/sql/predicates.rs b/datafusion/core/tests/sql/predicates.rs similarity index 100% rename from datafusion/tests/sql/predicates.rs rename to datafusion/core/tests/sql/predicates.rs diff --git a/datafusion/tests/sql/projection.rs b/datafusion/core/tests/sql/projection.rs similarity index 100% rename from datafusion/tests/sql/projection.rs rename to datafusion/core/tests/sql/projection.rs diff --git a/datafusion/tests/sql/references.rs b/datafusion/core/tests/sql/references.rs similarity index 100% rename from datafusion/tests/sql/references.rs rename to datafusion/core/tests/sql/references.rs diff --git a/datafusion/tests/sql/select.rs b/datafusion/core/tests/sql/select.rs similarity index 100% rename from datafusion/tests/sql/select.rs rename to datafusion/core/tests/sql/select.rs diff --git a/datafusion/tests/sql/timestamp.rs b/datafusion/core/tests/sql/timestamp.rs similarity index 100% rename from datafusion/tests/sql/timestamp.rs rename to datafusion/core/tests/sql/timestamp.rs diff --git a/datafusion/tests/sql/udf.rs b/datafusion/core/tests/sql/udf.rs similarity index 100% rename from datafusion/tests/sql/udf.rs rename to datafusion/core/tests/sql/udf.rs diff --git a/datafusion/tests/sql/unicode.rs b/datafusion/core/tests/sql/unicode.rs similarity index 100% rename from datafusion/tests/sql/unicode.rs rename to datafusion/core/tests/sql/unicode.rs diff --git a/datafusion/tests/sql/union.rs b/datafusion/core/tests/sql/union.rs similarity index 100% rename from datafusion/tests/sql/union.rs rename to datafusion/core/tests/sql/union.rs diff --git a/datafusion/tests/sql/wildcard.rs b/datafusion/core/tests/sql/wildcard.rs similarity index 100% rename from datafusion/tests/sql/wildcard.rs rename to datafusion/core/tests/sql/wildcard.rs diff --git a/datafusion/tests/sql/window.rs b/datafusion/core/tests/sql/window.rs similarity index 100% rename from datafusion/tests/sql/window.rs rename to datafusion/core/tests/sql/window.rs diff --git a/datafusion/tests/sql_integration.rs b/datafusion/core/tests/sql_integration.rs similarity index 100% rename from datafusion/tests/sql_integration.rs rename to datafusion/core/tests/sql_integration.rs diff --git a/datafusion/tests/statistics.rs b/datafusion/core/tests/statistics.rs similarity index 100% rename from datafusion/tests/statistics.rs rename to datafusion/core/tests/statistics.rs diff --git a/datafusion/tests/tpch-csv/customer.csv b/datafusion/core/tests/tpch-csv/customer.csv similarity index 100% rename from datafusion/tests/tpch-csv/customer.csv rename to datafusion/core/tests/tpch-csv/customer.csv diff --git a/datafusion/tests/tpch-csv/lineitem.csv b/datafusion/core/tests/tpch-csv/lineitem.csv similarity index 100% rename from datafusion/tests/tpch-csv/lineitem.csv rename to datafusion/core/tests/tpch-csv/lineitem.csv diff --git a/datafusion/tests/tpch-csv/nation.csv b/datafusion/core/tests/tpch-csv/nation.csv similarity index 100% rename from datafusion/tests/tpch-csv/nation.csv rename to datafusion/core/tests/tpch-csv/nation.csv diff --git a/datafusion/tests/tpch-csv/orders.csv b/datafusion/core/tests/tpch-csv/orders.csv similarity index 100% rename from datafusion/tests/tpch-csv/orders.csv rename to datafusion/core/tests/tpch-csv/orders.csv diff --git a/datafusion/tests/user_defined_plan.rs b/datafusion/core/tests/user_defined_plan.rs similarity index 100% rename from datafusion/tests/user_defined_plan.rs rename to datafusion/core/tests/user_defined_plan.rs diff --git a/datafusion-expr/Cargo.toml b/datafusion/expr/Cargo.toml similarity index 95% rename from datafusion-expr/Cargo.toml rename to datafusion/expr/Cargo.toml index de0d227132b8..68b218f7f3bd 100644 --- a/datafusion-expr/Cargo.toml +++ b/datafusion/expr/Cargo.toml @@ -37,5 +37,5 @@ path = "src/lib.rs" [dependencies] ahash = { version = "0.7", default-features = false } arrow = { version = "11", features = ["prettyprint"] } -datafusion-common = { path = "../datafusion-common", version = "7.0.0" } +datafusion-common = { path = "../common", version = "7.0.0" } sqlparser = "0.15" diff --git a/datafusion-expr/README.md b/datafusion/expr/README.md similarity index 100% rename from datafusion-expr/README.md rename to datafusion/expr/README.md diff --git a/datafusion-expr/src/accumulator.rs b/datafusion/expr/src/accumulator.rs similarity index 100% rename from datafusion-expr/src/accumulator.rs rename to datafusion/expr/src/accumulator.rs diff --git a/datafusion-expr/src/aggregate_function.rs b/datafusion/expr/src/aggregate_function.rs similarity index 100% rename from datafusion-expr/src/aggregate_function.rs rename to datafusion/expr/src/aggregate_function.rs diff --git a/datafusion-expr/src/built_in_function.rs b/datafusion/expr/src/built_in_function.rs similarity index 100% rename from datafusion-expr/src/built_in_function.rs rename to datafusion/expr/src/built_in_function.rs diff --git a/datafusion-expr/src/columnar_value.rs b/datafusion/expr/src/columnar_value.rs similarity index 100% rename from datafusion-expr/src/columnar_value.rs rename to datafusion/expr/src/columnar_value.rs diff --git a/datafusion-expr/src/expr.rs b/datafusion/expr/src/expr.rs similarity index 100% rename from datafusion-expr/src/expr.rs rename to datafusion/expr/src/expr.rs diff --git a/datafusion-expr/src/expr_fn.rs b/datafusion/expr/src/expr_fn.rs similarity index 100% rename from datafusion-expr/src/expr_fn.rs rename to datafusion/expr/src/expr_fn.rs diff --git a/datafusion-expr/src/function.rs b/datafusion/expr/src/function.rs similarity index 100% rename from datafusion-expr/src/function.rs rename to datafusion/expr/src/function.rs diff --git a/datafusion-expr/src/lib.rs b/datafusion/expr/src/lib.rs similarity index 100% rename from datafusion-expr/src/lib.rs rename to datafusion/expr/src/lib.rs diff --git a/datafusion-expr/src/literal.rs b/datafusion/expr/src/literal.rs similarity index 100% rename from datafusion-expr/src/literal.rs rename to datafusion/expr/src/literal.rs diff --git a/datafusion-expr/src/operator.rs b/datafusion/expr/src/operator.rs similarity index 100% rename from datafusion-expr/src/operator.rs rename to datafusion/expr/src/operator.rs diff --git a/datafusion-expr/src/signature.rs b/datafusion/expr/src/signature.rs similarity index 100% rename from datafusion-expr/src/signature.rs rename to datafusion/expr/src/signature.rs diff --git a/datafusion-expr/src/udaf.rs b/datafusion/expr/src/udaf.rs similarity index 100% rename from datafusion-expr/src/udaf.rs rename to datafusion/expr/src/udaf.rs diff --git a/datafusion-expr/src/udf.rs b/datafusion/expr/src/udf.rs similarity index 100% rename from datafusion-expr/src/udf.rs rename to datafusion/expr/src/udf.rs diff --git a/datafusion-expr/src/window_frame.rs b/datafusion/expr/src/window_frame.rs similarity index 100% rename from datafusion-expr/src/window_frame.rs rename to datafusion/expr/src/window_frame.rs diff --git a/datafusion-expr/src/window_function.rs b/datafusion/expr/src/window_function.rs similarity index 100% rename from datafusion-expr/src/window_function.rs rename to datafusion/expr/src/window_function.rs diff --git a/datafusion-jit/Cargo.toml b/datafusion/jit/Cargo.toml similarity index 93% rename from datafusion-jit/Cargo.toml rename to datafusion/jit/Cargo.toml index 69811c828c80..26780271dca0 100644 --- a/datafusion-jit/Cargo.toml +++ b/datafusion/jit/Cargo.toml @@ -40,5 +40,5 @@ cranelift = "0.82.0" cranelift-jit = "0.82.0" cranelift-module = "0.82.0" cranelift-native = "0.82.0" -datafusion-common = { path = "../datafusion-common", version = "7.0.0", features = ["jit"] } +datafusion-common = { path = "../common", version = "7.0.0", features = ["jit"] } parking_lot = "0.12" diff --git a/datafusion-jit/src/api.rs b/datafusion/jit/src/api.rs similarity index 100% rename from datafusion-jit/src/api.rs rename to datafusion/jit/src/api.rs diff --git a/datafusion-jit/src/ast.rs b/datafusion/jit/src/ast.rs similarity index 100% rename from datafusion-jit/src/ast.rs rename to datafusion/jit/src/ast.rs diff --git a/datafusion-jit/src/jit.rs b/datafusion/jit/src/jit.rs similarity index 100% rename from datafusion-jit/src/jit.rs rename to datafusion/jit/src/jit.rs diff --git a/datafusion-jit/src/lib.rs b/datafusion/jit/src/lib.rs similarity index 100% rename from datafusion-jit/src/lib.rs rename to datafusion/jit/src/lib.rs diff --git a/datafusion-physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml similarity index 93% rename from datafusion-physical-expr/Cargo.toml rename to datafusion/physical-expr/Cargo.toml index 485af3757d36..8f93c72a5b22 100644 --- a/datafusion-physical-expr/Cargo.toml +++ b/datafusion/physical-expr/Cargo.toml @@ -44,8 +44,8 @@ arrow = { version = "11", features = ["prettyprint"] } blake2 = { version = "^0.10.2", optional = true } blake3 = { version = "1.0", optional = true } chrono = { version = "0.4", default-features = false } -datafusion-common = { path = "../datafusion-common", version = "7.0.0" } -datafusion-expr = { path = "../datafusion-expr", version = "7.0.0" } +datafusion-common = { path = "../common", version = "7.0.0" } +datafusion-expr = { path = "../expr", version = "7.0.0" } hashbrown = { version = "0.12", features = ["raw"] } lazy_static = { version = "^1.4.0" } md-5 = { version = "^0.10.0", optional = true } diff --git a/datafusion-physical-expr/README.md b/datafusion/physical-expr/README.md similarity index 100% rename from datafusion-physical-expr/README.md rename to datafusion/physical-expr/README.md diff --git a/datafusion-physical-expr/src/aggregate_expr.rs b/datafusion/physical-expr/src/aggregate_expr.rs similarity index 100% rename from datafusion-physical-expr/src/aggregate_expr.rs rename to datafusion/physical-expr/src/aggregate_expr.rs diff --git a/datafusion-physical-expr/src/array_expressions.rs b/datafusion/physical-expr/src/array_expressions.rs similarity index 100% rename from datafusion-physical-expr/src/array_expressions.rs rename to datafusion/physical-expr/src/array_expressions.rs diff --git a/datafusion-physical-expr/src/coercion_rule/aggregate_rule.rs b/datafusion/physical-expr/src/coercion_rule/aggregate_rule.rs similarity index 100% rename from datafusion-physical-expr/src/coercion_rule/aggregate_rule.rs rename to datafusion/physical-expr/src/coercion_rule/aggregate_rule.rs diff --git a/datafusion-physical-expr/src/coercion_rule/binary_rule.rs b/datafusion/physical-expr/src/coercion_rule/binary_rule.rs similarity index 100% rename from datafusion-physical-expr/src/coercion_rule/binary_rule.rs rename to datafusion/physical-expr/src/coercion_rule/binary_rule.rs diff --git a/datafusion-physical-expr/src/coercion_rule/mod.rs b/datafusion/physical-expr/src/coercion_rule/mod.rs similarity index 100% rename from datafusion-physical-expr/src/coercion_rule/mod.rs rename to datafusion/physical-expr/src/coercion_rule/mod.rs diff --git a/datafusion-physical-expr/src/crypto_expressions.rs b/datafusion/physical-expr/src/crypto_expressions.rs similarity index 100% rename from datafusion-physical-expr/src/crypto_expressions.rs rename to datafusion/physical-expr/src/crypto_expressions.rs diff --git a/datafusion-physical-expr/src/datetime_expressions.rs b/datafusion/physical-expr/src/datetime_expressions.rs similarity index 100% rename from datafusion-physical-expr/src/datetime_expressions.rs rename to datafusion/physical-expr/src/datetime_expressions.rs diff --git a/datafusion-physical-expr/src/expressions/approx_distinct.rs b/datafusion/physical-expr/src/expressions/approx_distinct.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/approx_distinct.rs rename to datafusion/physical-expr/src/expressions/approx_distinct.rs diff --git a/datafusion-physical-expr/src/expressions/approx_median.rs b/datafusion/physical-expr/src/expressions/approx_median.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/approx_median.rs rename to datafusion/physical-expr/src/expressions/approx_median.rs diff --git a/datafusion-physical-expr/src/expressions/approx_percentile_cont.rs b/datafusion/physical-expr/src/expressions/approx_percentile_cont.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/approx_percentile_cont.rs rename to datafusion/physical-expr/src/expressions/approx_percentile_cont.rs diff --git a/datafusion-physical-expr/src/expressions/approx_percentile_cont_with_weight.rs b/datafusion/physical-expr/src/expressions/approx_percentile_cont_with_weight.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/approx_percentile_cont_with_weight.rs rename to datafusion/physical-expr/src/expressions/approx_percentile_cont_with_weight.rs diff --git a/datafusion-physical-expr/src/expressions/array_agg.rs b/datafusion/physical-expr/src/expressions/array_agg.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/array_agg.rs rename to datafusion/physical-expr/src/expressions/array_agg.rs diff --git a/datafusion-physical-expr/src/expressions/average.rs b/datafusion/physical-expr/src/expressions/average.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/average.rs rename to datafusion/physical-expr/src/expressions/average.rs diff --git a/datafusion-physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/binary.rs rename to datafusion/physical-expr/src/expressions/binary.rs diff --git a/datafusion-physical-expr/src/expressions/case.rs b/datafusion/physical-expr/src/expressions/case.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/case.rs rename to datafusion/physical-expr/src/expressions/case.rs diff --git a/datafusion-physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/cast.rs rename to datafusion/physical-expr/src/expressions/cast.rs diff --git a/datafusion-physical-expr/src/expressions/column.rs b/datafusion/physical-expr/src/expressions/column.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/column.rs rename to datafusion/physical-expr/src/expressions/column.rs diff --git a/datafusion-physical-expr/src/expressions/correlation.rs b/datafusion/physical-expr/src/expressions/correlation.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/correlation.rs rename to datafusion/physical-expr/src/expressions/correlation.rs diff --git a/datafusion-physical-expr/src/expressions/count.rs b/datafusion/physical-expr/src/expressions/count.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/count.rs rename to datafusion/physical-expr/src/expressions/count.rs diff --git a/datafusion-physical-expr/src/expressions/covariance.rs b/datafusion/physical-expr/src/expressions/covariance.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/covariance.rs rename to datafusion/physical-expr/src/expressions/covariance.rs diff --git a/datafusion-physical-expr/src/expressions/cume_dist.rs b/datafusion/physical-expr/src/expressions/cume_dist.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/cume_dist.rs rename to datafusion/physical-expr/src/expressions/cume_dist.rs diff --git a/datafusion-physical-expr/src/expressions/distinct_expressions.rs b/datafusion/physical-expr/src/expressions/distinct_expressions.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/distinct_expressions.rs rename to datafusion/physical-expr/src/expressions/distinct_expressions.rs diff --git a/datafusion-physical-expr/src/expressions/get_indexed_field.rs b/datafusion/physical-expr/src/expressions/get_indexed_field.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/get_indexed_field.rs rename to datafusion/physical-expr/src/expressions/get_indexed_field.rs diff --git a/datafusion-physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/in_list.rs rename to datafusion/physical-expr/src/expressions/in_list.rs diff --git a/datafusion-physical-expr/src/expressions/is_not_null.rs b/datafusion/physical-expr/src/expressions/is_not_null.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/is_not_null.rs rename to datafusion/physical-expr/src/expressions/is_not_null.rs diff --git a/datafusion-physical-expr/src/expressions/is_null.rs b/datafusion/physical-expr/src/expressions/is_null.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/is_null.rs rename to datafusion/physical-expr/src/expressions/is_null.rs diff --git a/datafusion-physical-expr/src/expressions/lead_lag.rs b/datafusion/physical-expr/src/expressions/lead_lag.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/lead_lag.rs rename to datafusion/physical-expr/src/expressions/lead_lag.rs diff --git a/datafusion-physical-expr/src/expressions/literal.rs b/datafusion/physical-expr/src/expressions/literal.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/literal.rs rename to datafusion/physical-expr/src/expressions/literal.rs diff --git a/datafusion-physical-expr/src/expressions/min_max.rs b/datafusion/physical-expr/src/expressions/min_max.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/min_max.rs rename to datafusion/physical-expr/src/expressions/min_max.rs diff --git a/datafusion-physical-expr/src/expressions/mod.rs b/datafusion/physical-expr/src/expressions/mod.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/mod.rs rename to datafusion/physical-expr/src/expressions/mod.rs diff --git a/datafusion-physical-expr/src/expressions/negative.rs b/datafusion/physical-expr/src/expressions/negative.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/negative.rs rename to datafusion/physical-expr/src/expressions/negative.rs diff --git a/datafusion-physical-expr/src/expressions/not.rs b/datafusion/physical-expr/src/expressions/not.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/not.rs rename to datafusion/physical-expr/src/expressions/not.rs diff --git a/datafusion-physical-expr/src/expressions/nth_value.rs b/datafusion/physical-expr/src/expressions/nth_value.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/nth_value.rs rename to datafusion/physical-expr/src/expressions/nth_value.rs diff --git a/datafusion-physical-expr/src/expressions/nullif.rs b/datafusion/physical-expr/src/expressions/nullif.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/nullif.rs rename to datafusion/physical-expr/src/expressions/nullif.rs diff --git a/datafusion-physical-expr/src/expressions/rank.rs b/datafusion/physical-expr/src/expressions/rank.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/rank.rs rename to datafusion/physical-expr/src/expressions/rank.rs diff --git a/datafusion-physical-expr/src/expressions/row_number.rs b/datafusion/physical-expr/src/expressions/row_number.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/row_number.rs rename to datafusion/physical-expr/src/expressions/row_number.rs diff --git a/datafusion-physical-expr/src/expressions/stats.rs b/datafusion/physical-expr/src/expressions/stats.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/stats.rs rename to datafusion/physical-expr/src/expressions/stats.rs diff --git a/datafusion-physical-expr/src/expressions/stddev.rs b/datafusion/physical-expr/src/expressions/stddev.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/stddev.rs rename to datafusion/physical-expr/src/expressions/stddev.rs diff --git a/datafusion-physical-expr/src/expressions/sum.rs b/datafusion/physical-expr/src/expressions/sum.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/sum.rs rename to datafusion/physical-expr/src/expressions/sum.rs diff --git a/datafusion-physical-expr/src/expressions/try_cast.rs b/datafusion/physical-expr/src/expressions/try_cast.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/try_cast.rs rename to datafusion/physical-expr/src/expressions/try_cast.rs diff --git a/datafusion-physical-expr/src/expressions/variance.rs b/datafusion/physical-expr/src/expressions/variance.rs similarity index 100% rename from datafusion-physical-expr/src/expressions/variance.rs rename to datafusion/physical-expr/src/expressions/variance.rs diff --git a/datafusion-physical-expr/src/field_util.rs b/datafusion/physical-expr/src/field_util.rs similarity index 100% rename from datafusion-physical-expr/src/field_util.rs rename to datafusion/physical-expr/src/field_util.rs diff --git a/datafusion-physical-expr/src/functions.rs b/datafusion/physical-expr/src/functions.rs similarity index 100% rename from datafusion-physical-expr/src/functions.rs rename to datafusion/physical-expr/src/functions.rs diff --git a/datafusion-physical-expr/src/hyperloglog/mod.rs b/datafusion/physical-expr/src/hyperloglog/mod.rs similarity index 100% rename from datafusion-physical-expr/src/hyperloglog/mod.rs rename to datafusion/physical-expr/src/hyperloglog/mod.rs diff --git a/datafusion-physical-expr/src/lib.rs b/datafusion/physical-expr/src/lib.rs similarity index 100% rename from datafusion-physical-expr/src/lib.rs rename to datafusion/physical-expr/src/lib.rs diff --git a/datafusion-physical-expr/src/math_expressions.rs b/datafusion/physical-expr/src/math_expressions.rs similarity index 100% rename from datafusion-physical-expr/src/math_expressions.rs rename to datafusion/physical-expr/src/math_expressions.rs diff --git a/datafusion-physical-expr/src/physical_expr.rs b/datafusion/physical-expr/src/physical_expr.rs similarity index 100% rename from datafusion-physical-expr/src/physical_expr.rs rename to datafusion/physical-expr/src/physical_expr.rs diff --git a/datafusion-physical-expr/src/regex_expressions.rs b/datafusion/physical-expr/src/regex_expressions.rs similarity index 100% rename from datafusion-physical-expr/src/regex_expressions.rs rename to datafusion/physical-expr/src/regex_expressions.rs diff --git a/datafusion-physical-expr/src/sort_expr.rs b/datafusion/physical-expr/src/sort_expr.rs similarity index 100% rename from datafusion-physical-expr/src/sort_expr.rs rename to datafusion/physical-expr/src/sort_expr.rs diff --git a/datafusion-physical-expr/src/string_expressions.rs b/datafusion/physical-expr/src/string_expressions.rs similarity index 100% rename from datafusion-physical-expr/src/string_expressions.rs rename to datafusion/physical-expr/src/string_expressions.rs diff --git a/datafusion-physical-expr/src/tdigest/mod.rs b/datafusion/physical-expr/src/tdigest/mod.rs similarity index 100% rename from datafusion-physical-expr/src/tdigest/mod.rs rename to datafusion/physical-expr/src/tdigest/mod.rs diff --git a/datafusion-physical-expr/src/unicode_expressions.rs b/datafusion/physical-expr/src/unicode_expressions.rs similarity index 100% rename from datafusion-physical-expr/src/unicode_expressions.rs rename to datafusion/physical-expr/src/unicode_expressions.rs diff --git a/datafusion-physical-expr/src/window/aggregate.rs b/datafusion/physical-expr/src/window/aggregate.rs similarity index 100% rename from datafusion-physical-expr/src/window/aggregate.rs rename to datafusion/physical-expr/src/window/aggregate.rs diff --git a/datafusion-physical-expr/src/window/built_in.rs b/datafusion/physical-expr/src/window/built_in.rs similarity index 100% rename from datafusion-physical-expr/src/window/built_in.rs rename to datafusion/physical-expr/src/window/built_in.rs diff --git a/datafusion-physical-expr/src/window/built_in_window_function_expr.rs b/datafusion/physical-expr/src/window/built_in_window_function_expr.rs similarity index 100% rename from datafusion-physical-expr/src/window/built_in_window_function_expr.rs rename to datafusion/physical-expr/src/window/built_in_window_function_expr.rs diff --git a/datafusion-physical-expr/src/window/mod.rs b/datafusion/physical-expr/src/window/mod.rs similarity index 100% rename from datafusion-physical-expr/src/window/mod.rs rename to datafusion/physical-expr/src/window/mod.rs diff --git a/datafusion-physical-expr/src/window/partition_evaluator.rs b/datafusion/physical-expr/src/window/partition_evaluator.rs similarity index 100% rename from datafusion-physical-expr/src/window/partition_evaluator.rs rename to datafusion/physical-expr/src/window/partition_evaluator.rs diff --git a/datafusion-physical-expr/src/window/window_expr.rs b/datafusion/physical-expr/src/window/window_expr.rs similarity index 100% rename from datafusion-physical-expr/src/window/window_expr.rs rename to datafusion/physical-expr/src/window/window_expr.rs diff --git a/datafusion-proto/Cargo.toml b/datafusion/proto/Cargo.toml similarity index 95% rename from datafusion-proto/Cargo.toml rename to datafusion/proto/Cargo.toml index b71e434531cc..5b1cdae72cb2 100644 --- a/datafusion-proto/Cargo.toml +++ b/datafusion/proto/Cargo.toml @@ -35,7 +35,7 @@ path = "src/lib.rs" [features] [dependencies] -datafusion = { path = "../datafusion", version = "7.0.0" } +datafusion = { path = "../core", version = "7.0.0" } prost = "0.9" [build-dependencies] diff --git a/datafusion-proto/build.rs b/datafusion/proto/build.rs similarity index 100% rename from datafusion-proto/build.rs rename to datafusion/proto/build.rs diff --git a/datafusion-proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto similarity index 100% rename from datafusion-proto/proto/datafusion.proto rename to datafusion/proto/proto/datafusion.proto diff --git a/datafusion-proto/src/from_proto.rs b/datafusion/proto/src/from_proto.rs similarity index 100% rename from datafusion-proto/src/from_proto.rs rename to datafusion/proto/src/from_proto.rs diff --git a/datafusion-proto/src/lib.rs b/datafusion/proto/src/lib.rs similarity index 100% rename from datafusion-proto/src/lib.rs rename to datafusion/proto/src/lib.rs diff --git a/datafusion-proto/src/to_proto.rs b/datafusion/proto/src/to_proto.rs similarity index 100% rename from datafusion-proto/src/to_proto.rs rename to datafusion/proto/src/to_proto.rs