Skip to content

Commit

Permalink
chore: update datafusion (delta-io#1114)
Browse files Browse the repository at this point in the history
# Description

A simple maintenance PR to update datafusion to the latest version. 

# Related Issue(s)
<!---
For example:

- closes delta-io#106
--->

# Documentation

<!---
Share links to useful documentation
--->
  • Loading branch information
roeap authored and chitralverma committed Mar 17, 2023
1 parent 19f607a commit 4afdaef
Show file tree
Hide file tree
Showing 5 changed files with 13 additions and 14 deletions.
2 changes: 1 addition & 1 deletion python/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ crate-type = ["cdylib"]
name = "deltalake._internal"

[dependencies]
arrow-schema = { version = "29", features = ["serde"] }
arrow-schema = { version = "31", features = ["serde"] }
chrono = "0"
env_logger = "0"
futures = "0.3"
Expand Down
4 changes: 2 additions & 2 deletions python/deltalake/_internal.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class MapType:
key_type: DataType,
value_type: DataType,
*,
value_contains_null: bool = True
value_contains_null: bool = True,
) -> None: ...
type: Literal["map"]
key_type: DataType
Expand All @@ -90,7 +90,7 @@ class Field:
type: DataType,
*,
nullable: bool = True,
metadata: Optional[Dict[str, Any]] = None
metadata: Optional[Dict[str, Any]] = None,
) -> None: ...
name: str
type: DataType
Expand Down
1 change: 0 additions & 1 deletion python/deltalake/table.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,6 @@ def version(self) -> int:
def files(
self, partition_filters: Optional[List[Tuple[str, str, Any]]] = None
) -> List[str]:

return self._table.files(self.__stringify_partition_values(partition_filters))

files.__doc__ = f"""
Expand Down
14 changes: 7 additions & 7 deletions rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ readme = "README.md"
edition = "2021"

[dependencies]
arrow = { version = "29", optional = true }
arrow = { version = "31", optional = true }
async-trait = "0.1"
bytes = "1"
chrono = { version = "0.4.22", default-features = false, features = ["clock"] }
Expand All @@ -29,7 +29,7 @@ num-traits = "0.2.15"
object_store = "0.5.3"
once_cell = "1.16.0"
parking_lot = "0.12"
parquet = { version = "29", features = ["async"], optional = true }
parquet = { version = "31", features = ["async"], optional = true }
parquet2 = { version = "0.17", optional = true }
percent-encoding = "2"
serde = { version = "1", features = ["derive"] }
Expand All @@ -50,10 +50,10 @@ rusoto_dynamodb = { version = "0.48", default-features = false, optional = true
rusoto_glue = { version = "0.48", default-features = false, optional = true }

# Datafusion
datafusion = { version = "16", optional = true }
datafusion-expr = { version = "16", optional = true }
datafusion-common = { version = "16", optional = true }
datafusion-proto = { version = "16", optional = true }
datafusion = { version = "17", optional = true }
datafusion-expr = { version = "17", optional = true }
datafusion-common = { version = "17", optional = true }
datafusion-proto = { version = "17", optional = true }

# NOTE dependencies only for integration tests
fs_extra = { version = "1.2.0", optional = true }
Expand Down Expand Up @@ -128,4 +128,4 @@ required-features = ["datafusion"]

[[example]]
name = "recordbatch-writer"
required-features = ["arrow"]
required-features = ["arrow"]
6 changes: 3 additions & 3 deletions rust/src/action/parquet_read/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -268,15 +268,15 @@ fn primitive_parquet_field_to_json_value(field: &Field) -> Result<serde_json::Va
}
}

fn convert_timestamp_millis_to_string(value: u64) -> Result<String, &'static str> {
fn convert_timestamp_millis_to_string(value: i64) -> Result<String, &'static str> {
let dt = Utc
.timestamp_opt((value / 1000) as i64, ((value % 1000) * 1000000) as u32)
.timestamp_opt(value / 1000, ((value % 1000) * 1000000) as u32)
.single()
.ok_or("Value out of bounds")?;
Ok(dt.to_rfc3339_opts(SecondsFormat::Millis, true))
}

fn convert_date_to_string(value: u32) -> Result<String, &'static str> {
fn convert_date_to_string(value: i32) -> Result<String, &'static str> {
static NUM_SECONDS_IN_DAY: i64 = 60 * 60 * 24;
let dt = Utc
.timestamp_opt(value as i64 * NUM_SECONDS_IN_DAY, 0)
Expand Down

0 comments on commit 4afdaef

Please sign in to comment.