Skip to content

Commit

Permalink
expose checkpoint interval and target file size
Browse files Browse the repository at this point in the history
  • Loading branch information
Ryan Aston committed Oct 12, 2023
1 parent ae3265c commit e041765
Showing 1 changed file with 22 additions and 0 deletions.
22 changes: 22 additions & 0 deletions rust/src/table_state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,15 @@ pub struct DeltaTableState {
min_writer_version: i32,
// table metadata corresponding to current version
current_metadata: Option<DeltaTableMetaData>,
// frequency for creating checkpoints
checkpoint_interval: i32,
// retention period for tombstones in milli-seconds
tombstone_retention_millis: i64,
// retention period for log entries in milli-seconds
log_retention_millis: i64,
enable_expired_log_cleanup: bool,
// target file size for optimization
target_file_size: i64,
}

impl DeltaTableState {
Expand Down Expand Up @@ -166,6 +170,11 @@ impl DeltaTableState {
&self.commit_infos
}

/// Frequency for creating checkpoints.
pub fn checkpoint_interval(&self) -> i32 {
self.checkpoint_interval
}

/// Retention of tombstone in milliseconds.
pub fn tombstone_retention_millis(&self) -> i64 {
self.tombstone_retention_millis
Expand All @@ -181,6 +190,11 @@ impl DeltaTableState {
self.enable_expired_log_cleanup
}

/// Target file size for optimizations.
pub fn target_file_size(&self) -> i64 {
self.target_file_size
}

/// Full list of tombstones (remove actions) representing files removed from table state).
pub fn all_tombstones(&self) -> &HashSet<action::Remove> {
&self.tombstones
Expand Down Expand Up @@ -288,9 +302,11 @@ impl DeltaTableState {
}

if new_state.current_metadata.is_some() {
self.checkpoint_interval = new_state.checkpoint_interval;
self.tombstone_retention_millis = new_state.tombstone_retention_millis;
self.log_retention_millis = new_state.log_retention_millis;
self.enable_expired_log_cleanup = new_state.enable_expired_log_cleanup;
self.target_file_size = new_state.target_file_size;
self.current_metadata = new_state.current_metadata.take();
}

Expand Down Expand Up @@ -341,11 +357,13 @@ impl DeltaTableState {
action::Action::metaData(v) => {
let md = DeltaTableMetaData::try_from(v)?;
let table_config = TableConfig(&md.configuration);
self.checkpoint_interval = table_config.checkpoint_interval();
self.tombstone_retention_millis =
table_config.deleted_file_retention_duration().as_millis() as i64;
self.log_retention_millis =
table_config.log_retention_duration().as_millis() as i64;
self.enable_expired_log_cleanup = table_config.enable_expired_log_cleanup();
self.target_file_size = table_config.target_file_size();
self.current_metadata = Some(md);
}
action::Action::txn(v) => {
Expand Down Expand Up @@ -416,9 +434,11 @@ mod tests {
min_reader_version: 0,
min_writer_version: 0,
current_metadata: None,
checkpoint_interval: 0,
tombstone_retention_millis: 0,
log_retention_millis: 0,
enable_expired_log_cleanup: false,
target_file_size: 0,
};
let bytes = serde_json::to_vec(&expected).unwrap();
let actual: DeltaTableState = serde_json::from_slice(&bytes).unwrap();
Expand All @@ -440,9 +460,11 @@ mod tests {
min_reader_version: 1,
min_writer_version: 1,
app_transaction_version,
checkpoint_interval: 0,
tombstone_retention_millis: 0,
log_retention_millis: 0,
enable_expired_log_cleanup: true,
target_file_size: 0,
};

let txn_action = action::Action::txn(action::Txn {
Expand Down

0 comments on commit e041765

Please sign in to comment.