feat: support display spill file stats

databendlabs · Jan 7, 2025 · 24faee1 · 24faee1
1 parent c44caad
commit 24faee1
Show file tree

Hide file tree

Showing 6 changed files with 32 additions and 23 deletions.
diff --git a/bindings/nodejs/src/lib.rs b/bindings/nodejs/src/lib.rs
@@ -548,8 +548,8 @@ impl ServerStats {
     }
 
     #[napi(getter)]
-    pub fn spill_file_count(&self) -> usize {
-        self.0.spill_file_count
+    pub fn spill_file_nums(&self) -> usize {
+        self.0.spill_file_nums
     }
 
     #[napi(getter)]

diff --git a/bindings/python/src/types.rs b/bindings/python/src/types.rs
@@ -361,8 +361,8 @@ impl ServerStats {
         self.0.write_bytes
     }
     #[getter]
-    pub fn spill_file_count(&self) -> f64 {
-        self.0.spill_file_count
+    pub fn spill_file_nums(&self) -> f64 {
+        self.0.spill_file_nums
     }
     #[getter]
     pub fn running_time_ms(&self) -> f64 {

diff --git a/cli/src/display.rs b/cli/src/display.rs
@@ -417,8 +417,12 @@ fn format_read_progress(ss: &ServerStats, elapsed: f64) -> String {
         HumanBytes(ss.read_bytes as u64),
         HumanBytes(ss.total_bytes as u64),
         HumanBytes((ss.read_bytes as f64 / elapsed) as u64),
-        if ss.spill_file_count > 0 {
-            format!(", spilled {} files", ss.spill_file_count)
+        if ss.spill_file_nums > 0 {
+            format!(
+                ", spilled {} files, {}",
+                ss.spill_file_nums,
+                HumanBytes(ss.spill_bytes as u64)
+            )
         } else {
             "".to_string()
         }
@@ -432,8 +436,12 @@ pub fn format_write_progress(ss: &ServerStats, elapsed: f64) -> String {
         humanize_count(ss.write_rows as f64 / elapsed),
         HumanBytes(ss.write_bytes as u64),
         HumanBytes((ss.write_bytes as f64 / elapsed) as u64),
-        if ss.spill_file_count > 0 {
-            format!(", spilled {} files", ss.spill_file_count)
+        if ss.spill_file_nums > 0 {
+            format!(
+                ", spilled {} files, {}",
+                ss.spill_file_nums,
+                HumanBytes(ss.spill_bytes as u64)
+            )
         } else {
             "".to_string()
         }

diff --git a/core/src/response.rs b/core/src/response.rs
@@ -30,7 +30,7 @@ pub struct Progresses {
     pub result_progress: ProgressValues,
     // make it optional for backward compatibility
     pub total_scan: Option<ProgressValues>,
-    pub spill_file_count: usize,
+    pub spill_progress: SpillProgress,
 }
 
 impl Progresses {
@@ -54,6 +54,12 @@ pub struct ProgressValues {
     pub bytes: usize,
 }
 
+#[derive(Debug, Clone, Deserialize, Serialize, Default)]
+pub struct SpillProgress {
+    pub file_nums: usize,
+    pub bytes: usize,
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct SchemaField {
     pub name: String,

diff --git a/driver/src/conn.rs b/driver/src/conn.rs
@@ -199,14 +199,10 @@ pub trait Connection: Send + Sync {
                 Err(e) => (entry.to_string_lossy().to_string(), e.to_string()),
             };
             let ss = ServerStats {
-                total_rows: 0,
-                total_bytes: 0,
-                read_rows: 0,
-                read_bytes: 0,
                 write_rows: total_count,
                 write_bytes: total_size,
-                spill_file_count: 0,
-                running_time_ms: 0.0,
+
+                ..Default::default()
             };
             results.push(Ok(RowWithStats::Stats(ss)));
             results.push(Ok(RowWithStats::Row(Row::from_vec(
@@ -256,14 +252,9 @@ pub trait Connection: Send + Sync {
                 Err(e) => (e.to_string(), 0),
             };
             let ss = ServerStats {
-                total_rows: 0,
-                total_bytes: 0,
                 read_rows: total_count,
                 read_bytes: total_size,
-                spill_file_count: 0,
-                write_rows: 0,
-                write_bytes: 0,
-                running_time_ms: 0.0,
+                ..Default::default()
             };
             results.push(Ok(RowWithStats::Stats(ss)));
             results.push(Ok(RowWithStats::Row(Row::from_vec(

diff --git a/sql/src/rows.rs b/sql/src/rows.rs
@@ -53,7 +53,10 @@ pub struct ServerStats {
     pub running_time_ms: f64,
 
     #[serde(default)]
-    pub spill_file_count: usize,
+    pub spill_file_nums: usize,
+
+    #[serde(default)]
+    pub spill_bytes: usize,
 }
 
 impl ServerStats {
@@ -76,7 +79,8 @@ impl From<databend_client::QueryStats> for ServerStats {
             read_bytes: stats.progresses.scan_progress.bytes,
             write_rows: stats.progresses.write_progress.rows,
             write_bytes: stats.progresses.write_progress.bytes,
-            spill_file_count: stats.progresses.spill_file_count,
+            spill_file_nums: stats.progresses.spill_progress.file_nums,
+            spill_bytes: stats.progresses.spill_progress.bytes,
             running_time_ms: stats.running_time_ms,
         };
         if let Some(total) = stats.progresses.total_scan {