perf(protobuf): migrate from pandas to polars (#22080)

hongbo-miao · Dec 28, 2024 · f5abd3e · f5abd3e
1 parent 51d6a22
commit f5abd3e
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 76 deletions.
diff --git a/computer-vision/hm-open3d/pyproject.toml b/computer-vision/hm-open3d/pyproject.toml
@@ -5,7 +5,6 @@ requires-python = "~=3.10.0"
 dependencies = [
   "addict==2.4.0",
   "open3d==0.18.0",
-  "pandas==2.2.3",
   "Pillow==11.0.0",
   "PyYAML==6.0.2",
   "scikit-learn==1.6.0",

diff --git a/computer-vision/hm-open3d/uv.lock b/computer-vision/hm-open3d/uv.lock
diff --git a/data-storage/hm-protobuf/pyproject.toml b/data-storage/hm-protobuf/pyproject.toml
@@ -4,7 +4,7 @@ version = "1.0.0"
 requires-python = "~=3.13.0"
 dependencies = [
   "numpy==2.2.1",
-  "pandas==2.2.3",
+  "polars==1.18.0",
   "protobuf==5.29.2",
 ]
 

diff --git a/data-storage/hm-protobuf/src/main.py b/data-storage/hm-protobuf/src/main.py
@@ -5,7 +5,7 @@
 from pathlib import Path
 from typing import Any
 
-import pandas as pd
+import polars as pl
 from protos.production.iot import motor_pb2
 
 
@@ -27,7 +27,7 @@ class ProtobufReader:
     def __init__(self, filename: Path) -> None:
         self.file = open(filename, "rb")
 
-    def get_dataframe(self) -> pd.DataFrame:
+    def get_dataframe(self) -> pl.DataFrame:
         data: list[dict[str, Any]] = []
         while True:
             size_data: bytes = self.file.read(4)
@@ -53,7 +53,7 @@ def get_dataframe(self) -> pd.DataFrame:
             }
             data.append(row)
 
-        df: pd.DataFrame = pd.DataFrame(data)
+        df: pl.DataFrame = pl.DataFrame(data)
         return df
 
     def close(self) -> None:
@@ -92,7 +92,7 @@ def main() -> None:
 
     # Read data
     reader: ProtobufReader = ProtobufReader(motor_data_path)
-    df: pd.DataFrame = reader.get_dataframe()
+    df: pl.DataFrame = reader.get_dataframe()
     reader.close()
     logging.info(df)
 

diff --git a/data-storage/hm-protobuf/uv.lock b/data-storage/hm-protobuf/uv.lock