processing_engine: initial implementation of Processing Engine plugin…

…s and triggers
influxdata · Dec 10, 2024 · d2ec446 · d2ec446
1 parent 3a66fe0
commit d2ec446
Show file tree

Hide file tree

Showing 31 changed files with 1,176 additions and 52 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -8,6 +8,8 @@ members = [
     "influxdb3_id",
     "influxdb3_load_generator",
     "influxdb3_process",
+    "influxdb3_processing_engine",
+    "influxdb3_py_api",
     "influxdb3_server",
     "influxdb3_telemetry",
     "influxdb3_test_helpers",
@@ -38,7 +40,7 @@ license = "MIT OR Apache-2.0"
 
 [workspace.dependencies]
 anyhow = "1.0"
-arrow = { version = "53.0.0", features = ["prettyprint", "chrono-tz"] }
+arrow = { version = "53.0.0", features = ["prettyprint", "chrono-tz", "pyarrow"] }
 arrow-array = "53.0.0"
 arrow-buffer = "53.0.0"
 arrow-csv = "53.0.0"

diff --git a/influxdb3/tests/server/flight.rs b/influxdb3/tests/server/flight.rs
@@ -108,20 +108,22 @@ async fn flight() -> Result<(), influxdb3_client::Error> {
 
         assert_batches_sorted_eq!(
             [
-                "+--------------+--------------------+---------------+------------+",
-                "| catalog_name | db_schema_name     | table_name    | table_type |",
-                "+--------------+--------------------+---------------+------------+",
-                "| public       | information_schema | columns       | VIEW       |",
-                "| public       | information_schema | df_settings   | VIEW       |",
-                "| public       | information_schema | schemata      | VIEW       |",
-                "| public       | information_schema | tables        | VIEW       |",
-                "| public       | information_schema | views         | VIEW       |",
-                "| public       | iox                | cpu           | BASE TABLE |",
-                "| public       | system             | last_caches   | BASE TABLE |",
-                "| public       | system             | meta_caches   | BASE TABLE |",
-                "| public       | system             | parquet_files | BASE TABLE |",
-                "| public       | system             | queries       | BASE TABLE |",
-                "+--------------+--------------------+---------------+------------+",
+                "+--------------+--------------------+----------------------------+------------+",
+                "| catalog_name | db_schema_name     | table_name                 | table_type |",
+                "+--------------+--------------------+----------------------------+------------+",
+                "| public       | information_schema | columns                    | VIEW       |",
+                "| public       | information_schema | df_settings                | VIEW       |",
+                "| public       | information_schema | schemata                   | VIEW       |",
+                "| public       | information_schema | tables                     | VIEW       |",
+                "| public       | information_schema | views                      | VIEW       |",
+                "| public       | iox                | cpu                        | BASE TABLE |",
+                "| public       | system             | last_caches                | BASE TABLE |",
+                "| public       | system             | meta_caches                | BASE TABLE |",
+                "| public       | system             | parquet_files              | BASE TABLE |",
+                "| public       | system             | processing_engine_plugins  | BASE TABLE |",
+                "| public       | system             | processing_engine_triggers | BASE TABLE |",
+                "| public       | system             | queries                    | BASE TABLE |",
+                "+--------------+--------------------+----------------------------+------------+",
             ],
             &batches
         );

diff --git a/influxdb3_cache/src/last_cache/mod.rs b/influxdb3_cache/src/last_cache/mod.rs
@@ -1267,6 +1267,8 @@ mod tests {
                 map.insert(TableId::from(1), "test_table_2".into());
                 map
             },
+            processing_engine_plugins: Default::default(),
+            processing_engine_triggers: Default::default(),
             deleted: false,
         };
         let table_id = TableId::from(0);

diff --git a/influxdb3_catalog/Cargo.toml b/influxdb3_catalog/Cargo.toml
@@ -15,6 +15,7 @@ iox_time.workspace = true
 # Local deps
 influxdb3_id = { path = "../influxdb3_id" }
 influxdb3_wal = { path = "../influxdb3_wal" }
+influxdb3_processing_engine = { path = "../influxdb3_processing_engine" }
 
 # crates.io dependencies
 arrow.workspace = true