Fix compiling ballista in standalone mode, add build to CI (#1839)

* Test ballista compiles with --standalone in CI * Fix compile * ignore broken test ! * add ticket reference
apache · Feb 16, 2022 · 0a1eee0 · 0a1eee0
1 parent e79472d
commit 0a1eee0
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 6 deletions.
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -142,6 +142,8 @@ jobs:
           cd ballista/rust
           # snmalloc requires cmake so build without default features
           cargo test --no-default-features --features sled
+          # Ensure also compiles in standalone mode
+          cargo test --no-default-features --features standalone
         env:
           CARGO_HOME: "/github/home/.cargo"
           CARGO_TARGET_DIR: "/github/home/target"

diff --git a/ballista/rust/client/src/context.rs b/ballista/rust/client/src/context.rs
@@ -70,6 +70,8 @@ impl BallistaContextState {
         concurrent_tasks: usize,
     ) -> ballista_core::error::Result<Self> {
         use ballista_core::serde::protobuf::scheduler_grpc_client::SchedulerGrpcClient;
+        use ballista_core::serde::protobuf::PhysicalPlanNode;
+        use ballista_core::serde::BallistaCodec;
 
         log::info!("Running in local mode. Scheduler will be run in-proc");
 
@@ -90,7 +92,16 @@ impl BallistaContextState {
             }
         };
 
-        ballista_executor::new_standalone_executor(scheduler, concurrent_tasks).await?;
+        let default_codec: BallistaCodec<LogicalPlanNode, PhysicalPlanNode> =
+            BallistaCodec::default();
+
+        ballista_executor::new_standalone_executor(
+            scheduler,
+            concurrent_tasks,
+            default_codec,
+        )
+        .await?;
+
         Ok(Self {
             config: config.clone(),
             scheduler_host: "localhost".to_string(),
@@ -458,13 +469,17 @@ mod tests {
 
     #[tokio::test]
     #[cfg(feature = "standalone")]
+    #[ignore]
+    // Tracking: https://github.com/apache/arrow-datafusion/issues/1840
     async fn test_task_stuck_when_referenced_task_failed() {
         use super::*;
         use datafusion::arrow::datatypes::Schema;
         use datafusion::arrow::util::pretty;
         use datafusion::datasource::file_format::csv::CsvFormat;
         use datafusion::datasource::file_format::parquet::ParquetFormat;
-        use datafusion::datasource::listing::{ListingOptions, ListingTable};
+        use datafusion::datasource::listing::{
+            ListingOptions, ListingTable, ListingTableConfig,
+        };
 
         use ballista_core::config::{
             BallistaConfigBuilder, BALLISTA_WITH_INFORMATION_SCHEMA,
@@ -502,12 +517,16 @@ mod tests {
                         collect_stat: x.collect_stat,
                         target_partitions: x.target_partitions,
                     };
-                    let error_table = ListingTable::new(
+
+                    let config = ListingTableConfig::new(
                         listing_table.object_store().clone(),
                         listing_table.table_path().to_string(),
-                        Arc::new(Schema::new(vec![])),
-                        error_options,
-                    );
+                    )
+                    .with_schema(Arc::new(Schema::new(vec![])))
+                    .with_listing_options(error_options);
+
+                    let error_table = ListingTable::try_new(config).unwrap();
+
                     // change the table to an error table
                     guard
                         .tables