WIP: Attempt to support arrow serialization

vl-convert-rs/src/converter.rs

@@ -1,4 +1,6 @@
-use crate::module_loader::import_map::{url_for_path, vega_themes_url, vega_url, VlVersion};
+use crate::module_loader::import_map::{
+    url_for_path, vega_themes_url, vega_url, VlVersion, ARROW_PATH,
+};
 use crate::module_loader::{VlConvertModuleLoader, IMPORT_MAP};
 use std::borrow::Cow;
 use std::collections::hash_map::Entry;
@@ -141,9 +143,38 @@ var vegaThemes;
 import('{vega_themes_url}').then((imported) => {{
     vegaThemes = imported;
 }})
+
+var arrow;
+import('{arrow_url}').then((imported) => {{
+    arrow = imported;
+}})
+
+async function base64UrlToArrayBuffer(b64Url) {{
+    const res = await fetch(b64Url);
+    return await res.arrayBuffer();
+}}
+
+async function deserializeArrowDatasets(spec) {{
+    // Handle data
+    for (const dataset of spec.data || []) {{
+         if (typeof dataset.url === "string" && dataset.url.startsWith("data:application/vnd.apache.arrow.file;base64,")) {{
+            dataset.values = arrow.tableFromIPC(await base64UrlToArrayBuffer(dataset.url));
+            delete dataset.url;
+        }}
+    }}
+
+    // Handle group marks recursively
+    for (const mark of spec.marks || []) {{
+        if (mark.type === "group") {{
+            await deserializeArrowDatasets(mark);
+        }}
+    }}
+}}
+
 "#,
                 vega_url = vega_url(),
                 vega_themes_url = vega_themes_url(),
+                arrow_url = url_for_path(ARROW_PATH),
             ));
 
             self.worker.execute_script("<anon>", import_code)?;
@@ -219,11 +250,13 @@ import('{url}').then((sg) => {{
             // Create and initialize svg function string
             let function_str = r#"
 function vegaToSvg(vgSpec) {
-    let runtime = vega.parse(vgSpec);
-    const baseURL = 'https://vega.github.io/vega-datasets/';
-    const loader = vega.loader({ mode: 'http', baseURL });
-    let view = new vega.View(runtime, {renderer: 'none', loader});
-    let svgPromise = view.toSVG().finally(() => { view.finalize() });
+    let svgPromise = deserializeArrowDatasets(vgSpec).then(() => {
+        let runtime = vega.parse(vgSpec);
+        const baseURL = 'https://vega.github.io/vega-datasets/';
+        const loader = vega.loader({ mode: 'http', baseURL });
+        let view = new vega.View(runtime, {renderer: 'none', loader});
+        return view.toSVG().finally(() => { view.finalize() });
+    });
     return svgPromise
 }
 "#;

vl-convert-rs/src/html.rs

@@ -1,4 +1,6 @@
-use crate::module_loader::import_map::{DEBOUNCE_PATH, SKYPACK_URL, VEGA_EMBED_PATH, VEGA_PATH};
+use crate::module_loader::import_map::{
+    ARROW_PATH, DEBOUNCE_PATH, SKYPACK_URL, VEGA_EMBED_PATH, VEGA_PATH, VEGA_SCHEMA_PATH,
+};
 use crate::module_loader::VlConvertBundleLoader;
 use crate::VlVersion;
 use deno_core::error::AnyError;
@@ -73,6 +75,8 @@ import vegaEmbed from "{SKYPACK_URL}{VEGA_EMBED_PATH}"
 import vega from "{SKYPACK_URL}{VEGA_PATH}"
 import vegaLite from "{SKYPACK_URL}{VEGA_LITE_PATH}"
 import lodashDebounce from "{SKYPACK_URL}{DEBOUNCE_PATH}"
+import vegaSchemaUrlParser from "{SKYPACK_URL}{VEGA_SCHEMA_PATH}"
+import * as arrow from "{SKYPACK_URL}{ARROW_PATH}"
 {snippet}
 "#,
         VEGA_LITE_PATH = vl_version.to_path()

vl-convert-rs/tests/test_specs.rs

@@ -410,6 +410,7 @@ mod test_png_no_theme {
 
     #[rstest(name, scale,
         case("circle_binned", 1.0),
+        case("circle_binned_b64", 1.0),
         case("circle_binned_base_url", 1.0),
         case("stacked_bar_h", 2.0),
         case("bar_chart_trellis_compact", 2.0),