Refactor crate structure (#241)

* Remove unused error conversion

* Extract error to vegafusion-common crate

* Rename vegafusion-rt-datafusion to vegafusion-runtime

* extract vegafusion-datafusion-udfs crate

* extract sql logic into vegafusion-sql crate

* Fix vegafusion-wasm package

* Fix compilation

* add new crates to bump_version.py

* make vegafusion-runtime not depend on full datafusion crate

* Pass Connection to TaskGraphRuntime

Remove dependency on vegafusion-sql from vegafusion-runtime

* restore sqlite connection begine sqlite-conn feature flag

* Remove unused dependencies from vegafusion-rumtime

* TaskGraphRuntime -> VegaFusionRuntime

* DataFrame query methods don't need to be async anymore

* Initial crate READMEs

* clippy fix

* remove native-tls dependency

* PyTaskGraphRuntime -> PyVegaFusionRuntime

.github/workflows/build_test.yml

@@ -57,7 +57,7 @@ jobs:
         with:
           node-version: '17'
       - name: Install test node dependencies
-        working-directory: vegafusion-rt-datafusion/tests/util/vegajs_runtime
+        working-directory: vegafusion-runtime/tests/util/vegajs_runtime
         run: |
           npm install
       - name: Run Rust Tests (Windows)
@@ -81,7 +81,7 @@ jobs:
         with:
           name: vegafusion-rt-test-images
           path: |
-            vegafusion-rt-datafusion/tests/output
+            vegafusion-runtime/tests/output
 
   build-vegafusion-server:
     strategy:

.gitignore

@@ -12,9 +12,9 @@ Cargo.bak.lock
 vega-fusion/tests/scratch/*
 
 vega-fusion/tests/util/vegajs_runtime/node_modules
-vegafusion-rt-datafusion/tests/util/vegajs_runtime/node_modules/
+vegafusion-runtime/tests/util/vegajs_runtime/node_modules/
 /vegafusion-wasm/node_modules/
-/vegafusion-rt-datafusion/tests/output
+/vegafusion-runtime/tests/output
 /python/vegafusion-jupyter/docs/source/_static/
 /python/vegafusion-jupyter/vegafusion_jupyter/nbextension/
 /.idea/
@@ -36,7 +36,7 @@ _vegafusion_data
 /javascript/vegafusion-embed/package/css/
 /javascript/vegafusion-embed/package/lib/
 /vegafusion-server/examples/chart_editor/node_modules/
-/vegafusion-rt-datafusion/tests/data/
+/vegafusion-runtime/tests/data/
 /scratch/
 .DS_Store
 __pycache__

BUILD.md

@@ -59,7 +59,7 @@ Note: The `npm` command is included in the `nodejs` conda-forge package installe
 
 From the repository root:
 ```bash
-cd vegafusion-rt-datafusion/tests/util/vegajs_runtime/
+cd vegafusion-runtime/tests/util/vegajs_runtime/
 npm install
 ```
 

Cargo.toml

@@ -1,8 +1,12 @@
 [workspace]
 
 members = [
+    "vegafusion-common",
     "vegafusion-core",
-    "vegafusion-rt-datafusion",
+    "vegafusion-runtime",
+    "vegafusion-dataframe",
+    "vegafusion-datafusion-udfs",
+    "vegafusion-sql",
     "vegafusion-python-embed",
     "vegafusion-wasm",
     "vegafusion-server",

automation/bump_version.py

@@ -15,8 +15,12 @@ def bump_version(version):
 
     # Handle Cargo.toml files
     cargo_packages = [
+        "vegafusion-common",
         "vegafusion-core",
-        "vegafusion-rt-datafusion",
+        "vegafusion-datafusion-udfs",
+        "vegafusion-dataframe",
+        "vegafusion-sql",
+        "vegafusion-runtime",
         "vegafusion-python-embed",
         "vegafusion-server",
         "vegafusion-wasm"

python/vegafusion/vegafusion/runtime.py

@@ -22,9 +22,9 @@ def __init__(self, cache_capacity, memory_limit, worker_threads):
     def embedded_runtime(self):
         if self._embedded_runtime is None:
             # Try to initialize an embedded runtime
-            from vegafusion_embed import PyTaskGraphRuntime
+            from vegafusion_embed import PyVegaFusionRuntime
 
-            self._embedded_runtime = PyTaskGraphRuntime(self.cache_capacity, self.memory_limit, self.worker_threads)
+            self._embedded_runtime = PyVegaFusionRuntime(self.cache_capacity, self.memory_limit, self.worker_threads)
         return self._embedded_runtime
 
     def grpc_connect(self, channel):