From 86e805eab7adc38ef5358479fc2897ac3e4af25d Mon Sep 17 00:00:00 2001 From: Aleksandr Golubov Date: Wed, 23 Nov 2022 16:23:06 +0500 Subject: [PATCH] Upgrade Metabase up to v1.44.6. Fixed some build issues. Allowed SQL modify queries (does't returns any results). --- .gitignore | 2 ++ README.md | 9 ++++---- build_docker_image.sh | 31 +++++++++++++++++++++++++++ build_image.sh | 24 --------------------- deps.edn | 8 +++---- docker/metabase/bin/docker/Dockerfile | 12 +++++++---- resources/metabase-plugin.yaml | 2 +- src/metabase/driver/duckdb.clj | 27 +++++++++++++++++++---- 8 files changed, 74 insertions(+), 41 deletions(-) mode change 100644 => 100755 README.md create mode 100755 build_docker_image.sh delete mode 100755 build_image.sh mode change 100644 => 100755 deps.edn mode change 100644 => 100755 docker/metabase/bin/docker/Dockerfile mode change 100644 => 100755 resources/metabase-plugin.yaml mode change 100644 => 100755 src/metabase/driver/duckdb.clj diff --git a/.gitignore b/.gitignore index ff71a43..e6da604 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ duckdb-driver.iml .lein-repl-history .cpcache +.clj-kondo +.lsp diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 6fc0147..c1a00ff --- a/README.md +++ b/README.md @@ -73,10 +73,11 @@ This will build a file called `target/duckdb.metabase-driver.jar`; copy this to ### Build in dev container using Visual Studio Code -Install the VSCode 'Remote - Containers' extension. Start the Docker engine. Open the project in the VSCode. You will be asked if you want to re-open the project in a dev container. Reopen the project in the container. Wait until it started. Start new VSCode terminal and build plugin the same way: +Install the VSCode 'Remote - Containers' extension. Start the Docker engine. Open the project in the VSCode. You will be asked if you want to re-open the project in a dev container. Reopen the project in the container. Wait until it started. Start new VSCode terminal and build plugin: -1. modify :paths in deps.edn, make them absolute -2. `vscode ➜ /workspaces/metabase_duckdb_driver (main ✗) $ clojure -X:build :project-dir "\"$(pwd)\""` +1. check the versions you need of Metabase and DuckDB in the `build_docker_image.sh` and `deps.edn` +2. modify :paths in deps.edn, make them absolute +3. `vscode ➜ /workspaces/metabase_duckdb_driver (main ✗) $ clojure -X:build :project-dir "\"$(pwd)\""` ## Configuring @@ -102,7 +103,7 @@ ORDER BY averageRating * numVotes DESC Unfortunately, DuckDB plugin does't work in the default Alpine based Metabase docker container due to some glibc problems. But it works in the Ubuntu based Metabase docker image. There is Ubuntu based image build script in the docker folder of this project. So, please, run Docker daemon in you host and: ```bash -./build_image.sh +./build_docker_image.sh ``` After a while, it will build the `metabase_duckdb` Ubuntu based image of Metabase with DuckDB plugin. Just run container of this image exposing 3000 port. diff --git a/build_docker_image.sh b/build_docker_image.sh new file mode 100755 index 0000000..716b1e6 --- /dev/null +++ b/build_docker_image.sh @@ -0,0 +1,31 @@ +# DOC +# +# This script builds docker image of Metabase based on Ubuntu. DuckDB pluging is included. +# +# The docker/metabase/bin/docker folder contains a bit modified files from the Metabase source. +# + +# Common +CUR=$PWD + +# Metabase +MB_VSN=v1.44.6 +MB_GIT_URL=https://github.com/metabase/metabase.git +MB_SRC_FOLDER=docker/metabase/source +MB_IMAGE_NAME=ubuntu_metabase +MB_DUCKDB_IMAGE_NAME=metabase_duckdb + +# Clone metabase source code +if [ ! -d "$MB_SRC_FOLDER/$MB_VSN" ] ; then + git clone --depth 1 --branch $MB_VSN $MB_GIT_URL $MB_SRC_FOLDER/$MB_VSN +fi + +# Copy ubuntu based docker files/sh script into source code of Metabase +yes | cp -rf docker/metabase/Dockerfile $MB_SRC_FOLDER/$MB_VSN +yes | cp -rf docker/metabase/bin/docker/* $MB_SRC_FOLDER/$MB_VSN/bin/docker + +# Build the Metabase Ubuntu based docker image +cd $MB_SRC_FOLDER/$MB_VSN && docker build -t $MB_IMAGE_NAME -f Dockerfile . && cd $CUR + +# Build the Metabase image with DuckDB plugin +docker build -t $MB_DUCKDB_IMAGE_NAME -f docker/Dockerfile . diff --git a/build_image.sh b/build_image.sh deleted file mode 100755 index 3aedadf..0000000 --- a/build_image.sh +++ /dev/null @@ -1,24 +0,0 @@ -# Metabase soure code -MB_SRC_FOLDER=docker/metabase/source -MB_GIT_URL=https://github.com/metabase/metabase.git -MB_IMAGE_NAME=ubuntu_metabase -MB_DUCKDB_IMAGE_NAME=metabase_duckdb - -# Clone metabase source code -if [ ! -d "$MB_SRC_FOLDER" ] ; then - git clone $MB_GIT_URL $MB_SRC_FOLDER -else - git -C $MB_SRC_FOLDER fetch - git -C $MB_SRC_FOLDER reset --hard HEAD - git -C $MB_SRC_FOLDER merge origin/master -fi - -# Copy ubuntu based docker files/sh script into source code of Metabase -yes | cp -rf docker/metabase/Dockerfile $MB_SRC_FOLDER -yes | cp -rf docker/metabase/bin/docker/* $MB_SRC_FOLDER/bin/docker - -# Build the Metabase Ubuntu based docker image -docker build -t $MB_IMAGE_NAME -f $MB_SRC_FOLDER/Dockerfile . - -# Build the Metabase image with DuckDB plugin -docker build -t $MB_DUCKDB_IMAGE_NAME -f docker/Dockerfile . diff --git a/deps.edn b/deps.edn old mode 100644 new mode 100755 index c00adc4..3186c24 --- a/deps.edn +++ b/deps.edn @@ -11,13 +11,13 @@ ; replace also the version in metabase-plugin.yaml metabase/metabase-core { :git/url "https://github.com/metabase/metabase.git" - :git/tag "v1.44.3" - :git/sha "7d50282" + :git/tag "v1.44.6" + :git/sha "5c6ae21" } metabase/build-drivers { :git/url "https://github.com/metabase/metabase.git" - :git/tag "v1.44.3" - :git/sha "7d50282" + :git/tag "v1.44.6" + :git/sha "5c6ae21" :deps/root "bin/build-drivers" } org.duckdb/duckdb_jdbc {:mvn/version "0.5.1"} diff --git a/docker/metabase/bin/docker/Dockerfile b/docker/metabase/bin/docker/Dockerfile old mode 100644 new mode 100755 index 6b11d71..39353ee --- a/docker/metabase/bin/docker/Dockerfile +++ b/docker/metabase/bin/docker/Dockerfile @@ -14,12 +14,16 @@ RUN apt-get update && \ /opt/java/openjdk/bin/keytool -noprompt -import -trustcacerts -alias azure-cert -file /app/certs/DigiCertGlobalRootG2.crt.pem -keystore /etc/ssl/certs/java/cacerts -keypass changeit -storepass changeit && \ mkdir -p /plugins && chmod a+rwx /plugins -# add Metabase script and uberjar -COPY --from=builder /home/circleci/target/uberjar/metabase.jar /app/ -COPY bin/docker/run_metabase.sh /app/ +# add Metabase jar & add our run script to the image +COPY ./metabase.jar ./run_metabase.sh /app/ # expose our default runtime port EXPOSE 3000 +# if you have an H2 database that you want to initialize the new Metabase +# instance with, mount it in the container as a volume that will match the +# pattern /app/initial*.db: +# $ docker run ... -v $PWD/metabase.db.mv.db:/app/initial.db.mv.db ... + # run it -ENTRYPOINT ["/app/run_metabase.sh"] \ No newline at end of file +ENTRYPOINT ["/app/run_metabase.sh"] diff --git a/resources/metabase-plugin.yaml b/resources/metabase-plugin.yaml old mode 100644 new mode 100755 index 11844f3..4b038b5 --- a/resources/metabase-plugin.yaml +++ b/resources/metabase-plugin.yaml @@ -1,6 +1,6 @@ info: name: Metabase DuckDB Driver - version: 1.0.0-SNAPSHOT-0.1.2 + version: 1.0.0-SNAPSHOT-0.1.3 description: Allows Metabase to connect to DuckDB databases. contact-info: name: Alexander Golubov diff --git a/src/metabase/driver/duckdb.clj b/src/metabase/driver/duckdb.clj old mode 100644 new mode 100755 index 90e38b0..e898db8 --- a/src/metabase/driver/duckdb.clj +++ b/src/metabase/driver/duckdb.clj @@ -5,7 +5,7 @@ [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] [metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]) - (:import [java.sql ResultSet Types])) + (:import [java.sql Statement ResultSet ResultSetMetaData Types])) (driver/register! :duckdb, :parent :sql-jdbc) @@ -14,9 +14,8 @@ (let [conn_details (merge {:classname "org.duckdb.DuckDBDriver" :subprotocol "duckdb" - :subname (or database_file "") - :read_only true} - (dissoc details :database_file))] + :subname (or database_file "")} + (dissoc details :database_file :port :engine))] conn_details)) (def ^:private database-type->base-type @@ -82,6 +81,26 @@ (fn [] (let [sqlTime (.getObject rs i java.sql.Time)] (.toLocalTime sqlTime)))) +(defn empty-rs [_] ; + (reify + ResultSet + (getMetaData [_] + (reify + ResultSetMetaData + (getColumnCount [_] 1) + (getColumnLabel [_ _idx] "WARNING") + (getColumnTypeName [_ _] "CHAR") + (getColumnType [_ _] Types/CHAR))) + (next [_] false) + (close [_]))) + +;; override native execute-statement! to make queries that does't returns ResultSet +(defmethod sql-jdbc.execute/execute-statement! :sql-jdbc + [_driver ^Statement stmt ^String sql] + (if (.execute stmt sql) + (.getResultSet stmt) + (empty-rs []))) + (defmethod driver/describe-database :duckdb [_ database] {:tables