feat(script): generate tpch data set

.gitignore

@@ -57,3 +57,6 @@ venv/
 __pycache__/
 
 *.zip
+
+# tpch data set
+benchmark/tpch/data

benchmark/tpch/README.md

@@ -0,0 +1,11 @@
+# Databend TPCH-Benchmark
+
+### TPCH DataSet
+Run the following command to generate tpch dataset:
+```shell
+# scale_factor: scale of the database population. scale 1.0 represents ~1 GB of data
+../../scripts/setup/dev_setup.sh -t <scale_factor>
+```
+
+### TPCH Benchmark
+**TBD**

scripts/setup/dev_setup.sh

@@ -329,6 +329,7 @@ function usage {
         -d Install development tools
         -p Install profile
         -s Install codegen tools
+        -t Install tpch data set
         -v Verbose mode
 EOF
 }
@@ -353,6 +354,7 @@ Build tools (since -b or no option was provided):
   * protobuf-compiler
   * thrift-compiler
   * openjdk
+  * tpch dataset for benchmark
 EOF
 	fi
 
@@ -379,6 +381,12 @@ Moreover, ~/.profile will be updated (since -p was provided).
 EOF
 	fi
 
+  if [[ "$INSTALL_TPCH_DATA" == "true" ]]; then
+    cat <<EOF
+Tpch dataset (since -t was provided):
+EOF
+  fi
+
 	cat <<EOF
 If you'd prefer to install these dependencies yourself, please exit this script
 now with Ctrl-C.
@@ -391,9 +399,10 @@ INSTALL_BUILD_TOOLS=false
 INSTALL_DEV_TOOLS=false
 INSTALL_PROFILE=false
 INSTALL_CODEGEN=false
+INSTALL_TPCH_DATA=false
 
 # parse args
-while getopts "ybdpsv" arg; do
+while getopts "ybdpstv" arg; do
 	case "$arg" in
 	y)
 		AUTO_APPROVE="true"
@@ -413,6 +422,10 @@ while getopts "ybdpsv" arg; do
 	v)
 		VERBOSE="true"
 		;;
+  t)
+    INSTALL_TPCH_DATA="true"
+    ;;
+
 	*)
 		usage
 		exit 0
@@ -427,6 +440,7 @@ fi
 if [[ "$INSTALL_BUILD_TOOLS" == "false" ]] &&
 	[[ "$INSTALL_DEV_TOOLS" == "false" ]] &&
 	[[ "$INSTALL_PROFILE" == "false" ]] &&
+	[[ "$INSTALL_TPCH_DATA" == "false" ]] &&
 	[[ "$INSTALL_CODEGEN" == "false" ]]; then
 	INSTALL_BUILD_TOOLS="true"
 fi
@@ -559,6 +573,23 @@ if [[ "$INSTALL_CODEGEN" == "true" ]]; then
 	"${PRE_COMMAND[@]}" python3 -m pip install --quiet coscmd PyYAML
 fi
 
+if [[ "$INSTALL_TPCH_DATA" == "true" ]]; then
+  # Construct a docker imagine to generate tpch-data
+  if [[ -z $2 ]]; then
+    docker build  -f scripts/setup/tpchdata.dockerfile -t databend:latest .
+  else
+    docker build  -f scripts/setup/tpchdata.dockerfile -t databend:latest --build-arg scale_factor=$2 .
+  fi
+  # Generate data into the ./data directory if it does not already exist
+  FILE=benchmark/tpch/data/customer.tbl
+  if test -f "$FILE"; then
+      echo "$FILE exists."
+  else
+    mkdir `pwd`/benchmark/tpch/data 2>/dev/null
+    docker run -v `pwd`/benchmark/tpch/data:/data --rm databend:latest
+  fi
+fi
+
 [[ "${AUTO_APPROVE}" == "false" ]] && cat <<EOF
 Finished installing all dependencies.
 

scripts/setup/run-tpch-dbgen.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cd /tpch-dbgen
+./dbgen -vf -s $1
+mv *.tbl /data

scripts/setup/tpchdata.dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:22.04
+ARG scale_factor=1
+ENV scale_factor=$scale_factor
+RUN apt-get update && \
+    apt-get install -y git build-essential
+
+# Use https://github.com/databricks/tpch-dbgen to generate data
+RUN git clone https://github.com/databricks/tpch-dbgen.git && cd tpch-dbgen && make
+
+WORKDIR /tpch-dbgen
+ADD scripts/setup/run-tpch-dbgen.sh /tpch-dbgen/
+
+VOLUME /data
+
+SHELL ["/bin/bash", "-c"]
+RUN chmod +x run-tpch-dbgen.sh
+ENTRYPOINT ./run-tpch-dbgen.sh $scale_factor