[POP-7131] Add dbt-full docker image

timescale · Sep 13, 2023 · f83c4f0 · f83c4f0
1 parent d16f9a3
commit f83c4f0
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 26 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -42,17 +42,30 @@ jobs:
       # snowflake adapter for dbt < 1.1 does not have prebuilt arm64 wheels, and building it
       # from source is very ardious and time consuming (due to pyarrow), so only build for
       # amd64 for those versions
-      - name: Set platforms
-        id: platforms
+      - name: Set build properties
+        id: properties
         run: |
           version=${{ matrix.version }}
-          if [ "${{ matrix.adapter }}" = "snowflake" ] && ([ "${version:0:1}" = "0" ] || [ "${version}" = "1.0" ]); then
+          adapter=${{ matrix.adapter }}
+          if ([ -z ${adapter} ] || [ "${adapter}" = "snowflake" ]) && ([ "${version:0:1}" = "0" ] || [ "${version}" = "1.0" ]); then
             platform="linux/amd64"
           else
             platform="linux/amd64,linux/arm64"
           fi
+          if [ -z ${adapter} ]; then
+            image_name="dbt-full"
+            requirements_file="requirements.txt"
+          else
+            image_name="dbt-${adapter}"
+            requirements_file="requirements-${adapter}.txt"
+          fi
+
           echo "platforms=${platform}"
+          echo "image_name=${image_name}"
+          echo "requirements_file=${requirements_file}"
           echo "platforms=${platform}" >> $GITHUB_OUTPUT
+          echo "image_name=${image_name}" >> $GITHUB_OUTPUT
+          echo "requirements_file=${requirements_file}" >> $GITHUB_OUTPUT
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -71,9 +84,9 @@ jobs:
         uses: docker/build-push-action@v3
         with:
           context: .
-          platforms: ${{ steps.platforms.outputs.platforms }}
+          platforms: ${{ steps.properties.outputs.platforms }}
           push: true
-          tags: ghcr.io/popsql/dbt-${{ matrix.adapter }}:${{ matrix.version }}
+          tags: ghcr.io/popsql/${{ steps.properties.outputs.image_name }}:${{ matrix.version }}
           build-args: |
-            DBT_ADAPTER=${{ matrix.adapter }}
+            REQUIREMENTS_FILE=${{ steps.properties.outputs.requirements_file }}
             DBT_VERSION=${{ matrix.version }}
diff --git a/Dockerfile b/Dockerfile
@@ -13,10 +13,10 @@ RUN apt-get update \
   && groupadd -f -g ${GID} -r dbt && useradd -g dbt -l -m -r -u ${UID} dbt \
   && python3 -m pip install -U wheel
 
+ARG REQUIREMENTS_FILE
 ARG DBT_VERSION
-ARG DBT_ADAPTER
 
-COPY requirements/${DBT_VERSION}/requirements-${DBT_ADAPTER}.txt /tmp/requirements.txt
+COPY requirements/${DBT_VERSION}/${REQUIREMENTS_FILE} /tmp/requirements.txt
 
 RUN  python3 -m pip install -r /tmp/requirements.txt && rm -f /tmp/requirements.txt
 

diff --git a/bin/build.sh b/bin/build.sh
@@ -15,8 +15,15 @@ adapter=$2
 
 "${SCRIPT_DIR}"/generate_requirements.sh "${version}" "${adapter}"
 
+if [ -z ${adapter} ]; then
+    requirements_file="requirements.txt"
+    image_name="dbt-full"
+else
+    requirements_file="requirements-${adapter}.txt"
+    image_name="dbt-${adapter}"
+fi
+
 docker build \
-    --build-arg DBT_VERSION="${version}" \
-    --build-arg DBT_ADAPTER="${adapter}" \
-    --tag "ghcr.io/popsql/dbt-${adapter}:${version}" \
+    --build-arg REQUIREMENTS_FILE="${requirements_file}" \
+    --tag "ghcr.io/popsql/${image_name}:${version}" \
     "${BASE_DIR}"
diff --git a/bin/generate_matrix.sh b/bin/generate_matrix.sh
@@ -14,15 +14,14 @@ for i in "${!versions[@]}"; do
     echo -n ', '
   fi
 
+  echo -n "{\"version\": \"${version}\", \"adapter\": \"\"}"
+
   requirements_dir="${BASE_REQUIREMENTS_DIR}/${version}"
   # shellcheck disable=SC2207
   adapters=($(grep "^dbt-*" "${requirements_dir}"/pyproject.toml | grep -v "core" | grep -v "rpc" | perl -p -e 's/^dbt-([a-z0-9]*).*$/\1/'))
   for j in "${!adapters[@]}"; do
-    if [ "${j}" -gt 0 ]; then
-      echo -n ', '
-    fi
     adapter=${adapters[$j]}
-    echo -n "{\"version\": \"${version}\", \"adapter\": \"${adapter}\"}"
+    echo -n ",{\"version\": \"${version}\", \"adapter\": \"${adapter}\"}"
   done
 done
 

diff --git a/bin/generate_requirements.sh b/bin/generate_requirements.sh
@@ -26,21 +26,30 @@ if [ ! -d "${requirements_dir}" ]; then
   exit 1
 fi
 
-if grep -Fxq "$adapter" "${requirements_dir}/pyproject.toml"; then
-  echo "Adapter ${adapter} for ${version} not found"
-  exit 1
-fi
+tmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'docker-dbt')
+
+if [ -z "${adapter}" ]; then
+  echo "Building requirements for ${version}"
+  cp "${requirements_dir}/pyproject.toml" "${tmpdir}/pyproject.toml"
+  requirements_file="requirements.txt"
+else
+  if grep -Fxq "$adapter" "${requirements_dir}/pyproject.toml"; then
+    echo "Adapter ${adapter} for ${version} not found"
+    exit 1
+  fi
 
-echo "Building requirements for ${version}/${adapter}"
+  echo "Building requirements for ${version}/${adapter}"
+
+  # note, we break ${adapter} out of the single quotes into its own double
+  # quotes section, as we want it to be expanded, but we also don't want
+  # bash expansion on the other parts of the string
+  perl -p -e 's/^dbt-(?!core|'"${adapter}"'|rpc).*\n$//' "${requirements_dir}"/pyproject.toml > "${tmpdir}"/pyproject.toml
+  requirements_file="requirements-${adapter}.txt"
+fi
 
-tmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'docker-dbt')
-# note, we break ${adapter} out of the single quotes into its own double
-# quotes section, as we want it to be expanded, but we also don't want
-# bash expansion on the other parts of the string
-perl -p -e 's/^dbt-(?!core|'"${adapter}"'|rpc).*\n$//' "${requirements_dir}"/pyproject.toml > "${tmpdir}"/pyproject.toml
 cp "${requirements_dir}"/poetry.lock "${tmpdir}"/poetry.lock
 pushd "${tmpdir}" > /dev/null
 poetry -q lock
-poetry export -o "${requirements_dir}/requirements-${adapter}".txt
+poetry export -o "${requirements_dir}/${requirements_file}"
 popd > /dev/null
 rm -rf "${tmpdir}"