chore: Export git repositories as a zip file for Devfile v2

Signed-off-by: Igor Vinokur <ivinokur@redhat.com>

build/dockerfiles/Dockerfile

@@ -20,8 +20,9 @@ RUN ./list_referenced_images.sh devfiles > /build/devfiles/external_images.txt
 RUN chmod -R g+rwX /build/devfiles
 
 FROM docker.io/node:16.13.0-alpine3.14 as dwtemplates
+RUN apk add --no-cache py-pip jq bash git && pip install yq
 COPY ./build/scripts /build/
-COPY ./devfiles /build/devfiles
+COPY ./devfiles /build/source_devfiles
 WORKDIR /build/
 RUN ./generate_devworkspace_templates.sh
 
@@ -37,7 +38,8 @@ RUN apk add --no-cache bash && \
     apk add --no-cache coreutils
 COPY .htaccess README.md /usr/local/apache2/htdocs/
 COPY --from=builder /build/devfiles /usr/local/apache2/htdocs/devfiles
-COPY --from=dwtemplates /build/out /usr/local/apache2/htdocs/devfiles
+COPY --from=dwtemplates /build/devfiles /usr/local/apache2/htdocs/devfiles
+COPY --from=dwtemplates /build/resources /usr/local/apache2/htdocs/resources
 COPY ./images /usr/local/apache2/htdocs/images
 COPY ./build/dockerfiles/entrypoint.sh /usr/bin/
 ENTRYPOINT ["/usr/bin/entrypoint.sh"]

build/dockerfiles/entrypoint.sh

@@ -116,6 +116,7 @@ fi
 # The below command will fail if any path contains whitespace
 readarray -t devfiles < <(find "${DEVFILES_DIR}" -name 'devfile.yaml')
 readarray -t metas < <(find "${DEVFILES_DIR}" -name 'meta.yaml')
+readarray -t templates < <(find "${DEVFILES_DIR}" -name 'devworkspace-che-theia-*.yaml')
 for devfile in "${devfiles[@]}"; do
   echo "Checking devfile $devfile"
   # Need to update each field separately in case they are not defined.
@@ -137,7 +138,7 @@ done
 if [ -n "$PUBLIC_URL" ]; then
   echo "Updating devfiles to point at internal project zip files"
   PUBLIC_URL=${PUBLIC_URL%/}
-  sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${PUBLIC_URL}|" "${devfiles[@]}" "${metas[@]}" "$INDEX_JSON"
+  sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${PUBLIC_URL}|" "${devfiles[@]}" "${metas[@]}" "${templates[@]}" "$INDEX_JSON"
 
   # Add PUBLIC_URL at the begining of 'icon' field and links ('self', 'eclipse/che-theia/latest' and 'eclipse/che-theia/next')
   sed -i "s|\"icon\": \"/images/|\"icon\": \"${PUBLIC_URL}/images/|" "$INDEX_JSON"
@@ -154,7 +155,7 @@ else
     SERVICE_HOST=$(env | grep DEVFILE_REGISTRY_SERVICE_HOST= | cut -d '=' -f 2)
     SERVICE_PORT=$(env | grep DEVFILE_REGISTRY_SERVICE_PORT= | cut -d '=' -f 2)
     URL="http://${SERVICE_HOST}:${SERVICE_PORT}"
-    sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${URL}|" "${devfiles[@]}" "${metas[@]}" "$INDEX_JSON"
+    sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${URL}|" "${devfiles[@]}" "${metas[@]}" "${templates[@]}" "$INDEX_JSON"
   fi
 fi
 

build/scripts/cache_projects.sh

@@ -19,44 +19,8 @@ DEVFILES_DIR="${1%/}"
 RESOURCES_DIR="${2%/}"
 TEMP_DIR="${RESOURCES_DIR}/devfiles_temp"
 TEMP_FILE="${TEMP_DIR}/temp.yaml"
-TEMP_REPO="${TEMP_DIR}/cloned"
 
-# Clone a git repository and create an archive zip at a specified location
-# Args:
-#   $1 - URL of git repo
-#   $2 - branch to archive
-#   $3 - destination path for the archived project zip file
-#   $4 - sparse checkout directory
-#   $5 - commitId
-function cache_project() {
-  local repo="$1"
-  local branch="$2"
-  local destination="$3"
-  local sparse_checkout_dir="$4"
-  local commitId="$5"
-
-  rm -fr "$TEMP_REPO"
-  if [[ ! "$commitId" ]] || [[ "$commitId" == "null" ]]; then
-    git clone "$repo" -b "$branch" --depth 1 "$TEMP_REPO" -q
-  else
-    git clone "$repo" "$TEMP_REPO" -q
-    pushd "$TEMP_REPO" &>/dev/null
-    git reset --hard "${commitId}"
-    popd &>/dev/null
-    # change branch name for the archive
-    branch="HEAD"
-  fi
-
-  pushd "$TEMP_REPO" &>/dev/null
-    if [ -n "$sparse_checkout_dir" ]; then
-      echo "    Using sparse checkout dir '$sparse_checkout_dir'"
-      git archive -9 "$branch" "$sparse_checkout_dir" -o "$destination"
-    else 
-      git archive -9 "$branch" -o "$destination"
-    fi
-  popd &>/dev/null
-  rm -rf "$TEMP_REPO"
-}
+source ./clone_and_zip.sh
 
 # Update devfile to refer to a locally stored zip instead of a public git repo
 # Args:
@@ -143,7 +107,7 @@ for devfile in "${devfiles[@]}"; do
     absolute_destination=$(realpath "$destination")
     # echo "    Caching project to $absolute_destination"
     echo "    Caching project from $location/blob/${branch} to $destination"
-    cache_project "$location" "$branch" "$absolute_destination" "$sparse_checkout_dir" "$commitId"
+    clone_and_zip "$location" "$branch" "$absolute_destination" "$sparse_checkout_dir" "$commitId"
 
     echo "    Updating devfile $devfile to point at cached project zip $destination"
     update_devfile "$devfile" "$project_name" "$destination"

build/scripts/clone_and_zip.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Copyright (c) 2018-2021 Red Hat, Inc.
+# This program and the accompanying materials are made
+# available under the terms of the Eclipse Public License 2.0
+# which is available at https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+
+TEMP_REPO="/tmp/cloned"
+
+# Clone a git repository and create an archive zip at a specified location
+# Args:
+#   $1 - URL of git repo
+#   $2 - branch to archive
+#   $3 - destination path for the archived project zip file
+#   $4 - sparse checkout directory
+#   $5 - commitId
+function clone_and_zip() {
+  local repo="$1"
+  local branch="$2"
+  local destination="$3"
+  local sparse_checkout_dir="$4"
+  local commitId="$5"
+
+  rm -rf "$TEMP_REPO"
+  if [[ ! "$commitId" ]] || [[ "$commitId" == "null" ]]; then
+    git clone "$repo" -b "$branch" --depth 1 "$TEMP_REPO" -q
+  else
+    git clone "$repo" "$TEMP_REPO" -q
+    pushd "$TEMP_REPO" &>/dev/null
+    git reset --hard "${commitId}"
+    popd &>/dev/null
+    # change branch name for the archive
+    branch="HEAD"
+  fi
+
+  pushd "$TEMP_REPO" &>/dev/null
+    if [ -n "$sparse_checkout_dir" ]; then
+      echo "    Using sparse checkout dir '$sparse_checkout_dir'"
+      git archive -9 "$branch" "$sparse_checkout_dir" -o "$destination"
+    else
+      git archive -9 "$branch" -o "$destination"
+    fi
+  popd &>/dev/null
+  rm -rf "$TEMP_REPO"
+}

build/scripts/generate_devworkspace_templates.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Copyright (c) 2021 Red Hat, Inc.
 # This program and the accompanying materials are made
@@ -8,18 +8,58 @@
 # SPDX-License-Identifier: EPL-2.0
 #
 
+source ./clone_and_zip.sh
+
+TEMP_FILE="${TEMP_DIR}/temp.yaml"
+
+# Update devfile template with a locally stored zip in a project section
+# Args:
+#   $1 - path to devfile template to update
+#   $2 - name of project to update within devfile template
+#   $3 - path to project zip
+function update_template() {
+  local devfile="$1"
+  local project_name="$2"
+  local zip_path="$3"
+  # The yq script below will set the project with $project_name to be a zip-type
+  # project with provided path. The location field contains a placeholder
+  # '{{ DEVFILE_REGISTRY_URL }}' which must be filled at runtime (see
+  # build/dockerfiles/entrypoint.sh script)
+  # shellcheck disable=SC2016
+  yq -y \
+  '.projects |= . + [
+    {
+      zip: {
+        location: "{{ DEVFILE_REGISTRY_URL }}\($PROJECT_PATH)"
+      },
+      name: $PROJECT_NAME
+    }
+  ]' "$devfile" \
+  --arg "PROJECT_NAME" "${project_name}" \
+  --arg "PROJECT_PATH" "${zip_path}" \
+  > "$TEMP_FILE"
+  echo "    Copying $TEMP_FILE -> $devfile"
+  mv "$TEMP_FILE" "$devfile"
+}
+
 npm install -g @eclipse-che/che-theia-devworkspace-handler@0.0.1-1637592995
-mkdir /build/out/
-for dir in /build/devfiles/*/
+mkdir /build/devfiles/
+mkdir -p /build/resources/v2/
+for dir in /build/source_devfiles/*/
 do
   devfile=$(grep "v2:" "${dir}"meta.yaml)
   if [ -n "$devfile" ]; then
     devfile=${devfile##*v2: }
+    devfile=${devfile%/}
     dir=${dir%/}
-    dir=/build/out/${dir##*/}
-    mkdir "${dir}"
+    name=${dir##*/}
+    dir=/build/devfiles/$name
+    mkdir "$dir"
     npx @eclipse-che/che-theia-devworkspace-handler --devfile-url:"${devfile}" --output-file:"${dir}"/devworkspace-che-theia-next.yaml
     npx @eclipse-che/che-theia-devworkspace-handler --devfile-url:"${devfile}" --editor:eclipse/che-theia/latest \
     --output-file:"${dir}"/devworkspace-che-theia-latest.yaml
+    clone_and_zip "${devfile%/tree*}.git" "${devfile##*/}" "/build/resources/v2/$name.zip"
+    update_template "${dir}"/devworkspace-che-theia-next.yaml "$name" "/resources/v2/$name.zip"
+    update_template "${dir}"/devworkspace-che-theia-latest.yaml "$name" "/resources/v2/$name.zip"
   fi
 done