chore: Export git repositories as a zip file for Devfile v2

build/dockerfiles/Dockerfile

@@ -32,6 +32,7 @@ RUN apk add --no-cache bash && \
     apk add --no-cache coreutils
 COPY .htaccess README.md /usr/local/apache2/htdocs/
 COPY --from=builder /build/devfiles /usr/local/apache2/htdocs/devfiles
+COPY --from=builder /build/resources /usr/local/apache2/htdocs/resources
 COPY ./images /usr/local/apache2/htdocs/images
 COPY ./build/dockerfiles/entrypoint.sh /usr/bin/
 ENTRYPOINT ["/usr/bin/entrypoint.sh"]

build/dockerfiles/entrypoint.sh

@@ -116,6 +116,7 @@ fi
 # The below command will fail if any path contains whitespace
 readarray -t devfiles < <(find "${DEVFILES_DIR}" -name 'devfile.yaml')
 readarray -t metas < <(find "${DEVFILES_DIR}" -name 'meta.yaml')
+readarray -t templates < <(find "${DEVFILES_DIR}" -name 'devworkspace-che-theia-*.yaml')
 for devfile in "${devfiles[@]}"; do
   echo "Checking devfile $devfile"
   # Need to update each field separately in case they are not defined.
@@ -137,7 +138,7 @@ done
 if [ -n "$PUBLIC_URL" ]; then
   echo "Updating devfiles to point at internal project zip files"
   PUBLIC_URL=${PUBLIC_URL%/}
-  sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${PUBLIC_URL}|" "${devfiles[@]}" "${metas[@]}" "$INDEX_JSON"
+  sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${PUBLIC_URL}|" "${devfiles[@]}" "${metas[@]}" "${templates[@]}" "$INDEX_JSON"
 
   # Add PUBLIC_URL at the begining of 'icon' field and links ('self', 'eclipse/che-theia/latest' and 'eclipse/che-theia/next')
   sed -i "s|\"icon\": \"/images/|\"icon\": \"${PUBLIC_URL}/images/|" "$INDEX_JSON"
@@ -154,7 +155,7 @@ else
     SERVICE_HOST=$(env | grep DEVFILE_REGISTRY_SERVICE_HOST= | cut -d '=' -f 2)
     SERVICE_PORT=$(env | grep DEVFILE_REGISTRY_SERVICE_PORT= | cut -d '=' -f 2)
     URL="http://${SERVICE_HOST}:${SERVICE_PORT}"
-    sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${URL}|" "${devfiles[@]}" "${metas[@]}" "$INDEX_JSON"
+    sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${URL}|" "${devfiles[@]}" "${metas[@]}" "${templates[@]}" "$INDEX_JSON"
   fi
 fi
 

build/scripts/cache_projects.sh

@@ -19,44 +19,9 @@ DEVFILES_DIR="${1%/}"
 RESOURCES_DIR="${2%/}"
 TEMP_DIR="${RESOURCES_DIR}/devfiles_temp"
 TEMP_FILE="${TEMP_DIR}/temp.yaml"
-TEMP_REPO="${TEMP_DIR}/cloned"
 
-# Clone a git repository and create an archive zip at a specified location
-# Args:
-#   $1 - URL of git repo
-#   $2 - branch to archive
-#   $3 - destination path for the archived project zip file
-#   $4 - sparse checkout directory
-#   $5 - commitId
-function cache_project() {
-  local repo="$1"
-  local branch="$2"
-  local destination="$3"
-  local sparse_checkout_dir="$4"
-  local commitId="$5"
-
-  rm -fr "$TEMP_REPO"
-  if [[ ! "$commitId" ]] || [[ "$commitId" == "null" ]]; then
-    git clone "$repo" -b "$branch" --depth 1 "$TEMP_REPO" -q
-  else
-    git clone "$repo" "$TEMP_REPO" -q
-    pushd "$TEMP_REPO" &>/dev/null
-    git reset --hard "${commitId}"
-    popd &>/dev/null
-    # change branch name for the archive
-    branch="HEAD"
-  fi
-
-  pushd "$TEMP_REPO" &>/dev/null
-    if [ -n "$sparse_checkout_dir" ]; then
-      echo "    Using sparse checkout dir '$sparse_checkout_dir'"
-      git archive -9 "$branch" "$sparse_checkout_dir" -o "$destination"
-    else 
-      git archive -9 "$branch" -o "$destination"
-    fi
-  popd &>/dev/null
-  rm -rf "$TEMP_REPO"
-}
+# shellcheck disable=SC1091
+source ./clone_and_zip.sh
 
 # Update devfile to refer to a locally stored zip instead of a public git repo
 # Args:
@@ -143,7 +108,7 @@ for devfile in "${devfiles[@]}"; do
     absolute_destination=$(realpath "$destination")
     # echo "    Caching project to $absolute_destination"
     echo "    Caching project from $location/blob/${branch} to $destination"
-    cache_project "$location" "$branch" "$absolute_destination" "$sparse_checkout_dir" "$commitId"
+    clone_and_zip "$location" "$branch" "$absolute_destination" "$sparse_checkout_dir" "$commitId"
 
     echo "    Updating devfile $devfile to point at cached project zip $destination"
     update_devfile "$devfile" "$project_name" "$destination"

build/scripts/clone_and_zip.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Copyright (c) 2018-2021 Red Hat, Inc.
+# This program and the accompanying materials are made
+# available under the terms of the Eclipse Public License 2.0
+# which is available at https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+
+set -e
+
+TEMP_REPO="/tmp/cloned"
+
+# Clone a git repository and create an archive zip at a specified location
+# Args:
+#   $1 - URL of git repo
+#   $2 - branch to archive
+#   $3 - destination path for the archived project zip file
+#   $4 - sparse checkout directory
+#   $5 - commitId
+function clone_and_zip() {
+  local repo="$1"
+  local branch="$2"
+  local destination="$3"
+  local sparse_checkout_dir="$4"
+  local commitId="$5"
+
+  rm -rf "$TEMP_REPO"
+  if [[ ! "$commitId" ]] || [[ "$commitId" == "null" ]]; then
+    git clone "$repo" -b "$branch" --depth 1 "$TEMP_REPO" -q
+  else
+    git clone "$repo" "$TEMP_REPO" -q
+    pushd "$TEMP_REPO" &>/dev/null
+    git reset --hard "${commitId}"
+    popd &>/dev/null
+    # change branch name for the archive
+    branch="HEAD"
+  fi
+
+  pushd "$TEMP_REPO" &>/dev/null
+    if [ -n "$sparse_checkout_dir" ]; then
+      echo "    Using sparse checkout dir '$sparse_checkout_dir'"
+      git archive -9 "$branch" "$sparse_checkout_dir" -o "$destination"
+    else
+      git archive -9 "$branch" -o "$destination"
+    fi
+  popd &>/dev/null
+  rm -rf "$TEMP_REPO"
+}

build/scripts/generate_devworkspace_templates.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Copyright (c) 2021 Red Hat, Inc.
 # This program and the accompanying materials are made
@@ -8,15 +8,31 @@
 # SPDX-License-Identifier: EPL-2.0
 #
 
-npm install -g @eclipse-che/che-theia-devworkspace-handler@0.0.1-1638274327
-mkdir /build/out/
+set -e
+
+# shellcheck disable=SC1091
+source ./clone_and_zip.sh
+
+npm install -g @eclipse-che/che-theia-devworkspace-handler@0.0.1-1638890256
+mkdir -p /build/resources/v2/
 for dir in /build/devfiles/*/
 do
-  devfile=$(grep "v2:" "${dir}"meta.yaml)
-  if [ -n "$devfile" ]; then
-    devfile=${devfile##*v2: }
-    npx @eclipse-che/che-theia-devworkspace-handler --devfile-url:"${devfile}" --output-file:"${dir}"/devworkspace-che-theia-next.yaml
-    npx @eclipse-che/che-theia-devworkspace-handler --devfile-url:"${devfile}" --editor:eclipse/che-theia/latest \
-    --output-file:"${dir}"/devworkspace-che-theia-latest.yaml
+  devfile_url=$(grep "v2:" "${dir}"meta.yaml) || :
+  if [ -n "$devfile_url" ]; then
+    devfile_url=${devfile_url##*v2: }
+    devfile_url=${devfile_url%/}
+    devfile_repo=${devfile_url%/tree*}
+    name=$(basename "${devfile_repo}")
+
+    npm_config_yes=true npx @eclipse-che/che-theia-devworkspace-handler --devfile-url:"${devfile_url}" \
+    --output-file:"${dir}"/devworkspace-che-theia-next.yaml \
+    --project.$name="{{ DEVFILE_REGISTRY_URL }}/resources/v2/"${name}".zip"
+
+    npm_config_yes=true npx @eclipse-che/che-theia-devworkspace-handler --devfile-url:"${devfile_url}" \
+    --editor:eclipse/che-theia/latest \
+    --output-file:"${dir}"/devworkspace-che-theia-latest.yaml \
+    --project.$name="{{ DEVFILE_REGISTRY_URL }}/resources/v2/"${name}".zip"
+
+    clone_and_zip "${devfile_repo}" "${devfile_url##*/}" "/build/resources/v2/$name.zip"
   fi
 done