feat: avoid dual build, repackage kubo as go-ipfs

See comments at the top of dists/go-ipfs/build-from-kubo.sh

Makefile

@@ -2,7 +2,10 @@ export NODE_OPTIONS="--unhandled-rejections=strict"
 
 all: deps releases all_dists site
 
-DISTS = $(notdir $(wildcard dists/*))
+# DISTS are sorted in reverse lex order
+# because 'kubo' MUST build before 'go-ipfs'
+reverse = $(if $1,$(call reverse,$(wordlist 2,999999,$1)) $(firstword $1))
+DISTS = $(call reverse,$(sort $(notdir $(wildcard dists/*))))
 
 NIGHTLY_IGNORED := $(shell cat ignored-during-nightly)
 DISTS_FILTERED = $(filter-out $(NIGHTLY_IGNORED),$(DISTS))

build-go.sh

@@ -46,7 +46,7 @@ function notice() {
 }
 
 # dep checks
-reqbins="jq zip tar go npm"
+reqbins="jq zip unzip tar go npm"
 for b in $reqbins
 do
 	if ! type "$b" > /dev/null; then
@@ -141,12 +141,10 @@ function doBuild() {
 	if ! (cd "$build_dir_name" && goBuild "$package" "$goos" "$goarch") > build-log; then
 		local logfi="$dir/build-log-$goos-$goarch"
 		cp "$build_dir_name/build-log" "$logfi"
-		warn "    failed. logfile at '$logfi'"
+		warn "    $binname failed. logfile at '$logfi'"
 		return 1
 	fi
 
-	notice "    $goos $goarch build succeeded!"
-
 	# copy dist assets if they exist
 	if [ -e "$GOPATH/src/$package/dist" ]; then
 		cp -r "$GOPATH/src/$package/dist/"* "$build_dir_name/"
@@ -156,8 +154,9 @@ function doBuild() {
 	if bundleDist "$dir/$binname" "$goos" "$build_dir_name"; then
 		buildDistInfo "$binname" "$dir"
 		rm -rf "$build_dir_name"
+		notice "    build $binname succeeded!"
 	else
-		warn "    failed to zip up output"
+		warn "    failed to build $binname"
 		success=1
 	fi
 
@@ -454,6 +453,9 @@ function startGoBuilds() {
 	notice "build complete!"
 }
 
+# Execute only when called directly (allows for sourcing)
+if [ "${BASH_SOURCE[0]}" -ef "$0" ]; then
 startGoBuilds "$1" "$2" "$3" "$4" "$5"
+fi
 
-# vim: noet
+# vim: ts=4:noet

dist.sh

@@ -86,7 +86,9 @@ case $1 in
 			# make sure latest go-ipfs release follows kubo
 			cat "dists/kubo/current" > "dists/go-ipfs/current"
 			# make sure go-ipfs has all new kubo releases (one directional sync)
-			diff "dists/kubo/versions" "dists/go-ipfs/versions" | grep '^<' | awk '{print $2}' | uniq >> "dists/go-ipfs/versions"
+			newreleases="$(mktemp)"
+			diff "dists/kubo/versions" "dists/go-ipfs/versions" | grep '^<' | awk '{print $2}' | uniq > "$newreleases"
+			cat "$newreleases" >> "dists/go-ipfs/versions"
 		fi
 
 		# error on old kubo name

dists/go-ipfs/Makefile

@@ -2,3 +2,8 @@ repo = github.com/ipfs/go-ipfs
 package = cmd/ipfs
 
 include ../../common.mk
+
+# we override dist recipe for go-ipfs to avoid bulding kubo binaries for the second time.
+# instead, we repackage existing kubo artifacts under the legacy go-ipfs name
+dist:
+	${relpath}/dists/go-ipfs/build-from-kubo.sh "${relpath}" "${distname}" "${repo}" "${package}" "${versions}"

dists/go-ipfs/build-from-kubo.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# This script replaces go build with repackaging of existing 'kubo' binaries
+# - go-ipfs is the old name of kubo, and we provide it for legacy reasons
+# - this script assumes kubo artifacts were built recently and are still
+#   in local "releases/kubo/${version}" - this is ok because nobody will build
+#   go-ipfs on their own, only kubo, and go-ipfs build happens automatically
+#   when someone adds new kubo release with './dist add-release kubo <version>'
+
+# Usage in Makefile is drop-in replacement for build-go.sh:
+# build-from-kubo.sh "${relpath}" "${distname}" "${repo}" "${package}" "${versions}"
+
+# path to the root of ipfs/distributions repo
+rootpath="$(realpath "$1")"
+
+distname="$2"
+repo="$3"
+package="$4"
+versions="$5"
+
+# import utility functions and variables from regular go build	script
+source "${rootpath}/build-go.sh"
+
+# override build step.
+# goBuild is skipped and we replaced it with repackaging of existing kubo artifacts
+# this way we build everything only once
+function doBuild() {
+	local goos=$1
+	local goarch=$2
+	local package=$3
+	local output=$4
+	local version=$5
+
+	local dir name binname
+
+	dir="$output"
+	name="$(basename "$(pwd)")"
+	binname="${name}_${version}_${goos}-${goarch}"
+
+	# local dir with just recently built kubo release
+	# that will be repackaged under go-ipfs name
+	kuboreleasedir="${rootpath}/releases/kubo/${version}"
+	kubobinname="kubo_${version}_${goos}-${goarch}"
+
+	echo "==> repackaging kubo to go-ipfs for $goos $goarch"
+
+	if [ -e "$dir/$binname" ]; then
+		echo "	  $dir/$binname exists, skipping build"
+		return
+	fi
+	echo "	  output to $dir/$binname"
+
+	local build_dir_name=$name
+	mkdir -p "$dir"
+
+	# unpack kubo package (it produces 'kubo/ipfs[.exe]')
+	case $(pkgType "$goos") in
+	zip)
+		unzip -oq "${kuboreleasedir}/${kubobinname}.zip"
+		;;
+	tar.gz)
+		tar xf "${kuboreleasedir}/${kubobinname}.tar.gz"
+		;;
+	esac
+
+	# remove any stale unpacked data
+	rm -rf "$build_dir_name"
+
+	# rename extracted directory to match name expected by build scripts (go-ipfs)
+	mv "kubo" "$build_dir_name"
+
+	# now (re)package it all up
+	if bundleDist "$dir/$binname" "$goos" "$build_dir_name"; then
+		buildDistInfo "$binname" "$dir"
+		rm -rf "$build_dir_name"
+		notice "	repackaging of $binname succeeded!"
+	else
+		fail "	  failed to repackage $binname"
+		success=1
+	fi
+
+	notice "	$goos $goarch repackaging succeeded!"
+
+	# output results to results table
+	echo "$target, $goos, $goarch, $success" >> "$output/results"
+}
+
+# run unmodified logic from build-go.sh
+startGoBuilds "${distname}" "${repo}" "${package}" "${versions}"
+# vim: ts=4:noet

dists/kubo/versions

@@ -1 +1,2 @@
 v0.13.0
+v0.14.0