feat: dual publishing go-ipfs and kubo

Makefile

@@ -2,7 +2,10 @@ export NODE_OPTIONS="--unhandled-rejections=strict"
 
 all: deps releases all_dists site
 
-DISTS = $(notdir $(wildcard dists/*))
+# DISTS are sorted in reverse lex order
+# because 'kubo' MUST build before 'go-ipfs'
+reverse = $(if $1,$(call reverse,$(wordlist 2,999999,$1)) $(firstword $1))
+DISTS = $(call reverse,$(sort $(notdir $(wildcard dists/*))))
 
 NIGHTLY_IGNORED := $(shell cat ignored-during-nightly)
 DISTS_FILTERED = $(filter-out $(NIGHTLY_IGNORED),$(DISTS))

build-go.sh

@@ -46,7 +46,7 @@ function notice() {
 }
 
 # dep checks
-reqbins="jq zip tar go npm"
+reqbins="jq zip unzip tar go npm"
 for b in $reqbins
 do
 	if ! type "$b" > /dev/null; then
@@ -141,12 +141,10 @@ function doBuild() {
 	if ! (cd "$build_dir_name" && goBuild "$package" "$goos" "$goarch") > build-log; then
 		local logfi="$dir/build-log-$goos-$goarch"
 		cp "$build_dir_name/build-log" "$logfi"
-		warn "    failed. logfile at '$logfi'"
+		warn "    $binname failed. logfile at '$logfi'"
 		return 1
 	fi
 
-	notice "    $goos $goarch build succeeded!"
-
 	# copy dist assets if they exist
 	if [ -e "$GOPATH/src/$package/dist" ]; then
 		cp -r "$GOPATH/src/$package/dist/"* "$build_dir_name/"
@@ -156,8 +154,9 @@ function doBuild() {
 	if bundleDist "$dir/$binname" "$goos" "$build_dir_name"; then
 		buildDistInfo "$binname" "$dir"
 		rm -rf "$build_dir_name"
+		notice "    build $binname succeeded!"
 	else
-		warn "    failed to zip up output"
+		warn "    failed to build $binname"
 		success=1
 	fi
 
@@ -454,6 +453,9 @@ function startGoBuilds() {
 	notice "build complete!"
 }
 
+# Execute only when called directly (allows for sourcing)
+if [ "${BASH_SOURCE[0]}" -ef "$0" ]; then
 startGoBuilds "$1" "$2" "$3" "$4" "$5"
+fi
 
-# vim: noet
+# vim: ts=4:noet

dist.sh

@@ -80,6 +80,29 @@ case $1 in
 
 		echo "$nvers" >> "dists/$dist/versions"
 
+		# legacy go-ipfs dist needs to be created for every new kubo release:
+		# https://github.com/ipfs/distributions/pull/717
+		if [ "$dist" == "kubo" ]; then
+			# use the same targets
+			cat "dists/kubo/build_matrix" > "dists/go-ipfs/build_matrix"
+			# make sure latest go-ipfs release follows kubo
+			cat "dists/kubo/current" > "dists/go-ipfs/current"
+			# make sure go-ipfs has all new kubo releases (one directional sync)
+			newreleases="$(mktemp)"
+			diff "dists/kubo/versions" "dists/go-ipfs/versions" | grep '^<' | awk '{print $2}' | uniq > "$newreleases"
+			cat "$newreleases" >> "dists/go-ipfs/versions"
+		fi
+
+		# error on old kubo name
+		if [ "$dist" == "go-ipfs" ]; then
+			echo "ERROR: go-ipfs is now named kubo, use the new name:"
+			echo
+			echo "$ dist.sh add-version kubo <version>"
+			echo
+			echo "(a backward-compatible go-ipfs release will be added automatically)"
+			exit 1
+		fi
+
 		# cd "dists/$dist" && make update_sources
 		# build-go will update sources as needed
 		cd "dists/$dist" && make
@@ -92,3 +115,4 @@ case $1 in
 		exit 1
 		;;
 esac
+## vim: sts=4:ts=4:sw=4:noet

dists/go-ipfs/Makefile

@@ -2,3 +2,8 @@ repo = github.com/ipfs/go-ipfs
 package = cmd/ipfs
 
 include ../../common.mk
+
+# we override dist recipe for go-ipfs to avoid bulding kubo binaries for the second time.
+# instead, we repackage existing kubo artifacts under the legacy go-ipfs name
+dist:
+	${relpath}/dists/go-ipfs/build-from-kubo.sh "${relpath}" "${distname}" "${repo}" "${package}" "${versions}"

dists/go-ipfs/build-from-kubo.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# This script replaces go build with repackaging of existing 'kubo' binaries
+# - go-ipfs is the old name of kubo, and we provide it for legacy reasons
+# - this script assumes kubo artifacts were built recently and are still
+#   in local "releases/kubo/${version}" - this is ok because nobody will build
+#   go-ipfs on their own, only kubo, and go-ipfs build happens automatically
+#   when someone adds new kubo release with './dist add-release kubo <version>'
+
+# Usage in Makefile is drop-in replacement for build-go.sh:
+# build-from-kubo.sh "${relpath}" "${distname}" "${repo}" "${package}" "${versions}"
+
+# path to the root of ipfs/distributions repo
+rootpath="$(realpath "$1")"
+
+distname="$2"
+repo="$3"
+package="$4"
+versions="$5"
+
+# import utility functions and variables from regular go build	script
+source "${rootpath}/build-go.sh"
+
+# override build step.
+# goBuild is skipped and we replaced it with repackaging of existing kubo artifacts
+# this way we build everything only once
+function doBuild() {
+	local goos=$1
+	local goarch=$2
+	local package=$3
+	local output=$4
+	local version=$5
+
+	local dir name binname
+
+	dir="$output"
+	name="$(basename "$(pwd)")"
+	binname="${name}_${version}_${goos}-${goarch}"
+
+	# local dir with just recently built kubo release
+	# that will be repackaged under go-ipfs name
+	kuboreleasedir="${rootpath}/releases/kubo/${version}"
+	kubobinname="kubo_${version}_${goos}-${goarch}"
+
+	echo "==> repackaging kubo to go-ipfs for $goos $goarch"
+
+	if [ -e "$dir/$binname" ]; then
+		echo "	  $dir/$binname exists, skipping build"
+		return
+	fi
+	echo "	  output to $dir/$binname"
+
+	local build_dir_name=$name
+	mkdir -p "$dir"
+
+	# unpack kubo package (it produces 'kubo/ipfs[.exe]')
+	case $(pkgType "$goos") in
+	zip)
+		unzip -oq "${kuboreleasedir}/${kubobinname}.zip"
+		;;
+	tar.gz)
+		tar xf "${kuboreleasedir}/${kubobinname}.tar.gz"
+		;;
+	esac
+
+	# remove any stale unpacked data
+	rm -rf "$build_dir_name"
+
+	# rename extracted directory to match name expected by build scripts (go-ipfs)
+	mv "kubo" "$build_dir_name"
+
+	# now (re)package it all up
+	if bundleDist "$dir/$binname" "$goos" "$build_dir_name"; then
+		buildDistInfo "$binname" "$dir"
+		rm -rf "$build_dir_name"
+		notice "	repackaging of $binname succeeded!"
+	else
+		fail "	  failed to repackage $binname"
+		success=1
+	fi
+
+	notice "	$goos $goarch repackaging succeeded!"
+
+	# output results to results table
+	echo "$target, $goos, $goarch, $success" >> "$output/results"
+}
+
+# run unmodified logic from build-go.sh
+startGoBuilds "${distname}" "${repo}" "${package}" "${versions}"
+# vim: ts=4:noet

dists/go-ipfs/versions

@@ -68,3 +68,4 @@ v0.12.2
 v0.13.0-rc1
 v0.13.0
 v0.13.1
+v0.14.0-rc1

dists/kubo/Makefile

@@ -0,0 +1,4 @@
+repo = github.com/ipfs/kubo
+package = cmd/ipfs
+
+include ../../common.mk

dists/kubo/build_matrix

@@ -0,0 +1,14 @@
+darwin amd64
+darwin arm64
+freebsd 386
+freebsd amd64
+freebsd arm
+openbsd 386
+openbsd amd64
+openbsd arm
+linux 386
+linux amd64
+linux arm
+linux arm64
+windows 386
+windows amd64

dists/kubo/current

@@ -0,0 +1 @@
+v0.14.0-rc1

dists/kubo/description

@@ -0,0 +1 @@
+kubo (go-ipfs) is the earliest and most widely used implementation of IPFS. It includes:\n- an IPFS daemon server\n- extensive command line tooling\n- an HTTP RPC API for controlling the node\n- an HTTP Gateway for serving content to HTTP browsers\n

dists/kubo/repo-name

@@ -0,0 +1 @@
+kubo

dists/kubo/repo-owner

@@ -0,0 +1 @@
+ipfs

dists/kubo/versions

@@ -0,0 +1 @@
+v0.14.0-rc1

ignored-during-nightly

@@ -7,4 +7,5 @@ fs-repo-5-to-6
 fs-repo-6-to-7
 fs-repo-7-to-8
 fs-repo-8-to-9
-fs-repo-9-to-10
+fs-repo-9-to-10
+go-ipfs

site/config.toml

@@ -7,7 +7,7 @@ title = "IPFS Distributions"
 taxonomies = {}
 
 [params]
-hiddenDists = ['gx', 'gx-go']
+hiddenDists = ['go-ipfs', 'gx', 'gx-go']
 newGoIpfsName = 'kubo'
 
 [params.targetMap]

site/layouts/_default/single.html

@@ -31,7 +31,7 @@
               </li>
               {{ range $key, $value := $.Site.Data.releases }}{{ if not (in $.Site.Params.hiddenDists $key) }}
                 <li class="d-sidebar-item nav-item">
-                  <a class="d-sidebar-link nav-link" href="#{{ $key }}">{{ if eq "go-ipfs" $key }}{{ $.Site.Params.newGoIpfsName }} ({{ $key }}){{ else }}{{ $key }}{{ end }}</a>
+                  <a class="d-sidebar-link nav-link" href="#{{ $key }}">{{ if eq "kubo" $key }}kubo <small style="color: var(--gray)">(go-ipfs)</small>{{ else }}{{ $key }}{{ end }}</a>
                 </li>
               {{ end }}{{ end }}
             </ul>
@@ -40,11 +40,11 @@
         <div class="col-md-9">
           {{ range $key, $value := $.Site.Data.releases }}{{ if not (in $.Site.Params.hiddenDists $key) }}
             {{ $data := $value.data }}
-            {{ if eq "go-ipfs" $key }}<div id="{{ $.Site.Params.newGoIpfsName }}"></div>{{ end }}
+            {{ if eq $.Site.Params.newGoIpfsName $key }}<div id="go-ipfs"></div>{{ end }}
             <div class="d-component" id="{{ $key }}">
               <div class="d-component-desc row">
                 <div class="col-md-7">
-                  <h1 class="d-component-desc-title">{{ if eq "go-ipfs" $key }}{{ $.Site.Params.newGoIpfsName }} (old name: {{ $key }}){{ else }}{{ $key }}{{ end }}</h1>
+                  <h1 class="d-component-desc-title">{{ $key }}{{ if eq $.Site.Params.newGoIpfsName $key }} <small style="color: var(--gray)">(go-ipfs)</small>{{ end }}</h1>
                   <h2 class="d-component-desc-sub-title">{{ $data.tagline }}</h2>
                   <div class="d-component-desc-body">{{ $data.description | markdownify }}</div>
                 </div>