Add BB downloading for OpenBLAS

.travis.yml

@@ -76,8 +76,8 @@ before_install:
         brew tap staticfloat/julia > /dev/null;
         brew rm --force $(brew deps --HEAD julia);
         brew install -v ccache gcc gmp mpfr pcre2 staticfloat/julia/openblas-julia staticfloat/julia/suite-sparse-julia staticfloat/juliadeps/libgfortran;
-        BUILDOPTS="-j3 USECLANG=1 USECCACHE=1 BINARYBUILDER_TRIPLET=x86_64-apple-darwin14 BINARYBUILDER_LLVM_ASSERTS=1";
-        BUILDOPTS="$BUILDOPTS USE_BINARYBUILDER_LLVM=1 LLVM_CONFIG=$TRAVIS_BUILD_DIR/usr/tools/llvm-config LLVM_SIZE=$TRAVIS_BUILD_DIR/usr/tools/llvm-size";
+        BUILDOPTS="-j3 USECLANG=1 USECCACHE=1 USE_BINARYBUILDER_LLVM=1 USE_BINARYBUILDER_OPENBLAS=1 BINARYBUILDER_LLVM_ASSERTS=1";
+        BUILDOPTS="$BUILDOPTS LLVM_CONFIG=$TRAVIS_BUILD_DIR/usr/tools/llvm-config LLVM_SIZE=$TRAVIS_BUILD_DIR/usr/tools/llvm-size";
         BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1";
         BUILDOPTS="$BUILDOPTS LIBBLAS=-lopenblas LIBBLASNAME=libopenblas LIBLAPACK=-lopenblas LIBLAPACKNAME=libopenblas";
         for lib in SUITESPARSE BLAS LAPACK GMP MPFR LIBUNWIND; do

Make.inc

@@ -232,11 +232,10 @@ INSTALL_F := $(JULIAHOME)/contrib/install.sh 644
 INSTALL_M := $(JULIAHOME)/contrib/install.sh 755
 
 # BinaryBuilder options
-# TODO: Autodiscover triplet
+USE_BINARYBUILDER_OPENBLAS := 0
 USE_BINARYBUILDER_LLVM := 0
 # Use the Assertions build
 BINARYBUILDER_LLVM_ASSERTS := 0
-BINARYBUILDER_TRIPLET :=
 
 # LLVM Options
 LLVMROOT := $(build_prefix)

contrib/fixup-libgfortran.sh

@@ -2,6 +2,7 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
 # Run as: fixup-libgfortran.sh [--verbose] <$private_libdir>
+FC=${FC:-gfortran}
 
 # If we're invoked with "--verbose", create a `debug` function that prints stuff out
 if [ "$1" = "--verbose" ] || [ "$1" = "-v" ]; then
@@ -22,7 +23,7 @@ if [ "$UNAME" = "Linux" ]; then
 elif [ "$UNAME" = "Darwin" ]; then
     SHLIB_EXT="dylib"
 else
-    echo "WARNING: Could not autodetect platform type ('uname -s' == $UNAME); assuming Linux" >&2
+    echo "WARNING: Could not autodetect platform type ('uname -s' = $UNAME); assuming Linux" >&2
     UNAME="Linux"
     SHLIB_EXT="so"
 fi
@@ -41,6 +42,20 @@ find_shlib()
     fi
 }
 
+find_shlib_dir()
+{
+    # Usually, on platforms like OSX we get full paths when linking.  However,
+    # if we are inspecting, say, BinaryBuilder-built OpenBLAS libraries, we will
+    # only get something like `@rpath/libgfortran.5.dylib` when inspecting the
+    # libraries.  We can, as a last resort, ask `$FC` directly what the full
+    # filepath for this library is, but only if we don't have a direct path to it:
+    if [ $(dirname "$1") = "@rpath" ]; then
+        dirname "$($FC -print-file-name="$(basename "$1")" 2>/dev/null)"
+    else
+        dirname "$1" 2>/dev/null
+    fi
+}
+
 # First, discover all the places where libgfortran/libgcc is, as well as their true SONAMES
 for lib in lapack blas openblas; do
     for private_libname in ${private_libdir}/lib$lib*.$SHLIB_EXT*; do
@@ -51,10 +66,11 @@ for lib in lapack blas openblas; do
         LIBQUADMATH_PATH=$(find_shlib "$private_libname" libquadmath)
 
         # Take the directories, add them onto LIBGFORTRAN_DIRS, which we use to
-        # search for these libraries in the future.
-        LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(dirname $LIBGFORTRAN_PATH 2>/dev/null)"
-        LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(dirname $LIBGCC_PATH 2>/dev/null)"
-        LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(dirname $LIBQUADMATH_PATH 2>/dev/null)"
+        # search for these libraries in the future.  If there is no directory, try
+        # asking `$FC` where such a file could be found.
+        LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(find_shlib_dir $LIBGFORTRAN_PATH)"
+        LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(find_shlib_dir $LIBGCC_PATH)"
+        LIBGFORTRAN_DIRS="$LIBGFORTRAN_DIRS $(find_shlib_dir $LIBQUADMATH_PATH)"
 
         # Save the SONAMES
         LIBGFORTRAN_SONAMES="$LIBGFORTRAN_SONAMES $(basename "$LIBGFORTRAN_PATH")"

contrib/normalize_triplet.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+
+import re, sys
+
+# This script designed to mimick `src/PlatformNames.jl` in `BinaryProvider.jl`, which has
+# a method `platform_key_abi()` to parse uname-like output into something standarized.
+
+if len(sys.argv) < 2:
+    print("Usage: %s <host triplet> [<gcc version>]")
+    sys.exit(1)
+
+arch_mapping = {
+    'x86_64': '(x86_|amd)64',
+    'i686': "i\\d86",
+    'aarch64': "aarch64",
+    'arm': "arm(v7l)?",
+    'powerpc64le': "p(ower)?pc64le",
+}
+platform_mapping = {
+    'darwin': "-apple-darwin[\\d\\.]*",
+    'freebsd': "-(.*-)?freebsd[\\d\\.]*",
+    'windows': "-w64-mingw32",
+    'linux': "-(.*-)?linux",
+}
+libc_mapping = {
+    'blank_libc': "",
+    'gnu': "-gnu",
+    'musl': "-musl",
+}
+call_abi_mapping = {
+    'blank_call_abi': "",
+    'eabihf': "eabihf",
+}
+gcc_version_mapping = {
+    'blank_gcc': "",
+    'gcc4': "-gcc4",
+    'gcc7': "-gcc7",
+    'gcc8': "-gcc8",
+}
+cxx_abi_mapping = {
+    'blank_cxx_abi': "",
+    'cxx03': "-cxx03",
+    'cxx11': "-cxx11",
+}
+
+# Helper function to collapse dictionary of mappings down into a regex of
+# named capture groups joined by "|" operators
+c = lambda mapping: "("+"|".join(["(?P<%s>%s)"%(k,v) for (k, v) in mapping.items()]) + ")"
+mondo_regex = re.compile(
+    "^"+
+    c(arch_mapping)+
+    c(platform_mapping)+
+    c(libc_mapping)+
+    c(call_abi_mapping)+
+    c(gcc_version_mapping)+
+    c(cxx_abi_mapping)+
+    "$"
+)
+
+# Apply our mondo regex to our input:
+m = mondo_regex.match(sys.argv[1])
+if m is None:
+    print("ERROR: Unmatchable platform string '%s'!"%(sys.argv[1]))
+    sys.exit(1)
+
+# Helper function to find the single named field within the giant regex
+# that is not `nothing` for each mapping we give it.
+def get_field(m, mapping):
+    g = m.groupdict()
+    for k in mapping:
+        if g[k] is not None:
+            return k
+
+arch = get_field(m, arch_mapping)
+platform = get_field(m, platform_mapping)
+libc = get_field(m, libc_mapping)
+call_abi = get_field(m, call_abi_mapping)
+gcc_version = get_field(m, gcc_version_mapping)
+cxx_abi = get_field(m, cxx_abi_mapping)
+
+def r(x):
+    x = x.replace("blank_call_abi", "")
+    x = x.replace("blank_gcc", "")
+    x = x.replace("blank_cxx_abi", "")
+    x = x.replace("blank_libc", "")
+    return x
+
+def p(x):
+    # These contain characters that can't be easily represented as
+    # capture group names, unfortunately:
+    os_remapping = {
+        'darwin': 'apple-darwin14',
+        'windows': 'w64-mingw32',
+        'freebsd': 'unknown-freebsd11.1',
+    }
+    x = r(x)
+    if x:
+        for k in os_remapping:
+            x = x.replace(k, os_remapping[k])
+        return '-' + x
+    return x
+
+# If the user passes in a GCC version (like 8.2.0) use that to force a
+# "-gcc8" tag at the end of the triplet, but only if it has otherwise
+# not been specified
+if gcc_version == "blank_gcc":
+    if len(sys.argv) == 3:
+        gcc_version = {
+            "4": "gcc4",
+            "5": "gcc4",
+            "6": "gcc4",
+            "7": "gcc7",
+            "8": "gcc8",
+        }[sys.argv[2][0]]
+
+
+print(arch+p(platform)+p(libc)+r(call_abi)+p(gcc_version)+p(cxx_abi))
+
+# Testing suite:
+# triplets="i686-w64-mingw32 x86_64-pc-linux-musl arm-linux-musleabihf x86_64-linux-gnu arm-linux-gnueabihf x86_64-apple-darwin14 x86_64-unknown-freebsd11.1"
+# for t in $triplets; do
+#     if [[ $(./normalize_triplet.py "$t") != "$t" ]]; then
+#         echo "ERROR: Failed test on $t"
+#     fi
+# done

contrib/windows/appveyor_build.sh

@@ -45,15 +45,13 @@ if [ "$ARCH" = x86_64 ]; then
   echo 'USE_BLAS64 = 1' >> Make.user
   echo 'LIBBLAS = -L$(JULIAHOME)/usr/bin -lopenblas64_' >> Make.user
   echo 'LIBBLASNAME = libopenblas64_' >> Make.user
-  echo 'BINARYBUILDER_TRIPLET = x86_64-w64-mingw32' >> Make.user
 else
   bits=32
   archsuffix=86
   exc=sjlj
   echo "override MARCH = pentium4" >> Make.user
   echo 'LIBBLAS = -L$(JULIAHOME)/usr/bin -lopenblas' >> Make.user
   echo 'LIBBLASNAME = libopenblas' >> Make.user
-  echo 'BINARYBUILDER_TRIPLET = i686-w64-mingw32' >> Make.user
 fi
 echo "override JULIA_CPU_TARGET=generic;native" >> Make.user
 
@@ -198,6 +196,7 @@ if [ -n "$USEMSVC" ]; then
 else
   # Use BinaryBuilder
   echo 'USE_BINARYBUILDER_LLVM = 1' >> Make.user
+  echo 'USE_BINARYBUILDER_OPENBLAS = 1' >> Make.user
   echo 'BINARYBUILDER_LLVM_ASSERTS = 1' >> Make.user
   echo 'override DEP_LIBS += llvm openlibm' >> Make.user
   export CCACHE_DIR=/cygdrive/c/ccache

deps/Makefile

@@ -12,6 +12,7 @@ include $(SRCDIR)/Versions.make
 include $(JULIAHOME)/Make.inc
 include $(SRCDIR)/tools/common.mk
 include $(SRCDIR)/tools/git-external.mk
+include $(SRCDIR)/tools/bb-install.mk
 
 # Special comments:
 #

deps/blas.mk

@@ -92,6 +92,8 @@ endif
 # Do not overwrite the "-j" flag
 OPENBLAS_BUILD_OPTS += MAKE_NB_JOBS=0
 
+ifneq ($(USE_BINARYBUILDER_OPENBLAS), 1)
+
 $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/build-configured: $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/source-extracted
 	perl -i -ple 's/^\s*(EXTRALIB\s*\+=\s*-lSystemStubs)\s*$$/# $$1/g' $(dir $<)/Makefile.system
 	echo 1 > $@
@@ -194,3 +196,21 @@ configure-lapack: extract-lapack
 compile-lapack: $(BUILDDIR)/lapack-$(LAPACK_VER)/build-compiled
 fastcheck-lapack: check-lapack
 check-lapack: $(BUILDDIR)/lapack-$(LAPACK_VER)/build-checked
+
+else # USE_BINARYBUILDER_OPENBLAS
+
+
+OPENBLAS_BB_URL_BASE := https://github.com/JuliaPackaging/Yggdrasil/releases/download/OpenBLAS-v$(OPENBLAS_VER)-$(OPENBLAS_BB_REL)
+OPENBLAS_BB_NAME := OpenBLAS.v$(OPENBLAS_VER)-$(OPENBLAS_BB_REL)
+
+$(eval $(call bb-install,openblas,OPENBLAS,true))
+get-lapack: get-openblas
+extract-lapack: extract-openblas
+configure-lapack: configure-openblas
+compile-lapack: compile-openblas
+fastcheck-lapack: fastcheck-openblas
+check-lapack: check-openblas
+clean-lapack: clean-openblas
+distclean-lapack: distclean-openblas
+install-lapack: install-openblas
+endif

deps/llvm.mk

@@ -513,37 +513,12 @@ ifeq ($(USE_POLLY),1)
 endif
 endif
 else # USE_BINARYBUILDER_LLVM
-LLVM_BB_URL_BASE := https://github.com/staticfloat/LLVMBuilder/releases/download
+LLVM_BB_URL_BASE := https://github.com/staticfloat/LLVMBuilder/releases/download/v$(LLVM_VER)-$(LLVM_BB_REL)
 ifneq ($(BINARYBUILDER_LLVM_ASSERTS), 1)
-LLVM_BB_NAME := LLVM
+LLVM_BB_NAME := LLVM.v$(LLVM_VER)
 else
-LLVM_BB_NAME := LLVM.asserts
+LLVM_BB_NAME := LLVM.asserts.v$(LLVM_VER)
 endif
-LLVM_BB_NAME := $(LLVM_BB_NAME).v$(LLVM_VER)
-LLVM_BB_URL := $(LLVM_BB_URL_BASE)/v$(LLVM_VER)-$(LLVM_BB_REL)/$(LLVM_BB_NAME).$(BINARYBUILDER_TRIPLET).tar.gz
 
-
-$(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL):
-	mkdir -p $@
-
-$(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)/LLVM.$(BINARYBUILDER_TRIPLET).tar.gz: | $(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)
-	$(JLDOWNLOAD) $@ $(LLVM_BB_URL)
-
-$(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)/build-compiled: | $(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)/LLVM.$(BINARYBUILDER_TRIPLET).tar.gz
-	echo 1 > $@
-
-$(eval $(call staged-install,llvm,llvm-$$(LLVM_VER)-$$(LLVM_BB_REL),,,,))
-
-#Override provision of stage tarball
-$(build_staging)/llvm-$(LLVM_VER)-$(LLVM_BB_REL).tgz: $(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)/LLVM.$(BINARYBUILDER_TRIPLET).tar.gz | $(build_staging)
-	cp $< $@
-
-clean-llvm:
-distclean-llvm:
-get-llvm:  $(BUILDDIR)/llvm-$(LLVM_VER)-$(LLVM_BB_REL)/LLVM.$(BINARYBUILDER_TRIPLET).tar.gz
-extract-llvm:
-configure-llvm:
-compile-llvm:
-fastcheck-llvm:
-check-llvm:
+$(eval $(call bb-install,llvm,LLVM,false))
 endif # USE_BINARYBUILDER_LLVM

deps/openblas.version

@@ -1,2 +1,4 @@
 OPENBLAS_BRANCH=v0.3.3
 OPENBLAS_SHA1=fd8d1868a126bb9f12bbc43b36ee30d1ba943fbb
+OPENBLAS_VER=0.3.3
+OPENBLAS_BB_REL=0

deps/tools/bb-install.mk

@@ -0,0 +1,33 @@
+define bb-install
+# If the user has signified that this is a GCC-multiversioned tarball, then generate the proper tarball
+ifeq ($(3),true)
+$(2)_BB_TRIPLET := $(shell python $(call cygpath_w,$(JULIAHOME)/contrib/normalize_triplet.py) $(or $(XC_HOST),$(XC_HOST),$(BUILD_MACHINE)) $(lastword $(shell $(FC) --version | head -1)))
+else
+$(2)_BB_TRIPLET := $(shell python $(call cygpath_w,$(JULIAHOME)/contrib/normalize_triplet.py) $(or $(XC_HOST),$(XC_HOST),$(BUILD_MACHINE)))
+endif
+$(2)_BB_URL := $$($(2)_BB_URL_BASE)/$$($(2)_BB_NAME).$$($(2)_BB_TRIPLET).tar.gz
+
+$$(BUILDDIR)/$(1)-$$($(2)_BB_NAME):
+	mkdir -p $$@
+
+$$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)/$(2).$$($(2)_BB_TRIPLET).tar.gz: | $$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)
+	$$(JLDOWNLOAD) $$@ $$($(2)_BB_URL)
+
+$$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)/build-compiled: | $$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)/$(2).$$($(2)_BB_TRIPLET).tar.gz
+	echo 1 > $$@
+
+$$(eval $$(call staged-install,$(1),$(1)-$$$$($(2)_BB_NAME),,,,))
+
+#Override provision of stage tarball
+$$(build_staging)/$(1)-$$($(2)_BB_NAME).tgz: $$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)/$(2).$$($(2)_BB_TRIPLET).tar.gz | $$(build_staging)
+	cp $$< $$@
+
+clean-$(1):
+distclean-$(1):
+get-$(1): $$(BUILDDIR)/$(1)-$$($(2)_BB_NAME)/$(2).$$($(2)_BB_TRIPLET).tar.gz
+extract-$(1):
+configure-$(1):
+compile-$(1):
+fastcheck-$(1):
+check-$(1):
+endef