Rollup merge of rust-lang#39111 - alexcrichton:more-cross-targets, r=…

…brson

travis: Expand the `cross` linux image

This expands the `cross` travis matrix entry with a few more targets that our
nightlies are building:

* x86_64-rumprun-netbsd
* arm-unknown-linux-musleabi
* arm-unknown-linux-musleabihf
* armv7-unknown-linux-musleabihf
* mips-unknown-linux-musl
* mipsel-unknown-linux-musl

This commit doesn't compile custom toolchains like our current cross-image does,
but instead compiles musl manually and then compiles libunwind manually (like
x86_64) for use for the ARM targets and just uses openwrt toolchains for the
mips targets.

cc rust-lang#38531

.travis.yml

@@ -13,7 +13,7 @@ osx_image: xcode8.2
 matrix:
   include:
     # Linux builders, all docker images
-    - env: IMAGE=arm-android DEPLOY=1
+    - env: IMAGE=android DEPLOY=1
     - env: IMAGE=cross DEPLOY=1
     - env: IMAGE=linux-tested-targets DEPLOY=1
     - env: IMAGE=dist-arm-linux DEPLOY=1

src/bootstrap/cc.rs

@@ -121,10 +121,14 @@ fn set_compiler(cfg: &mut gcc::Config,
         }
 
         "mips-unknown-linux-musl" => {
-            cfg.compiler("mips-linux-musl-gcc");
+            if cfg.get_compiler().path().to_str() == Some("gcc") {
+                cfg.compiler("mips-linux-musl-gcc");
+            }
         }
         "mipsel-unknown-linux-musl" => {
-            cfg.compiler("mipsel-linux-musl-gcc");
+            if cfg.get_compiler().path().to_str() == Some("gcc") {
+                cfg.compiler("mipsel-linux-musl-gcc");
+            }
         }
 
         t if t.contains("musl") => {

src/bootstrap/flags.rs

@@ -287,8 +287,8 @@ To learn more about a subcommand, run `./x.py <command> -h`
             build: m.opt_str("build").unwrap_or_else(|| {
                 env::var("BUILD").unwrap()
             }),
-            host: m.opt_strs("host"),
-            target: m.opt_strs("target"),
+            host: split(m.opt_strs("host")),
+            target: split(m.opt_strs("target")),
             config: cfg_file,
             src: m.opt_str("src").map(PathBuf::from),
             jobs: m.opt_str("jobs").map(|j| j.parse().unwrap()),
@@ -309,3 +309,7 @@ impl Subcommand {
         }
     }
 }
+
+fn split(s: Vec<String>) -> Vec<String> {
+    s.iter().flat_map(|s| s.split(',')).map(|s| s.to_string()).collect()
+}

src/ci/docker/arm-android/Dockerfile → src/ci/docker/android/Dockerfile

@@ -53,8 +53,4 @@ ENV RUST_CONFIGURE_ARGS \
 # to all the targets above eventually.
 ENV SCRIPT \
   python2.7 ../x.py test --target arm-linux-androideabi && \
-  python2.7 ../x.py dist \
-      --target arm-linux-androideabi \
-      --target armv7-linux-androideabi \
-      --target i686-linux-android \
-      --target aarch64-linux-android
+  python2.7 ../x.py dist --target $TARGETS

src/ci/docker/cross/Dockerfile

@@ -10,7 +10,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   git \
   cmake \
   sudo \
-  xz-utils
+  xz-utils \
+  zlib1g-dev \
+  g++-arm-linux-gnueabi \
+  g++-arm-linux-gnueabihf \
+  bzip2 \
+  patch
 
 ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
 RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
@@ -21,21 +26,50 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
     rm dumb-init_*.deb
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
+WORKDIR /tmp
+
+COPY build-rumprun.sh /tmp/
+RUN ./build-rumprun.sh
+
+COPY build-arm-musl.sh /tmp/
+RUN ./build-arm-musl.sh
+
+# originally from
+# https://downloads.openwrt.org/snapshots/trunk/ar71xx/generic/OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2
+RUN mkdir /usr/local/mips-linux-musl
+RUN curl -L https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2 | \
+      tar xjf - -C /usr/local/mips-linux-musl --strip-components=2
+RUN for file in /usr/local/mips-linux-musl/bin/mips-openwrt-linux-*; do \
+      ln -s $file /usr/local/bin/`basename $file`; \
+    done
+
+# Note that this originally came from:
+# https://downloads.openwrt.org/snapshots/trunk/malta/generic/OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2
+RUN mkdir /usr/local/mipsel-linux-musl
+RUN curl -L https://s3.amazonaws.com/rust-lang-ci/libc/OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2 | \
+      tar xjf - -C /usr/local/mipsel-linux-musl --strip-components=2
+RUN for file in /usr/local/mipsel-linux-musl/bin/mipsel-openwrt-linux-*; do \
+      ln -s $file /usr/local/bin/`basename $file`; \
+    done
+
 ENV TARGETS=asmjs-unknown-emscripten
 ENV TARGETS=$TARGETS,wasm32-unknown-emscripten
+ENV TARGETS=$TARGETS,x86_64-rumprun-netbsd
+ENV TARGETS=$TARGETS,mips-unknown-linux-musl
+ENV TARGETS=$TARGETS,mipsel-unknown-linux-musl
+ENV TARGETS=$TARGETS,arm-unknown-linux-musleabi
+ENV TARGETS=$TARGETS,arm-unknown-linux-musleabihf
+ENV TARGETS=$TARGETS,armv7-unknown-linux-musleabihf
+
+ENV CC_mipsel_unknown_linux_musl=mipsel-openwrt-linux-gcc \
+    CC_mips_unknown_linux_musl=mips-openwrt-linux-gcc
 
-#ENV TARGETS=$TARGETS,mips-unknown-linux-musl
-#ENV TARGETS=$TARGETS,arm-unknown-linux-musleabi
-#ENV TARGETS=$TARGETS,arm-unknown-linux-musleabihf
-#ENV TARGETS=$TARGETS,armv7-unknown-linux-musleabihf
-#ENV TARGETS=$TARGETS,x86_64-rumprun-netbsd
+# Suppress some warnings in the openwrt toolchains we downloaded
+ENV STAGING_DIR=/tmp
 
 ENV RUST_CONFIGURE_ARGS \
       --target=$TARGETS \
-      --enable-rustbuild
-
-# Just a smoke test in dist to see if this works for now, we should expand this
-# to all the targets above eventually.
-ENV SCRIPT \
-  python2.7 ../x.py build && \
-  python2.7 ../x.py dist --target wasm32-unknown-emscripten
+      --musl-root-arm=/usr/local/arm-linux-musleabi \
+      --musl-root-armhf=/usr/local/arm-linux-musleabihf \
+      --musl-root-armv7=/usr/local/armv7-linux-musleabihf
+ENV SCRIPT python2.7 ../x.py dist --target $TARGETS

src/ci/docker/cross/build-arm-musl.sh

@@ -0,0 +1,104 @@
+#!/bin/sh
+# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ex
+
+MUSL=1.1.16
+
+curl -O https://www.musl-libc.org/releases/musl-$MUSL.tar.gz
+tar xf musl-$MUSL.tar.gz
+cd musl-$MUSL
+CC=arm-linux-gnueabi-gcc \
+CFLAGS="-march=armv6 -marm" \
+    ./configure \
+        --prefix=/usr/local/arm-linux-musleabi \
+        --enable-wrapper=gcc
+make -j$(nproc)
+make install
+cd ..
+rm -rf musl-$MUSL
+
+tar xf musl-$MUSL.tar.gz
+cd musl-$MUSL
+CC=arm-linux-gnueabihf-gcc \
+CFLAGS="-march=armv6 -marm" \
+    ./configure \
+        --prefix=/usr/local/arm-linux-musleabihf \
+        --enable-wrapper=gcc
+make -j$(nproc)
+make install
+cd ..
+rm -rf musl-$MUSL
+
+tar xf musl-$MUSL.tar.gz
+cd musl-$MUSL
+CC=arm-linux-gnueabihf-gcc \
+CFLAGS="-march=armv7-a" \
+    ./configure \
+        --prefix=/usr/local/armv7-linux-musleabihf \
+        --enable-wrapper=gcc
+make -j$(nproc)
+make install
+cd ..
+rm -rf musl-$MUSL*
+
+ln -nsf ../arm-linux-musleabi/bin/musl-gcc /usr/local/bin/arm-linux-musleabi-gcc
+ln -nsf ../arm-linux-musleabihf/bin/musl-gcc /usr/local/bin/arm-linux-musleabihf-gcc
+ln -nsf ../armv7-linux-musleabihf/bin/musl-gcc /usr/local/bin/armv7-linux-musleabihf-gcc
+
+
+curl -L https://github.com/llvm-mirror/llvm/archive/release_39.tar.gz | tar xzf -
+curl -L https://github.com/llvm-mirror/libunwind/archive/release_39.tar.gz | tar xzf -
+
+mkdir libunwind-build
+cd libunwind-build
+cmake ../libunwind-release_39 \
+          -DLLVM_PATH=/tmp/llvm-release_39 \
+          -DLIBUNWIND_ENABLE_SHARED=0 \
+          -DCMAKE_C_COMPILER=arm-linux-gnueabi-gcc \
+          -DCMAKE_CXX_COMPILER=arm-linux-gnueabi-g++ \
+          -DCMAKE_C_FLAGS="-march=armv6 -marm" \
+          -DCMAKE_CXX_FLAGS="-march=armv6 -marm"
+make -j$(nproc)
+cp lib/libunwind.a /usr/local/arm-linux-musleabi/lib
+cd ..
+rm -rf libunwind-build
+
+mkdir libunwind-build
+cd libunwind-build
+cmake ../libunwind-release_39 \
+          -DLLVM_PATH=/tmp/llvm-release_39 \
+          -DLIBUNWIND_ENABLE_SHARED=0 \
+          -DCMAKE_C_COMPILER=arm-linux-gnueabihf-gcc \
+          -DCMAKE_CXX_COMPILER=arm-linux-gnueabihf-g++ \
+          -DCMAKE_C_FLAGS="-march=armv6 -marm" \
+          -DCMAKE_CXX_FLAGS="-march=armv6 -marm"
+make -j$(nproc)
+cp lib/libunwind.a /usr/local/arm-linux-musleabihf/lib
+cd ..
+rm -rf libunwind-build
+
+mkdir libunwind-build
+cd libunwind-build
+cmake ../libunwind-release_39 \
+          -DLLVM_PATH=/tmp/llvm-release_39 \
+          -DLIBUNWIND_ENABLE_SHARED=0 \
+          -DCMAKE_C_COMPILER=arm-linux-gnueabihf-gcc \
+          -DCMAKE_CXX_COMPILER=arm-linux-gnueabihf-g++ \
+          -DCMAKE_C_FLAGS="-march=armv7-a" \
+          -DCMAKE_CXX_FLAGS="-march=armv7-a"
+make -j$(nproc)
+cp lib/libunwind.a /usr/local/armv7-linux-musleabihf/lib
+cd ..
+rm -rf libunwind-build
+
+rm -rf libunwind-release_39
+rm -rf llvm-release_39

src/ci/docker/cross/build-rumprun.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ex
+
+hide_output() {
+  set +x
+  on_err="
+echo ERROR: An error was encountered with the build.
+cat /tmp/build.log
+exit 1
+"
+  trap "$on_err" ERR
+  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
+  PING_LOOP_PID=$!
+  $@ &> /tmp/build.log
+  trap - ERR
+  kill $PING_LOOP_PID
+  rm /tmp/build.log
+  set -x
+}
+
+
+git clone https://github.com/rumpkernel/rumprun
+cd rumprun
+git reset --hard 39a97f37a85e44c69b662f6b97b688fbe892603b
+git submodule update --init
+
+CC=cc hide_output ./build-rr.sh -d /usr/local hw
+cd ..
+rm -rf rumprun

src/ci/docker/dist-arm-linux/Dockerfile

@@ -24,11 +24,8 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
     rm dumb-init_*.deb
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
-ENV RUST_CONFIGURE_ARGS \
-      --host=arm-unknown-linux-gnueabi,arm-unknown-linux-gnueabihf
-ENV SCRIPT \
-      python2.7 ../x.py dist \
-        --host arm-unknown-linux-gnueabi \
-        --target arm-unknown-linux-gnueabi \
-        --host arm-unknown-linux-gnueabihf \
-        --target arm-unknown-linux-gnueabihf
+ENV HOSTS=arm-unknown-linux-gnueabi
+ENV HOSTS=$HOSTS,arm-unknown-linux-gnueabihf
+
+ENV RUST_CONFIGURE_ARGS --host=$HOSTS
+ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS

src/ci/docker/dist-armv7-aarch64-linux/Dockerfile

@@ -24,11 +24,8 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
     rm dumb-init_*.deb
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
-ENV RUST_CONFIGURE_ARGS \
-      --host=armv7-unknown-linux-gnueabihf,aarch64-unknown-linux-gnu
-ENV SCRIPT \
-      python2.7 ../x.py dist \
-        --host armv7-unknown-linux-gnueabihf \
-        --target armv7-unknown-linux-gnueabihf \
-        --host aarch64-unknown-linux-gnu \
-        --target aarch64-unknown-linux-gnu
+ENV HOSTS=armv7-unknown-linux-gnueabihf
+ENV HOSTS=$HOSTS,aarch64-unknown-linux-gnu
+
+ENV RUST_CONFIGURE_ARGS --host=$HOSTS
+ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS

src/ci/docker/dist-freebsd/Dockerfile

@@ -35,11 +35,8 @@ ENV \
     CC_i686_unknown_freebsd=i686-unknown-freebsd10-gcc \
     CXX_i686_unknown_freebsd=i686-unknown-freebsd10-g++
 
-ENV RUST_CONFIGURE_ARGS \
-        --host=x86_64-unknown-freebsd,i686-unknown-freebsd
-ENV SCRIPT \
-      python2.7 ../x.py dist \
-        --host x86_64-unknown-freebsd \
-        --target x86_64-unknown-freebsd \
-        --host i686-unknown-freebsd \
-        --target i686-unknown-freebsd
+ENV HOSTS=x86_64-unknown-freebsd
+ENV HOSTS=$HOSTS,i686-unknown-freebsd
+
+ENV RUST_CONFIGURE_ARGS --host=$HOSTS
+ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS

src/ci/docker/dist-mips-linux/Dockerfile

@@ -24,11 +24,8 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
     rm dumb-init_*.deb
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
-ENV RUST_CONFIGURE_ARGS \
-      --host=mips-unknown-linux-gnu,mipsel-unknown-linux-gnu
-ENV SCRIPT \
-      python2.7 ../x.py dist \
-        --host mips-unknown-linux-gnu \
-        --target mips-unknown-linux-gnu \
-        --host mipsel-unknown-linux-gnu \
-        --target mipsel-unknown-linux-gnu
+ENV HOSTS=mips-unknown-linux-gnu
+ENV HOSTS=$HOSTS,mipsel-unknown-linux-gnu
+
+ENV RUST_CONFIGURE_ARGS --host=$HOSTS
+ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS

src/ci/docker/dist-mips64-linux/Dockerfile

@@ -24,11 +24,8 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
     rm dumb-init_*.deb
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
-ENV RUST_CONFIGURE_ARGS \
-      --host=mips64-unknown-linux-gnuabi64,mips64el-unknown-linux-gnuabi64
-ENV SCRIPT \
-      python2.7 ../x.py dist \
-        --host mips64-unknown-linux-gnuabi64 \
-        --target mips64-unknown-linux-gnuabi64 \
-        --host mips64el-unknown-linux-gnuabi64 \
-        --target mips64el-unknown-linux-gnuabi64
+ENV HOSTS=mips64-unknown-linux-gnuabi64
+ENV HOSTS=$HOSTS,mips64el-unknown-linux-gnuabi64
+
+ENV RUST_CONFIGURE_ARGS --host=$HOSTS
+ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS

src/ci/docker/dist-powerpc-linux/Dockerfile

@@ -23,12 +23,10 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
     rm dumb-init_*.deb
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
-ENV RUST_CONFIGURE_ARGS \
-      --host=powerpc-unknown-linux-gnu
-ENV SCRIPT \
-      python2.7 ../x.py dist \
-        --host powerpc-unknown-linux-gnu \
-        --target powerpc-unknown-linux-gnu
+ENV HOSTS=powerpc-unknown-linux-gnu
+
+ENV RUST_CONFIGURE_ARGS --host=$HOSTS
+ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
 
 # FIXME(#36150) this will fail the bootstrap. Probably means something bad is
 #               happening!

src/ci/docker/dist-powerpc64-linux/Dockerfile

@@ -29,11 +29,8 @@ ENV \
     CC_powerpc64_unknown_linux_gnu=powerpc64-linux-gnu-gcc \
     CXX_powerpc64_unknown_linux_gnu=powerpc64-linux-gnu-g++
 
-ENV RUST_CONFIGURE_ARGS \
-      --host=powerpc64-unknown-linux-gnu,powerpc64le-unknown-linux-gnu
-ENV SCRIPT \
-      python2.7 ../x.py dist \
-        --host powerpc64-unknown-linux-gnu \
-        --target powerpc64-unknown-linux-gnu \
-        --host powerpc64le-unknown-linux-gnu \
-        --target powerpc64le-unknown-linux-gnu
+ENV HOSTS=powerpc64-unknown-linux-gnu
+ENV HOSTS=$HOSTS,powerpc64le-unknown-linux-gnu
+
+ENV RUST_CONFIGURE_ARGS --host=$HOSTS
+ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS