Merge pull request #1 from tsg/build_on_debian6

Linux compile on Debian 6.

Makefile

@@ -9,16 +9,28 @@ all: packetbeat/deb packetbeat/rpm packetbeat/darwin packetbeat/win packetbeat/b
 
 .PHONY: packetbeat topbeat
 packetbeat topbeat: xgo-image build
-	cd build && xgo -image=tudorg/beats-builder -static \
+	cd build && xgo -image=tudorg/beats-builder \
 		-before-build=../xgo-scripts/$@_before_build.sh \
 		-branch $(RELEASE) \
 		github.com/elastic/$@
 
-%/deb: % build/god-linux-386 build/god-linux-amd64 fpm-image
+.PHONY: packetbeat-linux topbeat-linux
+packetbeat-linux: xgo-image build
+	cd build && xgo -image=tudorg/beats-builder-deb6 \
+		-before-build=../xgo-scripts/packetbeat_before_build.sh \
+		-branch $(RELEASE) \
+		github.com/elastic/packetbeat
+topbeat-linux: xgo-image build
+	cd build && xgo -image=tudorg/beats-builder-deb6 \
+		-before-build=../xgo-scripts/topbeat_before_build.sh \
+		-branch $(RELEASE) \
+		github.com/elastic/topbeat
+
+%/deb: %-linux build/god-linux-386 build/god-linux-amd64 fpm-image
 	ARCH=386 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/debian/build.sh
 	ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/debian/build.sh
 
-%/rpm: % build/god-linux-386 build/god-linux-amd64 fpm-image
+%/rpm: %-linux build/god-linux-386 build/god-linux-amd64 fpm-image
 	ARCH=386 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/centos/build.sh
 	ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/centos/build.sh
 
@@ -28,7 +40,7 @@ packetbeat topbeat: xgo-image build
 %/win: % fpm-image
 	ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/windows/build.sh
 
-%/bin: % fpm-image
+%/bin: %-linux fpm-image
 	ARCH=386 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/binary/build.sh
 	ARCH=amd64 RELEASE=$(RELEASE) BEAT=$(@D) BUILDID=$(BUILDID) ./platforms/binary/build.sh
 
@@ -39,15 +51,16 @@ deps:
 
 .PHONY: xgo-image
 xgo-image:
-	docker build -t tudorg/beats-builder docker/xgo-image/
+	cd docker/xgo-image/; ./build.sh
+	cd docker/xgo-image-deb6/; ./build.sh
 
 .PHONY: fpm-image
 fpm-image:
-	docker build -t tudorg/fpm docker/fpm-image/
+	docker build --rm=true -t tudorg/fpm docker/fpm-image/
 
 .PHONY: go-daemon-image
 go-daemon-image:
-	docker build -t tudorg/go-daemon docker/go-daemon/
+	docker build --rm=true -t tudorg/go-daemon docker/go-daemon/
 
 build/god-linux-386 build/god-linux-amd64: go-daemon-image
 	docker run -v $(shell pwd)/build:/build tudorg/go-daemon
@@ -60,11 +73,15 @@ s3-nightlies-upload: all
 	echo $(BUILDID) > build/upload/build_id.txt
 	aws s3 cp --recursive --acl public-read build/upload s3://beats-nightlies
 
+.PHONY: release-upload
+release-upload:
+	aws s3 cp --recursive --acl public-read build/upload s3://download.elasticsearch.org/beats/
+
 .PHONY: run-interactive
 run-interactive:
 	docker run -t -i -v $(shell pwd)/build:/build \
 		-v $(shell pwd)/xgo-scripts/:/scripts \
-		--entrypoint=bash tudorg/beats-builder
+		--entrypoint=bash tudorg/beats-builder-deb6
 .PHONY: clean
 clean:
 	rm -rf build/ || true

docker/xgo-image-deb6/base/Dockerfile

@@ -0,0 +1,44 @@
+# Go cross compiler (xgo): Base cross-compilation layer
+# Copyright (c) 2014 Péter Szilágyi. All rights reserved.
+#
+# Released under the MIT license.
+
+FROM debian:6
+
+MAINTAINER Tudor Golubenco <tudor@elastic.co>
+
+# Configure the Go environment, since it's not going to change
+ENV PATH   /usr/local/go/bin:$PATH
+ENV GOPATH /go
+
+
+# Inject the remote file fetcher and checksum verifier
+ADD fetch.sh /fetch.sh
+ENV FETCH /fetch.sh
+RUN chmod +x $FETCH
+
+
+# Make sure apt-get is up to date and dependent packages are installed
+RUN \
+  apt-get update && \
+  apt-get install -y automake autogen build-essential ca-certificates \
+    gcc-multilib \
+    clang llvm-dev  libtool libxml2-dev uuid-dev libssl-dev pkg-config \
+    patch make xz-utils cpio wget unzip git mercurial bzr --no-install-recommends
+
+# Inject the Go package downloader and tool-chain bootstrapper
+ADD bootstrap.sh /bootstrap.sh
+ENV BOOTSTRAP /bootstrap.sh
+RUN chmod +x $BOOTSTRAP
+
+# Inject the C dependency cross compiler
+ADD build_deps.sh /build_deps.sh
+ENV BUILD_DEPS /build_deps.sh
+RUN chmod +x $BUILD_DEPS
+
+# Inject the container entry point, the build script
+ADD build.sh /build.sh
+ENV BUILD /build.sh
+RUN chmod +x $BUILD
+
+ENTRYPOINT ["/build.sh"]

docker/xgo-image-deb6/base/bootstrap.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Contains the Go tool-chain bootstrapper, that retrieves all the configured
+# distribution packages, extracts the binaries and deletes anything not needed.
+#
+# Usage: bootstrap.sh
+#
+# Needed environment variables:
+#   FETCH - Remote file fetcher and checksum verifier (injected by image)
+#   DIST_LINUX_64,  DIST_LINUX_64_SHA1  - 64 bit Linux Go binaries and checksum
+#   DIST_LINUX_32,  DIST_LINUX_32_SHA1  - 32 bit Linux Go binaries and checksum
+#   DIST_LINUX_ARM, DIST_LINUX_ARM_SHA1 - ARM v5 Linux Go binaries and checksum
+#   DIST_OSX_64,    DIST_OSX_64_SHA1    - 64 bit Mac OSX Go binaries and checksum
+#   DIST_OSX_32,    DIST_OSX_32_SHA1    - 32 bit Mac OSX Go binaries and checksum
+#   DIST_WIN_64,    DIST_WIN_64_SHA1    - 64 bit Windows Go binaries and checksum
+#   DIST_WIN_32,    DIST_WIN_32_SHA1    - 32 bit Windows Go binaries and checksum 
+set -e
+
+# Download and verify all the binary packages
+$FETCH $DIST_LINUX_64  $DIST_LINUX_64_SHA1
+$FETCH $DIST_LINUX_32  $DIST_LINUX_32_SHA1
+
+# Extract the 64 bit Linux package as the primary Go SDK
+tar -C /usr/local -xzf `basename $DIST_LINUX_64`
+
+# Extract all other packages as secondary ones, keeping only the binaries
+tar -C /usr/local --wildcards -xzf `basename $DIST_LINUX_32` go/pkg/linux_386*
+GOOS=linux GOARCH=386 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap
+
+# Delete all the intermediate downloaded files
+rm -f `basename $DIST_LINUX_64` `basename $DIST_LINUX_32`

docker/xgo-image-deb6/base/build.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+#
+# Contains the main cross compiler, that individually sets up each target build
+# platform, compiles all the C dependencies, then build the requested executable
+# itself.
+#
+# Usage: build.sh <import path>
+#
+# Needed environment variables:
+#   REPO_REMOTE - Optional VCS remote if not the primary repository is needed
+#   REPO_BRANCH - Optional VCS branch to use, if not the master branch
+#   DEPS        - Optional list of C dependency packages to build
+#   PACK        - Optional sub-package, if not the import path is being built
+#   OUT         - Optional output prefix to override the package name
+#   FLAG_V      - Optional verbosity flag to set on the Go builder
+#   FLAG_RACE   - Optional race flag to set on the Go builder
+
+# Download the canonical import path (may fail, don't allow failures beyond)
+echo "Fetching main repository $1..."
+mkdir -p $GOPATH/src/`dirname $1`
+cd $GOPATH/src/`dirname $1`
+git clone https://$1.git
+
+set -e
+
+cd $GOPATH/src/$1
+export GOPATH=$GOPATH:`pwd`/Godeps/_workspace
+
+# Switch over the code-base to another checkout if requested
+if [ "$REPO_REMOTE" != "" ]; then
+  echo "Switching over to remote $REPO_REMOTE..."
+  if [ -d ".git" ]; then
+    git remote set-url origin $REPO_REMOTE
+    git pull
+  elif [ -d ".hg" ]; then
+    echo -e "[paths]\ndefault = $REPO_REMOTE\n" >> .hg/hgrc
+    hg pull
+  fi
+fi
+
+if [ "$REPO_BRANCH" != "" ]; then
+  echo "Switching over to branch $REPO_BRANCH..."
+  if [ -d ".git" ]; then
+    git checkout $REPO_BRANCH
+  elif [ -d ".hg" ]; then
+    hg checkout $REPO_BRANCH
+  fi
+fi
+
+# Download all the C dependencies
+echo "Fetching dependencies..."
+mkdir -p /deps
+DEPS=($DEPS) && for dep in "${DEPS[@]}"; do
+  echo Downloading $dep
+  if [ "${dep##*.}" == "tar" ]; then wget -q $dep -O - | tar -C /deps -x; fi
+  if [ "${dep##*.}" == "gz" ]; then wget -q $dep -O - | tar -C /deps -xz; fi
+  if [ "${dep##*.}" == "bz2" ]; then wget -q $dep -O - | tar -C /deps -xj; fi
+done
+
+# Configure some global build parameters
+NAME=`basename $1/$PACK`
+if [ "$OUT" != "" ]; then
+  NAME=$OUT
+fi
+
+
+if [ "$FLAG_V" == "true" ]; then V=-v; fi
+if [ "$FLAG_RACE" == "true" ]; then R=-race; fi
+if [ "$STATIC" == "true" ]; then LDARGS=--ldflags\ \'-extldflags\ \"-static\"\'; fi
+
+if [ -n $BEFORE_BUILD ]; then
+	chmod +x /scripts/$BEFORE_BUILD
+	echo "Execute /scripts/$BEFORE_BUILD"
+	/scripts/$BEFORE_BUILD
+fi
+
+# Build for each platform individually
+echo "Compiling for linux/amd64..."
+HOST=x86_64-linux PREFIX=/usr/local $BUILD_DEPS /deps
+GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK
+sh -c "GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go build $V $R $LDARGS -o $NAME-linux-amd64$R ./$PACK"
+
+echo "Compiling for linux/386..."
+HOST=i686-linux PREFIX=/usr/local $BUILD_DEPS /deps
+GOOS=linux GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK
+sh -c "GOOS=linux GOARCH=386 CGO_ENABLED=1 go build $V $LDARGS -o $NAME-linux-386 ./$PACK"
+
+#echo "Compiling for linux/arm..."
+#CC=arm-linux-gnueabi-gcc HOST=arm-linux PREFIX=/usr/local/arm $BUILD_DEPS /deps
+#CC=arm-linux-gnueabi-gcc GOOS=linux GOARCH=arm CGO_ENABLED=1 GOARM=5 go get -d ./$PACK
+#CC=arm-linux-gnueabi-gcc GOOS=linux GOARCH=arm CGO_ENABLED=1 GOARM=5 go build $V -o $NAME-linux-arm ./$PACK
+
+#echo "Compiling for windows/amd64..."
+#CC=x86_64-w64-mingw32-gcc HOST=x86_64-w64-mingw32 PREFIX=/usr/x86_64-w64-mingw32 $BUILD_DEPS /deps
+#CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK
+#CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go build  $V $R -o $NAME-windows-amd64$R.exe ./$PACK
+
+#echo "Compiling for windows/386..."
+#CC=i686-w64-mingw32-gcc HOST=i686-w64-mingw32 PREFIX=/usr/i686-w64-mingw32 $BUILD_DEPS /deps
+#CC=i686-w64-mingw32-gcc GOOS=windows GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK
+#CC=i686-w64-mingw32-gcc GOOS=windows GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-windows-386.exe ./$PACK
+
+#echo "Compiling for darwin/amd64..."
+#CC=o64-clang HOST=x86_64-apple-darwin10 PREFIX=/usr/local $BUILD_DEPS /deps
+#CC=o64-clang GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK
+#CC=o64-clang GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go build $V $R -o $NAME-darwin-amd64$R ./$PACK
+
+#echo "Compiling for darwin/386..."
+#CC=o32-clang HOST=i386-apple-darwin10 PREFIX=/usr/local $BUILD_DEPS /deps
+#CC=o32-clang GOOS=darwin GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK
+#CC=o32-clang GOOS=darwin GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-darwin-386 ./$PACK
+
+echo "Moving binaries to host..."
+cp `ls -t | head -n 2` /build

docker/xgo-image-deb6/base/build_deps.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Contains the a dependency builder to iterate over all installed dependencies
+# and cross compile them to the requested target platform.
+# 
+# Usage: build_deps.sh <dependency folder>
+#
+# Needed environment variables:
+#   CC     - C cross compiler to use for the build
+#   HOST   - Target platform to build (used to find the needed tool-chains)
+#   PREFIX - File-system path where to install the built binaries
+set -e
+
+# Remove any previous build leftovers, and copy a fresh working set (clean doesn't work for cross compiling)
+rm -rf /deps-build && cp -r $1 /deps-build
+
+# Build all the dependencies (no order for now)
+for dep in `ls /deps-build`; do
+	echo "Configuring dependency $dep for $HOST..."
+	(cd /deps-build/$dep && ./configure --disable-shared --host=$HOST --prefix=$PREFIX --silent)
+
+	echo "Building dependency $dep for $HOST..."
+	(cd /deps-build/$dep && make --silent -j install)
+done
+
+# Remove any build artifacts
+rm -rf /deps-build

docker/xgo-image-deb6/base/fetch.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Contains a simple fetcher to download a file from a remote URL and verify its
+# SHA1 checksum.
+#
+# Usage: fetch.sh <remote URL> <SHA1 checksum>
+set -e
+
+# Pull the file from the remote URL
+file=`basename $1`
+echo "Downloading $1..."
+wget --no-check-certificate -q $1
+
+# Generate a desired checksum report and check against it
+echo "$2  $file" > $file.sum
+sha1sum -c $file.sum
+rm $file.sum

docker/xgo-image-deb6/beats-builder/Dockerfile

@@ -0,0 +1,18 @@
+FROM tudorg/xgo-deb6-1.4.2
+
+MAINTAINER Tudor Golubenco <tudor@elastic.co>
+
+# Get libpcap binaries for linux
+RUN \
+	mkdir -p /libpcap && \
+    wget http://ftp.de.debian.org/debian/pool/main/libp/libpcap/libpcap0.8-dev_1.1.1-2+squeeze1_i386.deb && \
+	dpkg -x libpcap0.8-dev_*_i386.deb /libpcap/i386 && \
+	wget http://ftp.de.debian.org/debian/pool/main/libp/libpcap/libpcap0.8-dev_1.1.1-2+squeeze1_amd64.deb && \
+	dpkg -x libpcap0.8-dev_*_amd64.deb /libpcap/amd64 && \
+	rm libpcap0.8-dev*.deb
+RUN \
+	apt-get update && \
+	apt-get install -y libpcap0.8-dev
+
+# add patch for gopacket
+ADD gopacket_pcap.patch /gopacket_pcap.patch

docker/xgo-image-deb6/beats-builder/gopacket_pcap.patch

@@ -0,0 +1,22 @@
+diff --git a/Godeps/_workspace/src/github.com/tsg/gopacket/pcap/pcap.go b/Godeps/_workspace/src/github.
+index d2465bb..7b23b84 100644
+--- a/Godeps/_workspace/src/github.com/tsg/gopacket/pcap/pcap.go
++++ b/Godeps/_workspace/src/github.com/tsg/gopacket/pcap/pcap.go
+@@ -8,12 +8,13 @@
+ package pcap
+
+ /*
+-#cgo linux LDFLAGS: -lpcap
++#cgo linux,386 LDFLAGS: /libpcap/i386/usr/lib/libpcap.a
++#cgo linux,amd64 LDFLAGS: /libpcap/amd64/usr/lib/libpcap.a
+ #cgo freebsd LDFLAGS: -lpcap
+ #cgo darwin LDFLAGS: -lpcap
+-#cgo windows CFLAGS: -I C:/WpdPack/Include
+-#cgo windows,386 LDFLAGS: -L C:/WpdPack/Lib -lwpcap
+-#cgo windows,amd64 LDFLAGS: -L C:/WpdPack/Lib/x64 -lwpcap
++#cgo windows CFLAGS: -I /libpcap/win/WpdPack/Include
++#cgo windows,386 LDFLAGS: -L /libpcap/win/WpdPack/Lib -lwpcap
++#cgo windows,amd64 LDFLAGS: -L /libpcap/win/WpdPack/Lib/x64 -lwpcap
+ #include <stdlib.h>
+ #include <pcap.h>
+

docker/xgo-image-deb6/build.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+docker build --rm=true -t tudorg/xgo-deb6-base base/ && \
+    docker build --rm=true -t tudorg/xgo-deb6-1.4.2 go-1.4.2/ &&
+    docker build --rm=true -t tudorg/beats-builder-deb6 beats-builder

docker/xgo-image-deb6/go-1.4.2/Dockerfile

@@ -0,0 +1,18 @@
+# Go cross compiler (xgo): Go 1.4.2 layer
+# Copyright (c) 2014 Péter Szilágyi. All rights reserved.
+#
+# Released under the MIT license.
+
+FROM tudorg/xgo-deb6-base
+
+MAINTAINER Tudor Golubenco <tudor@elastic.co>
+
+# Configure the Go packages and bootstrap them
+RUN \
+  export DIST_LINUX_64=https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz && \
+  export DIST_LINUX_64_SHA1=5020af94b52b65cc9b6f11d50a67e4bae07b0aff && \
+  \
+  export DIST_LINUX_32=https://storage.googleapis.com/golang/go1.4.2.linux-386.tar.gz && \
+  export DIST_LINUX_32_SHA1=50557248e89b6e38d395fda93b2f96b2b860a26a && \
+  \
+  $BOOTSTRAP