Use compiled Python library

Add script to download and compile Python from scratch. Currently, Linux
systems will use Python 3.6.6 and Mac OS systems 3.7. Once Python 3.7
compiles without SSL issues on Linux, the version should be bumped so
that ideally both operating system versions use the same Python.

The Python library is only compiled into a static library so that the
final binary should be as portable as possible without the need for
Python to be installed in a target environment.

Update Travis CI configuration to address the changed pre-reqs.

Change the way CFLAGS and LDFLAGS are populated in the Go source file
that uses the transpiled C file.

.gitignore

@@ -2,3 +2,4 @@ internal/**/updateYAML.c
 internal/**/updateYAML.go
 internal/**/__pycache__
 binaries
+third_party

.travis.yml

@@ -11,12 +11,8 @@ os:
 - osx
 
 install:
-- if [[ "$(uname)" == "Linux" ]]; then sudo apt-get update && sudo apt-get install -y build-essential python3 python3-pip python3-dev; fi
-- if [[ "$(uname)" == "Darwin" ]]; then brew update && brew install jq && ( brew outdated python || brew upgrade python ) && brew cleanup; fi
-- pip3 install --user 'ruamel.yaml<=0.15.42'
-- pip3 install --user 'cython'
-- if [[ "$(uname)" == "Linux" ]]; then export PATH="$PATH:$HOME/.local/bin"; fi
-- if [[ "$(uname)" == "Darwin" ]]; then if PYBIN=$(ls -d $HOME/Library/Python/3.?/bin); then export PATH="$PATH:$PYBIN"; fi; fi
+- if [[ "$(uname)" == "Linux" ]]; then sudo apt-get update >/dev/null && sudo apt-get install -y build-essential libssl-dev libffi-dev zlib1g-dev; fi
+- if [[ "$(uname)" == "Darwin" ]]; then brew update >/dev/null && brew install jq && ( brew outdated openssl || brew upgrade openssl ); fi
 - curl --silent --location https://goo.gl/g1CpPX | bash -s v1.0.2
 
 script:

Makefile

@@ -18,7 +18,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-.PHONY: all clean test pythontest gotest build upx
+.PHONY: all clean mrproper test pythontest gotest build upx
 
 os := $(shell uname | tr '[:upper:]' '[:lower:]')
 arch := $(shell uname -m | sed 's/x86_64/amd64/')
@@ -29,23 +29,34 @@ clean:
 	go clean -i -r -cache
 	rm -rf internal/pycgo/updateYAML.c internal/pycgo/updateYAML.go internal/pycgo/__pycache__ binaries
 
-internal/pycgo/updateYAML.c: internal/pycgo/updateYAML.py
-	cython -3 --embed=updateYAML --output-file internal/pycgo/updateYAML.c internal/pycgo/updateYAML.py
+mrproper: clean
+	rm -rf third_party
 
-internal/pycgo/updateYAML.go: internal/pycgo/updateYAML.go.template
-	@scripts/createGoSourceFileFromTemplate.sh
+gotest:
+	ginkgo -r --nodes 1 --randomizeAllSpecs --randomizeSuites --race --trace
 
-test: pythontest gotest
+pythontest: third_party/lib/python
+	third_party/lib/python/bin/python3 internal/pycgo/updateYAML_test.py
 
-pythontest:
-	python3 internal/pycgo/updateYAML_test.py
+test: gotest pythontest
 
-gotest:
-	ginkgo -r --nodes 1 --randomizeAllSpecs --randomizeSuites --race --trace
+third_party/lib/python:
+	@scripts/compilePythonLibrary.sh
+
+internal/pycgo/updateYAML.c: third_party/lib/python internal/pycgo/updateYAML.py
+	$${HOME}/.local/bin/cython -3 --embed=updateYAML --output-file internal/pycgo/updateYAML.c internal/pycgo/updateYAML.py
+
+internal/pycgo/updateYAML.go: third_party/lib/python internal/pycgo/updateYAML.go.template
+	@scripts/createGoSourceFileFromTemplate.sh
 
-build: internal/pycgo/updateYAML.c internal/pycgo/updateYAML.go
+build: third_party/lib/python internal/pycgo/updateYAML.c internal/pycgo/updateYAML.go
 	@mkdir -p binaries
-	go build -a -ldflags='-s -w' -o binaries/ytbx-$(os)-$(arch) cmd/ytbx/main.go
+	go build -ldflags='-s -w' -o binaries/ytbx-$(os)-$(arch) cmd/ytbx/main.go
+	@echo
+	@ls -lh binaries/ytbx-$(os)-$(arch)
+	@echo
+	@bash -c 'if [[ "$(os)" == "linux" ]]; then readelf -d binaries/ytbx-$(os)-$(arch); fi'
+	@bash -c 'if [[ "$(os)" == "darwin" ]]; then otool -L binaries/ytbx-$(os)-$(arch); fi'
 
 upx: build
 	upx -q binaries/ytbx-$(os)-$(arch)

scripts/compilePythonLibrary.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -euo pipefail
+
+BASEDIR="$(cd "$(dirname "$0")/.." && pwd)"
+
+setupDarwin() {
+  if [[ ! -d "${BASEDIR}/third_party/lib/python" ]]; then
+    mkdir -p "${BASEDIR}/third_party/src"
+    pushd "${BASEDIR}/third_party/src" >/dev/null
+
+    curl --silent --location https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz | tar -xzf -
+    pushd Python-3.7.0 >/dev/null
+
+    ./configure --prefix "${BASEDIR}/third_party/lib/python" --disable-shared --with-openssl="$(brew --prefix openssl)" --enable-optimizations
+    make --jobs
+    make install
+    popd >/dev/null
+
+    popd >/dev/null
+  fi
+}
+
+setupLinux() {
+  if [[ ! -d "${BASEDIR}/third_party/lib/python" ]]; then
+    mkdir -p "${BASEDIR}/third_party/src"
+    pushd "${BASEDIR}/third_party/src" >/dev/null
+
+    curl --silent --location https://www.python.org/ftp/python/3.6.6/Python-3.6.6.tar.xz | tar -xJf -
+    pushd Python-3.6.6 >/dev/null
+
+    ./configure --prefix "${BASEDIR}/third_party/lib/python" --disable-shared --enable-optimizations
+    make --jobs
+    make install
+    popd >/dev/null
+
+    popd >/dev/null
+  fi
+}
+
+case "$(uname)" in
+  Darwin)
+    setupDarwin
+    ;;
+
+  Linux)
+    setupLinux
+    ;;
+esac
+
+if ! (pip3 list | grep ruamel >/dev/null); then
+  "${BASEDIR}"/third_party/lib/python/bin/pip3 install --user --upgrade \
+    'pip' \
+    'setuptools' \
+    'wheel' \
+    'ruamel.yaml<=0.15.42' \
+    'cython'
+fi

scripts/createGoSourceFileFromTemplate.sh

@@ -4,6 +4,20 @@ set -euo pipefail
 
 BASEDIR="$(cd "$(dirname "$0")/.." && pwd)"
 
+# In case the local Python library is available
+if LIB_FILE="$(ls "${BASEDIR}"/third_party/lib/python/lib/libpython3*.a)"; then
+  LIB_NAME="$(basename "${LIB_FILE}" | sed -e 's/^lib//' -e 's/.a$//')"
+  INCLUDE_PATH="${BASEDIR}/third_party/lib/python/include/${LIB_NAME}"
+
+  sed \
+    -e "s:<cflags>:-O3 -pthread -I${INCLUDE_PATH}:" \
+    -e "s:<ldflags>:-L${BASEDIR}/third_party/lib/python/lib -l${LIB_NAME} -lm -ldl -lutil:" \
+    "${BASEDIR}/internal/pycgo/updateYAML.go.template" >"${BASEDIR}/internal/pycgo/updateYAML.go"
+
+  exit 0
+fi
+
+# Default case, use pkg-config
 sed \
   -e "s:<cflags>:$(pkg-config --cflags python3):" \
   -e "s:<ldflags>:$(pkg-config --static --libs python3):" \