cross-platform, cross-compile build toolchain

build.gradle

@@ -39,12 +39,12 @@ dependencies {
 }
 
 android {
-    compileSdkVersion 10
+    compileSdkVersion 19
     buildToolsVersion '24.0.3'
 
     defaultConfig {
-        minSdkVersion 10
-        targetSdkVersion 10
+        minSdkVersion 19
+        targetSdkVersion 19
 
         testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
     }

build_system/config_android.py

@@ -66,7 +66,68 @@ def build_j2v8_junit(config, arch):
     # this assumes that a proper target Android device or emulator is running
     # for the tests to execute on (platform + architecture must match the build settings)
     # TODO: for the cross-compile make sure to start up the appropriate emulator / shutdown after the test runs
+
+    # emu_start = [
+    #     "echo no | $ANDROID_HOME/tools/bin/avdmanager create avd -n test -k 'system-images;android-19;default;$ARCH'",
+    #     "$ANDROID_HOME/emulator/emulator64-x86 -avd test -noaudio -no-window -memory 1024 -gpu on -verbose -qemu -usbdevice tablet -vnc :0",
+    # ]
+    # TODO: gradlew on windows
+    run_tests = [
+        "gradle connectedCheck --info"
+    ]
+
+    # TODO: refactor to be able to check CLI / build params
+    params_cross_enabled = True
+    if (params_cross_enabled):
+        supervisord_conf = None
+        with open("./docker/android/supervisord.template.conf", 'r') as f:
+            supervisord_conf = f.read()
+
+        # TODO: generic inject env utility ???
+        supervisord_conf = supervisord_conf.replace("$TEST_CMDS", " && ".join(run_tests))
+
+        with open("./docker/android/supervisord.conf", 'w') as f:
+            f.write(supervisord_conf)
+
+        emulator_start = None
+        with open("./docker/android/start-emulator.template.sh", 'r') as f:
+            emulator_start = f.read()
+
+        # TODO: generic inject env utility ???
+        emulator_start = emulator_start.replace("$ARCH", arch)
+
+        with open("./docker/android/start-emulator.sh", 'w') as f:
+            f.write(emulator_start)
+
+        return [
+            "/usr/bin/supervisord -c /j2v8/docker/android/supervisord.conf"
+        ]
+    else:
+        # just run the tests
+        return run_tests
+
+    #if -x
+    # create supervisord conf
+    # p1 = android emu
+    # p2 = wait via adb for emu, then run tests
+    # run supervisord
+    #else
+    # just run tests
+
     return [
+        # "$ANDROID_HOME/tools/android create avd -f -n test -t android-19 --abi default/$ARCH",
+
+        # V1
+        # TODO: how to make this cross platform / cross compile aware
+        # "echo no | $ANDROID_HOME/tools/bin/avdmanager create avd -n test -k 'system-images;android-19;default;$ARCH'",
+        # "$ANDROID_HOME/emulator/emulator64-x86 -avd test -noaudio -no-window -memory 1024 -gpu on -verbose -qemu -usbdevice tablet -vnc :0",
+
+        # V2
+        # "echo no | /usr/local/android-sdk/tools/android create avd -f -n test -t android-19 --abi default/$ARCH",
+        # "echo no | /usr/local/android-sdk/tools/emulator64-$ARCH -avd test -noaudio -no-window -gpu off -verbose -qemu -usbdevice tablet -vnc :0",
+
+
+        # setup complete, now run the actual tests
         "gradlew connectedCheck --info"
         #set ANDROID_HOME=C:\PROGRA~2\Android\android-sdk
         #gradlew connectedCheck -Pandroid.testInstrumentationRunnerArguments.class=com.eclipsesource.v8.V8Test

build_system/cross_build.py

@@ -34,10 +34,9 @@ def cross_config(self, cross_config):
         self.configs['cross'] = cross_config
 
 class BuildConfig:
-    def __init__(self, name, platform, mounts = [], build = [], build_cwd = None, host_cwd = None):
+    def __init__(self, name, platform, build = [], build_cwd = None, host_cwd = None):
         self.name = name
         self.platform = platform
-        self.mounts = mounts
         self.build = build
         self.build_cwd = build_cwd
         self.host_cwd = host_cwd
@@ -82,12 +81,12 @@ def exec_cmd(self, cmd, config, arch):
         #os.popen(cmd).read()
 
     def inject_env(self, cmd, config, arch):
-        mounts = [self.get_mount_string(x) for x in config.mounts]
-        mounts_str = " ".join(mounts)
+        # mounts = [self.get_mount_string(x) for x in config.mounts]
+        # mounts_str = " ".join(mounts)
 
         return (cmd
             # substitute variables that can contain variables first
-            .replace("$MOUNTS", mounts_str)
+            # .replace("$MOUNTS", mounts_str)
             # substitute atomic variables later
             .replace("$BUILD_CWD", config.build_cwd or cwd)
             .replace("$HOST_CWD", config.host_cwd or "")

build_system/docker_build.py

@@ -26,17 +26,18 @@ def pre_build(self, config, arch):
         # this allows to download most of the maven dependencies for the build beforehand
         copy2("pom.xml", "./docker/shared")
         copy2("build.gradle", "./docker/shared")
+        copy2("src/main/AndroidManifest.xml", "./docker/android/AndroidManifest.xml")
 
         self.exec_host_cmd("docker build -f $PLATFORM/Dockerfile -t \"j2v8-$PLATFORM\" .", config, arch)
 
     def exec_build(self, config, arch, custom_cmd):
         print ("DOCKER building " + config.platform + "@" + arch + " => " + config.name)
 
-        docker_run_str = self.inject_env("docker run -v $CWD:/j2v8 --name j2v8.$PLATFORM.$ARCH j2v8-$PLATFORM ", config, arch)
+        docker_run_str = self.inject_env("docker run --privileged -P -v $CWD:/j2v8 --name j2v8.$PLATFORM.$ARCH j2v8-$PLATFORM ", config, arch)
         build_cmds_str = self.inject_env("/bin/bash -c \"cd $BUILD_CWD; " + (custom_cmd or "; ".join(config.build(config, arch))) + "\"", config, arch)
 
         docker_str = docker_run_str + build_cmds_str
-
+        print docker_str
         self.exec_host_cmd(docker_str, config, arch)
 
     def post_build(self, config, arch):

docker/android/.gitignore

@@ -0,0 +1,3 @@
+start-emulator.sh
+supervisord.conf
+AndroidManifest.xml

docker/android/Dockerfile

@@ -1,4 +1,5 @@
-FROM philcryer/min-jessie:latest
+# FROM philcryer/min-jessie:latest
+FROM ubuntu:16.04
 
 ENV NDK_VERSION r13b
 
@@ -8,8 +9,10 @@ ENV NDK=/build/android-ndk-$NDK_VERSION \
 ENV PATH=$PATH:/build/android-gcc-toolchain:$NDK
 ENV PATH=$PATH:/opt/cmake/bin
 
+# TODO: move to actual build command ?!
 ENV CCFLAGS=-fPIC CXXFLAGS=-fPIC
 
+# TODO: call into shared scripts for installing common build dependencies instead
 RUN echo "Preparing Debian packages..." && \
 	apt-get -qq update && \
 	DEBIAN_FRONTEND=noninteractive apt-get -qq install -y \
@@ -59,26 +62,81 @@ RUN echo "Preparing Gradle..." && \
 ENV GRADLE_HOME=/opt/gradle-3.5
 ENV PATH=$PATH:$GRADLE_HOME/bin
 
-RUN echo "Preparing Android SDK..." && \
-	mkdir -p /opt/android-sdk-linux && \
-	cd /opt/android-sdk-linux && \
-	curl -L -O https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip && \
-	unzip sdk-tools-linux-3859397.zip
+# V1 --------------------------------------------------
 
-ENV ANDROID_HOME=/opt/android-sdk-linux
+# ENV ANDROID_HOME=/opt/android-sdk
+# ENV PATH $PATH:$ANDROID_HOME/tools
+# ENV PATH $PATH:$ANDROID_HOME/platform-tools
 
-RUN yes | /opt/android-sdk-linux/tools/bin/sdkmanager --licenses
+# RUN echo "Preparing Android SDK..." && \
+# 	mkdir -p $ANDROID_HOME && \
+# 	cd $ANDROID_HOME && \
+# 	curl -L -O https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip && \
+# 	unzip sdk-tools-linux-3859397.zip && \
+# 	rm -rf sdk-tools-linux-3859397.zip
 
-# RUN /opt/android-sdk-linux/tools/bin/sdkmanager --update --verbose
-RUN /opt/android-sdk-linux/tools/bin/sdkmanager --verbose "build-tools;24.0.3" "platforms;android-10"
+# RUN yes | $ANDROID_HOME/tools/bin/sdkmanager --licenses
+
+# # RUN $ANDROID_HOME/tools/bin/sdkmanager --update --verbose
+# RUN $ANDROID_HOME/tools/bin/sdkmanager --verbose	\
+# 	"platform-tools"								\
+# 	"emulator"										\
+# 	"build-tools;25.0.3"							\
+# 	"platforms;android-19"							\
+# 	"system-images;android-19;default;armeabi-v7a"	\
+# 	"system-images;android-19;default;x86"
+
+# ENV LD_LIBRARY_PATH=$ANDROID_HOME/emulator/lib64/qt/lib:$ANDROID_HOME/emulator/lib64/gles_mesa
+
+# RUN DEBIAN_FRONTEND=noninteractive apt-get -qq install -y libxdamage1 libxfixes3 qemu-kvm cpu-checker supervisor
+
+
+
+# V2 --------------------------------------------------
+
+# TODO: move to packages at top
+RUN echo "Preparing Debian packages..." && \
+	apt-get -qq update && \
+	DEBIAN_FRONTEND=noninteractive apt-get -qq install -y \
+	wget supervisor
+
+RUN wget -qO- http://dl.google.com/android/android-sdk_r23-linux.tgz | \
+    tar xvz -C /usr/local/ && \
+    mv /usr/local/android-sdk-linux /usr/local/android-sdk && \
+    chown -R root:root /usr/local/android-sdk/
+
+# Add android tools and platform tools to PATH
+ENV ANDROID_HOME /usr/local/android-sdk
+ENV PATH $PATH:$ANDROID_HOME/tools
+ENV PATH $PATH:$ANDROID_HOME/platform-tools
+
+RUN ( sleep 4 && while [ 1 ]; do sleep 1; echo y; done ) | android update sdk --no-ui --force -a --filter \
+    build-tools-24.0.3,tools,platform-tools,android-19,extra-android-m2repository,sys-img-x86-android-19,sys-img-armeabi-v7a-android-19 && \
+    echo "y" | android update adb
+
+# Create fake keymap file
+RUN mkdir /usr/local/android-sdk/tools/keymaps && \
+    touch /usr/local/android-sdk/tools/keymaps/en-us
+
+EXPOSE 22
+EXPOSE 5037
+EXPOSE 5554
+EXPOSE 5555
+EXPOSE 5900
+
+#RUN android update sdk --no-ui --all --filter "build-tools-24.0.3"
+
+# RUN yes | /usr/local/android-sdk/tools/bin/sdkmanager --licenses
+
+# COPY ./android/licenses $ANDROID_HOME/licenses
+
+# sdk-build-tools require the license to be accepted
+RUN mkdir -p $ANDROID_HOME/licenses && echo -e "\n8933bad161af4178b1185d1a37fbf41ea5269c55\n" > $ANDROID_HOME/licenses/android-sdk-license
 
 RUN mkdir -p /temp
 
 # download the most critical gradle dependencies for the build beforehand
 COPY ./shared/build.gradle /temp
+COPY ./android/AndroidManifest.xml /temp/src/main/AndroidManifest.xml
 WORKDIR /temp
-# RUN export MVN_PLATFORM_NAME=temp && \
-#     export MVN_ARCH_NAME=temp && \
-#     mvn verify -DskipTests || true
-RUN cd /temp && gradle || true
-
+RUN cd /temp && gradle --dry-run

docker/android/kill_supervisor.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+import sys
+import os
+import signal
+
+def write_stdout(s):
+   sys.stdout.write(s)
+   sys.stdout.flush()
+def write_stderr(s):
+   sys.stderr.write(s)
+   sys.stderr.flush()
+def main():
+   while 1:
+       write_stdout('READY\n')
+       line = sys.stdin.readline()
+       write_stdout('This line kills supervisor: ' + line);
+       try:
+               pidfile = open('/var/run/supervisord.pid','r')
+               pid = int(pidfile.readline());
+               os.kill(pid, signal.SIGQUIT)
+       except Exception as e:
+               write_stdout('Could not kill supervisor: ' + e.strerror + '\n')
+       write_stdout('RESULT 2\nOK')
+if __name__ == '__main__':
+   main()
+   import sys

docker/android/start-emulator.template.sh

@@ -0,0 +1,2 @@
+echo no | $ANDROID_HOME/tools/android create avd -f -n test -t android-19 --abi default/$ARCH
+echo no | $ANDROID_HOME/tools/emulator64-$ARCH -avd test -noaudio -no-window -gpu off -verbose -qemu -usbdevice tablet -vnc :0

docker/android/supervisord.template.conf

@@ -0,0 +1,47 @@
+[supervisord]
+nodaemon=true
+pidfile = /var/run/supervisord.pid
+
+[program:emulator]
+command=/bin/bash -c "/j2v8/docker/android/start-emulator.sh"
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+startsecs = 0
+autorestart = false
+startretries = 1
+numprocs=1
+stopasgroup=true
+killasgroup=true
+
+[eventlistener:emulator_exit]
+command=/j2v8/docker/android/kill_supervisor.py
+process_name=emulator
+events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:tests]
+command=/bin/bash -c "/j2v8/docker/android/wait-for-emulator.sh; $TEST_CMDS"
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+startsecs = 0
+autorestart = false
+startretries = 1
+numprocs=1
+stopasgroup=true
+killasgroup=true
+
+[eventlistener:tests_exit]
+command=/j2v8/docker/android/kill_supervisor.py
+process_name=tests
+events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0

docker/android/wait-for-emulator.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+adb wait-for-device
+
+A=$(adb shell getprop sys.boot_completed | tr -d '\r')
+
+echo "Waiting for emulator to boot (this can take several minutes)"
+
+while [ "$A" != "1" ]; do
+        sleep 5
+        echo "$(date +%T) waiting for emulator to boot..."
+        A=$(adb shell getprop sys.boot_completed | tr -d '\r')
+done
+
+adb shell input keyevent 82