Run scons in CI (#14)

* try to run scons in azure pipelines

* sudo

* install capnp

* does this run

* also clean test runner

* remove makefiles

* this should build

Author: pd0wm

.dockerignore

@@ -0,0 +1 @@
+.sconsign.dblite

.gitignore

@@ -9,4 +9,5 @@ libcereal*.a
 libmessaging.*
 libmessaging_shared.*
 services.h
+.sconsign.dblite
 

Dockerfile

@@ -0,0 +1,19 @@
+from ubuntu:16.04
+
+RUN apt-get update && apt-get install -y libzmq3-dev clang wget git autoconf libtool curl make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl
+
+RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
+ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
+RUN pyenv install 3.7.3
+RUN pyenv global 3.7.3
+RUN pyenv rehash
+RUN pip3 install pyyaml==5.1.2 Cython==0.29.14 scons==3.1.1 pycapnp==0.6.4
+
+WORKDIR /project/cereal
+COPY install_capnp.sh .
+RUN ./install_capnp.sh
+
+ENV PYTHONPATH=/project
+
+COPY . .
+RUN scons -c && scons -j$(nproc)

Makefile

@@ -1,15 +1,18 @@
 Import('env', 'arch', 'zmq')
 
+gen_dir = Dir('gen')
+messaging_dir = Dir('messaging')
+
 # TODO: remove src-prefix and cereal from command string. can we set working directory?
-env.Command(["gen/c/include/c++.capnp.h", "gen/c/include/java.capnp.h"], [], "mkdir -p cereal/gen/c/include && touch $TARGETS")
+env.Command(["gen/c/include/c++.capnp.h", "gen/c/include/java.capnp.h"], [], "mkdir -p " + gen_dir.path + "/c/include && touch $TARGETS")
 env.Command(
   ['gen/c/car.capnp.c', 'gen/c/log.capnp.c', 'gen/c/car.capnp.h', 'gen/c/log.capnp.h'],
   ['car.capnp', 'log.capnp'],
-  'capnpc $SOURCES --src-prefix=cereal -o c:cereal/gen/c/')
+  'capnpc $SOURCES --src-prefix=cereal -o c:' + gen_dir.path + '/c/')
 env.Command(
   ['gen/cpp/car.capnp.c++', 'gen/cpp/log.capnp.c++', 'gen/cpp/car.capnp.h', 'gen/cpp/log.capnp.h'],
   ['car.capnp', 'log.capnp'],
-  'capnpc $SOURCES --src-prefix=cereal -o c++:cereal/gen/cpp/')
+  'capnpc $SOURCES --src-prefix=cereal -o c++:' + gen_dir.path + '/cpp/')
 
 env.Library('cereal', [
     'gen/c/car.capnp.c',
@@ -18,10 +21,12 @@ env.Library('cereal', [
     'gen/cpp/log.capnp.c++',
   ])
 
+
+cereal_dir = Dir('.')
 env.Command(
   ['services.h'],
   ['service_list.yaml', 'services.py'],
-  'python3 cereal/services.py > $TARGET')
+  'python3 ' + cereal_dir.path + '/services.py > $TARGET')
 
 messaging_deps = [
   'messaging/messaging.cc',
@@ -39,12 +44,16 @@ if arch == "aarch64":
 messaging_shared_lib = env.SharedLibrary('messaging_shared', messaging_deps, LIBS=shared_lib_shared_lib)
 env.Command(['messaging/messaging.so'], [messaging_shared_lib], "chmod 777 $SOURCES && ln -sf `realpath $SOURCES` $TARGET")
 
-env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=['messaging', 'zmq'])
+env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging_lib, 'zmq'])
 
 # different target?
 #env.Program('messaging/demo', ['messaging/demo.cc'], LIBS=['messaging', 'zmq'])
 
+
 env.Command(['messaging/messaging_pyx.so'],
   [messaging_lib, 'messaging/messaging_pyx_setup.py', 'messaging/messaging_pyx.pyx', 'messaging/messaging.pxd'],
-  "cd cereal/messaging && python3 messaging_pyx_setup.py build_ext --inplace")
+  "cd " + messaging_dir.path + " && python3 messaging_pyx_setup.py build_ext --inplace")
+
 
+if GetOption('test'):
+  env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging_lib])

SConscript

@@ -0,0 +1,49 @@
+import os
+import subprocess
+
+zmq = 'zmq'
+arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+
+cereal_dir = Dir('.')
+
+cpppath = [
+    cereal_dir,
+    '/usr/lib/include',
+]
+
+AddOption('--test',
+          action='store_true',
+          help='build test files')
+
+AddOption('--asan',
+          action='store_true',
+          help='turn on ASAN')
+
+ccflags_asan = ["-fsanitize=address", "-fno-omit-frame-pointer"] if GetOption('asan') else []
+ldflags_asan = ["-fsanitize=address"] if GetOption('asan') else []
+
+env = Environment(
+  ENV=os.environ,
+  CC='clang',
+  CXX='clang++',
+  CCFLAGS=[
+    "-g",
+    "-fPIC",
+    "-O2",
+    "-Werror=implicit-function-declaration",
+    "-Werror=incompatible-pointer-types",
+    "-Werror=int-conversion",
+    "-Werror=return-type",
+    "-Werror=format-extra-args",
+  ] + ccflags_asan,
+  LDFLAGS=ldflags_asan,
+  LINKFLAGS=ldflags_asan,
+
+  CFLAGS="-std=gnu11",
+  CXXFLAGS="-std=c++14",
+  CPPPATH=cpppath,
+)
+
+
+Export('env', 'zmq', 'arch')
+SConscript(['SConscript'])

SConstruct

@@ -3,10 +3,7 @@ pool:
 
 steps:
 - script: |
-    cd messaging
-    ASAN=1 make test_runner
-    ./test_runner -r junit -o tests.xml
+    docker build -t cereal .
+    docker run cereal bash -c "scons --test --asan -j$(nproc) && messaging/test_runner"
+
   displayName: 'Run Tests'
-- task: PublishTestResults@2
-  inputs:
-    testResultsFiles: 'messaging/tests.xml'

azure-pipelines.yml

@@ -8,7 +8,8 @@ tar xvf capnproto-c++-${VERSION}.tar.gz
 cd capnproto-c++-${VERSION}
 CXXFLAGS="-fPIC" ./configure
 
-make -j4
+make -j$(nproc)
+make install
 
 # manually build binaries statically
 g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnp src/capnp/compiler/module-loader.o src/capnp/compiler/capnp.o  ./.libs/libcapnpc.a ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread
@@ -18,7 +19,6 @@ g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -D
 g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnpc-capnp src/capnp/compiler/capnpc-capnp.o  ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread
 
 cp .libs/capnp /usr/local/bin/
-ln -s /usr/local/bin/capnp /usr/local/bin/capnpc
 cp .libs/capnpc-c++ /usr/local/bin/
 cp .libs/capnpc-capnp /usr/local/bin/
 cp .libs/*.a /usr/local/lib
@@ -30,7 +30,8 @@ cd c-capnproto
 git submodule update --init --recursive
 autoreconf -f -i -s
 CXXFLAGS="-fPIC" ./configure
-make -j4
+make -j$(nproc)
+make install
 
 # manually build binaries statically
 gcc -fPIC -o .libs/capnpc-c compiler/capnpc-c.o compiler/schema.capnp.o compiler/str.o  ./.libs/libcapnp_c.a

install_capnp.sh

@@ -7,3 +7,4 @@ test_runner
 *.a
 *.so
 messaging_pyx.cpp
+build/

messaging/.gitignore

@@ -1,10 +1,33 @@
 import os
 import subprocess
+import sysconfig
 from distutils.core import Extension, setup  # pylint: disable=import-error,no-name-in-module
 
 from Cython.Build import cythonize
+from Cython.Distutils import build_ext
+
+
+def get_ext_filename_without_platform_suffix(filename):
+  name, ext = os.path.splitext(filename)
+  ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
+
+  if ext_suffix == ext:
+    return filename
+
+  ext_suffix = ext_suffix.replace(ext, '')
+  idx = name.find(ext_suffix)
+
+  if idx == -1:
+    return filename
+  else:
+    return name[:idx] + ext
+
+
+class BuildExtWithoutPlatformSuffix(build_ext):
+  def get_ext_filename(self, ext_name):
+    filename = super().get_ext_filename(ext_name)
+    return get_ext_filename_without_platform_suffix(filename)
 
-from common.cython_hacks import BuildExtWithoutPlatformSuffix
 
 sourcefiles = ['messaging_pyx.pyx']
 extra_compile_args = ["-std=c++11"]