Use sccache with s3 to speed up builds

New binaries with Azure support

.circleci/config.yml

@@ -14,13 +14,19 @@ jobs:
           command: script/bootstrap
       - run:
           name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
+      - run:
+          name: Check sccache stats before build
+          command: script/sccache -s
       - run:
           name: Build shared library
           command: script/build -t $TARGET_ARCH -c $COMPONENT
       - run:
           name: Build FFmpeg
           command: script/build -t $TARGET_ARCH -c ffmpeg
+      - run:
+          name: Check sccache stats after build
+          command: script/sccache -s
       - run:
           name: Create distribution
           command: script/create-dist -t $TARGET_ARCH -c $COMPONENT
@@ -50,7 +56,7 @@ jobs:
           command: script/bootstrap
       - run:
           name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
       - run:
           name: Build static library
           command: script/build -t $TARGET_ARCH -c $COMPONENT
@@ -86,7 +92,7 @@ jobs:
           command: script/bootstrap
       - run:
           name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
       - run:
           name: Build shared library
           command: script/build -t $TARGET_ARCH -c $COMPONENT
@@ -122,7 +128,7 @@ jobs:
           command: script/bootstrap
       - run:
           name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
       - run:
           name: Build static library
           command: script/build -t $TARGET_ARCH -c $COMPONENT
@@ -158,7 +164,7 @@ jobs:
           command: script/bootstrap
       - run:
           name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
       - run:
           name: Build mksnapshot
           command: script/build -t $TARGET_ARCH -c native_mksnapshot
@@ -202,7 +208,7 @@ jobs:
           command: script/bootstrap
       - run:
           name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
       - run:
           name: Build static library
           command: script/build -t $TARGET_ARCH -c $COMPONENT
@@ -238,7 +244,7 @@ jobs:
           command: script/bootstrap
       - run:
           name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
       - run:
           name: Build mksnapshot
           command: script/build -t $TARGET_ARCH -c native_mksnapshot
@@ -282,7 +288,7 @@ jobs:
           command: script/bootstrap
       - run:
           name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
       - run:
           name: Build static library
           command: script/build -t $TARGET_ARCH -c $COMPONENT
@@ -318,7 +324,7 @@ jobs:
           command: script/bootstrap
       - run:
           name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
       - run:
           name: Build shared library
           command: script/build -t $TARGET_ARCH -c $COMPONENT
@@ -354,7 +360,7 @@ jobs:
           command: script/bootstrap
       - run:
           name: Update
-          command: script/update --clean -t $TARGET_ARCH
+          command: script/update --clean -t $TARGET_ARCH --use-bundled-sccache
       - run:
           name: Build static library
           command: script/build -t $TARGET_ARCH -c $COMPONENT

script/cibuild

@@ -18,6 +18,7 @@ S3_CREDENTIALS_FILE = os.path.join(JENKINS_ROOT, 'config', 's3credentials')
 def main():
   args = parse_args()
   skip_upload = args.skip_upload
+  use_sccache = args.use_sccache
 
   if (not skip_upload and
      not 'LIBCHROMIUMCONTENT_S3_ACCESS_KEY' in os.environ and
@@ -34,23 +35,25 @@ def main():
 
   if 'TARGET_ARCH' in os.environ:
     args = ['-t', os.environ['TARGET_ARCH']]
-    return run_ci(args, skip_upload, os.environ['COMPONENT'])
+    return run_ci(args, skip_upload, use_sccache, os.environ['COMPONENT'])
 
   if sys.platform in ['win32', 'cygwin']:
-    return (run_ci(['-t', 'x64'], skip_upload) or
-            run_ci(['-t', 'ia32'], skip_upload))
+    return (run_ci(['-t', 'x64'], skip_upload, use_sccache) or
+            run_ci(['-t', 'ia32'], skip_upload, use_sccache))
   elif sys.platform == 'linux2':
-    return (run_ci(['-t', 'x64'], skip_upload) or
-            run_ci(['-t', 'ia32'], skip_upload) or
-            run_ci(['-t', 'arm'], skip_upload) or
-            run_ci(['-t', 'arm64'], skip_upload))
+    return (run_ci(['-t', 'x64'], skip_upload, use_sccache) or
+            run_ci(['-t', 'ia32'], skip_upload, use_sccache) or
+            run_ci(['-t', 'arm'], skip_upload, use_sccache) or
+            run_ci(['-t', 'arm64'], skip_upload, use_sccache))
   else:
-    return run_ci([], skip_upload)
+    return run_ci([], skip_upload, use_sccache)
 
 def parse_args():
   parser = argparse.ArgumentParser(description='Run CI build')
   parser.add_argument('-s', '--skip_upload', action='store_true',
                       help='Skip uploading to S3')
+  parser.add_argument('--use_sccache', action='store_true',
+                      help='Use sccache binary stored in with the libcc repo.')
   return parser.parse_args()
 
 def copy_to_environment(credentials_file):
@@ -72,9 +75,12 @@ def os_version():
     return platform.platform()
 
 
-def run_ci(args, skip_upload, component=None):
+def run_ci(args, skip_upload, use_sccache, component=None):
   build_args = []
   component_args = []
+  update_args = ['--clean'] + args
+  if use_sccache:
+    update_args += ['--use-bundled-sccache']
   if component is not None:
     component_args = ['-c', component]
     build_args += component_args + ['ffmpeg']
@@ -88,7 +94,7 @@ def run_ci(args, skip_upload, component=None):
     os.environ.update(vs_env)
 
   return (run_script('bootstrap') or
-          run_script('update', ['--clean'] + args) or
+          run_script('update', update_args) or
           run_script('build', args + build_args) or
           run_script('create-dist', args + component_args) or
           run_script('upload', args, skip_upload))

script/cibuild.ps1

@@ -1,4 +1,4 @@
-param([switch]$skipUpload)
+param([switch]$skipUpload, [switch]$useSccache)
 function Run-Command([scriptblock]$Command, [switch]$Fatal, [switch]$Quiet) {
   $output = ""
   try {
@@ -33,8 +33,13 @@ function Run-Command([scriptblock]$Command, [switch]$Fatal, [switch]$Quiet) {
 }
 
 Write-Output ""
+$CommandLine = "python .\script\cibuild"
 if ($skipUpload) {
-  Run-Command -Fatal { python .\script\cibuild --skip_upload }
-} else {
-  Run-Command -Fatal { python .\script\cibuild }
+  $CommandLine += " --skip_upload"
 }
+
+if ($useSccache) {
+  $CommandLine += " --use_sccache"
+}
+$CICommand = [ScriptBlock]::Create($CommandLine)
+Run-Command -Fatal $CICommand

script/lib/sccache.py

@@ -5,7 +5,7 @@
 from config import TOOLS_DIR
 
 
-VERSION = '0.2.6'
+VERSION = 'aad2120'
 SUPPORTED_PLATFORMS = {
   'cygwin': 'windows',
   'darwin': 'mac',

script/sccache

@@ -1,12 +1,34 @@
 #!/usr/bin/env python
 
+import argparse
+import os
 import sys
 
 import lib.sccache as sccache
 
+def parse_args():
+  parser = argparse.ArgumentParser(description='Run sccache`')
+  parser.add_argument('--azure_container', help='Name of azure container to use')
+  parser.add_argument('--azure_connection', help='Name of azure connection to use')
+  parser.add_argument('-s', '--show-stats', action='store_true', help='Show cache statistics')
+  parser.add_argument('--start-server', action='store_true', help='start background server')
+  parser.add_argument('--stop-server', action='store_true', help='stop background server')
+  parser.add_argument('-V', '--version', action='store_true', help='Prints version information')
+  parser.add_argument('-z', '--zero-stats', action='store_true', help='zero statistics counters')
+  return parser.parse_args()
 
 def main(args_list):
-  return sccache.run(*args_list)
+  args = parse_args()
+  sccache_args = list(args_list)
+  if args.azure_container:
+    os.environ['SCCACHE_AZURE_BLOB_CONTAINER'] = args.azure_container
+    sccache_args.remove('--azure_container')
+    sccache_args.remove(args.azure_container)
+  if args.azure_connection:
+    os.environ['SCCACHE_AZURE_CONNECTION_STRING'] = args.azure_connection
+    sccache_args.remove('--azure_connection')
+    sccache_args.remove(args.azure_connection)
+  return sccache.run(*sccache_args)
 
 
 if __name__ == '__main__':

tools/sccache/README.md

@@ -3,7 +3,7 @@
 Shared Compilation Cache  
 https://github.com/mozilla/sccache
 
-### Building a portable binary version 0.2.6 on Mac
+### Building a portable binary version on Mac
 
 The goal is to build a binary that would not require `openssl` to be installed in the system. The project's docs provide [build instructions](https://github.com/mozilla/sccache#building-portable-binaries)  for that but they're quite laconic.
 
@@ -12,8 +12,8 @@ Old versions might not be available via [Homebrew](https://brew.sh) but `brew in
 2. Clone the repo:  
 `git clone https://github.com/mozilla/sccache.git`
 3. `cd sccache`
-4. Checkout version 0.2.6:  
-`git fetch --tags && git checkout 0.2.6`
+4. Checkout version aad212037ce6340839686da8a1bdf5455b143b1c:  
+`git fetch --tags && git checkout aad212037ce6340839686da8a1bdf5455b143b1c`
 5. `export OPENSSL_STATIC=yes`
 6. Build the project in a release mode:  
 `cargo build --release`.

vsts.yml

@@ -47,18 +47,18 @@ phases:
 
   - bash: |
      echo "===Updating for $TARGET_ARCH===" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
-     script/update --clean -t $TARGET_ARCH > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+     script/update --clean -t $TARGET_ARCH --use-bundled-sccache > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
     name: Update
 
   - bash: |
+     script/sccache --start-server --azure_container $(SCCACHE_AZURE_BLOB_CONTAINER) --azure_connection "$(SCCACHE_AZURE_CONNECTION_STRING)" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+     script/sccache -s > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
      echo "===Building $COMPONENT for $TARGET_ARCH===" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
      script/build -t $TARGET_ARCH -c $COMPONENT > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
-    name: Build_library
-
-  - bash: |
      echo "===Building ffmpeg for $TARGET_ARCH===" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
      script/build -t $TARGET_ARCH -c ffmpeg > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
-    name: Build_ffmpeg
+     script/sccache --stop-server > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
+    name: Build
 
   - bash: |
      echo "===Create $COMPONENT distribution for $TARGET_ARCH===" > >(tee -a buildlog.txt) 2> >(tee -a buildlog.txt >&2)
@@ -98,3 +98,4 @@ phases:
       fi
 
   - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    condition: always()