[dev/tooling] Add script to build wheels

.circleci/config.yml

@@ -877,28 +877,17 @@ jobs:
       - *save_cache_step
 
   deploy_dev:
-    # build only the nightly package
-    docker:
-      - image: circleci/python:3.6
-    resource_class: *resource_class
+    # DEV: Use CircleCI machine instead of docker since we need to be able to run docker commands directly
+    # DEV: Use machine instead of setup_remote_docker since we need to supporting mounting volumes for docker commands
+    machine:
+      docker_layer_caching: true
     steps:
       - checkout
-      - run: sudo apt-get -y install rake
-      - run: sudo pip install mkwheelhouse sphinx awscli wrapt
+      - run: sudo apt-get -y install rake python3 python3-pip
+      - run: sudo pip install mkwheelhouse sphinx awscli
       - run: S3_DIR=trace-dev rake release:docs
       - run: VERSION_SUFFIX=$CIRCLE_BRANCH$CIRCLE_BUILD_NUM S3_DIR=trace-dev rake release:wheel
 
-  deploy_experimental:
-    # build the *-dev branch releasing development docs
-    docker:
-      - image: circleci/python:3.6
-    resource_class: *resource_class
-    steps:
-      - checkout
-      - run: sudo apt-get -y install rake
-      - run: sudo pip install mkwheelhouse sphinx awscli wrapt
-      - run: VERSION_SUFFIX=$CIRCLE_BRANCH$CIRCLE_BUILD_NUM S3_DIR=trace-dev rake release:wheel
-
   jinja2:
     docker:
       - *test_runner
@@ -1207,10 +1196,4 @@ workflows:
             - wait_all_tests
           filters:
             branches:
-              only: /(master)/
-      - deploy_experimental:
-          requires:
-            - wait_all_tests
-          filters:
-            branches:
-              only: /(.*-dev)/
+              only: /(master|.*-dev)/

Rakefile

@@ -12,13 +12,12 @@ S3_BUCKET = "pypi.datadoghq.com"
 
 desc "release the a new wheel"
 task :'release:wheel' do
-  # Use mkwheelhouse to build the wheel, push it to S3 then update the repo index
-  # If at some point, we need only the 2 first steps:
-  #  - python setup.py bdist_wheel
-  #  - aws s3 cp dist/*.whl s3://pypi.datadoghq.com/#{s3_dir}/
   fail "Missing environment variable S3_DIR" if !S3_DIR or S3_DIR.empty?
 
-  # Use custom mkwheelhouse script to build and upload an sdist to S3 bucket
+  # Use custom script to build wheels and source distribution into dist/
+  sh "scripts/build-dist"
+
+  # Use custom `mkwheelhouse` to upload wheels and source distribution from dist/ to S3 bucket
   sh "scripts/mkwheelhouse"
 end
 
@@ -66,6 +65,7 @@ namespace :pypi do
 
   task :build => :clean do
     puts "building release in #{RELEASE_DIR}"
+    # TODO: Use `scripts/build-dist` instead to build sdist and wheels
     sh "python setup.py -q sdist -d #{RELEASE_DIR}"
   end
 

scripts/build-dist

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -ex
+
+# DEV: `${VERSION_SUFFIX-}` means don't fail if it doesn't exist, use empty string instead (which is fine)
+echo "Building with version suffix: ${VERSION_SUFFIX-}"
+
+# Determine where "../dist" is
+PARENT_DIR="$( cd "$(dirname "${0}")/../" ; pwd -P )"
+DIST_DIR="${PARENT_DIR}/dist"
+
+# Remove and recreate dist/ directory where our release wheels/source distribution will go
+rm -rf "${DIST_DIR}"
+mkdir "${DIST_DIR}"
+
+build_script=$(cat <<'EOF'
+set -ex
+
+# Build linux wheels from the source distribution we created
+for PYBIN in /opt/python/*/bin;
+do
+  "${PYBIN}/pip" wheel --no-deps /dd-trace-py/dist/*.tar.gz -w /dd-trace-py/dist
+done
+
+# Build manylinux wheels from the linux wheels we just created
+for whl in /dd-trace-py/dist/*-linux_${ARCH}.whl;
+do
+  auditwheel repair "${whl}" -w /dd-trace-py/dist
+done
+EOF
+)
+
+# First build a source distribution for our package
+python setup.py sdist --dist-dir dist
+
+# Build x86_64 linux and manylinux wheels
+# DEV: `${VERSION_SUFFIX-}` means don't fail if it doesn't exist, use empty string instead (which is fine)
+docker run -it --rm -v "${PARENT_DIR}:/dd-trace-py" -e "ARCH=x86_64" -e "VERSION_SUFFIX=${VERSION_SUFFIX-}" quay.io/pypa/manylinux1_x86_64 /bin/bash -c "${build_script}"
+
+# Build i686 linux and manylinux wheels
+# DEV: `${VERSION_SUFFIX-}` means don't fail if it doesn't exist, use empty string instead (which is fine)
+docker run -it --rm -v "${PARENT_DIR}:/dd-trace-py" -e "ARCH=i686" -e "VERSION_SUFFIX=${VERSION_SUFFIX-}" quay.io/pypa/manylinux1_i686 linux32 /bin/bash -c "${build_script}"

scripts/mkwheelhouse

@@ -1,25 +1,13 @@
 #!/usr/bin/env python
 import os
 import shutil
-import tempfile
 
 import mkwheelhouse
 
 S3_BUCKET = 'pypi.datadoghq.com'
 S3_DIR = os.environ['S3_DIR']
 
 
-# DEV: This is the same `mkwheelhouse.build_wheels` except we are running `python setup.py sdist` instead
-def build_sdist():
-    build_dir = tempfile.mkdtemp(prefix='mkwheelhouse-')
-    args = [
-        'python', 'setup.py', 'sdist',
-        '--dist-dir', build_dir,
-    ]
-    mkwheelhouse.spawn(args)
-    return build_dir
-
-
 # DEV: This is the same as `mkwheelhouse.Bucket.make_index`, except we include `*.whl` and `*.tar.gz` files
 def make_index(bucket):
     doc, tag, text = mkwheelhouse.yattag.Doc().tagtext()
@@ -46,10 +34,12 @@ def run():
         bucket.put('<!DOCTYPE html><html></html>', 'index.html', acl=acl)
 
     index_url = bucket.generate_url('index.html')
-    build_dir = build_sdist()
+
+    # We have already built the wheels and source distribution in ./dist/
+    build_dir = os.path.abspath('./dist/')
+
     bucket.sync(build_dir, acl=acl)
     bucket.put(make_index(bucket), key='index.html', acl=acl)
-    shutil.rmtree(build_dir)
     print('mkwheelhouse: index written to', index_url)