Skip to content

Commit

Permalink
ARROW-6920: [Packaging] Build python 3.8 wheels
Browse files Browse the repository at this point in the history
adds python3.8 wheels

as far as I can tell python3.8 isn't available for Conda yet (conda-forge/python-feedstock#274), so that's will have to be added later

Closes #5685 from sjhewitt/arrow-6920 and squashes the following commits:

feac494 <Sutou Kouhei> Add a TODO comment
908a396 <Sutou Kouhei> Check Docker version for debug
43833a3 <Sutou Kouhei> Add Python 3.8 support to package metadata
f3f9bfa <Krisztián Szűcs> Install tensorflow for 32 bit unicode width py27
cd9ae83 <Sutou Kouhei> Install TensorFlow for Python 2.7 manually
5293f2b <Sutou Kouhei> Pin base image and remove needless virtualenvs
344eecd <Sutou Kouhei> Use apache/arrow-dev again
99cad41 <Sutou Kouhei> Use the uploaded image
f0c8e69 <Sutou Kouhei> Link to libpython only on Windows
260fa79 <Sutou Kouhei> Add a missing empty line
52761b4 <Sutou Kouhei> libpython.so may not exist
5e5977e <Sutou Kouhei> Don't link to other Python libraries on non Windows
e047d54 <Sutou Kouhei> Don't use local function
10dac67 <Terence D. Honles> move plasma subprocesses to top level functions
da300ac <Sutou Kouhei> Don't use TTY
c4526d8 <Sutou Kouhei> Use more recent numpy
7ace0cb <Sutou Kouhei> Ensure using the latest image
655f88c <Sutou Kouhei> Use Azure Pipelines instead of Travis CI
2ef6213 <Sutou Kouhei> Use "call conda.bat activate/deactivate"
1c8427e <Sutou Kouhei> Use conda without call
99251ea <Sutou Kouhei> Use "conda activate/deactivate"
f403b6f <Sutou Kouhei> Use the latest multibuild
c818840 <Sutou Kouhei> Remove m suffix
e9f1408 <Sutou Kouhei> Set up Python 3.8
2ab6732 <Simon Hewitt> ARROW-6920:  Build python 3.8 wheels

Lead-authored-by: Sutou Kouhei <kou@clear-code.com>
Co-authored-by: Terence D. Honles <terence@honles.com>
Co-authored-by: Krisztián Szűcs <szucs.krisztian@gmail.com>
Co-authored-by: Simon Hewitt <si@sjhewitt.co.uk>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>
  • Loading branch information
4 people committed Dec 8, 2019
1 parent 7102d7e commit 6c6754a
Show file tree
Hide file tree
Showing 18 changed files with 234 additions and 314 deletions.
5 changes: 2 additions & 3 deletions cpp/src/arrow/python/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,9 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL
set_property(SOURCE pyarrow.cc APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-cast-qual ")
endif()

set(ARROW_PYTHON_SHARED_LINK_LIBS arrow_shared ${PYTHON_OTHER_LIBS})

set(ARROW_PYTHON_SHARED_LINK_LIBS arrow_shared)
if(WIN32)
set(ARROW_PYTHON_SHARED_LINK_LIBS ${ARROW_PYTHON_SHARED_LINK_LIBS} ${PYTHON_LIBRARIES})
list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS ${PYTHON_LIBRARIES} ${PYTHON_OTHER_LIBS})
endif()

set(ARROW_PYTHON_INCLUDES ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS})
Expand Down
92 changes: 92 additions & 0 deletions dev/tasks/python-wheels/azure.linux.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

jobs:
- job: linux
pool:
vmImage: ubuntu-latest
timeoutInMinutes: 360
steps:
- task: CondaEnvironment@1
inputs:
packageSpecs: 'click github3.py jinja2 jira pygit2 ruamel.yaml setuptools_scm toolz'
installOptions: '-c conda-forge'
updateConda: false

- script: |
set -ex
git clone --no-checkout {{ arrow.remote }} arrow
git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
git -C arrow checkout FETCH_HEAD
git -C arrow submodule update --init --recursive
displayName: Clone arrow
- script: |
set -ex
docker -v
docker-compose -v
cd arrow
BUILD_IMAGE=centos-python-{{ wheel_tag }}
docker-compose pull $BUILD_IMAGE
# TODO(kou): Uncomment this when we resolve "ADD never use cache" problem.
# docker-compose build $BUILD_IMAGE
docker-compose run \
-e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} \
-e PYTHON_VERSION="{{ python_version }}" \
-e UNICODE_WIDTH="{{ unicode_width }}" \
$BUILD_IMAGE
displayName: Build wheel
# auditwheel does always exit with 0 so it is mostly for debugging
# purposes
- script: |
set -ex
cd arrow
docker run \
-v $(pwd):/arrow quay.io/pypa/{{ wheel_tag }}_x86_64 \
/bin/bash -c \
"auditwheel show /arrow/python/{{ wheel_tag }}/dist/*.whl"
displayName: Audit wheel
- script: |
set -ex
cd arrow
test_args=""
{%- if test_remove_system_libs %}
test_args="${test_args} --remove-system-libs"
{%- endif %}
{%- for image in test_docker_images %}
docker run \
--shm-size 2G \
-v $(pwd):/arrow \
-e WHEEL_TAG="{{ wheel_tag }}" \
{{ image }} \
/arrow/dev/tasks/python-wheels/manylinux-test.sh ${test_args}
{%- endfor %}
displayName: Test wheel
- script: |
set -ex
python arrow/dev/tasks/crossbow.py \
--queue-path . \
--queue-remote {{ queue.remote_url }} \
upload-artifacts \
--pattern "arrow/python/{{ wheel_tag }}/dist/*" \
--sha {{ task.branch }} \
--tag {{ task.tag }}
env:
CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN)
displayName: Upload packages as a GitHub release
88 changes: 0 additions & 88 deletions dev/tasks/python-wheels/travis.linux.yml

This file was deleted.

4 changes: 2 additions & 2 deletions dev/tasks/python-wheels/travis.osx.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ env:
- MACOSX_DEPLOYMENT_TARGET="10.9"

before_install:
- git clone https://github.com/matthew-brett/multibuild # TODO pin it
- git -C multibuild checkout 4e7a9396e9a50731bb83fc0d16bb98fb0c4032d7
- git clone https://github.com/matthew-brett/multibuild
- git -C multibuild checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa

- git clone --no-checkout {{ arrow.remote }} arrow
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
Expand Down
10 changes: 5 additions & 5 deletions dev/tasks/python-wheels/win-build.bat
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,14 @@
@rem create conda environment for compiling
conda update --yes --quiet conda

call conda create -n wheel-build -q -y -c conda-forge ^
conda create -n wheel-build -q -y -c conda-forge ^
--file=%ARROW_SRC%\ci\conda_env_cpp.yml ^
--file=%ARROW_SRC%\ci\conda_env_gandiva.yml ^
python=%PYTHON_VERSION% ^
numpy=%NUMPY_VERSION% ^
|| exit /B

call activate wheel-build
call conda.bat activate wheel-build

@rem Cannot use conda_env_python.yml here because conda-forge has
@rem ceased providing up-to-date packages for Python 3.5
Expand Down Expand Up @@ -84,17 +84,17 @@ pushd %ARROW_SRC%\python
python setup.py bdist_wheel || exit /B
popd

call deactivate
call conda.bat deactivate

set ARROW_TEST_DATA=%ARROW_SRC%\testing\data

@rem test the wheel
@rem TODO For maximum reliability, we should test in a plain virtualenv instead.
call conda create -n wheel-test -c conda-forge -q -y ^
conda create -n wheel-test -c conda-forge -q -y ^
--file %ARROW_SRC%\ci\conda_env_python.yml ^
python=%PYTHON_VERSION% ^
numpy=%NUMPY_VERSION% || exit /B
call activate wheel-test
call conda.bat activate wheel-test

@rem install the built wheel
pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B
Expand Down
Loading

0 comments on commit 6c6754a

Please sign in to comment.