diff --git a/ci/docker/linux-apt-python-313-freethreading.dockerfile b/ci/docker/linux-apt-python-313-freethreading.dockerfile new file mode 100644 index 0000000000000..f5505e67f00bb --- /dev/null +++ b/ci/docker/linux-apt-python-313-freethreading.dockerfile @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +RUN apt-get update -y -q && \ + apt install -y -q --no-install-recommends software-properties-common gpg-agent && \ + add-apt-repository -y ppa:deadsnakes/ppa && \ + apt-get update -y -q && \ + apt install -y -q --no-install-recommends python3.13-dev python3.13-nogil python3.13-venv && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +COPY python/requirements-build.txt \ + python/requirements-test.txt \ + /arrow/python/ + +ENV ARROW_PYTHON_VENV /arrow-dev +RUN python3.13t -m venv ${ARROW_PYTHON_VENV} +RUN ${ARROW_PYTHON_VENV}/bin/python -m pip install -U pip setuptools wheel +RUN ${ARROW_PYTHON_VENV}/bin/python -m pip install \ + --pre \ + --prefer-binary \ + --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" \ + -r arrow/python/requirements-build.txt \ + -r arrow/python/requirements-test.txt + +# We want to run the PyArrow test suite with the GIL disabled, but cffi +# (more precisely, the `_cffi_backend` module) currently doesn't declare +# itself safe to run without the GIL. +# Therefore set PYTHON_GIL to 0. +ENV ARROW_ACERO=ON \ + ARROW_BUILD_STATIC=OFF \ + ARROW_BUILD_TESTS=OFF \ + ARROW_BUILD_UTILITIES=OFF \ + ARROW_COMPUTE=ON \ + ARROW_CSV=ON \ + ARROW_DATASET=ON \ + ARROW_FILESYSTEM=ON \ + ARROW_GDB=ON \ + ARROW_HDFS=ON \ + ARROW_JSON=ON \ + ARROW_USE_GLOG=OFF \ + PYTHON_GIL=0 diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 9ded6ee41ab4b..c1c15a3ff73fd 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1257,6 +1257,14 @@ tasks: PYTHON: "3.10" image: conda-python-cython2 + test-ubuntu-22.04-python-313-freethreading: + ci: github + template: docker-tests/github.linux.yml + params: + env: + UBUNTU: 22.04 + image: ubuntu-python-313-freethreading + test-debian-12-python-3-amd64: ci: github template: docker-tests/github.linux.yml diff --git a/docker-compose.yml b/docker-compose.yml index 36cf150f25f39..8721eef524a19 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -149,6 +149,7 @@ x-hierarchy: - ubuntu-lint - ubuntu-python - ubuntu-python-sdist-test + - ubuntu-python-313-freethreading - ubuntu-r - ubuntu-r-only-r - ubuntu-cpp-bundled @@ -1085,6 +1086,32 @@ services: /arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/python_sdist_test.sh /arrow" + ############################ Python free-threading ########################## + + ubuntu-python-313-freethreading: + # Usage: + # docker-compose build ubuntu-cpp + # docker-compose build ubuntu-python-313-freethreading + # docker-compose run --rm ubuntu-python-313-freethreading + # Parameters: + # ARCH: amd64, arm64v8, ... + # UBUNTU: 20.04, 22.04, 24.04 + image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-313-freethreading + build: + context: . + dockerfile: ci/docker/linux-apt-python-313-freethreading.dockerfile + cache_from: + - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-313-freethreading + args: + base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp + shm_size: *shm-size + environment: + <<: [*common, *ccache] + # Bundled build of OpenTelemetry needs a git client + ARROW_WITH_OPENTELEMETRY: "OFF" + volumes: *ubuntu-volumes + command: *python-command + ############################ Python wheels ################################## # See available versions at: diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index eda4ff4ca5f07..912719b20f0e4 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -21,6 +21,13 @@ cmake_minimum_required(VERSION 3.16) project(pyarrow) +# This is needed for 3.13 free-threading. CMake used to add Python +# include directories with `-isystem`, which led to some Python-internal +# includes to resolve to normal 3.13 includes (cause -isystem includes +# are searched after system directories), instead of 3.13-freethreading, +# which in turn meant that Py_GIL_DISABLED was not set. +set(CMAKE_NO_SYSTEM_FROM_IMPORTED ON) + set(PYARROW_VERSION "18.0.0-SNAPSHOT") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" PYARROW_BASE_VERSION "${PYARROW_VERSION}")