array_record pip package

PiperOrigin-RevId: 490568092

.bazelrc

@@ -1,8 +1,10 @@
 build -c opt
 build --cxxopt=-std=c++17
 build --host_cxxopt=-std=c++17
+build --experimental_repo_remote_exec
 
 # TODO(fchern): Use non-hardcode path.
 build --action_env=PYTHON_BIN_PATH="/usr/bin/python3"
+build --action_env=PYTHON_LIB_PATH="/usr/lib/python3"
 build --repo_env=PYTHON_BIN_PATH="/usr/bin/python3"
 build --python_path="/usr/bin/python3"

WORKSPACE

@@ -52,6 +52,7 @@ http_archive(
   name = "pybind11",
   build_file = "@pybind11_bazel//:pybind11.BUILD",
   strip_prefix = "pybind11-2.9.2",
+  sha256 = "d1646e6f70d8a3acb2ddd85ce1ed543b5dd579c68b8fb8e9638282af20edead8",
   urls = ["https://github.com/pybind/pybind11/archive/refs/tags/v2.9.2.zip"],
 )
 load("@pybind11_bazel//:python_configure.bzl", "python_configure")
@@ -133,3 +134,16 @@ http_archive(
     strip_prefix = "highwayhash-276dd7b4b6d330e4734b756e97ccfb1b69cc2e12",
     urls = ["https://github.com/google/highwayhash/archive/276dd7b4b6d330e4734b756e97ccfb1b69cc2e12.zip"],  # 2019-02-22
 )
+
+
+http_archive(
+    name = "org_tensorflow",
+    strip_prefix = "tensorflow-2.10.0",
+    sha256 = "d79a95ede8305f14a10dd0409a1e5a228849039c19ccfb90dfe8367295fd04e0",
+    urls = ["https://github.com/tensorflow/tensorflow/archive/v2.10.0.zip"],
+)
+
+# This import (along with the org_tensorflow archive) is necessary to provide the devtoolset-9 toolchain
+load("@org_tensorflow//tensorflow/tools/toolchains/remote_config:configs.bzl", "initialize_rbe_configs")  # buildifier: disable=load-on-top
+
+initialize_rbe_configs()

oss/README.md

@@ -0,0 +1,10 @@
+# Steps to build a new array_record pip package
+
+1. Update the version number in setup.py
+
+2. In workspace, run
+```
+./array_record/oss/runner.sh
+```
+
+3. Wheels are in /tmp/array_record, upload to PyPI

oss/build.Dockerfile

@@ -0,0 +1,49 @@
+# Constructs the environment within which we will build the array_record pip wheels.
+#
+# From /tmp/array_record,
+# ❯ DOCKER_BUILDKIT=1 docker build -t array_record:latest - < oss/build.Dockerfile
+# ❯ docker run --rm -it -v /tmp/array_recor:/tmp/array_reco \
+#      array_record:latest bash
+
+ARG base_image="tensorflow/build:2.10-python3.9"
+FROM $base_image
+LABEL maintainer="Array_record team <array-record@google.com>"
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install supplementary Python interpreters
+RUN mkdir /tmp/python
+RUN --mount=type=cache,target=/var/cache/apt \
+  apt update && \
+  apt install -yqq \
+    apt-utils \
+    build-essential \
+    checkinstall \
+    libffi-dev \
+    neovim
+
+# 3.9 is the built-in interpreter version in this image.
+RUN for v in 3.8.15; do \
+    wget "https://www.python.org/ftp/python/$v/Python-${v}.tar.xz" && \
+    rm -rf "/tmp/python${v}" && mkdir -p "/tmp/python${v}" && \
+    tar xvf "Python-${v}.tar.xz" -C "/tmp/python${v}" && \
+    cd "/tmp/python${v}/Python-${v}" && \
+    ./configure 2>&1 >/dev/null && \
+    make -j8 altinstall 2>&1 >/dev/null && \
+    ln -sf "/usr/local/bin/python${v%.*}" "/usr/bin/python${v%.*}"; \
+  done
+
+# For each python interpreter, install pip dependencies needed for array_record
+RUN --mount=type=cache,target=/root/.cache \
+  for p in 3.8 3.9; do \
+    python${p} -m pip install -U pip && \
+    python${p} -m pip install -U \
+      absl-py \
+      auditwheel \
+      patchelf \
+      setuptools \
+      twine \
+      wheel; \
+  done
+
+WORKDIR "/tmp/array_record"

oss/build_whl.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# build wheel for python version specified in $PYTHON
+
+set -e -x
+
+export PYTHON_MINOR_VERSION="${PYTHON_MINOR_VERSION}"
+PYTHON="python3${PYTHON_MINOR_VERSION:+.$PYTHON_MINOR_VERSION}"
+
+function write_to_bazelrc() {
+  echo "$1" >> .bazelrc
+}
+
+function main() {
+  # Remove .bazelrc if it already exists
+  [ -e .bazelrc ] && rm .bazelrc
+
+  write_to_bazelrc "build -c opt"
+  write_to_bazelrc "build --cxxopt=-std=c++17"
+  write_to_bazelrc "build --host_cxxopt=-std=c++17"
+  write_to_bazelrc "build --linkopt=\"-lrt -lm\""
+  write_to_bazelrc "build --experimental_repo_remote_exec"
+  write_to_bazelrc "build --action_env=PYTHON_BIN_PATH=\"/usr/bin/$PYTHON\""
+  write_to_bazelrc "build --action_env=PYTHON_LIB_PATH=\"/usr/lib/$PYTHON\""
+  write_to_bazelrc "build --python_path=\"/usr/bin/$PYTHON\""
+
+  bazel clean
+  bazel build $@ ...
+  bazel test $@ ...
+
+  DEST="/tmp/array_record_pip_pkg"
+  # Create the directory, then do dirname on a non-existent file inside it to
+  # give us an absolute paths with tilde characters resolved to the destination
+  # directory.
+  mkdir -p "${DEST}"
+  echo "=== destination directory: ${DEST}"
+
+  TMPDIR=$(mktemp -d -t tmp.XXXXXXXXXX)
+
+  echo $(date) : "=== Using tmpdir: ${TMPDIR}"
+  mkdir "${TMPDIR}/array_record"
+
+  echo "=== Copy array_record files"
+
+  cp setup.py "${TMPDIR}"
+  cp LICENSE "${TMPDIR}"
+  rsync -avm -L  --exclude="bazel-*/" . "${TMPDIR}/array_record"
+  rsync -avm -L  --include="*.so" --include="*_pb2.py" \
+    --exclude="*.runfiles" --exclude="*_obj" --include="*/" --exclude="*" \
+    bazel-bin/cpp "${TMPDIR}/array_record"
+  rsync -avm -L  --include="*.so" --include="*_pb2.py" \
+    --exclude="*.runfiles" --exclude="*_obj" --include="*/" --exclude="*" \
+    bazel-bin/python "${TMPDIR}/array_record"
+
+  pushd ${TMPDIR}
+  echo $(date) : "=== Building wheel"
+  ${PYTHON} setup.py bdist_wheel --python-tag py3${PYTHON_MINOR_VERSION}
+
+  echo $(date) : "=== Auditing wheel"
+  auditwheel repair --plat manylinux2014_x86_64 -w dist dist/*.whl
+  cp dist/*.whl "${DEST}"
+  popd
+
+  echo $(date) : "=== Output wheel file is in: ${DEST}"
+}
+
+main "$@"

oss/build_whl_runner.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# run build_whl.sh for different python versions
+
+set -x
+
+for p in 8 9
+do
+  update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.${p} 1
+
+  PYTHON_MINOR_VERSION=${p}  oss/build_whl.sh \
+    --crosstool_top=@sigbuild-r2.9-python3.${p}_config_cuda//crosstool:toolchain
+
+  update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.${p} 0
+done
+
+cp /tmp/array_record_pip_pkg/*.whl /tmp/array_record/

oss/runner.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# This script copy array_record from internal repo, build a docker, and build pip wheels
+
+set -e -x
+
+export TMP_FOLDER="/tmp/array_record"
+
+[ -f $TMP_FOLDER ] && rm -rf $TMP_FOLDER
+copybara array_record/oss/copy.bara.sky g3folder_to_gitfolder ../../ \
+  --init-history --folder-dir=$TMP_FOLDER --ignore-noop
+
+cd $TMP_FOLDER
+DOCKER_BUILDKIT=1 docker build --progress=plain --no-cache \
+  -t array_record:latest - < oss/build.Dockerfile
+
+docker run --rm -a stdin -a stdout -a stderr \
+  -v $TMP_FOLDER:/tmp/array_record --name array_record array_record:latest \
+  bash oss/build_whl_runner.sh
+
+ls $TMP_FOLDER/*.whl

setup.py

@@ -0,0 +1,26 @@
+"""Setup.py file for array_record."""
+
+from setuptools import find_packages
+from setuptools import setup
+
+setup(
+    name='array_record',
+    version='0.1.0',
+    description=(
+        'A file format that achieves a new frontier of IO efficiency'
+        ),
+    author='ArrayRecord team',
+    author_email='no-reply@google.com',
+    packages=find_packages(),
+    include_package_data=True,
+    package_data={'': ['*.so']},
+    python_requires='>=3.7',
+    install_requires=['absl-py'],
+    url='https://github.com/google/array_record',
+    license='Apache-2.0',
+    classifiers=[
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+    ],
+    zip_safe=False,
+)