Merge pull request #1 from timkpaine/tkp/linux

Fix CI
timkpaine · Aug 3, 2023 · e1ed4d1 · e1ed4d1
2 parents b823b9b + 6032a03
commit e1ed4d1
Show file tree

Hide file tree

Showing 59 changed files with 13,173 additions and 150 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - tkp/linux 
     tags:
       - v*
     paths-ignore:
@@ -81,6 +82,7 @@ jobs:
       matrix:
         os:
           - ubuntu-22.04
+          - macos-12
         python-version:
           - '3.9'
       fail-fast: false
@@ -102,7 +104,18 @@ jobs:
         uses: actions-ext/cpp/setup@v1
 
       - name: Install dependencies
-        run: make develop
+        run: make develop-macos
+        if: ${{ runner.os == 'macOS' }}
+
+      # - name: Install dependencies
+      #   run: make develop-cpp
+
+      - name: Install dependencies
+        run: make develop-arrow-ubuntu
+        if: ${{ runner.os == 'Linux' }}
+
+      - name: Install dependencies
+        run: make develop-py
 
       - name: Build project
         run: make build
@@ -210,8 +223,19 @@ jobs:
           version: ${{ matrix.python-version }}
 
       - name: Install dependencies
-        run: make develop
+        run: make develop-macos
+        if: ${{ runner.os == 'macOS' }}
 
+      # - name: Install dependencies
+      #   run: make develop-cpp
+
+      - name: Install dependencies
+        run: make develop-arrow-ubuntu
+        if: ${{ runner.os == 'Linux' }}
+
+      - name: Install dependencies
+        run: make develop-py
+
       - name: Run cibuildwheel
         uses: actions-ext/python/run-cibuildwheel@v1
         with:
@@ -257,8 +281,19 @@ jobs:
           version: ${{ matrix.python-version }}
 
       - name: Install dependencies
-        run: make develop
+        run: make develop-macos
+        if: ${{ runner.os == 'macOS' }}
+
+      # - name: Install dependencies
+      #   run: make develop-cpp
+
+      - name: Install dependencies
+        run: make develop-arrow-ubuntu
+        if: ${{ runner.os == 'Linux' }}
 
+      - name: Install dependencies
+        run: make develop-py
+
       - name: Build sdist
         run: make dist-sdist
 
@@ -353,8 +388,19 @@ jobs:
           version: ${{ matrix.python-version }}
 
       - name: Install dependencies
-        run: make develop
+        run: make develop-macos
+        if: ${{ runner.os == 'macOS' }}
+
+      # - name: Install dependencies
+      #   run: make develop-cpp
+
+      - name: Install dependencies
+        run: make develop-arrow-ubuntu
+        if: ${{ runner.os == 'Linux' }}
 
+      - name: Install dependencies
+        run: make develop-py
+
       - name: Download wheels
         uses: actions-ext/python/download-dist@v1
         with:
@@ -419,8 +465,19 @@ jobs:
           version: ${{ matrix.python-version }}
 
       - name: Install dependencies
-        run: make develop
+        run: make develop-macos
+        if: ${{ runner.os == 'macOS' }}
+
+      # - name: Install dependencies
+      #   run: make develop-cpp
+
+      - name: Install dependencies
+        run: make develop-arrow-ubuntu
+        if: ${{ runner.os == 'Linux' }}
 
+      - name: Install dependencies
+        run: make develop-py
+
       - name: Download wheels
         uses: actions-ext/python/download-dist@v1
         with:

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -45,18 +45,18 @@ endif()
 # Paths #
 #########
 # Custom CMake modules
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_SOURCE_DIR}/cmake/modules")
+list(PREPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/modules")
 
 # Base includes
-include_directories ("${CMAKE_SOURCE_DIR}/src")
+include_directories ("${PROJECT_SOURCE_DIR}/src")
 
 ###############################################################################################################
 # Build Configuration #
 #######################
 find_package (Color)
 
 if (NOT DEFINED CMAKE_BUILD_TYPE)
-  set(CMAKE_BUILD_TYPE "Debugma" CACHE STRING "Release/Debug build")
+  set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Release/Debug build")
 endif()
 if (NOT DEFINED PYTHON_VERSION)
   set(PYTHON_VERSION "3.9" CACHE STRING "Python version to build against")
@@ -65,6 +65,7 @@ option(BUILD_PYTHON "Build Python bindings" ON)
 option(USE_CCACHE "Use CCache for build" OFF)
 option(STATIC_PYTHON "Build against static python (no libraries)" OFF)
 option(PYODIDE "Build against pyodide" OFF)
+set(BUILD_SHARED_LIBS ON)
 
 # CCache setup
 if(USE_CCACHE)
@@ -104,24 +105,6 @@ if(WIN32)
     set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 endif()
 
-###############################################################################################################
-# Version #
-###########
-# Set version from cmake and extract latest hash if available
-set(ARROW_PYTHON_NOCOPY_VERSION_MAJOR ${PROJECT_VERSION_MAJOR})
-set(ARROW_PYTHON_NOCOPY_VERSION_MINOR ${PROJECT_VERSION_MINOR})
-set(ARROW_PYTHON_NOCOPY_VERSION_PATCH ${PROJECT_VERSION_PATCH})
-if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
-  # Get latest commit
-  execute_process(COMMAND git rev-parse HEAD
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    OUTPUT_VARIABLE ARROW_PYTHON_NOCOPY_VERSION_COMMIT_SHA)
-  # strip newline
-  string(REGEX REPLACE "\n$" "" ARROW_PYTHON_NOCOPY_VERSION_COMMIT_SHA "${ARROW_PYTHON_NOCOPY_VERSION_COMMIT_SHA}")
-else()
-  set(ARROW_PYTHON_NOCOPY_VERSION_COMMIT_SHA "release")
-endif()
-
 ###############################################################################################################
 # RPath #
 #########
@@ -148,6 +131,9 @@ endif()
 ###############################################################################################################
 # Flags #
 #########
+set(CMAKE_POSITION_INDEPENDENT_CODE On)
+add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
+
 # Compiler version flags
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++2a")
@@ -185,7 +171,12 @@ endif()
 ###############################################################################################################
 # Dependencies #
 ################
+message("${Red}${CMAKE_MODULE_PATH}${ColorReset}")
+
+
+# broken on mac
 find_package(Arrow REQUIRED)
+include_directories(${Arrow_INCLUDE_DIR})
 
 ###############################################################################################################
 # Python #
@@ -244,7 +235,7 @@ set(
 )
 
 add_library(arrow-python-nocopy SHARED ${PROJECT_SRCS})
-target_link_libraries(arrow-python-nocopy Arrow::arrow_static)
+target_link_libraries(arrow-python-nocopy PRIVATE ${Arrow_LIBRARY})
 set_target_properties(arrow-python-nocopy PROPERTIES PUBLIC_HEADER "${PROJECT_HDRS}")
 
 # export symbols
@@ -261,14 +252,46 @@ install(TARGETS arrow-python-nocopy EXPORT ArrowPythonNocopy LIBRARY DESTINATION
 if(BUILD_PYTHON)
   # Find Numpy
   find_package(NumPy REQUIRED)
+  include_directories(${NumPy_INCLUDE_DIR})
 
+  # Find PyArrow. We will link against it for the build only
+  # find_package(Pyarrow REQUIRED)
+  # include_directories(${PYARROW_INCLUDE_DIR})
+
   # Find PyBind11
-  find_package(pybind11 REQUIRED)
-  include_directories(${pybind11_INCLUDE_DIR})
-  include_directories("${PROJECT_SOURCE_DIR}/src/vendored/apache-arrow-12.0.1")
-
+   find_package(pybind11 REQUIRED)
+   include_directories(${pybind11_INCLUDE_DIR})
+   set(VENDORED_PYARROW_ROOT "${PROJECT_SOURCE_DIR}/src/vendored/apache-arrow-12.0.1")
+   include_directories(${VENDORED_PYARROW_ROOT})
+   set(VENDORED_PYARROW_SRCS
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/arrow_to_pandas.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/benchmark.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/common.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/csv.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/datetime.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/decimal.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/deserialize.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/extension_type.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/filesystem.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/flight.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/gdb.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/helpers.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/inference.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/init.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/io.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/ipc.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/numpy_convert.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/numpy_to_arrow.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/parquet_encryption.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/pyarrow.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/python_test.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/python_to_arrow.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/serialize.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/udf.cc
+  )
+
   # common functionality
-  add_library(common SHARED "${PROJECT_SOURCE_DIR}/src/apn-python/common.cpp")
+  add_library(common SHARED "${PROJECT_SOURCE_DIR}/src/apn-python/common.cpp" ${VENDORED_PYARROW_SRCS})
   set_target_properties(common PROPERTIES PUBLIC_HEADER "${PROJECT_SOURCE_DIR}/src/apn-python/common.hpp")
 
   # pybind11 extension
@@ -280,16 +303,16 @@ if(BUILD_PYTHON)
   set_target_properties(cpythonextension PROPERTIES PUBLIC_HEADER "${PROJECT_SOURCE_DIR}/src/apn-python/cpython.hpp")
 
   # Link to standalone/common library
-  target_link_libraries(common PUBLIC arrow-python-nocopy)
-  target_link_libraries(pybind11extension PUBLIC common)
-  target_link_libraries(cpythonextension PUBLIC common)
-  set_property(TARGET common PROPERTY INSTALL_RPATH "${module_origin_path}/lib")
-  set_property(TARGET pybind11extension PROPERTY INSTALL_RPATH "${module_origin_path}/lib")
-  set_property(TARGET cpythonextension PROPERTY INSTALL_RPATH "${module_origin_path}/lib")
+  target_link_libraries(common PRIVATE arrow-python-nocopy)
+  target_link_libraries(pybind11extension PRIVATE common)
+  target_link_libraries(cpythonextension PRIVATE common)
+  set_property(TARGET common PROPERTY INSTALL_RPATH "${module_origin_path}")
+  set_property(TARGET pybind11extension PROPERTY INSTALL_RPATH "${module_origin_path}")
+  set_property(TARGET cpythonextension PROPERTY INSTALL_RPATH "${module_origin_path}")
   set_property(TARGET cpythonextension PROPERTY PREFIX "")
 
   # install in python module
   install(TARGETS common EXPORT ArrowPythonNocopy LIBRARY DESTINATION lib PUBLIC_HEADER DESTINATION include/python)
-  install(TARGETS pybind11extension EXPORT ArrowPythonNocopy LIBRARY DESTINATION . PUBLIC_HEADER DESTINATION include/python)
-  install(TARGETS cpythonextension EXPORT ArrowPythonNocopy LIBRARY DESTINATION . PUBLIC_HEADER DESTINATION include/python)
+  install(TARGETS pybind11extension EXPORT ArrowPythonNocopy LIBRARY DESTINATION lib PUBLIC_HEADER DESTINATION include/python)
+  install(TARGETS cpythonextension EXPORT ArrowPythonNocopy LIBRARY DESTINATION lib PUBLIC_HEADER DESTINATION include/python)
 endif()
diff --git a/Makefile b/Makefile
@@ -18,6 +18,16 @@ develop-cpp:
 	./vcpkg/bootstrap-vcpkg.sh
 	./vcpkg/vcpkg install
 
+develop-arrow-ubuntu:
+	wget https://apache.jfrog.io/artifactory/arrow/$(shell lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(shell lsb_release --codename --short).deb
+	sudo apt install -y -V ./apache-arrow-apt-source-latest-$(shell lsb_release --codename --short).deb
+	sudo apt update
+	sudo apt install -y -V libarrow-dev # For C++
+	sudo apt install -y -V libarrow-glib-dev # For GLib (C)
+
+develop-macos:
+	brew install bison apache-arrow thrift
+
 develop: develop-cpp develop-py  ## Setup project for development
 
 .PHONY: build-py build-cpp build debug-py debug

diff --git a/arrow_python_nocopy/__init__.py b/arrow_python_nocopy/__init__.py
@@ -2,12 +2,14 @@
 import os.path
 import pyarrow as pa
 import pandas as pd
-from .pybind11extension import array_info, create_array
-from .pybind11extension import schema_info, create_schema
-# from .pybind11extension import table_info, create_table
-from .cpythonextension import array_info as array_info_cp, create_array as create_array_cp
-from .cpythonextension import schema_info as schema_info_cp, create_schema as create_schema_cp
-# from .pybind11extension import table_info as table_info_cp, create_table as create_table_cp
+from .lib.pybind11extension import array_info, create_array
+from .lib.pybind11extension import schema_info, create_schema
+
+# from .lib.pybind11extension import table_info, create_table
+from .lib.cpythonextension import array_info as array_info_cp, create_array as create_array_cp
+from .lib.cpythonextension import schema_info as schema_info_cp, create_schema as create_schema_cp
+
+# from .lib.cpythonextension import table_info as table_info_cp, create_table as create_table_cp
 
 
 __version__ = "0.1.0"
@@ -24,17 +26,21 @@ def bin_path():
 def lib_path():
     return os.path.abspath(os.path.join(os.path.dirname(__file__), "lib"))
 
+
 def _df():
     return pd.DataFrame({"a": pd.Series([1, 2, 3], dtype='Int32'), "b": pd.Series([1.1, 2.2, 3.3], dtype='Float32'), "c": pd.Series(["abc", "def", "ghi"], dtype=str)})
 
+
 def _table():
     return pa.Table.from_pandas(_df())
 
+
 def create_arrow_array_in_python():
     table = _table()
     array = table.columns['a']
     print(array_info(array))
 
+
 def create_arrow_array_in_cpp():
     return create_array()
 
@@ -44,6 +50,7 @@ def create_arrow_schema_in_python():
     schema = table.schema
     print(schema_info(schema))
 
+
 def create_arrow_schema_in_cpp():
     return create_schema()