wip linux

.github/workflows/build.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - tkp/linux 
     tags:
       - v*
     paths-ignore:
@@ -102,7 +103,15 @@ jobs:
         uses: actions-ext/cpp/setup@v1
 
       - name: Install dependencies
-        run: make develop
+        run: make develop-cpp
+        if: ${{ matrix.os != 'ubuntu-22.04' }}
+
+      - name: Install dependencies
+        run: make develop-arrow-ubuntu
+        if: ${{ matrix.os == 'ubuntu-22.04' }}
+
+      - name: Install dependencies
+        run: make develop-py
 
       - name: Build project
         run: make build

CMakeLists.txt

@@ -45,7 +45,7 @@ endif()
 # Paths #
 #########
 # Custom CMake modules
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_SOURCE_DIR}/cmake/modules")
+list(PREPEND CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_SOURCE_DIR}/cmake/modules")
 
 # Base includes
 include_directories ("${CMAKE_SOURCE_DIR}/src")
@@ -56,7 +56,7 @@ include_directories ("${CMAKE_SOURCE_DIR}/src")
 find_package (Color)
 
 if (NOT DEFINED CMAKE_BUILD_TYPE)
-  set(CMAKE_BUILD_TYPE "Debugma" CACHE STRING "Release/Debug build")
+  set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Release/Debug build")
 endif()
 if (NOT DEFINED PYTHON_VERSION)
   set(PYTHON_VERSION "3.9" CACHE STRING "Python version to build against")
@@ -104,24 +104,6 @@ if(WIN32)
     set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 endif()
 
-###############################################################################################################
-# Version #
-###########
-# Set version from cmake and extract latest hash if available
-set(ARROW_PYTHON_NOCOPY_VERSION_MAJOR ${PROJECT_VERSION_MAJOR})
-set(ARROW_PYTHON_NOCOPY_VERSION_MINOR ${PROJECT_VERSION_MINOR})
-set(ARROW_PYTHON_NOCOPY_VERSION_PATCH ${PROJECT_VERSION_PATCH})
-if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
-  # Get latest commit
-  execute_process(COMMAND git rev-parse HEAD
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    OUTPUT_VARIABLE ARROW_PYTHON_NOCOPY_VERSION_COMMIT_SHA)
-  # strip newline
-  string(REGEX REPLACE "\n$" "" ARROW_PYTHON_NOCOPY_VERSION_COMMIT_SHA "${ARROW_PYTHON_NOCOPY_VERSION_COMMIT_SHA}")
-else()
-  set(ARROW_PYTHON_NOCOPY_VERSION_COMMIT_SHA "release")
-endif()
-
 ###############################################################################################################
 # RPath #
 #########
@@ -148,6 +130,9 @@ endif()
 ###############################################################################################################
 # Flags #
 #########
+set(CMAKE_POSITION_INDEPENDENT_CODE On)
+add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
+
 # Compiler version flags
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++2a")
@@ -244,7 +229,7 @@ set(
 )
 
 add_library(arrow-python-nocopy SHARED ${PROJECT_SRCS})
-target_link_libraries(arrow-python-nocopy Arrow::arrow_static)
+target_link_libraries(arrow-python-nocopy PRIVATE Arrow::arrow_shared)
 set_target_properties(arrow-python-nocopy PROPERTIES PUBLIC_HEADER "${PROJECT_HDRS}")
 
 # export symbols
@@ -261,14 +246,47 @@ install(TARGETS arrow-python-nocopy EXPORT ArrowPythonNocopy LIBRARY DESTINATION
 if(BUILD_PYTHON)
   # Find Numpy
   find_package(NumPy REQUIRED)
+  include_directories(${NUMPY_INCLUDE_DIR})
 
+  # Find PyArrow. We will link against it for the build only
+  # find_package(Pyarrow REQUIRED)
+  # include_directories(${PYARROW_INCLUDE_DIR})
+
   # Find PyBind11
-  find_package(pybind11 REQUIRED)
-  include_directories(${pybind11_INCLUDE_DIR})
-  include_directories("${PROJECT_SOURCE_DIR}/src/vendored/apache-arrow-12.0.1")
-
+   find_package(pybind11 REQUIRED)
+   include_directories(${pybind11_INCLUDE_DIR})
+   set(VENDORED_PYARROW_ROOT "${PROJECT_SOURCE_DIR}/src/vendored/apache-arrow-12.0.1")
+   include_directories(${VENDORED_PYARROW_ROOT})
+   set(VENDORED_PYARROW_SRCS
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/arrow_to_pandas.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/benchmark.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/common.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/csv.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/datetime.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/decimal.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/deserialize.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/extension_type.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/filesystem.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/flight.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/gdb.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/helpers.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/inference.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/init.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/io.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/ipc.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/numpy_convert.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/numpy_to_arrow.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/parquet_encryption.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/pyarrow.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/python_test.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/python_to_arrow.cc
+     ${VENDORED_PYARROW_ROOT}/arrow/python/serialize.cc
+     # ${VENDORED_PYARROW_ROOT}/arrow/python/udf.cc
+  )
+
   # common functionality
-  add_library(common SHARED "${PROJECT_SOURCE_DIR}/src/apn-python/common.cpp")
+  add_library(common SHARED "${PROJECT_SOURCE_DIR}/src/apn-python/common.cpp" ${VENDORED_PYARROW_SRCS})
+  target_include_directories(common "${NUMPY_INCLUDE_DIR}")
   set_target_properties(common PROPERTIES PUBLIC_HEADER "${PROJECT_SOURCE_DIR}/src/apn-python/common.hpp")
 
   # pybind11 extension
@@ -280,12 +298,12 @@ if(BUILD_PYTHON)
   set_target_properties(cpythonextension PROPERTIES PUBLIC_HEADER "${PROJECT_SOURCE_DIR}/src/apn-python/cpython.hpp")
 
   # Link to standalone/common library
-  target_link_libraries(common PUBLIC arrow-python-nocopy)
-  target_link_libraries(pybind11extension PUBLIC common)
-  target_link_libraries(cpythonextension PUBLIC common)
-  set_property(TARGET common PROPERTY INSTALL_RPATH "${module_origin_path}/lib")
-  set_property(TARGET pybind11extension PROPERTY INSTALL_RPATH "${module_origin_path}/lib")
-  set_property(TARGET cpythonextension PROPERTY INSTALL_RPATH "${module_origin_path}/lib")
+  target_link_libraries(common PRIVATE arrow-python-nocopy)
+  target_link_libraries(pybind11extension PRIVATE common)
+  target_link_libraries(cpythonextension PRIVATE common)
+  set_property(TARGET common PROPERTY INSTALL_RPATH "${module_origin_path}:${module_origin_path}/lib")
+  set_property(TARGET pybind11extension PROPERTY INSTALL_RPATH "${module_origin_path}:${module_origin_path}/lib")
+  set_property(TARGET cpythonextension PROPERTY INSTALL_RPATH "${module_origin_path}:${module_origin_path}/lib")
   set_property(TARGET cpythonextension PROPERTY PREFIX "")
 
   # install in python module

Makefile

@@ -18,6 +18,13 @@ develop-cpp:
 	./vcpkg/bootstrap-vcpkg.sh
 	./vcpkg/vcpkg install
 
+develop-arrow-ubuntu:
+	wget https://apache.jfrog.io/artifactory/arrow/$(shell lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(shell lsb_release --codename --short).deb
+	sudo apt install -y -V ./apache-arrow-apt-source-latest-$(shell lsb_release --codename --short).deb
+	sudo apt update
+	sudo apt install -y -V libarrow-dev # For C++
+	sudo apt install -y -V libarrow-glib-dev # For GLib (C)
+
 develop: develop-cpp develop-py  ## Setup project for development
 
 .PHONY: build-py build-cpp build debug-py debug

cmake/modules/FindNumPy.cmake

@@ -18,12 +18,12 @@ execute_process(
           "from __future__ import print_function;import numpy;print(numpy.__version__, end='')"
   OUTPUT_VARIABLE __numpy_version)
 
-find_path(NUMPY_INCLUDE_DIR numpy/arrayobject.h
+find_path(NUMPY_INCLUDE_DIR numpy/numpyconfig.h
   HINTS "${__numpy_path}" "${PYTHON_INCLUDE_PATH}" NO_DEFAULT_PATH)
 
 if(NUMPY_INCLUDE_DIR)
   set(NUMPY_FOUND 1 CACHE INTERNAL "Python numpy found")
-endif(NUMPY_INCLUDE_DIR)
+endif()
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(NumPy REQUIRED_VARS NUMPY_INCLUDE_DIR

cmake/modules/FindPyarrow.cmake

@@ -0,0 +1,45 @@
+# Find the Pyarrow package
+# PYARROW_INCLUDE_DIR
+# PYARROW_LIB_DIR
+# PYARROW_LIBRARY
+# PYARROW_FOUND
+# will be set by this script
+
+cmake_minimum_required(VERSION 3.7.2)
+
+find_package( PythonInterp ${PYTHON_VERSION} EXACT REQUIRED )
+
+# Find out the include path
+execute_process(
+  COMMAND "${Python_EXECUTABLE}" -c
+          "from __future__ import print_function;import pyarrow;print(pyarrow.get_include(), end='')"
+          OUTPUT_VARIABLE __pyarrow_path)
+
+# Find out the library path
+execute_process(
+  COMMAND "${Python_EXECUTABLE}" -c
+          "from __future__ import print_function;import pyarrow;print(pyarrow.get_library_dirs()[0], end='')"
+          OUTPUT_VARIABLE __pyarrow_lib_path)
+
+# And the version
+execute_process(
+  COMMAND "${Python_EXECUTABLE}" -c
+          "from __future__ import print_function;import pyarrow;print(pyarrow.__version__, end='')"
+  OUTPUT_VARIABLE __pyarrow_version)
+
+find_path(PYARROW_INCLUDE_DIR arrow/python/pyarrow.h
+  HINTS "${__pyarrow_path}" "${PYTHON_INCLUDE_PATH}" NO_DEFAULT_PATH)
+
+find_path(PYARROW_LIB_DIR libarrow_python.so
+  HINTS "${__pyarrow_lib_path}" "${PYTHON_LIBRARY_PATH}" NO_DEFAULT_PATH)
+
+find_file(PYARROW_LIBRARY libarrow_python.so
+  HINTS "${__pyarrow_lib_path}" "${PYTHON_LIBRARY_PATH}" NO_DEFAULT_PATH)
+
+if(PYARROW_INCLUDE_DIR AND PYARROW_LIB_DIR AND PYARROW_LIBRARY)
+  set(PYARROW_FOUND 1 CACHE INTERNAL "Python pyarrow found")
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Pyarrow REQUIRED_VARS PYARROW_INCLUDE_DIR PYARROW_LIB_DIR PYARROW_LIBRARY
+                                        VERSION_VAR __pyarrow_version)

pyproject.toml

@@ -91,6 +91,7 @@ test-requires = [
 [tool.cibuildwheel.linux]
 archs = "x86_64 aarch64"
 skip = "*musllinux* *i686"
+before-all = "make develop-arrow-ubuntu && make develop-py"
 
 [tool.cibuildwheel.macos]
 archs = "x86_64 arm64"

setup.py

@@ -12,7 +12,7 @@
     )
 )
 
-if os.path.exists(vcpkg_config_file):
+if os.path.exists(vcpkg_config_file) and os.name != "posix":
     cmake_args.append(
         f"-DCMAKE_TOOLCHAIN_FILE={vcpkg_config_file}"
     )

src/apn-python/caster.hpp

@@ -8,7 +8,6 @@ namespace detail {
     PYBIND11_TYPE_CASTER(std::shared_ptr<arrow::Array>, const_name("pyarrow::Array"));
     /* Python->C++ */
     bool load(handle src, bool) {
-      arrow::py::import_pyarrow();
       PyObject* source = src.ptr();
       if(!arrow::py::is_array(source))
         return false;
@@ -21,7 +20,6 @@ namespace detail {
 
     /* C++ -> Python) */
     static handle cast(std::shared_ptr<arrow::Array> src, return_value_policy /* policy */, handle /* parent */) {
-      arrow::py::import_pyarrow();
       return arrow::py::wrap_array(src);
     }
   };
@@ -32,7 +30,6 @@ namespace detail {
     PYBIND11_TYPE_CASTER(std::shared_ptr<arrow::Schema>, const_name("pyarrow::Schema"));
     /* Python->C++ */
     bool load(handle src, bool) {
-      arrow::py::import_pyarrow();
       PyObject* source = src.ptr();
       if(!arrow::py::is_schema(source))
         return false;
@@ -45,7 +42,6 @@ namespace detail {
 
     /* C++ -> Python) */
     static handle cast(std::shared_ptr<arrow::Schema> src, return_value_policy /* policy */, handle /* parent */) {
-      arrow::py::import_pyarrow();
       return arrow::py::wrap_schema(src);
     }
   };
@@ -56,7 +52,6 @@ namespace detail {
     PYBIND11_TYPE_CASTER(std::shared_ptr<arrow::Table>, const_name("pyarrow::Table"));
     /* Python->C++ */
     bool load(handle src, bool) {
-      arrow::py::import_pyarrow();
       PyObject* source = src.ptr();
       if(!arrow::py::is_table(source))
         return false;
@@ -69,7 +64,6 @@ namespace detail {
 
     /* C++ -> Python) */
     static handle cast(std::shared_ptr<arrow::Table> src, return_value_policy /* policy */, handle /* parent */) {
-      arrow::py::import_pyarrow();
       return arrow::py::wrap_table(src);
     }
   };

src/apn-python/common.cpp

@@ -1,20 +1,19 @@
+#include <arrow/python/pyarrow.h>
 #include <arrow/c/bridge.h>
 #include <apn-python/common.hpp>
 #include <apn/bridge.h>
 
-std::string array_info_py(std::shared_ptr<arrow::Array> array) {
+char* array_info_py(std::shared_ptr<arrow::Array> array) {
   // ABI unstable!
   // return array_info(array);
-  char buffer[100];
+  char* buffer = new char[100];
   struct ArrowArray c_array;
   (void)ExportArray(*array, &c_array);
   array_info_cabi(&c_array, buffer, 100);
-  return std::string(buffer);
+  return buffer;
 }
 
 std::shared_ptr<arrow::Array> create_array_py() {
-  arrow::py::import_pyarrow();
-
   // ABI unstable!
   // std::shared_ptr<arrow::Array> arrow_array = create_array_cabi();
   struct ArrowArray c_array;
@@ -23,19 +22,17 @@ std::shared_ptr<arrow::Array> create_array_py() {
   return arrow_array;
 }
 
-std::string schema_info_py(std::shared_ptr<arrow::Schema> schema) {
+char* schema_info_py(std::shared_ptr<arrow::Schema> schema) {
   // ABI unstable!
   // return schema_info(schema);
-  char buffer[100];
+  char* buffer = new char[100];
   struct ArrowSchema c_schema;
   (void)arrow::ExportSchema(*schema, &c_schema);
   schema_info_cabi(&c_schema, buffer, 100);
-  return std::string(buffer);
+  return buffer;
 }
 
 std::shared_ptr<arrow::Schema> create_schema_py() {
-  arrow::py::import_pyarrow();
-
   // ABI unstable!
   // std::shared_ptr<arrow::Schema> arrow_schema = create_schema();
 
@@ -50,7 +47,6 @@ std::shared_ptr<arrow::Schema> create_schema_py() {
 // }
 
 // std::shared_ptr<arrow::Table> create_table_py() {
-//     arrow::py::import_pyarrow();
 //     std::shared_ptr<arrow::Table> arrow_table = create_table();
 //     return arrow_table;
 //     // PyObject* obj = arrow::py::wrap_table(arrow_table);

src/apn-python/common.hpp

@@ -4,11 +4,11 @@
 #include <arrow/python/pyarrow.h>
 #include <apn-common/exports.h>
 
-LIB_EXPORT std::string array_info_py(std::shared_ptr<arrow::Array> array);
+LIB_EXPORT char* array_info_py(std::shared_ptr<arrow::Array> array);
 LIB_EXPORT std::shared_ptr<arrow::Array> create_array_py();
 
-LIB_EXPORT std::string schema_info_py(std::shared_ptr<arrow::Schema> schema);
+LIB_EXPORT char* schema_info_py(std::shared_ptr<arrow::Schema> schema);
 LIB_EXPORT std::shared_ptr<arrow::Schema> create_schema_py();
 
-LIB_EXPORT std::string table_info_py(std::shared_ptr<arrow::Table> table);
+LIB_EXPORT char* table_info_py(std::shared_ptr<arrow::Table> table);
 LIB_EXPORT std::shared_ptr<arrow::Table> create_table_py();