wip linux

.github/workflows/build.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - tkp/linux 
     tags:
       - v*
     paths-ignore:
@@ -104,6 +105,10 @@ jobs:
       - name: Install dependencies
         run: make develop
 
+      - name: Install dependencies
+        run: sudo apt install libarrow-dev
+        if: ${{ matrix.os == 'ubuntu-22.04' }}
+
       - name: Build project
         run: make build
 

CMakeLists.txt

@@ -56,7 +56,7 @@ include_directories ("${CMAKE_SOURCE_DIR}/src")
 find_package (Color)
 
 if (NOT DEFINED CMAKE_BUILD_TYPE)
-  set(CMAKE_BUILD_TYPE "Debugma" CACHE STRING "Release/Debug build")
+  set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Release/Debug build")
 endif()
 if (NOT DEFINED PYTHON_VERSION)
   set(PYTHON_VERSION "3.9" CACHE STRING "Python version to build against")
@@ -104,24 +104,6 @@ if(WIN32)
     set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 endif()
 
-###############################################################################################################
-# Version #
-###########
-# Set version from cmake and extract latest hash if available
-set(ARROW_PYTHON_NOCOPY_VERSION_MAJOR ${PROJECT_VERSION_MAJOR})
-set(ARROW_PYTHON_NOCOPY_VERSION_MINOR ${PROJECT_VERSION_MINOR})
-set(ARROW_PYTHON_NOCOPY_VERSION_PATCH ${PROJECT_VERSION_PATCH})
-if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
-  # Get latest commit
-  execute_process(COMMAND git rev-parse HEAD
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    OUTPUT_VARIABLE ARROW_PYTHON_NOCOPY_VERSION_COMMIT_SHA)
-  # strip newline
-  string(REGEX REPLACE "\n$" "" ARROW_PYTHON_NOCOPY_VERSION_COMMIT_SHA "${ARROW_PYTHON_NOCOPY_VERSION_COMMIT_SHA}")
-else()
-  set(ARROW_PYTHON_NOCOPY_VERSION_COMMIT_SHA "release")
-endif()
-
 ###############################################################################################################
 # RPath #
 #########
@@ -148,6 +130,9 @@ endif()
 ###############################################################################################################
 # Flags #
 #########
+set(CMAKE_POSITION_INDEPENDENT_CODE On)
+add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
+
 # Compiler version flags
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++2a")
@@ -244,7 +229,7 @@ set(
 )
 
 add_library(arrow-python-nocopy SHARED ${PROJECT_SRCS})
-target_link_libraries(arrow-python-nocopy Arrow::arrow_static)
+target_link_libraries(arrow-python-nocopy PRIVATE Arrow::arrow_shared)
 set_target_properties(arrow-python-nocopy PROPERTIES PUBLIC_HEADER "${PROJECT_HDRS}")
 
 # export symbols
@@ -261,14 +246,46 @@ install(TARGETS arrow-python-nocopy EXPORT ArrowPythonNocopy LIBRARY DESTINATION
 if(BUILD_PYTHON)
   # Find Numpy
   find_package(NumPy REQUIRED)
+  include_directories(${NUMPY_INCLUDE_DIR})
 
+  # Find PyArrow. We will link against it for the build only
+  find_package(Pyarrow REQUIRED)
+  include_directories(${PYARROW_INCLUDE_DIR})
+
   # Find PyBind11
-  find_package(pybind11 REQUIRED)
-  include_directories(${pybind11_INCLUDE_DIR})
-  include_directories("${PROJECT_SOURCE_DIR}/src/vendored/apache-arrow-12.0.1")
-
+   find_package(pybind11 REQUIRED)
+   include_directories(${pybind11_INCLUDE_DIR})
+   set(VENDORED_PYARROW_ROOT "${PROJECT_SOURCE_DIR}/src/vendored/apache-arrow-12.0.1")
+   include_directories(${VENDORED_PYARROW_ROOT})
+   set(VENDORED_PYARROW_SRCS
+       # ${VENDORED_PYARROW_ROOT}/arrow/python/arrow_to_pandas.cc
+       # ${VENDORED_PYARROW_ROOT}/arrow/python/benchmark.cc
+       ${VENDORED_PYARROW_ROOT}/arrow/python/common.cc
+       # ${VENDORED_PYARROW_ROOT}/arrow/python/csv.cc
+       ${VENDORED_PYARROW_ROOT}/arrow/python/datetime.cc
+       ${VENDORED_PYARROW_ROOT}/arrow/python/decimal.cc
+       ${VENDORED_PYARROW_ROOT}/arrow/python/deserialize.cc
+       ${VENDORED_PYARROW_ROOT}/arrow/python/extension_type.cc
+       # ${VENDORED_PYARROW_ROOT}/arrow/python/filesystem.cc
+       # ${VENDORED_PYARROW_ROOT}/arrow/python/flight.cc
+       # ${VENDORED_PYARROW_ROOT}/arrow/python/gdb.cc
+       ${VENDORED_PYARROW_ROOT}/arrow/python/helpers.cc
+       ${VENDORED_PYARROW_ROOT}/arrow/python/inference.cc
+       ${VENDORED_PYARROW_ROOT}/arrow/python/init.cc
+       ${VENDORED_PYARROW_ROOT}/arrow/python/io.cc
+       ${VENDORED_PYARROW_ROOT}/arrow/python/ipc.cc
+       ${VENDORED_PYARROW_ROOT}/arrow/python/numpy_convert.cc
+       ${VENDORED_PYARROW_ROOT}/arrow/python/numpy_to_arrow.cc
+       # ${VENDORED_PYARROW_ROOT}/arrow/python/parquet_encryption.cc
+       ${VENDORED_PYARROW_ROOT}/arrow/python/pyarrow.cc
+       # ${VENDORED_PYARROW_ROOT}/arrow/python/python_test.cc
+       ${VENDORED_PYARROW_ROOT}/arrow/python/python_to_arrow.cc
+       ${VENDORED_PYARROW_ROOT}/arrow/python/serialize.cc
+       # ${VENDORED_PYARROW_ROOT}/arrow/python/udf.cc
+)
+
   # common functionality
-  add_library(common SHARED "${PROJECT_SOURCE_DIR}/src/apn-python/common.cpp")
+  add_library(common SHARED "${PROJECT_SOURCE_DIR}/src/apn-python/common.cpp" ${VENDORED_PYARROW_SRCS})
   set_target_properties(common PROPERTIES PUBLIC_HEADER "${PROJECT_SOURCE_DIR}/src/apn-python/common.hpp")
 
   # pybind11 extension
@@ -280,12 +297,12 @@ if(BUILD_PYTHON)
   set_target_properties(cpythonextension PROPERTIES PUBLIC_HEADER "${PROJECT_SOURCE_DIR}/src/apn-python/cpython.hpp")
 
   # Link to standalone/common library
-  target_link_libraries(common PUBLIC arrow-python-nocopy)
-  target_link_libraries(pybind11extension PUBLIC common)
-  target_link_libraries(cpythonextension PUBLIC common)
-  set_property(TARGET common PROPERTY INSTALL_RPATH "${module_origin_path}/lib")
-  set_property(TARGET pybind11extension PROPERTY INSTALL_RPATH "${module_origin_path}/lib")
-  set_property(TARGET cpythonextension PROPERTY INSTALL_RPATH "${module_origin_path}/lib")
+  target_link_libraries(common PRIVATE arrow-python-nocopy)
+  target_link_libraries(pybind11extension PRIVATE common)
+  target_link_libraries(cpythonextension PRIVATE common)
+  set_property(TARGET common PROPERTY INSTALL_RPATH "${module_origin_path}:${module_origin_path}/lib")
+  set_property(TARGET pybind11extension PROPERTY INSTALL_RPATH "${module_origin_path}:${module_origin_path}/lib")
+  set_property(TARGET cpythonextension PROPERTY INSTALL_RPATH "${module_origin_path}:${module_origin_path}/lib")
   set_property(TARGET cpythonextension PROPERTY PREFIX "")
 
   # install in python module

cmake/modules/FindPyarrow.cmake

@@ -0,0 +1,45 @@
+# Find the Pyarrow package
+# PYARROW_INCLUDE_DIR
+# PYARROW_LIB_DIR
+# PYARROW_LIBRARY
+# PYARROW_FOUND
+# will be set by this script
+
+cmake_minimum_required(VERSION 3.7.2)
+
+find_package( PythonInterp ${PYTHON_VERSION} EXACT REQUIRED )
+
+# Find out the include path
+execute_process(
+  COMMAND "${Python_EXECUTABLE}" -c
+          "from __future__ import print_function;import pyarrow;print(pyarrow.get_include(), end='')"
+          OUTPUT_VARIABLE __pyarrow_path)
+
+# Find out the library path
+execute_process(
+  COMMAND "${Python_EXECUTABLE}" -c
+          "from __future__ import print_function;import pyarrow;print(pyarrow.get_library_dirs()[0], end='')"
+          OUTPUT_VARIABLE __pyarrow_lib_path)
+
+# And the version
+execute_process(
+  COMMAND "${Python_EXECUTABLE}" -c
+          "from __future__ import print_function;import pyarrow;print(pyarrow.__version__, end='')"
+  OUTPUT_VARIABLE __pyarrow_version)
+
+find_path(PYARROW_INCLUDE_DIR arrow/python/pyarrow.h
+  HINTS "${__pyarrow_path}" "${PYTHON_INCLUDE_PATH}" NO_DEFAULT_PATH)
+
+find_path(PYARROW_LIB_DIR libarrow_python.so
+  HINTS "${__pyarrow_lib_path}" "${PYTHON_LIBRARY_PATH}" NO_DEFAULT_PATH)
+
+find_file(PYARROW_LIBRARY libarrow_python.so
+  HINTS "${__pyarrow_lib_path}" "${PYTHON_LIBRARY_PATH}" NO_DEFAULT_PATH)
+
+if(PYARROW_INCLUDE_DIR AND PYARROW_LIB_DIR AND PYARROW_LIBRARY)
+  set(PYARROW_FOUND 1 CACHE INTERNAL "Python pyarrow found")
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Pyarrow REQUIRED_VARS PYARROW_INCLUDE_DIR PYARROW_LIB_DIR PYARROW_LIBRARY
+                                        VERSION_VAR __pyarrow_version)

setup.py

@@ -12,7 +12,7 @@
     )
 )
 
-if os.path.exists(vcpkg_config_file):
+if os.path.exists(vcpkg_config_file) and os.name != "posix":
     cmake_args.append(
         f"-DCMAKE_TOOLCHAIN_FILE={vcpkg_config_file}"
     )

src/apn-python/caster.hpp

@@ -8,7 +8,6 @@ namespace detail {
     PYBIND11_TYPE_CASTER(std::shared_ptr<arrow::Array>, const_name("pyarrow::Array"));
     /* Python->C++ */
     bool load(handle src, bool) {
-      arrow::py::import_pyarrow();
       PyObject* source = src.ptr();
       if(!arrow::py::is_array(source))
         return false;
@@ -21,7 +20,6 @@ namespace detail {
 
     /* C++ -> Python) */
     static handle cast(std::shared_ptr<arrow::Array> src, return_value_policy /* policy */, handle /* parent */) {
-      arrow::py::import_pyarrow();
       return arrow::py::wrap_array(src);
     }
   };
@@ -32,7 +30,6 @@ namespace detail {
     PYBIND11_TYPE_CASTER(std::shared_ptr<arrow::Schema>, const_name("pyarrow::Schema"));
     /* Python->C++ */
     bool load(handle src, bool) {
-      arrow::py::import_pyarrow();
       PyObject* source = src.ptr();
       if(!arrow::py::is_schema(source))
         return false;
@@ -45,7 +42,6 @@ namespace detail {
 
     /* C++ -> Python) */
     static handle cast(std::shared_ptr<arrow::Schema> src, return_value_policy /* policy */, handle /* parent */) {
-      arrow::py::import_pyarrow();
       return arrow::py::wrap_schema(src);
     }
   };
@@ -56,7 +52,6 @@ namespace detail {
     PYBIND11_TYPE_CASTER(std::shared_ptr<arrow::Table>, const_name("pyarrow::Table"));
     /* Python->C++ */
     bool load(handle src, bool) {
-      arrow::py::import_pyarrow();
       PyObject* source = src.ptr();
       if(!arrow::py::is_table(source))
         return false;
@@ -69,7 +64,6 @@ namespace detail {
 
     /* C++ -> Python) */
     static handle cast(std::shared_ptr<arrow::Table> src, return_value_policy /* policy */, handle /* parent */) {
-      arrow::py::import_pyarrow();
       return arrow::py::wrap_table(src);
     }
   };

src/apn-python/common.cpp

@@ -1,20 +1,19 @@
+#include <arrow/python/pyarrow.h>
 #include <arrow/c/bridge.h>
 #include <apn-python/common.hpp>
 #include <apn/bridge.h>
 
-std::string array_info_py(std::shared_ptr<arrow::Array> array) {
+char* array_info_py(std::shared_ptr<arrow::Array> array) {
   // ABI unstable!
   // return array_info(array);
-  char buffer[100];
+  char* buffer = new char[100];
   struct ArrowArray c_array;
   (void)ExportArray(*array, &c_array);
   array_info_cabi(&c_array, buffer, 100);
-  return std::string(buffer);
+  return buffer;
 }
 
 std::shared_ptr<arrow::Array> create_array_py() {
-  arrow::py::import_pyarrow();
-
   // ABI unstable!
   // std::shared_ptr<arrow::Array> arrow_array = create_array_cabi();
   struct ArrowArray c_array;
@@ -23,19 +22,17 @@ std::shared_ptr<arrow::Array> create_array_py() {
   return arrow_array;
 }
 
-std::string schema_info_py(std::shared_ptr<arrow::Schema> schema) {
+char* schema_info_py(std::shared_ptr<arrow::Schema> schema) {
   // ABI unstable!
   // return schema_info(schema);
-  char buffer[100];
+  char* buffer = new char[100];
   struct ArrowSchema c_schema;
   (void)arrow::ExportSchema(*schema, &c_schema);
   schema_info_cabi(&c_schema, buffer, 100);
-  return std::string(buffer);
+  return buffer;
 }
 
 std::shared_ptr<arrow::Schema> create_schema_py() {
-  arrow::py::import_pyarrow();
-
   // ABI unstable!
   // std::shared_ptr<arrow::Schema> arrow_schema = create_schema();
 
@@ -50,7 +47,6 @@ std::shared_ptr<arrow::Schema> create_schema_py() {
 // }
 
 // std::shared_ptr<arrow::Table> create_table_py() {
-//     arrow::py::import_pyarrow();
 //     std::shared_ptr<arrow::Table> arrow_table = create_table();
 //     return arrow_table;
 //     // PyObject* obj = arrow::py::wrap_table(arrow_table);

src/apn-python/common.hpp

@@ -4,11 +4,11 @@
 #include <arrow/python/pyarrow.h>
 #include <apn-common/exports.h>
 
-LIB_EXPORT std::string array_info_py(std::shared_ptr<arrow::Array> array);
+LIB_EXPORT char* array_info_py(std::shared_ptr<arrow::Array> array);
 LIB_EXPORT std::shared_ptr<arrow::Array> create_array_py();
 
-LIB_EXPORT std::string schema_info_py(std::shared_ptr<arrow::Schema> schema);
+LIB_EXPORT char* schema_info_py(std::shared_ptr<arrow::Schema> schema);
 LIB_EXPORT std::shared_ptr<arrow::Schema> create_schema_py();
 
-LIB_EXPORT std::string table_info_py(std::shared_ptr<arrow::Table> table);
+LIB_EXPORT char* table_info_py(std::shared_ptr<arrow::Table> table);
 LIB_EXPORT std::shared_ptr<arrow::Table> create_table_py();

src/apn-python/cpython.cpp

@@ -1,5 +1,4 @@
 #include <apn-python/cpython.hpp>
-#include <iostream>
 
 static PyObject* _raise_error(PyObject* module) {
   PyErr_SetString(PyExc_TypeError, "Bad value provided");
@@ -8,7 +7,6 @@ static PyObject* _raise_error(PyObject* module) {
 
 PyObject* array_info_py_raw(PyObject* self, PyObject* args) {
   PyObject* source;
-  arrow::py::import_pyarrow();
 
   if(!PyArg_ParseTuple(args, "O", &source))
     return _raise_error(self);
@@ -21,8 +19,8 @@ PyObject* array_info_py_raw(PyObject* self, PyObject* args) {
   if(!result.ok())
     return _raise_error(self);
 
-  std::string ret_str = array_info_py(std::static_pointer_cast<arrow::Array>(result.ValueOrDie()));
-  return PyUnicode_FromStringAndSize(ret_str.c_str(), ret_str.length());
+  char* ret_str = array_info_py(std::static_pointer_cast<arrow::Array>(result.ValueOrDie()));
+  return PyUnicode_FromStringAndSize(ret_str, strlen(ret_str));
 }
 
 PyObject* create_array_py_raw(PyObject* self, PyObject* args) {
@@ -31,7 +29,6 @@ PyObject* create_array_py_raw(PyObject* self, PyObject* args) {
 
 PyObject* schema_info_py_raw(PyObject* self, PyObject* args) {
   PyObject* source;
-  arrow::py::import_pyarrow();
 
   // parse arguments
   if(!PyArg_ParseTuple(args, "O", &source))
@@ -45,8 +42,8 @@ PyObject* schema_info_py_raw(PyObject* self, PyObject* args) {
   if(!result.ok())
     return _raise_error(self);
 
-  std::string ret_str = schema_info_py(std::static_pointer_cast<arrow::Schema>(result.ValueOrDie()));
-  return PyUnicode_FromStringAndSize(ret_str.c_str(), ret_str.length());
+  char* ret_str = schema_info_py(std::static_pointer_cast<arrow::Schema>(result.ValueOrDie()));
+  return PyUnicode_FromStringAndSize(ret_str, strlen(ret_str));
 }
 
 PyObject* create_schema_py_raw(PyObject* self, PyObject* Py_UNUSED(args)) {

src/apn-python/cpython.hpp

@@ -5,7 +5,6 @@
 
 LIB_EXPORT PyObject* array_info_py_raw(PyObject*, PyObject*);
 LIB_EXPORT PyObject* create_array_py_raw(PyObject*, PyObject*);
-
 LIB_EXPORT PyObject* schema_info_py_raw(PyObject*, PyObject*);
 LIB_EXPORT PyObject* create_schema_py_raw(PyObject*, PyObject*);
 
@@ -24,6 +23,7 @@ static PyModuleDef cpythonextension_module = {
   PyModuleDef_HEAD_INIT, "cpythonextension", "cpython", -1, cpythonextension_methods};
 
 PyMODINIT_FUNC PyInit_cpythonextension(void) {
+  arrow::py::import_pyarrow();
   Py_Initialize();
   return PyModule_Create(&cpythonextension_module);
 }

src/apn-python/pybind11.cpp

@@ -1 +1,2 @@
 #include <apn-python/pybind11.hpp>
+