ARROW-3829: [Python] add __arrow_array__ protocol to support third-party array classes in conversion to Arrow

python/pyarrow/array.pxi

@@ -161,7 +161,9 @@ def array(object obj, type=None, mask=None, size=None, from_pandas=None,
     else:
         c_from_pandas = from_pandas
 
-    if _is_array_like(obj):
+    if hasattr(obj, '__arrow_array__'):
+        return obj.__arrow_array__(type=type)
+    elif _is_array_like(obj):
         if mask is not None:
             # out argument unused
             mask = get_series_values(mask, &is_pandas_object)
@@ -178,7 +180,9 @@ def array(object obj, type=None, mask=None, size=None, from_pandas=None,
                 mask = values.mask
                 values = values.data
 
-        if pandas_api.is_categorical(values):
+        if hasattr(values, '__arrow_array__'):
+            return values.__arrow_array__(type=type)
+        elif pandas_api.is_categorical(values):
             return DictionaryArray.from_arrays(
                 values.codes, values.categories.values,
                 mask=mask, ordered=values.ordered,

python/pyarrow/pandas-shim.pxi

@@ -30,7 +30,7 @@ cdef class _PandasAPIShim(object):
         object _loose_version, _version
         object _pd, _types_api, _compat_module
         object _data_frame, _index, _series, _categorical_type
-        object _datetimetz_type
+        object _datetimetz_type, _extension_array
         object _array_like_types
 
     def __init__(self):
@@ -54,9 +54,11 @@ cdef class _PandasAPIShim(object):
         self._index = pd.Index
         self._categorical_type = pd.Categorical
         self._series = pd.Series
+        self._extension_array = pd.api.extensions.ExtensionArray
 
         self._array_like_types = (self._series, self._index,
-                                  self._categorical_type)
+                                  self._categorical_type,
+                                  self._extension_array)
 
         self._version = pd.__version__
         from distutils.version import LooseVersion

python/pyarrow/tests/test_array.py

@@ -1565,6 +1565,27 @@ def test_array_from_large_pyints():
         pa.array([int(2 ** 63)])
 
 
+def test_array_protocol():
+
+    class MyArray:
+        def __init__(self, data):
+            self.data = data
+
+        def __arrow_array__(self, type=None):
+            return pa.array(self.data, type=type)
+
+    arr = MyArray(np.array([1, 2, 3], dtype='int64'))
+    result = pa.array(arr)
+    expected = pa.array([1, 2, 3], type=pa.int64())
+    assert result.equals(expected)
+    result = pa.array(arr, type=pa.int64())
+    expected = pa.array([1, 2, 3], type=pa.int64())
+    assert result.equals(expected)
+    result = pa.array(arr, type=pa.float64())
+    expected = pa.array([1, 2, 3], type=pa.float64())
+    assert result.equals(expected)
+
+
 def test_concat_array():
     concatenated = pa.concat_arrays(
         [pa.array([1, 2]), pa.array([3, 4])])

python/pyarrow/tests/test_pandas.py

@@ -2882,6 +2882,50 @@ def test_variable_dictionary_with_pandas():
         a.to_pandas()
 
 
+# ----------------------------------------------------------------------
+# Array protocol in pandas conversions tests
+
+
+def test_array_protocol():
+
+    def __arrow_array__(self, type=None):
+        return pa.array(self._data, mask=self._mask, type=type)
+
+    df = pd.DataFrame({'a': pd.Series([1, 2, None], dtype='Int64')})
+
+    # with latest pandas/arrow, trying to convert nullable integer errors
+    with pytest.raises(TypeError):
+        pa.table(df)
+
+    # patch IntegerArray with the protocol
+    pd.arrays.IntegerArray.__arrow_array__ = __arrow_array__
+
+    # default conversion
+    result = pa.table(df)
+    expected = pa.array([1, 2, None], pa.int64())
+    assert result[0].chunk(0).equals(expected)
+
+    # with specifying schema
+    schema = pa.schema([('a', pa.float64())])
+    result = pa.table(df, schema=schema)
+    expected2 = pa.array([1, 2, None], pa.float64())
+    assert result[0].chunk(0).equals(expected2)
+
+    # pass Series to pa.array
+    result = pa.array(df['a'])
+    assert result.equals(expected)
+    result = pa.array(df['a'], type=pa.float64())
+    assert result.equals(expected2)
+
+    # pass actual ExtensionArray to pa.array
+    result = pa.array(df['a'].values)
+    assert result.equals(expected)
+    result = pa.array(df['a'].values, type=pa.float64())
+    assert result.equals(expected2)
+
+    del pd.arrays.IntegerArray.__arrow_array__
+
+
 # ----------------------------------------------------------------------
 # Legacy metadata compatibility tests