diff --git a/ibis/backends/base/__init__.py b/ibis/backends/base/__init__.py index 73a6c5f44fb2..704bc3e63b74 100644 --- a/ibis/backends/base/__init__.py +++ b/ibis/backends/base/__init__.py @@ -245,6 +245,8 @@ def _import_pyarrow(): "Exporting to arrow formats requires `pyarrow` but it is not installed" ) else: + import pyarrow_hotfix # noqa: F401 + return pyarrow def to_pandas_batches( diff --git a/ibis/backends/base/sql/alchemy/__init__.py b/ibis/backends/base/sql/alchemy/__init__.py index 589d2c6c1913..8abc62900e80 100644 --- a/ibis/backends/base/sql/alchemy/__init__.py +++ b/ibis/backends/base/sql/alchemy/__init__.py @@ -286,6 +286,7 @@ def create_table( import pandas as pd import pyarrow as pa + import pyarrow_hotfix # noqa: F401 if isinstance(obj, (pd.DataFrame, pa.Table)): obj = ibis.memtable(obj) diff --git a/ibis/backends/bigquery/__init__.py b/ibis/backends/bigquery/__init__.py index 38606f2ff6b4..d9b3b8f7f743 100644 --- a/ibis/backends/bigquery/__init__.py +++ b/ibis/backends/bigquery/__init__.py @@ -685,6 +685,7 @@ def create_table( if obj is not None: import pyarrow as pa + import pyarrow_hotfix # noqa: F401 if isinstance(obj, (pd.DataFrame, pa.Table)): table = ibis.memtable(obj, schema=schema) diff --git a/ibis/backends/clickhouse/__init__.py b/ibis/backends/clickhouse/__init__.py index 99f6c699bba2..ed19a1c7ce6e 100644 --- a/ibis/backends/clickhouse/__init__.py +++ b/ibis/backends/clickhouse/__init__.py @@ -10,6 +10,7 @@ import clickhouse_connect as cc import pyarrow as pa +import pyarrow_hotfix # noqa: F401 import sqlalchemy as sa import sqlglot as sg import toolz diff --git a/ibis/backends/datafusion/__init__.py b/ibis/backends/datafusion/__init__.py index b9ac998532a2..f0514593bd78 100644 --- a/ibis/backends/datafusion/__init__.py +++ b/ibis/backends/datafusion/__init__.py @@ -9,6 +9,7 @@ import datafusion as df import pyarrow as pa import pyarrow.dataset as ds +import pyarrow_hotfix # noqa: F401 import sqlglot as sg from sqlglot import exp, transforms from sqlglot.dialects import Postgres diff --git a/ibis/backends/datafusion/udfs.py b/ibis/backends/datafusion/udfs.py index 0db6b6b4231b..7bf52a953623 100644 --- a/ibis/backends/datafusion/udfs.py +++ b/ibis/backends/datafusion/udfs.py @@ -5,6 +5,7 @@ import pyarrow as pa import pyarrow.compute as pc +import pyarrow_hotfix # noqa: F401 import ibis.expr.datatypes as dt # noqa: TCH001 diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py index 04fb675584ed..319ee4c56377 100644 --- a/ibis/backends/duckdb/__init__.py +++ b/ibis/backends/duckdb/__init__.py @@ -11,6 +11,7 @@ import duckdb import pyarrow as pa +import pyarrow_hotfix # noqa: F401 import sqlalchemy as sa import sqlglot as sg import toolz diff --git a/ibis/backends/flink/__init__.py b/ibis/backends/flink/__init__.py index c584a4cb24bd..83f6171ffaa2 100644 --- a/ibis/backends/flink/__init__.py +++ b/ibis/backends/flink/__init__.py @@ -312,6 +312,7 @@ def create_table( """ import pandas as pd import pyarrow as pa + import pyarrow_hotfix # noqa: F401 import ibis.expr.types as ir @@ -485,6 +486,7 @@ def insert( """ import pandas as pd import pyarrow as pa + import pyarrow_hotfix # noqa: F401 if isinstance(obj, ir.Table): expr = obj diff --git a/ibis/backends/impala/__init__.py b/ibis/backends/impala/__init__.py index e465b33cae3d..396daa8b60ff 100644 --- a/ibis/backends/impala/__init__.py +++ b/ibis/backends/impala/__init__.py @@ -1389,6 +1389,7 @@ def to_pyarrow( **kwargs: Any, ) -> pa.Table: import pyarrow as pa + import pyarrow_hotfix # noqa: F401 from ibis.formats.pyarrow import PyArrowData diff --git a/ibis/backends/pandas/__init__.py b/ibis/backends/pandas/__init__.py index 35b69fa87e2a..1b4573839d3b 100644 --- a/ibis/backends/pandas/__init__.py +++ b/ibis/backends/pandas/__init__.py @@ -6,6 +6,7 @@ import pandas as pd import pyarrow as pa +import pyarrow_hotfix # noqa: F401 import ibis.common.exceptions as com import ibis.config @@ -242,8 +243,6 @@ def _from_pandas(df: pd.DataFrame) -> pd.DataFrame: @classmethod def _convert_object(cls, obj: Any) -> Any: - import pyarrow as pa - if isinstance(obj, ir.Table): # Support memtables assert isinstance(obj.op(), ops.InMemoryTable) diff --git a/ibis/backends/pyspark/__init__.py b/ibis/backends/pyspark/__init__.py index 3f2432126ae7..4a40e7f79fcf 100644 --- a/ibis/backends/pyspark/__init__.py +++ b/ibis/backends/pyspark/__init__.py @@ -387,6 +387,7 @@ def create_table( """ import pandas as pd import pyarrow as pa + import pyarrow_hotfix # noqa: F401 if obj is None and schema is None: raise com.IbisError("The schema or obj parameter is required") @@ -827,6 +828,7 @@ def to_pyarrow( **kwargs: Any, ) -> pa.Table: import pyarrow as pa + import pyarrow_hotfix # noqa: F401 from ibis.formats.pyarrow import PyArrowData diff --git a/ibis/backends/snowflake/__init__.py b/ibis/backends/snowflake/__init__.py index 8418ac846bc6..2ac5df33cf9b 100644 --- a/ibis/backends/snowflake/__init__.py +++ b/ibis/backends/snowflake/__init__.py @@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Any import pyarrow as pa +import pyarrow_hotfix # noqa: F401 import sqlalchemy as sa import sqlglot as sg from packaging.version import parse as vparse diff --git a/ibis/backends/snowflake/tests/conftest.py b/ibis/backends/snowflake/tests/conftest.py index 55f917928a7f..4faac50223a0 100644 --- a/ibis/backends/snowflake/tests/conftest.py +++ b/ibis/backends/snowflake/tests/conftest.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any import pyarrow.parquet as pq +import pyarrow_hotfix # noqa: F401 import pytest import sqlalchemy as sa import sqlglot as sg diff --git a/ibis/backends/trino/__init__.py b/ibis/backends/trino/__init__.py index 686dc618fad0..34b5f315b8b4 100644 --- a/ibis/backends/trino/__init__.py +++ b/ibis/backends/trino/__init__.py @@ -351,6 +351,7 @@ def literal_compile(v): if obj is not None: import pyarrow as pa + import pyarrow_hotfix # noqa: F401 if isinstance(obj, (pd.DataFrame, pa.Table)): table = ibis.memtable(obj, schema=schema) diff --git a/ibis/examples/__init__.py b/ibis/examples/__init__.py index 24d46aaf1345..eb23d7d73580 100644 --- a/ibis/examples/__init__.py +++ b/ibis/examples/__init__.py @@ -52,6 +52,8 @@ def fetch( else: return backend.read_csv(path, table_name=table_name) else: + import pyarrow_hotfix # noqa: F401 + if path.endswith(".parquet"): import pyarrow.parquet diff --git a/ibis/expr/operations/relations.py b/ibis/expr/operations/relations.py index 5bb0466767fc..d379ad8675a9 100644 --- a/ibis/expr/operations/relations.py +++ b/ibis/expr/operations/relations.py @@ -141,6 +141,7 @@ def to_pyarrow(self, schema: Schema) -> pa.Table: # pragma: no cover def to_pyarrow_bytes(self, schema: Schema) -> bytes: import pyarrow as pa + import pyarrow_hotfix # noqa: F401 data = self.to_pyarrow(schema=schema) out = pa.BufferOutputStream() @@ -170,6 +171,7 @@ def to_frame(self) -> pd.DataFrame: def to_pyarrow(self, schema: Schema) -> pa.Table: import pyarrow as pa + import pyarrow_hotfix # noqa: F401 from ibis.formats.pyarrow import PyArrowSchema diff --git a/ibis/formats/pyarrow.py b/ibis/formats/pyarrow.py index 7fbb90b9b389..5784198db7a2 100644 --- a/ibis/formats/pyarrow.py +++ b/ibis/formats/pyarrow.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any import pyarrow as pa +import pyarrow_hotfix # noqa: F401 import ibis.expr.datatypes as dt from ibis.expr.schema import Schema diff --git a/poetry.lock b/poetry.lock index 1e3ea7a53516..d853ad2a22c3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3853,6 +3853,17 @@ files = [ [package.dependencies] numpy = ">=1.16.6" +[[package]] +name = "pyarrow-hotfix" +version = "0.4" +description = "" +optional = false +python-versions = ">=3.5" +files = [ + {file = "pyarrow_hotfix-0.4-py3-none-any.whl", hash = "sha256:aecaecaccbe4b3d7b7198d6634afe1c5eec3cbf89d1b551e9f85675f21b24921"}, + {file = "pyarrow_hotfix-0.4.tar.gz", hash = "sha256:eb6e2738ba6e023e93a5fd8078d12c437e90d79df87736f1b2ba1c46f0e242d1"}, +] + [[package]] name = "pyasn1" version = "0.5.0" @@ -6267,4 +6278,4 @@ visualization = ["graphviz"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "7200e140aabf8a6616d85f74f33aab840e2065d925d12617afb5ceecf2b81253" +content-hash = "36fc927f7e49d8794b2fd8dc260a83c09027955ad1c697adfa41a7479d63ca80" diff --git a/pyproject.toml b/pyproject.toml index eeca19812bf1..98def98921d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ pandas = ">=1.2.5,<3" parsy = ">=2,<3" pins = { version = ">=0.8.3,<1", extras = ["gcs"] } pyarrow = ">=2,<15" +pyarrow-hotfix = ">=0.4,<1" python-dateutil = ">=2.8.2,<3" pytz = ">=2022.7" rich = ">=12.4.4,<14" diff --git a/requirements-dev.txt b/requirements-dev.txt index f6e5ed81be43..a021d0b6d66f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -139,6 +139,7 @@ pure-eval==0.2.2 ; python_version >= "3.9" and python_version < "4.0" pure-sasl==0.6.2 ; python_version >= "3.9" and python_version < "4.0" py-cpuinfo==9.0.0 ; python_version >= "3.9" and python_version < "4.0" py4j==0.10.9.5 ; python_version >= "3.9" and python_version < "4.0" +pyarrow-hotfix==0.4 ; python_version >= "3.9" and python_version < "4.0" pyarrow==14.0.0 ; python_version >= "3.9" and python_version < "4.0" pyasn1-modules==0.3.0 ; python_version >= "3.9" and python_version < "4.0" pyasn1==0.5.0 ; python_version >= "3.9" and python_version < "4.0"