1
1
from typing import Callable , Dict , Iterable , Optional , Tuple
2
2
3
- from pyarrow import fs
3
+ from pyarrow ._fs import FileSystem
4
+ from pyarrow ._s3fs import S3FileSystem
4
5
from pyarrow .parquet import ParquetFile
5
6
6
7
from feast import type_map
@@ -138,7 +139,7 @@ def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]:
138
139
def get_table_column_names_and_types (
139
140
self , config : RepoConfig
140
141
) -> Iterable [Tuple [str , str ]]:
141
- filesystem , path = FileSource .prepare_path (
142
+ filesystem , path = FileSource .create_filesystem_and_path (
142
143
self .path , self ._file_options .s3_endpoint_override
143
144
)
144
145
schema = ParquetFile (
@@ -147,12 +148,14 @@ def get_table_column_names_and_types(
147
148
return zip (schema .names , map (str , schema .types ))
148
149
149
150
@staticmethod
150
- def prepare_path (path : str , s3_endpoint_override : str ):
151
+ def create_filesystem_and_path (
152
+ path : str , s3_endpoint_override : str
153
+ ) -> Tuple [Optional [FileSystem ], str ]:
151
154
if path .startswith ("s3://" ):
152
- s3 = fs . S3FileSystem (
155
+ s3fs = S3FileSystem (
153
156
endpoint_override = s3_endpoint_override if s3_endpoint_override else None
154
157
)
155
- return s3 , path .replace ("s3://" , "" )
158
+ return s3fs , path .replace ("s3://" , "" )
156
159
else :
157
160
return None , path
158
161
0 commit comments