Skip to content

Commit

Permalink
Added use_fastavro parameter for backward compatibility
Browse files Browse the repository at this point in the history
Even though use_fastavro can be passed as an input parameter, it would have no effect.
  • Loading branch information
AnandInguva committed Nov 8, 2021
1 parent 3f9bc65 commit cb6e47c
Showing 1 changed file with 17 additions and 3 deletions.
20 changes: 17 additions & 3 deletions sdks/python/apache_beam/io/avroio.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,12 @@
class ReadFromAvro(PTransform):
"""A :class:`~apache_beam.transforms.ptransform.PTransform` for reading avro
files."""
def __init__(self, file_pattern=None, min_bundle_size=0, validate=True):
def __init__(
self,
file_pattern=None,
min_bundle_size=0,
validate=True,
use_fastavro=True):
"""Initializes :class:`ReadFromAvro`.
Uses source :class:`~apache_beam.io._AvroSource` to read a set of Avro
Expand Down Expand Up @@ -128,6 +133,8 @@ def __init__(self, file_pattern=None, min_bundle_size=0, validate=True):
splitting the input into bundles.
validate (bool): flag to verify that the files exist during the pipeline
creation time.
use_fastavro (bool): This flag is left for API backwards compatibility
and no longer has an effect. Do not use.
"""
super().__init__()
self._source = _create_avro_source(
Expand All @@ -152,6 +159,7 @@ class ReadAllFromAvro(PTransform):
def __init__(
self,
min_bundle_size=0,
use_fastavro=True,
desired_bundle_size=DEFAULT_DESIRED_BUNDLE_SIZE,
with_filename=False,
label='ReadAllFiles'):
Expand All @@ -165,6 +173,8 @@ def __init__(
with_filename: If True, returns a Key Value with the key being the file
name and the value being the actual data. If False, it only returns
the data.
use_fastavro (bool): This flag is left for API backwards compatibility
and no longer has an effect. Do not use.
"""
source_from_file = partial(
_create_avro_source, min_bundle_size=min_bundle_size)
Expand Down Expand Up @@ -206,7 +216,8 @@ def advance_file_past_next_sync_marker(f, sync_marker):
data = f.read(buf_size)


def _create_avro_source(file_pattern=None, min_bundle_size=0, validate=False):
def _create_avro_source(
file_pattern=None, min_bundle_size=0, validate=False, use_fasvro=True):
return \
_FastAvroSource(
file_pattern=file_pattern,
Expand Down Expand Up @@ -274,7 +285,8 @@ def __init__(
file_name_suffix='',
num_shards=0,
shard_name_template=None,
mime_type='application/x-avro'):
mime_type='application/x-avro',
use_fastavro=True):
"""Initialize a WriteToAvro transform.
Args:
Expand All @@ -301,6 +313,8 @@ def __init__(
is '-SSSSS-of-NNNNN' if None is passed as the shard_name_template.
mime_type: The MIME type to use for the produced files, if the filesystem
supports specifying MIME types.
use_fastavro (bool): This flag is left for API backwards compatibility
and no longer has an effect. Do not use.
Returns:
A WriteToAvro transform usable for writing.
Expand Down

0 comments on commit cb6e47c

Please sign in to comment.