From 7ce19759b715c7af4af31340db2f626cbef6aac4 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 16 Mar 2023 09:23:47 -0700 Subject: [PATCH 01/16] added to_dict to filters --- google/cloud/bigtable/row_filters.py | 295 ++++++++++++++------------- 1 file changed, 156 insertions(+), 139 deletions(-) diff --git a/google/cloud/bigtable/row_filters.py b/google/cloud/bigtable/row_filters.py index 53192acc8..79728822d 100644 --- a/google/cloud/bigtable/row_filters.py +++ b/google/cloud/bigtable/row_filters.py @@ -13,14 +13,20 @@ # limitations under the License. """Filters for Google Cloud Bigtable Row classes.""" +from __future__ import annotations import struct +from typing import Any, TYPE_CHECKING from google.cloud._helpers import _microseconds_from_datetime # type: ignore from google.cloud._helpers import _to_bytes # type: ignore from google.cloud.bigtable_v2.types import data as data_v2_pb2 +if TYPE_CHECKING: + # import dependencies when type checking + from datetime import datetime + _PACK_I64 = struct.Struct(">q").pack @@ -35,6 +41,19 @@ class RowFilter(object): This class is a do-nothing base class for all row filters. """ + def to_pb(self) -> data_v2_pb2.RowFilter: + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_v2_pb2.RowFilter` + :returns: The converted current object. + """ + return data_v2_pb2.RowFilter(**self.to_dict()) + + def to_dict(self) -> dict[str, Any]: + """Converts the row filter to a dict representation.""" + # unimplemented on base class + raise NotImplementedError + class _BoolFilter(RowFilter): """Row filter that uses a boolean flag. @@ -43,7 +62,7 @@ class _BoolFilter(RowFilter): :param flag: An indicator if a setting is turned on or off. """ - def __init__(self, flag): + def __init__(self, flag: bool): self.flag = flag def __eq__(self, other): @@ -66,13 +85,9 @@ class SinkFilter(_BoolFilter): of a :class:`ConditionalRowFilter`. """ - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. - """ - return data_v2_pb2.RowFilter(sink=self.flag) + def to_dict(self) -> dict[str, Any]: + """Converts the row filter to a dict representation.""" + return {"sink": self.flag} class PassAllFilter(_BoolFilter): @@ -84,13 +99,9 @@ class PassAllFilter(_BoolFilter): completeness. """ - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. - """ - return data_v2_pb2.RowFilter(pass_all_filter=self.flag) + def to_dict(self) -> dict[str, Any]: + """Converts the row filter to a dict representation.""" + return {"pass_all_filter": self.flag} class BlockAllFilter(_BoolFilter): @@ -101,13 +112,9 @@ class BlockAllFilter(_BoolFilter): temporarily disabling just part of a filter. """ - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. - """ - return data_v2_pb2.RowFilter(block_all_filter=self.flag) + def to_dict(self) -> dict[str, Any]: + """Converts the row filter to a dict representation.""" + return {"block_all_filter": self.flag} class _RegexFilter(RowFilter): @@ -124,8 +131,8 @@ class _RegexFilter(RowFilter): will be encoded as ASCII. """ - def __init__(self, regex): - self.regex = _to_bytes(regex) + def __init__(self, regex: str | bytes): + self.regex: bytes = _to_bytes(regex) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -159,13 +166,9 @@ class RowKeyRegexFilter(_RegexFilter): since the row key is already specified. """ - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. - """ - return data_v2_pb2.RowFilter(row_key_regex_filter=self.regex) + def to_dict(self) -> dict[str, Any]: + """Converts the row filter to a dict representation.""" + return {"row_key_regex_filter": self.regex} class RowSampleFilter(RowFilter): @@ -176,8 +179,8 @@ class RowSampleFilter(RowFilter): interval ``(0, 1)`` The end points are excluded). """ - def __init__(self, sample): - self.sample = sample + def __init__(self, sample: float): + self.sample: float = sample def __eq__(self, other): if not isinstance(other, self.__class__): @@ -187,13 +190,9 @@ def __eq__(self, other): def __ne__(self, other): return not self == other - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. - """ - return data_v2_pb2.RowFilter(row_sample_filter=self.sample) + def to_dict(self) -> dict[str, Any]: + """Converts the row filter to a dict representation.""" + return {"row_sample_filter": self.sample} class FamilyNameRegexFilter(_RegexFilter): @@ -211,13 +210,9 @@ class FamilyNameRegexFilter(_RegexFilter): used as a literal. """ - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. - """ - return data_v2_pb2.RowFilter(family_name_regex_filter=self.regex) + def to_dict(self) -> dict[str, Any]: + """Converts the row filter to a dict representation.""" + return {"family_name_regex_filter": self.regex} class ColumnQualifierRegexFilter(_RegexFilter): @@ -241,13 +236,9 @@ class ColumnQualifierRegexFilter(_RegexFilter): match this regex (irrespective of column family). """ - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. - """ - return data_v2_pb2.RowFilter(column_qualifier_regex_filter=self.regex) + def to_dict(self) -> dict[str, Any]: + """Converts the row filter to a dict representation.""" + return {"column_qualifier_regex_filter": self.regex} class TimestampRange(object): @@ -262,9 +253,9 @@ class TimestampRange(object): range. If omitted, no upper bound is used. """ - def __init__(self, start=None, end=None): - self.start = start - self.end = end + def __init__(self, start: "datetime" | None = None, end: "datetime" | None = None): + self.start: "datetime" | None = start + self.end: "datetime" | None = end def __eq__(self, other): if not isinstance(other, self.__class__): @@ -274,23 +265,27 @@ def __eq__(self, other): def __ne__(self, other): return not self == other - def to_pb(self): + def to_pb(self) -> data_v2_pb2.TimestampRange: """Converts the :class:`TimestampRange` to a protobuf. :rtype: :class:`.data_v2_pb2.TimestampRange` :returns: The converted current object. """ + return data_v2_pb2.TimestampRange(**self.to_dict()) + + def to_dict(self) -> dict[str, int]: + """Converts the timestamp range to a dict representation.""" timestamp_range_kwargs = {} if self.start is not None: - timestamp_range_kwargs["start_timestamp_micros"] = ( - _microseconds_from_datetime(self.start) // 1000 * 1000 - ) + start_time = _microseconds_from_datetime(self.start) // 1000 * 1000 + timestamp_range_kwargs["start_timestamp_micros"] = start_time if self.end is not None: end_time = _microseconds_from_datetime(self.end) if end_time % 1000 != 0: + # if not a whole milisecond value, round up end_time = end_time // 1000 * 1000 + 1000 timestamp_range_kwargs["end_timestamp_micros"] = end_time - return data_v2_pb2.TimestampRange(**timestamp_range_kwargs) + return timestamp_range_kwargs class TimestampRangeFilter(RowFilter): @@ -300,8 +295,8 @@ class TimestampRangeFilter(RowFilter): :param range_: Range of time that cells should match against. """ - def __init__(self, range_): - self.range_ = range_ + def __init__(self, start: "datetime" | None = None, end: "datetime" | None = None): + self.range_: TimestampRange = TimestampRange(start, end) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -322,6 +317,10 @@ def to_pb(self): """ return data_v2_pb2.RowFilter(timestamp_range_filter=self.range_.to_pb()) + def to_dict(self) -> dict[str, Any]: + """Converts the row filter to a dict representation.""" + return {"timestamp_range_filter": self.range_.to_dict()} + class ColumnRangeFilter(RowFilter): """A row filter to restrict to a range of columns. @@ -362,29 +361,28 @@ class ColumnRangeFilter(RowFilter): def __init__( self, - column_family_id, - start_column=None, - end_column=None, - inclusive_start=None, - inclusive_end=None, + column_family_id: str, + start_column: bytes | None = None, + end_column: bytes | None = None, + inclusive_start: bool | None = None, + inclusive_end: bool | None = None, ): - self.column_family_id = column_family_id - if inclusive_start is None: inclusive_start = True elif start_column is None: raise ValueError( - "Inclusive start was specified but no " "start column was given." + "inclusive_start was specified but no start_column was given." ) - self.start_column = start_column - self.inclusive_start = inclusive_start - if inclusive_end is None: inclusive_end = True elif end_column is None: - raise ValueError( - "Inclusive end was specified but no " "end column was given." - ) + raise ValueError("inclusive_end was specified but no end_column was given.") + + self.column_family_id = column_family_id + + self.start_column = start_column + self.inclusive_start = inclusive_start + self.end_column = end_column self.inclusive_end = inclusive_end @@ -411,7 +409,13 @@ def to_pb(self): :rtype: :class:`.data_v2_pb2.RowFilter` :returns: The converted current object. """ - column_range_kwargs = {"family_name": self.column_family_id} + column_range = data_v2_pb2.ColumnRange(**self.range_to_dict()) + return data_v2_pb2.RowFilter(column_range_filter=column_range) + + def range_to_dict(self) -> dict[str, str | bytes]: + """Converts the column range range to a dict representation.""" + column_range_kwargs: dict[str, str | bytes] = {} + column_range_kwargs["family_name"] = self.column_family_id if self.start_column is not None: if self.inclusive_start: key = "start_qualifier_closed" @@ -424,9 +428,11 @@ def to_pb(self): else: key = "end_qualifier_open" column_range_kwargs[key] = _to_bytes(self.end_column) + return column_range_kwargs - column_range = data_v2_pb2.ColumnRange(**column_range_kwargs) - return data_v2_pb2.RowFilter(column_range_filter=column_range) + def to_dict(self) -> dict[str, Any]: + """Converts the row filter to a dict representation.""" + return {"column_range_filter": self.range_to_dict()} class ValueRegexFilter(_RegexFilter): @@ -450,13 +456,9 @@ class ValueRegexFilter(_RegexFilter): match this regex. String values will be encoded as ASCII. """ - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. - """ - return data_v2_pb2.RowFilter(value_regex_filter=self.regex) + def to_dict(self) -> dict[str, bytes]: + """Converts the row filter to a dict representation.""" + return {"value_regex_filter": self.regex} class ExactValueFilter(ValueRegexFilter): @@ -469,7 +471,7 @@ class ExactValueFilter(ValueRegexFilter): equivalent bytes, or an integer (which will be packed into 8-bytes). """ - def __init__(self, value): + def __init__(self, value: bytes | str | int): if isinstance(value, int): value = _PACK_I64(value) super(ExactValueFilter, self).__init__(value) @@ -510,25 +512,27 @@ class ValueRangeFilter(RowFilter): """ def __init__( - self, start_value=None, end_value=None, inclusive_start=None, inclusive_end=None + self, + start_value: bytes | int | None = None, + end_value: bytes | int | None = None, + inclusive_start: bool | None = None, + inclusive_end: bool | None = None, ): if inclusive_start is None: inclusive_start = True elif start_value is None: raise ValueError( - "Inclusive start was specified but no " "start value was given." + "inclusive_start was specified but no start_value was given." ) + if inclusive_end is None: + inclusive_end = True + elif end_value is None: + raise ValueError("inclusive_end was specified but no end_column was given.") if isinstance(start_value, int): start_value = _PACK_I64(start_value) self.start_value = start_value self.inclusive_start = inclusive_start - if inclusive_end is None: - inclusive_end = True - elif end_value is None: - raise ValueError( - "Inclusive end was specified but no " "end value was given." - ) if isinstance(end_value, int): end_value = _PACK_I64(end_value) self.end_value = end_value @@ -556,6 +560,11 @@ def to_pb(self): :rtype: :class:`.data_v2_pb2.RowFilter` :returns: The converted current object. """ + value_range = data_v2_pb2.ValueRange(**self.range_to_dict()) + return data_v2_pb2.RowFilter(value_range_filter=value_range) + + def range_to_dict(self) -> dict[str, bytes]: + """Converts the value range range to a dict representation.""" value_range_kwargs = {} if self.start_value is not None: if self.inclusive_start: @@ -569,9 +578,11 @@ def to_pb(self): else: key = "end_value_open" value_range_kwargs[key] = _to_bytes(self.end_value) + return value_range_kwargs - value_range = data_v2_pb2.ValueRange(**value_range_kwargs) - return data_v2_pb2.RowFilter(value_range_filter=value_range) + def to_dict(self) -> dict[str, Any]: + """Converts the row filter to a dict representation.""" + return {"value_range_filter": self.range_to_dict()} class _CellCountFilter(RowFilter): @@ -584,7 +595,7 @@ class _CellCountFilter(RowFilter): :param num_cells: An integer count / offset / limit. """ - def __init__(self, num_cells): + def __init__(self, num_cells: int): self.num_cells = num_cells def __eq__(self, other): @@ -603,13 +614,9 @@ class CellsRowOffsetFilter(_CellCountFilter): :param num_cells: Skips the first N cells of the row. """ - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. - """ - return data_v2_pb2.RowFilter(cells_per_row_offset_filter=self.num_cells) + def to_dict(self) -> dict[str, int]: + """Converts the row filter to a dict representation.""" + return {"cells_per_row_offset_filter": self.num_cells} class CellsRowLimitFilter(_CellCountFilter): @@ -619,13 +626,9 @@ class CellsRowLimitFilter(_CellCountFilter): :param num_cells: Matches only the first N cells of the row. """ - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. - """ - return data_v2_pb2.RowFilter(cells_per_row_limit_filter=self.num_cells) + def to_dict(self) -> dict[str, int]: + """Converts the row filter to a dict representation.""" + return {"cells_per_row_limit_filter": self.num_cells} class CellsColumnLimitFilter(_CellCountFilter): @@ -637,13 +640,9 @@ class CellsColumnLimitFilter(_CellCountFilter): timestamps of each cell. """ - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. - """ - return data_v2_pb2.RowFilter(cells_per_column_limit_filter=self.num_cells) + def to_dict(self) -> dict[str, int]: + """Converts the row filter to a dict representation.""" + return {"cells_per_column_limit_filter": self.num_cells} class StripValueTransformerFilter(_BoolFilter): @@ -655,13 +654,9 @@ class StripValueTransformerFilter(_BoolFilter): transformer than a generic query / filter. """ - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. - """ - return data_v2_pb2.RowFilter(strip_value_transformer=self.flag) + def to_dict(self) -> dict[str, Any]: + """Converts the row filter to a dict representation.""" + return {"strip_value_transformer": self.flag} class ApplyLabelFilter(RowFilter): @@ -683,7 +678,7 @@ class ApplyLabelFilter(RowFilter): ``[a-z0-9\\-]+``. """ - def __init__(self, label): + def __init__(self, label: str): self.label = label def __eq__(self, other): @@ -694,13 +689,9 @@ def __eq__(self, other): def __ne__(self, other): return not self == other - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. - """ - return data_v2_pb2.RowFilter(apply_label_transformer=self.label) + def to_dict(self) -> dict[str, str]: + """Converts the row filter to a dict representation.""" + return {"apply_label_transformer": self.label} class _FilterCombination(RowFilter): @@ -714,10 +705,10 @@ class _FilterCombination(RowFilter): :param filters: List of :class:`RowFilter` """ - def __init__(self, filters=None): + def __init__(self, filters: list[RowFilter] | None = None): if filters is None: filters = [] - self.filters = filters + self.filters: list[RowFilter] = filters def __eq__(self, other): if not isinstance(other, self.__class__): @@ -750,6 +741,10 @@ def to_pb(self): ) return data_v2_pb2.RowFilter(chain=chain) + def to_dict(self) -> dict[str, Any]: + """Converts the row filter to a dict representation.""" + return {"chain": {"filters": [f.to_dict() for f in self.filters]}} + class RowFilterUnion(_FilterCombination): """Union of row filters. @@ -775,6 +770,10 @@ def to_pb(self): ) return data_v2_pb2.RowFilter(interleave=interleave) + def to_dict(self) -> dict[str, Any]: + """Converts the row filter to a dict representation.""" + return {"interleave": {"filters": [f.to_dict() for f in self.filters]}} + class ConditionalRowFilter(RowFilter): """Conditional row filter which exhibits ternary behavior. @@ -806,7 +805,12 @@ class ConditionalRowFilter(RowFilter): will be returned in the false case. """ - def __init__(self, base_filter, true_filter=None, false_filter=None): + def __init__( + self, + base_filter: RowFilter, + true_filter: RowFilter | None = None, + false_filter: RowFilter | None = None, + ): self.base_filter = base_filter self.true_filter = true_filter self.false_filter = false_filter @@ -836,3 +840,16 @@ def to_pb(self): condition_kwargs["false_filter"] = self.false_filter.to_pb() condition = data_v2_pb2.RowFilter.Condition(**condition_kwargs) return data_v2_pb2.RowFilter(condition=condition) + + def condition_to_dict(self) -> dict[str, Any]: + """Converts the condition to a dict representation.""" + condition_kwargs = {"predicate_filter": self.base_filter.to_dict()} + if self.true_filter is not None: + condition_kwargs["true_filter"] = self.true_filter.to_dict() + if self.false_filter is not None: + condition_kwargs["false_filter"] = self.false_filter.to_dict() + return condition_kwargs + + def to_dict(self) -> dict[str, Any]: + """Converts the row filter to a dict representation.""" + return {"condition": self.condition_to_dict()} From e78dac7a90fedb7ce7d7d82a2e13e73910fce83d Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 16 Mar 2023 09:24:13 -0700 Subject: [PATCH 02/16] added filters tests --- tests/unit/test_row_filters.py | 1617 ++++++++++++++++++++++++++++++++ 1 file changed, 1617 insertions(+) create mode 100644 tests/unit/test_row_filters.py diff --git a/tests/unit/test_row_filters.py b/tests/unit/test_row_filters.py new file mode 100644 index 000000000..759e84c45 --- /dev/null +++ b/tests/unit/test_row_filters.py @@ -0,0 +1,1617 @@ +# Copyright 2016 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest + + +def test_bool_filter_constructor(): + from google.cloud.bigtable.row_filters import _BoolFilter + + flag = object() + row_filter = _BoolFilter(flag) + assert row_filter.flag is flag + + +def test_bool_filter___eq__type_differ(): + from google.cloud.bigtable.row_filters import _BoolFilter + + flag = object() + row_filter1 = _BoolFilter(flag) + row_filter2 = object() + assert not (row_filter1 == row_filter2) + + +def test_bool_filter___eq__same_value(): + from google.cloud.bigtable.row_filters import _BoolFilter + + flag = object() + row_filter1 = _BoolFilter(flag) + row_filter2 = _BoolFilter(flag) + assert row_filter1 == row_filter2 + + +def test_bool_filter___ne__same_value(): + from google.cloud.bigtable.row_filters import _BoolFilter + + flag = object() + row_filter1 = _BoolFilter(flag) + row_filter2 = _BoolFilter(flag) + assert not (row_filter1 != row_filter2) + + +def test_sink_filter_to_pb(): + from google.cloud.bigtable.row_filters import SinkFilter + + flag = True + row_filter = SinkFilter(flag) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(sink=flag) + assert pb_val == expected_pb + + +def test_sink_filter_to_dict(): + from google.cloud.bigtable.row_filters import SinkFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + flag = True + row_filter = SinkFilter(flag) + expected_dict = {"sink": flag} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_pass_all_filter_to_pb(): + from google.cloud.bigtable.row_filters import PassAllFilter + + flag = True + row_filter = PassAllFilter(flag) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(pass_all_filter=flag) + assert pb_val == expected_pb + + +def test_pass_all_filter_to_dict(): + from google.cloud.bigtable.row_filters import PassAllFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + flag = True + row_filter = PassAllFilter(flag) + expected_dict = {"pass_all_filter": flag} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_block_all_filter_to_pb(): + from google.cloud.bigtable.row_filters import BlockAllFilter + + flag = True + row_filter = BlockAllFilter(flag) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(block_all_filter=flag) + assert pb_val == expected_pb + + +def test_block_all_filter_to_dict(): + from google.cloud.bigtable.row_filters import BlockAllFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + flag = True + row_filter = BlockAllFilter(flag) + expected_dict = {"block_all_filter": flag} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_regex_filterconstructor(): + from google.cloud.bigtable.row_filters import _RegexFilter + + regex = b"abc" + row_filter = _RegexFilter(regex) + assert row_filter.regex is regex + + +def test_regex_filterconstructor_non_bytes(): + from google.cloud.bigtable.row_filters import _RegexFilter + + regex = "abc" + row_filter = _RegexFilter(regex) + assert row_filter.regex == b"abc" + + +def test_regex_filter__eq__type_differ(): + from google.cloud.bigtable.row_filters import _RegexFilter + + regex = b"def-rgx" + row_filter1 = _RegexFilter(regex) + row_filter2 = object() + assert not (row_filter1 == row_filter2) + + +def test_regex_filter__eq__same_value(): + from google.cloud.bigtable.row_filters import _RegexFilter + + regex = b"trex-regex" + row_filter1 = _RegexFilter(regex) + row_filter2 = _RegexFilter(regex) + assert row_filter1 == row_filter2 + + +def test_regex_filter__ne__same_value(): + from google.cloud.bigtable.row_filters import _RegexFilter + + regex = b"abc" + row_filter1 = _RegexFilter(regex) + row_filter2 = _RegexFilter(regex) + assert not (row_filter1 != row_filter2) + + +def test_row_key_regex_filter_to_pb(): + from google.cloud.bigtable.row_filters import RowKeyRegexFilter + + regex = b"row-key-regex" + row_filter = RowKeyRegexFilter(regex) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(row_key_regex_filter=regex) + assert pb_val == expected_pb + + +def test_row_key_regex_filter_to_dict(): + from google.cloud.bigtable.row_filters import RowKeyRegexFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + regex = b"row-key-regex" + row_filter = RowKeyRegexFilter(regex) + expected_dict = {"row_key_regex_filter": regex} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_row_sample_filter_constructor(): + from google.cloud.bigtable.row_filters import RowSampleFilter + + sample = object() + row_filter = RowSampleFilter(sample) + assert row_filter.sample is sample + + +def test_row_sample_filter___eq__type_differ(): + from google.cloud.bigtable.row_filters import RowSampleFilter + + sample = object() + row_filter1 = RowSampleFilter(sample) + row_filter2 = object() + assert not (row_filter1 == row_filter2) + + +def test_row_sample_filter___eq__same_value(): + from google.cloud.bigtable.row_filters import RowSampleFilter + + sample = object() + row_filter1 = RowSampleFilter(sample) + row_filter2 = RowSampleFilter(sample) + assert row_filter1 == row_filter2 + + +def test_row_sample_filter___ne__(): + from google.cloud.bigtable.row_filters import RowSampleFilter + + sample = object() + other_sample = object() + row_filter1 = RowSampleFilter(sample) + row_filter2 = RowSampleFilter(other_sample) + assert row_filter1 != row_filter2 + + +def test_row_sample_filter_to_pb(): + from google.cloud.bigtable.row_filters import RowSampleFilter + + sample = 0.25 + row_filter = RowSampleFilter(sample) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(row_sample_filter=sample) + assert pb_val == expected_pb + + +def test_family_name_regex_filter_to_pb(): + from google.cloud.bigtable.row_filters import FamilyNameRegexFilter + + regex = "family-regex" + row_filter = FamilyNameRegexFilter(regex) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(family_name_regex_filter=regex) + assert pb_val == expected_pb + + +def test_family_name_regex_filter_to_dict(): + from google.cloud.bigtable.row_filters import FamilyNameRegexFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + regex = "family-regex" + row_filter = FamilyNameRegexFilter(regex) + expected_dict = {"family_name_regex_filter": regex.encode()} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_column_qualifier_regex_filter_to_pb(): + from google.cloud.bigtable.row_filters import ColumnQualifierRegexFilter + + regex = b"column-regex" + row_filter = ColumnQualifierRegexFilter(regex) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(column_qualifier_regex_filter=regex) + assert pb_val == expected_pb + + +def test_column_qualifier_regex_filter_to_dict(): + from google.cloud.bigtable.row_filters import ColumnQualifierRegexFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + regex = b"column-regex" + row_filter = ColumnQualifierRegexFilter(regex) + expected_dict = {"column_qualifier_regex_filter": regex} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_timestamp_range_constructor(): + from google.cloud.bigtable.row_filters import TimestampRange + + start = object() + end = object() + time_range = TimestampRange(start=start, end=end) + assert time_range.start is start + assert time_range.end is end + + +def test_timestamp_range___eq__(): + from google.cloud.bigtable.row_filters import TimestampRange + + start = object() + end = object() + time_range1 = TimestampRange(start=start, end=end) + time_range2 = TimestampRange(start=start, end=end) + assert time_range1 == time_range2 + + +def test_timestamp_range___eq__type_differ(): + from google.cloud.bigtable.row_filters import TimestampRange + + start = object() + end = object() + time_range1 = TimestampRange(start=start, end=end) + time_range2 = object() + assert not (time_range1 == time_range2) + + +def test_timestamp_range___ne__same_value(): + from google.cloud.bigtable.row_filters import TimestampRange + + start = object() + end = object() + time_range1 = TimestampRange(start=start, end=end) + time_range2 = TimestampRange(start=start, end=end) + assert not (time_range1 != time_range2) + + +def _timestamp_range_to_pb_helper(pb_kwargs, start=None, end=None): + import datetime + from google.cloud._helpers import _EPOCH + from google.cloud.bigtable.row_filters import TimestampRange + + if start is not None: + start = _EPOCH + datetime.timedelta(microseconds=start) + if end is not None: + end = _EPOCH + datetime.timedelta(microseconds=end) + time_range = TimestampRange(start=start, end=end) + expected_pb = _TimestampRangePB(**pb_kwargs) + time_pb = time_range.to_pb() + assert time_pb.start_timestamp_micros == expected_pb.start_timestamp_micros + assert time_pb.end_timestamp_micros == expected_pb.end_timestamp_micros + assert time_pb == expected_pb + + +def test_timestamp_range_to_pb(): + start_micros = 30871234 + end_micros = 12939371234 + start_millis = start_micros // 1000 * 1000 + assert start_millis == 30871000 + end_millis = end_micros // 1000 * 1000 + 1000 + assert end_millis == 12939372000 + pb_kwargs = {} + pb_kwargs["start_timestamp_micros"] = start_millis + pb_kwargs["end_timestamp_micros"] = end_millis + _timestamp_range_to_pb_helper(pb_kwargs, start=start_micros, end=end_micros) + + +def test_timestamp_range_to_dict(): + from google.cloud.bigtable.row_filters import TimestampRange + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + import datetime + + row_filter = TimestampRange( + start=datetime.datetime(2019, 1, 1), end=datetime.datetime(2019, 1, 2) + ) + expected_dict = { + "start_timestamp_micros": 1546300800000000, + "end_timestamp_micros": 1546387200000000, + } + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.TimestampRange(**expected_dict) == expected_pb_value + + +def test_timestamp_range_to_pb_start_only(): + # Makes sure already milliseconds granularity + start_micros = 30871000 + start_millis = start_micros // 1000 * 1000 + assert start_millis == 30871000 + pb_kwargs = {} + pb_kwargs["start_timestamp_micros"] = start_millis + _timestamp_range_to_pb_helper(pb_kwargs, start=start_micros, end=None) + + +def test_timestamp_range_to_dict_start_only(): + from google.cloud.bigtable.row_filters import TimestampRange + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + import datetime + + row_filter = TimestampRange(start=datetime.datetime(2019, 1, 1)) + expected_dict = {"start_timestamp_micros": 1546300800000000} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.TimestampRange(**expected_dict) == expected_pb_value + + +def test_timestamp_range_to_pb_end_only(): + # Makes sure already milliseconds granularity + end_micros = 12939371000 + end_millis = end_micros // 1000 * 1000 + assert end_millis == 12939371000 + pb_kwargs = {} + pb_kwargs["end_timestamp_micros"] = end_millis + _timestamp_range_to_pb_helper(pb_kwargs, start=None, end=end_micros) + + +def test_timestamp_range_to_dict_end_only(): + from google.cloud.bigtable.row_filters import TimestampRange + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + import datetime + + row_filter = TimestampRange(end=datetime.datetime(2019, 1, 2)) + expected_dict = {"end_timestamp_micros": 1546387200000000} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.TimestampRange(**expected_dict) == expected_pb_value + + +def test_timestamp_range_filter___eq__type_differ(): + from google.cloud.bigtable.row_filters import TimestampRangeFilter + + range_ = object() + row_filter1 = TimestampRangeFilter(range_) + row_filter2 = object() + assert not (row_filter1 == row_filter2) + + +def test_timestamp_range_filter___eq__same_value(): + from google.cloud.bigtable.row_filters import TimestampRangeFilter + + range_ = object() + row_filter1 = TimestampRangeFilter(range_) + row_filter2 = TimestampRangeFilter(range_) + assert row_filter1 == row_filter2 + + +def test_timestamp_range_filter___ne__(): + from google.cloud.bigtable.row_filters import TimestampRangeFilter + + range_ = object() + other_range_ = object() + row_filter1 = TimestampRangeFilter(range_) + row_filter2 = TimestampRangeFilter(other_range_) + assert row_filter1 != row_filter2 + + +def test_timestamp_range_filter_to_pb(): + from google.cloud.bigtable.row_filters import TimestampRangeFilter + + row_filter = TimestampRangeFilter() + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(timestamp_range_filter=_TimestampRangePB()) + assert pb_val == expected_pb + + +def test_timestamp_range_filter_to_dict(): + from google.cloud.bigtable.row_filters import TimestampRangeFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + import datetime + + row_filter = TimestampRangeFilter( + start=datetime.datetime(2019, 1, 1), end=datetime.datetime(2019, 1, 2) + ) + expected_dict = { + "timestamp_range_filter": { + "start_timestamp_micros": 1546300800000000, + "end_timestamp_micros": 1546387200000000, + } + } + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_timestamp_range_filter_empty_to_dict(): + from google.cloud.bigtable.row_filters import TimestampRangeFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + row_filter = TimestampRangeFilter() + expected_dict = {"timestamp_range_filter": {}} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_column_range_filter_constructor_defaults(): + from google.cloud.bigtable.row_filters import ColumnRangeFilter + + column_family_id = object() + row_filter = ColumnRangeFilter(column_family_id) + assert row_filter.column_family_id is column_family_id + assert row_filter.start_column is None + assert row_filter.end_column is None + assert row_filter.inclusive_start + assert row_filter.inclusive_end + + +def test_column_range_filter_constructor_explicit(): + from google.cloud.bigtable.row_filters import ColumnRangeFilter + + column_family_id = object() + start_column = object() + end_column = object() + inclusive_start = object() + inclusive_end = object() + row_filter = ColumnRangeFilter( + column_family_id, + start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end, + ) + assert row_filter.column_family_id is column_family_id + assert row_filter.start_column is start_column + assert row_filter.end_column is end_column + assert row_filter.inclusive_start is inclusive_start + assert row_filter.inclusive_end is inclusive_end + + +def test_column_range_filter_constructor_(): + from google.cloud.bigtable.row_filters import ColumnRangeFilter + + column_family_id = object() + with pytest.raises(ValueError): + ColumnRangeFilter(column_family_id, inclusive_start=True) + + +def test_column_range_filter_constructor_bad_end(): + from google.cloud.bigtable.row_filters import ColumnRangeFilter + + column_family_id = object() + with pytest.raises(ValueError): + ColumnRangeFilter(column_family_id, inclusive_end=True) + + +def test_column_range_filter___eq__(): + from google.cloud.bigtable.row_filters import ColumnRangeFilter + + column_family_id = object() + start_column = object() + end_column = object() + inclusive_start = object() + inclusive_end = object() + row_filter1 = ColumnRangeFilter( + column_family_id, + start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end, + ) + row_filter2 = ColumnRangeFilter( + column_family_id, + start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end, + ) + assert row_filter1 == row_filter2 + + +def test_column_range_filter___eq__type_differ(): + from google.cloud.bigtable.row_filters import ColumnRangeFilter + + column_family_id = object() + row_filter1 = ColumnRangeFilter(column_family_id) + row_filter2 = object() + assert not (row_filter1 == row_filter2) + + +def test_column_range_filter___ne__(): + from google.cloud.bigtable.row_filters import ColumnRangeFilter + + column_family_id = object() + other_column_family_id = object() + start_column = object() + end_column = object() + inclusive_start = object() + inclusive_end = object() + row_filter1 = ColumnRangeFilter( + column_family_id, + start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end, + ) + row_filter2 = ColumnRangeFilter( + other_column_family_id, + start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end, + ) + assert row_filter1 != row_filter2 + + +def test_column_range_filter_to_pb(): + from google.cloud.bigtable.row_filters import ColumnRangeFilter + + column_family_id = "column-family-id" + row_filter = ColumnRangeFilter(column_family_id) + col_range_pb = _ColumnRangePB(family_name=column_family_id) + expected_pb = _RowFilterPB(column_range_filter=col_range_pb) + assert row_filter.to_pb() == expected_pb + + +def test_column_range_filter_to_dict(): + from google.cloud.bigtable.row_filters import ColumnRangeFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + column_family_id = "column-family-id" + row_filter = ColumnRangeFilter(column_family_id) + expected_dict = {"column_range_filter": {"family_name": column_family_id}} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_column_range_filter_to_pb_inclusive_start(): + from google.cloud.bigtable.row_filters import ColumnRangeFilter + + column_family_id = "column-family-id" + column = b"column" + row_filter = ColumnRangeFilter(column_family_id, start_column=column) + col_range_pb = _ColumnRangePB( + family_name=column_family_id, start_qualifier_closed=column + ) + expected_pb = _RowFilterPB(column_range_filter=col_range_pb) + assert row_filter.to_pb() == expected_pb + + +def test_column_range_filter_to_pb_exclusive_start(): + from google.cloud.bigtable.row_filters import ColumnRangeFilter + + column_family_id = "column-family-id" + column = b"column" + row_filter = ColumnRangeFilter( + column_family_id, start_column=column, inclusive_start=False + ) + col_range_pb = _ColumnRangePB( + family_name=column_family_id, start_qualifier_open=column + ) + expected_pb = _RowFilterPB(column_range_filter=col_range_pb) + assert row_filter.to_pb() == expected_pb + + +def test_column_range_filter_to_pb_inclusive_end(): + from google.cloud.bigtable.row_filters import ColumnRangeFilter + + column_family_id = "column-family-id" + column = b"column" + row_filter = ColumnRangeFilter(column_family_id, end_column=column) + col_range_pb = _ColumnRangePB( + family_name=column_family_id, end_qualifier_closed=column + ) + expected_pb = _RowFilterPB(column_range_filter=col_range_pb) + assert row_filter.to_pb() == expected_pb + + +def test_column_range_filter_to_pb_exclusive_end(): + from google.cloud.bigtable.row_filters import ColumnRangeFilter + + column_family_id = "column-family-id" + column = b"column" + row_filter = ColumnRangeFilter( + column_family_id, end_column=column, inclusive_end=False + ) + col_range_pb = _ColumnRangePB( + family_name=column_family_id, end_qualifier_open=column + ) + expected_pb = _RowFilterPB(column_range_filter=col_range_pb) + assert row_filter.to_pb() == expected_pb + + +def test_value_regex_filter_to_pb_w_bytes(): + from google.cloud.bigtable.row_filters import ValueRegexFilter + + value = regex = b"value-regex" + row_filter = ValueRegexFilter(value) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(value_regex_filter=regex) + assert pb_val == expected_pb + + +def test_value_regex_filter_to_dict_w_bytes(): + from google.cloud.bigtable.row_filters import ValueRegexFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + value = regex = b"value-regex" + row_filter = ValueRegexFilter(value) + expected_dict = {"value_regex_filter": regex} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_value_regex_filter_to_pb_w_str(): + from google.cloud.bigtable.row_filters import ValueRegexFilter + + value = "value-regex" + regex = value.encode("ascii") + row_filter = ValueRegexFilter(value) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(value_regex_filter=regex) + assert pb_val == expected_pb + + +def test_value_regex_filter_to_dict_w_str(): + from google.cloud.bigtable.row_filters import ValueRegexFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + value = "value-regex" + regex = value.encode("ascii") + row_filter = ValueRegexFilter(value) + expected_dict = {"value_regex_filter": regex} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_exact_value_filter_to_pb_w_bytes(): + from google.cloud.bigtable.row_filters import ExactValueFilter + + value = regex = b"value-regex" + row_filter = ExactValueFilter(value) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(value_regex_filter=regex) + assert pb_val == expected_pb + + +def test_exact_value_filter_to_dict_w_bytes(): + from google.cloud.bigtable.row_filters import ExactValueFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + value = regex = b"value-regex" + row_filter = ExactValueFilter(value) + expected_dict = {"value_regex_filter": regex} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_exact_value_filter_to_pb_w_str(): + from google.cloud.bigtable.row_filters import ExactValueFilter + + value = "value-regex" + regex = value.encode("ascii") + row_filter = ExactValueFilter(value) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(value_regex_filter=regex) + assert pb_val == expected_pb + + +def test_exact_value_filter_to_dict_w_str(): + from google.cloud.bigtable.row_filters import ExactValueFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + value = "value-regex" + regex = value.encode("ascii") + row_filter = ExactValueFilter(value) + expected_dict = {"value_regex_filter": regex} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_exact_value_filter_to_pb_w_int(): + import struct + from google.cloud.bigtable.row_filters import ExactValueFilter + + value = 1 + regex = struct.Struct(">q").pack(value) + row_filter = ExactValueFilter(value) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(value_regex_filter=regex) + assert pb_val == expected_pb + + +def test_exact_value_filter_to_dict_w_int(): + import struct + from google.cloud.bigtable.row_filters import ExactValueFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + value = 1 + regex = struct.Struct(">q").pack(value) + row_filter = ExactValueFilter(value) + expected_dict = {"value_regex_filter": regex} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_value_range_filter_constructor_defaults(): + from google.cloud.bigtable.row_filters import ValueRangeFilter + + row_filter = ValueRangeFilter() + + assert row_filter.start_value is None + assert row_filter.end_value is None + assert row_filter.inclusive_start + assert row_filter.inclusive_end + + +def test_value_range_filter_constructor_explicit(): + from google.cloud.bigtable.row_filters import ValueRangeFilter + + start_value = object() + end_value = object() + inclusive_start = object() + inclusive_end = object() + + row_filter = ValueRangeFilter( + start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end, + ) + + assert row_filter.start_value is start_value + assert row_filter.end_value is end_value + assert row_filter.inclusive_start is inclusive_start + assert row_filter.inclusive_end is inclusive_end + + +def test_value_range_filter_constructor_w_int_values(): + from google.cloud.bigtable.row_filters import ValueRangeFilter + import struct + + start_value = 1 + end_value = 10 + + row_filter = ValueRangeFilter(start_value=start_value, end_value=end_value) + + expected_start_value = struct.Struct(">q").pack(start_value) + expected_end_value = struct.Struct(">q").pack(end_value) + + assert row_filter.start_value == expected_start_value + assert row_filter.end_value == expected_end_value + assert row_filter.inclusive_start + assert row_filter.inclusive_end + + +def test_value_range_filter_constructor_bad_start(): + from google.cloud.bigtable.row_filters import ValueRangeFilter + + with pytest.raises(ValueError): + ValueRangeFilter(inclusive_start=True) + + +def test_value_range_filter_constructor_bad_end(): + from google.cloud.bigtable.row_filters import ValueRangeFilter + + with pytest.raises(ValueError): + ValueRangeFilter(inclusive_end=True) + + +def test_value_range_filter___eq__(): + from google.cloud.bigtable.row_filters import ValueRangeFilter + + start_value = object() + end_value = object() + inclusive_start = object() + inclusive_end = object() + row_filter1 = ValueRangeFilter( + start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end, + ) + row_filter2 = ValueRangeFilter( + start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end, + ) + assert row_filter1 == row_filter2 + + +def test_value_range_filter___eq__type_differ(): + from google.cloud.bigtable.row_filters import ValueRangeFilter + + row_filter1 = ValueRangeFilter() + row_filter2 = object() + assert not (row_filter1 == row_filter2) + + +def test_value_range_filter___ne__(): + from google.cloud.bigtable.row_filters import ValueRangeFilter + + start_value = object() + other_start_value = object() + end_value = object() + inclusive_start = object() + inclusive_end = object() + row_filter1 = ValueRangeFilter( + start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end, + ) + row_filter2 = ValueRangeFilter( + start_value=other_start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end, + ) + assert row_filter1 != row_filter2 + + +def test_value_range_filter_to_pb(): + from google.cloud.bigtable.row_filters import ValueRangeFilter + + row_filter = ValueRangeFilter() + expected_pb = _RowFilterPB(value_range_filter=_ValueRangePB()) + assert row_filter.to_pb() == expected_pb + + +def test_value_range_filter_to_dict(): + from google.cloud.bigtable.row_filters import ValueRangeFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + row_filter = ValueRangeFilter() + expected_dict = {"value_range_filter": {}} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_value_range_filter_to_pb_inclusive_start(): + from google.cloud.bigtable.row_filters import ValueRangeFilter + + value = b"some-value" + row_filter = ValueRangeFilter(start_value=value) + val_range_pb = _ValueRangePB(start_value_closed=value) + expected_pb = _RowFilterPB(value_range_filter=val_range_pb) + assert row_filter.to_pb() == expected_pb + + +def test_value_range_filter_to_pb_exclusive_start(): + from google.cloud.bigtable.row_filters import ValueRangeFilter + + value = b"some-value" + row_filter = ValueRangeFilter(start_value=value, inclusive_start=False) + val_range_pb = _ValueRangePB(start_value_open=value) + expected_pb = _RowFilterPB(value_range_filter=val_range_pb) + assert row_filter.to_pb() == expected_pb + + +def test_value_range_filter_to_pb_inclusive_end(): + from google.cloud.bigtable.row_filters import ValueRangeFilter + + value = b"some-value" + row_filter = ValueRangeFilter(end_value=value) + val_range_pb = _ValueRangePB(end_value_closed=value) + expected_pb = _RowFilterPB(value_range_filter=val_range_pb) + assert row_filter.to_pb() == expected_pb + + +def test_value_range_filter_to_pb_exclusive_end(): + from google.cloud.bigtable.row_filters import ValueRangeFilter + + value = b"some-value" + row_filter = ValueRangeFilter(end_value=value, inclusive_end=False) + val_range_pb = _ValueRangePB(end_value_open=value) + expected_pb = _RowFilterPB(value_range_filter=val_range_pb) + assert row_filter.to_pb() == expected_pb + + +def test_cell_count_constructor(): + from google.cloud.bigtable.row_filters import _CellCountFilter + + num_cells = object() + row_filter = _CellCountFilter(num_cells) + assert row_filter.num_cells is num_cells + + +def test_cell_count___eq__type_differ(): + from google.cloud.bigtable.row_filters import _CellCountFilter + + num_cells = object() + row_filter1 = _CellCountFilter(num_cells) + row_filter2 = object() + assert not (row_filter1 == row_filter2) + + +def test_cell_count___eq__same_value(): + from google.cloud.bigtable.row_filters import _CellCountFilter + + num_cells = object() + row_filter1 = _CellCountFilter(num_cells) + row_filter2 = _CellCountFilter(num_cells) + assert row_filter1 == row_filter2 + + +def test_cell_count___ne__same_value(): + from google.cloud.bigtable.row_filters import _CellCountFilter + + num_cells = object() + row_filter1 = _CellCountFilter(num_cells) + row_filter2 = _CellCountFilter(num_cells) + assert not (row_filter1 != row_filter2) + + +def test_cells_row_offset_filter_to_pb(): + from google.cloud.bigtable.row_filters import CellsRowOffsetFilter + + num_cells = 76 + row_filter = CellsRowOffsetFilter(num_cells) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(cells_per_row_offset_filter=num_cells) + assert pb_val == expected_pb + + +def test_cells_row_offset_filter_to_dict(): + from google.cloud.bigtable.row_filters import CellsRowOffsetFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + num_cells = 76 + row_filter = CellsRowOffsetFilter(num_cells) + expected_dict = {"cells_per_row_offset_filter": num_cells} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_cells_row_limit_filter_to_pb(): + from google.cloud.bigtable.row_filters import CellsRowLimitFilter + + num_cells = 189 + row_filter = CellsRowLimitFilter(num_cells) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(cells_per_row_limit_filter=num_cells) + assert pb_val == expected_pb + + +def test_cells_row_limit_filter_to_dict(): + from google.cloud.bigtable.row_filters import CellsRowLimitFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + num_cells = 189 + row_filter = CellsRowLimitFilter(num_cells) + expected_dict = {"cells_per_row_limit_filter": num_cells} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_cells_column_limit_filter_to_pb(): + from google.cloud.bigtable.row_filters import CellsColumnLimitFilter + + num_cells = 10 + row_filter = CellsColumnLimitFilter(num_cells) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(cells_per_column_limit_filter=num_cells) + assert pb_val == expected_pb + + +def test_cells_column_limit_filter_to_dict(): + from google.cloud.bigtable.row_filters import CellsColumnLimitFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + num_cells = 10 + row_filter = CellsColumnLimitFilter(num_cells) + expected_dict = {"cells_per_column_limit_filter": num_cells} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_strip_value_transformer_filter_to_pb(): + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + + flag = True + row_filter = StripValueTransformerFilter(flag) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(strip_value_transformer=flag) + assert pb_val == expected_pb + + +def test_strip_value_transformer_filter_to_dict(): + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + flag = True + row_filter = StripValueTransformerFilter(flag) + expected_dict = {"strip_value_transformer": flag} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_apply_label_filter_constructor(): + from google.cloud.bigtable.row_filters import ApplyLabelFilter + + label = object() + row_filter = ApplyLabelFilter(label) + assert row_filter.label is label + + +def test_apply_label_filter___eq__type_differ(): + from google.cloud.bigtable.row_filters import ApplyLabelFilter + + label = object() + row_filter1 = ApplyLabelFilter(label) + row_filter2 = object() + assert not (row_filter1 == row_filter2) + + +def test_apply_label_filter___eq__same_value(): + from google.cloud.bigtable.row_filters import ApplyLabelFilter + + label = object() + row_filter1 = ApplyLabelFilter(label) + row_filter2 = ApplyLabelFilter(label) + assert row_filter1 == row_filter2 + + +def test_apply_label_filter___ne__(): + from google.cloud.bigtable.row_filters import ApplyLabelFilter + + label = object() + other_label = object() + row_filter1 = ApplyLabelFilter(label) + row_filter2 = ApplyLabelFilter(other_label) + assert row_filter1 != row_filter2 + + +def test_apply_label_filter_to_pb(): + from google.cloud.bigtable.row_filters import ApplyLabelFilter + + label = "label" + row_filter = ApplyLabelFilter(label) + pb_val = row_filter.to_pb() + expected_pb = _RowFilterPB(apply_label_transformer=label) + assert pb_val == expected_pb + + +def test_apply_label_filter_to_dict(): + from google.cloud.bigtable.row_filters import ApplyLabelFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + label = "label" + row_filter = ApplyLabelFilter(label) + expected_dict = {"apply_label_transformer": label} + assert row_filter.to_dict() == expected_dict + expected_pb_value = row_filter.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_filter_combination_constructor_defaults(): + from google.cloud.bigtable.row_filters import _FilterCombination + + row_filter = _FilterCombination() + assert row_filter.filters == [] + + +def test_filter_combination_constructor_explicit(): + from google.cloud.bigtable.row_filters import _FilterCombination + + filters = object() + row_filter = _FilterCombination(filters=filters) + assert row_filter.filters is filters + + +def test_filter_combination___eq__(): + from google.cloud.bigtable.row_filters import _FilterCombination + + filters = object() + row_filter1 = _FilterCombination(filters=filters) + row_filter2 = _FilterCombination(filters=filters) + assert row_filter1 == row_filter2 + + +def test_filter_combination___eq__type_differ(): + from google.cloud.bigtable.row_filters import _FilterCombination + + filters = object() + row_filter1 = _FilterCombination(filters=filters) + row_filter2 = object() + assert not (row_filter1 == row_filter2) + + +def test_filter_combination___ne__(): + from google.cloud.bigtable.row_filters import _FilterCombination + + filters = object() + other_filters = object() + row_filter1 = _FilterCombination(filters=filters) + row_filter2 = _FilterCombination(filters=other_filters) + assert row_filter1 != row_filter2 + + +def test_row_filter_chain_to_pb(): + from google.cloud.bigtable.row_filters import RowFilterChain + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = RowFilterChain(filters=[row_filter1, row_filter2]) + filter_pb = row_filter3.to_pb() + + expected_pb = _RowFilterPB( + chain=_RowFilterChainPB(filters=[row_filter1_pb, row_filter2_pb]) + ) + assert filter_pb == expected_pb + + +def test_row_filter_chain_to_dict(): + from google.cloud.bigtable.row_filters import RowFilterChain + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_dict = row_filter1.to_dict() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_dict = row_filter2.to_dict() + + row_filter3 = RowFilterChain(filters=[row_filter1, row_filter2]) + filter_dict = row_filter3.to_dict() + + expected_dict = {"chain": {"filters": [row_filter1_dict, row_filter2_dict]}} + assert filter_dict == expected_dict + expected_pb_value = row_filter3.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_row_filter_chain_to_pb_nested(): + from google.cloud.bigtable.row_filters import CellsRowLimitFilter + from google.cloud.bigtable.row_filters import RowFilterChain + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) + + row_filter3 = RowFilterChain(filters=[row_filter1, row_filter2]) + row_filter3_pb = row_filter3.to_pb() + + row_filter4 = CellsRowLimitFilter(11) + row_filter4_pb = row_filter4.to_pb() + + row_filter5 = RowFilterChain(filters=[row_filter3, row_filter4]) + filter_pb = row_filter5.to_pb() + + expected_pb = _RowFilterPB( + chain=_RowFilterChainPB(filters=[row_filter3_pb, row_filter4_pb]) + ) + assert filter_pb == expected_pb + + +def test_row_filter_chain_to_dict_nested(): + from google.cloud.bigtable.row_filters import CellsRowLimitFilter + from google.cloud.bigtable.row_filters import RowFilterChain + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + row_filter1 = StripValueTransformerFilter(True) + + row_filter2 = RowSampleFilter(0.25) + + row_filter3 = RowFilterChain(filters=[row_filter1, row_filter2]) + row_filter3_dict = row_filter3.to_dict() + + row_filter4 = CellsRowLimitFilter(11) + row_filter4_dict = row_filter4.to_dict() + + row_filter5 = RowFilterChain(filters=[row_filter3, row_filter4]) + filter_dict = row_filter5.to_dict() + + expected_dict = {"chain": {"filters": [row_filter3_dict, row_filter4_dict]}} + assert filter_dict == expected_dict + expected_pb_value = row_filter5.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_row_filter_union_to_pb(): + from google.cloud.bigtable.row_filters import RowFilterUnion + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = RowFilterUnion(filters=[row_filter1, row_filter2]) + filter_pb = row_filter3.to_pb() + + expected_pb = _RowFilterPB( + interleave=_RowFilterInterleavePB(filters=[row_filter1_pb, row_filter2_pb]) + ) + assert filter_pb == expected_pb + + +def test_row_filter_union_to_dict(): + from google.cloud.bigtable.row_filters import RowFilterUnion + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_dict = row_filter1.to_dict() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_dict = row_filter2.to_dict() + + row_filter3 = RowFilterUnion(filters=[row_filter1, row_filter2]) + filter_dict = row_filter3.to_dict() + + expected_dict = {"interleave": {"filters": [row_filter1_dict, row_filter2_dict]}} + assert filter_dict == expected_dict + expected_pb_value = row_filter3.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_row_filter_union_to_pb_nested(): + from google.cloud.bigtable.row_filters import CellsRowLimitFilter + from google.cloud.bigtable.row_filters import RowFilterUnion + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) + + row_filter3 = RowFilterUnion(filters=[row_filter1, row_filter2]) + row_filter3_pb = row_filter3.to_pb() + + row_filter4 = CellsRowLimitFilter(11) + row_filter4_pb = row_filter4.to_pb() + + row_filter5 = RowFilterUnion(filters=[row_filter3, row_filter4]) + filter_pb = row_filter5.to_pb() + + expected_pb = _RowFilterPB( + interleave=_RowFilterInterleavePB(filters=[row_filter3_pb, row_filter4_pb]) + ) + assert filter_pb == expected_pb + + +def test_row_filter_union_to_dict_nested(): + from google.cloud.bigtable.row_filters import CellsRowLimitFilter + from google.cloud.bigtable.row_filters import RowFilterUnion + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + row_filter1 = StripValueTransformerFilter(True) + + row_filter2 = RowSampleFilter(0.25) + + row_filter3 = RowFilterUnion(filters=[row_filter1, row_filter2]) + row_filter3_dict = row_filter3.to_dict() + + row_filter4 = CellsRowLimitFilter(11) + row_filter4_dict = row_filter4.to_dict() + + row_filter5 = RowFilterUnion(filters=[row_filter3, row_filter4]) + filter_dict = row_filter5.to_dict() + + expected_dict = {"interleave": {"filters": [row_filter3_dict, row_filter4_dict]}} + assert filter_dict == expected_dict + expected_pb_value = row_filter5.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_conditional_row_filter_constructor(): + from google.cloud.bigtable.row_filters import ConditionalRowFilter + + base_filter = object() + true_filter = object() + false_filter = object() + cond_filter = ConditionalRowFilter( + base_filter, true_filter=true_filter, false_filter=false_filter + ) + assert cond_filter.base_filter is base_filter + assert cond_filter.true_filter is true_filter + assert cond_filter.false_filter is false_filter + + +def test_conditional_row_filter___eq__(): + from google.cloud.bigtable.row_filters import ConditionalRowFilter + + base_filter = object() + true_filter = object() + false_filter = object() + cond_filter1 = ConditionalRowFilter( + base_filter, true_filter=true_filter, false_filter=false_filter + ) + cond_filter2 = ConditionalRowFilter( + base_filter, true_filter=true_filter, false_filter=false_filter + ) + assert cond_filter1 == cond_filter2 + + +def test_conditional_row_filter___eq__type_differ(): + from google.cloud.bigtable.row_filters import ConditionalRowFilter + + base_filter = object() + true_filter = object() + false_filter = object() + cond_filter1 = ConditionalRowFilter( + base_filter, true_filter=true_filter, false_filter=false_filter + ) + cond_filter2 = object() + assert not (cond_filter1 == cond_filter2) + + +def test_conditional_row_filter___ne__(): + from google.cloud.bigtable.row_filters import ConditionalRowFilter + + base_filter = object() + other_base_filter = object() + true_filter = object() + false_filter = object() + cond_filter1 = ConditionalRowFilter( + base_filter, true_filter=true_filter, false_filter=false_filter + ) + cond_filter2 = ConditionalRowFilter( + other_base_filter, true_filter=true_filter, false_filter=false_filter + ) + assert cond_filter1 != cond_filter2 + + +def test_conditional_row_filter_to_pb(): + from google.cloud.bigtable.row_filters import ConditionalRowFilter + from google.cloud.bigtable.row_filters import CellsRowOffsetFilter + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = CellsRowOffsetFilter(11) + row_filter3_pb = row_filter3.to_pb() + + row_filter4 = ConditionalRowFilter( + row_filter1, true_filter=row_filter2, false_filter=row_filter3 + ) + filter_pb = row_filter4.to_pb() + + expected_pb = _RowFilterPB( + condition=_RowFilterConditionPB( + predicate_filter=row_filter1_pb, + true_filter=row_filter2_pb, + false_filter=row_filter3_pb, + ) + ) + assert filter_pb == expected_pb + + +def test_conditional_row_filter_to_dict(): + from google.cloud.bigtable.row_filters import ConditionalRowFilter + from google.cloud.bigtable.row_filters import CellsRowOffsetFilter + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_dict = row_filter1.to_dict() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_dict = row_filter2.to_dict() + + row_filter3 = CellsRowOffsetFilter(11) + row_filter3_dict = row_filter3.to_dict() + + row_filter4 = ConditionalRowFilter( + row_filter1, true_filter=row_filter2, false_filter=row_filter3 + ) + filter_dict = row_filter4.to_dict() + + expected_dict = { + "condition": { + "predicate_filter": row_filter1_dict, + "true_filter": row_filter2_dict, + "false_filter": row_filter3_dict, + } + } + assert filter_dict == expected_dict + expected_pb_value = row_filter4.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_conditional_row_filter_to_pb_true_only(): + from google.cloud.bigtable.row_filters import ConditionalRowFilter + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = ConditionalRowFilter(row_filter1, true_filter=row_filter2) + filter_pb = row_filter3.to_pb() + + expected_pb = _RowFilterPB( + condition=_RowFilterConditionPB( + predicate_filter=row_filter1_pb, true_filter=row_filter2_pb + ) + ) + assert filter_pb == expected_pb + + +def test_conditional_row_filter_to_dict_true_only(): + from google.cloud.bigtable.row_filters import ConditionalRowFilter + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_dict = row_filter1.to_dict() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_dict = row_filter2.to_dict() + + row_filter3 = ConditionalRowFilter(row_filter1, true_filter=row_filter2) + filter_dict = row_filter3.to_dict() + + expected_dict = { + "condition": { + "predicate_filter": row_filter1_dict, + "true_filter": row_filter2_dict, + } + } + assert filter_dict == expected_dict + expected_pb_value = row_filter3.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def test_conditional_row_filter_to_pb_false_only(): + from google.cloud.bigtable.row_filters import ConditionalRowFilter + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = ConditionalRowFilter(row_filter1, false_filter=row_filter2) + filter_pb = row_filter3.to_pb() + + expected_pb = _RowFilterPB( + condition=_RowFilterConditionPB( + predicate_filter=row_filter1_pb, false_filter=row_filter2_pb + ) + ) + assert filter_pb == expected_pb + + +def test_conditional_row_filter_to_dict_false_only(): + from google.cloud.bigtable.row_filters import ConditionalRowFilter + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_dict = row_filter1.to_dict() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_dict = row_filter2.to_dict() + + row_filter3 = ConditionalRowFilter(row_filter1, false_filter=row_filter2) + filter_dict = row_filter3.to_dict() + + expected_dict = { + "condition": { + "predicate_filter": row_filter1_dict, + "false_filter": row_filter2_dict, + } + } + assert filter_dict == expected_dict + expected_pb_value = row_filter3.to_pb() + assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value + + +def _ColumnRangePB(*args, **kw): + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + return data_v2_pb2.ColumnRange(*args, **kw) + + +def _RowFilterPB(*args, **kw): + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + return data_v2_pb2.RowFilter(*args, **kw) + + +def _RowFilterChainPB(*args, **kw): + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + return data_v2_pb2.RowFilter.Chain(*args, **kw) + + +def _RowFilterConditionPB(*args, **kw): + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + return data_v2_pb2.RowFilter.Condition(*args, **kw) + + +def _RowFilterInterleavePB(*args, **kw): + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + return data_v2_pb2.RowFilter.Interleave(*args, **kw) + + +def _TimestampRangePB(*args, **kw): + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + return data_v2_pb2.TimestampRange(*args, **kw) + + +def _ValueRangePB(*args, **kw): + from google.cloud.bigtable_v2.types import data as data_v2_pb2 + + return data_v2_pb2.ValueRange(*args, **kw) From 906ed3316a7429046b7c8b1b1dd8a310e1e93428 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 16 Mar 2023 09:24:21 -0700 Subject: [PATCH 03/16] ignore tests for mypy --- noxfile.py | 1 - owlbot.py | 1 - 2 files changed, 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index 94982fcfd..ed69bf85e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -133,7 +133,6 @@ def mypy(session): session.run( "mypy", "google/cloud/bigtable", - "tests/", "--check-untyped-defs", "--warn-unreachable", "--disallow-any-generics", diff --git a/owlbot.py b/owlbot.py index f56a0bd9e..b542b3246 100644 --- a/owlbot.py +++ b/owlbot.py @@ -171,7 +171,6 @@ def mypy(session): session.run( "mypy", "google/cloud/bigtable", - "tests/", "--check-untyped-defs", "--warn-unreachable", "--disallow-any-generics", From fcabba5767316b67131620e7bca871cb67314015 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 16 Mar 2023 13:19:22 -0700 Subject: [PATCH 04/16] added str and repr for row filters --- google/cloud/bigtable/row_filters.py | 54 ++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/google/cloud/bigtable/row_filters.py b/google/cloud/bigtable/row_filters.py index 79728822d..e6747950c 100644 --- a/google/cloud/bigtable/row_filters.py +++ b/google/cloud/bigtable/row_filters.py @@ -54,6 +54,9 @@ def to_dict(self) -> dict[str, Any]: # unimplemented on base class raise NotImplementedError + def __repr__(self) -> str: + return f"{self.__class__.__name__}()" + class _BoolFilter(RowFilter): """Row filter that uses a boolean flag. @@ -73,6 +76,9 @@ def __eq__(self, other): def __ne__(self, other): return not self == other + def __repr__(self) -> str: + return f"{self.__class__.__name__}(flag={self.flag})" + class SinkFilter(_BoolFilter): """Advanced row filter to skip parent filters. @@ -142,6 +148,9 @@ def __eq__(self, other): def __ne__(self, other): return not self == other + def __repr__(self) -> str: + return f"{self.__class__.__name__}(regex={self.regex!r})" + class RowKeyRegexFilter(_RegexFilter): """Row filter for a row key regular expression. @@ -194,6 +203,8 @@ def to_dict(self) -> dict[str, Any]: """Converts the row filter to a dict representation.""" return {"row_sample_filter": self.sample} + def __repr__(self) -> str: + return f"{self.__class__.__name__}(sample={self.sample})" class FamilyNameRegexFilter(_RegexFilter): """Row filter for a family name regular expression. @@ -287,6 +298,8 @@ def to_dict(self) -> dict[str, int]: timestamp_range_kwargs["end_timestamp_micros"] = end_time return timestamp_range_kwargs + def __repr__(self) -> str: + return f"{self.__class__.__name__}(start={self.start}, end={self.end})" class TimestampRangeFilter(RowFilter): """Row filter that limits cells to a range of time. @@ -321,6 +334,9 @@ def to_dict(self) -> dict[str, Any]: """Converts the row filter to a dict representation.""" return {"timestamp_range_filter": self.range_.to_dict()} + def __repr__(self) -> str: + return f"{self.__class__.__name__}(start={self.range_.start}, end={self.range_.end})" + class ColumnRangeFilter(RowFilter): """A row filter to restrict to a range of columns. @@ -434,6 +450,8 @@ def to_dict(self) -> dict[str, Any]: """Converts the row filter to a dict representation.""" return {"column_range_filter": self.range_to_dict()} + def __repr__(self) -> str: + return f"{self.__class__.__name__}(column_family_id={self.column_family_id}, start_column={self.start_column}, end_column={self.end_column}, inclusive_start={self.inclusive_start}, inclusive_end={self.inclusive_end})" class ValueRegexFilter(_RegexFilter): """Row filter for a value regular expression. @@ -476,6 +494,9 @@ def __init__(self, value: bytes | str | int): value = _PACK_I64(value) super(ExactValueFilter, self).__init__(value) + def __repr__(self) -> str: + return f"{self.__class__.__name__}(value={self.regex})" + class ValueRangeFilter(RowFilter): """A range of values to restrict to in a row filter. @@ -584,6 +605,9 @@ def to_dict(self) -> dict[str, Any]: """Converts the row filter to a dict representation.""" return {"value_range_filter": self.range_to_dict()} + def __repr__(self) -> str: + return f"{self.__class__.__name__}(start_value={self.start_value}, end_value={self.end_value}, inclusive_start={self.inclusive_start}, inclusive_end={self.inclusive_end})" + class _CellCountFilter(RowFilter): """Row filter that uses an integer count of cells. @@ -606,6 +630,9 @@ def __eq__(self, other): def __ne__(self, other): return not self == other + def __repr__(self) -> str: + return f"{self.__class__.__name__}(num_cells={self.num_cells})" + class CellsRowOffsetFilter(_CellCountFilter): """Row filter to skip cells in a row. @@ -693,6 +720,8 @@ def to_dict(self) -> dict[str, str]: """Converts the row filter to a dict representation.""" return {"apply_label_transformer": self.label} + def __repr__(self) -> str: + return f"{self.__class__.__name__}(label={self.label})" class _FilterCombination(RowFilter): """Chain of row filters. @@ -718,6 +747,16 @@ def __eq__(self, other): def __ne__(self, other): return not self == other + def __repr__(self) -> str: + return f"{self.__class__.__name__}(filters={[repr(f) for f in self.filters]})" + + def __str__(self) -> str: + output = [f"{self.__class__.__name__}(["] + for filter_ in self.filters: + filter_lines = f"{filter_},".splitlines() + output.extend([f" {line}" for line in filter_lines]) + output.append("])") + return "\n".join(output) class RowFilterChain(_FilterCombination): """Chain of row filters. @@ -853,3 +892,18 @@ def condition_to_dict(self) -> dict[str, Any]: def to_dict(self) -> dict[str, Any]: """Converts the row filter to a dict representation.""" return {"condition": self.condition_to_dict()} + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(base_filter={self.base_filter!r}, true_filter={self.true_filter!r}, false_filter={self.false_filter!r})" + + def __str__(self) -> str: + output = [f"{self.__class__.__name__}("] + for filter_type in ("base_filter", "true_filter", "false_filter"): + filter_ = getattr(self, filter_type) + if filter_ is None: + continue + # add the new filter set, adding indentations for readability + output.append(f" {filter_type}={filter_!r},") + output.append(")") + return "\n".join(output) + From 5367e8e4c0ae66a5dcd9a4e6f2641cc61a77e824 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 16 Mar 2023 13:24:27 -0700 Subject: [PATCH 05/16] renamed base filter to predicate filter --- google/cloud/bigtable/row_filters.py | 32 ++++++++++++++++------------ tests/unit/test_row_filters.py | 24 ++++++++++----------- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/google/cloud/bigtable/row_filters.py b/google/cloud/bigtable/row_filters.py index e6747950c..eefeef81f 100644 --- a/google/cloud/bigtable/row_filters.py +++ b/google/cloud/bigtable/row_filters.py @@ -206,6 +206,7 @@ def to_dict(self) -> dict[str, Any]: def __repr__(self) -> str: return f"{self.__class__.__name__}(sample={self.sample})" + class FamilyNameRegexFilter(_RegexFilter): """Row filter for a family name regular expression. @@ -301,6 +302,7 @@ def to_dict(self) -> dict[str, int]: def __repr__(self) -> str: return f"{self.__class__.__name__}(start={self.start}, end={self.end})" + class TimestampRangeFilter(RowFilter): """Row filter that limits cells to a range of time. @@ -453,6 +455,7 @@ def to_dict(self) -> dict[str, Any]: def __repr__(self) -> str: return f"{self.__class__.__name__}(column_family_id={self.column_family_id}, start_column={self.start_column}, end_column={self.end_column}, inclusive_start={self.inclusive_start}, inclusive_end={self.inclusive_end})" + class ValueRegexFilter(_RegexFilter): """Row filter for a value regular expression. @@ -723,6 +726,7 @@ def to_dict(self) -> dict[str, str]: def __repr__(self) -> str: return f"{self.__class__.__name__}(label={self.label})" + class _FilterCombination(RowFilter): """Chain of row filters. @@ -758,6 +762,7 @@ def __str__(self) -> str: output.append("])") return "\n".join(output) + class RowFilterChain(_FilterCombination): """Chain of row filters. @@ -817,40 +822,40 @@ def to_dict(self) -> dict[str, Any]: class ConditionalRowFilter(RowFilter): """Conditional row filter which exhibits ternary behavior. - Executes one of two filters based on another filter. If the ``base_filter`` + Executes one of two filters based on another filter. If the ``predicate_filter`` returns any cells in the row, then ``true_filter`` is executed. If not, then ``false_filter`` is executed. .. note:: - The ``base_filter`` does not execute atomically with the true and false + The ``predicate_filter`` does not execute atomically with the true and false filters, which may lead to inconsistent or unexpected results. Additionally, executing a :class:`ConditionalRowFilter` has poor performance on the server, especially when ``false_filter`` is set. - :type base_filter: :class:`RowFilter` - :param base_filter: The filter to condition on before executing the + :type predicate_filter: :class:`RowFilter` + :param predicate_filter: The filter to condition on before executing the true/false filters. :type true_filter: :class:`RowFilter` :param true_filter: (Optional) The filter to execute if there are any cells - matching ``base_filter``. If not provided, no results + matching ``predicate_filter``. If not provided, no results will be returned in the true case. :type false_filter: :class:`RowFilter` :param false_filter: (Optional) The filter to execute if there are no cells - matching ``base_filter``. If not provided, no results + matching ``predicate_filter``. If not provided, no results will be returned in the false case. """ def __init__( self, - base_filter: RowFilter, + predicate_filter: RowFilter, true_filter: RowFilter | None = None, false_filter: RowFilter | None = None, ): - self.base_filter = base_filter + self.predicate_filter = predicate_filter self.true_filter = true_filter self.false_filter = false_filter @@ -858,7 +863,7 @@ def __eq__(self, other): if not isinstance(other, self.__class__): return NotImplemented return ( - other.base_filter == self.base_filter + other.predicate_filter == self.predicate_filter and other.true_filter == self.true_filter and other.false_filter == self.false_filter ) @@ -872,7 +877,7 @@ def to_pb(self): :rtype: :class:`.data_v2_pb2.RowFilter` :returns: The converted current object. """ - condition_kwargs = {"predicate_filter": self.base_filter.to_pb()} + condition_kwargs = {"predicate_filter": self.predicate_filter.to_pb()} if self.true_filter is not None: condition_kwargs["true_filter"] = self.true_filter.to_pb() if self.false_filter is not None: @@ -882,7 +887,7 @@ def to_pb(self): def condition_to_dict(self) -> dict[str, Any]: """Converts the condition to a dict representation.""" - condition_kwargs = {"predicate_filter": self.base_filter.to_dict()} + condition_kwargs = {"predicate_filter": self.predicate_filter.to_dict()} if self.true_filter is not None: condition_kwargs["true_filter"] = self.true_filter.to_dict() if self.false_filter is not None: @@ -894,11 +899,11 @@ def to_dict(self) -> dict[str, Any]: return {"condition": self.condition_to_dict()} def __repr__(self) -> str: - return f"{self.__class__.__name__}(base_filter={self.base_filter!r}, true_filter={self.true_filter!r}, false_filter={self.false_filter!r})" + return f"{self.__class__.__name__}(predicate_filter={self.predicate_filter!r}, true_filter={self.true_filter!r}, false_filter={self.false_filter!r})" def __str__(self) -> str: output = [f"{self.__class__.__name__}("] - for filter_type in ("base_filter", "true_filter", "false_filter"): + for filter_type in ("predicate_filter", "true_filter", "false_filter"): filter_ = getattr(self, filter_type) if filter_ is None: continue @@ -906,4 +911,3 @@ def __str__(self) -> str: output.append(f" {filter_type}={filter_!r},") output.append(")") return "\n".join(output) - diff --git a/tests/unit/test_row_filters.py b/tests/unit/test_row_filters.py index 759e84c45..827460737 100644 --- a/tests/unit/test_row_filters.py +++ b/tests/unit/test_row_filters.py @@ -1361,13 +1361,13 @@ def test_row_filter_union_to_dict_nested(): def test_conditional_row_filter_constructor(): from google.cloud.bigtable.row_filters import ConditionalRowFilter - base_filter = object() + predicate_filter = object() true_filter = object() false_filter = object() cond_filter = ConditionalRowFilter( - base_filter, true_filter=true_filter, false_filter=false_filter + predicate_filter, true_filter=true_filter, false_filter=false_filter ) - assert cond_filter.base_filter is base_filter + assert cond_filter.predicate_filter is predicate_filter assert cond_filter.true_filter is true_filter assert cond_filter.false_filter is false_filter @@ -1375,14 +1375,14 @@ def test_conditional_row_filter_constructor(): def test_conditional_row_filter___eq__(): from google.cloud.bigtable.row_filters import ConditionalRowFilter - base_filter = object() + predicate_filter = object() true_filter = object() false_filter = object() cond_filter1 = ConditionalRowFilter( - base_filter, true_filter=true_filter, false_filter=false_filter + predicate_filter, true_filter=true_filter, false_filter=false_filter ) cond_filter2 = ConditionalRowFilter( - base_filter, true_filter=true_filter, false_filter=false_filter + predicate_filter, true_filter=true_filter, false_filter=false_filter ) assert cond_filter1 == cond_filter2 @@ -1390,11 +1390,11 @@ def test_conditional_row_filter___eq__(): def test_conditional_row_filter___eq__type_differ(): from google.cloud.bigtable.row_filters import ConditionalRowFilter - base_filter = object() + predicate_filter = object() true_filter = object() false_filter = object() cond_filter1 = ConditionalRowFilter( - base_filter, true_filter=true_filter, false_filter=false_filter + predicate_filter, true_filter=true_filter, false_filter=false_filter ) cond_filter2 = object() assert not (cond_filter1 == cond_filter2) @@ -1403,15 +1403,15 @@ def test_conditional_row_filter___eq__type_differ(): def test_conditional_row_filter___ne__(): from google.cloud.bigtable.row_filters import ConditionalRowFilter - base_filter = object() - other_base_filter = object() + predicate_filter = object() + other_predicate_filter = object() true_filter = object() false_filter = object() cond_filter1 = ConditionalRowFilter( - base_filter, true_filter=true_filter, false_filter=false_filter + predicate_filter, true_filter=true_filter, false_filter=false_filter ) cond_filter2 = ConditionalRowFilter( - other_base_filter, true_filter=true_filter, false_filter=false_filter + other_predicate_filter, true_filter=true_filter, false_filter=false_filter ) assert cond_filter1 != cond_filter2 From 22d476e01202f3fb73cfe471c75341fe0801271d Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 16 Mar 2023 13:26:32 -0700 Subject: [PATCH 06/16] fixed some print bytes formatting --- google/cloud/bigtable/row_filters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/google/cloud/bigtable/row_filters.py b/google/cloud/bigtable/row_filters.py index eefeef81f..569ff2c08 100644 --- a/google/cloud/bigtable/row_filters.py +++ b/google/cloud/bigtable/row_filters.py @@ -453,7 +453,7 @@ def to_dict(self) -> dict[str, Any]: return {"column_range_filter": self.range_to_dict()} def __repr__(self) -> str: - return f"{self.__class__.__name__}(column_family_id={self.column_family_id}, start_column={self.start_column}, end_column={self.end_column}, inclusive_start={self.inclusive_start}, inclusive_end={self.inclusive_end})" + return f"{self.__class__.__name__}(column_family_id={self.column_family_id}, start_column={self.start_column!r}, end_column={self.end_column!r}, inclusive_start={self.inclusive_start}, inclusive_end={self.inclusive_end})" class ValueRegexFilter(_RegexFilter): @@ -498,7 +498,7 @@ def __init__(self, value: bytes | str | int): super(ExactValueFilter, self).__init__(value) def __repr__(self) -> str: - return f"{self.__class__.__name__}(value={self.regex})" + return f"{self.__class__.__name__}(value={self.regex!r})" class ValueRangeFilter(RowFilter): @@ -609,7 +609,7 @@ def to_dict(self) -> dict[str, Any]: return {"value_range_filter": self.range_to_dict()} def __repr__(self) -> str: - return f"{self.__class__.__name__}(start_value={self.start_value}, end_value={self.end_value}, inclusive_start={self.inclusive_start}, inclusive_end={self.inclusive_end})" + return f"{self.__class__.__name__}(start_value={self.start_value!r}, end_value={self.end_value!r}, inclusive_start={self.inclusive_start}, inclusive_end={self.inclusive_end})" class _CellCountFilter(RowFilter): From 0e4eb5d02c6bd3b284a37164bd8279d1030e723c Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 16 Mar 2023 13:29:49 -0700 Subject: [PATCH 07/16] renamed family and qualifier variables to be consistent with new classes --- google/cloud/bigtable/row_filters.py | 64 +++++++------- tests/unit/test_row_filters.py | 124 ++++++++++++--------------- 2 files changed, 91 insertions(+), 97 deletions(-) diff --git a/google/cloud/bigtable/row_filters.py b/google/cloud/bigtable/row_filters.py index 569ff2c08..41a337bef 100644 --- a/google/cloud/bigtable/row_filters.py +++ b/google/cloud/bigtable/row_filters.py @@ -347,70 +347,72 @@ class ColumnRangeFilter(RowFilter): By default, we include them both, but this can be changed with optional flags. - :type column_family_id: str - :param column_family_id: The column family that contains the columns. Must + :type family_id: str + :param family_id: The column family that contains the columns. Must be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - :type start_column: bytes - :param start_column: The start of the range of columns. If no value is + :type start_qualifier: bytes + :param start_qualifier: The start of the range of columns. If no value is used, the backend applies no upper bound to the values. - :type end_column: bytes - :param end_column: The end of the range of columns. If no value is used, + :type end_qualifier: bytes + :param end_qualifier: The end of the range of columns. If no value is used, the backend applies no upper bound to the values. :type inclusive_start: bool :param inclusive_start: Boolean indicating if the start column should be included in the range (or excluded). Defaults - to :data:`True` if ``start_column`` is passed and + to :data:`True` if ``start_qualifier`` is passed and no ``inclusive_start`` was given. :type inclusive_end: bool :param inclusive_end: Boolean indicating if the end column should be included in the range (or excluded). Defaults - to :data:`True` if ``end_column`` is passed and + to :data:`True` if ``end_qualifier`` is passed and no ``inclusive_end`` was given. :raises: :class:`ValueError ` if ``inclusive_start`` - is set but no ``start_column`` is given or if ``inclusive_end`` - is set but no ``end_column`` is given + is set but no ``start_qualifier`` is given or if ``inclusive_end`` + is set but no ``end_qualifier`` is given """ def __init__( self, - column_family_id: str, - start_column: bytes | None = None, - end_column: bytes | None = None, + family_id: str, + start_qualifier: bytes | None = None, + end_qualifier: bytes | None = None, inclusive_start: bool | None = None, inclusive_end: bool | None = None, ): if inclusive_start is None: inclusive_start = True - elif start_column is None: + elif start_qualifier is None: raise ValueError( - "inclusive_start was specified but no start_column was given." + "inclusive_start was specified but no start_qualifier was given." ) if inclusive_end is None: inclusive_end = True - elif end_column is None: - raise ValueError("inclusive_end was specified but no end_column was given.") + elif end_qualifier is None: + raise ValueError( + "inclusive_end was specified but no end_qualifier was given." + ) - self.column_family_id = column_family_id + self.family_id = family_id - self.start_column = start_column + self.start_qualifier = start_qualifier self.inclusive_start = inclusive_start - self.end_column = end_column + self.end_qualifier = end_qualifier self.inclusive_end = inclusive_end def __eq__(self, other): if not isinstance(other, self.__class__): return NotImplemented return ( - other.column_family_id == self.column_family_id - and other.start_column == self.start_column - and other.end_column == self.end_column + other.family_id == self.family_id + and other.start_qualifier == self.start_qualifier + and other.end_qualifier == self.end_qualifier and other.inclusive_start == self.inclusive_start and other.inclusive_end == self.inclusive_end ) @@ -433,19 +435,19 @@ def to_pb(self): def range_to_dict(self) -> dict[str, str | bytes]: """Converts the column range range to a dict representation.""" column_range_kwargs: dict[str, str | bytes] = {} - column_range_kwargs["family_name"] = self.column_family_id - if self.start_column is not None: + column_range_kwargs["family_name"] = self.family_id + if self.start_qualifier is not None: if self.inclusive_start: key = "start_qualifier_closed" else: key = "start_qualifier_open" - column_range_kwargs[key] = _to_bytes(self.start_column) - if self.end_column is not None: + column_range_kwargs[key] = _to_bytes(self.start_qualifier) + if self.end_qualifier is not None: if self.inclusive_end: key = "end_qualifier_closed" else: key = "end_qualifier_open" - column_range_kwargs[key] = _to_bytes(self.end_column) + column_range_kwargs[key] = _to_bytes(self.end_qualifier) return column_range_kwargs def to_dict(self) -> dict[str, Any]: @@ -453,7 +455,7 @@ def to_dict(self) -> dict[str, Any]: return {"column_range_filter": self.range_to_dict()} def __repr__(self) -> str: - return f"{self.__class__.__name__}(column_family_id={self.column_family_id}, start_column={self.start_column!r}, end_column={self.end_column!r}, inclusive_start={self.inclusive_start}, inclusive_end={self.inclusive_end})" + return f"{self.__class__.__name__}(family_id={self.family_id}, start_qualifier={self.start_qualifier!r}, end_qualifier={self.end_qualifier!r}, inclusive_start={self.inclusive_start}, inclusive_end={self.inclusive_end})" class ValueRegexFilter(_RegexFilter): @@ -551,7 +553,9 @@ def __init__( if inclusive_end is None: inclusive_end = True elif end_value is None: - raise ValueError("inclusive_end was specified but no end_column was given.") + raise ValueError( + "inclusive_end was specified but no end_qualifier was given." + ) if isinstance(start_value, int): start_value = _PACK_I64(start_value) self.start_value = start_value diff --git a/tests/unit/test_row_filters.py b/tests/unit/test_row_filters.py index 827460737..c6fb396c6 100644 --- a/tests/unit/test_row_filters.py +++ b/tests/unit/test_row_filters.py @@ -473,11 +473,11 @@ def test_timestamp_range_filter_empty_to_dict(): def test_column_range_filter_constructor_defaults(): from google.cloud.bigtable.row_filters import ColumnRangeFilter - column_family_id = object() - row_filter = ColumnRangeFilter(column_family_id) - assert row_filter.column_family_id is column_family_id - assert row_filter.start_column is None - assert row_filter.end_column is None + family_id = object() + row_filter = ColumnRangeFilter(family_id) + assert row_filter.family_id is family_id + assert row_filter.start_qualifier is None + assert row_filter.end_qualifier is None assert row_filter.inclusive_start assert row_filter.inclusive_end @@ -485,21 +485,21 @@ def test_column_range_filter_constructor_defaults(): def test_column_range_filter_constructor_explicit(): from google.cloud.bigtable.row_filters import ColumnRangeFilter - column_family_id = object() - start_column = object() - end_column = object() + family_id = object() + start_qualifier = object() + end_qualifier = object() inclusive_start = object() inclusive_end = object() row_filter = ColumnRangeFilter( - column_family_id, - start_column=start_column, - end_column=end_column, + family_id, + start_qualifier=start_qualifier, + end_qualifier=end_qualifier, inclusive_start=inclusive_start, inclusive_end=inclusive_end, ) - assert row_filter.column_family_id is column_family_id - assert row_filter.start_column is start_column - assert row_filter.end_column is end_column + assert row_filter.family_id is family_id + assert row_filter.start_qualifier is start_qualifier + assert row_filter.end_qualifier is end_qualifier assert row_filter.inclusive_start is inclusive_start assert row_filter.inclusive_end is inclusive_end @@ -507,38 +507,38 @@ def test_column_range_filter_constructor_explicit(): def test_column_range_filter_constructor_(): from google.cloud.bigtable.row_filters import ColumnRangeFilter - column_family_id = object() + family_id = object() with pytest.raises(ValueError): - ColumnRangeFilter(column_family_id, inclusive_start=True) + ColumnRangeFilter(family_id, inclusive_start=True) def test_column_range_filter_constructor_bad_end(): from google.cloud.bigtable.row_filters import ColumnRangeFilter - column_family_id = object() + family_id = object() with pytest.raises(ValueError): - ColumnRangeFilter(column_family_id, inclusive_end=True) + ColumnRangeFilter(family_id, inclusive_end=True) def test_column_range_filter___eq__(): from google.cloud.bigtable.row_filters import ColumnRangeFilter - column_family_id = object() - start_column = object() - end_column = object() + family_id = object() + start_qualifier = object() + end_qualifier = object() inclusive_start = object() inclusive_end = object() row_filter1 = ColumnRangeFilter( - column_family_id, - start_column=start_column, - end_column=end_column, + family_id, + start_qualifier=start_qualifier, + end_qualifier=end_qualifier, inclusive_start=inclusive_start, inclusive_end=inclusive_end, ) row_filter2 = ColumnRangeFilter( - column_family_id, - start_column=start_column, - end_column=end_column, + family_id, + start_qualifier=start_qualifier, + end_qualifier=end_qualifier, inclusive_start=inclusive_start, inclusive_end=inclusive_end, ) @@ -548,8 +548,8 @@ def test_column_range_filter___eq__(): def test_column_range_filter___eq__type_differ(): from google.cloud.bigtable.row_filters import ColumnRangeFilter - column_family_id = object() - row_filter1 = ColumnRangeFilter(column_family_id) + family_id = object() + row_filter1 = ColumnRangeFilter(family_id) row_filter2 = object() assert not (row_filter1 == row_filter2) @@ -557,23 +557,23 @@ def test_column_range_filter___eq__type_differ(): def test_column_range_filter___ne__(): from google.cloud.bigtable.row_filters import ColumnRangeFilter - column_family_id = object() - other_column_family_id = object() - start_column = object() - end_column = object() + family_id = object() + other_family_id = object() + start_qualifier = object() + end_qualifier = object() inclusive_start = object() inclusive_end = object() row_filter1 = ColumnRangeFilter( - column_family_id, - start_column=start_column, - end_column=end_column, + family_id, + start_qualifier=start_qualifier, + end_qualifier=end_qualifier, inclusive_start=inclusive_start, inclusive_end=inclusive_end, ) row_filter2 = ColumnRangeFilter( - other_column_family_id, - start_column=start_column, - end_column=end_column, + other_family_id, + start_qualifier=start_qualifier, + end_qualifier=end_qualifier, inclusive_start=inclusive_start, inclusive_end=inclusive_end, ) @@ -583,9 +583,9 @@ def test_column_range_filter___ne__(): def test_column_range_filter_to_pb(): from google.cloud.bigtable.row_filters import ColumnRangeFilter - column_family_id = "column-family-id" - row_filter = ColumnRangeFilter(column_family_id) - col_range_pb = _ColumnRangePB(family_name=column_family_id) + family_id = "column-family-id" + row_filter = ColumnRangeFilter(family_id) + col_range_pb = _ColumnRangePB(family_name=family_id) expected_pb = _RowFilterPB(column_range_filter=col_range_pb) assert row_filter.to_pb() == expected_pb @@ -594,9 +594,9 @@ def test_column_range_filter_to_dict(): from google.cloud.bigtable.row_filters import ColumnRangeFilter from google.cloud.bigtable_v2.types import data as data_v2_pb2 - column_family_id = "column-family-id" - row_filter = ColumnRangeFilter(column_family_id) - expected_dict = {"column_range_filter": {"family_name": column_family_id}} + family_id = "column-family-id" + row_filter = ColumnRangeFilter(family_id) + expected_dict = {"column_range_filter": {"family_name": family_id}} assert row_filter.to_dict() == expected_dict expected_pb_value = row_filter.to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -605,12 +605,10 @@ def test_column_range_filter_to_dict(): def test_column_range_filter_to_pb_inclusive_start(): from google.cloud.bigtable.row_filters import ColumnRangeFilter - column_family_id = "column-family-id" + family_id = "column-family-id" column = b"column" - row_filter = ColumnRangeFilter(column_family_id, start_column=column) - col_range_pb = _ColumnRangePB( - family_name=column_family_id, start_qualifier_closed=column - ) + row_filter = ColumnRangeFilter(family_id, start_qualifier=column) + col_range_pb = _ColumnRangePB(family_name=family_id, start_qualifier_closed=column) expected_pb = _RowFilterPB(column_range_filter=col_range_pb) assert row_filter.to_pb() == expected_pb @@ -618,14 +616,12 @@ def test_column_range_filter_to_pb_inclusive_start(): def test_column_range_filter_to_pb_exclusive_start(): from google.cloud.bigtable.row_filters import ColumnRangeFilter - column_family_id = "column-family-id" + family_id = "column-family-id" column = b"column" row_filter = ColumnRangeFilter( - column_family_id, start_column=column, inclusive_start=False - ) - col_range_pb = _ColumnRangePB( - family_name=column_family_id, start_qualifier_open=column + family_id, start_qualifier=column, inclusive_start=False ) + col_range_pb = _ColumnRangePB(family_name=family_id, start_qualifier_open=column) expected_pb = _RowFilterPB(column_range_filter=col_range_pb) assert row_filter.to_pb() == expected_pb @@ -633,12 +629,10 @@ def test_column_range_filter_to_pb_exclusive_start(): def test_column_range_filter_to_pb_inclusive_end(): from google.cloud.bigtable.row_filters import ColumnRangeFilter - column_family_id = "column-family-id" + family_id = "column-family-id" column = b"column" - row_filter = ColumnRangeFilter(column_family_id, end_column=column) - col_range_pb = _ColumnRangePB( - family_name=column_family_id, end_qualifier_closed=column - ) + row_filter = ColumnRangeFilter(family_id, end_qualifier=column) + col_range_pb = _ColumnRangePB(family_name=family_id, end_qualifier_closed=column) expected_pb = _RowFilterPB(column_range_filter=col_range_pb) assert row_filter.to_pb() == expected_pb @@ -646,14 +640,10 @@ def test_column_range_filter_to_pb_inclusive_end(): def test_column_range_filter_to_pb_exclusive_end(): from google.cloud.bigtable.row_filters import ColumnRangeFilter - column_family_id = "column-family-id" + family_id = "column-family-id" column = b"column" - row_filter = ColumnRangeFilter( - column_family_id, end_column=column, inclusive_end=False - ) - col_range_pb = _ColumnRangePB( - family_name=column_family_id, end_qualifier_open=column - ) + row_filter = ColumnRangeFilter(family_id, end_qualifier=column, inclusive_end=False) + col_range_pb = _ColumnRangePB(family_name=family_id, end_qualifier_open=column) expected_pb = _RowFilterPB(column_range_filter=col_range_pb) assert row_filter.to_pb() == expected_pb From 73b187f163c737c4954b8685cfd8f3cd8c2265ed Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 16 Mar 2023 13:34:41 -0700 Subject: [PATCH 08/16] filter combination implements sequence abc --- google/cloud/bigtable/row_filters.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/google/cloud/bigtable/row_filters.py b/google/cloud/bigtable/row_filters.py index 41a337bef..6102dca61 100644 --- a/google/cloud/bigtable/row_filters.py +++ b/google/cloud/bigtable/row_filters.py @@ -731,7 +731,7 @@ def __repr__(self) -> str: return f"{self.__class__.__name__}(label={self.label})" -class _FilterCombination(RowFilter): +class _FilterCombination(RowFilter, Sequence[RowFilter]): """Chain of row filters. Sends rows through several filters in sequence. The filters are "chained" @@ -755,10 +755,21 @@ def __eq__(self, other): def __ne__(self, other): return not self == other + def __len__(self) -> int: + return len(self.filters) + + def __getitem__(self, index: int) -> RowFilter: + return self.filters[index] + def __repr__(self) -> str: return f"{self.__class__.__name__}(filters={[repr(f) for f in self.filters]})" def __str__(self) -> str: + """ + Returns a string representation of the filter chain. + + Adds line breaks between each sub-filter for readability. + """ output = [f"{self.__class__.__name__}(["] for filter_ in self.filters: filter_lines = f"{filter_},".splitlines() From d7f1f7f594c6a232d4365dadab26f9bf0e5d760f Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 16 Mar 2023 13:38:23 -0700 Subject: [PATCH 09/16] fixed mypy issues --- google/cloud/bigtable/row_filters.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/google/cloud/bigtable/row_filters.py b/google/cloud/bigtable/row_filters.py index 6102dca61..d6af363f0 100644 --- a/google/cloud/bigtable/row_filters.py +++ b/google/cloud/bigtable/row_filters.py @@ -17,7 +17,7 @@ import struct -from typing import Any, TYPE_CHECKING +from typing import Any, Sequence, TYPE_CHECKING, overload from google.cloud._helpers import _microseconds_from_datetime # type: ignore from google.cloud._helpers import _to_bytes # type: ignore @@ -758,7 +758,17 @@ def __ne__(self, other): def __len__(self) -> int: return len(self.filters) + @overload def __getitem__(self, index: int) -> RowFilter: + # overload signature for type checking + pass + + @overload + def __getitem__(self, index: slice) -> list[RowFilter]: + # overload signature for type checking + pass + + def __getitem__(self, index): return self.filters[index] def __repr__(self) -> str: From 451811997a12aefbffa376e68c8f9da8809a32c5 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 16 Mar 2023 13:44:29 -0700 Subject: [PATCH 10/16] made to_pb private --- google/cloud/bigtable/row_filters.py | 28 ++--- tests/unit/test_row_filters.py | 162 +++++++++++++-------------- 2 files changed, 95 insertions(+), 95 deletions(-) diff --git a/google/cloud/bigtable/row_filters.py b/google/cloud/bigtable/row_filters.py index d6af363f0..0fe78abed 100644 --- a/google/cloud/bigtable/row_filters.py +++ b/google/cloud/bigtable/row_filters.py @@ -41,7 +41,7 @@ class RowFilter(object): This class is a do-nothing base class for all row filters. """ - def to_pb(self) -> data_v2_pb2.RowFilter: + def _to_pb(self) -> data_v2_pb2.RowFilter: """Converts the row filter to a protobuf. :rtype: :class:`.data_v2_pb2.RowFilter` @@ -277,7 +277,7 @@ def __eq__(self, other): def __ne__(self, other): return not self == other - def to_pb(self) -> data_v2_pb2.TimestampRange: + def _to_pb(self) -> data_v2_pb2.TimestampRange: """Converts the :class:`TimestampRange` to a protobuf. :rtype: :class:`.data_v2_pb2.TimestampRange` @@ -321,7 +321,7 @@ def __eq__(self, other): def __ne__(self, other): return not self == other - def to_pb(self): + def _to_pb(self): """Converts the row filter to a protobuf. First converts the ``range_`` on the current object to a protobuf and @@ -330,7 +330,7 @@ def to_pb(self): :rtype: :class:`.data_v2_pb2.RowFilter` :returns: The converted current object. """ - return data_v2_pb2.RowFilter(timestamp_range_filter=self.range_.to_pb()) + return data_v2_pb2.RowFilter(timestamp_range_filter=self.range_._to_pb()) def to_dict(self) -> dict[str, Any]: """Converts the row filter to a dict representation.""" @@ -420,7 +420,7 @@ def __eq__(self, other): def __ne__(self, other): return not self == other - def to_pb(self): + def _to_pb(self): """Converts the row filter to a protobuf. First converts to a :class:`.data_v2_pb2.ColumnRange` and then uses it @@ -579,7 +579,7 @@ def __eq__(self, other): def __ne__(self, other): return not self == other - def to_pb(self): + def _to_pb(self): """Converts the row filter to a protobuf. First converts to a :class:`.data_v2_pb2.ValueRange` and then uses @@ -799,14 +799,14 @@ class RowFilterChain(_FilterCombination): :param filters: List of :class:`RowFilter` """ - def to_pb(self): + def _to_pb(self): """Converts the row filter to a protobuf. :rtype: :class:`.data_v2_pb2.RowFilter` :returns: The converted current object. """ chain = data_v2_pb2.RowFilter.Chain( - filters=[row_filter.to_pb() for row_filter in self.filters] + filters=[row_filter._to_pb() for row_filter in self.filters] ) return data_v2_pb2.RowFilter(chain=chain) @@ -828,14 +828,14 @@ class RowFilterUnion(_FilterCombination): :param filters: List of :class:`RowFilter` """ - def to_pb(self): + def _to_pb(self): """Converts the row filter to a protobuf. :rtype: :class:`.data_v2_pb2.RowFilter` :returns: The converted current object. """ interleave = data_v2_pb2.RowFilter.Interleave( - filters=[row_filter.to_pb() for row_filter in self.filters] + filters=[row_filter._to_pb() for row_filter in self.filters] ) return data_v2_pb2.RowFilter(interleave=interleave) @@ -896,17 +896,17 @@ def __eq__(self, other): def __ne__(self, other): return not self == other - def to_pb(self): + def _to_pb(self): """Converts the row filter to a protobuf. :rtype: :class:`.data_v2_pb2.RowFilter` :returns: The converted current object. """ - condition_kwargs = {"predicate_filter": self.predicate_filter.to_pb()} + condition_kwargs = {"predicate_filter": self.predicate_filter._to_pb()} if self.true_filter is not None: - condition_kwargs["true_filter"] = self.true_filter.to_pb() + condition_kwargs["true_filter"] = self.true_filter._to_pb() if self.false_filter is not None: - condition_kwargs["false_filter"] = self.false_filter.to_pb() + condition_kwargs["false_filter"] = self.false_filter._to_pb() condition = data_v2_pb2.RowFilter.Condition(**condition_kwargs) return data_v2_pb2.RowFilter(condition=condition) diff --git a/tests/unit/test_row_filters.py b/tests/unit/test_row_filters.py index c6fb396c6..d29ddcfd4 100644 --- a/tests/unit/test_row_filters.py +++ b/tests/unit/test_row_filters.py @@ -56,7 +56,7 @@ def test_sink_filter_to_pb(): flag = True row_filter = SinkFilter(flag) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(sink=flag) assert pb_val == expected_pb @@ -69,7 +69,7 @@ def test_sink_filter_to_dict(): row_filter = SinkFilter(flag) expected_dict = {"sink": flag} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -78,7 +78,7 @@ def test_pass_all_filter_to_pb(): flag = True row_filter = PassAllFilter(flag) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(pass_all_filter=flag) assert pb_val == expected_pb @@ -91,7 +91,7 @@ def test_pass_all_filter_to_dict(): row_filter = PassAllFilter(flag) expected_dict = {"pass_all_filter": flag} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -100,7 +100,7 @@ def test_block_all_filter_to_pb(): flag = True row_filter = BlockAllFilter(flag) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(block_all_filter=flag) assert pb_val == expected_pb @@ -113,7 +113,7 @@ def test_block_all_filter_to_dict(): row_filter = BlockAllFilter(flag) expected_dict = {"block_all_filter": flag} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -165,7 +165,7 @@ def test_row_key_regex_filter_to_pb(): regex = b"row-key-regex" row_filter = RowKeyRegexFilter(regex) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(row_key_regex_filter=regex) assert pb_val == expected_pb @@ -178,7 +178,7 @@ def test_row_key_regex_filter_to_dict(): row_filter = RowKeyRegexFilter(regex) expected_dict = {"row_key_regex_filter": regex} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -223,7 +223,7 @@ def test_row_sample_filter_to_pb(): sample = 0.25 row_filter = RowSampleFilter(sample) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(row_sample_filter=sample) assert pb_val == expected_pb @@ -233,7 +233,7 @@ def test_family_name_regex_filter_to_pb(): regex = "family-regex" row_filter = FamilyNameRegexFilter(regex) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(family_name_regex_filter=regex) assert pb_val == expected_pb @@ -246,7 +246,7 @@ def test_family_name_regex_filter_to_dict(): row_filter = FamilyNameRegexFilter(regex) expected_dict = {"family_name_regex_filter": regex.encode()} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -255,7 +255,7 @@ def test_column_qualifier_regex_filter_to_pb(): regex = b"column-regex" row_filter = ColumnQualifierRegexFilter(regex) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(column_qualifier_regex_filter=regex) assert pb_val == expected_pb @@ -268,7 +268,7 @@ def test_column_qualifier_regex_filter_to_dict(): row_filter = ColumnQualifierRegexFilter(regex) expected_dict = {"column_qualifier_regex_filter": regex} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -323,7 +323,7 @@ def _timestamp_range_to_pb_helper(pb_kwargs, start=None, end=None): end = _EPOCH + datetime.timedelta(microseconds=end) time_range = TimestampRange(start=start, end=end) expected_pb = _TimestampRangePB(**pb_kwargs) - time_pb = time_range.to_pb() + time_pb = time_range._to_pb() assert time_pb.start_timestamp_micros == expected_pb.start_timestamp_micros assert time_pb.end_timestamp_micros == expected_pb.end_timestamp_micros assert time_pb == expected_pb @@ -355,7 +355,7 @@ def test_timestamp_range_to_dict(): "end_timestamp_micros": 1546387200000000, } assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.TimestampRange(**expected_dict) == expected_pb_value @@ -377,7 +377,7 @@ def test_timestamp_range_to_dict_start_only(): row_filter = TimestampRange(start=datetime.datetime(2019, 1, 1)) expected_dict = {"start_timestamp_micros": 1546300800000000} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.TimestampRange(**expected_dict) == expected_pb_value @@ -399,7 +399,7 @@ def test_timestamp_range_to_dict_end_only(): row_filter = TimestampRange(end=datetime.datetime(2019, 1, 2)) expected_dict = {"end_timestamp_micros": 1546387200000000} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.TimestampRange(**expected_dict) == expected_pb_value @@ -435,7 +435,7 @@ def test_timestamp_range_filter_to_pb(): from google.cloud.bigtable.row_filters import TimestampRangeFilter row_filter = TimestampRangeFilter() - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(timestamp_range_filter=_TimestampRangePB()) assert pb_val == expected_pb @@ -455,7 +455,7 @@ def test_timestamp_range_filter_to_dict(): } } assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -466,7 +466,7 @@ def test_timestamp_range_filter_empty_to_dict(): row_filter = TimestampRangeFilter() expected_dict = {"timestamp_range_filter": {}} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -587,7 +587,7 @@ def test_column_range_filter_to_pb(): row_filter = ColumnRangeFilter(family_id) col_range_pb = _ColumnRangePB(family_name=family_id) expected_pb = _RowFilterPB(column_range_filter=col_range_pb) - assert row_filter.to_pb() == expected_pb + assert row_filter._to_pb() == expected_pb def test_column_range_filter_to_dict(): @@ -598,7 +598,7 @@ def test_column_range_filter_to_dict(): row_filter = ColumnRangeFilter(family_id) expected_dict = {"column_range_filter": {"family_name": family_id}} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -610,7 +610,7 @@ def test_column_range_filter_to_pb_inclusive_start(): row_filter = ColumnRangeFilter(family_id, start_qualifier=column) col_range_pb = _ColumnRangePB(family_name=family_id, start_qualifier_closed=column) expected_pb = _RowFilterPB(column_range_filter=col_range_pb) - assert row_filter.to_pb() == expected_pb + assert row_filter._to_pb() == expected_pb def test_column_range_filter_to_pb_exclusive_start(): @@ -623,7 +623,7 @@ def test_column_range_filter_to_pb_exclusive_start(): ) col_range_pb = _ColumnRangePB(family_name=family_id, start_qualifier_open=column) expected_pb = _RowFilterPB(column_range_filter=col_range_pb) - assert row_filter.to_pb() == expected_pb + assert row_filter._to_pb() == expected_pb def test_column_range_filter_to_pb_inclusive_end(): @@ -634,7 +634,7 @@ def test_column_range_filter_to_pb_inclusive_end(): row_filter = ColumnRangeFilter(family_id, end_qualifier=column) col_range_pb = _ColumnRangePB(family_name=family_id, end_qualifier_closed=column) expected_pb = _RowFilterPB(column_range_filter=col_range_pb) - assert row_filter.to_pb() == expected_pb + assert row_filter._to_pb() == expected_pb def test_column_range_filter_to_pb_exclusive_end(): @@ -645,7 +645,7 @@ def test_column_range_filter_to_pb_exclusive_end(): row_filter = ColumnRangeFilter(family_id, end_qualifier=column, inclusive_end=False) col_range_pb = _ColumnRangePB(family_name=family_id, end_qualifier_open=column) expected_pb = _RowFilterPB(column_range_filter=col_range_pb) - assert row_filter.to_pb() == expected_pb + assert row_filter._to_pb() == expected_pb def test_value_regex_filter_to_pb_w_bytes(): @@ -653,7 +653,7 @@ def test_value_regex_filter_to_pb_w_bytes(): value = regex = b"value-regex" row_filter = ValueRegexFilter(value) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(value_regex_filter=regex) assert pb_val == expected_pb @@ -666,7 +666,7 @@ def test_value_regex_filter_to_dict_w_bytes(): row_filter = ValueRegexFilter(value) expected_dict = {"value_regex_filter": regex} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -676,7 +676,7 @@ def test_value_regex_filter_to_pb_w_str(): value = "value-regex" regex = value.encode("ascii") row_filter = ValueRegexFilter(value) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(value_regex_filter=regex) assert pb_val == expected_pb @@ -690,7 +690,7 @@ def test_value_regex_filter_to_dict_w_str(): row_filter = ValueRegexFilter(value) expected_dict = {"value_regex_filter": regex} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -699,7 +699,7 @@ def test_exact_value_filter_to_pb_w_bytes(): value = regex = b"value-regex" row_filter = ExactValueFilter(value) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(value_regex_filter=regex) assert pb_val == expected_pb @@ -712,7 +712,7 @@ def test_exact_value_filter_to_dict_w_bytes(): row_filter = ExactValueFilter(value) expected_dict = {"value_regex_filter": regex} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -722,7 +722,7 @@ def test_exact_value_filter_to_pb_w_str(): value = "value-regex" regex = value.encode("ascii") row_filter = ExactValueFilter(value) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(value_regex_filter=regex) assert pb_val == expected_pb @@ -736,7 +736,7 @@ def test_exact_value_filter_to_dict_w_str(): row_filter = ExactValueFilter(value) expected_dict = {"value_regex_filter": regex} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -747,7 +747,7 @@ def test_exact_value_filter_to_pb_w_int(): value = 1 regex = struct.Struct(">q").pack(value) row_filter = ExactValueFilter(value) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(value_regex_filter=regex) assert pb_val == expected_pb @@ -762,7 +762,7 @@ def test_exact_value_filter_to_dict_w_int(): row_filter = ExactValueFilter(value) expected_dict = {"value_regex_filter": regex} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -888,7 +888,7 @@ def test_value_range_filter_to_pb(): row_filter = ValueRangeFilter() expected_pb = _RowFilterPB(value_range_filter=_ValueRangePB()) - assert row_filter.to_pb() == expected_pb + assert row_filter._to_pb() == expected_pb def test_value_range_filter_to_dict(): @@ -898,7 +898,7 @@ def test_value_range_filter_to_dict(): row_filter = ValueRangeFilter() expected_dict = {"value_range_filter": {}} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -909,7 +909,7 @@ def test_value_range_filter_to_pb_inclusive_start(): row_filter = ValueRangeFilter(start_value=value) val_range_pb = _ValueRangePB(start_value_closed=value) expected_pb = _RowFilterPB(value_range_filter=val_range_pb) - assert row_filter.to_pb() == expected_pb + assert row_filter._to_pb() == expected_pb def test_value_range_filter_to_pb_exclusive_start(): @@ -919,7 +919,7 @@ def test_value_range_filter_to_pb_exclusive_start(): row_filter = ValueRangeFilter(start_value=value, inclusive_start=False) val_range_pb = _ValueRangePB(start_value_open=value) expected_pb = _RowFilterPB(value_range_filter=val_range_pb) - assert row_filter.to_pb() == expected_pb + assert row_filter._to_pb() == expected_pb def test_value_range_filter_to_pb_inclusive_end(): @@ -929,7 +929,7 @@ def test_value_range_filter_to_pb_inclusive_end(): row_filter = ValueRangeFilter(end_value=value) val_range_pb = _ValueRangePB(end_value_closed=value) expected_pb = _RowFilterPB(value_range_filter=val_range_pb) - assert row_filter.to_pb() == expected_pb + assert row_filter._to_pb() == expected_pb def test_value_range_filter_to_pb_exclusive_end(): @@ -939,7 +939,7 @@ def test_value_range_filter_to_pb_exclusive_end(): row_filter = ValueRangeFilter(end_value=value, inclusive_end=False) val_range_pb = _ValueRangePB(end_value_open=value) expected_pb = _RowFilterPB(value_range_filter=val_range_pb) - assert row_filter.to_pb() == expected_pb + assert row_filter._to_pb() == expected_pb def test_cell_count_constructor(): @@ -982,7 +982,7 @@ def test_cells_row_offset_filter_to_pb(): num_cells = 76 row_filter = CellsRowOffsetFilter(num_cells) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(cells_per_row_offset_filter=num_cells) assert pb_val == expected_pb @@ -995,7 +995,7 @@ def test_cells_row_offset_filter_to_dict(): row_filter = CellsRowOffsetFilter(num_cells) expected_dict = {"cells_per_row_offset_filter": num_cells} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -1004,7 +1004,7 @@ def test_cells_row_limit_filter_to_pb(): num_cells = 189 row_filter = CellsRowLimitFilter(num_cells) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(cells_per_row_limit_filter=num_cells) assert pb_val == expected_pb @@ -1017,7 +1017,7 @@ def test_cells_row_limit_filter_to_dict(): row_filter = CellsRowLimitFilter(num_cells) expected_dict = {"cells_per_row_limit_filter": num_cells} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -1026,7 +1026,7 @@ def test_cells_column_limit_filter_to_pb(): num_cells = 10 row_filter = CellsColumnLimitFilter(num_cells) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(cells_per_column_limit_filter=num_cells) assert pb_val == expected_pb @@ -1039,7 +1039,7 @@ def test_cells_column_limit_filter_to_dict(): row_filter = CellsColumnLimitFilter(num_cells) expected_dict = {"cells_per_column_limit_filter": num_cells} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -1048,7 +1048,7 @@ def test_strip_value_transformer_filter_to_pb(): flag = True row_filter = StripValueTransformerFilter(flag) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(strip_value_transformer=flag) assert pb_val == expected_pb @@ -1061,7 +1061,7 @@ def test_strip_value_transformer_filter_to_dict(): row_filter = StripValueTransformerFilter(flag) expected_dict = {"strip_value_transformer": flag} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -1106,7 +1106,7 @@ def test_apply_label_filter_to_pb(): label = "label" row_filter = ApplyLabelFilter(label) - pb_val = row_filter.to_pb() + pb_val = row_filter._to_pb() expected_pb = _RowFilterPB(apply_label_transformer=label) assert pb_val == expected_pb @@ -1119,7 +1119,7 @@ def test_apply_label_filter_to_dict(): row_filter = ApplyLabelFilter(label) expected_dict = {"apply_label_transformer": label} assert row_filter.to_dict() == expected_dict - expected_pb_value = row_filter.to_pb() + expected_pb_value = row_filter._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -1172,13 +1172,13 @@ def test_row_filter_chain_to_pb(): from google.cloud.bigtable.row_filters import StripValueTransformerFilter row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() + row_filter1_pb = row_filter1._to_pb() row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() + row_filter2_pb = row_filter2._to_pb() row_filter3 = RowFilterChain(filters=[row_filter1, row_filter2]) - filter_pb = row_filter3.to_pb() + filter_pb = row_filter3._to_pb() expected_pb = _RowFilterPB( chain=_RowFilterChainPB(filters=[row_filter1_pb, row_filter2_pb]) @@ -1203,7 +1203,7 @@ def test_row_filter_chain_to_dict(): expected_dict = {"chain": {"filters": [row_filter1_dict, row_filter2_dict]}} assert filter_dict == expected_dict - expected_pb_value = row_filter3.to_pb() + expected_pb_value = row_filter3._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -1217,13 +1217,13 @@ def test_row_filter_chain_to_pb_nested(): row_filter2 = RowSampleFilter(0.25) row_filter3 = RowFilterChain(filters=[row_filter1, row_filter2]) - row_filter3_pb = row_filter3.to_pb() + row_filter3_pb = row_filter3._to_pb() row_filter4 = CellsRowLimitFilter(11) - row_filter4_pb = row_filter4.to_pb() + row_filter4_pb = row_filter4._to_pb() row_filter5 = RowFilterChain(filters=[row_filter3, row_filter4]) - filter_pb = row_filter5.to_pb() + filter_pb = row_filter5._to_pb() expected_pb = _RowFilterPB( chain=_RowFilterChainPB(filters=[row_filter3_pb, row_filter4_pb]) @@ -1253,7 +1253,7 @@ def test_row_filter_chain_to_dict_nested(): expected_dict = {"chain": {"filters": [row_filter3_dict, row_filter4_dict]}} assert filter_dict == expected_dict - expected_pb_value = row_filter5.to_pb() + expected_pb_value = row_filter5._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -1263,13 +1263,13 @@ def test_row_filter_union_to_pb(): from google.cloud.bigtable.row_filters import StripValueTransformerFilter row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() + row_filter1_pb = row_filter1._to_pb() row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() + row_filter2_pb = row_filter2._to_pb() row_filter3 = RowFilterUnion(filters=[row_filter1, row_filter2]) - filter_pb = row_filter3.to_pb() + filter_pb = row_filter3._to_pb() expected_pb = _RowFilterPB( interleave=_RowFilterInterleavePB(filters=[row_filter1_pb, row_filter2_pb]) @@ -1294,7 +1294,7 @@ def test_row_filter_union_to_dict(): expected_dict = {"interleave": {"filters": [row_filter1_dict, row_filter2_dict]}} assert filter_dict == expected_dict - expected_pb_value = row_filter3.to_pb() + expected_pb_value = row_filter3._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -1308,13 +1308,13 @@ def test_row_filter_union_to_pb_nested(): row_filter2 = RowSampleFilter(0.25) row_filter3 = RowFilterUnion(filters=[row_filter1, row_filter2]) - row_filter3_pb = row_filter3.to_pb() + row_filter3_pb = row_filter3._to_pb() row_filter4 = CellsRowLimitFilter(11) - row_filter4_pb = row_filter4.to_pb() + row_filter4_pb = row_filter4._to_pb() row_filter5 = RowFilterUnion(filters=[row_filter3, row_filter4]) - filter_pb = row_filter5.to_pb() + filter_pb = row_filter5._to_pb() expected_pb = _RowFilterPB( interleave=_RowFilterInterleavePB(filters=[row_filter3_pb, row_filter4_pb]) @@ -1344,7 +1344,7 @@ def test_row_filter_union_to_dict_nested(): expected_dict = {"interleave": {"filters": [row_filter3_dict, row_filter4_dict]}} assert filter_dict == expected_dict - expected_pb_value = row_filter5.to_pb() + expected_pb_value = row_filter5._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -1413,18 +1413,18 @@ def test_conditional_row_filter_to_pb(): from google.cloud.bigtable.row_filters import StripValueTransformerFilter row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() + row_filter1_pb = row_filter1._to_pb() row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() + row_filter2_pb = row_filter2._to_pb() row_filter3 = CellsRowOffsetFilter(11) - row_filter3_pb = row_filter3.to_pb() + row_filter3_pb = row_filter3._to_pb() row_filter4 = ConditionalRowFilter( row_filter1, true_filter=row_filter2, false_filter=row_filter3 ) - filter_pb = row_filter4.to_pb() + filter_pb = row_filter4._to_pb() expected_pb = _RowFilterPB( condition=_RowFilterConditionPB( @@ -1465,7 +1465,7 @@ def test_conditional_row_filter_to_dict(): } } assert filter_dict == expected_dict - expected_pb_value = row_filter4.to_pb() + expected_pb_value = row_filter4._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -1475,13 +1475,13 @@ def test_conditional_row_filter_to_pb_true_only(): from google.cloud.bigtable.row_filters import StripValueTransformerFilter row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() + row_filter1_pb = row_filter1._to_pb() row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() + row_filter2_pb = row_filter2._to_pb() row_filter3 = ConditionalRowFilter(row_filter1, true_filter=row_filter2) - filter_pb = row_filter3.to_pb() + filter_pb = row_filter3._to_pb() expected_pb = _RowFilterPB( condition=_RowFilterConditionPB( @@ -1513,7 +1513,7 @@ def test_conditional_row_filter_to_dict_true_only(): } } assert filter_dict == expected_dict - expected_pb_value = row_filter3.to_pb() + expected_pb_value = row_filter3._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value @@ -1523,13 +1523,13 @@ def test_conditional_row_filter_to_pb_false_only(): from google.cloud.bigtable.row_filters import StripValueTransformerFilter row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() + row_filter1_pb = row_filter1._to_pb() row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() + row_filter2_pb = row_filter2._to_pb() row_filter3 = ConditionalRowFilter(row_filter1, false_filter=row_filter2) - filter_pb = row_filter3.to_pb() + filter_pb = row_filter3._to_pb() expected_pb = _RowFilterPB( condition=_RowFilterConditionPB( @@ -1561,7 +1561,7 @@ def test_conditional_row_filter_to_dict_false_only(): } } assert filter_dict == expected_dict - expected_pb_value = row_filter3.to_pb() + expected_pb_value = row_filter3._to_pb() assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value From ad11eb3f360235bf61b3c1a83530e2a19496e010 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 17 Mar 2023 10:09:31 -0700 Subject: [PATCH 11/16] changed annotations --- google/cloud/bigtable/row_filters.py | 36 +++++++++++----------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/google/cloud/bigtable/row_filters.py b/google/cloud/bigtable/row_filters.py index 0fe78abed..35d5ed99b 100644 --- a/google/cloud/bigtable/row_filters.py +++ b/google/cloud/bigtable/row_filters.py @@ -44,8 +44,7 @@ class RowFilter(object): def _to_pb(self) -> data_v2_pb2.RowFilter: """Converts the row filter to a protobuf. - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. + Returns: The converted current object. """ return data_v2_pb2.RowFilter(**self.to_dict()) @@ -280,8 +279,7 @@ def __ne__(self, other): def _to_pb(self) -> data_v2_pb2.TimestampRange: """Converts the :class:`TimestampRange` to a protobuf. - :rtype: :class:`.data_v2_pb2.TimestampRange` - :returns: The converted current object. + Returns: The converted current object. """ return data_v2_pb2.TimestampRange(**self.to_dict()) @@ -321,14 +319,13 @@ def __eq__(self, other): def __ne__(self, other): return not self == other - def _to_pb(self): + def _to_pb(self) -> data_v2_pb2.RowFilter: """Converts the row filter to a protobuf. First converts the ``range_`` on the current object to a protobuf and then uses it in the ``timestamp_range_filter`` field. - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. + Returns: The converted current object. """ return data_v2_pb2.RowFilter(timestamp_range_filter=self.range_._to_pb()) @@ -420,14 +417,13 @@ def __eq__(self, other): def __ne__(self, other): return not self == other - def _to_pb(self): + def _to_pb(self) -> data_v2_pb2.RowFilter: """Converts the row filter to a protobuf. First converts to a :class:`.data_v2_pb2.ColumnRange` and then uses it in the ``column_range_filter`` field. - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. + Returns: The converted current object. """ column_range = data_v2_pb2.ColumnRange(**self.range_to_dict()) return data_v2_pb2.RowFilter(column_range_filter=column_range) @@ -579,14 +575,13 @@ def __eq__(self, other): def __ne__(self, other): return not self == other - def _to_pb(self): + def _to_pb(self) -> data_v2_pb2.RowFilter: """Converts the row filter to a protobuf. First converts to a :class:`.data_v2_pb2.ValueRange` and then uses it to create a row filter protobuf. - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. + Returns: The converted current object. """ value_range = data_v2_pb2.ValueRange(**self.range_to_dict()) return data_v2_pb2.RowFilter(value_range_filter=value_range) @@ -799,11 +794,10 @@ class RowFilterChain(_FilterCombination): :param filters: List of :class:`RowFilter` """ - def _to_pb(self): + def _to_pb(self) -> data_v2_pb2.RowFilter: """Converts the row filter to a protobuf. - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. + Returns: The converted current object. """ chain = data_v2_pb2.RowFilter.Chain( filters=[row_filter._to_pb() for row_filter in self.filters] @@ -828,11 +822,10 @@ class RowFilterUnion(_FilterCombination): :param filters: List of :class:`RowFilter` """ - def _to_pb(self): + def _to_pb(self) -> data_v2_pb2.RowFilter: """Converts the row filter to a protobuf. - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. + Returns: The converted current object. """ interleave = data_v2_pb2.RowFilter.Interleave( filters=[row_filter._to_pb() for row_filter in self.filters] @@ -896,11 +889,10 @@ def __eq__(self, other): def __ne__(self, other): return not self == other - def _to_pb(self): + def _to_pb(self) -> data_v2_pb2.RowFilter: """Converts the row filter to a protobuf. - :rtype: :class:`.data_v2_pb2.RowFilter` - :returns: The converted current object. + Returns: The converted current object. """ condition_kwargs = {"predicate_filter": self.predicate_filter._to_pb()} if self.true_filter is not None: From b4ce55118ed7c0d4d89fdb9426976a8d089d6d77 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 17 Mar 2023 11:18:11 -0700 Subject: [PATCH 12/16] added str and repr tests --- google/cloud/bigtable/row_filters.py | 11 +- tests/unit/test_row_filters.py | 393 +++++++++++++++++++++++++++ 2 files changed, 399 insertions(+), 5 deletions(-) diff --git a/google/cloud/bigtable/row_filters.py b/google/cloud/bigtable/row_filters.py index 35d5ed99b..65c33d4f5 100644 --- a/google/cloud/bigtable/row_filters.py +++ b/google/cloud/bigtable/row_filters.py @@ -334,7 +334,7 @@ def to_dict(self) -> dict[str, Any]: return {"timestamp_range_filter": self.range_.to_dict()} def __repr__(self) -> str: - return f"{self.__class__.__name__}(start={self.range_.start}, end={self.range_.end})" + return f"{self.__class__.__name__}(start={self.range_.start!r}, end={self.range_.end!r})" class ColumnRangeFilter(RowFilter): @@ -451,7 +451,7 @@ def to_dict(self) -> dict[str, Any]: return {"column_range_filter": self.range_to_dict()} def __repr__(self) -> str: - return f"{self.__class__.__name__}(family_id={self.family_id}, start_qualifier={self.start_qualifier!r}, end_qualifier={self.end_qualifier!r}, inclusive_start={self.inclusive_start}, inclusive_end={self.inclusive_end})" + return f"{self.__class__.__name__}(family_id='{self.family_id}', start_qualifier={self.start_qualifier!r}, end_qualifier={self.end_qualifier!r}, inclusive_start={self.inclusive_start}, inclusive_end={self.inclusive_end})" class ValueRegexFilter(_RegexFilter): @@ -767,7 +767,7 @@ def __getitem__(self, index): return self.filters[index] def __repr__(self) -> str: - return f"{self.__class__.__name__}(filters={[repr(f) for f in self.filters]})" + return f"{self.__class__.__name__}(filters={self.filters})" def __str__(self) -> str: """ @@ -778,7 +778,7 @@ def __str__(self) -> str: output = [f"{self.__class__.__name__}(["] for filter_ in self.filters: filter_lines = f"{filter_},".splitlines() - output.extend([f" {line}" for line in filter_lines]) + output.extend([f" {line}" for line in filter_lines]) output.append("])") return "\n".join(output) @@ -925,6 +925,7 @@ def __str__(self) -> str: if filter_ is None: continue # add the new filter set, adding indentations for readability - output.append(f" {filter_type}={filter_!r},") + filter_lines = f"{filter_type}={filter_},".splitlines() + output.extend(f" {line}" for line in filter_lines) output.append(")") return "\n".join(output) diff --git a/tests/unit/test_row_filters.py b/tests/unit/test_row_filters.py index d29ddcfd4..0bbe46c81 100644 --- a/tests/unit/test_row_filters.py +++ b/tests/unit/test_row_filters.py @@ -51,6 +51,16 @@ def test_bool_filter___ne__same_value(): assert not (row_filter1 != row_filter2) +def test_bool_filter___repr__(): + from google.cloud.bigtable.row_filters import _BoolFilter + + flag = True + row_filter = _BoolFilter(flag) + assert repr(row_filter) == "_BoolFilter(flag={})".format(flag) + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_sink_filter_to_pb(): from google.cloud.bigtable.row_filters import SinkFilter @@ -73,6 +83,16 @@ def test_sink_filter_to_dict(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_sink_filter___repr__(): + from google.cloud.bigtable.row_filters import SinkFilter + + flag = True + row_filter = SinkFilter(flag) + assert repr(row_filter) == "SinkFilter(flag={})".format(flag) + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_pass_all_filter_to_pb(): from google.cloud.bigtable.row_filters import PassAllFilter @@ -95,6 +115,16 @@ def test_pass_all_filter_to_dict(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_pass_all_filter___repr__(): + from google.cloud.bigtable.row_filters import PassAllFilter + + flag = True + row_filter = PassAllFilter(flag) + assert repr(row_filter) == "PassAllFilter(flag={})".format(flag) + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_block_all_filter_to_pb(): from google.cloud.bigtable.row_filters import BlockAllFilter @@ -117,6 +147,16 @@ def test_block_all_filter_to_dict(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_block_all_filter___repr__(): + from google.cloud.bigtable.row_filters import BlockAllFilter + + flag = True + row_filter = BlockAllFilter(flag) + assert repr(row_filter) == "BlockAllFilter(flag={})".format(flag) + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_regex_filterconstructor(): from google.cloud.bigtable.row_filters import _RegexFilter @@ -160,6 +200,16 @@ def test_regex_filter__ne__same_value(): assert not (row_filter1 != row_filter2) +def test_regex_filter___repr__(): + from google.cloud.bigtable.row_filters import _RegexFilter + + regex = b"abc" + row_filter = _RegexFilter(regex) + assert repr(row_filter) == "_RegexFilter(regex={})".format(regex) + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_row_key_regex_filter_to_pb(): from google.cloud.bigtable.row_filters import RowKeyRegexFilter @@ -182,6 +232,16 @@ def test_row_key_regex_filter_to_dict(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_row_key_regex_filter___repr__(): + from google.cloud.bigtable.row_filters import RowKeyRegexFilter + + regex = b"row-key-regex" + row_filter = RowKeyRegexFilter(regex) + assert repr(row_filter) == "RowKeyRegexFilter(regex={})".format(regex) + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_row_sample_filter_constructor(): from google.cloud.bigtable.row_filters import RowSampleFilter @@ -228,6 +288,16 @@ def test_row_sample_filter_to_pb(): assert pb_val == expected_pb +def test_row_sample_filter___repr__(): + from google.cloud.bigtable.row_filters import RowSampleFilter + + sample = 0.25 + row_filter = RowSampleFilter(sample) + assert repr(row_filter) == "RowSampleFilter(sample={})".format(sample) + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_family_name_regex_filter_to_pb(): from google.cloud.bigtable.row_filters import FamilyNameRegexFilter @@ -250,6 +320,17 @@ def test_family_name_regex_filter_to_dict(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_family_name_regex_filter___repr__(): + from google.cloud.bigtable.row_filters import FamilyNameRegexFilter + + regex = "family-regex" + row_filter = FamilyNameRegexFilter(regex) + expected = "FamilyNameRegexFilter(regex=b'family-regex')" + assert repr(row_filter) == expected + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_column_qualifier_regex_filter_to_pb(): from google.cloud.bigtable.row_filters import ColumnQualifierRegexFilter @@ -272,6 +353,16 @@ def test_column_qualifier_regex_filter_to_dict(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_column_qualifier_regex_filter___repr__(): + from google.cloud.bigtable.row_filters import ColumnQualifierRegexFilter + + regex = b"column-regex" + row_filter = ColumnQualifierRegexFilter(regex) + assert repr(row_filter) == "ColumnQualifierRegexFilter(regex={})".format(regex) + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_timestamp_range_constructor(): from google.cloud.bigtable.row_filters import TimestampRange @@ -403,6 +494,17 @@ def test_timestamp_range_to_dict_end_only(): assert data_v2_pb2.TimestampRange(**expected_dict) == expected_pb_value +def timestamp_range___repr__(): + from google.cloud.bigtable.row_filters import TimestampRange + + start = object() + end = object() + time_range = TimestampRange(start=start, end=end) + assert repr(time_range) == "TimestampRange(start={}, end={})".format(start, end) + assert repr(time_range) == str(time_range) + assert eval(repr(time_range)) == time_range + + def test_timestamp_range_filter___eq__type_differ(): from google.cloud.bigtable.row_filters import TimestampRangeFilter @@ -470,6 +572,21 @@ def test_timestamp_range_filter_empty_to_dict(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_timestamp_range_filter___repr__(): + from google.cloud.bigtable.row_filters import TimestampRangeFilter + import datetime + + start = datetime.datetime(2019, 1, 1) + end = datetime.datetime(2019, 1, 2) + row_filter = TimestampRangeFilter(start, end) + assert ( + repr(row_filter) + == f"TimestampRangeFilter(start={repr(start)}, end={repr(end)})" + ) + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_column_range_filter_constructor_defaults(): from google.cloud.bigtable.row_filters import ColumnRangeFilter @@ -648,6 +765,19 @@ def test_column_range_filter_to_pb_exclusive_end(): assert row_filter._to_pb() == expected_pb +def test_column_range_filter___repr__(): + from google.cloud.bigtable.row_filters import ColumnRangeFilter + + family_id = "column-family-id" + start_qualifier = b"column" + end_qualifier = b"column2" + row_filter = ColumnRangeFilter(family_id, start_qualifier, end_qualifier) + expected = "ColumnRangeFilter(family_id='column-family-id', start_qualifier=b'column', end_qualifier=b'column2', inclusive_start=True, inclusive_end=True)" + assert repr(row_filter) == expected + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_value_regex_filter_to_pb_w_bytes(): from google.cloud.bigtable.row_filters import ValueRegexFilter @@ -694,6 +824,17 @@ def test_value_regex_filter_to_dict_w_str(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_value_regex_filter___repr__(): + from google.cloud.bigtable.row_filters import ValueRegexFilter + + value = "value-regex" + row_filter = ValueRegexFilter(value) + expected = "ValueRegexFilter(regex=b'value-regex')" + assert repr(row_filter) == expected + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_exact_value_filter_to_pb_w_bytes(): from google.cloud.bigtable.row_filters import ExactValueFilter @@ -766,6 +907,17 @@ def test_exact_value_filter_to_dict_w_int(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_exact_value_filter___repr__(): + from google.cloud.bigtable.row_filters import ExactValueFilter + + value = "value-regex" + row_filter = ExactValueFilter(value) + expected = "ExactValueFilter(value=b'value-regex')" + assert repr(row_filter) == expected + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_value_range_filter_constructor_defaults(): from google.cloud.bigtable.row_filters import ValueRangeFilter @@ -942,6 +1094,20 @@ def test_value_range_filter_to_pb_exclusive_end(): assert row_filter._to_pb() == expected_pb +def test_value_range_filter___repr__(): + from google.cloud.bigtable.row_filters import ValueRangeFilter + + start_value = b"some-value" + end_value = b"some-other-value" + row_filter = ValueRangeFilter( + start_value=start_value, end_value=end_value, inclusive_end=False + ) + expected = "ValueRangeFilter(start_value=b'some-value', end_value=b'some-other-value', inclusive_start=True, inclusive_end=False)" + assert repr(row_filter) == expected + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_cell_count_constructor(): from google.cloud.bigtable.row_filters import _CellCountFilter @@ -977,6 +1143,16 @@ def test_cell_count___ne__same_value(): assert not (row_filter1 != row_filter2) +def test_cell_count___repr__(): + from google.cloud.bigtable.row_filters import _CellCountFilter + + row_filter = _CellCountFilter(10) + expected = "_CellCountFilter(num_cells=10)" + assert repr(row_filter) == expected + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_cells_row_offset_filter_to_pb(): from google.cloud.bigtable.row_filters import CellsRowOffsetFilter @@ -999,6 +1175,17 @@ def test_cells_row_offset_filter_to_dict(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_cells_row_offset_filter___repr__(): + from google.cloud.bigtable.row_filters import CellsRowOffsetFilter + + num_cells = 76 + row_filter = CellsRowOffsetFilter(num_cells) + expected = "CellsRowOffsetFilter(num_cells={})".format(num_cells) + assert repr(row_filter) == expected + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_cells_row_limit_filter_to_pb(): from google.cloud.bigtable.row_filters import CellsRowLimitFilter @@ -1021,6 +1208,17 @@ def test_cells_row_limit_filter_to_dict(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_cells_row_limit_filter___repr__(): + from google.cloud.bigtable.row_filters import CellsRowLimitFilter + + num_cells = 189 + row_filter = CellsRowLimitFilter(num_cells) + expected = "CellsRowLimitFilter(num_cells={})".format(num_cells) + assert repr(row_filter) == expected + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_cells_column_limit_filter_to_pb(): from google.cloud.bigtable.row_filters import CellsColumnLimitFilter @@ -1043,6 +1241,17 @@ def test_cells_column_limit_filter_to_dict(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_cells_column_limit_filter___repr__(): + from google.cloud.bigtable.row_filters import CellsColumnLimitFilter + + num_cells = 10 + row_filter = CellsColumnLimitFilter(num_cells) + expected = "CellsColumnLimitFilter(num_cells={})".format(num_cells) + assert repr(row_filter) == expected + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_strip_value_transformer_filter_to_pb(): from google.cloud.bigtable.row_filters import StripValueTransformerFilter @@ -1065,6 +1274,17 @@ def test_strip_value_transformer_filter_to_dict(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_strip_value_transformer_filter___repr__(): + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + + flag = True + row_filter = StripValueTransformerFilter(flag) + expected = "StripValueTransformerFilter(flag={})".format(flag) + assert repr(row_filter) == expected + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_apply_label_filter_constructor(): from google.cloud.bigtable.row_filters import ApplyLabelFilter @@ -1123,6 +1343,17 @@ def test_apply_label_filter_to_dict(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_apply_label_filter___repr__(): + from google.cloud.bigtable.row_filters import ApplyLabelFilter + + label = "label" + row_filter = ApplyLabelFilter(label) + expected = "ApplyLabelFilter(label={})".format(label) + assert repr(row_filter) == expected + assert repr(row_filter) == str(row_filter) + assert eval(repr(row_filter)) == row_filter + + def test_filter_combination_constructor_defaults(): from google.cloud.bigtable.row_filters import _FilterCombination @@ -1166,6 +1397,64 @@ def test_filter_combination___ne__(): assert row_filter1 != row_filter2 +def test_filter_combination_len(): + from google.cloud.bigtable.row_filters import _FilterCombination + + filters = [object(), object()] + row_filter = _FilterCombination(filters=filters) + assert len(row_filter) == len(filters) + + +def test_filter_combination_iter(): + from google.cloud.bigtable.row_filters import _FilterCombination + + filters = [object(), object()] + row_filter = _FilterCombination(filters=filters) + assert list(iter(row_filter)) == filters + for filter_, expected in zip(row_filter, filters): + assert filter_ is expected + + +def test_filter_combination___getitem__(): + from google.cloud.bigtable.row_filters import _FilterCombination + + filters = [object(), object()] + row_filter = _FilterCombination(filters=filters) + row_filter[0] is filters[0] + row_filter[1] is filters[1] + with pytest.raises(IndexError): + row_filter[2] + row_filter[:] is filters[:] + + +def test_filter_combination___repr__(): + from google.cloud.bigtable.row_filters import _FilterCombination + from google.cloud.bigtable.row_filters import BlockAllFilter + + filters = [BlockAllFilter(False), BlockAllFilter(True)] + row_filter = _FilterCombination(filters=filters) + expected = "_FilterCombination(filters={})".format(filters) + assert repr(row_filter) == expected + assert eval(repr(row_filter)) == row_filter + + +def test_filter_combination___str__(): + from google.cloud.bigtable.row_filters import _FilterCombination + from google.cloud.bigtable.row_filters import PassAllFilter + from google.cloud.bigtable.row_filters import RowKeyRegexFilter + + filters = [PassAllFilter(True), PassAllFilter(False)] + row_filter = _FilterCombination(filters=filters) + expected = "_FilterCombination([\n PassAllFilter(flag=True),\n PassAllFilter(flag=False),\n])" + assert str(row_filter) == expected + # test with nesting + filters = [PassAllFilter(True), _FilterCombination(filters=filters)] + filters = [RowKeyRegexFilter("foo"), _FilterCombination(filters=filters)] + row_filter = _FilterCombination(filters=filters) + expected = "_FilterCombination([\n RowKeyRegexFilter(regex=b'foo'),\n _FilterCombination([\n PassAllFilter(flag=True),\n _FilterCombination([\n PassAllFilter(flag=True),\n PassAllFilter(flag=False),\n ]),\n ]),\n])" + assert str(row_filter) == expected + + def test_row_filter_chain_to_pb(): from google.cloud.bigtable.row_filters import RowFilterChain from google.cloud.bigtable.row_filters import RowSampleFilter @@ -1257,6 +1546,37 @@ def test_row_filter_chain_to_dict_nested(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_row_filter_chain___repr__(): + from google.cloud.bigtable.row_filters import RowFilterChain + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) + + row_filter3 = RowFilterChain(filters=[row_filter1, row_filter2]) + expected = f"RowFilterChain(filters={[row_filter1, row_filter2]})" + assert repr(row_filter3) == expected + assert eval(repr(row_filter3)) == row_filter3 + + +def test_row_filter_chain___str__(): + from google.cloud.bigtable.row_filters import RowFilterChain + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) + + row_filter3 = RowFilterChain(filters=[row_filter1, row_filter2]) + expected = "RowFilterChain([\n StripValueTransformerFilter(flag=True),\n RowSampleFilter(sample=0.25),\n])" + assert str(row_filter3) == expected + # test nested + row_filter4 = RowFilterChain(filters=[row_filter3]) + expected = "RowFilterChain([\n RowFilterChain([\n StripValueTransformerFilter(flag=True),\n RowSampleFilter(sample=0.25),\n ]),\n])" + assert str(row_filter4) == expected + + def test_row_filter_union_to_pb(): from google.cloud.bigtable.row_filters import RowFilterUnion from google.cloud.bigtable.row_filters import RowSampleFilter @@ -1348,6 +1668,37 @@ def test_row_filter_union_to_dict_nested(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_row_filter_union___repr__(): + from google.cloud.bigtable.row_filters import RowFilterUnion + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) + + row_filter3 = RowFilterUnion(filters=[row_filter1, row_filter2]) + expected = "RowFilterUnion(filters=[StripValueTransformerFilter(flag=True), RowSampleFilter(sample=0.25)])" + assert repr(row_filter3) == expected + assert eval(repr(row_filter3)) == row_filter3 + + +def test_row_filter_union___str__(): + from google.cloud.bigtable.row_filters import RowFilterUnion + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) + + row_filter3 = RowFilterUnion(filters=[row_filter1, row_filter2]) + expected = "RowFilterUnion([\n StripValueTransformerFilter(flag=True),\n RowSampleFilter(sample=0.25),\n])" + assert str(row_filter3) == expected + # test nested + row_filter4 = RowFilterUnion(filters=[row_filter3]) + expected = "RowFilterUnion([\n RowFilterUnion([\n StripValueTransformerFilter(flag=True),\n RowSampleFilter(sample=0.25),\n ]),\n])" + assert str(row_filter4) == expected + + def test_conditional_row_filter_constructor(): from google.cloud.bigtable.row_filters import ConditionalRowFilter @@ -1565,6 +1916,48 @@ def test_conditional_row_filter_to_dict_false_only(): assert data_v2_pb2.RowFilter(**expected_dict) == expected_pb_value +def test_conditional_row_filter___repr__(): + from google.cloud.bigtable.row_filters import ConditionalRowFilter + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) + row_filter3 = ConditionalRowFilter(row_filter1, true_filter=row_filter2) + expected = ( + "ConditionalRowFilter(predicate_filter=StripValueTransformerFilter(" + "flag=True), true_filter=RowSampleFilter(sample=0.25), false_filter=None)" + ) + assert repr(row_filter3) == expected + assert eval(repr(row_filter3)) == row_filter3 + # test nested + row_filter4 = ConditionalRowFilter(row_filter3, true_filter=row_filter2) + expected = "ConditionalRowFilter(predicate_filter=ConditionalRowFilter(predicate_filter=StripValueTransformerFilter(flag=True), true_filter=RowSampleFilter(sample=0.25), false_filter=None), true_filter=RowSampleFilter(sample=0.25), false_filter=None)" + assert repr(row_filter4) == expected + assert eval(repr(row_filter4)) == row_filter4 + + +def test_conditional_row_filter___str__(): + from google.cloud.bigtable.row_filters import ConditionalRowFilter + from google.cloud.bigtable.row_filters import RowSampleFilter + from google.cloud.bigtable.row_filters import RowFilterUnion + from google.cloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) + row_filter3 = ConditionalRowFilter(row_filter1, true_filter=row_filter2) + expected = "ConditionalRowFilter(\n predicate_filter=StripValueTransformerFilter(flag=True),\n true_filter=RowSampleFilter(sample=0.25),\n)" + assert str(row_filter3) == expected + # test nested + row_filter4 = ConditionalRowFilter( + row_filter3, + true_filter=row_filter2, + false_filter=RowFilterUnion([row_filter1, row_filter2]), + ) + expected = "ConditionalRowFilter(\n predicate_filter=ConditionalRowFilter(\n predicate_filter=StripValueTransformerFilter(flag=True),\n true_filter=RowSampleFilter(sample=0.25),\n ),\n true_filter=RowSampleFilter(sample=0.25),\n false_filter=RowFilterUnion([\n StripValueTransformerFilter(flag=True),\n RowSampleFilter(sample=0.25),\n ]),\n)" + assert str(row_filter4) == expected + + def _ColumnRangePB(*args, **kw): from google.cloud.bigtable_v2.types import data as data_v2_pb2 From d60ec7fbcc346685579b98b5860cd478a61517d5 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 22 Mar 2023 11:11:20 -0700 Subject: [PATCH 13/16] update date in test file header Co-authored-by: Mariatta Wijaya --- tests/unit/test_row_filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_row_filters.py b/tests/unit/test_row_filters.py index 0bbe46c81..d7f1f1dae 100644 --- a/tests/unit/test_row_filters.py +++ b/tests/unit/test_row_filters.py @@ -1,4 +1,4 @@ -# Copyright 2016 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From a4ede76bb4b5c33f344c97be472e7fd6cdc2fe54 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 22 Mar 2023 14:07:44 -0700 Subject: [PATCH 14/16] made abstract classes for filters --- google/cloud/bigtable/row_filters.py | 13 +- tests/unit/test_row_filters.py | 345 +++++++++++++-------------- 2 files changed, 176 insertions(+), 182 deletions(-) diff --git a/google/cloud/bigtable/row_filters.py b/google/cloud/bigtable/row_filters.py index 65c33d4f5..6df7e7a9b 100644 --- a/google/cloud/bigtable/row_filters.py +++ b/google/cloud/bigtable/row_filters.py @@ -18,6 +18,7 @@ import struct from typing import Any, Sequence, TYPE_CHECKING, overload +from abc import ABC, abstractmethod from google.cloud._helpers import _microseconds_from_datetime # type: ignore from google.cloud._helpers import _to_bytes # type: ignore @@ -30,7 +31,7 @@ _PACK_I64 = struct.Struct(">q").pack -class RowFilter(object): +class RowFilter(ABC): """Basic filter to apply to cells in a row. These values can be combined via :class:`RowFilterChain`, @@ -48,16 +49,16 @@ def _to_pb(self) -> data_v2_pb2.RowFilter: """ return data_v2_pb2.RowFilter(**self.to_dict()) + @abstractmethod def to_dict(self) -> dict[str, Any]: """Converts the row filter to a dict representation.""" - # unimplemented on base class - raise NotImplementedError + pass def __repr__(self) -> str: return f"{self.__class__.__name__}()" -class _BoolFilter(RowFilter): +class _BoolFilter(RowFilter, ABC): """Row filter that uses a boolean flag. :type flag: bool @@ -122,7 +123,7 @@ def to_dict(self) -> dict[str, Any]: return {"block_all_filter": self.flag} -class _RegexFilter(RowFilter): +class _RegexFilter(RowFilter, ABC): """Row filter that uses a regular expression. The ``regex`` must be valid RE2 patterns. See Google's @@ -726,7 +727,7 @@ def __repr__(self) -> str: return f"{self.__class__.__name__}(label={self.label})" -class _FilterCombination(RowFilter, Sequence[RowFilter]): +class _FilterCombination(RowFilter, Sequence[RowFilter], ABC): """Chain of row filters. Sends rows through several filters in sequence. The filters are "chained" diff --git a/tests/unit/test_row_filters.py b/tests/unit/test_row_filters.py index d7f1f1dae..67160de2f 100644 --- a/tests/unit/test_row_filters.py +++ b/tests/unit/test_row_filters.py @@ -17,48 +17,34 @@ def test_bool_filter_constructor(): - from google.cloud.bigtable.row_filters import _BoolFilter - - flag = object() - row_filter = _BoolFilter(flag) - assert row_filter.flag is flag + for FilterType in _get_bool_filters(): + flag = True + row_filter = FilterType(flag) + assert row_filter.flag is flag def test_bool_filter___eq__type_differ(): - from google.cloud.bigtable.row_filters import _BoolFilter - - flag = object() - row_filter1 = _BoolFilter(flag) - row_filter2 = object() - assert not (row_filter1 == row_filter2) + for FilterType in _get_bool_filters(): + flag = object() + row_filter1 = FilterType(flag) + row_filter2 = object() + assert not (row_filter1 == row_filter2) def test_bool_filter___eq__same_value(): - from google.cloud.bigtable.row_filters import _BoolFilter - - flag = object() - row_filter1 = _BoolFilter(flag) - row_filter2 = _BoolFilter(flag) - assert row_filter1 == row_filter2 + for FilterType in _get_bool_filters(): + flag = object() + row_filter1 = FilterType(flag) + row_filter2 = FilterType(flag) + assert row_filter1 == row_filter2 def test_bool_filter___ne__same_value(): - from google.cloud.bigtable.row_filters import _BoolFilter - - flag = object() - row_filter1 = _BoolFilter(flag) - row_filter2 = _BoolFilter(flag) - assert not (row_filter1 != row_filter2) - - -def test_bool_filter___repr__(): - from google.cloud.bigtable.row_filters import _BoolFilter - - flag = True - row_filter = _BoolFilter(flag) - assert repr(row_filter) == "_BoolFilter(flag={})".format(flag) - assert repr(row_filter) == str(row_filter) - assert eval(repr(row_filter)) == row_filter + for FilterType in _get_bool_filters(): + flag = object() + row_filter1 = FilterType(flag) + row_filter2 = FilterType(flag) + assert not (row_filter1 != row_filter2) def test_sink_filter_to_pb(): @@ -158,56 +144,41 @@ def test_block_all_filter___repr__(): def test_regex_filterconstructor(): - from google.cloud.bigtable.row_filters import _RegexFilter - - regex = b"abc" - row_filter = _RegexFilter(regex) - assert row_filter.regex is regex + for FilterType in _get_regex_filters(): + regex = b"abc" + row_filter = FilterType(regex) + assert row_filter.regex == regex def test_regex_filterconstructor_non_bytes(): - from google.cloud.bigtable.row_filters import _RegexFilter - - regex = "abc" - row_filter = _RegexFilter(regex) - assert row_filter.regex == b"abc" + for FilterType in _get_regex_filters(): + regex = "abc" + row_filter = FilterType(regex) + assert row_filter.regex == b"abc" def test_regex_filter__eq__type_differ(): - from google.cloud.bigtable.row_filters import _RegexFilter - - regex = b"def-rgx" - row_filter1 = _RegexFilter(regex) - row_filter2 = object() - assert not (row_filter1 == row_filter2) + for FilterType in _get_regex_filters(): + regex = b"def-rgx" + row_filter1 = FilterType(regex) + row_filter2 = object() + assert not (row_filter1 == row_filter2) def test_regex_filter__eq__same_value(): - from google.cloud.bigtable.row_filters import _RegexFilter - - regex = b"trex-regex" - row_filter1 = _RegexFilter(regex) - row_filter2 = _RegexFilter(regex) - assert row_filter1 == row_filter2 + for FilterType in _get_regex_filters(): + regex = b"trex-regex" + row_filter1 = FilterType(regex) + row_filter2 = FilterType(regex) + assert row_filter1 == row_filter2 def test_regex_filter__ne__same_value(): - from google.cloud.bigtable.row_filters import _RegexFilter - - regex = b"abc" - row_filter1 = _RegexFilter(regex) - row_filter2 = _RegexFilter(regex) - assert not (row_filter1 != row_filter2) - - -def test_regex_filter___repr__(): - from google.cloud.bigtable.row_filters import _RegexFilter - - regex = b"abc" - row_filter = _RegexFilter(regex) - assert repr(row_filter) == "_RegexFilter(regex={})".format(regex) - assert repr(row_filter) == str(row_filter) - assert eval(repr(row_filter)) == row_filter + for FilterType in _get_regex_filters(): + regex = b"abc" + row_filter1 = FilterType(regex) + row_filter2 = FilterType(regex) + assert not (row_filter1 != row_filter2) def test_row_key_regex_filter_to_pb(): @@ -1109,48 +1080,34 @@ def test_value_range_filter___repr__(): def test_cell_count_constructor(): - from google.cloud.bigtable.row_filters import _CellCountFilter - - num_cells = object() - row_filter = _CellCountFilter(num_cells) - assert row_filter.num_cells is num_cells + for FilerType in _get_cell_count_filters(): + num_cells = object() + row_filter = FilerType(num_cells) + assert row_filter.num_cells is num_cells def test_cell_count___eq__type_differ(): - from google.cloud.bigtable.row_filters import _CellCountFilter - - num_cells = object() - row_filter1 = _CellCountFilter(num_cells) - row_filter2 = object() - assert not (row_filter1 == row_filter2) + for FilerType in _get_cell_count_filters(): + num_cells = object() + row_filter1 = FilerType(num_cells) + row_filter2 = object() + assert not (row_filter1 == row_filter2) def test_cell_count___eq__same_value(): - from google.cloud.bigtable.row_filters import _CellCountFilter - - num_cells = object() - row_filter1 = _CellCountFilter(num_cells) - row_filter2 = _CellCountFilter(num_cells) - assert row_filter1 == row_filter2 + for FilerType in _get_cell_count_filters(): + num_cells = object() + row_filter1 = FilerType(num_cells) + row_filter2 = FilerType(num_cells) + assert row_filter1 == row_filter2 def test_cell_count___ne__same_value(): - from google.cloud.bigtable.row_filters import _CellCountFilter - - num_cells = object() - row_filter1 = _CellCountFilter(num_cells) - row_filter2 = _CellCountFilter(num_cells) - assert not (row_filter1 != row_filter2) - - -def test_cell_count___repr__(): - from google.cloud.bigtable.row_filters import _CellCountFilter - - row_filter = _CellCountFilter(10) - expected = "_CellCountFilter(num_cells=10)" - assert repr(row_filter) == expected - assert repr(row_filter) == str(row_filter) - assert eval(repr(row_filter)) == row_filter + for FilerType in _get_cell_count_filters(): + num_cells = object() + row_filter1 = FilerType(num_cells) + row_filter2 = FilerType(num_cells) + assert not (row_filter1 != row_filter2) def test_cells_row_offset_filter_to_pb(): @@ -1355,104 +1312,80 @@ def test_apply_label_filter___repr__(): def test_filter_combination_constructor_defaults(): - from google.cloud.bigtable.row_filters import _FilterCombination - - row_filter = _FilterCombination() - assert row_filter.filters == [] + for FilterType in _get_filter_combination_filters(): + row_filter = FilterType() + assert row_filter.filters == [] def test_filter_combination_constructor_explicit(): - from google.cloud.bigtable.row_filters import _FilterCombination - - filters = object() - row_filter = _FilterCombination(filters=filters) - assert row_filter.filters is filters + for FilterType in _get_filter_combination_filters(): + filters = object() + row_filter = FilterType(filters=filters) + assert row_filter.filters is filters def test_filter_combination___eq__(): - from google.cloud.bigtable.row_filters import _FilterCombination - - filters = object() - row_filter1 = _FilterCombination(filters=filters) - row_filter2 = _FilterCombination(filters=filters) - assert row_filter1 == row_filter2 + for FilterType in _get_filter_combination_filters(): + filters = object() + row_filter1 = FilterType(filters=filters) + row_filter2 = FilterType(filters=filters) + assert row_filter1 == row_filter2 def test_filter_combination___eq__type_differ(): - from google.cloud.bigtable.row_filters import _FilterCombination - - filters = object() - row_filter1 = _FilterCombination(filters=filters) - row_filter2 = object() - assert not (row_filter1 == row_filter2) + for FilterType in _get_filter_combination_filters(): + filters = object() + row_filter1 = FilterType(filters=filters) + row_filter2 = object() + assert not (row_filter1 == row_filter2) def test_filter_combination___ne__(): - from google.cloud.bigtable.row_filters import _FilterCombination - - filters = object() - other_filters = object() - row_filter1 = _FilterCombination(filters=filters) - row_filter2 = _FilterCombination(filters=other_filters) - assert row_filter1 != row_filter2 + for FilterType in _get_filter_combination_filters(): + filters = object() + other_filters = object() + row_filter1 = FilterType(filters=filters) + row_filter2 = FilterType(filters=other_filters) + assert row_filter1 != row_filter2 def test_filter_combination_len(): - from google.cloud.bigtable.row_filters import _FilterCombination - - filters = [object(), object()] - row_filter = _FilterCombination(filters=filters) - assert len(row_filter) == len(filters) + for FilterType in _get_filter_combination_filters(): + filters = [object(), object()] + row_filter = FilterType(filters=filters) + assert len(row_filter) == len(filters) def test_filter_combination_iter(): - from google.cloud.bigtable.row_filters import _FilterCombination - - filters = [object(), object()] - row_filter = _FilterCombination(filters=filters) - assert list(iter(row_filter)) == filters - for filter_, expected in zip(row_filter, filters): - assert filter_ is expected + for FilterType in _get_filter_combination_filters(): + filters = [object(), object()] + row_filter = FilterType(filters=filters) + assert list(iter(row_filter)) == filters + for filter_, expected in zip(row_filter, filters): + assert filter_ is expected def test_filter_combination___getitem__(): - from google.cloud.bigtable.row_filters import _FilterCombination - - filters = [object(), object()] - row_filter = _FilterCombination(filters=filters) - row_filter[0] is filters[0] - row_filter[1] is filters[1] - with pytest.raises(IndexError): - row_filter[2] - row_filter[:] is filters[:] - - -def test_filter_combination___repr__(): - from google.cloud.bigtable.row_filters import _FilterCombination - from google.cloud.bigtable.row_filters import BlockAllFilter - - filters = [BlockAllFilter(False), BlockAllFilter(True)] - row_filter = _FilterCombination(filters=filters) - expected = "_FilterCombination(filters={})".format(filters) - assert repr(row_filter) == expected - assert eval(repr(row_filter)) == row_filter + for FilterType in _get_filter_combination_filters(): + filters = [object(), object()] + row_filter = FilterType(filters=filters) + row_filter[0] is filters[0] + row_filter[1] is filters[1] + with pytest.raises(IndexError): + row_filter[2] + row_filter[:] is filters[:] def test_filter_combination___str__(): - from google.cloud.bigtable.row_filters import _FilterCombination from google.cloud.bigtable.row_filters import PassAllFilter - from google.cloud.bigtable.row_filters import RowKeyRegexFilter - filters = [PassAllFilter(True), PassAllFilter(False)] - row_filter = _FilterCombination(filters=filters) - expected = "_FilterCombination([\n PassAllFilter(flag=True),\n PassAllFilter(flag=False),\n])" - assert str(row_filter) == expected - # test with nesting - filters = [PassAllFilter(True), _FilterCombination(filters=filters)] - filters = [RowKeyRegexFilter("foo"), _FilterCombination(filters=filters)] - row_filter = _FilterCombination(filters=filters) - expected = "_FilterCombination([\n RowKeyRegexFilter(regex=b'foo'),\n _FilterCombination([\n PassAllFilter(flag=True),\n _FilterCombination([\n PassAllFilter(flag=True),\n PassAllFilter(flag=False),\n ]),\n ]),\n])" - assert str(row_filter) == expected + for FilterType in _get_filter_combination_filters(): + filters = [PassAllFilter(True), PassAllFilter(False)] + row_filter = FilterType(filters=filters) + expected = ( + "([\n PassAllFilter(flag=True),\n PassAllFilter(flag=False),\n])" + ) + assert expected in str(row_filter) def test_row_filter_chain_to_pb(): @@ -1998,3 +1931,63 @@ def _ValueRangePB(*args, **kw): from google.cloud.bigtable_v2.types import data as data_v2_pb2 return data_v2_pb2.ValueRange(*args, **kw) + + +def _get_regex_filters(): + from google.cloud.bigtable.row_filters import ( + RowKeyRegexFilter, + FamilyNameRegexFilter, + ColumnQualifierRegexFilter, + ValueRegexFilter, + ExactValueFilter, + ) + + return [ + RowKeyRegexFilter, + FamilyNameRegexFilter, + ColumnQualifierRegexFilter, + ValueRegexFilter, + ExactValueFilter, + ] + + +def _get_bool_filters(): + from google.cloud.bigtable.row_filters import ( + SinkFilter, + PassAllFilter, + BlockAllFilter, + StripValueTransformerFilter, + ) + + return [ + SinkFilter, + PassAllFilter, + BlockAllFilter, + StripValueTransformerFilter, + ] + + +def _get_cell_count_filters(): + from google.cloud.bigtable.row_filters import ( + CellsRowLimitFilter, + CellsRowOffsetFilter, + CellsColumnLimitFilter, + ) + + return [ + CellsRowLimitFilter, + CellsRowOffsetFilter, + CellsColumnLimitFilter, + ] + + +def _get_filter_combination_filters(): + from google.cloud.bigtable.row_filters import ( + RowFilterChain, + RowFilterUnion, + ) + + return [ + RowFilterChain, + RowFilterUnion, + ] From 0e8d3b169aa781d6f827fc50018f08cb7db195ec Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 22 Mar 2023 14:11:43 -0700 Subject: [PATCH 15/16] test trying to construct abstract classes --- tests/unit/test_row_filters.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unit/test_row_filters.py b/tests/unit/test_row_filters.py index 67160de2f..d0fbad42f 100644 --- a/tests/unit/test_row_filters.py +++ b/tests/unit/test_row_filters.py @@ -16,6 +16,22 @@ import pytest +def test_abstract_class_constructors(): + from google.cloud.bigtable.row_filters import RowFilter + from google.cloud.bigtable.row_filters import _BoolFilter + from google.cloud.bigtable.row_filters import _FilterCombination + from google.cloud.bigtable.row_filters import _CellCountFilter + + with pytest.raises(TypeError): + RowFilter() + with pytest.raises(TypeError): + _BoolFilter(False) + with pytest.raises(TypeError): + _FilterCombination([]) + with pytest.raises(TypeError): + _CellCountFilter(0) + + def test_bool_filter_constructor(): for FilterType in _get_bool_filters(): flag = True From 8f30d8e80aeec5576a9626e3585a2296c8090827 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 22 Mar 2023 14:18:17 -0700 Subject: [PATCH 16/16] made cellcountfilter abc --- google/cloud/bigtable/row_filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/bigtable/row_filters.py b/google/cloud/bigtable/row_filters.py index 6df7e7a9b..48a1d4d8a 100644 --- a/google/cloud/bigtable/row_filters.py +++ b/google/cloud/bigtable/row_filters.py @@ -612,7 +612,7 @@ def __repr__(self) -> str: return f"{self.__class__.__name__}(start_value={self.start_value!r}, end_value={self.end_value!r}, inclusive_start={self.inclusive_start}, inclusive_end={self.inclusive_end})" -class _CellCountFilter(RowFilter): +class _CellCountFilter(RowFilter, ABC): """Row filter that uses an integer count of cells. The cell count is used as an offset or a limit for the number