diff --git a/.coveragerc b/.coveragerc index 77dd56b8e2e8..240882dc7d07 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,7 +1,5 @@ [report] omit = - */demo/* - */demo.py */_generated/*.py exclude_lines = # Re-enable the standard pragma diff --git a/docs/bigquery-usage.rst b/docs/bigquery-usage.rst index bb3e68d90060..f84bf638f3cb 100644 --- a/docs/bigquery-usage.rst +++ b/docs/bigquery-usage.rst @@ -220,8 +220,8 @@ Update all writable metadata for a table >>> dataset = client.dataset('dataset_name') >>> table = dataset.table(name='person_ages') >>> table.schema = [ - ... SchemaField(name='full_name', type='string', mode='required'), - ... SchemaField(name='age', type='int', mode='required)] + ... SchemaField('full_name', 'STRING', mode='required'), + ... SchemaField('age', 'INTEGER', mode='required)] >>> table.update() # API request Upload table data from a file: @@ -233,8 +233,8 @@ Upload table data from a file: >>> dataset = client.dataset('dataset_name') >>> table = dataset.table(name='person_ages') >>> table.schema = [ - ... SchemaField(name='full_name', type='string', mode='required'), - ... SchemaField(name='age', type='int', mode='required)] + ... SchemaField('full_name', 'STRING', mode='required'), + ... SchemaField('age', 'INTEGER', mode='required)] >>> with open('person_ages.csv', 'rb') as csv_file: ... table.upload_from_file(csv_file, CSV, ... create_disposition='CREATE_IF_NEEDED') @@ -384,8 +384,8 @@ Load data synchronously from a local CSV file into a new table: >>> client = bigquery.Client() >>> table = dataset.table(name='person_ages') >>> table.schema = [ - ... SchemaField(name='full_name', type='string', mode='required'), - ... SchemaField(name='age', type='int', mode='required)] + ... SchemaField('full_name', 'STRING', mode='required'), + ... SchemaField('age', 'INTEGER', mode='required)] >>> with open('/path/to/person_ages.csv', 'rb') as file_obj: ... reader = csv.reader(file_obj) ... rows = list(reader) @@ -405,8 +405,8 @@ the job locally: >>> client = bigquery.Client() >>> table = dataset.table(name='person_ages') >>> table.schema = [ - ... SchemaField(name='full_name', type='string', mode='required'), - ... SchemaField(name='age', type='int', mode='required)] + ... SchemaField('full_name', 'STRING', mode='required'), + ... SchemaField('age', 'INTEGER', mode='required)] >>> job = client.load_table_from_storage( ... 'load-from-storage-job', table, 'gs://bucket-name/object-prefix*') >>> job.source_format = 'CSV' diff --git a/docs/bigtable-client-intro.rst b/docs/bigtable-client-intro.rst new file mode 100644 index 000000000000..3f43371ec1db --- /dev/null +++ b/docs/bigtable-client-intro.rst @@ -0,0 +1,92 @@ +Base for Everything +=================== + +To use the API, the :class:`Client ` +class defines a high-level interface which handles authorization +and creating other objects: + +.. code:: python + + from gcloud.bigtable.client import Client + client = Client() + +Long-lived Defaults +------------------- + +When creating a :class:`Client `, the +``user_agent`` and ``timeout_seconds`` arguments have sensible +defaults +(:data:`DEFAULT_USER_AGENT ` and +:data:`DEFAULT_TIMEOUT_SECONDS `). +However, you may over-ride them and these will be used throughout all API +requests made with the ``client`` you create. + +Configuration +------------- + +- For an overview of authentication in ``gcloud-python``, + see :doc:`gcloud-auth`. + +- In addition to any authentication configuration, you can also set the + :envvar:`GCLOUD_PROJECT` environment variable for the Google Cloud Console + project you'd like to interact with. If your code is running in Google App + Engine or Google Compute Engine the project will be detected automatically. + (Setting this environment variable is not required, you may instead pass the + ``project`` explicitly when constructing a + :class:`Client `). + +- After configuring your environment, create a + :class:`Client ` + + .. code:: + + >>> from gcloud import bigtable + >>> client = bigtable.Client() + + or pass in ``credentials`` and ``project`` explicitly + + .. code:: + + >>> from gcloud import bigtable + >>> client = bigtable.Client(project='my-project', credentials=creds) + +.. tip:: + + Be sure to use the **Project ID**, not the **Project Number**. + +Admin API Access +---------------- + +If you'll be using your client to make `Cluster Admin`_ and `Table Admin`_ +API requests, you'll need to pass the ``admin`` argument: + +.. code:: python + + client = bigtable.Client(admin=True) + +Read-Only Mode +-------------- + +If on the other hand, you only have (or want) read access to the data, +you can pass the ``read_only`` argument: + +.. code:: python + + client = bigtable.Client(read_only=True) + +This will ensure that the +:data:`READ_ONLY_SCOPE ` is used +for API requests (so any accidental requests that would modify data will +fail). + +Next Step +--------- + +After a :class:`Client `, the next highest-level +object is a :class:`Cluster `. You'll need +one before you can interact with tables or data. + +Head next to learn about the :doc:`bigtable-cluster-api`. + +.. _Cluster Admin: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/tree/master/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1 +.. _Table Admin: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/tree/master/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1 diff --git a/docs/bigtable-client.rst b/docs/bigtable-client.rst new file mode 100644 index 000000000000..b765144a160d --- /dev/null +++ b/docs/bigtable-client.rst @@ -0,0 +1,7 @@ +Client +~~~~~~ + +.. automodule:: gcloud.bigtable.client + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/bigtable-cluster-api.rst b/docs/bigtable-cluster-api.rst new file mode 100644 index 000000000000..17fba96840db --- /dev/null +++ b/docs/bigtable-cluster-api.rst @@ -0,0 +1,181 @@ +Cluster Admin API +================= + +After creating a :class:`Client `, you can +interact with individual clusters, groups of clusters or available +zones for a project. + +List Clusters +------------- + +If you want a comprehensive list of all existing clusters, make a +`ListClusters`_ API request with +:meth:`Client.list_clusters() `: + +.. code:: python + + clusters = client.list_clusters() + +List Zones +---------- + +If you aren't sure which ``zone`` to create a cluster in, find out +which zones your project has access to with a `ListZones`_ API request +with :meth:`Client.list_zones() `: + +.. code:: python + + zones = client.list_zones() + +You can choose a :class:`string ` from among the result to pass to +the :class:`Cluster ` constructor. + +The available zones (as of February 2016) are + +.. code:: python + + >>> zones + [u'asia-east1-b', u'europe-west1-c', u'us-central1-c', u'us-central1-b'] + +Cluster Factory +--------------- + +To create a :class:`Cluster ` object: + +.. code:: python + + cluster = client.cluster(zone, cluster_id, + display_name=display_name, + serve_nodes=3) + +Both ``display_name`` and ``serve_nodes`` are optional. When not provided, +``display_name`` defaults to the ``cluster_id`` value and ``serve_nodes`` +defaults to the minimum allowed: +:data:`DEFAULT_SERVE_NODES `. + +Even if this :class:`Cluster ` already +has been created with the API, you'll want this object to use as a +parent of a :class:`Table ` just as the +:class:`Client ` is used as the parent of +a :class:`Cluster `. + +Create a new Cluster +-------------------- + +After creating the cluster object, make a `CreateCluster`_ API request +with :meth:`create() `: + +.. code:: python + + cluster.display_name = 'My very own cluster' + cluster.create() + +If you would like more than the minimum number of nodes +(:data:`DEFAULT_SERVE_NODES `) +in your cluster: + +.. code:: python + + cluster.serve_nodes = 10 + cluster.create() + +Check on Current Operation +-------------------------- + +.. note:: + + When modifying a cluster (via a `CreateCluster`_, `UpdateCluster`_ or + `UndeleteCluster`_ request), the Bigtable API will return a + `long-running operation`_ and a corresponding + :class:`Operation ` object + will be returned by each of + :meth:`create() `, + :meth:`update() ` and + :meth:`undelete() `. + +You can check if a long-running operation (for a +:meth:`create() `, +:meth:`update() ` or +:meth:`undelete() `) has finished +by making a `GetOperation`_ request with +:meth:`Operation.finished() `: + +.. code:: python + + >>> operation = cluster.create() + >>> operation.finished() + True + +.. note:: + + Once an :class:`Operation ` object + has returned :data:`True` from + :meth:`finished() `, the + object should not be re-used. Subsequent calls to + :meth:`finished() ` + will result in a :class:`ValueError `. + +Get metadata for an existing Cluster +------------------------------------ + +After creating the cluster object, make a `GetCluster`_ API request +with :meth:`reload() `: + +.. code:: python + + cluster.reload() + +This will load ``serve_nodes`` and ``display_name`` for the existing +``cluster`` in addition to the ``cluster_id``, ``zone`` and ``project`` +already set on the :class:`Cluster ` object. + +Update an existing Cluster +-------------------------- + +After creating the cluster object, make an `UpdateCluster`_ API request +with :meth:`update() `: + +.. code:: python + + client.display_name = 'New display_name' + cluster.update() + +Delete an existing Cluster +-------------------------- + +Make a `DeleteCluster`_ API request with +:meth:`delete() `: + +.. code:: python + + cluster.delete() + +Undelete a deleted Cluster +-------------------------- + +Make an `UndeleteCluster`_ API request with +:meth:`undelete() `: + +.. code:: python + + cluster.undelete() + +Next Step +--------- + +Now we go down the hierarchy from +:class:`Cluster ` to a +:class:`Table `. + +Head next to learn about the :doc:`bigtable-table-api`. + +.. _Cluster Admin API: https://cloud.google.com/bigtable/docs/creating-cluster +.. _CreateCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L66-L68 +.. _GetCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L38-L40 +.. _UpdateCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L93-L95 +.. _DeleteCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L109-L111 +.. _ListZones: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L33-L35 +.. _ListClusters: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L44-L46 +.. _GetOperation: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/longrunning/operations.proto#L43-L45 +.. _UndeleteCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L126-L128 +.. _long-running operation: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/longrunning/operations.proto#L73-L102 diff --git a/docs/bigtable-cluster.rst b/docs/bigtable-cluster.rst new file mode 100644 index 000000000000..9b88f2059d14 --- /dev/null +++ b/docs/bigtable-cluster.rst @@ -0,0 +1,7 @@ +Cluster +~~~~~~~ + +.. automodule:: gcloud.bigtable.cluster + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/bigtable-column-family.rst b/docs/bigtable-column-family.rst new file mode 100644 index 000000000000..b51d7a18e4ee --- /dev/null +++ b/docs/bigtable-column-family.rst @@ -0,0 +1,50 @@ +Column Families +=============== + +When creating a +:class:`ColumnFamily `, it is +possible to set garbage collection rules for expired data. + +By setting a rule, cells in the table matching the rule will be deleted +during periodic garbage collection (which executes opportunistically in the +background). + +The types +:class:`MaxAgeGCRule `, +:class:`MaxVersionsGCRule `, +:class:`GarbageCollectionRuleUnion ` and +:class:`GarbageCollectionRuleIntersection ` +can all be used as the optional ``gc_rule`` argument in the +:class:`ColumnFamily ` +constructor. This value is then used in the +:meth:`create() ` and +:meth:`update() ` methods. + +These rules can be nested arbitrarily, with a +:class:`MaxAgeGCRule ` or +:class:`MaxVersionsGCRule ` +at the lowest level of the nesting: + +.. code:: python + + import datetime + + max_age = datetime.timedelta(days=3) + rule1 = MaxAgeGCRule(max_age) + rule2 = MaxVersionsGCRule(1) + + # Make a composite that matches anything older than 3 days **AND** + # with more than 1 version. + rule3 = GarbageCollectionIntersection(rules=[rule1, rule2]) + + # Make another composite that matches our previous intersection + # **OR** anything that has more than 3 versions. + rule4 = GarbageCollectionRule(max_num_versions=3) + rule5 = GarbageCollectionUnion(rules=[rule3, rule4]) + +---- + +.. automodule:: gcloud.bigtable.column_family + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/bigtable-data-api.rst b/docs/bigtable-data-api.rst new file mode 100644 index 000000000000..779efa991886 --- /dev/null +++ b/docs/bigtable-data-api.rst @@ -0,0 +1,344 @@ +Data API +======== + +After creating a :class:`Table ` and some +column families, you are ready to store and retrieve data. + +Cells vs. Columns vs. Column Families ++++++++++++++++++++++++++++++++++++++ + +* As explained in the :doc:`table overview `, tables can + have many column families. +* As described below, a table can also have many rows which are + specified by row keys. +* Within a row, data is stored in a cell. A cell simply has a value (as + bytes) and a timestamp. The number of cells in each row can be + different, depending on what was stored in each row. +* Each cell lies in a column (**not** a column family). A column is really + just a more **specific** modifier within a column family. A column + can be present in every column family, in only one or anywhere in between. +* Within a column family there can be many columns. For example within + the column family ``foo`` we could have columns ``bar`` and ``baz``. + These would typically be represented as ``foo:bar`` and ``foo:baz``. + +Modifying Data +++++++++++++++ + +Since data is stored in cells, which are stored in rows, we +use the metaphor of a **row** in classes that are used to modify +(write, update, delete) data in a +:class:`Table `. + +Direct vs. Conditional vs. Append +--------------------------------- + +There are three ways to modify data in a table, described by the +`MutateRow`_, `CheckAndMutateRow`_ and `ReadModifyWriteRow`_ API +methods. + +* The **direct** way is via `MutateRow`_ which involves simply + adding, overwriting or deleting cells. The + :class:`DirectRow ` class + handles direct mutations. +* The **conditional** way is via `CheckAndMutateRow`_. This method + first checks if some filter is matched in a a given row, then + applies one of two sets of mutations, depending on if a match + occurred or not. (These mutation sets are called the "true + mutations" and "false mutations".) The + :class:`ConditionalRow ` class + handles conditional mutations. +* The **append** way is via `ReadModifyWriteRow`_. This simply + appends (as bytes) or increments (as an integer) data in a presumed + existing cell in a row. The + :class:`AppendRow ` class + handles append mutations. + +Row Factory +----------- + +A single factory can be used to create any of the three row types. +To create a :class:`DirectRow `: + +.. code:: python + + row = table.row(row_key) + +Unlike the previous string values we've used before, the row key must +be ``bytes``. + +To create a :class:`ConditionalRow `, +first create a :class:`RowFilter ` and +then + +.. code:: python + + cond_row = table.row(row_key, filter_=filter_) + +To create an :class:`AppendRow ` + +.. code:: python + + append_row = table.row(row_key, append=True) + +Building Up Mutations +--------------------- + +In all three cases, a set of mutations (or two sets) are built up +on a row before they are sent of in a batch via + +.. code:: python + + row.commit() + +Direct Mutations +---------------- + +Direct mutations can be added via one of four methods + +* :meth:`set_cell() ` allows a + single value to be written to a column + + .. code:: python + + row.set_cell(column_family_id, column, value, + timestamp=timestamp) + + If the ``timestamp`` is omitted, the current time on the Google Cloud + Bigtable server will be used when the cell is stored. + + The value can either by bytes or an integer (which will be converted to + bytes as a signed 64-bit integer). + +* :meth:`delete_cell() ` deletes + all cells (i.e. for all timestamps) in a given column + + .. code:: python + + row.delete_cell(column_family_id, column) + + Remember, this only happens in the ``row`` we are using. + + If we only want to delete cells from a limited range of time, a + :class:`TimestampRange ` can + be used + + .. code:: python + + row.delete_cell(column_family_id, column, + time_range=time_range) + +* :meth:`delete_cells() ` does + the same thing as + :meth:`delete_cell() ` + but accepts a list of columns in a column family rather than a single one. + + .. code:: python + + row.delete_cells(column_family_id, [column1, column2], + time_range=time_range) + + In addition, if we want to delete cells from every column in a column family, + the special :attr:`ALL_COLUMNS ` + value can be used + + .. code:: python + + row.delete_cells(column_family_id, row.ALL_COLUMNS, + time_range=time_range) + +* :meth:`delete() ` will delete the + entire row + + .. code:: python + + row.delete() + +Conditional Mutations +--------------------- + +Making **conditional** modifications is essentially identical +to **direct** modifications: it uses the exact same methods +to accumulate mutations. + +However, each mutation added must specify a ``state``: will the mutation be +applied if the filter matches or if it fails to match. + +For example: + +.. code:: python + + cond_row.set_cell(column_family_id, column, value, + timestamp=timestamp, state=True) + +will add to the set of true mutations. + +Append Mutations +---------------- + +Append mutations can be added via one of two methods + +* :meth:`append_cell_value() ` + appends a bytes value to an existing cell: + + .. code:: python + + append_row.append_cell_value(column_family_id, column, bytes_value) + +* :meth:`increment_cell_value() ` + increments an integer value in an existing cell: + + .. code:: python + + append_row.increment_cell_value(column_family_id, column, int_value) + + Since only bytes are stored in a cell, the cell value is decoded as + a signed 64-bit integer before being incremented. (This happens on + the Google Cloud Bigtable server, not in the library.) + +Notice that no timestamp was specified. This is because **append** mutations +operate on the latest value of the specified column. + +If there are no cells in the specified column, then the empty string (bytes +case) or zero (integer case) are the assumed values. + +Starting Fresh +-------------- + +If accumulated mutations need to be dropped, use + +.. code:: python + + row.clear() + +Reading Data +++++++++++++ + +Read Single Row from a Table +---------------------------- + +To make a `ReadRows`_ API request for a single row key, use +:meth:`Table.read_row() `: + +.. code:: python + + >>> row_data = table.read_row(row_key) + >>> row_data.cells + { + u'fam1': { + b'col1': [ + , + , + ], + b'col2': [ + , + ], + }, + u'fam2': { + b'col3': [ + , + , + , + ], + }, + } + >>> cell = row_data.cells[u'fam1'][b'col1'][0] + >>> cell + + >>> cell.value + b'val1' + >>> cell.timestamp + datetime.datetime(2016, 2, 27, 3, 41, 18, 122823, tzinfo=) + +Rather than returning a :class:`DirectRow ` +or similar class, this method returns a +:class:`PartialRowData ` +instance. This class is used for reading and parsing data rather than for +modifying data (as :class:`DirectRow ` is). + +A filter can also be applied to the results: + +.. code:: python + + row_data = table.read_row(row_key, filter_=filter_val) + +The allowable ``filter_`` values are the same as those used for a +:class:`ConditionalRow `. For +more information, see the +:meth:`Table.read_row() ` documentation. + +Stream Many Rows from a Table +----------------------------- + +To make a `ReadRows`_ API request for a stream of rows, use +:meth:`Table.read_rows() `: + +.. code:: python + + row_data = table.read_rows() + +Using gRPC over HTTP/2, a continual stream of responses will be delivered. +In particular + +* :meth:`consume_next() ` + pulls the next result from the stream, parses it and stores it on the + :class:`PartialRowsData ` instance +* :meth:`consume_all() ` + pulls results from the stream until there are no more +* :meth:`cancel() ` closes + the stream + +See the :class:`PartialRowsData ` +documentation for more information. + +As with +:meth:`Table.read_row() `, an optional +``filter_`` can be applied. In addition a ``start_key`` and / or ``end_key`` +can be supplied for the stream, a ``limit`` can be set and a boolean +``allow_row_interleaving`` can be specified to allow faster streamed results +at the potential cost of non-sequential reads. + +See the :meth:`Table.read_rows() ` +documentation for more information on the optional arguments. + +Sample Keys in a Table +---------------------- + +Make a `SampleRowKeys`_ API request with +:meth:`Table.sample_row_keys() `: + +.. code:: python + + keys_iterator = table.sample_row_keys() + +The returned row keys will delimit contiguous sections of the table of +approximately equal size, which can be used to break up the data for +distributed tasks like mapreduces. + +As with +:meth:`Table.read_rows() `, the +returned ``keys_iterator`` is connected to a cancellable HTTP/2 stream. + +The next key in the result can be accessed via + +.. code:: python + + next_key = keys_iterator.next() + +or all keys can be iterated over via + +.. code:: python + + for curr_key in keys_iterator: + do_something(curr_key) + +Just as with reading, the stream can be canceled: + +.. code:: python + + keys_iterator.cancel() + +.. _ReadRows: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L36-L38 +.. _SampleRowKeys: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L44-L46 +.. _MutateRow: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L50-L52 +.. _CheckAndMutateRow: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L62-L64 +.. _ReadModifyWriteRow: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L70-L72 diff --git a/docs/bigtable-row-data.rst b/docs/bigtable-row-data.rst new file mode 100644 index 000000000000..5ec98f932d1d --- /dev/null +++ b/docs/bigtable-row-data.rst @@ -0,0 +1,7 @@ +Row Data +~~~~~~~~ + +.. automodule:: gcloud.bigtable.row_data + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/bigtable-row-filters.rst b/docs/bigtable-row-filters.rst new file mode 100644 index 000000000000..b5e99eab6575 --- /dev/null +++ b/docs/bigtable-row-filters.rst @@ -0,0 +1,68 @@ +Bigtable Row Filters +==================== + +It is possible to use a +:class:`RowFilter ` +when adding mutations to a +:class:`ConditionalRow ` and when +reading row data with :meth:`read_row() ` +:meth:`read_rows() `. + +As laid out in the `RowFilter definition`_, the following basic filters +are provided: + +* :class:`SinkFilter <.row_filters.SinkFilter>` +* :class:`PassAllFilter <.row_filters.PassAllFilter>` +* :class:`BlockAllFilter <.row_filters.BlockAllFilter>` +* :class:`RowKeyRegexFilter <.row_filters.RowKeyRegexFilter>` +* :class:`RowSampleFilter <.row_filters.RowSampleFilter>` +* :class:`FamilyNameRegexFilter <.row_filters.FamilyNameRegexFilter>` +* :class:`ColumnQualifierRegexFilter <.row_filters.ColumnQualifierRegexFilter>` +* :class:`TimestampRangeFilter <.row_filters.TimestampRangeFilter>` +* :class:`ColumnRangeFilter <.row_filters.ColumnRangeFilter>` +* :class:`ValueRegexFilter <.row_filters.ValueRegexFilter>` +* :class:`ValueRangeFilter <.row_filters.ValueRangeFilter>` +* :class:`CellsRowOffsetFilter <.row_filters.CellsRowOffsetFilter>` +* :class:`CellsRowLimitFilter <.row_filters.CellsRowLimitFilter>` +* :class:`CellsColumnLimitFilter <.row_filters.CellsColumnLimitFilter>` +* :class:`StripValueTransformerFilter <.row_filters.StripValueTransformerFilter>` +* :class:`ApplyLabelFilter <.row_filters.ApplyLabelFilter>` + +In addition, these filters can be combined into composite filters with + +* :class:`RowFilterChain <.row_filters.RowFilterChain>` +* :class:`RowFilterUnion <.row_filters.RowFilterUnion>` +* :class:`ConditionalRowFilter <.row_filters.ConditionalRowFilter>` + +These rules can be nested arbitrarily, with a basic filter at the lowest +level. For example: + +.. code:: python + + # Filter in a specified column (matching any column family). + col1_filter = ColumnQualifierRegexFilter(b'columnbia') + + # Create a filter to label results. + label1 = u'label-red' + label1_filter = ApplyLabelFilter(label1) + + # Combine the filters to label all the cells in columnbia. + chain1 = RowFilterChain(filters=[col1_filter, label1_filter]) + + # Create a similar filter to label cells blue. + col2_filter = ColumnQualifierRegexFilter(b'columnseeya') + label2 = u'label-blue' + label2_filter = ApplyLabelFilter(label2) + chain2 = RowFilterChain(filters=[col2_filter, label2_filter]) + + # Bring our two labeled columns together. + row_filter = RowFilterUnion(filters=[chain1, chain2]) + +---- + +.. automodule:: gcloud.bigtable.row_filters + :members: + :undoc-members: + :show-inheritance: + +.. _RowFilter definition: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/1ff247c2e3b7cd0a2dd49071b2d95beaf6563092/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_data.proto#L195 diff --git a/docs/bigtable-row.rst b/docs/bigtable-row.rst new file mode 100644 index 000000000000..ae9835995bda --- /dev/null +++ b/docs/bigtable-row.rst @@ -0,0 +1,8 @@ +Bigtable Row +============ + +.. automodule:: gcloud.bigtable.row + :members: + :undoc-members: + :show-inheritance: + :inherited-members: diff --git a/docs/bigtable-table-api.rst b/docs/bigtable-table-api.rst new file mode 100644 index 000000000000..b3108da75a1b --- /dev/null +++ b/docs/bigtable-table-api.rst @@ -0,0 +1,165 @@ +Table Admin API +=============== + +After creating a :class:`Cluster `, you can +interact with individual tables, groups of tables or column families within +a table. + +List Tables +----------- + +If you want a comprehensive list of all existing tables in a cluster, make a +`ListTables`_ API request with +:meth:`Cluster.list_tables() `: + +.. code:: python + + >>> cluster.list_tables() + [, + ] + +Table Factory +------------- + +To create a :class:`Table ` object: + +.. code:: python + + table = cluster.table(table_id) + +Even if this :class:`Table ` already +has been created with the API, you'll want this object to use as a +parent of a :class:`ColumnFamily ` +or :class:`Row `. + +Create a new Table +------------------ + +After creating the table object, make a `CreateTable`_ API request +with :meth:`create() `: + +.. code:: python + + table.create() + +If you would to initially split the table into several tablets (Tablets are +similar to HBase regions): + +.. code:: python + + table.create(initial_split_keys=['s1', 's2']) + +Delete an existing Table +------------------------ + +Make a `DeleteTable`_ API request with +:meth:`delete() `: + +.. code:: python + + table.delete() + +Rename an existing Table +------------------------ + +Though the `RenameTable`_ API request is listed in the service +definition, requests to that method return:: + + BigtableTableService.RenameTable is not yet implemented + +We have implemented :meth:`rename() ` +but it will not work unless the backend supports the method. + +List Column Families in a Table +------------------------------- + +Though there is no **official** method for retrieving `column families`_ +associated with a table, the `GetTable`_ API method returns a +table object with the names of the column families. + +To retrieve the list of column families use +:meth:`list_column_families() `: + +.. code:: python + + column_families = table.list_column_families() + +Column Family Factory +--------------------- + +To create a +:class:`ColumnFamily ` object: + +.. code:: python + + column_family = table.column_family(column_family_id) + +There is no real reason to use this factory unless you intend to +create or delete a column family. + +In addition, you can specify an optional ``gc_rule`` (a +:class:`GarbageCollectionRule ` +or similar): + +.. code:: python + + column_family = table.column_family(column_family_id, + gc_rule=gc_rule) + +This rule helps the backend determine when and how to clean up old cells +in the column family. + +See :doc:`bigtable-column-family` for more information about +:class:`GarbageCollectionRule ` +and related classes. + +Create a new Column Family +-------------------------- + +After creating the column family object, make a `CreateColumnFamily`_ API +request with +:meth:`ColumnFamily.create() ` + +.. code:: python + + column_family.create() + +Delete an existing Column Family +-------------------------------- + +Make a `DeleteColumnFamily`_ API request with +:meth:`ColumnFamily.delete() ` + +.. code:: python + + column_family.delete() + +Update an existing Column Family +-------------------------------- + +Make an `UpdateColumnFamily`_ API request with +:meth:`ColumnFamily.delete() ` + +.. code:: python + + column_family.update() + +Next Step +--------- + +Now we go down the final step of the hierarchy from +:class:`Table ` to +:class:`Row ` as well as streaming +data directly via a :class:`Table `. + +Head next to learn about the :doc:`bigtable-data-api`. + +.. _ListTables: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L40-L42 +.. _CreateTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L35-L37 +.. _DeleteTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L50-L52 +.. _RenameTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L56-L58 +.. _GetTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L45-L47 +.. _CreateColumnFamily: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L61-L63 +.. _UpdateColumnFamily: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L66-L68 +.. _DeleteColumnFamily: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L71-L73 +.. _column families: https://cloud.google.com/bigtable/docs/schema-design#column_families_and_column_qualifiers diff --git a/docs/bigtable-table.rst b/docs/bigtable-table.rst new file mode 100644 index 000000000000..03ca332f9c9a --- /dev/null +++ b/docs/bigtable-table.rst @@ -0,0 +1,7 @@ +Table +~~~~~ + +.. automodule:: gcloud.bigtable.table + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/bigtable-usage.rst b/docs/bigtable-usage.rst new file mode 100644 index 000000000000..95faba854e69 --- /dev/null +++ b/docs/bigtable-usage.rst @@ -0,0 +1,25 @@ +Using the API +============= + +API requests are sent to the `Google Cloud Bigtable`_ API via RPC over HTTP/2. +In order to support this, we'll rely on `gRPC`_. We are working with the gRPC +team to rapidly make the install story more user-friendly. + +Get started by learning about the +:class:`Client ` on the +:doc:`bigtable-client-intro` page. + +In the hierarchy of API concepts + +* a :class:`Client ` owns a + :class:`Cluster ` +* a :class:`Cluster ` owns a + :class:`Table ` +* a :class:`Table ` owns a + :class:`ColumnFamily ` +* a :class:`Table ` owns a + :class:`Row ` + (and all the cells in the row) + +.. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs/ +.. _gRPC: http://www.grpc.io/ diff --git a/docs/happybase-batch.rst b/docs/happybase-batch.rst new file mode 100644 index 000000000000..c1fc86b9d6e0 --- /dev/null +++ b/docs/happybase-batch.rst @@ -0,0 +1,7 @@ +HappyBase Batch +~~~~~~~~~~~~~~~ + +.. automodule:: gcloud.bigtable.happybase.batch + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/happybase-connection.rst b/docs/happybase-connection.rst new file mode 100644 index 000000000000..01485bbdbde0 --- /dev/null +++ b/docs/happybase-connection.rst @@ -0,0 +1,7 @@ +HappyBase Connection +~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: gcloud.bigtable.happybase.connection + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/happybase-package.rst b/docs/happybase-package.rst new file mode 100644 index 000000000000..22e6134f0fa5 --- /dev/null +++ b/docs/happybase-package.rst @@ -0,0 +1,7 @@ +HappyBase Package +~~~~~~~~~~~~~~~~~ + +.. automodule:: gcloud.bigtable.happybase.__init__ + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/happybase-pool.rst b/docs/happybase-pool.rst new file mode 100644 index 000000000000..9390fd41c01d --- /dev/null +++ b/docs/happybase-pool.rst @@ -0,0 +1,7 @@ +HappyBase Connection Pool +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: gcloud.bigtable.happybase.pool + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/happybase-table.rst b/docs/happybase-table.rst new file mode 100644 index 000000000000..b5f477d8058d --- /dev/null +++ b/docs/happybase-table.rst @@ -0,0 +1,7 @@ +HappyBase Table +~~~~~~~~~~~~~~~ + +.. automodule:: gcloud.bigtable.happybase.table + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/index.rst b/docs/index.rst index c08588dfa8b1..dd0e6e90bf70 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -53,6 +53,29 @@ bigquery-table bigquery-query +.. toctree:: + :maxdepth: 0 + :hidden: + :caption: Cloud Bigtable + + bigtable-usage + HappyBase + bigtable-client-intro + bigtable-cluster-api + bigtable-table-api + bigtable-data-api + Client + bigtable-cluster + bigtable-table + bigtable-column-family + bigtable-row + bigtable-row-filters + bigtable-row-data + happybase-connection + happybase-pool + happybase-table + happybase-batch + .. toctree:: :maxdepth: 0 :hidden: diff --git a/docs/pubsub-usage.rst b/docs/pubsub-usage.rst index a18b981f416d..99d01d2cde50 100644 --- a/docs/pubsub-usage.rst +++ b/docs/pubsub-usage.rst @@ -186,8 +186,8 @@ List subscriptions for a topic: >>> from gcloud import pubsub >>> client = pubsub.Client() - >>> subscriptions, next_page_token = client.list_subscriptions( - ... topic_name='topic_name') # API request + >>> topic = client.topic('topic_name') + >>> subscriptions, next_page_token = topic.list_subscriptions() # API request >>> [subscription.name for subscription in subscriptions] ['subscription_name'] diff --git a/gcloud/_helpers.py b/gcloud/_helpers.py index ce561976dd24..dd85a54e405f 100644 --- a/gcloud/_helpers.py +++ b/gcloud/_helpers.py @@ -19,13 +19,14 @@ import calendar import datetime import os -from threading import local as Local +import re import socket import sys +from threading import local as Local from google.protobuf import timestamp_pb2 import six -from six.moves.http_client import HTTPConnection # pylint: disable=F0401 +from six.moves.http_client import HTTPConnection from gcloud.environment_vars import PROJECT @@ -388,6 +389,45 @@ def _datetime_to_pb_timestamp(when): return timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos) +def _name_from_project_path(path, project, template): + """Validate a URI path and get the leaf object's name. + + :type path: string + :param path: URI path containing the name. + + :type project: string + :param project: The project associated with the request. It is + included for validation purposes. + + :type template: string + :param template: Template regex describing the expected form of the path. + The regex must have two named groups, 'project' and + 'name'. + + :rtype: string + :returns: Name parsed from ``path``. + :raises: :class:`ValueError` if the ``path`` is ill-formed or if + the project from the ``path`` does not agree with the + ``project`` passed in. + """ + if isinstance(template, str): + template = re.compile(template) + + match = template.match(path) + + if not match: + raise ValueError('path "%s" did not match expected pattern "%s"' % ( + path, template.pattern,)) + + found_project = match.group('project') + if found_project != project: + raise ValueError( + 'Project from client (%s) should agree with ' + 'project from resource(%s).' % (project, found_project)) + + return match.group('name') + + try: from pytz import UTC # pylint: disable=unused-import,wrong-import-order except ImportError: diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py index dd8e003bc8d5..bbe1f1ccfdf9 100644 --- a/gcloud/bigquery/table.py +++ b/gcloud/bigquery/table.py @@ -631,6 +631,7 @@ def insert_data(self, row_ids=None, skip_invalid_rows=None, ignore_unknown_values=None, + template_suffix=None, client=None): """API call: insert table data via a POST request @@ -652,6 +653,13 @@ def insert_data(self, :type ignore_unknown_values: boolean or ``NoneType`` :param ignore_unknown_values: ignore columns beyond schema? + :type template_suffix: string or ``NoneType`` + :param template_suffix: treat ``name`` as a template table and provide + a suffix. BigQuery will create the table + `` + `` based on the + schema of the template table. See: + https://cloud.google.com/bigquery/streaming-data-into-bigquery#template-tables + :type client: :class:`gcloud.bigquery.client.Client` or ``NoneType`` :param client: the client to use. If not passed, falls back to the ``client`` stored on the current dataset. @@ -689,6 +697,9 @@ def insert_data(self, if ignore_unknown_values is not None: data['ignoreUnknownValues'] = ignore_unknown_values + if template_suffix is not None: + data['templateSuffix'] = template_suffix + response = client.connection.api_request( method='POST', path='%s/insertAll' % self.path, diff --git a/gcloud/bigquery/test_job.py b/gcloud/bigquery/test_job.py index 58953dbc81d5..64660706be2e 100644 --- a/gcloud/bigquery/test_job.py +++ b/gcloud/bigquery/test_job.py @@ -1,4 +1,3 @@ -# pylint: disable=C0302 # Copyright 2015 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/gcloud/bigquery/test_table.py b/gcloud/bigquery/test_table.py index 56c9416fc5cf..2069d2455fc2 100644 --- a/gcloud/bigquery/test_table.py +++ b/gcloud/bigquery/test_table.py @@ -1185,6 +1185,7 @@ def _row_data(row): SENT = { 'skipInvalidRows': True, 'ignoreUnknownValues': True, + 'templateSuffix': '20160303', 'rows': [{'insertId': index, 'json': _row_data(row)} for index, row in enumerate(ROWS)], } @@ -1194,7 +1195,9 @@ def _row_data(row): rows=ROWS, row_ids=[index for index, _ in enumerate(ROWS)], skip_invalid_rows=True, - ignore_unknown_values=True) + ignore_unknown_values=True, + template_suffix='20160303', + ) self.assertEqual(len(errors), 1) self.assertEqual(errors[0]['index'], 1) diff --git a/gcloud/bigtable/client.py b/gcloud/bigtable/client.py index 2913feaa68c3..c67ce2b7057f 100644 --- a/gcloud/bigtable/client.py +++ b/gcloud/bigtable/client.py @@ -27,6 +27,8 @@ """ +from pkg_resources import get_distribution + from grpc.beta import implementations from gcloud.bigtable._generated import bigtable_cluster_data_pb2 as data_pb2 @@ -75,7 +77,8 @@ DEFAULT_TIMEOUT_SECONDS = 10 """The default timeout to use for API requests.""" -DEFAULT_USER_AGENT = 'gcloud-bigtable-python' +DEFAULT_USER_AGENT = 'gcloud-python/{0}'.format( + get_distribution('gcloud').version) """The default user agent for API requests.""" diff --git a/gcloud/bigtable/cluster.py b/gcloud/bigtable/cluster.py index c04175a3a16c..95be153ad474 100644 --- a/gcloud/bigtable/cluster.py +++ b/gcloud/bigtable/cluster.py @@ -34,7 +34,6 @@ _OPERATION_NAME_RE = re.compile(r'^operations/projects/([^/]+)/zones/([^/]+)/' r'clusters/([a-z][-a-z0-9]*)/operations/' r'(?P\d+)$') -_DEFAULT_SERVE_NODES = 3 _TYPE_URL_BASE = 'type.googleapis.com/google.bigtable.' _ADMIN_TYPE_URL_BASE = _TYPE_URL_BASE + 'admin.cluster.v1.' _CLUSTER_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'CreateClusterMetadata' @@ -46,6 +45,9 @@ _UNDELETE_CREATE_METADATA: messages_pb2.UndeleteClusterMetadata, } +DEFAULT_SERVE_NODES = 3 +"""Default number of nodes to use when creating a cluster.""" + def _prepare_create_request(cluster): """Creates a protobuf request for a CreateCluster request. @@ -204,7 +206,7 @@ class Cluster(object): :type cluster_id: str :param cluster_id: The ID of the cluster. - :type client: :class:`.client.Client` + :type client: :class:`Client ` :param client: The client that owns the cluster. Provides authorization and a project ID. @@ -216,11 +218,11 @@ class Cluster(object): :type serve_nodes: int :param serve_nodes: (Optional) The number of nodes in the cluster. - Defaults to 3 (``_DEFAULT_SERVE_NODES``). + Defaults to :data:`DEFAULT_SERVE_NODES`. """ def __init__(self, zone, cluster_id, client, - display_name=None, serve_nodes=_DEFAULT_SERVE_NODES): + display_name=None, serve_nodes=DEFAULT_SERVE_NODES): self.zone = zone self.cluster_id = cluster_id self.display_name = display_name or cluster_id @@ -253,14 +255,16 @@ def from_pb(cls, cluster_pb, client): :type cluster_pb: :class:`bigtable_cluster_data_pb2.Cluster` :param cluster_pb: A cluster protobuf object. - :type client: :class:`.client.Client` + :type client: :class:`Client ` :param client: The client that owns the cluster. :rtype: :class:`Cluster` :returns: The cluster parsed from the protobuf response. :raises: :class:`ValueError ` if the cluster - name does not match :data:`_CLUSTER_NAME_RE` or if the parsed - project ID does not match the project ID on the client. + name does not match + ``projects/{project}/zones/{zone}/clusters/{cluster_id}`` + or if the parsed project ID does not match the project ID + on the client. """ match = _CLUSTER_NAME_RE.match(cluster_pb.name) if match is None: @@ -277,9 +281,8 @@ def from_pb(cls, cluster_pb, client): def copy(self): """Make a copy of this cluster. - Copies the local data stored as simple types but does not copy the - current state of any operations with the Cloud Bigtable API. Also - copies the client attached to this instance. + Copies the local data stored as simple types and copies the client + attached to this instance. :rtype: :class:`.Cluster` :returns: A copy of the current cluster. diff --git a/gcloud/bigtable/column_family.py b/gcloud/bigtable/column_family.py index 0d3aeda8c508..c0d9060316a4 100644 --- a/gcloud/bigtable/column_family.py +++ b/gcloud/bigtable/column_family.py @@ -157,7 +157,7 @@ def __eq__(self, other): return other.rules == self.rules def to_pb(self): - """Converts the union into a single gc rule as a protobuf. + """Converts the union into a single GC rule as a protobuf. :rtype: :class:`.data_pb2.GcRule` :returns: The converted current object. @@ -183,7 +183,7 @@ def __eq__(self, other): return other.rules == self.rules def to_pb(self): - """Converts the intersection into a single gc rule as a protobuf. + """Converts the intersection into a single GC rule as a protobuf. :rtype: :class:`.data_pb2.GcRule` :returns: The converted current object. diff --git a/gcloud/bigtable/happybase/__init__.py b/gcloud/bigtable/happybase/__init__.py index 4c5b52f80924..03e4d9215ff1 100644 --- a/gcloud/bigtable/happybase/__init__.py +++ b/gcloud/bigtable/happybase/__init__.py @@ -24,10 +24,18 @@ Bigtable API. As a result, the following instance methods and functions could not be implemented: -* :meth:`.Connection.enable_table` - no concept of enabled/disabled -* :meth:`.Connection.disable_table` - no concept of enabled/disabled -* :meth:`.Connection.is_table_enabled` - no concept of enabled/disabled -* :meth:`.Connection.compact_table` - table storage is opaque to user +* :meth:`Connection.enable_table() \ + ` - no + concept of enabled/disabled +* :meth:`Connection.disable_table() \ + ` - no + concept of enabled/disabled +* :meth:`Connection.is_table_enabled() \ + ` + - no concept of enabled/disabled +* :meth:`Connection.compact_table() \ + ` - + table storage is opaque to user * :func:`make_row() ` - helper needed for Thrift library * :func:`make_ordered_row() ` @@ -41,8 +49,9 @@ However, it's worth nothing this implementation was based off HappyBase 0.9. -In addition, many of the constants from :mod:`.connection` are specific -to HBase and are defined as :data:`None` in our module: +In addition, many of the constants from +:mod:`connection ` +are specific to HBase and are defined as :data:`None` in our module: * ``COMPAT_MODES`` * ``THRIFT_TRANSPORTS`` @@ -63,9 +72,12 @@ -------------------- * Since there is no concept of an enabled / disabled table, calling - :meth:`.Connection.delete_table` with ``disable=True`` can't be supported. + :meth:`Connection.delete_table() \ + ` + with ``disable=True`` can't be supported. Using that argument will result in a warning. -* The :class:`.Connection` constructor **disables** the use of several +* The :class:`Connection ` + constructor **disables** the use of several arguments and will print a warning if any of them are passed in as keyword arguments. The arguments are: @@ -74,9 +86,12 @@ * ``compat`` * ``transport`` * ``protocol`` -* In order to make :class:`.Connection` compatible with Cloud Bigtable, we - add a ``cluster`` keyword argument to allow user's to pass in their own - :class:`.Cluster` (which they can construct beforehand). +* In order to make + :class:`Connection ` + compatible with Cloud Bigtable, we add a ``cluster`` keyword argument to + allow users to pass in their own + :class:`Cluster ` (which they can + construct beforehand). For example: @@ -93,14 +108,16 @@ * Any uses of the ``wal`` (Write Ahead Log) argument will result in a warning as well. This includes uses in: - * :class:`.Batch` constructor - * :meth:`.Batch.put` - * :meth:`.Batch.delete` + * :class:`Batch ` + * :meth:`Batch.put() ` + * :meth:`Batch.delete() ` * :meth:`Table.put() ` * :meth:`Table.delete() ` * :meth:`Table.batch() ` factory -* When calling :meth:`.Connection.create_table`, the majority of HBase column - family options cannot be used. Among +* When calling + :meth:`Connection.create_table() \ + `, the + majority of HBase column family options cannot be used. Among * ``max_versions`` * ``compression`` @@ -113,9 +130,9 @@ Only ``max_versions`` and ``time_to_live`` are availabe in Cloud Bigtable (as - :class:`MaxVersionsGCRule ` - and - `MaxAgeGCRule `). + :class:`MaxVersionsGCRule ` + and + :class:`MaxAgeGCRule `). In addition to using a dictionary for specifying column family options, we also accept instances of :class:`.GarbageCollectionRule` or subclasses. @@ -131,7 +148,8 @@ not possible with Cloud Bigtable and will result in a :class:`TypeError `. However, the method now accepts instances of :class:`.RowFilter` and subclasses. -* :meth:`.Batch.delete` (and hence +* :meth:`Batch.delete() ` (and + hence :meth:`Table.delete() `) will fail with a :class:`ValueError ` when either a row or column family delete is attempted with a ``timestamp``. This is diff --git a/gcloud/bigtable/happybase/batch.py b/gcloud/bigtable/happybase/batch.py index 310b0ae582cc..25e6d073fc66 100644 --- a/gcloud/bigtable/happybase/batch.py +++ b/gcloud/bigtable/happybase/batch.py @@ -21,7 +21,7 @@ import six from gcloud._helpers import _datetime_from_microseconds -from gcloud.bigtable.row import TimestampRange +from gcloud.bigtable.row_filters import TimestampRange _WAL_SENTINEL = object() @@ -179,7 +179,7 @@ def _delete_columns(self, columns, row_object): strings). Each column name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` :type row_object: :class:`Row ` :param row_object: The row which will hold the delete mutations. @@ -213,7 +213,7 @@ def delete(self, row, columns=None, wal=_WAL_SENTINEL): strings). Each column name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` If not used, will delete the entire row. @@ -223,7 +223,7 @@ def delete(self, row, columns=None, wal=_WAL_SENTINEL): irrelevant for Cloud Bigtable since it does not have a Write Ahead Log. - :raises: If if the delete timestamp range is set on the + :raises: If the delete timestamp range is set on the current batch, but a full row delete is attempted. """ if wal is not _WAL_SENTINEL: @@ -287,7 +287,7 @@ def _get_column_pairs(columns, require_qualifier=False): strings). Each column name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` :type require_qualifier: bool :param require_qualifier: Boolean indicating if the columns should diff --git a/gcloud/bigtable/happybase/connection.py b/gcloud/bigtable/happybase/connection.py index b3abad5da841..c6ba0b2f8f96 100644 --- a/gcloud/bigtable/happybase/connection.py +++ b/gcloud/bigtable/happybase/connection.py @@ -50,7 +50,8 @@ def _get_cluster(timeout=None): """Gets cluster for the default project. Creates a client with the inferred credentials and project ID from - the local environment. Then uses :meth:`.Client.list_clusters` to + the local environment. Then uses + :meth:`.bigtable.client.Client.list_clusters` to get the unique cluster owned by the project. If the request fails for any reason, or if there isn't exactly one cluster @@ -94,7 +95,8 @@ class Connection(object): If you pass a ``cluster``, it will be :meth:`.Cluster.copy`-ed before being stored on the new connection. This also copies the - :class:`.Client` that created the :class:`.Cluster` instance and the + :class:`Client ` that created the + :class:`Cluster ` instance and the :class:`Credentials ` stored on the client. @@ -117,12 +119,13 @@ class Connection(object): :param table_prefix_separator: (Optional) Separator used with ``table_prefix``. Defaults to ``_``. - :type cluster: :class:`gcloud.bigtable.cluster.Cluster` + :type cluster: :class:`Cluster ` :param cluster: (Optional) A Cloud Bigtable cluster. The instance also owns a client for making gRPC requests to the Cloud Bigtable API. If not passed in, defaults to creating client with ``admin=True`` and using the ``timeout`` here for the - ``timeout_seconds`` argument to the :class:`.Client`` + ``timeout_seconds`` argument to the + :class:`Client ` constructor. The credentials for the client will be the implicit ones loaded from the environment. Then that client is used to retrieve all the clusters @@ -191,7 +194,8 @@ def open(self): """Open the underlying transport to Cloud Bigtable. This method opens the underlying HTTP/2 gRPC connection using a - :class:`.Client` bound to the :class:`.Cluster` owned by + :class:`Client ` bound to the + :class:`Cluster ` owned by this connection. """ self._cluster._client.start() @@ -200,7 +204,8 @@ def close(self): """Close the underlying transport to Cloud Bigtable. This method closes the underlying HTTP/2 gRPC connection using a - :class:`.Client` bound to the :class:`.Cluster` owned by + :class:`Client ` bound to the + :class:`Cluster ` owned by this connection. """ self._cluster._client.stop() @@ -233,7 +238,7 @@ def table(self, name, use_prefix=True): :type use_prefix: bool :param use_prefix: Whether to use the table prefix (if any). - :rtype: `Table ` + :rtype: :class:`Table ` :returns: Table instance owned by this connection. """ if use_prefix: @@ -362,8 +367,10 @@ def delete_table(self, name, disable=False): def enable_table(self, name): """Enable the specified table. - Cloud Bigtable has no concept of enabled / disabled tables so this - method does not work. It is provided simply for compatibility. + .. warning:: + + Cloud Bigtable has no concept of enabled / disabled tables so this + method does not work. It is provided simply for compatibility. :raises: :class:`NotImplementedError ` always @@ -374,8 +381,10 @@ def enable_table(self, name): def disable_table(self, name): """Disable the specified table. - Cloud Bigtable has no concept of enabled / disabled tables so this - method does not work. It is provided simply for compatibility. + .. warning:: + + Cloud Bigtable has no concept of enabled / disabled tables so this + method does not work. It is provided simply for compatibility. :raises: :class:`NotImplementedError ` always @@ -386,8 +395,10 @@ def disable_table(self, name): def is_table_enabled(self, name): """Return whether the specified table is enabled. - Cloud Bigtable has no concept of enabled / disabled tables so this - method does not work. It is provided simply for compatibility. + .. warning:: + + Cloud Bigtable has no concept of enabled / disabled tables so this + method does not work. It is provided simply for compatibility. :raises: :class:`NotImplementedError ` always @@ -398,8 +409,10 @@ def is_table_enabled(self, name): def compact_table(self, name, major=False): """Compact the specified table. - Cloud Bigtable does not support compacting a table, so this - method does not work. It is provided simply for compatibility. + .. warning:: + + Cloud Bigtable does not support compacting a table, so this + method does not work. It is provided simply for compatibility. :raises: :class:`NotImplementedError ` always diff --git a/gcloud/bigtable/happybase/pool.py b/gcloud/bigtable/happybase/pool.py index e2d897155adb..ab84724740a2 100644 --- a/gcloud/bigtable/happybase/pool.py +++ b/gcloud/bigtable/happybase/pool.py @@ -42,17 +42,19 @@ class ConnectionPool(object): .. note:: All keyword arguments are passed unmodified to the - :class:`.Connection` constructor **except** for ``autoconnect``. - This is because the ``open`` / ``closed`` status of a connection - is managed by the pool. In addition, if ``cluster`` is not passed, - the default / inferred cluster is determined by the pool and then - passed to each :class:`.Connection` that is created. + :class:`Connection <.happybase.connection.Connection>` constructor + **except** for ``autoconnect``. This is because the ``open`` / + ``closed`` status of a connection is managed by the pool. In addition, + if ``cluster`` is not passed, the default / inferred cluster is + determined by the pool and then passed to each + :class:`Connection <.happybase.connection.Connection>` that is created. :type size: int :param size: The maximum number of concurrently open connections. :type kwargs: dict - :param kwargs: Keyword arguments passed to :class:`.Connection` + :param kwargs: Keyword arguments passed to + :class:`Connection <.happybase.Connection>` constructor. :raises: :class:`TypeError ` if ``size`` @@ -88,7 +90,7 @@ def _acquire_connection(self, timeout=None): :param timeout: (Optional) Time (in seconds) to wait for a connection to open. - :rtype: :class:`.Connection` + :rtype: :class:`Connection <.happybase.Connection>` :returns: An active connection from the queue stored on the pool. :raises: :class:`NoConnectionsAvailable` if ``Queue.get`` fails before the ``timeout`` (only if a timeout is specified). @@ -109,13 +111,13 @@ def connection(self, timeout=None): pass # do something with the connection If ``timeout`` is omitted, this method waits forever for a connection - to become available. + to become available from the local queue. :type timeout: int :param timeout: (Optional) Time (in seconds) to wait for a connection to open. - :rtype: :class:`.Connection` + :rtype: :class:`Connection <.happybase.connection.Connection>` :returns: An active connection from the pool. :raises: :class:`NoConnectionsAvailable` if no connection can be retrieved from the pool before the ``timeout`` (only if diff --git a/gcloud/bigtable/happybase/table.py b/gcloud/bigtable/happybase/table.py index 342fb04a15ff..bf018ad1647f 100644 --- a/gcloud/bigtable/happybase/table.py +++ b/gcloud/bigtable/happybase/table.py @@ -30,14 +30,14 @@ from gcloud.bigtable.happybase.batch import _get_column_pairs from gcloud.bigtable.happybase.batch import _WAL_SENTINEL from gcloud.bigtable.happybase.batch import Batch -from gcloud.bigtable.row import CellsColumnLimitFilter -from gcloud.bigtable.row import ColumnQualifierRegexFilter -from gcloud.bigtable.row import FamilyNameRegexFilter -from gcloud.bigtable.row import RowFilterChain -from gcloud.bigtable.row import RowFilterUnion -from gcloud.bigtable.row import RowKeyRegexFilter -from gcloud.bigtable.row import TimestampRange -from gcloud.bigtable.row import TimestampRangeFilter +from gcloud.bigtable.row_filters import CellsColumnLimitFilter +from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter +from gcloud.bigtable.row_filters import FamilyNameRegexFilter +from gcloud.bigtable.row_filters import RowFilterChain +from gcloud.bigtable.row_filters import RowFilterUnion +from gcloud.bigtable.row_filters import RowKeyRegexFilter +from gcloud.bigtable.row_filters import TimestampRange +from gcloud.bigtable.row_filters import TimestampRangeFilter from gcloud.bigtable.table import Table as _LowLevelTable @@ -49,7 +49,7 @@ def make_row(cell_map, include_timestamp): """Make a row dict for a Thrift cell mapping. - .. note:: + .. warning:: This method is only provided for HappyBase compatibility, but does not actually work. @@ -74,7 +74,7 @@ def make_row(cell_map, include_timestamp): def make_ordered_row(sorted_columns, include_timestamp): """Make a row dict for sorted Thrift column results from scans. - .. note:: + .. warning:: This method is only provided for HappyBase compatibility, but does not actually work. @@ -103,7 +103,7 @@ class Table(object): :type name: str :param name: The name of the table. - :type connection: :class:`.Connection` + :type connection: :class:`Connection <.happybase.connection.Connection>` :param connection: The connection which has access to the table. """ @@ -136,9 +136,11 @@ def families(self): def regions(self): """Retrieve the regions for this table. - Cloud Bigtable does not give information about how a table is laid - out in memory, so regions so this method does not work. It is - provided simply for compatibility. + .. warning:: + + Cloud Bigtable does not give information about how a table is laid + out in memory, so this method does not work. It is + provided simply for compatibility. :raises: :class:`NotImplementedError ` always @@ -161,7 +163,7 @@ def row(self, row, columns=None, timestamp=None, include_timestamp=False): strings). Each column name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` :type timestamp: int :param timestamp: (Optional) Timestamp (in milliseconds since the @@ -206,7 +208,7 @@ def rows(self, rows, columns=None, timestamp=None, strings). Each column name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` :type timestamp: int :param timestamp: (Optional) Timestamp (in milliseconds since the @@ -319,9 +321,9 @@ def scan(self, row_start=None, row_stop=None, row_prefix=None, results to retrieve per request. The HBase scanner defaults to reading one record at a time, so this argument allows HappyBase to increase that number. However, the Cloud Bigtable API uses HTTP/2 streaming so - there is no concept of a batch. The ``sorted_columns`` flag tells - HBase to return columns in order, but Cloud Bigtable doesn't have - this feature.) + there is no concept of a batched scan. The ``sorted_columns`` flag + tells HBase to return columns in order, but Cloud Bigtable doesn't + have this feature.) :type row_start: str :param row_start: (Optional) Row key where the scanner should start @@ -344,9 +346,9 @@ def scan(self, row_start=None, row_stop=None, row_prefix=None, strings). Each column name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` - :type filter: :class:`.RowFilter` + :type filter: :class:`RowFilter ` :param filter: (Optional) An additional filter (beyond column and row range filters supported here). HappyBase / HBase users will have used this as an HBase filter string. See @@ -369,7 +371,7 @@ def scan(self, row_start=None, row_stop=None, row_prefix=None, :param kwargs: Remaining keyword arguments. Provided for HappyBase compatibility. - :raises: If ``limit`` is set but non-positive, or if row prefix is + :raises: If ``limit`` is set but non-positive, or if ``row_prefix`` is used with row start/stop, :class:`TypeError ` if a string ``filter`` is used. @@ -479,7 +481,7 @@ def delete(self, row, columns=None, timestamp=None, wal=_WAL_SENTINEL): strings). Each column name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` :type timestamp: int :param timestamp: (Optional) Timestamp (in milliseconds since the @@ -498,8 +500,9 @@ def batch(self, timestamp=None, batch_size=None, transaction=False, wal=_WAL_SENTINEL): """Create a new batch operation for this table. - This method returns a new :class:`.Batch` instance that can be used - for mass data manipulation. + This method returns a new + :class:`Batch <.happybase.batch.Batch>` instance that can be + used for mass data manipulation. :type timestamp: int :param timestamp: (Optional) Timestamp (in milliseconds since the @@ -512,10 +515,11 @@ def batch(self, timestamp=None, batch_size=None, transaction=False, :type transaction: bool :param transaction: Flag indicating if the mutations should be sent transactionally or not. If ``transaction=True`` and - an error occurs while a :class:`Batch` is active, - then none of the accumulated mutations will be - committed. If ``batch_size`` is set, the mutation - can't be transactional. + an error occurs while a + :class:`Batch <.happybase.batch.Batch>` is + active, then none of the accumulated mutations will + be committed. If ``batch_size`` is set, the + mutation can't be transactional. :type wal: object :param wal: Unused parameter (to be passed to the created batch). @@ -523,7 +527,7 @@ def batch(self, timestamp=None, batch_size=None, transaction=False, for Cloud Bigtable since it does not have a Write Ahead Log. - :rtype: :class:`gcloud.bigtable.happybase.batch.Batch` + :rtype: :class:`Batch ` :returns: A batch bound to this table. """ return Batch(self, timestamp=timestamp, batch_size=batch_size, @@ -554,6 +558,30 @@ def counter_get(self, row, column): # is correctly initialized if didn't exist yet. return self.counter_inc(row, column, value=0) + def counter_set(self, row, column, value=0): + """Set a counter column to a specific value. + + This method is provided in HappyBase, but we do not provide it here + because it defeats the purpose of using atomic increment and decrement + of a counter. + + :type row: str + :param row: Row key for the row we are setting a counter in. + + :type column: str + :param column: Column we are setting a value in; of + the form ``fam:col``. + + :type value: int + :param value: Value to set the counter to. + + :raises: :class:`NotImplementedError ` + always + """ + raise NotImplementedError('Table.counter_set will not be implemented. ' + 'Instead use the increment/decrement ' + 'methods along with counter_get.') + def counter_inc(self, row, column, value=1): """Atomically increment a counter column. @@ -575,12 +603,12 @@ def counter_inc(self, row, column, value=1): :rtype: int :returns: Counter value after incrementing. """ - row = self._low_level_table.row(row) + row = self._low_level_table.row(row, append=True) if isinstance(column, six.binary_type): column = column.decode('utf-8') column_family_id, column_qualifier = column.split(':') row.increment_cell_value(column_family_id, column_qualifier, value) - # See row.commit_modifications() will return a dictionary: + # See AppendRow.commit() will return a dictionary: # { # u'col-fam-id': { # b'col-name1': [ @@ -590,7 +618,7 @@ def counter_inc(self, row, column, value=1): # ... # }, # } - modified_cells = row.commit_modifications() + modified_cells = row.commit() # Get the cells in the modified column, column_cells = modified_cells[column_family_id][column_qualifier] # Make sure there is exactly one cell in the column. @@ -635,10 +663,12 @@ def _gc_rule_to_dict(gc_rule): Only does this if the garbage collection rule is: - * :class:`.MaxAgeGCRule` - * :class:`.MaxVersionsGCRule` - * Composite :class:`.GCRuleIntersection` with two rules, one each - of type :class:`.MaxAgeGCRule` and :class:`.MaxVersionsGCRule` + * :class:`gcloud.bigtable.column_family.MaxAgeGCRule` + * :class:`gcloud.bigtable.column_family.MaxVersionsGCRule` + * Composite :class:`gcloud.bigtable.column_family.GCRuleIntersection` + with two rules, one each of type + :class:`gcloud.bigtable.column_family.MaxAgeGCRule` and + :class:`gcloud.bigtable.column_family.MaxVersionsGCRule` Otherwise, just returns the input without change. @@ -647,7 +677,8 @@ def _gc_rule_to_dict(gc_rule): :param gc_rule: A garbage collection rule to convert to a dictionary (if possible). - :rtype: dict or :class:`.GarbageCollectionRule` + :rtype: dict or + :class:`gcloud.bigtable.column_family.GarbageCollectionRule` :returns: The converted garbage collection rule. """ result = gc_rule @@ -734,7 +765,8 @@ def _convert_to_time_range(timestamp=None): epoch). Intended to be used as the end of an HBase time range, which is exclusive. - :rtype: :class:`.TimestampRange`, :data:`NoneType ` + :rtype: :class:`gcloud.bigtable.row.TimestampRange`, + :data:`NoneType ` :returns: The timestamp range corresponding to the passed in ``timestamp``. """ @@ -760,7 +792,8 @@ def _cells_to_pairs(cells, include_timestamp=False): [(b'val1', 1456361486255), (b'val2', 1456361491927)] :type cells: list - :param cells: List of :class:`.Cell` returned from a read request. + :param cells: List of :class:`gcloud.bigtable.row_data.Cell` returned + from a read request. :type include_timestamp: bool :param include_timestamp: Flag to indicate if cell timestamps should be @@ -844,7 +877,7 @@ def _filter_chain_helper(column=None, versions=None, timestamp=None, :type filters: list :param filters: (Optional) List of existing filters to be extended. - :rtype: :class:`.RowFilter` + :rtype: :class:`RowFilter ` :returns: The chained filter created, or just a single filter if only one was needed. :raises: :class:`ValueError ` if there are no @@ -883,9 +916,9 @@ def _columns_filter_helper(columns): name can be either * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` + * a single column: ``fam:col`` - :rtype: :class:`.RowFilter` + :rtype: :class:`RowFilter ` :returns: The union filter created containing all of the matched columns. :raises: :class:`ValueError ` if there are no filters to union. @@ -916,7 +949,7 @@ def _row_keys_filter_helper(row_keys): :type row_keys: list :param row_keys: Iterable containing row keys (as strings). - :rtype: :class:`.RowFilter` + :rtype: :class:`RowFilter ` :returns: The union filter created containing all of the row keys. :raises: :class:`ValueError ` if there are no filters to union. diff --git a/gcloud/bigtable/happybase/test_batch.py b/gcloud/bigtable/happybase/test_batch.py index 40cc84ae4c67..cf2156f226b4 100644 --- a/gcloud/bigtable/happybase/test_batch.py +++ b/gcloud/bigtable/happybase/test_batch.py @@ -46,7 +46,7 @@ def test_constructor_defaults(self): def test_constructor_explicit(self): from gcloud._helpers import _datetime_from_microseconds - from gcloud.bigtable.row import TimestampRange + from gcloud.bigtable.row_filters import TimestampRange table = object() timestamp = 144185290431 diff --git a/gcloud/bigtable/happybase/test_table.py b/gcloud/bigtable/happybase/test_table.py index c6abb83b33d8..be18ec1bc014 100644 --- a/gcloud/bigtable/happybase/test_table.py +++ b/gcloud/bigtable/happybase/test_table.py @@ -871,10 +871,12 @@ def _counter_inc_helper(self, row, column, value, commit_result): table = self._makeOne(name, connection) # Mock the return values. table._low_level_table = _MockLowLevelTable() - table._low_level_table.row_values[row] = _MockLowLevelRow( + table._low_level_table.row_values[row] = row_obj = _MockLowLevelRow( row, commit_result=commit_result) + self.assertFalse(row_obj._append) result = table.counter_inc(row, column, value=value) + self.assertTrue(row_obj._append) incremented_value = value + _MockLowLevelRow.COUNTER_DEFAULT self.assertEqual(result, incremented_value) @@ -886,6 +888,17 @@ def _counter_inc_helper(self, row, column, value, commit_result): self.assertEqual(row_obj.counts, {tuple(column.split(':')): incremented_value}) + def test_counter_set(self): + name = 'table-name' + connection = None + table = self._makeOne(name, connection) + + row = 'row-key' + column = 'fam:col1' + value = 42 + with self.assertRaises(NotImplementedError): + table.counter_set(row, column, value=value) + def test_counter_inc(self): import struct @@ -1100,7 +1113,7 @@ def test_invalid_type(self): def test_success(self): from gcloud._helpers import _datetime_from_microseconds - from gcloud.bigtable.row import TimestampRange + from gcloud.bigtable.row_filters import TimestampRange timestamp = 1441928298571 ts_dt = _datetime_from_microseconds(1000 * timestamp) @@ -1206,7 +1219,7 @@ def test_no_filters(self): self._callFUT() def test_single_filter(self): - from gcloud.bigtable.row import CellsColumnLimitFilter + from gcloud.bigtable.row_filters import CellsColumnLimitFilter versions = 1337 result = self._callFUT(versions=versions) @@ -1216,7 +1229,7 @@ def test_single_filter(self): self.assertEqual(result.num_cells, versions) def test_existing_filters(self): - from gcloud.bigtable.row import CellsColumnLimitFilter + from gcloud.bigtable.row_filters import CellsColumnLimitFilter filters = [] versions = 1337 @@ -1231,9 +1244,9 @@ def test_existing_filters(self): def _column_helper(self, num_filters, versions=None, timestamp=None, column=None, col_fam=None, qual=None): - from gcloud.bigtable.row import ColumnQualifierRegexFilter - from gcloud.bigtable.row import FamilyNameRegexFilter - from gcloud.bigtable.row import RowFilterChain + from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter + from gcloud.bigtable.row_filters import FamilyNameRegexFilter + from gcloud.bigtable.row_filters import RowFilterChain if col_fam is None: col_fam = 'cf1' @@ -1252,8 +1265,8 @@ def _column_helper(self, num_filters, versions=None, timestamp=None, # Relies on the fact that RowFilter instances can # only have one value set. - self.assertEqual(fam_filter.regex, col_fam) - self.assertEqual(qual_filter.regex, qual) + self.assertEqual(fam_filter.regex, col_fam.encode('utf-8')) + self.assertEqual(qual_filter.regex, qual.encode('utf-8')) return result @@ -1269,7 +1282,7 @@ def test_column_unicode(self): col_fam=u'cfU', qual=u'qualN') def test_with_versions(self): - from gcloud.bigtable.row import CellsColumnLimitFilter + from gcloud.bigtable.row_filters import CellsColumnLimitFilter versions = 11 result = self._column_helper(num_filters=3, versions=versions) @@ -1282,8 +1295,8 @@ def test_with_versions(self): def test_with_timestamp(self): from gcloud._helpers import _datetime_from_microseconds - from gcloud.bigtable.row import TimestampRange - from gcloud.bigtable.row import TimestampRangeFilter + from gcloud.bigtable.row_filters import TimestampRange + from gcloud.bigtable.row_filters import TimestampRangeFilter timestamp = 1441928298571 result = self._column_helper(num_filters=3, timestamp=timestamp) @@ -1317,7 +1330,7 @@ def test_no_columns(self): self._callFUT(columns) def test_single_column(self): - from gcloud.bigtable.row import FamilyNameRegexFilter + from gcloud.bigtable.row_filters import FamilyNameRegexFilter col_fam = 'cf1' columns = [col_fam] @@ -1326,10 +1339,10 @@ def test_single_column(self): self.assertEqual(result, expected_result) def test_column_and_column_families(self): - from gcloud.bigtable.row import ColumnQualifierRegexFilter - from gcloud.bigtable.row import FamilyNameRegexFilter - from gcloud.bigtable.row import RowFilterChain - from gcloud.bigtable.row import RowFilterUnion + from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter + from gcloud.bigtable.row_filters import FamilyNameRegexFilter + from gcloud.bigtable.row_filters import RowFilterChain + from gcloud.bigtable.row_filters import RowFilterUnion col_fam1 = 'cf1' col_fam2 = 'cf2' @@ -1343,14 +1356,14 @@ def test_column_and_column_families(self): filter2 = result.filters[1] self.assertTrue(isinstance(filter1, FamilyNameRegexFilter)) - self.assertEqual(filter1.regex, col_fam1) + self.assertEqual(filter1.regex, col_fam1.encode('utf-8')) self.assertTrue(isinstance(filter2, RowFilterChain)) filter2a, filter2b = filter2.filters self.assertTrue(isinstance(filter2a, FamilyNameRegexFilter)) - self.assertEqual(filter2a.regex, col_fam2) + self.assertEqual(filter2a.regex, col_fam2.encode('utf-8')) self.assertTrue(isinstance(filter2b, ColumnQualifierRegexFilter)) - self.assertEqual(filter2b.regex, col_qual2) + self.assertEqual(filter2b.regex, col_qual2.encode('utf-8')) class Test__row_keys_filter_helper(unittest2.TestCase): @@ -1365,7 +1378,7 @@ def test_no_rows(self): self._callFUT(row_keys) def test_single_row(self): - from gcloud.bigtable.row import RowKeyRegexFilter + from gcloud.bigtable.row_filters import RowKeyRegexFilter row_key = b'row-key' row_keys = [row_key] @@ -1374,8 +1387,8 @@ def test_single_row(self): self.assertEqual(result, expected_result) def test_many_rows(self): - from gcloud.bigtable.row import RowFilterUnion - from gcloud.bigtable.row import RowKeyRegexFilter + from gcloud.bigtable.row_filters import RowFilterUnion + from gcloud.bigtable.row_filters import RowKeyRegexFilter row_key1 = b'row-key1' row_key2 = b'row-key2' @@ -1420,8 +1433,10 @@ def list_column_families(self): self.list_column_families_calls += 1 return self.column_families - def row(self, row_key): - return self.row_values[row_key] + def row(self, row_key, append=None): + result = self.row_values[row_key] + result._append = append + return result def read_row(self, *args, **kwargs): self.read_row_calls.append((args, kwargs)) @@ -1438,6 +1453,7 @@ class _MockLowLevelRow(object): def __init__(self, row_key, commit_result=None): self.row_key = row_key + self._append = False self.counts = {} self.commit_result = commit_result @@ -1446,7 +1462,7 @@ def increment_cell_value(self, column_family_id, column, int_value): self.COUNTER_DEFAULT) self.counts[(column_family_id, column)] = count + int_value - def commit_modifications(self): + def commit(self): return self.commit_result diff --git a/gcloud/bigtable/row.py b/gcloud/bigtable/row.py index d090cb071620..cb9ce2e67e3d 100644 --- a/gcloud/bigtable/row.py +++ b/gcloud/bigtable/row.py @@ -27,100 +27,78 @@ bigtable_service_messages_pb2 as messages_pb2) -_MAX_MUTATIONS = 100000 _PACK_I64 = struct.Struct('>q').pack +MAX_MUTATIONS = 100000 +"""The maximum number of mutations that a row can accumulate.""" + class Row(object): - """Representation of a Google Cloud Bigtable Row. + """Base representation of a Google Cloud Bigtable Row. - .. note:: + This class has three subclasses corresponding to the three + RPC methods for sending row mutations: - A :class:`Row` accumulates mutations locally via the :meth:`set_cell`, - :meth:`delete`, :meth:`delete_cell` and :meth:`delete_cells` methods. - To actually send these mutations to the Google Cloud Bigtable API, you - must call :meth:`commit`. If a ``filter_`` is set on the :class:`Row`, - the mutations must have an associated state: :data:`True` or - :data:`False`. The mutations will be applied conditionally, based on - whether the filter matches any cells in the :class:`Row` or not. + * :class:`DirectRow` for ``MutateRow`` + * :class:`ConditionalRow` for ``CheckAndMutateRow`` + * :class:`AppendRow` for ``ReadModifyWriteRow`` :type row_key: bytes :param row_key: The key for the current row. :type table: :class:`Table ` :param table: The table that owns the row. + """ + + def __init__(self, row_key, table): + self._row_key = _to_bytes(row_key) + self._table = table + + +class _SetDeleteRow(Row): + """Row helper for setting or deleting cell values. - :type filter_: :class:`RowFilter` - :param filter_: (Optional) Filter to be used for conditional mutations. - If a filter is set, then the :class:`Row` will accumulate - mutations for either a :data:`True` or :data:`False` state. - When :meth:`commit`-ed, the mutations for the :data:`True` - state will be applied if the filter matches any cells in - the row, otherwise the :data:`False` state will be. + Implements helper methods to add mutations to set or delete cell contents: + + * :meth:`set_cell` + * :meth:`delete` + * :meth:`delete_cell` + * :meth:`delete_cells` + + :type row_key: bytes + :param row_key: The key for the current row. + + :type table: :class:`Table ` + :param table: The table that owns the row. """ ALL_COLUMNS = object() """Sentinel value used to indicate all columns in a column family.""" - def __init__(self, row_key, table, filter_=None): - self._row_key = _to_bytes(row_key) - self._table = table - self._filter = filter_ - self._rule_pb_list = [] - if self._filter is None: - self._pb_mutations = [] - self._true_pb_mutations = None - self._false_pb_mutations = None - else: - self._pb_mutations = None - self._true_pb_mutations = [] - self._false_pb_mutations = [] - - def _get_mutations(self, state=None): + def _get_mutations(self, state): """Gets the list of mutations for a given state. - If the state is :data`None` but there is a filter set, then we've - reached an invalid state. Similarly if no filter is set but the - state is not :data:`None`. + This method intended to be implemented by subclasses. + + ``state`` may not need to be used by all subclasses. :type state: bool - :param state: (Optional) The state that the mutation should be - applied in. Unset if the mutation is not conditional, - otherwise :data:`True` or :data:`False`. + :param state: The state that the mutation should be + applied in. - :rtype: list - :returns: The list to add new mutations to (for the current state). - :raises: :class:`ValueError ` + :raises: :class:`NotImplementedError ` + always. """ - if state is None: - if self._filter is not None: - raise ValueError('A filter is set on the current row, but no ' - 'state given for the mutation') - return self._pb_mutations - else: - if self._filter is None: - raise ValueError('No filter was set on the current row, but a ' - 'state was given for the mutation') - if state: - return self._true_pb_mutations - else: - return self._false_pb_mutations - - def set_cell(self, column_family_id, column, value, timestamp=None, - state=None): - """Sets a value in this row. + raise NotImplementedError - The cell is determined by the ``row_key`` of the :class:`Row` and the - ``column``. The ``column`` must be in an existing - :class:`.column_family.ColumnFamily` (as determined by - ``column_family_id``). + def _set_cell(self, column_family_id, column, value, timestamp=None, + state=None): + """Helper for :meth:`set_cell` - .. note:: + Adds a mutation to set the value in a specific cell. - This method adds a mutation to the accumulated mutations on this - :class:`Row`, but does not make an API request. To actually - send an API request (with the mutations) to the Google Cloud - Bigtable API, call :meth:`commit`. + ``state`` is unused by :class:`DirectRow` but is used by + subclasses. :type column_family_id: str :param column_family_id: The column family that contains the column. @@ -140,9 +118,8 @@ def set_cell(self, column_family_id, column, value, timestamp=None, :param timestamp: (Optional) The timestamp of the operation. :type state: bool - :param state: (Optional) The state that the mutation should be - applied in. Unset if the mutation is not conditional, - otherwise :data:`True` or :data:`False`. + :param state: (Optional) The state that is passed along to + :meth:`_get_mutations`. """ column = _to_bytes(column) if isinstance(value, six.integer_types): @@ -165,49 +142,143 @@ def set_cell(self, column_family_id, column, value, timestamp=None, mutation_pb = data_pb2.Mutation(set_cell=mutation_val) self._get_mutations(state).append(mutation_pb) - def append_cell_value(self, column_family_id, column, value): - """Appends a value to an existing cell. + def _delete(self, state=None): + """Helper for :meth:`delete` - .. note:: + Adds a delete mutation (for the entire row) to the accumulated + mutations. - This method adds a read-modify rule protobuf to the accumulated - read-modify rules on this :class:`Row`, but does not make an API - request. To actually send an API request (with the rules) to the - Google Cloud Bigtable API, call :meth:`commit_modifications`. + ``state`` is unused by :class:`DirectRow` but is used by + subclasses. + + :type state: bool + :param state: (Optional) The state that is passed along to + :meth:`_get_mutations`. + """ + mutation_val = data_pb2.Mutation.DeleteFromRow() + mutation_pb = data_pb2.Mutation(delete_from_row=mutation_val) + self._get_mutations(state).append(mutation_pb) + + def _delete_cells(self, column_family_id, columns, time_range=None, + state=None): + """Helper for :meth:`delete_cell` and :meth:`delete_cells`. + + ``state`` is unused by :class:`DirectRow` but is used by + subclasses. :type column_family_id: str - :param column_family_id: The column family that contains the column. - Must be of the form - ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. + :param column_family_id: The column family that contains the column + or columns with cells being deleted. Must be + of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - :type column: bytes - :param column: The column within the column family where the cell - is located. + :type columns: :class:`list` of :class:`str` / + :func:`unicode `, or :class:`object` + :param columns: The columns within the column family that will have + cells deleted. If :attr:`ALL_COLUMNS` is used then + the entire column family will be deleted from the row. - :type value: bytes - :param value: The value to append to the existing value in the cell. If - the targeted cell is unset, it will be treated as - containing the empty string. + :type time_range: :class:`TimestampRange` + :param time_range: (Optional) The range of time within which cells + should be deleted. + + :type state: bool + :param state: (Optional) The state that is passed along to + :meth:`_get_mutations`. """ - column = _to_bytes(column) - value = _to_bytes(value) - rule_pb = data_pb2.ReadModifyWriteRule(family_name=column_family_id, - column_qualifier=column, - append_value=value) - self._rule_pb_list.append(rule_pb) + mutations_list = self._get_mutations(state) + if columns is self.ALL_COLUMNS: + mutation_val = data_pb2.Mutation.DeleteFromFamily( + family_name=column_family_id, + ) + mutation_pb = data_pb2.Mutation(delete_from_family=mutation_val) + mutations_list.append(mutation_pb) + else: + delete_kwargs = {} + if time_range is not None: + delete_kwargs['time_range'] = time_range.to_pb() - def increment_cell_value(self, column_family_id, column, int_value): - """Increments a value in an existing cell. + to_append = [] + for column in columns: + column = _to_bytes(column) + # time_range will never change if present, but the rest of + # delete_kwargs will + delete_kwargs.update( + family_name=column_family_id, + column_qualifier=column, + ) + mutation_val = data_pb2.Mutation.DeleteFromColumn( + **delete_kwargs) + mutation_pb = data_pb2.Mutation( + delete_from_column=mutation_val) + to_append.append(mutation_pb) - Assumes the value in the cell is stored as a 64 bit integer - serialized to bytes. + # We don't add the mutations until all columns have been + # processed without error. + mutations_list.extend(to_append) + + +class DirectRow(_SetDeleteRow): + """Google Cloud Bigtable Row for sending "direct" mutations. + + These mutations directly set or delete cell contents: + + * :meth:`set_cell` + * :meth:`delete` + * :meth:`delete_cell` + * :meth:`delete_cells` + + These methods can be used directly:: + + >>> row = table.row(b'row-key1') + >>> row.set_cell(u'fam', b'col1', b'cell-val') + >>> row.delete_cell(u'fam', b'col2') + + .. note:: + + A :class:`DirectRow` accumulates mutations locally via the + :meth:`set_cell`, :meth:`delete`, :meth:`delete_cell` and + :meth:`delete_cells` methods. To actually send these mutations to the + Google Cloud Bigtable API, you must call :meth:`commit`. + + :type row_key: bytes + :param row_key: The key for the current row. + + :type table: :class:`Table ` + :param table: The table that owns the row. + """ + + def __init__(self, row_key, table): + super(DirectRow, self).__init__(row_key, table) + self._pb_mutations = [] + + def _get_mutations(self, state): # pylint: disable=unused-argument + """Gets the list of mutations for a given state. + + ``state`` is unused by :class:`DirectRow` but is used by + subclasses. + + :type state: bool + :param state: The state that the mutation should be + applied in. + + :rtype: list + :returns: The list to add new mutations to (for the current state). + """ + return self._pb_mutations + + def set_cell(self, column_family_id, column, value, timestamp=None): + """Sets a value in this row. + + The cell is determined by the ``row_key`` of this :class:`DirectRow` + and the ``column``. The ``column`` must be in an existing + :class:`.ColumnFamily` (as determined by ``column_family_id``). .. note:: - This method adds a read-modify rule protobuf to the accumulated - read-modify rules on this :class:`Row`, but does not make an API - request. To actually send an API request (with the rules) to the - Google Cloud Bigtable API, call :meth:`commit_modifications`. + This method adds a mutation to the accumulated mutations on this + row, but does not make an API request. To actually + send an API request (with the mutations) to the Google Cloud + Bigtable API, call :meth:`commit`. :type column_family_id: str :param column_family_id: The column family that contains the column. @@ -218,47 +289,36 @@ def increment_cell_value(self, column_family_id, column, int_value): :param column: The column within the column family where the cell is located. - :type int_value: int - :param int_value: The value to increment the existing value in the cell - by. If the targeted cell is unset, it will be treated - as containing a zero. Otherwise, the targeted cell - must contain an 8-byte value (interpreted as a 64-bit - big-endian signed integer), or the entire request - will fail. + :type value: bytes or :class:`int` + :param value: The value to set in the cell. If an integer is used, + will be interpreted as a 64-bit big-endian signed + integer (8 bytes). + + :type timestamp: :class:`datetime.datetime` + :param timestamp: (Optional) The timestamp of the operation. """ - column = _to_bytes(column) - rule_pb = data_pb2.ReadModifyWriteRule(family_name=column_family_id, - column_qualifier=column, - increment_amount=int_value) - self._rule_pb_list.append(rule_pb) + self._set_cell(column_family_id, column, value, timestamp=timestamp, + state=None) - def delete(self, state=None): + def delete(self): """Deletes this row from the table. .. note:: This method adds a mutation to the accumulated mutations on this - :class:`Row`, but does not make an API request. To actually + row, but does not make an API request. To actually send an API request (with the mutations) to the Google Cloud Bigtable API, call :meth:`commit`. - - :type state: bool - :param state: (Optional) The state that the mutation should be - applied in. Unset if the mutation is not conditional, - otherwise :data:`True` or :data:`False`. """ - mutation_val = data_pb2.Mutation.DeleteFromRow() - mutation_pb = data_pb2.Mutation(delete_from_row=mutation_val) - self._get_mutations(state).append(mutation_pb) + self._delete(state=None) - def delete_cell(self, column_family_id, column, time_range=None, - state=None): + def delete_cell(self, column_family_id, column, time_range=None): """Deletes cell in this row. .. note:: This method adds a mutation to the accumulated mutations on this - :class:`Row`, but does not make an API request. To actually + row, but does not make an API request. To actually send an API request (with the mutations) to the Google Cloud Bigtable API, call :meth:`commit`. @@ -274,23 +334,17 @@ def delete_cell(self, column_family_id, column, time_range=None, :type time_range: :class:`TimestampRange` :param time_range: (Optional) The range of time within which cells should be deleted. - - :type state: bool - :param state: (Optional) The state that the mutation should be - applied in. Unset if the mutation is not conditional, - otherwise :data:`True` or :data:`False`. """ - self.delete_cells(column_family_id, [column], time_range=time_range, - state=state) + self._delete_cells(column_family_id, [column], time_range=time_range, + state=None) - def delete_cells(self, column_family_id, columns, time_range=None, - state=None): + def delete_cells(self, column_family_id, columns, time_range=None): """Deletes cells in this row. .. note:: This method adds a mutation to the accumulated mutations on this - :class:`Row`, but does not make an API request. To actually + row, but does not make an API request. To actually send an API request (with the mutations) to the Google Cloud Bigtable API, call :meth:`commit`. @@ -302,65 +356,38 @@ def delete_cells(self, column_family_id, columns, time_range=None, :type columns: :class:`list` of :class:`str` / :func:`unicode `, or :class:`object` :param columns: The columns within the column family that will have - cells deleted. If :attr:`Row.ALL_COLUMNS` is used then + cells deleted. If :attr:`ALL_COLUMNS` is used then the entire column family will be deleted from the row. :type time_range: :class:`TimestampRange` :param time_range: (Optional) The range of time within which cells should be deleted. - - :type state: bool - :param state: (Optional) The state that the mutation should be - applied in. Unset if the mutation is not conditional, - otherwise :data:`True` or :data:`False`. """ - mutations_list = self._get_mutations(state) - if columns is self.ALL_COLUMNS: - mutation_val = data_pb2.Mutation.DeleteFromFamily( - family_name=column_family_id, - ) - mutation_pb = data_pb2.Mutation(delete_from_family=mutation_val) - mutations_list.append(mutation_pb) - else: - delete_kwargs = {} - if time_range is not None: - delete_kwargs['time_range'] = time_range.to_pb() + self._delete_cells(column_family_id, columns, time_range=time_range, + state=None) - to_append = [] - for column in columns: - column = _to_bytes(column) - # time_range will never change if present, but the rest of - # delete_kwargs will - delete_kwargs.update( - family_name=column_family_id, - column_qualifier=column, - ) - mutation_val = data_pb2.Mutation.DeleteFromColumn( - **delete_kwargs) - mutation_pb = data_pb2.Mutation( - delete_from_column=mutation_val) - to_append.append(mutation_pb) + def commit(self): + """Makes a ``MutateRow`` API request. - # We don't add the mutations until all columns have been - # processed without error. - mutations_list.extend(to_append) + If no mutations have been created in the row, no request is made. - def _commit_mutate(self): - """Makes a ``MutateRow`` API request. + Mutations are applied atomically and in order, meaning that earlier + mutations can be masked / negated by later ones. Cells already present + in the row are left unchanged unless explicitly changed by a mutation. - Assumes no filter is set on the :class:`Row` and is meant to be called - by :meth:`commit`. + After committing the accumulated mutations, resets the local + mutations to an empty list. :raises: :class:`ValueError ` if the number of - mutations exceeds the ``_MAX_MUTATIONS``. + mutations exceeds the :data:`MAX_MUTATIONS`. """ - mutations_list = self._get_mutations() + mutations_list = self._get_mutations(None) num_mutations = len(mutations_list) if num_mutations == 0: return - if num_mutations > _MAX_MUTATIONS: + if num_mutations > MAX_MUTATIONS: raise ValueError('%d total mutations exceed the maximum allowable ' - '%d.' % (num_mutations, _MAX_MUTATIONS)) + '%d.' % (num_mutations, MAX_MUTATIONS)) request_pb = messages_pb2.MutateRowRequest( table_name=self._table.name, row_key=self._row_key, @@ -369,18 +396,99 @@ def _commit_mutate(self): # We expect a `google.protobuf.empty_pb2.Empty` client = self._table._cluster._client client._data_stub.MutateRow(request_pb, client.timeout_seconds) + self.clear() + + def clear(self): + """Removes all currently accumulated mutations on the current row.""" + del self._pb_mutations[:] + + +class ConditionalRow(_SetDeleteRow): + """Google Cloud Bigtable Row for sending mutations conditionally. + + Each mutation has an associated state: :data:`True` or :data:`False`. + When :meth:`commit`-ed, the mutations for the :data:`True` + state will be applied if the filter matches any cells in + the row, otherwise the :data:`False` state will be applied. + + A :class:`ConditionalRow` accumulates mutations in the same way a + :class:`DirectRow` does: + + * :meth:`set_cell` + * :meth:`delete` + * :meth:`delete_cell` + * :meth:`delete_cells` + + with the only change the extra ``state`` parameter:: + + >>> row_cond = table.row(b'row-key2', filter_=row_filter) + >>> row_cond.set_cell(u'fam', b'col', b'cell-val', state=True) + >>> row_cond.delete_cell(u'fam', b'col', state=False) + + .. note:: + + As with :class:`DirectRow`, to actually send these mutations to the + Google Cloud Bigtable API, you must call :meth:`commit`. - def _commit_check_and_mutate(self): + :type row_key: bytes + :param row_key: The key for the current row. + + :type table: :class:`Table ` + :param table: The table that owns the row. + + :type filter_: :class:`.RowFilter` + :param filter_: Filter to be used for conditional mutations. + """ + def __init__(self, row_key, table, filter_): + super(ConditionalRow, self).__init__(row_key, table) + self._filter = filter_ + self._true_pb_mutations = [] + self._false_pb_mutations = [] + + def _get_mutations(self, state): + """Gets the list of mutations for a given state. + + Over-ridden so that the state can be used in: + + * :meth:`set_cell` + * :meth:`delete` + * :meth:`delete_cell` + * :meth:`delete_cells` + + :type state: bool + :param state: The state that the mutation should be + applied in. + + :rtype: list + :returns: The list to add new mutations to (for the current state). + """ + if state: + return self._true_pb_mutations + else: + return self._false_pb_mutations + + def commit(self): """Makes a ``CheckAndMutateRow`` API request. - Assumes a filter is set on the :class:`Row` and is meant to be called - by :meth:`commit`. + If no mutations have been created in the row, no request is made. + + The mutations will be applied conditionally, based on whether the + filter matches any cells in the :class:`ConditionalRow` or not. (Each + method which adds a mutation has a ``state`` parameter for this + purpose.) + + Mutations are applied atomically and in order, meaning that earlier + mutations can be masked / negated by later ones. Cells already present + in the row are left unchanged unless explicitly changed by a mutation. + + After committing the accumulated mutations, resets the local + mutations. :rtype: bool :returns: Flag indicating if the filter was matched (which also indicates which set of mutations were applied by the server). :raises: :class:`ValueError ` if the number of - mutations exceeds the ``_MAX_MUTATIONS``. + mutations exceeds the :data:`MAX_MUTATIONS`. """ true_mutations = self._get_mutations(state=True) false_mutations = self._get_mutations(state=False) @@ -388,12 +496,12 @@ def _commit_check_and_mutate(self): num_false_mutations = len(false_mutations) if num_true_mutations == 0 and num_false_mutations == 0: return - if (num_true_mutations > _MAX_MUTATIONS or - num_false_mutations > _MAX_MUTATIONS): + if (num_true_mutations > MAX_MUTATIONS or + num_false_mutations > MAX_MUTATIONS): raise ValueError( 'Exceed the maximum allowable mutations (%d). Had %s true ' 'mutations and %d false mutations.' % ( - _MAX_MUTATIONS, num_true_mutations, num_false_mutations)) + MAX_MUTATIONS, num_true_mutations, num_false_mutations)) request_pb = messages_pb2.CheckAndMutateRowRequest( table_name=self._table.name, @@ -406,854 +514,298 @@ def _commit_check_and_mutate(self): client = self._table._cluster._client resp = client._data_stub.CheckAndMutateRow( request_pb, client.timeout_seconds) + self.clear() return resp.predicate_matched - def clear_mutations(self): - """Removes all currently accumulated mutations on the current row.""" - if self._filter is None: - del self._pb_mutations[:] - else: - del self._true_pb_mutations[:] - del self._false_pb_mutations[:] - - def commit(self): - """Makes a ``MutateRow`` or ``CheckAndMutateRow`` API request. - - If no mutations have been created in the row, no request is made. - - Mutations are applied atomically and in order, meaning that earlier - mutations can be masked / negated by later ones. Cells already present - in the row are left unchanged unless explicitly changed by a mutation. - - After committing the accumulated mutations, resets the local - mutations to an empty list. - - In the case that a filter is set on the :class:`Row`, the mutations - will be applied conditionally, based on whether the filter matches - any cells in the :class:`Row` or not. (Each method which adds a - mutation has a ``state`` parameter for this purpose.) - - :rtype: :class:`bool` or :data:`NoneType ` - :returns: :data:`None` if there is no filter, otherwise a flag - indicating if the filter was matched (which also - indicates which set of mutations were applied by the server). - :raises: :class:`ValueError ` if the number of - mutations exceeds the ``_MAX_MUTATIONS``. - """ - if self._filter is None: - result = self._commit_mutate() - else: - result = self._commit_check_and_mutate() - - # Reset mutations after commit-ing request. - self.clear_mutations() - - return result - - def clear_modification_rules(self): - """Removes all currently accumulated modifications on current row.""" - del self._rule_pb_list[:] - - def commit_modifications(self): - """Makes a ``ReadModifyWriteRow`` API request. - - This commits modifications made by :meth:`append_cell_value` and - :meth:`increment_cell_value`. If no modifications were made, makes - no API request and just returns ``{}``. - - Modifies a row atomically, reading the latest existing timestamp/value - from the specified columns and writing a new value by appending / - incrementing. The new cell created uses either the current server time - or the highest timestamp of a cell in that column (if it exceeds the - server time). - - :rtype: dict - :returns: The new contents of all modified cells. Returned as a - dictionary of column families, each of which holds a - dictionary of columns. Each column contains a list of cells - modified. Each cell is represented with a two-tuple with the - value (in bytes) and the timestamp for the cell. For example: - - .. code:: python - - { - u'col-fam-id': { - b'col-name1': [ - (b'cell-val', datetime.datetime(...)), - (b'cell-val-newer', datetime.datetime(...)), - ], - b'col-name2': [ - (b'altcol-cell-val', datetime.datetime(...)), - ], - }, - u'col-fam-id2': { - b'col-name3-but-other-fam': [ - (b'foo', datetime.datetime(...)), - ], - }, - } - """ - if len(self._rule_pb_list) == 0: - return {} - request_pb = messages_pb2.ReadModifyWriteRowRequest( - table_name=self._table.name, - row_key=self._row_key, - rules=self._rule_pb_list, - ) - # We expect a `.data_pb2.Row` - client = self._table._cluster._client - row_response = client._data_stub.ReadModifyWriteRow( - request_pb, client.timeout_seconds) - - # Reset modifications after commit-ing request. - self.clear_modification_rules() - - # NOTE: We expect row_response.key == self._row_key but don't check. - return _parse_rmw_row_response(row_response) - - -class RowFilter(object): - """Basic filter to apply to cells in a row. - - These values can be combined via :class:`RowFilterChain`, - :class:`RowFilterUnion` and :class:`ConditionalRowFilter`. - - .. note:: - - This class is a do-nothing base class for all row filters. - """ - - def __ne__(self, other): - return not self.__eq__(other) - - -class _BoolFilter(RowFilter): - """Row filter that uses a boolean flag. - - :type flag: bool - :param flag: An indicator if a setting is turned on or off. - """ - - def __init__(self, flag): - self.flag = flag - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.flag == self.flag - - -class SinkFilter(_BoolFilter): - """Advanced row filter to skip parent filters. - - :type flag: bool - :param flag: ADVANCED USE ONLY. Hook for introspection into the row filter. - Outputs all cells directly to the output of the read rather - than to any parent filter. Cannot be used within the - ``predicate_filter``, ``true_filter``, or ``false_filter`` - of a :class:`ConditionalRowFilter`. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(sink=self.flag) - - -class PassAllFilter(_BoolFilter): - """Row filter equivalent to not filtering at all. - - :type flag: bool - :param flag: Matches all cells, regardless of input. Functionally - equivalent to leaving ``filter`` unset, but included for - completeness. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(pass_all_filter=self.flag) - - -class BlockAllFilter(_BoolFilter): - """Row filter that doesn't match any cells. - - :type flag: bool - :param flag: Does not match any cells, regardless of input. Useful for - temporarily disabling just part of a filter. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(block_all_filter=self.flag) - - -class _RegexFilter(RowFilter): - """Row filter that uses a regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - :type regex: bytes or str - :param regex: A regular expression (RE2) for some row filter. - """ - - def __init__(self, regex): - self.regex = regex - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.regex == self.regex - - -class RowKeyRegexFilter(_RegexFilter): - """Row filter for a row key regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - .. note:: - - Special care need be used with the expression used. Since - each of these properties can contain arbitrary bytes, the ``\\C`` - escape sequence must be used if a true wildcard is desired. The ``.`` - character will not match the new line character ``\\n``, which may be - present in a binary value. - - :type regex: bytes - :param regex: A regular expression (RE2) to match cells from rows with row - keys that satisfy this regex. For a - ``CheckAndMutateRowRequest``, this filter is unnecessary - since the row key is already specified. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(row_key_regex_filter=self.regex) - - -class RowSampleFilter(RowFilter): - """Matches all cells from a row with probability p. - - :type sample: float - :param sample: The probability of matching a cell (must be in the - interval ``[0, 1]``). - """ - - def __init__(self, sample): - self.sample = sample - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.sample == self.sample - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(row_sample_filter=self.sample) - - -class FamilyNameRegexFilter(_RegexFilter): - """Row filter for a family name regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - :type regex: str - :param regex: A regular expression (RE2) to match cells from columns in a - given column family. For technical reasons, the regex must - not contain the ``':'`` character, even if it is not being - used as a literal. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(family_name_regex_filter=self.regex) - - -class ColumnQualifierRegexFilter(_RegexFilter): - """Row filter for a column qualifier regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - .. note:: - - Special care need be used with the expression used. Since - each of these properties can contain arbitrary bytes, the ``\\C`` - escape sequence must be used if a true wildcard is desired. The ``.`` - character will not match the new line character ``\\n``, which may be - present in a binary value. - - :type regex: bytes - :param regex: A regular expression (RE2) to match cells from column that - match this regex (irrespective of column family). - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(column_qualifier_regex_filter=self.regex) - - -class TimestampRange(object): - """Range of time with inclusive lower and exclusive upper bounds. - - :type start: :class:`datetime.datetime` - :param start: (Optional) The (inclusive) lower bound of the timestamp - range. If omitted, defaults to Unix epoch. - - :type end: :class:`datetime.datetime` - :param end: (Optional) The (exclusive) upper bound of the timestamp - range. If omitted, no upper bound is used. - """ - - def __init__(self, start=None, end=None): - self.start = start - self.end = end - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.start == self.start and - other.end == self.end) - - def __ne__(self, other): - return not self.__eq__(other) - - def to_pb(self): - """Converts the :class:`TimestampRange` to a protobuf. - - :rtype: :class:`.data_pb2.TimestampRange` - :returns: The converted current object. - """ - timestamp_range_kwargs = {} - if self.start is not None: - timestamp_range_kwargs['start_timestamp_micros'] = ( - _microseconds_from_datetime(self.start)) - if self.end is not None: - timestamp_range_kwargs['end_timestamp_micros'] = ( - _microseconds_from_datetime(self.end)) - return data_pb2.TimestampRange(**timestamp_range_kwargs) - - -class TimestampRangeFilter(RowFilter): - """Row filter that limits cells to a range of time. - - :type range_: :class:`TimestampRange` - :param range_: Range of time that cells should match against. - """ - - def __init__(self, range_): - self.range_ = range_ - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.range_ == self.range_ - - def to_pb(self): - """Converts the row filter to a protobuf. - - First converts the ``range_`` on the current object to a protobuf and - then uses it in the ``timestamp_range_filter`` field. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(timestamp_range_filter=self.range_.to_pb()) - - -class ColumnRangeFilter(RowFilter): - """A row filter to restrict to a range of columns. + # pylint: disable=arguments-differ + def set_cell(self, column_family_id, column, value, timestamp=None, + state=True): + """Sets a value in this row. - Both the start and end column can be included or excluded in the range. - By default, we include them both, but this can be changed with optional - flags. + The cell is determined by the ``row_key`` of this + :class:`ConditionalRow` and the ``column``. The ``column`` must be in + an existing :class:`.ColumnFamily` (as determined by + ``column_family_id``). - :type column_family_id: str - :param column_family_id: The column family that contains the columns. Must - be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. + .. note:: - :type start_column: bytes - :param start_column: The start of the range of columns. If no value is - used, the backend applies no upper bound to the - values. + This method adds a mutation to the accumulated mutations on this + row, but does not make an API request. To actually + send an API request (with the mutations) to the Google Cloud + Bigtable API, call :meth:`commit`. - :type end_column: bytes - :param end_column: The end of the range of columns. If no value is used, - the backend applies no upper bound to the values. + :type column_family_id: str + :param column_family_id: The column family that contains the column. + Must be of the form + ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - :type inclusive_start: bool - :param inclusive_start: Boolean indicating if the start column should be - included in the range (or excluded). Defaults - to :data:`True` if ``start_column`` is passed and - no ``inclusive_start`` was given. + :type column: bytes + :param column: The column within the column family where the cell + is located. - :type inclusive_end: bool - :param inclusive_end: Boolean indicating if the end column should be - included in the range (or excluded). Defaults - to :data:`True` if ``end_column`` is passed and - no ``inclusive_end`` was given. + :type value: bytes or :class:`int` + :param value: The value to set in the cell. If an integer is used, + will be interpreted as a 64-bit big-endian signed + integer (8 bytes). - :raises: :class:`ValueError ` if ``inclusive_start`` - is set but no ``start_column`` is given or if ``inclusive_end`` - is set but no ``end_column`` is given - """ + :type timestamp: :class:`datetime.datetime` + :param timestamp: (Optional) The timestamp of the operation. - def __init__(self, column_family_id, start_column=None, end_column=None, - inclusive_start=None, inclusive_end=None): - self.column_family_id = column_family_id - - if inclusive_start is None: - inclusive_start = True - elif start_column is None: - raise ValueError('Inclusive start was specified but no ' - 'start column was given.') - self.start_column = start_column - self.inclusive_start = inclusive_start - - if inclusive_end is None: - inclusive_end = True - elif end_column is None: - raise ValueError('Inclusive end was specified but no ' - 'end column was given.') - self.end_column = end_column - self.inclusive_end = inclusive_end - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.column_family_id == self.column_family_id and - other.start_column == self.start_column and - other.end_column == self.end_column and - other.inclusive_start == self.inclusive_start and - other.inclusive_end == self.inclusive_end) - - def to_pb(self): - """Converts the row filter to a protobuf. - - First converts to a :class:`.data_pb2.ColumnRange` and then uses it - in the ``column_range_filter`` field. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. + :type state: bool + :param state: (Optional) The state that the mutation should be + applied in. Defaults to :data:`True`. """ - column_range_kwargs = {'family_name': self.column_family_id} - if self.start_column is not None: - if self.inclusive_start: - key = 'start_qualifier_inclusive' - else: - key = 'start_qualifier_exclusive' - column_range_kwargs[key] = _to_bytes(self.start_column) - if self.end_column is not None: - if self.inclusive_end: - key = 'end_qualifier_inclusive' - else: - key = 'end_qualifier_exclusive' - column_range_kwargs[key] = _to_bytes(self.end_column) + self._set_cell(column_family_id, column, value, timestamp=timestamp, + state=state) - column_range = data_pb2.ColumnRange(**column_range_kwargs) - return data_pb2.RowFilter(column_range_filter=column_range) - - -class ValueRegexFilter(_RegexFilter): - """Row filter for a value regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - .. note:: - - Special care need be used with the expression used. Since - each of these properties can contain arbitrary bytes, the ``\\C`` - escape sequence must be used if a true wildcard is desired. The ``.`` - character will not match the new line character ``\\n``, which may be - present in a binary value. + def delete(self, state=True): + """Deletes this row from the table. - :type regex: bytes - :param regex: A regular expression (RE2) to match cells with values that - match this regex. - """ + .. note:: - def to_pb(self): - """Converts the row filter to a protobuf. + This method adds a mutation to the accumulated mutations on this + row, but does not make an API request. To actually + send an API request (with the mutations) to the Google Cloud + Bigtable API, call :meth:`commit`. - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. + :type state: bool + :param state: (Optional) The state that the mutation should be + applied in. Defaults to :data:`True`. """ - return data_pb2.RowFilter(value_regex_filter=self.regex) - - -class ValueRangeFilter(RowFilter): - """A range of values to restrict to in a row filter. - - Will only match cells that have values in this range. + self._delete(state=state) - Both the start and end value can be included or excluded in the range. - By default, we include them both, but this can be changed with optional - flags. + def delete_cell(self, column_family_id, column, time_range=None, + state=True): + """Deletes cell in this row. - :type start_value: bytes - :param start_value: The start of the range of values. If no value is used, - the backend applies no lower bound to the values. + .. note:: - :type end_value: bytes - :param end_value: The end of the range of values. If no value is used, - the backend applies no upper bound to the values. + This method adds a mutation to the accumulated mutations on this + row, but does not make an API request. To actually + send an API request (with the mutations) to the Google Cloud + Bigtable API, call :meth:`commit`. - :type inclusive_start: bool - :param inclusive_start: Boolean indicating if the start value should be - included in the range (or excluded). Defaults - to :data:`True` if ``start_value`` is passed and - no ``inclusive_start`` was given. + :type column_family_id: str + :param column_family_id: The column family that contains the column + or columns with cells being deleted. Must be + of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - :type inclusive_end: bool - :param inclusive_end: Boolean indicating if the end value should be - included in the range (or excluded). Defaults - to :data:`True` if ``end_value`` is passed and - no ``inclusive_end`` was given. + :type column: bytes + :param column: The column within the column family that will have a + cell deleted. - :raises: :class:`ValueError ` if ``inclusive_start`` - is set but no ``start_value`` is given or if ``inclusive_end`` - is set but no ``end_value`` is given - """ + :type time_range: :class:`TimestampRange` + :param time_range: (Optional) The range of time within which cells + should be deleted. - def __init__(self, start_value=None, end_value=None, - inclusive_start=None, inclusive_end=None): - if inclusive_start is None: - inclusive_start = True - elif start_value is None: - raise ValueError('Inclusive start was specified but no ' - 'start value was given.') - self.start_value = start_value - self.inclusive_start = inclusive_start - - if inclusive_end is None: - inclusive_end = True - elif end_value is None: - raise ValueError('Inclusive end was specified but no ' - 'end value was given.') - self.end_value = end_value - self.inclusive_end = inclusive_end - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.start_value == self.start_value and - other.end_value == self.end_value and - other.inclusive_start == self.inclusive_start and - other.inclusive_end == self.inclusive_end) - - def to_pb(self): - """Converts the row filter to a protobuf. - - First converts to a :class:`.data_pb2.ValueRange` and then uses - it to create a row filter protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. + :type state: bool + :param state: (Optional) The state that the mutation should be + applied in. Defaults to :data:`True`. """ - value_range_kwargs = {} - if self.start_value is not None: - if self.inclusive_start: - key = 'start_value_inclusive' - else: - key = 'start_value_exclusive' - value_range_kwargs[key] = _to_bytes(self.start_value) - if self.end_value is not None: - if self.inclusive_end: - key = 'end_value_inclusive' - else: - key = 'end_value_exclusive' - value_range_kwargs[key] = _to_bytes(self.end_value) - - value_range = data_pb2.ValueRange(**value_range_kwargs) - return data_pb2.RowFilter(value_range_filter=value_range) - - -class _CellCountFilter(RowFilter): - """Row filter that uses an integer count of cells. - - The cell count is used as an offset or a limit for the number - of results returned. - - :type num_cells: int - :param num_cells: An integer count / offset / limit. - """ - - def __init__(self, num_cells): - self.num_cells = num_cells - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.num_cells == self.num_cells + self._delete_cells(column_family_id, [column], time_range=time_range, + state=state) + def delete_cells(self, column_family_id, columns, time_range=None, + state=True): + """Deletes cells in this row. -class CellsRowOffsetFilter(_CellCountFilter): - """Row filter to skip cells in a row. - - :type num_cells: int - :param num_cells: Skips the first N cells of the row. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(cells_per_row_offset_filter=self.num_cells) + .. note:: + This method adds a mutation to the accumulated mutations on this + row, but does not make an API request. To actually + send an API request (with the mutations) to the Google Cloud + Bigtable API, call :meth:`commit`. -class CellsRowLimitFilter(_CellCountFilter): - """Row filter to limit cells in a row. + :type column_family_id: str + :param column_family_id: The column family that contains the column + or columns with cells being deleted. Must be + of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - :type num_cells: int - :param num_cells: Matches only the first N cells of the row. - """ + :type columns: :class:`list` of :class:`str` / + :func:`unicode `, or :class:`object` + :param columns: The columns within the column family that will have + cells deleted. If :attr:`ALL_COLUMNS` is used then the + entire column family will be deleted from the row. - def to_pb(self): - """Converts the row filter to a protobuf. + :type time_range: :class:`TimestampRange` + :param time_range: (Optional) The range of time within which cells + should be deleted. - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. + :type state: bool + :param state: (Optional) The state that the mutation should be + applied in. Defaults to :data:`True`. """ - return data_pb2.RowFilter(cells_per_row_limit_filter=self.num_cells) + self._delete_cells(column_family_id, columns, time_range=time_range, + state=state) + # pylint: enable=arguments-differ + def clear(self): + """Removes all currently accumulated mutations on the current row.""" + del self._true_pb_mutations[:] + del self._false_pb_mutations[:] -class CellsColumnLimitFilter(_CellCountFilter): - """Row filter to limit cells in a column. - :type num_cells: int - :param num_cells: Matches only the most recent N cells within each column. - This filters a (family name, column) pair, based on - timestamps of each cell. - """ +class AppendRow(Row): + """Google Cloud Bigtable Row for sending append mutations. - def to_pb(self): - """Converts the row filter to a protobuf. + These mutations are intended to augment the value of an existing cell + and uses the methods: - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(cells_per_column_limit_filter=self.num_cells) + * :meth:`append_cell_value` + * :meth:`increment_cell_value` + The first works by appending bytes and the second by incrementing an + integer (stored in the cell as 8 bytes). In either case, if the + cell is empty, assumes the default empty value (empty string for + bytes or and 0 for integer). -class StripValueTransformerFilter(_BoolFilter): - """Row filter that transforms cells into empty string (0 bytes). + :type row_key: bytes + :param row_key: The key for the current row. - :type flag: bool - :param flag: If :data:`True`, replaces each cell's value with the empty - string. As the name indicates, this is more useful as a - transformer than a generic query / filter. + :type table: :class:`Table ` + :param table: The table that owns the row. """ - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(strip_value_transformer=self.flag) - - -class ApplyLabelFilter(RowFilter): - """Filter to apply labels to cells. - - Intended to be used as an intermediate filter on a pre-existing filtered - result set. This was if two sets are combined, the label can tell where - the cell(s) originated.This allows the client to determine which results - were produced from which part of the filter. + def __init__(self, row_key, table): + super(AppendRow, self).__init__(row_key, table) + self._rule_pb_list = [] - .. note:: + def clear(self): + """Removes all currently accumulated modifications on current row.""" + del self._rule_pb_list[:] - Due to a technical limitation, it is not currently possible to apply - multiple labels to a cell. + def append_cell_value(self, column_family_id, column, value): + """Appends a value to an existing cell. - :type label: str - :param label: Label to apply to cells in the output row. Values must be - at most 15 characters long, and match the pattern - ``[a-z0-9\\-]+``. - """ + .. note:: - def __init__(self, label): - self.label = label + This method adds a read-modify rule protobuf to the accumulated + read-modify rules on this row, but does not make an API + request. To actually send an API request (with the rules) to the + Google Cloud Bigtable API, call :meth:`commit`. - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.label == self.label + :type column_family_id: str + :param column_family_id: The column family that contains the column. + Must be of the form + ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - def to_pb(self): - """Converts the row filter to a protobuf. + :type column: bytes + :param column: The column within the column family where the cell + is located. - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. + :type value: bytes + :param value: The value to append to the existing value in the cell. If + the targeted cell is unset, it will be treated as + containing the empty string. """ - return data_pb2.RowFilter(apply_label_transformer=self.label) - - -class _FilterCombination(RowFilter): - """Chain of row filters. - - Sends rows through several filters in sequence. The filters are "chained" - together to process a row. After the first filter is applied, the second - is applied to the filtered output and so on for subsequent filters. - - :type filters: list - :param filters: List of :class:`RowFilter` - """ - - def __init__(self, filters=None): - if filters is None: - filters = [] - self.filters = filters + column = _to_bytes(column) + value = _to_bytes(value) + rule_pb = data_pb2.ReadModifyWriteRule(family_name=column_family_id, + column_qualifier=column, + append_value=value) + self._rule_pb_list.append(rule_pb) - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.filters == self.filters + def increment_cell_value(self, column_family_id, column, int_value): + """Increments a value in an existing cell. + Assumes the value in the cell is stored as a 64 bit integer + serialized to bytes. -class RowFilterChain(_FilterCombination): - """Chain of row filters. + .. note:: - Sends rows through several filters in sequence. The filters are "chained" - together to process a row. After the first filter is applied, the second - is applied to the filtered output and so on for subsequent filters. + This method adds a read-modify rule protobuf to the accumulated + read-modify rules on this row, but does not make an API + request. To actually send an API request (with the rules) to the + Google Cloud Bigtable API, call :meth:`commit`. - :type filters: list - :param filters: List of :class:`RowFilter` - """ + :type column_family_id: str + :param column_family_id: The column family that contains the column. + Must be of the form + ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - def to_pb(self): - """Converts the row filter to a protobuf. + :type column: bytes + :param column: The column within the column family where the cell + is located. - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. + :type int_value: int + :param int_value: The value to increment the existing value in the cell + by. If the targeted cell is unset, it will be treated + as containing a zero. Otherwise, the targeted cell + must contain an 8-byte value (interpreted as a 64-bit + big-endian signed integer), or the entire request + will fail. """ - chain = data_pb2.RowFilter.Chain( - filters=[row_filter.to_pb() for row_filter in self.filters]) - return data_pb2.RowFilter(chain=chain) - - -class RowFilterUnion(_FilterCombination): - """Union of row filters. - - Sends rows through several filters simultaneously, then - merges / interleaves all the filtered results together. + column = _to_bytes(column) + rule_pb = data_pb2.ReadModifyWriteRule(family_name=column_family_id, + column_qualifier=column, + increment_amount=int_value) + self._rule_pb_list.append(rule_pb) - If multiple cells are produced with the same column and timestamp, - they will all appear in the output row in an unspecified mutual order. + def commit(self): + """Makes a ``ReadModifyWriteRow`` API request. - :type filters: list - :param filters: List of :class:`RowFilter` - """ + This commits modifications made by :meth:`append_cell_value` and + :meth:`increment_cell_value`. If no modifications were made, makes + no API request and just returns ``{}``. - def to_pb(self): - """Converts the row filter to a protobuf. + Modifies a row atomically, reading the latest existing + timestamp / value from the specified columns and writing a new value by + appending / incrementing. The new cell created uses either the current + server time or the highest timestamp of a cell in that column (if it + exceeds the server time). + + After committing the accumulated mutations, resets the local mutations. + + .. code:: python + + >>> append_row.commit() + { + u'col-fam-id': { + b'col-name1': [ + (b'cell-val', datetime.datetime(...)), + (b'cell-val-newer', datetime.datetime(...)), + ], + b'col-name2': [ + (b'altcol-cell-val', datetime.datetime(...)), + ], + }, + u'col-fam-id2': { + b'col-name3-but-other-fam': [ + (b'foo', datetime.datetime(...)), + ], + }, + } - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. + :rtype: dict + :returns: The new contents of all modified cells. Returned as a + dictionary of column families, each of which holds a + dictionary of columns. Each column contains a list of cells + modified. Each cell is represented with a two-tuple with the + value (in bytes) and the timestamp for the cell. + :raises: :class:`ValueError ` if the number of + mutations exceeds the :data:`MAX_MUTATIONS`. """ - interleave = data_pb2.RowFilter.Interleave( - filters=[row_filter.to_pb() for row_filter in self.filters]) - return data_pb2.RowFilter(interleave=interleave) - - -class ConditionalRowFilter(RowFilter): - """Conditional row filter which exhibits ternary behavior. - - Executes one of two filters based on another filter. If the ``base_filter`` - returns any cells in the row, then ``true_filter`` is executed. If not, - then ``false_filter`` is executed. - - .. note:: - - The ``base_filter`` does not execute atomically with the true and false - filters, which may lead to inconsistent or unexpected results. - - Additionally, executing a :class:`ConditionalRowFilter` has poor - performance on the server, especially when ``false_filter`` is set. - - :type base_filter: :class:`RowFilter` - :param base_filter: The filter to condition on before executing the - true/false filters. - - :type true_filter: :class:`RowFilter` - :param true_filter: (Optional) The filter to execute if there are any cells - matching ``base_filter``. If not provided, no results - will be returned in the true case. - - :type false_filter: :class:`RowFilter` - :param false_filter: (Optional) The filter to execute if there are no cells - matching ``base_filter``. If not provided, no results - will be returned in the false case. - """ - - def __init__(self, base_filter, true_filter=None, false_filter=None): - self.base_filter = base_filter - self.true_filter = true_filter - self.false_filter = false_filter - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.base_filter == self.base_filter and - other.true_filter == self.true_filter and - other.false_filter == self.false_filter) + num_mutations = len(self._rule_pb_list) + if num_mutations == 0: + return {} + if num_mutations > MAX_MUTATIONS: + raise ValueError('%d total append mutations exceed the maximum ' + 'allowable %d.' % (num_mutations, MAX_MUTATIONS)) + request_pb = messages_pb2.ReadModifyWriteRowRequest( + table_name=self._table.name, + row_key=self._row_key, + rules=self._rule_pb_list, + ) + # We expect a `.data_pb2.Row` + client = self._table._cluster._client + row_response = client._data_stub.ReadModifyWriteRow( + request_pb, client.timeout_seconds) - def to_pb(self): - """Converts the row filter to a protobuf. + # Reset modifications after commit-ing request. + self.clear() - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - condition_kwargs = {'predicate_filter': self.base_filter.to_pb()} - if self.true_filter is not None: - condition_kwargs['true_filter'] = self.true_filter.to_pb() - if self.false_filter is not None: - condition_kwargs['false_filter'] = self.false_filter.to_pb() - condition = data_pb2.RowFilter.Condition(**condition_kwargs) - return data_pb2.RowFilter(condition=condition) + # NOTE: We expect row_response.key == self._row_key but don't check. + return _parse_rmw_row_response(row_response) def _parse_rmw_row_response(row_response): diff --git a/gcloud/bigtable/row_filters.py b/gcloud/bigtable/row_filters.py new file mode 100644 index 000000000000..b7a1388b3a09 --- /dev/null +++ b/gcloud/bigtable/row_filters.py @@ -0,0 +1,764 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Filters for Google Cloud Bigtable Row classes.""" + + +from gcloud._helpers import _microseconds_from_datetime +from gcloud._helpers import _to_bytes +from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + +class RowFilter(object): + """Basic filter to apply to cells in a row. + + These values can be combined via :class:`RowFilterChain`, + :class:`RowFilterUnion` and :class:`ConditionalRowFilter`. + + .. note:: + + This class is a do-nothing base class for all row filters. + """ + + def __ne__(self, other): + return not self.__eq__(other) + + +class _BoolFilter(RowFilter): + """Row filter that uses a boolean flag. + + :type flag: bool + :param flag: An indicator if a setting is turned on or off. + """ + + def __init__(self, flag): + self.flag = flag + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.flag == self.flag + + +class SinkFilter(_BoolFilter): + """Advanced row filter to skip parent filters. + + :type flag: bool + :param flag: ADVANCED USE ONLY. Hook for introspection into the row filter. + Outputs all cells directly to the output of the read rather + than to any parent filter. Cannot be used within the + ``predicate_filter``, ``true_filter``, or ``false_filter`` + of a :class:`ConditionalRowFilter`. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(sink=self.flag) + + +class PassAllFilter(_BoolFilter): + """Row filter equivalent to not filtering at all. + + :type flag: bool + :param flag: Matches all cells, regardless of input. Functionally + equivalent to leaving ``filter`` unset, but included for + completeness. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(pass_all_filter=self.flag) + + +class BlockAllFilter(_BoolFilter): + """Row filter that doesn't match any cells. + + :type flag: bool + :param flag: Does not match any cells, regardless of input. Useful for + temporarily disabling just part of a filter. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(block_all_filter=self.flag) + + +class _RegexFilter(RowFilter): + """Row filter that uses a regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + :type regex: bytes or str + :param regex: A regular expression (RE2) for some row filter. + """ + + def __init__(self, regex): + self.regex = _to_bytes(regex) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.regex == self.regex + + +class RowKeyRegexFilter(_RegexFilter): + """Row filter for a row key regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + .. note:: + + Special care need be used with the expression used. Since + each of these properties can contain arbitrary bytes, the ``\\C`` + escape sequence must be used if a true wildcard is desired. The ``.`` + character will not match the new line character ``\\n``, which may be + present in a binary value. + + :type regex: bytes + :param regex: A regular expression (RE2) to match cells from rows with row + keys that satisfy this regex. For a + ``CheckAndMutateRowRequest``, this filter is unnecessary + since the row key is already specified. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(row_key_regex_filter=self.regex) + + +class RowSampleFilter(RowFilter): + """Matches all cells from a row with probability p. + + :type sample: float + :param sample: The probability of matching a cell (must be in the + interval ``[0, 1]``). + """ + + def __init__(self, sample): + self.sample = sample + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.sample == self.sample + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(row_sample_filter=self.sample) + + +class FamilyNameRegexFilter(_RegexFilter): + """Row filter for a family name regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + :type regex: str + :param regex: A regular expression (RE2) to match cells from columns in a + given column family. For technical reasons, the regex must + not contain the ``':'`` character, even if it is not being + used as a literal. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(family_name_regex_filter=self.regex) + + +class ColumnQualifierRegexFilter(_RegexFilter): + """Row filter for a column qualifier regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + .. note:: + + Special care need be used with the expression used. Since + each of these properties can contain arbitrary bytes, the ``\\C`` + escape sequence must be used if a true wildcard is desired. The ``.`` + character will not match the new line character ``\\n``, which may be + present in a binary value. + + :type regex: bytes + :param regex: A regular expression (RE2) to match cells from column that + match this regex (irrespective of column family). + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(column_qualifier_regex_filter=self.regex) + + +class TimestampRange(object): + """Range of time with inclusive lower and exclusive upper bounds. + + :type start: :class:`datetime.datetime` + :param start: (Optional) The (inclusive) lower bound of the timestamp + range. If omitted, defaults to Unix epoch. + + :type end: :class:`datetime.datetime` + :param end: (Optional) The (exclusive) upper bound of the timestamp + range. If omitted, no upper bound is used. + """ + + def __init__(self, start=None, end=None): + self.start = start + self.end = end + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return (other.start == self.start and + other.end == self.end) + + def __ne__(self, other): + return not self.__eq__(other) + + def to_pb(self): + """Converts the :class:`TimestampRange` to a protobuf. + + :rtype: :class:`.data_pb2.TimestampRange` + :returns: The converted current object. + """ + timestamp_range_kwargs = {} + if self.start is not None: + timestamp_range_kwargs['start_timestamp_micros'] = ( + _microseconds_from_datetime(self.start)) + if self.end is not None: + timestamp_range_kwargs['end_timestamp_micros'] = ( + _microseconds_from_datetime(self.end)) + return data_pb2.TimestampRange(**timestamp_range_kwargs) + + +class TimestampRangeFilter(RowFilter): + """Row filter that limits cells to a range of time. + + :type range_: :class:`TimestampRange` + :param range_: Range of time that cells should match against. + """ + + def __init__(self, range_): + self.range_ = range_ + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.range_ == self.range_ + + def to_pb(self): + """Converts the row filter to a protobuf. + + First converts the ``range_`` on the current object to a protobuf and + then uses it in the ``timestamp_range_filter`` field. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(timestamp_range_filter=self.range_.to_pb()) + + +class ColumnRangeFilter(RowFilter): + """A row filter to restrict to a range of columns. + + Both the start and end column can be included or excluded in the range. + By default, we include them both, but this can be changed with optional + flags. + + :type column_family_id: str + :param column_family_id: The column family that contains the columns. Must + be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. + + :type start_column: bytes + :param start_column: The start of the range of columns. If no value is + used, the backend applies no upper bound to the + values. + + :type end_column: bytes + :param end_column: The end of the range of columns. If no value is used, + the backend applies no upper bound to the values. + + :type inclusive_start: bool + :param inclusive_start: Boolean indicating if the start column should be + included in the range (or excluded). Defaults + to :data:`True` if ``start_column`` is passed and + no ``inclusive_start`` was given. + + :type inclusive_end: bool + :param inclusive_end: Boolean indicating if the end column should be + included in the range (or excluded). Defaults + to :data:`True` if ``end_column`` is passed and + no ``inclusive_end`` was given. + + :raises: :class:`ValueError ` if ``inclusive_start`` + is set but no ``start_column`` is given or if ``inclusive_end`` + is set but no ``end_column`` is given + """ + + def __init__(self, column_family_id, start_column=None, end_column=None, + inclusive_start=None, inclusive_end=None): + self.column_family_id = column_family_id + + if inclusive_start is None: + inclusive_start = True + elif start_column is None: + raise ValueError('Inclusive start was specified but no ' + 'start column was given.') + self.start_column = start_column + self.inclusive_start = inclusive_start + + if inclusive_end is None: + inclusive_end = True + elif end_column is None: + raise ValueError('Inclusive end was specified but no ' + 'end column was given.') + self.end_column = end_column + self.inclusive_end = inclusive_end + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return (other.column_family_id == self.column_family_id and + other.start_column == self.start_column and + other.end_column == self.end_column and + other.inclusive_start == self.inclusive_start and + other.inclusive_end == self.inclusive_end) + + def to_pb(self): + """Converts the row filter to a protobuf. + + First converts to a :class:`.data_pb2.ColumnRange` and then uses it + in the ``column_range_filter`` field. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + column_range_kwargs = {'family_name': self.column_family_id} + if self.start_column is not None: + if self.inclusive_start: + key = 'start_qualifier_inclusive' + else: + key = 'start_qualifier_exclusive' + column_range_kwargs[key] = _to_bytes(self.start_column) + if self.end_column is not None: + if self.inclusive_end: + key = 'end_qualifier_inclusive' + else: + key = 'end_qualifier_exclusive' + column_range_kwargs[key] = _to_bytes(self.end_column) + + column_range = data_pb2.ColumnRange(**column_range_kwargs) + return data_pb2.RowFilter(column_range_filter=column_range) + + +class ValueRegexFilter(_RegexFilter): + """Row filter for a value regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + .. note:: + + Special care need be used with the expression used. Since + each of these properties can contain arbitrary bytes, the ``\\C`` + escape sequence must be used if a true wildcard is desired. The ``.`` + character will not match the new line character ``\\n``, which may be + present in a binary value. + + :type regex: bytes + :param regex: A regular expression (RE2) to match cells with values that + match this regex. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(value_regex_filter=self.regex) + + +class ValueRangeFilter(RowFilter): + """A range of values to restrict to in a row filter. + + Will only match cells that have values in this range. + + Both the start and end value can be included or excluded in the range. + By default, we include them both, but this can be changed with optional + flags. + + :type start_value: bytes + :param start_value: The start of the range of values. If no value is used, + the backend applies no lower bound to the values. + + :type end_value: bytes + :param end_value: The end of the range of values. If no value is used, + the backend applies no upper bound to the values. + + :type inclusive_start: bool + :param inclusive_start: Boolean indicating if the start value should be + included in the range (or excluded). Defaults + to :data:`True` if ``start_value`` is passed and + no ``inclusive_start`` was given. + + :type inclusive_end: bool + :param inclusive_end: Boolean indicating if the end value should be + included in the range (or excluded). Defaults + to :data:`True` if ``end_value`` is passed and + no ``inclusive_end`` was given. + + :raises: :class:`ValueError ` if ``inclusive_start`` + is set but no ``start_value`` is given or if ``inclusive_end`` + is set but no ``end_value`` is given + """ + + def __init__(self, start_value=None, end_value=None, + inclusive_start=None, inclusive_end=None): + if inclusive_start is None: + inclusive_start = True + elif start_value is None: + raise ValueError('Inclusive start was specified but no ' + 'start value was given.') + self.start_value = start_value + self.inclusive_start = inclusive_start + + if inclusive_end is None: + inclusive_end = True + elif end_value is None: + raise ValueError('Inclusive end was specified but no ' + 'end value was given.') + self.end_value = end_value + self.inclusive_end = inclusive_end + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return (other.start_value == self.start_value and + other.end_value == self.end_value and + other.inclusive_start == self.inclusive_start and + other.inclusive_end == self.inclusive_end) + + def to_pb(self): + """Converts the row filter to a protobuf. + + First converts to a :class:`.data_pb2.ValueRange` and then uses + it to create a row filter protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + value_range_kwargs = {} + if self.start_value is not None: + if self.inclusive_start: + key = 'start_value_inclusive' + else: + key = 'start_value_exclusive' + value_range_kwargs[key] = _to_bytes(self.start_value) + if self.end_value is not None: + if self.inclusive_end: + key = 'end_value_inclusive' + else: + key = 'end_value_exclusive' + value_range_kwargs[key] = _to_bytes(self.end_value) + + value_range = data_pb2.ValueRange(**value_range_kwargs) + return data_pb2.RowFilter(value_range_filter=value_range) + + +class _CellCountFilter(RowFilter): + """Row filter that uses an integer count of cells. + + The cell count is used as an offset or a limit for the number + of results returned. + + :type num_cells: int + :param num_cells: An integer count / offset / limit. + """ + + def __init__(self, num_cells): + self.num_cells = num_cells + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.num_cells == self.num_cells + + +class CellsRowOffsetFilter(_CellCountFilter): + """Row filter to skip cells in a row. + + :type num_cells: int + :param num_cells: Skips the first N cells of the row. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(cells_per_row_offset_filter=self.num_cells) + + +class CellsRowLimitFilter(_CellCountFilter): + """Row filter to limit cells in a row. + + :type num_cells: int + :param num_cells: Matches only the first N cells of the row. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(cells_per_row_limit_filter=self.num_cells) + + +class CellsColumnLimitFilter(_CellCountFilter): + """Row filter to limit cells in a column. + + :type num_cells: int + :param num_cells: Matches only the most recent N cells within each column. + This filters a (family name, column) pair, based on + timestamps of each cell. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(cells_per_column_limit_filter=self.num_cells) + + +class StripValueTransformerFilter(_BoolFilter): + """Row filter that transforms cells into empty string (0 bytes). + + :type flag: bool + :param flag: If :data:`True`, replaces each cell's value with the empty + string. As the name indicates, this is more useful as a + transformer than a generic query / filter. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(strip_value_transformer=self.flag) + + +class ApplyLabelFilter(RowFilter): + """Filter to apply labels to cells. + + Intended to be used as an intermediate filter on a pre-existing filtered + result set. This way if two sets are combined, the label can tell where + the cell(s) originated.This allows the client to determine which results + were produced from which part of the filter. + + .. note:: + + Due to a technical limitation of the backend, it is not currently + possible to apply multiple labels to a cell. + + :type label: str + :param label: Label to apply to cells in the output row. Values must be + at most 15 characters long, and match the pattern + ``[a-z0-9\\-]+``. + """ + + def __init__(self, label): + self.label = label + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.label == self.label + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(apply_label_transformer=self.label) + + +class _FilterCombination(RowFilter): + """Chain of row filters. + + Sends rows through several filters in sequence. The filters are "chained" + together to process a row. After the first filter is applied, the second + is applied to the filtered output and so on for subsequent filters. + + :type filters: list + :param filters: List of :class:`RowFilter` + """ + + def __init__(self, filters=None): + if filters is None: + filters = [] + self.filters = filters + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.filters == self.filters + + +class RowFilterChain(_FilterCombination): + """Chain of row filters. + + Sends rows through several filters in sequence. The filters are "chained" + together to process a row. After the first filter is applied, the second + is applied to the filtered output and so on for subsequent filters. + + :type filters: list + :param filters: List of :class:`RowFilter` + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + chain = data_pb2.RowFilter.Chain( + filters=[row_filter.to_pb() for row_filter in self.filters]) + return data_pb2.RowFilter(chain=chain) + + +class RowFilterUnion(_FilterCombination): + """Union of row filters. + + Sends rows through several filters simultaneously, then + merges / interleaves all the filtered results together. + + If multiple cells are produced with the same column and timestamp, + they will all appear in the output row in an unspecified mutual order. + + :type filters: list + :param filters: List of :class:`RowFilter` + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + interleave = data_pb2.RowFilter.Interleave( + filters=[row_filter.to_pb() for row_filter in self.filters]) + return data_pb2.RowFilter(interleave=interleave) + + +class ConditionalRowFilter(RowFilter): + """Conditional row filter which exhibits ternary behavior. + + Executes one of two filters based on another filter. If the ``base_filter`` + returns any cells in the row, then ``true_filter`` is executed. If not, + then ``false_filter`` is executed. + + .. note:: + + The ``base_filter`` does not execute atomically with the true and false + filters, which may lead to inconsistent or unexpected results. + + Additionally, executing a :class:`ConditionalRowFilter` has poor + performance on the server, especially when ``false_filter`` is set. + + :type base_filter: :class:`RowFilter` + :param base_filter: The filter to condition on before executing the + true/false filters. + + :type true_filter: :class:`RowFilter` + :param true_filter: (Optional) The filter to execute if there are any cells + matching ``base_filter``. If not provided, no results + will be returned in the true case. + + :type false_filter: :class:`RowFilter` + :param false_filter: (Optional) The filter to execute if there are no cells + matching ``base_filter``. If not provided, no results + will be returned in the false case. + """ + + def __init__(self, base_filter, true_filter=None, false_filter=None): + self.base_filter = base_filter + self.true_filter = true_filter + self.false_filter = false_filter + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return (other.base_filter == self.base_filter and + other.true_filter == self.true_filter and + other.false_filter == self.false_filter) + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + condition_kwargs = {'predicate_filter': self.base_filter.to_pb()} + if self.true_filter is not None: + condition_kwargs['true_filter'] = self.true_filter.to_pb() + if self.false_filter is not None: + condition_kwargs['false_filter'] = self.false_filter.to_pb() + condition = data_pb2.RowFilter.Condition(**condition_kwargs) + return data_pb2.RowFilter(condition=condition) diff --git a/gcloud/bigtable/table.py b/gcloud/bigtable/table.py index 2abee12bc9d0..5815086d7c00 100644 --- a/gcloud/bigtable/table.py +++ b/gcloud/bigtable/table.py @@ -23,7 +23,9 @@ bigtable_service_messages_pb2 as data_messages_pb2) from gcloud.bigtable.column_family import _gc_rule_from_pb from gcloud.bigtable.column_family import ColumnFamily -from gcloud.bigtable.row import Row +from gcloud.bigtable.row import AppendRow +from gcloud.bigtable.row import ConditionalRow +from gcloud.bigtable.row import DirectRow from gcloud.bigtable.row_data import PartialRowData from gcloud.bigtable.row_data import PartialRowsData @@ -53,7 +55,7 @@ class Table(object): :type table_id: str :param table_id: The ID of the table. - :type cluster: :class:`.cluster.Cluster` + :type cluster: :class:`Cluster <.cluster.Cluster>` :param cluster: The cluster that owns the table. """ @@ -86,29 +88,47 @@ def column_family(self, column_family_id, gc_rule=None): :param column_family_id: The ID of the column family. Must be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - :type gc_rule: :class:`.column_family.GarbageCollectionRule` + :type gc_rule: :class:`.GarbageCollectionRule` :param gc_rule: (Optional) The garbage collection settings for this column family. - :rtype: :class:`.column_family.ColumnFamily` + :rtype: :class:`.ColumnFamily` :returns: A column family owned by this table. """ return ColumnFamily(column_family_id, self, gc_rule=gc_rule) - def row(self, row_key, filter_=None): + def row(self, row_key, filter_=None, append=False): """Factory to create a row associated with this table. + .. warning:: + + At most one of ``filter_`` and ``append`` can be used in a + :class:`Row`. + :type row_key: bytes :param row_key: The key for the row being created. :type filter_: :class:`.RowFilter` :param filter_: (Optional) Filter to be used for conditional mutations. - See :class:`.Row` for more details. + See :class:`.DirectRow` for more details. + + :type append: bool + :param append: (Optional) Flag to determine if the row should be used + for append mutations. - :rtype: :class:`.Row` + :rtype: :class:`.DirectRow` :returns: A row owned by this table. + :raises: :class:`ValueError ` if both + ``filter_`` and ``append`` are used. """ - return Row(row_key, self, filter_=filter_) + if append and filter_ is not None: + raise ValueError('At most one of filter_ and append can be set') + if append: + return AppendRow(row_key, self) + elif filter_ is not None: + return ConditionalRow(row_key, self, filter_=filter_) + else: + return DirectRow(row_key, self) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -199,7 +219,7 @@ def list_column_families(self): :rtype: dict :returns: Dictionary of column families attached to this table. Keys are strings (column family names) and values are - :class:`.column_family.ColumnFamily` instances. + :class:`.ColumnFamily` instances. :raises: :class:`ValueError ` if the column family name from the response does not agree with the computed name from the column family ID. @@ -228,7 +248,7 @@ def read_row(self, row_key, filter_=None): :type row_key: bytes :param row_key: The key of the row to read from. - :type filter_: :class:`.row.RowFilter` + :type filter_: :class:`.RowFilter` :param filter_: (Optional) The filter to apply to the contents of the row. If unset, returns the entire row. @@ -291,7 +311,7 @@ def read_rows(self, start_key=None, end_key=None, more than N rows. However, only N ``commit_row`` chunks will be sent. - :type filter_: :class:`.row.RowFilter` + :type filter_: :class:`.RowFilter` :param filter_: (Optional) The filter to apply to the contents of the specified row(s). If unset, reads every column in each row. @@ -368,7 +388,7 @@ def _create_row_request(table_name, row_key=None, start_key=None, end_key=None, The range will not include ``end_key``. If left empty, will be interpreted as an infinite string. - :type filter_: :class:`.row.RowFilter` + :type filter_: :class:`.RowFilter` :param filter_: (Optional) The filter to apply to the contents of the specified row(s). If unset, reads the entire table. diff --git a/gcloud/bigtable/test_cluster.py b/gcloud/bigtable/test_cluster.py index eba227a72db0..427a4ec9126b 100644 --- a/gcloud/bigtable/test_cluster.py +++ b/gcloud/bigtable/test_cluster.py @@ -210,7 +210,7 @@ def test_table_factory(self): def test__update_from_pb_success(self): from gcloud.bigtable._generated import ( bigtable_cluster_data_pb2 as data_pb2) - from gcloud.bigtable.cluster import _DEFAULT_SERVE_NODES + from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES display_name = 'display_name' serve_nodes = 8 @@ -221,7 +221,7 @@ def test__update_from_pb_success(self): cluster = self._makeOne(None, None, None) self.assertEqual(cluster.display_name, None) - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + self.assertEqual(cluster.serve_nodes, DEFAULT_SERVE_NODES) cluster._update_from_pb(cluster_pb) self.assertEqual(cluster.display_name, display_name) self.assertEqual(cluster.serve_nodes, serve_nodes) @@ -229,30 +229,30 @@ def test__update_from_pb_success(self): def test__update_from_pb_no_display_name(self): from gcloud.bigtable._generated import ( bigtable_cluster_data_pb2 as data_pb2) - from gcloud.bigtable.cluster import _DEFAULT_SERVE_NODES + from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES cluster_pb = data_pb2.Cluster(serve_nodes=331) cluster = self._makeOne(None, None, None) self.assertEqual(cluster.display_name, None) - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + self.assertEqual(cluster.serve_nodes, DEFAULT_SERVE_NODES) with self.assertRaises(ValueError): cluster._update_from_pb(cluster_pb) self.assertEqual(cluster.display_name, None) - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + self.assertEqual(cluster.serve_nodes, DEFAULT_SERVE_NODES) def test__update_from_pb_no_serve_nodes(self): from gcloud.bigtable._generated import ( bigtable_cluster_data_pb2 as data_pb2) - from gcloud.bigtable.cluster import _DEFAULT_SERVE_NODES + from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES cluster_pb = data_pb2.Cluster(display_name='name') cluster = self._makeOne(None, None, None) self.assertEqual(cluster.display_name, None) - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + self.assertEqual(cluster.serve_nodes, DEFAULT_SERVE_NODES) with self.assertRaises(ValueError): cluster._update_from_pb(cluster_pb) self.assertEqual(cluster.display_name, None) - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + self.assertEqual(cluster.serve_nodes, DEFAULT_SERVE_NODES) def test_from_pb_success(self): from gcloud.bigtable._generated import ( @@ -353,7 +353,7 @@ def test_reload(self): from gcloud.bigtable._generated import ( bigtable_cluster_service_messages_pb2 as messages_pb2) from gcloud.bigtable._testing import _FakeStub - from gcloud.bigtable.cluster import _DEFAULT_SERVE_NODES + from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES project = 'PROJECT' zone = 'zone' @@ -383,7 +383,7 @@ def test_reload(self): expected_result = None # reload() has no return value. # Check Cluster optional config values before. - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + self.assertEqual(cluster.serve_nodes, DEFAULT_SERVE_NODES) self.assertEqual(cluster.display_name, cluster_id) # Perform the method and check the result. diff --git a/gcloud/bigtable/test_row.py b/gcloud/bigtable/test_row.py index 71a62763bd32..9e6da708e6b6 100644 --- a/gcloud/bigtable/test_row.py +++ b/gcloud/bigtable/test_row.py @@ -16,11 +16,26 @@ import unittest2 -class TestRow(unittest2.TestCase): +class Test_SetDeleteRow(unittest2.TestCase): def _getTargetClass(self): - from gcloud.bigtable.row import Row - return Row + from gcloud.bigtable.row import _SetDeleteRow + return _SetDeleteRow + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test__get_mutations_virtual(self): + row = self._makeOne(b'row-key', None) + with self.assertRaises(NotImplementedError): + row._get_mutations(None) + + +class TestDirectRow(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row import DirectRow + return DirectRow def _makeOne(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) @@ -28,12 +43,11 @@ def _makeOne(self, *args, **kwargs): def test_constructor(self): row_key = b'row_key' table = object() - filter_ = object() - row = self._makeOne(row_key, table, filter_=filter_) + row = self._makeOne(row_key, table) self.assertEqual(row._row_key, row_key) self.assertTrue(row._table is table) - self.assertTrue(row._filter is filter_) + self.assertEqual(row._pb_mutations, []) def test_constructor_with_unicode(self): row_key = u'row_key' @@ -49,45 +63,12 @@ def test_constructor_with_non_bytes(self): with self.assertRaises(TypeError): self._makeOne(row_key, None) - def _get_mutations_helper(self, filter_=None, state=None): + def test__get_mutations(self): row_key = b'row_key' - row = self._makeOne(row_key, None, filter_=filter_) - # Mock the mutations with unique objects so we can compare. - row._pb_mutations = no_bool = object() - row._true_pb_mutations = true_mutations = object() - row._false_pb_mutations = false_mutations = object() + row = self._makeOne(row_key, None) - mutations = row._get_mutations(state) - return (no_bool, true_mutations, false_mutations), mutations - - def test__get_mutations_no_filter(self): - (no_bool, _, _), mutations = self._get_mutations_helper() - self.assertTrue(mutations is no_bool) - - def test__get_mutations_no_filter_bad_state(self): - state = object() # State should be null when no filter. - with self.assertRaises(ValueError): - self._get_mutations_helper(state=state) - - def test__get_mutations_with_filter_true_state(self): - filter_ = object() - state = True - (_, true_filter, _), mutations = self._get_mutations_helper( - filter_=filter_, state=state) - self.assertTrue(mutations is true_filter) - - def test__get_mutations_with_filter_false_state(self): - filter_ = object() - state = False - (_, _, false_filter), mutations = self._get_mutations_helper( - filter_=filter_, state=state) - self.assertTrue(mutations is false_filter) - - def test__get_mutations_with_filter_bad_state(self): - filter_ = object() - state = None - with self.assertRaises(ValueError): - self._get_mutations_helper(filter_=filter_, state=state) + row._pb_mutations = mutations = object() + self.assertTrue(mutations is row._get_mutations(None)) def _set_cell_helper(self, column=None, column_bytes=None, value=b'foobar', timestamp=None, @@ -152,40 +133,6 @@ def test_set_cell_with_non_null_timestamp(self): self._set_cell_helper(timestamp=timestamp, timestamp_micros=millis_granularity) - def test_append_cell_value(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - table = object() - row_key = b'row_key' - row = self._makeOne(row_key, table) - self.assertEqual(row._rule_pb_list, []) - - column = b'column' - column_family_id = u'column_family_id' - value = b'bytes-val' - row.append_cell_value(column_family_id, column, value) - expected_pb = data_pb2.ReadModifyWriteRule( - family_name=column_family_id, column_qualifier=column, - append_value=value) - self.assertEqual(row._rule_pb_list, [expected_pb]) - - def test_increment_cell_value(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - table = object() - row_key = b'row_key' - row = self._makeOne(row_key, table) - self.assertEqual(row._rule_pb_list, []) - - column = b'column' - column_family_id = u'column_family_id' - int_value = 281330 - row.increment_cell_value(column_family_id, column, int_value) - expected_pb = data_pb2.ReadModifyWriteRule( - family_name=column_family_id, column_qualifier=column, - increment_amount=int_value) - self.assertEqual(row._rule_pb_list, [expected_pb]) - def test_delete(self): from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 @@ -210,7 +157,7 @@ def __init__(self, *args, **kwargs): self._kwargs = [] # Replace the called method with one that logs arguments. - def delete_cells(self, *args, **kwargs): + def _delete_cells(self, *args, **kwargs): self._args.append(args) self._kwargs.append(kwargs) @@ -305,7 +252,7 @@ def test_delete_cells_no_time_range(self): def test_delete_cells_with_time_range(self): import datetime from gcloud._helpers import _EPOCH - from gcloud.bigtable.row import TimestampRange + from gcloud.bigtable.row_filters import TimestampRange microseconds = 30871000 # Makes sure already milliseconds granularity start = _EPOCH + datetime.timedelta(microseconds=microseconds) @@ -408,8 +355,6 @@ def test_commit(self): {}, )]) self.assertEqual(row._pb_mutations, []) - self.assertEqual(row._true_pb_mutations, None) - self.assertEqual(row._false_pb_mutations, None) def test_commit_too_many_mutations(self): from gcloud._testing import _Monkey @@ -420,7 +365,7 @@ def test_commit_too_many_mutations(self): row = self._makeOne(row_key, table) row._pb_mutations = [1, 2, 3] num_mutations = len(row._pb_mutations) - with _Monkey(MUT, _MAX_MUTATIONS=num_mutations - 1): + with _Monkey(MUT, MAX_MUTATIONS=num_mutations - 1): with self.assertRaises(ValueError): row.commit() @@ -442,17 +387,53 @@ def test_commit_no_mutations(self): # Make sure no request was sent. self.assertEqual(stub.method_calls, []) - def test_commit_with_filter(self): + +class TestConditionalRow(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row import ConditionalRow + return ConditionalRow + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + row_key = b'row_key' + table = object() + filter_ = object() + + row = self._makeOne(row_key, table, filter_=filter_) + self.assertEqual(row._row_key, row_key) + self.assertTrue(row._table is table) + self.assertTrue(row._filter is filter_) + self.assertEqual(row._true_pb_mutations, []) + self.assertEqual(row._false_pb_mutations, []) + + def test__get_mutations(self): + row_key = b'row_key' + filter_ = object() + row = self._makeOne(row_key, None, filter_=filter_) + + row._true_pb_mutations = true_mutations = object() + row._false_pb_mutations = false_mutations = object() + self.assertTrue(true_mutations is row._get_mutations(True)) + self.assertTrue(false_mutations is row._get_mutations(False)) + self.assertTrue(false_mutations is row._get_mutations(None)) + + def test_commit(self): from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 from gcloud.bigtable._generated import ( bigtable_service_messages_pb2 as messages_pb2) from gcloud.bigtable._testing import _FakeStub - from gcloud.bigtable.row import RowSampleFilter + from gcloud.bigtable.row_filters import RowSampleFilter row_key = b'row_key' table_name = 'projects/more-stuff' - column_family_id = u'column_family_id' - column = b'column' + column_family_id1 = u'column_family_id1' + column_family_id2 = u'column_family_id2' + column_family_id3 = u'column_family_id3' + column1 = b'column1' + column2 = b'column2' timeout_seconds = 262 client = _Client(timeout_seconds=timeout_seconds) table = _Table(table_name, client=client) @@ -463,26 +444,31 @@ def test_commit_with_filter(self): value1 = b'bytes-value' mutation1 = data_pb2.Mutation( set_cell=data_pb2.Mutation.SetCell( - family_name=column_family_id, - column_qualifier=column, + family_name=column_family_id1, + column_qualifier=column1, timestamp_micros=-1, # Default value. value=value1, ), ) - value2 = b'other-bytes' mutation2 = data_pb2.Mutation( - set_cell=data_pb2.Mutation.SetCell( - family_name=column_family_id, - column_qualifier=column, - timestamp_micros=-1, # Default value. - value=value2, + delete_from_row=data_pb2.Mutation.DeleteFromRow(), + ) + mutation3 = data_pb2.Mutation( + delete_from_column=data_pb2.Mutation.DeleteFromColumn( + family_name=column_family_id2, + column_qualifier=column2, + ), + ) + mutation4 = data_pb2.Mutation( + delete_from_family=data_pb2.Mutation.DeleteFromFamily( + family_name=column_family_id3, ), ) request_pb = messages_pb2.CheckAndMutateRowRequest( table_name=table_name, row_key=row_key, predicate_filter=row_filter.to_pb(), - true_mutations=[mutation1], + true_mutations=[mutation1, mutation3, mutation4], false_mutations=[mutation2], ) @@ -498,8 +484,10 @@ def test_commit_with_filter(self): expected_result = predicate_matched # Perform the method and check the result. - row.set_cell(column_family_id, column, value1, state=True) - row.set_cell(column_family_id, column, value2, state=False) + row.set_cell(column_family_id1, column1, value1, state=True) + row.delete(state=False) + row.delete_cell(column_family_id2, column2, state=True) + row.delete_cells(column_family_id3, row.ALL_COLUMNS, state=True) result = row.commit() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( @@ -507,11 +495,10 @@ def test_commit_with_filter(self): (request_pb, timeout_seconds), {}, )]) - self.assertEqual(row._pb_mutations, None) self.assertEqual(row._true_pb_mutations, []) self.assertEqual(row._false_pb_mutations, []) - def test_commit_with_filter_too_many_mutations(self): + def test_commit_too_many_mutations(self): from gcloud._testing import _Monkey from gcloud.bigtable import row as MUT @@ -521,11 +508,11 @@ def test_commit_with_filter_too_many_mutations(self): row = self._makeOne(row_key, table, filter_=filter_) row._true_pb_mutations = [1, 2, 3] num_mutations = len(row._true_pb_mutations) - with _Monkey(MUT, _MAX_MUTATIONS=num_mutations - 1): + with _Monkey(MUT, MAX_MUTATIONS=num_mutations - 1): with self.assertRaises(ValueError): row.commit() - def test_commit_with_filter_no_mutations(self): + def test_commit_no_mutations(self): from gcloud.bigtable._testing import _FakeStub row_key = b'row_key' @@ -545,7 +532,68 @@ def test_commit_with_filter_no_mutations(self): # Make sure no request was sent. self.assertEqual(stub.method_calls, []) - def test_commit_modifications(self): + +class TestAppendRow(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row import AppendRow + return AppendRow + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + row_key = b'row_key' + table = object() + + row = self._makeOne(row_key, table) + self.assertEqual(row._row_key, row_key) + self.assertTrue(row._table is table) + self.assertEqual(row._rule_pb_list, []) + + def test_clear(self): + row_key = b'row_key' + table = object() + row = self._makeOne(row_key, table) + row._rule_pb_list = [1, 2, 3] + row.clear() + self.assertEqual(row._rule_pb_list, []) + + def test_append_cell_value(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + table = object() + row_key = b'row_key' + row = self._makeOne(row_key, table) + self.assertEqual(row._rule_pb_list, []) + + column = b'column' + column_family_id = u'column_family_id' + value = b'bytes-val' + row.append_cell_value(column_family_id, column, value) + expected_pb = data_pb2.ReadModifyWriteRule( + family_name=column_family_id, column_qualifier=column, + append_value=value) + self.assertEqual(row._rule_pb_list, [expected_pb]) + + def test_increment_cell_value(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + table = object() + row_key = b'row_key' + row = self._makeOne(row_key, table) + self.assertEqual(row._rule_pb_list, []) + + column = b'column' + column_family_id = u'column_family_id' + int_value = 281330 + row.increment_cell_value(column_family_id, column, int_value) + expected_pb = data_pb2.ReadModifyWriteRule( + family_name=column_family_id, column_qualifier=column, + increment_amount=int_value) + self.assertEqual(row._rule_pb_list, [expected_pb]) + + def test_commit(self): from gcloud._testing import _Monkey from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 from gcloud.bigtable._generated import ( @@ -594,7 +642,7 @@ def mock_parse_rmw_row_response(row_response): # Perform the method and check the result. with _Monkey(MUT, _parse_rmw_row_response=mock_parse_rmw_row_response): row.append_cell_value(column_family_id, column, value) - result = row.commit_modifications() + result = row.commit() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( @@ -602,14 +650,10 @@ def mock_parse_rmw_row_response(row_response): (request_pb, timeout_seconds), {}, )]) - self.assertEqual(row._pb_mutations, []) - self.assertEqual(row._true_pb_mutations, None) - self.assertEqual(row._false_pb_mutations, None) - self.assertEqual(row_responses, [response_pb]) self.assertEqual(row._rule_pb_list, []) - def test_commit_modifications_no_rules(self): + def test_commit_no_rules(self): from gcloud.bigtable._testing import _FakeStub row_key = b'row_key' @@ -622,999 +666,23 @@ def test_commit_modifications_no_rules(self): client._data_stub = stub = _FakeStub() # Perform the method and check the result. - result = row.commit_modifications() + result = row.commit() self.assertEqual(result, {}) # Make sure no request was sent. self.assertEqual(stub.method_calls, []) + def test_commit_too_many_mutations(self): + from gcloud._testing import _Monkey + from gcloud.bigtable import row as MUT -class Test_BoolFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import _BoolFilter - return _BoolFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - flag = object() - row_filter = self._makeOne(flag) - self.assertTrue(row_filter.flag is flag) - - def test___eq__type_differ(self): - flag = object() - row_filter1 = self._makeOne(flag) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - flag = object() - row_filter1 = self._makeOne(flag) - row_filter2 = self._makeOne(flag) - self.assertEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - flag = object() - row_filter1 = self._makeOne(flag) - row_filter2 = self._makeOne(flag) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - - -class TestSinkFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import SinkFilter - return SinkFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(sink=flag) - self.assertEqual(pb_val, expected_pb) - - -class TestPassAllFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import PassAllFilter - return PassAllFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(pass_all_filter=flag) - self.assertEqual(pb_val, expected_pb) - - -class TestBlockAllFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import BlockAllFilter - return BlockAllFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(block_all_filter=flag) - self.assertEqual(pb_val, expected_pb) - - -class Test_RegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import _RegexFilter - return _RegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - regex = object() - row_filter = self._makeOne(regex) - self.assertTrue(row_filter.regex is regex) - - def test___eq__type_differ(self): - regex = object() - row_filter1 = self._makeOne(regex) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - regex = object() - row_filter1 = self._makeOne(regex) - row_filter2 = self._makeOne(regex) - self.assertEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - regex = object() - row_filter1 = self._makeOne(regex) - row_filter2 = self._makeOne(regex) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - - -class TestRowKeyRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import RowKeyRegexFilter - return RowKeyRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = b'row-key-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(row_key_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestRowSampleFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import RowSampleFilter - return RowSampleFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - sample = object() - row_filter = self._makeOne(sample) - self.assertTrue(row_filter.sample is sample) - - def test___eq__type_differ(self): - sample = object() - row_filter1 = self._makeOne(sample) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - sample = object() - row_filter1 = self._makeOne(sample) - row_filter2 = self._makeOne(sample) - self.assertEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - sample = 0.25 - row_filter = self._makeOne(sample) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(row_sample_filter=sample) - self.assertEqual(pb_val, expected_pb) - - -class TestFamilyNameRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import FamilyNameRegexFilter - return FamilyNameRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = u'family-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(family_name_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestColumnQualifierRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ColumnQualifierRegexFilter - return ColumnQualifierRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = b'column-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(column_qualifier_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestTimestampRange(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import TimestampRange - return TimestampRange - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - start = object() - end = object() - time_range = self._makeOne(start=start, end=end) - self.assertTrue(time_range.start is start) - self.assertTrue(time_range.end is end) - - def test___eq__(self): - start = object() - end = object() - time_range1 = self._makeOne(start=start, end=end) - time_range2 = self._makeOne(start=start, end=end) - self.assertEqual(time_range1, time_range2) - - def test___eq__type_differ(self): - start = object() - end = object() - time_range1 = self._makeOne(start=start, end=end) - time_range2 = object() - self.assertNotEqual(time_range1, time_range2) - - def test___ne__same_value(self): - start = object() - end = object() - time_range1 = self._makeOne(start=start, end=end) - time_range2 = self._makeOne(start=start, end=end) - comparison_val = (time_range1 != time_range2) - self.assertFalse(comparison_val) - - def _to_pb_helper(self, start_micros=None, end_micros=None): - import datetime - from gcloud._helpers import _EPOCH - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - pb_kwargs = {} - - start = None - if start_micros is not None: - start = _EPOCH + datetime.timedelta(microseconds=start_micros) - pb_kwargs['start_timestamp_micros'] = start_micros - end = None - if end_micros is not None: - end = _EPOCH + datetime.timedelta(microseconds=end_micros) - pb_kwargs['end_timestamp_micros'] = end_micros - time_range = self._makeOne(start=start, end=end) - - expected_pb = data_pb2.TimestampRange(**pb_kwargs) - self.assertEqual(time_range.to_pb(), expected_pb) - - def test_to_pb(self): - # Makes sure already milliseconds granularity - start_micros = 30871000 - end_micros = 12939371000 - self._to_pb_helper(start_micros=start_micros, - end_micros=end_micros) - - def test_to_pb_start_only(self): - # Makes sure already milliseconds granularity - start_micros = 30871000 - self._to_pb_helper(start_micros=start_micros) - - def test_to_pb_end_only(self): - # Makes sure already milliseconds granularity - end_micros = 12939371000 - self._to_pb_helper(end_micros=end_micros) - - -class TestTimestampRangeFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import TimestampRangeFilter - return TimestampRangeFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - range_ = object() - row_filter = self._makeOne(range_) - self.assertTrue(row_filter.range_ is range_) - - def test___eq__type_differ(self): - range_ = object() - row_filter1 = self._makeOne(range_) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - range_ = object() - row_filter1 = self._makeOne(range_) - row_filter2 = self._makeOne(range_) - self.assertEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import TimestampRange - - range_ = TimestampRange() - row_filter = self._makeOne(range_) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter( - timestamp_range_filter=data_pb2.TimestampRange()) - self.assertEqual(pb_val, expected_pb) - - -class TestColumnRangeFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ColumnRangeFilter - return ColumnRangeFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - column_family_id = object() - row_filter = self._makeOne(column_family_id) - self.assertTrue(row_filter.column_family_id is column_family_id) - self.assertEqual(row_filter.start_column, None) - self.assertEqual(row_filter.end_column, None) - self.assertTrue(row_filter.inclusive_start) - self.assertTrue(row_filter.inclusive_end) - - def test_constructor_explicit(self): - column_family_id = object() - start_column = object() - end_column = object() - inclusive_start = object() - inclusive_end = object() - row_filter = self._makeOne(column_family_id, start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertTrue(row_filter.column_family_id is column_family_id) - self.assertTrue(row_filter.start_column is start_column) - self.assertTrue(row_filter.end_column is end_column) - self.assertTrue(row_filter.inclusive_start is inclusive_start) - self.assertTrue(row_filter.inclusive_end is inclusive_end) - - def test_constructor_bad_start(self): - column_family_id = object() - self.assertRaises(ValueError, self._makeOne, - column_family_id, inclusive_start=True) - - def test_constructor_bad_end(self): - column_family_id = object() - self.assertRaises(ValueError, self._makeOne, - column_family_id, inclusive_end=True) - - def test___eq__(self): - column_family_id = object() - start_column = object() - end_column = object() - inclusive_start = object() - inclusive_end = object() - row_filter1 = self._makeOne(column_family_id, - start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - row_filter2 = self._makeOne(column_family_id, - start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - column_family_id = object() - row_filter1 = self._makeOne(column_family_id) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - row_filter = self._makeOne(column_family_id) - col_range_pb = data_pb2.ColumnRange(family_name=column_family_id) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_start(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, start_column=column) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - start_qualifier_inclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_start(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, start_column=column, - inclusive_start=False) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - start_qualifier_exclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_end(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, end_column=column) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - end_qualifier_inclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_end(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, end_column=column, - inclusive_end=False) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - end_qualifier_exclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - -class TestValueRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ValueRegexFilter - return ValueRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = b'value-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(value_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestValueRangeFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ValueRangeFilter - return ValueRangeFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - row_filter = self._makeOne() - self.assertEqual(row_filter.start_value, None) - self.assertEqual(row_filter.end_value, None) - self.assertTrue(row_filter.inclusive_start) - self.assertTrue(row_filter.inclusive_end) - - def test_constructor_explicit(self): - start_value = object() - end_value = object() - inclusive_start = object() - inclusive_end = object() - row_filter = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertTrue(row_filter.start_value is start_value) - self.assertTrue(row_filter.end_value is end_value) - self.assertTrue(row_filter.inclusive_start is inclusive_start) - self.assertTrue(row_filter.inclusive_end is inclusive_end) - - def test_constructor_bad_start(self): - self.assertRaises(ValueError, self._makeOne, inclusive_start=True) - - def test_constructor_bad_end(self): - self.assertRaises(ValueError, self._makeOne, inclusive_end=True) - - def test___eq__(self): - start_value = object() - end_value = object() - inclusive_start = object() - inclusive_end = object() - row_filter1 = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - row_filter2 = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - row_filter1 = self._makeOne() - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - row_filter = self._makeOne() - expected_pb = data_pb2.RowFilter( - value_range_filter=data_pb2.ValueRange()) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_start(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(start_value=value) - val_range_pb = data_pb2.ValueRange(start_value_inclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_start(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(start_value=value, inclusive_start=False) - val_range_pb = data_pb2.ValueRange(start_value_exclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_end(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(end_value=value) - val_range_pb = data_pb2.ValueRange(end_value_inclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_end(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(end_value=value, inclusive_end=False) - val_range_pb = data_pb2.ValueRange(end_value_exclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - -class Test_CellCountFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import _CellCountFilter - return _CellCountFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - num_cells = object() - row_filter = self._makeOne(num_cells) - self.assertTrue(row_filter.num_cells is num_cells) - - def test___eq__type_differ(self): - num_cells = object() - row_filter1 = self._makeOne(num_cells) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - num_cells = object() - row_filter1 = self._makeOne(num_cells) - row_filter2 = self._makeOne(num_cells) - self.assertEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - num_cells = object() - row_filter1 = self._makeOne(num_cells) - row_filter2 = self._makeOne(num_cells) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - - -class TestCellsRowOffsetFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import CellsRowOffsetFilter - return CellsRowOffsetFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - num_cells = 76 - row_filter = self._makeOne(num_cells) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(cells_per_row_offset_filter=num_cells) - self.assertEqual(pb_val, expected_pb) - - -class TestCellsRowLimitFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import CellsRowLimitFilter - return CellsRowLimitFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - num_cells = 189 - row_filter = self._makeOne(num_cells) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(cells_per_row_limit_filter=num_cells) - self.assertEqual(pb_val, expected_pb) - - -class TestCellsColumnLimitFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import CellsColumnLimitFilter - return CellsColumnLimitFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - num_cells = 10 - row_filter = self._makeOne(num_cells) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter( - cells_per_column_limit_filter=num_cells) - self.assertEqual(pb_val, expected_pb) - - -class TestStripValueTransformerFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import StripValueTransformerFilter - return StripValueTransformerFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(strip_value_transformer=flag) - self.assertEqual(pb_val, expected_pb) - - -class TestApplyLabelFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ApplyLabelFilter - return ApplyLabelFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - label = object() - row_filter = self._makeOne(label) - self.assertTrue(row_filter.label is label) - - def test___eq__type_differ(self): - label = object() - row_filter1 = self._makeOne(label) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - label = object() - row_filter1 = self._makeOne(label) - row_filter2 = self._makeOne(label) - self.assertEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - - label = u'label' - row_filter = self._makeOne(label) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(apply_label_transformer=label) - self.assertEqual(pb_val, expected_pb) - - -class Test_FilterCombination(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import _FilterCombination - return _FilterCombination - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - row_filter = self._makeOne() - self.assertEqual(row_filter.filters, []) - - def test_constructor_explicit(self): - filters = object() - row_filter = self._makeOne(filters=filters) - self.assertTrue(row_filter.filters is filters) - - def test___eq__(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = self._makeOne(filters=filters) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - -class TestRowFilterChain(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import RowFilterChain - return RowFilterChain - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - chain=data_pb2.RowFilter.Chain( - filters=[row_filter1_pb, row_filter2_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_nested(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import CellsRowLimitFilter - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter2 = RowSampleFilter(0.25) - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - row_filter3_pb = row_filter3.to_pb() - - row_filter4 = CellsRowLimitFilter(11) - row_filter4_pb = row_filter4.to_pb() - - row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) - filter_pb = row_filter5.to_pb() - - expected_pb = data_pb2.RowFilter( - chain=data_pb2.RowFilter.Chain( - filters=[row_filter3_pb, row_filter4_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - -class TestRowFilterUnion(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import RowFilterUnion - return RowFilterUnion - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - interleave=data_pb2.RowFilter.Interleave( - filters=[row_filter1_pb, row_filter2_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_nested(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import CellsRowLimitFilter - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter2 = RowSampleFilter(0.25) - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - row_filter3_pb = row_filter3.to_pb() - - row_filter4 = CellsRowLimitFilter(11) - row_filter4_pb = row_filter4.to_pb() - - row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) - filter_pb = row_filter5.to_pb() - - expected_pb = data_pb2.RowFilter( - interleave=data_pb2.RowFilter.Interleave( - filters=[row_filter3_pb, row_filter4_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - -class TestConditionalRowFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.bigtable.row import ConditionalRowFilter - return ConditionalRowFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - self.assertTrue(cond_filter.base_filter is base_filter) - self.assertTrue(cond_filter.true_filter is true_filter) - self.assertTrue(cond_filter.false_filter is false_filter) - - def test___eq__(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter1 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - cond_filter2 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - self.assertEqual(cond_filter1, cond_filter2) - - def test___eq__type_differ(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter1 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - cond_filter2 = object() - self.assertNotEqual(cond_filter1, cond_filter2) - - def test_to_pb(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import CellsRowOffsetFilter - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = CellsRowOffsetFilter(11) - row_filter3_pb = row_filter3.to_pb() - - row_filter4 = self._makeOne(row_filter1, true_filter=row_filter2, - false_filter=row_filter3) - filter_pb = row_filter4.to_pb() - - expected_pb = data_pb2.RowFilter( - condition=data_pb2.RowFilter.Condition( - predicate_filter=row_filter1_pb, - true_filter=row_filter2_pb, - false_filter=row_filter3_pb, - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_true_only(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(row_filter1, true_filter=row_filter2) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - condition=data_pb2.RowFilter.Condition( - predicate_filter=row_filter1_pb, - true_filter=row_filter2_pb, - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_false_only(self): - from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud.bigtable.row import RowSampleFilter - from gcloud.bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(row_filter1, false_filter=row_filter2) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - condition=data_pb2.RowFilter.Condition( - predicate_filter=row_filter1_pb, - false_filter=row_filter2_pb, - ), - ) - self.assertEqual(filter_pb, expected_pb) + row_key = b'row_key' + table = object() + row = self._makeOne(row_key, table) + row._rule_pb_list = [1, 2, 3] + num_mutations = len(row._rule_pb_list) + with _Monkey(MUT, MAX_MUTATIONS=num_mutations - 1): + with self.assertRaises(ValueError): + row.commit() class Test__parse_rmw_row_response(unittest2.TestCase): diff --git a/gcloud/bigtable/test_row_filters.py b/gcloud/bigtable/test_row_filters.py new file mode 100644 index 000000000000..aed90574683f --- /dev/null +++ b/gcloud/bigtable/test_row_filters.py @@ -0,0 +1,1010 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest2 + + +class Test_BoolFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import _BoolFilter + return _BoolFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + flag = object() + row_filter = self._makeOne(flag) + self.assertTrue(row_filter.flag is flag) + + def test___eq__type_differ(self): + flag = object() + row_filter1 = self._makeOne(flag) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + flag = object() + row_filter1 = self._makeOne(flag) + row_filter2 = self._makeOne(flag) + self.assertEqual(row_filter1, row_filter2) + + def test___ne__same_value(self): + flag = object() + row_filter1 = self._makeOne(flag) + row_filter2 = self._makeOne(flag) + comparison_val = (row_filter1 != row_filter2) + self.assertFalse(comparison_val) + + +class TestSinkFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import SinkFilter + return SinkFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(sink=flag) + self.assertEqual(pb_val, expected_pb) + + +class TestPassAllFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import PassAllFilter + return PassAllFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(pass_all_filter=flag) + self.assertEqual(pb_val, expected_pb) + + +class TestBlockAllFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import BlockAllFilter + return BlockAllFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(block_all_filter=flag) + self.assertEqual(pb_val, expected_pb) + + +class Test_RegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import _RegexFilter + return _RegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + regex = b'abc' + row_filter = self._makeOne(regex) + self.assertTrue(row_filter.regex is regex) + + def test_constructor_non_bytes(self): + regex = u'abc' + row_filter = self._makeOne(regex) + self.assertEqual(row_filter.regex, b'abc') + + def test___eq__type_differ(self): + regex = b'def-rgx' + row_filter1 = self._makeOne(regex) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + regex = b'trex-regex' + row_filter1 = self._makeOne(regex) + row_filter2 = self._makeOne(regex) + self.assertEqual(row_filter1, row_filter2) + + def test___ne__same_value(self): + regex = b'abc' + row_filter1 = self._makeOne(regex) + row_filter2 = self._makeOne(regex) + comparison_val = (row_filter1 != row_filter2) + self.assertFalse(comparison_val) + + +class TestRowKeyRegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import RowKeyRegexFilter + return RowKeyRegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + regex = b'row-key-regex' + row_filter = self._makeOne(regex) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(row_key_regex_filter=regex) + self.assertEqual(pb_val, expected_pb) + + +class TestRowSampleFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import RowSampleFilter + return RowSampleFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + sample = object() + row_filter = self._makeOne(sample) + self.assertTrue(row_filter.sample is sample) + + def test___eq__type_differ(self): + sample = object() + row_filter1 = self._makeOne(sample) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + sample = object() + row_filter1 = self._makeOne(sample) + row_filter2 = self._makeOne(sample) + self.assertEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + sample = 0.25 + row_filter = self._makeOne(sample) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(row_sample_filter=sample) + self.assertEqual(pb_val, expected_pb) + + +class TestFamilyNameRegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import FamilyNameRegexFilter + return FamilyNameRegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + regex = u'family-regex' + row_filter = self._makeOne(regex) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(family_name_regex_filter=regex) + self.assertEqual(pb_val, expected_pb) + + +class TestColumnQualifierRegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter + return ColumnQualifierRegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + regex = b'column-regex' + row_filter = self._makeOne(regex) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(column_qualifier_regex_filter=regex) + self.assertEqual(pb_val, expected_pb) + + +class TestTimestampRange(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import TimestampRange + return TimestampRange + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + start = object() + end = object() + time_range = self._makeOne(start=start, end=end) + self.assertTrue(time_range.start is start) + self.assertTrue(time_range.end is end) + + def test___eq__(self): + start = object() + end = object() + time_range1 = self._makeOne(start=start, end=end) + time_range2 = self._makeOne(start=start, end=end) + self.assertEqual(time_range1, time_range2) + + def test___eq__type_differ(self): + start = object() + end = object() + time_range1 = self._makeOne(start=start, end=end) + time_range2 = object() + self.assertNotEqual(time_range1, time_range2) + + def test___ne__same_value(self): + start = object() + end = object() + time_range1 = self._makeOne(start=start, end=end) + time_range2 = self._makeOne(start=start, end=end) + comparison_val = (time_range1 != time_range2) + self.assertFalse(comparison_val) + + def _to_pb_helper(self, start_micros=None, end_micros=None): + import datetime + from gcloud._helpers import _EPOCH + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + pb_kwargs = {} + + start = None + if start_micros is not None: + start = _EPOCH + datetime.timedelta(microseconds=start_micros) + pb_kwargs['start_timestamp_micros'] = start_micros + end = None + if end_micros is not None: + end = _EPOCH + datetime.timedelta(microseconds=end_micros) + pb_kwargs['end_timestamp_micros'] = end_micros + time_range = self._makeOne(start=start, end=end) + + expected_pb = data_pb2.TimestampRange(**pb_kwargs) + self.assertEqual(time_range.to_pb(), expected_pb) + + def test_to_pb(self): + # Makes sure already milliseconds granularity + start_micros = 30871000 + end_micros = 12939371000 + self._to_pb_helper(start_micros=start_micros, + end_micros=end_micros) + + def test_to_pb_start_only(self): + # Makes sure already milliseconds granularity + start_micros = 30871000 + self._to_pb_helper(start_micros=start_micros) + + def test_to_pb_end_only(self): + # Makes sure already milliseconds granularity + end_micros = 12939371000 + self._to_pb_helper(end_micros=end_micros) + + +class TestTimestampRangeFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import TimestampRangeFilter + return TimestampRangeFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + range_ = object() + row_filter = self._makeOne(range_) + self.assertTrue(row_filter.range_ is range_) + + def test___eq__type_differ(self): + range_ = object() + row_filter1 = self._makeOne(range_) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + range_ = object() + row_filter1 = self._makeOne(range_) + row_filter2 = self._makeOne(range_) + self.assertEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import TimestampRange + + range_ = TimestampRange() + row_filter = self._makeOne(range_) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter( + timestamp_range_filter=data_pb2.TimestampRange()) + self.assertEqual(pb_val, expected_pb) + + +class TestColumnRangeFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ColumnRangeFilter + return ColumnRangeFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor_defaults(self): + column_family_id = object() + row_filter = self._makeOne(column_family_id) + self.assertTrue(row_filter.column_family_id is column_family_id) + self.assertEqual(row_filter.start_column, None) + self.assertEqual(row_filter.end_column, None) + self.assertTrue(row_filter.inclusive_start) + self.assertTrue(row_filter.inclusive_end) + + def test_constructor_explicit(self): + column_family_id = object() + start_column = object() + end_column = object() + inclusive_start = object() + inclusive_end = object() + row_filter = self._makeOne(column_family_id, start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertTrue(row_filter.column_family_id is column_family_id) + self.assertTrue(row_filter.start_column is start_column) + self.assertTrue(row_filter.end_column is end_column) + self.assertTrue(row_filter.inclusive_start is inclusive_start) + self.assertTrue(row_filter.inclusive_end is inclusive_end) + + def test_constructor_bad_start(self): + column_family_id = object() + self.assertRaises(ValueError, self._makeOne, + column_family_id, inclusive_start=True) + + def test_constructor_bad_end(self): + column_family_id = object() + self.assertRaises(ValueError, self._makeOne, + column_family_id, inclusive_end=True) + + def test___eq__(self): + column_family_id = object() + start_column = object() + end_column = object() + inclusive_start = object() + inclusive_end = object() + row_filter1 = self._makeOne(column_family_id, + start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + row_filter2 = self._makeOne(column_family_id, + start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertEqual(row_filter1, row_filter2) + + def test___eq__type_differ(self): + column_family_id = object() + row_filter1 = self._makeOne(column_family_id) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + row_filter = self._makeOne(column_family_id) + col_range_pb = data_pb2.ColumnRange(family_name=column_family_id) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_inclusive_start(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, start_column=column) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + start_qualifier_inclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_exclusive_start(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, start_column=column, + inclusive_start=False) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + start_qualifier_exclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_inclusive_end(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, end_column=column) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + end_qualifier_inclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_exclusive_end(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, end_column=column, + inclusive_end=False) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + end_qualifier_exclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + +class TestValueRegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ValueRegexFilter + return ValueRegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + regex = b'value-regex' + row_filter = self._makeOne(regex) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(value_regex_filter=regex) + self.assertEqual(pb_val, expected_pb) + + +class TestValueRangeFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ValueRangeFilter + return ValueRangeFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor_defaults(self): + row_filter = self._makeOne() + self.assertEqual(row_filter.start_value, None) + self.assertEqual(row_filter.end_value, None) + self.assertTrue(row_filter.inclusive_start) + self.assertTrue(row_filter.inclusive_end) + + def test_constructor_explicit(self): + start_value = object() + end_value = object() + inclusive_start = object() + inclusive_end = object() + row_filter = self._makeOne(start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertTrue(row_filter.start_value is start_value) + self.assertTrue(row_filter.end_value is end_value) + self.assertTrue(row_filter.inclusive_start is inclusive_start) + self.assertTrue(row_filter.inclusive_end is inclusive_end) + + def test_constructor_bad_start(self): + self.assertRaises(ValueError, self._makeOne, inclusive_start=True) + + def test_constructor_bad_end(self): + self.assertRaises(ValueError, self._makeOne, inclusive_end=True) + + def test___eq__(self): + start_value = object() + end_value = object() + inclusive_start = object() + inclusive_end = object() + row_filter1 = self._makeOne(start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + row_filter2 = self._makeOne(start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertEqual(row_filter1, row_filter2) + + def test___eq__type_differ(self): + row_filter1 = self._makeOne() + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + row_filter = self._makeOne() + expected_pb = data_pb2.RowFilter( + value_range_filter=data_pb2.ValueRange()) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_inclusive_start(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + value = b'some-value' + row_filter = self._makeOne(start_value=value) + val_range_pb = data_pb2.ValueRange(start_value_inclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_exclusive_start(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + value = b'some-value' + row_filter = self._makeOne(start_value=value, inclusive_start=False) + val_range_pb = data_pb2.ValueRange(start_value_exclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_inclusive_end(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + value = b'some-value' + row_filter = self._makeOne(end_value=value) + val_range_pb = data_pb2.ValueRange(end_value_inclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_exclusive_end(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + value = b'some-value' + row_filter = self._makeOne(end_value=value, inclusive_end=False) + val_range_pb = data_pb2.ValueRange(end_value_exclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + +class Test_CellCountFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import _CellCountFilter + return _CellCountFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + num_cells = object() + row_filter = self._makeOne(num_cells) + self.assertTrue(row_filter.num_cells is num_cells) + + def test___eq__type_differ(self): + num_cells = object() + row_filter1 = self._makeOne(num_cells) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + num_cells = object() + row_filter1 = self._makeOne(num_cells) + row_filter2 = self._makeOne(num_cells) + self.assertEqual(row_filter1, row_filter2) + + def test___ne__same_value(self): + num_cells = object() + row_filter1 = self._makeOne(num_cells) + row_filter2 = self._makeOne(num_cells) + comparison_val = (row_filter1 != row_filter2) + self.assertFalse(comparison_val) + + +class TestCellsRowOffsetFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import CellsRowOffsetFilter + return CellsRowOffsetFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + num_cells = 76 + row_filter = self._makeOne(num_cells) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(cells_per_row_offset_filter=num_cells) + self.assertEqual(pb_val, expected_pb) + + +class TestCellsRowLimitFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import CellsRowLimitFilter + return CellsRowLimitFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + num_cells = 189 + row_filter = self._makeOne(num_cells) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(cells_per_row_limit_filter=num_cells) + self.assertEqual(pb_val, expected_pb) + + +class TestCellsColumnLimitFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import CellsColumnLimitFilter + return CellsColumnLimitFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + num_cells = 10 + row_filter = self._makeOne(num_cells) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter( + cells_per_column_limit_filter=num_cells) + self.assertEqual(pb_val, expected_pb) + + +class TestStripValueTransformerFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import StripValueTransformerFilter + return StripValueTransformerFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(strip_value_transformer=flag) + self.assertEqual(pb_val, expected_pb) + + +class TestApplyLabelFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ApplyLabelFilter + return ApplyLabelFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + label = object() + row_filter = self._makeOne(label) + self.assertTrue(row_filter.label is label) + + def test___eq__type_differ(self): + label = object() + row_filter1 = self._makeOne(label) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + label = object() + row_filter1 = self._makeOne(label) + row_filter2 = self._makeOne(label) + self.assertEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + + label = u'label' + row_filter = self._makeOne(label) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(apply_label_transformer=label) + self.assertEqual(pb_val, expected_pb) + + +class Test_FilterCombination(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import _FilterCombination + return _FilterCombination + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor_defaults(self): + row_filter = self._makeOne() + self.assertEqual(row_filter.filters, []) + + def test_constructor_explicit(self): + filters = object() + row_filter = self._makeOne(filters=filters) + self.assertTrue(row_filter.filters is filters) + + def test___eq__(self): + filters = object() + row_filter1 = self._makeOne(filters=filters) + row_filter2 = self._makeOne(filters=filters) + self.assertEqual(row_filter1, row_filter2) + + def test___eq__type_differ(self): + filters = object() + row_filter1 = self._makeOne(filters=filters) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + +class TestRowFilterChain(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import RowFilterChain + return RowFilterChain + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) + filter_pb = row_filter3.to_pb() + + expected_pb = data_pb2.RowFilter( + chain=data_pb2.RowFilter.Chain( + filters=[row_filter1_pb, row_filter2_pb], + ), + ) + self.assertEqual(filter_pb, expected_pb) + + def test_to_pb_nested(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import CellsRowLimitFilter + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) + + row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) + row_filter3_pb = row_filter3.to_pb() + + row_filter4 = CellsRowLimitFilter(11) + row_filter4_pb = row_filter4.to_pb() + + row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) + filter_pb = row_filter5.to_pb() + + expected_pb = data_pb2.RowFilter( + chain=data_pb2.RowFilter.Chain( + filters=[row_filter3_pb, row_filter4_pb], + ), + ) + self.assertEqual(filter_pb, expected_pb) + + +class TestRowFilterUnion(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import RowFilterUnion + return RowFilterUnion + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) + filter_pb = row_filter3.to_pb() + + expected_pb = data_pb2.RowFilter( + interleave=data_pb2.RowFilter.Interleave( + filters=[row_filter1_pb, row_filter2_pb], + ), + ) + self.assertEqual(filter_pb, expected_pb) + + def test_to_pb_nested(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import CellsRowLimitFilter + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) + + row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) + row_filter3_pb = row_filter3.to_pb() + + row_filter4 = CellsRowLimitFilter(11) + row_filter4_pb = row_filter4.to_pb() + + row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) + filter_pb = row_filter5.to_pb() + + expected_pb = data_pb2.RowFilter( + interleave=data_pb2.RowFilter.Interleave( + filters=[row_filter3_pb, row_filter4_pb], + ), + ) + self.assertEqual(filter_pb, expected_pb) + + +class TestConditionalRowFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.bigtable.row_filters import ConditionalRowFilter + return ConditionalRowFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + base_filter = object() + true_filter = object() + false_filter = object() + cond_filter = self._makeOne(base_filter, + true_filter=true_filter, + false_filter=false_filter) + self.assertTrue(cond_filter.base_filter is base_filter) + self.assertTrue(cond_filter.true_filter is true_filter) + self.assertTrue(cond_filter.false_filter is false_filter) + + def test___eq__(self): + base_filter = object() + true_filter = object() + false_filter = object() + cond_filter1 = self._makeOne(base_filter, + true_filter=true_filter, + false_filter=false_filter) + cond_filter2 = self._makeOne(base_filter, + true_filter=true_filter, + false_filter=false_filter) + self.assertEqual(cond_filter1, cond_filter2) + + def test___eq__type_differ(self): + base_filter = object() + true_filter = object() + false_filter = object() + cond_filter1 = self._makeOne(base_filter, + true_filter=true_filter, + false_filter=false_filter) + cond_filter2 = object() + self.assertNotEqual(cond_filter1, cond_filter2) + + def test_to_pb(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import CellsRowOffsetFilter + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = CellsRowOffsetFilter(11) + row_filter3_pb = row_filter3.to_pb() + + row_filter4 = self._makeOne(row_filter1, true_filter=row_filter2, + false_filter=row_filter3) + filter_pb = row_filter4.to_pb() + + expected_pb = data_pb2.RowFilter( + condition=data_pb2.RowFilter.Condition( + predicate_filter=row_filter1_pb, + true_filter=row_filter2_pb, + false_filter=row_filter3_pb, + ), + ) + self.assertEqual(filter_pb, expected_pb) + + def test_to_pb_true_only(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = self._makeOne(row_filter1, true_filter=row_filter2) + filter_pb = row_filter3.to_pb() + + expected_pb = data_pb2.RowFilter( + condition=data_pb2.RowFilter.Condition( + predicate_filter=row_filter1_pb, + true_filter=row_filter2_pb, + ), + ) + self.assertEqual(filter_pb, expected_pb) + + def test_to_pb_false_only(self): + from gcloud.bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud.bigtable.row_filters import RowSampleFilter + from gcloud.bigtable.row_filters import StripValueTransformerFilter + + row_filter1 = StripValueTransformerFilter(True) + row_filter1_pb = row_filter1.to_pb() + + row_filter2 = RowSampleFilter(0.25) + row_filter2_pb = row_filter2.to_pb() + + row_filter3 = self._makeOne(row_filter1, false_filter=row_filter2) + filter_pb = row_filter3.to_pb() + + expected_pb = data_pb2.RowFilter( + condition=data_pb2.RowFilter.Condition( + predicate_filter=row_filter1_pb, + false_filter=row_filter2_pb, + ), + ) + self.assertEqual(filter_pb, expected_pb) diff --git a/gcloud/bigtable/test_table.py b/gcloud/bigtable/test_table.py index 882db22c4d51..9fcdf21593b0 100644 --- a/gcloud/bigtable/test_table.py +++ b/gcloud/bigtable/test_table.py @@ -56,8 +56,20 @@ def test_column_family_factory(self): self.assertTrue(column_family.gc_rule is gc_rule) self.assertEqual(column_family._table, table) - def test_row_factory(self): - from gcloud.bigtable.row import Row + def test_row_factory_direct(self): + from gcloud.bigtable.row import DirectRow + + table_id = 'table-id' + table = self._makeOne(table_id, None) + row_key = b'row_key' + row = table.row(row_key) + + self.assertTrue(isinstance(row, DirectRow)) + self.assertEqual(row._row_key, row_key) + self.assertEqual(row._table, table) + + def test_row_factory_conditional(self): + from gcloud.bigtable.row import ConditionalRow table_id = 'table-id' table = self._makeOne(table_id, None) @@ -65,10 +77,27 @@ def test_row_factory(self): filter_ = object() row = table.row(row_key, filter_=filter_) - self.assertTrue(isinstance(row, Row)) + self.assertTrue(isinstance(row, ConditionalRow)) + self.assertEqual(row._row_key, row_key) + self.assertEqual(row._table, table) + + def test_row_factory_append(self): + from gcloud.bigtable.row import AppendRow + + table_id = 'table-id' + table = self._makeOne(table_id, None) + row_key = b'row_key' + row = table.row(row_key, append=True) + + self.assertTrue(isinstance(row, AppendRow)) self.assertEqual(row._row_key, row_key) self.assertEqual(row._table, table) - self.assertEqual(row._filter, filter_) + + def test_row_factory_failure(self): + table_id = 'table-id' + table = self._makeOne(table_id, None) + with self.assertRaises(ValueError): + table.row(b'row_key', filter_=object(), append=True) def test___eq__(self): table_id = 'table_id' @@ -553,7 +582,7 @@ def test_row_range_both_keys(self): def test_with_filter(self): from gcloud.bigtable._generated import ( bigtable_service_messages_pb2 as messages_pb2) - from gcloud.bigtable.row import RowSampleFilter + from gcloud.bigtable.row_filters import RowSampleFilter table_name = 'table_name' row_filter = RowSampleFilter(0.33) diff --git a/gcloud/connection.py b/gcloud/connection.py index f965497c2c9d..cb8f7840732a 100644 --- a/gcloud/connection.py +++ b/gcloud/connection.py @@ -17,7 +17,7 @@ import json from pkg_resources import get_distribution import six -from six.moves.urllib.parse import urlencode # pylint: disable=F0401 +from six.moves.urllib.parse import urlencode import httplib2 diff --git a/gcloud/credentials.py b/gcloud/credentials.py index da88afb906a1..3d95733285c6 100644 --- a/gcloud/credentials.py +++ b/gcloud/credentials.py @@ -17,29 +17,9 @@ import base64 import datetime import six -from six.moves.urllib.parse import urlencode # pylint: disable=F0401 - -try: - from OpenSSL import crypto -except ImportError: # pragma: NO COVER - # pyOpenSSL can't be installed on App Engine, but it will not - # be needed there since app_identity is used. - crypto = None +from six.moves.urllib.parse import urlencode from oauth2client import client -from oauth2client import crypt -from oauth2client.service_account import ServiceAccountCredentials -try: - from oauth2client.contrib.appengine import ( - AppAssertionCredentials as _GAECreds) -except ImportError: - class _GAECreds(object): - """Dummy class if not in App Engine environment.""" - -try: - from google.appengine.api import app_identity -except ImportError: - app_identity = None from gcloud._helpers import UTC from gcloud._helpers import _NOW @@ -102,95 +82,10 @@ def get_credentials(): return client.GoogleCredentials.get_application_default() -def _get_pem_key(credentials): - """Gets private key for a PEM payload from a credentials object. - - :type credentials: :class:`service_account.ServiceAccountCredentials`, - :param credentials: The credentials used to create a private key - for signing text. - - :rtype: :class:`OpenSSL.crypto.PKey` - :returns: A PKey object used to sign text. - :raises: `TypeError` if `credentials` is the wrong type. - `EnvironmentError` if `crypto` did not import successfully. - """ - if isinstance(credentials, ServiceAccountCredentials): - if credentials._private_key_pkcs12 is not None: - # Take our PKCS12 (.p12) text and convert to PEM text. - pem_text = crypt.pkcs12_key_as_pem( - credentials._private_key_pkcs12, - credentials._private_key_password) - else: - pem_text = credentials._private_key_pkcs8_pem - else: - raise TypeError((credentials, - 'not a valid service account credentials type')) - - if crypto is None: - raise EnvironmentError( - 'pyOpenSSL must be installed to load a private key') - return crypto.load_privatekey(crypto.FILETYPE_PEM, pem_text) - - -def _get_signature_bytes(credentials, string_to_sign): - """Uses crypto attributes of credentials to sign a string/bytes. - - :type credentials: :class:`service_account.ServiceAccountCredentials`, - :class:`_GAECreds` - :param credentials: The credentials used for signing text (typically - involves the creation of a PKey). - - :type string_to_sign: string - :param string_to_sign: The string to be signed by the credentials. - - :rtype: bytes - :returns: Signed bytes produced by the credentials. - :raises: `EnvironmentError` if `crypto` did not import successfully. - """ - if isinstance(credentials, _GAECreds): - _, signed_bytes = app_identity.sign_blob(string_to_sign) - return signed_bytes - else: - # Sign the string with the PKey. - pkey = _get_pem_key(credentials) - if not isinstance(string_to_sign, six.binary_type): - string_to_sign = string_to_sign.encode('utf-8') - if crypto is None: - raise EnvironmentError( - 'pyOpenSSL must be installed to sign content using a ' - 'private key') - return crypto.sign(pkey, string_to_sign, 'SHA256') - - -def _get_service_account_name(credentials): - """Determines service account name from a credentials object. - - :type credentials: :class:`service_account.ServiceAccountCredentials`, - :class:`_GAECreds` - :param credentials: The credentials used to determine the service - account name. - - :rtype: string - :returns: Service account name associated with the credentials. - :raises: :class:`ValueError` if the credentials are not a valid service - account type. - """ - service_account_name = None - if isinstance(credentials, ServiceAccountCredentials): - service_account_name = credentials.service_account_email - elif isinstance(credentials, _GAECreds): - service_account_name = app_identity.get_service_account_name() - - if service_account_name is None: - raise ValueError('Service account name could not be determined ' - 'from credentials') - return service_account_name - - def _get_signed_query_params(credentials, expiration, string_to_sign): """Gets query parameters for creating a signed URL. - :type credentials: :class:`service_account.ServiceAccountCredentials` + :type credentials: :class:`oauth2client.client.AssertionCredentials` :param credentials: The credentials used to create a private key for signing text. @@ -204,9 +99,9 @@ def _get_signed_query_params(credentials, expiration, string_to_sign): :returns: Query parameters matching the signing credentials with a signed payload. """ - signature_bytes = _get_signature_bytes(credentials, string_to_sign) + _, signature_bytes = credentials.sign_blob(string_to_sign) signature = base64.b64encode(signature_bytes) - service_account_name = _get_service_account_name(credentials) + service_account_name = credentials.service_account_email return { 'GoogleAccessId': service_account_name, 'Expires': str(expiration), @@ -246,6 +141,15 @@ def generate_signed_url(credentials, resource, expiration, response_disposition=None, generation=None): """Generate signed URL to provide query-string auth'n to a resource. + .. note:: + + Assumes ``credentials`` implements a ``sign_blob()`` method that takes + bytes to sign and returns a pair of the key ID (unused here) and the + signed bytes (this is abstract in the base class + :class:`oauth2client.client.AssertionCredentials`). Also assumes + ``credentials`` has a ``service_account_email`` property which + identifies the credentials. + .. note:: If you are on Google Compute Engine, you can't generate a signed URL. diff --git a/gcloud/datastore/demo/__init__.py b/gcloud/datastore/demo/__init__.py deleted file mode 100644 index a49240c57180..000000000000 --- a/gcloud/datastore/demo/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from gcloud.environment_vars import TESTS_PROJECT - -__all__ = ['PROJECT'] - -PROJECT = os.getenv(TESTS_PROJECT) diff --git a/gcloud/datastore/demo/__main__.py b/gcloud/datastore/demo/__main__.py deleted file mode 100644 index e5e55065a9fa..000000000000 --- a/gcloud/datastore/demo/__main__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pragma NO COVER -from gcloud import demo -from gcloud import datastore - -demo.DemoRunner.from_module(datastore).run() diff --git a/gcloud/datastore/demo/demo.py b/gcloud/datastore/demo/demo.py deleted file mode 100644 index 5814db57b88b..000000000000 --- a/gcloud/datastore/demo/demo.py +++ /dev/null @@ -1,119 +0,0 @@ -# Welcome to the gCloud Datastore Demo! (hit enter) -# We're going to walk through some of the basics... -# Don't worry though. You don't need to do anything, just keep hitting enter... - -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Let's start by importing the demo module and initializing our client. -from gcloud import datastore -from gcloud.datastore import demo - -client = datastore.Client(project=demo.PROJECT) - -# Let's create a new entity of type "Thing" and name it 'Toy': -key = client.key('Thing') -toy = datastore.Entity(key) -toy.update({'name': 'Toy'}) - -# Now let's save it to our datastore: -client.put(toy) - -# If we look it up by its key, we should find it... -print(client.get(toy.key)) - -# And we should be able to delete it... -client.delete(toy.key) - -# Since we deleted it, if we do another lookup it shouldn't be there again: -print(client.get(toy.key)) - -# Now let's try a more advanced query. -# First, let's create some entities. -SAMPLE_DATA = [ - (1234, 'Computer', 10), - (2345, 'Computer', 8), - (3456, 'Laptop', 10), - (4567, 'Printer', 11), - (5678, 'Printer', 12), - (6789, 'Computer', 13)] -sample_keys = [] -for id, name, age in SAMPLE_DATA: - key = client.key('Thing', id) - sample_keys.append(key) - entity = datastore.Entity(key) - entity['name'] = name - entity['age'] = age - client.put(entity) -# We'll start by look at all Thing entities: -query = client.query(kind='Thing') - -# Let's look at the first two. -print(list(query.fetch(limit=2))) - -# Now let's check for Thing entities named 'Computer' -query.add_filter('name', '=', 'Computer') -print(list(query.fetch())) - -# If you want to filter by multiple attributes, -# you can call .add_filter multiple times on the query. -query.add_filter('age', '=', 10) -print(list(query.fetch())) - -# Now delete them. -client.delete_multi(sample_keys) - -# You can also work inside a transaction. -# (Check the official docs for explanations of what's happening here.) -with client.transaction() as xact: - print('Creating and saving an entity...') - key = client.key('Thing', 'foo') - thing = datastore.Entity(key) - thing['age'] = 10 - xact.put(thing) - - print('Creating and saving another entity...') - key2 = client.key('Thing', 'bar') - thing2 = datastore.Entity(key2) - thing2['age'] = 15 - xact.put(thing2) - - print('Committing the transaction...') - -# Now that the transaction is commited, let's delete the entities. -client.delete_multi([key, key2]) - -# To rollback a transaction, just call .rollback() -with client.transaction() as xact: - key = client.key('Thing', 'another') - thing = datastore.Entity(key) - xact.put(thing) - xact.rollback() - -# Let's check if the entity was actually created: -created = client.get(key) -print('yes' if created else 'no') - -# Remember, a key won't be complete until the transaction is commited. -# That is, while inside the transaction block, thing.key will be incomplete. -with client.transaction() as xact: - key = client.key('Thing') # partial - thing = datastore.Entity(key) - xact.put(thing) - print(thing.key) # This will still be partial - -print(thing.key) # This will be complete - -# Now let's delete the entity. -client.delete(thing.key) diff --git a/gcloud/datastore/query.py b/gcloud/datastore/query.py index acc5f820955a..fba2f4204d11 100644 --- a/gcloud/datastore/query.py +++ b/gcloud/datastore/query.py @@ -207,7 +207,7 @@ def add_filter(self, property_name, operator, value): :type value: :class:`int`, :class:`str`, :class:`bool`, :class:`float`, :class:`NoneType`, - :class`datetime.datetime` + :class:`datetime.datetime` :param value: The value to filter on. :raises: :class:`ValueError` if ``operation`` is not one of the diff --git a/gcloud/demo.py b/gcloud/demo.py deleted file mode 100644 index 05b78a4ab8c4..000000000000 --- a/gcloud/demo.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pragma NO COVER -from code import interact -import itertools -import os.path -import sys -import time -from six.moves import input - - -class DemoRunner(object): - """An interactive runner of demo scripts.""" - - KEYPRESS_DELAY = 0.02 - GLOBALS, LOCALS = globals(), locals() - CODE, COMMENT = 'code', 'comment' - - def __init__(self, fp): - self.lines = [line.rstrip() for line in fp.readlines()] - - @classmethod - def from_module(cls, module): - path = os.path.join(os.path.dirname(module.__file__), - 'demo', 'demo.py') - - return cls(open(path, 'r')) - - def run(self): - line_groups = itertools.groupby(self.lines, self.get_line_type) - - newline = False # Don't use newline on the first statement. - for group_type, lines in line_groups: - if group_type == self.COMMENT: - self.write(lines, newline=newline) - newline = True - - elif group_type == self.CODE: - self.code(lines) - - interact('(Hit CTRL-D to exit...)', local=self.LOCALS) - - def wait(self): - input() - - @classmethod - def get_line_type(cls, line): - if line.startswith('#'): - return cls.COMMENT - else: - return cls.CODE - - def get_indent_level(self, line): - if not line.strip(): - return None - return len(line) - len(line.lstrip()) - - def _print(self, text='', newline=True): - sys.stdout.write(text) - if newline: - sys.stdout.write('\n') - - def write(self, lines, newline=True): - self._print(newline=newline) - self._print('\n'.join(lines), False) - self.wait() - - def code(self, lines): - code_lines = [] - - for line in lines: - indent = self.get_indent_level(line) - - # If we've completed a block, - # run whatever code was built up in code_lines. - if indent == 0: - self._execute_lines(code_lines) - code_lines = [] - - # Print the prefix for the line depending on the indentation level. - if indent == 0: - self._print('>>> ', False) - elif indent > 0: - self._print('\n... ', False) - elif indent is None: - continue - - # Break the line into the code section and the comment section. - if '#' in line: - code, comment = line.split('#', 2) - else: - code, comment = line, None - - # 'Type' out the comment section. - for char in code.rstrip(): - time.sleep(self.KEYPRESS_DELAY) - sys.stdout.write(char) - sys.stdout.flush() - - # Print the comment section (not typed out). - if comment: - sys.stdout.write(' # %s' % comment.strip()) - - # Add the current line to the list of lines to be run - # in this block. - code_lines.append(line) - - # If we had any code built up that wasn't part of a completed block - # (ie, the lines ended with an indented line), - # run that code. - if code_lines: - self._execute_lines(code_lines) - - def _execute_lines(self, lines): - if lines: - self.wait() - - # Yes, this is crazy unsafe... but it's demo code. - exec('\n'.join(lines), self.GLOBALS, self.LOCALS) diff --git a/gcloud/pubsub/_helpers.py b/gcloud/pubsub/_helpers.py index dad877c0f91b..7b87e73e0355 100644 --- a/gcloud/pubsub/_helpers.py +++ b/gcloud/pubsub/_helpers.py @@ -14,6 +14,26 @@ """Helper functions for shared behavior.""" +import re + +from gcloud._helpers import _name_from_project_path + + +_TOPIC_TEMPLATE = re.compile(r""" + projects/ # static prefix + (?P[^/]+) # initial letter, wordchars + hyphen + /topics/ # static midfix + (?P[^/]+) # initial letter, wordchars + allowed punc +""", re.VERBOSE) + + +_SUBSCRIPTION_TEMPLATE = re.compile(r""" + projects/ # static prefix + (?P[^/]+) # initial letter, wordchars + hyphen + /subscriptions/ # static midfix + (?P[^/]+) # initial letter, wordchars + allowed punc +""", re.VERBOSE) + def topic_name_from_path(path, project): """Validate a topic URI path and get the topic name. @@ -31,15 +51,23 @@ def topic_name_from_path(path, project): the project from the ``path`` does not agree with the ``project`` passed in. """ - # PATH = 'projects/%s/topics/%s' % (PROJECT, TOPIC_NAME) - path_parts = path.split('/') - if (len(path_parts) != 4 or path_parts[0] != 'projects' or - path_parts[2] != 'topics'): - raise ValueError('Expected path to be of the form ' - 'projects/{project}/topics/{topic_name}') - if (len(path_parts) != 4 or path_parts[0] != 'projects' or - path_parts[2] != 'topics' or path_parts[1] != project): - raise ValueError('Project from client should agree with ' - 'project from resource.') - - return path_parts[3] + return _name_from_project_path(path, project, _TOPIC_TEMPLATE) + + +def subscription_name_from_path(path, project): + """Validate a subscription URI path and get the subscription name. + + :type path: string + :param path: URI path for a subscription API request. + + :type project: string + :param project: The project associated with the request. It is + included for validation purposes. + + :rtype: string + :returns: subscription name parsed from ``path``. + :raises: :class:`ValueError` if the ``path`` is ill-formed or if + the project from the ``path`` does not agree with the + ``project`` passed in. + """ + return _name_from_project_path(path, project, _SUBSCRIPTION_TEMPLATE) diff --git a/gcloud/pubsub/client.py b/gcloud/pubsub/client.py index 3e1374927a70..ecafae99af7d 100644 --- a/gcloud/pubsub/client.py +++ b/gcloud/pubsub/client.py @@ -80,8 +80,7 @@ def list_topics(self, page_size=None, page_token=None): for resource in resp.get('topics', ())] return topics, resp.get('nextPageToken') - def list_subscriptions(self, page_size=None, page_token=None, - topic_name=None): + def list_subscriptions(self, page_size=None, page_token=None): """List subscriptions for the project associated with this client. See: @@ -99,10 +98,6 @@ def list_subscriptions(self, page_size=None, page_token=None, passed, the API will return the first page of topics. - :type topic_name: string - :param topic_name: limit results to subscriptions bound to the given - topic. - :rtype: tuple, (list, str) :returns: list of :class:`gcloud.pubsub.subscription.Subscription`, plus a "next page token" string: if not None, indicates that @@ -117,11 +112,7 @@ def list_subscriptions(self, page_size=None, page_token=None, if page_token is not None: params['pageToken'] = page_token - if topic_name is None: - path = '/projects/%s/subscriptions' % (self.project,) - else: - path = '/projects/%s/topics/%s/subscriptions' % (self.project, - topic_name) + path = '/projects/%s/subscriptions' % (self.project,) resp = self.connection.api_request(method='GET', path=path, query_params=params) diff --git a/gcloud/pubsub/test__helpers.py b/gcloud/pubsub/test__helpers.py index 514883a922d8..26514629f817 100644 --- a/gcloud/pubsub/test__helpers.py +++ b/gcloud/pubsub/test__helpers.py @@ -21,27 +21,37 @@ def _callFUT(self, path, project): from gcloud.pubsub._helpers import topic_name_from_path return topic_name_from_path(path, project) - def test_invalid_path_length(self): - PATH = 'projects/foo' - PROJECT = None - self.assertRaises(ValueError, self._callFUT, PATH, PROJECT) - - def test_invalid_path_format(self): - TOPIC_NAME = 'TOPIC_NAME' - PROJECT = 'PROJECT' - PATH = 'foo/%s/bar/%s' % (PROJECT, TOPIC_NAME) - self.assertRaises(ValueError, self._callFUT, PATH, PROJECT) - - def test_invalid_project(self): + def test_w_simple_name(self): TOPIC_NAME = 'TOPIC_NAME' - PROJECT1 = 'PROJECT1' - PROJECT2 = 'PROJECT2' - PATH = 'projects/%s/topics/%s' % (PROJECT1, TOPIC_NAME) - self.assertRaises(ValueError, self._callFUT, PATH, PROJECT2) + PROJECT = 'my-project-1234' + PATH = 'projects/%s/topics/%s' % (PROJECT, TOPIC_NAME) + topic_name = self._callFUT(PATH, PROJECT) + self.assertEqual(topic_name, TOPIC_NAME) - def test_valid_data(self): - TOPIC_NAME = 'TOPIC_NAME' - PROJECT = 'PROJECT' + def test_w_name_w_all_extras(self): + TOPIC_NAME = 'TOPIC_NAME-part.one~part.two%part-three' + PROJECT = 'my-project-1234' PATH = 'projects/%s/topics/%s' % (PROJECT, TOPIC_NAME) topic_name = self._callFUT(PATH, PROJECT) self.assertEqual(topic_name, TOPIC_NAME) + + +class Test_subscription_name_from_path(unittest2.TestCase): + + def _callFUT(self, path, project): + from gcloud.pubsub._helpers import subscription_name_from_path + return subscription_name_from_path(path, project) + + def test_w_simple_name(self): + SUBSCRIPTION_NAME = 'SUBSCRIPTION_NAME' + PROJECT = 'my-project-1234' + PATH = 'projects/%s/subscriptions/%s' % (PROJECT, SUBSCRIPTION_NAME) + subscription_name = self._callFUT(PATH, PROJECT) + self.assertEqual(subscription_name, SUBSCRIPTION_NAME) + + def test_w_name_w_all_extras(self): + SUBSCRIPTION_NAME = 'SUBSCRIPTION_NAME-part.one~part.two%part-three' + PROJECT = 'my-project-1234' + PATH = 'projects/%s/subscriptions/%s' % (PROJECT, SUBSCRIPTION_NAME) + topic_name = self._callFUT(PATH, PROJECT) + self.assertEqual(topic_name, SUBSCRIPTION_NAME) diff --git a/gcloud/pubsub/test_client.py b/gcloud/pubsub/test_client.py index 32b4674a7172..54d54cc72162 100644 --- a/gcloud/pubsub/test_client.py +++ b/gcloud/pubsub/test_client.py @@ -196,47 +196,6 @@ def test_list_subscriptions_w_missing_key(self): self.assertEqual(req['path'], '/projects/%s/subscriptions' % PROJECT) self.assertEqual(req['query_params'], {}) - def test_list_subscriptions_with_topic_name(self): - from gcloud.pubsub.subscription import Subscription - PROJECT = 'PROJECT' - CREDS = _Credentials() - - CLIENT_OBJ = self._makeOne(project=PROJECT, credentials=CREDS) - - SUB_NAME_1 = 'subscription_1' - SUB_PATH_1 = 'projects/%s/subscriptions/%s' % (PROJECT, SUB_NAME_1) - SUB_NAME_2 = 'subscription_2' - SUB_PATH_2 = 'projects/%s/subscriptions/%s' % (PROJECT, SUB_NAME_2) - TOPIC_NAME = 'topic_name' - TOPIC_PATH = 'projects/%s/topics/%s' % (PROJECT, TOPIC_NAME) - SUB_INFO = [{'name': SUB_PATH_1, 'topic': TOPIC_PATH}, - {'name': SUB_PATH_2, 'topic': TOPIC_PATH}] - TOKEN = 'TOKEN' - RETURNED = {'subscriptions': SUB_INFO, 'nextPageToken': TOKEN} - # Replace the connection on the client with one of our own. - CLIENT_OBJ.connection = _Connection(RETURNED) - - # Execute request. - subscriptions, next_page_token = CLIENT_OBJ.list_subscriptions( - topic_name=TOPIC_NAME) - # Test values are correct. - self.assertEqual(len(subscriptions), 2) - self.assertTrue(isinstance(subscriptions[0], Subscription)) - self.assertEqual(subscriptions[0].name, SUB_NAME_1) - self.assertEqual(subscriptions[0].topic.name, TOPIC_NAME) - self.assertTrue(isinstance(subscriptions[1], Subscription)) - self.assertEqual(subscriptions[1].name, SUB_NAME_2) - self.assertEqual(subscriptions[1].topic.name, TOPIC_NAME) - self.assertTrue(subscriptions[1].topic is subscriptions[0].topic) - self.assertEqual(next_page_token, TOKEN) - self.assertEqual(len(CLIENT_OBJ.connection._requested), 1) - req = CLIENT_OBJ.connection._requested[0] - self.assertEqual(req['method'], 'GET') - self.assertEqual(req['path'], - '/projects/%s/topics/%s/subscriptions' - % (PROJECT, TOPIC_NAME)) - self.assertEqual(req['query_params'], {}) - def test_topic(self): PROJECT = 'PROJECT' TOPIC_NAME = 'TOPIC_NAME' diff --git a/gcloud/pubsub/test_topic.py b/gcloud/pubsub/test_topic.py index b390104c26e2..fa0c908efacb 100644 --- a/gcloud/pubsub/test_topic.py +++ b/gcloud/pubsub/test_topic.py @@ -336,8 +336,7 @@ def test_subscription(self): TOPIC_NAME = 'topic_name' PROJECT = 'PROJECT' CLIENT = _Client(project=PROJECT) - topic = self._makeOne(TOPIC_NAME, - client=CLIENT) + topic = self._makeOne(TOPIC_NAME, client=CLIENT) SUBSCRIPTION_NAME = 'subscription_name' subscription = topic.subscription(SUBSCRIPTION_NAME) @@ -345,6 +344,114 @@ def test_subscription(self): self.assertEqual(subscription.name, SUBSCRIPTION_NAME) self.assertTrue(subscription.topic is topic) + def test_list_subscriptions_no_paging(self): + from gcloud.pubsub.subscription import Subscription + TOPIC_NAME = 'topic_name' + PROJECT = 'PROJECT' + SUB_NAME_1 = 'subscription_1' + SUB_PATH_1 = 'projects/%s/subscriptions/%s' % (PROJECT, SUB_NAME_1) + SUB_NAME_2 = 'subscription_2' + SUB_PATH_2 = 'projects/%s/subscriptions/%s' % (PROJECT, SUB_NAME_2) + TOPIC_NAME = 'topic_name' + SUBS_LIST = [SUB_PATH_1, SUB_PATH_2] + TOKEN = 'TOKEN' + RETURNED = {'subscriptions': SUBS_LIST, 'nextPageToken': TOKEN} + + conn = _Connection(RETURNED) + CLIENT = _Client(project=PROJECT, connection=conn) + topic = self._makeOne(TOPIC_NAME, client=CLIENT) + + # Execute request. + subscriptions, next_page_token = topic.list_subscriptions() + # Test values are correct. + self.assertEqual(len(subscriptions), 2) + + subscription = subscriptions[0] + self.assertTrue(isinstance(subscription, Subscription)) + self.assertEqual(subscriptions[0].name, SUB_NAME_1) + self.assertTrue(subscription.topic is topic) + + subscription = subscriptions[1] + self.assertTrue(isinstance(subscription, Subscription)) + self.assertEqual(subscriptions[1].name, SUB_NAME_2) + self.assertTrue(subscription.topic is topic) + + self.assertEqual(next_page_token, TOKEN) + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], + '/projects/%s/topics/%s/subscriptions' + % (PROJECT, TOPIC_NAME)) + self.assertEqual(req['query_params'], {}) + + def test_list_subscriptions_with_paging(self): + from gcloud.pubsub.subscription import Subscription + TOPIC_NAME = 'topic_name' + PROJECT = 'PROJECT' + SUB_NAME_1 = 'subscription_1' + SUB_PATH_1 = 'projects/%s/subscriptions/%s' % (PROJECT, SUB_NAME_1) + SUB_NAME_2 = 'subscription_2' + SUB_PATH_2 = 'projects/%s/subscriptions/%s' % (PROJECT, SUB_NAME_2) + TOPIC_NAME = 'topic_name' + SUBS_LIST = [SUB_PATH_1, SUB_PATH_2] + PAGE_SIZE = 10 + TOKEN = 'TOKEN' + RETURNED = {'subscriptions': SUBS_LIST} + + conn = _Connection(RETURNED) + CLIENT = _Client(project=PROJECT, connection=conn) + topic = self._makeOne(TOPIC_NAME, client=CLIENT) + + # Execute request. + subscriptions, next_page_token = topic.list_subscriptions( + page_size=PAGE_SIZE, page_token=TOKEN) + # Test values are correct. + self.assertEqual(len(subscriptions), 2) + + subscription = subscriptions[0] + self.assertTrue(isinstance(subscription, Subscription)) + self.assertEqual(subscriptions[0].name, SUB_NAME_1) + self.assertTrue(subscription.topic is topic) + + subscription = subscriptions[1] + self.assertTrue(isinstance(subscription, Subscription)) + self.assertEqual(subscriptions[1].name, SUB_NAME_2) + self.assertTrue(subscription.topic is topic) + + self.assertEqual(next_page_token, None) + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], + '/projects/%s/topics/%s/subscriptions' + % (PROJECT, TOPIC_NAME)) + self.assertEqual(req['query_params'], + {'pageSize': PAGE_SIZE, 'pageToken': TOKEN}) + + def test_list_subscriptions_missing_key(self): + TOPIC_NAME = 'topic_name' + PROJECT = 'PROJECT' + TOPIC_NAME = 'topic_name' + + conn = _Connection({}) + CLIENT = _Client(project=PROJECT, connection=conn) + topic = self._makeOne(TOPIC_NAME, client=CLIENT) + + # Execute request. + subscriptions, next_page_token = topic.list_subscriptions() + # Test values are correct. + self.assertEqual(len(subscriptions), 0) + self.assertEqual(next_page_token, None) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], + '/projects/%s/topics/%s/subscriptions' + % (PROJECT, TOPIC_NAME)) + self.assertEqual(req['query_params'], {}) + class TestBatch(unittest2.TestCase): diff --git a/gcloud/pubsub/topic.py b/gcloud/pubsub/topic.py index c4ce30645938..b2143b86be72 100644 --- a/gcloud/pubsub/topic.py +++ b/gcloud/pubsub/topic.py @@ -19,6 +19,7 @@ from gcloud._helpers import _datetime_to_rfc3339 from gcloud._helpers import _NOW from gcloud.exceptions import NotFound +from gcloud.pubsub._helpers import subscription_name_from_path from gcloud.pubsub._helpers import topic_name_from_path from gcloud.pubsub.subscription import Subscription @@ -212,6 +213,51 @@ def delete(self, client=None): client = self._require_client(client) client.connection.api_request(method='DELETE', path=self.path) + def list_subscriptions(self, page_size=None, page_token=None, client=None): + """List subscriptions for the project associated with this client. + + See: + https://cloud.google.com/pubsub/reference/rest/v1/projects.topics.subscriptions/list + + :type page_size: int + :param page_size: maximum number of topics to return, If not passed, + defaults to a value set by the API. + + :type page_token: string + :param page_token: opaque marker for the next "page" of topics. If not + passed, the API will return the first page of + topics. + + :type client: :class:`gcloud.pubsub.client.Client` or ``NoneType`` + :param client: the client to use. If not passed, falls back to the + ``client`` stored on the current topic. + + :rtype: tuple, (list, str) + :returns: list of :class:`gcloud.pubsub.subscription.Subscription`, + plus a "next page token" string: if not None, indicates that + more topics can be retrieved with another call (pass that + value as ``page_token``). + """ + client = self._require_client(client) + params = {} + + if page_size is not None: + params['pageSize'] = page_size + + if page_token is not None: + params['pageToken'] = page_token + + path = '/projects/%s/topics/%s/subscriptions' % ( + self.project, self.name) + + resp = client.connection.api_request(method='GET', path=path, + query_params=params) + subscriptions = [] + for sub_path in resp.get('subscriptions', ()): + sub_name = subscription_name_from_path(sub_path, self.project) + subscriptions.append(Subscription(sub_name, self)) + return subscriptions, resp.get('nextPageToken') + class Batch(object): """Context manager: collect messages to publish via a single API call. diff --git a/gcloud/storage/blob.py b/gcloud/storage/blob.py index c8ced994f856..fe73a9b1a9a0 100644 --- a/gcloud/storage/blob.py +++ b/gcloud/storage/blob.py @@ -22,7 +22,7 @@ import time import six -from six.moves.urllib.parse import quote # pylint: disable=F0401 +from six.moves.urllib.parse import quote from gcloud._helpers import _rfc3339_to_datetime from gcloud.credentials import generate_signed_url @@ -277,6 +277,11 @@ def delete(self, client=None): def download_to_file(self, file_obj, client=None): """Download the contents of this blob into a file-like object. + .. note:: + + If the server-set property, :attr:`media_link`, is not yet + initialized, makes an additional API request to load it. + :type file_obj: file :param file_obj: A file handle to which to write the blob's data. @@ -287,6 +292,9 @@ def download_to_file(self, file_obj, client=None): :raises: :class:`gcloud.exceptions.NotFound` """ client = self._require_client(client) + if self.media_link is None: # not yet loaded + self.reload() + download_url = self.media_link # Use apitools 'Download' facility. diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 589f8527fb3f..d4e82b61bdb5 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -167,9 +167,12 @@ def create(self, client=None): """ client = self._require_client(client) query_params = {'project': client.project} + properties = dict( + (key, self._properties[key]) for key in self._changes) + properties['name'] = self.name api_response = client.connection.api_request( method='POST', path='/b', query_params=query_params, - data={'name': self.name}, _target_object=self) + data=properties, _target_object=self) self._set_properties(api_response) @property diff --git a/gcloud/storage/demo/__init__.py b/gcloud/storage/demo/__init__.py deleted file mode 100644 index a5d4b20fc566..000000000000 --- a/gcloud/storage/demo/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from gcloud.environment_vars import TESTS_PROJECT - -__all__ = ['PROJECT_ID'] - -PROJECT_ID = os.getenv(TESTS_PROJECT) diff --git a/gcloud/storage/demo/__main__.py b/gcloud/storage/demo/__main__.py deleted file mode 100644 index ada9e7858624..000000000000 --- a/gcloud/storage/demo/__main__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pragma NO COVER -from gcloud import demo -from gcloud import storage - -demo.DemoRunner.from_module(storage).run() diff --git a/gcloud/storage/demo/demo.py b/gcloud/storage/demo/demo.py deleted file mode 100644 index f64c05f145d0..000000000000 --- a/gcloud/storage/demo/demo.py +++ /dev/null @@ -1,55 +0,0 @@ -# Welcome to the gCloud Storage Demo! (hit enter) -# We're going to walk through some of the basics... -# Don't worry though. You don't need to do anything, just keep hitting enter... - -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Let's start by importing the demo module and getting a client: -import time - -from gcloud import storage -from gcloud.storage import demo - -client = storage.Client(project=demo.PROJECT_ID) - -# OK, now let's look at all of the buckets... -print(list(client.list_buckets())) # This might take a second... - -# Now let's create a new bucket... -bucket_name = ("bucket-%s" % time.time()).replace(".", "") # Get rid of dots. -print(bucket_name) -bucket = client.create_bucket(bucket_name) -print(bucket) - -# Let's look at all of the buckets again... -print(list(client.list_buckets())) - -# How about we create a new blob inside this bucket. -blob = bucket.blob("my-new-file.txt") - -# Now let's put some data in there. -blob.upload_from_string("this is some data!") - -# ... and we can read that data back again. -print(blob.download_as_string()) - -# Now let's delete that blob. -print(blob.delete()) - -# And now that we're done, let's delete that bucket... -print(bucket.delete()) - -# Alright! That's all! -# Here's an interactive prompt for you now... diff --git a/gcloud/storage/test_blob.py b/gcloud/storage/test_blob.py index 868498555587..2d7778357a22 100644 --- a/gcloud/storage/test_blob.py +++ b/gcloud/storage/test_blob.py @@ -260,7 +260,7 @@ def test_generate_signed_url_w_method_arg(self): def test_exists_miss(self): from six.moves.http_client import NOT_FOUND NONESUCH = 'nonesuch' - not_found_response = {'status': NOT_FOUND} + not_found_response = ({'status': NOT_FOUND}, b'') connection = _Connection(not_found_response) client = _Client(connection) bucket = _Bucket(client) @@ -270,7 +270,7 @@ def test_exists_miss(self): def test_exists_hit(self): from six.moves.http_client import OK BLOB_NAME = 'blob-name' - found_response = {'status': OK} + found_response = ({'status': OK}, b'') connection = _Connection(found_response) client = _Client(connection) bucket = _Bucket(client) @@ -281,7 +281,7 @@ def test_exists_hit(self): def test_delete(self): from six.moves.http_client import NOT_FOUND BLOB_NAME = 'blob-name' - not_found_response = {'status': NOT_FOUND} + not_found_response = ({'status': NOT_FOUND}, b'') connection = _Connection(not_found_response) client = _Client(connection) bucket = _Bucket(client) @@ -291,6 +291,32 @@ def test_delete(self): self.assertFalse(blob.exists()) self.assertEqual(bucket._deleted, [(BLOB_NAME, None)]) + def test_download_to_file_wo_media_link(self): + from six.moves.http_client import OK + from six.moves.http_client import PARTIAL_CONTENT + from io import BytesIO + BLOB_NAME = 'blob-name' + MEDIA_LINK = 'http://example.com/media/' + chunk1_response = {'status': PARTIAL_CONTENT, + 'content-range': 'bytes 0-2/6'} + chunk2_response = {'status': OK, + 'content-range': 'bytes 3-5/6'} + connection = _Connection( + (chunk1_response, b'abc'), + (chunk2_response, b'def'), + ) + # Only the 'reload' request hits on this side: the others are done + # through the 'http' object. + reload_response = {'status': OK, 'content-type': 'application/json'} + connection._responses = [(reload_response, {"mediaLink": MEDIA_LINK})] + client = _Client(connection) + bucket = _Bucket(client) + blob = self._makeOne(BLOB_NAME, bucket=bucket) + fh = BytesIO() + blob.download_to_file(fh) + self.assertEqual(fh.getvalue(), b'abcdef') + self.assertEqual(blob.media_link, MEDIA_LINK) + def _download_to_file_helper(self, chunk_size=None): from six.moves.http_client import OK from six.moves.http_client import PARTIAL_CONTENT @@ -749,10 +775,11 @@ def test_upload_from_string_w_text(self): self.assertEqual(rq[0]['body'], ENCODED) def test_make_public(self): + from six.moves.http_client import OK from gcloud.storage.acl import _ACLEntity BLOB_NAME = 'blob-name' permissive = [{'entity': 'allUsers', 'role': _ACLEntity.READER_ROLE}] - after = {'acl': permissive} + after = ({'status': OK}, {'acl': permissive}) connection = _Connection(after) client = _Client(connection) bucket = _Bucket(client=client) @@ -1092,10 +1119,10 @@ def __init__(self, *responses): def api_request(self, **kw): from six.moves.http_client import NOT_FOUND from gcloud.exceptions import NotFound - result = self._respond(**kw) - if result.get('status') == NOT_FOUND: - raise NotFound(result) - return result + info, content = self._respond(**kw) + if info.get('status') == NOT_FOUND: + raise NotFound(info) + return content def build_api_url(self, path, query_params=None, api_base_url=API_BASE_URL): diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index 947b1f0e600d..d7f127c44898 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -190,6 +190,45 @@ def test_create_hit(self): self.assertEqual(kw['query_params'], {'project': PROJECT}) self.assertEqual(kw['data'], DATA) + def test_create_w_extra_properties(self): + BUCKET_NAME = 'bucket-name' + PROJECT = 'PROJECT' + CORS = [{ + 'maxAgeSeconds': 60, + 'methods': ['*'], + 'origin': ['https://example.com/frontend'], + 'responseHeader': ['X-Custom-Header'], + }] + LIFECYCLE_RULES = [{ + "action": {"type": "Delete"}, + "condition": {"age": 365} + }] + LOCATION = 'eu' + STORAGE_CLASS = 'NEARLINE' + DATA = { + 'name': BUCKET_NAME, + 'cors': CORS, + 'lifecycle': {'rule': LIFECYCLE_RULES}, + 'location': LOCATION, + 'storageClass': STORAGE_CLASS, + 'versioning': {'enabled': True}, + } + connection = _Connection(DATA) + client = _Client(connection, project=PROJECT) + bucket = self._makeOne(client=client, name=BUCKET_NAME) + bucket.cors = CORS + bucket.lifecycle_rules = LIFECYCLE_RULES + bucket.location = LOCATION + bucket.storage_class = STORAGE_CLASS + bucket.versioning_enabled = True + bucket.create() + + kw, = connection._requested + self.assertEqual(kw['method'], 'POST') + self.assertEqual(kw['path'], '/b') + self.assertEqual(kw['query_params'], {'project': PROJECT}) + self.assertEqual(kw['data'], DATA) + def test_acl_property(self): from gcloud.storage.acl import BucketACL bucket = self._makeOne() diff --git a/gcloud/streaming/http_wrapper.py b/gcloud/streaming/http_wrapper.py index eda580d355f5..17c3b67171db 100644 --- a/gcloud/streaming/http_wrapper.py +++ b/gcloud/streaming/http_wrapper.py @@ -12,8 +12,8 @@ import httplib2 import six -from six.moves import http_client # pylint: disable=F0401 -from six.moves.urllib import parse # pylint: disable=F0401 +from six.moves import http_client +from six.moves.urllib import parse from gcloud.streaming.exceptions import BadStatusCodeError from gcloud.streaming.exceptions import RequestError diff --git a/gcloud/test__helpers.py b/gcloud/test__helpers.py index 5b6f329d2cb8..4f2cb849c89d 100644 --- a/gcloud/test__helpers.py +++ b/gcloud/test__helpers.py @@ -526,6 +526,41 @@ def test_it(self): self.assertEqual(self._callFUT(dt_stamp), timestamp) +class Test__name_from_project_path(unittest2.TestCase): + + PROJECT = 'PROJECT' + THING_NAME = 'THING_NAME' + TEMPLATE = r'projects/(?P\w+)/things/(?P\w+)' + + def _callFUT(self, path, project, template): + from gcloud._helpers import _name_from_project_path + return _name_from_project_path(path, project, template) + + def test_w_invalid_path_length(self): + PATH = 'projects/foo' + with self.assertRaises(ValueError): + self._callFUT(PATH, None, self.TEMPLATE) + + def test_w_invalid_path_segments(self): + PATH = 'foo/%s/bar/%s' % (self.PROJECT, self.THING_NAME) + with self.assertRaises(ValueError): + self._callFUT(PATH, self.PROJECT, self.TEMPLATE) + + def test_w_mismatched_project(self): + PROJECT1 = 'PROJECT1' + PROJECT2 = 'PROJECT2' + PATH = 'projects/%s/things/%s' % (PROJECT1, self.THING_NAME) + with self.assertRaises(ValueError): + self._callFUT(PATH, PROJECT2, self.TEMPLATE) + + def test_w_valid_data_w_compiled_regex(self): + import re + template = re.compile(self.TEMPLATE) + PATH = 'projects/%s/things/%s' % (self.PROJECT, self.THING_NAME) + name = self._callFUT(PATH, self.PROJECT, template) + self.assertEqual(name, self.THING_NAME) + + class _AppIdentity(object): def __init__(self, app_id): diff --git a/gcloud/test_credentials.py b/gcloud/test_credentials.py index 45b85aa4336f..e4108cff11c9 100644 --- a/gcloud/test_credentials.py +++ b/gcloud/test_credentials.py @@ -15,53 +15,6 @@ import unittest2 -def _setup_appengine_import(test_case, app_identity): - import sys - import types - - GOOGLE = types.ModuleType('google') - GAE = types.ModuleType('appengine') - GAE_API = types.ModuleType('api') - GAE_EXT = types.ModuleType('ext') - GAE_EXT_WEBAPP = types.ModuleType('webapp') - GAE_EXT_WEBAPP_UTIL = types.ModuleType('util') - - GOOGLE.appengine = GAE - GAE.api = GAE_API - GAE.api.app_identity = app_identity - GAE.api.memcache = None - GAE.api.users = None - GAE.ext = GAE_EXT - GAE.ext.db = _MockDB - GAE.ext.webapp = GAE_EXT_WEBAPP - GAE.ext.webapp.util = GAE_EXT_WEBAPP_UTIL - GAE.ext.webapp.util.login_required = None - GAE.ext.webapp.util.run_wsgi_app = None - - test_case._PREV_GOOGLE_MODULE = sys.modules['google'] - - sys.modules['google'] = GOOGLE - sys.modules['google.appengine'] = GAE - sys.modules['google.appengine.api'] = GAE_API - sys.modules['google.appengine.ext'] = GAE_EXT - sys.modules['google.appengine.ext.webapp'] = GAE_EXT_WEBAPP - sys.modules['google.appengine.ext.webapp.util'] = GAE_EXT_WEBAPP_UTIL - sys.modules['webapp2'] = GAE_EXT_WEBAPP - - -def _teardown_appengine_import(test_case): - import sys - sys.modules.pop('google') - sys.modules.pop('google.appengine') - sys.modules.pop('google.appengine.api') - sys.modules.pop('google.appengine.ext') - sys.modules.pop('google.appengine.ext.webapp') - sys.modules.pop('google.appengine.ext.webapp.util') - sys.modules.pop('webapp2') - - sys.modules['google'] = test_case._PREV_GOOGLE_MODULE - - class Test_get_credentials(unittest2.TestCase): def _callFUT(self): @@ -102,7 +55,7 @@ def _generate_helper(self, response_type=None, response_disposition=None, def _get_signed_query_params(*args): credentials, expiration = args[:2] return { - 'GoogleAccessId': credentials.service_account_name, + 'GoogleAccessId': credentials.service_account_email, 'Expires': str(expiration), 'Signature': SIGNED, } @@ -123,7 +76,7 @@ def _get_signed_query_params(*args): self.assertEqual(params.pop('Signature'), [SIGNED.decode('ascii')]) self.assertEqual(params.pop('Expires'), ['1000']) self.assertEqual(params.pop('GoogleAccessId'), - [_Credentials.service_account_name]) + [CREDENTIALS.service_account_email]) if response_type is not None: self.assertEqual(params.pop('response-content-type'), [response_type]) @@ -148,173 +101,6 @@ def test_w_custom_fields(self): generation=generation) -class Test__get_signature_bytes(unittest2.TestCase): - - def setUp(self): - SERVICE_ACCOUNT_NAME = 'SERVICE_ACCOUNT_NAME' - self.APP_IDENTITY = _AppIdentity(SERVICE_ACCOUNT_NAME) - _setup_appengine_import(self, self.APP_IDENTITY) - - def tearDown(self): - _teardown_appengine_import(self) - - def _callFUT(self, credentials, string_to_sign): - from gcloud.credentials import _get_signature_bytes - return _get_signature_bytes(credentials, string_to_sign) - - def _run_with_fake_crypto(self, credentials, private_key_text, - string_to_sign): - import base64 - import six - from gcloud._testing import _Monkey - from gcloud import credentials as MUT - - crypt = _Crypt() - load_result = object() - sign_result = object() - openssl_crypto = _OpenSSLCrypto(load_result, sign_result) - - with _Monkey(MUT, crypt=crypt, crypto=openssl_crypto): - result = self._callFUT(credentials, string_to_sign) - - if crypt._pkcs12_key_as_pem_called: - self.assertEqual(crypt._private_key_text, private_key_text) - self.assertEqual(crypt._private_key_password, 'notasecret') - self.assertEqual(openssl_crypto._loaded, - [(openssl_crypto.FILETYPE_PEM, _Crypt._KEY)]) - else: - self.assertEqual(openssl_crypto._loaded, - [(openssl_crypto.FILETYPE_PEM, private_key_text)]) - - if not isinstance(string_to_sign, six.binary_type): - string_to_sign = string_to_sign.encode('utf-8') - self.assertEqual(openssl_crypto._signed, - [(load_result, string_to_sign, 'SHA256')]) - - self.assertEqual(result, sign_result) - - def test_p12_type(self): - from oauth2client.service_account import ServiceAccountCredentials - ACCOUNT_NAME = 'dummy_service_account_name' - PRIVATE_KEY_TEXT = b'dummy_private_key_text' - STRING_TO_SIGN = b'dummy_signature' - SIGNER = object() - CREDENTIALS = ServiceAccountCredentials( - ACCOUNT_NAME, SIGNER) - CREDENTIALS._private_key_pkcs12 = PRIVATE_KEY_TEXT - CREDENTIALS._private_key_password = 'notasecret' - self._run_with_fake_crypto(CREDENTIALS, PRIVATE_KEY_TEXT, - STRING_TO_SIGN) - - def test_p12_type_non_bytes_to_sign(self): - from oauth2client.service_account import ServiceAccountCredentials - ACCOUNT_NAME = 'dummy_service_account_name' - PRIVATE_KEY_TEXT = b'dummy_private_key_text' - STRING_TO_SIGN = u'dummy_signature' - SIGNER = object() - CREDENTIALS = ServiceAccountCredentials( - ACCOUNT_NAME, SIGNER) - CREDENTIALS._private_key_pkcs12 = PRIVATE_KEY_TEXT - CREDENTIALS._private_key_password = 'notasecret' - self._run_with_fake_crypto(CREDENTIALS, PRIVATE_KEY_TEXT, - STRING_TO_SIGN) - - def test_json_type(self): - from oauth2client import service_account - from gcloud._testing import _Monkey - - PRIVATE_KEY_TEXT = 'dummy_private_key_pkcs8_text' - STRING_TO_SIGN = b'dummy_signature' - SIGNER = object() - CREDENTIALS = service_account.ServiceAccountCredentials( - 'dummy_service_account_email', SIGNER) - CREDENTIALS._private_key_pkcs8_pem = PRIVATE_KEY_TEXT - self._run_with_fake_crypto(CREDENTIALS, PRIVATE_KEY_TEXT, - STRING_TO_SIGN) - - def test_gae_type(self): - # Relies on setUp fixing up App Engine imports. - from oauth2client.contrib.appengine import AppAssertionCredentials - from gcloud._testing import _Monkey - from gcloud import credentials - - APP_IDENTITY = self.APP_IDENTITY - CREDENTIALS = AppAssertionCredentials([]) - STRING_TO_SIGN = b'STRING_TO_SIGN' - - with _Monkey(credentials, _GAECreds=AppAssertionCredentials, - app_identity=APP_IDENTITY): - signed_bytes = self._callFUT(CREDENTIALS, b'STRING_TO_SIGN') - - self.assertEqual(signed_bytes, STRING_TO_SIGN) - self.assertEqual(APP_IDENTITY._strings_signed, [STRING_TO_SIGN]) - - def test_without_pyopenssl(self): - from gcloud._testing import _Monkey - from gcloud import credentials as credentials_mod - - mock_called = [] - credentials = object() - - def mock_pem_key(local_creds): - mock_called.append(local_creds) - - with _Monkey(credentials_mod, crypto=None, _get_pem_key=mock_pem_key): - with self.assertRaises(EnvironmentError): - self._callFUT(credentials, b'STRING_TO_SIGN') - self.assertEqual(mock_called, [credentials]) - - -class Test__get_service_account_name(unittest2.TestCase): - - def setUp(self): - SERVICE_ACCOUNT_NAME = 'SERVICE_ACCOUNT_NAME' - self.APP_IDENTITY = _AppIdentity(SERVICE_ACCOUNT_NAME) - _setup_appengine_import(self, self.APP_IDENTITY) - - def tearDown(self): - _teardown_appengine_import(self) - - def _callFUT(self, credentials): - from gcloud.credentials import _get_service_account_name - return _get_service_account_name(credentials) - - def test_bad_type(self): - from oauth2client.client import OAuth2Credentials - CREDENTIALS = OAuth2Credentials('bogus_token', 'bogus_id', - 'bogus_secret', 'bogus_refresh', - None, None, None) - self.assertRaises(ValueError, self._callFUT, CREDENTIALS) - - def test_service_account_type(self): - from oauth2client import service_account - - SERVICE_ACCOUNT_NAME = 'SERVICE_ACCOUNT_NAME' - SIGNER = object() - CREDENTIALS = service_account.ServiceAccountCredentials( - SERVICE_ACCOUNT_NAME, SIGNER) - - found = self._callFUT(CREDENTIALS) - self.assertEqual(found, SERVICE_ACCOUNT_NAME) - - def test_gae_type(self): - # Relies on setUp fixing up App Engine imports. - from oauth2client.contrib.appengine import AppAssertionCredentials - from gcloud._testing import _Monkey - from gcloud import credentials - - APP_IDENTITY = self.APP_IDENTITY - SERVICE_ACCOUNT_NAME = APP_IDENTITY.service_account_name - - CREDENTIALS = AppAssertionCredentials([]) - - with _Monkey(credentials, _GAECreds=AppAssertionCredentials, - app_identity=APP_IDENTITY): - found = self._callFUT(CREDENTIALS) - - self.assertEqual(found, SERVICE_ACCOUNT_NAME) - - class Test__get_signed_query_params(unittest2.TestCase): def _callFUT(self, credentials, expiration, string_to_sign): @@ -327,112 +113,21 @@ def test_it(self): from gcloud._testing import _Monkey from gcloud import credentials as MUT - _called_get_sig = [] SIG_BYTES = b'DEADBEEF' - - def mock_get_sig_bytes(creds, string_to_sign): - _called_get_sig.append((creds, string_to_sign)) - return SIG_BYTES - - _called_get_name = [] ACCOUNT_NAME = object() - - def mock_get_name(creds): - _called_get_name.append((creds,)) - return ACCOUNT_NAME - - CREDENTIALS = object() + CREDENTIALS = _Credentials(sign_result=SIG_BYTES, + service_account_email=ACCOUNT_NAME) EXPIRATION = 100 STRING_TO_SIGN = 'dummy_signature' - with _Monkey(MUT, _get_signature_bytes=mock_get_sig_bytes, - _get_service_account_name=mock_get_name): - result = self._callFUT(CREDENTIALS, EXPIRATION, - STRING_TO_SIGN) + result = self._callFUT(CREDENTIALS, EXPIRATION, + STRING_TO_SIGN) self.assertEqual(result, { 'GoogleAccessId': ACCOUNT_NAME, 'Expires': str(EXPIRATION), 'Signature': base64.b64encode(b'DEADBEEF'), }) - self.assertEqual(_called_get_sig, - [(CREDENTIALS, STRING_TO_SIGN)]) - self.assertEqual(_called_get_name, [(CREDENTIALS,)]) - - -class Test__get_pem_key(unittest2.TestCase): - - def _callFUT(self, credentials): - from gcloud.credentials import _get_pem_key - return _get_pem_key(credentials) - - def test_bad_argument(self): - self.assertRaises(TypeError, self._callFUT, None) - - def test_signed_jwt_for_p12(self): - from oauth2client import service_account - from gcloud._testing import _Monkey - from gcloud import credentials as MUT - - PRIVATE_KEY = b'dummy_private_key_text' - SIGNER = object() - credentials = service_account.ServiceAccountCredentials( - 'dummy_service_account_email', SIGNER) - credentials._private_key_pkcs12 = PRIVATE_KEY - credentials._private_key_password = password = 'password-nope' - - crypt = _Crypt() - load_result = object() - openssl_crypto = _OpenSSLCrypto(load_result, None) - - with _Monkey(MUT, crypt=crypt, crypto=openssl_crypto): - result = self._callFUT(credentials) - - self.assertEqual(crypt._private_key_text, PRIVATE_KEY) - self.assertEqual(crypt._private_key_password, password) - self.assertEqual(result, load_result) - self.assertEqual(openssl_crypto._loaded, - [(openssl_crypto.FILETYPE_PEM, _Crypt._KEY)]) - self.assertEqual(openssl_crypto._signed, []) - - def test_service_account_via_json_key(self): - from oauth2client import service_account - from gcloud._testing import _Monkey - from gcloud import credentials as MUT - - scopes = [] - - PRIVATE_TEXT = 'dummy_private_key_pkcs8_text' - SIGNER = object() - credentials = service_account.ServiceAccountCredentials( - 'dummy_service_account_email', SIGNER, scopes=scopes) - credentials._private_key_pkcs8_pem = PRIVATE_TEXT - - load_result = object() - openssl_crypto = _OpenSSLCrypto(load_result, None) - - with _Monkey(MUT, crypto=openssl_crypto): - result = self._callFUT(credentials) - - self.assertEqual(result, load_result) - self.assertEqual(openssl_crypto._loaded, - [(openssl_crypto.FILETYPE_PEM, PRIVATE_TEXT)]) - self.assertEqual(openssl_crypto._signed, []) - - def test_without_pyopenssl(self): - from oauth2client import service_account - from gcloud._testing import _Monkey - from gcloud import credentials as credentials_mod - - PRIVATE_TEXT = 'dummy_private_key_pkcs8_text' - SIGNER = object() - - credentials = service_account.ServiceAccountCredentials( - 'dummy_service_account_email', SIGNER) - credentials._private_key_pkcs8_pem = PRIVATE_TEXT - - with _Monkey(credentials_mod, crypto=None): - with self.assertRaises(EnvironmentError): - self._callFUT(credentials) + self.assertEqual(CREDENTIALS._signed, [STRING_TO_SIGN]) class Test__get_expiration_seconds(unittest2.TestCase): @@ -519,7 +214,16 @@ def test_w_timedelta_days(self): class _Credentials(object): - service_account_name = 'testing@example.com' + + def __init__(self, service_account_email='testing@example.com', + sign_result=''): + self.service_account_email = service_account_email + self._sign_result = sign_result + self._signed = [] + + def sign_blob(self, bytes_to_sign): + self._signed.append(bytes_to_sign) + return None, self._sign_result class _Client(object): @@ -534,65 +238,3 @@ def get_application_default(): return self._signed self.GoogleCredentials = GoogleCredentials - - -class _Crypt(object): - - _pkcs12_key_as_pem_called = False - _KEY = '__PEM__' - - def pkcs12_key_as_pem(self, private_key_text, private_key_password): - self._pkcs12_key_as_pem_called = True - self._private_key_text = private_key_text - self._private_key_password = private_key_password - return self._KEY - - -class _OpenSSLCrypto(object): - - FILETYPE_PEM = object() - - def __init__(self, load_result, sign_result): - self._loaded = [] - self._load_result = load_result - self._signed = [] - self._sign_result = sign_result - - def load_privatekey(self, key_type, key_text): - self._loaded.append((key_type, key_text)) - return self._load_result - - def sign(self, pkey, to_sign, sign_algo): - self._signed.append((pkey, to_sign, sign_algo)) - return self._sign_result - - -class _AppIdentity(object): - - def __init__(self, service_account_name): - self._strings_signed = [] - self.service_account_name = service_account_name - - def get_service_account_name(self): - return self.service_account_name - - def sign_blob(self, string_to_sign): - self._strings_signed.append(string_to_sign) - throwaway = object() - return throwaway, string_to_sign - - -class _MockDB(object): - - Model = object - Property = object - StringProperty = object - _stored = [] - - @staticmethod - def non_transactional(*args, **kwargs): - _MockDB._stored.append((args, kwargs)) # To please lint. - - def do_nothing_wrapper(func): - return func - return do_nothing_wrapper diff --git a/scripts/pylintrc_default b/scripts/pylintrc_default index 2d47e0b8d8fc..1f254578f5d7 100644 --- a/scripts/pylintrc_default +++ b/scripts/pylintrc_default @@ -2,7 +2,7 @@ # # NOTES: # -# - Rules for test / demo code are generated into 'pylintrc_reduced' +# - Rules for test code are generated into 'pylintrc_reduced' # as deltas from this configuration by the 'run_pylint.py' script. # # - 'RATIONALE: API mapping' as a defense for non-default settings is @@ -193,7 +193,7 @@ no-space-check = # Maximum number of lines in a module # DEFAULT: max-module-lines=1000 # RATIONALE: API-mapping -max-module-lines=1500 +max-module-lines=1200 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). @@ -313,6 +313,9 @@ good-names = i, j, k, ex, Run, _, # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis # DEFAULT: ignored-modules= +# RATIONALE: six aliases stuff for compatibility. +# google.protobuf fixes up namespace package "late". +ignored-modules = six, google.protobuf # List of classes names for which member attributes should not be checked # (useful for classes with attributes dynamically set). diff --git a/scripts/run_pylint.py b/scripts/run_pylint.py index 203995243b3c..fe5c772d4c5d 100644 --- a/scripts/run_pylint.py +++ b/scripts/run_pylint.py @@ -17,7 +17,7 @@ This runs pylint as a script via subprocess in two different subprocesses. The first lints the production/library code using the default rc file (PRODUCTION_RC). The second lints the -demo/test code using an rc file (TEST_RC) which allows more style +test code using an rc file (TEST_RC) which allows more style violations (hence it has a reduced number of style checks). """ @@ -65,7 +65,7 @@ } TEST_RC_REPLACEMENTS = { 'FORMAT': { - 'max-module-lines': 1800, + 'max-module-lines': 1700, }, } @@ -125,7 +125,7 @@ def is_production_filename(filename): :rtype: bool :returns: Boolean indicating production status. """ - return not ('demo' in filename or 'test' in filename) + return 'test' not in filename def get_files_for_linting(allow_limited=True): @@ -194,7 +194,7 @@ def get_python_files(all_files=None): :rtype: tuple :returns: A tuple containing two lists and a boolean. The first list - contains all production files, the next all test/demo files and + contains all production files, the next all test files and the boolean indicates if a restricted fileset was used. """ using_restricted = False @@ -242,7 +242,7 @@ def main(): library_files, non_library_files, using_restricted = get_python_files() try: lint_fileset(library_files, PRODUCTION_RC, 'library code') - lint_fileset(non_library_files, TEST_RC, 'test and demo code') + lint_fileset(non_library_files, TEST_RC, 'test code') except SystemExit: if not using_restricted: raise @@ -253,7 +253,7 @@ def main(): library_files, non_library_files, _ = get_python_files( all_files=all_files) lint_fileset(library_files, PRODUCTION_RC, 'library code') - lint_fileset(non_library_files, TEST_RC, 'test and demo code') + lint_fileset(non_library_files, TEST_RC, 'test code') if __name__ == '__main__': diff --git a/scripts/verify_included_modules.py b/scripts/verify_included_modules.py index e290a020114e..eb1a6f3571fe 100644 --- a/scripts/verify_included_modules.py +++ b/scripts/verify_included_modules.py @@ -30,20 +30,17 @@ OBJECT_INVENTORY_RELPATH = os.path.join('_build', 'html', 'objects.inv') IGNORED_PREFIXES = ('test_', '_') IGNORED_MODULES = frozenset([ - 'gcloud.bigtable.client', - 'gcloud.bigtable.cluster', - 'gcloud.bigtable.column_family', - 'gcloud.bigtable.happybase.batch', - 'gcloud.bigtable.happybase.connection', - 'gcloud.bigtable.happybase.pool', - 'gcloud.bigtable.happybase.table', - 'gcloud.bigtable.row', - 'gcloud.bigtable.row_data', - 'gcloud.bigtable.table', - 'gcloud.datastore.demo.demo', - 'gcloud.demo', + 'gcloud.__init__', + 'gcloud.bigquery.__init__', + 'gcloud.bigtable.__init__', + 'gcloud.datastore.__init__', + 'gcloud.dns.__init__', 'gcloud.iterator', - 'gcloud.storage.demo.demo', + 'gcloud.pubsub.__init__', + 'gcloud.resource_manager.__init__', + 'gcloud.search.__init__', + 'gcloud.storage.__init__', + 'gcloud.streaming.__init__', 'gcloud.streaming.buffered_stream', 'gcloud.streaming.exceptions', 'gcloud.streaming.http_wrapper', @@ -73,6 +70,8 @@ def is_valid_module(filename): """ if not filename.endswith('.py'): return False + if filename == '__init__.py': + return True for prefix in IGNORED_PREFIXES: if filename.startswith(prefix): return False diff --git a/setup.py b/setup.py index fde5845cad25..9b44b3bc1d25 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ import os - from setuptools import setup from setuptools import find_packages @@ -14,15 +13,16 @@ REQUIREMENTS = [ 'httplib2 >= 0.9.1', 'googleapis-common-protos', - 'oauth2client >= 2.0.0.post1', - 'protobuf >= 3.0.0b2', + 'oauth2client >= 2.0.1', + 'protobuf >= 3.0.0b2, != 3.0.0.b2.post1', 'pyOpenSSL', 'six', ] +GRPC_EXTRAS = ['grpcio >= 0.13.0'] setup( name='gcloud', - version='0.10.1', + version='0.11.0', description='API Client library for Google Cloud', author='Google Cloud Platform', author_email='jjg+gcloud-python@google.com', @@ -35,6 +35,7 @@ include_package_data=True, zip_safe=False, install_requires=REQUIREMENTS, + extras_require={'grpc': GRPC_EXTRAS}, classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', diff --git a/system_tests/bigtable.py b/system_tests/bigtable.py index 36ec696dad1d..39d60e0658f6 100644 --- a/system_tests/bigtable.py +++ b/system_tests/bigtable.py @@ -24,10 +24,10 @@ from gcloud._helpers import UTC from gcloud.bigtable.client import Client from gcloud.bigtable.column_family import MaxVersionsGCRule -from gcloud.bigtable.row import ApplyLabelFilter -from gcloud.bigtable.row import ColumnQualifierRegexFilter -from gcloud.bigtable.row import RowFilterChain -from gcloud.bigtable.row import RowFilterUnion +from gcloud.bigtable.row_filters import ApplyLabelFilter +from gcloud.bigtable.row_filters import ColumnQualifierRegexFilter +from gcloud.bigtable.row_filters import RowFilterChain +from gcloud.bigtable.row_filters import RowFilterUnion from gcloud.bigtable.row_data import Cell from gcloud.bigtable.row_data import PartialRowData from gcloud.environment_vars import TESTS_PROJECT @@ -332,7 +332,7 @@ def setUp(self): def tearDown(self): for row in self.rows_to_delete: - row.clear_mutations() + row.clear() row.delete() row.commit() diff --git a/system_tests/bigtable_happybase.py b/system_tests/bigtable_happybase.py index 06f6dcf4d8b8..a0f3366d7e2d 100644 --- a/system_tests/bigtable_happybase.py +++ b/system_tests/bigtable_happybase.py @@ -13,6 +13,8 @@ # limitations under the License. +import operator +import struct import time import unittest2 @@ -23,6 +25,8 @@ from gcloud.environment_vars import TESTS_PROJECT +_PACK_I64 = struct.Struct('>q').pack +_FIRST_ELT = operator.itemgetter(0) _helpers.PROJECT = TESTS_PROJECT ZONE = 'us-central1-c' NOW_MILLIS = int(1000 * time.time()) @@ -38,6 +42,13 @@ COL_FAM2: {'max_versions': 1, 'time_to_live': TTL_FOR_TEST}, COL_FAM3: {}, # use defaults } +ROW_KEY1 = 'row-key1' +ROW_KEY2 = 'row-key2a' +ROW_KEY3 = 'row-key2b' +COL1 = COL_FAM1 + ':qual1' +COL2 = COL_FAM1 + ':qual2' +COL3 = COL_FAM2 + ':qual1' +COL4 = COL_FAM3 + ':qual3' class Config(object): @@ -111,3 +122,740 @@ def test_create_table_failure(self): with self.assertRaises(ValueError): connection.create_table(ALT_TABLE_NAME, empty_families) self.assertFalse(ALT_TABLE_NAME in connection.tables()) + + +class BaseTableTest(unittest2.TestCase): + + def setUp(self): + self.rows_to_delete = [] + + def tearDown(self): + for row_key in self.rows_to_delete: + Config.TABLE.delete(row_key) + + +class TestTable_families(BaseTableTest): + + def test_families(self): + families = Config.TABLE.families() + + self.assertEqual(set(families.keys()), set(FAMILIES.keys())) + for col_fam, settings in FAMILIES.items(): + retrieved = families[col_fam] + for key, value in settings.items(): + self.assertEqual(retrieved[key], value) + + +class TestTable_row(BaseTableTest): + + def test_row_when_empty(self): + row1 = Config.TABLE.row(ROW_KEY1) + row2 = Config.TABLE.row(ROW_KEY2) + + self.assertEqual(row1, {}) + self.assertEqual(row2, {}) + + def test_row_with_columns(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + value3 = 'value3' + value4 = 'value4' + row1_data = { + COL1: value1, + COL2: value2, + COL3: value3, + COL4: value4, + } + + # Need to clean-up row1 after. + self.rows_to_delete.append(ROW_KEY1) + table.put(ROW_KEY1, row1_data) + + # Make sure the vanilla write succeeded. + row1 = table.row(ROW_KEY1) + self.assertEqual(row1, row1_data) + + # Pick out specific columns. + row1_diff_fams = table.row(ROW_KEY1, columns=[COL1, COL4]) + self.assertEqual(row1_diff_fams, {COL1: value1, COL4: value4}) + row1_single_col = table.row(ROW_KEY1, columns=[COL3]) + self.assertEqual(row1_single_col, {COL3: value3}) + row1_col_fam = table.row(ROW_KEY1, columns=[COL_FAM1]) + self.assertEqual(row1_col_fam, {COL1: value1, COL2: value2}) + row1_fam_qual_overlap1 = table.row(ROW_KEY1, columns=[COL1, COL_FAM1]) + self.assertEqual(row1_fam_qual_overlap1, {COL1: value1, COL2: value2}) + row1_fam_qual_overlap2 = table.row(ROW_KEY1, columns=[COL_FAM1, COL1]) + self.assertEqual(row1_fam_qual_overlap2, + {COL1: value1, COL2: value2}) + row1_multiple_col_fams = table.row(ROW_KEY1, + columns=[COL_FAM1, COL_FAM2]) + self.assertEqual(row1_multiple_col_fams, + {COL1: value1, COL2: value2, COL3: value3}) + + def test_row_with_timestamp(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + value3 = 'value3' + + # Need to clean-up row1 after. + self.rows_to_delete.append(ROW_KEY1) + table.put(ROW_KEY1, {COL1: value1}) + table.put(ROW_KEY1, {COL2: value2}) + table.put(ROW_KEY1, {COL3: value3}) + + # Make sure the vanilla write succeeded. + row1 = table.row(ROW_KEY1, include_timestamp=True) + ts1 = row1[COL1][1] + ts2 = row1[COL2][1] + ts3 = row1[COL3][1] + + expected_row = { + COL1: (value1, ts1), + COL2: (value2, ts2), + COL3: (value3, ts3), + } + self.assertEqual(row1, expected_row) + + # Make sure the timestamps are (strictly) ascending. + self.assertTrue(ts1 < ts2 < ts3) + + # Use timestamps to retrieve row. + first_two = table.row(ROW_KEY1, timestamp=ts2 + 1, + include_timestamp=True) + self.assertEqual(first_two, { + COL1: (value1, ts1), + COL2: (value2, ts2), + }) + first_one = table.row(ROW_KEY1, timestamp=ts2, + include_timestamp=True) + self.assertEqual(first_one, { + COL1: (value1, ts1), + }) + + +class TestTable_rows(BaseTableTest): + + def test_rows(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + value3 = 'value3' + row1_data = {COL1: value1, COL2: value2} + row2_data = {COL1: value3} + + # Need to clean-up row1 and row2 after. + self.rows_to_delete.append(ROW_KEY1) + self.rows_to_delete.append(ROW_KEY2) + table.put(ROW_KEY1, row1_data) + table.put(ROW_KEY2, row2_data) + + rows = sorted(table.rows([ROW_KEY1, ROW_KEY2]), key=_FIRST_ELT) + row1, row2 = rows + self.assertEqual(row1, (ROW_KEY1, row1_data)) + self.assertEqual(row2, (ROW_KEY2, row2_data)) + + def test_rows_with_returned_timestamps(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + value3 = 'value3' + row1_data = {COL1: value1, COL2: value2} + row2_data = {COL1: value3} + + # Need to clean-up row1 and row2 after. + self.rows_to_delete.append(ROW_KEY1) + self.rows_to_delete.append(ROW_KEY2) + with table.batch() as batch: + batch.put(ROW_KEY1, row1_data) + batch.put(ROW_KEY2, row2_data) + + rows = sorted(table.rows([ROW_KEY1, ROW_KEY2], include_timestamp=True), + key=_FIRST_ELT) + row1, row2 = rows + self.assertEqual(row1[0], ROW_KEY1) + self.assertEqual(row2[0], ROW_KEY2) + + # Drop the keys now that we have checked. + _, row1 = row1 + _, row2 = row2 + + ts = row1[COL1][1] + # All will have the same timestamp since we used batch. + expected_row1_result = {COL1: (value1, ts), COL2: (value2, ts)} + self.assertEqual(row1, expected_row1_result) + # NOTE: This method was written before Cloud Bigtable had the concept + # of batching, so each mutation is sent individually. (This + # will be the case until support for the MutateRows() RPC method + # is implemented.) Thus, the server-side timestamps correspond + # to separate calls to row.commit(). We could circumvent this by + # manually using the local time and storing it on mutations + # before sending. + ts3 = row2[COL1][1] + expected_row2_result = {COL1: (value3, ts3)} + self.assertEqual(row2, expected_row2_result) + + def test_rows_with_columns(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + value3 = 'value3' + row1_data = {COL1: value1, COL2: value2} + row2_data = {COL1: value3} + + # Need to clean-up row1 and row2 after. + self.rows_to_delete.append(ROW_KEY1) + self.rows_to_delete.append(ROW_KEY2) + table.put(ROW_KEY1, row1_data) + table.put(ROW_KEY2, row2_data) + + # Filter a single column present in both rows. + rows_col1 = sorted(table.rows([ROW_KEY1, ROW_KEY2], columns=[COL1]), + key=_FIRST_ELT) + row1, row2 = rows_col1 + self.assertEqual(row1, (ROW_KEY1, {COL1: value1})) + self.assertEqual(row2, (ROW_KEY2, {COL1: value3})) + + # Filter a column not present in one row. + rows_col2 = table.rows([ROW_KEY1, ROW_KEY2], columns=[COL2]) + self.assertEqual(rows_col2, [(ROW_KEY1, {COL2: value2})]) + + # Filter a column family. + rows_col_fam1 = sorted( + table.rows([ROW_KEY1, ROW_KEY2], columns=[COL_FAM1]), + key=_FIRST_ELT) + row1, row2 = rows_col_fam1 + self.assertEqual(row1, (ROW_KEY1, row1_data)) + self.assertEqual(row2, (ROW_KEY2, row2_data)) + + # Filter a column family with no entries. + rows_col_fam2 = table.rows([ROW_KEY1, ROW_KEY2], columns=[COL_FAM2]) + self.assertEqual(rows_col_fam2, []) + + # Filter a column family that overlaps with a column. + rows_col_fam_overlap1 = sorted(table.rows([ROW_KEY1, ROW_KEY2], + columns=[COL1, COL_FAM1]), + key=_FIRST_ELT) + row1, row2 = rows_col_fam_overlap1 + self.assertEqual(row1, (ROW_KEY1, row1_data)) + self.assertEqual(row2, (ROW_KEY2, row2_data)) + + # Filter a column family that overlaps with a column (opposite order). + rows_col_fam_overlap2 = sorted(table.rows([ROW_KEY1, ROW_KEY2], + columns=[COL_FAM1, COL1]), + key=_FIRST_ELT) + row1, row2 = rows_col_fam_overlap2 + self.assertEqual(row1, (ROW_KEY1, row1_data)) + self.assertEqual(row2, (ROW_KEY2, row2_data)) + + def test_rows_with_timestamp(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + value3 = 'value3' + value4 = 'value4' + + # Need to clean-up row1 and row2 after. + self.rows_to_delete.append(ROW_KEY1) + self.rows_to_delete.append(ROW_KEY2) + table.put(ROW_KEY1, {COL1: value1}) + table.put(ROW_KEY2, {COL1: value2}) + table.put(ROW_KEY1, {COL2: value3}) + table.put(ROW_KEY1, {COL4: value4}) + + # Just grab the timestamps + rows = sorted(table.rows([ROW_KEY1, ROW_KEY2], include_timestamp=True), + key=_FIRST_ELT) + row1, row2 = rows + self.assertEqual(row1[0], ROW_KEY1) + self.assertEqual(row2[0], ROW_KEY2) + _, row1 = row1 + _, row2 = row2 + ts1 = row1[COL1][1] + ts2 = row2[COL1][1] + ts3 = row1[COL2][1] + ts4 = row1[COL4][1] + + # Make sure the timestamps are (strictly) ascending. + self.assertTrue(ts1 < ts2 < ts3 < ts4) + + # Rows before the third timestamp (assumes exclusive endpoint). + rows = sorted(table.rows([ROW_KEY1, ROW_KEY2], timestamp=ts3, + include_timestamp=True), + key=_FIRST_ELT) + row1, row2 = rows + self.assertEqual(row1, (ROW_KEY1, {COL1: (value1, ts1)})) + self.assertEqual(row2, (ROW_KEY2, {COL1: (value2, ts2)})) + + # All writes (bump the exclusive endpoint by 1 millisecond). + rows = sorted(table.rows([ROW_KEY1, ROW_KEY2], timestamp=ts4 + 1, + include_timestamp=True), + key=_FIRST_ELT) + row1, row2 = rows + row1_all_data = { + COL1: (value1, ts1), + COL2: (value3, ts3), + COL4: (value4, ts4), + } + self.assertEqual(row1, (ROW_KEY1, row1_all_data)) + self.assertEqual(row2, (ROW_KEY2, {COL1: (value2, ts2)})) + + # First three writes, restricted to column 2. + rows = table.rows([ROW_KEY1, ROW_KEY2], timestamp=ts4, + columns=[COL2], include_timestamp=True) + self.assertEqual(rows, [(ROW_KEY1, {COL2: (value3, ts3)})]) + + +class TestTable_cells(BaseTableTest): + + def test_cells(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + value3 = 'value3' + + # Need to clean-up row1 after. + self.rows_to_delete.append(ROW_KEY1) + table.put(ROW_KEY1, {COL1: value1}) + table.put(ROW_KEY1, {COL1: value2}) + table.put(ROW_KEY1, {COL1: value3}) + + # Check with no extra arguments. + all_values = table.cells(ROW_KEY1, COL1) + self.assertEqual(all_values, [value3, value2, value1]) + + # Check the timestamp on all the cells. + all_cells = table.cells(ROW_KEY1, COL1, include_timestamp=True) + self.assertEqual(len(all_cells), 3) + + ts3 = all_cells[0][1] + ts2 = all_cells[1][1] + ts1 = all_cells[2][1] + self.assertEqual(all_cells, + [(value3, ts3), (value2, ts2), (value1, ts1)]) + + # Limit to the two latest cells. + latest_two = table.cells(ROW_KEY1, COL1, include_timestamp=True, + versions=2) + self.assertEqual(latest_two, [(value3, ts3), (value2, ts2)]) + + # Limit to cells before the 2nd timestamp (inclusive). + first_two = table.cells(ROW_KEY1, COL1, include_timestamp=True, + timestamp=ts2 + 1) + self.assertEqual(first_two, [(value2, ts2), (value1, ts1)]) + + # Limit to cells before the 2nd timestamp (exclusive). + first_cell = table.cells(ROW_KEY1, COL1, include_timestamp=True, + timestamp=ts2) + self.assertEqual(first_cell, [(value1, ts1)]) + + +class TestTable_scan(BaseTableTest): + + def test_scan_when_empty(self): + scan_result = list(Config.TABLE.scan()) + self.assertEqual(scan_result, []) + + def test_scan_single_row(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + row1_data = {COL1: value1, COL2: value2} + + # Need to clean-up row1 after. + self.rows_to_delete.append(ROW_KEY1) + table.put(ROW_KEY1, row1_data) + + scan_result = list(table.scan()) + self.assertEqual(scan_result, [(ROW_KEY1, row1_data)]) + + scan_result_cols = list(table.scan(columns=[COL1])) + self.assertEqual(scan_result_cols, [(ROW_KEY1, {COL1: value1})]) + + scan_result_ts = list(table.scan(include_timestamp=True)) + self.assertEqual(len(scan_result_ts), 1) + only_row = scan_result_ts[0] + self.assertEqual(only_row[0], ROW_KEY1) + row_values = only_row[1] + ts = row_values[COL1][1] + self.assertEqual(row_values, {COL1: (value1, ts), COL2: (value2, ts)}) + + def test_scan_filters(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + value3 = 'value3' + value4 = 'value4' + value5 = 'value5' + value6 = 'value6' + row1_data = {COL1: value1, COL2: value2} + row2_data = {COL2: value3, COL3: value4} + row3_data = {COL3: value5, COL4: value6} + + # Need to clean-up row1/2/3 after. + self.rows_to_delete.append(ROW_KEY1) + self.rows_to_delete.append(ROW_KEY2) + self.rows_to_delete.append(ROW_KEY3) + table.put(ROW_KEY1, row1_data) + table.put(ROW_KEY2, row2_data) + table.put(ROW_KEY3, row3_data) + + # Basic scan (no filters) + scan_result = list(table.scan()) + self.assertEqual(scan_result, [ + (ROW_KEY1, row1_data), + (ROW_KEY2, row2_data), + (ROW_KEY3, row3_data), + ]) + + # Limit the size of the scan + scan_result = list(table.scan(limit=1)) + self.assertEqual(scan_result, [ + (ROW_KEY1, row1_data), + ]) + + # Scan with a row prefix. + prefix = ROW_KEY2[:-1] + self.assertEqual(prefix, ROW_KEY3[:-1]) + scan_result_prefixed = list(table.scan(row_prefix=prefix)) + self.assertEqual(scan_result_prefixed, [ + (ROW_KEY2, row2_data), + (ROW_KEY3, row3_data), + ]) + + # Make sure our keys are sorted in order + row_keys = [ROW_KEY1, ROW_KEY2, ROW_KEY3] + self.assertEqual(row_keys, sorted(row_keys)) + + # row_start alone (inclusive) + scan_result_row_start = list(table.scan(row_start=ROW_KEY2)) + self.assertEqual(scan_result_row_start, [ + (ROW_KEY2, row2_data), + (ROW_KEY3, row3_data), + ]) + + # row_stop alone (exclusive) + scan_result_row_stop = list(table.scan(row_stop=ROW_KEY2)) + self.assertEqual(scan_result_row_stop, [ + (ROW_KEY1, row1_data), + ]) + + # Both row_start and row_stop + scan_result_row_stop_and_start = list( + table.scan(row_start=ROW_KEY1, row_stop=ROW_KEY3)) + self.assertEqual(scan_result_row_stop_and_start, [ + (ROW_KEY1, row1_data), + (ROW_KEY2, row2_data), + ]) + + def test_scan_timestamp(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + value3 = 'value3' + value4 = 'value4' + value5 = 'value5' + value6 = 'value6' + + # Need to clean-up row1/2/3 after. + self.rows_to_delete.append(ROW_KEY1) + self.rows_to_delete.append(ROW_KEY2) + self.rows_to_delete.append(ROW_KEY3) + table.put(ROW_KEY3, {COL4: value6}) + table.put(ROW_KEY2, {COL3: value4}) + table.put(ROW_KEY2, {COL2: value3}) + table.put(ROW_KEY1, {COL2: value2}) + table.put(ROW_KEY3, {COL3: value5}) + table.put(ROW_KEY1, {COL1: value1}) + + # Retrieve all the timestamps so we can filter with them. + scan_result = list(table.scan(include_timestamp=True)) + self.assertEqual(len(scan_result), 3) + row1, row2, row3 = scan_result + self.assertEqual(row1[0], ROW_KEY1) + self.assertEqual(row2[0], ROW_KEY2) + self.assertEqual(row3[0], ROW_KEY3) + + # Drop the keys now that we have checked. + _, row1 = row1 + _, row2 = row2 + _, row3 = row3 + + # These are numbered in order of insertion, **not** in + # the order of the values. + ts1 = row3[COL4][1] + ts2 = row2[COL3][1] + ts3 = row2[COL2][1] + ts4 = row1[COL2][1] + ts5 = row3[COL3][1] + ts6 = row1[COL1][1] + + self.assertEqual(row1, {COL1: (value1, ts6), COL2: (value2, ts4)}) + self.assertEqual(row2, {COL2: (value3, ts3), COL3: (value4, ts2)}) + self.assertEqual(row3, {COL3: (value5, ts5), COL4: (value6, ts1)}) + + # All cells before ts1 (exclusive) + scan_result_before_ts1 = list(table.scan(timestamp=ts1, + include_timestamp=True)) + self.assertEqual(scan_result_before_ts1, []) + + # All cells before ts2 (inclusive) + scan_result_before_ts2 = list(table.scan(timestamp=ts2 + 1, + include_timestamp=True)) + self.assertEqual(scan_result_before_ts2, [ + (ROW_KEY2, {COL3: (value4, ts2)}), + (ROW_KEY3, {COL4: (value6, ts1)}), + ]) + + # All cells before ts6 (exclusive) + scan_result_before_ts6 = list(table.scan(timestamp=ts6, + include_timestamp=True)) + self.assertEqual(scan_result_before_ts6, [ + (ROW_KEY1, {COL2: (value2, ts4)}), + (ROW_KEY2, {COL2: (value3, ts3), COL3: (value4, ts2)}), + (ROW_KEY3, {COL3: (value5, ts5), COL4: (value6, ts1)}), + ]) + + +class TestTable_put(BaseTableTest): + + def test_put(self): + value1 = 'value1' + value2 = 'value2' + row1_data = {COL1: value1, COL2: value2} + + # Need to clean-up row1 after. + self.rows_to_delete.append(ROW_KEY1) + Config.TABLE.put(ROW_KEY1, row1_data) + + row1 = Config.TABLE.row(ROW_KEY1) + self.assertEqual(row1, row1_data) + + # Check again, but this time with timestamps. + row1 = Config.TABLE.row(ROW_KEY1, include_timestamp=True) + timestamp1 = row1[COL1][1] + timestamp2 = row1[COL2][1] + self.assertEqual(timestamp1, timestamp2) + + row1_data_with_timestamps = {COL1: (value1, timestamp1), + COL2: (value2, timestamp2)} + self.assertEqual(row1, row1_data_with_timestamps) + + def test_put_with_timestamp(self): + value1 = 'value1' + value2 = 'value2' + row1_data = {COL1: value1, COL2: value2} + ts = NOW_MILLIS + + # Need to clean-up row1 after. + self.rows_to_delete.append(ROW_KEY1) + Config.TABLE.put(ROW_KEY1, row1_data, timestamp=ts) + + # Check again, but this time with timestamps. + row1 = Config.TABLE.row(ROW_KEY1, include_timestamp=True) + row1_data_with_timestamps = {COL1: (value1, ts), + COL2: (value2, ts)} + self.assertEqual(row1, row1_data_with_timestamps) + + +class TestTable_delete(BaseTableTest): + + def test_delete(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + row1_data = {COL1: value1, COL2: value2} + + # Need to clean-up row1 after. + self.rows_to_delete.append(ROW_KEY1) + table.put(ROW_KEY1, row1_data) + + row1 = table.row(ROW_KEY1) + self.assertEqual(row1, row1_data) + + table.delete(ROW_KEY1) + row1_after = table.row(ROW_KEY1) + self.assertEqual(row1_after, {}) + + def test_delete_with_columns(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + row1_data = {COL1: value1, COL2: value2} + + # Need to clean-up row1 after. + self.rows_to_delete.append(ROW_KEY1) + table.put(ROW_KEY1, row1_data) + + row1 = table.row(ROW_KEY1) + self.assertEqual(row1, row1_data) + + table.delete(ROW_KEY1, columns=[COL1]) + row1_after = table.row(ROW_KEY1) + self.assertEqual(row1_after, {COL2: value2}) + + def test_delete_with_column_family(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + value3 = 'value3' + row1_data = {COL1: value1, COL2: value2, COL4: value3} + + # Need to clean-up row1 after. + self.rows_to_delete.append(ROW_KEY1) + table.put(ROW_KEY1, row1_data) + + row1 = table.row(ROW_KEY1) + self.assertEqual(row1, row1_data) + + table.delete(ROW_KEY1, columns=[COL_FAM1]) + row1_after = table.row(ROW_KEY1) + self.assertEqual(row1_after, {COL4: value3}) + + def test_delete_with_columns_family_overlap(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + row1_data = {COL1: value1, COL2: value2} + + # Need to clean-up row1 after. + self.rows_to_delete.append(ROW_KEY1) + + # First go-around, use [COL_FAM1, COL1] + table.put(ROW_KEY1, row1_data) + row1 = table.row(ROW_KEY1) + self.assertEqual(row1, row1_data) + + table.delete(ROW_KEY1, columns=[COL_FAM1, COL1]) + row1_after = table.row(ROW_KEY1) + self.assertEqual(row1_after, {}) + + # Second go-around, use [COL1, COL_FAM1] + table.put(ROW_KEY1, row1_data) + row1 = table.row(ROW_KEY1) + self.assertEqual(row1, row1_data) + + table.delete(ROW_KEY1, columns=[COL1, COL_FAM1]) + row1_after = table.row(ROW_KEY1) + self.assertEqual(row1_after, {}) + + def test_delete_with_timestamp(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + + # Need to clean-up row1 after. + self.rows_to_delete.append(ROW_KEY1) + table.put(ROW_KEY1, {COL1: value1}) + table.put(ROW_KEY1, {COL2: value2}) + + row1 = table.row(ROW_KEY1, include_timestamp=True) + ts1 = row1[COL1][1] + ts2 = row1[COL2][1] + + self.assertTrue(ts1 < ts2) + + # NOTE: The Cloud Bigtable "Mutation.DeleteFromRow" mutation does + # not support timestamps. Even attempting to send one + # conditionally(via CheckAndMutateRowRequest) deletes the + # entire row. + # NOTE: Cloud Bigtable deletes **ALSO** use an inclusive timestamp + # at the endpoint, but only because we fake this when + # creating Batch._delete_range. + table.delete(ROW_KEY1, columns=[COL1, COL2], timestamp=ts1 - 1) + row1_after_early_delete = table.row(ROW_KEY1, include_timestamp=True) + self.assertEqual(row1_after_early_delete, row1) + + # NOTE: Cloud Bigtable deletes **ALSO** use an inclusive timestamp + # at the endpoint, but only because we fake this when + # creating Batch._delete_range. + table.delete(ROW_KEY1, columns=[COL1, COL2], timestamp=ts1) + row1_after_incl_delete = table.row(ROW_KEY1, include_timestamp=True) + self.assertEqual(row1_after_incl_delete, {COL2: (value2, ts2)}) + + def test_delete_with_columns_and_timestamp(self): + table = Config.TABLE + value1 = 'value1' + value2 = 'value2' + + # Need to clean-up row1 after. + self.rows_to_delete.append(ROW_KEY1) + table.put(ROW_KEY1, {COL1: value1}) + table.put(ROW_KEY1, {COL2: value2}) + + row1 = table.row(ROW_KEY1, include_timestamp=True) + ts1 = row1[COL1][1] + ts2 = row1[COL2][1] + + # Delete with conditions that have no matches. + table.delete(ROW_KEY1, timestamp=ts1, columns=[COL2]) + row1_after_delete = table.row(ROW_KEY1, include_timestamp=True) + # NOTE: COL2 is still present since it occurs after ts1 and + # COL1 is still present since it is not in `columns`. + self.assertEqual(row1_after_delete, row1) + + # Delete with conditions that have no matches. + # NOTE: Cloud Bigtable can't use a timestamp with column families + # since "Mutation.DeleteFromFamily" does not include a + # timestamp range. + # NOTE: Cloud Bigtable deletes **ALSO** use an inclusive timestamp + # at the endpoint, but only because we fake this when + # creating Batch._delete_range. + table.delete(ROW_KEY1, timestamp=ts1, columns=[COL1, COL2]) + row1_delete_fam = table.row(ROW_KEY1, include_timestamp=True) + # NOTE: COL2 is still present since it occurs after ts1 and + # COL1 is still present since it is not in `columns`. + self.assertEqual(row1_delete_fam, {COL2: (value2, ts2)}) + + +class TestTableCounterMethods(BaseTableTest): + + def test_counter_get(self): + table = Config.TABLE + + # Need to clean-up row1 after. + self.rows_to_delete.append(ROW_KEY1) + + self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), {}) + initial_counter = table.counter_get(ROW_KEY1, COL1) + self.assertEqual(initial_counter, 0) + + self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), + {COL1: _PACK_I64(0)}) + + def test_counter_inc(self): + table = Config.TABLE + + # Need to clean-up row1 after. + self.rows_to_delete.append(ROW_KEY1) + + self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), {}) + initial_counter = table.counter_get(ROW_KEY1, COL1) + self.assertEqual(initial_counter, 0) + + inc_value = 10 + updated_counter = table.counter_inc(ROW_KEY1, COL1, value=inc_value) + self.assertEqual(updated_counter, inc_value) + + # Check that the value is set (does not seem to occur on HBase). + self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), + {COL1: _PACK_I64(inc_value)}) + + def test_counter_dec(self): + table = Config.TABLE + + # Need to clean-up row1 after. + self.rows_to_delete.append(ROW_KEY1) + + self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), {}) + initial_counter = table.counter_get(ROW_KEY1, COL1) + self.assertEqual(initial_counter, 0) + + dec_value = 10 + updated_counter = table.counter_dec(ROW_KEY1, COL1, value=dec_value) + self.assertEqual(updated_counter, -dec_value) + + # Check that the value is set (does not seem to occur on HBase). + self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), + {COL1: _PACK_I64(-dec_value)}) diff --git a/system_tests/pubsub.py b/system_tests/pubsub.py index b4e73f161779..956a788c6d36 100644 --- a/system_tests/pubsub.py +++ b/system_tests/pubsub.py @@ -118,8 +118,7 @@ def test_list_subscriptions(self): self.assertFalse(topic.exists()) topic.create() self.to_delete.append(topic) - empty, _ = Config.CLIENT.list_subscriptions( - topic_name=DEFAULT_TOPIC_NAME) + empty, _ = topic.list_subscriptions() self.assertEqual(len(empty), 0) subscriptions_to_create = [ 'new%d' % (1000 * time.time(),), @@ -132,10 +131,9 @@ def test_list_subscriptions(self): self.to_delete.append(subscription) # Retrieve the subscriptions. - all_subscriptions, _ = Config.CLIENT.list_subscriptions() + all_subscriptions, _ = topic.list_subscriptions() created = [subscription for subscription in all_subscriptions - if subscription.name in subscriptions_to_create and - subscription.topic.name == DEFAULT_TOPIC_NAME] + if subscription.name in subscriptions_to_create] self.assertEqual(len(created), len(subscriptions_to_create)) def test_message_pull_mode_e2e(self): diff --git a/tox.ini b/tox.ini index a5539551bde3..976c176ae731 100644 --- a/tox.ini +++ b/tox.ini @@ -21,6 +21,18 @@ covercmd = --cover-branches \ --nocapture +# Until grpcio 0.13.1 ships, this environment is broken on UCS2 builds. +# See: https://github.com/grpc/grpc/issues/5280 and +# https://github.com/grpc/grpc/pull/5319 +#[testenv:py27] +#basepython = +# python2.7 +#deps = +# {[testenv]deps} +# grpcio >= 0.13.0 +#setenv = +# PYTHONPATH = + [testenv:cover] basepython = python2.7