Merge master into datastore-v1beta3

.coveragerc

@@ -1,7 +1,5 @@
 [report]
 omit =
-    */demo/*
-    */demo.py
     */_generated/*.py
 exclude_lines =
     # Re-enable the standard pragma

docs/bigquery-usage.rst

@@ -220,8 +220,8 @@ Update all writable metadata for a table
    >>> dataset = client.dataset('dataset_name')
    >>> table = dataset.table(name='person_ages')
    >>> table.schema = [
-   ...     SchemaField(name='full_name', type='string', mode='required'),
-   ...     SchemaField(name='age', type='int', mode='required)]
+   ...     SchemaField('full_name', 'STRING', mode='required'),
+   ...     SchemaField('age', 'INTEGER', mode='required)]
    >>> table.update()  # API request
 
 Upload table data from a file:
@@ -233,8 +233,8 @@ Upload table data from a file:
    >>> dataset = client.dataset('dataset_name')
    >>> table = dataset.table(name='person_ages')
    >>> table.schema = [
-   ...     SchemaField(name='full_name', type='string', mode='required'),
-   ...     SchemaField(name='age', type='int', mode='required)]
+   ...     SchemaField('full_name', 'STRING', mode='required'),
+   ...     SchemaField('age', 'INTEGER', mode='required)]
    >>> with open('person_ages.csv', 'rb') as csv_file:
    ...     table.upload_from_file(csv_file, CSV,
    ...                            create_disposition='CREATE_IF_NEEDED')
@@ -384,8 +384,8 @@ Load data synchronously from a local CSV file into a new table:
    >>> client = bigquery.Client()
    >>> table = dataset.table(name='person_ages')
    >>> table.schema = [
-   ...     SchemaField(name='full_name', type='string', mode='required'),
-   ...     SchemaField(name='age', type='int', mode='required)]
+   ...     SchemaField('full_name', 'STRING', mode='required'),
+   ...     SchemaField('age', 'INTEGER', mode='required)]
    >>> with open('/path/to/person_ages.csv', 'rb') as file_obj:
    ...     reader = csv.reader(file_obj)
    ...     rows = list(reader)
@@ -405,8 +405,8 @@ the job locally:
    >>> client = bigquery.Client()
    >>> table = dataset.table(name='person_ages')
    >>> table.schema = [
-   ...     SchemaField(name='full_name', type='string', mode='required'),
-   ...     SchemaField(name='age', type='int', mode='required)]
+   ...     SchemaField('full_name', 'STRING', mode='required'),
+   ...     SchemaField('age', 'INTEGER', mode='required)]
    >>> job = client.load_table_from_storage(
    ...     'load-from-storage-job', table, 'gs://bucket-name/object-prefix*')
    >>> job.source_format = 'CSV'

docs/bigtable-client-intro.rst

@@ -0,0 +1,92 @@
+Base for Everything
+===================
+
+To use the API, the :class:`Client <gcloud.bigtable.client.Client>`
+class defines a high-level interface which handles authorization
+and creating other objects:
+
+.. code:: python
+
+    from gcloud.bigtable.client import Client
+    client = Client()
+
+Long-lived Defaults
+-------------------
+
+When creating a :class:`Client <gcloud.bigtable.client.Client>`, the
+``user_agent`` and ``timeout_seconds`` arguments have sensible
+defaults
+(:data:`DEFAULT_USER_AGENT <gcloud.bigtable.client.DEFAULT_USER_AGENT>` and
+:data:`DEFAULT_TIMEOUT_SECONDS <gcloud.bigtable.client.DEFAULT_TIMEOUT_SECONDS>`).
+However, you may over-ride them and these will be used throughout all API
+requests made with the ``client`` you create.
+
+Configuration
+-------------
+
+- For an overview of authentication in ``gcloud-python``,
+  see :doc:`gcloud-auth`.
+
+- In addition to any authentication configuration, you can also set the
+  :envvar:`GCLOUD_PROJECT` environment variable for the Google Cloud Console
+  project you'd like to interact with. If your code is running in Google App
+  Engine or Google Compute Engine the project will be detected automatically.
+  (Setting this environment variable is not required, you may instead pass the
+  ``project`` explicitly when constructing a
+  :class:`Client <gcloud.storage.client.Client>`).
+
+- After configuring your environment, create a
+  :class:`Client <gcloud.storage.client.Client>`
+
+  .. code::
+
+     >>> from gcloud import bigtable
+     >>> client = bigtable.Client()
+
+  or pass in ``credentials`` and ``project`` explicitly
+
+  .. code::
+
+     >>> from gcloud import bigtable
+     >>> client = bigtable.Client(project='my-project', credentials=creds)
+
+.. tip::
+
+    Be sure to use the **Project ID**, not the **Project Number**.
+
+Admin API Access
+----------------
+
+If you'll be using your client to make `Cluster Admin`_ and `Table Admin`_
+API requests, you'll need to pass the ``admin`` argument:
+
+.. code:: python
+
+    client = bigtable.Client(admin=True)
+
+Read-Only Mode
+--------------
+
+If on the other hand, you only have (or want) read access to the data,
+you can pass the ``read_only`` argument:
+
+.. code:: python
+
+    client = bigtable.Client(read_only=True)
+
+This will ensure that the
+:data:`READ_ONLY_SCOPE <gcloud.bigtable.client.READ_ONLY_SCOPE>` is used
+for API requests (so any accidental requests that would modify data will
+fail).
+
+Next Step
+---------
+
+After a :class:`Client <gcloud.bigtable.client.Client>`, the next highest-level
+object is a :class:`Cluster <gcloud.bigtable.cluster.Cluster>`. You'll need
+one before you can interact with tables or data.
+
+Head next to learn about the :doc:`bigtable-cluster-api`.
+
+.. _Cluster Admin: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/tree/master/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1
+.. _Table Admin: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/tree/master/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1

docs/bigtable-client.rst

@@ -0,0 +1,7 @@
+Client
+~~~~~~
+
+.. automodule:: gcloud.bigtable.client
+  :members:
+  :undoc-members:
+  :show-inheritance:

docs/bigtable-cluster-api.rst

@@ -0,0 +1,181 @@
+Cluster Admin API
+=================
+
+After creating a :class:`Client <gcloud.bigtable.client.Client>`, you can
+interact with individual clusters, groups of clusters or available
+zones for a project.
+
+List Clusters
+-------------
+
+If you want a comprehensive list of all existing clusters, make a
+`ListClusters`_ API request with
+:meth:`Client.list_clusters() <gcloud.bigtable.client.Client.list_clusters>`:
+
+.. code:: python
+
+    clusters = client.list_clusters()
+
+List Zones
+----------
+
+If you aren't sure which ``zone`` to create a cluster in, find out
+which zones your project has access to with a `ListZones`_ API request
+with :meth:`Client.list_zones() <gcloud.bigtable.client.Client.list_zones>`:
+
+.. code:: python
+
+    zones = client.list_zones()
+
+You can choose a :class:`string <str>` from among the result to pass to
+the :class:`Cluster <gcloud.bigtable.cluster.Cluster>` constructor.
+
+The available zones (as of February 2016) are
+
+.. code:: python
+
+    >>> zones
+    [u'asia-east1-b', u'europe-west1-c', u'us-central1-c', u'us-central1-b']
+
+Cluster Factory
+---------------
+
+To create a :class:`Cluster <gcloud.bigtable.cluster.Cluster>` object:
+
+.. code:: python
+
+    cluster = client.cluster(zone, cluster_id,
+                             display_name=display_name,
+                             serve_nodes=3)
+
+Both ``display_name`` and ``serve_nodes`` are optional. When not provided,
+``display_name`` defaults to the ``cluster_id`` value and ``serve_nodes``
+defaults to the minimum allowed:
+:data:`DEFAULT_SERVE_NODES <gcloud.bigtable.cluster.DEFAULT_SERVE_NODES>`.
+
+Even if this :class:`Cluster <gcloud.bigtable.cluster.Cluster>` already
+has been created with the API, you'll want this object to use as a
+parent of a :class:`Table <gcloud.bigtable.table.Table>` just as the
+:class:`Client <gcloud.bigtable.client.Client>` is used as the parent of
+a :class:`Cluster <gcloud.bigtable.cluster.Cluster>`.
+
+Create a new Cluster
+--------------------
+
+After creating the cluster object, make a `CreateCluster`_ API request
+with :meth:`create() <gcloud.bigtable.cluster.Cluster.create>`:
+
+.. code:: python
+
+    cluster.display_name = 'My very own cluster'
+    cluster.create()
+
+If you would like more than the minimum number of nodes
+(:data:`DEFAULT_SERVE_NODES <gcloud.bigtable.cluster.DEFAULT_SERVE_NODES>`)
+in your cluster:
+
+.. code:: python
+
+    cluster.serve_nodes = 10
+    cluster.create()
+
+Check on Current Operation
+--------------------------
+
+.. note::
+
+    When modifying a cluster (via a `CreateCluster`_, `UpdateCluster`_ or
+    `UndeleteCluster`_ request), the Bigtable API will return a
+    `long-running operation`_ and a corresponding
+    :class:`Operation <gcloud.bigtable.cluster.Operation>` object
+    will be returned by each of
+    :meth:`create() <gcloud.bigtable.cluster.Cluster.create>`,
+    :meth:`update() <gcloud.bigtable.cluster.Cluster.update>` and
+    :meth:`undelete() <gcloud.bigtable.cluster.Cluster.undelete>`.
+
+You can check if a long-running operation (for a
+:meth:`create() <gcloud.bigtable.cluster.Cluster.create>`,
+:meth:`update() <gcloud.bigtable.cluster.Cluster.update>` or
+:meth:`undelete() <gcloud.bigtable.cluster.Cluster.undelete>`) has finished
+by making a `GetOperation`_ request with
+:meth:`Operation.finished() <gcloud.bigtable.cluster.Operation.finished>`:
+
+.. code:: python
+
+    >>> operation = cluster.create()
+    >>> operation.finished()
+    True
+
+.. note::
+
+    Once an :class:`Operation <gcloud.bigtable.cluster.Operation>` object
+    has returned :data:`True` from
+    :meth:`finished() <gcloud.bigtable.cluster.Operation.finished>`, the
+    object should not be re-used. Subsequent calls to
+    :meth:`finished() <gcloud.bigtable.cluster.Operation.finished>`
+    will result in a :class:`ValueError <exceptions.ValueError>`.
+
+Get metadata for an existing Cluster
+------------------------------------
+
+After creating the cluster object, make a `GetCluster`_ API request
+with :meth:`reload() <gcloud.bigtable.cluster.Cluster.reload>`:
+
+.. code:: python
+
+    cluster.reload()
+
+This will load ``serve_nodes`` and ``display_name`` for the existing
+``cluster`` in addition to the ``cluster_id``, ``zone`` and ``project``
+already set on the :class:`Cluster <gcloud.bigtable.cluster.Cluster>` object.
+
+Update an existing Cluster
+--------------------------
+
+After creating the cluster object, make an `UpdateCluster`_ API request
+with :meth:`update() <gcloud.bigtable.cluster.Cluster.update>`:
+
+.. code:: python
+
+    client.display_name = 'New display_name'
+    cluster.update()
+
+Delete an existing Cluster
+--------------------------
+
+Make a `DeleteCluster`_ API request with
+:meth:`delete() <gcloud.bigtable.cluster.Cluster.delete>`:
+
+.. code:: python
+
+    cluster.delete()
+
+Undelete a deleted Cluster
+--------------------------
+
+Make an `UndeleteCluster`_ API request with
+:meth:`undelete() <gcloud.bigtable.cluster.Cluster.undelete>`:
+
+.. code:: python
+
+    cluster.undelete()
+
+Next Step
+---------
+
+Now we go down the hierarchy from
+:class:`Cluster <gcloud.bigtable.cluster.Cluster>` to a
+:class:`Table <gcloud.bigtable.table.Table>`.
+
+Head next to learn about the :doc:`bigtable-table-api`.
+
+.. _Cluster Admin API: https://cloud.google.com/bigtable/docs/creating-cluster
+.. _CreateCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L66-L68
+.. _GetCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L38-L40
+.. _UpdateCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L93-L95
+.. _DeleteCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L109-L111
+.. _ListZones: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L33-L35
+.. _ListClusters: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L44-L46
+.. _GetOperation: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/longrunning/operations.proto#L43-L45
+.. _UndeleteCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L126-L128
+.. _long-running operation: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/2aae624081f652427052fb652d3ae43d8ac5bf5a/bigtable-protos/src/main/proto/google/longrunning/operations.proto#L73-L102

docs/bigtable-cluster.rst

@@ -0,0 +1,7 @@
+Cluster
+~~~~~~~
+
+.. automodule:: gcloud.bigtable.cluster
+  :members:
+  :undoc-members:
+  :show-inheritance:

docs/bigtable-column-family.rst

@@ -0,0 +1,50 @@
+Column Families
+===============
+
+When creating a
+:class:`ColumnFamily <gcloud.bigtable.column_family.ColumnFamily>`, it is
+possible to set garbage collection rules for expired data.
+
+By setting a rule, cells in the table matching the rule will be deleted
+during periodic garbage collection (which executes opportunistically in the
+background).
+
+The types
+:class:`MaxAgeGCRule <gcloud.bigtable.column_family.MaxAgeGCRule>`,
+:class:`MaxVersionsGCRule <gcloud.bigtable.column_family.MaxVersionsGCRule>`,
+:class:`GarbageCollectionRuleUnion <gcloud.bigtable.column_family.GarbageCollectionRuleUnion>` and
+:class:`GarbageCollectionRuleIntersection <gcloud.bigtable.column_family.GarbageCollectionRuleIntersection>`
+can all be used as the optional ``gc_rule`` argument in the
+:class:`ColumnFamily <gcloud.bigtable.column_family.ColumnFamily>`
+constructor. This value is then used in the
+:meth:`create() <gcloud.bigtable.column_family.ColumnFamily.create>` and
+:meth:`update() <gcloud.bigtable.column_family.ColumnFamily.update>` methods.
+
+These rules can be nested arbitrarily, with a
+:class:`MaxAgeGCRule <gcloud.bigtable.column_family.MaxAgeGCRule>` or
+:class:`MaxVersionsGCRule <gcloud.bigtable.column_family.MaxVersionsGCRule>`
+at the lowest level of the nesting:
+
+.. code:: python
+
+    import datetime
+
+    max_age = datetime.timedelta(days=3)
+    rule1 = MaxAgeGCRule(max_age)
+    rule2 = MaxVersionsGCRule(1)
+
+    # Make a composite that matches anything older than 3 days **AND**
+    # with more than 1 version.
+    rule3 = GarbageCollectionIntersection(rules=[rule1, rule2])
+
+    # Make another composite that matches our previous intersection
+    # **OR** anything that has more than 3 versions.
+    rule4 = GarbageCollectionRule(max_num_versions=3)
+    rule5 = GarbageCollectionUnion(rules=[rule3, rule4])
+
+----
+
+.. automodule:: gcloud.bigtable.column_family
+  :members:
+  :undoc-members:
+  :show-inheritance: