Skip to content

Commit

Permalink
Merge pull request #2060 from tseaver/bigquery-partition-tidy
Browse files Browse the repository at this point in the history
Slightly more explicit coverage for partition-related methods.
  • Loading branch information
tseaver authored Aug 9, 2016
2 parents eab9cd6 + 1e6996e commit 4dbd7f9
Show file tree
Hide file tree
Showing 11 changed files with 268 additions and 173 deletions.
7 changes: 7 additions & 0 deletions docs/bigquery-schema.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Schemas
~~~~~~~

.. automodule:: gcloud.bigquery.schema
:members:
:show-inheritance:

1 change: 1 addition & 0 deletions docs/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
bigquery-job
bigquery-table
bigquery-query
bigquery-schema

.. toctree::
:maxdepth: 0
Expand Down
2 changes: 1 addition & 1 deletion gcloud/bigquery/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
from gcloud.bigquery.connection import Connection
from gcloud.bigquery.dataset import AccessGrant
from gcloud.bigquery.dataset import Dataset
from gcloud.bigquery.table import SchemaField
from gcloud.bigquery.schema import SchemaField
from gcloud.bigquery.table import Table


Expand Down
2 changes: 1 addition & 1 deletion gcloud/bigquery/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from gcloud.exceptions import NotFound
from gcloud._helpers import _datetime_from_microseconds
from gcloud.bigquery.dataset import Dataset
from gcloud.bigquery.table import SchemaField
from gcloud.bigquery.schema import SchemaField
from gcloud.bigquery.table import Table
from gcloud.bigquery.table import _build_schema_resource
from gcloud.bigquery.table import _parse_schema_resource
Expand Down
52 changes: 52 additions & 0 deletions gcloud/bigquery/schema.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Scheamas for BigQuery tables / queries."""


class SchemaField(object):
"""Describe a single field within a table schema.
:type name: str
:param name: the name of the field
:type field_type: str
:param field_type: the type of the field (one of 'STRING', 'INTEGER',
'FLOAT', 'BOOLEAN', 'TIMESTAMP' or 'RECORD')
:type mode: str
:param mode: the type of the field (one of 'NULLABLE', 'REQUIRED',
or 'REPEATED')
:type description: str
:param description: optional description for the field
:type fields: list of :class:`SchemaField`, or None
:param fields: subfields (requires ``field_type`` of 'RECORD').
"""
def __init__(self, name, field_type, mode='NULLABLE', description=None,
fields=None):
self.name = name
self.field_type = field_type
self.mode = mode
self.description = description
self.fields = fields

def __eq__(self, other):
return (
self.name == other.name and
self.field_type.lower() == other.field_type.lower() and
self.mode == other.mode and
self.description == other.description and
self.fields == other.fields)
75 changes: 21 additions & 54 deletions gcloud/bigquery/table.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,49 +28,13 @@
from gcloud.streaming.http_wrapper import make_api_request
from gcloud.streaming.transfer import RESUMABLE_UPLOAD
from gcloud.streaming.transfer import Upload
from gcloud.bigquery.schema import SchemaField
from gcloud.bigquery._helpers import _rows_from_json


_MARKER = object()


class SchemaField(object):
"""Describe a single field within a table schema.
:type name: str
:param name: the name of the field
:type field_type: str
:param field_type: the type of the field (one of 'STRING', 'INTEGER',
'FLOAT', 'BOOLEAN', 'TIMESTAMP' or 'RECORD')
:type mode: str
:param mode: the type of the field (one of 'NULLABLE', 'REQUIRED',
or 'REPEATED')
:type description: str
:param description: optional description for the field
:type fields: list of :class:`SchemaField`, or None
:param fields: subfields (requires ``field_type`` of 'RECORD').
"""
def __init__(self, name, field_type, mode='NULLABLE', description=None,
fields=None):
self.name = name
self.field_type = field_type
self.mode = mode
self.description = description
self.fields = fields

def __eq__(self, other):
return (
self.name == other.name and
self.field_type.lower() == other.field_type.lower() and
self.mode == other.mode and
self.description == other.description and
self.fields == other.fields)


class Table(object):
"""Tables represent a set of rows whose values correspond to a schema.
Expand Down Expand Up @@ -245,25 +209,22 @@ def partitioning_type(self, value):
:type value: str
:param value: partitioning type only "DAY" is currently supported
"""
if not (isinstance(value, six.string_types)
and value.upper() == "DAY") and value is not None:
if value not in ('DAY', None):
raise ValueError("value must be one of ['DAY', None]")

self._properties.setdefault('timePartitioning', {})
if value is not None:
self._properties['timePartitioning']['type'] = value.upper()
if value is None:
self._properties.pop('timePartitioning', None)
else:
time_part = self._properties.setdefault('timePartitioning', {})
time_part['type'] = value.upper()

@property
def partition_expiration(self):
"""Expiration time in ms for a partition
:rtype: int, or ``NoneType``
:returns: Returns the time in ms for partition expiration
"""
expiry = None
if "timePartitioning" in self._properties:
time_part = self._properties.get("timePartitioning")
expiry = time_part.get("expirationMs")
return expiry
return self._properties.get('timePartitioning', {}).get('expirationMs')

@partition_expiration.setter
def partition_expiration(self, value):
Expand All @@ -272,13 +233,19 @@ def partition_expiration(self, value):
:type value: int
:param value: partition experiation time in ms
"""
if not isinstance(value, int):
raise ValueError("must be an integer representing millisseconds")
try:
self._properties["timePartitioning"]["expirationMs"] = value
except KeyError:
self._properties['timePartitioning'] = {'type': "DAY"}
self._properties["timePartitioning"]["expirationMs"] = value
if not isinstance(value, (int, type(None))):
raise ValueError(
"must be an integer representing millisseconds or None")

if value is None:
if 'timePartitioning' in self._properties:
self._properties['timePartitioning'].pop('expirationMs')
else:
try:
self._properties['timePartitioning']['expirationMs'] = value
except KeyError:
self._properties['timePartitioning'] = {'type': 'DAY'}
self._properties['timePartitioning']['expirationMs'] = value

@property
def description(self):
Expand Down
2 changes: 1 addition & 1 deletion gcloud/bigquery/test_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -747,7 +747,7 @@ def test_table_wo_schema(self):
self.assertEqual(table.schema, [])

def test_table_w_schema(self):
from gcloud.bigquery.table import SchemaField
from gcloud.bigquery.schema import SchemaField
from gcloud.bigquery.table import Table
conn = _Connection({})
client = _Client(project=self.PROJECT, connection=conn)
Expand Down
8 changes: 4 additions & 4 deletions gcloud/bigquery/test_job.py
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ def test_ctor(self):
self.assertTrue(job.write_disposition is None)

def test_ctor_w_schema(self):
from gcloud.bigquery.table import SchemaField
from gcloud.bigquery.schema import SchemaField
client = _Client(self.PROJECT)
table = _Table()
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
Expand All @@ -349,7 +349,7 @@ def test_schema_setter_non_list(self):
job.schema = object()

def test_schema_setter_invalid_field(self):
from gcloud.bigquery.table import SchemaField
from gcloud.bigquery.schema import SchemaField
client = _Client(self.PROJECT)
table = _Table()
job = self._makeOne(self.JOB_NAME, table, [self.SOURCE1], client)
Expand All @@ -358,7 +358,7 @@ def test_schema_setter_invalid_field(self):
job.schema = [full_name, object()]

def test_schema_setter(self):
from gcloud.bigquery.table import SchemaField
from gcloud.bigquery.schema import SchemaField
client = _Client(self.PROJECT)
table = _Table()
job = self._makeOne(self.JOB_NAME, table, [self.SOURCE1], client)
Expand Down Expand Up @@ -523,7 +523,7 @@ def test_begin_w_bound_client(self):
self._verifyResourceProperties(job, RESOURCE)

def test_begin_w_alternate_client(self):
from gcloud.bigquery.table import SchemaField
from gcloud.bigquery.schema import SchemaField
PATH = 'projects/%s/jobs' % self.PROJECT
RESOURCE = self._makeResource(ended=True)
LOAD_CONFIGURATION = {
Expand Down
2 changes: 1 addition & 1 deletion gcloud/bigquery/test_query.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ def _makeResource(self, complete=False):
return resource

def _verifySchema(self, query, resource):
from gcloud.bigquery.table import SchemaField
from gcloud.bigquery.schema import SchemaField
if 'schema' in resource:
fields = resource['schema']['fields']
self.assertEqual(len(query.schema), len(fields))
Expand Down
110 changes: 110 additions & 0 deletions gcloud/bigquery/test_schema.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest


class TestSchemaField(unittest.TestCase):

def _getTargetClass(self):
from gcloud.bigquery.schema import SchemaField
return SchemaField

def _makeOne(self, *args, **kw):
return self._getTargetClass()(*args, **kw)

def test_ctor_defaults(self):
field = self._makeOne('test', 'STRING')
self.assertEqual(field.name, 'test')
self.assertEqual(field.field_type, 'STRING')
self.assertEqual(field.mode, 'NULLABLE')
self.assertEqual(field.description, None)
self.assertEqual(field.fields, None)

def test_ctor_explicit(self):
field = self._makeOne('test', 'STRING', mode='REQUIRED',
description='Testing')
self.assertEqual(field.name, 'test')
self.assertEqual(field.field_type, 'STRING')
self.assertEqual(field.mode, 'REQUIRED')
self.assertEqual(field.description, 'Testing')
self.assertEqual(field.fields, None)

def test_ctor_subfields(self):
field = self._makeOne('phone_number', 'RECORD',
fields=[self._makeOne('area_code', 'STRING'),
self._makeOne('local_number', 'STRING')])
self.assertEqual(field.name, 'phone_number')
self.assertEqual(field.field_type, 'RECORD')
self.assertEqual(field.mode, 'NULLABLE')
self.assertEqual(field.description, None)
self.assertEqual(len(field.fields), 2)
self.assertEqual(field.fields[0].name, 'area_code')
self.assertEqual(field.fields[0].field_type, 'STRING')
self.assertEqual(field.fields[0].mode, 'NULLABLE')
self.assertEqual(field.fields[0].description, None)
self.assertEqual(field.fields[0].fields, None)
self.assertEqual(field.fields[1].name, 'local_number')
self.assertEqual(field.fields[1].field_type, 'STRING')
self.assertEqual(field.fields[1].mode, 'NULLABLE')
self.assertEqual(field.fields[1].description, None)
self.assertEqual(field.fields[1].fields, None)

def test___eq___name_mismatch(self):
field = self._makeOne('test', 'STRING')
other = self._makeOne('other', 'STRING')
self.assertNotEqual(field, other)

def test___eq___field_type_mismatch(self):
field = self._makeOne('test', 'STRING')
other = self._makeOne('test', 'INTEGER')
self.assertNotEqual(field, other)

def test___eq___mode_mismatch(self):
field = self._makeOne('test', 'STRING', mode='REQUIRED')
other = self._makeOne('test', 'STRING', mode='NULLABLE')
self.assertNotEqual(field, other)

def test___eq___description_mismatch(self):
field = self._makeOne('test', 'STRING', description='Testing')
other = self._makeOne('test', 'STRING', description='Other')
self.assertNotEqual(field, other)

def test___eq___fields_mismatch(self):
sub1 = self._makeOne('sub1', 'STRING')
sub2 = self._makeOne('sub2', 'STRING')
field = self._makeOne('test', 'RECORD', fields=[sub1])
other = self._makeOne('test', 'RECORD', fields=[sub2])
self.assertNotEqual(field, other)

def test___eq___hit(self):
field = self._makeOne('test', 'STRING', mode='REQUIRED',
description='Testing')
other = self._makeOne('test', 'STRING', mode='REQUIRED',
description='Testing')
self.assertEqual(field, other)

def test___eq___hit_case_diff_on_type(self):
field = self._makeOne('test', 'STRING', mode='REQUIRED',
description='Testing')
other = self._makeOne('test', 'string', mode='REQUIRED',
description='Testing')
self.assertEqual(field, other)

def test___eq___hit_w_fields(self):
sub1 = self._makeOne('sub1', 'STRING')
sub2 = self._makeOne('sub2', 'STRING')
field = self._makeOne('test', 'RECORD', fields=[sub1, sub2])
other = self._makeOne('test', 'RECORD', fields=[sub1, sub2])
self.assertEqual(field, other)
Loading

0 comments on commit 4dbd7f9

Please sign in to comment.