Skip to content

Commit 3357530

Browse files
tseavertseaver
committed
Merge pull request #1045 from tseaver/bigquery-flesh_out_api
Flesh out bigquery API
2 parents 99e04fd + 34a747c commit 3357530

File tree

7 files changed

+458
-35
lines changed

7 files changed

+458
-35
lines changed

‎gcloud/bigquery/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,5 @@
2424
from gcloud.bigquery.client import Client
2525
from gcloud.bigquery.connection import SCOPE
2626
from gcloud.bigquery.dataset import Dataset
27+
from gcloud.bigquery.table import SchemaField
28+
from gcloud.bigquery.table import Table

‎gcloud/bigquery/client.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,49 @@ class Client(JSONClient):
4343

4444
_connection_class = Connection
4545

46+
def list_datasets(self, include_all=False, max_results=None,
47+
page_token=None):
48+
"""List datasets for the project associated with this client.
49+
50+
See:
51+
https://cloud.google.com/bigquery/docs/reference/v2/datasets/list
52+
53+
:type include_all: boolean
54+
:param include_all: True if results include hidden datasets.
55+
56+
:type max_results: int
57+
:param max_results: maximum number of datasets to return, If not
58+
passed, defaults to a value set by the API.
59+
60+
:type page_token: string
61+
:param page_token: opaque marker for the next "page" of datasets. If
62+
not passed, the API will return the first page of
63+
datasets.
64+
65+
:rtype: tuple, (list, str)
66+
:returns: list of :class:`gcloud.bigquery.dataset.Dataset`, plus a
67+
"next page token" string: if the token is not None,
68+
indicates that more datasets can be retrieved with another
69+
call (pass that value as ``page_token``).
70+
"""
71+
params = {}
72+
73+
if include_all:
74+
params['all'] = True
75+
76+
if max_results is not None:
77+
params['maxResults'] = max_results
78+
79+
if page_token is not None:
80+
params['pageToken'] = page_token
81+
82+
path = '/projects/%s/datasets' % (self.project,)
83+
resp = self.connection.api_request(method='GET', path=path,
84+
query_params=params)
85+
datasets = [Dataset.from_api_repr(resource, self)
86+
for resource in resp['datasets']]
87+
return datasets, resp.get('nextPageToken')
88+
4689
def dataset(self, name):
4790
"""Construct a dataset bound to this client.
4891

‎gcloud/bigquery/dataset.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,29 @@ def location(self, value):
190190
raise ValueError("Pass a string, or None")
191191
self._properties['location'] = value
192192

193+
@classmethod
194+
def from_api_repr(cls, resource, client):
195+
"""Factory: construct a dataset given its API representation
196+
197+
:type resource: dict
198+
:param resource: dataset resource representation returned from the API
199+
200+
:type client: :class:`gcloud.bigquery.client.Client`
201+
:param client: Client which holds credentials and project
202+
configuration for the dataset.
203+
204+
:rtype: :class:`gcloud.bigquery.dataset.Dataset`
205+
:returns: Dataset parsed from ``resource``.
206+
"""
207+
if ('datasetReference' not in resource or
208+
'datasetId' not in resource['datasetReference']):
209+
raise KeyError('Resource lacks required identity information:'
210+
'["datasetReference"]["datasetId"]')
211+
name = resource['datasetReference']['datasetId']
212+
dataset = cls(name, client=client)
213+
dataset._set_properties(resource)
214+
return dataset
215+
193216
def _require_client(self, client):
194217
"""Check client or verify over-ride.
195218
@@ -357,6 +380,43 @@ def delete(self, client=None):
357380
client = self._require_client(client)
358381
client.connection.api_request(method='DELETE', path=self.path)
359382

383+
def list_tables(self, max_results=None, page_token=None):
384+
"""List tables for the project associated with this client.
385+
386+
See:
387+
https://cloud.google.com/bigquery/docs/reference/v2/tables/list
388+
389+
:type max_results: int
390+
:param max_results: maximum number of tables to return, If not
391+
passed, defaults to a value set by the API.
392+
393+
:type page_token: string
394+
:param page_token: opaque marker for the next "page" of datasets. If
395+
not passed, the API will return the first page of
396+
datasets.
397+
398+
:rtype: tuple, (list, str)
399+
:returns: list of :class:`gcloud.bigquery.table.Table`, plus a
400+
"next page token" string: if not None, indicates that
401+
more tables can be retrieved with another call (pass that
402+
value as ``page_token``).
403+
"""
404+
params = {}
405+
406+
if max_results is not None:
407+
params['maxResults'] = max_results
408+
409+
if page_token is not None:
410+
params['pageToken'] = page_token
411+
412+
path = '/projects/%s/datasets/%s/tables' % (self.project, self.name)
413+
connection = self._client.connection
414+
resp = connection.api_request(method='GET', path=path,
415+
query_params=params)
416+
tables = [Table.from_api_repr(resource, self)
417+
for resource in resp['tables']]
418+
return tables, resp.get('nextPageToken')
419+
360420
def table(self, name, schema=()):
361421
"""Construct a table bound to this dataset.
362422

‎gcloud/bigquery/table.py‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,28 @@ def view_query(self):
298298
"""Delete SQL query defining the table as a view."""
299299
self._properties.pop('view', None)
300300

301+
@classmethod
302+
def from_api_repr(cls, resource, dataset):
303+
"""Factory: construct a table given its API representation
304+
305+
:type resource: dict
306+
:param resource: table resource representation returned from the API
307+
308+
:type dataset: :class:`gcloud.bigquery.dataset.Dataset`
309+
:param dataset: The dataset containing the table.
310+
311+
:rtype: :class:`gcloud.bigquery.table.Table`
312+
:returns: Table parsed from ``resource``.
313+
"""
314+
if ('tableReference' not in resource or
315+
'tableId' not in resource['tableReference']):
316+
raise KeyError('Resource lacks required identity information:'
317+
'["tableReference"]["tableId"]')
318+
table_name = resource['tableReference']['tableId']
319+
table = cls(table_name, dataset=dataset)
320+
table._set_properties(resource)
321+
return table
322+
301323
def _require_client(self, client):
302324
"""Check client or verify over-ride.
303325
@@ -344,7 +366,7 @@ def _set_properties(self, api_response):
344366
"""
345367
self._properties.clear()
346368
cleaned = api_response.copy()
347-
schema = cleaned.pop('schema', {})
369+
schema = cleaned.pop('schema', {'fields': ()})
348370
self.schema = self._parse_schema_resource(schema)
349371
if 'creationTime' in cleaned:
350372
cleaned['creationTime'] = float(cleaned['creationTime'])

‎gcloud/bigquery/test_client.py‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,88 @@ def test_dataset(self):
4646
self.assertEqual(dataset.name, DATASET)
4747
self.assertTrue(dataset._client is client)
4848

49+
def test_list_datasets_defaults(self):
50+
from gcloud.bigquery.dataset import Dataset
51+
PROJECT = 'PROJECT'
52+
DATASET_1 = 'dataset_one'
53+
DATASET_2 = 'dataset_two'
54+
PATH = 'projects/%s/datasets' % PROJECT
55+
TOKEN = 'TOKEN'
56+
DATA = {
57+
'nextPageToken': TOKEN,
58+
'datasets': [
59+
{'kind': 'bigquery#dataset',
60+
'id': '%s:%s' % (PROJECT, DATASET_1),
61+
'datasetReference': {'datasetId': DATASET_1,
62+
'projectId': PROJECT},
63+
'friendlyName': None},
64+
{'kind': 'bigquery#dataset',
65+
'id': '%s:%s' % (PROJECT, DATASET_2),
66+
'datasetReference': {'datasetId': DATASET_2,
67+
'projectId': PROJECT},
68+
'friendlyName': 'Two'},
69+
]
70+
}
71+
creds = _Credentials()
72+
client = self._makeOne(PROJECT, creds)
73+
conn = client.connection = _Connection(DATA)
74+
75+
datasets, token = client.list_datasets()
76+
77+
self.assertEqual(len(datasets), len(DATA['datasets']))
78+
for found, expected in zip(datasets, DATA['datasets']):
79+
self.assertTrue(isinstance(found, Dataset))
80+
self.assertEqual(found.dataset_id, expected['id'])
81+
self.assertEqual(found.friendly_name, expected['friendlyName'])
82+
self.assertEqual(token, TOKEN)
83+
84+
self.assertEqual(len(conn._requested), 1)
85+
req = conn._requested[0]
86+
self.assertEqual(req['method'], 'GET')
87+
self.assertEqual(req['path'], '/%s' % PATH)
88+
89+
def test_list_datasets_explicit(self):
90+
from gcloud.bigquery.dataset import Dataset
91+
PROJECT = 'PROJECT'
92+
DATASET_1 = 'dataset_one'
93+
DATASET_2 = 'dataset_two'
94+
PATH = 'projects/%s/datasets' % PROJECT
95+
TOKEN = 'TOKEN'
96+
DATA = {
97+
'datasets': [
98+
{'kind': 'bigquery#dataset',
99+
'id': '%s:%s' % (PROJECT, DATASET_1),
100+
'datasetReference': {'datasetId': DATASET_1,
101+
'projectId': PROJECT},
102+
'friendlyName': None},
103+
{'kind': 'bigquery#dataset',
104+
'id': '%s:%s' % (PROJECT, DATASET_2),
105+
'datasetReference': {'datasetId': DATASET_2,
106+
'projectId': PROJECT},
107+
'friendlyName': 'Two'},
108+
]
109+
}
110+
creds = _Credentials()
111+
client = self._makeOne(PROJECT, creds)
112+
conn = client.connection = _Connection(DATA)
113+
114+
datasets, token = client.list_datasets(
115+
include_all=True, max_results=3, page_token=TOKEN)
116+
117+
self.assertEqual(len(datasets), len(DATA['datasets']))
118+
for found, expected in zip(datasets, DATA['datasets']):
119+
self.assertTrue(isinstance(found, Dataset))
120+
self.assertEqual(found.dataset_id, expected['id'])
121+
self.assertEqual(found.friendly_name, expected['friendlyName'])
122+
self.assertEqual(token, None)
123+
124+
self.assertEqual(len(conn._requested), 1)
125+
req = conn._requested[0]
126+
self.assertEqual(req['method'], 'GET')
127+
self.assertEqual(req['path'], '/%s' % PATH)
128+
self.assertEqual(req['query_params'],
129+
{'all': True, 'maxResults': 3, 'pageToken': TOKEN})
130+
49131

50132
class _Credentials(object):
51133

@@ -58,3 +140,15 @@ def create_scoped_required():
58140
def create_scoped(self, scope):
59141
self._scopes = scope
60142
return self
143+
144+
145+
class _Connection(object):
146+
147+
def __init__(self, *responses):
148+
self._responses = responses
149+
self._requested = []
150+
151+
def api_request(self, **kw):
152+
self._requested.append(kw)
153+
response, self._responses = self._responses[0], self._responses[1:]
154+
return response

0 commit comments

Comments
 (0)