From e004a766dbc37f90666fb30b36c1a1d4c384e665 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 8 Sep 2015 15:44:42 -0400 Subject: [PATCH 1/4] Fix conversion of backend-generated timestamp values. BQ returns them as strings, representing UNIX timestamp seconds-since-epoch to microsecond precision. Fixes #1125. --- gcloud/bigquery/table.py | 6 ++++-- gcloud/bigquery/test_table.py | 13 +++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py index 838e2c8b128e..e87cdf13bc79 100644 --- a/gcloud/bigquery/table.py +++ b/gcloud/bigquery/table.py @@ -18,6 +18,7 @@ import six +from gcloud._helpers import UTC from gcloud._helpers import _datetime_from_microseconds from gcloud._helpers import _millis_from_datetime from gcloud.exceptions import NotFound @@ -761,8 +762,9 @@ def _bool_from_json(value, field): def _datetime_from_json(value, field): if _not_null(value, field): - # Field value will be in milliseconds. - return _datetime_from_microseconds(1000.0 * float(value)) + # value will be a float in seconds, to microsecond precision, in UTC. + stamp = datetime.datetime.utcfromtimestamp(float(value)) + return stamp.replace(tzinfo=UTC) def _record_from_json(value, field): diff --git a/gcloud/bigquery/test_table.py b/gcloud/bigquery/test_table.py index 04f9524c232c..345c7d4cbdac 100644 --- a/gcloud/bigquery/test_table.py +++ b/gcloud/bigquery/test_table.py @@ -824,7 +824,6 @@ def test_fetch_data_w_bound_client(self): import datetime from gcloud._helpers import UTC from gcloud.bigquery.table import SchemaField - from gcloud._helpers import _millis_from_datetime PATH = 'projects/%s/datasets/%s/tables/%s/data' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) @@ -835,6 +834,11 @@ def test_fetch_data_w_bound_client(self): WHEN_2 = WHEN + datetime.timedelta(seconds=2) ROWS = 1234 TOKEN = 'TOKEN' + + def _bigquery_timestamp_float_repr(ts_float): + # Preserve microsecond precision for E+09 timestamps + return '%0.15E' % (ts_float,) + DATA = { 'totalRows': ROWS, 'pageToken': TOKEN, @@ -842,17 +846,17 @@ def test_fetch_data_w_bound_client(self): {'f': [ {'v': 'Phred Phlyntstone'}, {'v': '32'}, - {'v': _millis_from_datetime(WHEN)}, + {'v': _bigquery_timestamp_float_repr(WHEN_TS)}, ]}, {'f': [ {'v': 'Bharney Rhubble'}, {'v': '33'}, - {'v': _millis_from_datetime(WHEN_1)}, + {'v': _bigquery_timestamp_float_repr(WHEN_TS + 1)}, ]}, {'f': [ {'v': 'Wylma Phlyntstone'}, {'v': '29'}, - {'v': _millis_from_datetime(WHEN_2)}, + {'v': _bigquery_timestamp_float_repr(WHEN_TS + 2)}, ]}, {'f': [ {'v': 'Bhettye Rhubble'}, @@ -861,6 +865,7 @@ def test_fetch_data_w_bound_client(self): ]}, ] } + conn = _Connection(DATA) client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) From bda4cfb0c48777b4be3708ac0c56635bc8cc41c8 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 8 Sep 2015 15:46:44 -0400 Subject: [PATCH 2/4] Fix reference docs URLs. Addresses: FWIW in https://github.com/GoogleCloudPlatform/gcloud-python/issues/1125#issuecomment-138437728 --- gcloud/bigquery/table.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py index e87cdf13bc79..5383ffdd8c66 100644 --- a/gcloud/bigquery/table.py +++ b/gcloud/bigquery/table.py @@ -412,7 +412,7 @@ def create(self, client=None): """API call: create the dataset via a PUT request See: - https://cloud.google.com/bigquery/reference/rest/v2/tables/insert + https://cloud.google.com/bigquery/docs/reference/v2/tables/insert :type client: :class:`gcloud.bigquery.client.Client` or ``NoneType`` :param client: the client to use. If not passed, falls back to the @@ -553,7 +553,7 @@ def delete(self, client=None): """API call: delete the table via a DELETE request See: - https://cloud.google.com/bigquery/reference/rest/v2/tables/delete + https://cloud.google.com/bigquery/docs/reference/v2/tables/delete :type client: :class:`gcloud.bigquery.client.Client` or ``NoneType`` :param client: the client to use. If not passed, falls back to the @@ -566,7 +566,7 @@ def fetch_data(self, max_results=None, page_token=None, client=None): """API call: fetch the table data via a GET request See: - https://cloud.google.com/bigquery/reference/rest/v2/tabledata/list + https://cloud.google.com/bigquery/docs/reference/v2/tabledata/list .. note:: @@ -632,7 +632,7 @@ def insert_data(self, """API call: insert table data via a POST request See: - https://cloud.google.com/bigquery/reference/rest/v2/tabledata/insertAll + https://cloud.google.com/bigquery/docs/reference/v2/tabledata/insertAll :type rows: list of tuples :param rows: row data to be inserted From d77d956db70dd07cef57a958d03e61614242cf96 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 10 Sep 2015 17:08:01 -0400 Subject: [PATCH 3/4] Use utility function from 'gcloud._helpers' to convert timestamps. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1128#discussion_r39210175. --- gcloud/bigquery/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py index 5383ffdd8c66..d0e04048f762 100644 --- a/gcloud/bigquery/table.py +++ b/gcloud/bigquery/table.py @@ -763,7 +763,7 @@ def _bool_from_json(value, field): def _datetime_from_json(value, field): if _not_null(value, field): # value will be a float in seconds, to microsecond precision, in UTC. - stamp = datetime.datetime.utcfromtimestamp(float(value)) + stamp = _datetime_from_microseconds(1e6 * float(value)) return stamp.replace(tzinfo=UTC) From 8225dd882baf6ea071aed10e9f953a06ec715fd8 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 11 Sep 2015 11:19:49 -0400 Subject: [PATCH 4/4] Don't replace timestamp w/ UTC. '_datetime_from_microseconds' already does that. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1128#issuecomment-139380565 --- gcloud/bigquery/table.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py index d0e04048f762..5223d3ce6a8c 100644 --- a/gcloud/bigquery/table.py +++ b/gcloud/bigquery/table.py @@ -18,7 +18,6 @@ import six -from gcloud._helpers import UTC from gcloud._helpers import _datetime_from_microseconds from gcloud._helpers import _millis_from_datetime from gcloud.exceptions import NotFound @@ -763,8 +762,7 @@ def _bool_from_json(value, field): def _datetime_from_json(value, field): if _not_null(value, field): # value will be a float in seconds, to microsecond precision, in UTC. - stamp = _datetime_from_microseconds(1e6 * float(value)) - return stamp.replace(tzinfo=UTC) + return _datetime_from_microseconds(1e6 * float(value)) def _record_from_json(value, field):