From e16833fdd751ed5684e65d09c131503623f5c630 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 7 Mar 2017 18:46:21 -0500 Subject: [PATCH] Add system test for insert/fetch of rows w/ nested schema. Derived from schema / data from @tswast. Toward #3029. --- system_tests/bigquery.py | 67 +++++++++++++++++++++++---- system_tests/bigquery_test_data.json | 66 ++++++++++++++++++++++++++ system_tests/bigquery_test_data.jsonl | 3 ++ 3 files changed, 126 insertions(+), 10 deletions(-) create mode 100644 system_tests/bigquery_test_data.json create mode 100644 system_tests/bigquery_test_data.jsonl diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py index db2376753a7f..1033ca1607bc 100644 --- a/system_tests/bigquery.py +++ b/system_tests/bigquery.py @@ -12,13 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import base64 +import datetime import json import operator import os - +import time import unittest from google.cloud import bigquery +from google.cloud._helpers import UTC from google.cloud.exceptions import Forbidden from retry import RetryErrors @@ -40,6 +43,7 @@ def _make_dataset_name(prefix): def _load_json_schema(filename='bigquery_test_schema.json'): from google.cloud.bigquery.table import _parse_schema_resource + json_filename = os.path.join(WHERE, filename) with open(json_filename, 'r') as schema_file: @@ -289,9 +293,6 @@ def _fetch_single_page(table): return list(page) def test_insert_data_then_dump_table(self): - import datetime - from google.cloud._helpers import UTC - NOW_SECONDS = 1448911495.484366 NOW = datetime.datetime.utcfromtimestamp( NOW_SECONDS).replace(tzinfo=UTC) @@ -493,8 +494,6 @@ def _job_done(instance): # above). def test_sync_query_w_legacy_sql_types(self): - import datetime - from google.cloud._helpers import UTC naive = datetime.datetime(2016, 12, 5, 12, 41, 9) stamp = '%s %s' % (naive.date().isoformat(), naive.time().isoformat()) zoned = naive.replace(tzinfo=UTC) @@ -533,8 +532,6 @@ def test_sync_query_w_legacy_sql_types(self): self.assertEqual(query.rows[0][0], example['expected']) def test_sync_query_w_standard_sql_types(self): - import datetime - from google.cloud._helpers import UTC from google.cloud.bigquery._helpers import ArrayQueryParameter from google.cloud.bigquery._helpers import ScalarQueryParameter from google.cloud.bigquery._helpers import StructQueryParameter @@ -664,7 +661,7 @@ def test_dump_table_w_public_data(self): table = dataset.table(TABLE_NAME) self._fetch_single_page(table) - def test_create_table_nested_schema(self): + def test_create_table_insert_fetch_nested_schema(self): table_name = 'test_table' dataset = Config.CLIENT.dataset( _make_dataset_name('create_table_nested_schema')) @@ -673,8 +670,58 @@ def test_create_table_nested_schema(self): retry_403(dataset.create)() self.to_delete.append(dataset) - table = dataset.table(table_name, schema=_load_json_schema()) + schema = _load_json_schema() + table = dataset.table(table_name, schema=schema) table.create() self.to_delete.insert(0, table) self.assertTrue(table.exists()) self.assertEqual(table.name, table_name) + + to_insert = [] + # Data is in "JSON Lines" format, see http://jsonlines.org/ + json_filename = os.path.join(WHERE, 'bigquery_test_data.jsonl') + with open(json_filename) as rows_file: + for line in rows_file: + mapping = json.loads(line) + to_insert.append( + tuple(mapping[field.name] for field in schema)) + + errors = table.insert_data(to_insert) + self.assertEqual(len(errors), 0) + + retry = RetryResult(_has_rows, max_tries=8) + fetched = retry(self._fetch_single_page)(table) + self.assertEqual(len(fetched), len(to_insert)) + + for found, expected in zip(sorted(fetched), sorted(to_insert)): + self.assertEqual(found[0], expected[0]) # Name + self.assertEqual(found[1], int(expected[1])) # Age + self.assertEqual(found[2], expected[2]) # Weight + self.assertEqual(found[3], expected[3]) # IsMagic + + self.assertEqual(len(found[4]), len(expected[4])) # Spells + for f_spell, e_spell in zip(found[4], expected[4]): + self.assertEqual(f_spell['Name'], e_spell['Name']) + parts = time.strptime( + e_spell['LastUsed'], '%Y-%m-%d %H:%M:%S UTC') + e_used = datetime.datetime(*parts[0:6], tzinfo=UTC) + self.assertEqual(f_spell['LastUsed'], e_used) + self.assertEqual(f_spell['DiscoveredBy'], + e_spell['DiscoveredBy']) + self.assertEqual(f_spell['Properties'], e_spell['Properties']) + + e_icon = base64.standard_b64decode( + e_spell['Icon'].encode('ascii')) + self.assertEqual(f_spell['Icon'], e_icon) + + parts = time.strptime(expected[5], '%H:%M:%S') + e_teatime = datetime.time(*parts[3:6]) + self.assertEqual(found[5], e_teatime) # TeaTime + + parts = time.strptime(expected[6], '%Y-%m-%d') + e_nextvac = datetime.date(*parts[0:3]) + self.assertEqual(found[6], e_nextvac) # NextVacation + + parts = time.strptime(expected[7], '%Y-%m-%dT%H:%M:%S') + e_favtime = datetime.datetime(*parts[0:6]) + self.assertEqual(found[7], e_favtime) # FavoriteTime diff --git a/system_tests/bigquery_test_data.json b/system_tests/bigquery_test_data.json new file mode 100644 index 000000000000..ac854fb812d4 --- /dev/null +++ b/system_tests/bigquery_test_data.json @@ -0,0 +1,66 @@ +[ + { + "Age" : "111", + "Spells" : [], + "Name" : "Bilbo", + "Weight" : 67.2, + "TeaTime" : "10:00:00", + "NextVacation" : "2017-09-22", + "FavoriteTime" : "2031-04-01T05:09:27", + "IsMagic" : false + }, + { + "Age" : "1000", + "Name" : "Gandalf", + "Spells" : [ + { + "Name" : "Skydragon", + "Properties" : [ + { + "Power" : 1, + "Name" : "Flying" + }, + { + "Name" : "Creature", + "Power" : 1 + }, + { + "Power" : 11, + "Name" : "Explodey" + } + ], + "LastUsed" : "2015-10-31 23:59:56 UTC", + "Icon" : "iVBORw0KGgoAAAANSUhEUgAAAB4AAAAgCAYAAAAFQMh/AAAAAXNSR0IArs4c6QAAA9lJREFUSA21lk9OVEEQxvsRDImoiMG9mLjjCG5mEg7gEfQGsIcF7p0EDsBBSJiNO7ZsFRZqosb/QkSj7fer7ur33sw8GDFUUq+7q6vqq6qu7pkQzqG4EeI521e7FePVgM9cGPYwhCi6UO8qFOK+YY+Br66ujsmmxb84Yzwp6zCsxjJfWVkxnMsEMGuWHZ9Wcz11cM48hkq0vLwc1tbW4mAwqDpcdIqnMmgF0JMv2CiGnZ2dcHR0FA4PD8Pe3t5U/tx6bCSlb+JT8XfxT3HsUek0Li0tRdjWl+z6iRF+FNA1hXPDQ/IMNyRg3s8bD/OaZS+VP+9cOLSa64cA34oXZWagDkRzAaJxXaE+ufc4rCN7LrazZ2+8+STtpAL8WYDvpTaHKlkB2iQARMvb2+H27m4YaL7zaDtUw1BZAASi6T8T2UZnPZV2pvnJfCH5p8bewcGB6TrIfz8wBZgHQ83kjpuj6RBYQpuo09Tvmpd7TPe+ktZN8cKwS92KWXGuaqWowlYEwthtMcWOZUNJc8at+zuF/Xkqo69baS7P+AvWjYwJ4jyHXXsEnd74ZO/Pq+uXUuv6WNlso6cvnDsZB1V/unJab3D1/KrJDw9NCM9wHf2FK2ejTKMejnBHfGtfH7LGGCdQDqaqJgfgzWjXK1nYV4jRbPGnxUT7cqUaZfJrVZeOm9QmB21L6xXgbu/ScsYusJFMoU0x2fsamRJOd6kOYDRLUxv94ENZe8+0gM+0dyz+KgU7X8rLHHCIOZyrna4y6ykIu0YCs02TBXmk3PZssmEgaTxTo83xjCIjoE21h0Yah3MrV4+9kR8MaabGze+9NEILGAFE5nMOiiA32KnAr/sb7tED3nzlzC4dB38WMC+EjaqHfqvUKHi2gJPdWQ6AbH8hgyQ7QY6jvjj3QZWvX6pUAtduTX5Dss96Q7NI9RQRJeeKvRFbt0v2gb1Gx/PooJsztn1c1DqpAU3Hde2dB2aEHBhjgOFjMeDvxLafjQ3YZQSgOcHJZX611H45sGLHWvYTz9hiURlpNoBZvxb/Ft9lAQ1DmBfUiR+j1hAPkMBTE9L9+zLva1QvGFHurRBaZ5xLVitoBviiRkD/sIMDztKA5FA0b9/0OclzO2/XAQymJ0TcghZwEo9/AX8gMeAJMOvIsWWt5bwCoiFhVSllrdH0t5Q1JHAFlKJNkvTVdn2GHb9KdmacMT+d/Os05imJUccRX2YuZ93Sxf0Ilc4DPDeAq5SAvFEAY94cQc6BA26dzb4HWAJI4DPmQE5KCVUyvb2FcDZem7JdT2ggKUP3xX6n9XNq1DpzSf4Cy4ZqSlmM8d8AAAAASUVORK5CYII=", + "DiscoveredBy" : "Firebreather" + } + ], + "NextVacation" : "2666-06-06", + "TeaTime" : "15:00:00", + "Weight" : 198.6, + "FavoriteTime" : "2001-12-19T23:59:59", + "IsMagic" : true + }, + { + "Weight" : 128.3, + "TeaTime" : "12:00:00", + "NextVacation" : "2017-03-14", + "IsMagic" : true, + "FavoriteTime" : "2000-10-31T23:27:46", + "Age" : "17", + "Spells" : [ + { + "LastUsed" : "2017-02-14 12:07:23 UTC", + "Properties" : [ + { + "Name" : "Makes you look crazy", + "Power" : 1 + } + ], + "Icon" : "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAAAXNSR0IArs4c6QAAABxpRE9UAAAAAgAAAAAAAAAgAAAAKAAAACAAAAAgAAABxj2CfowAAAGSSURBVHgB7Jc9TsNAEIX3JDkCPUV6KlpKFHEGlD4nyA04ACUXQKTgCEipUnKGNEbP0otentayicZ24SlWs7tjO/N9u/5J2b2+NUtuZcnwYE8BuQPyGZAPwXwLLPk5kG+BJa9+fgfkh1B+CeancL4F8i2Q/wWm/S/w+XFoTseftn0dvhu0OXfhpM+AGvzcEiYVAFisPqE9zrETJhHAlXfg2lglMK9z0f3RBfB+ZyRUV3x+erzsEIjjOBqc1xtNAIrvguybV3A9lkVHxlEE6GrrPb/ZvAySwlUnfCmlPQ+R8JCExvGtcRQBLFwj4FGkznX1VYDKPG/f2/MjwCksXACgdNUxJjwK9xwl4JihOwTFR0kIF+CABEPRnvsvPFctMoYKqAFSAFaMwB4pp3Y+bodIYL9WmIAaIOHxo7W8wiHvAjTvhUeNwwSgeAeAABbqOewC5hBdwFD4+9+7puzXV9fS6/b1wwT4tsaYAhwOOQdUQch5vgZCeAhAv3ZM31yYAAUgvApQQQ6n5w6FB/RVe1jdJOAPAAD//1eMQwoAAAGQSURBVO1UMU4DQQy8X9AgWopIUINEkS4VlJQo4gvwAV7AD3gEH4iSgidESpWSXyyZExP5lr0c7K5PsXBhec/2+jzjuWtent9CLdtu1mG5+gjz+WNr7IsY7eH+tvO+xfuqk4vz7CH91edFaF5v9nb6dBKm13edvrL+0Lk5lMzJkQDeJSkkgHF6mR8CHwMHCQR/NAQQGD0BAlwK4FCefQiefq+A2Vn29tG7igLAfmwcnJu/nJy3BMQkMN9HEPr8AL3bfBv7Bp+7/SoExMDjZwKEJwmyhnnmQIQEBIlz2x0iKoAvJkAC6TsTIH6MqRrEWUMSZF2zAwqT4Eu/e6pzFAIkmNSZ4OFT+VYBIIF//UqbJwnF/4DU0GwOn8r/JQYCpPGufEfJuZiA37ycQw/5uFeqPq4pfR6FADmkBCXjfWdZj3NfXW58dAJyB9W65wRoMWulryvAyqa05nQFaDFrpa8rwMqmtOZ0BWgxa6WvK8DKprTmdAVoMWulryvAyqa05nQFaDFrpa8rwMqmtOb89wr4AtQ4aPoL6yVpAAAAAElFTkSuQmCC", + "Name" : "Talking cats", + "DiscoveredBy" : "Salem" + } + ], + "Name" : "Sabrina" + } +] diff --git a/system_tests/bigquery_test_data.jsonl b/system_tests/bigquery_test_data.jsonl new file mode 100644 index 000000000000..1da3f2309cae --- /dev/null +++ b/system_tests/bigquery_test_data.jsonl @@ -0,0 +1,3 @@ +{"Name":"Bilbo","Age":"111","Weight":67.2,"IsMagic":false,"Spells":[],"TeaTime":"10:00:00","NextVacation":"2017-09-22","FavoriteTime":"2031-04-01T05:09:27"} +{"Name":"Gandalf","Age":"1000","Weight":198.6,"IsMagic":true,"Spells":[{"Name": "Skydragon", "Icon":"iVBORw0KGgoAAAANSUhEUgAAAB4AAAAgCAYAAAAFQMh/AAAAAXNSR0IArs4c6QAAA9lJREFUSA21lk9OVEEQxvsRDImoiMG9mLjjCG5mEg7gEfQGsIcF7p0EDsBBSJiNO7ZsFRZqosb/QkSj7fer7ur33sw8GDFUUq+7q6vqq6qu7pkQzqG4EeI521e7FePVgM9cGPYwhCi6UO8qFOK+YY+Br66ujsmmxb84Yzwp6zCsxjJfWVkxnMsEMGuWHZ9Wcz11cM48hkq0vLwc1tbW4mAwqDpcdIqnMmgF0JMv2CiGnZ2dcHR0FA4PD8Pe3t5U/tx6bCSlb+JT8XfxT3HsUek0Li0tRdjWl+z6iRF+FNA1hXPDQ/IMNyRg3s8bD/OaZS+VP+9cOLSa64cA34oXZWagDkRzAaJxXaE+ufc4rCN7LrazZ2+8+STtpAL8WYDvpTaHKlkB2iQARMvb2+H27m4YaL7zaDtUw1BZAASi6T8T2UZnPZV2pvnJfCH5p8bewcGB6TrIfz8wBZgHQ83kjpuj6RBYQpuo09Tvmpd7TPe+ktZN8cKwS92KWXGuaqWowlYEwthtMcWOZUNJc8at+zuF/Xkqo69baS7P+AvWjYwJ4jyHXXsEnd74ZO/Pq+uXUuv6WNlso6cvnDsZB1V/unJab3D1/KrJDw9NCM9wHf2FK2ejTKMejnBHfGtfH7LGGCdQDqaqJgfgzWjXK1nYV4jRbPGnxUT7cqUaZfJrVZeOm9QmB21L6xXgbu/ScsYusJFMoU0x2fsamRJOd6kOYDRLUxv94ENZe8+0gM+0dyz+KgU7X8rLHHCIOZyrna4y6ykIu0YCs02TBXmk3PZssmEgaTxTo83xjCIjoE21h0Yah3MrV4+9kR8MaabGze+9NEILGAFE5nMOiiA32KnAr/sb7tED3nzlzC4dB38WMC+EjaqHfqvUKHi2gJPdWQ6AbH8hgyQ7QY6jvjj3QZWvX6pUAtduTX5Dss96Q7NI9RQRJeeKvRFbt0v2gb1Gx/PooJsztn1c1DqpAU3Hde2dB2aEHBhjgOFjMeDvxLafjQ3YZQSgOcHJZX611H45sGLHWvYTz9hiURlpNoBZvxb/Ft9lAQ1DmBfUiR+j1hAPkMBTE9L9+zLva1QvGFHurRBaZ5xLVitoBviiRkD/sIMDztKA5FA0b9/0OclzO2/XAQymJ0TcghZwEo9/AX8gMeAJMOvIsWWt5bwCoiFhVSllrdH0t5Q1JHAFlKJNkvTVdn2GHb9KdmacMT+d/Os05imJUccRX2YuZ93Sxf0Ilc4DPDeAq5SAvFEAY94cQc6BA26dzb4HWAJI4DPmQE5KCVUyvb2FcDZem7JdT2ggKUP3xX6n9XNq1DpzSf4Cy4ZqSlmM8d8AAAAASUVORK5CYII=","DiscoveredBy":"Firebreather","Properties":[{"Name":"Flying","Power":1},{"Name":"Creature","Power":1},{"Name":"Explodey","Power":11}],"LastUsed":"2015-10-31 23:59:56 UTC"}],"TeaTime":"15:00:00","NextVacation":"2666-06-06","FavoriteTime":"2001-12-19T23:59:59"} +{"Name":"Sabrina","Age":"17","Weight":128.3,"IsMagic":true,"Spells":[{"Name": "Talking cats", "Icon":"iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAAAXNSR0IArs4c6QAAABxpRE9UAAAAAgAAAAAAAAAgAAAAKAAAACAAAAAgAAABxj2CfowAAAGSSURBVHgB7Jc9TsNAEIX3JDkCPUV6KlpKFHEGlD4nyA04ACUXQKTgCEipUnKGNEbP0otentayicZ24SlWs7tjO/N9u/5J2b2+NUtuZcnwYE8BuQPyGZAPwXwLLPk5kG+BJa9+fgfkh1B+CeancL4F8i2Q/wWm/S/w+XFoTseftn0dvhu0OXfhpM+AGvzcEiYVAFisPqE9zrETJhHAlXfg2lglMK9z0f3RBfB+ZyRUV3x+erzsEIjjOBqc1xtNAIrvguybV3A9lkVHxlEE6GrrPb/ZvAySwlUnfCmlPQ+R8JCExvGtcRQBLFwj4FGkznX1VYDKPG/f2/MjwCksXACgdNUxJjwK9xwl4JihOwTFR0kIF+CABEPRnvsvPFctMoYKqAFSAFaMwB4pp3Y+bodIYL9WmIAaIOHxo7W8wiHvAjTvhUeNwwSgeAeAABbqOewC5hBdwFD4+9+7puzXV9fS6/b1wwT4tsaYAhwOOQdUQch5vgZCeAhAv3ZM31yYAAUgvApQQQ6n5w6FB/RVe1jdJOAPAAD//1eMQwoAAAGQSURBVO1UMU4DQQy8X9AgWopIUINEkS4VlJQo4gvwAV7AD3gEH4iSgidESpWSXyyZExP5lr0c7K5PsXBhec/2+jzjuWtent9CLdtu1mG5+gjz+WNr7IsY7eH+tvO+xfuqk4vz7CH91edFaF5v9nb6dBKm13edvrL+0Lk5lMzJkQDeJSkkgHF6mR8CHwMHCQR/NAQQGD0BAlwK4FCefQiefq+A2Vn29tG7igLAfmwcnJu/nJy3BMQkMN9HEPr8AL3bfBv7Bp+7/SoExMDjZwKEJwmyhnnmQIQEBIlz2x0iKoAvJkAC6TsTIH6MqRrEWUMSZF2zAwqT4Eu/e6pzFAIkmNSZ4OFT+VYBIIF//UqbJwnF/4DU0GwOn8r/JQYCpPGufEfJuZiA37ycQw/5uFeqPq4pfR6FADmkBCXjfWdZj3NfXW58dAJyB9W65wRoMWulryvAyqa05nQFaDFrpa8rwMqmtOZ0BWgxa6WvK8DKprTmdAVoMWulryvAyqa05nQFaDFrpa8rwMqmtOb89wr4AtQ4aPoL6yVpAAAAAElFTkSuQmCC","DiscoveredBy":"Salem","Properties":[{"Name":"Makes you look crazy","Power":1}],"LastUsed":"2017-02-14 12:07:23 UTC"}],"TeaTime":"12:00:00","NextVacation":"2017-03-14","FavoriteTime":"2000-10-31T23:27:46"}