Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

doc(bigquery): add code sample for scripting #9537

Merged
merged 3 commits into from
Oct 28, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions bigquery/docs/usage/queries.rst
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,16 @@ See BigQuery documentation for more information on
:dedent: 4
:start-after: [START bigquery_query_params_named]
:end-before: [END bigquery_query_params_named]

Run a script
^^^^^^^^^^^^

See BigQuery documentation for more information on `scripting in BigQuery
standard SQL
<https://cloud.google.com/bigquery/docs/reference/standard-sql/scripting>`_.

.. literalinclude:: ../samples/query_script.py
:language: python
:dedent: 4
:start-after: [START bigquery_query_script]
:end-before: [END bigquery_query_script]
69 changes: 69 additions & 0 deletions bigquery/samples/query_script.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


def query_script(client):
# [START bigquery_query_script]
# TODO(developer): Import the client library.
# from google.cloud import bigquery

# TODO(developer): Construct a BigQuery client object.
# client = bigquery.Client()

# Run a SQL script.
sql_script = """
-- Declare a variable to hold names as an array.
DECLARE top_names ARRAY<STRING>;

-- Build an array of the top 100 names from the year 2017.
SET top_names = (
SELECT ARRAY_AGG(name ORDER BY number DESC LIMIT 100)
FROM `bigquery-public-data.usa_names.usa_1910_2013`
WHERE year = 2000
);

-- Which names appear as words in Shakespeare's plays?
SELECT
name AS shakespeare_name
FROM UNNEST(top_names) AS name
WHERE name IN (
SELECT word
FROM `bigquery-public-data.samples.shakespeare`
);
"""
parent_job = client.query(sql_script)

# Wait for the whole script to finish.
rows_iterable = parent_job.result()
print("Script created {} child jobs.".format(parent_job.num_child_jobs))

# Fetch result rows for the final sub-job in the script.
rows = list(rows_iterable)
print(
"{} of the top 100 names from year 2000 also appear in Shakespeare's works.".format(
len(rows)
)
)

# Fetch jobs created by the SQL script.
child_jobs_iterable = client.list_jobs(parent_job=parent_job)
for child_job in child_jobs_iterable:
child_rows = list(child_job.result())
print(
"Child job with ID {} produced {} row(s).".format(
child_job.job_id, len(child_rows)
)
)

# [END bigquery_query_script]
plamut marked this conversation as resolved.
Show resolved Hide resolved
28 changes: 28 additions & 0 deletions bigquery/samples/tests/test_query_script.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from .. import query_script


def test_query_script(capsys, client):

query_script.query_script(client)
out, _ = capsys.readouterr()
assert "Script created 2 child jobs." in out
assert (
"53 of the top 100 names from year 2000 also appear in Shakespeare's works."
in out
)
assert "produced 53 row(s)" in out
assert "produced 1 row(s)" in out
31 changes: 0 additions & 31 deletions bigquery/tests/system.py
Original file line number Diff line number Diff line change
Expand Up @@ -431,37 +431,6 @@ def test_list_tables(self):
)
self.assertGreater(len(list(iterator)), 0)

def test_listing_scripting_jobs(self):
# Run a SQL script.
sql_script = """
-- Declare a variable to hold names as an array.
DECLARE top_names ARRAY<STRING>;
-- Build an array of the top 100 names from the year 2017.
SET top_names = (
SELECT ARRAY_AGG(name ORDER BY number DESC LIMIT 100)
FROM `bigquery-public-data.usa_names.usa_1910_current`
WHERE year = 2017
);
-- Which names appear as words in Shakespeare's plays?
SELECT
name AS shakespeare_name
FROM UNNEST(top_names) AS name
WHERE name IN (
SELECT word
FROM `bigquery-public-data.samples.shakespeare`
);
"""
parent_job = Config.CLIENT.query(sql_script, project=Config.CLIENT.project)
parent_job.result()

# Fetch jobs created by the SQL script.
child_jobs = list(Config.CLIENT.list_jobs(parent_job=parent_job))

assert parent_job.num_child_jobs > 0
assert len(child_jobs) == parent_job.num_child_jobs

def test_update_table(self):
dataset = self.temp_dataset(_make_dataset_id("update_table"))

Expand Down