From d610605d01185eb2e35834d3cf8aee20cf66b3f9 Mon Sep 17 00:00:00 2001 From: Charles Engelke Date: Wed, 21 Nov 2018 13:46:43 -0800 Subject: [PATCH] Revert "Update dataproc/submit_job_to_cluster.py" (#1864) * Revert "Remove test configs for non-testing directories (#1855)" This reverts commit 73a73321579337312e8ba85c34fe9c37b42b7f6e. * Revert "Auto-update dependencies. (#1846)" This reverts commit df8e623681e68f95c4671eadbd0b15d699898530. * Revert "Tweak slack sample (#1847)" This reverts commit a48c010481c166968d9f1bd58106054c5d1c58f9. * Revert "Non-client library example of constructing a Signed URL (#1837)" This reverts commit fc3284d995a8a35c473a207e80490fad265782af. * Revert "GCF samples: handle {empty JSON, GET} requests + remove commas (#1832)" This reverts commit 6928491ed3d52b0bec694e6b30257f08caac5f2b. * Revert "Correct the maintenance event types (#1830)" This reverts commit c22840fd23586349b7b665d851dea046a94ba7c7. * Revert "Fix GCF region tags (#1827)" This reverts commit 0fbfef27d35cea23ad0e20fd2c9df3e8a4a046cb. * Revert "Updated to Flask 1.0 (#1819)" This reverts commit d52ccf99503311bba2cec2881e8cb0f9b5a6f2bf. * Revert "Fix deprecation warning (#1801)" This reverts commit 981737e85f60eca5cc337f172249deddca9b291b. * Revert "Update submit_job_to_cluster.py (#1708)" This reverts commit cdab25b3b7a973bd9de864920d5fa5a79dce379c. --- dataproc/submit_job_to_cluster.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dataproc/submit_job_to_cluster.py b/dataproc/submit_job_to_cluster.py index ed49013d2752..f06d5981c167 100644 --- a/dataproc/submit_job_to_cluster.py +++ b/dataproc/submit_job_to_cluster.py @@ -34,6 +34,14 @@ def get_pyspark_file(filename): return f, os.path.basename(filename) +def get_region_from_zone(zone): + try: + region_as_list = zone.split('-')[:-1] + return '-'.join(region_as_list) + except (AttributeError, IndexError, ValueError): + raise ValueError('Invalid zone provided, please check your input.') + + def upload_pyspark_file(project_id, bucket_name, filename, file): """Uploads the PySpark file in this directory to the configured input bucket.""" @@ -191,7 +199,7 @@ def get_client(): def main(project_id, zone, cluster_name, bucket_name, pyspark_file=None, create_new_cluster=True): dataproc = get_client() - region = 'global' + region = get_region_from_zone(zone) try: if pyspark_file: spark_file, spark_filename = get_pyspark_file(pyspark_file)