From 66bc071fd5ec8b298b07918d4547bad76e077a4a Mon Sep 17 00:00:00 2001 From: Israel Herraiz Date: Thu, 29 Aug 2024 19:46:24 +0200 Subject: [PATCH 1/4] Avoid parsing all options just to grab an option that is already parsed. This fixes #32361 --- sdks/python/apache_beam/io/gcp/gcsio.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py index 919791e23305..17db1bd58115 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio.py +++ b/sdks/python/apache_beam/io/gcp/gcsio.py @@ -145,10 +145,8 @@ def __init__(self, storage_client=None, pipeline_options=None): pipeline_options = PipelineOptions.from_dictionary(pipeline_options) if storage_client is None: storage_client = create_storage_client(pipeline_options) - self.enable_read_bucket_metric = pipeline_options.get_all_options( - )['enable_bucket_read_metric_counter'] - self.enable_write_bucket_metric = pipeline_options.get_all_options( - )['enable_bucket_write_metric_counter'] + self.enable_read_bucket_metric = getattr(pipeline_options, 'enable_bucket_read_metric_counter', False) + self.enable_write_bucket_metric = getattr(pipeline_options, 'enable_bucket_write_metric_counter', False) self.client = storage_client self._rewrite_cb = None self.bucket_to_project_number = {} From 8fe3386c847072fd33f77e57a68b75d00e244bd1 Mon Sep 17 00:00:00 2001 From: Israel Herraiz Date: Thu, 29 Aug 2024 22:32:05 +0200 Subject: [PATCH 2/4] The GCS options are in GoogleCloudOptions, not generic PipelineOptions --- sdks/python/apache_beam/io/gcp/gcsio.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py index 17db1bd58115..a64db5b36401 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio.py +++ b/sdks/python/apache_beam/io/gcp/gcsio.py @@ -145,8 +145,11 @@ def __init__(self, storage_client=None, pipeline_options=None): pipeline_options = PipelineOptions.from_dictionary(pipeline_options) if storage_client is None: storage_client = create_storage_client(pipeline_options) - self.enable_read_bucket_metric = getattr(pipeline_options, 'enable_bucket_read_metric_counter', False) - self.enable_write_bucket_metric = getattr(pipeline_options, 'enable_bucket_write_metric_counter', False) + + google_cloud_options = pipeline_options.view_as(GoogleCloudOptions) + self.enable_read_bucket_metric = getattr(google_cloud_options, 'enable_bucket_read_metric_counter', False) + self.enable_write_bucket_metric = getattr(google_cloud_options, 'enable_bucket_write_metric_counter', False) + self.client = storage_client self._rewrite_cb = None self.bucket_to_project_number = {} From b0caa7976c3f55854a11450e4fb866cca2d9f079 Mon Sep 17 00:00:00 2001 From: Israel Herraiz Date: Thu, 29 Aug 2024 22:45:21 +0200 Subject: [PATCH 3/4] Fix formatting issues --- sdks/python/apache_beam/io/gcp/gcsio.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py index a64db5b36401..6e615e25906c 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio.py +++ b/sdks/python/apache_beam/io/gcp/gcsio.py @@ -147,8 +147,10 @@ def __init__(self, storage_client=None, pipeline_options=None): storage_client = create_storage_client(pipeline_options) google_cloud_options = pipeline_options.view_as(GoogleCloudOptions) - self.enable_read_bucket_metric = getattr(google_cloud_options, 'enable_bucket_read_metric_counter', False) - self.enable_write_bucket_metric = getattr(google_cloud_options, 'enable_bucket_write_metric_counter', False) + self.enable_read_bucket_metric = getattr( + google_cloud_options, 'enable_bucket_read_metric_counter', False) + self.enable_write_bucket_metric = getattr( + google_cloud_options, 'enable_bucket_write_metric_counter', False) self.client = storage_client self._rewrite_cb = None From aac398dab5c974ac60e8e40848c057f6207ae959 Mon Sep 17 00:00:00 2001 From: Israel Herraiz Date: Fri, 30 Aug 2024 09:55:14 +0200 Subject: [PATCH 4/4] Fix formatting issues (again) --- sdks/python/apache_beam/io/gcp/gcsio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py index 6e615e25906c..6b0470b82361 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio.py +++ b/sdks/python/apache_beam/io/gcp/gcsio.py @@ -148,9 +148,9 @@ def __init__(self, storage_client=None, pipeline_options=None): google_cloud_options = pipeline_options.view_as(GoogleCloudOptions) self.enable_read_bucket_metric = getattr( - google_cloud_options, 'enable_bucket_read_metric_counter', False) + google_cloud_options, 'enable_bucket_read_metric_counter', False) self.enable_write_bucket_metric = getattr( - google_cloud_options, 'enable_bucket_write_metric_counter', False) + google_cloud_options, 'enable_bucket_write_metric_counter', False) self.client = storage_client self._rewrite_cb = None