From 4d02e1d0750a430ed2f43663e00b98927d2cf190 Mon Sep 17 00:00:00 2001 From: Mike <45373284+munkhuushmgl@users.noreply.github.com> Date: Thu, 30 Apr 2020 17:53:21 -0700 Subject: [PATCH] translate: fix glossary leak issue (#3572) * fix glossary leak issue * removed try/catch from teardown methods, removed sample_ prefix from all other methods * added specific exceptions to tests, added backoff tags to tests * fixed the lint issues * reordered imports * moved backoff inside methd and removed Retry * corrected import nit --- translate/cloud-client/requirements-test.txt | 3 ++- .../translate_v3_batch_translate_text_test.py | 8 ++++-- ...batch_translate_text_with_glossary_test.py | 27 ++++++++++++++----- .../translate_v3_create_glossary_test.py | 23 +++++++++++----- .../translate_v3_detect_language.py | 2 +- .../translate_v3_detect_language_test.py | 2 +- .../translate_v3_get_glossary_test.py | 25 +++++++++++++---- .../translate_v3_get_supported_languages.py | 2 +- ...anslate_v3_get_supported_languages_test.py | 2 +- .../translate_v3_list_glossary.py | 2 +- .../translate_v3_list_glossary_test.py | 27 ++++++++++++++----- .../translate_v3_translate_text.py | 2 +- .../translate_v3_translate_text_test.py | 2 +- ...te_v3_translate_text_with_glossary_test.py | 25 +++++++++++++---- 14 files changed, 114 insertions(+), 38 deletions(-) diff --git a/translate/cloud-client/requirements-test.txt b/translate/cloud-client/requirements-test.txt index c6f9b64965ae..1fccf2dd32ea 100644 --- a/translate/cloud-client/requirements-test.txt +++ b/translate/cloud-client/requirements-test.txt @@ -1,2 +1,3 @@ +backoff==1.10.0 flaky==3.6.1 -pytest==5.3.2 +pytest==5.3.2 \ No newline at end of file diff --git a/translate/cloud-client/translate_v3_batch_translate_text_test.py b/translate/cloud-client/translate_v3_batch_translate_text_test.py index 9dda7c04789a..4fa8626a7363 100644 --- a/translate/cloud-client/translate_v3_batch_translate_text_test.py +++ b/translate/cloud-client/translate_v3_batch_translate_text_test.py @@ -13,11 +13,15 @@ # limitations under the License. import os -import pytest -import translate_v3_batch_translate_text import uuid + +import pytest + from google.cloud import storage +import translate_v3_batch_translate_text + + PROJECT_ID = os.environ["GCLOUD_PROJECT"] diff --git a/translate/cloud-client/translate_v3_batch_translate_text_with_glossary_test.py b/translate/cloud-client/translate_v3_batch_translate_text_with_glossary_test.py index e50105211892..9af648f758a9 100644 --- a/translate/cloud-client/translate_v3_batch_translate_text_with_glossary_test.py +++ b/translate/cloud-client/translate_v3_batch_translate_text_with_glossary_test.py @@ -13,12 +13,19 @@ # limitations under the License. import os -import pytest import uuid + +import backoff +import pytest + +from google.api_core.exceptions import DeadlineExceeded, GoogleAPICallError +from google.cloud.exceptions import NotFound +from google.cloud import storage + import translate_v3_batch_translate_text_with_glossary import translate_v3_create_glossary import translate_v3_delete_glossary -from google.cloud import storage + PROJECT_ID = os.environ["GCLOUD_PROJECT"] GLOSSARY_INPUT_URI = "gs://cloud-samples-data/translation/glossary_ja.csv" @@ -34,10 +41,18 @@ def glossary(): yield glossary_id - try: - translate_v3_delete_glossary.delete_glossary(PROJECT_ID, glossary_id) - except Exception: - pass + # cleanup + @backoff.on_exception( + backoff.expo, (DeadlineExceeded, GoogleAPICallError), max_time=60 + ) + def delete_glossary(): + try: + translate_v3_delete_glossary.delete_glossary( + PROJECT_ID, glossary_id) + except NotFound as e: + # Ignoring this case. + print("Got NotFound, detail: {}".format(str(e))) + delete_glossary() @pytest.fixture(scope="function") diff --git a/translate/cloud-client/translate_v3_create_glossary_test.py b/translate/cloud-client/translate_v3_create_glossary_test.py index b24488c80894..4b1e08ed640e 100644 --- a/translate/cloud-client/translate_v3_create_glossary_test.py +++ b/translate/cloud-client/translate_v3_create_glossary_test.py @@ -15,11 +15,16 @@ import os import uuid +import backoff import pytest +from google.api_core.exceptions import DeadlineExceeded, GoogleAPICallError +from google.cloud.exceptions import NotFound + import translate_v3_create_glossary import translate_v3_delete_glossary + PROJECT_ID = os.environ["GCLOUD_PROJECT"] GLOSSARY_INPUT_URI = "gs://cloud-samples-data/translation/glossary_ja.csv" @@ -36,9 +41,15 @@ def test_create_glossary(capsys): assert "Created:" in out assert "gs://cloud-samples-data/translation/glossary_ja.csv" in out finally: - # clean up after use - try: - translate_v3_delete_glossary.delete_glossary( - PROJECT_ID, glossary_id) - except Exception: - pass + # cleanup + @backoff.on_exception( + backoff.expo, (DeadlineExceeded, GoogleAPICallError), max_time=60 + ) + def delete_glossary(): + try: + translate_v3_delete_glossary.delete_glossary( + PROJECT_ID, glossary_id) + except NotFound as e: + # Ignoring this case. + print("Got NotFound, detail: {}".format(str(e))) + delete_glossary() diff --git a/translate/cloud-client/translate_v3_detect_language.py b/translate/cloud-client/translate_v3_detect_language.py index 9b92853cfca1..759eb41a4d30 100644 --- a/translate/cloud-client/translate_v3_detect_language.py +++ b/translate/cloud-client/translate_v3_detect_language.py @@ -16,7 +16,7 @@ from google.cloud import translate -def sample_detect_language(project_id="YOUR_PROJECT_ID"): +def detect_language(project_id="YOUR_PROJECT_ID"): """Detecting the language of a text string.""" client = translate.TranslationServiceClient() diff --git a/translate/cloud-client/translate_v3_detect_language_test.py b/translate/cloud-client/translate_v3_detect_language_test.py index 82620bc550c1..a538fc68aa6b 100644 --- a/translate/cloud-client/translate_v3_detect_language_test.py +++ b/translate/cloud-client/translate_v3_detect_language_test.py @@ -19,6 +19,6 @@ def test_detect_language(capsys): - translate_v3_detect_language.sample_detect_language(PROJECT_ID) + translate_v3_detect_language.detect_language(PROJECT_ID) out, _ = capsys.readouterr() assert "en" in out diff --git a/translate/cloud-client/translate_v3_get_glossary_test.py b/translate/cloud-client/translate_v3_get_glossary_test.py index 7a4aa167cead..8880bf6de5dd 100644 --- a/translate/cloud-client/translate_v3_get_glossary_test.py +++ b/translate/cloud-client/translate_v3_get_glossary_test.py @@ -13,11 +13,18 @@ # limitations under the License. import os +import uuid + +import backoff import pytest + +from google.api_core.exceptions import DeadlineExceeded, GoogleAPICallError +from google.cloud.exceptions import NotFound + import translate_v3_create_glossary import translate_v3_delete_glossary import translate_v3_get_glossary -import uuid + PROJECT_ID = os.environ["GCLOUD_PROJECT"] GLOSSARY_INPUT_URI = "gs://cloud-samples-data/translation/glossary_ja.csv" @@ -33,10 +40,18 @@ def glossary(): yield glossary_id - try: - translate_v3_delete_glossary.delete_glossary(PROJECT_ID, glossary_id) - except Exception: - pass + # cleanup + @backoff.on_exception( + backoff.expo, (DeadlineExceeded, GoogleAPICallError), max_time=60 + ) + def delete_glossary(): + try: + translate_v3_delete_glossary.delete_glossary( + PROJECT_ID, glossary_id) + except NotFound as e: + # Ignoring this case. + print("Got NotFound, detail: {}".format(str(e))) + delete_glossary() def test_get_glossary(capsys, glossary): diff --git a/translate/cloud-client/translate_v3_get_supported_languages.py b/translate/cloud-client/translate_v3_get_supported_languages.py index eb7d83666a43..6e15458d9cca 100644 --- a/translate/cloud-client/translate_v3_get_supported_languages.py +++ b/translate/cloud-client/translate_v3_get_supported_languages.py @@ -16,7 +16,7 @@ from google.cloud import translate -def sample_get_supported_languages(project_id="YOUR_PROJECT_ID"): +def get_supported_languages(project_id="YOUR_PROJECT_ID"): """Getting a list of supported language codes.""" client = translate.TranslationServiceClient() diff --git a/translate/cloud-client/translate_v3_get_supported_languages_test.py b/translate/cloud-client/translate_v3_get_supported_languages_test.py index b4ba4cd22b30..cd4e44a445c0 100644 --- a/translate/cloud-client/translate_v3_get_supported_languages_test.py +++ b/translate/cloud-client/translate_v3_get_supported_languages_test.py @@ -19,6 +19,6 @@ def test_list_languages(capsys): - translate_v3_get_supported_languages.sample_get_supported_languages(PROJECT_ID) + translate_v3_get_supported_languages.get_supported_languages(PROJECT_ID) out, _ = capsys.readouterr() assert "zh-CN" in out diff --git a/translate/cloud-client/translate_v3_list_glossary.py b/translate/cloud-client/translate_v3_list_glossary.py index e52d6b260ed9..53e51d85447f 100644 --- a/translate/cloud-client/translate_v3_list_glossary.py +++ b/translate/cloud-client/translate_v3_list_glossary.py @@ -16,7 +16,7 @@ from google.cloud import translate -def sample_list_glossaries(project_id="YOUR_PROJECT_ID"): +def list_glossaries(project_id="YOUR_PROJECT_ID"): """List Glossaries.""" client = translate.TranslationServiceClient() diff --git a/translate/cloud-client/translate_v3_list_glossary_test.py b/translate/cloud-client/translate_v3_list_glossary_test.py index de77daba8028..d0d38cde9855 100644 --- a/translate/cloud-client/translate_v3_list_glossary_test.py +++ b/translate/cloud-client/translate_v3_list_glossary_test.py @@ -13,11 +13,18 @@ # limitations under the License. import os +import uuid + +import backoff import pytest + +from google.api_core.exceptions import DeadlineExceeded, GoogleAPICallError +from google.cloud.exceptions import NotFound + import translate_v3_create_glossary import translate_v3_delete_glossary import translate_v3_list_glossary -import uuid + PROJECT_ID = os.environ["GCLOUD_PROJECT"] GLOSSARY_INPUT_URI = "gs://cloud-samples-data/translation/glossary_ja.csv" @@ -33,14 +40,22 @@ def glossary(): yield glossary_id - try: - translate_v3_delete_glossary.sample_delete_glossary(PROJECT_ID, glossary_id) - except Exception: - pass + # clean up + @backoff.on_exception( + backoff.expo, (DeadlineExceeded, GoogleAPICallError), max_time=60 + ) + def delete_glossary(): + try: + translate_v3_delete_glossary.delete_glossary( + PROJECT_ID, glossary_id) + except NotFound as e: + # Ignoring this case. + print("Got NotFound, detail: {}".format(str(e))) + delete_glossary() def test_list_glossary(capsys, glossary): - translate_v3_list_glossary.sample_list_glossaries(PROJECT_ID) + translate_v3_list_glossary.list_glossaries(PROJECT_ID) out, _ = capsys.readouterr() assert glossary in out assert "gs://cloud-samples-data/translation/glossary_ja.csv" in out diff --git a/translate/cloud-client/translate_v3_translate_text.py b/translate/cloud-client/translate_v3_translate_text.py index 77fcec26deeb..086f775a37fb 100644 --- a/translate/cloud-client/translate_v3_translate_text.py +++ b/translate/cloud-client/translate_v3_translate_text.py @@ -16,7 +16,7 @@ from google.cloud import translate -def sample_translate_text(text="YOUR_TEXT_TO_TRANSLATE", project_id="YOUR_PROJECT_ID"): +def translate_text(text="YOUR_TEXT_TO_TRANSLATE", project_id="YOUR_PROJECT_ID"): """Translating Text.""" client = translate.TranslationServiceClient() diff --git a/translate/cloud-client/translate_v3_translate_text_test.py b/translate/cloud-client/translate_v3_translate_text_test.py index a63190197818..096b8dd847fa 100644 --- a/translate/cloud-client/translate_v3_translate_text_test.py +++ b/translate/cloud-client/translate_v3_translate_text_test.py @@ -19,7 +19,7 @@ def test_translate_text(capsys): - translate_v3_translate_text.sample_translate_text( + translate_v3_translate_text.translate_text( "Hello World!", PROJECT_ID) out, _ = capsys.readouterr() assert "Bonjour le monde" in out diff --git a/translate/cloud-client/translate_v3_translate_text_with_glossary_test.py b/translate/cloud-client/translate_v3_translate_text_with_glossary_test.py index 72f9d64e2f8d..adf66836fde8 100644 --- a/translate/cloud-client/translate_v3_translate_text_with_glossary_test.py +++ b/translate/cloud-client/translate_v3_translate_text_with_glossary_test.py @@ -14,11 +14,18 @@ # limitations under the License. import os +import uuid + +import backoff import pytest + +from google.api_core.exceptions import DeadlineExceeded, GoogleAPICallError +from google.cloud.exceptions import NotFound + import translate_v3_create_glossary import translate_v3_delete_glossary import translate_v3_translate_text_with_glossary -import uuid + PROJECT_ID = os.environ["GCLOUD_PROJECT"] GLOSSARY_INPUT_URI = "gs://cloud-samples-data/translation/glossary_ja.csv" @@ -34,10 +41,18 @@ def glossary(): yield glossary_id - try: - translate_v3_delete_glossary.delete_glossary(PROJECT_ID, glossary_id) - except Exception: - pass + # cleanup + @backoff.on_exception( + backoff.expo, (DeadlineExceeded, GoogleAPICallError), max_time=60 + ) + def delete_glossary(): + try: + translate_v3_delete_glossary.delete_glossary( + PROJECT_ID, glossary_id) + except NotFound as e: + # Ignoring this case. + print("Got NotFound, detail: {}".format(str(e))) + delete_glossary() def test_translate_text_with_glossary(capsys, glossary):