From aaaf99c6b35a3b599b52ecdd9d61cd75e4b3cd8c Mon Sep 17 00:00:00 2001 From: Josema Camacho Date: Thu, 3 Sep 2020 17:47:54 +0200 Subject: [PATCH 1/4] Not found decorator for CARTO --- cartoframes/io/carto.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cartoframes/io/carto.py b/cartoframes/io/carto.py index d1a49add4..4fe14444c 100644 --- a/cartoframes/io/carto.py +++ b/cartoframes/io/carto.py @@ -1,4 +1,5 @@ """Functions to interact with the CARTO platform""" +import functools import math from pandas import DataFrame @@ -21,7 +22,22 @@ CSV_TO_CARTO_RATIO = 1.4 +def not_found(): + def decorator_func(func): + @functools.wraps(func) + def wrapper_func(*args, **kwargs): + try: + return func(*args, **kwargs) + + except CartoException: + log.error('User and/or table do not exist') + + return wrapper_func + return decorator_func + + @send_metrics('data_downloaded') +@not_found() def read_carto(source, credentials=None, limit=None, retry_times=3, schema=None, index_col=None, decode_geom=True, null_geom_value=None): """Read a table or a SQL query from the CARTO account. @@ -74,6 +90,7 @@ def read_carto(source, credentials=None, limit=None, retry_times=3, schema=None, @send_metrics('data_uploaded') +@not_found() def to_carto(dataframe, table_name, credentials=None, if_exists='fail', geom_col=None, index=False, index_label=None, cartodbfy=True, log_enabled=True, retry_times=3, max_upload_size=MAX_UPLOAD_SIZE_BYTES, skip_quota_warning=False): @@ -172,6 +189,7 @@ def to_carto(dataframe, table_name, credentials=None, if_exists='fail', geom_col return table_name +@not_found() def list_tables(credentials=None): """List all of the tables in the CARTO account. @@ -189,6 +207,7 @@ def list_tables(credentials=None): return context_manager.list_tables() +@not_found() def has_table(table_name, credentials=None, schema=None): """Check if the table exists in the CARTO account. @@ -213,6 +232,7 @@ def has_table(table_name, credentials=None, schema=None): return context_manager.has_table(table_name, schema) +@not_found() def delete_table(table_name, credentials=None, log_enabled=True): """Delete the table from the CARTO account. @@ -238,6 +258,7 @@ def delete_table(table_name, credentials=None, log_enabled=True): log.info('Table "{}" does not exist'.format(table_name)) +@not_found() def rename_table(table_name, new_table_name, credentials=None, if_exists='fail', log_enabled=True): """Rename a table in the CARTO account. @@ -270,6 +291,7 @@ def rename_table(table_name, new_table_name, credentials=None, if_exists='fail', log.info('Success! Table "{0}" renamed to table "{1}" correctly'.format(table_name, new_table_name)) +@not_found() def copy_table(table_name, new_table_name, credentials=None, if_exists='fail', log_enabled=True): """Copy a table into a new table in the CARTO account. @@ -303,6 +325,7 @@ def copy_table(table_name, new_table_name, credentials=None, if_exists='fail', l log.info('Success! Table "{0}" copied to table "{1}" correctly'.format(table_name, new_table_name)) +@not_found() def create_table_from_query(query, new_table_name, credentials=None, if_exists='fail', log_enabled=True): """Create a new table from an SQL query in the CARTO account. @@ -334,6 +357,7 @@ def create_table_from_query(query, new_table_name, credentials=None, if_exists=' log.info('Success! Table "{0}" created correctly'.format(new_table_name)) +@not_found() def describe_table(table_name, credentials=None, schema=None): """Describe the table in the CARTO account. @@ -372,6 +396,7 @@ def describe_table(table_name, credentials=None, schema=None): } +@not_found() def update_privacy_table(table_name, privacy, credentials=None, log_enabled=True): """Update the table information in the CARTO account. From 1899951b816fec9fdb662007ed7a58073d6a7681 Mon Sep 17 00:00:00 2001 From: Josema Camacho Date: Fri, 4 Sep 2020 10:13:56 +0200 Subject: [PATCH 2/4] Moved user/table not found exception to ContextManager and handling only NotFoundException subexception --- cartoframes/io/carto.py | 25 ------------------- cartoframes/io/managers/context_manager.py | 29 +++++++++++++++++++--- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/cartoframes/io/carto.py b/cartoframes/io/carto.py index 4fe14444c..d1a49add4 100644 --- a/cartoframes/io/carto.py +++ b/cartoframes/io/carto.py @@ -1,5 +1,4 @@ """Functions to interact with the CARTO platform""" -import functools import math from pandas import DataFrame @@ -22,22 +21,7 @@ CSV_TO_CARTO_RATIO = 1.4 -def not_found(): - def decorator_func(func): - @functools.wraps(func) - def wrapper_func(*args, **kwargs): - try: - return func(*args, **kwargs) - - except CartoException: - log.error('User and/or table do not exist') - - return wrapper_func - return decorator_func - - @send_metrics('data_downloaded') -@not_found() def read_carto(source, credentials=None, limit=None, retry_times=3, schema=None, index_col=None, decode_geom=True, null_geom_value=None): """Read a table or a SQL query from the CARTO account. @@ -90,7 +74,6 @@ def read_carto(source, credentials=None, limit=None, retry_times=3, schema=None, @send_metrics('data_uploaded') -@not_found() def to_carto(dataframe, table_name, credentials=None, if_exists='fail', geom_col=None, index=False, index_label=None, cartodbfy=True, log_enabled=True, retry_times=3, max_upload_size=MAX_UPLOAD_SIZE_BYTES, skip_quota_warning=False): @@ -189,7 +172,6 @@ def to_carto(dataframe, table_name, credentials=None, if_exists='fail', geom_col return table_name -@not_found() def list_tables(credentials=None): """List all of the tables in the CARTO account. @@ -207,7 +189,6 @@ def list_tables(credentials=None): return context_manager.list_tables() -@not_found() def has_table(table_name, credentials=None, schema=None): """Check if the table exists in the CARTO account. @@ -232,7 +213,6 @@ def has_table(table_name, credentials=None, schema=None): return context_manager.has_table(table_name, schema) -@not_found() def delete_table(table_name, credentials=None, log_enabled=True): """Delete the table from the CARTO account. @@ -258,7 +238,6 @@ def delete_table(table_name, credentials=None, log_enabled=True): log.info('Table "{}" does not exist'.format(table_name)) -@not_found() def rename_table(table_name, new_table_name, credentials=None, if_exists='fail', log_enabled=True): """Rename a table in the CARTO account. @@ -291,7 +270,6 @@ def rename_table(table_name, new_table_name, credentials=None, if_exists='fail', log.info('Success! Table "{0}" renamed to table "{1}" correctly'.format(table_name, new_table_name)) -@not_found() def copy_table(table_name, new_table_name, credentials=None, if_exists='fail', log_enabled=True): """Copy a table into a new table in the CARTO account. @@ -325,7 +303,6 @@ def copy_table(table_name, new_table_name, credentials=None, if_exists='fail', l log.info('Success! Table "{0}" copied to table "{1}" correctly'.format(table_name, new_table_name)) -@not_found() def create_table_from_query(query, new_table_name, credentials=None, if_exists='fail', log_enabled=True): """Create a new table from an SQL query in the CARTO account. @@ -357,7 +334,6 @@ def create_table_from_query(query, new_table_name, credentials=None, if_exists=' log.info('Success! Table "{0}" created correctly'.format(new_table_name)) -@not_found() def describe_table(table_name, credentials=None, schema=None): """Describe the table in the CARTO account. @@ -396,7 +372,6 @@ def describe_table(table_name, credentials=None, schema=None): } -@not_found() def update_privacy_table(table_name, privacy, credentials=None, log_enabled=True): """Update the table information in the CARTO account. diff --git a/cartoframes/io/managers/context_manager.py b/cartoframes/io/managers/context_manager.py index f9825e8f1..b24649947 100644 --- a/cartoframes/io/managers/context_manager.py +++ b/cartoframes/io/managers/context_manager.py @@ -1,3 +1,4 @@ +import functools import time import pandas as pd @@ -8,16 +9,16 @@ from carto.datasets import DatasetManager from carto.exceptions import CartoException, CartoRateLimitException from carto.sql import SQLClient, BatchSQLClient, CopySQLClient +from pyrestcli.exceptions import NotFoundException from ..dataset_info import DatasetInfo from ... import __version__ from ...auth.defaults import get_default_credentials from ...utils.logger import log from ...utils.geom_utils import encode_geometry_ewkb -from ...utils.utils import is_sql_query, check_credentials, encode_row, map_geom_type, PG_NULL, \ - double_quote -from ...utils.columns import get_dataframe_columns_info, get_query_columns_info, obtain_converters, \ - date_columns_names, normalize_name +from ...utils.utils import is_sql_query, check_credentials, encode_row, map_geom_type, PG_NULL, double_quote +from ...utils.columns import (get_dataframe_columns_info, get_query_columns_info, obtain_converters, date_columns_names, + normalize_name) DEFAULT_RETRY_TIMES = 3 @@ -43,6 +44,24 @@ def wrapper(*args, **kwargs): return wrapper +def not_found(): + def decorator_func(func): + @functools.wraps(func) + def wrapper_func(*args, **kwargs): + try: + return func(*args, **kwargs) + + except CartoException as e: + if hasattr(e, 'args') and isinstance(e.args, (list, tuple)) and type(e.args[0]) == NotFoundException: + raise NotFoundException('User and/or table do not exist') from None + + else: + raise e + + return wrapper_func + return decorator_func + + class ContextManager: def __init__(self, credentials): @@ -54,9 +73,11 @@ def __init__(self, credentials): self.copy_client = CopySQLClient(self.auth_client) self.batch_sql_client = BatchSQLClient(self.auth_client) + @not_found() def execute_query(self, query, parse_json=True, do_post=True, format=None, **request_args): return self.sql_client.send(query.strip(), parse_json, do_post, format, **request_args) + @not_found() def execute_long_running_query(self, query): return self.batch_sql_client.create_and_wait_for_completion(query.strip()) From 397336e61fa05e1f421449cfe33b1ad5012e188e Mon Sep 17 00:00:00 2001 From: Josema Camacho Date: Fri, 4 Sep 2020 10:27:04 +0200 Subject: [PATCH 3/4] Autocalled decorator --- cartoframes/io/managers/context_manager.py | 26 +++++++++------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/cartoframes/io/managers/context_manager.py b/cartoframes/io/managers/context_manager.py index b24649947..ed2fc18cb 100644 --- a/cartoframes/io/managers/context_manager.py +++ b/cartoframes/io/managers/context_manager.py @@ -1,4 +1,3 @@ -import functools import time import pandas as pd @@ -44,21 +43,18 @@ def wrapper(*args, **kwargs): return wrapper -def not_found(): - def decorator_func(func): - @functools.wraps(func) - def wrapper_func(*args, **kwargs): - try: - return func(*args, **kwargs) +def not_found(func): + def decorator_func(*args, **kwargs): + try: + return func(*args, **kwargs) - except CartoException as e: - if hasattr(e, 'args') and isinstance(e.args, (list, tuple)) and type(e.args[0]) == NotFoundException: - raise NotFoundException('User and/or table do not exist') from None + except CartoException as e: + if hasattr(e, 'args') and isinstance(e.args, (list, tuple)) and type(e.args[0]) == NotFoundException: + raise NotFoundException('User and/or table do not exist') from None - else: - raise e + else: + raise e - return wrapper_func return decorator_func @@ -73,11 +69,11 @@ def __init__(self, credentials): self.copy_client = CopySQLClient(self.auth_client) self.batch_sql_client = BatchSQLClient(self.auth_client) - @not_found() + @not_found def execute_query(self, query, parse_json=True, do_post=True, format=None, **request_args): return self.sql_client.send(query.strip(), parse_json, do_post, format, **request_args) - @not_found() + @not_found def execute_long_running_query(self, query): return self.batch_sql_client.create_and_wait_for_completion(query.strip()) From b2b5d157aa650836011a1e459e36351aefecc049 Mon Sep 17 00:00:00 2001 From: Josema Camacho Date: Fri, 4 Sep 2020 12:58:37 +0200 Subject: [PATCH 4/4] Returning generic exception when user or table not found --- cartoframes/io/managers/context_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cartoframes/io/managers/context_manager.py b/cartoframes/io/managers/context_manager.py index ed2fc18cb..15d3cf8ec 100644 --- a/cartoframes/io/managers/context_manager.py +++ b/cartoframes/io/managers/context_manager.py @@ -50,7 +50,7 @@ def decorator_func(*args, **kwargs): except CartoException as e: if hasattr(e, 'args') and isinstance(e.args, (list, tuple)) and type(e.args[0]) == NotFoundException: - raise NotFoundException('User and/or table do not exist') from None + raise Exception('Resource not found') from None else: raise e