From 5c9ff5b9d9624fcf4a40527e7eac40ae4de90ddc Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Tue, 22 Dec 2020 15:40:07 +0100 Subject: [PATCH 01/13] Seek files before retry --- deepomatic/api/http_helper.py | 33 ++++++++++++++++++++++++--------- deepomatic/api/version.py | 2 +- setup.py | 13 +++---------- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index 710f20a..0ae9453 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -209,15 +209,30 @@ def recursive_json_dump(prefix, obj, data_dict, omit_dot=False): return new_dict def send_request(self, requests_callable, *args, **kwargs): - # requests_callable must be a method from the requests module - # this is the timeout of requests module requests_timeout = kwargs.pop('timeout', self.requests_timeout) + + files = kwargs.pop('files', None) + print(files) + if files: + for f in files.values(): + # seek files before each retry + if hasattr(f, 'seek'): + f.seek(0) + print("SENDING REQUEST") + return requests_callable(*args, files=files, + timeout=requests_timeout, + verify=self.verify_ssl, + **kwargs) + + def maybe_retry_send_request(self, requests_callable, *args, **kwargs): + # requests_callable must be a method from the requests module + http_retry = kwargs.pop('http_retry', self.http_retry) - functor = functools.partial(requests_callable, *args, - verify=self.verify_ssl, - timeout=requests_timeout, **kwargs) + functor = functools.partial(self.send_request, + requests_callable, + *args, **kwargs) if http_retry is not None: return http_retry.retry(functor) @@ -274,10 +289,10 @@ def make_request(self, func, resource, params=None, data=None, if not resource.startswith('http'): resource = self.resource_prefix + resource - response = self.send_request(func, resource, *args, - params=params, data=data, - files=files, headers=headers, - stream=stream, **kwargs) + response = self.maybe_retry_send_request(func, resource, *args, + params=params, data=data, + files=files, headers=headers, + stream=stream, **kwargs) # Close opened files for file in opened_files: diff --git a/deepomatic/api/version.py b/deepomatic/api/version.py index f4f3cf1..a7fa864 100644 --- a/deepomatic/api/version.py +++ b/deepomatic/api/version.py @@ -1,6 +1,6 @@ __title__ = 'deepomatic-api' __description__ = 'Deepomatic API client' -__version__ = '0.9.2' +__version__ = '0.9.3' __author__ = 'deepomatic' __author_email__ = 'support@deepomatic.com' __url__ = 'https://github.com/deepomatic/deepomatic-client-python' diff --git a/setup.py b/setup.py index 58addf5..f0d6af7 100644 --- a/setup.py +++ b/setup.py @@ -2,17 +2,9 @@ import io from setuptools import find_packages, setup -try: - # for pip >= 10 - from pip._internal.req import parse_requirements -except ImportError: - # for pip <= 9.0.3 - from pip.req import parse_requirements - here = os.path.abspath(os.path.dirname(__file__)) - about = {} with io.open(os.path.join(here, 'deepomatic', 'api', 'version.py'), 'r', encoding='utf-8') as f: exec(f.read(), about) @@ -24,7 +16,8 @@ os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) # Read requirements -install_reqs = parse_requirements(os.path.join(here, 'requirements.txt'), session='hack') +with io.open(os.path.join(here, 'requirements.txt'), encoding='utf-8') as f: + requirements = f.readlines() namespaces = ['deepomatic'] @@ -43,7 +36,7 @@ long_description=README, long_description_content_type='text/markdown', data_files=[('', ['requirements.txt'])], - install_requires=[str(ir.req) for ir in install_reqs], + install_requires=requirements, python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", classifiers=[ 'Operating System :: OS Independent', From 08217fd164b6b8af0aa8d71dceb4b6ba11eea3f8 Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Tue, 22 Dec 2020 15:41:01 +0100 Subject: [PATCH 02/13] remove debug --- deepomatic/api/http_helper.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index 0ae9453..e94e200 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -213,13 +213,12 @@ def send_request(self, requests_callable, *args, **kwargs): requests_timeout = kwargs.pop('timeout', self.requests_timeout) files = kwargs.pop('files', None) - print(files) if files: for f in files.values(): # seek files before each retry if hasattr(f, 'seek'): f.seek(0) - print("SENDING REQUEST") + return requests_callable(*args, files=files, timeout=requests_timeout, verify=self.verify_ssl, From 0a69bfb236cc3c67c80b0bb918cbef39f41cf709 Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Tue, 22 Dec 2020 15:49:59 +0100 Subject: [PATCH 03/13] flake --- demo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo.py b/demo.py index 6847e0f..1f7013a 100644 --- a/demo.py +++ b/demo.py @@ -359,7 +359,7 @@ def download_file(url): filename = os.path.join(tempfile.gettempdir(), hashlib.sha1(url.encode()).hexdigest() + ext) if os.path.exists(filename): # avoid redownloading - logger.info("Skipping download of {}: file already exist in ".format(url, filename)) + logger.info("Skipping download of {}: file already exist in {}".format(url, filename)) return filename r = requests.get(url, stream=True) r.raise_for_status() From 813c694ca76376ce0a72d1a0c72c94c310741a83 Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Tue, 22 Dec 2020 17:23:54 +0100 Subject: [PATCH 04/13] Check unsupported files --- deepomatic/api/http_helper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index e94e200..a0a4bc7 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -214,10 +214,12 @@ def send_request(self, requests_callable, *args, **kwargs): files = kwargs.pop('files', None) if files: - for f in files.values(): + for key, f in files.items(): # seek files before each retry if hasattr(f, 'seek'): f.seek(0) + elif not isinstance(f, (string_types, bytes, bytearray)): + raise DeepomaticException("Unsupported file type '{}' for key '{}'".format(type(f), key)) return requests_callable(*args, files=files, timeout=requests_timeout, From 928941b40cde1cfd6278523404fd57639ad2a71b Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Tue, 22 Dec 2020 17:25:53 +0100 Subject: [PATCH 05/13] Move flake to requirement.dev.txt --- requirements.dev.txt | 1 + requirements.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index 6a2542d..a579324 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -3,3 +3,4 @@ pytest==4.6.5 pytest-cov==2.7.1 pytest-voluptuous==1.1.0 httpretty==0.9.6 +flake8==3.8.4 diff --git a/requirements.txt b/requirements.txt index 01fca15..9e16833 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,3 @@ promise>=2.1,<3 six>=1.10.0,<2 requests>=2.19.0,<3 # will not work below in python3 tenacity>=5.1,<6 -flake8>=3.7,<4 From 91cc9b50cc72402fbc0063d398e1aba0c1e01c97 Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Tue, 22 Dec 2020 17:26:37 +0100 Subject: [PATCH 06/13] Update deepomatic/api/http_helper.py Co-authored-by: Thomas Riccardi --- deepomatic/api/http_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index a0a4bc7..bf52882 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -219,7 +219,7 @@ def send_request(self, requests_callable, *args, **kwargs): if hasattr(f, 'seek'): f.seek(0) elif not isinstance(f, (string_types, bytes, bytearray)): - raise DeepomaticException("Unsupported file type '{}' for key '{}'".format(type(f), key)) + raise DeepomaticException("Unsupported file object type '{}' for key '{}'".format(type(f), key)) return requests_callable(*args, files=files, timeout=requests_timeout, From 355e540c34db82c9ccd5ab9606b0fefae81ae11b Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Tue, 22 Dec 2020 17:27:37 +0100 Subject: [PATCH 07/13] Update deepomatic/api/http_helper.py Co-authored-by: Thomas Riccardi --- deepomatic/api/http_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index bf52882..bece6e6 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -215,7 +215,7 @@ def send_request(self, requests_callable, *args, **kwargs): files = kwargs.pop('files', None) if files: for key, f in files.items(): - # seek files before each retry + # seek files before each retry, to avoid silently retrying with different input if hasattr(f, 'seek'): f.seek(0) elif not isinstance(f, (string_types, bytes, bytearray)): From baee39c3f28526032b6670d0014cc58606b818d0 Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Tue, 22 Dec 2020 17:47:32 +0100 Subject: [PATCH 08/13] handling file tuple --- deepomatic/api/http_helper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index bece6e6..e7b0055 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -215,6 +215,10 @@ def send_request(self, requests_callable, *args, **kwargs): files = kwargs.pop('files', None) if files: for key, f in files.items(): + # file can be a tuple + # if so, the fileobj is in second position + if isinstance(f, (tuple, list)): + f = f[1] # seek files before each retry, to avoid silently retrying with different input if hasattr(f, 'seek'): f.seek(0) From bbd8eaa87abcf7b05c8fcae8a4f2c75c202e3812 Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Tue, 22 Dec 2020 17:59:23 +0100 Subject: [PATCH 09/13] include scalar types --- deepomatic/api/http_helper.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index e7b0055..505c9f2 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -219,11 +219,17 @@ def send_request(self, requests_callable, *args, **kwargs): # if so, the fileobj is in second position if isinstance(f, (tuple, list)): f = f[1] + if isinstance(f, (string_types, bytes, bytearray, int, float, bool)): + continue + error = "Unsupported file object type '{}' for key '{}'".format(type(f), key) # seek files before each retry, to avoid silently retrying with different input if hasattr(f, 'seek'): + if hasattr('seekable') and not f.seakable(): + raise DeepomaticException("{}: not seakable".format(error) f.seek(0) - elif not isinstance(f, (string_types, bytes, bytearray)): - raise DeepomaticException("Unsupported file object type '{}' for key '{}'".format(type(f), key)) + continue + + raise DeepomaticException("{}: not a scalar or seakable.".format(error) return requests_callable(*args, files=files, timeout=requests_timeout, From 4044a087fd0aa0ed46352fa762d586a44a1328cb Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Tue, 22 Dec 2020 18:01:23 +0100 Subject: [PATCH 10/13] fix syntax --- deepomatic/api/http_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index 505c9f2..0c4dcd8 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -225,11 +225,11 @@ def send_request(self, requests_callable, *args, **kwargs): # seek files before each retry, to avoid silently retrying with different input if hasattr(f, 'seek'): if hasattr('seekable') and not f.seakable(): - raise DeepomaticException("{}: not seakable".format(error) + raise DeepomaticException("{}: not seakable".format(error)) f.seek(0) continue - raise DeepomaticException("{}: not a scalar or seakable.".format(error) + raise DeepomaticException("{}: not a scalar or seakable.".format(error)) return requests_callable(*args, files=files, timeout=requests_timeout, From 821e395a5e05ed93d5d7889f171451b525cc09ee Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Tue, 22 Dec 2020 18:05:09 +0100 Subject: [PATCH 11/13] fix syntax --- deepomatic/api/http_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index 0c4dcd8..1420b4b 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -224,7 +224,7 @@ def send_request(self, requests_callable, *args, **kwargs): error = "Unsupported file object type '{}' for key '{}'".format(type(f), key) # seek files before each retry, to avoid silently retrying with different input if hasattr(f, 'seek'): - if hasattr('seekable') and not f.seakable(): + if hasattr(f, 'seekable') and not f.seakable(): raise DeepomaticException("{}: not seakable".format(error)) f.seek(0) continue From 61db7857026b9c0501c59a9d6589447d663dfcd6 Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Tue, 22 Dec 2020 18:08:05 +0100 Subject: [PATCH 12/13] fix syntax --- deepomatic/api/http_helper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index 1420b4b..0b270de 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -224,12 +224,12 @@ def send_request(self, requests_callable, *args, **kwargs): error = "Unsupported file object type '{}' for key '{}'".format(type(f), key) # seek files before each retry, to avoid silently retrying with different input if hasattr(f, 'seek'): - if hasattr(f, 'seekable') and not f.seakable(): - raise DeepomaticException("{}: not seakable".format(error)) + if hasattr(f, 'seekable') and not f.seekable(): + raise DeepomaticException("{}: not seekable".format(error)) f.seek(0) continue - raise DeepomaticException("{}: not a scalar or seakable.".format(error)) + raise DeepomaticException("{}: not a scalar or seekable.".format(error)) return requests_callable(*args, files=files, timeout=requests_timeout, From 0b1cd5c12f0acdb6e3bc74b3b81c0643a51d7d95 Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Tue, 22 Dec 2020 18:14:34 +0100 Subject: [PATCH 13/13] handling None --- deepomatic/api/http_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index 0b270de..48f31bf 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -219,7 +219,7 @@ def send_request(self, requests_callable, *args, **kwargs): # if so, the fileobj is in second position if isinstance(f, (tuple, list)): f = f[1] - if isinstance(f, (string_types, bytes, bytearray, int, float, bool)): + if f is None or isinstance(f, (string_types, bytes, bytearray, int, float, bool)): continue error = "Unsupported file object type '{}' for key '{}'".format(type(f), key) # seek files before each retry, to avoid silently retrying with different input