Merge pull request #145 from xtrojak/142-pubchem-blocking-time

PubChem blocking time
RECETOX · May 11, 2023 · 0dccbc9 · 0dccbc9
2 parents 380f480 + eca2902
commit 0dccbc9
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 * general class Data for input handling [#141](https://github.com/RECETOX/MSMetaEnhancer/pull/141)
 * DataFrame class to read and handle tabular metadata input [#141](https://github.com/RECETOX/MSMetaEnhancer/pull/141)
+* implementation of blocking time in PubChem [#145](https://github.com/RECETOX/MSMetaEnhancer/pull/145)
 ### Changed
 * Spectra class is an instantiation of Data class [#141](https://github.com/RECETOX/MSMetaEnhancer/pull/141)
 * fix throttling freezing the app [#144](https://github.com/RECETOX/MSMetaEnhancer/pull/144)

diff --git a/MSMetaEnhancer/libs/converters/web/PubChem.py b/MSMetaEnhancer/libs/converters/web/PubChem.py
@@ -1,8 +1,9 @@
+import asyncio
 import json
-
-from MSMetaEnhancer.libs.converters.web.WebConverter import WebConverter
 from frozendict import frozendict
 
+from MSMetaEnhancer.libs.converters.web.WebConverter import WebConverter
+from MSMetaEnhancer.libs.utils.Generic import string_to_seconds
 from MSMetaEnhancer.libs.utils.Errors import UnknownResponse
 from MSMetaEnhancer.libs.utils.Throttler import Throttler
 
@@ -126,7 +127,9 @@ async def process_request(self, response, url, method):
         """
         result = await response.text()
         if 'X-Throttling-Control' in response.headers:
-            self.adjust_throttling(response.headers['X-Throttling-Control'])
+            sleep_time = self.adjust_throttling(response.headers['X-Throttling-Control'])
+            if sleep_time:
+                await asyncio.sleep(sleep_time)
         if response.ok:
             return result
         else:
@@ -156,15 +159,22 @@ def parse_pubchem_info(header):
             """
             indicators = header.split(',')
             blocked = False
+            sleep_time = 0
             if 'too many requests per second or blacklisted' in indicators[-1]:
                 blocked = True
-            return {'load': max([parse_status(indicator) for indicator in indicators[:3]]), 'blocked': blocked}
+            if 'Remaining blocking time' in indicators[-1]:
+                sleep_time = string_to_seconds(indicators[-1].split(': ')[1])
+                blocked = True
+            return {'load': max([parse_status(indicator) for indicator in indicators[:3]]), 
+                    'blocked': blocked,
+                    'sleep_time': sleep_time}
 
         status = parse_pubchem_info(throttling_header)
         if status['blocked'] or status['load'] > 75:
             self.throttler.decrease_limit()
         elif status['load'] < 25:
             self.throttler.increase_limit()
+        return status['sleep_time']
 
     def parse_attributes(self, response):
         """

diff --git a/MSMetaEnhancer/libs/utils/Generic.py b/MSMetaEnhancer/libs/utils/Generic.py
@@ -2,3 +2,10 @@ def escape_single_quotes(f):
     async def wrapper(self, arg):
         return await f(self, arg.replace("'", "\\'"))
     return wrapper
+
+
+def string_to_seconds(string):
+    """
+    Convert generic H:M:S string to seconds.
+    """
+    return sum(x * int(t) for x, t in zip([3600, 60, 1], string.split(":")))