Improvements and bugfixes to globus_auth_utils, particularly native a…

…pp login.

Additional core_utils functions and modifications to better support oauth2 token credential handling.
Various related fixes to download/backup/restore CLI/API modules.

deriva/core/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.8.6"
+__version__ = "0.8.7"
 
 from deriva.core.utils.core_utils import *
 from deriva.core.base_cli import BaseCLI, KeyValuePairArgs

deriva/core/deriva_binding.py

@@ -216,8 +216,8 @@ def set_credentials(self, credentials, server):
             self._session.cookies.set(cname, cval, domain=server, path='/')
         elif credentials and ('username' in credentials and 'password' in credentials):
             self.post_authn_session(credentials)
-        elif credentials and ('bearer-token' in credentials):
-            self._session.headers.update({'Authorization' : 'Bearer {token}'.format(token=credentials['bearer-token'])})
+        if credentials and ('bearer-token' in credentials):
+            self._session.headers.update({'Authorization': 'Bearer {token}'.format(token=credentials['bearer-token'])})
 
     def get_authn_session(self):
         headers = { 'deriva-client-context': self.dcctx.encoded() }

deriva/core/utils/core_utils.py

@@ -34,6 +34,7 @@
 DEFAULT_HEADERS = {}
 DEFAULT_CONFIG_PATH = os.path.join(os.path.expanduser('~'), '.deriva')
 DEFAULT_CREDENTIAL_FILE = os.path.join(DEFAULT_CONFIG_PATH, 'credential.json')
+DEFAULT_GLOBUS_CREDENTIAL_FILE = os.path.join(DEFAULT_CONFIG_PATH, 'globus-credential.json')
 DEFAULT_CONFIG_FILE = os.path.join(DEFAULT_CONFIG_PATH, 'config.json')
 DEFAULT_COOKIE_JAR_FILE = os.path.join(DEFAULT_CONFIG_PATH, 'cookies.txt')
 DEFAULT_SESSION_CONFIG = {
@@ -43,6 +44,7 @@
     "retry_status_forcelist": [500, 503, 504],
     "cookie_jar": DEFAULT_COOKIE_JAR_FILE
 }
+OAUTH2_SCOPES_KEY = "oauth2_scopes"
 DEFAULT_CONFIG = {
     "server":
     {
@@ -51,7 +53,8 @@
         "catalog_id": 1
     },
     "session": DEFAULT_SESSION_CONFIG,
-    "download_processor_whitelist": []
+    "download_processor_whitelist": [],
+    OAUTH2_SCOPES_KEY: {}
 }
 DEFAULT_CREDENTIAL = {}
 
@@ -244,22 +247,88 @@ def read_credential(credential_file=DEFAULT_CREDENTIAL_FILE, create_default=Fals
     return json.loads(credential, object_pairs_hook=OrderedDict)
 
 
-def get_credential(host, credential_file=DEFAULT_CREDENTIAL_FILE):
-    if credential_file is None:
-        credential_file = DEFAULT_CREDENTIAL_FILE
-    credentials = read_credential(credential_file)
-    return credentials.get(host, credentials.get(host.lower()))
+def get_credential(host,
+                   credential_file=DEFAULT_CREDENTIAL_FILE,
+                   globus_credential_file=DEFAULT_GLOBUS_CREDENTIAL_FILE,
+                   config_file=DEFAULT_CONFIG_FILE,
+                   requested_scope=None,
+                   force_scope_lookup=False,
+                   match_scope_tag="deriva-all"):
+    # load webauthn credentials first
+    credentials = read_credential(credential_file or DEFAULT_CREDENTIAL_FILE, create_default=True)
+    creds = credentials.get(host, credentials.get(host.lower(), dict()))
+
+    # if present, load globus credentials and merge
+    globus_credentials = read_credential(globus_credential_file or DEFAULT_GLOBUS_CREDENTIAL_FILE,
+                                         create_default=True, default=dict())
+    if globus_credentials:
+        scopes = get_oauth_scopes_for_host(host, config_file, force_refresh=force_scope_lookup)
+        for resource, g_creds in globus_credentials.items():
+            # 1. look for the explicitly requested scope in the token store, if specified
+            if requested_scope is not None and g_creds["scope"] == requested_scope:
+                creds["bearer-token"] = g_creds["access_token"]
+                break
+
+            # 2. try to determine the scope to use based on host-to-scope(s) mappings in the config file
+            if scopes:
+                for k, v in scopes.items():
+                    if v == g_creds["scope"]:
+                        if match_scope_tag is not None and match_scope_tag != k:
+                            continue
+                        creds["bearer-token"] = g_creds["access_token"]
+                        break
+                if creds.get("bearer-token"):
+                    break
+            else:
+                # 3. if we did not find any host-to-scope(s) mappings in the config file, just fall back and
+                # try to find a token by matching the hostname to the resource server name
+                if host.lower() == resource.lower():
+                    creds["bearer-token"] = g_creds["access_token"]
+                    break
+
+    return creds or None
+
+
+def get_oauth_scopes_for_host(host, config_file=DEFAULT_CONFIG_FILE, scheme="https", force_refresh=False):
+    config = read_config(config_file or DEFAULT_CONFIG_FILE, create_default=True)
+    required_scopes = config.get(OAUTH2_SCOPES_KEY)
+    result = dict()
+    # determine the scope to use based on host-to-scope(s) mappings in the config file
+    if required_scopes:
+        for hostname, scopes in required_scopes.items():
+            if host.lower() == hostname.lower():
+                result = scopes
+                break
+    if not result or force_refresh:
+        session = get_new_requests_session(session_config=DEFAULT_SESSION_CONFIG)
+        url = "%s://%s/authn/discovery" % (scheme or "https", host)
+        try:
+            r = session.get(url, headers=DEFAULT_HEADERS)
+            r.raise_for_status()
+            result = r.json().get(OAUTH2_SCOPES_KEY)
+            if result:
+                config[OAUTH2_SCOPES_KEY][host] = result
+                write_config(config_file or DEFAULT_CONFIG_FILE, config=config)
+        except Exception as e:
+            logging.debug("Unable to discover and/or update the \"%s\" mappings from [%s]. As a result, access to "
+                          "this host may be limited even though you may already be in possession of a valid login "
+                          "token for the relevant scopes. %s" % (OAUTH2_SCOPES_KEY, url, format_exception(e)))
+    return result
 
 
 def format_credential(token=None, oauth2_token=None, username=None, password=None):
-    if token:
-        return {"cookie": "webauthn=%s" % token}
-    elif oauth2_token:
-        return {"bearer-token": "%s" % oauth2_token}
-    elif username and password:
+    if username and password:
         return {"username": username, "password": password}
-    raise ValueError(
-        "Missing required argument(s): a supported authentication token or a username and password must be provided.")
+    credential = dict()
+    if token:
+        credential.update({"cookie": "webauthn=%s" % token})
+    if oauth2_token:
+        credential.update({"bearer-token": "%s" % oauth2_token})
+    if not credential:
+        raise ValueError(
+            "Missing required argument(s): a supported authentication token or a username/password must be provided.")
+
+    return credential
 
 
 def bootstrap():