feat(connector): simplify generator, add connect

dataprep/connector/__init__.py

@@ -1,6 +1,57 @@
-"""
-DataConnector
-"""
+"""Connector"""
+
+from typing import Any, Dict, Optional
+
 from .connector import Connector
+from .generator import ConfigGenerator, ConfigGeneratorUI
+
+__all__ = ["Connector", "ConfigGenerator", "ConfigGeneratorUI", "connect"]
+
+
+def connect(
+    config_path: str,
+    *,
+    update: bool = False,
+    _auth: Optional[Dict[str, Any]] = None,
+    _concurrency: int = 1,
+    **kwargs: Any,
+) -> Connector:
+    """Connect to a website.
+
+    Parameters
+    ----------
+    config_path
+        The path to the config. It can be hosted, e.g. "yelp", or from
+        local filesystem, e.g. "./yelp"
+    _auth: Optional[Dict[str, Any]] = None
+        The parameters for authentication, e.g. OAuth2
+    _concurrency: int = 5
+        The concurrency setting. By default it is 1 reqs/sec.
+    update: bool = True
+        Force update the config file even if the local version exists.
+    **kwargs
+        Parameters that shared by different queries.
+
+    Returns
+    -------
+    Connector
+        a Connector object.
+
+    Example
+    -------
+    >>> from dataprep.connector import connect
+    >>> dc = connect("yelp", _auth={"access_token": access_token}, _concurrency=3)
+    """
+    return Connector(config_path, update=update, _auth=_auth, _concurrency=_concurrency, **kwargs)
+
+
+def config_generator_ui(existing: Optional[Dict[str, Any]] = None) -> None:
+    """Create a Config Generator UI.
+
+    Parameters
+    ----------
+    existing: Optional[Dict[str, Any]] = None
+        Optionally pass in an existing configuration.
+    """
 
-__all__ = ["Connector"]
+    ConfigGeneratorUI(existing).display()

dataprep/connector/generator/generator.py

@@ -2,7 +2,7 @@
 
 from pathlib import Path
 from typing import Any, Dict, Optional, Union
-from urllib.parse import parse_qs, urlparse, urlunparse
+from urllib.parse import parse_qs, urlparse
 
 import requests
 from dataprep.connector.schema.base import BaseDef
@@ -39,7 +39,9 @@ def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
             self.config = ConfigState(ConfigDef(**config))
         self.storage = {}
 
-    def add_example(self, example: Dict[str, Any]) -> None:  # pylint: disable=too-many-locals
+    def add_example(  # pylint: disable=too-many-locals
+        self, example: Dict[str, Any], table_path: Optional[str] = None
+    ) -> None:  # pylint: disable=too-many-locals
         """Add an example to the generator. The example
         should be in the dictionary format.
 
@@ -65,7 +67,8 @@ class Example(TypedDict):
 
         params = example.get("params", {})
 
-        # Move url params to params
+        # Do sanity check on url. For all the parameters that already in the URL we keep them.
+        # For all the parameters that is not in the url we make it as free variables.
         parsed = urlparse(url)
 
         query_string = parse_qs(parsed.query)
@@ -74,9 +77,9 @@ class Example(TypedDict):
                 raise ValueError(
                     f"{key} appears in both url and params, but have different values."
                 )
-            params[key] = val
+            # params[key] = val
 
-        url = urlunparse((*parsed[:4], "", *parsed[5:]))
+        # url = urlunparse((*parsed[:4], "", *parsed[5:]))
         req = {
             "method": method,
             "url": url,
@@ -95,7 +98,7 @@ class Example(TypedDict):
             authdef.build(req, authparams, self.storage)
 
         # Send out request and construct config
-        config = _create_config(req)
+        config = _create_config(req, table_path)
 
         # Add pagination information into the config
         pagination = example.get("pagination")
@@ -125,7 +128,7 @@ def save(self, path: Union[str, Path]) -> None:
             f.write(self.to_string())
 
 
-def _create_config(req: Dict[str, Any]) -> ConfigDef:
+def _create_config(req: Dict[str, Any], table_path: Optional[str] = None) -> ConfigDef:
     resp = requests.request(
         req["method"].lower(),
         req["url"],
@@ -139,7 +142,8 @@ def _create_config(req: Dict[str, Any]) -> ConfigDef:
         )
     payload = resp.json()
 
-    table_path = search_table_path(payload)
+    if table_path is None:
+        table_path = search_table_path(payload)
 
     ret: Dict[str, Any] = {
         "version": 1,