feat(connector): allow parameters appear in url path

dataprep/connector/connector.py

@@ -14,10 +14,10 @@
 from jinja2 import Environment, StrictUndefined, UndefinedError
 from jsonpath_ng import parse as jparse
 
+from .config_manager import initialize_path
 from .errors import InvalidParameterError, RequestError, UniversalParameterOverridden
 from .implicit_database import ImplicitDatabase, ImplicitTable
 from .info import info
-from .config_manager import initialize_path
 from .ref import Ref
 from .schema import (
     FieldDef,
@@ -127,6 +127,8 @@ async def query(  # pylint: disable=too-many-locals
             else:
                 allowed_params.add(key)
 
+        allowed_params.update(self._impdb.tables[table].config.request.url_path_params())
+
         for key in where:
             if key not in allowed_params:
                 raise InvalidParameterError(key)
@@ -299,7 +301,7 @@ async def _fetch(  # pylint: disable=too-many-locals,too-many-branches,too-many-
 
         reqdef = table.config.request
         method = reqdef.method
-        url = reqdef.url
+
         req_data: Dict[str, Dict[str, Any]] = {
             "headers": {},
             "params": {},
@@ -364,6 +366,7 @@ async def _fetch(  # pylint: disable=too-many-locals,too-many-branches,too-many-
                     self._jenv,
                     merged_vars,
                 )
+
                 for ikey in instantiated_fields:
                     if ikey in req_data[key]:
                         warn(
@@ -378,6 +381,8 @@ async def _fetch(  # pylint: disable=too-many-locals,too-many-branches,too-many-
             if field_def is not None:
                 validate_fields(field_def, req_data[key])
 
+        url = reqdef.populate_url(merged_vars)
+
         await _throttler.acquire(_page)
 
         if _allowed_page is not None and int(_allowed_page) <= _page:

dataprep/connector/schema/defs.py

@@ -2,6 +2,7 @@
 import http.server
 import json
 import random
+import re
 import socket
 import socketserver
 import string
@@ -356,8 +357,14 @@ class SearchDef(BaseDef):
     key: str
 
 
-class RequestDef(BaseDef):
+class UrlDef(BaseDef):
     url: str
+    parse: bool = True
+    skip: Set[str] = Field(default_factory=set)
+
+
+class RequestDef(BaseDef):
+    url: Union[str, UrlDef]
     method: Method
 
     headers: Optional[Dict[str, FieldDefUnion]]
@@ -369,6 +376,38 @@ class RequestDef(BaseDef):
     pagination: Optional[PaginationDef]
     search: Optional[SearchDef]
 
+    def url_path_params(self) -> Set[str]:
+        if isinstance(self.url, str):
+            url = self.url
+            skip: Set[str] = set()
+        else:
+            if not self.url.parse:
+                return set()
+            url = self.url.url
+            skip = self.url.skip
+
+        ret: Set[str] = set()
+
+        for match in re.finditer(r"\{(?P<param>.*?)\}", url):
+            param = match.group("param")
+            if param:
+                ret.add(param)
+
+        return ret - skip
+
+    def populate_url(self, params: Dict[str, str]) -> str:
+        if isinstance(self.url, str):
+            url = self.url
+        else:
+            url = self.url.url
+            if not self.url.parse:
+                return url
+
+        for key, value in params.items():
+            url = url.replace("{" + key + "}", value)
+
+        return url
+
 
 class SchemaFieldDef(BaseDef):
     target: str