feat: add support for plugins (#67)

* chore(plugins): update dependencies

* feat(plugins): add support for plugins

* chore: remove editable option for pip install

* ci: add update to plugins in automation

* fix: abstract duplicates

* refactor: abstract stat scim scoped for configAPI

* refactor: remove all duplicates

* fix(admin-ui): typo on authserver config

* refactor: rename variable to avoid potential issue

The issue is reported as bug by sonarcloud.io

* fix: don't append hostname if existent

Co-authored-by: moabu <47318409+moabu@users.noreply.github.com>

automation/auto_update_build_date.py

@@ -12,37 +12,47 @@ def as_date(text):
     return as_date(new_build) > as_date(last_build)
 
 
+def update_image(image, source_url_env, build_date_env):
+    dfparser = DockerfileParser(f'../{image}')
+    version = dfparser.labels["version"]
+    try:
+        base_url = os.path.dirname(dfparser.envs[source_url_env])
+        pkg_url = os.path.basename(dfparser.envs[source_url_env])
+    except KeyError:
+        print(f'Docker image {image} does not contain any packages to update')
+        return False
+    session = HTMLSession()
+    req = session.get(base_url)
+    if not req.ok:
+        return
+
+    new_build = req.html.xpath(
+        f"//a[contains(@href, '{pkg_url}')]/../following-sibling::td",
+        first=True,
+    ).text
+
+    if should_update_build(dfparser.envs[build_date_env], new_build):
+        print(f"Updating {image} {build_date_env} to {new_build}")
+        # update Dockerfile in-place
+        dfparser.envs[build_date_env] = new_build
+    else:
+        print(f"No updates found for {image} {build_date_env}")
+
+
 def main():
     docker_image_folders = [name for name in os.listdir("../") if
                             os.path.isdir(os.path.join("../", name)) and "docker-jans" in name]
 
     for image in docker_image_folders:
-        dfparser = DockerfileParser(f'../{image}')
-        version = dfparser.labels["version"]
         try:
-            base_url = os.path.dirname(dfparser.envs["CN_SOURCE_URL"])
-            pkg_url = os.path.basename(dfparser.envs["CN_SOURCE_URL"])
+            update_image(image, "CN_SOURCE_URL", "CN_BUILD_DATE")
+            if image == "docker-jans-config-api":
+                update_image(image, "SCIM_PLUGIN_SOURCE_URL", "SCIM_PLUGIN_BUILD_DATE")
+                update_image(image, "ADMIN_UI_SOURCE_URL", "ADMIN_UI_PLUGIN_BUILD_DATE")
         except KeyError:
             print(f'Docker image {image} does not contain any packages to update')
             continue
 
-        session = HTMLSession()
-        req = session.get(base_url)
-        if not req.ok:
-            return
-
-        new_build = req.html.xpath(
-            f"//a[contains(@href, '{pkg_url}')]/../following-sibling::td",
-            first=True,
-        ).text
-
-        if should_update_build(dfparser.envs["CN_BUILD_DATE"], new_build):
-            print(f"Updating {image} CN_BUILD_DATE to {new_build}")
-            # update Dockerfile in-place
-            dfparser.envs["CN_BUILD_DATE"] = new_build
-        else:
-            print(f"No updates found for {image} CN_BUILD_DATE")
-
 
 if __name__ == "__main__":
     main()

docker-jans-auth-server/Dockerfile

@@ -51,8 +51,8 @@ RUN wget -q https://github.com/fabioz/PyDev.Debugger/archive/refs/tags/pydev_deb
 # ===========
 
 ENV CN_VERSION=1.0.0-SNAPSHOT
-ENV CN_BUILD_DATE='2021-11-19 02:13'
-ENV CN_SOURCE_URL=https://maven.jans.io/maven/io/jans/jans-auth-server/${CN_VERSION}/jans-auth-server-${CN_VERSION}.war
+ENV CN_BUILD_DATE='2021-11-29 04:07'
+ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-auth-server/${CN_VERSION}/jans-auth-server-${CN_VERSION}.war
 
 # Install Jans Auth
 RUN wget -q ${CN_SOURCE_URL} -O /tmp/jans-auth.war \
@@ -79,7 +79,7 @@ RUN wget -q https://repo1.maven.org/maven2/org/jsmpp/jsmpp/${JSMPP_VERSION}/jsmp
 
 COPY requirements.txt /app/requirements.txt
 RUN pip3 install -U pip \
-    && pip3 install --no-cache-dir -r /app/requirements.txt \
+    && pip3 install --no-cache-dir --default-timeout=300 -r /app/requirements.txt \
     && rm -rf /src/jans-pycloudlib/.git
 
 # =======
@@ -177,7 +177,8 @@ ENV CN_MAX_RAM_PERCENTAGE=75.0 \
     CN_SYNC_JKS_ENABLED=false \
     CN_SYNC_JKS_INTERVAL=30 \
     GOOGLE_PROJECT_ID="" \
-    GOOGLE_APPLICATION_CREDENTIALS=/etc/jans/conf/google-credentials.json
+    GOOGLE_APPLICATION_CREDENTIALS=/etc/jans/conf/google-credentials.json \
+    ADMIN_UI_JWKS=http://0.0.0.0:8080/jans-auth/restv1/jwks
 
 # ==========
 # misc stuff

docker-jans-auth-server/requirements.txt

@@ -1 +1 @@
--e git+https://github.com/JanssenProject/jans-pycloudlib@a6ce9a098be01b4edcb69fbeee7f0bf745130c44#egg=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@bd1a605999b2c560077cb19d7c80d93f6cba3f5d#egg=jans-pycloudlib&subdirectory=jans-pycloudlib

docker-jans-certmanager/Dockerfile

@@ -16,8 +16,8 @@ RUN apk update \
 
 # JAR files required to generate OpenID Connect keys
 ENV CN_VERSION=1.0.0-SNAPSHOT
-ENV CN_BUILD_DATE='2021-11-19 02:13'
-ENV CN_SOURCE_URL=https://maven.jans.io/maven/io/jans/jans-auth-client/${CN_VERSION}/jans-auth-client-${CN_VERSION}-jar-with-dependencies.jar
+ENV CN_BUILD_DATE='2021-11-29 04:02'
+ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-auth-client/${CN_VERSION}/jans-auth-client-${CN_VERSION}-jar-with-dependencies.jar
 
 RUN wget -q ${CN_SOURCE_URL} -P /app/javalibs/
 

docker-jans-certmanager/requirements.txt

@@ -1,2 +1,2 @@
 click==6.7
--e git+https://github.com/JanssenProject/jans-pycloudlib@a6ce9a098be01b4edcb69fbeee7f0bf745130c44#egg=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@bd1a605999b2c560077cb19d7c80d93f6cba3f5d#egg=jans-pycloudlib&subdirectory=jans-pycloudlib

docker-jans-client-api/Dockerfile

@@ -5,7 +5,7 @@ FROM alpine:3.13.6
 # ===============
 
 RUN apk update \
-    && apk add --no-cache openssl py3-pip tini curl openjdk11-jre-headless py3-cryptography py3-grpcio \
+    && apk add --no-cache openssl py3-pip tini curl openjdk11-jre-headless py3-cryptography py3-grpcio py3-psycopg2 \
     && apk add --no-cache --virtual build-deps unzip wget git \
     && mkdir -p /usr/java/latest \
     && ln -sf /usr/lib/jvm/default-jvm/jre /usr/java/latest/jre
@@ -16,7 +16,7 @@ RUN apk update \
 
 ENV CN_VERSION=1.0.0-SNAPSHOT
 ENV CN_BUILD_DATE='2021-10-08 06:18'
-ENV CN_SOURCE_URL=https://maven.jans.io/maven/io/jans/jans-client-api-server/${CN_VERSION}/jans-client-api-server-${CN_VERSION}-distribution.zip
+ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-client-api-server/${CN_VERSION}/jans-client-api-server-${CN_VERSION}-distribution.zip
 
 RUN wget -q ${CN_SOURCE_URL} -O /tmp/client-api.zip \
     && mkdir -p /opt/client-api \

docker-jans-client-api/requirements.txt

@@ -1,2 +1,2 @@
 ruamel.yaml==0.16.10
--e git+https://github.com/JanssenProject/jans-pycloudlib@132da3fcc2bbea66a4bd2a7c4bcb7a7b7a1e87e4#egg=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@bd1a605999b2c560077cb19d7c80d93f6cba3f5d#egg=jans-pycloudlib&subdirectory=jans-pycloudlib

docker-jans-config-api/.dockerignore

@@ -5,7 +5,7 @@
 # !certs
 !conf
 !jetty
-# !libs
+!plugins
 !scripts
 !LICENSE
 !requirements.txt

docker-jans-config-api/Dockerfile

@@ -31,8 +31,8 @@ RUN wget -q https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-home/${JETTY_
 # ==========
 
 ENV CN_VERSION=1.0.0-SNAPSHOT
-ENV CN_BUILD_DATE='2021-11-19 02:42'
-ENV CN_SOURCE_URL=https://maven.jans.io/maven/io/jans/jans-config-api-server/${CN_VERSION}/jans-config-api-server-${CN_VERSION}.war
+ENV CN_BUILD_DATE='2021-11-29 04:10'
+ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-config-api-server/${CN_VERSION}/jans-config-api-server-${CN_VERSION}.war
 
 # Install Jans Config API
 RUN wget -q ${CN_SOURCE_URL} -O /tmp/jans-config-api.war \
@@ -45,13 +45,27 @@ RUN wget -q ${CN_SOURCE_URL} -O /tmp/jans-config-api.war \
 # Ports exposed by config-api
 EXPOSE 8074
 
+# =======
+# Plugins
+# =======
+
+RUN mkdir -p /usr/share/java
+
+ENV SCIM_PLUGIN_BUILD_DATE='2021-11-29 04:11'
+ENV SCIM_PLUGIN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/scim-plugin/${CN_VERSION}/scim-plugin-${CN_VERSION}-distribution.jar
+RUN wget -q ${SCIM_PLUGIN_SOURCE_URL} -O /usr/share/java/scim-plugin.jar
+
+ENV ADMIN_UI_PLUGIN_BUILD_DATE='2021-11-29 04:10'
+ENV ADMIN_UI_SOURCE_URL=https://maven.jans.io/maven/io/jans/admin-ui-plugin/${CN_VERSION}/admin-ui-plugin-${CN_VERSION}-distribution.jar
+RUN wget -q ${ADMIN_UI_SOURCE_URL} -O /usr/share/java/admin-ui-plugin.jar
+
 # ======
 # Python
 # ======
 
 COPY requirements.txt /app/requirements.txt
 RUN pip3 install -U pip \
-    && pip3 install --no-cache-dir -r /app/requirements.txt \
+    && pip3 install --no-cache-dir --default-timeout=300 -r /app/requirements.txt \
     && rm -rf /src/jans-pycloudlib/.git
 
 # =======
@@ -152,16 +166,17 @@ LABEL name="Config API" \
     maintainer="Janssen Project <support@jans.io>" \
     vendor="Janssen" \
     version="1.0.0" \
-    release="b12" \
+    release="b11" \
     summary="Janssen Config API" \
     description=""
 
-RUN mkdir -p /etc/certs /app/templates/ /deploy /etc/jans/conf
+RUN mkdir -p /etc/certs /app/templates/ /deploy /etc/jans/conf ${JETTY_BASE}/jans-config-api/custom/libs ${JETTY_BASE}/jans-config-api/custom/config
 RUN touch /etc/hosts.back
 COPY jetty/jans-config-api.xml ${JETTY_BASE}/jans-config-api/webapps/
 COPY jetty/jetty-env.xml ${JETTY_BASE}/jans-config-api/webapps/jans-config-api/WEB-INF/
 COPY jetty/log4j2.xml ${JETTY_BASE}/jans-config-api/resources/
 COPY conf/*.tmpl /app/templates/
+COPY plugins /app/plugins
 COPY scripts /app/scripts
 RUN chmod +x /app/scripts/entrypoint.sh
 

docker-jans-config-api/README.md

@@ -70,6 +70,14 @@ The following environment variables are supported by the container:
 - `CN_GOOGLE_SPANNER_INSTANCE_ID`: Google Spanner instance ID.
 - `CN_GOOGLE_SPANNER_DATABASE_ID`: Google Spanner database ID.
 - `CN_CONFIG_API_APP_LOGGERS`: Custom logging configuration in JSON-string format with hash type (see [Configure app loggers](#configure-app-loggers) section for details).
+- `CN_CONFIG_API_PLUGINS`: Comma-separated plugin names that should be enabled (available plugins are `admin-ui` and `scim`).
+- `CN_TOKEN_SERVER_BASE_HOSTNAM`: Hostname of token server (default to `localhost`).
+- `CN_TOKEN_SERVER_AUTHZ_ENDPOINT`: Authorization endpoint at token server (default to `/jans-auth/authorize.htm`).
+- `CN_TOKEN_SERVER_TOKEN_ENDPOINT`: Token endpoint at token server (default to `/jans-auth/restv1/token`).
+- `CN_TOKEN_SERVER_INTROSPECTION_ENDPOINT`: Introspection endpoint at token server (default to `/jans-auth/restv1/introspection`).
+- `CN_TOKEN_SERVER_USERINFO_ENDPOINT`: User info endpoint at token server (default to `/jans-auth/restv1/userinfo`).
+- `CN_TOKEN_SERVER_CLIENT_ID`: Client ID registered at token server.
+- `CN_TOKEN_SERVER_CERT_FILE`: Path to token server certificate (default to `/etc/certs/token_server.crt`).
 
 ### Configure app loggers
 

docker-jans-config-api/plugins/admin-ui/auiConfiguration.properties.tmpl

@@ -0,0 +1,32 @@
+# auth server
+authserver.clientId=%(admin_ui_client_id)s
+authserver.clientSecret=%(admin_ui_client_pw)s
+authserver.authzBaseUrl=https://%(hostname)s/jans-auth/authorize.htm
+authserver.scope=openid+profile+email+user_name
+authserver.redirectUrl=https://%(hostname)s/admin
+authserver.frontChannelLogoutUrl=https://%(hostname)s/admin/logout
+authserver.postLogoutRedirectUri=https://%(hostname)s/admin
+authserver.tokenEndpoint=https://%(hostname)s/jans-auth/restv1/token
+authserver.introspectionEndpoint=https://%(hostname)s/jans-auth/restv1/introspection
+authserver.userInfoEndpoint=https://%(hostname)s/jans-auth/restv1/userinfo
+authserver.healthCheckUrl=https://%(hostname)s/jans-auth/sys/health-check
+authserver.endSessionEndpoint=https://%(hostname)s/jans-auth/restv1/end_session
+
+# token server
+tokenServer.clientId=%(token_server_admin_ui_client_id)s
+tokenServer.clientSecret=%(token_server_admin_ui_client_pw)s
+tokenServer.authzBaseUrl=%(token_server_authz_url)s
+tokenServer.scope=openid+profile+email+user_name
+tokenServer.redirectUrl=https://%(hostname)s/admin
+tokenServer.logoutUrl=https://%(hostname)s/admin/logout
+tokenServer.tokenEndpoint=%(token_server_token_url)s
+tokenServer.introspectionEndpoint=%(token_server_introspection_url)s
+tokenServer.userInfoEndpoint=%(token_server_userinfo_url)s
+
+# licenseSpring details
+licenseSpring.apiKey=%(admin_ui_license_api_key)s
+licenseSpring.productCode=%(admin_ui_license_product_code)s
+licenseSpring.sharedKey=%(admin_ui_license_shared_key)s
+licenseSpring.managementKey=%(admin_ui_license_management_key)s
+# TODO: set to true for production mode
+licenseSpring.enabled=false

docker-jans-config-api/requirements.txt

@@ -1 +1 @@
--e git+https://github.com/JanssenProject/jans-pycloudlib@a6ce9a098be01b4edcb69fbeee7f0bf745130c44#egg=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@bd1a605999b2c560077cb19d7c80d93f6cba3f5d#egg=jans-pycloudlib&subdirectory=jans-pycloudlib

docker-jans-config-api/scripts/bootstrap.py

@@ -15,37 +15,15 @@
 from jans.pycloudlib.persistence import render_sql_properties
 from jans.pycloudlib.persistence import render_spanner_properties
 from jans.pycloudlib.utils import cert_to_truststore
-from jans.pycloudlib.utils import safe_render
 
-from utils import get_injected_urls
 from settings import LOGGING_CONFIG
+from plugins import AdminUiPlugin
+from plugins import discover_plugins
 
 logging.config.dictConfig(LOGGING_CONFIG)
 logger = logging.getLogger("entrypoint")
 
 
-def render_app_properties(manager):
-    client_id = manager.config.get("jca_client_id")
-    client_encoded_pw = manager.secret.get("jca_client_encoded_pw")
-    hostname = manager.config.get("hostname")
-
-    approved_issuer = os.environ.get("CN_APPROVED_ISSUER") or f"https://{hostname}"
-
-    ctx = {
-        "jca_log_level": os.environ.get("CN_CONFIG_API_LOG_LEVEL", "INFO"),
-        "jca_client_id": client_id,
-        "jca_client_encoded_pw": client_encoded_pw,
-        "approved_issuer": approved_issuer,
-    }
-    ctx.update(get_injected_urls())
-
-    with open("/app/templates/application.properties.tmpl") as f:
-        txt = safe_render(f.read(), ctx)
-
-    with open("/opt/jans/jetty/jans-config-api/webapps/jans-config-api/WEB-INF/classes/application.properties", "w") as f:
-        f.write(txt)
-
-
 def main():
     manager = get_manager()
     persistence_type = os.environ.get("CN_PERSISTENCE_TYPE", "ldap")
@@ -101,13 +79,18 @@ def main():
         "changeit",
     )
 
-    # render_app_properties(manager)
-
     modify_jetty_xml()
     modify_webdefault_xml()
     modify_server_ini()
     configure_logging()
 
+    plugins = discover_plugins()
+    modify_config_api_xml(plugins)
+
+    if "admin-ui" in plugins:
+        admin_ui_plugin = AdminUiPlugin(manager)
+        admin_ui_plugin.setup()
+
 
 def modify_jetty_xml():
     fn = "/opt/jetty/etc/jetty.xml"
@@ -209,5 +192,19 @@ def configure_logging():
         f.write(tmpl.safe_substitute(config))
 
 
+def modify_config_api_xml(plugins=None):
+    plugins = plugins or []
+    fn = "/opt/jans/jetty/jans-config-api/webapps/jans-config-api.xml"
+
+    with open(fn) as f:
+        txt = f.read()
+
+    with open(fn, "w") as f:
+        ctx = {
+            "extra_classpath": ",".join([f"./custom/libs/{plugin}-plugin.jar" for plugin in plugins])
+        }
+        f.write(txt % ctx)
+
+
 if __name__ == "__main__":
     main()

docker-jans-config-api/scripts/entrypoint.sh

@@ -2,15 +2,25 @@
 
 set -e
 
+copy_builtin_plugins() {
+    if [ ! -f /opt/jans/jetty/jans-config-api/custom/libs/scim-plugin.jar ]; then
+        cp /usr/share/java/scim-plugin.jar /opt/jans/jetty/jans-config-api/custom/libs/
+    fi
+
+    if [ ! -f /opt/jans/jetty/jans-config-api/custom/libs/admin-ui-plugin.jar ]; then
+        cp /usr/share/java/admin-ui-plugin.jar /opt/jans/jetty/jans-config-api/custom/libs/
+    fi
+}
+
 python3 /app/scripts/wait.py
 
+copy_builtin_plugins
+
 if [ ! -f /deploy/touched  ]; then
     python3 /app/scripts/bootstrap.py
     touch /deploy/touched
 fi
 
-python3 /app/scripts/mod_context.py
-
 # run config-api
 mkdir -p /opt/jetty/temp
 cd /opt/jans/jetty/jans-config-api