[Marketplace] Simplify marketplace setting

contrib/kubespray/quick-start/services-configuration.yaml.template

@@ -7,6 +7,7 @@ cluster:
     data-path: "/datastorage"
     qos-switch: "{{ env["cfg"]["qos-switch"] | default('false') }}"
     docker-data-root: "{{ env['cfg']['docker_data_root'] | default('/mnt/docker') }}"
+    marketplace: "true"
 
   # the docker registry to store docker images that contain system services like frameworklauncher, hadoop, etc.
   docker-registry:
@@ -205,15 +206,15 @@ authentication:
 #         alertname: PAIJobGpuPercentLowerThan0_3For1h
 #   customized-receivers: # receivers are combination of several actions
 #   - name: "pai-email-admin-user-and-stop-job"
-#     actions: 
+#     actions:
 #       # the email template for `email-admin` and `email-user `can be chosen from ['general-template', 'kill-low-efficiency-job-alert']
 #       # if no template specified, 'general-template' will be used.
 #       email-admin:
-#       email-user:  
+#       email-user:
 #         template: 'kill-low-efficiency-job-alert'
 #       stop-jobs: # no parameters required for stop-jobs action
 #       tag-jobs:
-#         tags: 
+#         tags:
 #         - 'stopped-by-alert-manager'
 
 # uncomment following if you want to customize prometheus

deployment/quick-start/services-configuration.yaml.template

@@ -23,6 +23,7 @@ cluster:
   #  cluster-type: k8s
   #  # HDFS, zookeeper data path on your cluster machine.
   #  data-path: "/datastorage"
+  #  marketplace: "true"
 
   # the docker registry to store docker images that contain system services like frameworklauncher, hadoop, etc.
   docker-registry:
@@ -90,15 +91,15 @@ rest-server:
 #         alertname: PAIJobGpuPercentLowerThan0_3For1h
 #   customized-receivers: # receivers are combination of several actions
 #   - name: "pai-email-admin-user-and-stop-job"
-#     actions: 
+#     actions:
 #       # the email template for `email-admin` and `email-user `can be chosen from ['general-template', 'kill-low-efficiency-job-alert']
 #       # if no template specified, 'general-template' will be used.
 #       email-admin:
-#       email-user:  
+#       email-user:
 #         template: 'kill-low-efficiency-job-alert'
 #       stop-jobs: # no parameters required for stop-jobs action
 #       tag-jobs:
-#         tags: 
+#         tags:
 #         - 'stopped-by-alert-manager'
 
 # uncomment following if you want to customize prometheus

src/marketplace-restserver/config/marketplace-restserver.yaml

@@ -3,16 +3,15 @@
 
 service_type: "k8s"
 
-db_user: user
-db_password: passwd
-db: marketplace
+# db_user: user
+# db_password: passwd
+# db: marketplace
 # db_host: postgres
-db_port: 9291
+# db_port: 9291
 server-port: 9292
 
 azure_storage:
   type: blob
   storage_account: openpaimarketplace
-  connection_strings:
-  - '<connection_string>'
+  connection_strings: []
 azure_storage_json: ''

src/marketplace-restserver/deploy/marketplace-restserver.yaml.template

@@ -21,16 +21,32 @@ spec:
         image: {{ cluster_cfg["cluster"]["docker-registry"]["prefix"] }}marketplace-restserver:{{ cluster_cfg["cluster"]["docker-registry"]["tag"] }}
         imagePullPolicy: Always
         env:
+        - name: DB_HOST
+          value: {{ cluster_cfg["marketplace-restserver"]["db_host"] }}
         - name: DB_USERNAME
+{% if 'db_user' in cluster_cfg["marketplace-restserver"] %}
           value: {{ cluster_cfg["marketplace-restserver"]["db_user"] }}
+{% else %}
+          value: {{ cluster_cfg["marketplace-db"]["user"] }}
+{% endif %}
         - name: DB_PASSWORD
+{% if 'db_password' in cluster_cfg["marketplace-restserver"] %}
           value: {{ cluster_cfg["marketplace-restserver"]["db_password"] }}
+{% else %}
+          value: {{ cluster_cfg["marketplace-db"]["passwd"] }}
+{% endif %}
         - name: DATABASE
+{% if 'db' in cluster_cfg["marketplace-restserver"] %}
           value: {{ cluster_cfg["marketplace-restserver"]["db"] }}
-        - name: DB_HOST
-          value: {{ cluster_cfg["marketplace-restserver"]["db_host"] }}
+{% else %}
+          value: {{ cluster_cfg["marketplace-db"]["db"] }}
+{% endif %}
         - name: DB_PORT
-          value: "{{ cluster_cfg["marketplace-restserver"]["db_port"] }}"
+{% if 'db_port' in cluster_cfg["marketplace-restserver"] %}
+          value: '{{ cluster_cfg["marketplace-restserver"]["db_port"] }}'
+{% else %}
+          value: '{{ cluster_cfg["marketplace-db"]["port"] }}'
+{% endif %}
         - name: NODE_ENV
           value: production
         - name: PORT

src/marketplace-restserver/deploy/service.yaml

@@ -6,6 +6,7 @@ cluster-type:
 
 prerequisite:
   - cluster-configuration
+  - marketplace-db
 
 template-list:
   - marketplace-restserver.yaml

src/marketplace-webportal/config/marketplace_webportal.py

@@ -21,8 +21,6 @@ def run(self):
         api_port = self.service_conf['api-port']
         master_ip = [host['hostip'] for host in machine_list if host.get('pai-master') == 'true'][0]
         result['uri'] = 'http://{0}:{1}'.format(master_ip, server_port)
-        if 'marketplace_api_uri' not in result:
-            result['marketplace_api_uri'] = 'http://{0}:{1}/api'.format(master_ip, api_port)
         return result
 
     def validation_post(self, conf):

src/marketplace-webportal/deploy/marketplace-webportal.yaml.template

@@ -22,7 +22,15 @@ spec:
         imagePullPolicy: Always
         env:
         - name: MARKETPLACE_API_URL
+{%- if 'marketplace_api_uri' in cluster_cfg['marketplace-webportal'] %}
           value: {{ cluster_cfg["marketplace-webportal"]["marketplace_api_uri"] }}
+{%- else %}
+{% if "ssl" in cluster_cfg["pylon"] and cluster_cfg["pylon"]["ssl"] %}
+          value: "{{ cluster_cfg['pylon']['uri-https']}}/marketplace/api"
+{% else %}
+          value: "{{ cluster_cfg['pylon']['uri']}}/marketplace/api"
+{% endif %}
+{%- endif %}
         - name: SERVER_PORT
           value: "{{ cluster_cfg["marketplace-webportal"]["server-port"] }}"
         - name: NPM_INSTALL_TOKEN

src/marketplace-webportal/deploy/service.yaml

@@ -6,6 +6,7 @@ cluster-type:
 
 prerequisite:
   - cluster-configuration
+  - marketplace-restserver
 
 template-list:
   - marketplace-webportal.yaml

src/webportal/config/webportal.py

@@ -61,7 +61,7 @@ def apply_config(plugin):
         return {
             'server-port': server_port,
             'uri': uri,
-            'plugins': json.dumps([apply_config(plugin) for plugin in plugins]),
+            'plugins': [apply_config(plugin) for plugin in plugins],
             'webportal-address': master_ip,
             'enable-job-transfer': self.service_configuration['enable-job-transfer'],
         }

src/webportal/deploy/webportal.yaml.template

@@ -91,7 +91,24 @@ spec:
         - name: WEBPORTAL_PLUGINS
           # A raw JSON formatted value is required here.
           value: |
-            {{ cluster_cfg['webportal']['plugins'] }}
+            [
+{% if cluster_cfg['cluster']['common']['marketplace'] == "true" %}
+              {
+                "id": "marketplace",
+                "title": "Marketplace",
+{% if "ssl" in cluster_cfg["pylon"] and cluster_cfg["pylon"]["ssl"] %}
+                "uri": "{{ cluster_cfg['pylon']['uri-https']}}/marketplace/plugin.js"
+{% else %}
+                "uri": "{{ cluster_cfg['pylon']['uri']}}/marketplace/plugin.js"
+{% endif %}
+              },
+{% endif %}
+{% if cluster_cfg['webportal']['plugins']|length > 0 %}
+{% for item in cluster_cfg['webportal']['plugins'] %}
+                {{item}},
+{% endfor %}
+{% endif %}
+            ]
         ports:
         - name: webportal
           containerPort: 8080