Bug 1584321 - Handle static client scopes internally

With this change, each service declares its scopes in
`services/<service>/scopes.yml`, and that data is gathered during
generation and placed in a convenient place for the auth service.  The
Auth service then interpolates those scopes into the configured
STATIC_CLIENTS, preserving the supplied accessToken and applying the
service's configured azure accountId.

changelog/bug1584321-b.md

@@ -0,0 +1,6 @@
+level: major
+reference: bug 1584321
+---
+Scopes for the Taskcluster services themselves are now handled internally to the platform, although access tokens must still be managed as part of the deployment process.
+When deploying this version, remove all `scopes` and `description` properties from `static/taskcluster/..` clients in the array in the Auth service's `STATIC_CLIENTS` configuration.
+See [the new docs on static clients](https://docs.taskcluster.net/docs/manual/deploying/static-clients) for more background on this setting.

infrastructure/tooling/src/dev/helm.js

@@ -22,6 +22,15 @@ const actions = [
       if (!config.meta || !config.meta.deploymentPrefix) {
         throw new Error('Must have configured dev-config.yml to deploy.');
       }
+
+      if (config.auth && config.auth.static_clients) {
+        if (config.auth.static_clients.some(({clientId, scopes}) => clientId.startsWith('static/taskcluster/') && scopes)) {
+          throw new Error('`static/taskcluster/..` clients in auth.static_clients in `dev-config.yml` should not contain scopes');
+        }
+        if (config.auth.static_clients.some(({clientId, scopes}) => !clientId.startsWith('static/taskcluster/') && !scopes)) {
+          throw new Error('non-taskcluster static clients in auth.static_clients in `dev-config.yml` should scopes');
+        }
+      }
       return {'dev-config': config};
     },
   },
@@ -58,7 +67,7 @@ const actions = [
       if (requirements['helm-version'] === 2) {
         command.push('-n');
       }
-      command = command.concat(['taskcluster', '-f', './dev-config.yml', 'infrastructure/k8s']);
+      command = command.concat(['taskcluster', '-f', 'dev-config.yml', 'infrastructure/k8s']);
       return {
         'target-templates': await execCommand({
           command,

infrastructure/tooling/src/generate/generators/k8s.js

@@ -4,7 +4,6 @@ const glob = require('glob');
 const util = require('util');
 const yaml = require('js-yaml');
 const jsone = require('json-e');
-const config = require('taskcluster-lib-config');
 const rimraf = util.promisify(require('rimraf'));
 const mkdirp = util.promisify(require('mkdirp'));
 const {listServices, writeRepoFile, readRepoYAML, writeRepoYAML, writeRepoJSON, REPO_ROOT, configToSchema, configToExample} = require('../../utils');
@@ -140,34 +139,16 @@ exports.tasks.push({
 });
 
 SERVICES.forEach(name => {
-  exports.tasks.push({
-    title: `Get config options for ${name}`,
-    requires: [],
-    provides: [`configs-${name}`],
-    run: async (requirements, utils) => {
-      const envVars = config({
-        files: [{
-          path: path.join(REPO_ROOT, 'services', name, 'config.yml'),
-          required: true,
-        }],
-        getEnvVars: true,
-      });
-      return {
-        [`configs-${name}`]: envVars,
-      };
-    },
-  });
   exports.tasks.push({
     title: `Generate helm templates for ${name}`,
-    requires: [`configs-${name}`, 'k8s-templates'],
-    provides: [`ingresses-${name}`, `procslist-${name}`],
+    requires: [`configs-${name}`, `procslist-${name}`, 'k8s-templates'],
+    provides: [`ingresses-${name}`],
     run: async (requirements, utils) => {
-      const procs = await readRepoYAML(path.join('services', name, 'procs.yml'));
+      const procs = requirements[`procslist-${name}`];
       const templates = requirements['k8s-templates'];
       const vars = requirements[`configs-${name}`].map(v => v.var);
       return {
         [`ingresses-${name}`]: await renderTemplates(name, vars, procs, templates),
-        [`procslist-${name}`]: procs,
       };
     },
   });
@@ -254,6 +235,7 @@ exports.tasks.push({
   requires: [
     ...SERVICES.map(name => `configs-${name}`),
     ...SERVICES.map(name => `procslist-${name}`),
+    'static-clients',
   ],
   provides: [],
   run: async (requirements, utils) => {
@@ -362,9 +344,6 @@ exports.tasks.push({
       ...cfg,
     })));
 
-    const staticClients = [];
-    const serviceScopes = await readRepoYAML(path.join('infrastructure', 'tooling', 'static-clients.yml'));
-
     configs.forEach(cfg => {
       const confName = cfg.name.replace(/-/g, '_');
       exampleConfig[confName] = {};
@@ -393,14 +372,6 @@ exports.tasks.push({
         additionalProperties: false,
       };
 
-      if (serviceScopes[cfg.name]) {
-        staticClients.push({
-          clientId: `static/taskcluster/${cfg.name}`,
-          description: `Autogenerated, do not edit. Client for ${cfg.name}`,
-          scopes: serviceScopes[cfg.name],
-        });
-      }
-
       // Some services actually duplicate their config env vars in multiple places
       // so we de-dupe first. We use the variable name for this. If they've asked
       // for the same variable twice with different types then this is not our fault
@@ -462,17 +433,9 @@ exports.tasks.push({
       });
     });
 
-    // Now push root as well
-    staticClients.push({
-      clientId: 'static/taskcluster/root',
-      description: `Autogenerated, do not edit. Client for administering the deployment.`,
-      scopes: ['*'],
-    });
-
-    exampleConfig.auth.static_clients = staticClients.map(c => {
-      c.accessToken = '...';
-      return c;
-    });
+    // omit scopes and add a placeholder accessToken to each client
+    exampleConfig.auth.static_clients = requirements['static-clients']
+      .map(({scopes, ...c}) => ({...c, accessToken: '...'}));
 
     await writeRepoJSON(path.join(CHART_DIR, 'values.schema.json'), schema);
     await writeRepoYAML(path.join(CHART_DIR, 'values.yaml'), valuesYAML); // helm requires this to be "yaml"

infrastructure/tooling/src/generate/generators/metadata.js

@@ -0,0 +1,43 @@
+const _ = require('lodash');
+const path = require('path');
+const config = require('taskcluster-lib-config');
+const {listServices, readRepoYAML, REPO_ROOT} = require('../../utils');
+
+// We're not going to deploy login into k8s
+const SERVICES = listServices().filter(s => !['login'].includes(s));
+
+exports.tasks = [];
+
+SERVICES.forEach(name => {
+  exports.tasks.push({
+    title: `Fetch service metadata for ${name}`,
+    requires: [],
+    provides: [`configs-${name}`, `procslist-${name}`, `scopes-${name}`],
+    run: async (requirements, utils) => {
+      const envVars = config({
+        files: [{
+          path: path.join(REPO_ROOT, 'services', name, 'config.yml'),
+          required: true,
+        }],
+        getEnvVars: true,
+      });
+
+      const procs = await readRepoYAML(path.join('services', name, 'procs.yml'));
+
+      const scopesPath = path.join('services', name, 'scopes.yml');
+      let scopes = null;
+      try {
+        scopes = await readRepoYAML(scopesPath);
+      } catch (err) {
+        if (err.code !== 'ENOENT') {
+          throw err;
+        }
+      }
+      return {
+        [`configs-${name}`]: envVars,
+        [`procslist-${name}`]: procs,
+        [`scopes-${name}`]: scopes,
+      };
+    },
+  });
+});

infrastructure/tooling/src/generate/generators/static-clients.js

@@ -0,0 +1,46 @@
+const _ = require('lodash');
+const {listServices, writeRepoJSON} = require('../../utils');
+
+// We're not going to deploy login into k8s
+const SERVICES = listServices().filter(s => !['login'].includes(s));
+
+exports.tasks = [];
+
+exports.tasks.push({
+  title: 'Assemble static clients',
+  requires: [
+    ...SERVICES.map(name => `scopes-${name}`),
+  ],
+  provides: ['static-clients'],
+  run: async (requirements, utils) => {
+    const staticClients = [];
+    SERVICES.forEach(name => {
+      const scopes = requirements[`scopes-${name}`];
+      if (scopes) {
+        staticClients.push({
+          clientId: `static/taskcluster/${name}`,
+          scopes: scopes,
+        });
+      }
+    });
+
+    staticClients.push({
+      clientId: 'static/taskcluster/root',
+      scopes: ['*'],
+    });
+
+    return {'static-clients': staticClients};
+  },
+});
+
+exports.tasks.push({
+  title: 'Configure static client scopes',
+  requires: ['static-clients'],
+  provides: [],
+  run: async (requirements, utils) => {
+    const staticClients = requirements['static-clients'];
+    const staticScopes = staticClients.map(({clientId, scopes}) => ({clientId, scopes}));
+
+    writeRepoJSON('services/auth/src/static-scopes.json', staticScopes);
+  },
+});