Procedures for Kustomize-like overlays

.eslintrc

@@ -8,6 +8,10 @@
           "^std$"
         ]
       }
-    ]
+    ],
+    "no-bitwise": [
+      "error", {"allow": ["<<", ">>", "|", "&"]}
+    ],
+    "indent": ["error", 2, {"SwitchCase": 0, "CallExpression": {"arguments": "first"}}]
   }
 }

package.json

@@ -4,7 +4,9 @@
   "bugs": {
     "url": "https://github.com/jkcfg/kubernetes/issues"
   },
-  "dependencies": {},
+  "dependencies": {
+    "@jkcfg/mixins": "^0.1.2"
+  },
   "description": "jk Kubernetes library",
   "devDependencies": {
     "babel-jest": "^23.6.0",
@@ -34,11 +36,13 @@
   },
   "jest": {
     "testMatch": [
-      "<rootDir>/tests/**"
+      "<rootDir>/tests/*.test.js"
     ],
     "transform": {
       ".*": "<rootDir>/node_modules/babel-jest"
-    }
+    },
+    "transformIgnorePatterns": [],
+    "moduleDirectories": ["<rootDir>/node_modules"]
   },
   "version": "0.1.0"
 }

src/base64.js

@@ -0,0 +1,50 @@
+const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
+
+const chars = alphabet.split('');
+
+function encode(bytes) {
+  const triples = Math.floor(bytes.length / 3);
+  const tripleLen = triples * 3;
+  let encoded = '';
+  for (let i = 0; i < tripleLen; i += 3) {
+    const bits = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2];
+    const c1 = chars[bits >> 18];
+    const c2 = chars[(bits >> 12) & 0x3f];
+    const c3 = chars[(bits >> 6) & 0x3f];
+    const c4 = chars[bits & 0x3f];
+    encoded = `${encoded}${c1}${c2}${c3}${c4}`;
+  }
+  switch (bytes.length - tripleLen) {
+  case 1: {
+    // left with 8 bits; pad to 12 bits to get two six-bit characters
+    const last = bytes[bytes.length - 1];
+    const c1 = chars[last >> 2];
+    const c2 = chars[(last & 0x03) << 4];
+    encoded = `${encoded}${c1}${c2}==`;
+    break;
+  }
+  case 2: {
+    // left with 16 bits; pad to 18 bits to get three six-bit characters
+    const last2 = (bytes[bytes.length - 2] << 10) | (bytes[bytes.length - 1] << 2);
+    const c1 = chars[last2 >> 12];
+    const c2 = chars[(last2 >> 6) & 0x3f];
+    const c3 = chars[last2 & 0x3f];
+    encoded = `${encoded}${c1}${c2}${c3}=`;
+    break;
+  }
+  default:
+    break;
+  }
+  return encoded;
+}
+
+// Encode a native string (UTF16) of ASCII characters as an array of UTF8 bytes.
+function ascii2bytes(str) {
+  const result = new Array(str.length);
+  for (let i = 0; i < str.length; i += 1) {
+    result[i] = str.charCodeAt(i);
+  }
+  return result;
+}
+
+export { encode, ascii2bytes };

src/kubernetes.js

@@ -44,6 +44,15 @@ class ConfigMap {
   }
 }
 
+class Secret {
+  constructor(ns, name, data) {
+    this.apiVersion = 'v1';
+    this.kind = 'Secret';
+    this.metadata = new Meta(ns, name);
+    this.data = data;
+  }
+}
+
 export {
-  Container, Meta, Deployment, ConfigMap,
+  Container, Meta, Deployment, ConfigMap, Secret,
 };

src/overlay/data.js

@@ -0,0 +1,20 @@
+// dataFromFiles reads the contents of each file given and returns a
+// Map of the filenames to file contents.
+async function dataFromFiles(readEncoded, files) {
+  const result = new Map();
+  await Promise.all(files.map(f => readEncoded(f).then((c) => {
+    result.set(f, c);
+  })));
+  return result;
+}
+
+// Given `dir` and `read` procedures, construct a map of file basename
+// to file contents (as strings).
+const dataFromDir = ({ dir, read, Encoding }) => async function data(path) {
+  const d = dir(path);
+  const files = d.files.filter(({ isdir }) => !isdir).map(({ name }) => name);
+  const readFile = f => read(`${path}/${f}`, { encoding: Encoding.String });
+  return dataFromFiles(readFile, files);
+};
+
+export { dataFromFiles, dataFromDir };

src/overlay/generators.js

@@ -0,0 +1,71 @@
+import { dataFromFiles } from './data';
+import { basename } from '../path';
+import { ConfigMap, Secret } from '../kubernetes';
+import { encode as base64encode, ascii2bytes } from '../base64';
+
+const generateConfigMap = readStr => async function generate(config) {
+  const {
+    name,
+    files = [],
+    literals = [],
+  } = config;
+
+  const data = {};
+  literals.forEach((s) => {
+    const [k, v] = s.split('=');
+    data[k] = v;
+  });
+  const fileContents = dataFromFiles(readStr, files);
+  return fileContents.then((d) => {
+    d.forEach((v, k) => {
+      data[basename(k)] = v;
+    });
+    return new ConfigMap(undefined, name, data);
+  });
+};
+
+// In Kustomize, secrets are generally created from the result of
+// shelling out to some command (e.g., create an SSH key). Often these
+// won't be repeatable actions -- the usual mode of operation is such
+// that the value of the secret is cannot be used outside the
+// configuration. For example, a random password is constructed, and
+// supplied to both a server and the client that needs to connect to
+// it.
+//
+// Since we don't want to let the outside world in, with its icky
+// non-determinism, generateSecret here allows
+//
+//   - strings (so long as they are ASCII; supporting UTF8 is possible, but would
+//     need re-encoding)
+//   - files, read as bytes
+//
+// This changes how you can use generated secrets: instead of creating
+// shared secrets internal to the configuration, as above, it is
+// mostly for things supplied from outside, e.g., via
+// parameters. Instead of the config being variations on `command`
+// (see
+// https://github.com/kubernetes-sigs/kustomize/blob/master/pkg/types/kustomization.go),
+// there are much the same fields as for ConfigMaps, the difference
+// being that the values end up being encoded base64.
+const generateSecret = readBytes => async function generate(config) {
+  const {
+    name,
+    files = [],
+    literals = [],
+  } = config;
+
+  const data = {};
+  literals.forEach((s) => {
+    const [k, v] = s.split('=');
+    data[k] = base64encode(ascii2bytes(v));
+  });
+  const fileContents = dataFromFiles(readBytes, files);
+  return fileContents.then((d) => {
+    d.forEach((v, k) => {
+      data[basename(k)] = base64encode(v);
+    });
+    return new Secret(undefined, name, data);
+  });
+};
+
+export { generateConfigMap, generateSecret };

src/overlay/index.js

@@ -0,0 +1,71 @@
+// This module provides procedures for applying Kustomize-like
+// [overlays](https://github.com/kubernetes-sigs/kustomize/blob/master/docs/kustomization.yaml).
+
+// In Kustomize, a configuration is given in a `kustomize.yaml` file;
+// here we'll interpret an object (which can of course be loaded from
+// a file). In a `kustomize.yaml` you refer to files from which to
+// load or generate resource manifests, and transformations to apply
+// to all or some resources.
+//
+// The mechanism for composing configurations is to name `bases` in
+// the `kustomize.yaml` file; these are evaluated and included in the
+// resources.
+//
+//     Promise [Resource]
+//
+// There are different kinds of transformations, but they amount to
+//
+//     Resource -> Resource
+//
+// The approach taken here is
+//   0. load the file
+//   1. assemble all the transformations mentioned in various ways in the kustomize object;
+//   2. assemble all the resources, mentioned in various ways, in the kustomize object;
+//   2. run each resource through the transformations.
+//
+// Easy peasy!
+
+import { patchResource, commonMetadata } from './transforms';
+import { generateConfigMap, generateSecret } from './generators';
+
+const flatten = array => [].concat(...array);
+const pipeline = (...fns) => v => fns.reduce((acc, val) => val(acc), v);
+
+// overlay constructs an interpreter which takes an overlay object (as
+// would be parsed from a `kustomize.yaml`) and constructs a set of
+// resources to write out.
+const overlay = ({ read, Encoding }) => async function assemble(path, config) {
+  const readObj = f => read(`${path}/${f}`, { encoding: Encoding.JSON });
+  const readStr = f => read(`${path}/${f}`, { encoding: Encoding.String });
+  const readBytes = f => read(`${path}/${f}`, { encoding: Encoding.Bytes });
+  const {
+    resources: resourceFiles = [],
+    bases: baseFiles = [],
+    patches: patchFiles = [],
+    configMapGenerator = [],
+    secretGenerator = [],
+  } = config;
+
+  const patches = [];
+  patchFiles.forEach((f) => {
+    patches.push(readObj(f).then(patchResource));
+  });
+
+  // TODO: add the other kinds of transformation: imageTags,
+  // commonAnnotations, etc.
+
+  const resources = []; // :: [Promise [Resource]]
+  baseFiles.forEach((f) => {
+    const obj = readObj(`${f}/kustomize.yaml`);
+    resources.push(obj.then(o => assemble(`${path}/${f}`, o)));
+  });
+
+  resources.push(Promise.all(resourceFiles.map(readObj)));
+  resources.push(Promise.all(configMapGenerator.map(generateConfigMap(readStr))));
+  resources.push(Promise.all(secretGenerator.map(generateSecret(readBytes))));
+
+  const transform = pipeline(...await Promise.all(patches), commonMetadata(config));
+  return Promise.all(resources).then(flatten).then(rs => rs.map(transform));
+};
+
+export default overlay;

src/overlay/transforms.js

@@ -0,0 +1,45 @@
+import { patch, patches } from '@jkcfg/mixins/src/mixins';
+
+// resourceMatch returns a predicate which gives true if the given
+// object represents the same resource as `template`, false otherwise.
+function resourceMatch(target) {
+  // NaN is used for mandatory fields; if these are not present in the
+  // template, nothing will match it (since NaN does not equal
+  // anything, even itself).
+  const { apiVersion = NaN, kind = NaN, metadata = {} } = target;
+  const { name = NaN, namespace } = metadata;
+  return (obj) => {
+    const { apiVersion: v, kind: k, metadata: m } = obj;
+    if (v !== apiVersion || k !== kind) return false;
+    const { name: n, namespace: ns } = m;
+    if (n !== name || ns !== namespace) return false;
+    return true;
+  };
+}
+
+// patchResource returns a function that will patch the given object
+// if it refers to the same resource, and otherwise leave it
+// untouched.
+function patchResource(p) {
+  const match = resourceMatch(p);
+  return v => (match(v) ? patch(v, p) : v);
+}
+
+// commonMetadata returns a tranformation that will indiscriminately
+// add the given labels and annotations to every resource.
+function commonMetadata({ commonLabels = null, commonAnnotations = null }) {
+  // This isn't quite as cute as it could be; naively, just assembling a patch
+  //     { metadata: { labels: commonLabels, annotations: commonAnnotations }
+  // doesn't work, as it will assign null (or empty) values where they are not
+  // present.
+  const metaPatches = [];
+  if (commonLabels !== null) {
+    metaPatches.push({ metadata: { labels: commonLabels } });
+  }
+  if (commonAnnotations !== null) {
+    metaPatches.push({ metadata: { annotations: commonAnnotations } });
+  }
+  return patches(...metaPatches);
+}
+
+export { patchResource, commonMetadata };

src/path.js

@@ -0,0 +1,9 @@
+function basename(path) {
+  return path.substring(path.lastIndexOf('/') + 1);
+}
+
+function dirname(path) {
+  return path.substring(0, path.lastIndexOf('/'));
+}
+
+export { basename, dirname };

tests/base64.test.js

@@ -0,0 +1,23 @@
+import { encode, ascii2bytes } from '../src/base64';
+
+test('empty -> empty', () => {
+  expect(encode(new Uint8Array(0))).toEqual('');
+});
+
+test('ascii2bytes', () => {
+  expect(ascii2bytes('ABC')).toEqual([65, 66, 67]);
+});
+
+// from RFC 4648
+[
+  ['f', 'Zg=='],
+  ["fo", "Zm8="],
+  ["foo", "Zm9v"],
+  ["foob", "Zm9vYg=="],
+  ["fooba", "Zm9vYmE="],
+  ["foobar", "Zm9vYmFy"],
+].forEach(([input, output]) => {
+  test(`'${input}' -> '${output}' `, () => {
+    expect(encode(ascii2bytes(input))).toEqual(output);
+  });
+});