Add "schnauzer" generator for parameterized setting values

This lets settings be generated based on a template that reference other
settings.  The first example is regional container image URIs, where we replace
part with the value of settings.aws.region.

The template is stored as a new metadata key, next to the setting-generator
metadata, which schnauzer will read from the API.  For example, if the template
is "foo-{{ settings.bar }}" and "settings.bar" is set to "baz", then the
setting output will be "foo-baz".

To be able to reference other settings from sundog-spawned generators, we need
to commit settings that have already been patched, e.g. from user data.  To do
this, this commit eliminates the settings-committer service, instead using the
settings-committer binary as an ExecStartPre on any service that requires
committed settings.

packages/release/release.spec

@@ -40,6 +40,7 @@ Requires: %{_cross_os}kernel
 Requires: %{_cross_os}kernel-modules
 Requires: %{_cross_os}bork
 Requires: %{_cross_os}moondog
+Requires: %{_cross_os}schnauzer
 Requires: %{_cross_os}netdog
 Requires: %{_cross_os}signpost
 Requires: %{_cross_os}sundog

packages/workspaces/settings-applier.service

@@ -1,11 +1,12 @@
 [Unit]
 Description=Applies settings to create config files
-After=apiserver.service settings-committer.service
+After=storewolf.service sundog.service moondog.service apiserver.service
 Requires=apiserver.service
-Wants=settings-committer.service
+Wants=storewolf.service sundog.service moondog.service
 
 [Service]
 Type=oneshot
+ExecStartPre=/usr/bin/settings-committer
 ExecStart=/usr/bin/thar-be-settings --all
 RemainAfterExit=true
 StandardError=journal+console

packages/workspaces/sundog.service

@@ -6,6 +6,8 @@ Requires=network-online.target apiserver.service
 
 [Service]
 Type=oneshot
+# Commit first so we can use settings from user data.
+ExecStartPre=/usr/bin/settings-committer
 ExecStart=/usr/bin/sundog
 RemainAfterExit=true
 StandardError=journal+console

packages/workspaces/workspaces.spec

@@ -22,7 +22,6 @@ Source100: apiserver.service
 Source101: moondog.service
 Source102: sundog.service
 Source103: storewolf.service
-Source104: settings-committer.service
 Source105: settings-applier.service
 Source106: migrator.service
 Source107: host-containers@.service
@@ -64,6 +63,7 @@ Requires: %{_cross_os}apiserver = %{version}-%{release}
 %package -n %{_cross_os}sundog
 Summary: Updates settings dynamically based on user-specified generators
 Requires: %{_cross_os}apiserver = %{version}-%{release}
+Requires: %{_cross_os}schnauzer = %{version}-%{release}
 Requires: %{_cross_os}pluto = %{version}-%{release}
 Requires: %{_cross_os}bork = %{version}-%{release}
 %description -n %{_cross_os}sundog
@@ -74,6 +74,11 @@ Summary: Dynamic setting generator for updog
 %description -n %{_cross_os}bork
 %{summary}.
 
+%package -n %{_cross_os}schnauzer
+Summary: Setting generator for templated settings values.
+%description -n %{_cross_os}schnauzer
+%{summary}.
+
 %package -n %{_cross_os}pluto
 Summary: Dynamic setting generator for kubernetes
 %description -n %{_cross_os}pluto
@@ -145,6 +150,7 @@ mkdir bin
     -p moondog \
     -p netdog \
     -p sundog \
+    -p schnauzer \
     -p pluto \
     -p bork \
     -p thar-be-settings \
@@ -167,7 +173,7 @@ mkdir bin
 install -d %{buildroot}%{_cross_bindir}
 for p in \
   apiserver \
-  moondog netdog sundog pluto bork \
+  moondog netdog sundog schnauzer pluto bork \
   thar-be-settings servicedog host-containers \
   storewolf settings-committer \
   migrator \
@@ -204,7 +210,7 @@ install -p -m 0644 %{S:5} %{buildroot}%{_cross_templatedir}
 
 install -d %{buildroot}%{_cross_unitdir}
 install -p -m 0644 \
-  %{S:100} %{S:101} %{S:102} %{S:103} %{S:104} %{S:105} \
+  %{S:100} %{S:101} %{S:102} %{S:103} %{S:105} \
   %{S:106} %{S:107} %{S:108} %{S:109} %{S:110} \
   %{buildroot}%{_cross_unitdir}
 
@@ -233,6 +239,9 @@ install -p -m 0644 %{S:201} %{buildroot}%{_cross_tmpfilesdir}/host-containers.co
 %{_cross_bindir}/sundog
 %{_cross_unitdir}/sundog.service
 
+%files -n %{_cross_os}schnauzer
+%{_cross_bindir}/schnauzer
+
 %files -n %{_cross_os}pluto
 %{_cross_bindir}/pluto
 %dir %{_cross_datadir}/eks
@@ -264,7 +273,6 @@ install -p -m 0644 %{S:201} %{buildroot}%{_cross_tmpfilesdir}/host-containers.co
 
 %files -n %{_cross_os}settings-committer
 %{_cross_bindir}/settings-committer
-%{_cross_unitdir}/settings-committer.service
 
 %files -n %{_cross_os}growpart
 %{_cross_sbindir}/growpart

workspaces/Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "api/moondog",
     "api/netdog",
     "api/sundog",
+    "api/schnauzer",
     "api/pluto",
     "api/servicedog",
     "api/host-containers",

workspaces/api/README.md

@@ -101,7 +101,8 @@ Pluto generates settings needed for Kubernetes configuration, for example cluste
 
 [Further docs](settings-committer/)
 
-This service sends a commit request to the API, which moves all the pending settings from the above services into the live part of the data store.
+This binary sends a commit request to the API, which moves all the pending settings from the above services into the live part of the data store.
+It's called as a prerequisite of other services, like [sundog](#sundog) and [settings-applier](#settings-applier), that rely on settings being committed.
 
 ### settings-applier
 

workspaces/api/api-system.drawio

@@ -1 +1 @@
-<mxfile modified="2019-09-03T17:53:54.335Z" host="drawio.corp.amazon.com" agent="Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0" etag="1CiXTfHJy2oos7HLSLhY" version="10.9.7" type="device"><diagram id="vL4lszIxdcRJnpUGa12-" name="Page-1">3Vpbc+I2FP41zLQPZHzBhDwuue10m9lsmE7TvnSELYwmsuXKIkB+/UpYAlsS2AFDyOYhYx/JMvrOd65yx79OFvcUZNMHEkHc8Zxo0fFvOp531fP4fyFYFgLXcQeFJKYokrKNYITeoJoopTMUwbwykRGCGcqqwpCkKQxZRQYoJfPqtAnB1bdmIIaGYBQCbEr/RhGbSqnrOJuBrxDFU/nqQSAHEqAmS0E+BRGZl0T+bce/poSw4ipZXEMswFO4FM/dbRld/zAKU9bkgeET/Cv8/49v/Xs6Xdw8wRdvlHblKq8Az+SGO14f8/WGEXrll7G4TFBMASNUDfF3lEbl7thSQUbJLI2geKvLh+dTxOAoA6EYnXOScNmUJVgOm7tQPwlSBhclkdzVPSQJZHTJpyhO9YLikflGQQMJ+rSkGl9OA5IS8XqlDWr8QgL3DhB9A8ScowXnBE8+DhxlgRKIc8GqZ2AVAQZWGHHIDLi40WTiMlxixHGj9ZiNC4D/HK8FIHyJV7B/nzG+CmwP3CCogGtC27dAq+BuHdqgkS0TkkYk3mbKxvzfJojmjC87Fo7Kc0iKl7+reWOqP3kurkGO+k49/deu/CT87xtKAhnKIeV7MtDiy/EwB+uRmpCUydDJw2sryLmDKnIW3LxTkntg+thZQeQPpljPPTMPq35PGSrIGErjvBuSJEGMWbh2atgCv4Fl+ifFzcyF1riBLMPoDFDrXzZAzT0pap6BGk/FJyieURhdMEBjyLZG9SlcgJikHJsMUsR/kAjwSvqoRN5JoB14GrR+Q5cXHMvluWZeaYaINPoiqhyRImGQ5yjkaOQcdmaKS3itZtwhXItfTmY0hPUFhFR0bd4Ho0q5ZWqjhHawA2wKMWDotVqk2TQg3/BIUMpK0U3TtaOZR7Ft+VC5ptLW6WtBUlumQMVYZsWG9Z4PIIgt43s3QVIiUuJt7HAOYof/KdnRdTVf0NuXH13NX/vHY8hD79kbgbcfd9ffu/RfAB68r9ha3jchSBPXWqv8oKHy+2emfLct5fc/VvlmZD6h8vuf1PJ/FeXvmzvoYWBL8Ggldxj8Gg7Cb4sjR8wfrBwxexHNOFKfX3Iq0OWzSB8uAnX7T3nsZiFzi+Ju+Z5Mw7qXpsFmcFZccnuau/H2pJKr9Xn9y9NSyWzPGFQSPdhsq4KNQm19pAPGagXHrirV69Mg8JyGBdxVCwXcLk7ubsjKHgOfF8MUrs5Z8sbd2eaN2DN59OLiooXmsVPfEYhAPl03ZsTNIxB9r3Ql8Zye6JryGHZNMKGr5f3J6o/LCV8UMcGiwFl5O0pe1od/ohkhuIpCgL9gFIsFGcmsvmunpdQ3Jk7BW6uPvDJ4m+EZI4ZCVBsnZzDTejhC9Li5r21gowVUh7/2nOJALO11v+ov1p4GHMtDmEi/O0vbP/1SYHz6/EuPdXvn6Hr0PXZ1bnbqT6n+pq27M1d/T8+a91W/UesdW/3Wjy9AInzjKh7mS+5GE6FLSF9RCPP1sD1AKn/M2QAwhpjEFCSaY66MvcNDV6Og2zx8Ns/4trfmlWJUu9aS2NkY1z+W23YbpLslw5Ud1kpK0o4Je6YJ76Ta2diw35YLH3ywC7dFcCPrjWDG6dA4SSZpw4yY2w+rconCHL3JiklYXyY2voIiGHaCGy4BMmkNOdtWhq9nswmKIvH8EIMxxMP11zM2Wzepu9teDi7yus5F78rzKzpXddahpKw6mfXpilqBTCY5PE6j1swDhvJrGxpZTp1NtRfeWWlI+puyg5ai5sq3hYNqAab5+Ta/QdlVwdtOt71jOXrl1w/J0BoDU9vjUts8Ey+up+H9fXtXQXUhT7e89nz48yhyvjF89+P1v5F/G89vPPRkOSSb5cLqnOLLxBMmWVu9pMGWrQ7S004cbTZ0ZTOhFj4QsaLbggVZWV9rQrt0XVvzBOdkWJeae2tqWL6WZRmfquxtWPx28+F6MX3z+b9/+xM=</diagram></mxfile>
+<mxfile modified="2020-01-10T00:35:53.584Z" host="drawio.corp.amazon.com" agent="Mozilla/5.0 (X11; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0" etag="O4bPJ8mCazPHc_QchX88" version="12.4.8" type="device"><diagram id="vL4lszIxdcRJnpUGa12-" name="Page-1">3Vtbc5s4FP41ntl9cIaLIc5jru1st5O0np1N96WjgIw1kRERcmz3168EAgMStmyD67YPGTgSwnznO1epA/d2vvpAQTL7TEKIB44Vrgbu3cBxrkYO/ysE61xgW7aURBSFUrYRTNAPWEyU0gUKYVqbyAjBDCV1YUDiGAasJgOUkmV92pTg+lsTEEFFMAkAVqX/opDNpNS2rM3AR4iimXz12JMDc1BMloJ0BkKyrIjc+4F7Swlh+dV8dQuxAK/AJX/uoWW0/GEUxszkgZuv8J/g7a9P/gc6W919ha/OJB7KVd4BXsgPHjg+5uvdhOidX0bico4iChihxRB/R2VUfh1bF5BRsohDKN5q8+HlDDE4SUAgRpecJFw2Y3Msh9WvKH4SpAyuKiL5VR8gmUNG13xKwamRlz+y3ChoLEGfVVTjymlAUiIqV9qgxi8kcHuA6CogphwtuCR4+vPAKSxQAnEuWI0UrELAQIYRh0yBixtNIi6DNUYcN7obs5cc4L9fSgEIXqMM9scF46vA7sD1vBq4KrS+BtoC7s6h9YxsmZA4JFGbKSvz/5gimjK+7ItwVI5FYrz+s5j3QptPnotrkKOutZv+pSs/Cf99RUkgQSmk/JsUtPhyPMzB3UhNScxk6LTH3SBnj+vIaXBzTknuIm2oAMdD/hRFCwrDCwZoBFmr95jBFYhIzJFJIEX8BwlHUkifCpFzEkpe+nWnYY8sM2i93qBV45dKxTi8FtmUcMUYpCkKOBoph52p4gpe2YwHhHfil5IFDeDuREUqemd8gWEtrVO1UUHb2wI2hRgw9F5PBnUakG94IihmFStquB+r4Vbyz5YPVXO3xjp+wxgby+SoKMtkbCi/+QiC6CLL3gSJiQi9beywjmKH+0uyY1jaudTr6FB+DC/rC7n9MeTz6NmZgB9fHm4fh/Q/AD47H7G2jDAhiIlr3al8z1D5/pkp3+5K+f7PVb4amU+ofP8XtfzfRfmH5g7NMNASPCq5A1wh9iwjhbj+Vrm+W1Vv1kfHk/Hv4VLcrljVY8ahjyfWz/QptmmyeebqHzW1dqj6Fe/Ut/q1bUkwF9VY1mZI1ymDc6FLXjmjAKblsL7vUBSCnA0AY4hJRMG8UQ7Wxp42A7tqwpRR8lo2h7NynDutW4IJzV7uTrN/XE74GogJTL2t7mmPYtJr2KmmltQxzu+gltRrbryX4cqaIATprOwKdWPCjmrCW6l2NjY8rrcH3GZHxdiGbbexkttYqW8jvtIZcbO/GMKE88G4HUliw04jNyBWJxOFKfoBXrIJwvwS8eEZFN7NwLvjEoBRFItwwumWWb4wRBQAfC0H5igMxfM3GLxAfFM2lnXGrnJ3u8E0bb7c0pI/eVDdNdL5gqF1Mbpy3JrOi9roWFbWmVQ2BIoVyHSawn5qCzURuJGNaBpqWqaq2nP3XGhIOpyqh5Yic+Xr4sGmsW2pjr7L9mzp6DUNWVvjl5y+PH3h2I9J0YyBqfr7rTw5EzfezMP9AxMxuxnfm5bXnQ9/noTWJ4Yfvrx/n7j30fLOQV81fZ1FKqzOyjftTphlHWMzctS5bCZLqg1d6Uyog70gLbqnapxsU+3OGsc7a8O6PDA7KpOhssDtLTmavi0f366DZOpPlnfrx+vv936g3attqF6kFsmeyYGGCq3mMBrVIbB1xYNuI8q+7MAgtKgYNJOOR6VdH1tJ3R8U2oBmUEdVoNj5fe6ekLUmHmNN3qFD5qqvvMPolAPPRhmKIz4vgjHMDi+lxjWG+emGM3n04uKigxMZ1u4gXKvTxc0TYDx6x5nEsUbmvY9608TRpNuMJOZVVFsRtb9Fd8FbbRxVq+EELxhRFFIkUSmDSSN3EqI9UqYpWsHiRKU+VzgSy5aYrAkj2iM2fXkI9YiNAvKBxxk4cHSd7UFceMXtt+rYZh8iu2vfiNjmpHcWO6YbnlKTZ5K6DR2vSZRD29OO1czeTnsgQg3O6SI/VdfqXQ3Ou6XCRfKglXNG6UccGenbWwmGEb2LI3HaX7xfqtNLLuw3zwma5sJdNNK37kFVOZYnNekQJAlGmubXEWwzJJd5bOifXPx2c3g+t+zNf0Fw7/8H</diagram></mxfile>

workspaces/api/apiserver/src/server/mod.rs

@@ -81,7 +81,8 @@ where
             .service(
                 web::scope("/metadata")
                     .route("/affected-services", web::get().to(get_affected_services))
-                    .route("/setting-generators", web::get().to(get_setting_generators)),
+                    .route("/setting-generators", web::get().to(get_setting_generators))
+                    .route("/templates", web::get().to(get_templates)),
             )
             .service(web::scope("/services").route("", web::get().to(get_services)))
             .service(
@@ -230,6 +231,23 @@ fn get_setting_generators(data: web::Data<SharedDataStore>) -> Result<MetadataRe
     Ok(MetadataResponse(resp))
 }
 
+/// Get the template metadata for a list of data keys
+fn get_templates(
+    query: web::Query<HashMap<String, String>>,
+    data: web::Data<SharedDataStore>,
+) -> Result<MetadataResponse> {
+    if let Some(keys_str) = query.get("keys") {
+        let data_keys = comma_separated("keys", keys_str)?;
+        let datastore = data.ds.read().ok().context(error::DataStoreLock)?;
+        let resp =
+            controller::get_metadata_for_data_keys(&*datastore, "template", &data_keys)?;
+
+        Ok(MetadataResponse(resp))
+    } else {
+        return error::MissingInput { input: "keys" }.fail();
+    }
+}
+
 /// Get all services, or if 'names' is specified, services with those names
 fn get_services(
     query: web::Query<HashMap<String, String>>,

workspaces/api/schnauzer/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "schnauzer"
+version = "0.1.0"
+authors = ["Tom Kirchner <tjk@amazon.com>"]
+edition = "2018"
+publish = false
+build = "build.rs"
+
+[dependencies]
+apiclient = { path = "../apiclient" }
+base64 = "0.11"
+handlebars = "3.0"
+http = "0.1"
+log = "0.4"
+models = { path = "../../models" }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1"
+snafu = "0.6"
+
+[build-dependencies]
+cargo-readme = "3.1"

workspaces/api/schnauzer/README.md

@@ -0,0 +1,20 @@
+# schnauzer
+
+Current version: 0.1.0
+
+## Introduction
+
+schnauzer is called by sundog as a setting generator.
+Its sole parameter is the name of the setting to generate.
+
+The setting we're generating is expected to have a metadata key already set: "template".
+"template" is an arbitrary string with mustache template variables that reference other settings.
+
+For example, if we're generating "settings.x" and we have template "foo-{{ settings.bar }}", we look up the value of "settings.bar" in the API.
+If the returned value is "baz", our generated value will be "foo-baz".
+
+(The name "schnauzer" comes from the fact that Schnauzers are search and rescue dogs (similar to this search and replace task) and because they have mustaches.)
+
+## Colophon
+
+This text was generated from `README.tpl` using [cargo-readme](https://crates.io/crates/cargo-readme), and includes the rustdoc from `src/main.rs`.

workspaces/api/schnauzer/README.tpl

@@ -0,0 +1,9 @@
+# {{crate}}
+
+Current version: {{version}}
+
+{{readme}}
+
+## Colophon
+
+This text was generated from `README.tpl` using [cargo-readme](https://crates.io/crates/cargo-readme), and includes the rustdoc from `src/main.rs`.

workspaces/api/schnauzer/build.rs

@@ -0,0 +1,32 @@
+// Automatically generate README.md from rustdoc.
+
+use std::env;
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+
+fn main() {
+    // Check for environment variable "SKIP_README". If it is set,
+    // skip README generation
+    if env::var_os("SKIP_README").is_some() {
+        return;
+    }
+
+    let mut source = File::open("src/main.rs").unwrap();
+    let mut template = File::open("README.tpl").unwrap();
+
+    let content = cargo_readme::generate_readme(
+        &PathBuf::from("."), // root
+        &mut source,         // source
+        Some(&mut template), // template
+        // The "add x" arguments don't apply when using a template.
+        true,  // add title
+        false, // add badges
+        false, // add license
+        true,  // indent headings
+    )
+    .unwrap();
+
+    let mut readme = File::create("README.md").unwrap();
+    readme.write_all(content.as_bytes()).unwrap();
+}