diff --git a/cmd/cli/manager.go b/cmd/cli/manager.go index 5f0ded79e..36c19c7e1 100644 --- a/cmd/cli/manager.go +++ b/cmd/cli/manager.go @@ -110,6 +110,7 @@ func managerCommand(plugins func() discovery.Plugins) *cobra.Command { // Check the list of plugins for _, gp := range groups { + endpoint, err := plugins().Find(gp.Plugin) if err != nil { return err @@ -127,6 +128,9 @@ func managerCommand(plugins func() discovery.Plugins) *cobra.Command { // TODO(chungers) -- we need to enforce and confirm the type of this. // Right now we assume the RPC endpoint is indeed a group. target, err := group_plugin.NewClient(endpoint.Address) + + log.Debugln("For group", gp.Plugin, "address=", endpoint.Address, "err=", err, "spec=", spec) + if err != nil { return err } diff --git a/examples/flavor/swarm/README.md b/examples/flavor/swarm/README.md index c48af4ee4..0d1b84ffe 100644 --- a/examples/flavor/swarm/README.md +++ b/examples/flavor/swarm/README.md @@ -4,29 +4,93 @@ InfraKit Flavor Plugin - Swarm A [reference](/README.md#reference-implementations) implementation of a Flavor Plugin that creates a Docker cluster in [Swarm Mode](https://docs.docker.com/engine/swarm/). +## Schema & Templates -## Schema - -Here's a skeleton of this Plugin's schema: +This plugin has a schema that looks like this: ```json { - "Type": "", - "Attachments": {}, - "DockerRestartCommand": "" + "InitScriptTemplateURL": "http://your.github.io/your/project/swarm/worker-init.sh", + "SwarmJoinIP": "192.168.2.200", + "Docker" : { + "Host" : "tcp://192.168.2.200:4243" + } + } +``` +Note that the Docker connection information, as well as what IP in the Swarm the managers and workers should use +to join the swarm, are now part of the plugin configuration. + +This plugin makes heavy use of Golang template to enable customization of instance behavior on startup. For example, +the `InitScriptTemplateURL` field above is a URL where a init script template is served. The plugin will fetch this +template from the URL and processe the template to render the final init script for the instance. + +The plugin exposes a set of template functions that can be used, along with primitives already in [Golang template] +(https://golang.org/pkg/text/template/) and functions from [Sprig](https://github.com/Masterminds/sprig#functions). +This makes it possible to have complex templates for generating the user data / init script of the instances. + +For example, this is a template for the init script of a manager node: + +``` +#!/bin/sh +set -o errexit +set -o nounset +set -o xtrace + +{{/* Install Docker */}} +{{ include "install-docker.sh" }} + +mkdir -p /etc/docker +cat << EOF > /etc/docker/daemon.json +{ + "labels": {{ INFRAKIT_LABELS | to_json }} } +EOF + +{{/* Reload the engine labels */}} +kill -s HUP $(cat /var/run/docker.pid) +sleep 5 + +{{ if eq INSTANCE_LOGICAL_ID SPEC.SwarmJoinIP }} + + {{/* The first node of the special allocations will initialize the swarm. */}} + docker swarm init --advertise-addr {{ INSTANCE_LOGICAL_ID }} + + # Tell Docker to listen on port 4243 for remote API access. This is optional. + echo DOCKER_OPTS="\"-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock\"" >> /etc/default/docker + + # Restart Docker to let port listening take effect. + service docker restart + +{{ else }} + + {{/* The rest of the nodes will join as followers in the manager group. */}} + docker swarm join --token {{ SWARM_JOIN_TOKENS.Manager }} {{ SPEC.SwarmJoinIP }}:2377 + +{{ end }} ``` -The supported fields are: -* `Type`: The Swarm mode node type, `manager` or `worker` -* `Attachments`: A mapping from logical IDs to arrays of attachment IDs to associate. The instance plugin being used - defines the meaning of attachments and how to attach them. -* `DockerRestartCommand`: A shell command that will restart the Docker daemon, used when adding daemon labels +There are tags such as `{{ SWARM_JOIN_TOKENS.Manager }}` or `{{ INSTANCE_LOGICAL_ID }}`: these are made available by the +plugin and they are evaluated / interpolated during the `Prepare` phase of the plugin. The plugin will substitute +these 'placeholders' with actual values. The templating engine also supports inclusion of other templates / files, as +seen in the `{{ include "install-docker.sh" }}` tag above. This makes it easy to embed actual shell scripts, and other +texts, without painful and complicated escapes to meet the JSON syntax requirements. For example, the 'include' tag +above will embed the `install-docker.sh` template/file: -## Example +``` +# Tested on Ubuntu/trusty + +apt-get update -y +apt-get upgrade -y +wget -qO- https://get.docker.com/ | sh + +# Tell Docker to listen on port 4243 for remote API access. This is optional. +echo DOCKER_OPTS=\"-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock\" >> /etc/default/docker -Begin by building plugin [binaries](/README.md#binaries). +# Restart Docker to let port listening take effect. +service docker restart -### Security +``` + +### A Word on Security Since Swarm Mode uses [join-tokens](https://docs.docker.com/engine/swarm/join-nodes/) to authorize nodes, initializing the Swarm requires: @@ -39,58 +103,172 @@ We recommend approach (b) for anything but demonstration purposes unless the Doc Docker socket**. -### Running +### Building & Running -- An Example -Start required plugins: +There are scripts in this directory to illustrate how to start up the InfraKit plugin ensemble and examples for creating +a Docker swarm via vagrant. +Building the binaries - do this from the top level project directory: ```shell -$ build/infrakit-group-default +make binaries ``` +Start required plugins. We use the `infrakit plugin start` utility and a `plugins.json` to start up all the plugins, +along with the InfraKit manager: + ```shell -$ build/infrakit-flavor-vanilla +~/projects/src/github.com/docker/infrakit$ examples/flavor/swarm/start-plugins.sh +Starting up manager +Starting up group-stateless +INFO[0000] Waiting for manager to start: { + "Cmd" : "infrakit-manager --name group --proxy-for-group group-stateless os --leader-file /Users/me/.infrakit/leader --store-dir /Users/me/.infrakit/configs > /Users/me/.infrakit/logs/manager.log 2>&1" + } +INFO[0000] OS launcher: Plugin manager setPgId= true starting infrakit-manager --name group --proxy-for-group group-stateless os --leader-file /Users/me/.infrakit/leader --store-dir /Users/me/.infrakit/configs > /Users/me/.infrakit/logs/manager.log 2>&1 +INFO[0000] Running /bin/sh /bin/sh -c infrakit-manager --name group --proxy-for-group group-stateless os --leader-file /Users/me/.infrakit/leader --store-dir /Users/me/.infrakit/configs > /Users/me/.infrakit/logs/manager.log 2>&1 +INFO[0000] Starting with sh= infrakit-manager --name group --proxy-for-group group-stateless os --leader-file /Users/me/.infrakit/leader --store-dir /Users/me/.infrakit/configs > /Users/me/.infrakit/logs/manager.log 2>&1 +INFO[0000] Waiting for group-stateless to start: { + "Cmd" : "infrakit-group-default --poll-interval 10s --name group-stateless --log 5 > /Users/me/.infrakit/logs/group-stateless.log 2>&1" + } +INFO[0000] OS launcher: Plugin group-stateless setPgId= true starting infrakit-group-default --poll-interval 10s --name group-stateless --log 5 > /Users/me/.infrakit/logs/group-stateless.log 2>&1 +INFO[0000] Running /bin/sh /bin/sh -c infrakit-group-default --poll-interval 10s --name group-stateless --log 5 > /Users/me/.infrakit/logs/group-stateless.log 2>&1 +manager started. +Starting up flavor-swarm +INFO[0000] Starting with sh= infrakit-group-default --poll-interval 10s --name group-stateless --log 5 > /Users/me/.infrakit/logs/group-stateless.log 2>&1 +INFO[0000] Waiting for flavor-swarm to start: { + "Cmd" : "infrakit-flavor-swarm --log 5 > /Users/me/.infrakit/logs/flavor-swarm.log 2>&1" + } +INFO[0000] OS launcher: Plugin flavor-swarm setPgId= true starting infrakit-flavor-swarm --log 5 > /Users/me/.infrakit/logs/flavor-swarm.log 2>&1 +INFO[0000] Running /bin/sh /bin/sh -c infrakit-flavor-swarm --log 5 > /Users/me/.infrakit/logs/flavor-swarm.log 2>&1 +group-stateless started. +Starting up instance-vagrant +INFO[0000] Starting with sh= infrakit-flavor-swarm --log 5 > /Users/me/.infrakit/logs/flavor-swarm.log 2>&1 +INFO[0000] Waiting for instance-vagrant to start: { + "Cmd" : "infrakit-instance-vagrant --log 5 > /Users/me/.infrakit/logs/instance-vagrant.log 2>&1" + } +INFO[0000] OS launcher: Plugin instance-vagrant setPgId= true starting infrakit-instance-vagrant --log 5 > /Users/me/.infrakit/logs/instance-vagrant.log 2>&1 +INFO[0000] Running /bin/sh /bin/sh -c infrakit-instance-vagrant --log 5 > /Users/me/.infrakit/logs/instance-vagrant.log 2>&1 +flavor-swarm started. +INFO[0000] Starting with sh= infrakit-instance-vagrant --log 5 > /Users/me/.infrakit/logs/instance-vagrant.log 2>&1 +instance-vagrant started. +Plugins started. +Do something like: infrakit manager commit file:///Users/me/projects/src/github.com/docker/infrakit/examples/flavor/swarm/groups-fast.json + + ``` +Now start up the cluster comprised of a manager and a worker group. In this case, see `groups-fast.json` where we will create +a manager group of 3 nodes and a worker group of 3 nodes. The topology in this is a single ensemble of infrakit running on +your local machine that manages 6 vagrant vms running Docker in Swarm Mode. The `groups-fast.json` is named fast because +we are using a Vagrant box (image) that already has Docker installed. A slower version, that uses just `ubuntu/trusty64` and +a full Docker install, can be found in `groups.json`. + ```shell -$ build/infrakit-instance-vagrant +~/projects/src/github.com/docker/infrakit$ infrakit manager commit file:///Users/davidchung/projects/src/github.com/docker/infrakit/examples/flavor/swarm/groups-fast.json +INFO[0000] Found manager group is leader = true +INFO[0000] Found manager as group at /Users/davidchung/.infrakit/plugins/group +INFO[0000] Using file:///Users/davidchung/projects/src/github.com/docker/infrakit/examples/flavor/swarm/groups-fast.json for reading template + +Group swarm-workers with plugin group plan: Managing 3 instances +Group swarm-managers with plugin group plan: Managing 3 instances +``` + +Now it will take some time for the entire cluster to come up. During this, you may want to see the Vagrant and Swarm plugins +in action. If you look at the `plugins.json` you will see that the plugins are started with `stdout` and `stderr` being +directed to a `{{env "INFRAKIT_HOME"}}/logs` directory. The `$INFRAKIT_HOME` environment variable is set by the `start-plugins.sh` +script and is set to `~/.infrakit`. + +So, to look at the logs, just do this in another terminal: + +``` +tail -f ~/.infrakit/logs/*.log ``` +You will see the sequential interactions of the plugins. Here's an example: + ```shell -$ build/infrakit-flavor-swarm --host tcp://192.168.2.200:4243 + +==> group-stateless.log <== +time="2017-01-31T16:38:22-08:00" level=debug msg="Received response HTTP/1.1 200 OK\r\nContent-Length: 1288\r\nContent-Type: text/plain; charset=utf-8\r\nDate: Wed, 01 Feb 2017 00:38:22 GMT\r\n\r\n{\"jsonrpc\":\"2.0\",\"result\":{\"Type\":\"manager\",\"Spec\":{\"Properties\":{\"Box\":\"ubuntu/trusty64\"},\"Tags\":{\"infrakit-link\":\"HmjVIS7jEMNMu3mU\",\"infrakit-link-context\":\"swarm/t9vg5zqmdtw8ovhniwifszwbw/manager\",\"infrakit.config_sha\":\"UBS6BBMA84vUjjtMceeraQGP_eQ=\",\"infrakit.group\":\"swarm-managers\",\"swarm-id\":\"t9vg5zqmdtw8ovhniwifszwbw\"},\"Init\":\"#!/bin/sh\\nset -o errexit\\nset -o nounset\\nset -o xtrace\\n\\n\\n\\n# Tested on Ubuntu/trusty\\n\\napt-get update -y\\napt-get upgrade -y\\nwget -qO- https://get.docker.com/ | sh\\n\\n# Tell Docker to listen on port 4243 for remote API access. This is optional.\\necho DOCKER_OPTS=\\\\\\\"-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock\\\\\\\" \\u003e\\u003e /etc/default/docker\\n\\n# Restart Docker to let port listening take effect.\\nservice docker restart\\n\\n\\nmkdir -p /etc/docker\\ncat \\u003c\\u003c EOF \\u003e /etc/docker/daemon.json\\n{\\n \\\"labels\\\": [\\n \\\"infrakit-link=HmjVIS7jEMNMu3mU\\\",\\n \\\"infrakit-link-context=swarm/t9vg5zqmdtw8ovhniwifszwbw/manager\\\"\\n]\\n}\\nEOF\\n\\n\\nkill -s HUP $(cat /var/run/docker.pid)\\nsleep 5\\n\\n\\n\\n \\n \\n\\n \\n docker swarm join 192.168.2.200:4243 --token SWMTKN-1-69316t20viiyh4bwwg8ae2rx6p6obqojfnnxbwfo4dbn3b0npx-68n9c4khokso84f3unn53c15h\\n\\n\\n\\n\\n\",\"LogicalID\":\"192.168.2.202\",\"Attachments\":null}},\"id\":204406773822125757}\n" +time="2017-01-31T16:38:22-08:00" level=debug msg="Received response HTTP/1.1 200 OK\r\nContent-Length: 1289\r\nContent-Type: text/plain; charset=utf-8\r\nDate: Wed, 01 Feb 2017 00:38:22 GMT\r\n\r\n{\"jsonrpc\":\"2.0\",\"result\":{\"Type\":\"manager\",\"Spec\":{\"Properties\":{\"Box\":\"ubuntu/trusty64\"},\"Tags\":{\"infrakit-link\":\"gOII3YTODwN55QNZ\",\"infrakit-link-context\":\"swarm/t9vg5zqmdtw8ovhniwifszwbw/manager\",\"infrakit.config_sha\":\"UBS6BBMA84vUjjtMceeraQGP_eQ=\",\"infrakit.group\":\"swarm-managers\",\"swarm-id\":\"t9vg5zqmdtw8ovhniwifszwbw\"},\"Init\":\"#!/bin/sh\\nset -o errexit\\nset -o nounset\\nset -o xtrace\\n\\n\\n\\n# Tested on Ubuntu/trusty\\n\\napt-get update -y\\napt-get upgrade -y\\nwget -qO- https://get.docker.com/ | sh\\n\\n# Tell Docker to listen on port 4243 for remote API access. This is optional.\\necho DOCKER_OPTS=\\\\\\\"-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock\\\\\\\" \\u003e\\u003e /etc/default/docker\\n\\n# Restart Docker to let port listening take effect.\\nservice docker restart\\n\\n\\nmkdir -p /etc/docker\\ncat \\u003c\\u003c EOF \\u003e /etc/docker/daemon.json\\n{\\n \\\"labels\\\": [\\n \\\"infrakit-link=gOII3YTODwN55QNZ\\\",\\n \\\"infrakit-link-context=swarm/t9vg5zqmdtw8ovhniwifszwbw/manager\\\"\\n]\\n}\\nEOF\\n\\n\\nkill -s HUP $(cat /var/run/docker.pid)\\nsleep 5\\n\\n\\n\\n \\n \\n\\n \\n docker swarm join 192.168.2.200:4243 --token SWMTKN-1-69316t20viiyh4bwwg8ae2rx6p6obqojfnnxbwfo4dbn3b0npx-68n9c4khokso84f3unn53c15h\\n\\n\\n\\n\\n\",\"LogicalID\":\"192.168.2.201\",\"Attachments\":null}},\"id\":5501896949660804411}\n" +time="2017-01-31T16:38:22-08:00" level=debug msg="Sending request POST / HTTP/1.1\r\nHost: a\r\nContent-Type: application/json\r\n\r\n{\"jsonrpc\":\"2.0\",\"method\":\"Instance.Provision\",\"params\":{\"Type\":\"\",\"Spec\":{\"Properties\":{\"Box\":\"ubuntu/trusty64\"},\"Tags\":{\"infrakit-link\":\"HmjVIS7jEMNMu3mU\",\"infrakit-link-context\":\"swarm/t9vg5zqmdtw8ovhniwifszwbw/manager\",\"infrakit.config_sha\":\"UBS6BBMA84vUjjtMceeraQGP_eQ=\",\"infrakit.group\":\"swarm-managers\",\"swarm-id\":\"t9vg5zqmdtw8ovhniwifszwbw\"},\"Init\":\"#!/bin/sh\\nset -o errexit\\nset -o nounset\\nset -o xtrace\\n\\n\\n\\n# Tested on Ubuntu/trusty\\n\\napt-get update -y\\napt-get upgrade -y\\nwget -qO- https://get.docker.com/ | sh\\n\\n# Tell Docker to listen on port 4243 for remote API access. This is optional.\\necho DOCKER_OPTS=\\\\\\\"-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock\\\\\\\" \\u003e\\u003e /etc/default/docker\\n\\n# Restart Docker to let port listening take effect.\\nservice docker restart\\n\\n\\nmkdir -p /etc/docker\\ncat \\u003c\\u003c EOF \\u003e /etc/docker/daemon.json\\n{\\n \\\"labels\\\": [\\n \\\"infrakit-link=HmjVIS7jEMNMu3mU\\\",\\n \\\"infrakit-link-context=swarm/t9vg5zqmdtw8ovhniwifszwbw/manager\\\"\\n]\\n}\\nEOF\\n\\n\\nkill -s HUP $(cat /var/run/docker.pid)\\nsleep 5\\n\\n\\n\\n \\n \\n\\n \\n docker swarm join 192.168.2.200:4243 --token SWMTKN-1-69316t20viiyh4bwwg8ae2rx6p6obqojfnnxbwfo4dbn3b0npx-68n9c4khokso84f3unn53c15h\\n\\n\\n\\n\\n\",\"LogicalID\":\"192.168.2.202\",\"Attachments\":null}},\"id\":3237913983924951943}" +time="2017-01-31T16:38:22-08:00" level=debug msg="Sending request POST / HTTP/1.1\r\nHost: a\r\nContent-Type: application/json\r\n\r\n{\"jsonrpc\":\"2.0\",\"method\":\"Instance.Provision\",\"params\":{\"Type\":\"\",\"Spec\":{\"Properties\":{\"Box\":\"ubuntu/trusty64\"},\"Tags\":{\"infrakit-link\":\"gOII3YTODwN55QNZ\",\"infrakit-link-context\":\"swarm/t9vg5zqmdtw8ovhniwifszwbw/manager\",\"infrakit.config_sha\":\"UBS6BBMA84vUjjtMceeraQGP_eQ=\",\"infrakit.group\":\"swarm-managers\",\"swarm-id\":\"t9vg5zqmdtw8ovhniwifszwbw\"},\"Init\":\"#!/bin/sh\\nset -o errexit\\nset -o nounset\\nset -o xtrace\\n\\n\\n\\n# Tested on Ubuntu/trusty\\n\\napt-get update -y\\napt-get upgrade -y\\nwget -qO- https://get.docker.com/ | sh\\n\\n# Tell Docker to listen on port 4243 for remote API access. This is optional.\\necho DOCKER_OPTS=\\\\\\\"-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock\\\\\\\" \\u003e\\u003e /etc/default/docker\\n\\n# Restart Docker to let port listening take effect.\\nservice docker restart\\n\\n\\nmkdir -p /etc/docker\\ncat \\u003c\\u003c EOF \\u003e /etc/docker/daemon.json\\n{\\n \\\"labels\\\": [\\n \\\"infrakit-link=gOII3YTODwN55QNZ\\\",\\n \\\"infrakit-link-context=swarm/t9vg5zqmdtw8ovhniwifszwbw/manager\\\"\\n]\\n}\\nEOF\\n\\n\\nkill -s HUP $(cat /var/run/docker.pid)\\nsleep 5\\n\\n\\n\\n \\n \\n\\n \\n docker swarm join 192.168.2.200:4243 --token SWMTKN-1-69316t20viiyh4bwwg8ae2rx6p6obqojfnnxbwfo4dbn3b0npx-68n9c4khokso84f3unn53c15h\\n\\n\\n\\n\\n\",\"LogicalID\":\"192.168.2.201\",\"Attachments\":null}},\"id\":823635831294852014}" + +==> instance-vagrant.log <== +time="2017-01-31T16:38:22-08:00" level=debug msg="Received request POST / HTTP/1.1\r\nHost: a\r\nAccept-Encoding: gzip\r\nContent-Length: 1311\r\nContent-Type: application/json\r\nUser-Agent: Go-http-client/1.1\r\n\r\n{\"jsonrpc\":\"2.0\",\"method\":\"Instance.Provision\",\"params\":{\"Type\":\"\",\"Spec\":{\"Properties\":{\"Box\":\"ubuntu/trusty64\"},\"Tags\":{\"infrakit-link\":\"HmjVIS7jEMNMu3mU\",\"infrakit-link-context\":\"swarm/t9vg5zqmdtw8ovhniwifszwbw/manager\",\"infrakit.config_sha\":\"UBS6BBMA84vUjjtMceeraQGP_eQ=\",\"infrakit.group\":\"swarm-managers\",\"swarm-id\":\"t9vg5zqmdtw8ovhniwifszwbw\"},\"Init\":\"#!/bin/sh\\nset -o errexit\\nset -o nounset\\nset -o xtrace\\n\\n\\n\\n# Tested on Ubuntu/trusty\\n\\napt-get update -y\\napt-get upgrade -y\\nwget -qO- https://get.docker.com/ | sh\\n\\n# Tell Docker to listen on port 4243 for remote API access. This is optional.\\necho DOCKER_OPTS=\\\\\\\"-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock\\\\\\\" \\u003e\\u003e /etc/default/docker\\n\\n# Restart Docker to let port listening take effect.\\nservice docker restart\\n\\n\\nmkdir -p /etc/docker\\ncat \\u003c\\u003c EOF \\u003e /etc/docker/daemon.json\\n{\\n \\\"labels\\\": [\\n \\\"infrakit-link=HmjVIS7jEMNMu3mU\\\",\\n \\\"infrakit-link-context=swarm/t9vg5zqmdtw8ovhniwifszwbw/manager\\\"\\n]\\n}\\nEOF\\n\\n\\nkill -s HUP $(cat /var/run/docker.pid)\\nsleep 5\\n\\n\\n\\n \\n \\n\\n \\n docker swarm join 192.168.2.200:4243 --token SWMTKN-1-69316t20viiyh4bwwg8ae2rx6p6obqojfnnxbwfo4dbn3b0npx-68n9c4khokso84f3unn53c15h\\n\\n\\n\\n\\n\",\"LogicalID\":\"192.168.2.202\",\"Attachments\":null}},\"id\":3237913983924951943}" +time="2017-01-31T16:38:22-08:00" level=debug msg="Received request POST / HTTP/1.1\r\nHost: a\r\nAccept-Encoding: gzip\r\nContent-Length: 1310\r\nContent-Type: application/json\r\nUser-Agent: Go-http-client/1.1\r\n\r\n{\"jsonrpc\":\"2.0\",\"method\":\"Instance.Provision\",\"params\":{\"Type\":\"\",\"Spec\":{\"Properties\":{\"Box\":\"ubuntu/trusty64\"},\"Tags\":{\"infrakit-link\":\"gOII3YTODwN55QNZ\",\"infrakit-link-context\":\"swarm/t9vg5zqmdtw8ovhniwifszwbw/manager\",\"infrakit.config_sha\":\"UBS6BBMA84vUjjtMceeraQGP_eQ=\",\"infrakit.group\":\"swarm-managers\",\"swarm-id\":\"t9vg5zqmdtw8ovhniwifszwbw\"},\"Init\":\"#!/bin/sh\\nset -o errexit\\nset -o nounset\\nset -o xtrace\\n\\n\\n\\n# Tested on Ubuntu/trusty\\n\\napt-get update -y\\napt-get upgrade -y\\nwget -qO- https://get.docker.com/ | sh\\n\\n# Tell Docker to listen on port 4243 for remote API access. This is optional.\\necho DOCKER_OPTS=\\\\\\\"-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock\\\\\\\" \\u003e\\u003e /etc/default/docker\\n\\n# Restart Docker to let port listening take effect.\\nservice docker restart\\n\\n\\nmkdir -p /etc/docker\\ncat \\u003c\\u003c EOF \\u003e /etc/docker/daemon.json\\n{\\n \\\"labels\\\": [\\n \\\"infrakit-link=gOII3YTODwN55QNZ\\\",\\n \\\"infrakit-link-context=swarm/t9vg5zqmdtw8ovhniwifszwbw/manager\\\"\\n]\\n}\\nEOF\\n\\n\\nkill -s HUP $(cat /var/run/docker.pid)\\nsleep 5\\n\\n\\n\\n \\n \\n\\n \\n docker swarm join 192.168.2.200:4243 --token SWMTKN-1-69316t20viiyh4bwwg8ae2rx6p6obqojfnnxbwfo4dbn3b0npx-68n9c4khokso84f3unn53c15h\\n\\n\\n\\n\\n\",\"LogicalID\":\"192.168.2.201\",\"Attachments\":null}},\"id\":823635831294852014}" +time="2017-01-31T16:38:25-08:00" level=info msg="Vagrant STDOUT: Bringing machine 'default' up with 'virtualbox' provider...\n" +time="2017-01-31T16:38:25-08:00" level=info msg="Vagrant STDOUT: Bringing machine 'default' up with 'virtualbox' provider...\n" +time="2017-01-31T16:38:25-08:00" level=info msg="Vagrant STDOUT: ==> default: Importing base box 'ubuntu/trusty64'...\n" +time="2017-01-31T16:38:25-08:00" level=info msg="Vagrant STDOUT: ==> default: Importing base box 'ubuntu/trusty64'...\n" +time="2017-01-31T16:38:40-08:00" level=info msg="Vagrant STDOUT: \r\x1b[KProgress: 90%\r\x1b[K==> default: Matching MAC address for NAT networking...\n" +time="2017-01-31T16:38:40-08:00" level=info msg="Vagrant STDOUT: \r\x1b[KProgress: 90%\r\x1b[K==> default: Matching MAC address for NAT networking...\n" +time="2017-01-31T16:38:40-08:00" level=info msg="Vagrant STDOUT: ==> default: Checking if box 'ubuntu/trusty64' is up to date...\n" +time="2017-01-31T16:38:40-08:00" level=info msg="Vagrant STDOUT: ==> default: Checking if box 'ubuntu/trusty64' is up to date...\n" + ``` -Note that the Swarm Plugin is configured with a Docker host. This is used to determine where the join tokens are -fetched from. In this case, we are pointing at the yet-to-be-created Swarm manager node. +That's a lot of log entries! You can always lower the volume by adjusting the `--log` parameters in the `plugins.json` +for the next time. -Next, create the [manager node](swarm-vagrant-manager.json) and initialize the cluster: +After some time, you should be able to see the Swarm cluster up and running by doing: ```shell -$ build/infrakit group commit examples/flavor/swarm/swarm-vagrant-manager.json +vagrant global-status ``` -Once the first node has been successfully created, confirm that the Swarm was initialized: +or conveniently in a separate window, + ```shell -$ docker -H tcp://192.168.2.200:4243 node ls -ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS -exid5ftbv15pgqkfzastnpw9n * infrakit Ready Active Leader +watch -d vagrant global-status # using watch to monitor the vagrant vms ``` - -Now the [worker group](swarm-vagrant-workers.json) may be created: + +Now check the swarm: + ```shell -$ build/infrakit group commit examples/flavor/swarm/swarm-vagrant-workers.json +~/projects/src/github.com/docker/infrakit$ docker -H tcp://192.168.2.200:4243 node ls +ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS +1ag3s3m1avdahg19933io5xjr * infrakit Ready Active Leader +3pbow8fnfwf0y5d8pvibpx7or localhost Ready Active +45x97w5jipsm9xonprgj5393y localhost Ready Active Reachable +6v91xdon6e6lommqne9eeb776 localhost Ready Active +c4z4a4h3p2jz5vwg36zy2rww9 localhost Ready Active +ezkfjfjqmphi90daup2ur1yas localhost Ready Active Reachable ``` -Once completed, the cluster contains two nodes: +Or use Infrakit `group describe` to see the instances: ```shell -$ docker -H tcp://192.168.2.200:4243 node ls -ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS -39hrnf71gzjve3slg51i6mjs4 localhost Ready Active -exid5ftbv15pgqkfzastnpw9n * infrakit Ready Active Leader +~/projects/src/github.com/docker/infrakit$ infrakit group describe swarm-managers +ID LOGICAL TAGS +infrakit-240440289 192.168.2.201 infrakit-link-context=swarm/37phvxcyelv8js1lyqi76hnau/manager,infrakit-link=nbix0txooYIoyiUQ,infrakit.config_sha=mt39WMxI1MX4mFIg03moQjy4OjA=,infrakit.group=swarm-managers,swarm-id=37phvxcyelv8js1lyqi76hnau +infrakit-428836874 192.168.2.202 infrakit-link-context=swarm/37phvxcyelv8js1lyqi76hnau/manager,infrakit-link=dmVwcQ7w49aTj6K7,infrakit.config_sha=mt39WMxI1MX4mFIg03moQjy4OjA=,infrakit.group=swarm-managers,swarm-id=37phvxcyelv8js1lyqi76hnau +infrakit-541061527 192.168.2.200 infrakit-link-context=swarm/?/manager,infrakit-link=wz5NEiBoC02DtO4q,infrakit.config_sha=mt39WMxI1MX4mFIg03moQjy4OjA=,infrakit.group=swarm-managers,swarm-id=? +~/projects/src/github.com/docker/infrakit$ infrakit group describe swarm-workers +ID LOGICAL TAGS +infrakit-177710302 - infrakit-link-context=swarm/37phvxcyelv8js1lyqi76hnau/worker,infrakit-link=Q9Pu1jBVSbwMF8mz,infrakit.config_sha=TbwYlrX9Efh6_wIrLKG6B7Zd24s=,infrakit.group=swarm-workers,swarm-id= +infrakit-266730661 - infrakit-link-context=swarm/37phvxcyelv8js1lyqi76hnau/worker,infrakit-link=BQe9jpmy5cx24XVd,infrakit.config_sha=TbwYlrX9Efh6_wIrLKG6B7Zd24s=,infrakit.group=swarm-workers,swarm-id= +infrakit-782909388 - infrakit-link-context=swarm/37phvxcyelv8js1lyqi76hnau/worker,infrakit-link=jjAgxzSlQrpEVLo3,infrakit.config_sha=TbwYlrX9Efh6_wIrLKG6B7Zd24s=,infrakit.group=swarm-workers,swarm-id= +~/projects/src/github.com/docker/infrakit$ ``` -Finally, clean up the resources: +We can clean up vms after this brief demo: + ```shell -$ build/infrakit group destroy swarm-workers +~/projects/src/github.com/docker/infrakit$ infrakit group destroy swarm-workers +destroy swarm-workers initiated +~/projects/src/github.com/docker/infrakit$ infrakit group destroy swarm-managers +destroy swarm-managers initiated +``` -$ build/infrakit group destroy swarm-managers +And stop all the plugins: + +``` +~/projects/src/github.com/docker/infrakit$ infrakit plugin stop --all +INFO[0000] Stopping flavor-swarm at PID= 69525 +INFO[0000] Process for flavor-swarm exited +INFO[0000] Stopping group at PID= 69522 +INFO[0000] Process for group exited +INFO[0000] Stopping group-stateless at PID= 69524 +INFO[0000] Process for group-stateless exited +INFO[0000] Stopping instance-vagrant at PID= 69527 +INFO[0000] Process for instance-vagrant exited ``` diff --git a/examples/flavor/swarm/e2e-plugins.json b/examples/flavor/swarm/e2e-plugins.json deleted file mode 100644 index ac4f97e37..000000000 --- a/examples/flavor/swarm/e2e-plugins.json +++ /dev/null @@ -1,51 +0,0 @@ -[ - { - "Plugin" : "group-default", - "Launch" : { - "os": { - "Cmd" : "infrakit-group-default --poll-interval 500ms --name group-stateless --log 5 > {{env "LOG_DIR"}}/group-default-{{unixtime}}.log 2>&1 &", - "SamePgID" : true - } - } - } - , - { - "Plugin" : "instance-file", - "Launch" : { - "os" : { - "Cmd" : "infrakit-instance-file --dir {{env "TUTORIAL_DIR"}} --log 5 > {{env "LOG_DIR"}}/instance-file-{{unixtime}}.log 2>&1", - "SamePgID" : true - } - } - } - , - { - "Plugin" : "instance-vagrant", - "Launch" : { - "os" : { - "Cmd" : "infrakit-instance-vagrant --log 5 > {{env "LOG_DIR"}}/instance-vagrant-{{unixtime}}.log 2>&1", - "SamePgID" : true - } - } - } - , - { - "Plugin" : "flavor-vanilla", - "Launch" : { - "os" : { - "Cmd" : "infrakit-flavor-vanilla --log 5 > {{env "LOG_DIR"}}/flavor-vanilla-{{unixtime}}.log 2>&1", - "SamePgID" : true - } - } - } - , - { - "Plugin" : "flavor-swarm", - "Launch" : { - "os" : { - "Cmd" : "infrakit-flavor-swarm --host {{env "SWARM_MANAGER"}} --log 5 > {{env "LOG_DIR"}}/flavor-swarm-{{unixtime}}.log 2>&1", - "SamePgID" : true - } - } - } -] diff --git a/examples/flavor/swarm/e2e.sh b/examples/flavor/swarm/e2e.sh index c3e9e6aed..e9c48b94f 100755 --- a/examples/flavor/swarm/e2e.sh +++ b/examples/flavor/swarm/e2e.sh @@ -23,43 +23,39 @@ cleanup() { } trap cleanup EXIT +INFRAKIT_HOME=${INFRAKIT_HOME:-~/.infrakit} + # infrakit directories -plugins=~/.infrakit/plugins +plugins=$INFRAKIT_HOME/plugins mkdir -p $plugins rm -rf $plugins/* -configstore=~/.infrakit/configs +configstore=$INFRAKIT_HOME/configs mkdir -p $configstore rm -rf $configstore/* # set the leader -- for os / file based leader detection for manager -leaderfile=~/.infrakit/leader +leaderfile=$INFRAKIT_HOME/leader echo group > $leaderfile -# start up multiple instances of manager -- typically we want multiple SETS of plugins and managers -# but here for simplicity just start up with multiple managers and one set of plugins -infrakit-manager --name group --proxy-for-group group-stateless os --leader-file $leaderfile --store-dir $configstore & -infrakit-manager --name group1 --proxy-for-group group-stateless os --leader-file $leaderfile --store-dir $configstore & -infrakit-manager --name group2 --proxy-for-group group-stateless os --leader-file $leaderfile --store-dir $configstore & - -sleep 5 # manager needs to detect leadership - # location of logfiles when plugins are started by the plugin cli # the config json below expects LOG_DIR as an environment variable -LOG_DIR=~/.infrakit/logs +LOG_DIR=$INFRAKIT_HOME/logs mkdir -p $LOG_DIR # see the config josn 'e2e-test-plugins.json' for reference of environment variable E2E_SWARM_DIR -E2E_SWARM_DIR=~/.infrakit/e2e_swarm +E2E_SWARM_DIR=$INFRAKIT_HOME/e2e_swarm mkdir -p $E2E_SWARM_DIR rm -rf $E2E_SWARM_DIR/* +export INFRAKIT_HOME=$INFRAKIT_HOME export LOG_DIR=$LOG_DIR export E2E_SWARM_DIR=$E2E_SWARM_DIR export SWARM_MANAGER="tcp://192.168.2.200:4243" + # note -- on exit, this won't clean up the plugins started by the cli since they will be in a separate process group -infrakit plugin start --wait --config-url file:///$PWD/examples/flavor/swarm/e2e-plugins.json --os group-default instance-vagrant flavor-swarm flavor-vanilla & +infrakit plugin start --wait --config-url file:///$PWD/examples/flavor/swarm/e2e-plugins.json --os manager group-stateless instance-file instance-vagrant flavor-swarm flavor-vanilla & starterpid=$! echo "plugin start pid=$starterpid" diff --git a/examples/flavor/swarm/flavor.go b/examples/flavor/swarm/flavor.go index 0dc760b3b..5521234ed 100644 --- a/examples/flavor/swarm/flavor.go +++ b/examples/flavor/swarm/flavor.go @@ -1,19 +1,21 @@ package main import ( - "bytes" - "errors" "fmt" + "time" log "github.com/Sirupsen/logrus" docker_types "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/client" + "github.com/docker/go-connections/tlsconfig" group_types "github.com/docker/infrakit/pkg/plugin/group/types" "github.com/docker/infrakit/pkg/spi/flavor" "github.com/docker/infrakit/pkg/spi/instance" "github.com/docker/infrakit/pkg/template" "github.com/docker/infrakit/pkg/types" + "github.com/docker/infrakit/pkg/util/docker" "golang.org/x/net/context" ) @@ -21,20 +23,189 @@ const ( ebsAttachment string = "ebs" ) -type swarmFlavor struct { - client client.APIClient - initScript *template.Template +// Spec is the value passed in the `Properties` field of configs +type Spec struct { + + // Attachments indicate the devices that are to be attached to the instance + Attachments map[instance.LogicalID][]instance.Attachment + + // InitScriptTemplateURL overrides the template specified when the plugin started up. + InitScriptTemplateURL string + + // SwarmJoinIP is the IP for managers and workers to join + SwarmJoinIP string + + // Docker holds the connection params to the Docker engine for join tokens, etc. + Docker ConnectInfo +} + +// ConnectInfo holds the connection parameters for connecting to a Docker engine to get join tokens, etc. +type ConnectInfo struct { + Host string + TLS *tlsconfig.Options } -type schema struct { - Attachments map[instance.LogicalID][]instance.Attachment - DockerRestartCommand string +// DockerClient checks the validity of input spec and connects to Docker engine +func DockerClient(spec Spec) (client.APIClient, error) { + if spec.Docker.Host == "" && spec.Docker.TLS == nil { + return nil, fmt.Errorf("no docker connect info") + } + tls := spec.Docker.TLS + if tls == nil { + tls = &tlsconfig.Options{} + } + + return docker.NewDockerClient(spec.Docker.Host, tls) +} + +// baseFlavor is the base implementation. The manager / worker implementations will provide override. +type baseFlavor struct { + getDockerClient func(Spec) (client.APIClient, error) + initScript *template.Template +} + +// Validate checks the configuration of flavor plugin. +func (s *baseFlavor) Validate(flavorProperties *types.Any, allocation group_types.AllocationMethod) error { + if flavorProperties == nil { + return fmt.Errorf("missing config") + } + + spec := Spec{} + err := flavorProperties.Decode(&spec) + + if err != nil { + return err + } + + if spec.Docker.Host == "" && spec.Docker.TLS == nil { + return fmt.Errorf("no docker connect info") + } + + if spec.InitScriptTemplateURL != "" { + _, err := template.NewTemplate(spec.InitScriptTemplateURL, defaultTemplateOptions) + if err != nil { + return err + } + } + + if err := validateIDsAndAttachments(allocation.LogicalIDs, spec.Attachments); err != nil { + return err + } + + return nil } -func parseProperties(flavorProperties *types.Any) (schema, error) { - s := schema{} - err := flavorProperties.Decode(&s) - return s, err +// Healthy determines whether an instance is healthy. This is determined by whether it has successfully joined the +// Swarm. +func (s *baseFlavor) Healthy(flavorProperties *types.Any, inst instance.Description) (flavor.Health, error) { + if flavorProperties == nil { + return flavor.Unknown, fmt.Errorf("missing config") + } + spec := Spec{} + err := flavorProperties.Decode(&spec) + if err != nil { + return flavor.Unknown, err + } + dockerClient, err := s.getDockerClient(spec) + if err != nil { + return flavor.Unknown, err + } + return healthy(dockerClient, inst) +} + +func (s *baseFlavor) prepare(role string, flavorProperties *types.Any, instanceSpec instance.Spec, + allocation group_types.AllocationMethod) (instance.Spec, error) { + + spec := Spec{} + err := flavorProperties.Decode(&spec) + if err != nil { + return instanceSpec, err + } + + initTemplate := s.initScript + + if spec.InitScriptTemplateURL != "" { + + t, err := template.NewTemplate(spec.InitScriptTemplateURL, defaultTemplateOptions) + if err != nil { + return instanceSpec, err + } + + initTemplate = t + log.Infoln("Using", spec.InitScriptTemplateURL, "for init script template") + } + + var swarmID, initScript string + var swarmStatus *swarm.Swarm + var node *swarm.Node + var link *types.Link + + for i := 0; ; i++ { + log.Debugln(role, ">>>", i, "Querying docker swarm") + + dockerClient, err := s.getDockerClient(spec) + if err != nil { + log.Warningln("Cannot connect to Docker:", err) + continue + } + + swarmStatus, node, err = swarmState(dockerClient) + if err != nil { + log.Warningln("Worker prepare:", err) + } + + swarmID := "?" + if swarmStatus != nil { + swarmID = swarmStatus.ID + } + + link = types.NewLink().WithContext("swarm/" + swarmID + "/" + role) + context := &templateContext{ + flavorSpec: spec, + instanceSpec: instanceSpec, + allocation: allocation, + swarmStatus: swarmStatus, + nodeInfo: node, + link: *link, + } + + initScript, err = initTemplate.Render(context) + + log.Debugln(role, ">>> context.retries =", context.retries, "err=", err, "i=", i) + + if err == nil { + break + } + + if context.retries == 0 || i == context.retries { + log.Warningln("Retries exceeded and error:", err) + return instanceSpec, err + } + + log.Infoln("Going to wait for swarm to be ready. i=", i) + time.Sleep(context.poll) + } + + log.Debugln(role, "init script:", initScript) + + instanceSpec.Init = initScript + + if instanceSpec.LogicalID != nil { + if attachments, exists := spec.Attachments[*instanceSpec.LogicalID]; exists { + instanceSpec.Attachments = append(instanceSpec.Attachments, attachments...) + } + } + + // TODO(wfarner): Use the cluster UUID to scope instances for this swarm separately from instances in another + // swarm. This will require plumbing back to Scaled (membership tags). + instanceSpec.Tags["swarm-id"] = swarmID + link.WriteMap(instanceSpec.Tags) + + return instanceSpec, nil +} + +func (s *baseFlavor) Drain(flavorProperties *types.Any, inst instance.Description) error { + return nil } func validateIDsAndAttachments(logicalIDs []instance.LogicalID, @@ -88,54 +259,148 @@ func validateIDsAndAttachments(logicalIDs []instance.LogicalID, return nil } -const ( - // associationTag is a machine tag added to associate machines with Swarm nodes. - associationTag = "swarm-association-id" -) +func swarmState(docker client.APIClient) (status *swarm.Swarm, node *swarm.Node, err error) { + ctx := context.Background() + info, err := docker.Info(ctx) + if err != nil { + log.Warningln("Err docker info:", err) + status = nil + node = nil + return + } + n, _, err := docker.NodeInspectWithRaw(ctx, info.Swarm.NodeID) + if err != nil { + log.Warningln("Err node inspect:", err) + return + } -func generateInitScript(templ *template.Template, - joinIP, joinToken, associationID, restartCommand string) (string, error) { + node = &n - var buffer bytes.Buffer - err := templ.Execute(&buffer, map[string]string{ - "MY_IP": joinIP, - "JOIN_TOKEN": joinToken, - "ASSOCIATION_ID": associationID, - "RESTART_DOCKER": restartCommand, - }) + s, err := docker.SwarmInspect(ctx) if err != nil { - return "", err + log.Warningln("Err swarm inspect:", err) + return } - return buffer.String(), nil + status = &s + return } -func (s swarmFlavor) Validate(flavorProperties *types.Any, allocation group_types.AllocationMethod) error { - properties, err := parseProperties(flavorProperties) - if err != nil { - return err - } - if properties.DockerRestartCommand == "" { - return errors.New("DockerRestartCommand must be specified") - } - if err := validateIDsAndAttachments(allocation.LogicalIDs, properties.Attachments); err != nil { - return err +type templateContext struct { + flavorSpec Spec + instanceSpec instance.Spec + allocation group_types.AllocationMethod + swarmStatus *swarm.Swarm + nodeInfo *swarm.Node + link types.Link + retries int + poll time.Duration +} + +// Funcs implements the template.Context interface +func (c *templateContext) Funcs() []template.Function { + return []template.Function{ + { + Name: "SWARM_CONNECT_RETRIES", + Description: "Connect to the swarm manager", + Func: func(retries int, wait string) interface{} { + c.retries = retries + poll, err := time.ParseDuration(wait) + if err != nil { + poll = 1 * time.Minute + } + c.poll = poll + return "" + }, + }, + { + Name: "SPEC", + Description: "The flavor spec as found in Properties field of the config JSON", + Func: func() interface{} { + return c.flavorSpec + }, + }, + { + Name: "INSTANCE_LOGICAL_ID", + Description: "The logical id for the instance being prepared; can be empty if no logical ids are set (cattle).", + Func: func() string { + if c.instanceSpec.LogicalID != nil { + return string(*c.instanceSpec.LogicalID) + } + return "" + }, + }, + { + Name: "ALLOCATIONS", + Description: "The allocations contain fields such as the size of the group or the list of logical ids.", + Func: func() interface{} { + return c.allocation + }, + }, + { + Name: "INFRAKIT_LABELS", + Description: "The label name to use for linking an InfraKit managed resource somewhere else.", + Func: func() []string { + return c.link.KVPairs() + }, + }, + { + Name: "SWARM_MANAGER_IP", + Description: "The label name to use for linking an InfraKit managed resource somewhere else.", + Func: func() (string, error) { + if c.nodeInfo == nil { + return "", fmt.Errorf("cannot prepare: no node info") + } + if c.nodeInfo.ManagerStatus == nil { + return "", fmt.Errorf("cannot prepare: no manager status") + } + return c.nodeInfo.ManagerStatus.Addr, nil + }, + }, + { + Name: "SWARM_INITIALIZED", + Description: "Returns true if the swarm has been initialized.", + Func: func() bool { + if c.nodeInfo == nil { + return false + } + return c.nodeInfo.ManagerStatus != nil + }, + }, + { + Name: "SWARM_JOIN_TOKENS", + Description: "Returns the swarm JoinTokens object, with either .Manager or .Worker fields", + Func: func() (interface{}, error) { + if c.swarmStatus == nil { + return nil, fmt.Errorf("cannot prepare: no swarm status") + } + return c.swarmStatus.JoinTokens, nil + }, + }, + { + Name: "SWARM_CLUSTER_ID", + Description: "Returns the swarm cluster UUID", + Func: func() (interface{}, error) { + if c.swarmStatus == nil { + return nil, fmt.Errorf("cannot prepare: no swarm status") + } + return c.swarmStatus.ID, nil + }, + }, } - return nil } // Healthy determines whether an instance is healthy. This is determined by whether it has successfully joined the // Swarm. -func healthy(client client.APIClient, - flavorProperties *types.Any, inst instance.Description) (flavor.Health, error) { +func healthy(client client.APIClient, inst instance.Description) (flavor.Health, error) { - associationID, exists := inst.Tags[associationTag] - if !exists { + link := types.NewLinkFromMap(inst.Tags) + if !link.Valid() { log.Info("Reporting unhealthy for instance without an association tag", inst.ID) return flavor.Unhealthy, nil } filter := filters.NewArgs() - filter.Add("label", fmt.Sprintf("%s=%s", associationTag, associationID)) + filter.Add("label", fmt.Sprintf("%s=%s", link.Label(), link.Value())) nodes, err := client.NodeList(context.Background(), docker_types.NodeListOptions{Filters: filter}) if err != nil { @@ -151,7 +416,7 @@ func healthy(client client.APIClient, return flavor.Healthy, nil default: - log.Warnf("Expected at most one node with label %s, but found %s", associationID, nodes) + log.Warnf("Expected at most one node with label %s, but found %s", link.Value(), nodes) return flavor.Healthy, nil } } diff --git a/examples/flavor/swarm/flavor_test.go b/examples/flavor/swarm/flavor_test.go index 3815b4937..0c6d51198 100644 --- a/examples/flavor/swarm/flavor_test.go +++ b/examples/flavor/swarm/flavor_test.go @@ -7,6 +7,7 @@ import ( docker_types "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/swarm" + docker_client "github.com/docker/docker/client" mock_client "github.com/docker/infrakit/pkg/mock/docker/docker/client" group_types "github.com/docker/infrakit/pkg/plugin/group/types" "github.com/docker/infrakit/pkg/spi/flavor" @@ -17,8 +18,8 @@ import ( "github.com/stretchr/testify/require" ) -func templ() *template.Template { - t, err := template.NewTemplate("str://"+DefaultInitScriptTemplate, template.Options{}) +func templ(tpl string) *template.Template { + t, err := template.NewTemplate("str://"+tpl, template.Options{}) if err != nil { panic(err) } @@ -29,26 +30,30 @@ func TestValidate(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - managerFlavor := NewManagerFlavor(mock_client.NewMockAPIClient(ctrl), templ()) - workerFlavor := NewWorkerFlavor(mock_client.NewMockAPIClient(ctrl), templ()) + managerFlavor := NewManagerFlavor(func(Spec) (docker_client.APIClient, error) { + return mock_client.NewMockAPIClient(ctrl), nil + }, templ(DefaultManagerInitScriptTemplate)) + workerFlavor := NewWorkerFlavor(func(Spec) (docker_client.APIClient, error) { + return mock_client.NewMockAPIClient(ctrl), nil + }, templ(DefaultWorkerInitScriptTemplate)) require.NoError(t, workerFlavor.Validate( - types.AnyString(`{"DockerRestartCommand": "systemctl restart docker"}`), + types.AnyString(`{"Docker" : {"Host":"unix:///var/run/docker.sock"}}`), group_types.AllocationMethod{Size: 5})) require.NoError(t, managerFlavor.Validate( - types.AnyString(`{"DockerRestartCommand": "systemctl restart docker"}`), + types.AnyString(`{"Docker" : {"Host":"unix:///var/run/docker.sock"}}`), group_types.AllocationMethod{LogicalIDs: []instance.LogicalID{"127.0.0.1"}})) // Logical ID with multiple attachments is allowed. require.NoError(t, managerFlavor.Validate( types.AnyString(`{ - "DockerRestartCommand": "systemctl restart docker", + "Docker" : {"Host":"unix:///var/run/docker.sock"}, "Attachments": {"127.0.0.1": [{"ID": "a", "Type": "ebs"}, {"ID": "b", "Type": "ebs"}]}}`), group_types.AllocationMethod{LogicalIDs: []instance.LogicalID{"127.0.0.1"}})) // Logical ID used more than once. err := managerFlavor.Validate( - types.AnyString(`{"DockerRestartCommand": "systemctl restart docker"}`), + types.AnyString(`{"Docker":{"Host":"unix:///var/run/docker.sock"}}`), group_types.AllocationMethod{LogicalIDs: []instance.LogicalID{"127.0.0.1", "127.0.0.1", "127.0.0.2"}}) require.Error(t, err) require.Equal(t, "LogicalID 127.0.0.1 specified more than once", err.Error()) @@ -56,7 +61,7 @@ func TestValidate(t *testing.T) { // Attachment cannot be associated with multiple Logical IDs. err = managerFlavor.Validate( types.AnyString(`{ - "DockerRestartCommand": "systemctl restart docker", + "Docker" : {"Host":"unix:///var/run/docker.sock"}, "Attachments": {"127.0.0.1": [{"ID": "a", "Type": "ebs"}], "127.0.0.2": [{"ID": "a", "Type": "ebs"}]}}`), group_types.AllocationMethod{LogicalIDs: []instance.LogicalID{"127.0.0.1", "127.0.0.2", "127.0.0.3"}}) require.Error(t, err) @@ -65,7 +70,7 @@ func TestValidate(t *testing.T) { // Unsupported Attachment Type. err = managerFlavor.Validate( types.AnyString(`{ - "DockerRestartCommand": "systemctl restart docker", + "Docker" : {"Host":"unix:///var/run/docker.sock"}, "Attachments": {"127.0.0.1": [{"ID": "a", "Type": "keyboard"}]}}`), group_types.AllocationMethod{LogicalIDs: []instance.LogicalID{"127.0.0.1"}}) require.Error(t, err) @@ -78,7 +83,9 @@ func TestWorker(t *testing.T) { client := mock_client.NewMockAPIClient(ctrl) - flavorImpl := NewWorkerFlavor(client, templ()) + flavorImpl := NewWorkerFlavor(func(Spec) (docker_client.APIClient, error) { + return client, nil + }, templ(DefaultWorkerInitScriptTemplate)) swarmInfo := swarm.Swarm{ ClusterInfo: swarm.ClusterInfo{ID: "ClusterUUID"}, @@ -87,12 +94,11 @@ func TestWorker(t *testing.T) { Worker: "WorkerToken", }, } - client.EXPECT().SwarmInspect(gomock.Any()).Return(swarmInfo, nil) - - client.EXPECT().Info(gomock.Any()).Return(infoResponse, nil) + client.EXPECT().SwarmInspect(gomock.Any()).Return(swarmInfo, nil).AnyTimes() + client.EXPECT().Info(gomock.Any()).Return(infoResponse, nil).AnyTimes() nodeInfo := swarm.Node{ManagerStatus: &swarm.ManagerStatus{Addr: "1.2.3.4"}} - client.EXPECT().NodeInspectWithRaw(gomock.Any(), nodeID).Return(nodeInfo, nil, nil) + client.EXPECT().NodeInspectWithRaw(gomock.Any(), nodeID).Return(nodeInfo, nil, nil).AnyTimes() details, err := flavorImpl.Prepare( types.AnyString(`{}`), @@ -100,15 +106,18 @@ func TestWorker(t *testing.T) { group_types.AllocationMethod{Size: 5}) require.NoError(t, err) require.Equal(t, "b", details.Tags["a"]) - associationID := details.Tags[associationTag] - require.NotEqual(t, "", associationID) + + link := types.NewLinkFromMap(details.Tags) + require.True(t, link.Valid()) + require.True(t, len(link.KVPairs()) > 0) // Perform a rudimentary check to ensure that the expected fields are in the InitScript, without having any // other knowledge about the script structure. + associationID := link.Value() + associationTag := link.Label() require.Contains(t, details.Init, associationID) require.Contains(t, details.Init, swarmInfo.JoinTokens.Worker) require.NotContains(t, details.Init, swarmInfo.JoinTokens.Manager) - require.Contains(t, details.Init, nodeInfo.ManagerStatus.Addr) require.Empty(t, details.Attachments) @@ -140,7 +149,9 @@ func TestManager(t *testing.T) { client := mock_client.NewMockAPIClient(ctrl) - flavorImpl := NewManagerFlavor(client, templ()) + flavorImpl := NewManagerFlavor(func(Spec) (docker_client.APIClient, error) { + return client, nil + }, templ(DefaultManagerInitScriptTemplate)) swarmInfo := swarm.Swarm{ ClusterInfo: swarm.ClusterInfo{ID: "ClusterUUID"}, @@ -149,12 +160,11 @@ func TestManager(t *testing.T) { Worker: "WorkerToken", }, } - client.EXPECT().SwarmInspect(gomock.Any()).Return(swarmInfo, nil) - - client.EXPECT().Info(gomock.Any()).Return(infoResponse, nil) + client.EXPECT().SwarmInspect(gomock.Any()).Return(swarmInfo, nil).AnyTimes() + client.EXPECT().Info(gomock.Any()).Return(infoResponse, nil).AnyTimes() nodeInfo := swarm.Node{ManagerStatus: &swarm.ManagerStatus{Addr: "1.2.3.4"}} - client.EXPECT().NodeInspectWithRaw(gomock.Any(), nodeID).Return(nodeInfo, nil, nil) + client.EXPECT().NodeInspectWithRaw(gomock.Any(), nodeID).Return(nodeInfo, nil, nil).AnyTimes() id := instance.LogicalID("127.0.0.1") details, err := flavorImpl.Prepare( @@ -163,15 +173,28 @@ func TestManager(t *testing.T) { group_types.AllocationMethod{LogicalIDs: []instance.LogicalID{"127.0.0.1"}}) require.NoError(t, err) require.Equal(t, "b", details.Tags["a"]) - associationID := details.Tags[associationTag] - require.NotEqual(t, "", associationID) + + link := types.NewLinkFromMap(details.Tags) + require.True(t, link.Valid()) + require.True(t, len(link.KVPairs()) > 0) // Perform a rudimentary check to ensure that the expected fields are in the InitScript, without having any // other knowledge about the script structure. + + associationID := link.Value() + associationTag := link.Label() require.Contains(t, details.Init, associationID) + + // another instance -- note that this id is not the first in the allocation list of logical ids. + id = instance.LogicalID("172.200.100.2") + details, err = flavorImpl.Prepare( + types.AnyString(`{"Attachments": {"172.200.100.2": [{"ID": "a", "Type": "gpu"}]}}`), + instance.Spec{Tags: map[string]string{"a": "b"}, LogicalID: &id}, + group_types.AllocationMethod{LogicalIDs: []instance.LogicalID{"172.200.100.1", "172.200.100.2"}}) + require.NoError(t, err) + require.Contains(t, details.Init, swarmInfo.JoinTokens.Manager) require.NotContains(t, details.Init, swarmInfo.JoinTokens.Worker) - require.Contains(t, details.Init, nodeInfo.ManagerStatus.Addr) require.Equal(t, []instance.Attachment{{ID: "a", Type: "gpu"}}, details.Attachments) diff --git a/examples/flavor/swarm/groups-fast.json b/examples/flavor/swarm/groups-fast.json new file mode 100644 index 000000000..d7bb38abd --- /dev/null +++ b/examples/flavor/swarm/groups-fast.json @@ -0,0 +1,61 @@ +{{/* This config uses vagrant box that is pre-installed with Docker for faster performance. */}} +[ + { + "Plugin": "group", + "Properties": { + "ID": "swarm-workers", + "Properties": { + "Allocation": { + "Size": 3 + }, + "Flavor": { + "Plugin": "flavor-swarm/worker", + "Properties": { + "InitScriptTemplateURL": "file://{{env "PWD"}}/examples/flavor/swarm/worker-init-fast.sh", + "SwarmJoinIP": "192.168.2.200", + "Docker" : { + "Host" : "tcp://192.168.2.200:4243" + } + } + }, + "Instance": { + "Plugin": "instance-vagrant", + "Properties": { + "Box": "williamyeh/ubuntu-trusty64-docker" {{/* Already has Docker installed */}} + } + } + } + } + }, + { + "Plugin": "group", + "Properties": { + "ID": "swarm-managers", + "Properties": { + "Allocation": { + "LogicalIDs": [ + "192.168.2.200", + "192.168.2.201", + "192.168.2.202" + ] + }, + "Flavor": { + "Plugin": "flavor-swarm/manager", + "Properties": { + "InitScriptTemplateURL": "file://{{env "PWD"}}/examples/flavor/swarm/manager-init-fast.sh", + "SwarmJoinIP": "192.168.2.200", + "Docker" : { + "Host" : "tcp://192.168.2.200:4243" + } + } + }, + "Instance": { + "Plugin": "instance-vagrant", + "Properties": { + "Box": "williamyeh/ubuntu-trusty64-docker" {{/* Already has Docker installed */}} + } + } + } + } + } +] diff --git a/examples/flavor/swarm/groups.json b/examples/flavor/swarm/groups.json new file mode 100644 index 000000000..24c3162d2 --- /dev/null +++ b/examples/flavor/swarm/groups.json @@ -0,0 +1,61 @@ +{{/* This config uses full install of Docker so it's slower. */}} +[ + { + "Plugin": "group", + "Properties": { + "ID": "swarm-workers", + "Properties": { + "Allocation": { + "Size": 3 + }, + "Flavor": { + "Plugin": "flavor-swarm/worker", + "Properties": { + "InitScriptTemplateURL": "file://{{env "PWD"}}/examples/flavor/swarm/worker-init.sh", + "SwarmJoinIP": "192.168.2.200", + "Docker" : { + "Host" : "tcp://192.168.2.200:4243" + } + } + }, + "Instance": { + "Plugin": "instance-vagrant", + "Properties": { + "Box": "ubuntu/trusty64" + } + } + } + } + }, + { + "Plugin": "group", + "Properties": { + "ID": "swarm-managers", + "Properties": { + "Allocation": { + "LogicalIDs": [ + "192.168.2.200", + "192.168.2.201", + "192.168.2.202" + ] + }, + "Flavor": { + "Plugin": "flavor-swarm/manager", + "Properties": { + "InitScriptTemplateURL": "file://{{env "PWD"}}/examples/flavor/swarm/manager-init.sh", + "SwarmJoinIP": "192.168.2.200", + "Docker" : { + "Host" : "tcp://192.168.2.200:4243" + } + } + }, + "Instance": { + "Plugin": "instance-vagrant", + "Properties": { + "Box": "ubuntu/trusty64" + } + } + } + } + } +] diff --git a/examples/flavor/swarm/install-docker.sh b/examples/flavor/swarm/install-docker.sh new file mode 100644 index 000000000..e0d5fb8c1 --- /dev/null +++ b/examples/flavor/swarm/install-docker.sh @@ -0,0 +1,12 @@ + +# Tested on Ubuntu/trusty + +apt-get update -y +apt-get upgrade -y +wget -qO- https://get.docker.com/ | sh + +# Tell Docker to listen on port 4243 for remote API access. This is optional. +echo DOCKER_OPTS=\"-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock\" >> /etc/default/docker + +# Restart Docker to let port listening take effect. +service docker restart diff --git a/examples/flavor/swarm/main.go b/examples/flavor/swarm/main.go index 92479c6c7..e6e8a4622 100644 --- a/examples/flavor/swarm/main.go +++ b/examples/flavor/swarm/main.go @@ -4,7 +4,6 @@ import ( "os" log "github.com/Sirupsen/logrus" - "github.com/docker/go-connections/tlsconfig" "github.com/docker/infrakit/pkg/cli" "github.com/docker/infrakit/pkg/discovery" flavor_plugin "github.com/docker/infrakit/pkg/rpc/flavor" @@ -21,6 +20,10 @@ func init() { }) } +var defaultTemplateOptions = template.Options{ + SocketDir: discovery.Dir(), +} + func main() { cmd := &cobra.Command{ @@ -29,52 +32,26 @@ func main() { } name := cmd.Flags().String("name", "flavor-swarm", "Plugin name to advertise for discovery") logLevel := cmd.Flags().Int("log", cli.DefaultLogLevel, "Logging level. 0 is least verbose. Max is 5") - host := cmd.Flags().String("host", "unix:///var/run/docker.sock", "Docker host") - caFile := cmd.Flags().String("tlscacert", "", "TLS CA cert file path") - certFile := cmd.Flags().String("tlscert", "", "TLS cert file path") - tlsKey := cmd.Flags().String("tlskey", "", "TLS key file path") - insecureSkipVerify := cmd.Flags().Bool("tlsverify", true, "True to skip TLS") - initScriptTemplURL := cmd.Flags().String("init-template", "", "Init script template file, in URL form") + managerInitScriptTemplURL := cmd.Flags().String("manager-init-template", "", "URL, init script template for managers") + workerInitScriptTemplURL := cmd.Flags().String("worker-init-template", "", "URL, init script template for workers") cmd.RunE = func(c *cobra.Command, args []string) error { cli.SetLogLevel(*logLevel) - dockerClient, err := docker.NewDockerClient(*host, &tlsconfig.Options{ - CAFile: *caFile, - CertFile: *certFile, - KeyFile: *tlsKey, - InsecureSkipVerify: *insecureSkipVerify, - }) - log.Infoln("Connect to docker", host, "err=", err) + mt, err := getTemplate(*managerInitScriptTemplURL, DefaultManagerInitScriptTemplate, defaultTemplateOptions) if err != nil { return err } - - opts := template.Options{ - SocketDir: discovery.Dir(), - } - - var templ *template.Template - if *initScriptTemplURL == "" { - t, err := template.NewTemplate("str://"+DefaultInitScriptTemplate, opts) - if err != nil { - return err - } - templ = t - } else { - - t, err := template.NewTemplate(*initScriptTemplURL, opts) - if err != nil { - return err - } - templ = t + wt, err := getTemplate(*workerInitScriptTemplURL, DefaultWorkerInitScriptTemplate, defaultTemplateOptions) + if err != nil { + return err } cli.RunPlugin(*name, flavor_plugin.PluginServerWithTypes( map[string]flavor.Plugin{ - "manager": NewManagerFlavor(dockerClient, templ), - "worker": NewWorkerFlavor(dockerClient, templ), + "manager": NewManagerFlavor(DockerClient, mt), + "worker": NewWorkerFlavor(DockerClient, wt), })) return nil } @@ -88,24 +65,11 @@ func main() { } } -const ( - // DefaultInitScriptTemplate is the default template for the init script which - // the flavor injects into the user data of the instance to configure Docker Swarm. - DefaultInitScriptTemplate = ` -#!/bin/sh -set -o errexit -set -o nounset -set -o xtrace - -mkdir -p /etc/docker -cat << EOF > /etc/docker/daemon.json -{ - "labels": ["swarm-association-id={{.ASSOCIATION_ID}}"] +func getTemplate(url string, defaultTemplate string, opts template.Options) (t *template.Template, err error) { + if url == "" { + t, err = template.NewTemplate("str://"+defaultTemplate, opts) + return + } + t, err = template.NewTemplate(url, opts) + return } -EOF - -{{.RESTART_DOCKER}} - -docker swarm join {{.MY_IP}} --token {{.JOIN_TOKEN}} -` -) diff --git a/examples/flavor/swarm/manager-init-fast.sh b/examples/flavor/swarm/manager-init-fast.sh new file mode 100644 index 000000000..1ff89e31f --- /dev/null +++ b/examples/flavor/swarm/manager-init-fast.sh @@ -0,0 +1,33 @@ +#!/bin/sh +set -o errexit +set -o nounset +set -o xtrace + +mkdir -p /etc/docker +cat << EOF > /etc/docker/daemon.json +{ + "labels": {{ INFRAKIT_LABELS | to_json }} +} +EOF + +{{/* Reload the engine labels */}} +kill -s HUP $(cat /var/run/docker.pid) +sleep 5 + +{{ if eq INSTANCE_LOGICAL_ID SPEC.SwarmJoinIP }} + + {{/* The first node of the special allocations will initialize the swarm. */}} + docker swarm init --advertise-addr {{ INSTANCE_LOGICAL_ID }} + + # Tell Docker to listen on port 4243 for remote API access. This is optional. + echo DOCKER_OPTS="\"-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock\"" >> /etc/default/docker + + # Restart Docker to let port listening take effect. + service docker restart + +{{ else }} + + {{/* The rest of the nodes will join as followers in the manager group. */}} + docker swarm join --token {{ SWARM_JOIN_TOKENS.Manager }} {{ SPEC.SwarmJoinIP }}:2377 + +{{ end }} diff --git a/examples/flavor/swarm/manager-init.sh b/examples/flavor/swarm/manager-init.sh new file mode 100644 index 000000000..781d5a41b --- /dev/null +++ b/examples/flavor/swarm/manager-init.sh @@ -0,0 +1,36 @@ +#!/bin/sh +set -o errexit +set -o nounset +set -o xtrace + +{{/* Install Docker */}} +{{ include "install-docker.sh" }} + +mkdir -p /etc/docker +cat << EOF > /etc/docker/daemon.json +{ + "labels": {{ INFRAKIT_LABELS | to_json }} +} +EOF + +{{/* Reload the engine labels */}} +kill -s HUP $(cat /var/run/docker.pid) +sleep 5 + +{{ if eq INSTANCE_LOGICAL_ID SPEC.SwarmJoinIP }} + + {{/* The first node of the special allocations will initialize the swarm. */}} + docker swarm init --advertise-addr {{ INSTANCE_LOGICAL_ID }} + + # Tell Docker to listen on port 4243 for remote API access. This is optional. + echo DOCKER_OPTS="\"-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock\"" >> /etc/default/docker + + # Restart Docker to let port listening take effect. + service docker restart + +{{ else }} + + {{/* The rest of the nodes will join as followers in the manager group. */}} + docker swarm join --token {{ SWARM_JOIN_TOKENS.Manager }} {{ SPEC.SwarmJoinIP }}:2377 + +{{ end }} diff --git a/examples/flavor/swarm/manager.go b/examples/flavor/swarm/manager.go index decf90fef..58830cd15 100644 --- a/examples/flavor/swarm/manager.go +++ b/examples/flavor/swarm/manager.go @@ -2,124 +2,51 @@ package main import ( "errors" - "fmt" log "github.com/Sirupsen/logrus" "github.com/docker/docker/client" group_types "github.com/docker/infrakit/pkg/plugin/group/types" - "github.com/docker/infrakit/pkg/plugin/group/util" "github.com/docker/infrakit/pkg/spi/flavor" "github.com/docker/infrakit/pkg/spi/instance" "github.com/docker/infrakit/pkg/template" "github.com/docker/infrakit/pkg/types" - "golang.org/x/net/context" ) // NewManagerFlavor creates a flavor.Plugin that creates manager and worker nodes connected in a swarm. -func NewManagerFlavor(dockerClient client.APIClient, templ *template.Template) flavor.Plugin { - return &managerFlavor{client: dockerClient, initScript: templ} +func NewManagerFlavor(connect func(Spec) (client.APIClient, error), templ *template.Template) flavor.Plugin { + return &managerFlavor{&baseFlavor{initScript: templ, getDockerClient: connect}} } type managerFlavor struct { - client client.APIClient - initScript *template.Template + *baseFlavor } func (s *managerFlavor) Validate(flavorProperties *types.Any, allocation group_types.AllocationMethod) error { - properties, err := parseProperties(flavorProperties) - if err != nil { + + if err := s.baseFlavor.Validate(flavorProperties, allocation); err != nil { return err } - if properties.DockerRestartCommand == "" { - return errors.New("DockerRestartCommand must be specified") + spec := Spec{} + err := flavorProperties.Decode(&spec) + if err != nil { + return err } - numIDs := len(allocation.LogicalIDs) - if numIDs != 1 && numIDs != 3 && numIDs != 5 { - return errors.New("Must have 1, 3, or 5 manager logical IDs") + if len(allocation.LogicalIDs)%2 == 0 { + return errors.New("must have odd number for quorum") } for _, id := range allocation.LogicalIDs { - if att, exists := properties.Attachments[id]; !exists || len(att) == 0 { + if att, exists := spec.Attachments[id]; !exists || len(att) == 0 { log.Warnf("LogicalID %s has no attachments, which is needed for durability", id) } } - - if err := validateIDsAndAttachments(allocation.LogicalIDs, properties.Attachments); err != nil { - return err - } return nil } -// Healthy determines whether an instance is healthy. This is determined by whether it has successfully joined the -// Swarm. -func (s *managerFlavor) Healthy(flavorProperties *types.Any, inst instance.Description) (flavor.Health, error) { - return healthy(s.client, flavorProperties, inst) -} - +// Prepare sets up the provisioner / instance plugin's spec based on information about the swarm to join. func (s *managerFlavor) Prepare(flavorProperties *types.Any, - spec instance.Spec, allocation group_types.AllocationMethod) (instance.Spec, error) { - - properties, err := parseProperties(flavorProperties) - if err != nil { - return spec, err - } - - swarmStatus, err := s.client.SwarmInspect(context.Background()) - if err != nil { - return spec, fmt.Errorf("Failed to fetch Swarm join tokens: %s", err) - } - - nodeInfo, err := s.client.Info(context.Background()) - if err != nil { - return spec, fmt.Errorf("Failed to fetch node self info: %s", err) - } - - self, _, err := s.client.NodeInspectWithRaw(context.Background(), nodeInfo.Swarm.NodeID) - if err != nil { - return spec, fmt.Errorf("Failed to fetch Swarm node status: %s", err) - } - - if self.ManagerStatus == nil { - return spec, errors.New( - "Swarm node status did not include manager status. Need to run 'docker swarm init`?") - } - - associationID := util.RandomAlphaNumericString(8) - spec.Tags[associationTag] = associationID - - if spec.LogicalID == nil { - return spec, errors.New("Manager nodes require a LogicalID, " + - "which will be used as an assigned private IP address") - } - - initScript, err := generateInitScript( - s.initScript, - self.ManagerStatus.Addr, - swarmStatus.JoinTokens.Manager, - associationID, - properties.DockerRestartCommand) - if err != nil { - return spec, err - } - spec.Init = initScript - - if spec.LogicalID != nil { - if attachments, exists := properties.Attachments[*spec.LogicalID]; exists { - spec.Attachments = append(spec.Attachments, attachments...) - } - } - - // TODO(wfarner): Use the cluster UUID to scope instances for this swarm separately from instances in another - // swarm. This will require plumbing back to Scaled (membership tags). - spec.Tags["swarm-id"] = swarmStatus.ID - - return spec, nil -} - -// Drain only explicitly remove worker nodes, not manager nodes. Manager nodes are assumed to have an -// attached volume for state, and fixed IP addresses. This allows them to rejoin as the same node. -func (s *managerFlavor) Drain(flavorProperties *types.Any, inst instance.Description) error { - return nil + instanceSpec instance.Spec, allocation group_types.AllocationMethod) (instance.Spec, error) { + return s.baseFlavor.prepare("manager", flavorProperties, instanceSpec, allocation) } diff --git a/examples/flavor/swarm/plugins.json b/examples/flavor/swarm/plugins.json new file mode 100644 index 000000000..7ec06c89b --- /dev/null +++ b/examples/flavor/swarm/plugins.json @@ -0,0 +1,37 @@ +[ + { + "Plugin" : "manager", + "Launch" : { + "os": { + "Cmd" : "infrakit-manager --name group --proxy-for-group group-stateless os --leader-file {{env "INFRAKIT_HOME"}}/leader --store-dir {{env "INFRAKIT_HOME"}}/configs > {{env "INFRAKIT_HOME"}}/logs/manager.log 2>&1" + } + } + } + , + { + "Plugin" : "group-stateless", + "Launch" : { + "os": { + "Cmd" : "infrakit-group-default --poll-interval 10s --name group-stateless --log 5 > {{env "INFRAKIT_HOME"}}/logs/group-stateless.log 2>&1" + } + } + } + , + { + "Plugin" : "instance-vagrant", + "Launch" : { + "os" : { + "Cmd" : "infrakit-instance-vagrant --log 5 > {{env "INFRAKIT_HOME"}}/logs/instance-vagrant.log 2>&1" + } + } + } + , + { + "Plugin" : "flavor-swarm", + "Launch" : { + "os" : { + "Cmd" : "infrakit-flavor-swarm --log 5 > {{env "INFRAKIT_HOME"}}/logs/flavor-swarm.log 2>&1" + } + } + } +] diff --git a/examples/flavor/swarm/start-plugins.sh b/examples/flavor/swarm/start-plugins.sh new file mode 100755 index 000000000..c861b4d7b --- /dev/null +++ b/examples/flavor/swarm/start-plugins.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset + +HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$HERE/../../.." + +export PATH=$PWD/build:$PATH + +INFRAKIT_HOME=${INFRAKIT_HOME:-~/.infrakit} + +# infrakit directories +plugins=$INFRAKIT_HOME/plugins +mkdir -p $plugins +rm -rf $plugins/* + +configstore=$INFRAKIT_HOME/configs +mkdir -p $configstore +rm -rf $configstore/* + +logs=$INFRAKIT_HOME/logs +mkdir -p $logs + +# set the leader -- for os / file based leader detection for manager +leaderfile=$INFRAKIT_HOME/leader +echo group > $leaderfile + +export INFRAKIT_HOME=$INFRAKIT_HOME + +infrakit plugin start --config-url file:///$PWD/examples/flavor/swarm/plugins.json --exec os \ + manager \ + group-stateless \ + flavor-swarm \ + instance-vagrant + +sleep 5 + +echo "Plugins started." +echo "Do something like: infrakit manager commit file://$PWD/examples/flavor/swarm/groups-fast.json" diff --git a/examples/flavor/swarm/swarm-manager-template.json b/examples/flavor/swarm/swarm-manager-template.json new file mode 100644 index 000000000..aee4cf82f --- /dev/null +++ b/examples/flavor/swarm/swarm-manager-template.json @@ -0,0 +1,35 @@ +#!/bin/sh +set -o errexit +set -o nounset +set -o xtrace + +{{/* */}} +{{ include "install-docker.sh" }} + +mkdir -p /etc/docker +cat << EOF > /etc/docker/daemon.json +{ + "labels": {{ INFRAKIT_LABELS | to_json }} +} +EOF + +{{/* Reload the engine labels */}} +kill -s HUP $(cat /var/run/docker.pid) +sleep 5 + +{{ if index INSTANCE_LOGICAL_ID ALLOCATIONS.LogicalIDs | eq 0 }} + +{{/* The first node of the special allocations will initialize the swarm. */}} +docker swarm init --advertise-addr {{ INSTANCE_LOGICAL_ID }}:4243 + +{{ else }} + + {{/* retries when trying to get the join tokens */}} + {{ SWARM_CONNECT_RETRIES 10 "30s" }} + + {{/* The rest of the nodes will join as followers in the manager group. */}} + docker swarm join {{ SWARM_MANAGER_IP }} --token {{ SWARM_JOIN_TOKENS.Manager }} + + + +{{ end }} diff --git a/examples/flavor/swarm/swarm-vagrant-manager2.json b/examples/flavor/swarm/swarm-vagrant-manager2.json new file mode 100644 index 000000000..3dc5a5a11 --- /dev/null +++ b/examples/flavor/swarm/swarm-vagrant-manager2.json @@ -0,0 +1,20 @@ +{ + "ID": "swarm-managers", + "Properties": { + "Allocation": { + "LogicalIDs": ["192.168.2.200", "192.168.2.201", "192.168.2.202"] + }, + "Instance": { + "Plugin": "instance-vagrant", + "Properties": { + "Box": "ubuntu/trusty64" + } + }, + "Flavor": { + "Plugin": "flavor-swarm/manager", + "Properties": { + "InitScriptTemplateURL" : "file:///Users/davidchung/projects/src/github.com/docker/infrakit/examples/flavor/swarm/swarm-manager-template.json" + } + } + } +} diff --git a/examples/flavor/swarm/swarm-vagrant-workers2.json b/examples/flavor/swarm/swarm-vagrant-workers2.json new file mode 100644 index 000000000..9be6d990f --- /dev/null +++ b/examples/flavor/swarm/swarm-vagrant-workers2.json @@ -0,0 +1,20 @@ +{ + "ID": "swarm-workers", + "Properties": { + "Allocation": { + "Size": 2 + }, + "Instance": { + "Plugin": "instance-vagrant", + "Properties": { + "Box": "ubuntu/trusty64" + } + }, + "Flavor": { + "Plugin": "flavor-swarm/worker", + "Properties": { + "InitScriptTemplateURL" : "file:///Users/davidchung/projects/src/github.com/docker/infrakit/examples/flavor/swarm/swarm-worker-template.json" + } + } + } +} diff --git a/examples/flavor/swarm/templates.go b/examples/flavor/swarm/templates.go new file mode 100644 index 000000000..f6cbb59c7 --- /dev/null +++ b/examples/flavor/swarm/templates.go @@ -0,0 +1,65 @@ +package main + +const ( + // DefaultManagerInitScriptTemplate is the default template for the init script which + // the flavor injects into the user data of the instance to configure Docker Swarm Managers + DefaultManagerInitScriptTemplate = ` +#!/bin/sh +set -o errexit +set -o nounset +set -o xtrace + +mkdir -p /etc/docker +cat << EOF > /etc/docker/daemon.json +{ + "labels": {{ INFRAKIT_LABELS | to_json }} +} +EOF + +{{/* Reload the engine labels */}} +kill -s HUP $(cat /var/run/docker.pid) +sleep 5 + +{{ if eq INSTANCE_LOGICAL_ID SPEC.SwarmJoinIP }} + + {{/* The first node of the special allocations will initialize the swarm. */}} + docker swarm init --advertise-addr {{ INSTANCE_LOGICAL_ID }} + + # Tell Docker to listen on port 4243 for remote API access. This is optional. + echo DOCKER_OPTS="\"-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock\"" >> /etc/default/docker + + # Restart Docker to let port listening take effect. + service docker restart + +{{ else }} + + {{/* The rest of the nodes will join as followers in the manager group. */}} + docker swarm join --token {{ SWARM_JOIN_TOKENS.Manager }} {{ SPEC.SwarmJoinIP }}:2377 + +{{ end }} +` + + // DefaultWorkerInitScriptTemplate is the default template for the init script which + // the flavor injects into the user data of the instance to configure Docker Swarm. + DefaultWorkerInitScriptTemplate = ` +#!/bin/sh +set -o errexit +set -o nounset +set -o xtrace + +mkdir -p /etc/docker +cat << EOF > /etc/docker/daemon.json +{ + "labels": {{ INFRAKIT_LABELS | to_json }} +} +EOF + +# Tell engine to reload labels +kill -s HUP $(cat /var/run/docker.pid) + +sleep 5 + +docker swarm join --token {{ SWARM_JOIN_TOKENS.Worker }} {{ SPEC.SwarmJoinIP }}:2377 + +` +) diff --git a/examples/flavor/swarm/worker-init-fast.sh b/examples/flavor/swarm/worker-init-fast.sh new file mode 100644 index 000000000..5261fc3db --- /dev/null +++ b/examples/flavor/swarm/worker-init-fast.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -o errexit +set -o nounset +set -o xtrace + +mkdir -p /etc/docker +cat << EOF > /etc/docker/daemon.json +{ + "labels": {{ INFRAKIT_LABELS | to_json }} +} +EOF + +# Tell engine to reload labels +kill -s HUP $(cat /var/run/docker.pid) + +sleep 5 + +docker swarm join --token {{ SWARM_JOIN_TOKENS.Worker }} {{ SPEC.SwarmJoinIP }}:2377 diff --git a/examples/flavor/swarm/worker-init.sh b/examples/flavor/swarm/worker-init.sh new file mode 100644 index 000000000..4ee7436fe --- /dev/null +++ b/examples/flavor/swarm/worker-init.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -o errexit +set -o nounset +set -o xtrace + +{{/* Install Docker */}} +{{ include "install-docker.sh" }} + +mkdir -p /etc/docker +cat << EOF > /etc/docker/daemon.json +{ + "labels": {{ INFRAKIT_LABELS | to_json }} +} +EOF + +# Tell engine to reload labels +kill -s HUP $(cat /var/run/docker.pid) + +sleep 5 + +docker swarm join --token {{ SWARM_JOIN_TOKENS.Worker }} {{ SPEC.SwarmJoinIP }}:2377 diff --git a/examples/flavor/swarm/worker.go b/examples/flavor/swarm/worker.go index 4c20db8c2..54b24f806 100644 --- a/examples/flavor/swarm/worker.go +++ b/examples/flavor/swarm/worker.go @@ -1,14 +1,15 @@ package main import ( - "errors" "fmt" + //"time" + log "github.com/Sirupsen/logrus" docker_types "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" + //"github.com/docker/docker/api/types/swarm" "github.com/docker/docker/client" group_types "github.com/docker/infrakit/pkg/plugin/group/types" - "github.com/docker/infrakit/pkg/plugin/group/util" "github.com/docker/infrakit/pkg/spi/flavor" "github.com/docker/infrakit/pkg/spi/instance" "github.com/docker/infrakit/pkg/template" @@ -17,105 +18,46 @@ import ( ) // NewWorkerFlavor creates a flavor.Plugin that creates manager and worker nodes connected in a swarm. -func NewWorkerFlavor(dockerClient client.APIClient, templ *template.Template) flavor.Plugin { - return &workerFlavor{client: dockerClient, initScript: templ} +func NewWorkerFlavor(connect func(Spec) (client.APIClient, error), templ *template.Template) flavor.Plugin { + return &workerFlavor{&baseFlavor{initScript: templ, getDockerClient: connect}} } type workerFlavor struct { - client client.APIClient - initScript *template.Template + *baseFlavor } -func (s *workerFlavor) Validate(flavorProperties *types.Any, allocation group_types.AllocationMethod) error { - properties, err := parseProperties(flavorProperties) - if err != nil { - return err - } - - if properties.DockerRestartCommand == "" { - return errors.New("DockerRestartCommand must be specified") - } - - if err := validateIDsAndAttachments(allocation.LogicalIDs, properties.Attachments); err != nil { - return err - } - - return nil -} - -// Healthy determines whether an instance is healthy. This is determined by whether it has successfully joined the -// Swarm. -func (s *workerFlavor) Healthy(flavorProperties *types.Any, inst instance.Description) (flavor.Health, error) { - return healthy(s.client, flavorProperties, inst) -} - -func (s *workerFlavor) Prepare(flavorProperties *types.Any, spec instance.Spec, +// Prepare sets up the provisioner / instance plugin's spec based on information about the swarm to join. +func (s *workerFlavor) Prepare(flavorProperties *types.Any, instanceSpec instance.Spec, allocation group_types.AllocationMethod) (instance.Spec, error) { + return s.baseFlavor.prepare("worker", flavorProperties, instanceSpec, allocation) +} - properties, err := parseProperties(flavorProperties) - if err != nil { - return spec, err - } - - swarmStatus, err := s.client.SwarmInspect(context.Background()) - if err != nil { - return spec, fmt.Errorf("Failed to fetch Swarm join tokens: %s", err) - } - - nodeInfo, err := s.client.Info(context.Background()) - if err != nil { - return spec, fmt.Errorf("Failed to fetch node self info: %s", err) +// Drain in the case of worker will force a node removal in the swarm. +func (s *workerFlavor) Drain(flavorProperties *types.Any, inst instance.Description) error { + if flavorProperties == nil { + return fmt.Errorf("missing config") } - self, _, err := s.client.NodeInspectWithRaw(context.Background(), nodeInfo.Swarm.NodeID) + spec := Spec{} + err := flavorProperties.Decode(&spec) if err != nil { - return spec, fmt.Errorf("Failed to fetch Swarm node status: %s", err) - } - - if self.ManagerStatus == nil { - return spec, errors.New( - "Swarm node status did not include manager status. Need to run 'docker swarm init`?") + return err } - associationID := util.RandomAlphaNumericString(8) - spec.Tags[associationTag] = associationID - - initScript, err := - generateInitScript( - s.initScript, - self.ManagerStatus.Addr, - swarmStatus.JoinTokens.Worker, - associationID, - properties.DockerRestartCommand) + dockerClient, err := s.baseFlavor.getDockerClient(spec) if err != nil { - return spec, err - } - spec.Init = initScript - - if spec.LogicalID != nil { - if attachments, exists := properties.Attachments[*spec.LogicalID]; exists { - spec.Attachments = append(spec.Attachments, attachments...) - } + return err } - // TODO(wfarner): Use the cluster UUID to scope instances for this swarm separately from instances in another - // swarm. This will require plumbing back to Scaled (membership tags). - spec.Tags["swarm-id"] = swarmStatus.ID - - return spec, nil -} - -func (s *workerFlavor) Drain(flavorProperties *types.Any, inst instance.Description) error { - - associationID, exists := inst.Tags[associationTag] - if !exists { + link := types.NewLinkFromMap(inst.Tags) + if !link.Valid() { return fmt.Errorf("Unable to drain %s without an association tag", inst.ID) } filter := filters.NewArgs() - filter.Add("label", fmt.Sprintf("%s=%s", associationTag, associationID)) + filter.Add("label", fmt.Sprintf("%s=%s", link.Label(), link.Value())) - nodes, err := s.client.NodeList(context.Background(), docker_types.NodeListOptions{Filters: filter}) + nodes, err := dockerClient.NodeList(context.Background(), docker_types.NodeListOptions{Filters: filter}) if err != nil { return err } @@ -125,7 +67,8 @@ func (s *workerFlavor) Drain(flavorProperties *types.Any, inst instance.Descript return fmt.Errorf("Unable to drain %s, not found in swarm", inst.ID) case len(nodes) == 1: - err := s.client.NodeRemove( + log.Debugln("Docker NodeRemove", nodes[0].ID) + err := dockerClient.NodeRemove( context.Background(), nodes[0].ID, docker_types.NodeRemoveOptions{Force: true}) @@ -136,6 +79,6 @@ func (s *workerFlavor) Drain(flavorProperties *types.Any, inst instance.Descript return nil default: - return fmt.Errorf("Expected at most one node with label %s, but found %s", associationID, nodes) + return fmt.Errorf("Expected at most one node with label %s, but found %s", link.Value(), nodes) } } diff --git a/pkg/plugin/group/group.go b/pkg/plugin/group/group.go index 38a4b1b3f..15544b8c2 100644 --- a/pkg/plugin/group/group.go +++ b/pkg/plugin/group/group.go @@ -74,7 +74,7 @@ func (p *plugin) CommitGroup(config group.Spec, pretend bool) (string, error) { // TODO(wfarner): Don't hold the lock - this is a blocking operation. updatePlan, err := context.supervisor.PlanUpdate(context.scaled, context.settings, settings) if err != nil { - return updatePlan.Explain(), err + return "unable to fulfill request", err } if !pretend { diff --git a/pkg/rpc/manager/rpc_test.go b/pkg/rpc/manager/rpc_test.go new file mode 100644 index 000000000..4287570f1 --- /dev/null +++ b/pkg/rpc/manager/rpc_test.go @@ -0,0 +1,86 @@ +package manager + +import ( + "errors" + "io/ioutil" + "path" + "testing" + + "github.com/docker/infrakit/pkg/manager" + "github.com/docker/infrakit/pkg/rpc/server" + "github.com/stretchr/testify/require" +) + +type testPlugin struct { + // IsLeader returns true if manager is leader + DoIsLeader func() (bool, error) +} + +func (t *testPlugin) IsLeader() (bool, error) { + return t.DoIsLeader() +} + +func tempSocket() string { + dir, err := ioutil.TempDir("", "infrakit-test-") + if err != nil { + panic(err) + } + + return path.Join(dir, "manager-impl-test") +} + +func must(m manager.Manager, err error) manager.Manager { + if err != nil { + panic(err) + } + return m +} + +func TestManagerIsLeader(t *testing.T) { + socketPath := tempSocket() + + rawActual := make(chan bool, 1) + expect := true + + server, err := server.StartPluginAtPath(socketPath, PluginServer(&testPlugin{ + DoIsLeader: func() (bool, error) { + + rawActual <- expect + + return expect, nil + }, + })) + require.NoError(t, err) + + actual, err := must(NewClient(socketPath)).IsLeader() + require.NoError(t, err) + + server.Stop() + + require.Equal(t, expect, <-rawActual) + require.Equal(t, expect, actual) +} + +func TestManagerIsLeaderError(t *testing.T) { + socketPath := tempSocket() + + called := make(chan struct{}) + expect := errors.New("backend-error") + + server, err := server.StartPluginAtPath(socketPath, PluginServer(&testPlugin{ + DoIsLeader: func() (bool, error) { + + close(called) + + return false, expect + }, + })) + require.NoError(t, err) + + _, err = must(NewClient(socketPath)).IsLeader() + require.Error(t, err) + <-called + + server.Stop() + +} diff --git a/pkg/template/template.go b/pkg/template/template.go index 0605ab76e..3c3d85998 100644 --- a/pkg/template/template.go +++ b/pkg/template/template.go @@ -197,7 +197,7 @@ func makeTemplateFunc(ctx Context, f interface{}) (interface{}, error) { return nil, fmt.Errorf("not a function:%v", f) } - if ff.Type().In(0).AssignableTo(contextType) { + if ff.Type().NumIn() > 0 && ff.Type().In(0).AssignableTo(contextType) { in := make([]reflect.Type, ff.Type().NumIn()-1) // exclude the context param out := make([]reflect.Type, ff.Type().NumOut()) diff --git a/pkg/template/template_test.go b/pkg/template/template_test.go index 441081d9e..a67076369 100644 --- a/pkg/template/template_test.go +++ b/pkg/template/template_test.go @@ -137,6 +137,14 @@ func (s *context) Funcs() []Function { return c.(*context).Bool }, }, + { + Name: "invokes", + Description: "prints the invokes count", + Func: func() int { + s.invokes++ + return s.invokes + }, + }, } } diff --git a/pkg/template/types.go b/pkg/template/types.go deleted file mode 100644 index 38cdfe449..000000000 --- a/pkg/template/types.go +++ /dev/null @@ -1 +0,0 @@ -package template diff --git a/pkg/types/link.go b/pkg/types/link.go new file mode 100644 index 000000000..0e172cd3a --- /dev/null +++ b/pkg/types/link.go @@ -0,0 +1,124 @@ +package types + +import ( + "fmt" + "math/rand" + "time" +) + +const ( + letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" +) + +func init() { + rand.Seed(int64(time.Now().Nanosecond())) +} + +// Link is a struct that represents an association between an infrakit managed resource +// and an entity in some other system. The mechanism of linkage is via labels or tags +// on both sides. +type Link struct { + value string + context string +} + +// NewLink creates a link +func NewLink() *Link { + return &Link{ + value: randomAlphaNumericString(16), + } +} + +// NewLinkFromMap constructs a link from data in the map +func NewLinkFromMap(m map[string]string) *Link { + l := &Link{} + if v, has := m["infrakit-link"]; has { + l.value = v + } + + if v, has := m["infrakit-link-context"]; has { + l.context = v + } + return l +} + +// Valid returns true if the link value is set +func (l Link) Valid() bool { + return l.value != "" +} + +// Value returns the value of the link +func (l Link) Value() string { + return l.value +} + +// Label returns the label to look for the link +func (l Link) Label() string { + return "infrakit-link" +} + +// Context returns the context of the link +func (l Link) Context() string { + return l.context +} + +// WithContext sets a context for this link +func (l *Link) WithContext(s string) *Link { + l.context = s + return l +} + +// KVPairs returns the link representation as a slice of Key=Value pairs +func (l *Link) KVPairs() []string { + out := []string{} + for k, v := range l.Map() { + out = append(out, fmt.Sprintf("%s=%s", k, v)) + } + return out +} + +// Map returns a representation that is easily converted to JSON or YAML +func (l *Link) Map() map[string]string { + return map[string]string{ + "infrakit-link": l.value, + "infrakit-link-context": l.context, + } +} + +// WriteMap writes to the target map. This will overwrite values of same key +func (l *Link) WriteMap(target map[string]string) { + for k, v := range l.Map() { + target[k] = v + } +} + +// InMap returns true if the link is contained in the map +func (l *Link) InMap(m map[string]string) bool { + c, has := m["infrakit-link-context"] + if !has { + return false + } + if c != l.context { + return false + } + + v, has := m["infrakit-link"] + if !has { + return false + } + return v == l.value +} + +// Equal returns true if the links are the same - same value and context +func (l *Link) Equal(other Link) bool { + return l.value == other.value && l.context == other.context +} + +// randomAlphaNumericString generates a non-secure random alpha-numeric string of a given length. +func randomAlphaNumericString(length int) string { + b := make([]byte, length) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +}