From 567a3e757fbadd652540628095a59868f773c27f Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 1 May 2023 15:32:32 -0500 Subject: [PATCH 1/3] make much simpler config parsing, see #31 --- README.md | 51 +- app/config/config.go | 367 ++++--- app/config/config_test.go | 907 ++++++++++++------ .../testdata/playbook-with-inventory.yml | 44 + app/main.go | 37 +- app/main_test.go | 8 +- spot-example.yml | 9 +- 7 files changed, 853 insertions(+), 570 deletions(-) create mode 100644 app/config/testdata/playbook-with-inventory.yml diff --git a/README.md b/README.md index 0a2b8f98..1926c6f2 100644 --- a/README.md +++ b/README.md @@ -65,19 +65,22 @@ SimploTask supports the following command-line options: ## Example playbook ```yaml -user: umputun -ssh_key: keys/id_rsa +user: umputun # default ssh user. Can be overridden by -u flag or by inventory or host definition +ssh_key: keys/id_rsa # ssh key +inventory: /etc/spot/inventory.yml # default inventory file. Can be overridden by --inventory-file flag # list of targets, i.e. hosts, inventory files or inventory URLs targets: prod: - hosts: [{host: "h1.example.com", user: "user2"}, {"h2.example.com", port: 2222}] + hosts: # list of hosts, user, name and port optional. + - {host: "h1.example.com", user: "user2", name: "h1"} + - {host: "h2.example.com", port: 2222} staging: - inventory_file: {location: "/srv/etc/inventory.yml", groups: ["staging"]} + groups: ["dev", "staging"] # list of groups from inventory file dev: - inventory_url: {location: "http://localhost:8080/inventory", groups: ["dev"]} - dev_and_staging: - inventory_file: {location: "testdata/inventory"} + groups: ["dev"] # list of groups from inventory file + all: + groups: ["all"] # all hosts from all groups from inventory file # list of tasks, i.e. commands to execute tasks: @@ -211,9 +214,9 @@ By using this approach, SimploTask enables users to write and execute more compl Targets are used to define the remote hosts to execute the tasks on. Targets can be defined in the playbook file or passed as a command-line argument. The following target types are supported: -- `hosts`: a list of destination host names or IP addresses, with optional port and username, to execute the tasks on. Example: `hosts: [{host: "h1.example.com", user: test}, {host: "h2.example.com", "port": 2222}]`. If no user is specified, the user defined in the top section of the playbook file (or override) will be used. If no port is specified, port 22 will be used. -- `inventory_file`: a path to the inventory file to use and groups to use. Example: `inventory_file: {"location": "testdata/inventory", "groups": [{"gr1", "gr2"}] }`. If `groups` not defined all the groups will be used. The [inventory file](#inventory-file-format) contains a list of host names or IP addresses, one per line with optional `[group]` values. -- `inventory_url`: a URL to the inventory file to use. Example: `inventory_url: {"location": "http://localhost:8080/inventory"}`. The response contains a list of host names or IP addresses, one per line. The same support for groups as for `inventory_file` is available. +- `hosts`: a list of destination host names or IP addresses, with optional port and username, to execute the tasks on. Example: `hosts: [{host: "h1.example.com", user: "test", name: "h1}, {host: "h2.example.com", "port": 2222}]`. If no user is specified, the user defined in the top section of the playbook file (or override) will be used. If no port is specified, port 22 will be used. +- `groups`: a list of groups from inventory to use. Example: `groups: ["dev", "staging"}`. Special group `all` combines all the groups. The [inventory file](#inventory-file-format) contains a list of hosts and groups with hosts. + Targets contains environments each of which represents a set of hosts, for example: @@ -222,25 +225,28 @@ targets: prod: hosts: [{host: "h1.example.com", user: "test"}, {"h2.example.com", "port": 2222}] staging: - inventory_file: {location: "testdata/inventory", groups: ["staging"]} + groups: ["staging"] dev: - inventory_url: {location: "http://localhost:8080/inventory", groups: ["dev", "staging"]} + groups: ["dev", "staging"] + all-servers: + groups: ["all"] ``` ### Target overrides There are several ways to override or alter the target defined in the playbook file: -- `--inventory-file` set hosts from the provided inventory file. Example: `--inventory-file=inventory.yml`. -- `--inventory-url` set hosts from the provided inventory URL. Example: `--inventory-url=http://localhost:8080/inventory`. -- `--filter`, `-i`: Set the allowed hosts using the provided name or host address. This flag acts as a filter for the hosts defined in the playbook file or inventory. For instance, if a user has a playbook file with 10 hosts but only wants to execute the tasks on 3 of them, the `--host` flag can be used to specify (filter) the desired host names and host addresses to execute the tasks on. Example usage: `--host=h1.example.com --host=h2.example.com -h=my-cool-host`. - +- `--inventory` set hosts from the provided inventory file or url. Example: `--inventory=inventory.yml` or `--inventory=http://localhost:8080/inventory`. +- `--target` set groups from inventory or directly hosts to run playbook on. Example: `--target=prod` (will run on all hosts in group `prod`) or `--target=example.com:2222` (will run on host `example.com` with port `2222`). +- `--user` set the ssh user to run the playbook on remote hosts. Example: `--user=test`. +- `--key` set the ssh key to run the playbook on remote hosts. Example: `--key=/path/to/key`. -### Inventory file format +### Inventory The inventory file is a simple yml what can represent a list of hosts or a list of groups with hosts. In case if both groups and hosts defined, the hosts will be merged with groups and will add a new group named `hosts`. - +By default, inventory loaded from the file/url set in `SPOT_INVENTORY` environment variable. This is the lowest priority and can be overridden by `inventory` from the playbook (next priority) and `--inventory` flag (highest priority) +. This is an example of the inventory file with groups ```yaml @@ -255,7 +261,7 @@ groups: - {host: "h6.example.com", user: "user3", name: "h6"} ``` -In case if port not defined, the default port 22 will be used. If user not defined, the playbooks user will be used. +In case if port not defined, the default port 22 will be used. If user not defined, the playbook's user will be used. note: the `name` field is optional and used only to make reports/log more readable. @@ -271,6 +277,7 @@ hosts: ``` This format is useful when you want to define a list of hosts without groups. +In each case inventory automatically merged and a special group `all` will be created that contains all the hosts. ## Runtime variables @@ -304,11 +311,11 @@ tasks: ## Adhoc commands -SimploTask supports adhoc commands that can be executed on the remote hosts. This is useful when all is needed is to execute a command on the remote hosts without creating a playbook file. This command passed as `--cmd=` flag and should always be accompanied by the `--tarbget=` (`-d ) flags. Example: `spot --cmd="ls -la" -d h1.example.com -d h2.example.com`. +SimploTask supports adhoc commands that can be executed on the remote hosts. This is useful when all is needed is to execute a command on the remote hosts without creating a playbook file. This command optionally passed as a first argument, i.e. `spot "la -la /tmp` and should always be accompanied by the `--target=` (`-d `) flags. Example: `spot "ls -la" -d h1.example.com -d h2.example.com`. -All other overrides can be used with adhoc commands as well, for example `--user`and `--key` to specify the user and sshkey to use when connecting to the remote hosts. By default, SimploTask will use the current user and the default ssh key. Inventory can be passed to such commands as well, for example `--inventory-file=inventory.yml` or `--inventory-url=http://localhost:8080/inventory` as well as `--filter` (`-f`) to filter the hosts to execute the command on. +All other overrides can be used with adhoc commands as well, for example `--user`and `--key` to specify the user and sshkey to use when connecting to the remote hosts. By default, SimploTask will use the current user and the default ssh key. Inventory can be passed to such commands as well, for example `--inventory=inventory.yml`. -Adhoc commands always set `verbose` to `true` automatically, so the user can see the output of the command. +Adhoc commands always sets `verbose` to `true` automatically, so the user can see the output of the command. ## Rolling Updates diff --git a/app/config/config.go b/app/config/config.go index dfed9888..8ab3c9f0 100644 --- a/app/config/config.go +++ b/app/config/config.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "log" - "net" "net/http" "os" "sort" @@ -20,20 +19,21 @@ import ( // PlayBook defines top-level config yaml type PlayBook struct { - User string `yaml:"user"` - SSHKey string `yaml:"ssh_key"` - Targets map[string]Target `yaml:"targets"` - Tasks []Task `yaml:"tasks"` - - overrides *Overrides + User string `yaml:"user"` // ssh user + SSHKey string `yaml:"ssh_key"` // ssh key + Inventory string `yaml:"inventory"` // inventory file or url + Targets map[string]Target `yaml:"targets"` // list of targets/environments + Tasks []Task `yaml:"tasks"` // list of tasks + + inventory *InventoryData // loaded inventory + overrides *Overrides // overrides passed from cli } // Target defines hosts to run commands on type Target struct { - Name string `yaml:"name"` - Hosts []Destination `yaml:"hosts"` - InventoryFile Inventory `yaml:"inventory_file"` - InventoryURL Inventory `yaml:"inventory_url"` + Name string `yaml:"name"` + Hosts []Destination `yaml:"hosts"` // direct list of hosts to run commands on, no need to use inventory + Groups []string `yaml:"groups"` // list of groups to run commands on, matches to inventory } // Task defines multiple commands runs together @@ -98,18 +98,10 @@ type WaitInternal struct { // Overrides defines override for task passed from cli type Overrides struct { - User string - FilterHosts []string - InventoryFile string - InventoryURL string - Environment map[string]string - AdHocCommand string -} - -// Inventory defines external inventory file or url -type Inventory struct { - Groups []string `yaml:"groups"` - Location string `yaml:"location"` + User string + Inventory string + Environment map[string]string + AdHocCommand string } // InventoryData defines inventory data format @@ -119,12 +111,24 @@ type InventoryData struct { } // New makes new config from yml -func New(fname string, overrides *Overrides) (*PlayBook, error) { - res := &PlayBook{overrides: overrides} +func New(fname string, overrides *Overrides) (res *PlayBook, err error) { + res = &PlayBook{ + overrides: overrides, + inventory: &InventoryData{Groups: make(map[string][]Destination)}, + } + + // load playbook data, err := os.ReadFile(fname) // nolint if err != nil { if overrides != nil && overrides.AdHocCommand != "" { - return res, nil // no config file but adhoc set, just return empty config with overrides + // no config file but adhoc set, just return empty config with overrides + if overrides.Inventory != "" { // load inventory if set in cli + res.inventory, err = res.loadInventory(overrides.Inventory) + if err != nil { + return nil, fmt.Errorf("can't load inventory %s: %w", overrides.Inventory, err) + } + } + return res, nil } return nil, fmt.Errorf("can't read config %s: %w", fname, err) } @@ -133,17 +137,8 @@ func New(fname string, overrides *Overrides) (*PlayBook, error) { return nil, fmt.Errorf("can't unmarshal config %s: %w", fname, err) } - names := make(map[string]bool) - for i, t := range res.Tasks { - if t.Name == "" { - log.Printf("[WARN] missing name for task #%d", i) - return nil, fmt.Errorf("task name is required") - } - if names[t.Name] { - log.Printf("[WARN] duplicate task name %q", t.Name) - return nil, fmt.Errorf("duplicate task name %q", t.Name) - } - names[t.Name] = true + if err = res.checkConfig(); err != nil { + return nil, fmt.Errorf("config %s is invalid: %w", fname, err) } log.Printf("[INFO] playbook loaded with %d tasks", len(res.Tasks)) @@ -152,6 +147,25 @@ func New(fname string, overrides *Overrides) (*PlayBook, error) { log.Printf("[DEBUG] load task %s command %s", tsk.Name, c.Name) } } + + // load inventory if set + inventoryLoc := os.Getenv("SPOT_INVENTORY") // default inventory location from env + if res.Inventory != "" { + inventoryLoc = res.Inventory // inventory set in playbook + } + if overrides != nil && overrides.Inventory != "" { + inventoryLoc = overrides.Inventory // inventory set in cli overrides + } + if inventoryLoc != "" { // load inventory if set. if not set, assume direct hosts in targets are used + res.inventory, err = res.loadInventory(inventoryLoc) + if err != nil { + return nil, fmt.Errorf("can't load inventory %s: %w", inventoryLoc, err) + } + } + if len(res.inventory.Groups) > 0 { // even with hosts only it will make a group "all" + log.Printf("[INFO] inventory loaded with %d hosts", len(res.inventory.Groups["all"])) + } + return res, nil } @@ -211,210 +225,144 @@ func (p *PlayBook) Task(name string) (*Task, error) { // TargetHosts returns target hosts for given target name. // It applies overrides if any set and also retrieves hosts from inventory file or url if any set. func (p *PlayBook) TargetHosts(name string) ([]Destination, error) { - - loadInventoryFile := func(fname string, grs []string) ([]Destination, error) { - fh, err := os.Open(fname) // nolint - if err != nil { - return nil, fmt.Errorf("can't open inventory file %s: %w", fname, err) - } - defer fh.Close() // nolint - hosts, err := p.parseInventory(fh, grs) - if err != nil { - return nil, fmt.Errorf("can't parse inventory file %s: %w", fname, err) - } - return hosts, nil - } - - loadInventoryURL := func(url string, grs []string) ([]Destination, error) { - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Get(url) - if err != nil { - return nil, fmt.Errorf("can't get inventory from http %s: %w", url, err) - } - defer resp.Body.Close() // nolint - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("can't get inventory from http %s, status: %s", url, resp.Status) - } - hosts, err := p.parseInventory(resp.Body, grs) - if err != nil { - return nil, fmt.Errorf("can't parse inventory from http %s: %w", url, err) - } - return hosts, nil - } - user := p.User // default user from playbook if p.overrides != nil && p.overrides.User != "" { user = p.overrides.User // override user if set } - // check if we have overrides for inventory file, this is second priority - if p.overrides != nil && p.overrides.InventoryFile != "" { - res, err := loadInventoryFile(p.overrides.InventoryFile, nil) - if err != nil { - return nil, err - } - return p.filterHosts(res, p.overrides), nil - } - // check if we have overrides for inventory http, this is third priority - if p.overrides != nil && p.overrides.InventoryURL != "" { - res, err := loadInventoryURL(p.overrides.InventoryURL, nil) - if err != nil { - return nil, err - } - return p.filterHosts(res, p.overrides), nil - } - - // no overrides, check if we have target in config - t, ok := p.Targets[name] - if !ok { - // no target, check if it is a host and if so return it as a single host target - isValidTarget := func(name string) bool { - if ip := net.ParseIP(name); ip != nil { - return true - } - if strings.Contains(name, ".") || strings.HasPrefix(name, "localhost") { - return true + t, ok := p.Targets[name] // get target from playbook + if ok { + // we have found target in playbook, check it is host or group + if len(t.Hosts) > 0 { + // target is hosts + res := make([]Destination, len(t.Hosts)) + for i, h := range t.Hosts { + if h.Port == 0 { + h.Port = 22 // default port is 22 if not set + } + if h.User == "" { + h.User = user // default user is playbook's user or override, if not set + } + res[i] = h } - return false - }(name) - - if isValidTarget { - if !strings.Contains(name, ":") { - return []Destination{{Host: name, Port: 22, User: user}}, nil // default port is 22 if not set + return res, nil + } + if len(t.Groups) > 0 { + // target is group, get hosts from inventory + if p.inventory == nil { + return nil, fmt.Errorf("inventory is not loaded") } - elems := strings.Split(name, ":") - port, err := strconv.Atoi(elems[1]) - if err != nil { - return nil, fmt.Errorf("can't parse port %s: %w", elems[1], err) + res := make([]Destination, 0) + for _, g := range t.Groups { + // we don't set default port and user here, as they are set in inventory already + res = append(res, p.inventory.Groups[g]...) } - return []Destination{{Host: elems[0], Port: port, User: user}}, nil // it is a host, sent as ip + return res, nil } - return nil, fmt.Errorf("target %s not found", name) + return nil, fmt.Errorf("target %q has no hosts or groups", name) } - // target found, check if it has hosts - if len(t.Hosts) > 0 { - res := make([]Destination, len(t.Hosts)) - for i, h := range t.Hosts { - if h.Port == 0 { - h.Port = 22 // default port is 22 if not set - } - if h.User == "" { - h.User = user // default user is playbook's user or override, if not set - } - res[i] = h - } - return p.filterHosts(res, p.overrides), nil + // target not found in playbook + + // try first as group in inventory + hosts, ok := p.inventory.Groups[name] + if ok { + res := make([]Destination, len(hosts)) + copy(res, hosts) + return res, nil } - // target has no hosts, check if it has inventory file - if t.InventoryFile.Location != "" { - res, err := loadInventoryFile(t.InventoryFile.Location, t.InventoryFile.Groups) - if err != nil { - return nil, fmt.Errorf("can't load inventory file %s: %w", t.InventoryFile.Location, err) + // try as single host in inventory + for _, h := range p.inventory.Groups["all"] { + if h.Host == name { + return []Destination{h}, nil } - return p.filterHosts(res, p.overrides), nil } - // target has no hosts, check if it has inventory http - if t.InventoryURL.Location != "" { - res, err := loadInventoryURL(t.InventoryURL.Location, t.InventoryFile.Groups) + // try as single host or host:port + if strings.Contains(name, ":") { + elems := strings.Split(name, ":") + port, err := strconv.Atoi(elems[1]) if err != nil { - return nil, fmt.Errorf("can't load inventory http %s: %w", t.InventoryURL.Location, err) + return nil, fmt.Errorf("can't parse port %s: %w", elems[1], err) } - return p.filterHosts(res, p.overrides), nil + return []Destination{{Host: elems[0], Port: port, User: user}}, nil } - if t.Hosts == nil { - return nil, fmt.Errorf("target %s has no hosts", name) - } - - return p.filterHosts(t.Hosts, p.overrides), nil + // we assume it is a host name, with default port 22 + return []Destination{{Host: name, Port: 22, User: user}}, nil } -// filterHosts filters hosts by host name first and if not matched, by address. -func (p *PlayBook) filterHosts(inp []Destination, overrides *Overrides) []Destination { - if overrides == nil || len(overrides.FilterHosts) == 0 { // no filter, return all - return inp - } - filter := overrides.FilterHosts - res := []Destination{} - matchedNames := map[string]bool{} // map of matched names - - // first filter by name - for _, h := range inp { - for _, f := range filter { - if h.Name == f { - res = append(res, h) - matchedNames[h.Name] = true - } - } - } +// loadInventoryFile loads inventory from file and returns a struct with groups. +// Hosts, if presented, are loaded to the group "all". All the other groups are loaded to "all" +// as well and also to their own group. +func (p *PlayBook) loadInventory(loc string) (*InventoryData, error) { - // then filter by address - for _, h := range inp { - if matchedNames[h.Name] { - continue // already matched by name, skip - } - for _, f := range filter { - if h.Host == f { - res = append(res, h) - } + // get reader for inventory file or url + var rdr io.Reader + if strings.HasPrefix(loc, "http") { // location is a url + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Get(loc) + if err != nil { + return nil, fmt.Errorf("can't get inventory from http %s: %w", loc, err) } - } - - return res -} - -// parseInventory parses inventory yml file or url and returns a list of hosts for the specified group. -// user is optional, if not set, it is assumed to be defined in playbook. name is optional too. -func (p *PlayBook) parseInventory(r io.Reader, groups []string) ([]Destination, error) { - contains := func(s []string, e string) bool { - if len(s) == 0 { - return true + defer resp.Body.Close() //nolint + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("can't get inventory from http %s, status: %s", loc, resp.Status) } - for _, a := range s { - if a == e { - return true - } + rdr = resp.Body + } else { // location is a file + f, err := os.Open(loc) //nolint + if err != nil { + return nil, fmt.Errorf("can't open inventory file %s: %w", loc, err) } - return false + defer f.Close() //nolint + rdr = f } var data InventoryData - if err := yaml.NewDecoder(r).Decode(&data); err != nil { + if err := yaml.NewDecoder(rdr).Decode(&data); err != nil { return nil, fmt.Errorf("inventory decoder failed: %w", err) } - if len(data.Hosts) == 0 && len(data.Groups) == 0 { - return nil, fmt.Errorf("no hosts or groups defined in inventory") - } - - res := []Destination{} - if len(data.Hosts) > 0 { // hosts defined directly, use them - res = append(res, data.Hosts...) - } - - if len(data.Hosts) == 0 { // no hosts defined directly, use groups - for grName, hosts := range data.Groups { - if !contains(groups, grName) { + if len(data.Groups) > 0 { + // create group "all" with all hosts from all groups + data.Groups["all"] = []Destination{} + for key, g := range data.Groups { + if key == "all" { continue } - res = append(res, hosts...) + data.Groups["all"] = append(data.Groups["all"], g...) } } - - sort.Slice(res, func(i, j int) bool { return res[i].Host < res[j].Host }) - for i, h := range res { - if h.Port == 0 { - res[i].Port = 22 // default port is 22 if not set + if len(data.Hosts) > 0 { + // add hosts to group "all" + if data.Groups == nil { + data.Groups = make(map[string][]Destination) } - if h.User == "" { - res[i].User = p.User // default user is playbook's user or override, if not set in inventory + if _, ok := data.Groups["all"]; !ok { + data.Groups["all"] = []Destination{} } + data.Groups["all"] = append(data.Groups["all"], data.Hosts...) } + sort.Slice(data.Groups["all"], func(i, j int) bool { + return data.Groups["all"][i].Host < data.Groups["all"][j].Host + }) - return res, nil + // set default port and user if not set for inventory groups + // note: we don't care about hosts anymore, they are used only for parsing and are not used in the playbook directly + for _, gr := range data.Groups { + for i := range gr { + if gr[i].Port == 0 { + gr[i].Port = 22 // default port is 22 if not set + } + if gr[i].User == "" { + gr[i].User = p.User // default user is playbook's user or override, if not set + } + } + } + + return &data, nil } // GetScript returns a script string and an io.Reader based on the command being single line or multiline. @@ -503,3 +451,20 @@ func (cmd *Cmd) genEnv() []string { sort.Slice(envs, func(i, j int) bool { return envs[i] < envs[j] }) return envs } + +// checkConfig checks validity of config +func (p *PlayBook) checkConfig() error { + names := make(map[string]bool) + for i, t := range p.Tasks { + if t.Name == "" { + log.Printf("[WARN] missing name for task #%d", i) + return fmt.Errorf("task name is required") + } + if names[t.Name] { + log.Printf("[WARN] duplicate task name %q", t.Name) + return fmt.Errorf("duplicate task name %q", t.Name) + } + names[t.Name] = true + } + return nil +} diff --git a/app/config/config_test.go b/app/config/config_test.go index f05c0043..64986c4a 100644 --- a/app/config/config_test.go +++ b/app/config/config_test.go @@ -26,6 +26,48 @@ func TestNew(t *testing.T) { assert.Equal(t, "deploy-remark42", tsk.Name, "task name") }) + t.Run("inventory from env", func(t *testing.T) { + err := os.Setenv("SPOT_INVENTORY", "testdata/hosts-with-groups.yml") + require.NoError(t, err) + defer os.Unsetenv("SPOT_INVENTORY") + + c, err := New("testdata/f1.yml", nil) + require.NoError(t, err) + require.NotNil(t, c.inventory) + assert.Len(t, c.inventory.Groups["all"], 7, "7 hosts in inventory") + assert.Len(t, c.inventory.Groups["gr2"], 3, "3 hosts in gr2 group") + assert.Equal(t, Destination{Name: "h5", Host: "h5.example.com", Port: 2233, User: "umputun"}, c.inventory.Groups["gr2"][0]) + }) + + t.Run("inventory from playbook", func(t *testing.T) { + c, err := New("testdata/playbook-with-inventory.yml", nil) + require.NoError(t, err) + require.NotNil(t, c.inventory) + assert.Len(t, c.inventory.Groups["all"], 5, "5 hosts in inventory") + assert.Equal(t, Destination{Name: "h2", Host: "h2.example.com", Port: 2233, User: "umputun"}, + c.inventory.Groups["all"][0]) + }) + + t.Run("inventory from overrides", func(t *testing.T) { + c, err := New("testdata/f1.yml", &Overrides{Inventory: "testdata/hosts-with-groups.yml"}) + require.NoError(t, err) + require.NotNil(t, c.inventory) + assert.Len(t, c.inventory.Groups["all"], 7, "7 hosts in inventory") + assert.Len(t, c.inventory.Groups["gr2"], 3, "3 hosts in gr2 group") + assert.Equal(t, Destination{Name: "h5", Host: "h5.example.com", Port: 2233, User: "umputun"}, c.inventory.Groups["gr2"][0]) + }) + + t.Run("inventory from overrides with env and playbook", func(t *testing.T) { + err := os.Setenv("SPOT_INVENTORY", "testdata/inventory_env.yml") + require.NoError(t, err) + defer os.Unsetenv("SPOT_INVENTORY") + + c, err := New("testdata/playbook-with-inventory.yml", &Overrides{Inventory: "testdata/hosts-without-groups.yml"}) + require.NoError(t, err) + require.NotNil(t, c.inventory) + assert.Len(t, c.inventory.Groups["all"], 5, "5 hosts in inventory") + }) + t.Run("adhoc mode", func(t *testing.T) { c, err := New("no-such-thing", &Overrides{AdHocCommand: "echo 123", User: "umputun"}) require.NoError(t, err) @@ -278,378 +320,605 @@ func TestCmd_getScriptFile(t *testing.T) { } } -func TestPlaybook_TargetHosts(t *testing.T) { +func TestTargetHosts(t *testing.T) { + p := &PlayBook{ - User: "default_user", + User: "defaultuser", Targets: map[string]Target{ "target1": { - Hosts: []Destination{ - {Host: "host1", Port: 22, User: "user1"}, - {Host: "host2", Port: 2222}, - {Host: "host3", Name: "host3_name", Port: 2020, User: "user3"}, - }, + Name: "target1", + Hosts: []Destination{{Host: "host1.example.com", Port: 22}}, }, "target2": { - InventoryFile: Inventory{Location: "testdata/hosts-with-groups.yml", Groups: []string{"gr1"}}, + Name: "target2", + Groups: []string{"group1"}, + }, + }, + inventory: &InventoryData{ + Groups: map[string][]Destination{ + "all": { + {Host: "host1.example.com", Port: 22, User: "user1"}, + {Host: "host2.example.com", Port: 22, User: "defaultuser"}, + {Host: "host3.example.com", Port: 22, User: "defaultuser"}, + }, + "group1": { + {Host: "host2.example.com", Port: 2222, User: "defaultuser", Name: "host1"}, + }, }, - "target3": {}, - "target4": { - InventoryFile: Inventory{Location: "testdata/hosts-with-groups.yml"}, + Hosts: []Destination{ + {Host: "host3.example.com", Port: 22, Name: "host3"}, }, }, } - tests := []struct { - name string - targetName string - overrides *Overrides - want []Destination - wantErr bool + testCases := []struct { + name string + targetName string + overrides *Overrides + expected []Destination + expectError bool }{ { - name: "target from config", - targetName: "target1", - want: []Destination{ - {Host: "host1", Port: 22, User: "user1"}, - {Host: "host2", Port: 2222, User: "default_user"}, - {Name: "host3_name", Host: "host3", Port: 2020, User: "user3"}, - }, - wantErr: false, - }, - { - name: "overrides target hosts from inventory, name match", - targetName: "target4", - overrides: &Overrides{ - FilterHosts: []string{"h6", "h5"}, - }, - want: []Destination{ - {Name: "h5", Host: "h5.example.com", Port: 2233, User: "default_user"}, - {Name: "h6", Host: "h6.example.com", Port: 22, User: "user3"}, - }, - wantErr: false, - }, - { - name: "overrides target hosts from inventory address match", - targetName: "target4", - overrides: &Overrides{ - FilterHosts: []string{"h5.example.com", "h7.example.com"}, - }, - want: []Destination{ - {Name: "h5", Host: "h5.example.com", Port: 2233, User: "default_user"}, - {Name: "", Host: "h7.example.com", Port: 22, User: "user3"}, - }, - wantErr: false, - }, - { - name: "overrides target hosts direct, name and address match", + name: "target with hosts", targetName: "target1", - overrides: &Overrides{ - FilterHosts: []string{"host3_name", "bad-host", "host2"}, - }, - want: []Destination{ - {Name: "host3_name", Host: "host3", Port: 2020, User: "user3"}, - {Name: "", Host: "host2", Port: 2222, User: "default_user"}, - }, - wantErr: false, - }, - { - name: "overrides target hosts direct, address match", - targetName: "target1", - overrides: &Overrides{ - FilterHosts: []string{"host1", "bad-host", "host2"}, - }, - want: []Destination{ - {Name: "", Host: "host1", Port: 22, User: "user1"}, - {Name: "", Host: "host2", Port: 2222, User: "default_user"}, - }, - wantErr: false, - }, - { - name: "target not found", - targetName: "nonexistent", - wantErr: true, - }, - { - name: "target without anything defined", - targetName: "target3", - wantErr: true, - }, - { - name: "target as ip", - targetName: "127.0.0.1:2222", - want: []Destination{ - {Host: "127.0.0.1", Port: 2222, User: "default_user"}, - }, - wantErr: false, - }, - { - name: "target as ip, no port", - targetName: "127.0.0.1", - want: []Destination{ - {Host: "127.0.0.1", Port: 22, User: "default_user"}, + expected: []Destination{ + { + Host: "host1.example.com", + Port: 22, + User: "defaultuser", + }, }, - wantErr: false, + expectError: false, }, { - name: "target as fqdn", - targetName: "example.com:2222", - want: []Destination{ - {Host: "example.com", Port: 2222, User: "default_user"}, + name: "target with groups", + targetName: "target2", + expected: []Destination{ + { + Host: "host2.example.com", + Port: 2222, + User: "defaultuser", + Name: "host1", + }, }, - wantErr: false, + expectError: false, }, { - name: "target as fqdn, no port", - targetName: "host.example.com", - want: []Destination{ - {Host: "host.example.com", Port: 22, User: "default_user"}, + name: "target as group from inventory", + targetName: "group1", + expected: []Destination{ + { + Host: "host2.example.com", + Port: 2222, + User: "defaultuser", + Name: "host1", + }, }, - wantErr: false, + expectError: false, }, { - name: "target as localhost with port", - targetName: "localhost:50958", - want: []Destination{ - {Host: "localhost", Port: 50958, User: "default_user"}, + name: "target as single host from inventory", + targetName: "host3.example.com", + expected: []Destination{ + { + Host: "host3.example.com", + Port: 22, + User: "defaultuser", + }, }, - wantErr: false, + expectError: false, }, { - name: "valid target with inventory file", - targetName: "target2", - want: []Destination{ - {Host: "h1.example.com", Port: 22, User: "default_user", Name: "h1"}, - {Host: "h2.example.com", Port: 2233, User: "default_user", Name: "h2"}, - {Host: "h3.example.com", Port: 22, User: "user1"}, - {Host: "h4.example.com", Port: 22, User: "user2", Name: "h4"}, + name: "target as single host with port", + targetName: "host4.example.com:2222", + expected: []Destination{ + { + Host: "host4.example.com", + Port: 2222, + User: "defaultuser", + }, }, - wantErr: false, + expectError: false, }, { - name: "overrides inventory file", - targetName: "target2", - overrides: &Overrides{ - InventoryFile: "testdata/override_inventory.yml", - }, - want: []Destination{ - {Host: "host3", Port: 22, User: "default_user"}, - {Host: "host4", Port: 2222, User: "user2"}, - }, - wantErr: false, + name: "invalid host:port format", + targetName: "host5.example.com:invalid", + expectError: true, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p.overrides = tt.overrides - got, err := p.TargetHosts(tt.targetName) - if tt.wantErr { + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + p.overrides = tc.overrides + res, err := p.TargetHosts(tc.targetName) + if tc.expectError { require.Error(t, err) } else { require.NoError(t, err) - assert.Equal(t, tt.want, got) + require.Equal(t, tc.expected, res) } }) } } -func TestPlayBook_TargetHostsOverrides(t *testing.T) { - - t.Run("override hosts with file", func(t *testing.T) { - c, err := New("testdata/f1.yml", &Overrides{InventoryFile: "testdata/hosts-without-groups.yml"}) - require.NoError(t, err) - res, err := c.TargetHosts("blah") - require.NoError(t, err) - assert.Equal(t, []Destination{ - {Name: "h2", Host: "h2.example.com", Port: 2233, User: "umputun"}, - {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, - {Name: "h4", Host: "h4.example.com", Port: 22, User: "user2"}, - {Name: "hh1", Host: "hh1.example.com", Port: 22, User: "umputun"}, - {Name: "hh2", Host: "hh2.example.com", Port: 2233, User: "user1"}, - }, res) - }) - - t.Run("override hosts with file, filtered", func(t *testing.T) { - c, err := New("testdata/f1.yml", &Overrides{InventoryFile: "testdata/hosts-without-groups.yml", FilterHosts: []string{"h2", "h3"}}) - require.NoError(t, err) - res, err := c.TargetHosts("blah") - require.NoError(t, err) - assert.Equal(t, []Destination{ - {Name: "h2", Host: "h2.example.com", Port: 2233, User: "umputun"}, - {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, - }, res) - }) - - t.Run("override hosts with file not found", func(t *testing.T) { - c, err := New("testdata/f1.yml", &Overrides{InventoryFile: "testdata/hosts_not_found"}) - require.NoError(t, err) - _, err = c.TargetHosts("blah") - require.ErrorContains(t, err, "no such file or directory") - t.Log(err) - }) - - t.Run("override hosts with http", func(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fh, err := os.Open("testdata/hosts-without-groups.yml") - require.NoError(t, err) - defer fh.Close() - _, err = io.Copy(w, fh) - require.NoError(t, err) - })) - defer ts.Close() - c, err := New("testdata/f1.yml", &Overrides{InventoryURL: ts.URL}) - require.NoError(t, err) - res, err := c.TargetHosts("blah") - require.NoError(t, err) - assert.Equal(t, []Destination{ - {Name: "h2", Host: "h2.example.com", Port: 2233, User: "umputun"}, - {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, - {Name: "h4", Host: "h4.example.com", Port: 22, User: "user2"}, - {Name: "hh1", Host: "hh1.example.com", Port: 22, User: "umputun"}, - {Name: "hh2", Host: "hh2.example.com", Port: 2233, User: "user1"}, - }, res) - }) - - t.Run("override hosts with http, filtered", func(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fh, err := os.Open("testdata/hosts-without-groups.yml") - require.NoError(t, err) - defer fh.Close() - _, err = io.Copy(w, fh) - require.NoError(t, err) - })) - defer ts.Close() - c, err := New("testdata/f1.yml", &Overrides{InventoryURL: ts.URL, FilterHosts: []string{"h3", "h4.example.com"}}) - require.NoError(t, err) - res, err := c.TargetHosts("blah") - require.NoError(t, err) - assert.Equal(t, []Destination{ - {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, - {Name: "h4", Host: "h4.example.com", Port: 22, User: "user2"}, - }, res) - }) - t.Run("override hosts with http failed", func(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - defer ts.Close() - c, err := New("testdata/f1.yml", &Overrides{InventoryURL: ts.URL}) - require.NoError(t, err) - _, err = c.TargetHosts("blah") - require.ErrorContains(t, err, "status: 500 Internal Server Error") - t.Log(err) - }) -} +// +// func TestPlaybook_TargetHosts(t *testing.T) { +// p := &PlayBook{ +// User: "default_user", +// inventory: &InventoryData{ +// Groups: map[string][]Destination{ +// "gr1": {{Host: "host1", Port: 22, User: "user11"}}, +// "gr2": {{Host: "host2", Port: 2222, User: "default_user"}}, +// "all": { +// {Host: "host1", Port: 22, User: "user11"}, +// {Host: "host2", Port: 2222, User: "default_user"}, +// }, +// }, +// }, +// Targets: map[string]Target{ +// "target1": { +// Hosts: []Destination{ +// {Host: "host1", Port: 22, User: "user1"}, +// {Host: "host2", Port: 2222}, +// {Host: "host3", Name: "host3_name", Port: 2020, User: "user3"}, +// }, +// }, +// "target2": { +// Groups: []string{"gr1"}, +// }, +// "target3": {}, +// "target4": { +// Groups: []string{"all"}, +// }, +// }, +// } +// +// tests := []struct { +// name string +// targetName string +// overrides *Overrides +// want []Destination +// wantErr bool +// }{ +// { +// name: "target from config", +// targetName: "target1", +// want: []Destination{ +// {Host: "host1", Port: 22, User: "user1"}, +// {Host: "host2", Port: 2222, User: "default_user"}, +// {Name: "host3_name", Host: "host3", Port: 2020, User: "user3"}, +// }, +// wantErr: false, +// }, +// { +// name: "overrides target hosts from inventory, name match", +// targetName: "target4", +// overrides: &Overrides{ +// FilterHosts: []string{"h6", "h5"}, +// }, +// want: []Destination{ +// {Name: "h5", Host: "h5.example.com", Port: 2233, User: "default_user"}, +// {Name: "h6", Host: "h6.example.com", Port: 22, User: "user3"}, +// }, +// wantErr: false, +// }, +// { +// name: "overrides target hosts from inventory address match", +// targetName: "target4", +// overrides: &Overrides{ +// FilterHosts: []string{"h5.example.com", "h7.example.com"}, +// }, +// want: []Destination{ +// {Name: "h5", Host: "h5.example.com", Port: 2233, User: "default_user"}, +// {Name: "", Host: "h7.example.com", Port: 22, User: "user3"}, +// }, +// wantErr: false, +// }, +// { +// name: "overrides target hosts direct, name and address match", +// targetName: "target1", +// overrides: &Overrides{ +// FilterHosts: []string{"host3_name", "bad-host", "host2"}, +// }, +// want: []Destination{ +// {Name: "host3_name", Host: "host3", Port: 2020, User: "user3"}, +// {Name: "", Host: "host2", Port: 2222, User: "default_user"}, +// }, +// wantErr: false, +// }, +// { +// name: "overrides target hosts direct, address match", +// targetName: "target1", +// overrides: &Overrides{ +// FilterHosts: []string{"host1", "bad-host", "host2"}, +// }, +// want: []Destination{ +// {Name: "", Host: "host1", Port: 22, User: "user1"}, +// {Name: "", Host: "host2", Port: 2222, User: "default_user"}, +// }, +// wantErr: false, +// }, +// { +// name: "target not found", +// targetName: "nonexistent", +// wantErr: true, +// }, +// { +// name: "target without anything defined", +// targetName: "target3", +// wantErr: true, +// }, +// { +// name: "target as ip", +// targetName: "127.0.0.1:2222", +// want: []Destination{ +// {Host: "127.0.0.1", Port: 2222, User: "default_user"}, +// }, +// wantErr: false, +// }, +// { +// name: "target as ip, no port", +// targetName: "127.0.0.1", +// want: []Destination{ +// {Host: "127.0.0.1", Port: 22, User: "default_user"}, +// }, +// wantErr: false, +// }, +// { +// name: "target as fqdn", +// targetName: "example.com:2222", +// want: []Destination{ +// {Host: "example.com", Port: 2222, User: "default_user"}, +// }, +// wantErr: false, +// }, +// { +// name: "target as fqdn, no port", +// targetName: "host.example.com", +// want: []Destination{ +// {Host: "host.example.com", Port: 22, User: "default_user"}, +// }, +// wantErr: false, +// }, +// { +// name: "target as localhost with port", +// targetName: "localhost:50958", +// want: []Destination{ +// {Host: "localhost", Port: 50958, User: "default_user"}, +// }, +// wantErr: false, +// }, +// { +// name: "valid target with inventory file", +// targetName: "target2", +// want: []Destination{ +// {Host: "h1.example.com", Port: 22, User: "default_user", Name: "h1"}, +// {Host: "h2.example.com", Port: 2233, User: "default_user", Name: "h2"}, +// {Host: "h3.example.com", Port: 22, User: "user1"}, +// {Host: "h4.example.com", Port: 22, User: "user2", Name: "h4"}, +// }, +// wantErr: false, +// }, +// { +// name: "overrides inventory file", +// targetName: "target2", +// overrides: &Overrides{ +// InventoryFile: "testdata/override_inventory.yml", +// }, +// want: []Destination{ +// {Host: "host3", Port: 22, User: "default_user"}, +// {Host: "host4", Port: 2222, User: "user2"}, +// }, +// wantErr: false, +// }, +// } +// +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// p.overrides = tt.overrides +// got, err := p.TargetHosts(tt.targetName) +// if tt.wantErr { +// require.Error(t, err) +// } else { +// require.NoError(t, err) +// assert.Equal(t, tt.want, got) +// } +// }) +// } +// } + +// +// func TestPlayBook_TargetHostsOverrides(t *testing.T) { +// +// t.Run("override hosts with file", func(t *testing.T) { +// c, err := New("testdata/f1.yml", &Overrides{InventoryFile: "testdata/hosts-without-groups.yml"}) +// require.NoError(t, err) +// res, err := c.TargetHosts("blah") +// require.NoError(t, err) +// assert.Equal(t, []Destination{ +// {Name: "h2", Host: "h2.example.com", Port: 2233, User: "umputun"}, +// {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, +// {Name: "h4", Host: "h4.example.com", Port: 22, User: "user2"}, +// {Name: "hh1", Host: "hh1.example.com", Port: 22, User: "umputun"}, +// {Name: "hh2", Host: "hh2.example.com", Port: 2233, User: "user1"}, +// }, res) +// }) +// +// t.Run("override hosts with file, filtered", func(t *testing.T) { +// c, err := New("testdata/f1.yml", &Overrides{InventoryFile: "testdata/hosts-without-groups.yml", FilterHosts: []string{"h2", "h3"}}) +// require.NoError(t, err) +// res, err := c.TargetHosts("blah") +// require.NoError(t, err) +// assert.Equal(t, []Destination{ +// {Name: "h2", Host: "h2.example.com", Port: 2233, User: "umputun"}, +// {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, +// }, res) +// }) +// +// t.Run("override hosts with file not found", func(t *testing.T) { +// c, err := New("testdata/f1.yml", &Overrides{InventoryFile: "testdata/hosts_not_found"}) +// require.NoError(t, err) +// _, err = c.TargetHosts("blah") +// require.ErrorContains(t, err, "no such file or directory") +// t.Log(err) +// }) +// +// t.Run("override hosts with http", func(t *testing.T) { +// ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// fh, err := os.Open("testdata/hosts-without-groups.yml") +// require.NoError(t, err) +// defer fh.Close() +// _, err = io.Copy(w, fh) +// require.NoError(t, err) +// })) +// defer ts.Close() +// c, err := New("testdata/f1.yml", &Overrides{InventoryURL: ts.URL}) +// require.NoError(t, err) +// res, err := c.TargetHosts("blah") +// require.NoError(t, err) +// assert.Equal(t, []Destination{ +// {Name: "h2", Host: "h2.example.com", Port: 2233, User: "umputun"}, +// {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, +// {Name: "h4", Host: "h4.example.com", Port: 22, User: "user2"}, +// {Name: "hh1", Host: "hh1.example.com", Port: 22, User: "umputun"}, +// {Name: "hh2", Host: "hh2.example.com", Port: 2233, User: "user1"}, +// }, res) +// }) +// +// t.Run("override hosts with http, filtered", func(t *testing.T) { +// ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// fh, err := os.Open("testdata/hosts-without-groups.yml") +// require.NoError(t, err) +// defer fh.Close() +// _, err = io.Copy(w, fh) +// require.NoError(t, err) +// })) +// defer ts.Close() +// c, err := New("testdata/f1.yml", &Overrides{InventoryURL: ts.URL, FilterHosts: []string{"h3", "h4.example.com"}}) +// require.NoError(t, err) +// res, err := c.TargetHosts("blah") +// require.NoError(t, err) +// assert.Equal(t, []Destination{ +// {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, +// {Name: "h4", Host: "h4.example.com", Port: 22, User: "user2"}, +// }, res) +// }) +// t.Run("override hosts with http failed", func(t *testing.T) { +// ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// w.WriteHeader(http.StatusInternalServerError) +// })) +// defer ts.Close() +// c, err := New("testdata/f1.yml", &Overrides{InventoryURL: ts.URL}) +// require.NoError(t, err) +// _, err = c.TargetHosts("blah") +// require.ErrorContains(t, err, "status: 500 Internal Server Error") +// t.Log(err) +// }) +// } +// +// func TestPlayBook_parseInventoryGroups(t *testing.T) { +// playbook := &PlayBook{User: "defaultUser"} +// +// tests := []struct { +// name string +// inventory string +// groups []string +// want []Destination +// }{ +// { +// name: "all groups", +// inventory: "testdata/hosts-with-groups.yml", +// groups: nil, +// want: []Destination{ +// {Host: "h1.example.com", Port: 22, User: "defaultUser", Name: "h1"}, +// {Host: "h2.example.com", Port: 2233, User: "defaultUser", Name: "h2"}, +// {Host: "h3.example.com", Port: 22, User: "user1"}, +// {Host: "h4.example.com", Port: 22, User: "user2", Name: "h4"}, +// {Host: "h5.example.com", Port: 2233, User: "defaultUser", Name: "h5"}, +// {Host: "h6.example.com", Port: 22, User: "user3", Name: "h6"}, +// {Host: "h7.example.com", Port: 22, User: "user3"}, +// }, +// }, +// { +// name: "group 1", +// inventory: "testdata/hosts-with-groups.yml", +// groups: []string{"gr1"}, +// want: []Destination{ +// {Host: "h1.example.com", Port: 22, User: "defaultUser", Name: "h1"}, +// {Host: "h2.example.com", Port: 2233, User: "defaultUser", Name: "h2"}, +// {Host: "h3.example.com", Port: 22, User: "user1"}, +// {Host: "h4.example.com", Port: 22, User: "user2", Name: "h4"}, +// }, +// }, +// { +// name: "group 2", +// inventory: "testdata/hosts-with-groups.yml", +// groups: []string{"gr2"}, +// want: []Destination{ +// {Host: "h5.example.com", Port: 2233, User: "defaultUser", Name: "h5"}, +// {Host: "h6.example.com", Port: 22, User: "user3", Name: "h6"}, +// {Host: "h7.example.com", Port: 22, User: "user3"}, +// }, +// }, +// { +// name: "group 1 and 2", +// inventory: "testdata/hosts-with-groups.yml", +// groups: []string{"gr1", "gr2"}, +// want: []Destination{ +// {Host: "h1.example.com", Port: 22, User: "defaultUser", Name: "h1"}, +// {Host: "h2.example.com", Port: 2233, User: "defaultUser", Name: "h2"}, +// {Host: "h3.example.com", Port: 22, User: "user1"}, +// {Host: "h4.example.com", Port: 22, User: "user2", Name: "h4"}, +// {Host: "h5.example.com", Port: 2233, User: "defaultUser", Name: "h5"}, +// {Host: "h6.example.com", Port: 22, User: "user3", Name: "h6"}, +// {Host: "h7.example.com", Port: 22, User: "user3"}, +// }, +// }, +// { +// name: "empty group", +// inventory: "testdata/hosts-with-groups.yml", +// groups: []string{}, +// want: []Destination{ +// {Host: "h1.example.com", Port: 22, User: "defaultUser", Name: "h1"}, +// {Host: "h2.example.com", Port: 2233, User: "defaultUser", Name: "h2"}, +// {Host: "h3.example.com", Port: 22, User: "user1"}, +// {Host: "h4.example.com", Port: 22, User: "user2", Name: "h4"}, +// {Host: "h5.example.com", Port: 2233, User: "defaultUser", Name: "h5"}, +// {Host: "h6.example.com", Port: 22, User: "user3", Name: "h6"}, +// {Host: "h7.example.com", Port: 22, User: "user3"}, +// }, +// }, +// { +// name: "non-existent group", +// inventory: "testdata/hosts-with-groups.yml", +// groups: []string{"non-existent"}, +// want: []Destination{}, +// }, +// { +// name: "hosts inventory", +// inventory: "testdata/hosts-without-groups.yml", +// want: []Destination{ +// {Name: "h2", Host: "h2.example.com", Port: 2233, User: "defaultUser"}, +// {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, +// {Name: "h4", Host: "h4.example.com", Port: 22, User: "user2"}, +// {Name: "hh1", Host: "hh1.example.com", Port: 22, User: "defaultUser"}, +// {Name: "hh2", Host: "hh2.example.com", Port: 2233, User: "user1"}}, +// }, +// { +// name: "hosts inventory but group name set", +// inventory: "testdata/hosts-without-groups.yml", +// groups: []string{"some"}, +// want: []Destination{ +// {Name: "h2", Host: "h2.example.com", Port: 2233, User: "defaultUser"}, +// {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, +// {Name: "h4", Host: "h4.example.com", Port: 22, User: "user2"}, +// {Name: "hh1", Host: "hh1.example.com", Port: 22, User: "defaultUser"}, +// {Name: "hh2", Host: "hh2.example.com", Port: 2233, User: "user1"}}, +// }, +// } +// +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// reader, err := os.Open(tt.inventory) +// require.NoError(t, err) +// defer reader.Close() +// got, err := playbook.parseInventory(reader, tt.groups) +// require.NoError(t, err) +// assert.Equal(t, tt.want, got) +// }) +// } +// } + +func TestPlayBook_loadInventory(t *testing.T) { + // create temporary inventory file + tmpFile, err := os.CreateTemp("", "inventory-*.yaml") + require.NoError(t, err) + defer os.Remove(tmpFile.Name()) + + _, err = tmpFile.WriteString(`--- +groups: + group1: + - host: example.com + port: 22 + group2: + - host: another.com +hosts: + - {host: one.example.com, port: 2222} +`) + require.NoError(t, err) -func TestPlayBook_parseInventoryGroups(t *testing.T) { - playbook := &PlayBook{User: "defaultUser"} + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeFile(w, r, tmpFile.Name()) + })) + defer ts.Close() - tests := []struct { - name string - inventory string - groups []string - want []Destination + testCases := []struct { + name string + loc string + expectError bool }{ { - name: "all groups", - inventory: "testdata/hosts-with-groups.yml", - groups: nil, - want: []Destination{ - {Host: "h1.example.com", Port: 22, User: "defaultUser", Name: "h1"}, - {Host: "h2.example.com", Port: 2233, User: "defaultUser", Name: "h2"}, - {Host: "h3.example.com", Port: 22, User: "user1"}, - {Host: "h4.example.com", Port: 22, User: "user2", Name: "h4"}, - {Host: "h5.example.com", Port: 2233, User: "defaultUser", Name: "h5"}, - {Host: "h6.example.com", Port: 22, User: "user3", Name: "h6"}, - {Host: "h7.example.com", Port: 22, User: "user3"}, - }, - }, - { - name: "group 1", - inventory: "testdata/hosts-with-groups.yml", - groups: []string{"gr1"}, - want: []Destination{ - {Host: "h1.example.com", Port: 22, User: "defaultUser", Name: "h1"}, - {Host: "h2.example.com", Port: 2233, User: "defaultUser", Name: "h2"}, - {Host: "h3.example.com", Port: 22, User: "user1"}, - {Host: "h4.example.com", Port: 22, User: "user2", Name: "h4"}, - }, - }, - { - name: "group 2", - inventory: "testdata/hosts-with-groups.yml", - groups: []string{"gr2"}, - want: []Destination{ - {Host: "h5.example.com", Port: 2233, User: "defaultUser", Name: "h5"}, - {Host: "h6.example.com", Port: 22, User: "user3", Name: "h6"}, - {Host: "h7.example.com", Port: 22, User: "user3"}, - }, - }, - { - name: "group 1 and 2", - inventory: "testdata/hosts-with-groups.yml", - groups: []string{"gr1", "gr2"}, - want: []Destination{ - {Host: "h1.example.com", Port: 22, User: "defaultUser", Name: "h1"}, - {Host: "h2.example.com", Port: 2233, User: "defaultUser", Name: "h2"}, - {Host: "h3.example.com", Port: 22, User: "user1"}, - {Host: "h4.example.com", Port: 22, User: "user2", Name: "h4"}, - {Host: "h5.example.com", Port: 2233, User: "defaultUser", Name: "h5"}, - {Host: "h6.example.com", Port: 22, User: "user3", Name: "h6"}, - {Host: "h7.example.com", Port: 22, User: "user3"}, - }, - }, - { - name: "empty group", - inventory: "testdata/hosts-with-groups.yml", - groups: []string{}, - want: []Destination{ - {Host: "h1.example.com", Port: 22, User: "defaultUser", Name: "h1"}, - {Host: "h2.example.com", Port: 2233, User: "defaultUser", Name: "h2"}, - {Host: "h3.example.com", Port: 22, User: "user1"}, - {Host: "h4.example.com", Port: 22, User: "user2", Name: "h4"}, - {Host: "h5.example.com", Port: 2233, User: "defaultUser", Name: "h5"}, - {Host: "h6.example.com", Port: 22, User: "user3", Name: "h6"}, - {Host: "h7.example.com", Port: 22, User: "user3"}, - }, + name: "load from file", + loc: tmpFile.Name(), }, { - name: "non-existent group", - inventory: "testdata/hosts-with-groups.yml", - groups: []string{"non-existent"}, - want: []Destination{}, + name: "load from url", + loc: ts.URL, }, { - name: "hosts inventory", - inventory: "testdata/hosts-without-groups.yml", - want: []Destination{ - {Name: "h2", Host: "h2.example.com", Port: 2233, User: "defaultUser"}, - {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, - {Name: "h4", Host: "h4.example.com", Port: 22, User: "user2"}, - {Name: "hh1", Host: "hh1.example.com", Port: 22, User: "defaultUser"}, - {Name: "hh2", Host: "hh2.example.com", Port: 2233, User: "user1"}}, + name: "invalid url", + loc: "http://not-a-valid-url", + expectError: true, }, { - name: "hosts inventory but group name set", - inventory: "testdata/hosts-without-groups.yml", - groups: []string{"some"}, - want: []Destination{ - {Name: "h2", Host: "h2.example.com", Port: 2233, User: "defaultUser"}, - {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, - {Name: "h4", Host: "h4.example.com", Port: 22, User: "user2"}, - {Name: "hh1", Host: "hh1.example.com", Port: 22, User: "defaultUser"}, - {Name: "hh2", Host: "hh2.example.com", Port: 2233, User: "user1"}}, + name: "file not found", + loc: "nonexistent-file.yaml", + expectError: true, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - reader, err := os.Open(tt.inventory) - require.NoError(t, err) - defer reader.Close() - got, err := playbook.parseInventory(reader, tt.groups) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + p := &PlayBook{ + User: "testuser", + } + inv, err := p.loadInventory(tc.loc) + + if tc.expectError { + require.Error(t, err) + return + } + require.NoError(t, err) - assert.Equal(t, tt.want, got) + assert.NotNil(t, inv) + require.Len(t, inv.Groups, 3) + require.Len(t, inv.Hosts, 1) + + // check "all" group + allGroup := inv.Groups["all"] + require.Len(t, allGroup, 3, "all group should contain all hosts") + assert.Equal(t, "another.com", allGroup[0].Host) + assert.Equal(t, 22, allGroup[0].Port) + assert.Equal(t, "example.com", allGroup[1].Host) + assert.Equal(t, 22, allGroup[1].Port) + assert.Equal(t, "one.example.com", allGroup[2].Host) + assert.Equal(t, 2222, allGroup[2].Port) + + // check "group1" + group1 := inv.Groups["group1"] + require.Len(t, group1, 1) + assert.Equal(t, "example.com", group1[0].Host) + assert.Equal(t, 22, group1[0].Port) + + // check "group2" + group2 := inv.Groups["group2"] + require.Len(t, group2, 1) + assert.Equal(t, "another.com", group2[0].Host) + assert.Equal(t, 22, group2[0].Port) + + // check hosts + assert.Equal(t, "one.example.com", inv.Hosts[0].Host) + assert.Equal(t, 2222, inv.Hosts[0].Port) }) } } diff --git a/app/config/testdata/playbook-with-inventory.yml b/app/config/testdata/playbook-with-inventory.yml new file mode 100644 index 00000000..4f6c2dae --- /dev/null +++ b/app/config/testdata/playbook-with-inventory.yml @@ -0,0 +1,44 @@ +user: umputun +inventory: "testdata/hosts-without-groups.yml" + +targets: + remark42: + hosts: [{name: "h1", host: "h1.example.com"}, {host: "h2.example.com"}] + staging: + groups: ["all"] + + +tasks: + - name: deploy-remark42 + commands: + - name: wait + script: sleep 5 + + - name: copy configuration + copy: {"src": "/local/remark42.yml", "dst": "/srv/remark42.yml", "mkdir": true} + + - name: some local command + options: {local: true} + script: | + ls -la /srv + du -hcs /srv + + - name: git + before: "echo before git" + after: "echo after git" + onerror: "echo onerror git" + script: | + git clone https://example.com/remark42.git /srv || true # clone if doesn't exists, but don't fail if exists + cd /srv + git pull + + - name: docker + options: {no_auto: true} + script: | + docker pull umputun/remark42:latest + docker stop remark42 || true + docker rm remark42 || true + docker run -d --name remark42 -p 8080:8080 umputun/remark42:latest + env: + FOO: bar + BAR: qux \ No newline at end of file diff --git a/app/main.go b/app/main.go index 8d7aae64..6f6065e2 100644 --- a/app/main.go +++ b/app/main.go @@ -24,32 +24,29 @@ import ( ) type options struct { + PositionalArgs struct { + AdHocCmd string `positional-arg-name:"command" description:"run ad-hoc command on target hosts"` + } `positional-args:"yes" positional-optional:"yes"` + PlaybookFile string `short:"p" long:"file" env:"SPOT_FILE" description:"playbook file" default:"spot.yml"` TaskName string `short:"t" long:"task" description:"task name"` Targets []string `short:"d" long:"target" description:"target name" default:"default"` Concurrent int `short:"c" long:"concurrent" description:"concurrent tasks" default:"1"` SSHTimeout time.Duration `long:"timeout" description:"ssh timeout" default:"30s"` - // target overrides - Filter []string `short:"f" long:"filter" description:"filter target hosts"` - InventoryFile string `long:"inventory-file" env:"SPOT_INVENTORY" description:"inventory file"` - InventoryURL string `long:"inventory-url" description:"inventory http url"` - - // connection overrides - SSHUser string `short:"u" long:"user" description:"ssh user"` - SSHKey string `short:"k" long:"key" description:"ssh key"` - - Env map[string]string `short:"e" long:"env" description:"environment variables for all commands"` + // overrides + Inventory string `short:"i" long:"inventory" description:"inventory file or url"` + SSHUser string `short:"u" long:"user" description:"ssh user"` + SSHKey string `short:"k" long:"key" description:"ssh key"` + Env map[string]string `short:"e" long:"env" description:"environment variables for all commands"` // commands filter Skip []string `long:"skip" description:"skip commands"` Only []string `long:"only" description:"run only commands"` - AdHocCmd string `long:"cmd" description:"run ad-hoc command on target hosts"` - Verbose bool `short:"v" long:"verbose" description:"verbose mode"` Dbg bool `long:"dbg" description:"debug mode"` - Help bool `long:"help" description:"show help"` + Help bool `short:"h" long:"help" description:"show help"` } var revision = "latest" @@ -85,12 +82,10 @@ func run(opts options) error { defer cancel() overrides := config.Overrides{ - InventoryFile: opts.InventoryFile, - InventoryURL: opts.InventoryURL, - Environment: opts.Env, - User: opts.SSHUser, - FilterHosts: opts.Filter, - AdHocCommand: opts.AdHocCmd, + Inventory: opts.Inventory, + Environment: opts.Env, + User: opts.SSHUser, + AdHocCommand: opts.PositionalArgs.AdHocCmd, } conf, err := config.New(opts.PlaybookFile, &overrides) @@ -98,7 +93,7 @@ func run(opts options) error { return fmt.Errorf("can't read config: %w", err) } - if opts.AdHocCmd != "" { + if opts.PositionalArgs.AdHocCmd != "" { if err = adHocConf(opts, conf, &defaultUserInfoProvider{}); err != nil { return fmt.Errorf("can't setup ad-hoc config: %w", err) } @@ -119,7 +114,7 @@ func run(opts options) error { } errs := new(multierror.Error) - if opts.AdHocCmd != "" { // run ad-hoc command + if opts.PositionalArgs.AdHocCmd != "" { // run ad-hoc command r.Verbose = true // always verbose for ad-hoc for _, targetName := range opts.Targets { if err := runTaskForTarget(ctx, r, "ad-hoc", targetName); err != nil { diff --git a/app/main_test.go b/app/main_test.go index baa1e8b0..a10ecf93 100644 --- a/app/main_test.go +++ b/app/main_test.go @@ -54,11 +54,11 @@ func Test_runAdhoc(t *testing.T) { defer teardown() opts := options{ - SSHUser: "test", - SSHKey: "runner/testdata/test_ssh_key", - Targets: []string{hostAndPort}, - AdHocCmd: "echo hello", + SSHUser: "test", + SSHKey: "runner/testdata/test_ssh_key", + Targets: []string{hostAndPort}, } + opts.PositionalArgs.AdHocCmd = "echo hello" setupLog(true) err := run(opts) require.NoError(t, err) diff --git a/spot-example.yml b/spot-example.yml index 6c534f04..e08af319 100644 --- a/spot-example.yml +++ b/spot-example.yml @@ -1,13 +1,16 @@ user: app ssh_key: ~/.ssh/id_rsa +inventory: "testdata/hosts-with-groups.yml" targets: prod: - hosts: [{host: "h1.example.com", port: 2222, user: "app-user"}, {host: "h2.example.com"}] + hosts: + - {host: "h1.example.com", port: 2222, user: "app-user"} + - {host: "h2.example.com"} staging: - inventory_file: {location: "inventory.yml"} + groups: ["staging", "dev"] dev: - inventory_url: {location: "http://localhost:8080/inventory", groups: ["dev", "test"]} + groups: ["dev"] tasks: From 81d80f1c812eb718bf394a520acc38bf1749e6ba Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 1 May 2023 16:14:20 -0500 Subject: [PATCH 2/3] add match buy host address --- app/config/config.go | 17 ++++++++++++----- app/config/config_test.go | 21 +++++++++++++++++---- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/app/config/config.go b/app/config/config.go index 8ab3c9f0..5dddb658 100644 --- a/app/config/config.go +++ b/app/config/config.go @@ -272,9 +272,16 @@ func (p *PlayBook) TargetHosts(name string) ([]Destination, error) { return res, nil } - // try as single host in inventory + // try as single host name in inventory for _, h := range p.inventory.Groups["all"] { - if h.Host == name { + if strings.EqualFold(h.Name, name) { + return []Destination{h}, nil + } + } + + // try as a single host address in inventory + for _, h := range p.inventory.Groups["all"] { + if strings.EqualFold(h.Host, name) { return []Destination{h}, nil } } @@ -306,17 +313,17 @@ func (p *PlayBook) loadInventory(loc string) (*InventoryData, error) { if err != nil { return nil, fmt.Errorf("can't get inventory from http %s: %w", loc, err) } - defer resp.Body.Close() //nolint + defer resp.Body.Close() // nolint if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("can't get inventory from http %s, status: %s", loc, resp.Status) } rdr = resp.Body } else { // location is a file - f, err := os.Open(loc) //nolint + f, err := os.Open(loc) // nolint if err != nil { return nil, fmt.Errorf("can't open inventory file %s: %w", loc, err) } - defer f.Close() //nolint + defer f.Close() // nolint rdr = f } diff --git a/app/config/config_test.go b/app/config/config_test.go index 64986c4a..616f810c 100644 --- a/app/config/config_test.go +++ b/app/config/config_test.go @@ -338,11 +338,11 @@ func TestTargetHosts(t *testing.T) { Groups: map[string][]Destination{ "all": { {Host: "host1.example.com", Port: 22, User: "user1"}, - {Host: "host2.example.com", Port: 22, User: "defaultuser"}, + {Host: "host2.example.com", Port: 22, User: "defaultuser", Name: "host2"}, {Host: "host3.example.com", Port: 22, User: "defaultuser"}, }, "group1": { - {Host: "host2.example.com", Port: 2222, User: "defaultuser", Name: "host1"}, + {Host: "host2.example.com", Port: 2222, User: "defaultuser", Name: "host2"}, }, }, Hosts: []Destination{ @@ -378,7 +378,7 @@ func TestTargetHosts(t *testing.T) { Host: "host2.example.com", Port: 2222, User: "defaultuser", - Name: "host1", + Name: "host2", }, }, expectError: false, @@ -391,7 +391,7 @@ func TestTargetHosts(t *testing.T) { Host: "host2.example.com", Port: 2222, User: "defaultuser", - Name: "host1", + Name: "host2", }, }, expectError: false, @@ -420,6 +420,19 @@ func TestTargetHosts(t *testing.T) { }, expectError: false, }, + { + name: "target as single host address", + targetName: "host2.example.com", + expected: []Destination{ + { + Host: "host2.example.com", + Port: 22, + User: "defaultuser", + Name: "host2", + }, + }, + expectError: false, + }, { name: "invalid host:port format", targetName: "host5.example.com:invalid", From 8778becf970099d310c0d8148cd7155d6e459c3d Mon Sep 17 00:00:00 2001 From: Umputun Date: Mon, 1 May 2023 16:54:48 -0500 Subject: [PATCH 3/3] fix issue with user override --- app/config/config.go | 32 ++- app/config/config_test.go | 497 +++----------------------------------- 2 files changed, 56 insertions(+), 473 deletions(-) diff --git a/app/config/config.go b/app/config/config.go index 5dddb658..8db95110 100644 --- a/app/config/config.go +++ b/app/config/config.go @@ -225,9 +225,15 @@ func (p *PlayBook) Task(name string) (*Task, error) { // TargetHosts returns target hosts for given target name. // It applies overrides if any set and also retrieves hosts from inventory file or url if any set. func (p *PlayBook) TargetHosts(name string) ([]Destination, error) { - user := p.User // default user from playbook - if p.overrides != nil && p.overrides.User != "" { - user = p.overrides.User // override user if set + + userOveride := func(u string) string { + if p.overrides != nil && p.overrides.User != "" { + return p.overrides.User + } + if u != "" { + return u + } + return p.User } t, ok := p.Targets[name] // get target from playbook @@ -240,9 +246,7 @@ func (p *PlayBook) TargetHosts(name string) ([]Destination, error) { if h.Port == 0 { h.Port = 22 // default port is 22 if not set } - if h.User == "" { - h.User = user // default user is playbook's user or override, if not set - } + h.User = userOveride(h.User) res[i] = h } return res, nil @@ -269,20 +273,28 @@ func (p *PlayBook) TargetHosts(name string) ([]Destination, error) { if ok { res := make([]Destination, len(hosts)) copy(res, hosts) + for i, r := range res { + r.User = userOveride(r.User) + res[i] = r + } return res, nil } // try as single host name in inventory for _, h := range p.inventory.Groups["all"] { if strings.EqualFold(h.Name, name) { - return []Destination{h}, nil + res := []Destination{h} + res[0].User = userOveride(h.User) + return res, nil } } // try as a single host address in inventory for _, h := range p.inventory.Groups["all"] { if strings.EqualFold(h.Host, name) { - return []Destination{h}, nil + res := []Destination{h} + res[0].User = userOveride(h.User) + return res, nil } } @@ -293,11 +305,11 @@ func (p *PlayBook) TargetHosts(name string) ([]Destination, error) { if err != nil { return nil, fmt.Errorf("can't parse port %s: %w", elems[1], err) } - return []Destination{{Host: elems[0], Port: port, User: user}}, nil + return []Destination{{Host: elems[0], Port: port, User: userOveride("")}}, nil } // we assume it is a host name, with default port 22 - return []Destination{{Host: name, Port: 22, User: user}}, nil + return []Destination{{Host: name, Port: 22, User: userOveride("")}}, nil } // loadInventoryFile loads inventory from file and returns a struct with groups. diff --git a/app/config/config_test.go b/app/config/config_test.go index 616f810c..fa5116a1 100644 --- a/app/config/config_test.go +++ b/app/config/config_test.go @@ -325,21 +325,15 @@ func TestTargetHosts(t *testing.T) { p := &PlayBook{ User: "defaultuser", Targets: map[string]Target{ - "target1": { - Name: "target1", - Hosts: []Destination{{Host: "host1.example.com", Port: 22}}, - }, - "target2": { - Name: "target2", - Groups: []string{"group1"}, - }, + "target1": {Name: "target1", Hosts: []Destination{{Host: "host1.example.com", Port: 22}}}, + "target2": {Name: "target2", Groups: []string{"group1"}}, }, inventory: &InventoryData{ Groups: map[string][]Destination{ "all": { {Host: "host1.example.com", Port: 22, User: "user1"}, {Host: "host2.example.com", Port: 22, User: "defaultuser", Name: "host2"}, - {Host: "host3.example.com", Port: 22, User: "defaultuser"}, + {Host: "host3.example.com", Port: 22, User: "defaultuser", Name: "host3"}, }, "group1": { {Host: "host2.example.com", Port: 2222, User: "defaultuser", Name: "host2"}, @@ -359,84 +353,49 @@ func TestTargetHosts(t *testing.T) { expectError bool }{ { - name: "target with hosts", - targetName: "target1", - expected: []Destination{ - { - Host: "host1.example.com", - Port: 22, - User: "defaultuser", - }, - }, - expectError: false, + "target with hosts", "target1", nil, + []Destination{{Host: "host1.example.com", Port: 22, User: "defaultuser"}}, + false, }, { - name: "target with groups", - targetName: "target2", - expected: []Destination{ - { - Host: "host2.example.com", - Port: 2222, - User: "defaultuser", - Name: "host2", - }, - }, - expectError: false, + "target with groups", "target2", nil, + []Destination{{Host: "host2.example.com", Port: 2222, User: "defaultuser", Name: "host2"}}, + false, }, { - name: "target as group from inventory", - targetName: "group1", - expected: []Destination{ - { - Host: "host2.example.com", - Port: 2222, - User: "defaultuser", - Name: "host2", - }, - }, - expectError: false, + "target as group from inventory", "group1", nil, + []Destination{{Host: "host2.example.com", Port: 2222, User: "defaultuser", Name: "host2"}}, + false, }, { - name: "target as single host from inventory", - targetName: "host3.example.com", - expected: []Destination{ - { - Host: "host3.example.com", - Port: 22, - User: "defaultuser", - }, - }, - expectError: false, + "target as single host by name from inventory", "host3", nil, + []Destination{{Host: "host3.example.com", Port: 22, User: "defaultuser", Name: "host3"}}, + false, }, { - name: "target as single host with port", - targetName: "host4.example.com:2222", - expected: []Destination{ - { - Host: "host4.example.com", - Port: 2222, - User: "defaultuser", - }, - }, - expectError: false, + "target as single host from inventory", "host3.example.com", nil, + []Destination{{Host: "host3.example.com", Port: 22, User: "defaultuser", Name: "host3"}}, + false, }, { - name: "target as single host address", - targetName: "host2.example.com", - expected: []Destination{ - { - Host: "host2.example.com", - Port: 22, - User: "defaultuser", - Name: "host2", - }, - }, - expectError: false, + "target as single host with port", "host4.example.com:2222", nil, + []Destination{{Host: "host4.example.com", Port: 2222, User: "defaultuser"}}, + false, }, { - name: "invalid host:port format", - targetName: "host5.example.com:invalid", - expectError: true, + "target as single host address", "host2.example.com", nil, + []Destination{{Host: "host2.example.com", Port: 22, User: "defaultuser", Name: "host2"}}, + false, + }, + {"invalid host:port format", "host5.example.com:invalid", nil, nil, true}, + {"random host without a port", "host5.example.com", nil, + []Destination{{Host: "host5.example.com", Port: 22, User: "defaultuser"}}, + false, + }, + { + "user override", "host3", &Overrides{User: "overriddenuser"}, + []Destination{{Host: "host3.example.com", Port: 22, User: "overriddenuser", Name: "host3"}}, + false, }, } @@ -454,394 +413,6 @@ func TestTargetHosts(t *testing.T) { } } -// -// func TestPlaybook_TargetHosts(t *testing.T) { -// p := &PlayBook{ -// User: "default_user", -// inventory: &InventoryData{ -// Groups: map[string][]Destination{ -// "gr1": {{Host: "host1", Port: 22, User: "user11"}}, -// "gr2": {{Host: "host2", Port: 2222, User: "default_user"}}, -// "all": { -// {Host: "host1", Port: 22, User: "user11"}, -// {Host: "host2", Port: 2222, User: "default_user"}, -// }, -// }, -// }, -// Targets: map[string]Target{ -// "target1": { -// Hosts: []Destination{ -// {Host: "host1", Port: 22, User: "user1"}, -// {Host: "host2", Port: 2222}, -// {Host: "host3", Name: "host3_name", Port: 2020, User: "user3"}, -// }, -// }, -// "target2": { -// Groups: []string{"gr1"}, -// }, -// "target3": {}, -// "target4": { -// Groups: []string{"all"}, -// }, -// }, -// } -// -// tests := []struct { -// name string -// targetName string -// overrides *Overrides -// want []Destination -// wantErr bool -// }{ -// { -// name: "target from config", -// targetName: "target1", -// want: []Destination{ -// {Host: "host1", Port: 22, User: "user1"}, -// {Host: "host2", Port: 2222, User: "default_user"}, -// {Name: "host3_name", Host: "host3", Port: 2020, User: "user3"}, -// }, -// wantErr: false, -// }, -// { -// name: "overrides target hosts from inventory, name match", -// targetName: "target4", -// overrides: &Overrides{ -// FilterHosts: []string{"h6", "h5"}, -// }, -// want: []Destination{ -// {Name: "h5", Host: "h5.example.com", Port: 2233, User: "default_user"}, -// {Name: "h6", Host: "h6.example.com", Port: 22, User: "user3"}, -// }, -// wantErr: false, -// }, -// { -// name: "overrides target hosts from inventory address match", -// targetName: "target4", -// overrides: &Overrides{ -// FilterHosts: []string{"h5.example.com", "h7.example.com"}, -// }, -// want: []Destination{ -// {Name: "h5", Host: "h5.example.com", Port: 2233, User: "default_user"}, -// {Name: "", Host: "h7.example.com", Port: 22, User: "user3"}, -// }, -// wantErr: false, -// }, -// { -// name: "overrides target hosts direct, name and address match", -// targetName: "target1", -// overrides: &Overrides{ -// FilterHosts: []string{"host3_name", "bad-host", "host2"}, -// }, -// want: []Destination{ -// {Name: "host3_name", Host: "host3", Port: 2020, User: "user3"}, -// {Name: "", Host: "host2", Port: 2222, User: "default_user"}, -// }, -// wantErr: false, -// }, -// { -// name: "overrides target hosts direct, address match", -// targetName: "target1", -// overrides: &Overrides{ -// FilterHosts: []string{"host1", "bad-host", "host2"}, -// }, -// want: []Destination{ -// {Name: "", Host: "host1", Port: 22, User: "user1"}, -// {Name: "", Host: "host2", Port: 2222, User: "default_user"}, -// }, -// wantErr: false, -// }, -// { -// name: "target not found", -// targetName: "nonexistent", -// wantErr: true, -// }, -// { -// name: "target without anything defined", -// targetName: "target3", -// wantErr: true, -// }, -// { -// name: "target as ip", -// targetName: "127.0.0.1:2222", -// want: []Destination{ -// {Host: "127.0.0.1", Port: 2222, User: "default_user"}, -// }, -// wantErr: false, -// }, -// { -// name: "target as ip, no port", -// targetName: "127.0.0.1", -// want: []Destination{ -// {Host: "127.0.0.1", Port: 22, User: "default_user"}, -// }, -// wantErr: false, -// }, -// { -// name: "target as fqdn", -// targetName: "example.com:2222", -// want: []Destination{ -// {Host: "example.com", Port: 2222, User: "default_user"}, -// }, -// wantErr: false, -// }, -// { -// name: "target as fqdn, no port", -// targetName: "host.example.com", -// want: []Destination{ -// {Host: "host.example.com", Port: 22, User: "default_user"}, -// }, -// wantErr: false, -// }, -// { -// name: "target as localhost with port", -// targetName: "localhost:50958", -// want: []Destination{ -// {Host: "localhost", Port: 50958, User: "default_user"}, -// }, -// wantErr: false, -// }, -// { -// name: "valid target with inventory file", -// targetName: "target2", -// want: []Destination{ -// {Host: "h1.example.com", Port: 22, User: "default_user", Name: "h1"}, -// {Host: "h2.example.com", Port: 2233, User: "default_user", Name: "h2"}, -// {Host: "h3.example.com", Port: 22, User: "user1"}, -// {Host: "h4.example.com", Port: 22, User: "user2", Name: "h4"}, -// }, -// wantErr: false, -// }, -// { -// name: "overrides inventory file", -// targetName: "target2", -// overrides: &Overrides{ -// InventoryFile: "testdata/override_inventory.yml", -// }, -// want: []Destination{ -// {Host: "host3", Port: 22, User: "default_user"}, -// {Host: "host4", Port: 2222, User: "user2"}, -// }, -// wantErr: false, -// }, -// } -// -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// p.overrides = tt.overrides -// got, err := p.TargetHosts(tt.targetName) -// if tt.wantErr { -// require.Error(t, err) -// } else { -// require.NoError(t, err) -// assert.Equal(t, tt.want, got) -// } -// }) -// } -// } - -// -// func TestPlayBook_TargetHostsOverrides(t *testing.T) { -// -// t.Run("override hosts with file", func(t *testing.T) { -// c, err := New("testdata/f1.yml", &Overrides{InventoryFile: "testdata/hosts-without-groups.yml"}) -// require.NoError(t, err) -// res, err := c.TargetHosts("blah") -// require.NoError(t, err) -// assert.Equal(t, []Destination{ -// {Name: "h2", Host: "h2.example.com", Port: 2233, User: "umputun"}, -// {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, -// {Name: "h4", Host: "h4.example.com", Port: 22, User: "user2"}, -// {Name: "hh1", Host: "hh1.example.com", Port: 22, User: "umputun"}, -// {Name: "hh2", Host: "hh2.example.com", Port: 2233, User: "user1"}, -// }, res) -// }) -// -// t.Run("override hosts with file, filtered", func(t *testing.T) { -// c, err := New("testdata/f1.yml", &Overrides{InventoryFile: "testdata/hosts-without-groups.yml", FilterHosts: []string{"h2", "h3"}}) -// require.NoError(t, err) -// res, err := c.TargetHosts("blah") -// require.NoError(t, err) -// assert.Equal(t, []Destination{ -// {Name: "h2", Host: "h2.example.com", Port: 2233, User: "umputun"}, -// {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, -// }, res) -// }) -// -// t.Run("override hosts with file not found", func(t *testing.T) { -// c, err := New("testdata/f1.yml", &Overrides{InventoryFile: "testdata/hosts_not_found"}) -// require.NoError(t, err) -// _, err = c.TargetHosts("blah") -// require.ErrorContains(t, err, "no such file or directory") -// t.Log(err) -// }) -// -// t.Run("override hosts with http", func(t *testing.T) { -// ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { -// fh, err := os.Open("testdata/hosts-without-groups.yml") -// require.NoError(t, err) -// defer fh.Close() -// _, err = io.Copy(w, fh) -// require.NoError(t, err) -// })) -// defer ts.Close() -// c, err := New("testdata/f1.yml", &Overrides{InventoryURL: ts.URL}) -// require.NoError(t, err) -// res, err := c.TargetHosts("blah") -// require.NoError(t, err) -// assert.Equal(t, []Destination{ -// {Name: "h2", Host: "h2.example.com", Port: 2233, User: "umputun"}, -// {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, -// {Name: "h4", Host: "h4.example.com", Port: 22, User: "user2"}, -// {Name: "hh1", Host: "hh1.example.com", Port: 22, User: "umputun"}, -// {Name: "hh2", Host: "hh2.example.com", Port: 2233, User: "user1"}, -// }, res) -// }) -// -// t.Run("override hosts with http, filtered", func(t *testing.T) { -// ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { -// fh, err := os.Open("testdata/hosts-without-groups.yml") -// require.NoError(t, err) -// defer fh.Close() -// _, err = io.Copy(w, fh) -// require.NoError(t, err) -// })) -// defer ts.Close() -// c, err := New("testdata/f1.yml", &Overrides{InventoryURL: ts.URL, FilterHosts: []string{"h3", "h4.example.com"}}) -// require.NoError(t, err) -// res, err := c.TargetHosts("blah") -// require.NoError(t, err) -// assert.Equal(t, []Destination{ -// {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, -// {Name: "h4", Host: "h4.example.com", Port: 22, User: "user2"}, -// }, res) -// }) -// t.Run("override hosts with http failed", func(t *testing.T) { -// ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { -// w.WriteHeader(http.StatusInternalServerError) -// })) -// defer ts.Close() -// c, err := New("testdata/f1.yml", &Overrides{InventoryURL: ts.URL}) -// require.NoError(t, err) -// _, err = c.TargetHosts("blah") -// require.ErrorContains(t, err, "status: 500 Internal Server Error") -// t.Log(err) -// }) -// } -// -// func TestPlayBook_parseInventoryGroups(t *testing.T) { -// playbook := &PlayBook{User: "defaultUser"} -// -// tests := []struct { -// name string -// inventory string -// groups []string -// want []Destination -// }{ -// { -// name: "all groups", -// inventory: "testdata/hosts-with-groups.yml", -// groups: nil, -// want: []Destination{ -// {Host: "h1.example.com", Port: 22, User: "defaultUser", Name: "h1"}, -// {Host: "h2.example.com", Port: 2233, User: "defaultUser", Name: "h2"}, -// {Host: "h3.example.com", Port: 22, User: "user1"}, -// {Host: "h4.example.com", Port: 22, User: "user2", Name: "h4"}, -// {Host: "h5.example.com", Port: 2233, User: "defaultUser", Name: "h5"}, -// {Host: "h6.example.com", Port: 22, User: "user3", Name: "h6"}, -// {Host: "h7.example.com", Port: 22, User: "user3"}, -// }, -// }, -// { -// name: "group 1", -// inventory: "testdata/hosts-with-groups.yml", -// groups: []string{"gr1"}, -// want: []Destination{ -// {Host: "h1.example.com", Port: 22, User: "defaultUser", Name: "h1"}, -// {Host: "h2.example.com", Port: 2233, User: "defaultUser", Name: "h2"}, -// {Host: "h3.example.com", Port: 22, User: "user1"}, -// {Host: "h4.example.com", Port: 22, User: "user2", Name: "h4"}, -// }, -// }, -// { -// name: "group 2", -// inventory: "testdata/hosts-with-groups.yml", -// groups: []string{"gr2"}, -// want: []Destination{ -// {Host: "h5.example.com", Port: 2233, User: "defaultUser", Name: "h5"}, -// {Host: "h6.example.com", Port: 22, User: "user3", Name: "h6"}, -// {Host: "h7.example.com", Port: 22, User: "user3"}, -// }, -// }, -// { -// name: "group 1 and 2", -// inventory: "testdata/hosts-with-groups.yml", -// groups: []string{"gr1", "gr2"}, -// want: []Destination{ -// {Host: "h1.example.com", Port: 22, User: "defaultUser", Name: "h1"}, -// {Host: "h2.example.com", Port: 2233, User: "defaultUser", Name: "h2"}, -// {Host: "h3.example.com", Port: 22, User: "user1"}, -// {Host: "h4.example.com", Port: 22, User: "user2", Name: "h4"}, -// {Host: "h5.example.com", Port: 2233, User: "defaultUser", Name: "h5"}, -// {Host: "h6.example.com", Port: 22, User: "user3", Name: "h6"}, -// {Host: "h7.example.com", Port: 22, User: "user3"}, -// }, -// }, -// { -// name: "empty group", -// inventory: "testdata/hosts-with-groups.yml", -// groups: []string{}, -// want: []Destination{ -// {Host: "h1.example.com", Port: 22, User: "defaultUser", Name: "h1"}, -// {Host: "h2.example.com", Port: 2233, User: "defaultUser", Name: "h2"}, -// {Host: "h3.example.com", Port: 22, User: "user1"}, -// {Host: "h4.example.com", Port: 22, User: "user2", Name: "h4"}, -// {Host: "h5.example.com", Port: 2233, User: "defaultUser", Name: "h5"}, -// {Host: "h6.example.com", Port: 22, User: "user3", Name: "h6"}, -// {Host: "h7.example.com", Port: 22, User: "user3"}, -// }, -// }, -// { -// name: "non-existent group", -// inventory: "testdata/hosts-with-groups.yml", -// groups: []string{"non-existent"}, -// want: []Destination{}, -// }, -// { -// name: "hosts inventory", -// inventory: "testdata/hosts-without-groups.yml", -// want: []Destination{ -// {Name: "h2", Host: "h2.example.com", Port: 2233, User: "defaultUser"}, -// {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, -// {Name: "h4", Host: "h4.example.com", Port: 22, User: "user2"}, -// {Name: "hh1", Host: "hh1.example.com", Port: 22, User: "defaultUser"}, -// {Name: "hh2", Host: "hh2.example.com", Port: 2233, User: "user1"}}, -// }, -// { -// name: "hosts inventory but group name set", -// inventory: "testdata/hosts-without-groups.yml", -// groups: []string{"some"}, -// want: []Destination{ -// {Name: "h2", Host: "h2.example.com", Port: 2233, User: "defaultUser"}, -// {Name: "h3", Host: "h3.example.com", Port: 22, User: "user1"}, -// {Name: "h4", Host: "h4.example.com", Port: 22, User: "user2"}, -// {Name: "hh1", Host: "hh1.example.com", Port: 22, User: "defaultUser"}, -// {Name: "hh2", Host: "hh2.example.com", Port: 2233, User: "user1"}}, -// }, -// } -// -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// reader, err := os.Open(tt.inventory) -// require.NoError(t, err) -// defer reader.Close() -// got, err := playbook.parseInventory(reader, tt.groups) -// require.NoError(t, err) -// assert.Equal(t, tt.want, got) -// }) -// } -// } - func TestPlayBook_loadInventory(t *testing.T) { // create temporary inventory file tmpFile, err := os.CreateTemp("", "inventory-*.yaml")