Skip to content

Commit

Permalink
feat(plugin): add timeout support for SSH plugin (#376)
Browse files Browse the repository at this point in the history 
Closes #209

Signed-off-by: Romain Beuque <556072+rbeuque74@users.noreply.github.com>
Signed-off-by: William Poussier <william.poussier@gmail.com>
  • Loading branch information
@rbeuque74
rbeuque74 authored May 2, 2023
1 parent 4355bad commit 25b0cea
Show file tree
Hide file tree
Showing 3 changed files with 69 additions and 24 deletions.
3 changes: 3 additions & 0 deletions hack/template-schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,9 @@
"manual-lastline"
],
"default": "auto-result"
},
"timeout": {
"type": "string"
}
}
},
Expand Down
26 changes: 14 additions & 12 deletions pkg/plugins/builtin/ssh/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,19 @@ The step will be considered successful if the script returns exit code 0, otherw

## Configuration

|Fields|Description
|---|---
| `user` | username for the connection
| `target` | address of the remote machine
| `hops` | a list of intermediate addresses (bastions)
| `script` | multiline text, commands to be run on the machine's shell
| `output_mode` | indicates how to retrieve the output values ; valid values are: `auto-result` (default), `disabled`, `manual-delimiters`, `manual-lastline`
| `result` | an object to extract the values of variables from the machine's shell (only used when `output_mode` is configured to `auto-result`)
| `output_manual_delimiters` | array of 2 strings ; look for a JSON formatted string in the script output between specific delimiters (only used when `output_mode` is configured to `manual-delimiters`)
| `ssh_key` | private ssh key, preferably retrieved from {{.config}}
| `ssh_key_passphrase` | passphrase for the key, if any
| `exit_codes_unrecoverable` | a list of non-zero exit codes (1, 2, 3, ...) or ranges (1-10, ...) which should be considered unrecoverable and halt execution ; these will be returned to the main engine as a `CLIENT_ERROR`
| Fields | Description |
|----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `user` | username for the connection |
| `target` | address of the remote machine |
| `hops` | a list of intermediate addresses (bastions) |
| `script` | multiline text, commands to be run on the machine's shell |
| `output_mode` | indicates how to retrieve the output values ; valid values are: `auto-result` (default), `disabled`, `manual-delimiters`, `manual-lastline` |
| `result` | an object to extract the values of variables from the machine's shell (only used when `output_mode` is configured to `auto-result`) |
| `output_manual_delimiters` | array of 2 strings ; look for a JSON formatted string in the script output between specific delimiters (only used when `output_mode` is configured to `manual-delimiters`) |
| `ssh_key` | private ssh key, preferably retrieved from {{.config}} |
| `ssh_key_passphrase` | passphrase for the key, if any |
| `exit_codes_unrecoverable` | a list of non-zero exit codes (1, 2, 3, ...) or ranges (1-10, ...) which should be considered unrecoverable and halt execution ; these will be returned to the main engine as a `CLIENT_ERROR` |
| `timeout` | defines the maximum duration of the SSH session (connection time not included). Default to `3600s` (5 minutes). |

## Example

Expand Down Expand Up @@ -52,6 +53,7 @@ action:
- "1-10"
- "100"
- "110"
timeout: 30s
```
## Requirements
Expand Down
64 changes: 52 additions & 12 deletions pkg/plugins/builtin/ssh/ssh.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@ import (
"encoding/json"
"fmt"
"net"
"strconv"
"strings"
"time"

"github.com/juju/errors"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/ssh"

"github.com/ovh/utask/pkg/plugins/builtin/scriptutil"
Expand All @@ -17,8 +19,9 @@ import (

// connection configuration values
const (
MaxHops = 10
ConnTimeout = 10 * time.Second
MaxHops = 10
ConnTimeout = 10 * time.Second
DefaultCmdTimeout = 5 * time.Minute
)

// ssh plugin opens an ssh connection and runs commands on target machine
Expand All @@ -27,6 +30,7 @@ var (
taskplugin.WithConfig(configssh, ConfigSSH{}),
taskplugin.WithResources(resourcesssh),
)
ErrSessionTimeout = errors.New("ssh session has not terminated before timeout")
)

// ConfigSSH is the data needed to perform an SSH action
Expand All @@ -41,6 +45,7 @@ type ConfigSSH struct {
Key string `json:"ssh_key"`
KeyPassphrase string `json:"ssh_key_passphrase"`
ExitCodesUnrecoverable []string `json:"exit_codes_unrecoverable"`
Timeout string `json:"timeout,omitempty"`
}

func resourcesssh(i interface{}) []string {
Expand Down Expand Up @@ -76,6 +81,16 @@ func configssh(i interface{}) error {
return fmt.Errorf("ssh too many hops (max %d)", MaxHops)
}

if cfg.Timeout != "" {
dur, err := time.ParseDuration(cfg.Timeout)
if err != nil {
return fmt.Errorf("can't parse timeout field %q: %s", cfg.Timeout, err.Error())
}
if dur < 0 {
return errors.New("timeout must be positive")
}
}

switch cfg.OutputMode {
case "":
// default will have to be reset in execssh as config modification will not be persisted
Expand Down Expand Up @@ -133,6 +148,15 @@ func execssh(stepName string, i interface{}, ctx interface{}) (interface{}, inte
return nil, nil, errors.NewBadRequest(err, "ssh plugin: private key")
}

var executionTimeout time.Duration

if cfg.Timeout != "" {
// Can skip error, value already validated.
executionTimeout, _ = time.ParseDuration(cfg.Timeout)
} else {
executionTimeout = DefaultCmdTimeout
}

config := &ssh.ClientConfig{
User: cfg.User,
Auth: []ssh.AuthMethod{
Expand Down Expand Up @@ -226,26 +250,44 @@ trap printResultJSON EXIT
extraCmd = execStr
}

stdoutstderr, err := session.CombinedOutput(extraCmd)
if err != nil {
exitErr, ok := err.(*ssh.ExitError)
exit := make(chan struct{}, 1)
timer := time.NewTimer(executionTimeout)

go func() {
select {
case <-timer.C:
err := session.Signal(ssh.SIGKILL)
if err != nil {
logrus.Warnf("session signal error: %s", err)
}
case <-exit:
}
}()
cmdOutput, cmdErr := session.CombinedOutput(extraCmd)
if !timer.Stop() {
logrus.Debugf("session run error: %s", cmdErr)
cmdErr = ErrSessionTimeout
}
close(exit)

if cmdErr != nil {
exitErr, ok := cmdErr.(*ssh.ExitError)
if ok {
exitCode = exitErr.Waitmsg.ExitStatus()
exitSignal = exitErr.Waitmsg.Signal()
exitMessage = exitErr.Waitmsg.Msg()
} else {
return nil, nil, err
return nil, nil, cmdErr
}
}
outStr := string(cmdOutput)

outStr := string(stdoutstderr)
metadata := map[string]interface{}{
"output": outStr,
"exit_code": fmt.Sprint(exitCode),
"exit_code": strconv.Itoa(exitCode),
"exit_signal": exitSignal,
"exit_msg": exitMessage,
}

output := make(map[string]interface{})

if resultLine, err := scriptutil.ParseOutput(outStr, cfg.OutputMode, cfg.OutputManualDelimiters); err != nil {
Expand All @@ -256,10 +298,8 @@ trap printResultJSON EXIT
return nil, metadata, err
}
}

if exitCode != 0 {
return output, metadata, scriptutil.FormatErrorExitCode(exitCode, cfg.ExitCodesUnrecoverable, err)
return output, metadata, scriptutil.FormatErrorExitCode(exitCode, cfg.ExitCodesUnrecoverable, cmdErr)
}

return output, metadata, nil
}

0 comments on commit 25b0cea

Please sign in to comment.