Skip to content

Commit

Permalink
Add agent: 25x / 90x faster for Sudo / SSH (#2)
Browse files Browse the repository at this point in the history 
* Add agent

This reverts commit f80d5e5.

* simplify makefile

* comment out draft code

* change wording

* fix gitignore

* fix clean

* draft chmod

* add Chown

* add Lookup

* add LookupGroup

* extract function

* add Lstat

* add Mkdir

* add readfile

* draft run

* add Run

* extract function

* add writefile

* decrease agent size

* add missing close

* fix agent shutdown

* fix TODOs

* sort env

* add shutdown route

* run tests faster

* fix test

* extract function

* fix cross-compile
  • Loading branch information
@fornellas
fornellas authored Apr 10, 2023
1 parent e061a06 commit 1565e6e
Show file tree
Hide file tree
Showing 28 changed files with 1,616 additions and 338 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,8 @@
/cover.html
/cover.lcov
/cover.txt
/host/agent/agent_*_*
/host/agent/agent_*_*.gz
/host/agent_*_*_gz.go
/resonance.*.*
/version/.version
44 changes: 41 additions & 3 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ GOOS:

GOARCH_NATIVE_SHELL := case $$(uname -m) in i[23456]86) echo 386;; x86_64) echo amd64;; armv6l|armv7l) echo arm;; aarch64) echo arm64;; *) echo Unknown machine $$(uname -m) 1>&2 ; exit 1 ;; esac
GOARCH_NATIVE := $(shell $(GOARCH_NATIVE_SHELL))
ifneq ($(.SHELLSTATUS),0)
$(error GOARCH failed! output was $(GOARCH))
endif

export GOARCH ?= $(GOARCH_NATIVE)
ifneq ($(.SHELLSTATUS),0)
$(error GOARCH_NATIVE failed! output was $(GOARCH_NATIVE))
Expand Down Expand Up @@ -60,6 +64,8 @@ GOMODCACHE:
# management of local users
GO_BUILD_FLAGS := -tags osusergo

GOARCHS_AGENT := 386 amd64 arm arm64

GOIMPORTS := $(GO) run golang.org/x/tools/cmd/goimports
GOIMPORTS_LOCAL := github.com/fornellas/resonance/

Expand Down Expand Up @@ -105,7 +111,7 @@ GCOV2LCOV := $(GO) run github.com/jandelgado/gcov2lcov
RRB := $(GO) run github.com/fornellas/rrb
RRB_DEBOUNCE ?= 500ms
RRB_LOG_LEVEL ?= info
RRB_IGNORE_PATTERN ?= '.cache/**/*'
RRB_IGNORE_PATTERN ?= '.cache/**/*,host/agent_*_*_gz.go'
RRB_PATTERN ?= '**/*.{go},Makefile'
RRB_EXTRA_CMD ?= true

Expand Down Expand Up @@ -239,7 +245,7 @@ test-help:
help: test-help

.PHONY: test
test:
test: build-agent-$(GOARCH_NATIVE)

# gotest

Expand Down Expand Up @@ -295,8 +301,40 @@ build-help:
@echo 'build: build everything'
help: build-help

.PHONY: build-agent-%
build-agent-%: go-generate
GOARCH=$* GOOS=linux $(GO) build -o host/agent/agent_linux_$* $(GO_BUILD_FLAGS) ./host/agent/
gzip < host/agent/agent_linux_$* > host/agent/agent_linux_$*.gz
cat << EOF > host/agent_linux_$*_gz.go
package host
import _ "embed"
//go:embed agent/agent_linux_$*.gz
var agent_linux_$* []byte
func init() {
AgentBinGz["linux.$*"] = agent_linux_$*
}
EOF
build-agent: $(foreach GOARCH,$(GOARCHS_AGENT),build-agent-$(GOARCH))

.PHONY: clean-agent-%
clean-agent-%:
rm -f host/agent/agent_linux_$*
rm -f host/agent/agent_linux_$*.gz
rm -rf host/agent_linux_$*_gz.go
clean-agent: $(foreach GOARCH,$(GOARCHS_AGENT),clean-agent-$(GOARCH))
clean: clean-agent
build: clean-agent
go-generate: clean-agent
goimports: clean-agent
go-mod-tidy: clean-agent
go-get-u: clean-agent
staticcheck: clean-agent
misspell: clean-agent
gocyclo: clean-agent
go-vet: clean-agent

.PHONY: build
build: go go-generate
build: go go-generate build-agent
$(GO) build -o resonance.$(GOOS).$(GOARCH) $(GO_BUILD_FLAGS) .

.PHONY: clean-build
Expand Down
28 changes: 28 additions & 0 deletions cli/lib/lib.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,33 @@ var hostname string
var defaultHostname = ""
var sudo bool
var defaultSudo = false
var disableAgent bool
var defaultDisableAgent = false

func resetCommon() {
hostname = defaultHostname
sudo = defaultSudo
disableAgent = defaultDisableAgent
}

func wrapHost(ctx context.Context, hst host.Host) (host.Host, error) {
var err error
if sudo {
hst, err = host.NewSudo(ctx, hst)
if err != nil {
return nil, err
}
}

if !disableAgent && hostname != "" {
var err error
hst, err = host.NewAgent(ctx, hst)
if err != nil {
return nil, err
}
}

return hst, nil
}

func addHostFlagsCommon(cmd *cobra.Command) {
Expand All @@ -31,6 +54,11 @@ func addHostFlagsCommon(cmd *cobra.Command) {
&sudo, "sudo", "", defaultSudo,
"Use sudo when interacting with host",
)

cmd.Flags().BoolVarP(
&disableAgent, "disable-agent", "", defaultDisableAgent,
"Disables copying temporary a small agent to remote hosts. This can make things very slow, as without the agent, iteraction require running multiple commands. The only (unusual) use case for this is when the host architecture is not supported by the agent.",
)
}

var stateRoot string
Expand Down
9 changes: 1 addition & 8 deletions cli/lib/lib_darwin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,7 @@ func GetHost(ctx context.Context) (host.Host, error) {
return nil, err
}

if sudo {
hst, err = host.NewSudo(ctx, hst)
if err != nil {
return nil, err
}
}

return hst, nil
return wrapHost(ctx, hst)
}

func AddHostFlags(cmd *cobra.Command) {
Expand Down
9 changes: 1 addition & 8 deletions cli/lib/lib_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,7 @@ func GetHost(ctx context.Context) (host.Host, error) {
return nil, errors.New("must provide either --localhost or --hostname")
}

if sudo {
hst, err = host.NewSudo(ctx, hst)
if err != nil {
return nil, err
}
}

return hst, nil
return wrapHost(ctx, hst)
}

func AddHostFlags(cmd *cobra.Command) {
Expand Down
8 changes: 4 additions & 4 deletions cli/tests/host/test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"os/user"
"testing"

"github.com/fornellas/resonance/host"
"github.com/fornellas/resonance/host/types"
"github.com/fornellas/resonance/log"
)

Expand Down Expand Up @@ -54,8 +54,8 @@ type FuncRemove struct {
}

type FuncRun struct {
Cmd host.Cmd
ReturnWaitStatus host.WaitStatus
Cmd types.Cmd
ReturnWaitStatus types.WaitStatus
ReturnStdout string
ReturnStderr string
ReturnError error
Expand Down Expand Up @@ -186,7 +186,7 @@ func (t Test) Remove(ctx context.Context, name string) error {
return funcCall.Remove.ReturnError
}

func (t Test) Run(ctx context.Context, cmd host.Cmd) (host.WaitStatus, string, string, error) {
func (t Test) Run(ctx context.Context, cmd types.Cmd) (types.WaitStatus, string, string, error) {
logger := log.GetLogger(ctx)
logger.Debugf("Run %s", cmd)
funcCall := t.getFuncCall()
Expand Down
10 changes: 6 additions & 4 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,19 @@ require (
github.com/fornellas/rrb v0.1.0
github.com/fzipp/gocyclo v0.6.0
github.com/gliderlabs/ssh v0.3.5
github.com/gorilla/mux v1.8.0
github.com/jandelgado/gcov2lcov v1.0.5
github.com/kylelemons/godebug v1.1.0
github.com/openconfig/goyang v1.2.0
github.com/rakyll/gotest v0.0.6
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.7.0
github.com/stretchr/testify v1.8.2
golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d
go.uber.org/multierr v1.11.0
golang.org/x/crypto v0.8.0
golang.org/x/net v0.9.0
golang.org/x/term v0.7.0
golang.org/x/tools v0.7.0
golang.org/x/tools v0.8.0
gopkg.in/yaml.v3 v3.0.1
honnef.co/go/tools v0.4.3
)
Expand All @@ -30,7 +33,6 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.18 // indirect
github.com/nxadm/tail v1.4.8 // indirect
Expand All @@ -41,7 +43,7 @@ require (
golang.org/x/exp/typeparams v0.0.0-20230321023759-10a507213a29 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/sys v0.7.0 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
golang.org/x/text v0.9.0 // indirect
)

replace gopkg.in/yaml.v3 v3.0.1 => github.com/fornellas/yaml v0.0.0-20230305130802-4e6a4cc61a50
25 changes: 13 additions & 12 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,12 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/protobuf v3.11.4+incompatible/go.mod h1:lUQ9D1ePzbH2PrIS7ob/bjm9HXyH5WHB0Akwh7URreM=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jandelgado/gcov2lcov v1.0.5 h1:rkBt40h0CVK4oCb8Dps950gvfd1rYvQ8+cWa346lVU0=
github.com/jandelgado/gcov2lcov v1.0.5/go.mod h1:NnSxK6TMlg1oGDBfGelGbjgorT5/L3cchlbtgFYZSss=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
Expand Down Expand Up @@ -100,10 +97,13 @@ github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/williammartin/subreaper v0.0.0-20181101193406-731d9ece6883 h1:m8FhqozUpxMLUEeZ8PswV/pD1M4CoP8yAauTHvveoL0=
github.com/williammartin/subreaper v0.0.0-20181101193406-731d9ece6883/go.mod h1:jgqr305WXwkGQIAPYqA4EwWTMSVslVFqpYX/+YkiLXc=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d h1:3qF+Z8Hkrw9sOhrFHti9TlB1Hkac1x+DNRkv0XQiFjo=
golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp/typeparams v0.0.0-20230321023759-10a507213a29 h1:e7LhZmJ631l59keHP9ssC3sgSn3/oiEHKHKXDkimURY=
golang.org/x/exp/typeparams v0.0.0-20230321023759-10a507213a29/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
Expand All @@ -120,7 +120,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down Expand Up @@ -150,15 +151,16 @@ golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
Expand All @@ -172,9 +174,8 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
Expand Down
Loading

0 comments on commit 1565e6e

Please sign in to comment.