Skip to content

Commit

Permalink
Windows Update Support (#444)
Browse files Browse the repository at this point in the history
On unix, we replace the old binary with the new, and then exec. Windows disallows _both_ of those steps. Instead, we can do some moves, and then exit and let the service manager restart.

See ADR and example code in tools/upgrade-exec-service-testing for technical details
  • Loading branch information
directionless authored Mar 15, 2019
1 parent 6204043 commit 4c0320d
Show file tree
Hide file tree
Showing 34 changed files with 1,142 additions and 176 deletions.
20 changes: 14 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -40,19 +40,27 @@ osqueryi-tables: table.ext
extension: .pre-build
go run cmd/make/make.go -targets=extension


xp: xp-launcher xp-extension xp-grpc-extension

xp-%: .pre-build
xp-%:
$(MAKE) -j darwin-xp-$* windows-xp-$* linux-xp-$*

darwin-xp-%: .pre-build
go run cmd/make/make.go -targets=$* -linkstamp -os=darwin

linux-xp-%: .pre-build
go run cmd/make/make.go -targets=$* -linkstamp -os=linux

windows-xp-%: .pre-build
go run cmd/make/make.go -targets=$* -linkstamp -os=windows


codesign-darwin:
codesign-darwin: xp
codesign --force -s "${CODESIGN_IDENTITY}" -v ./build/darwin/launcher
codesign --force -s "${CODESIGN_IDENTITY}" -v ./build/darwin/osquery-extension.ext

codesign: xp codesign-darwin
codesign: codesign-darwin

package-builder: .pre-build deps
go run cmd/make/make.go -targets=package-builder -linkstamp
Expand Down Expand Up @@ -100,18 +108,18 @@ lint: \
lint-go-vet \
lint-go-nakedret

lint-go-deadcode:
lint-go-deadcode: deps-go
deadcode cmd/ pkg/

lint-misspell:
lint-misspell: deps-go
git ls-files \
| grep -v pkg/simulator/testdata/bad_symlink \
| xargs misspell -error -f 'misspell: {{ .Filename }}:{{ .Line }}:{{ .Column }}:corrected {{ printf "%q" .Original }} to {{ printf "%q" .Corrected }}'

lint-go-vet:
go vet ./cmd/... ./pkg/...

lint-go-nakedret:
lint-go-nakedret: deps-go
nakedret ./...


Expand Down
36 changes: 20 additions & 16 deletions cmd/launcher/launcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,24 @@ func runLauncher(ctx context.Context, cancel func(), opts *options, logger log.L
// create a rungroup for all the actors we create to allow for easy start/stop
var runGroup run.Group

// Create a channel for signals
sigChannel := make(chan os.Signal, 1)

// Add a rungroup to catch things on the sigChannel
// Attach a notifier for os.Interrupt
runGroup.Add(func() error {
signal.Notify(sigChannel, os.Interrupt)
select {
case <-sigChannel:
level.Info(logger).Log("msg", "beginnning shutdown via signal")
return nil
}
}, func(err error) {
level.Info(logger).Log("msg", "interrupted", "err", err, "stack", fmt.Sprintf("%+v", err))
cancel()
close(sigChannel)
})

var client service.KolideService
{
switch opts.transport {
Expand Down Expand Up @@ -298,6 +316,7 @@ func runLauncher(ctx context.Context, cancel func(), opts *options, logger log.L
NotaryURL: opts.notaryServerURL,
MirrorURL: opts.mirrorServerURL,
HTTPClient: httpClient,
SigChannel: sigChannel,
}

// create an updater for osquery
Expand All @@ -315,7 +334,7 @@ func runLauncher(ctx context.Context, cancel func(), opts *options, logger log.L
launcherUpdater, err := createUpdater(
ctx,
launcherPath,
launcherFinalizer(logger, runnerShutdown),
updateFinalizer(logger, runnerShutdown),
logger,
config,
)
Expand All @@ -325,21 +344,6 @@ func runLauncher(ctx context.Context, cancel func(), opts *options, logger log.L
runGroup.Add(launcherUpdater.Execute, launcherUpdater.Interrupt)
}

// Create the signal notifier and add it to the rungroup
sig := make(chan os.Signal, 1)
runGroup.Add(func() error {
signal.Notify(sig, os.Interrupt)
select {
case <-sig:
level.Info(logger).Log("msg", "beginnning shutdown")
return nil
}
}, func(err error) {
level.Info(logger).Log("msg", "interrupted", "err", err, "stack", fmt.Sprintf("%+v", err))
cancel()
close(sig)
})

err = runGroup.Run()
return errors.Wrap(err, "run service")
}
Expand Down
10 changes: 9 additions & 1 deletion cmd/launcher/svc_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,10 +84,18 @@ func (w *winSvc) Execute(args []string, r <-chan svc.ChangeRequest, changes chan
go func() {
err := runLauncher(ctx, cancel, w.opts, w.logger)
if err != nil {
level.Info(w.logger).Log("err", err, "stack", fmt.Sprintf("%+v", err))
level.Info(w.logger).Log("msg", "runLauncher exited", "err", err, "stack", fmt.Sprintf("%+v", err))
changes <- svc.Status{State: svc.Stopped, Accepts: cmdsAccepted}
os.Exit(1)
}

// If we get here, it means runLauncher returned nil. If we do
// nothing, the service is left running, but with no
// functionality. Instead, signal that as a stop to the service
// manager, and exit. We rely on the service manager to restart.
level.Info(w.logger).Log("msg", "runLauncher exited cleanly")
changes <- svc.Status{State: svc.Stopped, Accepts: cmdsAccepted}
os.Exit(0)
}()

for {
Expand Down
34 changes: 34 additions & 0 deletions cmd/launcher/updater-finalizer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
// +build !windows

package main

import (
"fmt"
"os"
"syscall"

"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/pkg/errors"
)

// updateFinalizer finalizes a launcher update. It assume the new
// binary has been copied into place, and calls exec, so we start a
// new running launcher in our place.
func updateFinalizer(logger log.Logger, shutdownOsquery func() error) func() error {
return func() error {
if err := shutdownOsquery(); err != nil {
level.Info(logger).Log(
"method", "updateFinalizer",
"err", err,
"stack", fmt.Sprintf("%+v", err),
)
}
// replace launcher
level.Info(logger).Log("msg", "Exec for updated launcher")
if err := syscall.Exec(os.Args[0], os.Args, os.Environ()); err != nil {
return errors.Wrap(err, "restarting launcher")
}
return nil
}
}
31 changes: 31 additions & 0 deletions cmd/launcher/updater-finalizer_windows.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// +build windows

package main

import (
"fmt"

"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/pkg/errors"
)

// updateFinalizer finalizes a launcher update. As windows does not
// support an exec, we exit so the service manager will restart
// us. Exit(0) might be more correct, but that's harder to plumb
// through this stack. So, return an error here to trigger an exit
// higher in the stack.
func updateFinalizer(logger log.Logger, shutdownOsquery func() error) func() error {
return func() error {
if err := shutdownOsquery(); err != nil {
level.Info(logger).Log(
"msg", "calling shutdownOsquery",
"method", "updateFinalizer",
"err", err,
"stack", fmt.Sprintf("%+v", err),
)
}
level.Info(logger).Log("msg", "Exit for updated launcher")
return errors.New("Exiting launcher to allow a service manager restart")
}
}
25 changes: 5 additions & 20 deletions cmd/launcher/updater.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"fmt"
"net/http"
"os"
"syscall"
"time"

"github.com/go-kit/kit/log"
Expand All @@ -26,6 +25,8 @@ type updaterConfig struct {
MirrorURL string

HTTPClient *http.Client

SigChannel chan os.Signal
}

func createUpdater(
Expand All @@ -46,6 +47,7 @@ func createUpdater(
autoupdate.WithMirrorURL(config.MirrorURL),
autoupdate.WithFinalizer(finalizer),
autoupdate.WithUpdateChannel(config.UpdateChannel),
autoupdate.WithSigChannel(config.SigChannel),
)
if err != nil {
return nil, err
Expand All @@ -59,8 +61,8 @@ func createUpdater(
Execute: func() error {
level.Info(logger).Log("msg", "updater started")

// run the updater and set the stop function so that the interrupt has aaccess to it
stop, err = updater.Run(tuf.WithFrequency(config.AutoupdateInterval))
// run the updater and set the stop function so that the interrupt has access to it
stop, err = updater.Run(tuf.WithFrequency(config.AutoupdateInterval), tuf.WithLogger(logger))
if err != nil {
return errors.Wrap(err, "running updater")
}
Expand All @@ -79,20 +81,3 @@ func createUpdater(
},
}, nil
}

func launcherFinalizer(logger log.Logger, shutdownOsquery func() error) func() error {
return func() error {
if err := shutdownOsquery(); err != nil {
level.Info(logger).Log(
"method", "launcherFinalizer",
"err", err,
"stack", fmt.Sprintf("%+v", err),
)
}
// replace launcher
if err := syscall.Exec(os.Args[0], os.Args, os.Environ()); err != nil {
return errors.Wrap(err, "restarting launcher")
}
return nil
}
}
78 changes: 78 additions & 0 deletions docs/architecture/2019-03-11_autoupdate.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# Launcher Auto Update Process

## Authors

- seph ([@directionless](https://github.com/directionless))

## Status

Accepted (March 11, 2019)

## Context

One of the features of Launcher is it's ability to securely update
osquery and itself. The unix implementation is a straightforward
`exec` implementation. However, Windows does not have an `exec`.

This ADR documents the current implementation, and a solution for
windows.

## Decision

New software versions are distributed using [The Update Framework
(TUF)](https://theupdateframework.github.io/). We use a Go client
library that we [built in-house](https://github.com/kolide/updater).

Launcher periodically checks for new updates. When a new version is
detected, it is downloaded into a staging directory, and then the
running binary is replaced. This code can be found in
[autoupdate.go](/pkg/autoupdate/autoupdate.go)

On Unix, launcher calls `syscall.Exec` to replace the current
executable without a new process, using the new binary. This code can
be found in [updater.go](/cmd/launcher/updater.go)

### Windows Variation

There are two needed changes on Windows.

First, Windows does not support replacing a running binary on
disk. Attempting will result in a `Permission Denied` error. A
workaround is to rename the old version, and then place them new one
in the correct location. This has the drawback of losing atomicity.

Second, Windows does not support `exec`. Instead, we will exit
launcher, and assume the service manager will restart. Empirically, it
will start the new binary on the configured path.

Exiting launcher is hard to navigate. Things inside TUF are buried
deep in routines. Simple returning an error isn't enough. While we
could call `os.Exit` that seems abrupt. So instead, we plumb the
signal channel through, and signal a `os.Interrupt` on it. (Note that
it's not a _signal_ in the posix sense. It's a constant sent to a
channel)

### Example Code

There an [example service](/tools/upgrade-exec-service-testing/)
exploring these mechanisms. See code for further comments and
discussion.

## Consequences

One consequence of this approach, is that the installed file is
_updated_ in place. While the new binary is verified by Launcher, this
may look like corruption in some packaging systems.

The update process on Windows is based on the service manager
restarting the service. We don't believe there's a downside here, but
it does increase restart counts. Due to implementation limitations of
WiX, the shortest recovery period is 1 day.

Due to the nature of this update process, updates that depend on
command line flag changes, require a re-installation of Launcher. They
are handled outside this update process.

The act of moving a new binary over a running old one (as we do on
unix) results in the running binary no longer being disk. This can
trigger notices in some monitoring software, for example, osquery.
6 changes: 3 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ require (
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 // indirect
github.com/ghodss/yaml v1.0.0
github.com/go-bindata/go-bindata v1.0.0
github.com/go-kit/kit v0.7.0
github.com/go-kit/kit v0.8.0
github.com/gogo/protobuf v1.2.0
github.com/golang/protobuf v1.2.0
github.com/google/certificate-transparency-go v1.0.21 // indirect
Expand All @@ -40,11 +40,11 @@ require (
github.com/jinzhu/gorm v1.9.1 // indirect
github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect
github.com/jinzhu/now v0.0.0-20181116074157-8ec929ed50c3 // indirect
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 // indirect
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1
github.com/knightsc/system_policy v1.1.1-0.20190125011806-04a47ae55cf7
github.com/kolide/kit v0.0.0-20181124013649-bd1a9de64d48
github.com/kolide/osquery-go v0.0.0-20190113061206-be0a8de4cf1d
github.com/kolide/updater v0.0.0-20181127001321-d5bab28ddf67
github.com/kolide/updater v0.0.0-20190315001611-15bbc19b5b80
github.com/kr/pty v1.1.2
github.com/mattn/go-sqlite3 v1.10.0
github.com/miekg/pkcs11 v0.0.0-20171207072458-57296583125b // indirect
Expand Down
10 changes: 8 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,18 @@ github.com/go-bindata/go-bindata v1.0.0 h1:DZ34txDXWn1DyWa+vQf7V9ANc2ILTtrEjtlsd
github.com/go-bindata/go-bindata v1.0.0/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
github.com/go-kit/kit v0.7.0 h1:ApufNmWF1H6/wUbAG81hZOHmqwd0zRf8mNfLjYj/064=
github.com/go-kit/kit v0.7.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.7.0 h1:S04+lLfST9FvL8dl4R31wVUC/paZp/WQZbLmUgWboGw=
github.com/go-stack/stack v1.7.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
Expand Down Expand Up @@ -115,8 +121,8 @@ github.com/kolide/kit v0.0.0-20181124013649-bd1a9de64d48 h1:n8FMcks5eBrK5+LGMw5W
github.com/kolide/kit v0.0.0-20181124013649-bd1a9de64d48/go.mod h1:wdtviHFxzZ59XeVE7KnsKbj/F+g9J5Cbx1mcXEdLKgY=
github.com/kolide/osquery-go v0.0.0-20190113061206-be0a8de4cf1d h1:2/+lLTfZan4IGrD8TGC1ZVkehWLtrRuah47FASAnifE=
github.com/kolide/osquery-go v0.0.0-20190113061206-be0a8de4cf1d/go.mod h1:umPEbeG8R8RDJpQ0EMyRdj+6pfnJK4FTzjafwgovDUk=
github.com/kolide/updater v0.0.0-20181127001321-d5bab28ddf67 h1:vq66QzvrIQtUmRDqPl1kHYJT4z25fNNz+Pn9QDiCh9Y=
github.com/kolide/updater v0.0.0-20181127001321-d5bab28ddf67/go.mod h1:arAVA89BRUlXkEhRyXcO6p66LK+HulKv+2WKjiWzwtM=
github.com/kolide/updater v0.0.0-20190315001611-15bbc19b5b80 h1:XFzdAHvTlbQoHZdEgOEiFt93eyfXP6VZCwH5p+lPpBg=
github.com/kolide/updater v0.0.0-20190315001611-15bbc19b5b80/go.mod h1:x3dEGYbZovhD1t8OwEgdyu/4ZCvrn9QvkbPtOZnul8k=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
Expand Down
Loading

0 comments on commit 4c0320d

Please sign in to comment.