Skip to content

Commit

Permalink
bottlerocket host update api integration
Browse files Browse the repository at this point in the history
Adds support for updating via the Bottlerocket Update API.
The new platform is used if the node is labeled
bottlerocket.aws/updater-interface-version=2.0.0
  • Loading branch information
etungsten committed Jul 21, 2020
1 parent 56f691f commit edf81ed
Show file tree
Hide file tree
Showing 10 changed files with 634 additions and 26 deletions.
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,15 @@ module github.com/bottlerocket-os/bottlerocket-update-operator
go 1.12

require (
github.com/coreos/go-systemd/v22 v22.0.0
github.com/godbus/dbus/v5 v5.0.3
github.com/Masterminds/semver v1.5.0
github.com/google/go-cmp v0.3.1 // indirect
github.com/googleapis/gnostic v0.3.1 // indirect
github.com/imdario/mergo v0.3.7 // indirect
github.com/karlseguin/ccache v2.0.3+incompatible
github.com/karlseguin/expect v1.0.1 // indirect
github.com/pkg/errors v0.8.1
github.com/sirupsen/logrus v1.4.2
github.com/stretchr/testify v1.3.0
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 // indirect
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 // indirect
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect
Expand Down
6 changes: 2 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
Expand All @@ -25,8 +27,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd/v22 v22.0.0 h1:XJIw/+VlJ+87J+doOxznsAWIdmWuViOVhkQamW5YV28=
github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down Expand Up @@ -58,8 +58,6 @@ github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nA
github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME=
github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
Expand Down
7 changes: 1 addition & 6 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/controller"
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/k8sutil"
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/logging"
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/platform/updog"
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/sigcontext"
"github.com/pkg/errors"
"k8s.io/client-go/kubernetes"
Expand Down Expand Up @@ -89,11 +88,7 @@ func runController(ctx context.Context, kube kubernetes.Interface, nodeName stri

func runAgent(ctx context.Context, kube kubernetes.Interface, nodeName string) error {
log := logging.New("agent")
platform, err := updog.New()
if err != nil {
return errors.WithMessage(err, "could not setup platform for agent")
}
a, err := agent.New(log, kube, platform, nodeName)
a, err := agent.New(log, kube, nodeName)
if err != nil {
return err
}
Expand Down
36 changes: 31 additions & 5 deletions pkg/agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import (
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/marker"
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/nodestream"
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/platform"
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/platform/api"
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/platform/updog"
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/workgroup"

"github.com/pkg/errors"
Expand Down Expand Up @@ -67,18 +69,42 @@ type proc interface {
KillProcess() error
}

func New(log logging.Logger, kube kubernetes.Interface, plat platform.Platform, nodeName string) (*Agent, error) {
func New(log logging.Logger, kube kubernetes.Interface, nodeName string) (*Agent, error) {
if nodeName == "" {
return nil, errors.New("nodeName must be provided for Agent to manage")
}
var nodeclient corev1.NodeInterface
if kube != nil {
nodeclient = kube.CoreV1().Nodes()
var platform platform.Platform
nodeclient = kube.CoreV1().Nodes()
// Determine which platform to use depending on the updater interface version
var node, err = nodeclient.Get(nodeName, v1meta.GetOptions{})
if err != nil {
return nil, errors.New("failed to retrieve node information")
}
// Get the updater interface version from the node label
var platformVersion = node.Labels[marker.UpdaterInterfaceVersionKey]
switch platformVersion {
default:
// If the updater interface version is not specified, default to
// using Updog as the platform
log.Warn("unknown platform version specified, defaulting to using updog")
fallthrough
case "1.0.0":
platform, err = updog.New()
if err != nil {
return nil, errors.WithMessage(err, "could not setup Updog platform for agent")
}
case "2.0.0":
platform, err = api.New()
if err != nil {
return nil, errors.WithMessage(err, "could not setup Update API platform for agent")
}
}

return &Agent{
log: log,
kube: kube,
platform: plat,
platform: platform,
poster: &k8sPoster{log, nodeclient},
proc: &osProc{},
nodeName: nodeName,
Expand Down Expand Up @@ -314,7 +340,7 @@ func (a *Agent) realize(in *intent.Intent) error {

case marker.NodeActionUnknown, marker.NodeActionStabilize:
log.Debug("sitrep")
_, err = a.platform.Status()
err = platform.Ping(a.platform)
if err != nil {
break
}
Expand Down
17 changes: 12 additions & 5 deletions pkg/agent/agent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@ import (
"testing"

"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/intent"
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/intent/cache"
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/internal/intents"
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/internal/testoutput"
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/logging"
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/marker"
"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/platform"

"gotest.tools/assert"
)

Expand Down Expand Up @@ -82,12 +84,17 @@ func testAgent(t *testing.T) (*Agent, *testHooks) {
Platform: &testPlatform{},
Proc: &testProc{},
}
a, err := New(testoutput.Logger(t, logging.New("agent")), nil, hooks.Platform, intents.NodeName)
if err != nil {
panic(err)
log := testoutput.Logger(t, logging.New("agent"))
a := &Agent{
log: log,
kube: nil,
platform: hooks.Platform,
poster: hooks.Poster,
proc: hooks.Proc,
nodeName: intents.NodeName,
lastCache: cache.NewLastCache(),
tracker: newPostTracker(),
}
a.poster = hooks.Poster
a.proc = hooks.Proc
return a, hooks
}

Expand Down
215 changes: 215 additions & 0 deletions pkg/platform/api/api.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
package api

import (
"context"
"encoding/json"
"io/ioutil"
"net"
"net/http"
"time"

"github.com/pkg/errors"

"github.com/bottlerocket-os/bottlerocket-update-operator/pkg/logging"
)

const bottlerocketAPISock = "/run/api.sock"

// The minimum required host Bottlerocket OS version is v0.4.1 because that's is when the Update API
// was first added. https://github.com/bottlerocket-os/bottlerocket/releases/tag/v0.4.1
const minimumRequiredOSVer = "0.4.1"

type updateState string

const (
stateIdle updateState = "Idle"
stateAvailable updateState = "Available"
stateStaged updateState = "Staged"
stateReady updateState = "Ready"
)

type updateImage struct {
Arch string `json:"arch"`
Version string `json:"version"`
Variant string `json:"variant"`
}

func (ui *updateImage) Identifier() interface{} {
return ui.Version
}

type stagedImage struct {
Image updateImage `json:"image"`
NextToBoot bool `json:"next_to_boot"`
}

type updateCommand string

const (
commandRefresh updateCommand = "refresh"
commandPrepare updateCommand = "prepare"
commandActivate updateCommand = "activate"
commandDeactivate updateCommand = "deactivate"
)

type commandStatus string

const (
statusSuccess commandStatus = "Success"
Failed commandStatus = "Failed"
Unknown commandStatus = "Unknown"
)

type commandResult struct {
CmdType updateCommand `json:"cmd_type"`
CmdStatus commandStatus `json:"cmd_status"`
Timestamp string `json:"timestamp"`
ExitStatus *int32 `json:"exit_status"`
Stderr *string `json:"stderr"`
}

type updateStatus struct {
UpdateState updateState `json:"update_state"`
AvailableUpdates []string `json:"available_updates"`
ChosenUpdate *updateImage `json:"chosen_update"`
ActivePartition *stagedImage `json:"active_partition"`
StagingPartition *stagedImage `json:"staging_partition"`
MostRecentCommand *commandResult `json:"most_recent_command"`
}

type apiClient struct {
log logging.Logger
httpClient *http.Client
}

func newAPIClient() *apiClient {
return &apiClient{log: logging.New("update-api"), httpClient: &http.Client{
Transport: &http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
dialer := net.Dialer{}
return dialer.DialContext(ctx, "unix", bottlerocketAPISock)
},
},
// Set a 10 second timeout for all requests
Timeout: 10 * time.Second,
},
}
}

func (c *apiClient) do(req *http.Request) (*http.Response, error) {
var response *http.Response
const maxAttempts = 5
attempts := 0
// Retry up to 5 times in case the Update API is busy; Waiting 10 seconds between each attempt.
for ; attempts < maxAttempts; attempts++ {
var err error
response, err = c.httpClient.Do(req)
if err != nil {
return nil, errors.Wrapf(err, "update API request error")
}
if response.StatusCode >= 200 && response.StatusCode < 300 {
// Response OK
break
} else if response.StatusCode == 423 {
if attempts < maxAttempts-1 {
c.log.Info("API server busy, retrying in 10 seconds ...")
// Retry after ten seconds if we get a 423 Locked response (update API busy)
time.Sleep(10 * time.Second)
continue
}
}
// API response was a non-transient error, bail out.
return response, errors.Errorf("bad http response, status code: %d", response.StatusCode)
}
if attempts == 5 {
return nil, errors.New("update API unavailable: retries exhausted")
}
return response, nil
}

func (c *apiClient) Get(path string) (*http.Response, error) {
req, err := http.NewRequest(http.MethodGet, "http://unix"+path, nil)
if err != nil {
return nil, err
}
c.log.WithField("path", path).WithField("method", http.MethodGet).Debugf("update API request")
return c.do(req)
}

func (c *apiClient) Post(path string) (*http.Response, error) {
req, err := http.NewRequest(http.MethodPost, "http://unix"+path, http.NoBody)
if err != nil {
return nil, err
}
c.log.WithField("path", path).WithField("method", http.MethodPost).Debugf("update API request")
return c.do(req)
}

// GetUpdateStatus returns the update status from the update API
func (c *apiClient) GetUpdateStatus() (*updateStatus, error) {
response, err := c.Get("/updates/status")
if err != nil {
return nil, err
}

var updateStatus updateStatus
body, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, err
}
err = json.Unmarshal(body, &updateStatus)
if err != nil {
return nil, err
}
return &updateStatus, nil
}

func (c *apiClient) GetMostRecentCommand() (*commandResult, error) {
updateStatus, err := c.GetUpdateStatus()
if err != nil {
return nil, err
}
return updateStatus.MostRecentCommand, nil
}

type oSInfo struct {
VersionID string `json:"version_id"`
}

func (c *apiClient) GetOSInfo() (*oSInfo, error) {
response, err := c.Get("/os")
if err != nil {
return nil, err
}

var osInfo oSInfo
body, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, err
}
err = json.Unmarshal(body, &osInfo)
if err != nil {
return nil, err
}
return &osInfo, nil
}

func (c *apiClient) RefreshUpdates() error {
_, err := c.Post("/actions/refresh-updates")
return err
}

func (c *apiClient) PrepareUpdate() error {
_, err := c.Post("/actions/prepare-update")
return err
}

func (c *apiClient) ActivateUpdate() error {
_, err := c.Post("/actions/activate-update")
return err
}

func (c *apiClient) Reboot() error {
_, err := c.Post("/actions/reboot")
return err
}
Loading

0 comments on commit edf81ed

Please sign in to comment.