From 0163f19ba5d65a6144a6afab8c9ce9eaee517827 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 1 Sep 2022 11:50:44 -0400 Subject: [PATCH 1/3] migrate autopilot implementation to raft-autopilot Nomad's original autopilot was importing from a private package in Consul. It has been moved out to a shared library. Switch Nomad to use this library so that we can eliminate the import of Consul, which is necessary to build Nomad ENT with the current version of the Consul SDK. This also will let us pick up autopilot improvements shared with Consul more easily. --- .changelog/14441.txt | 3 + command/agent/operator_endpoint.go | 11 +- go.mod | 2 +- go.sum | 66 +-------- nomad/autopilot.go | 229 +++++++++++++++++++++++------ nomad/autopilot_oss.go | 26 ++++ nomad/autopilot_test.go | 20 ++- nomad/leader.go | 6 +- nomad/leader_test.go | 82 +++++------ nomad/operator_endpoint.go | 13 +- nomad/serf.go | 3 +- nomad/server.go | 13 +- nomad/server_setup_oss.go | 12 +- nomad/stats_fetcher.go | 76 ++++++---- nomad/stats_fetcher_test.go | 16 +- nomad/status_endpoint.go | 3 +- nomad/structs/autopilot.go | 82 +++++++++++ 17 files changed, 447 insertions(+), 216 deletions(-) create mode 100644 .changelog/14441.txt create mode 100644 nomad/autopilot_oss.go create mode 100644 nomad/structs/autopilot.go diff --git a/.changelog/14441.txt b/.changelog/14441.txt new file mode 100644 index 000000000000..1a3419981336 --- /dev/null +++ b/.changelog/14441.txt @@ -0,0 +1,3 @@ +```release-note:improvement +autopilot: upgrade to raft-autopilot library +``` diff --git a/command/agent/operator_endpoint.go b/command/agent/operator_endpoint.go index 224ddabe27e3..4c8d278cd1d6 100644 --- a/command/agent/operator_endpoint.go +++ b/command/agent/operator_endpoint.go @@ -2,21 +2,20 @@ package agent import ( "context" + "fmt" "io" "net" "net/http" - "strings" - - "fmt" "strconv" + "strings" "time" - "github.com/hashicorp/consul/agent/consul/autopilot" "github.com/hashicorp/go-msgpack/codec" + "github.com/hashicorp/raft" + "github.com/hashicorp/nomad/api" cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/nomad/structs" - "github.com/hashicorp/raft" ) // OperatorRequest is used route operator/raft API requests to the implementing @@ -184,7 +183,7 @@ func (s *HTTPServer) OperatorServerHealth(resp http.ResponseWriter, req *http.Re return nil, nil } - var reply autopilot.OperatorHealthReply + var reply structs.OperatorHealthReply if err := s.agent.RPC("Operator.ServerHealth", &args, &reply); err != nil { return nil, err } diff --git a/go.mod b/go.mod index 01f5c2d0f12a..a352fd879578 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,6 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/gosuri/uilive v0.0.4 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/hashicorp/consul v1.7.8 github.com/hashicorp/consul-template v0.29.3-0.20220829190305-21d2c9bb9752 github.com/hashicorp/consul/api v1.14.0 github.com/hashicorp/consul/sdk v0.11.0 @@ -78,6 +77,7 @@ require ( github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 github.com/hashicorp/nomad/api v0.0.0-20220829153708-e1e5bb1dcefb github.com/hashicorp/raft v1.3.9 + github.com/hashicorp/raft-autopilot v0.1.6 github.com/hashicorp/raft-boltdb/v2 v2.2.0 github.com/hashicorp/serf v0.9.7 github.com/hashicorp/vault/api v1.7.2 diff --git a/go.sum b/go.sum index 51cc87fed41b..79bdc0c74444 100644 --- a/go.sum +++ b/go.sum @@ -55,7 +55,6 @@ github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v10.15.3+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= @@ -128,14 +127,12 @@ github.com/Microsoft/hcsshim v0.9.3/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfy github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.0.1/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/abdullin/seq v0.0.0-20160510034733-d5467c17e7af h1:DBNMBMuMiWYu0b+8KMJuWmfCkcxl09JwdlqwDZZ6U14= @@ -162,7 +159,6 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= -github.com/armon/go-metrics v0.3.4/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.4.0 h1:yCQqn7dwca4ITXb+CbubHmedzaQYHhNhrEXLYUeEe8Q= @@ -236,7 +232,6 @@ github.com/circonus-labs/circonusllhist v0.1.3 h1:TJH+oke8D16535+jHExHj4nQvzlZrj github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200313221541-5f7e5dd04533/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 h1:hzAQntlaYRkVSFEfj9OTWlVV1H155FMD8BTKktLv0QI= @@ -347,7 +342,6 @@ github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19 github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/coredns/coredns v1.1.2/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= @@ -411,7 +405,6 @@ github.com/docker/docker v20.10.17+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05 github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o= github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c= -github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= @@ -431,7 +424,6 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/go-bindata-assetfs v0.0.0-20160803192304-e1a2a7ec64b0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/elazarl/go-bindata-assetfs v1.0.1-0.20200509193318-234c15e7648f h1:AwZUiMWfYSmIiHdFJIubTSs8BFIFoMmUFbeuwBzHIPs= github.com/elazarl/go-bindata-assetfs v1.0.1-0.20200509193318-234c15e7648f/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= @@ -442,7 +434,6 @@ github.com/endocrimes/go-winio v0.4.13-0.20190628114223-fb47a8b41948/go.mod h1:q github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.5/go.mod h1:OXl5to++W0ctG+EHWTFUjiypVxC/Y4VLc/KFU+al13s= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= @@ -486,14 +477,12 @@ github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-ldap/ldap/v3 v3.1.10/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= @@ -509,7 +498,6 @@ github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= @@ -573,7 +561,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -665,14 +652,10 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul v1.7.8 h1:hp308KxAf3zWoGuwp2e+0UUhrm6qHjeBQk3jCZ+bjcY= -github.com/hashicorp/consul v1.7.8/go.mod h1:urbfGaVZDmnXC6geg0LYPh/SRUk1E8nfmDHpz+Q0nLw= github.com/hashicorp/consul-template v0.29.3-0.20220829190305-21d2c9bb9752 h1:VjEbNw/ZtuaQRz3HOHIeinO7qZKX3XIPO33A9tIcsZI= github.com/hashicorp/consul-template v0.29.3-0.20220829190305-21d2c9bb9752/go.mod h1:aiT2d9ReQd7VtFZJELlt1SfEOiiRRpca9Ot/jcyWQps= -github.com/hashicorp/consul/api v1.4.0/go.mod h1:xc8u05kyMa3Wjr9eEAsIAo3dg8+LywT5E/Cl7cNS5nU= github.com/hashicorp/consul/api v1.14.0 h1:Y64GIJ8hYTu+tuGekwO4G4ardXoiCivX9wv1iP/kihk= github.com/hashicorp/consul/api v1.14.0/go.mod h1:bcaw5CSZ7NE9qfOfKCI1xb7ZKjzu/MyvQkCLTfqLqxQ= -github.com/hashicorp/consul/sdk v0.4.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= github.com/hashicorp/consul/sdk v0.10.0/go.mod h1:yPkX5Q6CsxTFMjQQDJwzeNmUUF5NUGGbrDsv9wTb8cw= github.com/hashicorp/consul/sdk v0.11.0 h1:HRzj8YSCln2yGgCumN5CL8lYlD3gBurnervJRJAZyC4= github.com/hashicorp/consul/sdk v0.11.0/go.mod h1:yPkX5Q6CsxTFMjQQDJwzeNmUUF5NUGGbrDsv9wTb8cw= @@ -682,7 +665,6 @@ github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FK github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-bexpr v0.1.2/go.mod h1:ANbpTX1oAql27TZkKVeW8p1w8NTdnyzPe/0qqPCKohU= github.com/hashicorp/go-bexpr v0.1.11 h1:6DqdA/KBjurGby9yTY0bmkathya0lfwF2SeuubCI7dY= github.com/hashicorp/go-bexpr v0.1.11/go.mod h1:f03lAo0duBlDIUMGCuad8oLcgejw4m7U+N8T+6Kz1AE= github.com/hashicorp/go-checkpoint v0.0.0-20171009173528-1545e56e46de h1:XDCSythtg8aWSRSO29uwhgh7b127fWr+m5SemqjSUL8= @@ -691,7 +673,6 @@ github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-connlimit v0.2.0/go.mod h1:OUj9FGL1tPIhl/2RCfzYHrIiWj+VVPGNyVPnUX8AqS0= github.com/hashicorp/go-connlimit v0.3.0 h1:oAojHGjFxUTTTA8c5XXnDqWJ2HLuWbDiBPTpWvNzvqM= github.com/hashicorp/go-connlimit v0.3.0/go.mod h1:OUj9FGL1tPIhl/2RCfzYHrIiWj+VVPGNyVPnUX8AqS0= github.com/hashicorp/go-cty-funcs v0.0.0-20200930094925-2721b1e36840 h1:kgvybwEeu0SXktbB2y3uLHX9lklLo+nzUwh59A3jzQc= @@ -703,8 +684,6 @@ github.com/hashicorp/go-envparse v0.0.0-20180119215841-310ca1881b22/go.mod h1:/N github.com/hashicorp/go-gatedio v0.5.0 h1:Jm1X5yP4yCqqWj5L1TgW7iZwCVPGtVc+mro5r/XX7Tg= github.com/hashicorp/go-getter v1.6.1 h1:NASsgP4q6tL94WH6nJxKWj8As2H/2kop/bB1d8JMyRY= github.com/hashicorp/go-getter v1.6.1/go.mod h1:IZCrswsZPeWv9IkVnLElzRU/gz/QPi6pZHn4tv6vbwA= -github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= -github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= @@ -713,15 +692,12 @@ github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39 github.com/hashicorp/go-hclog v1.2.2 h1:ihRI7YFwcZdiSD7SIenIhHfQH3OuDvWerAUBZbeQS3M= github.com/hashicorp/go-hclog v1.2.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.2.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-kms-wrapping/entropy v0.1.0/go.mod h1:d1g9WGtAunDNpek8jUIEJnBlbgKS1N2Q61QkHiZyR1g= github.com/hashicorp/go-kms-wrapping/v2 v2.0.5 h1:rOFDv+3k05mnW0oaDLffhVUwg03Csn0mvfO98Wdd2bE= github.com/hashicorp/go-kms-wrapping/v2 v2.0.5/go.mod h1:sDQAfwJGv25uGPZA04x87ERglCG6avnRcBT9wYoMII8= -github.com/hashicorp/go-memdb v1.0.3/go.mod h1:LWQ8R70vPrS4OEY9k28D2z8/Zzyu34NVzeRibGAzHO0= github.com/hashicorp/go-memdb v1.3.3 h1:oGfEWrFuxtIUF3W2q/Jzt6G85TrMk9ey6XfYLvVe1Wo= github.com/hashicorp/go-memdb v1.3.3/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -733,16 +709,12 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= github.com/hashicorp/go-plugin v1.4.3 h1:DXmvivbWD5qdiBts9TpBC7BYL1Aia5sxbRgQB+v6UZM= github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= -github.com/hashicorp/go-raftchunking v0.6.1/go.mod h1:cGlg3JtDy7qy6c/3Bu660Mic1JF+7lWqIwCFSb08fX0= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4= github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= -github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= @@ -779,7 +751,6 @@ github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= @@ -788,46 +759,38 @@ github.com/hashicorp/hcl v1.0.1-0.20201016140508-a07e7d50bbee h1:8B4HqvMUtYSjsGk github.com/hashicorp/hcl v1.0.1-0.20201016140508-a07e7d50bbee/go.mod h1:gwlu9+/P9MmKtYrMsHeFRZPXj2CTPm11TDnMeaRHS7g= github.com/hashicorp/hcl/v2 v2.9.2-0.20220525143345-ab3cae0737bc h1:32lGaCPq5JPYNgFFTjl/cTIar9UWWxCbimCs5G2hMHg= github.com/hashicorp/hcl/v2 v2.9.2-0.20220525143345-ab3cae0737bc/go.mod h1:odKNpEeZv3COD+++SQcPyACuKOlM5eBoQlzRyN5utIQ= -github.com/hashicorp/hil v0.0.0-20160711231837-1e86c6b523c5/go.mod h1:KHvg/R2/dPtaePb16oW4qIyzkMxXOL38xjRN64adsts= github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/mdns v1.0.4 h1:sY0CMhFmjIPDMlTB+HfymFHCaYLhgifZ0QhjaYKD/UQ= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/memberlist v0.3.1 h1:MXgUXLqva1QvpVEDQW1IQLG0wivQAtmFlHRQ+1vWZfM= github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 h1:lc3c72qGlIMDqQpQH82Y4vaglRMMFdJbziYWriR4UcE= github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69/go.mod h1:/z+jUGRBlwVpUZfjute9jWaF6/HuhjuFQuL1YXzVD1Q= github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= -github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= -github.com/hashicorp/raft v1.1.2/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= +github.com/hashicorp/raft v1.2.0/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft v1.3.9 h1:9yuo1aR0bFTr1cw7pj3S2Bk6MhJCsnr2NAxvIBrP2x4= github.com/hashicorp/raft v1.3.9/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= +github.com/hashicorp/raft-autopilot v0.1.6 h1:C1q3RNF2FfXNZfHWbvVAu0QixaQK8K5pX4O5lh+9z4I= +github.com/hashicorp/raft-autopilot v0.1.6/go.mod h1:Af4jZBwaNOI+tXfIqIdbcAnh/UyyqIMj/pOISIfhArw= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea h1:RxcPJuutPRM8PUOyiweMmkuNO+RJyfy2jds2gfvgNmU= github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I= github.com/hashicorp/raft-boltdb/v2 v2.2.0 h1:/CVN9LSAcH50L3yp2TsPFIpeyHn1m3VF6kiutlDE3Nw= github.com/hashicorp/raft-boltdb/v2 v2.2.0/go.mod h1:SgPUD5TP20z/bswEr210SnkUFvQP/YjKV95aaiTbeMQ= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hashicorp/serf v0.9.3/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.7 h1:hkdgbqizGQHuU5IPqYM1JdSMV8nKfpuOnZYXssk9muY= github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= -github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= github.com/hashicorp/vault/api v1.7.2 h1:kawHE7s/4xwrdKbkmwQi0wYaIeUhk5ueek7ljuezCVQ= github.com/hashicorp/vault/api v1.7.2/go.mod h1:xbfA+1AvxFseDzxxdWaL0uO99n1+tndus4GCrtouy0M= github.com/hashicorp/vault/api/auth/kubernetes v0.2.0 h1:ScdzRAF8JZIdJYP4oprZKsIS4GSTCaTP4iG2JJlJDvA= github.com/hashicorp/vault/api/auth/kubernetes v0.2.0/go.mod h1:2BKADs9mwqAycDK/6tiHRh2sX0SPnC0DN4wHjJoAirw= -github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= github.com/hashicorp/vault/sdk v0.5.1 h1:zly/TmNgOXCGgWIRA8GojyXzG817POtVh3uzIwzZx+8= github.com/hashicorp/vault/sdk v0.5.1/go.mod h1:DoGraE9kKGNcVgPmTuX357Fm6WAx1Okvde8Vp3dPDoU= github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443 h1:O/pT5C1Q3mVXMyuqg7yuAWUg/jMZR1/0QTzTRdNR6Uw= github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443/go.mod h1:bEpDU35nTu0ey1EXjwNwPjI9xErAsoOCmcMb9GKvyxo= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= -github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 h1:xixZ2bWeofWV68J+x6AzmKuVM/JWCQwkWm6GW/MUR6I= github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hexdigest/gowrap v1.1.7/go.mod h1:Z+nBFUDLa01iaNM+/jzoOA1JJ7sm51rnYFauKFUB5fs= @@ -973,21 +936,16 @@ github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b h1:9+ke9YJ9KGWw5AN github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-testing-interface v1.0.3/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/hashstructure v1.1.0 h1:P6P1hdjqAAknpY/M1CGipelZgp+4y9ja9kmUZPXP+H0= github.com/mitchellh/hashstructure v1.1.0/go.mod h1:xUDAozZz0Wmdiufv0uyhnHkUTN6/6d8ulp4AwfLKrmA= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -995,7 +953,6 @@ github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQ github.com/mitchellh/pointerstructure v1.2.1 h1:ZhBBeX8tSlRpu/FFhXH4RC4OJzFlqsQhoHZAz4x7TIw= github.com/mitchellh/pointerstructure v1.2.1/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= @@ -1104,7 +1061,6 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -1165,7 +1121,6 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rboyer/safeio v0.2.1/go.mod h1:Cq/cEPK+YXFn622lsQ0K4KsPZSPtaptHHEldsy7Fmig= github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03 h1:Wdi9nwnhFNAlseAOekn6B5G/+GMtks9UKbvRU/CMM/o= github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03/go.mod h1:gRAiPF5C5Nd0eyyRdqIu9qTiFSoZzpTq727b5B8fkkU= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -1196,10 +1151,8 @@ github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2 github.com/seccomp/libseccomp-golang v0.10.0/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v0.0.0-20181107111621-48177ef5f880/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil/v3 v3.22.7 h1:flKnuCMfUUrO+oAvwAd6GKZgnPzr098VA/UJ14nhJd4= github.com/shirou/gopsutil/v3 v3.22.7/go.mod h1:s648gW4IywYzUfE/KjXxUsqrqx/T2xO5VqOXxONeRfI= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shoenig/test v0.3.1 h1:dhGZztS6nQuvJ0o0RtUiQHaEO4hhArh/WmWwik3Ols0= github.com/shoenig/test v0.3.1/go.mod h1:xYtyGBC5Q3kzCNyJg/SjgNpfAa2kvmgA0i5+lQso8x0= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= @@ -1346,6 +1299,7 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -1363,7 +1317,6 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191106202628-ed6320f186d4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200422194213-44a606286825/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1517,12 +1470,10 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1637,7 +1588,6 @@ golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fq golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -1677,6 +1627,7 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1770,7 +1721,6 @@ google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11K google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1837,12 +1787,10 @@ google.golang.org/genproto v0.0.0-20220314164441-57ef72a4c106 h1:ErU+UA6wxadoU8n google.golang.org/genproto v0.0.0-20220314164441-57ef72a4c106/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= @@ -1889,7 +1837,6 @@ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1941,12 +1888,10 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.0.0-20190325185214-7544f9db76f6/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/apimachinery v0.0.0-20190223001710-c182ff3b9841/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= @@ -1958,7 +1903,6 @@ k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/client-go v8.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= diff --git a/nomad/autopilot.go b/nomad/autopilot.go index 5e42fff54b35..fd846a981978 100644 --- a/nomad/autopilot.go +++ b/nomad/autopilot.go @@ -3,10 +3,12 @@ package nomad import ( "context" "fmt" + "strconv" metrics "github.com/armon/go-metrics" - "github.com/hashicorp/consul/agent/consul/autopilot" + "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/raft" + autopilot "github.com/hashicorp/raft-autopilot" "github.com/hashicorp/serf/serf" ) @@ -20,63 +22,56 @@ const ( AutopilotVersionTag = "ap_version" ) -// AutopilotDelegate is a Nomad delegate for autopilot operations. +// AutopilotDelegate is a Nomad delegate for autopilot operations. It implements +// the autopilot.ApplicationIntegration interface and methods required for that +// have been documented as such below. type AutopilotDelegate struct { server *Server } +var _ autopilot.ApplicationIntegration = &AutopilotDelegate{} + +// AutopilotConfig is used to retrieve the latest configuration from the Nomad +// delegate. This method is required to implement the ApplicationIntegration +// interface. func (d *AutopilotDelegate) AutopilotConfig() *autopilot.Config { c := d.server.getOrCreateAutopilotConfig() if c == nil { return nil } - conf := &autopilot.Config{ CleanupDeadServers: c.CleanupDeadServers, LastContactThreshold: c.LastContactThreshold, MaxTrailingLogs: c.MaxTrailingLogs, MinQuorum: c.MinQuorum, ServerStabilizationTime: c.ServerStabilizationTime, - DisableUpgradeMigration: c.DisableUpgradeMigration, - ModifyIndex: c.ModifyIndex, - CreateIndex: c.CreateIndex, - } - - if c.EnableRedundancyZones { - conf.RedundancyZoneTag = AutopilotRZTag - } - if c.EnableCustomUpgrades { - conf.UpgradeVersionTag = AutopilotVersionTag + Ext: autopilotConfigExt(c), } return conf } -func (d *AutopilotDelegate) FetchStats(ctx context.Context, servers []serf.Member) map[string]*autopilot.ServerStats { +// FetchServerStats will be called by autopilot to request Nomad fetch the +// server stats out of band. This method is required to implement the +// ApplicationIntegration interface +func (d *AutopilotDelegate) FetchServerStats(ctx context.Context, servers map[raft.ServerID]*autopilot.Server) map[raft.ServerID]*autopilot.ServerStats { return d.server.statsFetcher.Fetch(ctx, servers) } -func (d *AutopilotDelegate) IsServer(m serf.Member) (*autopilot.ServerInfo, error) { - ok, parts := isNomadServer(m) - if !ok || parts.Region != d.server.Region() { - return nil, nil - } - - server := &autopilot.ServerInfo{ - Name: m.Name, - ID: parts.ID, - Addr: parts.Addr, - Build: parts.Build, - Status: m.Status, - } - return server, nil +// KnownServers will be called by autopilot to request the list of servers known +// to Nomad. This method is required to implement the ApplicationIntegration +// interface +func (d *AutopilotDelegate) KnownServers() map[raft.ServerID]*autopilot.Server { + return d.server.autopilotServers() } -// NotifyHealth heartbeats a metric for monitoring if we're the leader. -func (d *AutopilotDelegate) NotifyHealth(health autopilot.OperatorHealthReply) { +// NotifyState will be called when the autopilot state is updated. The Nomad +// leader heartbeats a metric for monitoring based on this information. This +// method is required to implement the ApplicationIntegration interface +func (d *AutopilotDelegate) NotifyState(state *autopilot.State) { if d.server.raft.State() == raft.Leader { - metrics.SetGauge([]string{"nomad", "autopilot", "failure_tolerance"}, float32(health.FailureTolerance)) - if health.Healthy { + metrics.SetGauge([]string{"nomad", "autopilot", "failure_tolerance"}, float32(state.FailureTolerance)) + if state.Healthy { metrics.SetGauge([]string{"nomad", "autopilot", "healthy"}, 1) } else { metrics.SetGauge([]string{"nomad", "autopilot", "healthy"}, 0) @@ -84,24 +79,170 @@ func (d *AutopilotDelegate) NotifyHealth(health autopilot.OperatorHealthReply) { } } -func (d *AutopilotDelegate) PromoteNonVoters(conf *autopilot.Config, health autopilot.OperatorHealthReply) ([]raft.Server, error) { - future := d.server.raft.GetConfiguration() - if err := future.Error(); err != nil { - return nil, fmt.Errorf("failed to get raft configuration: %v", err) +// RemoveFailedServer will be called by autopilot to notify Nomad to remove the +// server in a failed state. This method is required to implement the +// ApplicationIntegration interface. (Note this is expected to return +// immediately so we'll spawn a goroutine for it.) +func (d *AutopilotDelegate) RemoveFailedServer(failedSrv *autopilot.Server) { + go func() { + err := d.server.RemoveFailedNode(failedSrv.Name) + if err != nil { + d.server.logger.Error("could not remove failed server", "server", string(failedSrv.ID)) + } + }() +} + +// MinRaftProtocol returns the lowest supported Raft protocol among alive +// servers +func (s *Server) MinRaftProtocol() (int, error) { + return minRaftProtocol(s.serf.Members(), isNomadServer) +} + +// GetClusterHealth is used to get the current health of the servers, as known +// by the leader. +func (s *Server) GetClusterHealth() *structs.OperatorHealthReply { + + state := s.autopilot.GetState() + if state == nil { + // this behavior seems odd but its functionally equivalent to 1.8.5 where if + // autopilot didn't have a health reply yet it would just return no error + return nil + } + + health := &structs.OperatorHealthReply{ + Healthy: state.Healthy, + FailureTolerance: state.FailureTolerance, + } + + for _, srv := range state.Servers { + srvHealth := structs.ServerHealth{ + ID: string(srv.Server.ID), + Name: srv.Server.Name, + Address: string(srv.Server.Address), + Version: srv.Server.Version, + Leader: srv.State == autopilot.RaftLeader, + Voter: srv.State == autopilot.RaftLeader || srv.State == autopilot.RaftVoter, + LastContact: srv.Stats.LastContact, + LastTerm: srv.Stats.LastTerm, + LastIndex: srv.Stats.LastIndex, + Healthy: srv.Health.Healthy, + StableSince: srv.Health.StableSince, + } + + switch srv.Server.NodeStatus { + case autopilot.NodeAlive: + srvHealth.SerfStatus = serf.StatusAlive + case autopilot.NodeLeft: + srvHealth.SerfStatus = serf.StatusLeft + case autopilot.NodeFailed: + srvHealth.SerfStatus = serf.StatusFailed + default: + srvHealth.SerfStatus = serf.StatusNone + } + + health.Servers = append(health.Servers, srvHealth) + } + + return health +} + +// ------------------- +// helper functions + +func minRaftProtocol(members []serf.Member, serverFunc func(serf.Member) (bool, *serverParts)) (int, error) { + minVersion := -1 + for _, m := range members { + if m.Status != serf.StatusAlive { + continue + } + + ok, server := serverFunc(m) + if !ok { + return -1, fmt.Errorf("not a Nomad server") + } + if server == nil { + continue + } + + vsn, ok := m.Tags["raft_vsn"] + if !ok { + vsn = "1" + } + raftVsn, err := strconv.Atoi(vsn) + if err != nil { + return -1, err + } + + if minVersion == -1 || raftVsn < minVersion { + minVersion = raftVsn + } } - return autopilot.PromoteStableServers(conf, health, future.Configuration().Servers), nil + if minVersion == -1 { + return minVersion, fmt.Errorf("No servers found") + } + + return minVersion, nil } -func (d *AutopilotDelegate) Raft() *raft.Raft { - return d.server.raft +func (s *Server) autopilotServers() map[raft.ServerID]*autopilot.Server { + servers := make(map[raft.ServerID]*autopilot.Server) + + for _, member := range s.serf.Members() { + srv, err := s.autopilotServer(member) + if err != nil { + s.logger.Warn("Error parsing server info", "name", member.Name, "error", err) + continue + } else if srv == nil { + // this member was a client + continue + } + + servers[srv.ID] = srv + } + + return servers } -func (d *AutopilotDelegate) SerfLAN() *serf.Serf { - return d.server.serf +func (s *Server) autopilotServer(m serf.Member) (*autopilot.Server, error) { + ok, srv := isNomadServer(m) + if !ok { + return nil, nil + } + + return s.autopilotServerFromMetadata(srv) } -func (d *AutopilotDelegate) SerfWAN() *serf.Serf { - // serf WAN isn't supported in nomad yet - return nil +func (s *Server) autopilotServerFromMetadata(srv *serverParts) (*autopilot.Server, error) { + server := &autopilot.Server{ + Name: srv.Name, + ID: raft.ServerID(srv.ID), + Address: raft.ServerAddress(srv.Addr.String()), + Version: srv.Build.String(), + RaftVersion: srv.RaftVersion, + Ext: s.autopilotServerExt(srv), + } + + switch srv.Status { + case serf.StatusLeft: + server.NodeStatus = autopilot.NodeLeft + case serf.StatusAlive, serf.StatusLeaving: + // we want to treat leaving as alive to prevent autopilot from + // prematurely removing the node. + server.NodeStatus = autopilot.NodeAlive + case serf.StatusFailed: + server.NodeStatus = autopilot.NodeFailed + default: + server.NodeStatus = autopilot.NodeUnknown + } + + members := s.serf.Members() + for _, member := range members { + if member.Name == srv.Name { + server.Meta = member.Tags + break + } + } + + return server, nil } diff --git a/nomad/autopilot_oss.go b/nomad/autopilot_oss.go new file mode 100644 index 000000000000..f18781aed476 --- /dev/null +++ b/nomad/autopilot_oss.go @@ -0,0 +1,26 @@ +//go:build !ent +// +build !ent + +package nomad + +import ( + autopilot "github.com/hashicorp/raft-autopilot" + + "github.com/hashicorp/nomad/nomad/structs" +) + +func (s *Server) autopilotPromoter() autopilot.Promoter { + return autopilot.DefaultPromoter() +} + +// autopilotServerExt returns the autopilot-enterprise.Server extensions needed +// for ENT feature support, but this is the empty OSS implementation. +func (s *Server) autopilotServerExt(_ *serverParts) interface{} { + return nil +} + +// autopilotConfigExt returns the autopilot-enterprise.Config extensions needed +// for ENT feature support, but this is the empty OSS implementation. +func autopilotConfigExt(_ *structs.AutopilotConfig) interface{} { + return nil +} diff --git a/nomad/autopilot_test.go b/nomad/autopilot_test.go index 8be118a583b5..91fce363fe8d 100644 --- a/nomad/autopilot_test.go +++ b/nomad/autopilot_test.go @@ -5,12 +5,13 @@ import ( "testing" "time" - "github.com/hashicorp/consul/agent/consul/autopilot" + // TODO: replace this with our own helper "github.com/hashicorp/consul/sdk/testutil/retry" - "github.com/hashicorp/nomad/ci" - "github.com/hashicorp/nomad/testutil" "github.com/hashicorp/raft" "github.com/hashicorp/serf/serf" + + "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/testutil" ) // wantPeers determines whether the server has the given @@ -21,7 +22,13 @@ func wantPeers(s *Server, peers int) error { return err } - n := autopilot.NumPeers(future.Configuration()) + var n int + for _, server := range future.Configuration().Servers { + if server.Suffrage == raft.Voter { + n++ + } + } + if got, want := n, peers; got != want { return fmt.Errorf("server %v: got %d peers want %d\n\tservers: %#+v", s.config.NodeName, got, want, future.Configuration().Servers) } @@ -68,7 +75,6 @@ func wantRaft(servers []*Server) error { func TestAutopilot_CleanupDeadServer(t *testing.T) { ci.Parallel(t) - t.Run("raft_v2", func(t *testing.T) { testCleanupDeadServer(t, 2) }) t.Run("raft_v3", func(t *testing.T) { testCleanupDeadServer(t, 3) }) } @@ -351,9 +357,9 @@ func TestAutopilot_PromoteNonVoter(t *testing.T) { if servers[1].Suffrage != raft.Nonvoter { r.Fatalf("bad: %v", servers) } - health := s1.autopilot.GetServerHealth(string(servers[1].ID)) + health := s1.autopilot.GetServerHealth(raft.ServerID(servers[1].ID)) if health == nil { - r.Fatalf("nil health, %v", s1.autopilot.GetClusterHealth()) + r.Fatalf("nil health, %v", s1.GetClusterHealth()) } if !health.Healthy { r.Fatalf("bad: %v", health) diff --git a/nomad/leader.go b/nomad/leader.go index 50913e96ac5c..950931ce9fb0 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -288,7 +288,7 @@ func (s *Server) establishLeadership(stopCh chan struct{}) error { // Initialize and start the autopilot routine s.getOrCreateAutopilotConfig() - s.autopilot.Start() + s.autopilot.Start(s.shutdownCtx) // Initialize scheduler configuration. schedulerConfig := s.getOrCreateSchedulerConfig() @@ -1302,7 +1302,7 @@ func (s *Server) addRaftPeer(m serf.Member, parts *serverParts) error { // but we want to avoid doing that if possible to prevent useless Raft // log entries. If the address is the same but the ID changed, remove the // old server before adding the new one. - minRaftProtocol, err := s.autopilot.MinRaftProtocol() + minRaftProtocol, err := s.MinRaftProtocol() if err != nil { return err } @@ -1372,7 +1372,7 @@ func (s *Server) removeRaftPeer(m serf.Member, parts *serverParts) error { return err } - minRaftProtocol, err := s.autopilot.MinRaftProtocol() + minRaftProtocol, err := s.MinRaftProtocol() if err != nil { return err } diff --git a/nomad/leader_test.go b/nomad/leader_test.go index 5289d413cb4b..fb367733490f 100644 --- a/nomad/leader_test.go +++ b/nomad/leader_test.go @@ -1175,7 +1175,7 @@ func TestLeader_UpgradeRaftVersion(t *testing.T) { } for _, s := range []*Server{s1, s3} { - minVer, err := s.autopilot.MinRaftProtocol() + minVer, err := s.MinRaftProtocol() if err != nil { t.Fatal(err) } @@ -1227,16 +1227,8 @@ func TestLeader_UpgradeRaftVersion(t *testing.T) { func TestLeader_Reelection(t *testing.T) { ci.Parallel(t) - raftProtocols := []int{1, 2, 3} - for _, p := range raftProtocols { - t.Run(fmt.Sprintf("Leader Election - Protocol version %d", p), func(t *testing.T) { - leaderElectionTest(t, raft.ProtocolVersion(p)) - }) - } - -} + const raftProtocol = 3 -func leaderElectionTest(t *testing.T, raftProtocol raft.ProtocolVersion) { s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 c.RaftConfig.ProtocolVersion = raftProtocol @@ -1298,19 +1290,20 @@ func TestLeader_RollRaftServer(t *testing.T) { ci.SkipSlow(t, "flaky on GHA; #12358") s1, cleanupS1 := TestServer(t, func(c *Config) { - c.RaftConfig.ProtocolVersion = 2 + c.BootstrapExpect = 3 + c.RaftConfig.ProtocolVersion = 3 }) defer cleanupS1() s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 - c.RaftConfig.ProtocolVersion = 2 + c.RaftConfig.ProtocolVersion = 3 }) defer cleanupS2() s3, cleanupS3 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 - c.RaftConfig.ProtocolVersion = 2 + c.RaftConfig.ProtocolVersion = 3 }) defer cleanupS3() @@ -1330,25 +1323,14 @@ func TestLeader_RollRaftServer(t *testing.T) { s.RemoveFailedNode(s1.config.NodeID) retry.Run(t, func(r *retry.R) { - minVer, err := s.autopilot.MinRaftProtocol() - if err != nil { - r.Fatal(err) - } - if got, want := minVer, 2; got != want { - r.Fatalf("got min raft version %d want %d", got, want) - } - configFuture := s.raft.GetConfiguration() - if err != nil { - r.Fatal(err) - } if len(configFuture.Configuration().Servers) != 2 { r.Fatalf("expected 2 servers, got %d", len(configFuture.Configuration().Servers)) } }) } - // Replace the dead server with one running raft protocol v3 + // Replace the dead server s4, cleanupS4 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 c.RaftConfig.ProtocolVersion = 3 @@ -1357,7 +1339,19 @@ func TestLeader_RollRaftServer(t *testing.T) { TestJoin(t, s2, s3, s4) servers[0] = s4 - // Kill the second v2 server + for _, s := range []*Server{s3, s4} { + retry.RunWith(&retry.Counter{ + Count: int(10 * testutil.TestMultiplier()), + Wait: time.Duration(testutil.TestMultiplier()) * time.Second, + }, t, func(r *retry.R) { + configFuture := s.raft.GetConfiguration() + if len(configFuture.Configuration().Servers) != 3 { + r.Fatalf("expected 3 servers, got %d", len(configFuture.Configuration().Servers)) + } + }) + } + + // Kill the second server s2.Shutdown() for _, s := range []*Server{s3, s4} { @@ -1367,24 +1361,14 @@ func TestLeader_RollRaftServer(t *testing.T) { Count: int(10 * testutil.TestMultiplier()), Wait: time.Duration(testutil.TestMultiplier()) * time.Second, }, t, func(r *retry.R) { - minVer, err := s.autopilot.MinRaftProtocol() - if err != nil { - r.Fatal(err) - } - if got, want := minVer, 2; got != want { - r.Fatalf("got min raft version %d want %d", got, want) - } - configFuture := s.raft.GetConfiguration() - if err != nil { - r.Fatal(err) - } - if len(configFuture.Configuration().Servers) != 2 { + if len(configFuture.Configuration().Servers) != 3 { r.Fatalf("expected 2 servers, got %d", len(configFuture.Configuration().Servers)) } }) } - // Replace another dead server with one running raft protocol v3 + + // Replace the 2nd dead server s5, cleanupS5 := TestServer(t, func(c *Config) { c.BootstrapExpect = 3 c.RaftConfig.ProtocolVersion = 3 @@ -1393,17 +1377,29 @@ func TestLeader_RollRaftServer(t *testing.T) { TestJoin(t, s3, s4, s5) servers[1] = s5 - // Kill the last v2 server, now minRaftProtocol should be 3 + for _, s := range []*Server{s3, s4, s5} { + retry.RunWith(&retry.Counter{ + Count: int(10 * testutil.TestMultiplier()), + Wait: time.Duration(testutil.TestMultiplier()) * time.Second, + }, t, func(r *retry.R) { + configFuture := s.raft.GetConfiguration() + if len(configFuture.Configuration().Servers) != 3 { + r.Fatalf("expected 3 servers, got %d", len(configFuture.Configuration().Servers)) + } + }) + } + + // Kill the last old server s3.Shutdown() for _, s := range []*Server{s4, s5} { - s.RemoveFailedNode(s2.config.NodeID) + s.RemoveFailedNode(s3.config.NodeID) retry.RunWith(&retry.Counter{ Count: int(10 * testutil.TestMultiplier()), Wait: time.Duration(testutil.TestMultiplier()) * time.Second, }, t, func(r *retry.R) { - minVer, err := s.autopilot.MinRaftProtocol() + minVer, err := s.MinRaftProtocol() if err != nil { r.Fatal(err) } @@ -1415,7 +1411,7 @@ func TestLeader_RollRaftServer(t *testing.T) { if err != nil { r.Fatal(err) } - if len(configFuture.Configuration().Servers) != 2 { + if len(configFuture.Configuration().Servers) != 3 { r.Fatalf("expected 2 servers, got %d", len(configFuture.Configuration().Servers)) } }) diff --git a/nomad/operator_endpoint.go b/nomad/operator_endpoint.go index 40c27669eac6..b21f3256b2e7 100644 --- a/nomad/operator_endpoint.go +++ b/nomad/operator_endpoint.go @@ -9,13 +9,12 @@ import ( log "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-msgpack/codec" + "github.com/hashicorp/raft" + "github.com/hashicorp/serf/serf" - "github.com/hashicorp/consul/agent/consul/autopilot" cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/helper/snapshot" "github.com/hashicorp/nomad/nomad/structs" - "github.com/hashicorp/raft" - "github.com/hashicorp/serf/serf" ) // Operator endpoint is used to perform low-level operator tasks for Nomad. @@ -183,7 +182,7 @@ REMOVE: // doing if you are calling this. If you remove a peer that's known to // Serf, for example, it will come back when the leader does a reconcile // pass. - minRaftProtocol, err := op.srv.autopilot.MinRaftProtocol() + minRaftProtocol, err := op.srv.MinRaftProtocol() if err != nil { return err } @@ -270,7 +269,7 @@ func (op *Operator) AutopilotSetConfiguration(args *structs.AutopilotSetConfigRe } // ServerHealth is used to get the current health of the servers. -func (op *Operator) ServerHealth(args *structs.GenericRequest, reply *autopilot.OperatorHealthReply) error { +func (op *Operator) ServerHealth(args *structs.GenericRequest, reply *structs.OperatorHealthReply) error { // This must be sent to the leader, so we fix the args since we are // re-using a structure where we don't support all the options. args.AllowStale = false @@ -288,7 +287,7 @@ func (op *Operator) ServerHealth(args *structs.GenericRequest, reply *autopilot. } // Exit early if the min Raft version is too low - minRaftProtocol, err := op.srv.autopilot.MinRaftProtocol() + minRaftProtocol, err := op.srv.MinRaftProtocol() if err != nil { return fmt.Errorf("error getting server raft protocol versions: %s", err) } @@ -296,7 +295,7 @@ func (op *Operator) ServerHealth(args *structs.GenericRequest, reply *autopilot. return fmt.Errorf("all servers must have raft_protocol set to 3 or higher to use this endpoint") } - *reply = op.srv.autopilot.GetClusterHealth() + *reply = *op.srv.GetClusterHealth() return nil } diff --git a/nomad/serf.go b/nomad/serf.go index 9a6d31a87c4f..adcf80a28827 100644 --- a/nomad/serf.go +++ b/nomad/serf.go @@ -196,7 +196,8 @@ func (s *Server) maybeBootstrap() { // Attempt a live bootstrap! var configuration raft.Configuration var addrs []string - minRaftVersion, err := s.autopilot.MinRaftProtocol() + + minRaftVersion, err := s.MinRaftProtocol() if err != nil { s.logger.Error("failed to read server raft versions", "error", err) } diff --git a/nomad/server.go b/nomad/server.go index f32f7d8808ad..32df0f448b70 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -20,11 +20,16 @@ import ( "time" "github.com/armon/go-metrics" - "github.com/hashicorp/consul/agent/consul/autopilot" consulapi "github.com/hashicorp/consul/api" log "github.com/hashicorp/go-hclog" multierror "github.com/hashicorp/go-multierror" lru "github.com/hashicorp/golang-lru" + "github.com/hashicorp/raft" + autopilot "github.com/hashicorp/raft-autopilot" + raftboltdb "github.com/hashicorp/raft-boltdb/v2" + "github.com/hashicorp/serf/serf" + "go.etcd.io/bbolt" + "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/codec" @@ -38,10 +43,6 @@ import ( "github.com/hashicorp/nomad/nomad/structs/config" "github.com/hashicorp/nomad/nomad/volumewatcher" "github.com/hashicorp/nomad/scheduler" - "github.com/hashicorp/raft" - raftboltdb "github.com/hashicorp/raft-boltdb/v2" - "github.com/hashicorp/serf/serf" - "go.etcd.io/bbolt" ) const ( @@ -752,7 +753,7 @@ func (s *Server) Leave() error { // for some sane period of time. isLeader := s.IsLeader() if isLeader && numPeers > 1 { - minRaftProtocol, err := s.autopilot.MinRaftProtocol() + minRaftProtocol, err := s.MinRaftProtocol() if err != nil { return err } diff --git a/nomad/server_setup_oss.go b/nomad/server_setup_oss.go index 7abf7a41ed6a..5e0684729a04 100644 --- a/nomad/server_setup_oss.go +++ b/nomad/server_setup_oss.go @@ -4,7 +4,7 @@ package nomad import ( - "github.com/hashicorp/consul/agent/consul/autopilot" + autopilot "github.com/hashicorp/raft-autopilot" ) type EnterpriseState struct{} @@ -20,7 +20,15 @@ func (es *EnterpriseState) ReloadLicense(_ *Config) error { func (s *Server) setupEnterprise(config *Config) error { // Set up the OSS version of autopilot apDelegate := &AutopilotDelegate{s} - s.autopilot = autopilot.NewAutopilot(s.logger, apDelegate, config.AutopilotInterval, config.ServerHealthInterval) + + s.autopilot = autopilot.New( + s.raft, + apDelegate, + autopilot.WithLogger(s.logger), + autopilot.WithReconcileInterval(config.AutopilotInterval), + autopilot.WithUpdateInterval(config.ServerHealthInterval), + autopilot.WithPromoter(s.autopilotPromoter()), + ) return nil } diff --git a/nomad/stats_fetcher.go b/nomad/stats_fetcher.go index 9967698b6cab..d06bffba5e18 100644 --- a/nomad/stats_fetcher.go +++ b/nomad/stats_fetcher.go @@ -2,13 +2,15 @@ package nomad import ( "context" + "net" "sync" log "github.com/hashicorp/go-hclog" + "github.com/hashicorp/raft" + autopilot "github.com/hashicorp/raft-autopilot" - "github.com/hashicorp/consul/agent/consul/autopilot" "github.com/hashicorp/nomad/helper/pool" - "github.com/hashicorp/serf/serf" + "github.com/hashicorp/nomad/nomad/structs" ) // StatsFetcher has two functions for autopilot. First, lets us fetch all the @@ -22,7 +24,7 @@ type StatsFetcher struct { logger log.Logger pool *pool.ConnPool region string - inflight map[string]struct{} + inflight map[raft.ServerID]struct{} inflightLock sync.Mutex } @@ -32,7 +34,7 @@ func NewStatsFetcher(logger log.Logger, pool *pool.ConnPool, region string) *Sta logger: logger.Named("stats_fetcher"), pool: pool, region: region, - inflight: make(map[string]struct{}), + inflight: make(map[raft.ServerID]struct{}), } } @@ -40,39 +42,47 @@ func NewStatsFetcher(logger log.Logger, pool *pool.ConnPool, region string) *Sta // cancel this when the context is canceled because we only want one in-flight // RPC to each server, so we let it finish and then clean up the in-flight // tracking. -func (f *StatsFetcher) fetch(server *serverParts, replyCh chan *autopilot.ServerStats) { +func (f *StatsFetcher) fetch(server *autopilot.Server, replyCh chan *autopilot.ServerStats) { var args struct{} - var reply autopilot.ServerStats - err := f.pool.RPC(f.region, server.Addr, "Status.RaftStats", &args, &reply) + var reply structs.RaftStats + + // defer some cleanup to notify everything else that the fetching is no longer occurring + // this is easier than trying to make the conditionals line up just right. + defer func() { + f.inflightLock.Lock() + delete(f.inflight, server.ID) + f.inflightLock.Unlock() + }() + + addr, err := net.ResolveTCPAddr("tcp", string(server.Address)) if err != nil { - f.logger.Warn("failed retrieving server health", "server", server.Name, "error", err) - } else { - replyCh <- &reply + f.logger.Warn("error resolving TCP address for server", + "address", server.Address, + "error", err) + return } - f.inflightLock.Lock() - delete(f.inflight, server.ID) - f.inflightLock.Unlock() + err = f.pool.RPC(f.region, addr, "Status.RaftStats", &args, &reply) + if err != nil { + f.logger.Warn("error getting server health", "server", server.Name, "error", err) + return + } + + replyCh <- reply.ToAutopilotServerStats() } // Fetch will attempt to query all the servers in parallel. -func (f *StatsFetcher) Fetch(ctx context.Context, members []serf.Member) map[string]*autopilot.ServerStats { +func (f *StatsFetcher) Fetch(ctx context.Context, servers map[raft.ServerID]*autopilot.Server) map[raft.ServerID]*autopilot.ServerStats { type workItem struct { - server *serverParts + server *autopilot.Server replyCh chan *autopilot.ServerStats } - var servers []*serverParts - for _, s := range members { - if ok, parts := isNomadServer(s); ok { - servers = append(servers, parts) - } - } // Skip any servers that have inflight requests. var work []*workItem f.inflightLock.Lock() - for _, server := range servers { - if _, ok := f.inflight[server.ID]; ok { + for id, server := range servers { + if _, ok := f.inflight[id]; ok { f.logger.Warn("failed retrieving server health; last request still outstanding", "server", server.Name) } else { workItem := &workItem{ @@ -80,7 +90,7 @@ func (f *StatsFetcher) Fetch(ctx context.Context, members []serf.Member) map[str replyCh: make(chan *autopilot.ServerStats, 1), } work = append(work, workItem) - f.inflight[server.ID] = struct{}{} + f.inflight[id] = struct{}{} go f.fetch(workItem.server, workItem.replyCh) } } @@ -88,14 +98,28 @@ func (f *StatsFetcher) Fetch(ctx context.Context, members []serf.Member) map[str // Now wait for the results to come in, or for the context to be // canceled. - replies := make(map[string]*autopilot.ServerStats) + replies := make(map[raft.ServerID]*autopilot.ServerStats) for _, workItem := range work { + // Drain the reply first if there is one. + select { + case reply := <-workItem.replyCh: + replies[workItem.server.ID] = reply + continue + default: + } + select { case reply := <-workItem.replyCh: replies[workItem.server.ID] = reply case <-ctx.Done(): - f.logger.Warn("failed retrieving server health", "server", workItem.server.Name, "error", ctx.Err()) + f.logger.Warn("failed retrieving server health", + "server", workItem.server.Name, "error", ctx.Err()) + + f.inflightLock.Lock() + delete(f.inflight, workItem.server.ID) + f.inflightLock.Unlock() + } } return replies diff --git a/nomad/stats_fetcher_test.go b/nomad/stats_fetcher_test.go index 36362d604e45..442ff4477d83 100644 --- a/nomad/stats_fetcher_test.go +++ b/nomad/stats_fetcher_test.go @@ -7,6 +7,7 @@ import ( "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/testutil" + "github.com/hashicorp/raft" ) func TestStatsFetcher(t *testing.T) { @@ -47,13 +48,14 @@ func TestStatsFetcher(t *testing.T) { func() { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - stats := s1.statsFetcher.Fetch(ctx, s1.Members()) + + stats := s1.statsFetcher.Fetch(ctx, s1.autopilotServers()) if len(stats) != 3 { t.Fatalf("bad: %#v", stats) } for id, stat := range stats { switch id { - case s1.config.NodeID, s2.config.NodeID, s3.config.NodeID: + case raft.ServerID(s1.config.NodeID), raft.ServerID(s2.config.NodeID), raft.ServerID(s3.config.NodeID): // OK default: t.Fatalf("bad: %s", id) @@ -68,20 +70,20 @@ func TestStatsFetcher(t *testing.T) { // Fake an in-flight request to server 3 and make sure we don't fetch // from it. func() { - s1.statsFetcher.inflight[string(s3.config.NodeID)] = struct{}{} - defer delete(s1.statsFetcher.inflight, string(s3.config.NodeID)) + s1.statsFetcher.inflight[raft.ServerID(s3.config.NodeID)] = struct{}{} + defer delete(s1.statsFetcher.inflight, raft.ServerID(s3.config.NodeID)) ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - stats := s1.statsFetcher.Fetch(ctx, s1.Members()) + stats := s1.statsFetcher.Fetch(ctx, s1.autopilotServers()) if len(stats) != 2 { t.Fatalf("bad: %#v", stats) } for id, stat := range stats { switch id { - case s1.config.NodeID, s2.config.NodeID: + case raft.ServerID(s1.config.NodeID), raft.ServerID(s2.config.NodeID): // OK - case s3.config.NodeID: + case raft.ServerID(s3.config.NodeID): t.Fatalf("bad") default: t.Fatalf("bad: %s", id) diff --git a/nomad/status_endpoint.go b/nomad/status_endpoint.go index 7b87afee7fb9..88fa754a27d9 100644 --- a/nomad/status_endpoint.go +++ b/nomad/status_endpoint.go @@ -7,7 +7,6 @@ import ( log "github.com/hashicorp/go-hclog" - "github.com/hashicorp/consul/agent/consul/autopilot" "github.com/hashicorp/nomad/nomad/structs" ) @@ -97,7 +96,7 @@ func (s *Status) Members(args *structs.GenericRequest, reply *structs.ServerMemb } // RaftStats is used by Autopilot to query the raft stats of the local server. -func (s *Status) RaftStats(args struct{}, reply *autopilot.ServerStats) error { +func (s *Status) RaftStats(args struct{}, reply *structs.RaftStats) error { stats := s.srv.raft.Stats() var err error diff --git a/nomad/structs/autopilot.go b/nomad/structs/autopilot.go new file mode 100644 index 000000000000..7520e084abc4 --- /dev/null +++ b/nomad/structs/autopilot.go @@ -0,0 +1,82 @@ +package structs + +import ( + "time" + + autopilot "github.com/hashicorp/raft-autopilot" + "github.com/hashicorp/serf/serf" +) + +// OperatorHealthReply is a representation of the overall health of the cluster +type OperatorHealthReply struct { + // Healthy is true if all the servers in the cluster are healthy. + Healthy bool + + // FailureTolerance is the number of healthy servers that could be lost without + // an outage occurring. + FailureTolerance int + + // Servers holds the health of each server. + Servers []ServerHealth +} + +// ServerHealth is the health (from the leader's point of view) of a server. +type ServerHealth struct { + // ID is the raft ID of the server. + ID string + + // Name is the node name of the server. + Name string + + // Address is the address of the server. + Address string + + // The status of the SerfHealth check for the server. + SerfStatus serf.MemberStatus + + // Version is the Nomad version of the server. + Version string + + // Leader is whether this server is currently the leader. + Leader bool + + // LastContact is the time since this node's last contact with the leader. + LastContact time.Duration + + // LastTerm is the highest leader term this server has a record of in its Raft log. + LastTerm uint64 + + // LastIndex is the last log index this server has a record of in its Raft log. + LastIndex uint64 + + // Healthy is whether or not the server is healthy according to the current + // Autopilot config. + Healthy bool + + // Voter is whether this is a voting server. + Voter bool + + // StableSince is the last time this server's Healthy value changed. + StableSince time.Time +} + +// RaftStats holds miscellaneous Raft metrics for a server, used by autopilot. +type RaftStats struct { + // LastContact is the time since this node's last contact with the leader. + LastContact string + + // LastTerm is the highest leader term this server has a record of in its Raft log. + LastTerm uint64 + + // LastIndex is the last log index this server has a record of in its Raft log. + LastIndex uint64 +} + +func (s *RaftStats) ToAutopilotServerStats() *autopilot.ServerStats { + duration, _ := time.ParseDuration(s.LastContact) + return &autopilot.ServerStats{ + LastContact: duration, + LastTerm: s.LastTerm, + LastIndex: s.LastIndex, + } +} From 09d3a8c28a1daaaf037af8c95dd8cde4ebcf7f9d Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 1 Sep 2022 13:16:35 -0400 Subject: [PATCH 2/3] address comments from review --- nomad/autopilot.go | 6 ++---- nomad/autopilot_test.go | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nomad/autopilot.go b/nomad/autopilot.go index fd846a981978..3d56ef4df687 100644 --- a/nomad/autopilot.go +++ b/nomad/autopilot.go @@ -23,14 +23,12 @@ const ( ) // AutopilotDelegate is a Nomad delegate for autopilot operations. It implements -// the autopilot.ApplicationIntegration interface and methods required for that -// have been documented as such below. +// the autopilot.ApplicationIntegration interface, and the methods required for +// that interface have been documented as such below. type AutopilotDelegate struct { server *Server } -var _ autopilot.ApplicationIntegration = &AutopilotDelegate{} - // AutopilotConfig is used to retrieve the latest configuration from the Nomad // delegate. This method is required to implement the ApplicationIntegration // interface. diff --git a/nomad/autopilot_test.go b/nomad/autopilot_test.go index 91fce363fe8d..417474126c5b 100644 --- a/nomad/autopilot_test.go +++ b/nomad/autopilot_test.go @@ -8,12 +8,15 @@ import ( // TODO: replace this with our own helper "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/hashicorp/raft" + autopilot "github.com/hashicorp/raft-autopilot" "github.com/hashicorp/serf/serf" "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/testutil" ) +var _ autopilot.ApplicationIntegration = (*AutopilotDelegate)(nil) + // wantPeers determines whether the server has the given // number of voting raft peers. func wantPeers(s *Server, peers int) error { From 45aa6981209b119a9f42214e640c72d8c4230378 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 1 Sep 2022 13:27:07 -0400 Subject: [PATCH 3/3] pad out timing for RollRaftServer test --- nomad/leader_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nomad/leader_test.go b/nomad/leader_test.go index fb367733490f..2eab55473c0a 100644 --- a/nomad/leader_test.go +++ b/nomad/leader_test.go @@ -1342,7 +1342,7 @@ func TestLeader_RollRaftServer(t *testing.T) { for _, s := range []*Server{s3, s4} { retry.RunWith(&retry.Counter{ Count: int(10 * testutil.TestMultiplier()), - Wait: time.Duration(testutil.TestMultiplier()) * time.Second, + Wait: time.Duration(testutil.TestMultiplier()) * time.Second * 2, }, t, func(r *retry.R) { configFuture := s.raft.GetConfiguration() if len(configFuture.Configuration().Servers) != 3 { @@ -1359,7 +1359,7 @@ func TestLeader_RollRaftServer(t *testing.T) { retry.RunWith(&retry.Counter{ Count: int(10 * testutil.TestMultiplier()), - Wait: time.Duration(testutil.TestMultiplier()) * time.Second, + Wait: time.Duration(testutil.TestMultiplier()) * time.Second * 2, }, t, func(r *retry.R) { configFuture := s.raft.GetConfiguration() if len(configFuture.Configuration().Servers) != 3 { @@ -1380,7 +1380,7 @@ func TestLeader_RollRaftServer(t *testing.T) { for _, s := range []*Server{s3, s4, s5} { retry.RunWith(&retry.Counter{ Count: int(10 * testutil.TestMultiplier()), - Wait: time.Duration(testutil.TestMultiplier()) * time.Second, + Wait: time.Duration(testutil.TestMultiplier()) * time.Second * 2, }, t, func(r *retry.R) { configFuture := s.raft.GetConfiguration() if len(configFuture.Configuration().Servers) != 3 { @@ -1397,7 +1397,7 @@ func TestLeader_RollRaftServer(t *testing.T) { retry.RunWith(&retry.Counter{ Count: int(10 * testutil.TestMultiplier()), - Wait: time.Duration(testutil.TestMultiplier()) * time.Second, + Wait: time.Duration(testutil.TestMultiplier()) * time.Second * 2, }, t, func(r *retry.R) { minVer, err := s.MinRaftProtocol() if err != nil {