diff --git a/.words b/.words index 0e9b4992b2a..b1eaddc2e11 100644 --- a/.words +++ b/.words @@ -104,3 +104,5 @@ PermitWithoutStream __lostleader ErrConnClosing unfreed +grpcAddr +clientURLs diff --git a/Documentation/dev-guide/api_reference_v3.md b/Documentation/dev-guide/api_reference_v3.md index 2d2000d8a72..e88906b0edc 100644 --- a/Documentation/dev-guide/api_reference_v3.md +++ b/Documentation/dev-guide/api_reference_v3.md @@ -37,6 +37,7 @@ This is a generated documentation. Please read the proto files for more. | MemberRemove | MemberRemoveRequest | MemberRemoveResponse | MemberRemove removes an existing member from the cluster. | | MemberUpdate | MemberUpdateRequest | MemberUpdateResponse | MemberUpdate updates the member configuration. | | MemberList | MemberListRequest | MemberListResponse | MemberList lists all the members in the cluster. | +| MemberPromote | MemberPromoteRequest | MemberPromoteResponse | MemberPromote promotes a member from raft learner (non-voting) to raft voting member. | @@ -609,6 +610,7 @@ Empty field. | name | name is the human-readable name of the member. If the member is not started, the name will be an empty string. | string | | peerURLs | peerURLs is the list of URLs the member exposes to the cluster for communication. | (slice of) string | | clientURLs | clientURLs is the list of URLs the member exposes to clients for communication. If the member is not started, clientURLs will be empty. | (slice of) string | +| isLearner | isLearner indicates if the member is raft learner. | bool | @@ -617,6 +619,7 @@ Empty field. | Field | Description | Type | | ----- | ----------- | ---- | | peerURLs | peerURLs is the list of URLs the added member will use to communicate with the cluster. | (slice of) string | +| isLearner | isLearner indicates if the added member is raft learner. | bool | @@ -645,6 +648,23 @@ Empty field. +##### message `MemberPromoteRequest` (etcdserver/etcdserverpb/rpc.proto) + +| Field | Description | Type | +| ----- | ----------- | ---- | +| ID | ID is the member ID of the member to promote. | uint64 | + + + +##### message `MemberPromoteResponse` (etcdserver/etcdserverpb/rpc.proto) + +| Field | Description | Type | +| ----- | ----------- | ---- | +| header | | ResponseHeader | +| members | members is a list of all members after promoting the member. | (slice of) Member | + + + ##### message `MemberRemoveRequest` (etcdserver/etcdserverpb/rpc.proto) | Field | Description | Type | @@ -819,6 +839,7 @@ Empty field. | raftAppliedIndex | raftAppliedIndex is the current raft applied index of the responding member. | uint64 | | errors | errors contains alarm/health information and status. | (slice of) string | | dbSizeInUse | dbSizeInUse is the size of the backend database logically in use, in bytes, of the responding member. | int64 | +| isLearner | isLearner indicates if the member is raft learner. | bool | diff --git a/Documentation/dev-guide/apispec/swagger/rpc.swagger.json b/Documentation/dev-guide/apispec/swagger/rpc.swagger.json index a76c9d3a8b1..acb38527ef0 100644 --- a/Documentation/dev-guide/apispec/swagger/rpc.swagger.json +++ b/Documentation/dev-guide/apispec/swagger/rpc.swagger.json @@ -501,6 +501,33 @@ } } }, + "/v3/cluster/member/promote": { + "post": { + "tags": [ + "Cluster" + ], + "summary": "MemberPromote promotes a member from raft learner (non-voting) to raft voting member.", + "operationId": "MemberPromote", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/etcdserverpbMemberPromoteRequest" + } + } + ], + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "$ref": "#/definitions/etcdserverpbMemberPromoteResponse" + } + } + } + } + }, "/v3/cluster/member/remove": { "post": { "tags": [ @@ -820,7 +847,7 @@ "200": { "description": "A successful response.(streaming responses)", "schema": { - "$ref": "#/definitions/etcdserverpbLeaseKeepAliveResponse" + "$ref": "#/x-stream-definitions/etcdserverpbLeaseKeepAliveResponse" } } } @@ -1009,7 +1036,7 @@ "200": { "description": "A successful response.(streaming responses)", "schema": { - "$ref": "#/definitions/etcdserverpbSnapshotResponse" + "$ref": "#/x-stream-definitions/etcdserverpbSnapshotResponse" } } } @@ -1091,7 +1118,7 @@ "200": { "description": "A successful response.(streaming responses)", "schema": { - "$ref": "#/definitions/etcdserverpbWatchResponse" + "$ref": "#/x-stream-definitions/etcdserverpbWatchResponse" } } } @@ -1882,6 +1909,11 @@ "type": "string" } }, + "isLearner": { + "description": "isLearner indicates if the member is raft learner.", + "type": "boolean", + "format": "boolean" + }, "name": { "description": "name is the human-readable name of the member. If the member is not started, the name will be an empty string.", "type": "string" @@ -1898,6 +1930,11 @@ "etcdserverpbMemberAddRequest": { "type": "object", "properties": { + "isLearner": { + "description": "isLearner indicates if the added member is raft learner.", + "type": "boolean", + "format": "boolean" + }, "peerURLs": { "description": "peerURLs is the list of URLs the added member will use to communicate with the cluster.", "type": "array", @@ -1944,6 +1981,31 @@ } } }, + "etcdserverpbMemberPromoteRequest": { + "type": "object", + "properties": { + "ID": { + "description": "ID is the member ID of the member to promote.", + "type": "string", + "format": "uint64" + } + } + }, + "etcdserverpbMemberPromoteResponse": { + "type": "object", + "properties": { + "header": { + "$ref": "#/definitions/etcdserverpbResponseHeader" + }, + "members": { + "description": "members is a list of all members after promoting the member.", + "type": "array", + "items": { + "$ref": "#/definitions/etcdserverpbMember" + } + } + } + }, "etcdserverpbMemberRemoveRequest": { "type": "object", "properties": { @@ -2266,6 +2328,11 @@ "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, + "isLearner": { + "description": "isLearner indicates if the member is raft learner.", + "type": "boolean", + "format": "boolean" + }, "leader": { "description": "leader is the member ID which the responding member believes is the current leader.", "type": "string", @@ -2508,6 +2575,43 @@ "format": "int64" } } + }, + "protobufAny": { + "type": "object", + "properties": { + "type_url": { + "type": "string" + }, + "value": { + "type": "string", + "format": "byte" + } + } + }, + "runtimeStreamError": { + "type": "object", + "properties": { + "details": { + "type": "array", + "items": { + "$ref": "#/definitions/protobufAny" + } + }, + "grpc_code": { + "type": "integer", + "format": "int32" + }, + "http_code": { + "type": "integer", + "format": "int32" + }, + "http_status": { + "type": "string" + }, + "message": { + "type": "string" + } + } } }, "securityDefinitions": { @@ -2521,5 +2625,43 @@ { "ApiKey": [] } - ] + ], + "x-stream-definitions": { + "etcdserverpbLeaseKeepAliveResponse": { + "properties": { + "error": { + "$ref": "#/definitions/runtimeStreamError" + }, + "result": { + "$ref": "#/definitions/etcdserverpbLeaseKeepAliveResponse" + } + }, + "title": "Stream result of etcdserverpbLeaseKeepAliveResponse", + "type": "object" + }, + "etcdserverpbSnapshotResponse": { + "properties": { + "error": { + "$ref": "#/definitions/runtimeStreamError" + }, + "result": { + "$ref": "#/definitions/etcdserverpbSnapshotResponse" + } + }, + "title": "Stream result of etcdserverpbSnapshotResponse", + "type": "object" + }, + "etcdserverpbWatchResponse": { + "properties": { + "error": { + "$ref": "#/definitions/runtimeStreamError" + }, + "result": { + "$ref": "#/definitions/etcdserverpbWatchResponse" + } + }, + "title": "Stream result of etcdserverpbWatchResponse", + "type": "object" + } + } } \ No newline at end of file diff --git a/Documentation/dev-guide/apispec/swagger/v3election.swagger.json b/Documentation/dev-guide/apispec/swagger/v3election.swagger.json index b0d33ad080b..a8d08ceaf11 100644 --- a/Documentation/dev-guide/apispec/swagger/v3election.swagger.json +++ b/Documentation/dev-guide/apispec/swagger/v3election.swagger.json @@ -77,7 +77,7 @@ "200": { "description": "A successful response.(streaming responses)", "schema": { - "$ref": "#/definitions/v3electionpbLeaderResponse" + "$ref": "#/x-stream-definitions/v3electionpbLeaderResponse" } } }, @@ -212,6 +212,43 @@ } } }, + "protobufAny": { + "type": "object", + "properties": { + "type_url": { + "type": "string" + }, + "value": { + "type": "string", + "format": "byte" + } + } + }, + "runtimeStreamError": { + "type": "object", + "properties": { + "grpc_code": { + "type": "integer", + "format": "int32" + }, + "http_code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + }, + "http_status": { + "type": "string" + }, + "details": { + "type": "array", + "items": { + "$ref": "#/definitions/protobufAny" + } + } + } + }, "v3electionpbCampaignRequest": { "type": "object", "properties": { @@ -330,5 +367,19 @@ } } } + }, + "x-stream-definitions": { + "v3electionpbLeaderResponse": { + "type": "object", + "properties": { + "result": { + "$ref": "#/definitions/v3electionpbLeaderResponse" + }, + "error": { + "$ref": "#/definitions/runtimeStreamError" + } + }, + "title": "Stream result of v3electionpbLeaderResponse" + } } } diff --git a/Documentation/op-guide/runtime-configuration.md b/Documentation/op-guide/runtime-configuration.md index 96966707291..09898e70881 100644 --- a/Documentation/op-guide/runtime-configuration.md +++ b/Documentation/op-guide/runtime-configuration.md @@ -123,6 +123,48 @@ The new member will run as a part of the cluster and immediately begin catching If adding multiple members the best practice is to configure a single member at a time and verify it starts correctly before adding more new members. If adding a new member to a 1-node cluster, the cluster cannot make progress before the new member starts because it needs two members as majority to agree on the consensus. This behavior only happens between the time `etcdctl member add` informs the cluster about the new member and the new member successfully establishing a connection to the existing one. +#### Add a new member as learner + +Starting from v3.4, etcd supports adding a new member as learner / non-voting member. +The motivation and design can be found in [design doc](https://etcd.readthedocs.io/en/latest/server-learner.html). +In order to make the process of adding a new member safer, +and to reduce cluster downtime when the new member is added, it is recommended that the new member is added to cluster +as a learner until it catches up. This can be described as a three step process: + + * Add the new member as learner via [gRPC members API][member-api-grpc] or the `etcdctl member add --learner` command. + Note that v2 [HTTP member API][member-api] does not support this feature. (If user wants to use HTTP, + etcd provides a JSON [gRPC gateway][grpc-gateway], which serves a RESTful proxy that translates HTTP/JSON requests into gRPC messages.) + + * Start the new member with the new cluster configuration, including a list of the updated members (existing members + the new member). + This step is exactly the same as before. + + * Promote the newly added learner to voting member via [gRPC members API][member-api-grpc] or the `etcdctl member promote` command. + etcd server validates promote request to ensure its operational safety. + Only after its raft log has caught up to leader’s can learner be promoted to a voting member. + If a learner member has not caught up to leader's raft log, member promote request will fail + (see [error cases when promoting a member] section for more details). + In this case, user should wait and retry later. + +In v3.4, etcd server limits the number of learners that cluster can have to one. The main consideration is to limit the +extra workload on leader due to propagating data from leader to learner. + +Use `etcdctl member add` with flag `--learner` to add new member to cluster as learner. + +```sh +$ etcdctl member add infra3 --peer-urls=http://10.0.1.13:2380 --learner +Member 9bf1b35fc7761a23 added to cluster a7ef944b95711739 + +ETCD_NAME="infra3" +ETCD_INITIAL_CLUSTER="infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380,infra3=http://10.0.1.13:2380" +ETCD_INITIAL_CLUSTER_STATE=existing +``` + +After new etcd process is started for the newly added learner member, use `etcdctl member promote` to promote learner to voting member. +``` +$ etcdctl member promote 9bf1b35fc7761a23 +Member 9e29bbaa45d74461 promoted in cluster a7ef944b95711739 +``` + #### Error cases when adding members In the following case a new host is not included in the list of enumerated nodes. If this is a new cluster, the node must be added to the list of initial cluster members. @@ -153,6 +195,35 @@ etcd: this member has been permanently removed from the cluster. Exiting. exit 1 ``` +#### Error cases when adding a learner member + +Cannot add learner to cluster if the cluster already has 1 learner (v3.4). +``` +$ etcdctl member add infra4 --peer-urls=http://10.0.1.14:2380 --learner +Error: etcdserver: too many learner members in cluster +``` + +#### Error cases when promoting a learner member + +Learner can only be promoted to voting member if it is caught up with leader. +``` +$ etcdctl member promote 9bf1b35fc7761a23 +Error: etcdserver: can only promote a learner member which catches up with leader +``` + +Promoting a member that is not a learner will fail. +``` +$ etcdctl member promote 9bf1b35fc7761a23 +Error: etcdserver: can only promote a learner member +``` + +Promoting a member that does not exist in cluster will fail. +``` +$ etcdctl member promote 12345abcde +Error: etcdserver: member not found +``` + + ### Strict reconfiguration check mode (`-strict-reconfig-check`) As described in the above, the best practice of adding new members is to configure a single member at a time and verify it starts correctly before adding more new members. This step by step approach is very important because if newly added members is not configured correctly (for example the peer URLs are incorrect), the cluster can lose quorum. The quorum loss happens since the newly added member are counted in the quorum even if that member is not reachable from other existing members. Also quorum loss might happen if there is a connectivity issue or there are operational issues. @@ -173,3 +244,5 @@ It is enabled by default. [member migration]: ../v2/admin_guide.md#member-migration [remove member]: #remove-a-member [runtime-reconf]: runtime-reconf-design.md +[grpc-gateway]: https://github.com/grpc-ecosystem/grpc-gateway +[error cases when promoting a member]: #error-cases-when-promoting-a-learner-member diff --git a/clientv3/cluster.go b/clientv3/cluster.go index d497c0578a9..8008c9554b5 100644 --- a/clientv3/cluster.go +++ b/clientv3/cluster.go @@ -24,11 +24,12 @@ import ( ) type ( - Member pb.Member - MemberListResponse pb.MemberListResponse - MemberAddResponse pb.MemberAddResponse - MemberRemoveResponse pb.MemberRemoveResponse - MemberUpdateResponse pb.MemberUpdateResponse + Member pb.Member + MemberListResponse pb.MemberListResponse + MemberAddResponse pb.MemberAddResponse + MemberRemoveResponse pb.MemberRemoveResponse + MemberUpdateResponse pb.MemberUpdateResponse + MemberPromoteResponse pb.MemberPromoteResponse ) type Cluster interface { @@ -36,13 +37,16 @@ type Cluster interface { MemberList(ctx context.Context) (*MemberListResponse, error) // MemberAdd adds a new member into the cluster. - MemberAdd(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error) + MemberAdd(ctx context.Context, peerAddrs []string, isLearner bool) (*MemberAddResponse, error) // MemberRemove removes an existing member from the cluster. MemberRemove(ctx context.Context, id uint64) (*MemberRemoveResponse, error) // MemberUpdate updates the peer addresses of the member. MemberUpdate(ctx context.Context, id uint64, peerAddrs []string) (*MemberUpdateResponse, error) + + // MemberPromote promotes a member from raft learner (non-voting) to raft voting member. + MemberPromote(ctx context.Context, id uint64) (*MemberPromoteResponse, error) } type cluster struct { @@ -66,13 +70,16 @@ func NewClusterFromClusterClient(remote pb.ClusterClient, c *Client) Cluster { return api } -func (c *cluster) MemberAdd(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error) { +func (c *cluster) MemberAdd(ctx context.Context, peerAddrs []string, isLearner bool) (*MemberAddResponse, error) { // fail-fast before panic in rafthttp if _, err := types.NewURLs(peerAddrs); err != nil { return nil, err } - r := &pb.MemberAddRequest{PeerURLs: peerAddrs} + r := &pb.MemberAddRequest{ + PeerURLs: peerAddrs, + IsLearner: isLearner, + } resp, err := c.remote.MemberAdd(ctx, r, c.callOpts...) if err != nil { return nil, toErr(ctx, err) @@ -112,3 +119,12 @@ func (c *cluster) MemberList(ctx context.Context) (*MemberListResponse, error) { } return nil, toErr(ctx, err) } + +func (c *cluster) MemberPromote(ctx context.Context, id uint64) (*MemberPromoteResponse, error) { + r := &pb.MemberPromoteRequest{ID: id} + resp, err := c.remote.MemberPromote(ctx, r, c.callOpts...) + if err != nil { + return nil, toErr(ctx, err) + } + return (*MemberPromoteResponse)(resp), nil +} diff --git a/clientv3/example_cluster_test.go b/clientv3/example_cluster_test.go index 279ea64ac69..f29a40f8b0d 100644 --- a/clientv3/example_cluster_test.go +++ b/clientv3/example_cluster_test.go @@ -51,7 +51,7 @@ func ExampleCluster_memberAdd() { defer cli.Close() peerURLs := endpoints[2:] - mresp, err := cli.MemberAdd(context.Background(), peerURLs) + mresp, err := cli.MemberAdd(context.Background(), peerURLs, false) if err != nil { log.Fatal(err) } diff --git a/clientv3/integration/cluster_test.go b/clientv3/integration/cluster_test.go index 37d773a4b3b..9639a3ed9bf 100644 --- a/clientv3/integration/cluster_test.go +++ b/clientv3/integration/cluster_test.go @@ -19,6 +19,7 @@ import ( "reflect" "strings" "testing" + "time" "go.etcd.io/etcd/integration" "go.etcd.io/etcd/pkg/testutil" @@ -52,7 +53,7 @@ func TestMemberAdd(t *testing.T) { capi := clus.RandClient() urls := []string{"http://127.0.0.1:1234"} - resp, err := capi.MemberAdd(context.Background(), urls) + resp, err := capi.MemberAdd(context.Background(), urls, false) if err != nil { t.Fatalf("failed to add member %v", err) } @@ -76,7 +77,7 @@ func TestMemberAddWithExistingURLs(t *testing.T) { } existingURL := resp.Members[0].PeerURLs[0] - _, err = capi.MemberAdd(context.Background(), []string{existingURL}) + _, err = capi.MemberAdd(context.Background(), []string{existingURL}, false) expectedErrKeywords := "Peer URLs already exists" if err == nil { t.Fatalf("expecting add member to fail, got no error") @@ -174,7 +175,7 @@ func TestMemberAddUpdateWrongURLs(t *testing.T) { {"localhost:1234"}, } for i := range tt { - _, err := capi.MemberAdd(context.Background(), tt[i]) + _, err := capi.MemberAdd(context.Background(), tt[i], false) if err == nil { t.Errorf("#%d: MemberAdd err = nil, but error", i) } @@ -184,3 +185,151 @@ func TestMemberAddUpdateWrongURLs(t *testing.T) { } } } + +func TestMemberAddForLearner(t *testing.T) { + defer testutil.AfterTest(t) + + clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) + defer clus.Terminate(t) + + capi := clus.RandClient() + + urls := []string{"http://127.0.0.1:1234"} + isLearner := true + resp, err := capi.MemberAdd(context.Background(), urls, isLearner) + if err != nil { + t.Fatalf("failed to add member %v", err) + } + + if resp.Member.IsLearner != isLearner { + t.Errorf("Added a member with IsLearner = %v, got %v", isLearner, resp.Member.IsLearner) + } + + numberOfLearners := 0 + for _, m := range resp.Members { + if m.IsLearner { + numberOfLearners++ + } + } + if numberOfLearners != 1 { + t.Errorf("Added 1 learner node to cluster, got %d", numberOfLearners) + } +} + +func TestMemberPromote(t *testing.T) { + defer testutil.AfterTest(t) + + clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) + defer clus.Terminate(t) + + // member promote request can be sent to any server in cluster, + // the request will be auto-forwarded to leader on server-side. + // This test explicitly includes the server-side forwarding by + // sending the request to follower. + leaderIdx := clus.WaitLeader(t) + followerIdx := (leaderIdx + 1) % 3 + capi := clus.Client(followerIdx) + + urls := []string{"http://127.0.0.1:1234"} + isLearner := true + memberAddResp, err := capi.MemberAdd(context.Background(), urls, isLearner) + if err != nil { + t.Fatalf("failed to add member %v", err) + } + + if memberAddResp.Member.IsLearner != isLearner { + t.Fatalf("Added a member with IsLearner = %v, got %v", isLearner, memberAddResp.Member.IsLearner) + } + learnerID := memberAddResp.Member.ID + + numberOfLearners := 0 + for _, m := range memberAddResp.Members { + if m.IsLearner { + numberOfLearners++ + } + } + if numberOfLearners != 1 { + t.Fatalf("Added 1 learner node to cluster, got %d", numberOfLearners) + } + + // learner is not started yet. Expect learner progress check to fail. + // As the result, member promote request will fail. + _, err = capi.MemberPromote(context.Background(), learnerID) + expectedErrKeywords := "can only promote a learner member which catches up with leader" + if err == nil { + t.Fatalf("expecting promote not ready learner to fail, got no error") + } + if !strings.Contains(err.Error(), expectedErrKeywords) { + t.Fatalf("expecting error to contain %s, got %s", expectedErrKeywords, err.Error()) + } + + // create and launch learner member based on the response of V3 Member Add API. + // (the response has information on peer urls of the existing members in cluster) + learnerMember := clus.MustNewMember(t, memberAddResp) + clus.Members = append(clus.Members, learnerMember) + if err := learnerMember.Launch(); err != nil { + t.Fatal(err) + } + + // retry until promote succeed or timeout + timeout := time.After(5 * time.Second) + for { + select { + case <-time.After(500 * time.Millisecond): + case <-timeout: + t.Errorf("failed all attempts to promote learner member, last error: %v", err) + break + } + + _, err = capi.MemberPromote(context.Background(), learnerID) + // successfully promoted learner + if err == nil { + break + } + // if member promote fails due to learner not ready, retry. + // otherwise fails the test. + if !strings.Contains(err.Error(), expectedErrKeywords) { + t.Fatalf("unexpected error when promoting learner member: %v", err) + } + } +} + +// TestMaxLearnerInCluster verifies that the maximum number of learners allowed in a cluster is 1 +func TestMaxLearnerInCluster(t *testing.T) { + defer testutil.AfterTest(t) + + // 1. start with a cluster with 3 voting member and 0 learner member + clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) + defer clus.Terminate(t) + + // 2. adding a learner member should succeed + resp1, err := clus.Client(0).MemberAdd(context.Background(), []string{"http://127.0.0.1:1234"}, true) + if err != nil { + t.Fatalf("failed to add learner member %v", err) + } + numberOfLearners := 0 + for _, m := range resp1.Members { + if m.IsLearner { + numberOfLearners++ + } + } + if numberOfLearners != 1 { + t.Fatalf("Added 1 learner node to cluster, got %d", numberOfLearners) + } + + // 3. cluster has 3 voting member and 1 learner, adding another learner should fail + _, err = clus.Client(0).MemberAdd(context.Background(), []string{"http://127.0.0.1:2345"}, true) + if err == nil { + t.Fatalf("expect member add to fail, got no error") + } + expectedErrKeywords := "too many learner members in cluster" + if !strings.Contains(err.Error(), expectedErrKeywords) { + t.Fatalf("expecting error to contain %s, got %s", expectedErrKeywords, err.Error()) + } + + // 4. cluster has 3 voting member and 1 learner, adding a voting member should succeed + _, err = clus.Client(0).MemberAdd(context.Background(), []string{"http://127.0.0.1:3456"}, false) + if err != nil { + t.Errorf("failed to add member %v", err) + } +} diff --git a/clientv3/integration/kv_test.go b/clientv3/integration/kv_test.go index 636bcd27a96..3a461d6d8bd 100644 --- a/clientv3/integration/kv_test.go +++ b/clientv3/integration/kv_test.go @@ -971,3 +971,128 @@ func TestKVLargeRequests(t *testing.T) { clus.Terminate(t) } } + +// TestKVForLearner ensures learner member only accepts serializable read request. +func TestKVForLearner(t *testing.T) { + defer testutil.AfterTest(t) + + clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) + defer clus.Terminate(t) + + // we have to add and launch learner member after initial cluster was created, because + // bootstrapping a cluster with learner member is not supported. + clus.AddAndLaunchLearnerMember(t) + + learners, err := clus.GetLearnerMembers() + if err != nil { + t.Fatalf("failed to get the learner members in cluster: %v", err) + } + if len(learners) != 1 { + t.Fatalf("added 1 learner to cluster, got %d", len(learners)) + } + + if len(clus.Members) != 4 { + t.Fatalf("expecting 4 members in cluster after adding the learner member, got %d", len(clus.Members)) + } + // note: + // 1. clus.Members[3] is the newly added learner member, which was appended to clus.Members + // 2. we are using member's grpcAddr instead of clientURLs as the endpoint for clientv3.Config, + // because the implementation of integration test has diverged from embed/etcd.go. + learnerEp := clus.Members[3].GRPCAddr() + cfg := clientv3.Config{ + Endpoints: []string{learnerEp}, + DialTimeout: 5 * time.Second, + DialOptions: []grpc.DialOption{grpc.WithBlock()}, + } + // this client only has endpoint of the learner member + cli, err := clientv3.New(cfg) + if err != nil { + t.Fatalf("failed to create clientv3: %v", err) + } + defer cli.Close() + + // wait until learner member is ready + <-clus.Members[3].ReadyNotify() + + tests := []struct { + op clientv3.Op + wErr bool + }{ + { + op: clientv3.OpGet("foo", clientv3.WithSerializable()), + wErr: false, + }, + { + op: clientv3.OpGet("foo"), + wErr: true, + }, + { + op: clientv3.OpPut("foo", "bar"), + wErr: true, + }, + { + op: clientv3.OpDelete("foo"), + wErr: true, + }, + { + op: clientv3.OpTxn([]clientv3.Cmp{clientv3.Compare(clientv3.CreateRevision("foo"), "=", 0)}, nil, nil), + wErr: true, + }, + } + + for idx, test := range tests { + _, err := cli.Do(context.TODO(), test.op) + if err != nil && !test.wErr { + t.Errorf("%d: expect no error, got %v", idx, err) + } + if err == nil && test.wErr { + t.Errorf("%d: expect error, got nil", idx) + } + } +} + +// TestBalancerSupportLearner verifies that balancer's retry and failover mechanism supports cluster with learner member +func TestBalancerSupportLearner(t *testing.T) { + defer testutil.AfterTest(t) + + clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) + defer clus.Terminate(t) + + // we have to add and launch learner member after initial cluster was created, because + // bootstrapping a cluster with learner member is not supported. + clus.AddAndLaunchLearnerMember(t) + + learners, err := clus.GetLearnerMembers() + if err != nil { + t.Fatalf("failed to get the learner members in cluster: %v", err) + } + if len(learners) != 1 { + t.Fatalf("added 1 learner to cluster, got %d", len(learners)) + } + + // clus.Members[3] is the newly added learner member, which was appended to clus.Members + learnerEp := clus.Members[3].GRPCAddr() + cfg := clientv3.Config{ + Endpoints: []string{learnerEp}, + DialTimeout: 5 * time.Second, + DialOptions: []grpc.DialOption{grpc.WithBlock()}, + } + cli, err := clientv3.New(cfg) + if err != nil { + t.Fatalf("failed to create clientv3: %v", err) + } + defer cli.Close() + + // wait until learner member is ready + <-clus.Members[3].ReadyNotify() + + if _, err := cli.Get(context.Background(), "foo"); err == nil { + t.Fatalf("expect Get request to learner to fail, got no error") + } + + eps := []string{learnerEp, clus.Members[0].GRPCAddr()} + cli.SetEndpoints(eps...) + if _, err := cli.Get(context.Background(), "foo"); err != nil { + t.Errorf("expect no error (balancer should retry when request to learner fails), got error: %v", err) + } +} diff --git a/clientv3/retry.go b/clientv3/retry.go index 38ad00ac9a0..6da7abf475c 100644 --- a/clientv3/retry.go +++ b/clientv3/retry.go @@ -183,6 +183,10 @@ func (rcc *retryClusterClient) MemberUpdate(ctx context.Context, in *pb.MemberUp return rcc.cc.MemberUpdate(ctx, in, opts...) } +func (rcc *retryClusterClient) MemberPromote(ctx context.Context, in *pb.MemberPromoteRequest, opts ...grpc.CallOption) (resp *pb.MemberPromoteResponse, err error) { + return rcc.cc.MemberPromote(ctx, in, opts...) +} + type retryMaintenanceClient struct { mc pb.MaintenanceClient } diff --git a/clientv3/snapshot/member_test.go b/clientv3/snapshot/member_test.go index a42066a5637..f7db982e31b 100644 --- a/clientv3/snapshot/member_test.go +++ b/clientv3/snapshot/member_test.go @@ -55,7 +55,7 @@ func TestSnapshotV3RestoreMultiMemberAdd(t *testing.T) { urls := newEmbedURLs(2) newCURLs, newPURLs := urls[:1], urls[1:] - if _, err = cli.MemberAdd(context.Background(), []string{newPURLs[0].String()}); err != nil { + if _, err = cli.MemberAdd(context.Background(), []string{newPURLs[0].String()}, false); err != nil { t.Fatal(err) } diff --git a/etcdctl/ctlv3/command/ep_command.go b/etcdctl/ctlv3/command/ep_command.go index 58c2c7e9659..b04285927b0 100644 --- a/etcdctl/ctlv3/command/ep_command.go +++ b/etcdctl/ctlv3/command/ep_command.go @@ -60,7 +60,7 @@ func newEpStatusCommand() *cobra.Command { Use: "status", Short: "Prints out the status of endpoints specified in `--endpoints` flag", Long: `When --write-out is set to simple, this command prints out comma-separated status lists for each endpoint. -The items in the lists are endpoint, ID, version, db size, is leader, raft term, raft index. +The items in the lists are endpoint, ID, version, db size, is leader, is learner, raft term, raft index, raft applied index, errors. `, Run: epStatusCommandFunc, } diff --git a/etcdctl/ctlv3/command/member_command.go b/etcdctl/ctlv3/command/member_command.go index 5b2119aba50..99452b5b684 100644 --- a/etcdctl/ctlv3/command/member_command.go +++ b/etcdctl/ctlv3/command/member_command.go @@ -23,7 +23,10 @@ import ( "github.com/spf13/cobra" ) -var memberPeerURLs string +var ( + memberPeerURLs string + isLearner bool +) // NewMemberCommand returns the cobra command for "member". func NewMemberCommand() *cobra.Command { @@ -36,6 +39,7 @@ func NewMemberCommand() *cobra.Command { mc.AddCommand(NewMemberRemoveCommand()) mc.AddCommand(NewMemberUpdateCommand()) mc.AddCommand(NewMemberListCommand()) + mc.AddCommand(NewMemberPromoteCommand()) return mc } @@ -50,6 +54,7 @@ func NewMemberAddCommand() *cobra.Command { } cc.Flags().StringVar(&memberPeerURLs, "peer-urls", "", "comma separated peer URLs for the new member.") + cc.Flags().BoolVar(&isLearner, "learner", false, "indicates if the new member is raft learner") return cc } @@ -86,7 +91,7 @@ func NewMemberListCommand() *cobra.Command { Use: "list", Short: "Lists all members in the cluster", Long: `When --write-out is set to simple, this command prints out comma-separated member lists for each endpoint. -The items in the lists are ID, Status, Name, Peer Addrs, Client Addrs. +The items in the lists are ID, Status, Name, Peer Addrs, Client Addrs, Is Learner. `, Run: memberListCommandFunc, @@ -95,6 +100,20 @@ The items in the lists are ID, Status, Name, Peer Addrs, Client Addrs. return cc } +// NewMemberPromoteCommand returns the cobra command for "member promote". +func NewMemberPromoteCommand() *cobra.Command { + cc := &cobra.Command{ + Use: "promote ", + Short: "Promotes a non-voting member in the cluster", + Long: `Promotes a non-voting learner member to a voting one in the cluster. +`, + + Run: memberPromoteCommandFunc, + } + + return cc +} + // memberAddCommandFunc executes the "member add" command. func memberAddCommandFunc(cmd *cobra.Command, args []string) { if len(args) < 1 { @@ -118,7 +137,7 @@ func memberAddCommandFunc(cmd *cobra.Command, args []string) { urls := strings.Split(memberPeerURLs, ",") ctx, cancel := commandCtx(cmd) cli := mustClientFromCmd(cmd) - resp, err := cli.MemberAdd(ctx, urls) + resp, err := cli.MemberAdd(ctx, urls, isLearner) cancel() if err != nil { ExitWithError(ExitError, err) @@ -225,3 +244,23 @@ func memberListCommandFunc(cmd *cobra.Command, args []string) { display.MemberList(*resp) } + +// memberPromoteCommandFunc executes the "member promote" command. +func memberPromoteCommandFunc(cmd *cobra.Command, args []string) { + if len(args) != 1 { + ExitWithError(ExitBadArgs, fmt.Errorf("member ID is not provided")) + } + + id, err := strconv.ParseUint(args[0], 16, 64) + if err != nil { + ExitWithError(ExitBadArgs, fmt.Errorf("bad member ID arg (%v), expecting ID in Hex", err)) + } + + ctx, cancel := commandCtx(cmd) + resp, err := mustClientFromCmd(cmd).MemberPromote(ctx, id) + cancel() + if err != nil { + ExitWithError(ExitError, err) + } + display.MemberPromote(id, *resp) +} diff --git a/etcdctl/ctlv3/command/printer.go b/etcdctl/ctlv3/command/printer.go index cd87811e420..2793c5fa4b8 100644 --- a/etcdctl/ctlv3/command/printer.go +++ b/etcdctl/ctlv3/command/printer.go @@ -42,6 +42,7 @@ type printer interface { MemberAdd(v3.MemberAddResponse) MemberRemove(id uint64, r v3.MemberRemoveResponse) MemberUpdate(id uint64, r v3.MemberUpdateResponse) + MemberPromote(id uint64, r v3.MemberPromoteResponse) MemberList(v3.MemberListResponse) EndpointHealth([]epHealth) @@ -158,18 +159,23 @@ func (p *printerUnsupported) DBStatus(snapshot.Status) { p.p(nil) } func (p *printerUnsupported) MoveLeader(leader, target uint64, r v3.MoveLeaderResponse) { p.p(nil) } func makeMemberListTable(r v3.MemberListResponse) (hdr []string, rows [][]string) { - hdr = []string{"ID", "Status", "Name", "Peer Addrs", "Client Addrs"} + hdr = []string{"ID", "Status", "Name", "Peer Addrs", "Client Addrs", "Is Learner"} for _, m := range r.Members { status := "started" if len(m.Name) == 0 { status = "unstarted" } + isLearner := "false" + if m.IsLearner { + isLearner = "true" + } rows = append(rows, []string{ fmt.Sprintf("%x", m.ID), status, m.Name, strings.Join(m.PeerURLs, ","), strings.Join(m.ClientURLs, ","), + isLearner, }) } return hdr, rows @@ -189,7 +195,8 @@ func makeEndpointHealthTable(healthList []epHealth) (hdr []string, rows [][]stri } func makeEndpointStatusTable(statusList []epStatus) (hdr []string, rows [][]string) { - hdr = []string{"endpoint", "ID", "version", "db size", "is leader", "raft term", "raft index", "raft applied index", "errors"} + hdr = []string{"endpoint", "ID", "version", "db size", "is leader", "is learner", "raft term", + "raft index", "raft applied index", "errors"} for _, status := range statusList { rows = append(rows, []string{ status.Ep, @@ -197,6 +204,7 @@ func makeEndpointStatusTable(statusList []epStatus) (hdr []string, rows [][]stri status.Resp.Version, humanize.Bytes(uint64(status.Resp.DbSize)), fmt.Sprint(status.Resp.Leader == status.Resp.Header.MemberId), + fmt.Sprint(status.Resp.IsLearner), fmt.Sprint(status.Resp.RaftTerm), fmt.Sprint(status.Resp.RaftIndex), fmt.Sprint(status.Resp.RaftAppliedIndex), diff --git a/etcdctl/ctlv3/command/printer_fields.go b/etcdctl/ctlv3/command/printer_fields.go index 7f2d4e5892d..d0c69998053 100644 --- a/etcdctl/ctlv3/command/printer_fields.go +++ b/etcdctl/ctlv3/command/printer_fields.go @@ -137,6 +137,7 @@ func (p *fieldsPrinter) MemberList(r v3.MemberListResponse) { for _, u := range m.ClientURLs { fmt.Printf("\"ClientURL\" : %q\n", u) } + fmt.Println(`"IsLearner" :`, m.IsLearner) fmt.Println() } } @@ -157,6 +158,7 @@ func (p *fieldsPrinter) EndpointStatus(eps []epStatus) { fmt.Printf("\"Version\" : %q\n", ep.Resp.Version) fmt.Println(`"DBSize" :`, ep.Resp.DbSize) fmt.Println(`"Leader" :`, ep.Resp.Leader) + fmt.Println(`"IsLearner" :`, ep.Resp.IsLearner) fmt.Println(`"RaftIndex" :`, ep.Resp.RaftIndex) fmt.Println(`"RaftTerm" :`, ep.Resp.RaftTerm) fmt.Println(`"RaftAppliedIndex" :`, ep.Resp.RaftAppliedIndex) diff --git a/etcdctl/ctlv3/command/printer_simple.go b/etcdctl/ctlv3/command/printer_simple.go index 93aa7941268..b321f850b57 100644 --- a/etcdctl/ctlv3/command/printer_simple.go +++ b/etcdctl/ctlv3/command/printer_simple.go @@ -136,6 +136,10 @@ func (s *simplePrinter) MemberUpdate(id uint64, r v3.MemberUpdateResponse) { fmt.Printf("Member %16x updated in cluster %16x\n", id, r.Header.ClusterId) } +func (s *simplePrinter) MemberPromote(id uint64, r v3.MemberPromoteResponse) { + fmt.Printf("Member %16x promoted in cluster %16x\n", id, r.Header.ClusterId) +} + func (s *simplePrinter) MemberList(resp v3.MemberListResponse) { _, rows := makeMemberListTable(resp) for _, row := range rows { diff --git a/etcdserver/api/etcdhttp/peer.go b/etcdserver/api/etcdhttp/peer.go index 9f3eac352ef..6c61bf5d510 100644 --- a/etcdserver/api/etcdhttp/peer.go +++ b/etcdserver/api/etcdhttp/peer.go @@ -16,56 +16,82 @@ package etcdhttp import ( "encoding/json" + "fmt" "net/http" + "strconv" + "strings" "go.etcd.io/etcd/etcdserver" "go.etcd.io/etcd/etcdserver/api" + "go.etcd.io/etcd/etcdserver/api/membership" "go.etcd.io/etcd/etcdserver/api/rafthttp" "go.etcd.io/etcd/lease/leasehttp" + "go.etcd.io/etcd/pkg/types" "go.uber.org/zap" ) const ( - peerMembersPrefix = "/members" + peerMembersPath = "/members" + peerMemberPromotePrefix = "/members/promote/" ) // NewPeerHandler generates an http.Handler to handle etcd peer requests. func NewPeerHandler(lg *zap.Logger, s etcdserver.ServerPeer) http.Handler { - return newPeerHandler(lg, s.Cluster(), s.RaftHandler(), s.LeaseHandler()) + return newPeerHandler(lg, s, s.RaftHandler(), s.LeaseHandler()) } -func newPeerHandler(lg *zap.Logger, cluster api.Cluster, raftHandler http.Handler, leaseHandler http.Handler) http.Handler { - mh := &peerMembersHandler{ - lg: lg, - cluster: cluster, - } +func newPeerHandler(lg *zap.Logger, s etcdserver.Server, raftHandler http.Handler, leaseHandler http.Handler) http.Handler { + peerMembersHandler := newPeerMembersHandler(lg, s.Cluster()) + peerMemberPromoteHandler := newPeerMemberPromoteHandler(lg, s) mux := http.NewServeMux() mux.HandleFunc("/", http.NotFound) mux.Handle(rafthttp.RaftPrefix, raftHandler) mux.Handle(rafthttp.RaftPrefix+"/", raftHandler) - mux.Handle(peerMembersPrefix, mh) + mux.Handle(peerMembersPath, peerMembersHandler) + mux.Handle(peerMemberPromotePrefix, peerMemberPromoteHandler) if leaseHandler != nil { mux.Handle(leasehttp.LeasePrefix, leaseHandler) mux.Handle(leasehttp.LeaseInternalPrefix, leaseHandler) } - mux.HandleFunc(versionPath, versionHandler(cluster, serveVersion)) + mux.HandleFunc(versionPath, versionHandler(s.Cluster(), serveVersion)) return mux } +func newPeerMembersHandler(lg *zap.Logger, cluster api.Cluster) http.Handler { + return &peerMembersHandler{ + lg: lg, + cluster: cluster, + } +} + type peerMembersHandler struct { lg *zap.Logger cluster api.Cluster } +func newPeerMemberPromoteHandler(lg *zap.Logger, s etcdserver.Server) http.Handler { + return &peerMemberPromoteHandler{ + lg: lg, + cluster: s.Cluster(), + server: s, + } +} + +type peerMemberPromoteHandler struct { + lg *zap.Logger + cluster api.Cluster + server etcdserver.Server +} + func (h *peerMembersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if !allowMethod(w, r, "GET") { return } w.Header().Set("X-Etcd-Cluster-ID", h.cluster.ID().String()) - if r.URL.Path != peerMembersPrefix { + if r.URL.Path != peerMembersPath { http.Error(w, "bad path", http.StatusBadRequest) return } @@ -79,3 +105,55 @@ func (h *peerMembersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } } + +func (h *peerMemberPromoteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if !allowMethod(w, r, "POST") { + return + } + w.Header().Set("X-Etcd-Cluster-ID", h.cluster.ID().String()) + + if !strings.HasPrefix(r.URL.Path, peerMemberPromotePrefix) { + http.Error(w, "bad path", http.StatusBadRequest) + return + } + idStr := strings.TrimPrefix(r.URL.Path, peerMemberPromotePrefix) + id, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + http.Error(w, fmt.Sprintf("member %s not found in cluster", idStr), http.StatusNotFound) + return + } + + resp, err := h.server.PromoteMember(r.Context(), id) + if err != nil { + switch err { + case membership.ErrIDNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case membership.ErrMemberNotLearner: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + case etcdserver.ErrLearnerNotReady: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + WriteError(h.lg, w, r, err) + } + if h.lg != nil { + h.lg.Warn( + "failed to promote a member", + zap.String("member-id", types.ID(id).String()), + zap.Error(err), + ) + } else { + plog.Errorf("error promoting member %s (%v)", types.ID(id).String(), err) + } + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(resp); err != nil { + if h.lg != nil { + h.lg.Warn("failed to encode members response", zap.Error(err)) + } else { + plog.Warningf("failed to encode members response (%v)", err) + } + } +} diff --git a/etcdserver/api/etcdhttp/peer_test.go b/etcdserver/api/etcdhttp/peer_test.go index 095aa5da849..8d890c0b585 100644 --- a/etcdserver/api/etcdhttp/peer_test.go +++ b/etcdserver/api/etcdhttp/peer_test.go @@ -15,19 +15,24 @@ package etcdhttp import ( + "context" "encoding/json" + "fmt" "io/ioutil" "net/http" "net/http/httptest" "path" "sort" + "strings" "testing" "go.uber.org/zap" "github.com/coreos/go-semver/semver" + "go.etcd.io/etcd/etcdserver/api" "go.etcd.io/etcd/etcdserver/api/membership" "go.etcd.io/etcd/etcdserver/api/rafthttp" + pb "go.etcd.io/etcd/etcdserver/etcdserverpb" "go.etcd.io/etcd/pkg/testutil" "go.etcd.io/etcd/pkg/types" ) @@ -51,13 +56,34 @@ func (c *fakeCluster) Members() []*membership.Member { func (c *fakeCluster) Member(id types.ID) *membership.Member { return c.members[uint64(id)] } func (c *fakeCluster) Version() *semver.Version { return nil } +type fakeServer struct { + cluster api.Cluster +} + +func (s *fakeServer) AddMember(ctx context.Context, memb membership.Member) ([]*membership.Member, error) { + return nil, fmt.Errorf("AddMember not implemented in fakeServer") +} +func (s *fakeServer) RemoveMember(ctx context.Context, id uint64) ([]*membership.Member, error) { + return nil, fmt.Errorf("RemoveMember not implemented in fakeServer") +} +func (s *fakeServer) UpdateMember(ctx context.Context, updateMemb membership.Member) ([]*membership.Member, error) { + return nil, fmt.Errorf("UpdateMember not implemented in fakeServer") +} +func (s *fakeServer) PromoteMember(ctx context.Context, id uint64) ([]*membership.Member, error) { + return nil, fmt.Errorf("PromoteMember not implemented in fakeServer") +} +func (s *fakeServer) ClusterVersion() *semver.Version { return nil } +func (s *fakeServer) Cluster() api.Cluster { return s.cluster } +func (s *fakeServer) Alarms() []*pb.AlarmMember { return nil } + +var fakeRaftHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("test data")) +}) + // TestNewPeerHandlerOnRaftPrefix tests that NewPeerHandler returns a handler that // handles raft-prefix requests well. func TestNewPeerHandlerOnRaftPrefix(t *testing.T) { - h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("test data")) - }) - ph := newPeerHandler(zap.NewExample(), &fakeCluster{}, h, nil) + ph := newPeerHandler(zap.NewExample(), &fakeServer{cluster: &fakeCluster{}}, fakeRaftHandler, nil) srv := httptest.NewServer(ph) defer srv.Close() @@ -80,6 +106,7 @@ func TestNewPeerHandlerOnRaftPrefix(t *testing.T) { } } +// TestServeMembersFails ensures peerMembersHandler only accepts GET request func TestServeMembersFails(t *testing.T) { tests := []struct { method string @@ -89,6 +116,10 @@ func TestServeMembersFails(t *testing.T) { "POST", http.StatusMethodNotAllowed, }, + { + "PUT", + http.StatusMethodNotAllowed, + }, { "DELETE", http.StatusMethodNotAllowed, @@ -100,8 +131,12 @@ func TestServeMembersFails(t *testing.T) { } for i, tt := range tests { rw := httptest.NewRecorder() - h := &peerMembersHandler{cluster: nil} - h.ServeHTTP(rw, &http.Request{Method: tt.method}) + h := newPeerMembersHandler(nil, &fakeCluster{}) + req, err := http.NewRequest(tt.method, "", nil) + if err != nil { + t.Fatalf("#%d: failed to create http request: %v", i, err) + } + h.ServeHTTP(rw, req) if rw.Code != tt.wcode { t.Errorf("#%d: code=%d, want %d", i, rw.Code, tt.wcode) } @@ -115,7 +150,7 @@ func TestServeMembersGet(t *testing.T) { id: 1, members: map[uint64]*membership.Member{1: &memb1, 2: &memb2}, } - h := &peerMembersHandler{cluster: cluster} + h := newPeerMembersHandler(nil, cluster) msb, err := json.Marshal([]membership.Member{memb1, memb2}) if err != nil { t.Fatal(err) @@ -128,8 +163,8 @@ func TestServeMembersGet(t *testing.T) { wct string wbody string }{ - {peerMembersPrefix, http.StatusOK, "application/json", wms}, - {path.Join(peerMembersPrefix, "bad"), http.StatusBadRequest, "text/plain; charset=utf-8", "bad path\n"}, + {peerMembersPath, http.StatusOK, "application/json", wms}, + {path.Join(peerMembersPath, "bad"), http.StatusBadRequest, "text/plain; charset=utf-8", "bad path\n"}, } for i, tt := range tests { @@ -156,3 +191,90 @@ func TestServeMembersGet(t *testing.T) { } } } + +// TestServeMemberPromoteFails ensures peerMemberPromoteHandler only accepts POST request +func TestServeMemberPromoteFails(t *testing.T) { + tests := []struct { + method string + wcode int + }{ + { + "GET", + http.StatusMethodNotAllowed, + }, + { + "PUT", + http.StatusMethodNotAllowed, + }, + { + "DELETE", + http.StatusMethodNotAllowed, + }, + { + "BAD", + http.StatusMethodNotAllowed, + }, + } + for i, tt := range tests { + rw := httptest.NewRecorder() + h := newPeerMemberPromoteHandler(nil, &fakeServer{cluster: &fakeCluster{}}) + req, err := http.NewRequest(tt.method, "", nil) + if err != nil { + t.Fatalf("#%d: failed to create http request: %v", i, err) + } + h.ServeHTTP(rw, req) + if rw.Code != tt.wcode { + t.Errorf("#%d: code=%d, want %d", i, rw.Code, tt.wcode) + } + } +} + +// TestNewPeerHandlerOnMembersPromotePrefix verifies the request with members promote prefix is routed correctly +func TestNewPeerHandlerOnMembersPromotePrefix(t *testing.T) { + ph := newPeerHandler(zap.NewExample(), &fakeServer{cluster: &fakeCluster{}}, fakeRaftHandler, nil) + srv := httptest.NewServer(ph) + defer srv.Close() + + tests := []struct { + path string + wcode int + checkBody bool + wKeyWords string + }{ + { + // does not contain member id in path + peerMemberPromotePrefix, + http.StatusNotFound, + false, + "", + }, + { + // try to promote member id = 1 + peerMemberPromotePrefix + "1", + http.StatusInternalServerError, + true, + "PromoteMember not implemented in fakeServer", + }, + } + for i, tt := range tests { + req, err := http.NewRequest("POST", srv.URL+tt.path, nil) + if err != nil { + t.Fatalf("failed to create request: %v", err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("failed to get http response: %v", err) + } + body, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + t.Fatalf("unexpected ioutil.ReadAll error: %v", err) + } + if resp.StatusCode != tt.wcode { + t.Fatalf("#%d: code = %d, want %d", i, resp.StatusCode, tt.wcode) + } + if tt.checkBody && strings.Contains(string(body), tt.wKeyWords) { + t.Errorf("#%d: body: %s, want body to contain keywords: %s", i, string(body), tt.wKeyWords) + } + } +} diff --git a/etcdserver/api/membership/cluster.go b/etcdserver/api/membership/cluster.go index 65ea46edc54..0affa754433 100644 --- a/etcdserver/api/membership/cluster.go +++ b/etcdserver/api/membership/cluster.go @@ -40,6 +40,8 @@ import ( "go.uber.org/zap" ) +const maxLearners = 1 + // RaftCluster is a list of Members that belong to the same raft cluster type RaftCluster struct { lg *zap.Logger @@ -59,10 +61,18 @@ type RaftCluster struct { removed map[types.ID]bool } +// ConfigChangeContext represents a context for confChange. +type ConfigChangeContext struct { + Member + IsPromote bool `json:"isPromote"` +} + +// NewClusterFromURLsMap creates a new raft cluster using provided urls map. Currently, it does not support creating +// cluster with raft learner member. func NewClusterFromURLsMap(lg *zap.Logger, token string, urlsmap types.URLsMap) (*RaftCluster, error) { c := NewCluster(lg, token) for name, urls := range urlsmap { - m := NewMember(name, urls, token, nil) + m := NewMember(name, urls, token, nil, false) if _, ok := c.members[m.ID]; ok { return nil, fmt.Errorf("member exists with identical ID %v", m) } @@ -112,6 +122,19 @@ func (c *RaftCluster) Member(id types.ID) *Member { return c.members[id].Clone() } +func (c *RaftCluster) VotingMembers() []*Member { + c.Lock() + defer c.Unlock() + var ms MembersByID + for _, m := range c.members { + if !m.IsLearner { + ms = append(ms, m.Clone()) + } + } + sort.Sort(ms) + return []*Member(ms) +} + // MemberByName returns a Member with the given name if exists. // If more than one member has the given name, it will panic. func (c *RaftCluster) MemberByName(name string) *Member { @@ -259,30 +282,53 @@ func (c *RaftCluster) ValidateConfigurationChange(cc raftpb.ConfChange) error { return ErrIDRemoved } switch cc.Type { - case raftpb.ConfChangeAddNode: - if members[id] != nil { - return ErrIDExists - } + case raftpb.ConfChangeAddNode, raftpb.ConfChangeAddLearnerNode: urls := make(map[string]bool) for _, m := range members { for _, u := range m.PeerURLs { urls[u] = true } } - m := new(Member) - if err := json.Unmarshal(cc.Context, m); err != nil { + + confChangeContext := new(ConfigChangeContext) + if err := json.Unmarshal(cc.Context, confChangeContext); err != nil { if c.lg != nil { - c.lg.Panic("failed to unmarshal member", zap.Error(err)) + c.lg.Panic("failed to unmarshal confChangeContext", zap.Error(err)) } else { - plog.Panicf("unmarshal member should never fail: %v", err) + plog.Panicf("unmarshal confChangeContext should never fail: %v", err) } } - for _, u := range m.PeerURLs { - if urls[u] { - return ErrPeerURLexists + + if confChangeContext.IsPromote { // promoting a learner member to voting member + if members[id] == nil { + return ErrIDNotFound + } + if !members[id].IsLearner { + return ErrMemberNotLearner + } + } else { // adding a new member + if members[id] != nil { + return ErrIDExists } - } + for _, u := range confChangeContext.PeerURLs { + if urls[u] { + return ErrPeerURLexists + } + } + + if confChangeContext.Member.IsLearner { // the new member is a learner + numLearners := 0 + for _, m := range members { + if m.IsLearner { + numLearners++ + } + } + if numLearners+1 > maxLearners { + return ErrTooManyLearners + } + } + } case raftpb.ConfChangeRemoveNode: if members[id] == nil { return ErrIDNotFound @@ -432,6 +478,30 @@ func (c *RaftCluster) UpdateAttributes(id types.ID, attr Attributes) { } } +// PromoteMember marks the member's IsLearner RaftAttributes to false. +func (c *RaftCluster) PromoteMember(id types.ID) { + c.Lock() + defer c.Unlock() + + c.members[id].RaftAttributes.IsLearner = false + if c.v2store != nil { + mustUpdateMemberInStore(c.v2store, c.members[id]) + } + if c.be != nil { + mustSaveMemberToBackend(c.be, c.members[id]) + } + + if c.lg != nil { + c.lg.Info( + "promote member", + zap.String("cluster-id", c.cid.String()), + zap.String("local-member-id", c.localID.String()), + ) + } else { + plog.Noticef("promote member %s in cluster %s", id, c.cid) + } +} + func (c *RaftCluster) UpdateRaftAttributes(id types.ID, raftAttr RaftAttributes) { c.Lock() defer c.Unlock() @@ -505,11 +575,11 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*zap.Logger, *s onSet(c.lg, ver) } -func (c *RaftCluster) IsReadyToAddNewMember() bool { +func (c *RaftCluster) IsReadyToAddVotingMember() bool { nmembers := 1 nstarted := 0 - for _, member := range c.members { + for _, member := range c.VotingMembers() { if member.IsStarted() { nstarted++ } @@ -546,11 +616,11 @@ func (c *RaftCluster) IsReadyToAddNewMember() bool { return true } -func (c *RaftCluster) IsReadyToRemoveMember(id uint64) bool { +func (c *RaftCluster) IsReadyToRemoveVotingMember(id uint64) bool { nmembers := 0 nstarted := 0 - for _, member := range c.members { + for _, member := range c.VotingMembers() { if uint64(member.ID) == id { continue } @@ -580,6 +650,36 @@ func (c *RaftCluster) IsReadyToRemoveMember(id uint64) bool { return true } +func (c *RaftCluster) IsReadyToPromoteMember(id uint64) bool { + nmembers := 1 + nstarted := 0 + + for _, member := range c.VotingMembers() { + if member.IsStarted() { + nstarted++ + } + nmembers++ + } + + nquorum := nmembers/2 + 1 + if nstarted < nquorum { + if c.lg != nil { + c.lg.Warn( + "rejecting member promote; started member will be less than quorum", + zap.Int("number-of-started-member", nstarted), + zap.Int("quorum", nquorum), + zap.String("cluster-id", c.cid.String()), + zap.String("local-member-id", c.localID.String()), + ) + } else { + plog.Warningf("Reject promote member request: the number of started member (%d) will be less than the quorum number of the cluster (%d)", nstarted, nquorum) + } + return false + } + + return true +} + func membersFromStore(lg *zap.Logger, st v2store.Store) (map[types.ID]*Member, map[types.ID]bool) { members := make(map[types.ID]*Member) removed := make(map[types.ID]bool) @@ -691,3 +791,44 @@ func mustDetectDowngrade(lg *zap.Logger, cv *semver.Version) { } } } + +// IsLearner returns if the local member is raft learner +func (c *RaftCluster) IsLearner() bool { + c.Lock() + defer c.Unlock() + localMember, ok := c.members[c.localID] + if !ok { + if c.lg != nil { + c.lg.Panic( + "failed to find local ID in cluster members", + zap.String("cluster-id", c.cid.String()), + zap.String("local-member-id", c.localID.String()), + ) + } else { + plog.Panicf("failed to find local ID %s in cluster %s", c.localID.String(), c.cid.String()) + } + } + return localMember.IsLearner +} + +// IsMemberExist returns if the member with the given id exists in cluster. +func (c *RaftCluster) IsMemberExist(id types.ID) bool { + c.Lock() + defer c.Unlock() + _, ok := c.members[id] + return ok +} + +// VotingMemberIDs returns the ID of voting members in cluster. +func (c *RaftCluster) VotingMemberIDs() []types.ID { + c.Lock() + defer c.Unlock() + var ids []types.ID + for _, m := range c.members { + if !m.IsLearner { + ids = append(ids, m.ID) + } + } + sort.Sort(types.IDSlice(ids)) + return ids +} diff --git a/etcdserver/api/membership/cluster_test.go b/etcdserver/api/membership/cluster_test.go index 7aed5aec244..55b72ee08c2 100644 --- a/etcdserver/api/membership/cluster_test.go +++ b/etcdserver/api/membership/cluster_test.go @@ -290,6 +290,12 @@ func TestClusterValidateConfigurationChange(t *testing.T) { t.Fatal(err) } + attr = RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 1)}} + ctx1, err := json.Marshal(&Member{ID: types.ID(1), RaftAttributes: attr}) + if err != nil { + t.Fatal(err) + } + attr = RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 5)}} ctx5, err := json.Marshal(&Member{ID: types.ID(5), RaftAttributes: attr}) if err != nil { @@ -308,6 +314,16 @@ func TestClusterValidateConfigurationChange(t *testing.T) { t.Fatal(err) } + ctx3, err := json.Marshal(&ConfigChangeContext{Member: Member{ID: types.ID(3), RaftAttributes: attr}, IsPromote: true}) + if err != nil { + t.Fatal(err) + } + + ctx6, err := json.Marshal(&ConfigChangeContext{Member: Member{ID: types.ID(6), RaftAttributes: attr}, IsPromote: true}) + if err != nil { + t.Fatal(err) + } + tests := []struct { cc raftpb.ConfChange werr error @@ -335,8 +351,9 @@ func TestClusterValidateConfigurationChange(t *testing.T) { }, { raftpb.ConfChange{ - Type: raftpb.ConfChangeAddNode, - NodeID: 1, + Type: raftpb.ConfChangeAddNode, + NodeID: 1, + Context: ctx1, }, ErrIDExists, }, @@ -388,6 +405,22 @@ func TestClusterValidateConfigurationChange(t *testing.T) { }, nil, }, + { + raftpb.ConfChange{ + Type: raftpb.ConfChangeAddNode, + NodeID: 3, + Context: ctx3, + }, + ErrMemberNotLearner, + }, + { + raftpb.ConfChange{ + Type: raftpb.ConfChangeAddNode, + NodeID: 6, + Context: ctx6, + }, + ErrIDNotFound, + }, } for i, tt := range tests { err := cl.ValidateConfigurationChange(tt.cc) @@ -472,6 +505,29 @@ func TestClusterAddMember(t *testing.T) { } } +func TestClusterAddMemberAsLearner(t *testing.T) { + st := mockstore.NewRecorder() + c := newTestCluster(nil) + c.SetStore(st) + c.AddMember(newTestMemberAsLearner(1, nil, "node1", nil)) + + wactions := []testutil.Action{ + { + Name: "Create", + Params: []interface{}{ + path.Join(StoreMembersPrefix, "1", "raftAttributes"), + false, + `{"peerURLs":null,"isLearner":true}`, + false, + v2store.TTLOptionSet{ExpireTime: v2store.Permanent}, + }, + }, + } + if g := st.Action(); !reflect.DeepEqual(g, wactions) { + t.Errorf("actions = %v, want %v", g, wactions) + } +} + func TestClusterMembers(t *testing.T) { cls := &RaftCluster{ members: map[types.ID]*Member{ @@ -570,7 +626,7 @@ func newTestCluster(membs []*Member) *RaftCluster { func stringp(s string) *string { return &s } -func TestIsReadyToAddNewMember(t *testing.T) { +func TestIsReadyToAddVotingMember(t *testing.T) { tests := []struct { members []*Member want bool @@ -641,16 +697,38 @@ func TestIsReadyToAddNewMember(t *testing.T) { []*Member{}, false, }, + { + // 2 voting members ready in cluster with 2 voting members and 2 unstarted learner member, should succeed + // (the status of learner members does not affect the readiness of adding voting member) + []*Member{ + newTestMember(1, nil, "1", nil), + newTestMember(2, nil, "2", nil), + newTestMemberAsLearner(3, nil, "", nil), + newTestMemberAsLearner(4, nil, "", nil), + }, + true, + }, + { + // 1 voting member ready in cluster with 2 voting members and 2 ready learner member, should fail + // (the status of learner members does not affect the readiness of adding voting member) + []*Member{ + newTestMember(1, nil, "1", nil), + newTestMember(2, nil, "", nil), + newTestMemberAsLearner(3, nil, "3", nil), + newTestMemberAsLearner(4, nil, "4", nil), + }, + false, + }, } for i, tt := range tests { c := newTestCluster(tt.members) - if got := c.IsReadyToAddNewMember(); got != tt.want { + if got := c.IsReadyToAddVotingMember(); got != tt.want { t.Errorf("%d: isReadyToAddNewMember returned %t, want %t", i, got, tt.want) } } } -func TestIsReadyToRemoveMember(t *testing.T) { +func TestIsReadyToRemoveVotingMember(t *testing.T) { tests := []struct { members []*Member removeID uint64 @@ -726,10 +804,57 @@ func TestIsReadyToRemoveMember(t *testing.T) { 4, true, }, + { + // 1 voting members ready in cluster with 1 voting member and 1 ready learner, + // removing voting member should fail + // (the status of learner members does not affect the readiness of removing voting member) + []*Member{ + newTestMember(1, nil, "1", nil), + newTestMemberAsLearner(2, nil, "2", nil), + }, + 1, + false, + }, + { + // 1 voting members ready in cluster with 2 voting member and 1 ready learner, + // removing ready voting member should fail + // (the status of learner members does not affect the readiness of removing voting member) + []*Member{ + newTestMember(1, nil, "1", nil), + newTestMember(2, nil, "", nil), + newTestMemberAsLearner(3, nil, "3", nil), + }, + 1, + false, + }, + { + // 1 voting members ready in cluster with 2 voting member and 1 ready learner, + // removing unstarted voting member should be fine. (Actual operation will fail) + // (the status of learner members does not affect the readiness of removing voting member) + []*Member{ + newTestMember(1, nil, "1", nil), + newTestMember(2, nil, "", nil), + newTestMemberAsLearner(3, nil, "3", nil), + }, + 2, + true, + }, + { + // 1 voting members ready in cluster with 2 voting member and 1 unstarted learner, + // removing not-ready voting member should be fine. (Actual operation will fail) + // (the status of learner members does not affect the readiness of removing voting member) + []*Member{ + newTestMember(1, nil, "1", nil), + newTestMember(2, nil, "", nil), + newTestMemberAsLearner(3, nil, "", nil), + }, + 2, + true, + }, } for i, tt := range tests { c := newTestCluster(tt.members) - if got := c.IsReadyToRemoveMember(tt.removeID); got != tt.want { + if got := c.IsReadyToRemoveVotingMember(tt.removeID); got != tt.want { t.Errorf("%d: isReadyToAddNewMember returned %t, want %t", i, got, tt.want) } } diff --git a/etcdserver/api/membership/errors.go b/etcdserver/api/membership/errors.go index ee5777383a2..8f6fe504e4b 100644 --- a/etcdserver/api/membership/errors.go +++ b/etcdserver/api/membership/errors.go @@ -21,10 +21,12 @@ import ( ) var ( - ErrIDRemoved = errors.New("membership: ID removed") - ErrIDExists = errors.New("membership: ID exists") - ErrIDNotFound = errors.New("membership: ID not found") - ErrPeerURLexists = errors.New("membership: peerURL exists") + ErrIDRemoved = errors.New("membership: ID removed") + ErrIDExists = errors.New("membership: ID exists") + ErrIDNotFound = errors.New("membership: ID not found") + ErrPeerURLexists = errors.New("membership: peerURL exists") + ErrMemberNotLearner = errors.New("membership: can only promote a learner member") + ErrTooManyLearners = errors.New("membership: too many learner members in cluster") ) func isKeyNotFound(err error) bool { diff --git a/etcdserver/api/membership/member.go b/etcdserver/api/membership/member.go index 6a3e79305f2..74c5fec7723 100644 --- a/etcdserver/api/membership/member.go +++ b/etcdserver/api/membership/member.go @@ -35,6 +35,8 @@ type RaftAttributes struct { // PeerURLs is the list of peers in the raft cluster. // TODO(philips): ensure these are URLs PeerURLs []string `json:"peerURLs"` + // IsLearner indicates if the member is raft learner. + IsLearner bool `json:"isLearner,omitempty"` } // Attributes represents all the non-raft related attributes of an etcd member. @@ -51,10 +53,13 @@ type Member struct { // NewMember creates a Member without an ID and generates one based on the // cluster name, peer URLs, and time. This is used for bootstrapping/adding new member. -func NewMember(name string, peerURLs types.URLs, clusterName string, now *time.Time) *Member { +func NewMember(name string, peerURLs types.URLs, clusterName string, now *time.Time, isLearner bool) *Member { m := &Member{ - RaftAttributes: RaftAttributes{PeerURLs: peerURLs.StringSlice()}, - Attributes: Attributes{Name: name}, + RaftAttributes: RaftAttributes{ + PeerURLs: peerURLs.StringSlice(), + IsLearner: isLearner, + }, + Attributes: Attributes{Name: name}, } var b []byte @@ -88,6 +93,9 @@ func (m *Member) Clone() *Member { } mm := &Member{ ID: m.ID, + RaftAttributes: RaftAttributes{ + IsLearner: m.IsLearner, + }, Attributes: Attributes{ Name: m.Name, }, diff --git a/etcdserver/api/membership/member_test.go b/etcdserver/api/membership/member_test.go index bbbb88986c3..8e2d487307f 100644 --- a/etcdserver/api/membership/member_test.go +++ b/etcdserver/api/membership/member_test.go @@ -36,17 +36,17 @@ func TestMemberTime(t *testing.T) { mem *Member id types.ID }{ - {NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.8:2379"}}, "", nil), 14544069596553697298}, + {NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.8:2379"}}, "", nil, false), 14544069596553697298}, // Same ID, different name (names shouldn't matter) - {NewMember("memfoo", []url.URL{{Scheme: "http", Host: "10.0.0.8:2379"}}, "", nil), 14544069596553697298}, + {NewMember("memfoo", []url.URL{{Scheme: "http", Host: "10.0.0.8:2379"}}, "", nil, false), 14544069596553697298}, // Same ID, different Time - {NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.8:2379"}}, "", timeParse("1984-12-23T15:04:05Z")), 2448790162483548276}, + {NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.8:2379"}}, "", timeParse("1984-12-23T15:04:05Z"), false), 2448790162483548276}, // Different cluster name - {NewMember("mcm1", []url.URL{{Scheme: "http", Host: "10.0.0.8:2379"}}, "etcd", timeParse("1984-12-23T15:04:05Z")), 6973882743191604649}, - {NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}}, "", timeParse("1984-12-23T15:04:05Z")), 1466075294948436910}, + {NewMember("mcm1", []url.URL{{Scheme: "http", Host: "10.0.0.8:2379"}}, "etcd", timeParse("1984-12-23T15:04:05Z"), false), 6973882743191604649}, + {NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}}, "", timeParse("1984-12-23T15:04:05Z"), false), 1466075294948436910}, // Order shouldn't matter - {NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}, {Scheme: "http", Host: "10.0.0.2:2379"}}, "", nil), 16552244735972308939}, - {NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.2:2379"}, {Scheme: "http", Host: "10.0.0.1:2379"}}, "", nil), 16552244735972308939}, + {NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}, {Scheme: "http", Host: "10.0.0.2:2379"}}, "", nil, false), 16552244735972308939}, + {NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.2:2379"}, {Scheme: "http", Host: "10.0.0.1:2379"}}, "", nil, false), 16552244735972308939}, } for i, tt := range tests { if tt.mem.ID != tt.id { @@ -113,3 +113,11 @@ func newTestMember(id uint64, peerURLs []string, name string, clientURLs []strin Attributes: Attributes{Name: name, ClientURLs: clientURLs}, } } + +func newTestMemberAsLearner(id uint64, peerURLs []string, name string, clientURLs []string) *Member { + return &Member{ + ID: types.ID(id), + RaftAttributes: RaftAttributes{PeerURLs: peerURLs, IsLearner: true}, + Attributes: Attributes{Name: name, ClientURLs: clientURLs}, + } +} diff --git a/etcdserver/api/v2http/client.go b/etcdserver/api/v2http/client.go index 1d1e592b25d..61de6abac1e 100644 --- a/etcdserver/api/v2http/client.go +++ b/etcdserver/api/v2http/client.go @@ -238,7 +238,7 @@ func (h *membersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } now := h.clock.Now() - m := membership.NewMember("", req.PeerURLs, "", &now) + m := membership.NewMember("", req.PeerURLs, "", &now, false) // does not support adding learner via v2http _, err := h.server.AddMember(ctx, *m) switch { case err == membership.ErrIDExists || err == membership.ErrPeerURLexists: diff --git a/etcdserver/api/v2http/client_test.go b/etcdserver/api/v2http/client_test.go index 5e1d80701e1..5bab6c45fa9 100644 --- a/etcdserver/api/v2http/client_test.go +++ b/etcdserver/api/v2http/client_test.go @@ -132,6 +132,11 @@ func (s *serverRecorder) UpdateMember(_ context.Context, m membership.Member) ([ return nil, nil } +func (s *serverRecorder) PromoteMember(_ context.Context, id uint64) ([]*membership.Member, error) { + s.actions = append(s.actions, action{name: "PromoteMember", params: []interface{}{id}}) + return nil, nil +} + type action struct { name string params []interface{} @@ -168,6 +173,9 @@ func (rs *resServer) RemoveMember(_ context.Context, _ uint64) ([]*membership.Me func (rs *resServer) UpdateMember(_ context.Context, _ membership.Member) ([]*membership.Member, error) { return nil, nil } +func (rs *resServer) PromoteMember(_ context.Context, _ uint64) ([]*membership.Member, error) { + return nil, nil +} func boolp(b bool) *bool { return &b } diff --git a/etcdserver/api/v2http/http_test.go b/etcdserver/api/v2http/http_test.go index 0e42c26ca09..f93071048d0 100644 --- a/etcdserver/api/v2http/http_test.go +++ b/etcdserver/api/v2http/http_test.go @@ -74,6 +74,9 @@ func (fs *errServer) RemoveMember(ctx context.Context, id uint64) ([]*membership func (fs *errServer) UpdateMember(ctx context.Context, m membership.Member) ([]*membership.Member, error) { return nil, fs.err } +func (fs *errServer) PromoteMember(ctx context.Context, id uint64) ([]*membership.Member, error) { + return nil, fs.err +} func TestWriteError(t *testing.T) { // nil error should not panic diff --git a/etcdserver/api/v2v3/server.go b/etcdserver/api/v2v3/server.go index fd5ad0744a8..f3845feda20 100644 --- a/etcdserver/api/v2v3/server.go +++ b/etcdserver/api/v2v3/server.go @@ -63,7 +63,7 @@ func (s *v2v3Server) Leader() types.ID { } func (s *v2v3Server) AddMember(ctx context.Context, memb membership.Member) ([]*membership.Member, error) { - resp, err := s.c.MemberAdd(ctx, memb.PeerURLs) + resp, err := s.c.MemberAdd(ctx, memb.PeerURLs, memb.IsLearner) if err != nil { return nil, err } @@ -78,6 +78,14 @@ func (s *v2v3Server) RemoveMember(ctx context.Context, id uint64) ([]*membership return v3MembersToMembership(resp.Members), nil } +func (s *v2v3Server) PromoteMember(ctx context.Context, id uint64) ([]*membership.Member, error) { + resp, err := s.c.MemberPromote(ctx, id) + if err != nil { + return nil, err + } + return v3MembersToMembership(resp.Members), nil +} + func (s *v2v3Server) UpdateMember(ctx context.Context, m membership.Member) ([]*membership.Member, error) { resp, err := s.c.MemberUpdate(ctx, uint64(m.ID), m.PeerURLs) if err != nil { @@ -92,7 +100,8 @@ func v3MembersToMembership(v3membs []*pb.Member) []*membership.Member { membs[i] = &membership.Member{ ID: types.ID(m.ID), RaftAttributes: membership.RaftAttributes{ - PeerURLs: m.PeerURLs, + PeerURLs: m.PeerURLs, + IsLearner: m.IsLearner, }, Attributes: membership.Attributes{ Name: m.Name, diff --git a/etcdserver/api/v3rpc/interceptor.go b/etcdserver/api/v3rpc/interceptor.go index 16865e22c75..b046244077f 100644 --- a/etcdserver/api/v3rpc/interceptor.go +++ b/etcdserver/api/v3rpc/interceptor.go @@ -48,6 +48,10 @@ func newUnaryInterceptor(s *etcdserver.EtcdServer) grpc.UnaryServerInterceptor { return nil, rpctypes.ErrGRPCNotCapable } + if s.IsMemberExist(s.ID()) && s.IsLearner() && !isRPCEnabledForLearner(req) { + return nil, rpctypes.ErrGPRCNotSupportedForLearner + } + md, ok := metadata.FromIncomingContext(ctx) if ok { if ks := md[rpctypes.MetadataRequireLeaderKey]; len(ks) > 0 && ks[0] == rpctypes.MetadataHasLeader { @@ -190,6 +194,10 @@ func newStreamInterceptor(s *etcdserver.EtcdServer) grpc.StreamServerInterceptor return rpctypes.ErrGRPCNotCapable } + if s.IsMemberExist(s.ID()) && s.IsLearner() { // learner does not support stream RPC + return rpctypes.ErrGPRCNotSupportedForLearner + } + md, ok := metadata.FromIncomingContext(ss.Context()) if ok { if ks := md[rpctypes.MetadataRequireLeaderKey]; len(ks) > 0 && ks[0] == rpctypes.MetadataHasLeader { diff --git a/etcdserver/api/v3rpc/maintenance.go b/etcdserver/api/v3rpc/maintenance.go index 002d1c7fa1b..c51271ac0fe 100644 --- a/etcdserver/api/v3rpc/maintenance.go +++ b/etcdserver/api/v3rpc/maintenance.go @@ -55,6 +55,10 @@ type AuthGetter interface { AuthStore() auth.AuthStore } +type ClusterStatusGetter interface { + IsLearner() bool +} + type maintenanceServer struct { lg *zap.Logger rg etcdserver.RaftStatusGetter @@ -63,10 +67,11 @@ type maintenanceServer struct { a Alarmer lt LeaderTransferrer hdr header + cs ClusterStatusGetter } func NewMaintenanceServer(s *etcdserver.EtcdServer) pb.MaintenanceServer { - srv := &maintenanceServer{lg: s.Cfg.Logger, rg: s, kg: s, bg: s, a: s, lt: s, hdr: newHeader(s)} + srv := &maintenanceServer{lg: s.Cfg.Logger, rg: s, kg: s, bg: s, a: s, lt: s, hdr: newHeader(s), cs: s} return &authMaintenanceServer{srv, s} } @@ -179,6 +184,7 @@ func (ms *maintenanceServer) Status(ctx context.Context, ar *pb.StatusRequest) ( RaftTerm: ms.rg.Term(), DbSize: ms.bg.Backend().Size(), DbSizeInUse: ms.bg.Backend().SizeInUse(), + IsLearner: ms.cs.IsLearner(), } if resp.Leader == raft.None { resp.Errors = append(resp.Errors, etcdserver.ErrNoLeader.Error()) diff --git a/etcdserver/api/v3rpc/member.go b/etcdserver/api/v3rpc/member.go index 49baa76ee40..4cbc7e4edce 100644 --- a/etcdserver/api/v3rpc/member.go +++ b/etcdserver/api/v3rpc/member.go @@ -45,15 +45,19 @@ func (cs *ClusterServer) MemberAdd(ctx context.Context, r *pb.MemberAddRequest) } now := time.Now() - m := membership.NewMember("", urls, "", &now) + m := membership.NewMember("", urls, "", &now, r.IsLearner) membs, merr := cs.server.AddMember(ctx, *m) if merr != nil { return nil, togRPCError(merr) } return &pb.MemberAddResponse{ - Header: cs.header(), - Member: &pb.Member{ID: uint64(m.ID), PeerURLs: m.PeerURLs}, + Header: cs.header(), + Member: &pb.Member{ + ID: uint64(m.ID), + PeerURLs: m.PeerURLs, + IsLearner: m.IsLearner, + }, Members: membersToProtoMembers(membs), }, nil } @@ -83,6 +87,14 @@ func (cs *ClusterServer) MemberList(ctx context.Context, r *pb.MemberListRequest return &pb.MemberListResponse{Header: cs.header(), Members: membs}, nil } +func (cs *ClusterServer) MemberPromote(ctx context.Context, r *pb.MemberPromoteRequest) (*pb.MemberPromoteResponse, error) { + membs, err := cs.server.PromoteMember(ctx, r.ID) + if err != nil { + return nil, togRPCError(err) + } + return &pb.MemberPromoteResponse{Header: cs.header(), Members: membersToProtoMembers(membs)}, nil +} + func (cs *ClusterServer) header() *pb.ResponseHeader { return &pb.ResponseHeader{ClusterId: uint64(cs.cluster.ID()), MemberId: uint64(cs.server.ID()), RaftTerm: cs.server.Term()} } @@ -95,6 +107,7 @@ func membersToProtoMembers(membs []*membership.Member) []*pb.Member { ID: uint64(membs[i].ID), PeerURLs: membs[i].PeerURLs, ClientURLs: membs[i].ClientURLs, + IsLearner: membs[i].IsLearner, } } return protoMembs diff --git a/etcdserver/api/v3rpc/rpctypes/error.go b/etcdserver/api/v3rpc/rpctypes/error.go index 9e45cea5b6e..2076b4aa63e 100644 --- a/etcdserver/api/v3rpc/rpctypes/error.go +++ b/etcdserver/api/v3rpc/rpctypes/error.go @@ -40,6 +40,9 @@ var ( ErrGRPCMemberNotEnoughStarted = status.New(codes.FailedPrecondition, "etcdserver: re-configuration failed due to not enough started members").Err() ErrGRPCMemberBadURLs = status.New(codes.InvalidArgument, "etcdserver: given member URLs are invalid").Err() ErrGRPCMemberNotFound = status.New(codes.NotFound, "etcdserver: member not found").Err() + ErrGRPCMemberNotLearner = status.New(codes.FailedPrecondition, "etcdserver: can only promote a learner member").Err() + ErrGRPCLearnerNotReady = status.New(codes.FailedPrecondition, "etcdserver: can only promote a learner member which catches up with leader").Err() + ErrGRPCTooManyLearners = status.New(codes.FailedPrecondition, "etcdserver: too many learner members in cluster").Err() ErrGRPCRequestTooLarge = status.New(codes.InvalidArgument, "etcdserver: request is too large").Err() ErrGRPCRequestTooManyRequests = status.New(codes.ResourceExhausted, "etcdserver: too many requests").Err() @@ -69,6 +72,8 @@ var ( ErrGRPCTimeoutDueToConnectionLost = status.New(codes.Unavailable, "etcdserver: request timed out, possibly due to connection lost").Err() ErrGRPCUnhealthy = status.New(codes.Unavailable, "etcdserver: unhealthy cluster").Err() ErrGRPCCorrupt = status.New(codes.DataLoss, "etcdserver: corrupt cluster").Err() + ErrGPRCNotSupportedForLearner = status.New(codes.Unavailable, "etcdserver: rpc not supported for learner").Err() + ErrGRPCBadLeaderTransferee = status.New(codes.FailedPrecondition, "etcdserver: bad leader transferee").Err() errStringToError = map[string]error{ ErrorDesc(ErrGRPCEmptyKey): ErrGRPCEmptyKey, @@ -91,6 +96,9 @@ var ( ErrorDesc(ErrGRPCMemberNotEnoughStarted): ErrGRPCMemberNotEnoughStarted, ErrorDesc(ErrGRPCMemberBadURLs): ErrGRPCMemberBadURLs, ErrorDesc(ErrGRPCMemberNotFound): ErrGRPCMemberNotFound, + ErrorDesc(ErrGRPCMemberNotLearner): ErrGRPCMemberNotLearner, + ErrorDesc(ErrGRPCLearnerNotReady): ErrGRPCLearnerNotReady, + ErrorDesc(ErrGRPCTooManyLearners): ErrGRPCTooManyLearners, ErrorDesc(ErrGRPCRequestTooLarge): ErrGRPCRequestTooLarge, ErrorDesc(ErrGRPCRequestTooManyRequests): ErrGRPCRequestTooManyRequests, @@ -120,6 +128,8 @@ var ( ErrorDesc(ErrGRPCTimeoutDueToConnectionLost): ErrGRPCTimeoutDueToConnectionLost, ErrorDesc(ErrGRPCUnhealthy): ErrGRPCUnhealthy, ErrorDesc(ErrGRPCCorrupt): ErrGRPCCorrupt, + ErrorDesc(ErrGPRCNotSupportedForLearner): ErrGPRCNotSupportedForLearner, + ErrorDesc(ErrGRPCBadLeaderTransferee): ErrGRPCBadLeaderTransferee, } ) @@ -144,6 +154,9 @@ var ( ErrMemberNotEnoughStarted = Error(ErrGRPCMemberNotEnoughStarted) ErrMemberBadURLs = Error(ErrGRPCMemberBadURLs) ErrMemberNotFound = Error(ErrGRPCMemberNotFound) + ErrMemberNotLearner = Error(ErrGRPCMemberNotLearner) + ErrMemberLearnerNotReady = Error(ErrGRPCLearnerNotReady) + ErrTooManyLearners = Error(ErrGRPCTooManyLearners) ErrRequestTooLarge = Error(ErrGRPCRequestTooLarge) ErrTooManyRequests = Error(ErrGRPCRequestTooManyRequests) @@ -173,6 +186,7 @@ var ( ErrTimeoutDueToConnectionLost = Error(ErrGRPCTimeoutDueToConnectionLost) ErrUnhealthy = Error(ErrGRPCUnhealthy) ErrCorrupt = Error(ErrGRPCCorrupt) + ErrBadLeaderTransferee = Error(ErrGRPCBadLeaderTransferee) ) // EtcdError defines gRPC server errors. diff --git a/etcdserver/api/v3rpc/util.go b/etcdserver/api/v3rpc/util.go index 5887dfeba44..d6579d3e436 100644 --- a/etcdserver/api/v3rpc/util.go +++ b/etcdserver/api/v3rpc/util.go @@ -22,6 +22,7 @@ import ( "go.etcd.io/etcd/etcdserver" "go.etcd.io/etcd/etcdserver/api/membership" "go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes" + pb "go.etcd.io/etcd/etcdserver/etcdserverpb" "go.etcd.io/etcd/lease" "go.etcd.io/etcd/mvcc" @@ -34,7 +35,10 @@ var toGRPCErrorMap = map[error]error{ membership.ErrIDNotFound: rpctypes.ErrGRPCMemberNotFound, membership.ErrIDExists: rpctypes.ErrGRPCMemberExist, membership.ErrPeerURLexists: rpctypes.ErrGRPCPeerURLExist, + membership.ErrMemberNotLearner: rpctypes.ErrGRPCMemberNotLearner, + membership.ErrTooManyLearners: rpctypes.ErrGRPCTooManyLearners, etcdserver.ErrNotEnoughStartedMembers: rpctypes.ErrMemberNotEnoughStarted, + etcdserver.ErrLearnerNotReady: rpctypes.ErrGRPCLearnerNotReady, mvcc.ErrCompacted: rpctypes.ErrGRPCCompacted, mvcc.ErrFutureRev: rpctypes.ErrGRPCFutureRev, @@ -52,6 +56,7 @@ var toGRPCErrorMap = map[error]error{ etcdserver.ErrUnhealthy: rpctypes.ErrGRPCUnhealthy, etcdserver.ErrKeyNotFound: rpctypes.ErrGRPCKeyNotFound, etcdserver.ErrCorrupt: rpctypes.ErrGRPCCorrupt, + etcdserver.ErrBadLeaderTransferee: rpctypes.ErrGRPCBadLeaderTransferee, lease.ErrLeaseNotFound: rpctypes.ErrGRPCLeaseNotFound, lease.ErrLeaseExists: rpctypes.ErrGRPCLeaseExist, @@ -116,3 +121,15 @@ func isClientCtxErr(ctxErr error, err error) bool { } return false } + +// in v3.4, learner is allowed to serve serializable read and endpoint status +func isRPCEnabledForLearner(req interface{}) bool { + switch r := req.(type) { + case *pb.StatusRequest: + return true + case *pb.RangeRequest: + return r.Serializable + default: + return false + } +} diff --git a/etcdserver/cluster_util.go b/etcdserver/cluster_util.go index eecb890e6dd..f92706cb7a1 100644 --- a/etcdserver/cluster_util.go +++ b/etcdserver/cluster_util.go @@ -15,11 +15,13 @@ package etcdserver import ( + "context" "encoding/json" "fmt" "io/ioutil" "net/http" "sort" + "strings" "time" "go.etcd.io/etcd/etcdserver/api/membership" @@ -355,3 +357,51 @@ func getVersion(lg *zap.Logger, m *membership.Member, rt http.RoundTripper) (*ve } return nil, err } + +func promoteMemberHTTP(ctx context.Context, url string, id uint64, peerRt http.RoundTripper) ([]*membership.Member, error) { + cc := &http.Client{Transport: peerRt} + // TODO: refactor member http handler code + // cannot import etcdhttp, so manually construct url + requestUrl := url + "/members/promote/" + fmt.Sprintf("%d", id) + req, err := http.NewRequest("POST", requestUrl, nil) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + resp, err := cc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusRequestTimeout { + return nil, ErrTimeout + } + if resp.StatusCode == http.StatusPreconditionFailed { + // both ErrMemberNotLearner and ErrLearnerNotReady have same http status code + if strings.Contains(string(b), ErrLearnerNotReady.Error()) { + return nil, ErrLearnerNotReady + } + if strings.Contains(string(b), membership.ErrMemberNotLearner.Error()) { + return nil, membership.ErrMemberNotLearner + } + return nil, fmt.Errorf("member promote: unknown error(%s)", string(b)) + } + if resp.StatusCode == http.StatusNotFound { + return nil, membership.ErrIDNotFound + } + + if resp.StatusCode != http.StatusOK { // all other types of errors + return nil, fmt.Errorf("member promote: unknown error(%s)", string(b)) + } + + var membs []*membership.Member + if err := json.Unmarshal(b, &membs); err != nil { + return nil, err + } + return membs, nil +} diff --git a/etcdserver/errors.go b/etcdserver/errors.go index 8cec52a177b..e5ab4bd7142 100644 --- a/etcdserver/errors.go +++ b/etcdserver/errors.go @@ -29,6 +29,7 @@ var ( ErrTimeoutLeaderTransfer = errors.New("etcdserver: request timed out, leader transfer took too long") ErrLeaderChanged = errors.New("etcdserver: leader changed") ErrNotEnoughStartedMembers = errors.New("etcdserver: re-configuration failed due to not enough started members") + ErrLearnerNotReady = errors.New("etcdserver: can only promote a learner member which catches up with leader") ErrNoLeader = errors.New("etcdserver: no leader") ErrNotLeader = errors.New("etcdserver: not leader") ErrRequestTooLarge = errors.New("etcdserver: request is too large") @@ -37,6 +38,7 @@ var ( ErrUnhealthy = errors.New("etcdserver: unhealthy cluster") ErrKeyNotFound = errors.New("etcdserver: key not found") ErrCorrupt = errors.New("etcdserver: corrupt cluster") + ErrBadLeaderTransferee = errors.New("etcdserver: bad leader transferee") ) type DiscoveryError struct { diff --git a/etcdserver/etcdserverpb/etcdserver.pb.go b/etcdserver/etcdserverpb/etcdserver.pb.go index f5134b9f7c4..9e9b42ceac7 100644 --- a/etcdserver/etcdserverpb/etcdserver.pb.go +++ b/etcdserver/etcdserverpb/etcdserver.pb.go @@ -64,6 +64,8 @@ MemberUpdateResponse MemberListRequest MemberListResponse + MemberPromoteRequest + MemberPromoteResponse DefragmentRequest DefragmentResponse MoveLeaderRequest diff --git a/etcdserver/etcdserverpb/gw/rpc.pb.gw.go b/etcdserver/etcdserverpb/gw/rpc.pb.gw.go index babe2130555..904c32187fb 100644 --- a/etcdserver/etcdserverpb/gw/rpc.pb.gw.go +++ b/etcdserver/etcdserverpb/gw/rpc.pb.gw.go @@ -341,6 +341,19 @@ func request_Cluster_MemberList_0(ctx context.Context, marshaler runtime.Marshal } +func request_Cluster_MemberPromote_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.ClusterClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq etcdserverpb.MemberPromoteRequest + var metadata runtime.ServerMetadata + + if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil && err != io.EOF { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + + msg, err := client.MemberPromote(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err + +} + func request_Maintenance_Alarm_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.MaintenanceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var protoReq etcdserverpb.AlarmRequest var metadata runtime.ServerMetadata @@ -1399,6 +1412,35 @@ func RegisterClusterHandlerClient(ctx context.Context, mux *runtime.ServeMux, cl }) + mux.Handle("POST", pattern_Cluster_MemberPromote_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + if cn, ok := w.(http.CloseNotifier); ok { + go func(done <-chan struct{}, closed <-chan bool) { + select { + case <-done: + case <-closed: + cancel() + } + }(ctx.Done(), cn.CloseNotify()) + } + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + rctx, err := runtime.AnnotateContext(ctx, mux, req) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := request_Cluster_MemberPromote_0(rctx, inboundMarshaler, client, req, pathParams) + ctx = runtime.NewServerMetadataContext(ctx, md) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + + forward_Cluster_MemberPromote_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + + }) + return nil } @@ -1410,6 +1452,8 @@ var ( pattern_Cluster_MemberUpdate_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "cluster", "member", "update"}, "")) pattern_Cluster_MemberList_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "cluster", "member", "list"}, "")) + + pattern_Cluster_MemberPromote_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "cluster", "member", "promote"}, "")) ) var ( @@ -1420,6 +1464,8 @@ var ( forward_Cluster_MemberUpdate_0 = runtime.ForwardResponseMessage forward_Cluster_MemberList_0 = runtime.ForwardResponseMessage + + forward_Cluster_MemberPromote_0 = runtime.ForwardResponseMessage ) // RegisterMaintenanceHandlerFromEndpoint is same as RegisterMaintenanceHandler but diff --git a/etcdserver/etcdserverpb/rpc.pb.go b/etcdserver/etcdserverpb/rpc.pb.go index 3e15079e6f7..6d728a59c81 100644 --- a/etcdserver/etcdserverpb/rpc.pb.go +++ b/etcdserver/etcdserverpb/rpc.pb.go @@ -211,7 +211,7 @@ func (x AlarmRequest_AlarmAction) String() string { return proto.EnumName(AlarmRequest_AlarmAction_name, int32(x)) } func (AlarmRequest_AlarmAction) EnumDescriptor() ([]byte, []int) { - return fileDescriptorRpc, []int{52, 0} + return fileDescriptorRpc, []int{54, 0} } type ResponseHeader struct { @@ -2186,6 +2186,8 @@ type Member struct { PeerURLs []string `protobuf:"bytes,3,rep,name=peerURLs" json:"peerURLs,omitempty"` // clientURLs is the list of URLs the member exposes to clients for communication. If the member is not started, clientURLs will be empty. ClientURLs []string `protobuf:"bytes,4,rep,name=clientURLs" json:"clientURLs,omitempty"` + // isLearner indicates if the member is raft learner. + IsLearner bool `protobuf:"varint,5,opt,name=isLearner,proto3" json:"isLearner,omitempty"` } func (m *Member) Reset() { *m = Member{} } @@ -2221,9 +2223,18 @@ func (m *Member) GetClientURLs() []string { return nil } +func (m *Member) GetIsLearner() bool { + if m != nil { + return m.IsLearner + } + return false +} + type MemberAddRequest struct { // peerURLs is the list of URLs the added member will use to communicate with the cluster. PeerURLs []string `protobuf:"bytes,1,rep,name=peerURLs" json:"peerURLs,omitempty"` + // isLearner indicates if the added member is raft learner. + IsLearner bool `protobuf:"varint,2,opt,name=isLearner,proto3" json:"isLearner,omitempty"` } func (m *MemberAddRequest) Reset() { *m = MemberAddRequest{} } @@ -2238,6 +2249,13 @@ func (m *MemberAddRequest) GetPeerURLs() []string { return nil } +func (m *MemberAddRequest) GetIsLearner() bool { + if m != nil { + return m.IsLearner + } + return false +} + type MemberAddResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // member is the member information for the added member. @@ -2398,13 +2416,55 @@ func (m *MemberListResponse) GetMembers() []*Member { return nil } +type MemberPromoteRequest struct { + // ID is the member ID of the member to promote. + ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` +} + +func (m *MemberPromoteRequest) Reset() { *m = MemberPromoteRequest{} } +func (m *MemberPromoteRequest) String() string { return proto.CompactTextString(m) } +func (*MemberPromoteRequest) ProtoMessage() {} +func (*MemberPromoteRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{48} } + +func (m *MemberPromoteRequest) GetID() uint64 { + if m != nil { + return m.ID + } + return 0 +} + +type MemberPromoteResponse struct { + Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` + // members is a list of all members after promoting the member. + Members []*Member `protobuf:"bytes,2,rep,name=members" json:"members,omitempty"` +} + +func (m *MemberPromoteResponse) Reset() { *m = MemberPromoteResponse{} } +func (m *MemberPromoteResponse) String() string { return proto.CompactTextString(m) } +func (*MemberPromoteResponse) ProtoMessage() {} +func (*MemberPromoteResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{49} } + +func (m *MemberPromoteResponse) GetHeader() *ResponseHeader { + if m != nil { + return m.Header + } + return nil +} + +func (m *MemberPromoteResponse) GetMembers() []*Member { + if m != nil { + return m.Members + } + return nil +} + type DefragmentRequest struct { } func (m *DefragmentRequest) Reset() { *m = DefragmentRequest{} } func (m *DefragmentRequest) String() string { return proto.CompactTextString(m) } func (*DefragmentRequest) ProtoMessage() {} -func (*DefragmentRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{48} } +func (*DefragmentRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{50} } type DefragmentResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` @@ -2413,7 +2473,7 @@ type DefragmentResponse struct { func (m *DefragmentResponse) Reset() { *m = DefragmentResponse{} } func (m *DefragmentResponse) String() string { return proto.CompactTextString(m) } func (*DefragmentResponse) ProtoMessage() {} -func (*DefragmentResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{49} } +func (*DefragmentResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{51} } func (m *DefragmentResponse) GetHeader() *ResponseHeader { if m != nil { @@ -2430,7 +2490,7 @@ type MoveLeaderRequest struct { func (m *MoveLeaderRequest) Reset() { *m = MoveLeaderRequest{} } func (m *MoveLeaderRequest) String() string { return proto.CompactTextString(m) } func (*MoveLeaderRequest) ProtoMessage() {} -func (*MoveLeaderRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{50} } +func (*MoveLeaderRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{52} } func (m *MoveLeaderRequest) GetTargetID() uint64 { if m != nil { @@ -2446,7 +2506,7 @@ type MoveLeaderResponse struct { func (m *MoveLeaderResponse) Reset() { *m = MoveLeaderResponse{} } func (m *MoveLeaderResponse) String() string { return proto.CompactTextString(m) } func (*MoveLeaderResponse) ProtoMessage() {} -func (*MoveLeaderResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{51} } +func (*MoveLeaderResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{53} } func (m *MoveLeaderResponse) GetHeader() *ResponseHeader { if m != nil { @@ -2470,7 +2530,7 @@ type AlarmRequest struct { func (m *AlarmRequest) Reset() { *m = AlarmRequest{} } func (m *AlarmRequest) String() string { return proto.CompactTextString(m) } func (*AlarmRequest) ProtoMessage() {} -func (*AlarmRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{52} } +func (*AlarmRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{54} } func (m *AlarmRequest) GetAction() AlarmRequest_AlarmAction { if m != nil { @@ -2503,7 +2563,7 @@ type AlarmMember struct { func (m *AlarmMember) Reset() { *m = AlarmMember{} } func (m *AlarmMember) String() string { return proto.CompactTextString(m) } func (*AlarmMember) ProtoMessage() {} -func (*AlarmMember) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{53} } +func (*AlarmMember) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{55} } func (m *AlarmMember) GetMemberID() uint64 { if m != nil { @@ -2528,7 +2588,7 @@ type AlarmResponse struct { func (m *AlarmResponse) Reset() { *m = AlarmResponse{} } func (m *AlarmResponse) String() string { return proto.CompactTextString(m) } func (*AlarmResponse) ProtoMessage() {} -func (*AlarmResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{54} } +func (*AlarmResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{56} } func (m *AlarmResponse) GetHeader() *ResponseHeader { if m != nil { @@ -2550,7 +2610,7 @@ type StatusRequest struct { func (m *StatusRequest) Reset() { *m = StatusRequest{} } func (m *StatusRequest) String() string { return proto.CompactTextString(m) } func (*StatusRequest) ProtoMessage() {} -func (*StatusRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{55} } +func (*StatusRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{57} } type StatusResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` @@ -2570,12 +2630,14 @@ type StatusResponse struct { Errors []string `protobuf:"bytes,8,rep,name=errors" json:"errors,omitempty"` // dbSizeInUse is the size of the backend database logically in use, in bytes, of the responding member. DbSizeInUse int64 `protobuf:"varint,9,opt,name=dbSizeInUse,proto3" json:"dbSizeInUse,omitempty"` + // isLearner indicates if the member is raft learner. + IsLearner bool `protobuf:"varint,10,opt,name=isLearner,proto3" json:"isLearner,omitempty"` } func (m *StatusResponse) Reset() { *m = StatusResponse{} } func (m *StatusResponse) String() string { return proto.CompactTextString(m) } func (*StatusResponse) ProtoMessage() {} -func (*StatusResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{56} } +func (*StatusResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{58} } func (m *StatusResponse) GetHeader() *ResponseHeader { if m != nil { @@ -2640,13 +2702,20 @@ func (m *StatusResponse) GetDbSizeInUse() int64 { return 0 } +func (m *StatusResponse) GetIsLearner() bool { + if m != nil { + return m.IsLearner + } + return false +} + type AuthEnableRequest struct { } func (m *AuthEnableRequest) Reset() { *m = AuthEnableRequest{} } func (m *AuthEnableRequest) String() string { return proto.CompactTextString(m) } func (*AuthEnableRequest) ProtoMessage() {} -func (*AuthEnableRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{57} } +func (*AuthEnableRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{59} } type AuthDisableRequest struct { } @@ -2654,7 +2723,7 @@ type AuthDisableRequest struct { func (m *AuthDisableRequest) Reset() { *m = AuthDisableRequest{} } func (m *AuthDisableRequest) String() string { return proto.CompactTextString(m) } func (*AuthDisableRequest) ProtoMessage() {} -func (*AuthDisableRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{58} } +func (*AuthDisableRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{60} } type AuthenticateRequest struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -2664,7 +2733,7 @@ type AuthenticateRequest struct { func (m *AuthenticateRequest) Reset() { *m = AuthenticateRequest{} } func (m *AuthenticateRequest) String() string { return proto.CompactTextString(m) } func (*AuthenticateRequest) ProtoMessage() {} -func (*AuthenticateRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{59} } +func (*AuthenticateRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{61} } func (m *AuthenticateRequest) GetName() string { if m != nil { @@ -2688,7 +2757,7 @@ type AuthUserAddRequest struct { func (m *AuthUserAddRequest) Reset() { *m = AuthUserAddRequest{} } func (m *AuthUserAddRequest) String() string { return proto.CompactTextString(m) } func (*AuthUserAddRequest) ProtoMessage() {} -func (*AuthUserAddRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{60} } +func (*AuthUserAddRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{62} } func (m *AuthUserAddRequest) GetName() string { if m != nil { @@ -2711,7 +2780,7 @@ type AuthUserGetRequest struct { func (m *AuthUserGetRequest) Reset() { *m = AuthUserGetRequest{} } func (m *AuthUserGetRequest) String() string { return proto.CompactTextString(m) } func (*AuthUserGetRequest) ProtoMessage() {} -func (*AuthUserGetRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{61} } +func (*AuthUserGetRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{63} } func (m *AuthUserGetRequest) GetName() string { if m != nil { @@ -2728,7 +2797,7 @@ type AuthUserDeleteRequest struct { func (m *AuthUserDeleteRequest) Reset() { *m = AuthUserDeleteRequest{} } func (m *AuthUserDeleteRequest) String() string { return proto.CompactTextString(m) } func (*AuthUserDeleteRequest) ProtoMessage() {} -func (*AuthUserDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{62} } +func (*AuthUserDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{64} } func (m *AuthUserDeleteRequest) GetName() string { if m != nil { @@ -2748,7 +2817,7 @@ func (m *AuthUserChangePasswordRequest) Reset() { *m = AuthUserChangePas func (m *AuthUserChangePasswordRequest) String() string { return proto.CompactTextString(m) } func (*AuthUserChangePasswordRequest) ProtoMessage() {} func (*AuthUserChangePasswordRequest) Descriptor() ([]byte, []int) { - return fileDescriptorRpc, []int{63} + return fileDescriptorRpc, []int{65} } func (m *AuthUserChangePasswordRequest) GetName() string { @@ -2775,7 +2844,7 @@ type AuthUserGrantRoleRequest struct { func (m *AuthUserGrantRoleRequest) Reset() { *m = AuthUserGrantRoleRequest{} } func (m *AuthUserGrantRoleRequest) String() string { return proto.CompactTextString(m) } func (*AuthUserGrantRoleRequest) ProtoMessage() {} -func (*AuthUserGrantRoleRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{64} } +func (*AuthUserGrantRoleRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{66} } func (m *AuthUserGrantRoleRequest) GetUser() string { if m != nil { @@ -2799,7 +2868,7 @@ type AuthUserRevokeRoleRequest struct { func (m *AuthUserRevokeRoleRequest) Reset() { *m = AuthUserRevokeRoleRequest{} } func (m *AuthUserRevokeRoleRequest) String() string { return proto.CompactTextString(m) } func (*AuthUserRevokeRoleRequest) ProtoMessage() {} -func (*AuthUserRevokeRoleRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{65} } +func (*AuthUserRevokeRoleRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{67} } func (m *AuthUserRevokeRoleRequest) GetName() string { if m != nil { @@ -2823,7 +2892,7 @@ type AuthRoleAddRequest struct { func (m *AuthRoleAddRequest) Reset() { *m = AuthRoleAddRequest{} } func (m *AuthRoleAddRequest) String() string { return proto.CompactTextString(m) } func (*AuthRoleAddRequest) ProtoMessage() {} -func (*AuthRoleAddRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{66} } +func (*AuthRoleAddRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{68} } func (m *AuthRoleAddRequest) GetName() string { if m != nil { @@ -2839,7 +2908,7 @@ type AuthRoleGetRequest struct { func (m *AuthRoleGetRequest) Reset() { *m = AuthRoleGetRequest{} } func (m *AuthRoleGetRequest) String() string { return proto.CompactTextString(m) } func (*AuthRoleGetRequest) ProtoMessage() {} -func (*AuthRoleGetRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{67} } +func (*AuthRoleGetRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{69} } func (m *AuthRoleGetRequest) GetRole() string { if m != nil { @@ -2854,7 +2923,7 @@ type AuthUserListRequest struct { func (m *AuthUserListRequest) Reset() { *m = AuthUserListRequest{} } func (m *AuthUserListRequest) String() string { return proto.CompactTextString(m) } func (*AuthUserListRequest) ProtoMessage() {} -func (*AuthUserListRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{68} } +func (*AuthUserListRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{70} } type AuthRoleListRequest struct { } @@ -2862,7 +2931,7 @@ type AuthRoleListRequest struct { func (m *AuthRoleListRequest) Reset() { *m = AuthRoleListRequest{} } func (m *AuthRoleListRequest) String() string { return proto.CompactTextString(m) } func (*AuthRoleListRequest) ProtoMessage() {} -func (*AuthRoleListRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{69} } +func (*AuthRoleListRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{71} } type AuthRoleDeleteRequest struct { Role string `protobuf:"bytes,1,opt,name=role,proto3" json:"role,omitempty"` @@ -2871,7 +2940,7 @@ type AuthRoleDeleteRequest struct { func (m *AuthRoleDeleteRequest) Reset() { *m = AuthRoleDeleteRequest{} } func (m *AuthRoleDeleteRequest) String() string { return proto.CompactTextString(m) } func (*AuthRoleDeleteRequest) ProtoMessage() {} -func (*AuthRoleDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{70} } +func (*AuthRoleDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{72} } func (m *AuthRoleDeleteRequest) GetRole() string { if m != nil { @@ -2891,7 +2960,7 @@ func (m *AuthRoleGrantPermissionRequest) Reset() { *m = AuthRoleGrantPer func (m *AuthRoleGrantPermissionRequest) String() string { return proto.CompactTextString(m) } func (*AuthRoleGrantPermissionRequest) ProtoMessage() {} func (*AuthRoleGrantPermissionRequest) Descriptor() ([]byte, []int) { - return fileDescriptorRpc, []int{71} + return fileDescriptorRpc, []int{73} } func (m *AuthRoleGrantPermissionRequest) GetName() string { @@ -2918,7 +2987,7 @@ func (m *AuthRoleRevokePermissionRequest) Reset() { *m = AuthRoleRevokeP func (m *AuthRoleRevokePermissionRequest) String() string { return proto.CompactTextString(m) } func (*AuthRoleRevokePermissionRequest) ProtoMessage() {} func (*AuthRoleRevokePermissionRequest) Descriptor() ([]byte, []int) { - return fileDescriptorRpc, []int{72} + return fileDescriptorRpc, []int{74} } func (m *AuthRoleRevokePermissionRequest) GetRole() string { @@ -2949,7 +3018,7 @@ type AuthEnableResponse struct { func (m *AuthEnableResponse) Reset() { *m = AuthEnableResponse{} } func (m *AuthEnableResponse) String() string { return proto.CompactTextString(m) } func (*AuthEnableResponse) ProtoMessage() {} -func (*AuthEnableResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{73} } +func (*AuthEnableResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{75} } func (m *AuthEnableResponse) GetHeader() *ResponseHeader { if m != nil { @@ -2965,7 +3034,7 @@ type AuthDisableResponse struct { func (m *AuthDisableResponse) Reset() { *m = AuthDisableResponse{} } func (m *AuthDisableResponse) String() string { return proto.CompactTextString(m) } func (*AuthDisableResponse) ProtoMessage() {} -func (*AuthDisableResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{74} } +func (*AuthDisableResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{76} } func (m *AuthDisableResponse) GetHeader() *ResponseHeader { if m != nil { @@ -2983,7 +3052,7 @@ type AuthenticateResponse struct { func (m *AuthenticateResponse) Reset() { *m = AuthenticateResponse{} } func (m *AuthenticateResponse) String() string { return proto.CompactTextString(m) } func (*AuthenticateResponse) ProtoMessage() {} -func (*AuthenticateResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{75} } +func (*AuthenticateResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{77} } func (m *AuthenticateResponse) GetHeader() *ResponseHeader { if m != nil { @@ -3006,7 +3075,7 @@ type AuthUserAddResponse struct { func (m *AuthUserAddResponse) Reset() { *m = AuthUserAddResponse{} } func (m *AuthUserAddResponse) String() string { return proto.CompactTextString(m) } func (*AuthUserAddResponse) ProtoMessage() {} -func (*AuthUserAddResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{76} } +func (*AuthUserAddResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{78} } func (m *AuthUserAddResponse) GetHeader() *ResponseHeader { if m != nil { @@ -3023,7 +3092,7 @@ type AuthUserGetResponse struct { func (m *AuthUserGetResponse) Reset() { *m = AuthUserGetResponse{} } func (m *AuthUserGetResponse) String() string { return proto.CompactTextString(m) } func (*AuthUserGetResponse) ProtoMessage() {} -func (*AuthUserGetResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{77} } +func (*AuthUserGetResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{79} } func (m *AuthUserGetResponse) GetHeader() *ResponseHeader { if m != nil { @@ -3046,7 +3115,7 @@ type AuthUserDeleteResponse struct { func (m *AuthUserDeleteResponse) Reset() { *m = AuthUserDeleteResponse{} } func (m *AuthUserDeleteResponse) String() string { return proto.CompactTextString(m) } func (*AuthUserDeleteResponse) ProtoMessage() {} -func (*AuthUserDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{78} } +func (*AuthUserDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{80} } func (m *AuthUserDeleteResponse) GetHeader() *ResponseHeader { if m != nil { @@ -3063,7 +3132,7 @@ func (m *AuthUserChangePasswordResponse) Reset() { *m = AuthUserChangePa func (m *AuthUserChangePasswordResponse) String() string { return proto.CompactTextString(m) } func (*AuthUserChangePasswordResponse) ProtoMessage() {} func (*AuthUserChangePasswordResponse) Descriptor() ([]byte, []int) { - return fileDescriptorRpc, []int{79} + return fileDescriptorRpc, []int{81} } func (m *AuthUserChangePasswordResponse) GetHeader() *ResponseHeader { @@ -3080,7 +3149,7 @@ type AuthUserGrantRoleResponse struct { func (m *AuthUserGrantRoleResponse) Reset() { *m = AuthUserGrantRoleResponse{} } func (m *AuthUserGrantRoleResponse) String() string { return proto.CompactTextString(m) } func (*AuthUserGrantRoleResponse) ProtoMessage() {} -func (*AuthUserGrantRoleResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{80} } +func (*AuthUserGrantRoleResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{82} } func (m *AuthUserGrantRoleResponse) GetHeader() *ResponseHeader { if m != nil { @@ -3096,7 +3165,7 @@ type AuthUserRevokeRoleResponse struct { func (m *AuthUserRevokeRoleResponse) Reset() { *m = AuthUserRevokeRoleResponse{} } func (m *AuthUserRevokeRoleResponse) String() string { return proto.CompactTextString(m) } func (*AuthUserRevokeRoleResponse) ProtoMessage() {} -func (*AuthUserRevokeRoleResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{81} } +func (*AuthUserRevokeRoleResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{83} } func (m *AuthUserRevokeRoleResponse) GetHeader() *ResponseHeader { if m != nil { @@ -3112,7 +3181,7 @@ type AuthRoleAddResponse struct { func (m *AuthRoleAddResponse) Reset() { *m = AuthRoleAddResponse{} } func (m *AuthRoleAddResponse) String() string { return proto.CompactTextString(m) } func (*AuthRoleAddResponse) ProtoMessage() {} -func (*AuthRoleAddResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{82} } +func (*AuthRoleAddResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{84} } func (m *AuthRoleAddResponse) GetHeader() *ResponseHeader { if m != nil { @@ -3129,7 +3198,7 @@ type AuthRoleGetResponse struct { func (m *AuthRoleGetResponse) Reset() { *m = AuthRoleGetResponse{} } func (m *AuthRoleGetResponse) String() string { return proto.CompactTextString(m) } func (*AuthRoleGetResponse) ProtoMessage() {} -func (*AuthRoleGetResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{83} } +func (*AuthRoleGetResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{85} } func (m *AuthRoleGetResponse) GetHeader() *ResponseHeader { if m != nil { @@ -3153,7 +3222,7 @@ type AuthRoleListResponse struct { func (m *AuthRoleListResponse) Reset() { *m = AuthRoleListResponse{} } func (m *AuthRoleListResponse) String() string { return proto.CompactTextString(m) } func (*AuthRoleListResponse) ProtoMessage() {} -func (*AuthRoleListResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{84} } +func (*AuthRoleListResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{86} } func (m *AuthRoleListResponse) GetHeader() *ResponseHeader { if m != nil { @@ -3177,7 +3246,7 @@ type AuthUserListResponse struct { func (m *AuthUserListResponse) Reset() { *m = AuthUserListResponse{} } func (m *AuthUserListResponse) String() string { return proto.CompactTextString(m) } func (*AuthUserListResponse) ProtoMessage() {} -func (*AuthUserListResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{85} } +func (*AuthUserListResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{87} } func (m *AuthUserListResponse) GetHeader() *ResponseHeader { if m != nil { @@ -3200,7 +3269,7 @@ type AuthRoleDeleteResponse struct { func (m *AuthRoleDeleteResponse) Reset() { *m = AuthRoleDeleteResponse{} } func (m *AuthRoleDeleteResponse) String() string { return proto.CompactTextString(m) } func (*AuthRoleDeleteResponse) ProtoMessage() {} -func (*AuthRoleDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{86} } +func (*AuthRoleDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{88} } func (m *AuthRoleDeleteResponse) GetHeader() *ResponseHeader { if m != nil { @@ -3217,7 +3286,7 @@ func (m *AuthRoleGrantPermissionResponse) Reset() { *m = AuthRoleGrantPe func (m *AuthRoleGrantPermissionResponse) String() string { return proto.CompactTextString(m) } func (*AuthRoleGrantPermissionResponse) ProtoMessage() {} func (*AuthRoleGrantPermissionResponse) Descriptor() ([]byte, []int) { - return fileDescriptorRpc, []int{87} + return fileDescriptorRpc, []int{89} } func (m *AuthRoleGrantPermissionResponse) GetHeader() *ResponseHeader { @@ -3235,7 +3304,7 @@ func (m *AuthRoleRevokePermissionResponse) Reset() { *m = AuthRoleRevoke func (m *AuthRoleRevokePermissionResponse) String() string { return proto.CompactTextString(m) } func (*AuthRoleRevokePermissionResponse) ProtoMessage() {} func (*AuthRoleRevokePermissionResponse) Descriptor() ([]byte, []int) { - return fileDescriptorRpc, []int{88} + return fileDescriptorRpc, []int{90} } func (m *AuthRoleRevokePermissionResponse) GetHeader() *ResponseHeader { @@ -3294,6 +3363,8 @@ func init() { proto.RegisterType((*MemberUpdateResponse)(nil), "etcdserverpb.MemberUpdateResponse") proto.RegisterType((*MemberListRequest)(nil), "etcdserverpb.MemberListRequest") proto.RegisterType((*MemberListResponse)(nil), "etcdserverpb.MemberListResponse") + proto.RegisterType((*MemberPromoteRequest)(nil), "etcdserverpb.MemberPromoteRequest") + proto.RegisterType((*MemberPromoteResponse)(nil), "etcdserverpb.MemberPromoteResponse") proto.RegisterType((*DefragmentRequest)(nil), "etcdserverpb.DefragmentRequest") proto.RegisterType((*DefragmentResponse)(nil), "etcdserverpb.DefragmentResponse") proto.RegisterType((*MoveLeaderRequest)(nil), "etcdserverpb.MoveLeaderRequest") @@ -3938,6 +4009,8 @@ type ClusterClient interface { MemberUpdate(ctx context.Context, in *MemberUpdateRequest, opts ...grpc.CallOption) (*MemberUpdateResponse, error) // MemberList lists all the members in the cluster. MemberList(ctx context.Context, in *MemberListRequest, opts ...grpc.CallOption) (*MemberListResponse, error) + // MemberPromote promotes a member from raft learner (non-voting) to raft voting member. + MemberPromote(ctx context.Context, in *MemberPromoteRequest, opts ...grpc.CallOption) (*MemberPromoteResponse, error) } type clusterClient struct { @@ -3984,6 +4057,15 @@ func (c *clusterClient) MemberList(ctx context.Context, in *MemberListRequest, o return out, nil } +func (c *clusterClient) MemberPromote(ctx context.Context, in *MemberPromoteRequest, opts ...grpc.CallOption) (*MemberPromoteResponse, error) { + out := new(MemberPromoteResponse) + err := grpc.Invoke(ctx, "/etcdserverpb.Cluster/MemberPromote", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // Server API for Cluster service type ClusterServer interface { @@ -3995,6 +4077,8 @@ type ClusterServer interface { MemberUpdate(context.Context, *MemberUpdateRequest) (*MemberUpdateResponse, error) // MemberList lists all the members in the cluster. MemberList(context.Context, *MemberListRequest) (*MemberListResponse, error) + // MemberPromote promotes a member from raft learner (non-voting) to raft voting member. + MemberPromote(context.Context, *MemberPromoteRequest) (*MemberPromoteResponse, error) } func RegisterClusterServer(s *grpc.Server, srv ClusterServer) { @@ -4073,6 +4157,24 @@ func _Cluster_MemberList_Handler(srv interface{}, ctx context.Context, dec func( return interceptor(ctx, in, info, handler) } +func _Cluster_MemberPromote_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(MemberPromoteRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ClusterServer).MemberPromote(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/etcdserverpb.Cluster/MemberPromote", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ClusterServer).MemberPromote(ctx, req.(*MemberPromoteRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _Cluster_serviceDesc = grpc.ServiceDesc{ ServiceName: "etcdserverpb.Cluster", HandlerType: (*ClusterServer)(nil), @@ -4093,6 +4195,10 @@ var _Cluster_serviceDesc = grpc.ServiceDesc{ MethodName: "MemberList", Handler: _Cluster_MemberList_Handler, }, + { + MethodName: "MemberPromote", + Handler: _Cluster_MemberPromote_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "rpc.proto", @@ -6741,6 +6847,16 @@ func (m *Member) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } + if m.IsLearner { + dAtA[i] = 0x28 + i++ + if m.IsLearner { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -6774,6 +6890,16 @@ func (m *MemberAddRequest) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } + if m.IsLearner { + dAtA[i] = 0x10 + i++ + if m.IsLearner { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -7026,6 +7152,69 @@ func (m *MemberListResponse) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *MemberPromoteRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *MemberPromoteRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.ID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.ID)) + } + return i, nil +} + +func (m *MemberPromoteResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *MemberPromoteResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Header != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) + n39, err := m.Header.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n39 + } + if len(m.Members) > 0 { + for _, msg := range m.Members { + dAtA[i] = 0x12 + i++ + i = encodeVarintRpc(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + func (m *DefragmentRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -7063,11 +7252,11 @@ func (m *DefragmentResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n39, err := m.Header.MarshalTo(dAtA[i:]) + n40, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n39 + i += n40 } return i, nil } @@ -7114,11 +7303,11 @@ func (m *MoveLeaderResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n40, err := m.Header.MarshalTo(dAtA[i:]) + n41, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n40 + i += n41 } return i, nil } @@ -7203,11 +7392,11 @@ func (m *AlarmResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n41, err := m.Header.MarshalTo(dAtA[i:]) + n42, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n41 + i += n42 } if len(m.Alarms) > 0 { for _, msg := range m.Alarms { @@ -7261,11 +7450,11 @@ func (m *StatusResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n42, err := m.Header.MarshalTo(dAtA[i:]) + n43, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n42 + i += n43 } if len(m.Version) > 0 { dAtA[i] = 0x12 @@ -7318,6 +7507,16 @@ func (m *StatusResponse) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintRpc(dAtA, i, uint64(m.DbSizeInUse)) } + if m.IsLearner { + dAtA[i] = 0x50 + i++ + if m.IsLearner { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -7688,11 +7887,11 @@ func (m *AuthRoleGrantPermissionRequest) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintRpc(dAtA, i, uint64(m.Perm.Size())) - n43, err := m.Perm.MarshalTo(dAtA[i:]) + n44, err := m.Perm.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n43 + i += n44 } return i, nil } @@ -7752,11 +7951,11 @@ func (m *AuthEnableResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n44, err := m.Header.MarshalTo(dAtA[i:]) + n45, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n44 + i += n45 } return i, nil } @@ -7780,11 +7979,11 @@ func (m *AuthDisableResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n45, err := m.Header.MarshalTo(dAtA[i:]) + n46, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n45 + i += n46 } return i, nil } @@ -7808,11 +8007,11 @@ func (m *AuthenticateResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n46, err := m.Header.MarshalTo(dAtA[i:]) + n47, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n46 + i += n47 } if len(m.Token) > 0 { dAtA[i] = 0x12 @@ -7842,11 +8041,11 @@ func (m *AuthUserAddResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n47, err := m.Header.MarshalTo(dAtA[i:]) + n48, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n47 + i += n48 } return i, nil } @@ -7870,11 +8069,11 @@ func (m *AuthUserGetResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n48, err := m.Header.MarshalTo(dAtA[i:]) + n49, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n48 + i += n49 } if len(m.Roles) > 0 { for _, s := range m.Roles { @@ -7913,11 +8112,11 @@ func (m *AuthUserDeleteResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n49, err := m.Header.MarshalTo(dAtA[i:]) + n50, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n49 + i += n50 } return i, nil } @@ -7941,11 +8140,11 @@ func (m *AuthUserChangePasswordResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n50, err := m.Header.MarshalTo(dAtA[i:]) + n51, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n50 + i += n51 } return i, nil } @@ -7969,11 +8168,11 @@ func (m *AuthUserGrantRoleResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n51, err := m.Header.MarshalTo(dAtA[i:]) + n52, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n51 + i += n52 } return i, nil } @@ -7997,11 +8196,11 @@ func (m *AuthUserRevokeRoleResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n52, err := m.Header.MarshalTo(dAtA[i:]) + n53, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n52 + i += n53 } return i, nil } @@ -8025,11 +8224,11 @@ func (m *AuthRoleAddResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n53, err := m.Header.MarshalTo(dAtA[i:]) + n54, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n53 + i += n54 } return i, nil } @@ -8053,11 +8252,11 @@ func (m *AuthRoleGetResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n54, err := m.Header.MarshalTo(dAtA[i:]) + n55, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n54 + i += n55 } if len(m.Perm) > 0 { for _, msg := range m.Perm { @@ -8093,11 +8292,11 @@ func (m *AuthRoleListResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n55, err := m.Header.MarshalTo(dAtA[i:]) + n56, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n55 + i += n56 } if len(m.Roles) > 0 { for _, s := range m.Roles { @@ -8136,11 +8335,11 @@ func (m *AuthUserListResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n56, err := m.Header.MarshalTo(dAtA[i:]) + n57, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n56 + i += n57 } if len(m.Users) > 0 { for _, s := range m.Users { @@ -8179,11 +8378,11 @@ func (m *AuthRoleDeleteResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n57, err := m.Header.MarshalTo(dAtA[i:]) + n58, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n57 + i += n58 } return i, nil } @@ -8207,11 +8406,11 @@ func (m *AuthRoleGrantPermissionResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n58, err := m.Header.MarshalTo(dAtA[i:]) + n59, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n58 + i += n59 } return i, nil } @@ -8235,11 +8434,11 @@ func (m *AuthRoleRevokePermissionResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintRpc(dAtA, i, uint64(m.Header.Size())) - n59, err := m.Header.MarshalTo(dAtA[i:]) + n60, err := m.Header.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n59 + i += n60 } return i, nil } @@ -9016,6 +9215,9 @@ func (m *Member) Size() (n int) { n += 1 + l + sovRpc(uint64(l)) } } + if m.IsLearner { + n += 2 + } return n } @@ -9028,6 +9230,9 @@ func (m *MemberAddRequest) Size() (n int) { n += 1 + l + sovRpc(uint64(l)) } } + if m.IsLearner { + n += 2 + } return n } @@ -9129,6 +9334,31 @@ func (m *MemberListResponse) Size() (n int) { return n } +func (m *MemberPromoteRequest) Size() (n int) { + var l int + _ = l + if m.ID != 0 { + n += 1 + sovRpc(uint64(m.ID)) + } + return n +} + +func (m *MemberPromoteResponse) Size() (n int) { + var l int + _ = l + if m.Header != nil { + l = m.Header.Size() + n += 1 + l + sovRpc(uint64(l)) + } + if len(m.Members) > 0 { + for _, e := range m.Members { + l = e.Size() + n += 1 + l + sovRpc(uint64(l)) + } + } + return n +} + func (m *DefragmentRequest) Size() (n int) { var l int _ = l @@ -9248,6 +9478,9 @@ func (m *StatusResponse) Size() (n int) { if m.DbSizeInUse != 0 { n += 1 + sovRpc(uint64(m.DbSizeInUse)) } + if m.IsLearner { + n += 2 + } return n } @@ -14629,6 +14862,26 @@ func (m *Member) Unmarshal(dAtA []byte) error { } m.ClientURLs = append(m.ClientURLs, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IsLearner", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.IsLearner = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) @@ -14708,6 +14961,26 @@ func (m *MemberAddRequest) Unmarshal(dAtA []byte) error { } m.PeerURLs = append(m.PeerURLs, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IsLearner", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.IsLearner = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) @@ -15435,6 +15708,189 @@ func (m *MemberListResponse) Unmarshal(dAtA []byte) error { } return nil } +func (m *MemberPromoteRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: MemberPromoteRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: MemberPromoteRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) + } + m.ID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipRpc(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRpc + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *MemberPromoteResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: MemberPromoteResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: MemberPromoteResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Header == nil { + m.Header = &ResponseHeader{} + } + if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Members", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Members = append(m.Members, &Member{}) + if err := m.Members[len(m.Members)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRpc(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRpc + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *DefragmentRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -16313,6 +16769,26 @@ func (m *StatusResponse) Unmarshal(dAtA []byte) error { break } } + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IsLearner", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.IsLearner = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) @@ -19305,245 +19781,250 @@ var ( func init() { proto.RegisterFile("rpc.proto", fileDescriptorRpc) } var fileDescriptorRpc = []byte{ - // 3836 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x5b, 0xdd, 0x6f, 0x23, 0xc9, - 0x71, 0xd7, 0x90, 0xe2, 0x57, 0xf1, 0x43, 0x54, 0xeb, 0x63, 0x29, 0xee, 0xae, 0x56, 0xd7, 0xbb, - 0x7b, 0xab, 0xdb, 0xbd, 0x13, 0x6d, 0xd9, 0x4e, 0x80, 0x4d, 0xe2, 0x58, 0x2b, 0xf1, 0x56, 0x3a, - 0x69, 0x45, 0xdd, 0x88, 0xda, 0xfb, 0x80, 0x11, 0x61, 0x44, 0xf6, 0x4a, 0x13, 0x91, 0x33, 0xf4, - 0xcc, 0x90, 0x2b, 0x5d, 0x82, 0x38, 0x30, 0x9c, 0x00, 0xc9, 0xa3, 0x0d, 0x04, 0xc9, 0x43, 0x9e, - 0x82, 0x20, 0xf0, 0x43, 0x80, 0xbc, 0x05, 0xc8, 0x5f, 0x90, 0xb7, 0x24, 0xc8, 0x3f, 0x10, 0x5c, - 0xfc, 0x92, 0xff, 0x22, 0xe8, 0xaf, 0x99, 0x9e, 0x2f, 0x69, 0x6d, 0xfa, 0xfc, 0x22, 0x4d, 0x57, - 0x57, 0x57, 0x55, 0x57, 0x77, 0x57, 0x55, 0xff, 0x66, 0x08, 0x25, 0x67, 0xd4, 0xdb, 0x18, 0x39, - 0xb6, 0x67, 0xa3, 0x0a, 0xf1, 0x7a, 0x7d, 0x97, 0x38, 0x13, 0xe2, 0x8c, 0xce, 0x9a, 0x8b, 0xe7, - 0xf6, 0xb9, 0xcd, 0x3a, 0x5a, 0xf4, 0x89, 0xf3, 0x34, 0x57, 0x28, 0x4f, 0x6b, 0x38, 0xe9, 0xf5, - 0xd8, 0x9f, 0xd1, 0x59, 0xeb, 0x72, 0x22, 0xba, 0xee, 0xb2, 0x2e, 0x63, 0xec, 0x5d, 0xb0, 0x3f, - 0xa3, 0x33, 0xf6, 0x4f, 0x74, 0xde, 0x3b, 0xb7, 0xed, 0xf3, 0x01, 0x69, 0x19, 0x23, 0xb3, 0x65, - 0x58, 0x96, 0xed, 0x19, 0x9e, 0x69, 0x5b, 0x2e, 0xef, 0xc5, 0x7f, 0xa1, 0x41, 0x4d, 0x27, 0xee, - 0xc8, 0xb6, 0x5c, 0xb2, 0x4b, 0x8c, 0x3e, 0x71, 0xd0, 0x7d, 0x80, 0xde, 0x60, 0xec, 0x7a, 0xc4, - 0x39, 0x35, 0xfb, 0x0d, 0x6d, 0x4d, 0x5b, 0x9f, 0xd5, 0x4b, 0x82, 0xb2, 0xd7, 0x47, 0x77, 0xa1, - 0x34, 0x24, 0xc3, 0x33, 0xde, 0x9b, 0x61, 0xbd, 0x45, 0x4e, 0xd8, 0xeb, 0xa3, 0x26, 0x14, 0x1d, - 0x32, 0x31, 0x5d, 0xd3, 0xb6, 0x1a, 0xd9, 0x35, 0x6d, 0x3d, 0xab, 0xfb, 0x6d, 0x3a, 0xd0, 0x31, - 0xde, 0x78, 0xa7, 0x1e, 0x71, 0x86, 0x8d, 0x59, 0x3e, 0x90, 0x12, 0xba, 0xc4, 0x19, 0xe2, 0x9f, + // 3907 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x5b, 0x5b, 0x6f, 0x23, 0xc9, + 0x75, 0x56, 0x93, 0xe2, 0xed, 0xf0, 0x22, 0xaa, 0x24, 0xcd, 0x70, 0x38, 0x33, 0x1a, 0x6d, 0xcd, + 0xce, 0xae, 0x76, 0x66, 0x57, 0xb4, 0x65, 0x3b, 0x01, 0x26, 0x89, 0x63, 0x8d, 0xc4, 0x9d, 0xd1, + 0x4a, 0x23, 0x6a, 0x5b, 0xd4, 0xec, 0x05, 0x46, 0x84, 0x16, 0x59, 0x92, 0x3a, 0x22, 0xbb, 0xe9, + 0xee, 0x26, 0x47, 0xda, 0x5c, 0x1c, 0x18, 0x8e, 0x81, 0xe4, 0xd1, 0x06, 0x82, 0xe4, 0x21, 0x4f, + 0x41, 0x10, 0xf8, 0x21, 0xcf, 0x01, 0xf2, 0x0b, 0xf2, 0x94, 0x0b, 0xf2, 0x07, 0x82, 0x8d, 0x5f, + 0x92, 0x5f, 0x61, 0xd4, 0xad, 0xbb, 0xfa, 0x46, 0x8d, 0xcd, 0xdd, 0x7d, 0x91, 0xba, 0x4e, 0x9d, + 0x3a, 0xe7, 0xd4, 0xa9, 0xaa, 0x73, 0x4e, 0x7f, 0x5d, 0x84, 0x92, 0x33, 0xea, 0x6d, 0x8c, 0x1c, + 0xdb, 0xb3, 0x51, 0x85, 0x78, 0xbd, 0xbe, 0x4b, 0x9c, 0x09, 0x71, 0x46, 0xa7, 0xcd, 0xe5, 0x73, + 0xfb, 0xdc, 0x66, 0x1d, 0x2d, 0xfa, 0xc4, 0x79, 0x9a, 0x77, 0x28, 0x4f, 0x6b, 0x38, 0xe9, 0xf5, + 0xd8, 0x9f, 0xd1, 0x69, 0xeb, 0x72, 0x22, 0xba, 0xee, 0xb2, 0x2e, 0x63, 0xec, 0x5d, 0xb0, 0x3f, + 0xa3, 0x53, 0xf6, 0x4f, 0x74, 0xde, 0x3b, 0xb7, 0xed, 0xf3, 0x01, 0x69, 0x19, 0x23, 0xb3, 0x65, + 0x58, 0x96, 0xed, 0x19, 0x9e, 0x69, 0x5b, 0x2e, 0xef, 0xc5, 0x7f, 0xa9, 0x41, 0x4d, 0x27, 0xee, + 0xc8, 0xb6, 0x5c, 0xf2, 0x82, 0x18, 0x7d, 0xe2, 0xa0, 0xfb, 0x00, 0xbd, 0xc1, 0xd8, 0xf5, 0x88, + 0x73, 0x62, 0xf6, 0x1b, 0xda, 0x9a, 0xb6, 0x3e, 0xaf, 0x97, 0x04, 0x65, 0xb7, 0x8f, 0xee, 0x42, + 0x69, 0x48, 0x86, 0xa7, 0xbc, 0x37, 0xc3, 0x7a, 0x8b, 0x9c, 0xb0, 0xdb, 0x47, 0x4d, 0x28, 0x3a, + 0x64, 0x62, 0xba, 0xa6, 0x6d, 0x35, 0xb2, 0x6b, 0xda, 0x7a, 0x56, 0xf7, 0xdb, 0x74, 0xa0, 0x63, + 0x9c, 0x79, 0x27, 0x1e, 0x71, 0x86, 0x8d, 0x79, 0x3e, 0x90, 0x12, 0xba, 0xc4, 0x19, 0xe2, 0x9f, 0xe6, 0xa0, 0xa2, 0x1b, 0xd6, 0x39, 0xd1, 0xc9, 0x8f, 0xc6, 0xc4, 0xf5, 0x50, 0x1d, 0xb2, 0x97, - 0xe4, 0x9a, 0xa9, 0xaf, 0xe8, 0xf4, 0x91, 0x8f, 0xb7, 0xce, 0xc9, 0x29, 0xb1, 0xb8, 0xe2, 0x0a, - 0x1d, 0x6f, 0x9d, 0x93, 0xb6, 0xd5, 0x47, 0x8b, 0x90, 0x1b, 0x98, 0x43, 0xd3, 0x13, 0x5a, 0x79, - 0x23, 0x64, 0xce, 0x6c, 0xc4, 0x9c, 0x6d, 0x00, 0xd7, 0x76, 0xbc, 0x53, 0xdb, 0xe9, 0x13, 0xa7, - 0x91, 0x5b, 0xd3, 0xd6, 0x6b, 0x9b, 0x8f, 0x36, 0xd4, 0x85, 0xd8, 0x50, 0x0d, 0xda, 0x38, 0xb6, - 0x1d, 0xaf, 0x43, 0x79, 0xf5, 0x92, 0x2b, 0x1f, 0xd1, 0xc7, 0x50, 0x66, 0x42, 0x3c, 0xc3, 0x39, - 0x27, 0x5e, 0x23, 0xcf, 0xa4, 0x3c, 0xbe, 0x45, 0x4a, 0x97, 0x31, 0xeb, 0x4c, 0x3d, 0x7f, 0x46, - 0x18, 0x2a, 0x2e, 0x71, 0x4c, 0x63, 0x60, 0x7e, 0x65, 0x9c, 0x0d, 0x48, 0xa3, 0xb0, 0xa6, 0xad, - 0x17, 0xf5, 0x10, 0x8d, 0xce, 0xff, 0x92, 0x5c, 0xbb, 0xa7, 0xb6, 0x35, 0xb8, 0x6e, 0x14, 0x19, - 0x43, 0x91, 0x12, 0x3a, 0xd6, 0xe0, 0x9a, 0x2d, 0x9a, 0x3d, 0xb6, 0x3c, 0xde, 0x5b, 0x62, 0xbd, - 0x25, 0x46, 0x61, 0xdd, 0xeb, 0x50, 0x1f, 0x9a, 0xd6, 0xe9, 0xd0, 0xee, 0x9f, 0xfa, 0x0e, 0x01, - 0xe6, 0x90, 0xda, 0xd0, 0xb4, 0x5e, 0xd9, 0x7d, 0x5d, 0xba, 0x85, 0x72, 0x1a, 0x57, 0x61, 0xce, - 0xb2, 0xe0, 0x34, 0xae, 0x54, 0xce, 0x0d, 0x58, 0xa0, 0x32, 0x7b, 0x0e, 0x31, 0x3c, 0x12, 0x30, - 0x57, 0x18, 0xf3, 0xfc, 0xd0, 0xb4, 0xb6, 0x59, 0x4f, 0x88, 0xdf, 0xb8, 0x8a, 0xf1, 0x57, 0x05, - 0xbf, 0x71, 0x15, 0xe6, 0xc7, 0x1b, 0x50, 0xf2, 0x7d, 0x8e, 0x8a, 0x30, 0x7b, 0xd8, 0x39, 0x6c, - 0xd7, 0x67, 0x10, 0x40, 0x7e, 0xeb, 0x78, 0xbb, 0x7d, 0xb8, 0x53, 0xd7, 0x50, 0x19, 0x0a, 0x3b, - 0x6d, 0xde, 0xc8, 0xe0, 0x17, 0x00, 0x81, 0x77, 0x51, 0x01, 0xb2, 0xfb, 0xed, 0x2f, 0xea, 0x33, - 0x94, 0xe7, 0x75, 0x5b, 0x3f, 0xde, 0xeb, 0x1c, 0xd6, 0x35, 0x3a, 0x78, 0x5b, 0x6f, 0x6f, 0x75, - 0xdb, 0xf5, 0x0c, 0xe5, 0x78, 0xd5, 0xd9, 0xa9, 0x67, 0x51, 0x09, 0x72, 0xaf, 0xb7, 0x0e, 0x4e, - 0xda, 0xf5, 0x59, 0xfc, 0x73, 0x0d, 0xaa, 0x62, 0xbd, 0xf8, 0x99, 0x40, 0xdf, 0x85, 0xfc, 0x05, - 0x3b, 0x17, 0x6c, 0x2b, 0x96, 0x37, 0xef, 0x45, 0x16, 0x37, 0x74, 0x76, 0x74, 0xc1, 0x8b, 0x30, - 0x64, 0x2f, 0x27, 0x6e, 0x23, 0xb3, 0x96, 0x5d, 0x2f, 0x6f, 0xd6, 0x37, 0xf8, 0x81, 0xdd, 0xd8, - 0x27, 0xd7, 0xaf, 0x8d, 0xc1, 0x98, 0xe8, 0xb4, 0x13, 0x21, 0x98, 0x1d, 0xda, 0x0e, 0x61, 0x3b, - 0xb6, 0xa8, 0xb3, 0x67, 0xba, 0x8d, 0xd9, 0xa2, 0x89, 0xdd, 0xca, 0x1b, 0xf8, 0x17, 0x1a, 0xc0, - 0xd1, 0xd8, 0x4b, 0x3f, 0x1a, 0x8b, 0x90, 0x9b, 0x50, 0xc1, 0xe2, 0x58, 0xf0, 0x06, 0x3b, 0x13, - 0xc4, 0x70, 0x89, 0x7f, 0x26, 0x68, 0x03, 0xdd, 0x81, 0xc2, 0xc8, 0x21, 0x93, 0xd3, 0xcb, 0x09, - 0x53, 0x52, 0xd4, 0xf3, 0xb4, 0xb9, 0x3f, 0x41, 0xef, 0x41, 0xc5, 0x3c, 0xb7, 0x6c, 0x87, 0x9c, - 0x72, 0x59, 0x39, 0xd6, 0x5b, 0xe6, 0x34, 0x66, 0xb7, 0xc2, 0xc2, 0x05, 0xe7, 0x55, 0x96, 0x03, - 0x4a, 0xc2, 0x16, 0x94, 0x99, 0xa9, 0x53, 0xb9, 0xef, 0x83, 0xc0, 0xc6, 0x0c, 0x1b, 0x16, 0x77, - 0xa1, 0xb0, 0x1a, 0xff, 0x10, 0xd0, 0x0e, 0x19, 0x10, 0x8f, 0x4c, 0x13, 0x3d, 0x14, 0x9f, 0x64, - 0x55, 0x9f, 0xe0, 0x9f, 0x69, 0xb0, 0x10, 0x12, 0x3f, 0xd5, 0xb4, 0x1a, 0x50, 0xe8, 0x33, 0x61, - 0xdc, 0x82, 0xac, 0x2e, 0x9b, 0xe8, 0x19, 0x14, 0x85, 0x01, 0x6e, 0x23, 0x9b, 0xb2, 0x69, 0x0a, - 0xdc, 0x26, 0x17, 0xff, 0x22, 0x03, 0x25, 0x31, 0xd1, 0xce, 0x08, 0x6d, 0x41, 0xd5, 0xe1, 0x8d, - 0x53, 0x36, 0x1f, 0x61, 0x51, 0x33, 0x3d, 0x08, 0xed, 0xce, 0xe8, 0x15, 0x31, 0x84, 0x91, 0xd1, - 0xef, 0x41, 0x59, 0x8a, 0x18, 0x8d, 0x3d, 0xe1, 0xf2, 0x46, 0x58, 0x40, 0xb0, 0xff, 0x76, 0x67, - 0x74, 0x10, 0xec, 0x47, 0x63, 0x0f, 0x75, 0x61, 0x51, 0x0e, 0xe6, 0xb3, 0x11, 0x66, 0x64, 0x99, - 0x94, 0xb5, 0xb0, 0x94, 0xf8, 0x52, 0xed, 0xce, 0xe8, 0x48, 0x8c, 0x57, 0x3a, 0x55, 0x93, 0xbc, - 0x2b, 0x1e, 0xbc, 0x63, 0x26, 0x75, 0xaf, 0xac, 0xb8, 0x49, 0xdd, 0x2b, 0xeb, 0x45, 0x09, 0x0a, - 0xa2, 0x85, 0xff, 0x35, 0x03, 0x20, 0x57, 0xa3, 0x33, 0x42, 0x3b, 0x50, 0x73, 0x44, 0x2b, 0xe4, - 0xad, 0xbb, 0x89, 0xde, 0x12, 0x8b, 0x38, 0xa3, 0x57, 0xe5, 0x20, 0x6e, 0xdc, 0xf7, 0xa1, 0xe2, - 0x4b, 0x09, 0x1c, 0xb6, 0x92, 0xe0, 0x30, 0x5f, 0x42, 0x59, 0x0e, 0xa0, 0x2e, 0xfb, 0x0c, 0x96, - 0xfc, 0xf1, 0x09, 0x3e, 0x7b, 0xef, 0x06, 0x9f, 0xf9, 0x02, 0x17, 0xa4, 0x04, 0xd5, 0x6b, 0xaa, - 0x61, 0x81, 0xdb, 0x56, 0x12, 0xdc, 0x16, 0x37, 0x8c, 0x3a, 0x0e, 0x68, 0xbe, 0xe4, 0x4d, 0xfc, - 0x7f, 0x59, 0x28, 0x6c, 0xdb, 0xc3, 0x91, 0xe1, 0xd0, 0xd5, 0xc8, 0x3b, 0xc4, 0x1d, 0x0f, 0x3c, - 0xe6, 0xae, 0xda, 0xe6, 0xc3, 0xb0, 0x44, 0xc1, 0x26, 0xff, 0xeb, 0x8c, 0x55, 0x17, 0x43, 0xe8, - 0x60, 0x91, 0x1e, 0x33, 0xef, 0x30, 0x58, 0x24, 0x47, 0x31, 0x44, 0x1e, 0xe4, 0x6c, 0x70, 0x90, - 0x9b, 0x50, 0x98, 0x10, 0x27, 0x48, 0xe9, 0xbb, 0x33, 0xba, 0x24, 0xa0, 0x0f, 0x60, 0x2e, 0x9a, - 0x5e, 0x72, 0x82, 0xa7, 0xd6, 0x0b, 0x67, 0xa3, 0x87, 0x50, 0x09, 0xe5, 0xb8, 0xbc, 0xe0, 0x2b, - 0x0f, 0x95, 0x14, 0xb7, 0x2c, 0xe3, 0x2a, 0xcd, 0xc7, 0x95, 0xdd, 0x19, 0x19, 0x59, 0x97, 0x65, - 0x64, 0x2d, 0x8a, 0x51, 0x22, 0xb6, 0x86, 0x82, 0xcc, 0x0f, 0xc2, 0x41, 0x06, 0xff, 0x00, 0xaa, - 0x21, 0x07, 0xd1, 0xbc, 0xd3, 0xfe, 0xf4, 0x64, 0xeb, 0x80, 0x27, 0xa9, 0x97, 0x2c, 0x2f, 0xe9, - 0x75, 0x8d, 0xe6, 0xba, 0x83, 0xf6, 0xf1, 0x71, 0x3d, 0x83, 0xaa, 0x50, 0x3a, 0xec, 0x74, 0x4f, - 0x39, 0x57, 0x16, 0xbf, 0xf4, 0x25, 0x88, 0x24, 0xa7, 0xe4, 0xb6, 0x19, 0x25, 0xb7, 0x69, 0x32, - 0xb7, 0x65, 0x82, 0xdc, 0xc6, 0xd2, 0xdc, 0x41, 0x7b, 0xeb, 0xb8, 0x5d, 0x9f, 0x7d, 0x51, 0x83, - 0x0a, 0xf7, 0xef, 0xe9, 0xd8, 0xa2, 0xa9, 0xf6, 0x1f, 0x34, 0x80, 0xe0, 0x34, 0xa1, 0x16, 0x14, - 0x7a, 0x5c, 0x4f, 0x43, 0x63, 0xc1, 0x68, 0x29, 0x71, 0xc9, 0x74, 0xc9, 0x85, 0xbe, 0x0d, 0x05, - 0x77, 0xdc, 0xeb, 0x11, 0x57, 0xa6, 0xbc, 0x3b, 0xd1, 0x78, 0x28, 0xa2, 0x95, 0x2e, 0xf9, 0xe8, - 0x90, 0x37, 0x86, 0x39, 0x18, 0xb3, 0x04, 0x78, 0xf3, 0x10, 0xc1, 0x87, 0xff, 0x4e, 0x83, 0xb2, - 0xb2, 0x79, 0x7f, 0xcd, 0x20, 0x7c, 0x0f, 0x4a, 0xcc, 0x06, 0xd2, 0x17, 0x61, 0xb8, 0xa8, 0x07, - 0x04, 0xf4, 0x3b, 0x50, 0x92, 0x27, 0x40, 0x46, 0xe2, 0x46, 0xb2, 0xd8, 0xce, 0x48, 0x0f, 0x58, - 0xf1, 0x3e, 0xcc, 0x33, 0xaf, 0xf4, 0x68, 0x71, 0x2d, 0xfd, 0xa8, 0x96, 0x9f, 0x5a, 0xa4, 0xfc, - 0x6c, 0x42, 0x71, 0x74, 0x71, 0xed, 0x9a, 0x3d, 0x63, 0x20, 0xac, 0xf0, 0xdb, 0xf8, 0x13, 0x40, - 0xaa, 0xb0, 0x69, 0xa6, 0x8b, 0xab, 0x50, 0xde, 0x35, 0xdc, 0x0b, 0x61, 0x12, 0x7e, 0x06, 0x55, - 0xda, 0xdc, 0x7f, 0xfd, 0x0e, 0x36, 0xb2, 0xcb, 0x81, 0xe4, 0x9e, 0xca, 0xe7, 0x08, 0x66, 0x2f, - 0x0c, 0xf7, 0x82, 0x4d, 0xb4, 0xaa, 0xb3, 0x67, 0xf4, 0x01, 0xd4, 0x7b, 0x7c, 0x92, 0xa7, 0x91, - 0x2b, 0xc3, 0x9c, 0xa0, 0xfb, 0x95, 0xe0, 0xe7, 0x50, 0xe1, 0x73, 0xf8, 0x4d, 0x1b, 0x81, 0xe7, - 0x61, 0xee, 0xd8, 0x32, 0x46, 0xee, 0x85, 0x2d, 0xb3, 0x1b, 0x9d, 0x74, 0x3d, 0xa0, 0x4d, 0xa5, - 0xf1, 0x09, 0xcc, 0x39, 0x64, 0x68, 0x98, 0x96, 0x69, 0x9d, 0x9f, 0x9e, 0x5d, 0x7b, 0xc4, 0x15, - 0x17, 0xa6, 0x9a, 0x4f, 0x7e, 0x41, 0xa9, 0xd4, 0xb4, 0xb3, 0x81, 0x7d, 0x26, 0xc2, 0x1c, 0x7b, - 0xc6, 0x7f, 0x99, 0x81, 0xca, 0x67, 0x86, 0xd7, 0x93, 0x4b, 0x87, 0xf6, 0xa0, 0xe6, 0x07, 0x37, - 0x46, 0x11, 0xb6, 0x44, 0x52, 0x2c, 0x1b, 0x23, 0x4b, 0x69, 0x99, 0x1d, 0xab, 0x3d, 0x95, 0xc0, - 0x44, 0x19, 0x56, 0x8f, 0x0c, 0x7c, 0x51, 0x99, 0x74, 0x51, 0x8c, 0x51, 0x15, 0xa5, 0x12, 0x50, - 0x07, 0xea, 0x23, 0xc7, 0x3e, 0x77, 0x88, 0xeb, 0xfa, 0xc2, 0x78, 0x1a, 0xc3, 0x09, 0xc2, 0x8e, - 0x04, 0x6b, 0x20, 0x6e, 0x6e, 0x14, 0x26, 0xbd, 0x98, 0x0b, 0xea, 0x19, 0x1e, 0x9c, 0xfe, 0x2b, - 0x03, 0x28, 0x3e, 0xa9, 0x5f, 0xb5, 0xc4, 0x7b, 0x0c, 0x35, 0xd7, 0x33, 0x9c, 0xd8, 0x66, 0xab, - 0x32, 0xaa, 0x1f, 0xf1, 0x9f, 0x80, 0x6f, 0xd0, 0xa9, 0x65, 0x7b, 0xe6, 0x9b, 0x6b, 0x51, 0x25, - 0xd7, 0x24, 0xf9, 0x90, 0x51, 0x51, 0x1b, 0x0a, 0x6f, 0xcc, 0x81, 0x47, 0x1c, 0xb7, 0x91, 0x5b, - 0xcb, 0xae, 0xd7, 0x36, 0x9f, 0xdd, 0xb6, 0x0c, 0x1b, 0x1f, 0x33, 0xfe, 0xee, 0xf5, 0x88, 0xe8, - 0x72, 0xac, 0x5a, 0x79, 0xe6, 0x43, 0xd5, 0xf8, 0x0a, 0x14, 0xdf, 0x52, 0x11, 0xf4, 0x96, 0x5d, - 0xe0, 0xc5, 0x22, 0x6b, 0xf3, 0x4b, 0xf6, 0x1b, 0xc7, 0x38, 0x1f, 0x12, 0xcb, 0x93, 0xf7, 0x40, - 0xd9, 0xc6, 0x8f, 0x01, 0x02, 0x35, 0x34, 0xe4, 0x1f, 0x76, 0x8e, 0x4e, 0xba, 0xf5, 0x19, 0x54, - 0x81, 0xe2, 0x61, 0x67, 0xa7, 0x7d, 0xd0, 0xa6, 0xf9, 0x01, 0xb7, 0xa4, 0x4b, 0x43, 0x6b, 0xa9, - 0xea, 0xd4, 0x42, 0x3a, 0xf1, 0x32, 0x2c, 0x26, 0x2d, 0x20, 0xad, 0x45, 0xab, 0x62, 0x97, 0x4e, - 0x75, 0x54, 0x54, 0xd5, 0x99, 0xf0, 0x74, 0x1b, 0x50, 0xe0, 0xbb, 0xb7, 0x2f, 0x8a, 0x73, 0xd9, - 0xa4, 0x8e, 0xe0, 0x9b, 0x91, 0xf4, 0xc5, 0x2a, 0xf9, 0xed, 0xc4, 0xf0, 0x92, 0x4b, 0x0c, 0x2f, - 0xe8, 0x21, 0x54, 0xfd, 0xd3, 0x60, 0xb8, 0xa2, 0x16, 0x28, 0xe9, 0x15, 0xb9, 0xd1, 0x29, 0x2d, - 0xe4, 0xf4, 0x42, 0xd8, 0xe9, 0xe8, 0x31, 0xe4, 0xc9, 0x84, 0x58, 0x9e, 0xdb, 0x28, 0xb3, 0x8c, - 0x51, 0x95, 0xb5, 0x7b, 0x9b, 0x52, 0x75, 0xd1, 0x89, 0xbf, 0x07, 0xf3, 0xec, 0x8e, 0xf4, 0xd2, - 0x31, 0x2c, 0xf5, 0x32, 0xd7, 0xed, 0x1e, 0x08, 0x77, 0xd3, 0x47, 0x54, 0x83, 0xcc, 0xde, 0x8e, - 0x70, 0x42, 0x66, 0x6f, 0x07, 0xff, 0x44, 0x03, 0xa4, 0x8e, 0x9b, 0xca, 0xcf, 0x11, 0xe1, 0x52, - 0x7d, 0x36, 0x50, 0xbf, 0x08, 0x39, 0xe2, 0x38, 0xb6, 0xc3, 0x3c, 0x5a, 0xd2, 0x79, 0x03, 0x3f, - 0x12, 0x36, 0xe8, 0x64, 0x62, 0x5f, 0xfa, 0x67, 0x90, 0x4b, 0xd3, 0x7c, 0x53, 0xf7, 0x61, 0x21, - 0xc4, 0x35, 0x55, 0xe6, 0xfa, 0x18, 0xe6, 0x98, 0xb0, 0xed, 0x0b, 0xd2, 0xbb, 0x1c, 0xd9, 0xa6, - 0x15, 0xd3, 0x47, 0x57, 0x2e, 0x08, 0xb0, 0x74, 0x1e, 0x7c, 0x62, 0x15, 0x9f, 0xd8, 0xed, 0x1e, - 0xe0, 0x2f, 0x60, 0x39, 0x22, 0x47, 0x9a, 0xff, 0x87, 0x50, 0xee, 0xf9, 0x44, 0x57, 0xd4, 0x3a, - 0xf7, 0xc3, 0xc6, 0x45, 0x87, 0xaa, 0x23, 0x70, 0x07, 0xee, 0xc4, 0x44, 0x4f, 0x35, 0xe7, 0x27, - 0xb0, 0xc4, 0x04, 0xee, 0x13, 0x32, 0xda, 0x1a, 0x98, 0x93, 0x54, 0x4f, 0x8f, 0xc4, 0xa4, 0x14, - 0xc6, 0x6f, 0x76, 0x5f, 0xe0, 0xdf, 0x17, 0x1a, 0xbb, 0xe6, 0x90, 0x74, 0xed, 0x83, 0x74, 0xdb, - 0x68, 0x36, 0xbb, 0x24, 0xd7, 0xae, 0x28, 0x6b, 0xd8, 0x33, 0xfe, 0x47, 0x4d, 0xb8, 0x4a, 0x1d, - 0xfe, 0x0d, 0xef, 0xe4, 0x55, 0x80, 0x73, 0x7a, 0x64, 0x48, 0x9f, 0x76, 0x70, 0x44, 0x45, 0xa1, - 0xf8, 0x76, 0xd2, 0xf8, 0x5d, 0x11, 0x76, 0x2e, 0x8a, 0x7d, 0xce, 0xfe, 0xf8, 0x51, 0xee, 0x3e, - 0x94, 0x19, 0xe1, 0xd8, 0x33, 0xbc, 0xb1, 0x1b, 0x5b, 0x8c, 0x3f, 0x13, 0xdb, 0x5e, 0x0e, 0x9a, - 0x6a, 0x5e, 0xdf, 0x86, 0x3c, 0xbb, 0x4c, 0xc8, 0x52, 0x7a, 0x25, 0x61, 0x3f, 0x72, 0x3b, 0x74, - 0xc1, 0x88, 0x2f, 0x20, 0xff, 0x8a, 0x21, 0xb0, 0x8a, 0x65, 0xb3, 0x72, 0x29, 0x2c, 0x63, 0xc8, - 0x71, 0xa1, 0x92, 0xce, 0x9e, 0x59, 0xe5, 0x49, 0x88, 0x73, 0xa2, 0x1f, 0xf0, 0x0a, 0xb7, 0xa4, - 0xfb, 0x6d, 0xea, 0xb2, 0xde, 0xc0, 0x24, 0x96, 0xc7, 0x7a, 0x67, 0x59, 0xaf, 0x42, 0xc1, 0x1b, - 0x50, 0xe7, 0x9a, 0xb6, 0xfa, 0x7d, 0xa5, 0x82, 0xf4, 0xe5, 0x69, 0x61, 0x79, 0xf8, 0x9f, 0x34, - 0x98, 0x57, 0x06, 0x4c, 0xe5, 0x98, 0x0f, 0x21, 0xcf, 0x71, 0x66, 0x51, 0xac, 0x2c, 0x86, 0x47, - 0x71, 0x35, 0xba, 0xe0, 0x41, 0x1b, 0x50, 0xe0, 0x4f, 0xb2, 0x8c, 0x4f, 0x66, 0x97, 0x4c, 0xf8, - 0x31, 0x2c, 0x08, 0x12, 0x19, 0xda, 0x49, 0x7b, 0x9b, 0x39, 0x14, 0xff, 0x29, 0x2c, 0x86, 0xd9, - 0xa6, 0x9a, 0x92, 0x62, 0x64, 0xe6, 0x5d, 0x8c, 0xdc, 0x92, 0x46, 0x9e, 0x8c, 0xfa, 0x4a, 0x29, - 0x14, 0x5d, 0x75, 0x75, 0x45, 0x32, 0x91, 0x15, 0xf1, 0x27, 0x20, 0x45, 0xfc, 0x56, 0x27, 0xb0, - 0x20, 0xb7, 0xc3, 0x81, 0xe9, 0xfa, 0x15, 0xf7, 0x57, 0x80, 0x54, 0xe2, 0x6f, 0xdb, 0xa0, 0x1d, - 0x22, 0x13, 0xb9, 0x34, 0xe8, 0x13, 0x40, 0x2a, 0x71, 0xaa, 0x88, 0xde, 0x82, 0xf9, 0x57, 0xf6, - 0x84, 0x86, 0x06, 0x4a, 0x0d, 0x8e, 0x0c, 0xbf, 0x7f, 0xfb, 0xcb, 0xe6, 0xb7, 0xa9, 0x72, 0x75, - 0xc0, 0x54, 0xca, 0xff, 0x43, 0x83, 0xca, 0xd6, 0xc0, 0x70, 0x86, 0x52, 0xf1, 0xf7, 0x21, 0xcf, - 0x6f, 0x95, 0x02, 0xc8, 0x79, 0x3f, 0x2c, 0x46, 0xe5, 0xe5, 0x8d, 0x2d, 0x7e, 0x07, 0x15, 0xa3, - 0xa8, 0xe1, 0xe2, 0x5d, 0xcf, 0x4e, 0xe4, 0xdd, 0xcf, 0x0e, 0xfa, 0x08, 0x72, 0x06, 0x1d, 0xc2, - 0x42, 0x70, 0x2d, 0x7a, 0x9f, 0x67, 0xd2, 0x58, 0xed, 0xcb, 0xb9, 0xf0, 0x77, 0xa1, 0xac, 0x68, - 0x40, 0x05, 0xc8, 0xbe, 0x6c, 0x8b, 0x42, 0x75, 0x6b, 0xbb, 0xbb, 0xf7, 0x9a, 0x03, 0x19, 0x35, - 0x80, 0x9d, 0xb6, 0xdf, 0xce, 0xe0, 0xcf, 0xc5, 0x28, 0x11, 0xef, 0x54, 0x7b, 0xb4, 0x34, 0x7b, - 0x32, 0xef, 0x64, 0xcf, 0x15, 0x54, 0xc5, 0xf4, 0xa7, 0x0d, 0xdf, 0x4c, 0x5e, 0x4a, 0xf8, 0x56, - 0x8c, 0xd7, 0x05, 0x23, 0x9e, 0x83, 0xaa, 0x08, 0xe8, 0x62, 0xff, 0xfd, 0x4b, 0x06, 0x6a, 0x92, - 0x32, 0x2d, 0xe0, 0x2c, 0xb1, 0x32, 0x9e, 0x01, 0x7c, 0xa4, 0x6c, 0x19, 0xf2, 0xfd, 0xb3, 0x63, - 0xf3, 0x2b, 0xf9, 0x72, 0x40, 0xb4, 0x28, 0x7d, 0xc0, 0xf5, 0xf0, 0x37, 0x74, 0xa2, 0x85, 0xee, - 0xf1, 0x97, 0x77, 0x7b, 0x56, 0x9f, 0x5c, 0xb1, 0x3a, 0x7a, 0x56, 0x0f, 0x08, 0x0c, 0x44, 0x10, - 0x6f, 0xf2, 0x58, 0xf1, 0xac, 0xbc, 0xd9, 0x43, 0x4f, 0xa1, 0x4e, 0x9f, 0xb7, 0x46, 0xa3, 0x81, - 0x49, 0xfa, 0x5c, 0x40, 0x81, 0xf1, 0xc4, 0xe8, 0x54, 0x3b, 0x2b, 0x37, 0xdd, 0x46, 0x91, 0x85, - 0x2d, 0xd1, 0x42, 0x6b, 0x50, 0xe6, 0xf6, 0xed, 0x59, 0x27, 0x2e, 0x61, 0xaf, 0xb7, 0xb2, 0xba, - 0x4a, 0xa2, 0xe7, 0x78, 0x6b, 0xec, 0x5d, 0xb4, 0x2d, 0xe3, 0x6c, 0x20, 0xe3, 0x22, 0x4d, 0xe6, - 0x94, 0xb8, 0x63, 0xba, 0x2a, 0xb5, 0x0d, 0x0b, 0x94, 0x4a, 0x2c, 0xcf, 0xec, 0x29, 0x41, 0x54, - 0xa6, 0x4a, 0x2d, 0x92, 0x2a, 0x0d, 0xd7, 0x7d, 0x6b, 0x3b, 0x7d, 0xe1, 0x40, 0xbf, 0x8d, 0x77, - 0xb8, 0xf0, 0x13, 0x37, 0x94, 0x0c, 0x7f, 0x55, 0x29, 0xeb, 0x81, 0x94, 0x97, 0xc4, 0xbb, 0x41, - 0x0a, 0x7e, 0x06, 0x4b, 0x92, 0x53, 0x40, 0xbe, 0x37, 0x30, 0x77, 0xe0, 0xbe, 0x64, 0xde, 0xbe, - 0xa0, 0x57, 0xe0, 0x23, 0xa1, 0xf0, 0xd7, 0xb5, 0xf3, 0x05, 0x34, 0x7c, 0x3b, 0xd9, 0x35, 0xc4, - 0x1e, 0xa8, 0x06, 0x8c, 0x5d, 0xb1, 0x33, 0x4b, 0x3a, 0x7b, 0xa6, 0x34, 0xc7, 0x1e, 0xf8, 0x85, - 0x07, 0x7d, 0xc6, 0xdb, 0xb0, 0x22, 0x65, 0x88, 0x0b, 0x42, 0x58, 0x48, 0xcc, 0xa0, 0x24, 0x21, - 0xc2, 0x61, 0x74, 0xe8, 0xcd, 0x6e, 0x57, 0x39, 0xc3, 0xae, 0x65, 0x32, 0x35, 0x45, 0xe6, 0x12, - 0xdf, 0x11, 0xd4, 0x30, 0x35, 0x2f, 0x09, 0x32, 0x15, 0xa0, 0x92, 0xc5, 0x42, 0x50, 0x72, 0x6c, - 0x21, 0x62, 0xa2, 0x7f, 0x08, 0xab, 0xbe, 0x11, 0xd4, 0x6f, 0x47, 0xc4, 0x19, 0x9a, 0xae, 0xab, - 0x80, 0x84, 0x49, 0x13, 0x7f, 0x1f, 0x66, 0x47, 0x44, 0x44, 0xae, 0xf2, 0x26, 0xda, 0xe0, 0x6f, - 0xf5, 0x37, 0x94, 0xc1, 0xac, 0x1f, 0xf7, 0xe1, 0x81, 0x94, 0xce, 0x3d, 0x9a, 0x28, 0x3e, 0x6a, - 0x94, 0x84, 0x4e, 0x32, 0x29, 0xd0, 0x49, 0x36, 0x02, 0x5c, 0x7f, 0xc2, 0x1d, 0x29, 0xcf, 0xd6, - 0x54, 0x19, 0x69, 0x9f, 0xfb, 0xd4, 0x3f, 0x92, 0x53, 0x09, 0x3b, 0x83, 0xc5, 0xf0, 0x49, 0x9e, - 0x2a, 0x58, 0x2e, 0x42, 0xce, 0xb3, 0x2f, 0x89, 0x0c, 0x95, 0xbc, 0x21, 0x0d, 0xf6, 0x8f, 0xf9, - 0x54, 0x06, 0x1b, 0x81, 0x30, 0xb6, 0x25, 0xa7, 0xb5, 0x97, 0xae, 0xa6, 0x2c, 0xf1, 0x78, 0x03, - 0x1f, 0xc2, 0x72, 0x34, 0x4c, 0x4c, 0x65, 0xf2, 0x6b, 0xbe, 0x81, 0x93, 0x22, 0xc9, 0x54, 0x72, - 0x3f, 0x0d, 0x82, 0x81, 0x12, 0x50, 0xa6, 0x12, 0xa9, 0x43, 0x33, 0x29, 0xbe, 0xfc, 0x26, 0xf6, - 0xab, 0x1f, 0x6e, 0xa6, 0x12, 0xe6, 0x06, 0xc2, 0xa6, 0x5f, 0xfe, 0x20, 0x46, 0x64, 0x6f, 0x8c, - 0x11, 0xe2, 0x90, 0x04, 0x51, 0xec, 0x1b, 0xd8, 0x74, 0x42, 0x47, 0x10, 0x40, 0xa7, 0xd5, 0x41, - 0x73, 0x88, 0xaf, 0x83, 0x35, 0xe4, 0xc6, 0x56, 0xc3, 0xee, 0x54, 0x8b, 0xf1, 0x59, 0x10, 0x3b, - 0x63, 0x91, 0x79, 0x2a, 0xc1, 0x9f, 0xc3, 0x5a, 0x7a, 0x50, 0x9e, 0x46, 0xf2, 0xd3, 0x16, 0x94, - 0xfc, 0xb2, 0x55, 0xf9, 0x22, 0xa6, 0x0c, 0x85, 0xc3, 0xce, 0xf1, 0xd1, 0xd6, 0x76, 0x9b, 0x7f, - 0x12, 0xb3, 0xdd, 0xd1, 0xf5, 0x93, 0xa3, 0x6e, 0x3d, 0xb3, 0xf9, 0xcb, 0x2c, 0x64, 0xf6, 0x5f, - 0xa3, 0x2f, 0x20, 0xc7, 0xdf, 0x0f, 0xdf, 0xf0, 0x51, 0x40, 0xf3, 0xa6, 0x57, 0xe0, 0xf8, 0xce, - 0x4f, 0xfe, 0xfb, 0x97, 0x3f, 0xcf, 0xcc, 0xe3, 0x4a, 0x6b, 0xf2, 0x9d, 0xd6, 0xe5, 0xa4, 0xc5, - 0x72, 0xc3, 0x73, 0xed, 0x29, 0xfa, 0x14, 0xb2, 0x47, 0x63, 0x0f, 0xa5, 0x7e, 0x2c, 0xd0, 0x4c, - 0x7f, 0x2b, 0x8e, 0x97, 0x98, 0xd0, 0x39, 0x0c, 0x42, 0xe8, 0x68, 0xec, 0x51, 0x91, 0x3f, 0x82, - 0xb2, 0xfa, 0x4e, 0xfb, 0xd6, 0x2f, 0x08, 0x9a, 0xb7, 0xbf, 0x2f, 0xc7, 0xf7, 0x99, 0xaa, 0x3b, - 0x18, 0x09, 0x55, 0xfc, 0xad, 0xbb, 0x3a, 0x8b, 0xee, 0x95, 0x85, 0x52, 0xbf, 0x2f, 0x68, 0xa6, - 0xbf, 0x42, 0x8f, 0xcd, 0xc2, 0xbb, 0xb2, 0xa8, 0xc8, 0x3f, 0x16, 0x6f, 0xcf, 0x7b, 0x1e, 0x7a, - 0x90, 0xf0, 0xf6, 0x54, 0x7d, 0x4f, 0xd8, 0x5c, 0x4b, 0x67, 0x10, 0x4a, 0xee, 0x31, 0x25, 0xcb, - 0x78, 0x5e, 0x28, 0xe9, 0xf9, 0x2c, 0xcf, 0xb5, 0xa7, 0x9b, 0x3d, 0xc8, 0x31, 0x0c, 0x1e, 0x7d, - 0x29, 0x1f, 0x9a, 0x09, 0x2f, 0x23, 0x52, 0x16, 0x3a, 0x84, 0xde, 0xe3, 0x45, 0xa6, 0xa8, 0x86, - 0x4b, 0x54, 0x11, 0x43, 0xe0, 0x9f, 0x6b, 0x4f, 0xd7, 0xb5, 0x6f, 0x69, 0x9b, 0xff, 0x9c, 0x83, - 0x1c, 0x03, 0x9f, 0xd0, 0x25, 0x40, 0x80, 0x47, 0x47, 0x67, 0x17, 0x43, 0xb8, 0xa3, 0xb3, 0x8b, - 0x43, 0xd9, 0xb8, 0xc9, 0x94, 0x2e, 0xe2, 0x39, 0xaa, 0x94, 0x61, 0x5a, 0x2d, 0x06, 0xd3, 0x51, - 0x3f, 0xfe, 0x95, 0x26, 0xb0, 0x37, 0x7e, 0x96, 0x50, 0x92, 0xb4, 0x10, 0x28, 0x1d, 0xdd, 0x0e, - 0x09, 0x80, 0x34, 0xfe, 0x1e, 0x53, 0xd8, 0xc2, 0xf5, 0x40, 0xa1, 0xc3, 0x38, 0x9e, 0x6b, 0x4f, - 0xbf, 0x6c, 0xe0, 0x05, 0xe1, 0xe5, 0x48, 0x0f, 0xfa, 0x31, 0xd4, 0xc2, 0xa0, 0x2b, 0x7a, 0x98, - 0xa0, 0x2b, 0x8a, 0xdd, 0x36, 0x1f, 0xdd, 0xcc, 0x24, 0x6c, 0x5a, 0x65, 0x36, 0x09, 0xe5, 0x5c, - 0xf3, 0x25, 0x21, 0x23, 0x83, 0x32, 0x89, 0x35, 0x40, 0x7f, 0xaf, 0x09, 0x4c, 0x3c, 0x40, 0x51, - 0x51, 0x92, 0xf4, 0x18, 0x46, 0xdb, 0x7c, 0x7c, 0x0b, 0x97, 0x30, 0xe2, 0x0f, 0x98, 0x11, 0xbf, - 0x8b, 0x17, 0x03, 0x23, 0x3c, 0x73, 0x48, 0x3c, 0x5b, 0x58, 0xf1, 0xe5, 0x3d, 0x7c, 0x27, 0xe4, - 0x9c, 0x50, 0x6f, 0xb0, 0x58, 0x1c, 0x09, 0x4d, 0x5c, 0xac, 0x10, 0xb2, 0x9a, 0xb8, 0x58, 0x61, - 0x18, 0x35, 0x69, 0xb1, 0x38, 0xee, 0x99, 0xb4, 0x58, 0x7e, 0xcf, 0x26, 0xfb, 0x7e, 0x85, 0x7f, - 0xb5, 0x8a, 0x6c, 0x28, 0xf9, 0x28, 0x24, 0x5a, 0x4d, 0x42, 0x84, 0x82, 0xbb, 0x44, 0xf3, 0x41, - 0x6a, 0xbf, 0x30, 0xe8, 0x3d, 0x66, 0xd0, 0x5d, 0xbc, 0x4c, 0x35, 0x8b, 0x0f, 0x63, 0x5b, 0x1c, - 0x76, 0x68, 0x19, 0xfd, 0x3e, 0x75, 0xc4, 0x9f, 0x40, 0x45, 0x85, 0x09, 0xd1, 0x7b, 0x89, 0x28, - 0x94, 0x8a, 0x34, 0x36, 0xf1, 0x4d, 0x2c, 0x42, 0xf3, 0x23, 0xa6, 0x79, 0x15, 0xaf, 0x24, 0x68, - 0x76, 0x18, 0x6b, 0x48, 0x39, 0x87, 0xf8, 0x92, 0x95, 0x87, 0x10, 0xc4, 0x64, 0xe5, 0x61, 0x84, - 0xf0, 0x46, 0xe5, 0x63, 0xc6, 0x4a, 0x95, 0xbb, 0x00, 0x01, 0x98, 0x87, 0x12, 0x7d, 0xa9, 0x5c, - 0xa6, 0xa2, 0xc1, 0x21, 0x8e, 0x03, 0x62, 0xcc, 0xd4, 0x8a, 0x7d, 0x17, 0x51, 0x3b, 0x30, 0x5d, - 0x1a, 0x24, 0x36, 0xff, 0x3a, 0x0f, 0xe5, 0x57, 0x86, 0x69, 0x79, 0xc4, 0x32, 0xac, 0x1e, 0x41, - 0x67, 0x90, 0x63, 0x89, 0x32, 0x1a, 0x07, 0x55, 0x7c, 0x2b, 0x1a, 0x07, 0x43, 0xe0, 0x0f, 0x5e, - 0x63, 0x5a, 0x9b, 0x78, 0x89, 0x6a, 0x1d, 0x06, 0xa2, 0x5b, 0x0c, 0xb3, 0xa1, 0x13, 0x7d, 0x03, - 0x79, 0xf1, 0x3a, 0x20, 0x22, 0x28, 0x84, 0xe5, 0x34, 0xef, 0x25, 0x77, 0x26, 0x6d, 0x25, 0x55, - 0x8d, 0xcb, 0xf8, 0xa8, 0x9e, 0x09, 0x40, 0x00, 0x46, 0x46, 0x1d, 0x1a, 0xc3, 0x2e, 0x9b, 0x6b, - 0xe9, 0x0c, 0x42, 0xe7, 0x63, 0xa6, 0xf3, 0x01, 0x6e, 0x46, 0x75, 0xf6, 0x7d, 0x5e, 0xaa, 0xf7, - 0x8f, 0x60, 0x76, 0xd7, 0x70, 0x2f, 0x50, 0x24, 0xf5, 0x29, 0x1f, 0x93, 0x34, 0x9b, 0x49, 0x5d, - 0x42, 0xcb, 0x03, 0xa6, 0x65, 0x85, 0x47, 0x12, 0x55, 0xcb, 0x85, 0xe1, 0xd2, 0x9c, 0x82, 0xfa, - 0x90, 0xe7, 0xdf, 0x96, 0x44, 0xfd, 0x17, 0xfa, 0x3e, 0x25, 0xea, 0xbf, 0xf0, 0xe7, 0x28, 0xb7, - 0x6b, 0x19, 0x41, 0x51, 0x7e, 0xcc, 0x81, 0x22, 0x6f, 0xf6, 0x22, 0x1f, 0x7e, 0x34, 0x57, 0xd3, - 0xba, 0x85, 0xae, 0x87, 0x4c, 0xd7, 0x7d, 0xdc, 0x88, 0xad, 0x95, 0xe0, 0x7c, 0xae, 0x3d, 0xfd, - 0x96, 0x86, 0x7e, 0x0c, 0x10, 0xe0, 0xb7, 0xb1, 0x03, 0x10, 0x85, 0x82, 0x63, 0x07, 0x20, 0x06, - 0xfd, 0xe2, 0x0d, 0xa6, 0x77, 0x1d, 0x3f, 0x8c, 0xea, 0xf5, 0x1c, 0xc3, 0x72, 0xdf, 0x10, 0xe7, - 0x23, 0x8e, 0xd1, 0xb9, 0x17, 0xe6, 0x88, 0x1e, 0x86, 0x7f, 0x9b, 0x83, 0x59, 0x5a, 0x80, 0xd2, - 0x3c, 0x1d, 0xdc, 0xdb, 0xa3, 0x96, 0xc4, 0xd0, 0xb2, 0xa8, 0x25, 0xf1, 0x2b, 0x7f, 0x38, 0x4f, - 0xb3, 0x9f, 0x1b, 0x10, 0xc6, 0x40, 0x1d, 0x6d, 0x43, 0x59, 0xb9, 0xd8, 0xa3, 0x04, 0x61, 0x61, - 0x18, 0x2e, 0x1a, 0xf9, 0x13, 0x50, 0x01, 0x7c, 0x97, 0xe9, 0x5b, 0xe2, 0x91, 0x9f, 0xe9, 0xeb, - 0x73, 0x0e, 0xaa, 0xf0, 0x2d, 0x54, 0xd4, 0xcb, 0x3f, 0x4a, 0x90, 0x17, 0x81, 0xf8, 0xa2, 0x51, - 0x2e, 0x09, 0x3b, 0x08, 0x1f, 0x7c, 0xff, 0x27, 0x15, 0x92, 0x8d, 0x2a, 0x1e, 0x40, 0x41, 0xa0, - 0x01, 0x49, 0xb3, 0x0c, 0xe3, 0x81, 0x49, 0xb3, 0x8c, 0x40, 0x09, 0xe1, 0xda, 0x8e, 0x69, 0xa4, - 0x17, 0x1e, 0x99, 0x49, 0x84, 0xb6, 0x97, 0xc4, 0x4b, 0xd3, 0x16, 0x80, 0x5b, 0x69, 0xda, 0x94, - 0xcb, 0x66, 0x9a, 0xb6, 0x73, 0xe2, 0x89, 0xe3, 0x22, 0x2f, 0x71, 0x28, 0x45, 0x98, 0x1a, 0xbd, - 0xf1, 0x4d, 0x2c, 0x49, 0xa5, 0x77, 0xa0, 0x50, 0x84, 0x6e, 0x74, 0x05, 0x10, 0x60, 0x15, 0xd1, - 0x7a, 0x2a, 0x11, 0xf0, 0x8c, 0xd6, 0x53, 0xc9, 0x70, 0x47, 0x38, 0x34, 0x04, 0x7a, 0x79, 0xe5, - 0x4f, 0x35, 0xff, 0x4c, 0x03, 0x14, 0x87, 0x35, 0xd0, 0xb3, 0x64, 0xe9, 0x89, 0x30, 0x6a, 0xf3, - 0xc3, 0x77, 0x63, 0x4e, 0x8a, 0xf6, 0x81, 0x49, 0x3d, 0xc6, 0x3d, 0x7a, 0x4b, 0x8d, 0xfa, 0x73, - 0x0d, 0xaa, 0x21, 0x4c, 0x04, 0xbd, 0x9f, 0xb2, 0xa6, 0x11, 0x14, 0xb6, 0xf9, 0xe4, 0x56, 0xbe, - 0xa4, 0x42, 0x53, 0xd9, 0x01, 0xb2, 0xe2, 0xfe, 0xa9, 0x06, 0xb5, 0x30, 0x86, 0x82, 0x52, 0x64, - 0xc7, 0x50, 0xdc, 0xe6, 0xfa, 0xed, 0x8c, 0x37, 0x2f, 0x4f, 0x50, 0x6c, 0x0f, 0xa0, 0x20, 0x50, - 0x97, 0xa4, 0x8d, 0x1f, 0xc6, 0x7f, 0x93, 0x36, 0x7e, 0x04, 0xb2, 0x49, 0xd8, 0xf8, 0x8e, 0x3d, - 0x20, 0xca, 0x31, 0x13, 0xb0, 0x4c, 0x9a, 0xb6, 0x9b, 0x8f, 0x59, 0x04, 0xd3, 0x49, 0xd3, 0x16, - 0x1c, 0x33, 0x89, 0xc7, 0xa0, 0x14, 0x61, 0xb7, 0x1c, 0xb3, 0x28, 0x9c, 0x93, 0x70, 0xcc, 0x98, - 0x42, 0xe5, 0x98, 0x05, 0xc8, 0x49, 0xd2, 0x31, 0x8b, 0xc1, 0xd9, 0x49, 0xc7, 0x2c, 0x0e, 0xbe, - 0x24, 0xac, 0x23, 0xd3, 0x1b, 0x3a, 0x66, 0x0b, 0x09, 0x20, 0x0b, 0xfa, 0x30, 0xc5, 0x89, 0x89, - 0x28, 0x79, 0xf3, 0xa3, 0x77, 0xe4, 0x4e, 0xdd, 0xe3, 0xdc, 0xfd, 0x72, 0x8f, 0xff, 0x8d, 0x06, - 0x8b, 0x49, 0x00, 0x0d, 0x4a, 0xd1, 0x93, 0x82, 0xae, 0x37, 0x37, 0xde, 0x95, 0xfd, 0x66, 0x6f, - 0xf9, 0xbb, 0xfe, 0x45, 0xfd, 0xdf, 0xbf, 0x5e, 0xd5, 0xfe, 0xf3, 0xeb, 0x55, 0xed, 0x7f, 0xbe, - 0x5e, 0xd5, 0xfe, 0xf6, 0x7f, 0x57, 0x67, 0xce, 0xf2, 0xec, 0x87, 0x7a, 0xdf, 0xf9, 0xff, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xc6, 0xc3, 0xa2, 0xb2, 0x2f, 0x38, 0x00, 0x00, + 0xe4, 0x9a, 0xa9, 0xaf, 0xe8, 0xf4, 0x91, 0x8f, 0xb7, 0xce, 0xc9, 0x09, 0xb1, 0xb8, 0xe2, 0x0a, + 0x1d, 0x6f, 0x9d, 0x93, 0xb6, 0xd5, 0x47, 0xcb, 0x90, 0x1b, 0x98, 0x43, 0xd3, 0x13, 0x5a, 0x79, + 0x23, 0x64, 0xce, 0x7c, 0xc4, 0x9c, 0x6d, 0x00, 0xd7, 0x76, 0xbc, 0x13, 0xdb, 0xe9, 0x13, 0xa7, + 0x91, 0x5b, 0xd3, 0xd6, 0x6b, 0x9b, 0x6f, 0x6f, 0xa8, 0x0b, 0xb1, 0xa1, 0x1a, 0xb4, 0x71, 0x64, + 0x3b, 0x5e, 0x87, 0xf2, 0xea, 0x25, 0x57, 0x3e, 0xa2, 0x0f, 0xa1, 0xcc, 0x84, 0x78, 0x86, 0x73, + 0x4e, 0xbc, 0x46, 0x9e, 0x49, 0x79, 0x74, 0x83, 0x94, 0x2e, 0x63, 0xd6, 0x99, 0x7a, 0xfe, 0x8c, + 0x30, 0x54, 0x5c, 0xe2, 0x98, 0xc6, 0xc0, 0xfc, 0xc2, 0x38, 0x1d, 0x90, 0x46, 0x61, 0x4d, 0x5b, + 0x2f, 0xea, 0x21, 0x1a, 0x9d, 0xff, 0x25, 0xb9, 0x76, 0x4f, 0x6c, 0x6b, 0x70, 0xdd, 0x28, 0x32, + 0x86, 0x22, 0x25, 0x74, 0xac, 0xc1, 0x35, 0x5b, 0x34, 0x7b, 0x6c, 0x79, 0xbc, 0xb7, 0xc4, 0x7a, + 0x4b, 0x8c, 0xc2, 0xba, 0xd7, 0xa1, 0x3e, 0x34, 0xad, 0x93, 0xa1, 0xdd, 0x3f, 0xf1, 0x1d, 0x02, + 0xcc, 0x21, 0xb5, 0xa1, 0x69, 0xbd, 0xb4, 0xfb, 0xba, 0x74, 0x0b, 0xe5, 0x34, 0xae, 0xc2, 0x9c, + 0x65, 0xc1, 0x69, 0x5c, 0xa9, 0x9c, 0x1b, 0xb0, 0x44, 0x65, 0xf6, 0x1c, 0x62, 0x78, 0x24, 0x60, + 0xae, 0x30, 0xe6, 0xc5, 0xa1, 0x69, 0x6d, 0xb3, 0x9e, 0x10, 0xbf, 0x71, 0x15, 0xe3, 0xaf, 0x0a, + 0x7e, 0xe3, 0x2a, 0xcc, 0x8f, 0x37, 0xa0, 0xe4, 0xfb, 0x1c, 0x15, 0x61, 0xfe, 0xa0, 0x73, 0xd0, + 0xae, 0xcf, 0x21, 0x80, 0xfc, 0xd6, 0xd1, 0x76, 0xfb, 0x60, 0xa7, 0xae, 0xa1, 0x32, 0x14, 0x76, + 0xda, 0xbc, 0x91, 0xc1, 0xcf, 0x00, 0x02, 0xef, 0xa2, 0x02, 0x64, 0xf7, 0xda, 0x9f, 0xd5, 0xe7, + 0x28, 0xcf, 0xab, 0xb6, 0x7e, 0xb4, 0xdb, 0x39, 0xa8, 0x6b, 0x74, 0xf0, 0xb6, 0xde, 0xde, 0xea, + 0xb6, 0xeb, 0x19, 0xca, 0xf1, 0xb2, 0xb3, 0x53, 0xcf, 0xa2, 0x12, 0xe4, 0x5e, 0x6d, 0xed, 0x1f, + 0xb7, 0xeb, 0xf3, 0xf8, 0x17, 0x1a, 0x54, 0xc5, 0x7a, 0xf1, 0x33, 0x81, 0xbe, 0x0b, 0xf9, 0x0b, + 0x76, 0x2e, 0xd8, 0x56, 0x2c, 0x6f, 0xde, 0x8b, 0x2c, 0x6e, 0xe8, 0xec, 0xe8, 0x82, 0x17, 0x61, + 0xc8, 0x5e, 0x4e, 0xdc, 0x46, 0x66, 0x2d, 0xbb, 0x5e, 0xde, 0xac, 0x6f, 0xf0, 0x03, 0xbb, 0xb1, + 0x47, 0xae, 0x5f, 0x19, 0x83, 0x31, 0xd1, 0x69, 0x27, 0x42, 0x30, 0x3f, 0xb4, 0x1d, 0xc2, 0x76, + 0x6c, 0x51, 0x67, 0xcf, 0x74, 0x1b, 0xb3, 0x45, 0x13, 0xbb, 0x95, 0x37, 0xf0, 0x2f, 0x35, 0x80, + 0xc3, 0xb1, 0x97, 0x7e, 0x34, 0x96, 0x21, 0x37, 0xa1, 0x82, 0xc5, 0xb1, 0xe0, 0x0d, 0x76, 0x26, + 0x88, 0xe1, 0x12, 0xff, 0x4c, 0xd0, 0x06, 0xba, 0x0d, 0x85, 0x91, 0x43, 0x26, 0x27, 0x97, 0x13, + 0xa6, 0xa4, 0xa8, 0xe7, 0x69, 0x73, 0x6f, 0x82, 0xde, 0x82, 0x8a, 0x79, 0x6e, 0xd9, 0x0e, 0x39, + 0xe1, 0xb2, 0x72, 0xac, 0xb7, 0xcc, 0x69, 0xcc, 0x6e, 0x85, 0x85, 0x0b, 0xce, 0xab, 0x2c, 0xfb, + 0x94, 0x84, 0x2d, 0x28, 0x33, 0x53, 0x67, 0x72, 0xdf, 0x7b, 0x81, 0x8d, 0x19, 0x36, 0x2c, 0xee, + 0x42, 0x61, 0x35, 0xfe, 0x21, 0xa0, 0x1d, 0x32, 0x20, 0x1e, 0x99, 0x25, 0x7a, 0x28, 0x3e, 0xc9, + 0xaa, 0x3e, 0xc1, 0x3f, 0xd7, 0x60, 0x29, 0x24, 0x7e, 0xa6, 0x69, 0x35, 0xa0, 0xd0, 0x67, 0xc2, + 0xb8, 0x05, 0x59, 0x5d, 0x36, 0xd1, 0x13, 0x28, 0x0a, 0x03, 0xdc, 0x46, 0x36, 0x65, 0xd3, 0x14, + 0xb8, 0x4d, 0x2e, 0xfe, 0x65, 0x06, 0x4a, 0x62, 0xa2, 0x9d, 0x11, 0xda, 0x82, 0xaa, 0xc3, 0x1b, + 0x27, 0x6c, 0x3e, 0xc2, 0xa2, 0x66, 0x7a, 0x10, 0x7a, 0x31, 0xa7, 0x57, 0xc4, 0x10, 0x46, 0x46, + 0xbf, 0x07, 0x65, 0x29, 0x62, 0x34, 0xf6, 0x84, 0xcb, 0x1b, 0x61, 0x01, 0xc1, 0xfe, 0x7b, 0x31, + 0xa7, 0x83, 0x60, 0x3f, 0x1c, 0x7b, 0xa8, 0x0b, 0xcb, 0x72, 0x30, 0x9f, 0x8d, 0x30, 0x23, 0xcb, + 0xa4, 0xac, 0x85, 0xa5, 0xc4, 0x97, 0xea, 0xc5, 0x9c, 0x8e, 0xc4, 0x78, 0xa5, 0x53, 0x35, 0xc9, + 0xbb, 0xe2, 0xc1, 0x3b, 0x66, 0x52, 0xf7, 0xca, 0x8a, 0x9b, 0xd4, 0xbd, 0xb2, 0x9e, 0x95, 0xa0, + 0x20, 0x5a, 0xf8, 0x5f, 0x32, 0x00, 0x72, 0x35, 0x3a, 0x23, 0xb4, 0x03, 0x35, 0x47, 0xb4, 0x42, + 0xde, 0xba, 0x9b, 0xe8, 0x2d, 0xb1, 0x88, 0x73, 0x7a, 0x55, 0x0e, 0xe2, 0xc6, 0x7d, 0x1f, 0x2a, + 0xbe, 0x94, 0xc0, 0x61, 0x77, 0x12, 0x1c, 0xe6, 0x4b, 0x28, 0xcb, 0x01, 0xd4, 0x65, 0x9f, 0xc0, + 0x8a, 0x3f, 0x3e, 0xc1, 0x67, 0x6f, 0x4d, 0xf1, 0x99, 0x2f, 0x70, 0x49, 0x4a, 0x50, 0xbd, 0xa6, + 0x1a, 0x16, 0xb8, 0xed, 0x4e, 0x82, 0xdb, 0xe2, 0x86, 0x51, 0xc7, 0x01, 0xcd, 0x97, 0xbc, 0x89, + 0xff, 0x2f, 0x0b, 0x85, 0x6d, 0x7b, 0x38, 0x32, 0x1c, 0xba, 0x1a, 0x79, 0x87, 0xb8, 0xe3, 0x81, + 0xc7, 0xdc, 0x55, 0xdb, 0x7c, 0x18, 0x96, 0x28, 0xd8, 0xe4, 0x7f, 0x9d, 0xb1, 0xea, 0x62, 0x08, + 0x1d, 0x2c, 0xd2, 0x63, 0xe6, 0x0d, 0x06, 0x8b, 0xe4, 0x28, 0x86, 0xc8, 0x83, 0x9c, 0x0d, 0x0e, + 0x72, 0x13, 0x0a, 0x13, 0xe2, 0x04, 0x29, 0xfd, 0xc5, 0x9c, 0x2e, 0x09, 0xe8, 0x3d, 0x58, 0x88, + 0xa6, 0x97, 0x9c, 0xe0, 0xa9, 0xf5, 0xc2, 0xd9, 0xe8, 0x21, 0x54, 0x42, 0x39, 0x2e, 0x2f, 0xf8, + 0xca, 0x43, 0x25, 0xc5, 0xdd, 0x92, 0x71, 0x95, 0xe6, 0xe3, 0xca, 0x8b, 0x39, 0x19, 0x59, 0x6f, + 0xc9, 0xc8, 0x5a, 0x14, 0xa3, 0x44, 0x6c, 0x0d, 0x05, 0x99, 0x1f, 0x84, 0x83, 0x0c, 0xfe, 0x01, + 0x54, 0x43, 0x0e, 0xa2, 0x79, 0xa7, 0xfd, 0xf1, 0xf1, 0xd6, 0x3e, 0x4f, 0x52, 0xcf, 0x59, 0x5e, + 0xd2, 0xeb, 0x1a, 0xcd, 0x75, 0xfb, 0xed, 0xa3, 0xa3, 0x7a, 0x06, 0x55, 0xa1, 0x74, 0xd0, 0xe9, + 0x9e, 0x70, 0xae, 0x2c, 0x7e, 0xee, 0x4b, 0x10, 0x49, 0x4e, 0xc9, 0x6d, 0x73, 0x4a, 0x6e, 0xd3, + 0x64, 0x6e, 0xcb, 0x04, 0xb9, 0x8d, 0xa5, 0xb9, 0xfd, 0xf6, 0xd6, 0x51, 0xbb, 0x3e, 0xff, 0xac, + 0x06, 0x15, 0xee, 0xdf, 0x93, 0xb1, 0x45, 0x53, 0xed, 0x3f, 0x68, 0x00, 0xc1, 0x69, 0x42, 0x2d, + 0x28, 0xf4, 0xb8, 0x9e, 0x86, 0xc6, 0x82, 0xd1, 0x4a, 0xe2, 0x92, 0xe9, 0x92, 0x0b, 0x7d, 0x1b, + 0x0a, 0xee, 0xb8, 0xd7, 0x23, 0xae, 0x4c, 0x79, 0xb7, 0xa3, 0xf1, 0x50, 0x44, 0x2b, 0x5d, 0xf2, + 0xd1, 0x21, 0x67, 0x86, 0x39, 0x18, 0xb3, 0x04, 0x38, 0x7d, 0x88, 0xe0, 0xc3, 0x7f, 0xa7, 0x41, + 0x59, 0xd9, 0xbc, 0xbf, 0x65, 0x10, 0xbe, 0x07, 0x25, 0x66, 0x03, 0xe9, 0x8b, 0x30, 0x5c, 0xd4, + 0x03, 0x02, 0xfa, 0x1d, 0x28, 0xc9, 0x13, 0x20, 0x23, 0x71, 0x23, 0x59, 0x6c, 0x67, 0xa4, 0x07, + 0xac, 0x78, 0x0f, 0x16, 0x99, 0x57, 0x7a, 0xb4, 0xb8, 0x96, 0x7e, 0x54, 0xcb, 0x4f, 0x2d, 0x52, + 0x7e, 0x36, 0xa1, 0x38, 0xba, 0xb8, 0x76, 0xcd, 0x9e, 0x31, 0x10, 0x56, 0xf8, 0x6d, 0xfc, 0x11, + 0x20, 0x55, 0xd8, 0x2c, 0xd3, 0xc5, 0x55, 0x28, 0xbf, 0x30, 0xdc, 0x0b, 0x61, 0x12, 0x7e, 0x02, + 0x55, 0xda, 0xdc, 0x7b, 0xf5, 0x06, 0x36, 0xb2, 0x97, 0x03, 0xc9, 0x3d, 0x93, 0xcf, 0x11, 0xcc, + 0x5f, 0x18, 0xee, 0x05, 0x9b, 0x68, 0x55, 0x67, 0xcf, 0xe8, 0x3d, 0xa8, 0xf7, 0xf8, 0x24, 0x4f, + 0x22, 0xaf, 0x0c, 0x0b, 0x82, 0xee, 0x57, 0x82, 0x9f, 0x42, 0x85, 0xcf, 0xe1, 0xab, 0x36, 0x02, + 0x2f, 0xc2, 0xc2, 0x91, 0x65, 0x8c, 0xdc, 0x0b, 0x5b, 0x66, 0x37, 0x3a, 0xe9, 0x7a, 0x40, 0x9b, + 0x49, 0xe3, 0xbb, 0xb0, 0xe0, 0x90, 0xa1, 0x61, 0x5a, 0xa6, 0x75, 0x7e, 0x72, 0x7a, 0xed, 0x11, + 0x57, 0xbc, 0x30, 0xd5, 0x7c, 0xf2, 0x33, 0x4a, 0xa5, 0xa6, 0x9d, 0x0e, 0xec, 0x53, 0x11, 0xe6, + 0xd8, 0x33, 0xfe, 0x59, 0x06, 0x2a, 0x9f, 0x18, 0x5e, 0x4f, 0x2e, 0x1d, 0xda, 0x85, 0x9a, 0x1f, + 0xdc, 0x18, 0x45, 0xd8, 0x12, 0x49, 0xb1, 0x6c, 0x8c, 0x2c, 0xa5, 0x65, 0x76, 0xac, 0xf6, 0x54, + 0x02, 0x13, 0x65, 0x58, 0x3d, 0x32, 0xf0, 0x45, 0x65, 0xd2, 0x45, 0x31, 0x46, 0x55, 0x94, 0x4a, + 0x40, 0x1d, 0xa8, 0x8f, 0x1c, 0xfb, 0xdc, 0x21, 0xae, 0xeb, 0x0b, 0xe3, 0x69, 0x0c, 0x27, 0x08, + 0x3b, 0x14, 0xac, 0x81, 0xb8, 0x85, 0x51, 0x98, 0xf4, 0x6c, 0x21, 0xa8, 0x67, 0x78, 0x70, 0xfa, + 0xaf, 0x0c, 0xa0, 0xf8, 0xa4, 0x7e, 0xd3, 0x12, 0xef, 0x11, 0xd4, 0x5c, 0xcf, 0x70, 0x62, 0x9b, + 0xad, 0xca, 0xa8, 0x7e, 0xc4, 0x7f, 0x17, 0x7c, 0x83, 0x4e, 0x2c, 0xdb, 0x33, 0xcf, 0xae, 0x45, + 0x95, 0x5c, 0x93, 0xe4, 0x03, 0x46, 0x45, 0x6d, 0x28, 0x9c, 0x99, 0x03, 0x8f, 0x38, 0x6e, 0x23, + 0xb7, 0x96, 0x5d, 0xaf, 0x6d, 0x3e, 0xb9, 0x69, 0x19, 0x36, 0x3e, 0x64, 0xfc, 0xdd, 0xeb, 0x11, + 0xd1, 0xe5, 0x58, 0xb5, 0xf2, 0xcc, 0x87, 0xaa, 0xf1, 0x3b, 0x50, 0x7c, 0x4d, 0x45, 0xd0, 0xb7, + 0xec, 0x02, 0x2f, 0x16, 0x59, 0x9b, 0xbf, 0x64, 0x9f, 0x39, 0xc6, 0xf9, 0x90, 0x58, 0x9e, 0x7c, + 0x0f, 0x94, 0x6d, 0xfc, 0x08, 0x20, 0x50, 0x43, 0x43, 0xfe, 0x41, 0xe7, 0xf0, 0xb8, 0x5b, 0x9f, + 0x43, 0x15, 0x28, 0x1e, 0x74, 0x76, 0xda, 0xfb, 0x6d, 0x9a, 0x1f, 0x70, 0x4b, 0xba, 0x34, 0xb4, + 0x96, 0xaa, 0x4e, 0x2d, 0xa4, 0x13, 0xdf, 0x82, 0xe5, 0xa4, 0x05, 0xa4, 0xb5, 0x68, 0x55, 0xec, + 0xd2, 0x99, 0x8e, 0x8a, 0xaa, 0x3a, 0x13, 0x9e, 0x6e, 0x03, 0x0a, 0x7c, 0xf7, 0xf6, 0x45, 0x71, + 0x2e, 0x9b, 0xd4, 0x11, 0x7c, 0x33, 0x92, 0xbe, 0x58, 0x25, 0xbf, 0x9d, 0x18, 0x5e, 0x72, 0x89, + 0xe1, 0x05, 0x3d, 0x84, 0xaa, 0x7f, 0x1a, 0x0c, 0x57, 0xd4, 0x02, 0x25, 0xbd, 0x22, 0x37, 0x3a, + 0xa5, 0x85, 0x9c, 0x5e, 0x08, 0x3b, 0x1d, 0x3d, 0x82, 0x3c, 0x99, 0x10, 0xcb, 0x73, 0x1b, 0x65, + 0x96, 0x31, 0xaa, 0xb2, 0x76, 0x6f, 0x53, 0xaa, 0x2e, 0x3a, 0xf1, 0xf7, 0x60, 0x91, 0xbd, 0x23, + 0x3d, 0x77, 0x0c, 0x4b, 0x7d, 0x99, 0xeb, 0x76, 0xf7, 0x85, 0xbb, 0xe9, 0x23, 0xaa, 0x41, 0x66, + 0x77, 0x47, 0x38, 0x21, 0xb3, 0xbb, 0x83, 0x7f, 0xa2, 0x01, 0x52, 0xc7, 0xcd, 0xe4, 0xe7, 0x88, + 0x70, 0xa9, 0x3e, 0x1b, 0xa8, 0x5f, 0x86, 0x1c, 0x71, 0x1c, 0xdb, 0x61, 0x1e, 0x2d, 0xe9, 0xbc, + 0x81, 0xdf, 0x16, 0x36, 0xe8, 0x64, 0x62, 0x5f, 0xfa, 0x67, 0x90, 0x4b, 0xd3, 0x7c, 0x53, 0xf7, + 0x60, 0x29, 0xc4, 0x35, 0x53, 0xe6, 0xfa, 0x10, 0x16, 0x98, 0xb0, 0xed, 0x0b, 0xd2, 0xbb, 0x1c, + 0xd9, 0xa6, 0x15, 0xd3, 0x47, 0x57, 0x2e, 0x08, 0xb0, 0x74, 0x1e, 0x7c, 0x62, 0x15, 0x9f, 0xd8, + 0xed, 0xee, 0xe3, 0xcf, 0xe0, 0x56, 0x44, 0x8e, 0x34, 0xff, 0x0f, 0xa1, 0xdc, 0xf3, 0x89, 0xae, + 0xa8, 0x75, 0xee, 0x87, 0x8d, 0x8b, 0x0e, 0x55, 0x47, 0xe0, 0x0e, 0xdc, 0x8e, 0x89, 0x9e, 0x69, + 0xce, 0xef, 0xc2, 0x0a, 0x13, 0xb8, 0x47, 0xc8, 0x68, 0x6b, 0x60, 0x4e, 0x52, 0x3d, 0x3d, 0x12, + 0x93, 0x52, 0x18, 0xbf, 0xde, 0x7d, 0x81, 0x7f, 0x5f, 0x68, 0xec, 0x9a, 0x43, 0xd2, 0xb5, 0xf7, + 0xd3, 0x6d, 0xa3, 0xd9, 0xec, 0x92, 0x5c, 0xbb, 0xa2, 0xac, 0x61, 0xcf, 0xf8, 0x1f, 0x35, 0xe1, + 0x2a, 0x75, 0xf8, 0xd7, 0xbc, 0x93, 0x57, 0x01, 0xce, 0xe9, 0x91, 0x21, 0x7d, 0xda, 0xc1, 0x11, + 0x15, 0x85, 0xe2, 0xdb, 0x49, 0xe3, 0x77, 0x45, 0xd8, 0xb9, 0x2c, 0xf6, 0x39, 0xfb, 0xe3, 0x47, + 0xb9, 0xfb, 0x50, 0x66, 0x84, 0x23, 0xcf, 0xf0, 0xc6, 0x6e, 0x6c, 0x31, 0xfe, 0x5c, 0x6c, 0x7b, + 0x39, 0x68, 0xa6, 0x79, 0x7d, 0x1b, 0xf2, 0xec, 0x65, 0x42, 0x96, 0xd2, 0x77, 0x12, 0xf6, 0x23, + 0xb7, 0x43, 0x17, 0x8c, 0xf8, 0x67, 0x1a, 0xe4, 0x5f, 0x32, 0x08, 0x56, 0x31, 0x6d, 0x5e, 0xae, + 0x85, 0x65, 0x0c, 0x39, 0x30, 0x54, 0xd2, 0xd9, 0x33, 0x2b, 0x3d, 0x09, 0x71, 0x8e, 0xf5, 0x7d, + 0x5e, 0xe2, 0x96, 0x74, 0xbf, 0x4d, 0x7d, 0xd6, 0x1b, 0x98, 0xc4, 0xf2, 0x58, 0xef, 0x3c, 0xeb, + 0x55, 0x28, 0xb4, 0x7a, 0x36, 0xdd, 0x7d, 0x62, 0x38, 0x96, 0x00, 0x4d, 0x8b, 0x7a, 0x40, 0xc0, + 0xfb, 0x50, 0xe7, 0x76, 0x6c, 0xf5, 0xfb, 0x4a, 0x81, 0xe9, 0x6b, 0xd3, 0x22, 0xda, 0x42, 0xd2, + 0x32, 0x51, 0x69, 0xff, 0xa4, 0xc1, 0xa2, 0x22, 0x6e, 0x26, 0xaf, 0xbe, 0x0f, 0x79, 0x0e, 0x52, + 0x8b, 0x4a, 0x67, 0x39, 0x3c, 0x8a, 0xab, 0xd1, 0x05, 0x0f, 0xda, 0x80, 0x02, 0x7f, 0x92, 0xef, + 0x00, 0xc9, 0xec, 0x92, 0x09, 0x3f, 0x82, 0x25, 0x41, 0x22, 0x43, 0x3b, 0xe9, 0x60, 0xb0, 0xc5, + 0xc0, 0x7f, 0x0a, 0xcb, 0x61, 0xb6, 0x99, 0xa6, 0xa4, 0x18, 0x99, 0x79, 0x13, 0x23, 0xb7, 0xa4, + 0x91, 0xc7, 0xa3, 0xbe, 0x52, 0x47, 0x45, 0x77, 0x8c, 0xba, 0x5e, 0x99, 0xf0, 0x7a, 0x05, 0x13, + 0x90, 0x22, 0xbe, 0xd1, 0x09, 0x2c, 0xc9, 0xed, 0xb0, 0x6f, 0xba, 0x7e, 0xb9, 0xfe, 0x05, 0x20, + 0x95, 0xf8, 0x8d, 0x1a, 0xf4, 0x8e, 0x74, 0xc7, 0xa1, 0x63, 0x0f, 0xed, 0x54, 0x97, 0xe2, 0x3f, + 0x83, 0x95, 0x08, 0xdf, 0x37, 0xed, 0xb7, 0x1d, 0x22, 0x8b, 0x15, 0xe9, 0xb7, 0x8f, 0x00, 0xa9, + 0xc4, 0x99, 0xb2, 0x56, 0x0b, 0x16, 0x5f, 0xda, 0x13, 0x1a, 0xfe, 0x28, 0x35, 0x38, 0xf7, 0x1c, + 0x63, 0xf0, 0x5d, 0xe1, 0xb7, 0xa9, 0x72, 0x75, 0xc0, 0x4c, 0xca, 0xff, 0x43, 0x83, 0xca, 0xd6, + 0xc0, 0x70, 0x86, 0x52, 0xf1, 0xf7, 0x21, 0xcf, 0xdf, 0x9c, 0x05, 0x58, 0xf5, 0x4e, 0x58, 0x8c, + 0xca, 0xcb, 0x1b, 0x5b, 0xfc, 0x3d, 0x5b, 0x8c, 0xa2, 0x86, 0x8b, 0xef, 0x59, 0x3b, 0x91, 0xef, + 0x5b, 0x3b, 0xe8, 0x03, 0xc8, 0x19, 0x74, 0x08, 0x4b, 0x33, 0xb5, 0x28, 0x66, 0xc1, 0xa4, 0xb1, + 0xfa, 0x9e, 0x73, 0xe1, 0xef, 0x42, 0x59, 0xd1, 0x80, 0x0a, 0x90, 0x7d, 0xde, 0x16, 0xc5, 0xf8, + 0xd6, 0x76, 0x77, 0xf7, 0x15, 0x07, 0x6b, 0x6a, 0x00, 0x3b, 0x6d, 0xbf, 0x9d, 0xc1, 0x9f, 0x8a, + 0x51, 0x22, 0xa4, 0xab, 0xf6, 0x68, 0x69, 0xf6, 0x64, 0xde, 0xc8, 0x9e, 0x2b, 0xa8, 0x8a, 0xe9, + 0xcf, 0x9a, 0xa2, 0x98, 0xbc, 0x94, 0x14, 0xa5, 0x18, 0xaf, 0x0b, 0x46, 0xbc, 0x00, 0x55, 0x91, + 0xb4, 0xc4, 0xfe, 0xfb, 0xf7, 0x0c, 0xd4, 0x24, 0x65, 0x56, 0x50, 0x5d, 0xe2, 0x81, 0x3c, 0xc9, + 0xf9, 0x68, 0xe0, 0x2d, 0xc8, 0xf7, 0x4f, 0x8f, 0xcc, 0x2f, 0xe4, 0x07, 0x10, 0xd1, 0xa2, 0xf4, + 0x01, 0xd7, 0xc3, 0xbf, 0x42, 0x8a, 0x16, 0xcd, 0x46, 0x8e, 0x71, 0xe6, 0xed, 0x5a, 0x7d, 0x72, + 0xc5, 0x72, 0xdb, 0xbc, 0x1e, 0x10, 0x18, 0x50, 0x22, 0xbe, 0x56, 0xb2, 0x17, 0x04, 0xe5, 0xeb, + 0x25, 0x7a, 0x0c, 0x75, 0xfa, 0xbc, 0x35, 0x1a, 0x0d, 0x4c, 0xd2, 0xe7, 0x02, 0x0a, 0x8c, 0x27, + 0x46, 0xa7, 0xda, 0x59, 0x49, 0xed, 0x36, 0x8a, 0x2c, 0xba, 0x8a, 0x16, 0x5a, 0x83, 0x32, 0xb7, + 0x6f, 0xd7, 0x3a, 0x76, 0x09, 0xfb, 0x84, 0x97, 0xd5, 0x55, 0x52, 0x38, 0x5b, 0x42, 0x34, 0x5b, + 0x2e, 0xc1, 0xe2, 0xd6, 0xd8, 0xbb, 0x68, 0x5b, 0xc6, 0xe9, 0x40, 0x46, 0x22, 0x5a, 0xce, 0x50, + 0xe2, 0x8e, 0xe9, 0xaa, 0xd4, 0x36, 0x2c, 0x51, 0x2a, 0xb1, 0x3c, 0xb3, 0xa7, 0x64, 0x02, 0x59, + 0x2b, 0x68, 0x91, 0x5a, 0xc1, 0x70, 0xdd, 0xd7, 0xb6, 0xd3, 0x17, 0xee, 0xf5, 0xdb, 0x78, 0x87, + 0x0b, 0x3f, 0x76, 0x43, 0xf9, 0xfe, 0x37, 0x95, 0xb2, 0x1e, 0x48, 0x79, 0x4e, 0xbc, 0x29, 0x52, + 0xf0, 0x13, 0x58, 0x91, 0x9c, 0x02, 0xf4, 0x9e, 0xc2, 0xdc, 0x81, 0xfb, 0x92, 0x79, 0xfb, 0xc2, + 0xb0, 0xce, 0xc9, 0xa1, 0x50, 0xf8, 0xdb, 0xda, 0xf9, 0x0c, 0x1a, 0xbe, 0x9d, 0xec, 0x45, 0xcc, + 0x1e, 0xa8, 0x06, 0x8c, 0x5d, 0xb1, 0x6f, 0x4b, 0x3a, 0x7b, 0xa6, 0x34, 0xc7, 0x1e, 0xf8, 0x95, + 0x17, 0x7d, 0xc6, 0xdb, 0x70, 0x47, 0xca, 0x10, 0xaf, 0x48, 0x61, 0x21, 0x31, 0x83, 0x92, 0x84, + 0x08, 0x87, 0xd1, 0xa1, 0xd3, 0xdd, 0xae, 0x72, 0x86, 0x5d, 0xcb, 0x64, 0x6a, 0x8a, 0xcc, 0x15, + 0xbe, 0x23, 0xa8, 0x61, 0x6a, 0x72, 0x15, 0x64, 0x2a, 0x40, 0x25, 0x8b, 0x85, 0xa0, 0xe4, 0xd8, + 0x42, 0xc4, 0x44, 0xff, 0x10, 0x56, 0x7d, 0x23, 0xa8, 0xdf, 0x0e, 0x89, 0x33, 0x34, 0x5d, 0x57, + 0x81, 0x49, 0x93, 0x26, 0xfe, 0x0e, 0xcc, 0x8f, 0x88, 0x88, 0x6b, 0xe5, 0x4d, 0xb4, 0xc1, 0xef, + 0x35, 0x6c, 0x28, 0x83, 0x59, 0x3f, 0xee, 0xc3, 0x03, 0x29, 0x9d, 0x7b, 0x34, 0x51, 0x7c, 0xd4, + 0x28, 0x09, 0x1e, 0x65, 0x52, 0xc0, 0xa3, 0x6c, 0x04, 0xba, 0xff, 0x88, 0x3b, 0x52, 0x9e, 0xad, + 0x99, 0xf2, 0xd5, 0x1e, 0xf7, 0xa9, 0x7f, 0x24, 0x67, 0x12, 0x76, 0x0a, 0xcb, 0xe1, 0x93, 0x3c, + 0x53, 0x28, 0x5d, 0x86, 0x9c, 0x67, 0x5f, 0x12, 0x19, 0x48, 0x79, 0x43, 0x1a, 0xec, 0x1f, 0xf3, + 0x99, 0x0c, 0x36, 0x02, 0x61, 0x6c, 0x4b, 0xce, 0x6a, 0x2f, 0x5d, 0x4d, 0x59, 0xa7, 0xf2, 0x06, + 0x3e, 0x80, 0x5b, 0xd1, 0x30, 0x31, 0x93, 0xc9, 0xaf, 0xf8, 0x06, 0x4e, 0x8a, 0x24, 0x33, 0xc9, + 0xfd, 0x38, 0x08, 0x06, 0x4a, 0x40, 0x99, 0x49, 0xa4, 0x0e, 0xcd, 0xa4, 0xf8, 0xf2, 0x55, 0xec, + 0x57, 0x3f, 0xdc, 0xcc, 0x24, 0xcc, 0x0d, 0x84, 0xcd, 0xbe, 0xfc, 0x41, 0x8c, 0xc8, 0x4e, 0x8d, + 0x11, 0xe2, 0x90, 0x04, 0x51, 0xec, 0x6b, 0xd8, 0x74, 0x42, 0x47, 0x10, 0x40, 0x67, 0xd5, 0x41, + 0x73, 0x88, 0xaf, 0x83, 0x35, 0xe4, 0xc6, 0x56, 0xc3, 0xee, 0x4c, 0x8b, 0xf1, 0x49, 0x10, 0x3b, + 0x63, 0x91, 0x79, 0x26, 0xc1, 0x9f, 0xc2, 0x5a, 0x7a, 0x50, 0x9e, 0x45, 0xf2, 0xe3, 0x16, 0x94, + 0xfc, 0xa2, 0x56, 0xb9, 0x13, 0x54, 0x86, 0xc2, 0x41, 0xe7, 0xe8, 0x70, 0x6b, 0xbb, 0xcd, 0x2f, + 0x05, 0x6d, 0x77, 0x74, 0xfd, 0xf8, 0xb0, 0x5b, 0xcf, 0x6c, 0xfe, 0x2a, 0x0b, 0x99, 0xbd, 0x57, + 0xe8, 0x33, 0xc8, 0xf1, 0x2f, 0xe4, 0x53, 0xae, 0x45, 0x34, 0xa7, 0x5d, 0x02, 0xc0, 0xb7, 0x7f, + 0xf2, 0xdf, 0xbf, 0xfa, 0x45, 0x66, 0x11, 0x57, 0x5a, 0x93, 0xef, 0xb4, 0x2e, 0x27, 0x2d, 0x96, + 0x1b, 0x9e, 0x6a, 0x8f, 0xd1, 0xc7, 0x90, 0x3d, 0x1c, 0x7b, 0x28, 0xf5, 0xba, 0x44, 0x33, 0xfd, + 0x5e, 0x00, 0x5e, 0x61, 0x42, 0x17, 0x30, 0x08, 0xa1, 0xa3, 0xb1, 0x47, 0x45, 0xfe, 0x08, 0xca, + 0xea, 0x57, 0xfd, 0x1b, 0xef, 0x50, 0x34, 0x6f, 0xbe, 0x31, 0x80, 0xef, 0x33, 0x55, 0xb7, 0x31, + 0x12, 0xaa, 0xf8, 0xbd, 0x03, 0x75, 0x16, 0xdd, 0x2b, 0x0b, 0xa5, 0xde, 0xb0, 0x68, 0xa6, 0x5f, + 0x22, 0x88, 0xcd, 0xc2, 0xbb, 0xb2, 0xa8, 0xc8, 0x3f, 0x16, 0xf7, 0x07, 0x7a, 0x1e, 0x7a, 0x90, + 0xf0, 0xfd, 0x58, 0xfd, 0x52, 0xda, 0x5c, 0x4b, 0x67, 0x10, 0x4a, 0xee, 0x31, 0x25, 0xb7, 0xf0, + 0xa2, 0x50, 0xd2, 0xf3, 0x59, 0x9e, 0x6a, 0x8f, 0x37, 0x7b, 0x90, 0x63, 0x5f, 0x21, 0xd0, 0xe7, + 0xf2, 0xa1, 0x99, 0xf0, 0x39, 0x26, 0x65, 0xa1, 0x43, 0xdf, 0x2f, 0xf0, 0x32, 0x53, 0x54, 0xc3, + 0x25, 0xaa, 0x88, 0x7d, 0x83, 0x78, 0xaa, 0x3d, 0x5e, 0xd7, 0xbe, 0xa5, 0x6d, 0xfe, 0x73, 0x0e, + 0x72, 0x0c, 0x7e, 0x43, 0x97, 0x00, 0x01, 0x22, 0x1f, 0x9d, 0x5d, 0x0c, 0xe3, 0x8f, 0xce, 0x2e, + 0x0e, 0xe6, 0xe3, 0x26, 0x53, 0xba, 0x8c, 0x17, 0xa8, 0x52, 0x86, 0xea, 0xb5, 0x18, 0x50, 0x49, + 0xfd, 0xf8, 0x57, 0x9a, 0x40, 0x1f, 0xf9, 0x59, 0x42, 0x49, 0xd2, 0x42, 0xb0, 0x7c, 0x74, 0x3b, + 0x24, 0x40, 0xf2, 0xf8, 0x7b, 0x4c, 0x61, 0x0b, 0xd7, 0x03, 0x85, 0x0e, 0xe3, 0x78, 0xaa, 0x3d, + 0xfe, 0xbc, 0x81, 0x97, 0x84, 0x97, 0x23, 0x3d, 0xe8, 0xc7, 0x50, 0x0b, 0xc3, 0xce, 0xe8, 0x61, + 0x82, 0xae, 0x28, 0x7a, 0xdd, 0x7c, 0x7b, 0x3a, 0x93, 0xb0, 0x69, 0x95, 0xd9, 0x24, 0x94, 0x73, + 0xcd, 0x97, 0x84, 0x8c, 0x0c, 0xca, 0x24, 0xd6, 0x00, 0xfd, 0xbd, 0x26, 0xbe, 0x0a, 0x04, 0x38, + 0x32, 0x4a, 0x92, 0x1e, 0x43, 0xa9, 0x9b, 0x8f, 0x6e, 0xe0, 0x12, 0x46, 0xfc, 0x01, 0x33, 0xe2, + 0x77, 0xf1, 0x72, 0x60, 0x84, 0x67, 0x0e, 0x89, 0x67, 0x0b, 0x2b, 0x3e, 0xbf, 0x87, 0x6f, 0x87, + 0x9c, 0x13, 0xea, 0x0d, 0x16, 0x8b, 0x63, 0xc1, 0x89, 0x8b, 0x15, 0xc2, 0x96, 0x13, 0x17, 0x2b, + 0x0c, 0x24, 0x27, 0x2d, 0x16, 0x47, 0x7e, 0x93, 0x16, 0xcb, 0xef, 0xd9, 0xfc, 0xff, 0x79, 0x28, + 0x6c, 0xf3, 0x7b, 0xbb, 0xc8, 0x86, 0x92, 0x0f, 0xa5, 0xa2, 0xd5, 0x24, 0xbc, 0x28, 0x78, 0x97, + 0x68, 0x3e, 0x48, 0xed, 0x17, 0x06, 0xbd, 0xc5, 0x0c, 0xba, 0x8b, 0x6f, 0x51, 0xcd, 0xe2, 0x6a, + 0x70, 0x8b, 0x83, 0x12, 0x2d, 0xa3, 0xdf, 0xa7, 0x8e, 0xf8, 0x13, 0xa8, 0xa8, 0x58, 0x27, 0x7a, + 0x2b, 0x11, 0xa3, 0x52, 0xe1, 0xd2, 0x26, 0x9e, 0xc6, 0x22, 0x34, 0xbf, 0xcd, 0x34, 0xaf, 0xe2, + 0x3b, 0x09, 0x9a, 0x1d, 0xc6, 0x1a, 0x52, 0xce, 0x71, 0xca, 0x64, 0xe5, 0x21, 0x18, 0x34, 0x59, + 0x79, 0x18, 0xe6, 0x9c, 0xaa, 0x7c, 0xcc, 0x58, 0xa9, 0x72, 0x17, 0x20, 0x40, 0x24, 0x51, 0xa2, + 0x2f, 0x95, 0x97, 0xa9, 0x68, 0x70, 0x88, 0x83, 0x99, 0x18, 0x33, 0xb5, 0x62, 0xdf, 0x45, 0xd4, + 0x0e, 0x4c, 0xd7, 0xe3, 0x07, 0xb3, 0x1a, 0x82, 0x18, 0x51, 0xe2, 0x7c, 0xc2, 0x38, 0x65, 0xf3, + 0xe1, 0x54, 0x1e, 0xa1, 0xfd, 0x11, 0xd3, 0xfe, 0x00, 0x37, 0x13, 0xb4, 0x8f, 0x38, 0x2f, 0xdd, + 0x6c, 0x7f, 0x9d, 0x87, 0xf2, 0x4b, 0xc3, 0xb4, 0x3c, 0x62, 0x19, 0x56, 0x8f, 0xa0, 0x53, 0xc8, + 0xb1, 0x4c, 0x1d, 0x0d, 0xc4, 0x2a, 0xfc, 0x16, 0x0d, 0xc4, 0x21, 0x6c, 0x0a, 0xaf, 0x31, 0xc5, + 0x4d, 0xbc, 0x42, 0x15, 0x0f, 0x03, 0xd1, 0x2d, 0x06, 0x29, 0xd1, 0x49, 0x9f, 0x41, 0x5e, 0x7c, + 0x91, 0x89, 0x08, 0x0a, 0x41, 0x4d, 0xcd, 0x7b, 0xc9, 0x9d, 0x49, 0x7b, 0x59, 0x55, 0xe3, 0x32, + 0x3e, 0xaa, 0x67, 0x02, 0x10, 0x60, 0xa5, 0xd1, 0x15, 0x8d, 0x41, 0xab, 0xcd, 0xb5, 0x74, 0x86, + 0x24, 0x9f, 0xaa, 0x3a, 0xfb, 0x3e, 0x2f, 0xd5, 0xfb, 0x47, 0x30, 0xff, 0xc2, 0x70, 0x2f, 0x50, + 0x24, 0xf7, 0x2a, 0xf7, 0x79, 0x9a, 0xcd, 0xa4, 0x2e, 0xa1, 0xe5, 0x01, 0xd3, 0x72, 0x87, 0x87, + 0x32, 0x55, 0xcb, 0x85, 0xe1, 0xd2, 0xa4, 0x86, 0xfa, 0x90, 0xe7, 0xd7, 0x7b, 0xa2, 0xfe, 0x0b, + 0x5d, 0x11, 0x8a, 0xfa, 0x2f, 0x7c, 0x23, 0xe8, 0x66, 0x2d, 0x23, 0x28, 0xca, 0xfb, 0x34, 0x28, + 0xf2, 0x71, 0x35, 0x72, 0xf7, 0xa6, 0xb9, 0x9a, 0xd6, 0x2d, 0x74, 0x3d, 0x64, 0xba, 0xee, 0xe3, + 0x46, 0x6c, 0xad, 0x04, 0xe7, 0x53, 0xed, 0xf1, 0xb7, 0x34, 0xf4, 0x63, 0x80, 0x00, 0x5e, 0x8e, + 0x9d, 0xc0, 0x28, 0x52, 0x1d, 0x3b, 0x81, 0x31, 0x64, 0x1a, 0x6f, 0x30, 0xbd, 0xeb, 0xf8, 0x61, + 0x54, 0xaf, 0xe7, 0x18, 0x96, 0x7b, 0x46, 0x9c, 0x0f, 0x38, 0x84, 0xe8, 0x5e, 0x98, 0x23, 0x7a, + 0x18, 0xfe, 0x75, 0x01, 0xe6, 0x69, 0x05, 0x4c, 0x0b, 0x85, 0x00, 0x38, 0x88, 0x5a, 0x12, 0x83, + 0xeb, 0xa2, 0x96, 0xc4, 0x31, 0x87, 0x70, 0xa1, 0xc0, 0x7e, 0xf1, 0x41, 0x18, 0x03, 0x75, 0xb4, + 0x0d, 0x65, 0x05, 0x59, 0x40, 0x09, 0xc2, 0xc2, 0x38, 0x60, 0x34, 0xf5, 0x24, 0xc0, 0x12, 0xf8, + 0x2e, 0xd3, 0xb7, 0xc2, 0x53, 0x0f, 0xd3, 0xd7, 0xe7, 0x1c, 0x54, 0xe1, 0x6b, 0xa8, 0xa8, 0xe8, + 0x03, 0x4a, 0x90, 0x17, 0xc1, 0x18, 0xa3, 0x61, 0x36, 0x09, 0xbc, 0x08, 0x1f, 0x7c, 0xff, 0x57, + 0x2d, 0x92, 0x8d, 0x2a, 0x1e, 0x40, 0x41, 0xc0, 0x11, 0x49, 0xb3, 0x0c, 0x03, 0x92, 0x49, 0xb3, + 0x8c, 0x60, 0x19, 0xe1, 0xe2, 0x92, 0x69, 0xa4, 0x6f, 0x5c, 0x32, 0x95, 0x09, 0x6d, 0xcf, 0x89, + 0x97, 0xa6, 0x2d, 0x40, 0xd7, 0xd2, 0xb4, 0x29, 0x6f, 0xbb, 0x69, 0xda, 0xce, 0x89, 0x27, 0x8e, + 0x8b, 0x7c, 0x8b, 0x44, 0x29, 0xc2, 0xd4, 0xf4, 0x81, 0xa7, 0xb1, 0x24, 0xd5, 0xfe, 0x81, 0x42, + 0x99, 0x3b, 0xae, 0x00, 0x02, 0xb0, 0x24, 0x5a, 0xd0, 0x25, 0x22, 0xae, 0xd1, 0x82, 0x2e, 0x19, + 0x6f, 0x09, 0x87, 0x86, 0x40, 0x2f, 0x7f, 0xf5, 0xa0, 0x9a, 0x7f, 0xae, 0x01, 0x8a, 0xe3, 0x2a, + 0xe8, 0x49, 0xb2, 0xf4, 0x44, 0x1c, 0xb7, 0xf9, 0xfe, 0x9b, 0x31, 0x27, 0x45, 0xfb, 0xc0, 0xa4, + 0x1e, 0xe3, 0x1e, 0xbd, 0xa6, 0x46, 0xfd, 0x85, 0x06, 0xd5, 0x10, 0x28, 0x83, 0xde, 0x49, 0x59, + 0xd3, 0x08, 0x0c, 0xdc, 0x7c, 0xf7, 0x46, 0xbe, 0xa4, 0x4a, 0x57, 0xd9, 0x01, 0xb2, 0xe4, 0xff, + 0xa9, 0x06, 0xb5, 0x30, 0x88, 0x83, 0x52, 0x64, 0xc7, 0x60, 0xe4, 0xe6, 0xfa, 0xcd, 0x8c, 0xd3, + 0x97, 0x27, 0xa8, 0xf6, 0x07, 0x50, 0x10, 0xb0, 0x4f, 0xd2, 0xc6, 0x0f, 0x03, 0xd0, 0x49, 0x1b, + 0x3f, 0x82, 0x19, 0x25, 0x6c, 0x7c, 0xc7, 0x1e, 0x10, 0xe5, 0x98, 0x09, 0x5c, 0x28, 0x4d, 0xdb, + 0xf4, 0x63, 0x16, 0x01, 0x95, 0xd2, 0xb4, 0x05, 0xc7, 0x4c, 0x02, 0x42, 0x28, 0x45, 0xd8, 0x0d, + 0xc7, 0x2c, 0x8a, 0x27, 0x25, 0x1c, 0x33, 0xa6, 0x50, 0x39, 0x66, 0x01, 0x74, 0x93, 0x74, 0xcc, + 0x62, 0x78, 0x7a, 0xd2, 0x31, 0x8b, 0xa3, 0x3f, 0x09, 0xeb, 0xc8, 0xf4, 0x86, 0x8e, 0xd9, 0x52, + 0x02, 0xca, 0x83, 0xde, 0x4f, 0x71, 0x62, 0x22, 0x4c, 0xdf, 0xfc, 0xe0, 0x0d, 0xb9, 0x53, 0xf7, + 0x38, 0x77, 0xbf, 0xdc, 0xe3, 0x7f, 0xa3, 0xc1, 0x72, 0x12, 0x42, 0x84, 0x52, 0xf4, 0xa4, 0xc0, + 0xfb, 0xcd, 0x8d, 0x37, 0x65, 0x9f, 0xee, 0x2d, 0x7f, 0xd7, 0x3f, 0xab, 0xff, 0xdb, 0x97, 0xab, + 0xda, 0x7f, 0x7e, 0xb9, 0xaa, 0xfd, 0xcf, 0x97, 0xab, 0xda, 0xdf, 0xfe, 0xef, 0xea, 0xdc, 0x69, + 0x9e, 0xfd, 0x56, 0xf2, 0x3b, 0xbf, 0x0e, 0x00, 0x00, 0xff, 0xff, 0xad, 0x27, 0xa6, 0xe1, 0xb2, + 0x39, 0x00, 0x00, } diff --git a/etcdserver/etcdserverpb/rpc.proto b/etcdserver/etcdserverpb/rpc.proto index 8060ca0160f..6fbf08d3cb1 100644 --- a/etcdserver/etcdserverpb/rpc.proto +++ b/etcdserver/etcdserverpb/rpc.proto @@ -165,6 +165,14 @@ service Cluster { body: "*" }; } + + // MemberPromote promotes a member from raft learner (non-voting) to raft voting member. + rpc MemberPromote(MemberPromoteRequest) returns (MemberPromoteResponse) { + option (google.api.http) = { + post: "/v3/cluster/member/promote" + body: "*" + }; + } } service Maintenance { @@ -846,11 +854,15 @@ message Member { repeated string peerURLs = 3; // clientURLs is the list of URLs the member exposes to clients for communication. If the member is not started, clientURLs will be empty. repeated string clientURLs = 4; + // isLearner indicates if the member is raft learner. + bool isLearner = 5; } message MemberAddRequest { // peerURLs is the list of URLs the added member will use to communicate with the cluster. repeated string peerURLs = 1; + // isLearner indicates if the added member is raft learner. + bool isLearner = 2; } message MemberAddResponse { @@ -894,6 +906,17 @@ message MemberListResponse { repeated Member members = 2; } +message MemberPromoteRequest { + // ID is the member ID of the member to promote. + uint64 ID = 1; +} + +message MemberPromoteResponse { + ResponseHeader header = 1; + // members is a list of all members after promoting the member. + repeated Member members = 2; +} + message DefragmentRequest { } @@ -967,6 +990,8 @@ message StatusResponse { repeated string errors = 8; // dbSizeInUse is the size of the backend database logically in use, in bytes, of the responding member. int64 dbSizeInUse = 9; + // isLearner indicates if the member is raft learner. + bool isLearner = 10; } message AuthEnableRequest { diff --git a/etcdserver/server.go b/etcdserver/server.go index 5a97b8341ef..be659eba4de 100644 --- a/etcdserver/server.go +++ b/etcdserver/server.go @@ -97,6 +97,8 @@ const ( maxPendingRevokes = 16 recommendedMaxRequestBytes = 10 * 1024 * 1024 + + readyPercent = 0.9 ) var ( @@ -156,6 +158,11 @@ type Server interface { // UpdateMember attempts to update an existing member in the cluster. It will // return ErrIDNotFound if the member ID does not exist. UpdateMember(ctx context.Context, updateMemb membership.Member) ([]*membership.Member, error) + // PromoteMember attempts to promote a non-voting node to a voting node. It will + // return ErrIDNotFound if the member ID does not exist. + // return ErrLearnerNotReady if the member are not ready. + // return ErrMemberNotLearner if the member is not a learner. + PromoteMember(ctx context.Context, id uint64) ([]*membership.Member, error) // ClusterVersion is the cluster-wide minimum major.minor version. // Cluster version is set to the min version that an etcd member is @@ -1372,16 +1379,16 @@ func (s *EtcdServer) triggerSnapshot(ep *etcdProgress) { ep.snapi = ep.appliedi } -func (s *EtcdServer) isMultiNode() bool { - return s.cluster != nil && len(s.cluster.MemberIDs()) > 1 -} - func (s *EtcdServer) isLeader() bool { return uint64(s.ID()) == s.Lead() } // MoveLeader transfers the leader to the given transferee. func (s *EtcdServer) MoveLeader(ctx context.Context, lead, transferee uint64) error { + if !s.cluster.IsMemberExist(types.ID(transferee)) || s.cluster.Member(types.ID(transferee)).IsLearner { + return ErrBadLeaderTransferee + } + now := time.Now() interval := time.Duration(s.Cfg.TickMs) * time.Millisecond @@ -1435,20 +1442,20 @@ func (s *EtcdServer) TransferLeadership() error { return nil } - if !s.isMultiNode() { + if s.cluster == nil || len(s.cluster.VotingMemberIDs()) <= 1 { if lg := s.getLogger(); lg != nil { lg.Info( - "skipped leadership transfer; it's a single-node cluster", + "skipped leadership transfer for single voting member cluster", zap.String("local-member-id", s.ID().String()), zap.String("current-leader-member-id", types.ID(s.Lead()).String()), ) } else { - plog.Printf("skipped leadership transfer for single member cluster") + plog.Printf("skipped leadership transfer for single voting member cluster") } return nil } - transferee, ok := longestConnected(s.r.transport, s.cluster.MemberIDs()) + transferee, ok := longestConnected(s.r.transport, s.cluster.VotingMemberIDs()) if !ok { return ErrUnhealthy } @@ -1544,50 +1551,67 @@ func (s *EtcdServer) AddMember(ctx context.Context, memb membership.Member) ([]* return nil, err } - if s.Cfg.StrictReconfigCheck { - // by default StrictReconfigCheck is enabled; reject new members if unhealthy - if !s.cluster.IsReadyToAddNewMember() { - if lg := s.getLogger(); lg != nil { - lg.Warn( - "rejecting member add request; not enough healthy members", - zap.String("local-member-id", s.ID().String()), - zap.String("requested-member-add", fmt.Sprintf("%+v", memb)), - zap.Error(ErrNotEnoughStartedMembers), - ) - } else { - plog.Warningf("not enough started members, rejecting member add %+v", memb) - } - return nil, ErrNotEnoughStartedMembers - } - - if !isConnectedFullySince(s.r.transport, time.Now().Add(-HealthInterval), s.ID(), s.cluster.Members()) { - if lg := s.getLogger(); lg != nil { - lg.Warn( - "rejecting member add request; local member has not been connected to all peers, reconfigure breaks active quorum", - zap.String("local-member-id", s.ID().String()), - zap.String("requested-member-add", fmt.Sprintf("%+v", memb)), - zap.Error(ErrUnhealthy), - ) - } else { - plog.Warningf("not healthy for reconfigure, rejecting member add %+v", memb) - } - return nil, ErrUnhealthy - } - } - // TODO: move Member to protobuf type b, err := json.Marshal(memb) if err != nil { return nil, err } + + // by default StrictReconfigCheck is enabled; reject new members if unhealthy. + if err := s.mayAddMember(memb); err != nil { + return nil, err + } + cc := raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: uint64(memb.ID), Context: b, } + + if memb.IsLearner { + cc.Type = raftpb.ConfChangeAddLearnerNode + } + return s.configure(ctx, cc) } +func (s *EtcdServer) mayAddMember(memb membership.Member) error { + if !s.Cfg.StrictReconfigCheck { + return nil + } + + // protect quorum when adding voting member + if !memb.IsLearner && !s.cluster.IsReadyToAddVotingMember() { + if lg := s.getLogger(); lg != nil { + lg.Warn( + "rejecting member add request; not enough healthy members", + zap.String("local-member-id", s.ID().String()), + zap.String("requested-member-add", fmt.Sprintf("%+v", memb)), + zap.Error(ErrNotEnoughStartedMembers), + ) + } else { + plog.Warningf("not enough started members, rejecting member add %+v", memb) + } + return ErrNotEnoughStartedMembers + } + + if !isConnectedFullySince(s.r.transport, time.Now().Add(-HealthInterval), s.ID(), s.cluster.VotingMembers()) { + if lg := s.getLogger(); lg != nil { + lg.Warn( + "rejecting member add request; local member has not been connected to all peers, reconfigure breaks active quorum", + zap.String("local-member-id", s.ID().String()), + zap.String("requested-member-add", fmt.Sprintf("%+v", memb)), + zap.Error(ErrUnhealthy), + ) + } else { + plog.Warningf("not healthy for reconfigure, rejecting member add %+v", memb) + } + return ErrUnhealthy + } + + return nil +} + func (s *EtcdServer) RemoveMember(ctx context.Context, id uint64) ([]*membership.Member, error) { if err := s.checkMembershipOperationPermission(ctx); err != nil { return nil, err @@ -1605,12 +1629,143 @@ func (s *EtcdServer) RemoveMember(ctx context.Context, id uint64) ([]*membership return s.configure(ctx, cc) } +// PromoteMember promotes a learner node to a voting node. +func (s *EtcdServer) PromoteMember(ctx context.Context, id uint64) ([]*membership.Member, error) { + resp, err := s.promoteMember(ctx, id) + if err != ErrNotLeader { + return resp, err + } + + cctx, cancel := context.WithTimeout(ctx, s.Cfg.ReqTimeout()) + defer cancel() + // forward to leader + for cctx.Err() == nil { + leader, err := s.waitLeader(cctx) + if err != nil { + return nil, err + } + for _, url := range leader.PeerURLs { + resp, err := promoteMemberHTTP(cctx, url, id, s.peerRt) + if err == nil { + return resp, nil + } + // If member promotion failed, return early. Otherwise keep retry. + if err == ErrLearnerNotReady || err == membership.ErrIDNotFound || err == membership.ErrMemberNotLearner { + return nil, err + } + } + } + + if cctx.Err() == context.DeadlineExceeded { + return nil, ErrTimeout + } + return nil, ErrCanceled +} + +func (s *EtcdServer) promoteMember(ctx context.Context, id uint64) ([]*membership.Member, error) { + if err := s.checkMembershipOperationPermission(ctx); err != nil { + return nil, err + } + + // check if we can promote this learner. + if err := s.mayPromoteMember(types.ID(id)); err != nil { + return nil, err + } + + // build the context for the promote confChange. mark IsLearner to false and IsPromote to true. + promoteChangeContext := membership.ConfigChangeContext{ + Member: membership.Member{ + ID: types.ID(id), + }, + IsPromote: true, + } + + b, err := json.Marshal(promoteChangeContext) + if err != nil { + return nil, err + } + + cc := raftpb.ConfChange{ + Type: raftpb.ConfChangeAddNode, + NodeID: id, + Context: b, + } + + return s.configure(ctx, cc) +} + +func (s *EtcdServer) mayPromoteMember(id types.ID) error { + err := s.isLearnerReady(uint64(id)) + if err != nil { + return err + } + + if !s.Cfg.StrictReconfigCheck { + return nil + } + if !s.cluster.IsReadyToPromoteMember(uint64(id)) { + if lg := s.getLogger(); lg != nil { + lg.Warn( + "rejecting member promote request; not enough healthy members", + zap.String("local-member-id", s.ID().String()), + zap.String("requested-member-remove-id", id.String()), + zap.Error(ErrNotEnoughStartedMembers), + ) + } else { + plog.Warningf("not enough started members, rejecting promote member %s", id) + } + return ErrNotEnoughStartedMembers + } + + return nil +} + +// check whether the learner catches up with leader or not. +// Note: it will return nil if member is not found in cluster or if member is not learner. +// These two conditions will be checked before apply phase later. +func (s *EtcdServer) isLearnerReady(id uint64) error { + rs := s.raftStatus() + + // leader's raftStatus.Progress is not nil + if rs.Progress == nil { + return ErrNotLeader + } + + var learnerMatch uint64 + isFound := false + leaderID := rs.ID + for memberID, progress := range rs.Progress { + if id == memberID { + // check its status + learnerMatch = progress.Match + isFound = true + break + } + } + + if isFound { + leaderMatch := rs.Progress[leaderID].Match + // the learner's Match not caught up with leader yet + if float64(learnerMatch) < float64(leaderMatch)*readyPercent { + return ErrLearnerNotReady + } + } + + return nil +} + func (s *EtcdServer) mayRemoveMember(id types.ID) error { if !s.Cfg.StrictReconfigCheck { return nil } - if !s.cluster.IsReadyToRemoveMember(uint64(id)) { + isLearner := s.cluster.IsMemberExist(id) && s.cluster.Member(id).IsLearner + // no need to check quorum when removing non-voting member + if isLearner { + return nil + } + + if !s.cluster.IsReadyToRemoveVotingMember(uint64(id)) { if lg := s.getLogger(); lg != nil { lg.Warn( "rejecting member remove request; not enough healthy members", @@ -1630,7 +1785,7 @@ func (s *EtcdServer) mayRemoveMember(id types.ID) error { } // protect quorum if some members are down - m := s.cluster.Members() + m := s.cluster.VotingMembers() active := numConnectedSince(s.r.transport, time.Now().Add(-HealthInterval), s.ID(), m) if (active - 1) < 1+((len(m)-1)/2) { if lg := s.getLogger(); lg != nil { @@ -2054,29 +2209,34 @@ func (s *EtcdServer) applyConfChange(cc raftpb.ConfChange, confState *raftpb.Con lg := s.getLogger() *confState = *s.r.ApplyConfChange(cc) switch cc.Type { - case raftpb.ConfChangeAddNode: - m := new(membership.Member) - if err := json.Unmarshal(cc.Context, m); err != nil { + case raftpb.ConfChangeAddNode, raftpb.ConfChangeAddLearnerNode: + confChangeContext := new(membership.ConfigChangeContext) + if err := json.Unmarshal(cc.Context, confChangeContext); err != nil { if lg != nil { lg.Panic("failed to unmarshal member", zap.Error(err)) } else { plog.Panicf("unmarshal member should never fail: %v", err) } } - if cc.NodeID != uint64(m.ID) { + if cc.NodeID != uint64(confChangeContext.Member.ID) { if lg != nil { lg.Panic( "got different member ID", zap.String("member-id-from-config-change-entry", types.ID(cc.NodeID).String()), - zap.String("member-id-from-message", m.ID.String()), + zap.String("member-id-from-message", confChangeContext.Member.ID.String()), ) } else { plog.Panicf("nodeID should always be equal to member ID") } } - s.cluster.AddMember(m) - if m.ID != s.id { - s.r.transport.AddPeer(m.ID, m.PeerURLs) + if confChangeContext.IsPromote { + s.cluster.PromoteMember(confChangeContext.Member.ID) + } else { + s.cluster.AddMember(&confChangeContext.Member) + + if confChangeContext.Member.ID != s.id { + s.r.transport.AddPeer(confChangeContext.Member.ID, confChangeContext.PeerURLs) + } } case raftpb.ConfChangeRemoveNode: @@ -2434,3 +2594,18 @@ func (s *EtcdServer) Alarms() []*pb.AlarmMember { func (s *EtcdServer) Logger() *zap.Logger { return s.lg } + +// IsLearner returns if the local member is raft learner +func (s *EtcdServer) IsLearner() bool { + return s.cluster.IsLearner() +} + +// IsMemberExist returns if the member with the given id exists in cluster. +func (s *EtcdServer) IsMemberExist(id types.ID) bool { + return s.cluster.IsMemberExist(id) +} + +// raftStatus returns the raft status of this etcd node. +func (s *EtcdServer) raftStatus() raft.Status { + return s.r.Node.Status() +} diff --git a/etcdserver/server_test.go b/etcdserver/server_test.go index 46a5363f815..fd3555de887 100644 --- a/etcdserver/server_test.go +++ b/etcdserver/server_test.go @@ -508,35 +508,57 @@ func TestApplyConfChangeError(t *testing.T) { } cl.RemoveMember(4) + attr := membership.RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 1)}} + ctx, err := json.Marshal(&membership.Member{ID: types.ID(1), RaftAttributes: attr}) + if err != nil { + t.Fatal(err) + } + + attr = membership.RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 4)}} + ctx4, err := json.Marshal(&membership.Member{ID: types.ID(1), RaftAttributes: attr}) + if err != nil { + t.Fatal(err) + } + + attr = membership.RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 5)}} + ctx5, err := json.Marshal(&membership.Member{ID: types.ID(1), RaftAttributes: attr}) + if err != nil { + t.Fatal(err) + } + tests := []struct { cc raftpb.ConfChange werr error }{ { raftpb.ConfChange{ - Type: raftpb.ConfChangeAddNode, - NodeID: 4, + Type: raftpb.ConfChangeAddNode, + NodeID: 4, + Context: ctx4, }, membership.ErrIDRemoved, }, { raftpb.ConfChange{ - Type: raftpb.ConfChangeUpdateNode, - NodeID: 4, + Type: raftpb.ConfChangeUpdateNode, + NodeID: 4, + Context: ctx4, }, membership.ErrIDRemoved, }, { raftpb.ConfChange{ - Type: raftpb.ConfChangeAddNode, - NodeID: 1, + Type: raftpb.ConfChangeAddNode, + NodeID: 1, + Context: ctx, }, membership.ErrIDExists, }, { raftpb.ConfChange{ - Type: raftpb.ConfChangeRemoveNode, - NodeID: 5, + Type: raftpb.ConfChangeRemoveNode, + NodeID: 5, + Context: ctx5, }, membership.ErrIDNotFound, }, @@ -553,7 +575,7 @@ func TestApplyConfChangeError(t *testing.T) { if err != tt.werr { t.Errorf("#%d: applyConfChange error = %v, want %v", i, err, tt.werr) } - cc := raftpb.ConfChange{Type: tt.cc.Type, NodeID: raft.None} + cc := raftpb.ConfChange{Type: tt.cc.Type, NodeID: raft.None, Context: tt.cc.Context} w := []testutil.Action{ { Name: "ApplyConfChange", @@ -634,7 +656,7 @@ func TestApplyConfigChangeUpdatesConsistIndex(t *testing.T) { if err != nil { t.Fatal(err) } - m := membership.NewMember("", urls, "", &now) + m := membership.NewMember("", urls, "", &now, false) m.ID = types.ID(2) b, err := json.Marshal(m) if err != nil { @@ -1564,23 +1586,23 @@ func TestGetOtherPeerURLs(t *testing.T) { }{ { []*membership.Member{ - membership.NewMember("1", types.MustNewURLs([]string{"http://10.0.0.1:1"}), "a", nil), + membership.NewMember("1", types.MustNewURLs([]string{"http://10.0.0.1:1"}), "a", nil, false), }, []string{}, }, { []*membership.Member{ - membership.NewMember("1", types.MustNewURLs([]string{"http://10.0.0.1:1"}), "a", nil), - membership.NewMember("2", types.MustNewURLs([]string{"http://10.0.0.2:2"}), "a", nil), - membership.NewMember("3", types.MustNewURLs([]string{"http://10.0.0.3:3"}), "a", nil), + membership.NewMember("1", types.MustNewURLs([]string{"http://10.0.0.1:1"}), "a", nil, false), + membership.NewMember("2", types.MustNewURLs([]string{"http://10.0.0.2:2"}), "a", nil, false), + membership.NewMember("3", types.MustNewURLs([]string{"http://10.0.0.3:3"}), "a", nil, false), }, []string{"http://10.0.0.2:2", "http://10.0.0.3:3"}, }, { []*membership.Member{ - membership.NewMember("1", types.MustNewURLs([]string{"http://10.0.0.1:1"}), "a", nil), - membership.NewMember("3", types.MustNewURLs([]string{"http://10.0.0.3:3"}), "a", nil), - membership.NewMember("2", types.MustNewURLs([]string{"http://10.0.0.2:2"}), "a", nil), + membership.NewMember("1", types.MustNewURLs([]string{"http://10.0.0.1:1"}), "a", nil, false), + membership.NewMember("3", types.MustNewURLs([]string{"http://10.0.0.3:3"}), "a", nil, false), + membership.NewMember("2", types.MustNewURLs([]string{"http://10.0.0.2:2"}), "a", nil, false), }, []string{"http://10.0.0.2:2", "http://10.0.0.3:3"}, }, diff --git a/etcdserver/v3_server.go b/etcdserver/v3_server.go index 74d98096144..b2084618b8a 100644 --- a/etcdserver/v3_server.go +++ b/etcdserver/v3_server.go @@ -260,7 +260,11 @@ func (s *EtcdServer) LeaseRenew(ctx context.Context, id lease.LeaseID) (int64, e } } } - return -1, ErrTimeout + + if cctx.Err() == context.DeadlineExceeded { + return -1, ErrTimeout + } + return -1, ErrCanceled } func (s *EtcdServer) LeaseTimeToLive(ctx context.Context, r *pb.LeaseTimeToLiveRequest) (*pb.LeaseTimeToLiveResponse, error) { @@ -303,7 +307,11 @@ func (s *EtcdServer) LeaseTimeToLive(ctx context.Context, r *pb.LeaseTimeToLiveR } } } - return nil, ErrTimeout + + if cctx.Err() == context.DeadlineExceeded { + return nil, ErrTimeout + } + return nil, ErrCanceled } func (s *EtcdServer) LeaseLeases(ctx context.Context, r *pb.LeaseLeasesRequest) (*pb.LeaseLeasesResponse, error) { diff --git a/functional/tester/case_sigquit_remove.go b/functional/tester/case_sigquit_remove.go index 6c3a795153b..6946e9d6f9e 100644 --- a/functional/tester/case_sigquit_remove.go +++ b/functional/tester/case_sigquit_remove.go @@ -133,7 +133,7 @@ func recover_SIGQUIT_ETCD_AND_REMOVE_DATA(clus *Cluster, idx1 int) error { } defer cli2.Close() - _, err = cli2.MemberAdd(context.Background(), clus.Members[idx1].Etcd.AdvertisePeerURLs) + _, err = cli2.MemberAdd(context.Background(), clus.Members[idx1].Etcd.AdvertisePeerURLs, false) clus.lg.Info( "member add before fresh restart", zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint), diff --git a/functional/tester/case_sigquit_remove_quorum.go b/functional/tester/case_sigquit_remove_quorum.go index 5fc78cdd3a4..ef63635e985 100644 --- a/functional/tester/case_sigquit_remove_quorum.go +++ b/functional/tester/case_sigquit_remove_quorum.go @@ -191,7 +191,7 @@ func (c *fetchSnapshotCaseQuorum) Recover(clus *Cluster) error { zap.Strings("peer-urls", clus.Members[idx].Etcd.AdvertisePeerURLs), ) ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - _, err := leaderc.MemberAdd(ctx, clus.Members[idx].Etcd.AdvertisePeerURLs) + _, err := leaderc.MemberAdd(ctx, clus.Members[idx].Etcd.AdvertisePeerURLs, false) cancel() clus.lg.Info( "member add request DONE", diff --git a/integration/cluster.go b/integration/cluster.go index 5585250e0ae..963e0a8ba6d 100644 --- a/integration/cluster.go +++ b/integration/cluster.go @@ -559,6 +559,8 @@ type member struct { clientMaxCallSendMsgSize int clientMaxCallRecvMsgSize int useIP bool + + isLearner bool } func (m *member) GRPCAddr() string { return m.grpcAddr } @@ -1164,6 +1166,10 @@ func (m *member) RecoverPartition(t testing.TB, others ...*member) { } } +func (m *member) ReadyNotify() <-chan struct{} { + return m.s.ReadyNotify() +} + func MustNewHTTPClient(t testing.TB, eps []string, tls *transport.TLSInfo) client.Client { cfgtls := transport.TLSInfo{} if tls != nil { @@ -1272,3 +1278,136 @@ type grpcAPI struct { // Election is the election API for the client's connection. Election epb.ElectionClient } + +// GetLearnerMembers returns the list of learner members in cluster using MemberList API. +func (c *ClusterV3) GetLearnerMembers() ([]*pb.Member, error) { + cli := c.Client(0) + resp, err := cli.MemberList(context.Background()) + if err != nil { + return nil, fmt.Errorf("failed to list member %v", err) + } + var learners []*pb.Member + for _, m := range resp.Members { + if m.IsLearner { + learners = append(learners, m) + } + } + return learners, nil +} + +// AddAndLaunchLearnerMember creates a leaner member, adds it to cluster +// via v3 MemberAdd API, and then launches the new member. +func (c *ClusterV3) AddAndLaunchLearnerMember(t testing.TB) { + m := c.mustNewMember(t) + m.isLearner = true + + scheme := schemeFromTLSInfo(c.cfg.PeerTLS) + peerURLs := []string{scheme + "://" + m.PeerListeners[0].Addr().String()} + + cli := c.Client(0) + _, err := cli.MemberAdd(context.Background(), peerURLs, m.isLearner) + if err != nil { + t.Fatalf("failed to add learner member %v", err) + } + + m.InitialPeerURLsMap = types.URLsMap{} + for _, mm := range c.Members { + m.InitialPeerURLsMap[mm.Name] = mm.PeerURLs + } + m.InitialPeerURLsMap[m.Name] = m.PeerURLs + m.NewCluster = false + + if err := m.Launch(); err != nil { + t.Fatal(err) + } + + c.Members = append(c.Members, m) + + c.waitMembersMatch(t) +} + +// getMembers returns a list of members in cluster, in format of etcdserverpb.Member +func (c *ClusterV3) getMembers() []*pb.Member { + var mems []*pb.Member + for _, m := range c.Members { + mem := &pb.Member{ + Name: m.Name, + PeerURLs: m.PeerURLs.StringSlice(), + ClientURLs: m.ClientURLs.StringSlice(), + IsLearner: m.isLearner, + } + mems = append(mems, mem) + } + return mems +} + +// waitMembersMatch waits until v3rpc MemberList returns the 'same' members info as the +// local 'c.Members', which is the local recording of members in the testing cluster. With +// the exception that the local recording c.Members does not have info on Member.ID, which +// is generated when the member is been added to cluster. +// +// Note: +// A successful match means the Member.clientURLs are matched. This means member has already +// finished publishing its server attributes to cluster. Publishing attributes is a cluster-wide +// write request (in v2 server). Therefore, at this point, any raft log entries prior to this +// would have already been applied. +// +// If a new member was added to an existing cluster, at this point, it has finished publishing +// its own server attributes to the cluster. And therefore by the same argument, it has already +// applied the raft log entries (especially those of type raftpb.ConfChangeType). At this point, +// the new member has the correct view of the cluster configuration. +// +// Special note on learner member: +// Learner member is only added to a cluster via v3rpc MemberAdd API (as of v3.4). When starting +// the learner member, its initial view of the cluster created by peerURLs map does not have info +// on whether or not the new member itself is learner. But at this point, a successful match does +// indicate that the new learner member has applied the raftpb.ConfChangeAddLearnerNode entry +// which was used to add the learner itself to the cluster, and therefore it has the correct info +// on learner. +func (c *ClusterV3) waitMembersMatch(t testing.TB) { + wMembers := c.getMembers() + sort.Sort(SortableProtoMemberSliceByPeerURLs(wMembers)) + cli := c.Client(0) + for { + resp, err := cli.MemberList(context.Background()) + if err != nil { + t.Fatalf("failed to list member %v", err) + } + + if len(resp.Members) != len(wMembers) { + continue + } + sort.Sort(SortableProtoMemberSliceByPeerURLs(resp.Members)) + for _, m := range resp.Members { + m.ID = 0 + } + if reflect.DeepEqual(resp.Members, wMembers) { + return + } + + time.Sleep(tickDuration) + } +} + +type SortableProtoMemberSliceByPeerURLs []*pb.Member + +func (p SortableProtoMemberSliceByPeerURLs) Len() int { return len(p) } +func (p SortableProtoMemberSliceByPeerURLs) Less(i, j int) bool { + return p[i].PeerURLs[0] < p[j].PeerURLs[0] +} +func (p SortableProtoMemberSliceByPeerURLs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +// MustNewMember creates a new member instance based on the response of V3 Member Add API. +func (c *ClusterV3) MustNewMember(t testing.TB, resp *clientv3.MemberAddResponse) *member { + m := c.mustNewMember(t) + m.isLearner = resp.Member.IsLearner + m.NewCluster = false + + m.InitialPeerURLsMap = types.URLsMap{} + for _, mm := range c.Members { + m.InitialPeerURLsMap[mm.Name] = mm.PeerURLs + } + m.InitialPeerURLsMap[m.Name] = types.MustNewURLs(resp.Member.PeerURLs) + + return m +} diff --git a/integration/v3_leadership_test.go b/integration/v3_leadership_test.go index 1dcb98b676f..29818113a14 100644 --- a/integration/v3_leadership_test.go +++ b/integration/v3_leadership_test.go @@ -16,6 +16,7 @@ package integration import ( "context" + "strings" "testing" "time" @@ -106,3 +107,70 @@ func TestMoveLeaderError(t *testing.T) { t.Errorf("err = %v, want %v", err, rpctypes.ErrGRPCNotLeader) } } + +// TestMoveLeaderToLearnerError ensures that leader transfer to learner member will fail. +func TestMoveLeaderToLearnerError(t *testing.T) { + defer testutil.AfterTest(t) + + clus := NewClusterV3(t, &ClusterConfig{Size: 3}) + defer clus.Terminate(t) + + // we have to add and launch learner member after initial cluster was created, because + // bootstrapping a cluster with learner member is not supported. + clus.AddAndLaunchLearnerMember(t) + + learners, err := clus.GetLearnerMembers() + if err != nil { + t.Fatalf("failed to get the learner members in cluster: %v", err) + } + if len(learners) != 1 { + t.Fatalf("added 1 learner to cluster, got %d", len(learners)) + } + + learnerID := learners[0].ID + leaderIdx := clus.WaitLeader(t) + cli := clus.Client(leaderIdx) + _, err = cli.MoveLeader(context.Background(), learnerID) + if err == nil { + t.Fatalf("expecting leader transfer to learner to fail, got no error") + } + expectedErrKeywords := "bad leader transferee" + if !strings.Contains(err.Error(), expectedErrKeywords) { + t.Errorf("expecting error to contain %s, got %s", expectedErrKeywords, err.Error()) + } +} + +// TestTransferLeadershipWithLearner ensures TransferLeadership does not timeout due to learner is +// automatically picked by leader as transferee. +func TestTransferLeadershipWithLearner(t *testing.T) { + defer testutil.AfterTest(t) + + clus := NewClusterV3(t, &ClusterConfig{Size: 1}) + defer clus.Terminate(t) + + clus.AddAndLaunchLearnerMember(t) + + learners, err := clus.GetLearnerMembers() + if err != nil { + t.Fatalf("failed to get the learner members in cluster: %v", err) + } + if len(learners) != 1 { + t.Fatalf("added 1 learner to cluster, got %d", len(learners)) + } + + leaderIdx := clus.WaitLeader(t) + errCh := make(chan error, 1) + go func() { + // note that this cluster has 1 leader and 1 learner. TransferLeadership should return nil. + // Leadership transfer is skipped in cluster with 1 voting member. + errCh <- clus.Members[leaderIdx].s.TransferLeadership() + }() + select { + case err := <-errCh: + if err != nil { + t.Errorf("got error during leadership transfer: %v", err) + } + case <-time.After(5 * time.Second): + t.Error("timed out waiting for leader transition") + } +} diff --git a/proxy/grpcproxy/adapter/cluster_client_adapter.go b/proxy/grpcproxy/adapter/cluster_client_adapter.go index 248dffd6461..73a6fdfcba5 100644 --- a/proxy/grpcproxy/adapter/cluster_client_adapter.go +++ b/proxy/grpcproxy/adapter/cluster_client_adapter.go @@ -43,3 +43,7 @@ func (s *cls2clc) MemberUpdate(ctx context.Context, r *pb.MemberUpdateRequest, o func (s *cls2clc) MemberRemove(ctx context.Context, r *pb.MemberRemoveRequest, opts ...grpc.CallOption) (*pb.MemberRemoveResponse, error) { return s.cls.MemberRemove(ctx, r) } + +func (s *cls2clc) MemberPromote(ctx context.Context, r *pb.MemberPromoteRequest, opts ...grpc.CallOption) (*pb.MemberPromoteResponse, error) { + return s.cls.MemberPromote(ctx, r) +} diff --git a/proxy/grpcproxy/cluster.go b/proxy/grpcproxy/cluster.go index ebab4ba446a..bce02c76ee0 100644 --- a/proxy/grpcproxy/cluster.go +++ b/proxy/grpcproxy/cluster.go @@ -25,6 +25,7 @@ import ( "go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes" pb "go.etcd.io/etcd/etcdserver/etcdserverpb" + "errors" "golang.org/x/time/rate" gnaming "google.golang.org/grpc/naming" ) @@ -108,7 +109,7 @@ func (cp *clusterProxy) monitor(wa gnaming.Watcher) { } func (cp *clusterProxy) MemberAdd(ctx context.Context, r *pb.MemberAddRequest) (*pb.MemberAddResponse, error) { - mresp, err := cp.clus.MemberAdd(ctx, r.PeerURLs) + mresp, err := cp.clus.MemberAdd(ctx, r.PeerURLs, r.IsLearner) if err != nil { return nil, err } @@ -175,3 +176,8 @@ func (cp *clusterProxy) MemberList(ctx context.Context, r *pb.MemberListRequest) resp := (pb.MemberListResponse)(*mresp) return &resp, err } + +func (cp *clusterProxy) MemberPromote(ctx context.Context, r *pb.MemberPromoteRequest) (*pb.MemberPromoteResponse, error) { + // TODO: implement + return nil, errors.New("not implemented") +} diff --git a/tests/e2e/ctl_v3_auth_test.go b/tests/e2e/ctl_v3_auth_test.go index ea4d3797dda..e47660ac8c5 100644 --- a/tests/e2e/ctl_v3_auth_test.go +++ b/tests/e2e/ctl_v3_auth_test.go @@ -510,13 +510,13 @@ func authTestMemberAdd(cx ctlCtx) { peerURL := fmt.Sprintf("http://localhost:%d", etcdProcessBasePort+11) // ordinary user cannot add a new member cx.user, cx.pass = "test-user", "pass" - if err := ctlV3MemberAdd(cx, peerURL); err == nil { + if err := ctlV3MemberAdd(cx, peerURL, false); err == nil { cx.t.Fatalf("ordinary user must not be allowed to add a member") } // root can add a new member cx.user, cx.pass = "root", "root" - if err := ctlV3MemberAdd(cx, peerURL); err != nil { + if err := ctlV3MemberAdd(cx, peerURL, false); err != nil { cx.t.Fatal(err) } } diff --git a/tests/e2e/ctl_v3_member_test.go b/tests/e2e/ctl_v3_member_test.go index 417f8672b67..61961c9f672 100644 --- a/tests/e2e/ctl_v3_member_test.go +++ b/tests/e2e/ctl_v3_member_test.go @@ -59,9 +59,10 @@ func TestCtlV3MemberAddClientTLS(t *testing.T) { testCtl(t, memberAddTest, withC func TestCtlV3MemberAddClientAutoTLS(t *testing.T) { testCtl(t, memberAddTest, withCfg(configClientAutoTLS)) } -func TestCtlV3MemberAddPeerTLS(t *testing.T) { testCtl(t, memberAddTest, withCfg(configPeerTLS)) } -func TestCtlV3MemberUpdate(t *testing.T) { testCtl(t, memberUpdateTest) } -func TestCtlV3MemberUpdateNoTLS(t *testing.T) { testCtl(t, memberUpdateTest, withCfg(configNoTLS)) } +func TestCtlV3MemberAddPeerTLS(t *testing.T) { testCtl(t, memberAddTest, withCfg(configPeerTLS)) } +func TestCtlV3MemberAddForLearner(t *testing.T) { testCtl(t, memberAddForLearnerTest) } +func TestCtlV3MemberUpdate(t *testing.T) { testCtl(t, memberUpdateTest) } +func TestCtlV3MemberUpdateNoTLS(t *testing.T) { testCtl(t, memberUpdateTest, withCfg(configNoTLS)) } func TestCtlV3MemberUpdateClientTLS(t *testing.T) { testCtl(t, memberUpdateTest, withCfg(configClientTLS)) } @@ -122,13 +123,22 @@ func ctlV3MemberRemove(cx ctlCtx, ep, memberID, clusterID string) error { } func memberAddTest(cx ctlCtx) { - if err := ctlV3MemberAdd(cx, fmt.Sprintf("http://localhost:%d", etcdProcessBasePort+11)); err != nil { + if err := ctlV3MemberAdd(cx, fmt.Sprintf("http://localhost:%d", etcdProcessBasePort+11), false); err != nil { cx.t.Fatal(err) } } -func ctlV3MemberAdd(cx ctlCtx, peerURL string) error { +func memberAddForLearnerTest(cx ctlCtx) { + if err := ctlV3MemberAdd(cx, fmt.Sprintf("http://localhost:%d", etcdProcessBasePort+11), true); err != nil { + cx.t.Fatal(err) + } +} + +func ctlV3MemberAdd(cx ctlCtx, peerURL string, isLearner bool) error { cmdArgs := append(cx.PrefixArgs(), "member", "add", "newmember", fmt.Sprintf("--peer-urls=%s", peerURL)) + if isLearner { + cmdArgs = append(cmdArgs, "--learner") + } return spawnWithExpect(cmdArgs, " added to cluster ") }