etcdserver: add auth revision to AuthStatus to improve observability …

…and testability

Documentation/dev-guide/api_reference_v3.md

@@ -253,6 +253,7 @@ Empty field.
 | ----- | ----------- | ---- |
 | header |  | ResponseHeader |
 | enabled |  | bool |
+| authRevision | authRevision is the current revision of auth store | uint64 |
 
 
 

Documentation/dev-guide/apispec/swagger/rpc.swagger.json

@@ -1731,6 +1731,11 @@
     "etcdserverpbAuthStatusResponse": {
       "type": "object",
       "properties": {
+        "authRevision": {
+          "type": "string",
+          "format": "uint64",
+          "title": "authRevision is the current revision of auth store"
+        },
         "enabled": {
           "type": "boolean",
           "format": "boolean"

etcdctl/ctlv3/command/auth_command.go

@@ -56,7 +56,7 @@ func authStatusCommandFunc(cmd *cobra.Command, args []string) {
 		ExitWithError(ExitError, err)
 	}
 
-	fmt.Println("Authentication Status:", result.Enabled)
+	display.AuthStatus(*result)
 }
 
 func newAuthEnableCommand() *cobra.Command {

etcdctl/ctlv3/command/printer.go

@@ -67,6 +67,8 @@ type printer interface {
 	UserGrantRole(user string, role string, r v3.AuthUserGrantRoleResponse)
 	UserRevokeRole(user string, role string, r v3.AuthUserRevokeRoleResponse)
 	UserDelete(user string, r v3.AuthUserDeleteResponse)
+
+	AuthStatus(r v3.AuthStatusResponse)
 }
 
 func NewPrinter(printerType string, isHex bool) printer {
@@ -141,6 +143,9 @@ func (p *printerRPC) UserRevokeRole(_ string, _ string, r v3.AuthUserRevokeRoleR
 func (p *printerRPC) UserDelete(_ string, r v3.AuthUserDeleteResponse) {
 	p.p((*pb.AuthUserDeleteResponse)(&r))
 }
+func (p *printerRPC) AuthStatus(r v3.AuthStatusResponse) {
+	p.p((*pb.AuthStatusResponse)(&r))
+}
 
 type printerUnsupported struct{ printerRPC }
 

etcdctl/ctlv3/command/printer_simple.go

@@ -285,3 +285,8 @@ func (s *simplePrinter) UserList(r v3.AuthUserListResponse) {
 		fmt.Printf("%s\n", user)
 	}
 }
+
+func (s *simplePrinter) AuthStatus(r v3.AuthStatusResponse) {
+	fmt.Println("Authentication Status:", r.Enabled)
+	fmt.Println("AuthRevision:", r.AuthRevision)
+}

etcdserver/apply.go

@@ -716,7 +716,8 @@ func (a *applierV3backend) AuthDisable() (*pb.AuthDisableResponse, error) {
 
 func (a *applierV3backend) AuthStatus() (*pb.AuthStatusResponse, error) {
 	enabled := a.s.AuthStore().IsAuthEnabled()
-	return &pb.AuthStatusResponse{Header: newHeader(a.s), Enabled: enabled}, nil
+	authRevision := a.s.AuthStore().Revision()
+	return &pb.AuthStatusResponse{Header: newHeader(a.s), Enabled: enabled, AuthRevision: authRevision}, nil
 }
 
 func (a *applierV3backend) Authenticate(r *pb.InternalAuthenticateRequest) (*pb.AuthenticateResponse, error) {

etcdserver/etcdserverpb/rpc.proto

@@ -1093,6 +1093,8 @@ message AuthDisableResponse {
 message AuthStatusResponse {
   ResponseHeader header = 1;
   bool enabled = 2;
+  // authRevision is the current revision of auth store
+  uint64 authRevision = 3;
 }
 
 message AuthenticateResponse {

tests/e2e/ctl_v3_auth_test.go

@@ -15,8 +15,10 @@
 package e2e
 
 import (
+	"context"
 	"fmt"
 	"os"
+	"syscall"
 	"testing"
 	"time"
 
@@ -25,6 +27,7 @@ import (
 
 func TestCtlV3AuthEnable(t *testing.T)              { testCtl(t, authEnableTest) }
 func TestCtlV3AuthDisable(t *testing.T)             { testCtl(t, authDisableTest) }
+func TestCtlV3AuthStatus(t *testing.T)              { testCtl(t, authStatusTest) }
 func TestCtlV3AuthWriteKey(t *testing.T)            { testCtl(t, authCredWriteKeyTest) }
 func TestCtlV3AuthRoleUpdate(t *testing.T)          { testCtl(t, authRoleUpdateTest) }
 func TestCtlV3AuthUserDeleteDuringOps(t *testing.T) { testCtl(t, authUserDeleteDuringOpsTest) }
@@ -69,6 +72,7 @@ func TestCtlV3AuthJWTExpire(t *testing.T) { testCtl(t, authTestJWTExpire, withCf
 func TestCtlV3AuthCertCNAndUsernameNoPassword(t *testing.T) {
 	testCtl(t, authTestCertCNAndUsernameNoPassword, withCfg(configClientTLSCertAuth))
 }
+func TestCtlV3AuthRevisionConsistency(t *testing.T) { testCtl(t, authTestRevisionConsistency) }
 
 func authEnableTest(cx ctlCtx) {
 	if err := authEnable(cx); err != nil {
@@ -141,6 +145,32 @@ func ctlV3AuthDisable(cx ctlCtx) error {
 	return spawnWithExpect(cmdArgs, "Authentication Disabled")
 }
 
+func authStatusTest(cx ctlCtx) {
+	cmdArgs := append(cx.PrefixArgs(), "auth", "status")
+	if err := spawnWithExpects(cmdArgs, "Authentication Status: false", "AuthRevision:"); err != nil {
+		cx.t.Fatal(err)
+	}
+
+	if err := authEnable(cx); err != nil {
+		cx.t.Fatal(err)
+	}
+
+	cx.user, cx.pass = "root", "root"
+	cmdArgs = append(cx.PrefixArgs(), "auth", "status")
+
+	if err := spawnWithExpects(cmdArgs, "Authentication Status: true", "AuthRevision:"); err != nil {
+		cx.t.Fatal(err)
+	}
+
+	cmdArgs = append(cx.PrefixArgs(), "auth", "status", "--write-out", "json")
+	if err := spawnWithExpect(cmdArgs, "enabled"); err != nil {
+		cx.t.Fatal(err)
+	}
+	if err := spawnWithExpect(cmdArgs, "authRevision"); err != nil {
+		cx.t.Fatal(err)
+	}
+}
+
 func authCredWriteKeyTest(cx ctlCtx) {
 	// baseline key to check for failed puts
 	if err := ctlV3Put(cx, "foo", "a", ""); err != nil {
@@ -1130,3 +1160,52 @@ func authTestJWTExpire(cx ctlCtx) {
 		cx.t.Error(err)
 	}
 }
+
+func authTestRevisionConsistency(cx ctlCtx) {
+	if err := authEnable(cx); err != nil {
+		cx.t.Fatal(err)
+	}
+	cx.user, cx.pass = "root", "root"
+
+	// add user
+	if err := ctlV3User(cx, []string{"add", "test-user", "--interactive=false"}, "User test-user created", []string{"pass"}); err != nil {
+		cx.t.Fatal(err)
+	}
+	// delete the same user
+	if err := ctlV3User(cx, []string{"delete", "test-user"}, "User test-user deleted", []string{}); err != nil {
+		cx.t.Fatal(err)
+	}
+
+	// get node0 auth revision
+	node0 := cx.epc.procs[0]
+	endpoint := node0.EndpointsV3()[0]
+	cli, err := clientv3.New(clientv3.Config{Endpoints: []string{endpoint}, Username: cx.user, Password: cx.pass, DialTimeout: 3 * time.Second})
+	if err != nil {
+		cx.t.Fatal(err)
+	}
+	defer cli.Close()
+
+	sresp, err := cli.AuthStatus(context.TODO())
+	if err != nil {
+		cx.t.Fatal(err)
+	}
+	oldAuthRevision := sresp.AuthRevision
+
+	// restart the node
+	node0.WithStopSignal(syscall.SIGINT)
+	if err := node0.Restart(); err != nil {
+		cx.t.Fatal(err)
+	}
+
+	// get node0 auth revision again
+	sresp, err = cli.AuthStatus(context.TODO())
+	if err != nil {
+		cx.t.Fatal(err)
+	}
+	newAuthRevision := sresp.AuthRevision
+
+	// assert AuthRevision equal
+	if newAuthRevision != oldAuthRevision {
+		cx.t.Fatalf("auth revison shouldn't change when restarting etcd, expected: %d, got: %d", oldAuthRevision, newAuthRevision)
+	}
+}