[sdk#999] Make sandbox entries restartable

pkg/networkservice/chains/client/client_heal_test.go

@@ -29,7 +29,6 @@ import (
 
 	"github.com/networkservicemesh/sdk/pkg/networkservice/chains/client"
 	"github.com/networkservicemesh/sdk/pkg/networkservice/chains/endpoint"
-	"github.com/networkservicemesh/sdk/pkg/tools/grpcutils"
 	"github.com/networkservicemesh/sdk/pkg/tools/sandbox"
 )
 
@@ -52,7 +51,7 @@ func TestClientHeal(t *testing.T) {
 
 	serverCancel()
 	require.Eventually(t, func() bool {
-		return grpcutils.CheckURLFree(serverURL)
+		return sandbox.CheckURLFree(serverURL)
 	}, time.Second, time.Millisecond*10)
 	require.NoError(t, ctx.Err())
 

pkg/networkservice/chains/nsmgr/heal_test.go

@@ -27,8 +27,6 @@ import (
 
 	"github.com/networkservicemesh/api/pkg/api/registry"
 
-	"github.com/networkservicemesh/sdk/pkg/networkservice/chains/nsmgr"
-	"github.com/networkservicemesh/sdk/pkg/tools/grpcutils"
 	"github.com/networkservicemesh/sdk/pkg/tools/sandbox"
 )
 
@@ -78,10 +76,8 @@ func testNSMGRHealEndpoint(t *testing.T, nodeNum int) {
 
 	nseReg := defaultRegistryEndpoint(nsReg.Name)
 
-	nseCtx, nseCtxCancel := context.WithCancel(context.Background())
-
 	counter := &counterServer{}
-	domain.Nodes[nodeNum].NewEndpoint(nseCtx, nseReg, sandbox.GenerateTestToken, counter)
+	nse := domain.Nodes[nodeNum].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, counter)
 
 	request := defaultRequest(nsReg.Name)
 
@@ -91,7 +87,7 @@ func testNSMGRHealEndpoint(t *testing.T, nodeNum int) {
 	require.NoError(t, err)
 	require.Equal(t, 1, counter.UniqueRequests())
 
-	nseCtxCancel()
+	nse.Cancel()
 
 	nseReg2 := defaultRegistryEndpoint(nsReg.Name)
 	nseReg2.Name += "-2"
@@ -143,18 +139,10 @@ func testNSMGRHealForwarder(t *testing.T, nodeNum int) {
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
-	var forwarderCtxCancel context.CancelFunc
 	domain := sandbox.NewBuilder(ctx, t).
 		SetNodesCount(2).
 		SetNSMgrProxySupplier(nil).
 		SetRegistryProxySupplier(nil).
-		SetNodeSetup(func(ctx context.Context, node *sandbox.Node, nodeN int) {
-			if nodeN != nodeNum {
-				sandbox.SetupDefaultNode(ctx, node, nsmgr.NewServer)
-			} else {
-				forwarderCtxCancel = setupCancelableForwarderNode(ctx, node)
-			}
-		}).
 		Build()
 
 	nsRegistryClient := domain.NewNSRegistryClient(ctx, sandbox.GenerateTestToken)
@@ -173,7 +161,10 @@ func testNSMGRHealForwarder(t *testing.T, nodeNum int) {
 	require.NoError(t, err)
 	require.Equal(t, 1, counter.UniqueRequests())
 
-	forwarderCtxCancel()
+	for _, forwarder := range domain.Nodes[nodeNum].Forwarders {
+		forwarder.Cancel()
+		break
+	}
 
 	forwarderReg := &registry.NetworkServiceEndpoint{
 		Name: sandbox.UniqueName("forwarder-2"),
@@ -197,17 +188,6 @@ func testNSMGRHealForwarder(t *testing.T, nodeNum int) {
 	require.Equal(t, 1, counter.UniqueCloses())
 }
 
-func setupCancelableForwarderNode(ctx context.Context, node *sandbox.Node) context.CancelFunc {
-	node.NewNSMgr(ctx, sandbox.UniqueName("nsmgr"), nil, sandbox.GenerateTestToken, nsmgr.NewServer)
-
-	forwarderCtx, forwarderCtxCancel := context.WithCancel(ctx)
-	node.NewForwarder(forwarderCtx, &registry.NetworkServiceEndpoint{
-		Name: sandbox.UniqueName("forwarder"),
-	}, sandbox.GenerateTestToken)
-
-	return forwarderCtxCancel
-}
-
 func TestNSMGR_HealNSMgr(t *testing.T) {
 	var samples = []struct {
 		name     string
@@ -239,18 +219,10 @@ func testNSMGRHealNSMgr(t *testing.T, nodeNum int, restored bool) {
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
-	var nsmgrCtxCancel context.CancelFunc
 	domain := sandbox.NewBuilder(ctx, t).
 		SetNodesCount(3).
 		SetNSMgrProxySupplier(nil).
 		SetRegistryProxySupplier(nil).
-		SetNodeSetup(func(ctx context.Context, node *sandbox.Node, nodeN int) {
-			if nodeN != nodeNum {
-				sandbox.SetupDefaultNode(ctx, node, nsmgr.NewServer)
-			} else {
-				nsmgrCtxCancel = setupCancellableNSMgrNode(ctx, node)
-			}
-		}).
 		Build()
 
 	nsRegistryClient := domain.NewNSRegistryClient(ctx, sandbox.GenerateTestToken)
@@ -275,19 +247,10 @@ func testNSMGRHealNSMgr(t *testing.T, nodeNum int, restored bool) {
 		nseReg2.Name += "-2"
 
 		domain.Nodes[2].NewEndpoint(ctx, nseReg2, sandbox.GenerateTestToken, counter)
-	}
-
-	nsmgrCtxCancel()
-
-	if restored {
-		mgr := domain.Nodes[nodeNum].NSMgr
-
-		// Wait grpc unblock the port
-		require.Eventually(t, func() bool {
-			return grpcutils.CheckURLFree(domain.Nodes[0].NSMgr.URL)
-		}, timeout, tick)
 
-		domain.Nodes[nodeNum].NewNSMgr(ctx, mgr.Name, mgr.URL, sandbox.GenerateTestToken, nsmgr.NewServer)
+		domain.Nodes[nodeNum].NSMgr.Cancel()
+	} else {
+		domain.Nodes[nodeNum].NSMgr.Restart()
 	}
 
 	if restored {
@@ -320,16 +283,50 @@ func testNSMGRHealNSMgr(t *testing.T, nodeNum int, restored bool) {
 	}
 }
 
-func setupCancellableNSMgrNode(ctx context.Context, node *sandbox.Node) context.CancelFunc {
-	nsmgrCtx, nsmgrCtxCancel := context.WithCancel(ctx)
+func TestNSMGR_HealRegistry(t *testing.T) {
+	t.Cleanup(func() { goleak.VerifyNone(t) })
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	domain := sandbox.NewBuilder(ctx, t).
+		SetNodesCount(1).
+		SetNSMgrProxySupplier(nil).
+		SetRegistryProxySupplier(nil).
+		Build()
+
+	nsRegistryClient := domain.NewNSRegistryClient(ctx, sandbox.GenerateTestToken)
+
+	nsReg, err := nsRegistryClient.Register(ctx, defaultRegistryService())
+	require.NoError(t, err)
+
+	nseReg := defaultRegistryEndpoint(nsReg.Name)
+
+	counter := &counterServer{}
+	domain.Nodes[0].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, counter)
 
-	node.NewNSMgr(nsmgrCtx, sandbox.UniqueName("nsmgr"), nil, sandbox.GenerateTestToken, nsmgr.NewServer)
+	request := defaultRequest(nsReg.Name)
+
+	nsc := domain.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken)
 
-	node.NewForwarder(ctx, &registry.NetworkServiceEndpoint{
-		Name: sandbox.UniqueName("forwarder"),
-	}, sandbox.GenerateTestToken)
+	conn, err := nsc.Request(ctx, request.Clone())
+	require.NoError(t, err)
+
+	// 1. Restart Registry
+	domain.Registry.Restart()
+
+	// 2. Check refresh
+	request.Connection = conn
+	_, err = nsc.Request(ctx, request.Clone())
+	require.NoError(t, err)
+
+	// 3. Check new client request
+	nsc = domain.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken)
+
+	_, err = nsc.Request(ctx, request.Clone())
+	require.NoError(t, err)
 
-	return nsmgrCtxCancel
+	require.Equal(t, int32(3), atomic.LoadInt32(&counter.Requests))
 }
 
 func TestNSMGR_CloseHeal(t *testing.T) {

pkg/networkservice/chains/nsmgr/suite_test.go

@@ -161,7 +161,7 @@ func (s *nsmgrSuite) Test_SelectsRestartingEndpointUsecase() {
 	})
 	require.NoError(t, err)
 
-	nseRegistryClient := registryclient.NewNetworkServiceEndpointRegistryClient(ctx, s.domain.Nodes[0].URL(),
+	nseRegistryClient := registryclient.NewNetworkServiceEndpointRegistryClient(ctx, sandbox.CloneURL(s.domain.Nodes[0].NSMgr.URL),
 		registryclient.WithDialOptions(sandbox.DefaultDialOptions(sandbox.GenerateTestToken)...))
 
 	nseReg, err = nseRegistryClient.Register(ctx, nseReg)
@@ -311,7 +311,6 @@ func (s *nsmgrSuite) Test_ConnectToDeadNSEUsecase() {
 	t := s.T()
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	nseCtx, killNse := context.WithCancel(ctx)
 	defer cancel()
 
 	nsReg, err := s.nsRegistryClient.Register(ctx, defaultRegistryService())
@@ -320,7 +319,7 @@ func (s *nsmgrSuite) Test_ConnectToDeadNSEUsecase() {
 	nseReg := defaultRegistryEndpoint(nsReg.Name)
 	counter := &counterServer{}
 
-	s.domain.Nodes[0].NewEndpoint(nseCtx, nseReg, sandbox.GenerateTestToken, counter)
+	nse := s.domain.Nodes[0].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, counter)
 
 	nsc := s.domain.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken)
 
@@ -335,7 +334,7 @@ func (s *nsmgrSuite) Test_ConnectToDeadNSEUsecase() {
 	require.Equal(t, int32(1), atomic.LoadInt32(&counter.Requests))
 	require.Equal(t, 5, len(conn.Path.PathSegments))
 
-	killNse()
+	nse.Cancel()
 
 	// Simulate refresh from client
 	refreshRequest := request.Clone()
@@ -813,7 +812,7 @@ func newPassThroughEndpoint(
 
 	var additionalFunctionality []networkservice.NetworkServiceServer
 	if hasClientFunctionality {
-		additionalFunctionality = additionalFunctionalityChain(ctx, node.URL(), name, labels)
+		additionalFunctionality = additionalFunctionalityChain(ctx, node.NSMgr.URL, name, labels)
 	}
 
 	if counter != nil {

pkg/networkservice/chains/nsmgrproxy/server_test.go

@@ -488,7 +488,7 @@ func Test_Interdomain_PassThroughUsecase(t *testing.T) {
 			// Passtrough to the node i-1
 			additionalFunctionality = []networkservice.NetworkServiceServer{
 				chain.NewNetworkServiceServer(
-					clienturl.NewServer(clusters[i].Nodes[0].URL()),
+					clienturl.NewServer(clusters[i].Nodes[0].NSMgr.URL),
 					connect.NewServer(ctx,
 						client.NewClientFactory(client.WithAdditionalFunctionality(
 							newPassTroughClient(fmt.Sprintf("my-service-remote-%v@cluster%v", i-1, i-1)),

pkg/networkservice/common/connect/server_cancel_test.go

@@ -102,7 +102,7 @@ func TestConnect_CancelDuringRequest(t *testing.T) {
 	}
 	domain.Nodes[0].NewEndpoint(ctx, nseReg2, sandbox.GenerateTestToken,
 		chain.NewNetworkServiceServer(
-			clienturl.NewServer(domain.Nodes[0].URL()),
+			clienturl.NewServer(domain.Nodes[0].NSMgr.URL),
 			connect.NewServer(ctx,
 				clientFactory,
 				connect.WithDialTimeout(sandbox.DialTimeout),

pkg/registry/common/connect/ns_client_test.go

@@ -31,7 +31,7 @@ import (
 	"github.com/networkservicemesh/sdk/pkg/registry/common/connect"
 	"github.com/networkservicemesh/sdk/pkg/registry/common/memory"
 	"github.com/networkservicemesh/sdk/pkg/registry/core/streamchannel"
-	"github.com/networkservicemesh/sdk/pkg/tools/grpcutils"
+	"github.com/networkservicemesh/sdk/pkg/tools/sandbox"
 )
 
 func TestConnectNSClient(t *testing.T) {
@@ -111,7 +111,7 @@ func TestConnectNSClient_Restart(t *testing.T) {
 	// 2. Restart remote
 	serverCancel()
 	require.Eventually(t, func() bool {
-		return grpcutils.CheckURLFree(u)
+		return sandbox.CheckURLFree(u)
 	}, time.Second, 10*time.Millisecond)
 
 	require.NoError(t, startNSServer(ctx, u, mem))

pkg/registry/common/connect/nse_client_test.go

@@ -31,7 +31,7 @@ import (
 	"github.com/networkservicemesh/sdk/pkg/registry/common/connect"
 	"github.com/networkservicemesh/sdk/pkg/registry/common/memory"
 	"github.com/networkservicemesh/sdk/pkg/registry/core/streamchannel"
-	"github.com/networkservicemesh/sdk/pkg/tools/grpcutils"
+	"github.com/networkservicemesh/sdk/pkg/tools/sandbox"
 )
 
 func TestConnectNSEClient(t *testing.T) {
@@ -111,7 +111,7 @@ func TestConnectNSEClient_Restart(t *testing.T) {
 	// 2. Restart remote
 	serverCancel()
 	require.Eventually(t, func() bool {
-		return grpcutils.CheckURLFree(u)
+		return sandbox.CheckURLFree(u)
 	}, time.Second, 10*time.Millisecond)
 
 	require.NoError(t, startNSEServer(ctx, u, mem))

pkg/registry/common/heal/find_test.go

@@ -28,7 +28,6 @@ import (
 
 	"github.com/networkservicemesh/sdk/pkg/networkservice/chains/nsmgr"
 	registryclient "github.com/networkservicemesh/sdk/pkg/registry/chains/client"
-	"github.com/networkservicemesh/sdk/pkg/tools/grpcutils"
 	"github.com/networkservicemesh/sdk/pkg/tools/sandbox"
 )
 
@@ -55,7 +54,7 @@ func TestHealClient_FindTest(t *testing.T) {
 	// 1. Create NS, NSE find clients
 	findCtx, findCancel := context.WithCancel(ctx)
 
-	nsRegistryClient := registryclient.NewNetworkServiceRegistryClient(ctx, domain.Nodes[0].URL(),
+	nsRegistryClient := registryclient.NewNetworkServiceRegistryClient(ctx, sandbox.CloneURL(domain.Nodes[0].NSMgr.URL),
 		registryclient.WithDialOptions(sandbox.DefaultDialOptions(sandbox.GenerateTestToken)...))
 
 	nsStream, err := nsRegistryClient.Find(findCtx, &registry.NetworkServiceQuery{
@@ -64,7 +63,7 @@ func TestHealClient_FindTest(t *testing.T) {
 	})
 	require.NoError(t, err)
 
-	nseRegistryClient := registryclient.NewNetworkServiceEndpointRegistryClient(ctx, domain.Nodes[0].URL(),
+	nseRegistryClient := registryclient.NewNetworkServiceEndpointRegistryClient(ctx, sandbox.CloneURL(domain.Nodes[0].NSMgr.URL),
 		registryclient.WithDialOptions(sandbox.DefaultDialOptions(sandbox.GenerateTestToken)...))
 
 	nseStream, err := nseRegistryClient.Find(findCtx, &registry.NetworkServiceEndpointQuery{
@@ -78,7 +77,7 @@ func TestHealClient_FindTest(t *testing.T) {
 
 	nsmgrCancel()
 	require.Eventually(t, func() bool {
-		return grpcutils.CheckURLFree(mgr.URL)
+		return sandbox.CheckURLFree(mgr.URL)
 	}, time.Second, 100*time.Millisecond)
 
 	mgr = domain.Nodes[0].NewNSMgr(ctx, mgr.Name, mgr.URL, sandbox.GenerateTestToken, nsmgr.NewServer)

pkg/tools/grpcutils/url.go

@@ -93,12 +93,3 @@ func TargetToNetAddr(target string) (network, addr string) {
 
 	return network, target
 }
-
-// CheckURLFree returns is given url is free for Listen
-func CheckURLFree(u *url.URL) bool {
-	ln, err := net.Listen(TargetToNetAddr(URLToTarget(u)))
-	if err == nil {
-		err = ln.Close()
-	}
-	return err == nil
-}