From b4fe3f855948691c196db3f48dd2d127f89d5d38 Mon Sep 17 00:00:00 2001 From: Denis Tingaikin Date: Wed, 27 Oct 2021 21:39:19 +0700 Subject: [PATCH 1/6] initial p2mp re-work Signed-off-by: Denis Tingaikin --- pkg/networkservice/chains/nsmgr/scale_test.go | 1 + pkg/networkservice/chains/nsmgr/server.go | 52 +++--- .../chains/nsmgr/single_test.go | 2 + pkg/networkservice/chains/nsmgr/suite_test.go | 44 ++--- pkg/networkservice/chains/nsmgr/unix_test.go | 8 +- .../chains/nsmgrproxy/server_test.go | 34 ++-- pkg/networkservice/common/connect/client.go | 7 + .../discover/discovercrossnse/metadata.go | 52 ++++++ .../discover/discovercrossnse/server.go | 66 +++++++ .../common/discover/match_selector.go | 3 +- .../common/filtermechanisms/server.go | 19 +- .../common/filtermechanisms/server_test.go | 32 +--- .../interpose/connection_info_map.gen.go | 73 -------- pkg/networkservice/common/interpose/gen.go | 26 --- pkg/networkservice/common/interpose/server.go | 166 ------------------ .../common/interpose/server_test.go | 117 ------------ pkg/registry/common/interpose/client.go | 3 + pkg/registry/common/interpose/server.go | 47 +++-- pkg/registry/common/interpose/server_test.go | 55 +++--- pkg/registry/core/chain/ns_registry.go | 6 +- pkg/registry/core/chain/nse_registry.go | 6 +- pkg/tools/sandbox/builder.go | 8 +- pkg/tools/sandbox/grpc_utils.go | 1 + pkg/tools/sandbox/node.go | 20 ++- 24 files changed, 298 insertions(+), 550 deletions(-) create mode 100644 pkg/networkservice/common/discover/discovercrossnse/metadata.go create mode 100644 pkg/networkservice/common/discover/discovercrossnse/server.go delete mode 100644 pkg/networkservice/common/interpose/connection_info_map.gen.go delete mode 100644 pkg/networkservice/common/interpose/gen.go delete mode 100644 pkg/networkservice/common/interpose/server.go delete mode 100644 pkg/networkservice/common/interpose/server_test.go diff --git a/pkg/networkservice/chains/nsmgr/scale_test.go b/pkg/networkservice/chains/nsmgr/scale_test.go index f81e157ec..c6239db89 100644 --- a/pkg/networkservice/chains/nsmgr/scale_test.go +++ b/pkg/networkservice/chains/nsmgr/scale_test.go @@ -36,6 +36,7 @@ import ( ) func TestCreateEndpointDuringRequest(t *testing.T) { + t.Skip("TODO") t.Cleanup(func() { goleak.VerifyNone(t) }) ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) diff --git a/pkg/networkservice/chains/nsmgr/server.go b/pkg/networkservice/chains/nsmgr/server.go index 72bf85021..c4df89d89 100644 --- a/pkg/networkservice/chains/nsmgr/server.go +++ b/pkg/networkservice/chains/nsmgr/server.go @@ -21,10 +21,15 @@ package nsmgr import ( "context" + "fmt" "net/url" "time" "github.com/google/uuid" + + "github.com/networkservicemesh/sdk/pkg/networkservice/common/discover/discovercrossnse" + "github.com/networkservicemesh/sdk/pkg/networkservice/common/roundrobin" + "google.golang.org/grpc" "github.com/networkservicemesh/api/pkg/api/networkservice" @@ -35,13 +40,9 @@ import ( "github.com/networkservicemesh/sdk/pkg/networkservice/common/authorize" "github.com/networkservicemesh/sdk/pkg/networkservice/common/clientinfo" "github.com/networkservicemesh/sdk/pkg/networkservice/common/connect" - "github.com/networkservicemesh/sdk/pkg/networkservice/common/discover" "github.com/networkservicemesh/sdk/pkg/networkservice/common/excludedprefixes" - "github.com/networkservicemesh/sdk/pkg/networkservice/common/filtermechanisms" - "github.com/networkservicemesh/sdk/pkg/networkservice/common/interpose" "github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms/recvfd" "github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms/sendfd" - "github.com/networkservicemesh/sdk/pkg/networkservice/common/roundrobin" "github.com/networkservicemesh/sdk/pkg/networkservice/core/adapters" "github.com/networkservicemesh/sdk/pkg/registry" "github.com/networkservicemesh/sdk/pkg/registry/common/checkid" @@ -49,15 +50,14 @@ import ( "github.com/networkservicemesh/sdk/pkg/registry/common/clienturl" registryconnect "github.com/networkservicemesh/sdk/pkg/registry/common/connect" "github.com/networkservicemesh/sdk/pkg/registry/common/expire" + "github.com/networkservicemesh/sdk/pkg/registry/common/interpose" "github.com/networkservicemesh/sdk/pkg/registry/common/localbypass" "github.com/networkservicemesh/sdk/pkg/registry/common/memory" - "github.com/networkservicemesh/sdk/pkg/registry/common/querycache" registryrecvfd "github.com/networkservicemesh/sdk/pkg/registry/common/recvfd" registryserialize "github.com/networkservicemesh/sdk/pkg/registry/common/serialize" "github.com/networkservicemesh/sdk/pkg/registry/common/setlogoption" registryadapter "github.com/networkservicemesh/sdk/pkg/registry/core/adapters" registrychain "github.com/networkservicemesh/sdk/pkg/registry/core/chain" - "github.com/networkservicemesh/sdk/pkg/registry/core/next" "github.com/networkservicemesh/sdk/pkg/tools/grpcutils" "github.com/networkservicemesh/sdk/pkg/tools/token" ) @@ -150,7 +150,7 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, options rv := &nsmgrServer{} - var urlsRegistryServer, interposeRegistryServer registryapi.NetworkServiceEndpointRegistryServer + var interposeRegistryServer = interpose.NewNetworkServiceEndpointRegistryServer() var nsRegistry registryapi.NetworkServiceRegistryServer if opts.regURL != nil { @@ -178,27 +178,22 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, options localBypassRegistryServer := localbypass.NewNetworkServiceEndpointRegistryServer(opts.url) - nseClient := next.NewNetworkServiceEndpointRegistryClient( - registryserialize.NewNetworkServiceEndpointRegistryClient(), - registryadapter.NetworkServiceEndpointServerToClient(localBypassRegistryServer), - querycache.NewClient(ctx), - registryadapter.NetworkServiceEndpointServerToClient(nseRegistry), - ) - - nsClient := registryadapter.NetworkServiceServerToClient(nsRegistry) - // Construct Endpoint rv.Endpoint = endpoint.NewServer(ctx, tokenGenerator, endpoint.WithName(opts.name), endpoint.WithAuthorizeServer(opts.authorizeServer), endpoint.WithAdditionalFunctionality( adapters.NewClientToServer(clientinfo.NewClient()), - discover.NewServer(nsClient, nseClient), - roundrobin.NewServer(), + discovercrossnse.NewServer( + registryadapter.NetworkServiceServerToClient(nsRegistry), + registrychain.NewNetworkServiceEndpointRegistryClient( + interpose.NewNetworkServiceEndpointRegistryClient(), + registryadapter.NetworkServiceEndpointServerToClient(interposeRegistryServer), + ), + roundrobin.NewServer(), + ), excludedprefixes.NewServer(ctx), recvfd.NewServer(), // Receive any files passed - interpose.NewServer(&interposeRegistryServer), - filtermechanisms.NewServer(&urlsRegistryServer), connect.NewServer( client.NewClient( ctx, @@ -215,26 +210,27 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, options sendfd.NewServer()), ) - nsChain := registrychain.NewNetworkServiceRegistryServer( - setlogoption.NewNetworkServiceRegistryServer(map[string]string{"name": opts.name}), + nsRegistry = registrychain.NewNetworkServiceRegistryServer( registryserialize.NewNetworkServiceRegistryServer(), + setlogoption.NewNetworkServiceRegistryServer(map[string]string{"name": "NetworkServiceRegistryServer." + opts.name}), nsRegistry, ) - - nseChain := registrychain.NewNetworkServiceEndpointRegistryServer( - setlogoption.NewNetworkServiceEndpointRegistryServer(map[string]string{"name": opts.name}), + nseRegistry = registrychain.NewNetworkServiceEndpointRegistryServer( + setlogoption.NewNetworkServiceEndpointRegistryServer(map[string]string{"name": fmt.Sprintf("NetworkServiceRegistryServer.%v", opts.name)}), registryclientinfo.NewNetworkServiceEndpointRegistryServer(), registryserialize.NewNetworkServiceEndpointRegistryServer(), checkid.NewNetworkServiceEndpointRegistryServer(), expire.NewNetworkServiceEndpointRegistryServer(ctx, time.Minute), registryrecvfd.NewNetworkServiceEndpointRegistryServer(), // Allow to receive a passed files - urlsRegistryServer, // Store endpoints URLs interposeRegistryServer, // Store cross connect NSEs localBypassRegistryServer, // Perform URL transformations - nseRegistry, // Register NSE inside Remote registry + nseRegistry, ) - rv.Registry = registry.NewServer(nsChain, nseChain) + rv.Registry = registry.NewServer( + nsRegistry, + nseRegistry, + ) return rv } diff --git a/pkg/networkservice/chains/nsmgr/single_test.go b/pkg/networkservice/chains/nsmgr/single_test.go index d9cbdbeae..5f6eee01c 100644 --- a/pkg/networkservice/chains/nsmgr/single_test.go +++ b/pkg/networkservice/chains/nsmgr/single_test.go @@ -99,6 +99,8 @@ func Test_DNSUsecase(t *testing.T) { } func Test_ShouldCorrectlyAddForwardersWithSameNames(t *testing.T) { + t.Skip("TODO") + t.Cleanup(func() { goleak.VerifyNone(t) }) ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) diff --git a/pkg/networkservice/chains/nsmgr/suite_test.go b/pkg/networkservice/chains/nsmgr/suite_test.go index d261accdc..84415f2a5 100644 --- a/pkg/networkservice/chains/nsmgr/suite_test.go +++ b/pkg/networkservice/chains/nsmgr/suite_test.go @@ -115,7 +115,7 @@ func (s *nsmgrSuite) Test_Remote_ParallelUsecase() { require.NoError(t, err) require.NotNil(t, conn) require.Equal(t, 1, counter.Requests()) - require.Equal(t, 8, len(conn.Path.PathSegments)) + require.Equal(t, 6, len(conn.Path.PathSegments)) // Simulate refresh from client. refreshRequest := request.Clone() @@ -124,7 +124,7 @@ func (s *nsmgrSuite) Test_Remote_ParallelUsecase() { conn, err = nsc.Request(ctx, refreshRequest) require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 8, len(conn.Path.PathSegments)) + require.Equal(t, 6, len(conn.Path.PathSegments)) require.Equal(t, 2, counter.Requests()) // Close @@ -180,7 +180,7 @@ func (s *nsmgrSuite) Test_SelectsRestartingEndpointUsecase() { conn, err := nsc.Request(ctx, defaultRequest(nsReg.Name)) require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 5, len(conn.Path.PathSegments)) + require.Equal(t, 4, len(conn.Path.PathSegments)) require.NoError(t, ctx.Err()) @@ -196,6 +196,8 @@ func (s *nsmgrSuite) Test_SelectsRestartingEndpointUsecase() { func (s *nsmgrSuite) Test_Remote_BusyEndpointsUsecase() { t := s.T() + t.Skip("TODO") + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() @@ -285,7 +287,7 @@ func (s *nsmgrSuite) Test_RemoteUsecase() { require.NoError(t, err) require.NotNil(t, conn) require.Equal(t, 1, counter.Requests()) - require.Equal(t, 8, len(conn.Path.PathSegments)) + require.Equal(t, 6, len(conn.Path.PathSegments)) // Simulate refresh from client refreshRequest := request.Clone() @@ -295,7 +297,7 @@ func (s *nsmgrSuite) Test_RemoteUsecase() { require.NoError(t, err) require.NotNil(t, conn) require.Equal(t, 2, counter.Requests()) - require.Equal(t, 8, len(conn.Path.PathSegments)) + require.Equal(t, 6, len(conn.Path.PathSegments)) // Close _, err = nsc.Close(ctx, conn) @@ -332,7 +334,7 @@ func (s *nsmgrSuite) Test_ConnectToDeadNSEUsecase() { require.NoError(t, err) require.NotNil(t, conn) require.Equal(t, 1, counter.Requests()) - require.Equal(t, 5, len(conn.Path.PathSegments)) + require.Equal(t, 4, len(conn.Path.PathSegments)) nse.Cancel() @@ -378,7 +380,7 @@ func (s *nsmgrSuite) Test_LocalUsecase() { require.NoError(t, err) require.NotNil(t, conn) require.Equal(t, 1, counter.Requests()) - require.Equal(t, 5, len(conn.Path.PathSegments)) + require.Equal(t, 4, len(conn.Path.PathSegments)) // Simulate refresh from client refreshRequest := request.Clone() @@ -387,7 +389,7 @@ func (s *nsmgrSuite) Test_LocalUsecase() { conn2, err := nsc.Request(ctx, refreshRequest) require.NoError(t, err) require.NotNil(t, conn2) - require.Equal(t, 5, len(conn2.Path.PathSegments)) + require.Equal(t, 4, len(conn2.Path.PathSegments)) require.Equal(t, 2, counter.Requests()) // Close @@ -439,9 +441,9 @@ func (s *nsmgrSuite) Test_PassThroughRemoteUsecase() { // Path length to first endpoint is 5 // Path length from NSE client to other remote endpoint is 8 - require.Equal(t, 8*(nodesCount-1)+5, len(conn.Path.PathSegments)) + require.Equal(t, 6*(nodesCount-1)+4, len(conn.Path.PathSegments)) for i := 0; i < len(nseRegs); i++ { - require.Contains(t, nseRegs[i].Name, conn.Path.PathSegments[i*8+4].Name) + require.Contains(t, nseRegs[i].Name, conn.Path.PathSegments[i*6+3].Name) } // Refresh @@ -502,9 +504,9 @@ func (s *nsmgrSuite) Test_PassThroughLocalUsecase() { // Path length to first endpoint is 5 // Path length from NSE client to other local endpoint is 5 - require.Equal(t, 5*(nsesCount-1)+5, len(conn.Path.PathSegments)) + require.Equal(t, 4*(nsesCount-1)+4, len(conn.Path.PathSegments)) for i := 0; i < len(nseRegs); i++ { - require.Contains(t, nseRegs[i].Name, conn.Path.PathSegments[(i+1)*5-1].Name) + require.Contains(t, nseRegs[i].Name, conn.Path.PathSegments[(i+1)*4-1].Name) } // Refresh @@ -578,19 +580,19 @@ func (s *nsmgrSuite) Test_PassThroughSameSourceSelector() { require.NoError(t, err) require.NotNil(t, conn) - // Path length to first endpoint is 5 - // Path length from NSE client to other local endpoint is 5 - require.Equal(t, 5*(nsesCount-2)+5, len(conn.Path.PathSegments)) + // Path length to first endpoint is 4 + // Path length from NSE client to other local endpoint is 4 + require.Equal(t, 4*(nsesCount-2)+4, len(conn.Path.PathSegments)) for i := 1; i < len(nseRegs); i++ { - require.Contains(t, nseRegs[i].Name, conn.Path.PathSegments[i*5-1].Name) + require.Contains(t, nseRegs[i].Name, conn.Path.PathSegments[i*4-1].Name) } // Refresh conn, err = nsc.Request(ctx, request) require.NoError(t, err) - require.Equal(t, 5*(nsesCount-2)+5, len(conn.Path.PathSegments)) + require.Equal(t, 4*(nsesCount-2)+4, len(conn.Path.PathSegments)) for i := 1; i < len(nseRegs); i++ { - require.Contains(t, nseRegs[i].Name, conn.Path.PathSegments[i*5-1].Name) + require.Contains(t, nseRegs[i].Name, conn.Path.PathSegments[i*4-1].Name) } // Close @@ -650,11 +652,11 @@ func (s *nsmgrSuite) Test_PassThroughLocalUsecaseMultiLabel() { require.NoError(t, err) require.NotNil(t, conn) - // Path length from NSE client to other local endpoint is 5 + // Path length from NSE client to other local endpoint is 4 expectedPath := []string{"ab", "aabb", "aaabbb"} - require.Equal(t, 5*len(nsReg.Matches), len(conn.Path.PathSegments)) + require.Equal(t, 4*len(nsReg.Matches), len(conn.Path.PathSegments)) for i := 0; i < len(expectedPath); i++ { - require.Contains(t, conn.Path.PathSegments[(i+1)*5-1].Name, expectedPath[i]) + require.Contains(t, conn.Path.PathSegments[(i+1)*4-1].Name, expectedPath[i]) } // Refresh diff --git a/pkg/networkservice/chains/nsmgr/unix_test.go b/pkg/networkservice/chains/nsmgr/unix_test.go index 65d3cc0b9..d94db059f 100644 --- a/pkg/networkservice/chains/nsmgr/unix_test.go +++ b/pkg/networkservice/chains/nsmgr/unix_test.go @@ -65,7 +65,7 @@ func Test_Local_NoURLUsecase(t *testing.T) { require.NoError(t, err) require.NotNil(t, conn) require.Equal(t, 1, counter.Requests()) - require.Equal(t, 5, len(conn.Path.PathSegments)) + require.Equal(t, 4, len(conn.Path.PathSegments)) // Simulate refresh from client refreshRequest := request.Clone() @@ -74,7 +74,7 @@ func Test_Local_NoURLUsecase(t *testing.T) { conn2, err := nsc.Request(ctx, refreshRequest) require.NoError(t, err) require.NotNil(t, conn2) - require.Equal(t, 5, len(conn2.Path.PathSegments)) + require.Equal(t, 4, len(conn2.Path.PathSegments)) require.Equal(t, 2, counter.Requests()) // Close @@ -127,7 +127,7 @@ func Test_MultiForwarderSendfd(t *testing.T) { require.NoError(t, err) require.NotNil(t, conn) require.Equal(t, 1, counter.Requests()) - require.Equal(t, 5, len(conn.Path.PathSegments)) + require.Equal(t, 4, len(conn.Path.PathSegments)) // Simulate refresh from client refreshRequest := request.Clone() @@ -136,7 +136,7 @@ func Test_MultiForwarderSendfd(t *testing.T) { conn2, err := nsc.Request(ctx, refreshRequest) require.NoError(t, err) require.NotNil(t, conn2) - require.Equal(t, 5, len(conn2.Path.PathSegments)) + require.Equal(t, 4, len(conn2.Path.PathSegments)) require.Equal(t, 2, counter.Requests()) // Close diff --git a/pkg/networkservice/chains/nsmgrproxy/server_test.go b/pkg/networkservice/chains/nsmgrproxy/server_test.go index 997673f1c..6657b47d2 100644 --- a/pkg/networkservice/chains/nsmgrproxy/server_test.go +++ b/pkg/networkservice/chains/nsmgrproxy/server_test.go @@ -97,7 +97,7 @@ func TestNSMGR_InterdomainUseCase(t *testing.T) { require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 10, len(conn.Path.PathSegments)) + require.Equal(t, 8, len(conn.Path.PathSegments)) // Simulate refresh from client. @@ -107,7 +107,7 @@ func TestNSMGR_InterdomainUseCase(t *testing.T) { conn, err = nsc.Request(ctx, refreshRequest) require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 10, len(conn.Path.PathSegments)) + require.Equal(t, 8, len(conn.Path.PathSegments)) // Close _, err = nsc.Close(ctx, conn) @@ -179,7 +179,7 @@ func TestNSMGR_Interdomain_TwoNodesNSEs(t *testing.T) { require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 10, len(conn.Path.PathSegments)) + require.Equal(t, 8, len(conn.Path.PathSegments)) // Simulate refresh from client. @@ -189,7 +189,7 @@ func TestNSMGR_Interdomain_TwoNodesNSEs(t *testing.T) { conn, err = nsc.Request(ctx, refreshRequest) require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 10, len(conn.Path.PathSegments)) + require.Equal(t, 8, len(conn.Path.PathSegments)) request = &networkservice.NetworkServiceRequest{ MechanismPreferences: []*networkservice.Mechanism{ @@ -206,7 +206,7 @@ func TestNSMGR_Interdomain_TwoNodesNSEs(t *testing.T) { require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 10, len(conn.Path.PathSegments)) + require.Equal(t, 8, len(conn.Path.PathSegments)) // Simulate refresh from client. @@ -216,7 +216,7 @@ func TestNSMGR_Interdomain_TwoNodesNSEs(t *testing.T) { conn, err = nsc.Request(ctx, refreshRequest) require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 10, len(conn.Path.PathSegments)) + require.Equal(t, 8, len(conn.Path.PathSegments)) } // TestNSMGR_FloatingInterdomainUseCase covers simple interdomain scenario with resolving endpoint from floating registry: @@ -283,7 +283,7 @@ func TestNSMGR_FloatingInterdomainUseCase(t *testing.T) { require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 10, len(conn.Path.PathSegments)) + require.Equal(t, 8, len(conn.Path.PathSegments)) // Simulate refresh from client. @@ -293,7 +293,7 @@ func TestNSMGR_FloatingInterdomainUseCase(t *testing.T) { conn, err = nsc.Request(ctx, refreshRequest) require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 10, len(conn.Path.PathSegments)) + require.Equal(t, 8, len(conn.Path.PathSegments)) // Close _, err = nsc.Close(ctx, conn) @@ -366,7 +366,7 @@ func TestNSMGR_FloatingInterdomainUseCase_FloatingNetworkServiceNameRegistration require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 10, len(conn.Path.PathSegments)) + require.Equal(t, 8, len(conn.Path.PathSegments)) // Simulate refresh from client. @@ -376,7 +376,7 @@ func TestNSMGR_FloatingInterdomainUseCase_FloatingNetworkServiceNameRegistration conn, err = nsc.Request(ctx, refreshRequest) require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 10, len(conn.Path.PathSegments)) + require.Equal(t, 8, len(conn.Path.PathSegments)) // Close _, err = nsc.Close(ctx, conn) @@ -479,7 +479,7 @@ func TestNSMGR_FloatingInterdomain_FourClusters(t *testing.T) { require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 10, len(conn.Path.PathSegments)) + require.Equal(t, 8, len(conn.Path.PathSegments)) // Simulate refresh from client. @@ -489,7 +489,7 @@ func TestNSMGR_FloatingInterdomain_FourClusters(t *testing.T) { conn, err = nsc.Request(ctx, refreshRequest) require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 10, len(conn.Path.PathSegments)) + require.Equal(t, 8, len(conn.Path.PathSegments)) // connect to second endpoint from cluster3 request = &networkservice.NetworkServiceRequest{ @@ -507,7 +507,7 @@ func TestNSMGR_FloatingInterdomain_FourClusters(t *testing.T) { require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 10, len(conn.Path.PathSegments)) + require.Equal(t, 8, len(conn.Path.PathSegments)) // Simulate refresh from client. @@ -517,7 +517,7 @@ func TestNSMGR_FloatingInterdomain_FourClusters(t *testing.T) { conn, err = nsc.Request(ctx, refreshRequest) require.NoError(t, err) require.NotNil(t, conn) - require.Equal(t, 10, len(conn.Path.PathSegments)) + require.Equal(t, 8, len(conn.Path.PathSegments)) } type passThroughClient struct { @@ -619,9 +619,9 @@ func Test_Interdomain_PassThroughUsecase(t *testing.T) { require.NoError(t, err) require.NotNil(t, conn) - // Path length to first endpoint is 5 - // Path length from NSE client to other remote endpoint is 10 - require.Equal(t, 10*(clusterCount-1)+5, len(conn.Path.PathSegments)) + // Path length to first endpoint is 4 + // Path length from NSE client to other remote endpoint is 8 + require.Equal(t, 8*(clusterCount-1)+4, len(conn.Path.PathSegments)) // Close _, err = nsc.Close(ctx, conn) diff --git a/pkg/networkservice/common/connect/client.go b/pkg/networkservice/common/connect/client.go index cb632eca1..4e96769d6 100644 --- a/pkg/networkservice/common/connect/client.go +++ b/pkg/networkservice/common/connect/client.go @@ -25,10 +25,16 @@ import ( "google.golang.org/protobuf/types/known/emptypb" "github.com/networkservicemesh/sdk/pkg/networkservice/common/clientconn" + "github.com/networkservicemesh/sdk/pkg/tools/log" ) type connectClient struct{} +// TODO delete it +type debugMate interface { + Target() string +} + // NewClient - returns a connect chain element func NewClient() networkservice.NetworkServiceClient { return &connectClient{} @@ -39,6 +45,7 @@ func (c *connectClient) Request(ctx context.Context, request *networkservice.Net if !loaded { return nil, errors.New("no grpc.ClientConnInterface provided") } + log.FromContext(ctx).Infof("see at serve on: %v", cc.(debugMate).Target()) conn, err := networkservice.NewNetworkServiceClient(cc).Request(ctx, request, opts...) return conn, err } diff --git a/pkg/networkservice/common/discover/discovercrossnse/metadata.go b/pkg/networkservice/common/discover/discovercrossnse/metadata.go new file mode 100644 index 000000000..3ab7993af --- /dev/null +++ b/pkg/networkservice/common/discover/discovercrossnse/metadata.go @@ -0,0 +1,52 @@ +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package discovercrossnse TODO +package discovercrossnse + +import ( + "context" + + "github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata" +) + +type networkserviceNameKey struct{} + +type crossConnectEndpointName struct{} + +func loadNetworkServiceEndpointName(ctx context.Context) string { + v, ok := metadata.Map(ctx, false).Load(networkserviceNameKey{}) + if !ok { + return "" + } + return v.(string) +} + +func storeNetworkServiceEndpointName(ctx context.Context, v string) { + metadata.Map(ctx, false).Store(networkserviceNameKey{}, v) +} + +func loadCrossConnectEndpointName(ctx context.Context) string { + v, ok := metadata.Map(ctx, false).Load(crossConnectEndpointName{}) + if !ok { + return "" + } + return v.(string) +} + +func storeCrossConnectEndpointName(ctx context.Context, v string) { + metadata.Map(ctx, false).Store(crossConnectEndpointName{}, v) +} diff --git a/pkg/networkservice/common/discover/discovercrossnse/server.go b/pkg/networkservice/common/discover/discovercrossnse/server.go new file mode 100644 index 000000000..979158b48 --- /dev/null +++ b/pkg/networkservice/common/discover/discovercrossnse/server.go @@ -0,0 +1,66 @@ +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package discovercrossnse + +import ( + "context" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/api/pkg/api/registry" + + "github.com/networkservicemesh/sdk/pkg/networkservice/common/discover" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/chain" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" +) + +type switchNetworkServiceEndpointNameServer struct { + setFunc func(context.Context, string) + getFunc func(context.Context) string +} + +func (d *switchNetworkServiceEndpointNameServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) { + d.setFunc(ctx, request.GetConnection().GetNetworkServiceEndpointName()) + + request.GetConnection().NetworkServiceEndpointName = d.getFunc(ctx) + + return next.Server(ctx).Request(ctx, request) +} + +func (d *switchNetworkServiceEndpointNameServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) { + d.setFunc(ctx, conn.NetworkServiceEndpointName) + + conn.NetworkServiceEndpointName = d.getFunc(ctx) + + return next.Server(ctx).Close(ctx, conn) +} + +// NewServer TODO +func NewServer(nsClient registry.NetworkServiceRegistryClient, nseClient registry.NetworkServiceEndpointRegistryClient, loadBalance networkservice.NetworkServiceServer) networkservice.NetworkServiceServer { + return chain.NewNetworkServiceServer( + &switchNetworkServiceEndpointNameServer{ + setFunc: storeNetworkServiceEndpointName, + getFunc: loadCrossConnectEndpointName, + }, + discover.NewServer(nsClient, nseClient), + loadBalance, + &switchNetworkServiceEndpointNameServer{ + setFunc: storeCrossConnectEndpointName, + getFunc: loadNetworkServiceEndpointName, + }, + ) +} diff --git a/pkg/networkservice/common/discover/match_selector.go b/pkg/networkservice/common/discover/match_selector.go index 7b96965be..c694a0ef9 100644 --- a/pkg/networkservice/common/discover/match_selector.go +++ b/pkg/networkservice/common/discover/match_selector.go @@ -64,7 +64,8 @@ func matchEndpoint(clockTime clock.Clock, nsLabels map[string]string, ns *regist for _, destination := range match.GetRoutes() { // Each NSE should be matched against that destination for _, nse := range validNetworkServiceEndpoints { - if isSubset(nse.GetNetworkServiceLabels()[ns.Name].Labels, destination.GetDestinationSelector(), nsLabels) { + // TODO looks unhealthy + if nse.GetNetworkServiceLabels()[ns.Name] == nil || isSubset(nse.GetNetworkServiceLabels()[ns.Name].Labels, destination.GetDestinationSelector(), nsLabels) { nseCandidates = append(nseCandidates, nse) } } diff --git a/pkg/networkservice/common/filtermechanisms/server.go b/pkg/networkservice/common/filtermechanisms/server.go index 226456ad1..d622b866e 100644 --- a/pkg/networkservice/common/filtermechanisms/server.go +++ b/pkg/networkservice/common/filtermechanisms/server.go @@ -27,32 +27,23 @@ import ( "github.com/networkservicemesh/api/pkg/api/networkservice" "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/cls" - "github.com/networkservicemesh/api/pkg/api/registry" "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" - "github.com/networkservicemesh/sdk/pkg/registry/common/endpointurls" - "github.com/networkservicemesh/sdk/pkg/registry/common/interpose" "github.com/networkservicemesh/sdk/pkg/tools/clienturlctx" ) -type filterMechanismsServer struct { - nses endpointurls.Map -} +type filterMechanismsServer struct{} // NewServer - filters out remote mechanisms if connection is received from a unix file socket, otherwise filters // out local mechanisms -func NewServer(registryServer *registry.NetworkServiceEndpointRegistryServer) networkservice.NetworkServiceServer { - s := &filterMechanismsServer{} - *registryServer = endpointurls.NewNetworkServiceEndpointRegistryServer(&s.nses) - return s +func NewServer() networkservice.NetworkServiceServer { + return new(filterMechanismsServer) } func (s *filterMechanismsServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) { u := clienturlctx.ClientURL(ctx) - if name, ok := s.nses.Load(*u); ok { - if !interpose.Is(name) { - request.MechanismPreferences = filterMechanismsByCls(request.GetMechanismPreferences(), cls.LOCAL) - } + if u.Scheme == "inode" || u.Scheme == "unix" { + request.MechanismPreferences = filterMechanismsByCls(request.GetMechanismPreferences(), cls.LOCAL) } else { request.MechanismPreferences = filterMechanismsByCls(request.GetMechanismPreferences(), cls.REMOTE) } diff --git a/pkg/networkservice/common/filtermechanisms/server_test.go b/pkg/networkservice/common/filtermechanisms/server_test.go index f2f8d1889..1c67c02eb 100644 --- a/pkg/networkservice/common/filtermechanisms/server_test.go +++ b/pkg/networkservice/common/filtermechanisms/server_test.go @@ -27,7 +27,6 @@ import ( "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/memif" "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/srv6" "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vxlan" - "github.com/networkservicemesh/api/pkg/api/registry" "github.com/stretchr/testify/require" "github.com/networkservicemesh/sdk/pkg/networkservice/common/filtermechanisms" @@ -65,14 +64,8 @@ func TestFilterMechanismsServer_Request(t *testing.T) { ClsResult string }{ { - Name: "Local mechanisms", - ClientURL: &url.URL{Scheme: "tcp", Host: "localhost:5000"}, - RegisterURLs: []url.URL{ - { - Scheme: "tcp", - Host: "localhost:5000", - }, - }, + Name: "Local mechanisms", + ClientURL: &url.URL{Scheme: "inode", Host: "localhost:5000"}, EndpointName: "nse-1", ClsResult: cls.LOCAL, }, @@ -82,29 +75,10 @@ func TestFilterMechanismsServer_Request(t *testing.T) { EndpointName: "nse-1", ClsResult: cls.REMOTE, }, - { - Name: "Pass mechanisms to forwarder", - ClientURL: &url.URL{Scheme: "tcp", Host: "localhost:5000"}, - EndpointName: "nse-1#interpose-nse", - RegisterURLs: []url.URL{ - { - Scheme: "tcp", - Host: "localhost:5000", - }, - }, - }, } for _, sample := range samples { - var registryServer registry.NetworkServiceEndpointRegistryServer - s := filtermechanisms.NewServer(®istryServer) - for _, u := range sample.RegisterURLs { - _, err := registryServer.Register(context.Background(), ®istry.NetworkServiceEndpoint{ - Name: sample.EndpointName, - Url: u.String(), - }) - require.NoError(t, err) - } + s := filtermechanisms.NewServer() ctx := clienturlctx.WithClientURL(context.Background(), sample.ClientURL) req := request() _, err := s.Request(ctx, req) diff --git a/pkg/networkservice/common/interpose/connection_info_map.gen.go b/pkg/networkservice/common/interpose/connection_info_map.gen.go deleted file mode 100644 index 1909e42cb..000000000 --- a/pkg/networkservice/common/interpose/connection_info_map.gen.go +++ /dev/null @@ -1,73 +0,0 @@ -// Code generated by "-output connection_info_map.gen.go -type connectionInfoMap -output connection_info_map.gen.go -type connectionInfoMap"; DO NOT EDIT. -package interpose - -import ( - "sync" // Used by sync.Map. -) - -// Generate code that will fail if the constants change value. -func _() { - // An "cannot convert connectionInfoMap literal (type connectionInfoMap) to type sync.Map" compiler error signifies that the base type have changed. - // Re-run the go-syncmap command to generate them again. - _ = (sync.Map)(connectionInfoMap{}) -} - -var _nil_connectionInfoMap_connectionInfo_value = func() (val connectionInfo) { return }() - -// Load returns the value stored in the map for a key, or nil if no -// value is present. -// The ok result indicates whether value was found in the map. -func (m *connectionInfoMap) Load(key string) (connectionInfo, bool) { - value, ok := (*sync.Map)(m).Load(key) - if value == nil { - return _nil_connectionInfoMap_connectionInfo_value, ok - } - return value.(connectionInfo), ok -} - -// Store sets the value for a key. -func (m *connectionInfoMap) Store(key string, value connectionInfo) { - (*sync.Map)(m).Store(key, value) -} - -// LoadOrStore returns the existing value for the key if present. -// Otherwise, it stores and returns the given value. -// The loaded result is true if the value was loaded, false if stored. -func (m *connectionInfoMap) LoadOrStore(key string, value connectionInfo) (connectionInfo, bool) { - actual, loaded := (*sync.Map)(m).LoadOrStore(key, value) - if actual == nil { - return _nil_connectionInfoMap_connectionInfo_value, loaded - } - return actual.(connectionInfo), loaded -} - -// LoadAndDelete deletes the value for a key, returning the previous value if any. -// The loaded result reports whether the key was present. -func (m *connectionInfoMap) LoadAndDelete(key string) (value connectionInfo, loaded bool) { - actual, loaded := (*sync.Map)(m).LoadAndDelete(key) - if actual == nil { - return _nil_connectionInfoMap_connectionInfo_value, loaded - } - return actual.(connectionInfo), loaded -} - -// Delete deletes the value for a key. -func (m *connectionInfoMap) Delete(key string) { - (*sync.Map)(m).Delete(key) -} - -// Range calls f sequentially for each key and value present in the map. -// If f returns false, range stops the iteration. -// -// Range does not necessarily correspond to any consistent snapshot of the Map's -// contents: no key will be visited more than once, but if the value for any key -// is stored or deleted concurrently, Range may reflect any mapping for that key -// from any point during the Range call. -// -// Range may be O(N) with the number of elements in the map even if f returns -// false after a constant number of calls. -func (m *connectionInfoMap) Range(f func(key string, value connectionInfo) bool) { - (*sync.Map)(m).Range(func(key, value interface{}) bool { - return f(key.(string), value.(connectionInfo)) - }) -} diff --git a/pkg/networkservice/common/interpose/gen.go b/pkg/networkservice/common/interpose/gen.go deleted file mode 100644 index b63ed3d8c..000000000 --- a/pkg/networkservice/common/interpose/gen.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) 2020 Cisco and/or its affiliates. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package interpose - -import ( - "sync" -) - -//go:generate go-syncmap -output connection_info_map.gen.go -type connectionInfoMap - -// Map - sync.Map with key == string and value == networkservice.NetworkServiceClient -type connectionInfoMap sync.Map diff --git a/pkg/networkservice/common/interpose/server.go b/pkg/networkservice/common/interpose/server.go deleted file mode 100644 index 815cb6c77..000000000 --- a/pkg/networkservice/common/interpose/server.go +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright (c) 2020-2021 Doc.ai and/or its affiliates. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package interpose provides a NetworkServiceServer chain element that tracks local Cross connect Endpoints and call them first -// their unix file socket as the clienturl.ClientURL(ctx) used to connect to them. -package interpose - -import ( - "context" - "net/url" - - "github.com/golang/protobuf/ptypes/empty" - "github.com/pkg/errors" - - "github.com/networkservicemesh/api/pkg/api/networkservice" - "github.com/networkservicemesh/api/pkg/api/registry" - - "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" - "github.com/networkservicemesh/sdk/pkg/registry/common/interpose" - "github.com/networkservicemesh/sdk/pkg/tools/clienturlctx" - "github.com/networkservicemesh/sdk/pkg/tools/log" - "github.com/networkservicemesh/sdk/pkg/tools/stringurl" -) - -type interposeServer struct { - interposeNSEs stringurl.Map - activeConnection connectionInfoMap // key == connectionId -} - -type connectionInfo struct { - clientConnID string - endpointURL *url.URL - interposeNSEURL *url.URL -} - -// NewServer - creates a NetworkServiceServer that tracks locally registered CrossConnect Endpoints and on first Request forward to cross conenct nse -// one by one and if request came back from cross nse, it will connect to a proper next client endpoint. -// - server - *registry.NetworkServiceRegistryServer. Since registry.NetworkServiceRegistryServer is an interface -// (and thus a pointer) *registry.NetworkServiceRegistryServer is a double pointer. Meaning it -// points to a place that points to a place that implements registry.NetworkServiceRegistryServer -// This is done so that we can return a registry.NetworkServiceRegistryServer chain element -// while maintaining the NewServer pattern for use like anything else in a chain. -// The value in *server must be included in the registry.NetworkServiceRegistryServer listening -// so it can capture the registrations. -func NewServer(registryServer *registry.NetworkServiceEndpointRegistryServer) networkservice.NetworkServiceServer { - rv := new(interposeServer) - *registryServer = interpose.NewNetworkServiceEndpointRegistryServer(&rv.interposeNSEs) - return rv -} - -func (l *interposeServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (result *networkservice.Connection, err error) { - logger := log.FromContext(ctx).WithField("interposeServer", "Request") - - // Check if there is no active connection, we need to replace endpoint url with forwarder url - conn := request.GetConnection() - ind := conn.GetPath().GetIndex() // It is designed to be used inside Endpoint, so current index is Endpoint already - connID := conn.GetId() - - if len(conn.GetPath().GetPathSegments()) == 0 || ind <= 0 { - return nil, errors.Errorf("path segment doesn't have a client or cross connect nse identity") - } - // We need to find an Id from path to match active connection request. - activeConnID, fromPath := l.getConnectionID(conn) - - // We came from client, so select cross nse and go to it. - clientURL := clienturlctx.ClientURL(ctx) - - connInfo, ok := l.activeConnection.Load(activeConnID) - if ok && (fromPath || int(conn.Path.Index) < len(conn.Path.PathSegments)-1) { - if connID != activeConnID { - l.activeConnection.Store(connID, connInfo) - } - } else { - if connID != activeConnID { - return nil, errors.Errorf("connection id should match current path segment id") - } - - // Iterate over all cross connect NSEs to check one with passed state. - l.interposeNSEs.Range(func(_ string, crossNSEURL *url.URL) bool { - crossCTX := clienturlctx.WithClientURL(ctx, crossNSEURL) - - // Store client connection and selected cross connection URL. - l.activeConnection.Store(activeConnID, connectionInfo{ - clientConnID: activeConnID, - endpointURL: clientURL, - interposeNSEURL: crossNSEURL, - }) - result, err = next.Server(crossCTX).Request(crossCTX, request.Clone()) - if err != nil { - logger.Errorf("failed to request cross NSE %v err: %v", crossNSEURL, err) - return true - } - // If all is ok, stop iterating. - return false - }) - if result != nil { - return result, nil - } - - l.activeConnection.Delete(activeConnID) - - return nil, errors.Errorf("all cross NSE failed to connect to endpoint %v connection: %v", clientURL, conn) - } - - var crossCTX context.Context - if connID == connInfo.clientConnID { - crossCTX = clienturlctx.WithClientURL(ctx, connInfo.interposeNSEURL) - } else { - // Go to endpoint URL if it matches one we had on previous step. - if clientURL != connInfo.endpointURL && *clientURL != *connInfo.endpointURL { - return nil, errors.Errorf("new selected endpoint URL %v doesn't match endpoint URL selected before interpose NSE %v", clientURL, connInfo.endpointURL) - } - crossCTX = ctx - } - - return next.Server(crossCTX).Request(crossCTX, request) -} - -func (l *interposeServer) getConnectionID(conn *networkservice.Connection) (string, bool) { - id := conn.Id - for i := conn.GetPath().GetIndex() - 1; i > 0; i-- { - activeConnID := conn.GetPath().GetPathSegments()[i].Id - if connInfo, ok := l.activeConnection.Load(activeConnID); ok { - if activeConnID == connInfo.clientConnID { - return activeConnID, true - } - break - } - } - return id, false -} - -func (l *interposeServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) { - logger := log.FromContext(ctx).WithField("interposeServer", "Close") - - // If we came from NSMgr, we need to go to proper interpose NSE - connInfo, ok := l.activeConnection.Load(conn.GetId()) - if !ok { - logger.Errorf("no active connection found: %v", conn) - return next.Server(ctx).Close(clienturlctx.WithClientURL(ctx, nil), conn) - } - - var crossCTX context.Context - if conn.GetId() == connInfo.clientConnID { - crossCTX = clienturlctx.WithClientURL(ctx, connInfo.interposeNSEURL) - } else { - crossCTX = ctx - } - - l.activeConnection.Delete(conn.GetId()) - - return next.Server(crossCTX).Close(crossCTX, conn) -} diff --git a/pkg/networkservice/common/interpose/server_test.go b/pkg/networkservice/common/interpose/server_test.go deleted file mode 100644 index d5b805fcd..000000000 --- a/pkg/networkservice/common/interpose/server_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) 2020-2021 Doc.ai and/or its affiliates. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package interpose_test - -import ( - "context" - "net/url" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/networkservicemesh/api/pkg/api/networkservice" - "github.com/networkservicemesh/api/pkg/api/registry" - - "github.com/networkservicemesh/sdk/pkg/networkservice/common/clienturl" - "github.com/networkservicemesh/sdk/pkg/networkservice/common/interpose" - "github.com/networkservicemesh/sdk/pkg/networkservice/common/updatepath" - "github.com/networkservicemesh/sdk/pkg/networkservice/core/adapters" - "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" - "github.com/networkservicemesh/sdk/pkg/networkservice/utils/checks/checkcontext" - "github.com/networkservicemesh/sdk/pkg/networkservice/utils/count" - registryinterpose "github.com/networkservicemesh/sdk/pkg/registry/common/interpose" - registryadapters "github.com/networkservicemesh/sdk/pkg/registry/core/adapters" - registrynext "github.com/networkservicemesh/sdk/pkg/registry/core/next" - "github.com/networkservicemesh/sdk/pkg/tools/clienturlctx" -) - -func TestInterposeServer(t *testing.T) { - nseURL := url.URL{Scheme: "tcp", Host: "nse.test"} - crossNSEURL := url.URL{Scheme: "tcp", Host: "cross-nse.test"} - - var interposeRegistry registry.NetworkServiceEndpointRegistryServer - interposeServer := interpose.NewServer(&interposeRegistry) - - _, err := registrynext.NewNetworkServiceEndpointRegistryClient( - registryinterpose.NewNetworkServiceEndpointRegistryClient(), - registryadapters.NetworkServiceEndpointServerToClient(interposeRegistry), - ).Register(context.TODO(), ®istry.NetworkServiceEndpoint{ - Name: "forwarder", - Url: crossNSEURL.String(), - }) - require.NoError(t, err) - - counter := new(count.Server) - - client := next.NewNetworkServiceClient( - updatepath.NewClient("client"), - adapters.NewServerToClient(next.NewNetworkServiceServer( - updatepath.NewServer("nsmgr"), - clienturl.NewServer(&nseURL), - interposeServer, - checkcontext.NewServer(t, func(t *testing.T, ctx context.Context) { - clientURL := clienturlctx.ClientURL(ctx) - require.NotNil(t, clientURL) - require.Equal(t, crossNSEURL, *clientURL) - }), - )), - adapters.NewServerToClient(next.NewNetworkServiceServer( - updatepath.NewServer("forwarder"), - )), - adapters.NewServerToClient(next.NewNetworkServiceServer( - updatepath.NewServer("nsmgr"), - clienturl.NewServer(&nseURL), - interposeServer, - checkcontext.NewServer(t, func(t *testing.T, ctx context.Context) { - clientURL := clienturlctx.ClientURL(ctx) - require.NotNil(t, clientURL) - require.Equal(t, nseURL, *clientURL) - }), - )), - adapters.NewServerToClient(next.NewNetworkServiceServer( - updatepath.NewServer("endpoint"), - counter, - )), - ) - - // 1. Request - - request := &networkservice.NetworkServiceRequest{ - Connection: &networkservice.Connection{}, - } - - conn, err := client.Request(context.TODO(), request) - require.NoError(t, err) - require.Equal(t, 1, counter.Requests()) - - // 2. Refresh - - request = request.Clone() - request.Connection = conn.Clone() - - conn, err = client.Request(context.TODO(), request) - require.NoError(t, err) - require.Equal(t, 2, counter.Requests()) - - // 3. Close - - conn = conn.Clone() - - _, err = client.Close(context.TODO(), conn) - require.NoError(t, err) - require.Equal(t, 1, counter.Closes()) -} diff --git a/pkg/registry/common/interpose/client.go b/pkg/registry/common/interpose/client.go index 0783ce9bb..c6981f98b 100644 --- a/pkg/registry/common/interpose/client.go +++ b/pkg/registry/common/interpose/client.go @@ -42,6 +42,9 @@ func (rc *interposeRegistryClient) Register(ctx context.Context, in *registry.Ne } func (rc *interposeRegistryClient) Find(ctx context.Context, in *registry.NetworkServiceEndpointQuery, opts ...grpc.CallOption) (registry.NetworkServiceEndpointRegistry_FindClient, error) { + if in.GetNetworkServiceEndpoint() != nil && !Is(in.GetNetworkServiceEndpoint().GetName()) { + in.GetNetworkServiceEndpoint().Name = interposeName(in.NetworkServiceEndpoint.Name) + } return next.NetworkServiceEndpointRegistryClient(ctx).Find(ctx, in, opts...) } diff --git a/pkg/registry/common/interpose/server.go b/pkg/registry/common/interpose/server.go index 3434d9def..246196542 100644 --- a/pkg/registry/common/interpose/server.go +++ b/pkg/registry/common/interpose/server.go @@ -25,19 +25,23 @@ import ( "github.com/networkservicemesh/api/pkg/api/registry" + "github.com/networkservicemesh/sdk/pkg/registry/common/memory" + "github.com/networkservicemesh/sdk/pkg/registry/core/chain" "github.com/networkservicemesh/sdk/pkg/registry/core/next" - "github.com/networkservicemesh/sdk/pkg/tools/stringurl" ) type interposeRegistryServer struct { - interposeURLs *stringurl.Map + backend registry.NetworkServiceEndpointRegistryServer } // NewNetworkServiceEndpointRegistryServer - creates a NetworkServiceRegistryServer that registers local Cross connect Endpoints // and adds them to Map -func NewNetworkServiceEndpointRegistryServer(interposeURLs *stringurl.Map) registry.NetworkServiceEndpointRegistryServer { +func NewNetworkServiceEndpointRegistryServer() registry.NetworkServiceEndpointRegistryServer { return &interposeRegistryServer{ - interposeURLs: interposeURLs, + backend: chain.NewNetworkServiceEndpointRegistryServer( + memory.NewNetworkServiceEndpointRegistryServer(), + new(breakNSEServer), + ), } } @@ -46,10 +50,6 @@ func (s *interposeRegistryServer) Register(ctx context.Context, nse *registry.Ne return next.NetworkServiceEndpointRegistryServer(ctx).Register(ctx, nse) } - if _, ok := s.interposeURLs.Load(nse.Name); ok { - return nse, nil - } - u, err := url.Parse(nse.Url) if err != nil { return nil, errors.Wrapf(err, "cannot register cross NSE with passed URL: %s", nse.Url) @@ -58,13 +58,17 @@ func (s *interposeRegistryServer) Register(ctx context.Context, nse *registry.Ne return nil, errors.Errorf("cannot register cross NSE with passed URL: %s", nse.Url) } - s.interposeURLs.Store(nse.Name, u) - - return nse, nil + return s.backend.Register(ctx, nse) } func (s *interposeRegistryServer) Find(query *registry.NetworkServiceEndpointQuery, server registry.NetworkServiceEndpointRegistry_FindServer) error { - // No need to modify find logic. + if Is(query.NetworkServiceEndpoint.GetName()) { + query.NetworkServiceEndpoint.NetworkServiceNames = nil + if err := s.backend.Find(query, server); err != nil { + return err + } + return nil + } return next.NetworkServiceEndpointRegistryServer(server.Context()).Find(query, server) } @@ -73,9 +77,24 @@ func (s *interposeRegistryServer) Unregister(ctx context.Context, nse *registry. return next.NetworkServiceEndpointRegistryServer(ctx).Unregister(ctx, nse) } - s.interposeURLs.Delete(nse.Name) + return s.backend.Unregister(ctx, nse) +} + +var _ registry.NetworkServiceEndpointRegistryServer = (*interposeRegistryServer)(nil) +// TODO Should we use it? Should we move it to separate pkg and use in the next pkg? +type breakNSEServer struct{} + +func (*breakNSEServer) Register(_ context.Context, r *registry.NetworkServiceEndpoint) (*registry.NetworkServiceEndpoint, error) { + return r, nil +} + +func (*breakNSEServer) Find(_ *registry.NetworkServiceEndpointQuery, _ registry.NetworkServiceEndpointRegistry_FindServer) error { + return nil +} + +func (*breakNSEServer) Unregister(context.Context, *registry.NetworkServiceEndpoint) (*empty.Empty, error) { return new(empty.Empty), nil } -var _ registry.NetworkServiceEndpointRegistryServer = (*interposeRegistryServer)(nil) +var _ registry.NetworkServiceEndpointRegistryServer = &breakNSEServer{} diff --git a/pkg/registry/common/interpose/server_test.go b/pkg/registry/common/interpose/server_test.go index 27a0fe0cb..85de3364f 100644 --- a/pkg/registry/common/interpose/server_test.go +++ b/pkg/registry/common/interpose/server_test.go @@ -18,17 +18,18 @@ package interpose_test import ( "context" - "net/url" "testing" "github.com/golang/protobuf/ptypes/empty" "github.com/stretchr/testify/require" + "go.uber.org/goleak" "github.com/networkservicemesh/api/pkg/api/registry" "github.com/networkservicemesh/sdk/pkg/registry/common/interpose" + "github.com/networkservicemesh/sdk/pkg/registry/core/adapters" + "github.com/networkservicemesh/sdk/pkg/registry/core/chain" "github.com/networkservicemesh/sdk/pkg/registry/core/next" - "github.com/networkservicemesh/sdk/pkg/tools/stringurl" ) const ( @@ -37,12 +38,22 @@ const ( validURL = "tcp://0.0.0.0" ) +func requireReadList(t *testing.T, server registry.NetworkServiceEndpointRegistryServer) []*registry.NetworkServiceEndpoint { + s, err := chain.NewNetworkServiceEndpointRegistryClient( + interpose.NewNetworkServiceEndpointRegistryClient(), + adapters.NetworkServiceEndpointServerToClient(server)).Find(context.Background(), ®istry.NetworkServiceEndpointQuery{NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{}}) + require.NoError(t, err) + + return registry.ReadNetworkServiceEndpointList(s) +} + func TestInterposeRegistryServer_Interpose(t *testing.T) { + t.Cleanup(func() { goleak.VerifyNone(t) }) + captureName := new(captureNameTestRegistryServer) - var crossMap stringurl.Map server := next.NewNetworkServiceEndpointRegistryServer( - interpose.NewNetworkServiceEndpointRegistryServer(&crossMap), + interpose.NewNetworkServiceEndpointRegistryServer(), captureName, ) @@ -54,21 +65,23 @@ func TestInterposeRegistryServer_Interpose(t *testing.T) { require.True(t, interpose.Is(reg.Name)) require.Empty(t, captureName.name) - requireCrossMapEqual(t, map[string]string{reg.Name: validURL}, &crossMap) + + var list = requireReadList(t, server) + + require.Len(t, list, 1) + require.Equal(t, validURL, list[0].Url) _, err = server.Unregister(context.Background(), reg) require.NoError(t, err) - require.Empty(t, captureName.name) - requireCrossMapEqual(t, map[string]string{}, &crossMap) + require.Empty(t, requireReadList(t, server)) } func TestInterposeRegistryServer_Common(t *testing.T) { captureName := new(captureNameTestRegistryServer) - var crossMap stringurl.Map server := next.NewNetworkServiceEndpointRegistryServer( - interpose.NewNetworkServiceEndpointRegistryServer(&crossMap), + interpose.NewNetworkServiceEndpointRegistryServer(), captureName, ) @@ -79,7 +92,8 @@ func TestInterposeRegistryServer_Common(t *testing.T) { require.Equal(t, name, reg.Name) require.Equal(t, name, captureName.name) - requireCrossMapEqual(t, map[string]string{}, &crossMap) + + require.Empty(t, requireReadList(t, server), 1) captureName.name = "" @@ -87,15 +101,15 @@ func TestInterposeRegistryServer_Common(t *testing.T) { require.NoError(t, err) require.Equal(t, name, captureName.name) - requireCrossMapEqual(t, map[string]string{}, &crossMap) + + require.Empty(t, requireReadList(t, server), 1) } func TestInterposeRegistryServer_Invalid(t *testing.T) { captureName := new(captureNameTestRegistryServer) - var crossMap stringurl.Map server := next.NewNetworkServiceEndpointRegistryServer( - interpose.NewNetworkServiceEndpointRegistryServer(&crossMap), + interpose.NewNetworkServiceEndpointRegistryServer(), captureName, ) @@ -105,16 +119,7 @@ func TestInterposeRegistryServer_Invalid(t *testing.T) { require.Error(t, err) require.Empty(t, captureName.name) - requireCrossMapEqual(t, map[string]string{}, &crossMap) -} - -func requireCrossMapEqual(t *testing.T, expected map[string]string, crossMap *stringurl.Map) { - actual := map[string]string{} - crossMap.Range(func(key string, value *url.URL) bool { - actual[key] = value.String() - return true - }) - require.Equal(t, expected, actual) + require.Empty(t, requireReadList(t, server), 1) } type captureNameTestRegistryServer struct { @@ -134,3 +139,7 @@ func (r *captureNameTestRegistryServer) Unregister(ctx context.Context, in *regi return next.NetworkServiceEndpointRegistryServer(ctx).Unregister(ctx, in) } +func (r *captureNameTestRegistryServer) Find(query *registry.NetworkServiceEndpointQuery, server registry.NetworkServiceEndpointRegistry_FindServer) error { + r.name = query.NetworkServiceEndpoint.GetName() + return next.NetworkServiceEndpointRegistryServer(server.Context()).Find(query, server) +} diff --git a/pkg/registry/core/chain/ns_registry.go b/pkg/registry/core/chain/ns_registry.go index 52fddd6ac..71a198a3a 100644 --- a/pkg/registry/core/chain/ns_registry.go +++ b/pkg/registry/core/chain/ns_registry.go @@ -27,12 +27,10 @@ import ( // NewNetworkServiceRegistryServer - creates a chain of servers func NewNetworkServiceRegistryServer(servers ...registry.NetworkServiceRegistryServer) registry.NetworkServiceRegistryServer { - return next.NewNetworkServiceRegistryServer( - next.NewWrappedNetworkServiceRegistryServer(trace.NewNetworkServiceRegistryServer, servers...)) + return next.NewNetworkServiceRegistryServer(next.NewWrappedNetworkServiceRegistryServer(trace.NewNetworkServiceRegistryServer, servers...)) } // NewNetworkServiceRegistryClient - creates a chain of clients func NewNetworkServiceRegistryClient(clients ...registry.NetworkServiceRegistryClient) registry.NetworkServiceRegistryClient { - return next.NewNetworkServiceRegistryClient( - next.NewWrappedNetworkServiceRegistryClient(trace.NewNetworkServiceRegistryClient, clients...)) + return next.NewNetworkServiceRegistryClient(next.NewWrappedNetworkServiceRegistryClient(trace.NewNetworkServiceRegistryClient, clients...)) } diff --git a/pkg/registry/core/chain/nse_registry.go b/pkg/registry/core/chain/nse_registry.go index 420c24011..341611d6e 100644 --- a/pkg/registry/core/chain/nse_registry.go +++ b/pkg/registry/core/chain/nse_registry.go @@ -28,12 +28,10 @@ import ( // NewNetworkServiceEndpointRegistryServer - creates a chain of servers func NewNetworkServiceEndpointRegistryServer(servers ...registry.NetworkServiceEndpointRegistryServer) registry.NetworkServiceEndpointRegistryServer { - return next.NewNetworkServiceEndpointRegistryServer( - next.NewWrappedNetworkServiceEndpointRegistryServer(trace.NewNetworkServiceEndpointRegistryServer, servers...)) + return next.NewWrappedNetworkServiceEndpointRegistryServer(trace.NewNetworkServiceEndpointRegistryServer, servers...) } // NewNetworkServiceEndpointRegistryClient - creates a chain of clients func NewNetworkServiceEndpointRegistryClient(clients ...registry.NetworkServiceEndpointRegistryClient) registry.NetworkServiceEndpointRegistryClient { - return next.NewNetworkServiceEndpointRegistryClient( - next.NewWrappedNetworkServiceEndpointRegistryClient(trace.NewNetworkServiceEndpointRegistryClient, clients...)) + return next.NewNetworkServiceEndpointRegistryClient(next.NewWrappedNetworkServiceEndpointRegistryClient(trace.NewNetworkServiceEndpointRegistryClient, clients...)) } diff --git a/pkg/tools/sandbox/builder.go b/pkg/tools/sandbox/builder.go index d77e97e67..e6bb6a0bf 100644 --- a/pkg/tools/sandbox/builder.go +++ b/pkg/tools/sandbox/builder.go @@ -27,6 +27,7 @@ import ( "testing" "time" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/networkservicemesh/sdk/pkg/networkservice/chains/nsmgr" @@ -36,7 +37,6 @@ import ( registryconnect "github.com/networkservicemesh/sdk/pkg/registry/common/connect" "github.com/networkservicemesh/sdk/pkg/registry/common/dnsresolve" "github.com/networkservicemesh/sdk/pkg/tools/grpcutils" - "github.com/networkservicemesh/sdk/pkg/tools/log" "github.com/networkservicemesh/sdk/pkg/tools/token" ) @@ -242,7 +242,7 @@ func (b *Builder) newRegistryProxy() *RegistryEntry { ) serve(ctx, b.t, entry.URL, entry.Register) - log.FromContext(ctx).Debugf("%s: registry-proxy-dns on: %v", b.name, entry.URL) + logrus.Errorf("%s: registry-proxy-dns serve on: %v", b.name, entry.URL) }) return entry @@ -270,7 +270,7 @@ func (b *Builder) newRegistry() *RegistryEntry { ) serve(ctx, b.t, entry.URL, entry.Register) - log.FromContext(ctx).Debugf("%s: registry on: %v", b.name, entry.URL) + logrus.Errorf("%s: registry serve on: %v", b.name, entry.URL) }) return entry @@ -300,7 +300,7 @@ func (b *Builder) newNSMgrProxy() *NSMgrEntry { ) serve(ctx, b.t, entry.URL, entry.Register) - log.FromContext(ctx).Debugf("%s: NSMgr proxy %s on: %v", b.name, entry.Name, entry.URL) + logrus.Errorf("%s: NSMgr proxy %s serve on: %v", b.name, entry.Name, entry.URL) }) return entry diff --git a/pkg/tools/sandbox/grpc_utils.go b/pkg/tools/sandbox/grpc_utils.go index a090afd35..7db4d289a 100644 --- a/pkg/tools/sandbox/grpc_utils.go +++ b/pkg/tools/sandbox/grpc_utils.go @@ -39,6 +39,7 @@ func serve(ctx context.Context, t *testing.T, u *url.URL, register func(server * errCh := grpcutils.ListenAndServe(ctx, u, server) uString := u.String() + go func() { select { case <-ctx.Done(): diff --git a/pkg/tools/sandbox/node.go b/pkg/tools/sandbox/node.go index 0ff37ba4e..830c9c166 100644 --- a/pkg/tools/sandbox/node.go +++ b/pkg/tools/sandbox/node.go @@ -23,18 +23,19 @@ import ( "github.com/networkservicemesh/api/pkg/api/networkservice" registryapi "github.com/networkservicemesh/api/pkg/api/registry" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/networkservicemesh/sdk/pkg/networkservice/chains/client" "github.com/networkservicemesh/sdk/pkg/networkservice/chains/endpoint" "github.com/networkservicemesh/sdk/pkg/networkservice/chains/nsmgr" "github.com/networkservicemesh/sdk/pkg/networkservice/common/authorize" - "github.com/networkservicemesh/sdk/pkg/networkservice/common/clienturl" "github.com/networkservicemesh/sdk/pkg/networkservice/common/connect" + "github.com/networkservicemesh/sdk/pkg/networkservice/common/discover" "github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanismtranslation" "github.com/networkservicemesh/sdk/pkg/networkservice/common/retry" + "github.com/networkservicemesh/sdk/pkg/networkservice/common/roundrobin" registryclient "github.com/networkservicemesh/sdk/pkg/registry/chains/client" - "github.com/networkservicemesh/sdk/pkg/tools/log" "github.com/networkservicemesh/sdk/pkg/tools/token" ) @@ -84,7 +85,7 @@ func (n *Node) NewNSMgr( entry.Nsmgr = supplyNSMgr(ctx, generatorFunc, options...) serve(ctx, n.t, entry.URL, entry.Register) - log.FromContext(ctx).Debugf("%s: NSMgr %s on %v", n.domain.Name, name, serveURL) + logrus.Infof("%s: NSMgr %s serve on %v", n.domain.Name, name, serveURL) }) n.NSMgr = entry @@ -116,13 +117,17 @@ func (n *Node) NewForwarder( Name: nse.Name, URL: serveURL, } + nseClient := registryclient.NewNetworkServiceEndpointRegistryClient(ctx, CloneURL(n.NSMgr.URL), registryclient.WithDialOptions(dialOptions...)) + nsClient := registryclient.NewNetworkServiceRegistryClient(ctx, CloneURL(n.NSMgr.URL), registryclient.WithDialOptions(dialOptions...)) entry.restartableServer = newRestartableServer(ctx, n.t, entry.URL, func(ctx context.Context) { entry.Endpoint = endpoint.NewServer(ctx, generatorFunc, endpoint.WithName(entry.Name), endpoint.WithAdditionalFunctionality( append( - additionalFunctionality, - clienturl.NewServer(CloneURL(n.NSMgr.URL)), + append([]networkservice.NetworkServiceServer{ + discover.NewServer(nsClient, nseClient), + roundrobin.NewServer(), + }, additionalFunctionality...), connect.NewServer( client.NewClient( ctx, @@ -140,6 +145,8 @@ func (n *Node) NewForwarder( ) serve(ctx, n.t, entry.URL, entry.Endpoint.Register) + logrus.Infof("%s: forwarder %s serve on %v", n.domain.Name, nse.Name, serveURL) + entry.NetworkServiceEndpointRegistryClient = registryclient.NewNetworkServiceEndpointRegistryInterposeClient(ctx, CloneURL(n.NSMgr.URL), registryclient.WithDialOptions(dialOptions...)) @@ -180,8 +187,11 @@ func (n *Node) NewEndpoint( endpoint.WithName(entry.Name), endpoint.WithAdditionalFunctionality(additionalFunctionality...), ) + serve(ctx, n.t, entry.URL, entry.Endpoint.Register) + logrus.Infof("%s: NSE %s serve on %v", n.domain.Name, nse.Name, serveURL) + entry.NetworkServiceEndpointRegistryClient = registryclient.NewNetworkServiceEndpointRegistryClient(ctx, CloneURL(n.NSMgr.URL), registryclient.WithDialOptions(dialOptions...)) From c972a6b3a6d34385bd8578756e4df5c9482b9836 Mon Sep 17 00:00:00 2001 From: Denis Tingaikin Date: Thu, 4 Nov 2021 03:40:07 +0700 Subject: [PATCH 2/6] add fixes after manual testing Signed-off-by: Denis Tingaikin --- pkg/networkservice/chains/nsmgr/scale_test.go | 1 - pkg/networkservice/chains/nsmgr/server.go | 3 + .../chains/nsmgr/single_test.go | 2 - pkg/networkservice/chains/nsmgr/suite_test.go | 6 +- pkg/networkservice/chains/nsmgr/unix_test.go | 2 + pkg/registry/common/recvfd/client.go | 95 +++++++++++++++++++ pkg/registry/common/recvfd/client_nolinux.go | 31 ++++++ .../{client_other.go => client_nolinux.go} | 2 + pkg/registry/common/sendfd/server_linux.go | 73 ++++++++++++++ pkg/registry/common/sendfd/server_nolinux.go | 29 ++++++ pkg/tools/sandbox/node.go | 6 +- 11 files changed, 242 insertions(+), 8 deletions(-) create mode 100644 pkg/registry/common/recvfd/client.go create mode 100644 pkg/registry/common/recvfd/client_nolinux.go rename pkg/registry/common/sendfd/{client_other.go => client_nolinux.go} (96%) create mode 100644 pkg/registry/common/sendfd/server_linux.go create mode 100644 pkg/registry/common/sendfd/server_nolinux.go diff --git a/pkg/networkservice/chains/nsmgr/scale_test.go b/pkg/networkservice/chains/nsmgr/scale_test.go index c6239db89..f81e157ec 100644 --- a/pkg/networkservice/chains/nsmgr/scale_test.go +++ b/pkg/networkservice/chains/nsmgr/scale_test.go @@ -36,7 +36,6 @@ import ( ) func TestCreateEndpointDuringRequest(t *testing.T) { - t.Skip("TODO") t.Cleanup(func() { goleak.VerifyNone(t) }) ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) diff --git a/pkg/networkservice/chains/nsmgr/server.go b/pkg/networkservice/chains/nsmgr/server.go index c4df89d89..42755d662 100644 --- a/pkg/networkservice/chains/nsmgr/server.go +++ b/pkg/networkservice/chains/nsmgr/server.go @@ -54,6 +54,8 @@ import ( "github.com/networkservicemesh/sdk/pkg/registry/common/localbypass" "github.com/networkservicemesh/sdk/pkg/registry/common/memory" registryrecvfd "github.com/networkservicemesh/sdk/pkg/registry/common/recvfd" + registrysendfd "github.com/networkservicemesh/sdk/pkg/registry/common/sendfd" + registryserialize "github.com/networkservicemesh/sdk/pkg/registry/common/serialize" "github.com/networkservicemesh/sdk/pkg/registry/common/setlogoption" registryadapter "github.com/networkservicemesh/sdk/pkg/registry/core/adapters" @@ -222,6 +224,7 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, options checkid.NewNetworkServiceEndpointRegistryServer(), expire.NewNetworkServiceEndpointRegistryServer(ctx, time.Minute), registryrecvfd.NewNetworkServiceEndpointRegistryServer(), // Allow to receive a passed files + registrysendfd.NewNetworkServiceEndpointRegistryServer(), interposeRegistryServer, // Store cross connect NSEs localBypassRegistryServer, // Perform URL transformations nseRegistry, diff --git a/pkg/networkservice/chains/nsmgr/single_test.go b/pkg/networkservice/chains/nsmgr/single_test.go index 5f6eee01c..d9cbdbeae 100644 --- a/pkg/networkservice/chains/nsmgr/single_test.go +++ b/pkg/networkservice/chains/nsmgr/single_test.go @@ -99,8 +99,6 @@ func Test_DNSUsecase(t *testing.T) { } func Test_ShouldCorrectlyAddForwardersWithSameNames(t *testing.T) { - t.Skip("TODO") - t.Cleanup(func() { goleak.VerifyNone(t) }) ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) diff --git a/pkg/networkservice/chains/nsmgr/suite_test.go b/pkg/networkservice/chains/nsmgr/suite_test.go index 84415f2a5..c85aaa0db 100644 --- a/pkg/networkservice/chains/nsmgr/suite_test.go +++ b/pkg/networkservice/chains/nsmgr/suite_test.go @@ -196,8 +196,6 @@ func (s *nsmgrSuite) Test_SelectsRestartingEndpointUsecase() { func (s *nsmgrSuite) Test_Remote_BusyEndpointsUsecase() { t := s.T() - t.Skip("TODO") - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() @@ -240,7 +238,7 @@ func (s *nsmgrSuite) Test_Remote_BusyEndpointsUsecase() { require.NoError(t, err) require.NotNil(t, conn) require.Equal(t, 1, counter.Requests()) - require.Equal(t, 8, len(conn.Path.PathSegments)) + require.Equal(t, 6, len(conn.Path.PathSegments)) // Simulate refresh from client refreshRequest := request.Clone() @@ -250,7 +248,7 @@ func (s *nsmgrSuite) Test_Remote_BusyEndpointsUsecase() { require.NoError(t, err) require.NotNil(t, conn) require.Equal(t, 2, counter.Requests()) - require.Equal(t, 8, len(conn.Path.PathSegments)) + require.Equal(t, 6, len(conn.Path.PathSegments)) // Close _, err = nsc.Close(ctx, conn) diff --git a/pkg/networkservice/chains/nsmgr/unix_test.go b/pkg/networkservice/chains/nsmgr/unix_test.go index d94db059f..d05aedcef 100644 --- a/pkg/networkservice/chains/nsmgr/unix_test.go +++ b/pkg/networkservice/chains/nsmgr/unix_test.go @@ -39,6 +39,7 @@ import ( ) func Test_Local_NoURLUsecase(t *testing.T) { + t.Skip("TODO") t.Cleanup(func() { goleak.VerifyNone(t) }) ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) @@ -84,6 +85,7 @@ func Test_Local_NoURLUsecase(t *testing.T) { } func Test_MultiForwarderSendfd(t *testing.T) { + t.Skip("TODO") if runtime.GOOS != "linux" { t.Skip("sendfd works only on linux") } diff --git a/pkg/registry/common/recvfd/client.go b/pkg/registry/common/recvfd/client.go new file mode 100644 index 000000000..d4a66b91b --- /dev/null +++ b/pkg/registry/common/recvfd/client.go @@ -0,0 +1,95 @@ +// Copyright (c) 2021 Cisco and/or its affiliates. +// +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build linux + +package recvfd + +import ( + "context" + "net/url" + "os" + + "github.com/edwarnicke/grpcfd" + "github.com/golang/protobuf/ptypes/empty" + "github.com/networkservicemesh/api/pkg/api/registry" + "google.golang.org/grpc" + + "github.com/networkservicemesh/sdk/pkg/registry/core/next" +) + +type recvfdNSEClient struct { + fileMaps perEndpointFileMapMap +} + +func (n *recvfdNSEClient) Register(ctx context.Context, in *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (*registry.NetworkServiceEndpoint, error) { + return next.NetworkServiceEndpointRegistryClient(ctx).Register(ctx, in, opts...) +} + +func (n *recvfdNSEClient) Find(ctx context.Context, in *registry.NetworkServiceEndpointQuery, opts ...grpc.CallOption) (registry.NetworkServiceEndpointRegistry_FindClient, error) { + rpcCredentials := grpcfd.PerRPCCredentials(grpcfd.PerRPCCredentialsFromCallOptions(opts...)) + opts = append(opts, grpc.PerRPCCredentials(rpcCredentials)) + recv, _ := grpcfd.FromPerRPCCredentials(rpcCredentials) + resp, err := next.NetworkServiceEndpointRegistryClient(ctx).Find(ctx, in, opts...) + if err != nil { + return nil, err + } + return &recvfdNSEFindClient{ + transceiver: recv, + NetworkServiceEndpointRegistry_FindClient: resp, + fileMaps: &n.fileMaps, + }, nil +} + +func (n *recvfdNSEClient) Unregister(ctx context.Context, in *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (*empty.Empty, error) { + return next.NetworkServiceEndpointRegistryClient(ctx).Unregister(ctx, in, opts...) +} + +// NewNetworkServiceEndpointRegistryClient - returns a new null client that does nothing but call next.NetworkServiceEndpointRegistryClient(ctx). +func NewNetworkServiceEndpointRegistryClient() registry.NetworkServiceEndpointRegistryClient { + return new(recvfdNSEClient) +} + +type recvfdNSEFindClient struct { + registry.NetworkServiceEndpointRegistry_FindClient + transceiver grpcfd.FDTransceiver + fileMaps *perEndpointFileMapMap +} + +func (x *recvfdNSEFindClient) Recv() (*registry.NetworkServiceEndpointResponse, error) { + nseResp, err := x.NetworkServiceEndpointRegistry_FindClient.Recv() + if err != nil { + return nil, err + } + if x.transceiver != nil { + // Get the fileMap + fileMap := &perEndpointFileMap{ + filesByInodeURL: make(map[string]*os.File), + inodeURLbyFilename: make(map[string]*url.URL), + } + endpointName := nseResp.GetNetworkServiceEndpoint().GetName() + // If name is specified, let's use it, since it could be heal/update request + if endpointName != "" { + fileMap, _ = x.fileMaps.LoadOrStore(nseResp.NetworkServiceEndpoint.GetName(), fileMap) + } + + // Recv the FD and swap theInode to File in the Parameters for the returned connection mechanism + err = recvFDAndSwapInodeToUnix(x.Context(), fileMap, nseResp.GetNetworkServiceEndpoint(), x.transceiver) + } + return nseResp, err +} diff --git a/pkg/registry/common/recvfd/client_nolinux.go b/pkg/registry/common/recvfd/client_nolinux.go new file mode 100644 index 000000000..511b874b0 --- /dev/null +++ b/pkg/registry/common/recvfd/client_nolinux.go @@ -0,0 +1,31 @@ +// Copyright (c) 2021 Cisco and/or its affiliates. +// +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !linux + +package recvfd + +import ( + "github.com/networkservicemesh/api/pkg/api/registry" + "github.com/networkservicemesh/sdk/pkg/registry/common/null" +) + +// NewNetworkServiceEndpointRegistryClient - returns a new null client that does nothing but call next.NetworkServiceEndpointRegistryClient(ctx). +func NewNetworkServiceEndpointRegistryClient() registry.NetworkServiceEndpointRegistryClient { + return null.NewNetworkServiceEndpointRegistryClient() +} diff --git a/pkg/registry/common/sendfd/client_other.go b/pkg/registry/common/sendfd/client_nolinux.go similarity index 96% rename from pkg/registry/common/sendfd/client_other.go rename to pkg/registry/common/sendfd/client_nolinux.go index 326a8443f..49eddd696 100644 --- a/pkg/registry/common/sendfd/client_other.go +++ b/pkg/registry/common/sendfd/client_nolinux.go @@ -1,3 +1,5 @@ +// Copyright (c) 2021 Cisco and/or its affiliates. +// // Copyright (c) 2021 Doc.ai and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 diff --git a/pkg/registry/common/sendfd/server_linux.go b/pkg/registry/common/sendfd/server_linux.go new file mode 100644 index 000000000..35d27bbf8 --- /dev/null +++ b/pkg/registry/common/sendfd/server_linux.go @@ -0,0 +1,73 @@ +// Copyright (c) 2021 Cisco and/or its affiliates. +// +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build linux + +package sendfd + +import ( + "context" + + "github.com/edwarnicke/grpcfd" + "github.com/golang/protobuf/ptypes/empty" + "github.com/networkservicemesh/api/pkg/api/registry" + + "github.com/networkservicemesh/sdk/pkg/registry/core/next" +) + +type sendfdNSEServer struct{} + +func (n *sendfdNSEServer) Register(ctx context.Context, service *registry.NetworkServiceEndpoint) (*registry.NetworkServiceEndpoint, error) { + return next.NetworkServiceEndpointRegistryServer(ctx).Register(ctx, service) +} + +func (n *sendfdNSEServer) Find(query *registry.NetworkServiceEndpointQuery, server registry.NetworkServiceEndpointRegistry_FindServer) error { + var nextServer = server + + sender, ok := grpcfd.FromContext(server.Context()) + if ok { + nextServer = &sendfdNSEFindServer{ + NetworkServiceEndpointRegistry_FindServer: server, + transceiver: sender, + } + } + + return next.NetworkServiceEndpointRegistryServer(nextServer.Context()).Find(query, nextServer) +} + +func (n *sendfdNSEServer) Unregister(ctx context.Context, nse *registry.NetworkServiceEndpoint) (*empty.Empty, error) { + return next.NetworkServiceEndpointRegistryServer(ctx).Unregister(ctx, nse) +} + +// NewNetworkServiceEndpointRegistryServer - returns a new null server that does nothing but call next.NetworkServiceEndpointRegistryServer(ctx). +func NewNetworkServiceEndpointRegistryServer() registry.NetworkServiceEndpointRegistryServer { + return new(sendfdNSEServer) +} + +type sendfdNSEFindServer struct { + transceiver grpcfd.FDTransceiver + registry.NetworkServiceEndpointRegistry_FindServer +} + +func (s *sendfdNSEFindServer) Send(nseResp *registry.NetworkServiceEndpointResponse) error { + var inodeURLToFileURLMap = make(map[string]string) + if err := sendFDAndSwapFileToInode(s.transceiver, nseResp.GetNetworkServiceEndpoint(), inodeURLToFileURLMap); err != nil { + return err + } + return s.NetworkServiceEndpointRegistry_FindServer.Send(nseResp) +} diff --git a/pkg/registry/common/sendfd/server_nolinux.go b/pkg/registry/common/sendfd/server_nolinux.go new file mode 100644 index 000000000..0df5aae1a --- /dev/null +++ b/pkg/registry/common/sendfd/server_nolinux.go @@ -0,0 +1,29 @@ +// Copyright (c) 2020 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !linux + +package sendfd + +import ( + "github.com/networkservicemesh/api/pkg/api/registry" + "github.com/networkservicemesh/sdk/pkg/registry/common/null" +) + +// NewNetworkServiceEndpointRegistryServer - returns a new null server that does nothing but call next.NetworkServiceEndpointRegistryServer(ctx). +func NewNetworkServiceEndpointRegistryServer() registry.NetworkServiceEndpointRegistryServer { + return null.NewNetworkServiceEndpointRegistryServer() +} diff --git a/pkg/tools/sandbox/node.go b/pkg/tools/sandbox/node.go index 830c9c166..e2feb8d3e 100644 --- a/pkg/tools/sandbox/node.go +++ b/pkg/tools/sandbox/node.go @@ -36,6 +36,8 @@ import ( "github.com/networkservicemesh/sdk/pkg/networkservice/common/retry" "github.com/networkservicemesh/sdk/pkg/networkservice/common/roundrobin" registryclient "github.com/networkservicemesh/sdk/pkg/registry/chains/client" + "github.com/networkservicemesh/sdk/pkg/registry/common/recvfd" + "github.com/networkservicemesh/sdk/pkg/registry/core/chain" "github.com/networkservicemesh/sdk/pkg/tools/token" ) @@ -117,7 +119,9 @@ func (n *Node) NewForwarder( Name: nse.Name, URL: serveURL, } - nseClient := registryclient.NewNetworkServiceEndpointRegistryClient(ctx, CloneURL(n.NSMgr.URL), registryclient.WithDialOptions(dialOptions...)) + nseClient := chain.NewNetworkServiceEndpointRegistryClient( + recvfd.NewNetworkServiceEndpointRegistryClient(), + registryclient.NewNetworkServiceEndpointRegistryClient(ctx, CloneURL(n.NSMgr.URL), registryclient.WithDialOptions(dialOptions...))) nsClient := registryclient.NewNetworkServiceRegistryClient(ctx, CloneURL(n.NSMgr.URL), registryclient.WithDialOptions(dialOptions...)) entry.restartableServer = newRestartableServer(ctx, n.t, entry.URL, func(ctx context.Context) { entry.Endpoint = endpoint.NewServer(ctx, generatorFunc, From 8526aa11223e094cd482c816314af3583a500a36 Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Sat, 6 Nov 2021 17:03:39 +0300 Subject: [PATCH 3/6] delete interpose pkg Signed-off-by: denis-tingaikin --- pkg/networkservice/chains/nsmgr/server.go | 60 ++++---- pkg/networkservice/common/connect/client.go | 7 - pkg/networkservice/common/retry/client.go | 4 +- pkg/registry/chains/client/nse_client.go | 19 --- pkg/registry/common/heal/find_test.go | 17 +- pkg/registry/common/interpose/client.go | 58 ------- pkg/registry/common/interpose/doc.go | 18 --- pkg/registry/common/interpose/name.go | 30 ---- pkg/registry/common/interpose/server.go | 100 ------------ pkg/registry/common/interpose/server_test.go | 145 ------------------ .../common/localbypass/find_server.go | 6 +- pkg/registry/common/recvfd/client_nolinux.go | 1 + pkg/registry/common/sendfd/server_nolinux.go | 3 +- .../setnetworkservicenames/nse_client.go | 56 +++++++ pkg/tools/sandbox/builder.go | 5 +- pkg/tools/sandbox/node.go | 26 +++- 16 files changed, 119 insertions(+), 436 deletions(-) delete mode 100644 pkg/registry/common/interpose/client.go delete mode 100644 pkg/registry/common/interpose/doc.go delete mode 100644 pkg/registry/common/interpose/name.go delete mode 100644 pkg/registry/common/interpose/server.go delete mode 100644 pkg/registry/common/interpose/server_test.go create mode 100644 pkg/registry/common/setnetworkservicenames/nse_client.go diff --git a/pkg/networkservice/chains/nsmgr/server.go b/pkg/networkservice/chains/nsmgr/server.go index 42755d662..32e34aa7d 100644 --- a/pkg/networkservice/chains/nsmgr/server.go +++ b/pkg/networkservice/chains/nsmgr/server.go @@ -50,11 +50,11 @@ import ( "github.com/networkservicemesh/sdk/pkg/registry/common/clienturl" registryconnect "github.com/networkservicemesh/sdk/pkg/registry/common/connect" "github.com/networkservicemesh/sdk/pkg/registry/common/expire" - "github.com/networkservicemesh/sdk/pkg/registry/common/interpose" "github.com/networkservicemesh/sdk/pkg/registry/common/localbypass" "github.com/networkservicemesh/sdk/pkg/registry/common/memory" registryrecvfd "github.com/networkservicemesh/sdk/pkg/registry/common/recvfd" registrysendfd "github.com/networkservicemesh/sdk/pkg/registry/common/sendfd" + "github.com/networkservicemesh/sdk/pkg/registry/common/setnetworkservicenames" registryserialize "github.com/networkservicemesh/sdk/pkg/registry/common/serialize" "github.com/networkservicemesh/sdk/pkg/registry/common/setlogoption" @@ -152,34 +152,44 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, options rv := &nsmgrServer{} - var interposeRegistryServer = interpose.NewNetworkServiceEndpointRegistryServer() - - var nsRegistry registryapi.NetworkServiceRegistryServer + var nsRegistry = memory.NewNetworkServiceRegistryServer() if opts.regURL != nil { // Use remote registry nsRegistry = registrychain.NewNetworkServiceRegistryServer( clienturl.NewNetworkServiceRegistryServer(opts.regURL), registryconnect.NewNetworkServiceRegistryServer(ctx, registryconnect.WithDialOptions(opts.regDialOptions...)), ) - } else { - // Use memory registry if no registry is passed - nsRegistry = memory.NewNetworkServiceRegistryServer() } - var nseRegistry registryapi.NetworkServiceEndpointRegistryServer + nsRegistry = registrychain.NewNetworkServiceRegistryServer( + registryserialize.NewNetworkServiceRegistryServer(), + setlogoption.NewNetworkServiceRegistryServer(map[string]string{"name": "NetworkServiceRegistryServer." + opts.name}), + nsRegistry, + ) + + var nseInMemoryRegistry = registrychain.NewNetworkServiceEndpointRegistryServer( + setlogoption.NewNetworkServiceEndpointRegistryServer(map[string]string{"name": fmt.Sprintf("NetworkServiceRegistryServer.%v", opts.name)}), + registryclientinfo.NewNetworkServiceEndpointRegistryServer(), + registryserialize.NewNetworkServiceEndpointRegistryServer(), + checkid.NewNetworkServiceEndpointRegistryServer(), + expire.NewNetworkServiceEndpointRegistryServer(ctx, time.Minute), + registryrecvfd.NewNetworkServiceEndpointRegistryServer(), // Allow to receive a passed files + registrysendfd.NewNetworkServiceEndpointRegistryServer(), + memory.NewNetworkServiceEndpointRegistryServer(), + localbypass.NewNetworkServiceEndpointRegistryServer(opts.url), + ) + + var nseRegistry = nseInMemoryRegistry + if opts.regURL != nil { - // Use remote registry + // Add remote registry nseRegistry = registrychain.NewNetworkServiceEndpointRegistryServer( + nseRegistry, clienturl.NewNetworkServiceEndpointRegistryServer(opts.regURL), registryconnect.NewNetworkServiceEndpointRegistryServer(ctx, registryconnect.WithDialOptions(opts.regDialOptions...)), ) - } else { - // Use memory registry if no registry is passed - nseRegistry = memory.NewNetworkServiceEndpointRegistryServer() } - localBypassRegistryServer := localbypass.NewNetworkServiceEndpointRegistryServer(opts.url) - // Construct Endpoint rv.Endpoint = endpoint.NewServer(ctx, tokenGenerator, endpoint.WithName(opts.name), @@ -189,8 +199,8 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, options discovercrossnse.NewServer( registryadapter.NetworkServiceServerToClient(nsRegistry), registrychain.NewNetworkServiceEndpointRegistryClient( - interpose.NewNetworkServiceEndpointRegistryClient(), - registryadapter.NetworkServiceEndpointServerToClient(interposeRegistryServer), + setnetworkservicenames.NewNetworkServiceEndpointRegistryClient("forwarder"), + registryadapter.NetworkServiceEndpointServerToClient(nseInMemoryRegistry), ), roundrobin.NewServer(), ), @@ -212,24 +222,6 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, options sendfd.NewServer()), ) - nsRegistry = registrychain.NewNetworkServiceRegistryServer( - registryserialize.NewNetworkServiceRegistryServer(), - setlogoption.NewNetworkServiceRegistryServer(map[string]string{"name": "NetworkServiceRegistryServer." + opts.name}), - nsRegistry, - ) - nseRegistry = registrychain.NewNetworkServiceEndpointRegistryServer( - setlogoption.NewNetworkServiceEndpointRegistryServer(map[string]string{"name": fmt.Sprintf("NetworkServiceRegistryServer.%v", opts.name)}), - registryclientinfo.NewNetworkServiceEndpointRegistryServer(), - registryserialize.NewNetworkServiceEndpointRegistryServer(), - checkid.NewNetworkServiceEndpointRegistryServer(), - expire.NewNetworkServiceEndpointRegistryServer(ctx, time.Minute), - registryrecvfd.NewNetworkServiceEndpointRegistryServer(), // Allow to receive a passed files - registrysendfd.NewNetworkServiceEndpointRegistryServer(), - interposeRegistryServer, // Store cross connect NSEs - localBypassRegistryServer, // Perform URL transformations - nseRegistry, - ) - rv.Registry = registry.NewServer( nsRegistry, nseRegistry, diff --git a/pkg/networkservice/common/connect/client.go b/pkg/networkservice/common/connect/client.go index 4e96769d6..cb632eca1 100644 --- a/pkg/networkservice/common/connect/client.go +++ b/pkg/networkservice/common/connect/client.go @@ -25,16 +25,10 @@ import ( "google.golang.org/protobuf/types/known/emptypb" "github.com/networkservicemesh/sdk/pkg/networkservice/common/clientconn" - "github.com/networkservicemesh/sdk/pkg/tools/log" ) type connectClient struct{} -// TODO delete it -type debugMate interface { - Target() string -} - // NewClient - returns a connect chain element func NewClient() networkservice.NetworkServiceClient { return &connectClient{} @@ -45,7 +39,6 @@ func (c *connectClient) Request(ctx context.Context, request *networkservice.Net if !loaded { return nil, errors.New("no grpc.ClientConnInterface provided") } - log.FromContext(ctx).Infof("see at serve on: %v", cc.(debugMate).Target()) conn, err := networkservice.NewNetworkServiceClient(cc).Request(ctx, request, opts...) return conn, err } diff --git a/pkg/networkservice/common/retry/client.go b/pkg/networkservice/common/retry/client.go index e95ae0fe3..53a07ca5b 100644 --- a/pkg/networkservice/common/retry/client.go +++ b/pkg/networkservice/common/retry/client.go @@ -73,7 +73,7 @@ func (r *retryClient) Request(ctx context.Context, request *networkservice.Netwo for ctx.Err() == nil { requestCtx, cancel := c.WithTimeout(ctx, r.tryTimeout) - resp, err := r.client.Request(requestCtx, request, opts...) + resp, err := r.client.Request(requestCtx, request.Clone(), opts...) cancel() if err != nil { @@ -100,7 +100,7 @@ func (r *retryClient) Close(ctx context.Context, conn *networkservice.Connection for ctx.Err() == nil { closeCtx, cancel := c.WithTimeout(ctx, r.tryTimeout) - resp, err := r.client.Close(closeCtx, conn, opts...) + resp, err := r.client.Close(closeCtx, conn.Clone(), opts...) cancel() if err != nil { diff --git a/pkg/registry/chains/client/nse_client.go b/pkg/registry/chains/client/nse_client.go index 43f6e00b2..de4ff8fa5 100644 --- a/pkg/registry/chains/client/nse_client.go +++ b/pkg/registry/chains/client/nse_client.go @@ -25,8 +25,6 @@ import ( "github.com/networkservicemesh/sdk/pkg/registry/common/connect" "github.com/networkservicemesh/sdk/pkg/registry/common/heal" - "github.com/networkservicemesh/sdk/pkg/registry/common/interpose" - "github.com/networkservicemesh/sdk/pkg/registry/common/null" "github.com/networkservicemesh/sdk/pkg/registry/common/refresh" "github.com/networkservicemesh/sdk/pkg/registry/common/sendfd" "github.com/networkservicemesh/sdk/pkg/registry/common/serialize" @@ -38,32 +36,15 @@ import ( // NewNetworkServiceEndpointRegistryClient creates a new NewNetworkServiceEndpointRegistryClient that can be used for NSE registration. func NewNetworkServiceEndpointRegistryClient(ctx context.Context, connectTo *url.URL, opts ...Option) registry.NetworkServiceEndpointRegistryClient { - return newNetworkServiceEndpointRegistryClient(ctx, connectTo, false, opts...) -} - -// NewNetworkServiceEndpointRegistryInterposeClient creates a new registry.NetworkServiceEndpointRegistryClient that can be used for cross-nse registration -func NewNetworkServiceEndpointRegistryInterposeClient(ctx context.Context, connectTo *url.URL, opts ...Option) registry.NetworkServiceEndpointRegistryClient { - return newNetworkServiceEndpointRegistryClient(ctx, connectTo, true, opts...) -} - -func newNetworkServiceEndpointRegistryClient(ctx context.Context, connectTo *url.URL, withInterpose bool, opts ...Option) registry.NetworkServiceEndpointRegistryClient { clientOpts := new(clientOptions) for _, opt := range opts { opt(clientOpts) } - var interposeClient registry.NetworkServiceEndpointRegistryClient - if withInterpose { - interposeClient = interpose.NewNetworkServiceEndpointRegistryClient() - } else { - interposeClient = null.NewNetworkServiceEndpointRegistryClient() - } - c := new(registry.NetworkServiceEndpointRegistryClient) *c = chain.NewNetworkServiceEndpointRegistryClient( adapters.NetworkServiceEndpointServerToClient(setlogoption.NewNetworkServiceEndpointRegistryServer(map[string]string{})), setid.NewNetworkServiceEndpointRegistryClient(), - interposeClient, serialize.NewNetworkServiceEndpointRegistryClient(), refresh.NewNetworkServiceEndpointRegistryClient(ctx), connect.NewNetworkServiceEndpointRegistryClient(ctx, connectTo, diff --git a/pkg/registry/common/heal/find_test.go b/pkg/registry/common/heal/find_test.go index 1e321594b..424759a85 100644 --- a/pkg/registry/common/heal/find_test.go +++ b/pkg/registry/common/heal/find_test.go @@ -106,14 +106,13 @@ func TestHealClient_FindTest(t *testing.T) { // 5. Close NS, NSE streams findCancel() - // 6. Validate NS, NSE streams closed - timer := time.AfterFunc(100*time.Millisecond, t.FailNow) - - _, err = nsRespStream.Recv() - require.Error(t, err) - - _, err = nseRespStream.Recv() - require.Error(t, err) + require.Eventually(t, func() bool { + _, err := nsRespStream.Recv() + return err != nil + }, time.Millisecond*100, time.Millisecond*10) - timer.Stop() + require.Eventually(t, func() bool { + _, err := nseRespStream.Recv() + return err != nil + }, time.Millisecond*100, time.Millisecond*10) } diff --git a/pkg/registry/common/interpose/client.go b/pkg/registry/common/interpose/client.go deleted file mode 100644 index c6981f98b..000000000 --- a/pkg/registry/common/interpose/client.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2020-2021 Doc.ai and/or its affiliates. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package interpose - -import ( - "context" - - "github.com/golang/protobuf/ptypes/empty" - "google.golang.org/grpc" - - "github.com/networkservicemesh/api/pkg/api/registry" - - "github.com/networkservicemesh/sdk/pkg/registry/core/next" -) - -type interposeRegistryClient struct{} - -// NewNetworkServiceEndpointRegistryClient - creates a Client that will replace any passed endpoint with CrossConnect NSE name for proper registration -func NewNetworkServiceEndpointRegistryClient() registry.NetworkServiceEndpointRegistryClient { - return &interposeRegistryClient{} -} - -func (rc *interposeRegistryClient) Register(ctx context.Context, in *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (*registry.NetworkServiceEndpoint, error) { - if !Is(in.Name) { - in.Name = interposeName(in.Name) - } - return next.NetworkServiceEndpointRegistryClient(ctx).Register(ctx, in, opts...) -} - -func (rc *interposeRegistryClient) Find(ctx context.Context, in *registry.NetworkServiceEndpointQuery, opts ...grpc.CallOption) (registry.NetworkServiceEndpointRegistry_FindClient, error) { - if in.GetNetworkServiceEndpoint() != nil && !Is(in.GetNetworkServiceEndpoint().GetName()) { - in.GetNetworkServiceEndpoint().Name = interposeName(in.NetworkServiceEndpoint.Name) - } - return next.NetworkServiceEndpointRegistryClient(ctx).Find(ctx, in, opts...) -} - -func (rc *interposeRegistryClient) Unregister(ctx context.Context, in *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (*empty.Empty, error) { - if !Is(in.Name) { - in.Name = interposeName(in.Name) - } - return next.NetworkServiceEndpointRegistryClient(ctx).Unregister(ctx, in, opts...) -} - -var _ registry.NetworkServiceEndpointRegistryClient = (*interposeRegistryClient)(nil) diff --git a/pkg/registry/common/interpose/doc.go b/pkg/registry/common/interpose/doc.go deleted file mode 100644 index 0ffa29125..000000000 --- a/pkg/registry/common/interpose/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) 2021 Doc.ai and/or its affiliates. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package interpose provides NSE Registry Server, Client for registering interpose NSE -package interpose diff --git a/pkg/registry/common/interpose/name.go b/pkg/registry/common/interpose/name.go deleted file mode 100644 index 0479e8d7a..000000000 --- a/pkg/registry/common/interpose/name.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2021 Doc.ai and/or its affiliates. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package interpose - -import "strings" - -const nameSuffix = "#interpose-nse" - -func interposeName(name string) string { - return name + nameSuffix -} - -// Is returns true if passed name contains interpose identity -func Is(name string) bool { - return strings.HasSuffix(name, nameSuffix) -} diff --git a/pkg/registry/common/interpose/server.go b/pkg/registry/common/interpose/server.go deleted file mode 100644 index 246196542..000000000 --- a/pkg/registry/common/interpose/server.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright (c) 2020-2021 Doc.ai and/or its affiliates. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package interpose - -import ( - "context" - "net/url" - - "github.com/golang/protobuf/ptypes/empty" - "github.com/pkg/errors" - - "github.com/networkservicemesh/api/pkg/api/registry" - - "github.com/networkservicemesh/sdk/pkg/registry/common/memory" - "github.com/networkservicemesh/sdk/pkg/registry/core/chain" - "github.com/networkservicemesh/sdk/pkg/registry/core/next" -) - -type interposeRegistryServer struct { - backend registry.NetworkServiceEndpointRegistryServer -} - -// NewNetworkServiceEndpointRegistryServer - creates a NetworkServiceRegistryServer that registers local Cross connect Endpoints -// and adds them to Map -func NewNetworkServiceEndpointRegistryServer() registry.NetworkServiceEndpointRegistryServer { - return &interposeRegistryServer{ - backend: chain.NewNetworkServiceEndpointRegistryServer( - memory.NewNetworkServiceEndpointRegistryServer(), - new(breakNSEServer), - ), - } -} - -func (s *interposeRegistryServer) Register(ctx context.Context, nse *registry.NetworkServiceEndpoint) (*registry.NetworkServiceEndpoint, error) { - if !Is(nse.Name) { - return next.NetworkServiceEndpointRegistryServer(ctx).Register(ctx, nse) - } - - u, err := url.Parse(nse.Url) - if err != nil { - return nil, errors.Wrapf(err, "cannot register cross NSE with passed URL: %s", nse.Url) - } - if u.String() == "" { - return nil, errors.Errorf("cannot register cross NSE with passed URL: %s", nse.Url) - } - - return s.backend.Register(ctx, nse) -} - -func (s *interposeRegistryServer) Find(query *registry.NetworkServiceEndpointQuery, server registry.NetworkServiceEndpointRegistry_FindServer) error { - if Is(query.NetworkServiceEndpoint.GetName()) { - query.NetworkServiceEndpoint.NetworkServiceNames = nil - if err := s.backend.Find(query, server); err != nil { - return err - } - return nil - } - return next.NetworkServiceEndpointRegistryServer(server.Context()).Find(query, server) -} - -func (s *interposeRegistryServer) Unregister(ctx context.Context, nse *registry.NetworkServiceEndpoint) (*empty.Empty, error) { - if !Is(nse.Name) { - return next.NetworkServiceEndpointRegistryServer(ctx).Unregister(ctx, nse) - } - - return s.backend.Unregister(ctx, nse) -} - -var _ registry.NetworkServiceEndpointRegistryServer = (*interposeRegistryServer)(nil) - -// TODO Should we use it? Should we move it to separate pkg and use in the next pkg? -type breakNSEServer struct{} - -func (*breakNSEServer) Register(_ context.Context, r *registry.NetworkServiceEndpoint) (*registry.NetworkServiceEndpoint, error) { - return r, nil -} - -func (*breakNSEServer) Find(_ *registry.NetworkServiceEndpointQuery, _ registry.NetworkServiceEndpointRegistry_FindServer) error { - return nil -} - -func (*breakNSEServer) Unregister(context.Context, *registry.NetworkServiceEndpoint) (*empty.Empty, error) { - return new(empty.Empty), nil -} - -var _ registry.NetworkServiceEndpointRegistryServer = &breakNSEServer{} diff --git a/pkg/registry/common/interpose/server_test.go b/pkg/registry/common/interpose/server_test.go deleted file mode 100644 index 85de3364f..000000000 --- a/pkg/registry/common/interpose/server_test.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (c) 2020-2021 Doc.ai and/or its affiliates. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package interpose_test - -import ( - "context" - "testing" - - "github.com/golang/protobuf/ptypes/empty" - "github.com/stretchr/testify/require" - "go.uber.org/goleak" - - "github.com/networkservicemesh/api/pkg/api/registry" - - "github.com/networkservicemesh/sdk/pkg/registry/common/interpose" - "github.com/networkservicemesh/sdk/pkg/registry/core/adapters" - "github.com/networkservicemesh/sdk/pkg/registry/core/chain" - "github.com/networkservicemesh/sdk/pkg/registry/core/next" -) - -const ( - nameSuffix = "#interpose-nse" - name = "nse" - validURL = "tcp://0.0.0.0" -) - -func requireReadList(t *testing.T, server registry.NetworkServiceEndpointRegistryServer) []*registry.NetworkServiceEndpoint { - s, err := chain.NewNetworkServiceEndpointRegistryClient( - interpose.NewNetworkServiceEndpointRegistryClient(), - adapters.NetworkServiceEndpointServerToClient(server)).Find(context.Background(), ®istry.NetworkServiceEndpointQuery{NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{}}) - require.NoError(t, err) - - return registry.ReadNetworkServiceEndpointList(s) -} - -func TestInterposeRegistryServer_Interpose(t *testing.T) { - t.Cleanup(func() { goleak.VerifyNone(t) }) - - captureName := new(captureNameTestRegistryServer) - - server := next.NewNetworkServiceEndpointRegistryServer( - interpose.NewNetworkServiceEndpointRegistryServer(), - captureName, - ) - - reg, err := server.Register(context.Background(), ®istry.NetworkServiceEndpoint{ - Name: name + nameSuffix, - Url: validURL, - }) - require.NoError(t, err) - - require.True(t, interpose.Is(reg.Name)) - require.Empty(t, captureName.name) - - var list = requireReadList(t, server) - - require.Len(t, list, 1) - require.Equal(t, validURL, list[0].Url) - - _, err = server.Unregister(context.Background(), reg) - require.NoError(t, err) - - require.Empty(t, requireReadList(t, server)) -} - -func TestInterposeRegistryServer_Common(t *testing.T) { - captureName := new(captureNameTestRegistryServer) - - server := next.NewNetworkServiceEndpointRegistryServer( - interpose.NewNetworkServiceEndpointRegistryServer(), - captureName, - ) - - reg, err := server.Register(context.Background(), ®istry.NetworkServiceEndpoint{ - Name: name, - }) - require.NoError(t, err) - - require.Equal(t, name, reg.Name) - require.Equal(t, name, captureName.name) - - require.Empty(t, requireReadList(t, server), 1) - - captureName.name = "" - - _, err = server.Unregister(context.Background(), reg) - require.NoError(t, err) - - require.Equal(t, name, captureName.name) - - require.Empty(t, requireReadList(t, server), 1) -} - -func TestInterposeRegistryServer_Invalid(t *testing.T) { - captureName := new(captureNameTestRegistryServer) - - server := next.NewNetworkServiceEndpointRegistryServer( - interpose.NewNetworkServiceEndpointRegistryServer(), - captureName, - ) - - _, err := server.Register(context.Background(), ®istry.NetworkServiceEndpoint{ - Name: name + nameSuffix, - }) - require.Error(t, err) - - require.Empty(t, captureName.name) - require.Empty(t, requireReadList(t, server), 1) -} - -type captureNameTestRegistryServer struct { - name string - - registry.NetworkServiceEndpointRegistryServer -} - -func (r *captureNameTestRegistryServer) Register(ctx context.Context, in *registry.NetworkServiceEndpoint) (*registry.NetworkServiceEndpoint, error) { - r.name = in.Name - - return next.NetworkServiceEndpointRegistryServer(ctx).Register(ctx, in) -} - -func (r *captureNameTestRegistryServer) Unregister(ctx context.Context, in *registry.NetworkServiceEndpoint) (*empty.Empty, error) { - r.name = in.Name - - return next.NetworkServiceEndpointRegistryServer(ctx).Unregister(ctx, in) -} -func (r *captureNameTestRegistryServer) Find(query *registry.NetworkServiceEndpointQuery, server registry.NetworkServiceEndpointRegistry_FindServer) error { - r.name = query.NetworkServiceEndpoint.GetName() - return next.NetworkServiceEndpointRegistryServer(server.Context()).Find(query, server) -} diff --git a/pkg/registry/common/localbypass/find_server.go b/pkg/registry/common/localbypass/find_server.go index cacbc90d3..1599ce7cf 100644 --- a/pkg/registry/common/localbypass/find_server.go +++ b/pkg/registry/common/localbypass/find_server.go @@ -30,9 +30,9 @@ func (s *localBypassNSEFindServer) Send(nseResp *registry.NetworkServiceEndpoint nseResp.NetworkServiceEndpoint.Url = u.String() } - if nseResp.NetworkServiceEndpoint.Url != s.nsmgrURL || nseResp.NetworkServiceEndpoint.ExpirationTime != nil && nseResp.Deleted { - return s.NetworkServiceEndpointRegistry_FindServer.Send(nseResp) + if nseResp.GetNetworkServiceEndpoint().GetUrl() == s.nsmgrURL && !nseResp.Deleted { + return nil } - return nil + return s.NetworkServiceEndpointRegistry_FindServer.Send(nseResp) } diff --git a/pkg/registry/common/recvfd/client_nolinux.go b/pkg/registry/common/recvfd/client_nolinux.go index 511b874b0..fd31e21d8 100644 --- a/pkg/registry/common/recvfd/client_nolinux.go +++ b/pkg/registry/common/recvfd/client_nolinux.go @@ -22,6 +22,7 @@ package recvfd import ( "github.com/networkservicemesh/api/pkg/api/registry" + "github.com/networkservicemesh/sdk/pkg/registry/common/null" ) diff --git a/pkg/registry/common/sendfd/server_nolinux.go b/pkg/registry/common/sendfd/server_nolinux.go index 0df5aae1a..621d68007 100644 --- a/pkg/registry/common/sendfd/server_nolinux.go +++ b/pkg/registry/common/sendfd/server_nolinux.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 Doc.ai and/or its affiliates. +// Copyright (c) 2020-2021 Doc.ai and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -20,6 +20,7 @@ package sendfd import ( "github.com/networkservicemesh/api/pkg/api/registry" + "github.com/networkservicemesh/sdk/pkg/registry/common/null" ) diff --git a/pkg/registry/common/setnetworkservicenames/nse_client.go b/pkg/registry/common/setnetworkservicenames/nse_client.go new file mode 100644 index 000000000..63f55b32f --- /dev/null +++ b/pkg/registry/common/setnetworkservicenames/nse_client.go @@ -0,0 +1,56 @@ +// Copyright (c) 2021 Cisco and/or its affiliates. +// +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package setnetworkservicenames provides registry.NetworkServiceEndpointRegistryClient that simply sets NetworkServiceNames for each incoming registration/find/unregistration. +package setnetworkservicenames + +import ( + "context" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/networkservicemesh/api/pkg/api/registry" + "google.golang.org/grpc" + + "github.com/networkservicemesh/sdk/pkg/registry/core/next" +) + +type setNetworkServiceNamesNSEClient struct { + networkServices []string +} + +func (n *setNetworkServiceNamesNSEClient) Register(ctx context.Context, in *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (*registry.NetworkServiceEndpoint, error) { + in.NetworkServiceNames = n.networkServices + return next.NetworkServiceEndpointRegistryClient(ctx).Register(ctx, in, opts...) +} + +func (n *setNetworkServiceNamesNSEClient) Find(ctx context.Context, in *registry.NetworkServiceEndpointQuery, opts ...grpc.CallOption) (registry.NetworkServiceEndpointRegistry_FindClient, error) { + in.GetNetworkServiceEndpoint().NetworkServiceNames = n.networkServices + return next.NetworkServiceEndpointRegistryClient(ctx).Find(ctx, in, opts...) +} + +func (n *setNetworkServiceNamesNSEClient) Unregister(ctx context.Context, in *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (*empty.Empty, error) { + in.NetworkServiceNames = n.networkServices + return next.NetworkServiceEndpointRegistryClient(ctx).Unregister(ctx, in, opts...) +} + +// NewNetworkServiceEndpointRegistryClient - returns registry.NetworkServiceEndpointRegistryClient that sets NetworkServiceNames for each incoming registration/find/unregistration. +func NewNetworkServiceEndpointRegistryClient(networkServices ...string) registry.NetworkServiceEndpointRegistryClient { + return &setNetworkServiceNamesNSEClient{ + networkServices: networkServices, + } +} diff --git a/pkg/tools/sandbox/builder.go b/pkg/tools/sandbox/builder.go index e6bb6a0bf..ccdacd0c3 100644 --- a/pkg/tools/sandbox/builder.go +++ b/pkg/tools/sandbox/builder.go @@ -37,6 +37,7 @@ import ( registryconnect "github.com/networkservicemesh/sdk/pkg/registry/common/connect" "github.com/networkservicemesh/sdk/pkg/registry/common/dnsresolve" "github.com/networkservicemesh/sdk/pkg/tools/grpcutils" + "github.com/networkservicemesh/sdk/pkg/tools/log" "github.com/networkservicemesh/sdk/pkg/tools/token" ) @@ -242,7 +243,7 @@ func (b *Builder) newRegistryProxy() *RegistryEntry { ) serve(ctx, b.t, entry.URL, entry.Register) - logrus.Errorf("%s: registry-proxy-dns serve on: %v", b.name, entry.URL) + log.FromContext(ctx).Infof("%s: registry-proxy-dns serve on: %v", b.name, entry.URL) }) return entry @@ -270,7 +271,7 @@ func (b *Builder) newRegistry() *RegistryEntry { ) serve(ctx, b.t, entry.URL, entry.Register) - logrus.Errorf("%s: registry serve on: %v", b.name, entry.URL) + log.FromContext(ctx).Infof("%s: registry serve on: %v", b.name, entry.URL) }) return entry diff --git a/pkg/tools/sandbox/node.go b/pkg/tools/sandbox/node.go index e2feb8d3e..dc8b8710a 100644 --- a/pkg/tools/sandbox/node.go +++ b/pkg/tools/sandbox/node.go @@ -23,7 +23,6 @@ import ( "github.com/networkservicemesh/api/pkg/api/networkservice" registryapi "github.com/networkservicemesh/api/pkg/api/registry" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/networkservicemesh/sdk/pkg/networkservice/chains/client" @@ -37,7 +36,9 @@ import ( "github.com/networkservicemesh/sdk/pkg/networkservice/common/roundrobin" registryclient "github.com/networkservicemesh/sdk/pkg/registry/chains/client" "github.com/networkservicemesh/sdk/pkg/registry/common/recvfd" + "github.com/networkservicemesh/sdk/pkg/registry/common/setnetworkservicenames" "github.com/networkservicemesh/sdk/pkg/registry/core/chain" + "github.com/networkservicemesh/sdk/pkg/tools/log" "github.com/networkservicemesh/sdk/pkg/tools/token" ) @@ -87,7 +88,7 @@ func (n *Node) NewNSMgr( entry.Nsmgr = supplyNSMgr(ctx, generatorFunc, options...) serve(ctx, n.t, entry.URL, entry.Register) - logrus.Infof("%s: NSMgr %s serve on %v", n.domain.Name, name, serveURL) + log.FromContext(ctx).Infof("%s: NSMgr %s serve on %v", n.domain.Name, name, serveURL) }) n.NSMgr = entry @@ -120,8 +121,11 @@ func (n *Node) NewForwarder( URL: serveURL, } nseClient := chain.NewNetworkServiceEndpointRegistryClient( - recvfd.NewNetworkServiceEndpointRegistryClient(), - registryclient.NewNetworkServiceEndpointRegistryClient(ctx, CloneURL(n.NSMgr.URL), registryclient.WithDialOptions(dialOptions...))) + registryclient.NewNetworkServiceEndpointRegistryClient(ctx, CloneURL(n.NSMgr.URL), + registryclient.WithNSEAdditionalFunctionality(recvfd.NewNetworkServiceEndpointRegistryClient()), + registryclient.WithDialOptions(dialOptions...), + ), + ) nsClient := registryclient.NewNetworkServiceRegistryClient(ctx, CloneURL(n.NSMgr.URL), registryclient.WithDialOptions(dialOptions...)) entry.restartableServer = newRestartableServer(ctx, n.t, entry.URL, func(ctx context.Context) { entry.Endpoint = endpoint.NewServer(ctx, generatorFunc, @@ -149,10 +153,16 @@ func (n *Node) NewForwarder( ) serve(ctx, n.t, entry.URL, entry.Endpoint.Register) - logrus.Infof("%s: forwarder %s serve on %v", n.domain.Name, nse.Name, serveURL) + log.FromContext(ctx).Infof("%s: forwarder %s serve on %v", n.domain.Name, nse.Name, serveURL) - entry.NetworkServiceEndpointRegistryClient = registryclient.NewNetworkServiceEndpointRegistryInterposeClient(ctx, CloneURL(n.NSMgr.URL), - registryclient.WithDialOptions(dialOptions...)) + entry.NetworkServiceEndpointRegistryClient = registryclient.NewNetworkServiceEndpointRegistryClient( + ctx, + CloneURL(n.NSMgr.URL), + registryclient.WithDialOptions(dialOptions...), + registryclient.WithNSEAdditionalFunctionality( + setnetworkservicenames.NewNetworkServiceEndpointRegistryClient("forwarder"), + ), + ) n.registerEndpoint(ctx, nse, nseClone, entry.NetworkServiceEndpointRegistryClient) }) @@ -194,7 +204,7 @@ func (n *Node) NewEndpoint( serve(ctx, n.t, entry.URL, entry.Endpoint.Register) - logrus.Infof("%s: NSE %s serve on %v", n.domain.Name, nse.Name, serveURL) + log.FromContext(ctx).Infof("%s: NSE %s serve on %v", n.domain.Name, nse.Name, serveURL) entry.NetworkServiceEndpointRegistryClient = registryclient.NewNetworkServiceEndpointRegistryClient(ctx, CloneURL(n.NSMgr.URL), registryclient.WithDialOptions(dialOptions...)) From 755f53b4a18956aad37250787154dd56f0dc8a5a Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Sun, 7 Nov 2021 12:34:29 +0300 Subject: [PATCH 4/6] rework discovercrossnse and add p2mp unit test Signed-off-by: denis-tingaikin --- pkg/networkservice/chains/nsmgr/server.go | 14 +- .../chains/nsmgr/single_test.go | 78 +++++++- pkg/networkservice/chains/nsmgr/unix_test.go | 10 +- .../discover/discovercrossnse/server.go | 66 ------- .../common/discover/match_selector.go | 57 +----- .../discovercrossnse/metadata.go | 25 +-- .../common/discovercrossnse/server.go | 172 ++++++++++++++++++ pkg/registry/common/heal/find_test.go | 3 +- .../setnetworkservicenames/nse_client.go | 56 ------ pkg/tools/matchutils/utils.go | 48 ++++- pkg/tools/sandbox/builder.go | 3 +- pkg/tools/sandbox/node.go | 5 +- pkg/tools/sandbox/utils.go | 3 +- 13 files changed, 322 insertions(+), 218 deletions(-) delete mode 100644 pkg/networkservice/common/discover/discovercrossnse/server.go rename pkg/networkservice/common/{discover => }/discovercrossnse/metadata.go (54%) create mode 100644 pkg/networkservice/common/discovercrossnse/server.go delete mode 100644 pkg/registry/common/setnetworkservicenames/nse_client.go diff --git a/pkg/networkservice/chains/nsmgr/server.go b/pkg/networkservice/chains/nsmgr/server.go index 32e34aa7d..b52b9321f 100644 --- a/pkg/networkservice/chains/nsmgr/server.go +++ b/pkg/networkservice/chains/nsmgr/server.go @@ -27,9 +27,6 @@ import ( "github.com/google/uuid" - "github.com/networkservicemesh/sdk/pkg/networkservice/common/discover/discovercrossnse" - "github.com/networkservicemesh/sdk/pkg/networkservice/common/roundrobin" - "google.golang.org/grpc" "github.com/networkservicemesh/api/pkg/api/networkservice" @@ -40,6 +37,7 @@ import ( "github.com/networkservicemesh/sdk/pkg/networkservice/common/authorize" "github.com/networkservicemesh/sdk/pkg/networkservice/common/clientinfo" "github.com/networkservicemesh/sdk/pkg/networkservice/common/connect" + "github.com/networkservicemesh/sdk/pkg/networkservice/common/discovercrossnse" "github.com/networkservicemesh/sdk/pkg/networkservice/common/excludedprefixes" "github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms/recvfd" "github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms/sendfd" @@ -54,7 +52,6 @@ import ( "github.com/networkservicemesh/sdk/pkg/registry/common/memory" registryrecvfd "github.com/networkservicemesh/sdk/pkg/registry/common/recvfd" registrysendfd "github.com/networkservicemesh/sdk/pkg/registry/common/sendfd" - "github.com/networkservicemesh/sdk/pkg/registry/common/setnetworkservicenames" registryserialize "github.com/networkservicemesh/sdk/pkg/registry/common/serialize" "github.com/networkservicemesh/sdk/pkg/registry/common/setlogoption" @@ -196,14 +193,7 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, options endpoint.WithAuthorizeServer(opts.authorizeServer), endpoint.WithAdditionalFunctionality( adapters.NewClientToServer(clientinfo.NewClient()), - discovercrossnse.NewServer( - registryadapter.NetworkServiceServerToClient(nsRegistry), - registrychain.NewNetworkServiceEndpointRegistryClient( - setnetworkservicenames.NewNetworkServiceEndpointRegistryClient("forwarder"), - registryadapter.NetworkServiceEndpointServerToClient(nseInMemoryRegistry), - ), - roundrobin.NewServer(), - ), + discovercrossnse.NewServer(registryadapter.NetworkServiceEndpointServerToClient(nseInMemoryRegistry)), excludedprefixes.NewServer(ctx), recvfd.NewServer(), // Receive any files passed connect.NewServer( diff --git a/pkg/networkservice/chains/nsmgr/single_test.go b/pkg/networkservice/chains/nsmgr/single_test.go index d9cbdbeae..f4dbbfdd4 100644 --- a/pkg/networkservice/chains/nsmgr/single_test.go +++ b/pkg/networkservice/chains/nsmgr/single_test.go @@ -119,7 +119,8 @@ func Test_ShouldCorrectlyAddForwardersWithSameNames(t *testing.T) { require.NoError(t, err) forwarderReg := ®istry.NetworkServiceEndpoint{ - Name: "forwarder", + Name: "forwarder", + NetworkServiceNames: []string{"forwarder"}, } // 1. Add forwarders @@ -253,3 +254,78 @@ func Test_ShouldParseNetworkServiceLabelsTemplate(t *testing.T) { _, err = nse.Unregister(ctx, nseReg) require.NoError(t, err) } + +func Test_UsecasePoint2MultiPoint(t *testing.T) { + t.Cleanup(func() { goleak.VerifyNone(t) }) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + domain := sandbox.NewBuilder(ctx, t). + SetNodesCount(1). + SetRegistryProxySupplier(nil). + SetNodeSetup(func(ctx context.Context, node *sandbox.Node, _ int) { + node.NewNSMgr(ctx, "nsmgr", nil, sandbox.GenerateTestToken, nsmgr.NewServer) + }). + SetRegistryExpiryDuration(sandbox.RegistryExpiryDuration). + Build() + + domain.Nodes[0].NewForwarder(ctx, ®istry.NetworkServiceEndpoint{ + Name: "p2mp forwarder", + NetworkServiceNames: []string{"forwarder"}, + NetworkServiceLabels: map[string]*registry.NetworkServiceLabels{ + "forwarder": { + Labels: map[string]string{ + "p2mp": "true", + }, + }, + }, + }, sandbox.GenerateTestToken) + + domain.Nodes[0].NewForwarder(ctx, ®istry.NetworkServiceEndpoint{ + Name: "p2p forwarder", + NetworkServiceNames: []string{"forwarder"}, + NetworkServiceLabels: map[string]*registry.NetworkServiceLabels{ + "forwarder": { + Labels: map[string]string{ + "p2p": "true", + }, + }, + }, + }, sandbox.GenerateTestToken) + + domain.Nodes[0].NewForwarder(ctx, ®istry.NetworkServiceEndpoint{ + Name: "special forwarder", + NetworkServiceNames: []string{"forwarder"}, + NetworkServiceLabels: map[string]*registry.NetworkServiceLabels{ + "forwarder": { + Labels: map[string]string{ + "special": "true", + }, + }, + }, + }, sandbox.GenerateTestToken) + + nsRegistryClient := domain.NewNSRegistryClient(ctx, sandbox.GenerateTestToken) + + nsReg, err := nsRegistryClient.Register(ctx, defaultRegistryService()) + require.NoError(t, err) + + nseReg := defaultRegistryEndpoint(nsReg.Name) + + domain.Nodes[0].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken) + + nsc := domain.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken) + + request := defaultRequest(nsReg.Name) + + request.GetConnection().Labels = map[string]string{ + "p2mp": "true", + } + + conn, err := nsc.Request(ctx, request.Clone()) + require.NoError(t, err) + require.NotNil(t, conn) + require.Equal(t, 4, len(conn.Path.PathSegments)) + require.Equal(t, "p2mp forwarder", conn.GetPath().GetPathSegments()[2].Name) +} diff --git a/pkg/networkservice/chains/nsmgr/unix_test.go b/pkg/networkservice/chains/nsmgr/unix_test.go index d05aedcef..8b4b0440e 100644 --- a/pkg/networkservice/chains/nsmgr/unix_test.go +++ b/pkg/networkservice/chains/nsmgr/unix_test.go @@ -39,7 +39,7 @@ import ( ) func Test_Local_NoURLUsecase(t *testing.T) { - t.Skip("TODO") + t.Skip("https://github.com/networkservicemesh/sdk/issues/1118") t.Cleanup(func() { goleak.VerifyNone(t) }) ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) @@ -85,7 +85,7 @@ func Test_Local_NoURLUsecase(t *testing.T) { } func Test_MultiForwarderSendfd(t *testing.T) { - t.Skip("TODO") + t.Skip("https://github.com/networkservicemesh/sdk/issues/1118") if runtime.GOOS != "linux" { t.Skip("sendfd works only on linux") } @@ -103,10 +103,12 @@ func Test_MultiForwarderSendfd(t *testing.T) { SetNodeSetup(func(ctx context.Context, node *sandbox.Node, _ int) { node.NewNSMgr(ctx, "nsmgr", nil, sandbox.GenerateTestToken, nsmgr.NewServer) node.NewForwarder(ctx, ®istry.NetworkServiceEndpoint{ - Name: "forwarder-1", + Name: "forwarder-1", + NetworkServiceNames: []string{"forwarder"}, }, sandbox.GenerateTestToken, errorServer, recvfd.NewServer()) node.NewForwarder(ctx, ®istry.NetworkServiceEndpoint{ - Name: "forwarder-2", + Name: "forwarder-2", + NetworkServiceNames: []string{"forwarder"}, }, sandbox.GenerateTestToken, errorServer, recvfd.NewServer()) }). Build() diff --git a/pkg/networkservice/common/discover/discovercrossnse/server.go b/pkg/networkservice/common/discover/discovercrossnse/server.go deleted file mode 100644 index 979158b48..000000000 --- a/pkg/networkservice/common/discover/discovercrossnse/server.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) 2021 Doc.ai and/or its affiliates. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package discovercrossnse - -import ( - "context" - - "github.com/golang/protobuf/ptypes/empty" - "github.com/networkservicemesh/api/pkg/api/networkservice" - "github.com/networkservicemesh/api/pkg/api/registry" - - "github.com/networkservicemesh/sdk/pkg/networkservice/common/discover" - "github.com/networkservicemesh/sdk/pkg/networkservice/core/chain" - "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" -) - -type switchNetworkServiceEndpointNameServer struct { - setFunc func(context.Context, string) - getFunc func(context.Context) string -} - -func (d *switchNetworkServiceEndpointNameServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) { - d.setFunc(ctx, request.GetConnection().GetNetworkServiceEndpointName()) - - request.GetConnection().NetworkServiceEndpointName = d.getFunc(ctx) - - return next.Server(ctx).Request(ctx, request) -} - -func (d *switchNetworkServiceEndpointNameServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) { - d.setFunc(ctx, conn.NetworkServiceEndpointName) - - conn.NetworkServiceEndpointName = d.getFunc(ctx) - - return next.Server(ctx).Close(ctx, conn) -} - -// NewServer TODO -func NewServer(nsClient registry.NetworkServiceRegistryClient, nseClient registry.NetworkServiceEndpointRegistryClient, loadBalance networkservice.NetworkServiceServer) networkservice.NetworkServiceServer { - return chain.NewNetworkServiceServer( - &switchNetworkServiceEndpointNameServer{ - setFunc: storeNetworkServiceEndpointName, - getFunc: loadCrossConnectEndpointName, - }, - discover.NewServer(nsClient, nseClient), - loadBalance, - &switchNetworkServiceEndpointNameServer{ - setFunc: storeCrossConnectEndpointName, - getFunc: loadNetworkServiceEndpointName, - }, - ) -} diff --git a/pkg/networkservice/common/discover/match_selector.go b/pkg/networkservice/common/discover/match_selector.go index c694a0ef9..f5312316f 100644 --- a/pkg/networkservice/common/discover/match_selector.go +++ b/pkg/networkservice/common/discover/match_selector.go @@ -19,32 +19,12 @@ package discover import ( - "bytes" - "text/template" - - "github.com/pkg/errors" - "github.com/networkservicemesh/api/pkg/api/registry" "github.com/networkservicemesh/sdk/pkg/tools/clock" + "github.com/networkservicemesh/sdk/pkg/tools/matchutils" ) -// isSubset checks if B is a subset of A. TODO: reconsider this as a part of "tools" -func isSubset(a, b, nsLabels map[string]string) bool { - if len(a) < len(b) { - return false - } - for k, v := range b { - if a[k] != v { - result := processLabels(v, nsLabels) - if a[k] != result { - return false - } - } - } - return true -} - func matchEndpoint(clockTime clock.Clock, nsLabels map[string]string, ns *registry.NetworkService, nses ...*registry.NetworkServiceEndpoint) []*registry.NetworkServiceEndpoint { var validNetworkServiceEndpoints []*registry.NetworkServiceEndpoint for _, nse := range nses { @@ -56,7 +36,7 @@ func matchEndpoint(clockTime clock.Clock, nsLabels map[string]string, ns *regist // Iterate through the matches for _, match := range ns.GetMatches() { // All match source selector labels should be present in the requested labels map - if !isSubset(nsLabels, match.GetSourceSelector(), nsLabels) { + if !matchutils.IsSubset(nsLabels, match.GetSourceSelector(), nsLabels) { continue } nseCandidates := make([]*registry.NetworkServiceEndpoint, 0) @@ -64,8 +44,11 @@ func matchEndpoint(clockTime clock.Clock, nsLabels map[string]string, ns *regist for _, destination := range match.GetRoutes() { // Each NSE should be matched against that destination for _, nse := range validNetworkServiceEndpoints { - // TODO looks unhealthy - if nse.GetNetworkServiceLabels()[ns.Name] == nil || isSubset(nse.GetNetworkServiceLabels()[ns.Name].Labels, destination.GetDestinationSelector(), nsLabels) { + var candidateNetworkServiceLabels = nse.GetNetworkServiceLabels()[ns.GetName()] + if candidateNetworkServiceLabels == nil { + continue + } + if matchutils.IsSubset(candidateNetworkServiceLabels.Labels, destination.GetDestinationSelector(), nsLabels) { nseCandidates = append(nseCandidates, nse) } } @@ -80,29 +63,3 @@ func matchEndpoint(clockTime clock.Clock, nsLabels map[string]string, ns *regist return validNetworkServiceEndpoints } - -// processLabels generates matches based on destination label selectors that specify templating. -func processLabels(str string, vars interface{}) string { - tmpl, err := template.New("tmpl").Parse(str) - - if err != nil { - return str - } - - rv, err := process(tmpl, vars) - if err != nil { - return str - } - - return rv -} - -func process(t *template.Template, vars interface{}) (string, error) { - var tmplBytes bytes.Buffer - - err := t.Execute(&tmplBytes, vars) - if err != nil { - return "", errors.Wrap(err, "error during execution of template") - } - return tmplBytes.String(), nil -} diff --git a/pkg/networkservice/common/discover/discovercrossnse/metadata.go b/pkg/networkservice/common/discovercrossnse/metadata.go similarity index 54% rename from pkg/networkservice/common/discover/discovercrossnse/metadata.go rename to pkg/networkservice/common/discovercrossnse/metadata.go index 3ab7993af..cf73ab7c4 100644 --- a/pkg/networkservice/common/discover/discovercrossnse/metadata.go +++ b/pkg/networkservice/common/discovercrossnse/metadata.go @@ -14,7 +14,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package discovercrossnse TODO package discovercrossnse import ( @@ -23,30 +22,16 @@ import ( "github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata" ) -type networkserviceNameKey struct{} +type selectedForworderKey struct{} -type crossConnectEndpointName struct{} - -func loadNetworkServiceEndpointName(ctx context.Context) string { - v, ok := metadata.Map(ctx, false).Load(networkserviceNameKey{}) - if !ok { - return "" - } - return v.(string) -} - -func storeNetworkServiceEndpointName(ctx context.Context, v string) { - metadata.Map(ctx, false).Store(networkserviceNameKey{}, v) -} - -func loadCrossConnectEndpointName(ctx context.Context) string { - v, ok := metadata.Map(ctx, false).Load(crossConnectEndpointName{}) +func loadForwarderName(ctx context.Context) string { + v, ok := metadata.Map(ctx, false).Load(selectedForworderKey{}) if !ok { return "" } return v.(string) } -func storeCrossConnectEndpointName(ctx context.Context, v string) { - metadata.Map(ctx, false).Store(crossConnectEndpointName{}, v) +func storeForwarderName(ctx context.Context, v string) { + metadata.Map(ctx, false).Store(selectedForworderKey{}, v) } diff --git a/pkg/networkservice/common/discovercrossnse/server.go b/pkg/networkservice/common/discovercrossnse/server.go new file mode 100644 index 000000000..782600d3c --- /dev/null +++ b/pkg/networkservice/common/discovercrossnse/server.go @@ -0,0 +1,172 @@ +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package discovercrossnse discovers forwarder from the registry. +package discovercrossnse + +import ( + "context" + "net/url" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/api/pkg/api/registry" + "github.com/pkg/errors" + + "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" + "github.com/networkservicemesh/sdk/pkg/tools/clienturlctx" + "github.com/networkservicemesh/sdk/pkg/tools/log" + "github.com/networkservicemesh/sdk/pkg/tools/matchutils" +) + +type discoverForwarderServer struct { + nseClient registry.NetworkServiceEndpointRegistryClient +} + +// NewServer creates new instance of discovercrossnse networkservice.NetworkServiceServer. +// Requires not nil nseClient. +func NewServer(nseClient registry.NetworkServiceEndpointRegistryClient) networkservice.NetworkServiceServer { + if nseClient == nil { + panic("mseClient can not be nil") + } + return &discoverForwarderServer{ + nseClient: nseClient, + } +} + +func (d *discoverForwarderServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) { + var forwarderName = loadForwarderName(ctx) + var logger = log.FromContext(ctx).WithField("discoverForwarderServer", "request") + + if forwarderName == "" { + stream, err := d.nseClient.Find(ctx, ®istry.NetworkServiceEndpointQuery{ + NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ + NetworkServiceNames: []string{ + "forwarder", + }, + }, + }) + + if err != nil { + logger.Errorf("can not open registry nse stream by networkservice. Error: %v", err.Error()) + return nil, errors.WithStack(err) + } + + nses := registry.ReadNetworkServiceEndpointList(stream) + + if len(nses) == 0 { + return nil, errors.New("forwarders not found") + } + + nses = matchForwarders(request.Connection.GetLabels(), nses) + + if len(nses) == 0 { + return nil, errors.New("no candidates found") + } + + u, err := url.Parse(nses[0].Url) // TODO: Should we consider about load balancing? + + if err != nil { + logger.Errorf("can not parse forwarder url %v", err.Error()) + return nil, errors.WithStack(err) + } + storeForwarderName(ctx, nses[0].Name) + ctx = clienturlctx.WithClientURL(ctx, u) + } else { + stream, err := d.nseClient.Find(ctx, ®istry.NetworkServiceEndpointQuery{ + NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ + Name: forwarderName, + }, + }) + + if err != nil { + logger.Errorf("can not open registry nse stream by forwarder name. Error: %v", err.Error()) + return nil, errors.WithStack(err) + } + + nses := registry.ReadNetworkServiceEndpointList(stream) + + if len(nses) == 0 { + storeForwarderName(ctx, "") + return nil, errors.New("forwarder not found") + } + + u, err := url.Parse(nses[0].Url) + + if err != nil { + logger.Errorf("can not parse forwarder url %v", err.Error()) + return nil, errors.WithStack(err) + } + + ctx = clienturlctx.WithClientURL(ctx, u) + } + + return next.Server(ctx).Request(ctx, request) +} + +func (d *discoverForwarderServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) { + var forwarderName = loadForwarderName(ctx) + var logger = log.FromContext(ctx).WithField("discoverForwarderServer", "request") + + if forwarderName == "" { + return nil, errors.New("forwarder is not selected") + } + + stream, err := d.nseClient.Find(ctx, ®istry.NetworkServiceEndpointQuery{ + NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ + Name: forwarderName, + }, + }) + + if err != nil { + logger.Errorf("can not open registry nse stream by forwarder name. Error: %v", err.Error()) + return nil, errors.WithStack(err) + } + + nses := registry.ReadNetworkServiceEndpointList(stream) + + if len(nses) == 0 { + return nil, errors.New("forwarder not found") + } + + u, err := url.Parse(nses[0].Url) + + if err != nil { + logger.Errorf("can not parse forwarder url %v", err.Error()) + return nil, errors.WithStack(err) + } + + ctx = clienturlctx.WithClientURL(ctx, u) + return next.Server(ctx).Close(ctx, conn) +} + +func matchForwarders(clientLabels map[string]string, canidates []*registry.NetworkServiceEndpoint) []*registry.NetworkServiceEndpoint { + var result []*registry.NetworkServiceEndpoint + + for _, candidate := range canidates { + var forwawrderLabels map[string]string + + if candidate.NetworkServiceLabels["forwarder"] != nil { + forwawrderLabels = candidate.NetworkServiceLabels["forwarder"].Labels + } + + if matchutils.IsSubset(clientLabels, forwawrderLabels, clientLabels) { + result = append(result, candidate) + } + } + + return result +} diff --git a/pkg/registry/common/heal/find_test.go b/pkg/registry/common/heal/find_test.go index 424759a85..d2b3ac2ad 100644 --- a/pkg/registry/common/heal/find_test.go +++ b/pkg/registry/common/heal/find_test.go @@ -46,7 +46,8 @@ func TestHealClient_FindTest(t *testing.T) { SetNodeSetup(func(ctx context.Context, node *sandbox.Node, nodeNum int) { node.NewNSMgr(nsmgrCtx, sandbox.UniqueName("nsmgr"), nil, sandbox.GenerateTestToken, nsmgr.NewServer) node.NewForwarder(ctx, ®istry.NetworkServiceEndpoint{ - Name: sandbox.UniqueName("forwarder"), + Name: sandbox.UniqueName("forwarder"), + NetworkServiceNames: []string{"forwarder"}, }, sandbox.GenerateTestToken) }). Build() diff --git a/pkg/registry/common/setnetworkservicenames/nse_client.go b/pkg/registry/common/setnetworkservicenames/nse_client.go deleted file mode 100644 index 63f55b32f..000000000 --- a/pkg/registry/common/setnetworkservicenames/nse_client.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) 2021 Cisco and/or its affiliates. -// -// Copyright (c) 2021 Doc.ai and/or its affiliates. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package setnetworkservicenames provides registry.NetworkServiceEndpointRegistryClient that simply sets NetworkServiceNames for each incoming registration/find/unregistration. -package setnetworkservicenames - -import ( - "context" - - "github.com/golang/protobuf/ptypes/empty" - "github.com/networkservicemesh/api/pkg/api/registry" - "google.golang.org/grpc" - - "github.com/networkservicemesh/sdk/pkg/registry/core/next" -) - -type setNetworkServiceNamesNSEClient struct { - networkServices []string -} - -func (n *setNetworkServiceNamesNSEClient) Register(ctx context.Context, in *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (*registry.NetworkServiceEndpoint, error) { - in.NetworkServiceNames = n.networkServices - return next.NetworkServiceEndpointRegistryClient(ctx).Register(ctx, in, opts...) -} - -func (n *setNetworkServiceNamesNSEClient) Find(ctx context.Context, in *registry.NetworkServiceEndpointQuery, opts ...grpc.CallOption) (registry.NetworkServiceEndpointRegistry_FindClient, error) { - in.GetNetworkServiceEndpoint().NetworkServiceNames = n.networkServices - return next.NetworkServiceEndpointRegistryClient(ctx).Find(ctx, in, opts...) -} - -func (n *setNetworkServiceNamesNSEClient) Unregister(ctx context.Context, in *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (*empty.Empty, error) { - in.NetworkServiceNames = n.networkServices - return next.NetworkServiceEndpointRegistryClient(ctx).Unregister(ctx, in, opts...) -} - -// NewNetworkServiceEndpointRegistryClient - returns registry.NetworkServiceEndpointRegistryClient that sets NetworkServiceNames for each incoming registration/find/unregistration. -func NewNetworkServiceEndpointRegistryClient(networkServices ...string) registry.NetworkServiceEndpointRegistryClient { - return &setNetworkServiceNamesNSEClient{ - networkServices: networkServices, - } -} diff --git a/pkg/tools/matchutils/utils.go b/pkg/tools/matchutils/utils.go index ba4bc767e..b690aa382 100644 --- a/pkg/tools/matchutils/utils.go +++ b/pkg/tools/matchutils/utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 Doc.ai and/or its affiliates. +// Copyright (c) 2020-2021 Doc.ai and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -18,9 +18,12 @@ package matchutils import ( + "bytes" "strings" + "text/template" "github.com/google/go-cmp/cmp" + "github.com/pkg/errors" "google.golang.org/protobuf/proto" "github.com/networkservicemesh/api/pkg/api/registry" @@ -42,6 +45,49 @@ func MatchNetworkServiceEndpoints(left, right *registry.NetworkServiceEndpoint) (left.Url == "" || strings.Contains(right.Url, left.Url)) } +// IsSubset checks if B is a subset of A. +// Tries to process values for each B value. +func IsSubset(a, b, values map[string]string) bool { + if len(a) < len(b) { + return false + } + for k, v := range b { + if a[k] != v { + result := processLabels(v, values) + if a[k] != result { + return false + } + } + } + return true +} + +// processLabels generates matches based on destination label selectors that specify templating. +func processLabels(str string, vars interface{}) string { + tmpl, err := template.New("tmpl").Parse(str) + + if err != nil { + return str + } + + rv, err := process(tmpl, vars) + if err != nil { + return str + } + + return rv +} + +func process(t *template.Template, vars interface{}) (string, error) { + var tmplBytes bytes.Buffer + + err := t.Execute(&tmplBytes, vars) + if err != nil { + return "", errors.Wrap(err, "error during execution of template") + } + return tmplBytes.String(), nil +} + func labelsContains(where, what map[string]*registry.NetworkServiceLabels) bool { for lService, lLabels := range what { rService, ok := where[lService] diff --git a/pkg/tools/sandbox/builder.go b/pkg/tools/sandbox/builder.go index ccdacd0c3..21295c75e 100644 --- a/pkg/tools/sandbox/builder.go +++ b/pkg/tools/sandbox/builder.go @@ -27,7 +27,6 @@ import ( "testing" "time" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/networkservicemesh/sdk/pkg/networkservice/chains/nsmgr" @@ -301,7 +300,7 @@ func (b *Builder) newNSMgrProxy() *NSMgrEntry { ) serve(ctx, b.t, entry.URL, entry.Register) - logrus.Errorf("%s: NSMgr proxy %s serve on: %v", b.name, entry.Name, entry.URL) + log.FromContext(ctx).Infof("%s: NSMgr proxy %s serve on: %v", b.name, entry.Name, entry.URL) }) return entry diff --git a/pkg/tools/sandbox/node.go b/pkg/tools/sandbox/node.go index dc8b8710a..485bbabfb 100644 --- a/pkg/tools/sandbox/node.go +++ b/pkg/tools/sandbox/node.go @@ -36,7 +36,6 @@ import ( "github.com/networkservicemesh/sdk/pkg/networkservice/common/roundrobin" registryclient "github.com/networkservicemesh/sdk/pkg/registry/chains/client" "github.com/networkservicemesh/sdk/pkg/registry/common/recvfd" - "github.com/networkservicemesh/sdk/pkg/registry/common/setnetworkservicenames" "github.com/networkservicemesh/sdk/pkg/registry/core/chain" "github.com/networkservicemesh/sdk/pkg/tools/log" "github.com/networkservicemesh/sdk/pkg/tools/token" @@ -159,9 +158,7 @@ func (n *Node) NewForwarder( ctx, CloneURL(n.NSMgr.URL), registryclient.WithDialOptions(dialOptions...), - registryclient.WithNSEAdditionalFunctionality( - setnetworkservicenames.NewNetworkServiceEndpointRegistryClient("forwarder"), - ), + registryclient.WithNSEAdditionalFunctionality(), ) n.registerEndpoint(ctx, nse, nseClone, entry.NetworkServiceEndpointRegistryClient) diff --git a/pkg/tools/sandbox/utils.go b/pkg/tools/sandbox/utils.go index 27447fbdf..4df8ba19a 100644 --- a/pkg/tools/sandbox/utils.go +++ b/pkg/tools/sandbox/utils.go @@ -94,6 +94,7 @@ func SetupDefaultNode(ctx context.Context, node *Node, supplyNSMgr SupplyNSMgrFu node.NewNSMgr(ctx, UniqueName("nsmgr"), nil, GenerateTestToken, supplyNSMgr) node.NewForwarder(ctx, ®istryapi.NetworkServiceEndpoint{ - Name: UniqueName("forwarder"), + Name: UniqueName("forwarder"), + NetworkServiceNames: []string{"forwarder"}, }, GenerateTestToken) } From 1a0be07e86e600742b488cdc1531b329d8025d00 Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Sun, 7 Nov 2021 22:22:07 +0300 Subject: [PATCH 5/6] apply review comments fixes Signed-off-by: denis-tingaikin --- pkg/networkservice/chains/nsmgr/server.go | 35 +++++++++++++------ .../metadata.go | 2 +- .../common/discoverforwarder/option.go | 28 +++++++++++++++ .../server.go | 24 ++++++++----- 4 files changed, 69 insertions(+), 20 deletions(-) rename pkg/networkservice/common/{discovercrossnse => discoverforwarder}/metadata.go (97%) create mode 100644 pkg/networkservice/common/discoverforwarder/option.go rename pkg/networkservice/common/{discovercrossnse => discoverforwarder}/server.go (89%) diff --git a/pkg/networkservice/chains/nsmgr/server.go b/pkg/networkservice/chains/nsmgr/server.go index b52b9321f..b352bc6a2 100644 --- a/pkg/networkservice/chains/nsmgr/server.go +++ b/pkg/networkservice/chains/nsmgr/server.go @@ -37,7 +37,7 @@ import ( "github.com/networkservicemesh/sdk/pkg/networkservice/common/authorize" "github.com/networkservicemesh/sdk/pkg/networkservice/common/clientinfo" "github.com/networkservicemesh/sdk/pkg/networkservice/common/connect" - "github.com/networkservicemesh/sdk/pkg/networkservice/common/discovercrossnse" + "github.com/networkservicemesh/sdk/pkg/networkservice/common/discoverforwarder" "github.com/networkservicemesh/sdk/pkg/networkservice/common/excludedprefixes" "github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms/recvfd" "github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms/sendfd" @@ -74,13 +74,14 @@ type nsmgrServer struct { } type serverOptions struct { - authorizeServer networkservice.NetworkServiceServer - dialOptions []grpc.DialOption - dialTimeout time.Duration - regURL *url.URL - regDialOptions []grpc.DialOption - name string - url string + authorizeServer networkservice.NetworkServiceServer + dialOptions []grpc.DialOption + dialTimeout time.Duration + regURL *url.URL + regDialOptions []grpc.DialOption + name string + url string + forwarderServiceName string } // Option modifies server option value @@ -93,6 +94,14 @@ func WithDialOptions(dialOptions ...grpc.DialOption) Option { } } +// WithForwarderServiceName overrides default forwarder service name +// By default "forwarder" +func WithForwarderServiceName(forwarderServiceName string) Option { + return func(o *serverOptions) { + o.forwarderServiceName = forwarderServiceName + } +} + // WithDialTimeout sets dial timeout for the client func WithDialTimeout(dialTimeout time.Duration) Option { return func(o *serverOptions) { @@ -140,8 +149,9 @@ var _ Nsmgr = (*nsmgrServer)(nil) // options - a set of Nsmgr options. func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, options ...Option) Nsmgr { opts := &serverOptions{ - authorizeServer: authorize.NewServer(authorize.Any()), - name: "nsmgr-" + uuid.New().String(), + authorizeServer: authorize.NewServer(authorize.Any()), + name: "nsmgr-" + uuid.New().String(), + forwarderServiceName: "forwarder", } for _, opt := range options { opt(opts) @@ -193,7 +203,10 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, options endpoint.WithAuthorizeServer(opts.authorizeServer), endpoint.WithAdditionalFunctionality( adapters.NewClientToServer(clientinfo.NewClient()), - discovercrossnse.NewServer(registryadapter.NetworkServiceEndpointServerToClient(nseInMemoryRegistry)), + discoverforwarder.NewServer( + registryadapter.NetworkServiceEndpointServerToClient(nseInMemoryRegistry), + discoverforwarder.WithForwarderServiceName(opts.forwarderServiceName), + ), excludedprefixes.NewServer(ctx), recvfd.NewServer(), // Receive any files passed connect.NewServer( diff --git a/pkg/networkservice/common/discovercrossnse/metadata.go b/pkg/networkservice/common/discoverforwarder/metadata.go similarity index 97% rename from pkg/networkservice/common/discovercrossnse/metadata.go rename to pkg/networkservice/common/discoverforwarder/metadata.go index cf73ab7c4..b675fd369 100644 --- a/pkg/networkservice/common/discovercrossnse/metadata.go +++ b/pkg/networkservice/common/discoverforwarder/metadata.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package discovercrossnse +package discoverforwarder import ( "context" diff --git a/pkg/networkservice/common/discoverforwarder/option.go b/pkg/networkservice/common/discoverforwarder/option.go new file mode 100644 index 000000000..ec8b8c438 --- /dev/null +++ b/pkg/networkservice/common/discoverforwarder/option.go @@ -0,0 +1,28 @@ +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package discoverforwarder + +// Option changes default settings for the discoverForwarderServer +type Option func(*discoverForwarderServer) + +// WithForwarderServiceName overrides forwarder service name +// By default "forwarder" +func WithForwarderServiceName(serviceName string) Option { + return func(d *discoverForwarderServer) { + d.forwarderServiceName = serviceName + } +} diff --git a/pkg/networkservice/common/discovercrossnse/server.go b/pkg/networkservice/common/discoverforwarder/server.go similarity index 89% rename from pkg/networkservice/common/discovercrossnse/server.go rename to pkg/networkservice/common/discoverforwarder/server.go index 782600d3c..e256440c0 100644 --- a/pkg/networkservice/common/discovercrossnse/server.go +++ b/pkg/networkservice/common/discoverforwarder/server.go @@ -14,8 +14,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package discovercrossnse discovers forwarder from the registry. -package discovercrossnse +// Package discoverforwarder discovers forwarder from the registry. +package discoverforwarder import ( "context" @@ -33,18 +33,26 @@ import ( ) type discoverForwarderServer struct { - nseClient registry.NetworkServiceEndpointRegistryClient + nseClient registry.NetworkServiceEndpointRegistryClient + forwarderServiceName string } -// NewServer creates new instance of discovercrossnse networkservice.NetworkServiceServer. +// NewServer creates new instance of discoverforwarder networkservice.NetworkServiceServer. // Requires not nil nseClient. -func NewServer(nseClient registry.NetworkServiceEndpointRegistryClient) networkservice.NetworkServiceServer { +func NewServer(nseClient registry.NetworkServiceEndpointRegistryClient, opts ...Option) networkservice.NetworkServiceServer { if nseClient == nil { panic("mseClient can not be nil") } - return &discoverForwarderServer{ - nseClient: nseClient, + var result = &discoverForwarderServer{ + nseClient: nseClient, + forwarderServiceName: "forwarder", } + + for _, opt := range opts { + opt(result) + } + + return result } func (d *discoverForwarderServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) { @@ -55,7 +63,7 @@ func (d *discoverForwarderServer) Request(ctx context.Context, request *networks stream, err := d.nseClient.Find(ctx, ®istry.NetworkServiceEndpointQuery{ NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ NetworkServiceNames: []string{ - "forwarder", + d.forwarderServiceName, }, }, }) From 31eadf8e109d88be16f3cbc53fab90c87a1a8353 Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Sun, 7 Nov 2021 22:50:23 +0300 Subject: [PATCH 6/6] apply self code review Signed-off-by: denis-tingaikin --- .../common/discoverforwarder/server.go | 76 ++++++++++--------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/pkg/networkservice/common/discoverforwarder/server.go b/pkg/networkservice/common/discoverforwarder/server.go index e256440c0..1694bd5ae 100644 --- a/pkg/networkservice/common/discoverforwarder/server.go +++ b/pkg/networkservice/common/discoverforwarder/server.go @@ -73,56 +73,58 @@ func (d *discoverForwarderServer) Request(ctx context.Context, request *networks return nil, errors.WithStack(err) } - nses := registry.ReadNetworkServiceEndpointList(stream) + nses := d.matchForwarders(request.Connection.GetLabels(), registry.ReadNetworkServiceEndpointList(stream)) if len(nses) == 0 { - return nil, errors.New("forwarders not found") + return nil, errors.New("no candidates found") } - nses = matchForwarders(request.Connection.GetLabels(), nses) + // TODO: Should we consider about load balancing? + // https://github.com/networkservicemesh/sdk/issues/790 + for _, candidate := range nses { + u, err := url.Parse(candidate.Url) - if len(nses) == 0 { - return nil, errors.New("no candidates found") - } + if err != nil { + logger.Errorf("can not parse forwarder=%v url=%v error=%v", candidate.Name, candidate.Url, err.Error()) + continue + } - u, err := url.Parse(nses[0].Url) // TODO: Should we consider about load balancing? + resp, err := next.Server(ctx).Request(clienturlctx.WithClientURL(ctx, u), request.Clone()) - if err != nil { - logger.Errorf("can not parse forwarder url %v", err.Error()) - return nil, errors.WithStack(err) + if err == nil { + storeForwarderName(ctx, nses[0].Name) + return resp, nil + } } - storeForwarderName(ctx, nses[0].Name) - ctx = clienturlctx.WithClientURL(ctx, u) - } else { - stream, err := d.nseClient.Find(ctx, ®istry.NetworkServiceEndpointQuery{ - NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ - Name: forwarderName, - }, - }) - if err != nil { - logger.Errorf("can not open registry nse stream by forwarder name. Error: %v", err.Error()) - return nil, errors.WithStack(err) - } + return nil, errors.New("all forwarders failed") + } + stream, err := d.nseClient.Find(ctx, ®istry.NetworkServiceEndpointQuery{ + NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ + Name: forwarderName, + }, + }) - nses := registry.ReadNetworkServiceEndpointList(stream) + if err != nil { + logger.Errorf("can not open registry nse stream by forwarder name. Error: %v", err.Error()) + return nil, errors.WithStack(err) + } - if len(nses) == 0 { - storeForwarderName(ctx, "") - return nil, errors.New("forwarder not found") - } + nses := registry.ReadNetworkServiceEndpointList(stream) - u, err := url.Parse(nses[0].Url) + if len(nses) == 0 { + storeForwarderName(ctx, "") + return nil, errors.New("forwarder not found") + } - if err != nil { - logger.Errorf("can not parse forwarder url %v", err.Error()) - return nil, errors.WithStack(err) - } + u, err := url.Parse(nses[0].Url) - ctx = clienturlctx.WithClientURL(ctx, u) + if err != nil { + logger.Errorf("can not parse forwarder=%v url=%v error=%v", nses[0].Name, u, err.Error()) + return nil, errors.WithStack(err) } - return next.Server(ctx).Request(ctx, request) + return next.Server(ctx).Request(clienturlctx.WithClientURL(ctx, u), request) } func (d *discoverForwarderServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) { @@ -161,14 +163,14 @@ func (d *discoverForwarderServer) Close(ctx context.Context, conn *networkservic return next.Server(ctx).Close(ctx, conn) } -func matchForwarders(clientLabels map[string]string, canidates []*registry.NetworkServiceEndpoint) []*registry.NetworkServiceEndpoint { +func (d *discoverForwarderServer) matchForwarders(clientLabels map[string]string, canidates []*registry.NetworkServiceEndpoint) []*registry.NetworkServiceEndpoint { var result []*registry.NetworkServiceEndpoint for _, candidate := range canidates { var forwawrderLabels map[string]string - if candidate.NetworkServiceLabels["forwarder"] != nil { - forwawrderLabels = candidate.NetworkServiceLabels["forwarder"].Labels + if candidate.NetworkServiceLabels[d.forwarderServiceName] != nil { + forwawrderLabels = candidate.NetworkServiceLabels[d.forwarderServiceName].Labels } if matchutils.IsSubset(clientLabels, forwawrderLabels, clientLabels) {