[CPPS][NoNetworkUVM][PoC] Create UVM without networking setup, setup …

…network during workload container creation

cmd/containerd-shim-runhcs-v1/pod.go

@@ -117,6 +117,10 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques
 		case *uvm.OptionsLCOW:
 			lopts = (opts).(*uvm.OptionsLCOW)
 			lopts.BundleDirectory = req.Bundle
+			log.G(ctx).WithField("tid", req.ID).Debug("Before calling uvm.createLCOW(), setting preprovisioneduvm to true for testing.")
+			// Hack: Set this to true to test preprovisioned uvm
+			lopts.PreprovisionedUvm = true
+			log.G(ctx).WithField("tid", req.ID).Debug("Calling uvm.createLCOW()")
 			parent, err = uvm.CreateLCOW(ctx, lopts)
 			if err != nil {
 				return nil, err
@@ -191,8 +195,15 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques
 			cid = id
 		}
 		caAddr := fmt.Sprintf(uvm.ComputeAgentAddrFmt, cid)
-		if err := parent.CreateAndAssignNetworkSetup(ctx, caAddr, cid); err != nil {
-			return nil, err
+
+		// Skip network setup if this is a preprovisioned uvm.
+		if lopts.PreprovisionedUvm == false {
+			log.G(ctx).WithField("tid", req.ID).Debug("Calling parent.CreateAndAssignNetworkSetup()")
+			if err := parent.CreateAndAssignNetworkSetup(ctx, caAddr, cid); err != nil {
+				return nil, err
+			}
+		} else {
+			log.G(ctx).Debug("Skip calling parent.CreateAndAssignNetworkSetup() if preprovisioned Uvm")
 		}
 	}
 
@@ -302,6 +313,7 @@ func (p *pod) ID() string {
 }
 
 func (p *pod) CreateTask(ctx context.Context, req *task.CreateTaskRequest, s *specs.Spec) (_ shimTask, err error) {
+	log.G(ctx).Debug("Inside pod.go::CreateTask -> Printing p.id", p.id)
 	if req.ID == p.id {
 		return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "task with id: '%s' already exists", req.ID)
 	}
@@ -356,6 +368,24 @@ func (p *pod) CreateTask(ctx context.Context, req *task.CreateTaskRequest, s *sp
 			sid)
 	}
 
+	// For preprovisioned UVMs, we skip creating the container during the pre-provision stage.
+	// Subsequently, we try to create and assign network for the UVM during the creation of
+	// (non-sandbox) container.
+	var parent *uvm.UtilityVM
+	parent = p.host
+	if parent != nil && ct != oci.KubernetesContainerTypeSandbox {
+		cid := req.ID
+		if id, ok := s.Annotations[annotations.NcproxyContainerID]; ok {
+			cid = id
+		}
+		caAddr := fmt.Sprintf(uvm.ComputeAgentAddrFmt, cid)
+
+		log.G(ctx).Debug("pod.go:CreateTask -> Calling parent.CreateAndAssignNetworkSetup()")
+		if err := parent.CreateAndAssignNetworkSetup(ctx, caAddr, cid); err != nil {
+			return nil, err
+		}
+	}
+
 	st, err := newHcsTask(ctx, p.events, p.host, false, req, s)
 	if err != nil {
 		return nil, err

cmd/containerd-shim-runhcs-v1/service_internal.go

@@ -11,6 +11,7 @@ import (
 
 	runhcsopts "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
 	"github.com/Microsoft/hcsshim/internal/extendedtask"
+	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/oci"
 	"github.com/Microsoft/hcsshim/internal/shimdiag"
 	containerd_v1_types "github.com/containerd/containerd/api/types/task"
@@ -70,6 +71,7 @@ func (s *service) stateInternal(ctx context.Context, req *task.StateRequest) (*t
 }
 
 func (s *service) createInternal(ctx context.Context, req *task.CreateTaskRequest) (*task.CreateTaskResponse, error) {
+	log.G(ctx).Debug("Inside service_internal::createInternal")
 	setupDebuggerEvent()
 
 	shimOpts := &runhcsopts.Options{}
@@ -159,6 +161,7 @@ func (s *service) createInternal(ctx context.Context, req *task.CreateTaskReques
 	if s.isSandbox {
 		pod, err := s.getPod()
 		if err == nil {
+			log.G(ctx).Debug("Inside service_internal::createInternal -> Pod found, Printing Pod ID", pod.ID())
 			// The POD sandbox was previously created. Unlock and forward to the POD
 			s.cl.Unlock()
 			t, err := pod.CreateTask(ctx, req, &spec)
@@ -169,6 +172,7 @@ func (s *service) createInternal(ctx context.Context, req *task.CreateTaskReques
 			resp.Pid = uint32(e.Pid())
 			return resp, nil
 		}
+		log.G(ctx).Debug("Inside service_internal::createInternal -> Pod not found, creating pod")
 		pod, err = createPod(ctx, s.events, req, &spec)
 		if err != nil {
 			s.cl.Unlock()
@@ -179,6 +183,7 @@ func (s *service) createInternal(ctx context.Context, req *task.CreateTaskReques
 		resp.Pid = uint32(e.Pid())
 		s.taskOrPod.Store(pod)
 	} else {
+		log.G(ctx).Debug("Inside service_internal::createInternal -> Not sandbox, creating standalone task")
 		t, err := newHcsStandaloneTask(ctx, s.events, req, &spec)
 		if err != nil {
 			s.cl.Unlock()

cmd/containerd-shim-runhcs-v1/task_hcs.go

@@ -213,7 +213,9 @@ func newHcsTask(
 		return nil, err
 	}
 
+	log.G(ctx).Debug("Inside task_hcs/newHCSTask")
 	container, resources, err := createContainer(ctx, req.ID, owner, netNS, s, parent, shimOpts, req.Rootfs)
+
 	if err != nil {
 		return nil, err
 	}

cmd/ncproxy/ncproxy.go

@@ -63,6 +63,7 @@ func newGRPCService(agentCache *computeAgentCache, ncproxyNetworking *ncproxysto
 var _ ncproxygrpc.NetworkConfigProxyServer = &grpcService{}
 
 func (s *grpcService) AddNIC(ctx context.Context, req *ncproxygrpc.AddNICRequest) (_ *ncproxygrpc.AddNICResponse, err error) {
+	log.G(ctx).Debug("Inside ncproxy/AddNIC")
 	ctx, span := oc.StartSpan(ctx, "AddNIC")
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
@@ -307,6 +308,7 @@ func (s *grpcService) DeleteNIC(ctx context.Context, req *ncproxygrpc.DeleteNICR
 }
 
 func (s *grpcService) CreateNetwork(ctx context.Context, req *ncproxygrpc.CreateNetworkRequest) (_ *ncproxygrpc.CreateNetworkResponse, err error) {
+	log.G(ctx).Debug("Inside CreateNetwork")
 	ctx, span := oc.StartSpan(ctx, "CreateNetwork")
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()

internal/gcs/container.go

@@ -37,6 +37,7 @@ var _ cow.Container = &Container{}
 // CreateContainer creates a container using ID `cid` and `cfg`. The request
 // will likely not be cancellable even if `ctx` becomes done.
 func (gc *GuestConnection) CreateContainer(ctx context.Context, cid string, config interface{}) (_ *Container, err error) {
+	log.G(ctx).Debug("Inside GuestConnection CreateContainer")
 	ctx, span := oc.StartSpan(ctx, "gcs::GuestConnection::CreateContainer", oc.WithClientSpanKind)
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()

internal/hcsoci/create.go

@@ -85,6 +85,7 @@ func validateContainerConfig(ctx context.Context, coi *createOptionsInternal) er
 }
 
 func initializeCreateOptions(ctx context.Context, createOptions *CreateOptions) (*createOptionsInternal, error) {
+	log.G(ctx).Debug("Inside initializeCreateOptions")
 	coi := &createOptionsInternal{
 		CreateOptions: createOptions,
 		actualID:      createOptions.ID,
@@ -125,9 +126,12 @@ func initializeCreateOptions(ctx context.Context, createOptions *CreateOptions)
 // configureSandboxNetwork creates a new network namespace for the pod (sandbox)
 // if required and then adds that namespace to the pod.
 func configureSandboxNetwork(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, ct oci.KubernetesContainerType) error {
+	log.G(ctx).Debug("Inside configureSandboxNetwork")
 	if coi.NetworkNamespace != "" {
+		log.G(ctx).Debug("coi.NetworkNamespace is not empty")
 		r.SetNetNS(coi.NetworkNamespace)
 	} else {
+		log.G(ctx).Debug("Invoking createNetworkNamespace")
 		err := createNetworkNamespace(ctx, coi, r)
 		if err != nil {
 			return err
@@ -139,25 +143,28 @@ func configureSandboxNetwork(ctx context.Context, coi *createOptionsInternal, r
 		// Only add the network namespace to a standalone or sandbox
 		// container but not a workload container in a sandbox that inherits
 		// the namespace.
-		if ct == oci.KubernetesContainerTypeNone || ct == oci.KubernetesContainerTypeSandbox {
-			if err := coi.HostingSystem.ConfigureNetworking(ctx, coi.actualNetworkNamespace); err != nil {
-				// No network setup type was specified for this UVM. Create and assign one here unless
-				// we received a different error.
-				if err == uvm.ErrNoNetworkSetup {
-					if err := coi.HostingSystem.CreateAndAssignNetworkSetup(ctx, "", ""); err != nil {
-						return err
-					}
-					if err := coi.HostingSystem.ConfigureNetworking(ctx, coi.actualNetworkNamespace); err != nil {
-						return err
-					}
-				} else {
+		// However in case of preprovisioned UVM, we skip network creation for sandbox
+		// container. Hence we configure the network in case of workload container.
+		log.G(ctx).Debug("Container type: ", ct)
+		//if ct == oci.KubernetesContainerTypeNone || ct == oci.KubernetesContainerTypeSandbox {
+		if err := coi.HostingSystem.ConfigureNetworking(ctx, coi.actualNetworkNamespace); err != nil {
+			// No network setup type was specified for this UVM. Create and assign one here unless
+			// we received a different error.
+			if err == uvm.ErrNoNetworkSetup {
+				if err := coi.HostingSystem.CreateAndAssignNetworkSetup(ctx, "", ""); err != nil {
 					return err
 				}
+				if err := coi.HostingSystem.ConfigureNetworking(ctx, coi.actualNetworkNamespace); err != nil {
+					return err
+				}
+			} else {
+				return err
 			}
-			r.SetAddedNetNSToVM(true)
 		}
+		r.SetAddedNetNSToVM(true)
+		//}
 	}
-
+	log.G(ctx).Debug("Network namespace: ", coi.actualNetworkNamespace)
 	return nil
 }
 
@@ -168,6 +175,9 @@ func configureSandboxNetwork(ctx context.Context, coi *createOptionsInternal, r
 // release the resources on failure, so that the client can make the necessary
 // call to release resources that have been allocated as part of calling this function.
 func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.Container, _ *resources.Resources, err error) {
+	// log.G(ctx).Debug("Inside CreateContainer")
+	// return nil, nil, nil
+
 	coi, err := initializeCreateOptions(ctx, createOptions)
 	if err != nil {
 		return nil, nil, err
@@ -210,13 +220,18 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C
 	isSandbox := ct == oci.KubernetesContainerTypeSandbox
 
 	// Create a network namespace if necessary.
+	log.G(ctx).Debug("hcsshim: IsSandboxContainer:", isSandbox)
 	if coi.Spec.Windows != nil &&
 		coi.Spec.Windows.Network != nil &&
-		schemaversion.IsV21(coi.actualSchemaVersion) {
+		schemaversion.IsV21(coi.actualSchemaVersion) &&
+		!isSandbox {
+		log.G(ctx).Debug("hcsshim::CreateContainer configuring sandbox network for workload container")
 		err = configureSandboxNetwork(ctx, coi, r, ct)
 		if err != nil {
 			return nil, r, fmt.Errorf("failure while creating namespace for container: %s", err)
 		}
+	} else {
+		log.G(ctx).Debug("hcsshim::CreateContainer skipping network creation for sandbox container")
 	}
 
 	var hcsDocument, gcsDocument interface{}
@@ -269,15 +284,16 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C
 		}
 	}
 
-	log.G(ctx).Debug("hcsshim::CreateContainer creating compute system")
 	if gcsDocument != nil {
+		log.G(ctx).Debug("hcsshim::CreateContainer creating container")
 		c, err := coi.HostingSystem.CreateContainer(ctx, coi.actualID, gcsDocument)
 		if err != nil {
 			return nil, r, err
 		}
 		return c, r, nil
 	}
 
+	log.G(ctx).Debug("hcsshim::CreateContainer creating compute system")
 	system, err := hcs.CreateComputeSystem(ctx, coi.actualID, hcsDocument)
 	if err != nil {
 		return nil, r, err

internal/hcsoci/hcsdoc_lcow.go

@@ -17,6 +17,7 @@ import (
 
 func createLCOWSpec(ctx context.Context, coi *createOptionsInternal) (*specs.Spec, error) {
 	// Remarshal the spec to perform a deep copy.
+	log.G(ctx).Debug("Inside createLCOWSpec")
 	j, err := json.Marshal(coi.Spec)
 	if err != nil {
 		return nil, err
@@ -31,7 +32,12 @@ func createLCOWSpec(ctx context.Context, coi *createOptionsInternal) (*specs.Spe
 	// network namespace and windows devices
 	spec.Windows = nil
 	if coi.Spec.Windows != nil {
-		setWindowsNetworkNamespace(coi, spec)
+		if coi.actualNetworkNamespace != "" {
+			log.G(ctx).Debug("Setting up Windows Network Namespace")
+			setWindowsNetworkNamespace(coi, spec, ctx)
+		} else {
+			log.G(ctx).Debug("Skip setting up Windows Network Namespace")
+		}
 		setWindowsDevices(coi, spec)
 	}
 
@@ -55,12 +61,14 @@ func createLCOWSpec(ctx context.Context, coi *createOptionsInternal) (*specs.Spe
 	return spec, nil
 }
 
-func setWindowsNetworkNamespace(coi *createOptionsInternal, spec *specs.Spec) {
+func setWindowsNetworkNamespace(coi *createOptionsInternal, spec *specs.Spec, ctx context.Context) {
+	log.G(ctx).Debug("Inside setWindowsNetworkNamespace")
 	if coi.Spec.Windows.Network != nil &&
 		coi.Spec.Windows.Network.NetworkNamespace != "" {
 		if spec.Windows == nil {
 			spec.Windows = &specs.Windows{}
 		}
+		log.G(ctx).Debug("Network namespace from coi.Spec: ", coi.Spec.Windows.Network.NetworkNamespace)
 		spec.Windows.Network = &specs.WindowsNetwork{
 			NetworkNamespace: coi.Spec.Windows.Network.NetworkNamespace,
 		}

internal/hcsoci/network.go

@@ -14,6 +14,7 @@ import (
 )
 
 func createNetworkNamespace(ctx context.Context, coi *createOptionsInternal, r *resources.Resources) error {
+	log.G(ctx).Debug("Inside createNetworkNamespace")
 	op := "hcsoci::createNetworkNamespace"
 	l := log.G(ctx).WithField(logfields.ContainerID, coi.ID)
 	l.Debug(op + " - Begin")

internal/oci/uvm.go

@@ -261,6 +261,7 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) (
 		lopts.UVMReferenceInfoFile = parseAnnotationsString(s.Annotations, annotations.UVMReferenceInfoFile, lopts.UVMReferenceInfoFile)
 		lopts.KernelBootOptions = parseAnnotationsString(s.Annotations, annotations.KernelBootOptions, lopts.KernelBootOptions)
 		lopts.DisableTimeSyncService = ParseAnnotationsBool(ctx, s.Annotations, annotations.DisableLCOWTimeSyncService, lopts.DisableTimeSyncService)
+		lopts.PreprovisionedUvm = ParseAnnotationsBool(ctx, s.Annotations, annotations.PreprovisionedUvm, lopts.PreprovisionedUvm)
 		handleAnnotationPreferredRootFSType(ctx, s.Annotations, lopts)
 		handleAnnotationKernelDirectBoot(ctx, s.Annotations, lopts)
 

internal/uvm/create.go

@@ -252,6 +252,7 @@ func (uvm *UtilityVM) Close() (err error) {
 // CreateContainer creates a container in the utility VM.
 func (uvm *UtilityVM) CreateContainer(ctx context.Context, id string, settings interface{}) (cow.Container, error) {
 	if uvm.gc != nil {
+		log.G(ctx).Debug("Inside uVM CreateContainer")
 		c, err := uvm.gc.CreateContainer(ctx, id, settings)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create container %s: %s", id, err)

internal/uvm/create_lcow.go

@@ -120,6 +120,7 @@ type OptionsLCOW struct {
 	VPCIEnabled             bool                // Whether the kernel should enable pci
 	EnableScratchEncryption bool                // Whether the scratch should be encrypted
 	DisableTimeSyncService  bool                // Disables the time synchronization service
+	PreprovisionedUvm       bool                // Whether the UVM is preprovisioned
 }
 
 // defaultLCOWOSBootFilesPath returns the default path used to locate the LCOW
@@ -781,6 +782,7 @@ func CreateLCOW(ctx context.Context, opts *OptionsLCOW) (_ *UtilityVM, err error
 		encryptScratch:          opts.EnableScratchEncryption,
 		noWritableFileShares:    opts.NoWritableFileShares,
 		confidentialUVMOptions:  opts.ConfidentialOptions,
+		preprovisionedUvm:       opts.PreprovisionedUvm,
 	}
 
 	defer func() {

internal/uvm/network.go

@@ -132,16 +132,24 @@ var ErrNoNetworkSetup = errors.New("no network setup present for UVM")
 //
 // `addr` is an optional parameter
 func (uvm *UtilityVM) CreateAndAssignNetworkSetup(ctx context.Context, addr, containerID string) (err error) {
+	log.G(ctx).Debug("Inside CreateAndAssignNetworkSetup")
+	// if uvm.preprovisionedUvm {
+	// 	log.G(ctx).Debug("Skipping CreateAndAssignNetworkSetup")
+	// 	return nil
+	// }
+
 	if uvm.NCProxyEnabled() {
 		if addr == "" || containerID == "" {
 			return errors.New("received empty field(s) for external network setup")
 		}
+		log.G(ctx).Debug("creating external network setup for UVM")
 		setup, err := NewExternalNetworkSetup(ctx, uvm, addr, containerID)
 		if err != nil {
 			return err
 		}
 		uvm.networkSetup = setup
 	} else {
+		log.G(ctx).Debug("creating internal network setup for UVM")
 		uvm.networkSetup = NewInternalNetworkSetup(uvm)
 	}
 	return nil

internal/uvm/types.go

@@ -33,16 +33,17 @@ type namespaceInfo struct {
 
 // UtilityVM is the object used by clients representing a utility VM
 type UtilityVM struct {
-	id               string               // Identifier for the utility VM (user supplied or generated)
-	runtimeID        guid.GUID            // Hyper-V VM ID
-	owner            string               // Owner for the utility VM (user supplied or generated)
-	operatingSystem  string               // "windows" or "linux"
-	hcsSystem        *hcs.System          // The handle to the compute system
-	gcListener       net.Listener         // The GCS connection listener
-	gc               *gcs.GuestConnection // The GCS connection
-	processorCount   int32
-	physicallyBacked bool       // If the uvm is backed by physical memory and not virtual memory
-	m                sync.Mutex // Lock for adding/removing devices
+	id                string               // Identifier for the utility VM (user supplied or generated)
+	runtimeID         guid.GUID            // Hyper-V VM ID
+	owner             string               // Owner for the utility VM (user supplied or generated)
+	operatingSystem   string               // "windows" or "linux"
+	hcsSystem         *hcs.System          // The handle to the compute system
+	gcListener        net.Listener         // The GCS connection listener
+	gc                *gcs.GuestConnection // The GCS connection
+	processorCount    int32
+	physicallyBacked  bool       // If the uvm is backed by physical memory and not virtual memory
+	m                 sync.Mutex // Lock for adding/removing devices
+	preprovisionedUvm bool
 
 	exitErr error
 	exitCh  chan struct{}

pkg/annotations/annotations.go

@@ -289,6 +289,9 @@ const (
 	// DumpDirectoryPath provides a path to the directory in which dumps for a UVM will be collected in
 	// case the UVM crashes.
 	DumpDirectoryPath = "io.microsoft.virtualmachine.dump-directory-path"
+
+	// Preprovisioned UVMs are UVMs that are created and started before a container is created.
+	PreprovisionedUvm = "io.microsoft.virtualmachine.preprovisioned"
 )
 
 // AnnotationExpansions maps annotations that will be expanded into an array of