Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Do Not Merge][CPPS][hcsshim][PoC] Create UVM without networking setup, set network during container creation #1782

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 32 additions & 2 deletions cmd/containerd-shim-runhcs-v1/pod.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,10 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques
case *uvm.OptionsLCOW:
lopts = (opts).(*uvm.OptionsLCOW)
lopts.BundleDirectory = req.Bundle
log.G(ctx).WithField("tid", req.ID).Debug("Before calling uvm.createLCOW(), setting preprovisioneduvm to true for testing.")
// Hack: Set this to true to test preprovisioned uvm
lopts.PreprovisionedUvm = true
log.G(ctx).WithField("tid", req.ID).Debug("Calling uvm.createLCOW()")
parent, err = uvm.CreateLCOW(ctx, lopts)
if err != nil {
return nil, err
Expand Down Expand Up @@ -191,8 +195,15 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques
cid = id
}
caAddr := fmt.Sprintf(uvm.ComputeAgentAddrFmt, cid)
if err := parent.CreateAndAssignNetworkSetup(ctx, caAddr, cid); err != nil {
return nil, err

// Skip network setup if this is a preprovisioned uvm.
if lopts.PreprovisionedUvm == false {
log.G(ctx).WithField("tid", req.ID).Debug("Calling parent.CreateAndAssignNetworkSetup()")
if err := parent.CreateAndAssignNetworkSetup(ctx, caAddr, cid); err != nil {
return nil, err
}
} else {
log.G(ctx).Debug("Skip calling parent.CreateAndAssignNetworkSetup() if preprovisioned Uvm")
}
}

Expand Down Expand Up @@ -302,6 +313,7 @@ func (p *pod) ID() string {
}

func (p *pod) CreateTask(ctx context.Context, req *task.CreateTaskRequest, s *specs.Spec) (_ shimTask, err error) {
log.G(ctx).Debug("Inside pod.go::CreateTask -> Printing p.id", p.id)
if req.ID == p.id {
return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "task with id: '%s' already exists", req.ID)
}
Expand Down Expand Up @@ -356,6 +368,24 @@ func (p *pod) CreateTask(ctx context.Context, req *task.CreateTaskRequest, s *sp
sid)
}

// For preprovisioned UVMs, we skip creating the container during the pre-provision stage.
// Subsequently, we try to create and assign network for the UVM during the creation of
// (non-sandbox) container.
var parent *uvm.UtilityVM
parent = p.host
if parent != nil && ct != oci.KubernetesContainerTypeSandbox {
cid := req.ID
if id, ok := s.Annotations[annotations.NcproxyContainerID]; ok {
cid = id
}
caAddr := fmt.Sprintf(uvm.ComputeAgentAddrFmt, cid)

log.G(ctx).Debug("pod.go:CreateTask -> Calling parent.CreateAndAssignNetworkSetup()")
if err := parent.CreateAndAssignNetworkSetup(ctx, caAddr, cid); err != nil {
return nil, err
}
}

st, err := newHcsTask(ctx, p.events, p.host, false, req, s)
if err != nil {
return nil, err
Expand Down
5 changes: 5 additions & 0 deletions cmd/containerd-shim-runhcs-v1/service_internal.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

runhcsopts "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
"github.com/Microsoft/hcsshim/internal/extendedtask"
"github.com/Microsoft/hcsshim/internal/log"
"github.com/Microsoft/hcsshim/internal/oci"
"github.com/Microsoft/hcsshim/internal/shimdiag"
containerd_v1_types "github.com/containerd/containerd/api/types/task"
Expand Down Expand Up @@ -70,6 +71,7 @@ func (s *service) stateInternal(ctx context.Context, req *task.StateRequest) (*t
}

func (s *service) createInternal(ctx context.Context, req *task.CreateTaskRequest) (*task.CreateTaskResponse, error) {
log.G(ctx).Debug("Inside service_internal::createInternal")
setupDebuggerEvent()

shimOpts := &runhcsopts.Options{}
Expand Down Expand Up @@ -159,6 +161,7 @@ func (s *service) createInternal(ctx context.Context, req *task.CreateTaskReques
if s.isSandbox {
pod, err := s.getPod()
if err == nil {
log.G(ctx).Debug("Inside service_internal::createInternal -> Pod found, Printing Pod ID", pod.ID())
// The POD sandbox was previously created. Unlock and forward to the POD
s.cl.Unlock()
t, err := pod.CreateTask(ctx, req, &spec)
Expand All @@ -169,6 +172,7 @@ func (s *service) createInternal(ctx context.Context, req *task.CreateTaskReques
resp.Pid = uint32(e.Pid())
return resp, nil
}
log.G(ctx).Debug("Inside service_internal::createInternal -> Pod not found, creating pod")
pod, err = createPod(ctx, s.events, req, &spec)
if err != nil {
s.cl.Unlock()
Expand All @@ -179,6 +183,7 @@ func (s *service) createInternal(ctx context.Context, req *task.CreateTaskReques
resp.Pid = uint32(e.Pid())
s.taskOrPod.Store(pod)
} else {
log.G(ctx).Debug("Inside service_internal::createInternal -> Not sandbox, creating standalone task")
t, err := newHcsStandaloneTask(ctx, s.events, req, &spec)
if err != nil {
s.cl.Unlock()
Expand Down
2 changes: 2 additions & 0 deletions cmd/containerd-shim-runhcs-v1/task_hcs.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,9 @@ func newHcsTask(
return nil, err
}

log.G(ctx).Debug("Inside task_hcs/newHCSTask")
container, resources, err := createContainer(ctx, req.ID, owner, netNS, s, parent, shimOpts, req.Rootfs)

if err != nil {
return nil, err
}
Expand Down
2 changes: 2 additions & 0 deletions cmd/ncproxy/ncproxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ func newGRPCService(agentCache *computeAgentCache, ncproxyNetworking *ncproxysto
var _ ncproxygrpc.NetworkConfigProxyServer = &grpcService{}

func (s *grpcService) AddNIC(ctx context.Context, req *ncproxygrpc.AddNICRequest) (_ *ncproxygrpc.AddNICResponse, err error) {
log.G(ctx).Debug("Inside ncproxy/AddNIC")
ctx, span := oc.StartSpan(ctx, "AddNIC")
defer span.End()
defer func() { oc.SetSpanStatus(span, err) }()
Expand Down Expand Up @@ -307,6 +308,7 @@ func (s *grpcService) DeleteNIC(ctx context.Context, req *ncproxygrpc.DeleteNICR
}

func (s *grpcService) CreateNetwork(ctx context.Context, req *ncproxygrpc.CreateNetworkRequest) (_ *ncproxygrpc.CreateNetworkResponse, err error) {
log.G(ctx).Debug("Inside CreateNetwork")
ctx, span := oc.StartSpan(ctx, "CreateNetwork")
defer span.End()
defer func() { oc.SetSpanStatus(span, err) }()
Expand Down
1 change: 1 addition & 0 deletions internal/gcs/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ var _ cow.Container = &Container{}
// CreateContainer creates a container using ID `cid` and `cfg`. The request
// will likely not be cancellable even if `ctx` becomes done.
func (gc *GuestConnection) CreateContainer(ctx context.Context, cid string, config interface{}) (_ *Container, err error) {
log.G(ctx).Debug("Inside GuestConnection CreateContainer")
ctx, span := oc.StartSpan(ctx, "gcs::GuestConnection::CreateContainer", oc.WithClientSpanKind)
defer span.End()
defer func() { oc.SetSpanStatus(span, err) }()
Expand Down
48 changes: 32 additions & 16 deletions internal/hcsoci/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ func validateContainerConfig(ctx context.Context, coi *createOptionsInternal) er
}

func initializeCreateOptions(ctx context.Context, createOptions *CreateOptions) (*createOptionsInternal, error) {
log.G(ctx).Debug("Inside initializeCreateOptions")
coi := &createOptionsInternal{
CreateOptions: createOptions,
actualID: createOptions.ID,
Expand Down Expand Up @@ -125,9 +126,12 @@ func initializeCreateOptions(ctx context.Context, createOptions *CreateOptions)
// configureSandboxNetwork creates a new network namespace for the pod (sandbox)
// if required and then adds that namespace to the pod.
func configureSandboxNetwork(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, ct oci.KubernetesContainerType) error {
log.G(ctx).Debug("Inside configureSandboxNetwork")
if coi.NetworkNamespace != "" {
log.G(ctx).Debug("coi.NetworkNamespace is not empty")
r.SetNetNS(coi.NetworkNamespace)
} else {
log.G(ctx).Debug("Invoking createNetworkNamespace")
err := createNetworkNamespace(ctx, coi, r)
if err != nil {
return err
Expand All @@ -139,25 +143,28 @@ func configureSandboxNetwork(ctx context.Context, coi *createOptionsInternal, r
// Only add the network namespace to a standalone or sandbox
// container but not a workload container in a sandbox that inherits
// the namespace.
if ct == oci.KubernetesContainerTypeNone || ct == oci.KubernetesContainerTypeSandbox {
if err := coi.HostingSystem.ConfigureNetworking(ctx, coi.actualNetworkNamespace); err != nil {
// No network setup type was specified for this UVM. Create and assign one here unless
// we received a different error.
if err == uvm.ErrNoNetworkSetup {
if err := coi.HostingSystem.CreateAndAssignNetworkSetup(ctx, "", ""); err != nil {
return err
}
if err := coi.HostingSystem.ConfigureNetworking(ctx, coi.actualNetworkNamespace); err != nil {
return err
}
} else {
// However in case of preprovisioned UVM, we skip network creation for sandbox
// container. Hence we configure the network in case of workload container.
log.G(ctx).Debug("Container type: ", ct)
//if ct == oci.KubernetesContainerTypeNone || ct == oci.KubernetesContainerTypeSandbox {
if err := coi.HostingSystem.ConfigureNetworking(ctx, coi.actualNetworkNamespace); err != nil {
// No network setup type was specified for this UVM. Create and assign one here unless
// we received a different error.
if err == uvm.ErrNoNetworkSetup {
if err := coi.HostingSystem.CreateAndAssignNetworkSetup(ctx, "", ""); err != nil {
return err
}
if err := coi.HostingSystem.ConfigureNetworking(ctx, coi.actualNetworkNamespace); err != nil {
return err
}
} else {
return err
}
r.SetAddedNetNSToVM(true)
}
r.SetAddedNetNSToVM(true)
//}
}

log.G(ctx).Debug("Network namespace: ", coi.actualNetworkNamespace)
return nil
}

Expand All @@ -168,6 +175,9 @@ func configureSandboxNetwork(ctx context.Context, coi *createOptionsInternal, r
// release the resources on failure, so that the client can make the necessary
// call to release resources that have been allocated as part of calling this function.
func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.Container, _ *resources.Resources, err error) {
// log.G(ctx).Debug("Inside CreateContainer")
// return nil, nil, nil

coi, err := initializeCreateOptions(ctx, createOptions)
if err != nil {
return nil, nil, err
Expand Down Expand Up @@ -210,13 +220,18 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C
isSandbox := ct == oci.KubernetesContainerTypeSandbox

// Create a network namespace if necessary.
log.G(ctx).Debug("hcsshim: IsSandboxContainer:", isSandbox)
if coi.Spec.Windows != nil &&
coi.Spec.Windows.Network != nil &&
schemaversion.IsV21(coi.actualSchemaVersion) {
schemaversion.IsV21(coi.actualSchemaVersion) &&
!isSandbox {
log.G(ctx).Debug("hcsshim::CreateContainer configuring sandbox network for workload container")
err = configureSandboxNetwork(ctx, coi, r, ct)
if err != nil {
return nil, r, fmt.Errorf("failure while creating namespace for container: %s", err)
}
} else {
log.G(ctx).Debug("hcsshim::CreateContainer skipping network creation for sandbox container")
}

var hcsDocument, gcsDocument interface{}
Expand Down Expand Up @@ -269,15 +284,16 @@ func CreateContainer(ctx context.Context, createOptions *CreateOptions) (_ cow.C
}
}

log.G(ctx).Debug("hcsshim::CreateContainer creating compute system")
if gcsDocument != nil {
log.G(ctx).Debug("hcsshim::CreateContainer creating container")
c, err := coi.HostingSystem.CreateContainer(ctx, coi.actualID, gcsDocument)
if err != nil {
return nil, r, err
}
return c, r, nil
}

log.G(ctx).Debug("hcsshim::CreateContainer creating compute system")
system, err := hcs.CreateComputeSystem(ctx, coi.actualID, hcsDocument)
if err != nil {
return nil, r, err
Expand Down
12 changes: 10 additions & 2 deletions internal/hcsoci/hcsdoc_lcow.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (

func createLCOWSpec(ctx context.Context, coi *createOptionsInternal) (*specs.Spec, error) {
// Remarshal the spec to perform a deep copy.
log.G(ctx).Debug("Inside createLCOWSpec")
j, err := json.Marshal(coi.Spec)
if err != nil {
return nil, err
Expand All @@ -31,7 +32,12 @@ func createLCOWSpec(ctx context.Context, coi *createOptionsInternal) (*specs.Spe
// network namespace and windows devices
spec.Windows = nil
if coi.Spec.Windows != nil {
setWindowsNetworkNamespace(coi, spec)
if coi.actualNetworkNamespace != "" {
log.G(ctx).Debug("Setting up Windows Network Namespace")
setWindowsNetworkNamespace(coi, spec, ctx)
} else {
log.G(ctx).Debug("Skip setting up Windows Network Namespace")
}
setWindowsDevices(coi, spec)
}

Expand All @@ -55,12 +61,14 @@ func createLCOWSpec(ctx context.Context, coi *createOptionsInternal) (*specs.Spe
return spec, nil
}

func setWindowsNetworkNamespace(coi *createOptionsInternal, spec *specs.Spec) {
func setWindowsNetworkNamespace(coi *createOptionsInternal, spec *specs.Spec, ctx context.Context) {
log.G(ctx).Debug("Inside setWindowsNetworkNamespace")
if coi.Spec.Windows.Network != nil &&
coi.Spec.Windows.Network.NetworkNamespace != "" {
if spec.Windows == nil {
spec.Windows = &specs.Windows{}
}
log.G(ctx).Debug("Network namespace from coi.Spec: ", coi.Spec.Windows.Network.NetworkNamespace)
spec.Windows.Network = &specs.WindowsNetwork{
NetworkNamespace: coi.Spec.Windows.Network.NetworkNamespace,
}
Expand Down
1 change: 1 addition & 0 deletions internal/hcsoci/network.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
)

func createNetworkNamespace(ctx context.Context, coi *createOptionsInternal, r *resources.Resources) error {
log.G(ctx).Debug("Inside createNetworkNamespace")
op := "hcsoci::createNetworkNamespace"
l := log.G(ctx).WithField(logfields.ContainerID, coi.ID)
l.Debug(op + " - Begin")
Expand Down
1 change: 1 addition & 0 deletions internal/oci/uvm.go
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,7 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) (
lopts.UVMReferenceInfoFile = parseAnnotationsString(s.Annotations, annotations.UVMReferenceInfoFile, lopts.UVMReferenceInfoFile)
lopts.KernelBootOptions = parseAnnotationsString(s.Annotations, annotations.KernelBootOptions, lopts.KernelBootOptions)
lopts.DisableTimeSyncService = ParseAnnotationsBool(ctx, s.Annotations, annotations.DisableLCOWTimeSyncService, lopts.DisableTimeSyncService)
lopts.PreprovisionedUvm = ParseAnnotationsBool(ctx, s.Annotations, annotations.PreprovisionedUvm, lopts.PreprovisionedUvm)
handleAnnotationPreferredRootFSType(ctx, s.Annotations, lopts)
handleAnnotationKernelDirectBoot(ctx, s.Annotations, lopts)

Expand Down
1 change: 1 addition & 0 deletions internal/uvm/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,7 @@ func (uvm *UtilityVM) Close() (err error) {
// CreateContainer creates a container in the utility VM.
func (uvm *UtilityVM) CreateContainer(ctx context.Context, id string, settings interface{}) (cow.Container, error) {
if uvm.gc != nil {
log.G(ctx).Debug("Inside uVM CreateContainer")
c, err := uvm.gc.CreateContainer(ctx, id, settings)
if err != nil {
return nil, fmt.Errorf("failed to create container %s: %s", id, err)
Expand Down
2 changes: 2 additions & 0 deletions internal/uvm/create_lcow.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ type OptionsLCOW struct {
VPCIEnabled bool // Whether the kernel should enable pci
EnableScratchEncryption bool // Whether the scratch should be encrypted
DisableTimeSyncService bool // Disables the time synchronization service
PreprovisionedUvm bool // Whether the UVM is preprovisioned
}

// defaultLCOWOSBootFilesPath returns the default path used to locate the LCOW
Expand Down Expand Up @@ -781,6 +782,7 @@ func CreateLCOW(ctx context.Context, opts *OptionsLCOW) (_ *UtilityVM, err error
encryptScratch: opts.EnableScratchEncryption,
noWritableFileShares: opts.NoWritableFileShares,
confidentialUVMOptions: opts.ConfidentialOptions,
preprovisionedUvm: opts.PreprovisionedUvm,
}

defer func() {
Expand Down
8 changes: 8 additions & 0 deletions internal/uvm/network.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,16 +132,24 @@ var ErrNoNetworkSetup = errors.New("no network setup present for UVM")
//
// `addr` is an optional parameter
func (uvm *UtilityVM) CreateAndAssignNetworkSetup(ctx context.Context, addr, containerID string) (err error) {
log.G(ctx).Debug("Inside CreateAndAssignNetworkSetup")
// if uvm.preprovisionedUvm {
// log.G(ctx).Debug("Skipping CreateAndAssignNetworkSetup")
// return nil
// }

if uvm.NCProxyEnabled() {
if addr == "" || containerID == "" {
return errors.New("received empty field(s) for external network setup")
}
log.G(ctx).Debug("creating external network setup for UVM")
setup, err := NewExternalNetworkSetup(ctx, uvm, addr, containerID)
if err != nil {
return err
}
uvm.networkSetup = setup
} else {
log.G(ctx).Debug("creating internal network setup for UVM")
uvm.networkSetup = NewInternalNetworkSetup(uvm)
}
return nil
Expand Down
21 changes: 11 additions & 10 deletions internal/uvm/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,16 +33,17 @@ type namespaceInfo struct {

// UtilityVM is the object used by clients representing a utility VM
type UtilityVM struct {
id string // Identifier for the utility VM (user supplied or generated)
runtimeID guid.GUID // Hyper-V VM ID
owner string // Owner for the utility VM (user supplied or generated)
operatingSystem string // "windows" or "linux"
hcsSystem *hcs.System // The handle to the compute system
gcListener net.Listener // The GCS connection listener
gc *gcs.GuestConnection // The GCS connection
processorCount int32
physicallyBacked bool // If the uvm is backed by physical memory and not virtual memory
m sync.Mutex // Lock for adding/removing devices
id string // Identifier for the utility VM (user supplied or generated)
runtimeID guid.GUID // Hyper-V VM ID
owner string // Owner for the utility VM (user supplied or generated)
operatingSystem string // "windows" or "linux"
hcsSystem *hcs.System // The handle to the compute system
gcListener net.Listener // The GCS connection listener
gc *gcs.GuestConnection // The GCS connection
processorCount int32
physicallyBacked bool // If the uvm is backed by physical memory and not virtual memory
m sync.Mutex // Lock for adding/removing devices
preprovisionedUvm bool

exitErr error
exitCh chan struct{}
Expand Down
3 changes: 3 additions & 0 deletions pkg/annotations/annotations.go
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,9 @@ const (
// DumpDirectoryPath provides a path to the directory in which dumps for a UVM will be collected in
// case the UVM crashes.
DumpDirectoryPath = "io.microsoft.virtualmachine.dump-directory-path"

// Preprovisioned UVMs are UVMs that are created and started before a container is created.
PreprovisionedUvm = "io.microsoft.virtualmachine.preprovisioned"
)

// AnnotationExpansions maps annotations that will be expanded into an array of
Expand Down