Skip to content

Commit

Permalink
cross-node migration works
Browse files Browse the repository at this point in the history
  • Loading branch information
@kevpar
kevpar committed Dec 19, 2024
1 parent 641f6db commit 019e2a2
Show file tree
Hide file tree
Showing 8 changed files with 791 additions and 405 deletions.
9 changes: 9 additions & 0 deletions internal/core/core.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,3 +80,12 @@ type LayersResource struct {
ResourceID string
ContainerID string
}

type Replacements struct {
Layers []*LayersReplacement
}

type LayersReplacement struct {
ResourceID string
Layers *layers.LCOWLayers2
}
67 changes: 50 additions & 17 deletions internal/core/linuxvm/migrator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/Microsoft/hcsshim/internal/core"
"github.com/Microsoft/hcsshim/internal/guestmanager"
"github.com/Microsoft/hcsshim/internal/hns"
"github.com/Microsoft/hcsshim/internal/layers"
statepkg "github.com/Microsoft/hcsshim/internal/state"
vm "github.com/Microsoft/hcsshim/internal/vm2"
vmpkg "github.com/Microsoft/hcsshim/internal/vm2"
Expand All @@ -29,20 +30,15 @@ func (s *Sandbox) LMPrepare(ctx context.Context) (_ *statepkg.SandboxState, _ *c
// s.LMCancel(ctx)
}
}()
g, err := guid.NewV4()
if err != nil {
return nil, nil, err
}
resources := &core.Resources{Layers: []*core.LayersResource{{ResourceID: g.String(), ContainerID: "SandboxID"}}}
for cid := range s.ctrs {
var resources core.Resources
var intResources []*statepkg.Resource
for id, r := range s.translator.resources {
g, err := guid.NewV4()
if err != nil {
return nil, nil, err
}
resources.Layers = append(resources.Layers, &core.LayersResource{ResourceID: g.String(), ContainerID: cid})
}
var intResources []*statepkg.Resource
for id, r := range s.translator.resources {
resources.Layers = append(resources.Layers, &core.LayersResource{ResourceID: g.String(), ContainerID: id})

var roLayers []*statepkg.Resource_Layers_SCSI
for _, l := range r.readOnlyLayers {
roLayers = append(roLayers, &statepkg.Resource_Layers_SCSI{
Expand All @@ -51,9 +47,9 @@ func (s *Sandbox) LMPrepare(ctx context.Context) (_ *statepkg.SandboxState, _ *c
})
}
intR := &statepkg.Resource{
ResourceId: g.String(),
Type: &statepkg.Resource_Layers_{
Layers: &statepkg.Resource_Layers{
TaskId: id,
Scratch: &statepkg.Resource_Layers_SCSI{
Controller: uint32(r.scratchLayer.controller),
Lun: uint32(r.scratchLayer.lun),
Expand All @@ -74,6 +70,7 @@ func (s *Sandbox) LMPrepare(ctx context.Context) (_ *statepkg.SandboxState, _ *c
}
}
s.state = &statepkg.SandboxState{
SandboxId: s.id,
Vm: &statepkg.VMState{
Config: statepkg.VMConfigFromInternal(vmConfig),
CompatInfo: compatInfo,
Expand All @@ -83,7 +80,7 @@ func (s *Sandbox) LMPrepare(ctx context.Context) (_ *statepkg.SandboxState, _ *c
Ifaces: s.ifaces,
Containers: containers,
}
return s.state, resources, nil
return s.state, &resources, nil
}

func (s *Sandbox) LMTransfer(ctx context.Context, socket uintptr) (core.Migrated, error) {
Expand All @@ -99,6 +96,7 @@ func (s *Sandbox) LMTransfer(ctx context.Context, socket uintptr) (core.Migrated

type migrated struct {
vm *vm.VM
sandboxID string
sandboxContainer *statepkg.Container
agentConfig *statepkg.GCState
newNetNS string
Expand All @@ -109,7 +107,7 @@ func (m *migrated) LMComplete(ctx context.Context) (core.Sandbox, error) {
if err := m.vm.LMFinalize(ctx, true); err != nil {
return nil, err
}
return newSandbox(ctx, m.vm, m.sandboxContainer, m.agentConfig, m.newNetNS, m.oldIfaces)
return newSandbox(ctx, m.vm, m.sandboxID, m.sandboxContainer, m.agentConfig, m.newNetNS, m.oldIfaces)
}

func (m *migrated) LMKill(ctx context.Context) error {
Expand All @@ -125,10 +123,44 @@ type migrator struct {
netns string
}

func NewMigrator(ctx context.Context, id string, config *statepkg.SandboxState, netns string, annos map[string]string) (_ core.Migrator, err error) {
func NewMigrator(ctx context.Context, id string, config *statepkg.SandboxState, netns string, annos map[string]string, replacements *core.Replacements) (_ core.Migrator, err error) {
logrus.WithField("config", config).Info("creating lm sandbox with config")
vmConfig := statepkg.VMConfigToInternal(config.Vm.Config)
vmConfig.Serial = annos["io.microsoft.virtualmachine.console.pipe"]

for _, replacement := range replacements.Layers {
for _, resource := range config.Vm.Resources {
if replacement.ResourceID == resource.ResourceId {
resource, ok := resource.Type.(*statepkg.Resource_Layers_)
if !ok {
return nil, fmt.Errorf("resource %s must be layers", replacement.ResourceID)
}
if len(replacement.Layers.Layers) != len(resource.Layers.ReadOnlyLayers) {
return nil, fmt.Errorf("mismatched number of layers in resource %s", replacement.ResourceID)
}
replace := func(controller, lun uint, replacement layers.LCOWLayer2) error {
att := vmConfig.SCSI[controller][lun]
switch v := replacement.(type) {
case *layers.LCOWLayerVHD:
att.Path = v.VHDPath
default:
return fmt.Errorf("invalid layer type: %T", v)
}
vmConfig.SCSI[controller][lun] = att
return nil
}
if err := replace(uint(resource.Layers.Scratch.Controller), uint(resource.Layers.Scratch.Lun), replacement.Layers.Scratch); err != nil {
return nil, fmt.Errorf("error replacing resource %s: %w", replacement.ResourceID, err)
}
for i, resourceLayer := range resource.Layers.ReadOnlyLayers {
if err := replace(uint(resourceLayer.Controller), uint(resourceLayer.Lun), replacement.Layers.Layers[i]); err != nil {
return nil, fmt.Errorf("error replacing resource %s: %w", replacement.ResourceID, err)
}
}
}
}
}

vmID := fmt.Sprintf("%s@vm", id)
for _, controller := range vmConfig.SCSI {
for _, att := range controller {
Expand Down Expand Up @@ -156,14 +188,15 @@ func (m *migrator) LMTransfer(ctx context.Context, socket uintptr) (core.Migrate
}
return &migrated{
vm: m.vm,
sandboxContainer: m.sandboxState.Containers["SANDBOX"],
sandboxID: m.sandboxState.SandboxId,
sandboxContainer: m.sandboxState.Containers[m.sandboxState.SandboxId],
agentConfig: m.sandboxState.Agent,
newNetNS: m.netns,
oldIfaces: m.sandboxState.Ifaces,
}, nil
}

func newSandbox(ctx context.Context, vm *vm.VM, sandboxContainer *statepkg.Container, agentConfig *statepkg.GCState, newNetNS string, oldIFaces []*statepkg.GuestInterface) (core.Sandbox, error) {
func newSandbox(ctx context.Context, vm *vm.VM, sandboxID string, sandboxContainer *statepkg.Container, agentConfig *statepkg.GCState, newNetNS string, oldIFaces []*statepkg.GuestInterface) (core.Sandbox, error) {
gm, err := guestmanager.NewLinuxManagerFromState(
func(port uint32) (net.Listener, error) { return vm.ListenHVSocket(winio.VsockServiceID(port)) },
agentConfig)
Expand Down Expand Up @@ -215,7 +248,7 @@ func newSandbox(ctx context.Context, vm *vm.VM, sandboxContainer *statepkg.Conta

waitCtx, waitCancel := context.WithCancel(context.Background())
gt := newGuestThing(gm)
pauseCtr, err := restoreContainer(ctx, gt, waitCtx, "SANDBOX", sandboxContainer.InitPid, nil)
pauseCtr, err := restoreContainer(ctx, gt, waitCtx, sandboxID, sandboxContainer.InitPid, nil)
if err != nil {
return nil, err
}
Expand Down
6 changes: 4 additions & 2 deletions internal/core/linuxvm/sandbox.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ var (
)

type Sandbox struct {
id string
vm *vmpkg.VM
gm *guestmanager.LinuxManager
gt *guestThing
Expand Down Expand Up @@ -208,7 +209,7 @@ func NewSandbox(ctx context.Context, id string, l *layers.LCOWLayers2, spec *spe
return nil, err
}
ctrConfig := &core.LinuxCtrConfig{
ID: "SANDBOX",
ID: id,
Layers: l,
Spec: newSpec,
}
Expand All @@ -228,13 +229,14 @@ func NewSandbox(ctx context.Context, id string, l *layers.LCOWLayers2, spec *spe
}

return &Sandbox{
id: id,
vm: vm,
gm: gm,
gt: gt,
translator: translator,
pauseCtr: pauseCtr,
ctrs: map[string]*ctr{
"SANDBOX": pauseCtr,
id: pauseCtr,
},
allowMigration: allowMigration,
waitCh: make(chan struct{}),
Expand Down
Loading

0 comments on commit 019e2a2

Please sign in to comment.