pillar: Adopt pillar to use native cpupinning

Update pillar's handling of QEMU guests to take advantage of native CPU pinning
options introduced in the newer version of QEMU (7.2). This eliminates the need
for our custom patches to QEMU for CPU pinning. The following changes are included:
Removing custom qemu cpu-pinning patches to include the native qemu-pinning feature (1)
Refactoring CPU pinning to use integer slices instead of comma-separated strings (2)
Updating QEMU command-line arguments to include native CPU pinning options (3)
Ensuring compatibility with both KVM and Xen hypervisors (4)
Changing 15-qemu-Set-the-affinity-of-QEMU-threads-according-to-t.patch to include the native qemu cpupinning (5)

This change allows us to leverage upstream QEMU improvements and simplifies
the codebase by removing custom patches and complex string manipulations. Tested with QEMU version 8.0.4

Signed-off-by: Roja Eswaran <roja@zededa.com>

pkg/pillar/cmd/domainmgr/domainmgr.go

@@ -1243,38 +1243,24 @@ func setCgroupCpuset(config *types.DomainConfig, status *types.DomainStatus) err
 		log.Warnf("Failed to find cgroups directory for %s", config.DisplayName)
 		return nil
 	}
-	err = controller.Update(&specs.LinuxResources{CPU: &specs.LinuxCPU{Cpus: status.VmConfig.CPUs}})
+	// Convert a list of CPUs to a CPU string
+	cpuStrings := make([]string, 0)
+	for _, cpu := range status.VmConfig.CPUs {
+		cpuStrings = append(cpuStrings, strconv.Itoa(cpu))
+	}
+	cpuMask := strings.Join(cpuStrings, ",")
+
+	err = controller.Update(&specs.LinuxResources{CPU: &specs.LinuxCPU{Cpus: cpuMask}})
 	if err != nil {
 		log.Warnf("Failed to update CPU set for %s", config.DisplayName)
 		return err
 	}
-	log.Functionf("Adjust the cgroups cpuset of %s to %s", config.DisplayName, status.VmConfig.CPUs)
+	log.Functionf("Adjust the cgroups cpuset of %s to %v", config.DisplayName, status.VmConfig.CPUs)
 	return nil
 }
 
-// constructNonPinnedCpumaskString returns a cpumask that contains at least CPUs reserved for the system
-// services. Hence, it can never be empty.
-func constructNonPinnedCpumaskString(ctx *domainContext) string {
-	result := ""
-	for _, cpu := range ctx.cpuAllocator.GetAllFree() {
-		addToMask(cpu, &result)
-	}
-	return result
-}
-
-func addToMask(cpu int, s *string) {
-	if s == nil {
-		return
-	}
-	if *s == "" {
-		*s = fmt.Sprintf("%d", cpu)
-	} else {
-		*s = fmt.Sprintf("%s,%d", *s, cpu)
-	}
-}
-
 func updateNonPinnedCPUs(ctx *domainContext, config *types.DomainConfig, status *types.DomainStatus) error {
-	status.VmConfig.CPUs = constructNonPinnedCpumaskString(ctx)
+	status.VmConfig.CPUs = ctx.cpuAllocator.GetAllFree()
 	err := setCgroupCpuset(config, status)
 	if err != nil {
 		return errors.New("failed to redistribute CPUs between VMs, can affect the inter-VM isolation")
@@ -1292,23 +1278,23 @@ func assignCPUs(ctx *domainContext, config *types.DomainConfig, status *types.Do
 			return errors.New("failed to allocate necessary amount of CPUs")
 		}
 		for _, cpu := range cpusToAssign {
-			addToMask(cpu, &status.VmConfig.CPUs)
+			status.VmConfig.CPUs = append(status.VmConfig.CPUs, cpu)
 		}
 	} else { // VM has no pinned CPUs, assign all the CPUs from the shared set
-		status.VmConfig.CPUs = constructNonPinnedCpumaskString(ctx)
+		status.VmConfig.CPUs = ctx.cpuAllocator.GetAllFree()
 	}
 	return nil
 }
 
 // releaseCPUs releases the CPUs that were previously assigned to the VM.
 // The cpumask in the *status* is updated accordingly, and the CPUs are released in the CPUAllocator context.
 func releaseCPUs(ctx *domainContext, config *types.DomainConfig, status *types.DomainStatus) {
-	if ctx.cpuPinningSupported && config.VmConfig.CPUsPinned && status.VmConfig.CPUs != "" {
+	if ctx.cpuPinningSupported && config.VmConfig.CPUsPinned && status.VmConfig.CPUs != nil {
 		if err := ctx.cpuAllocator.Free(config.UUIDandVersion.UUID); err != nil {
 			log.Errorf("Failed to free CPUs for %s: %s", config.DisplayName, err)
 		}
 	}
-	status.VmConfig.CPUs = ""
+	status.VmConfig.CPUs = nil
 }
 
 func handleCreate(ctx *domainContext, key string, config *types.DomainConfig) {
@@ -1330,7 +1316,7 @@ func handleCreate(ctx *domainContext, key string, config *types.DomainConfig) {
 		Service:        config.Service,
 	}
 
-	status.VmConfig.CPUs = ""
+	status.VmConfig.CPUs = make([]int, 0)
 
 	// Note that the -emu interface doesn't exist until after boot of the domU, but we
 	// initialize the VifList here with the VifUsed.
@@ -1545,7 +1531,7 @@ func doActivate(ctx *domainContext, config types.DomainConfig,
 			publishDomainStatus(ctx, status)
 			return
 		}
-		log.Functionf("CPUs for %s assigned: %s", config.DisplayName, status.VmConfig.CPUs)
+		log.Functionf("CPUs for %s assigned: %v", config.DisplayName, status.VmConfig.CPUs)
 	}
 
 	if errDescription := reserveAdapters(ctx, config); errDescription != nil {
@@ -1932,7 +1918,7 @@ func doCleanup(ctx *domainContext, status *types.DomainStatus) {
 			}
 			triggerCPUNotification()
 		}
-		status.VmConfig.CPUs = ""
+		status.VmConfig.CPUs = nil
 	}
 	releaseAdapters(ctx, status.IoAdapterList, status.UUIDandVersion.UUID,
 		status)

pkg/pillar/containerd/oci.go

@@ -447,8 +447,12 @@ func (s *ociSpec) UpdateFromDomain(dom *types.DomainConfig, status *types.Domain
 		s.Linux.Resources.Memory.Limit = &m
 		s.Linux.Resources.CPU.Period = &p
 		s.Linux.Resources.CPU.Quota = &q
-		if status.VmConfig.CPUs != "" {
-			s.Linux.Resources.CPU.Cpus = status.VmConfig.CPUs
+		if len(status.VmConfig.CPUs) != 0 {
+			cpusAsString := make([]string, len(status.VmConfig.CPUs))
+			for i, cpu := range status.VmConfig.CPUs {
+				cpusAsString[i] = fmt.Sprintf("%d", cpu)
+			}
+			s.Linux.Resources.CPU.Cpus = strings.Join(cpusAsString, ",")
 		}
 
 		s.Linux.CgroupsPath = fmt.Sprintf("/%s/%s", ctrdServicesNamespace, dom.GetTaskName())

pkg/pillar/hypervisor/kvm.go

@@ -846,6 +846,20 @@ func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig
 		"-readconfig", file.Name(),
 		"-pidfile", kvmStateDir+domainName+"/pid")
 
+	// Add CPUs affinity as a parameter to qemu.
+	// It's not supported to be configured in the .ini file so we need to add it here.
+	// The arguments are in the format of: -object thread-context,id=tc1,cpu-affinity=0-1,cpu-affinity=6-7
+	// The thread-context object is introduced in qemu 7.2
+	if config.CPUsPinned {
+		// Create the thread-context object string
+		threadContext := "thread-context,id=tc1"
+		for _, cpu := range status.CPUs {
+			// Add the cpu-affinity arguments to the thread-context object
+			threadContext += fmt.Sprintf(",cpu-affinity=%d", cpu)
+		}
+		args = append(args, "-object", threadContext)
+	}
+
 	spec, err := ctx.setupSpec(&status, &config, status.OCIConfigDir)
 
 	if err != nil {

pkg/pillar/hypervisor/xen.go

@@ -260,8 +260,12 @@ func (ctx xenContext) CreateDomConfig(domainName string,
 		maxCpus = vCpus
 	}
 	file.WriteString(fmt.Sprintf("maxvcpus = %d\n", maxCpus))
-	if config.CPUs != "" {
-		file.WriteString(fmt.Sprintf("cpus = \"%s\"\n", config.CPUs))
+	if len(config.CPUs) > 0 {
+		cpusString := make([]string, 0)
+		for _, curCPU := range config.CPUs {
+			cpusString = append(cpusString, strconv.Itoa(curCPU))
+		}
+		file.WriteString(fmt.Sprintf("cpus = \"%s\"\n", strings.Join(cpusString, ",")))
 	}
 	if config.DeviceTree != "" {
 		file.WriteString(fmt.Sprintf("device_tree = \"%s\"\n",

pkg/pillar/types/domainmgrtypes.go

@@ -242,7 +242,7 @@ type VmConfig struct {
 	ExtraArgs  string // added to bootargs
 	BootLoader string // default ""
 	// For CPU pinning
-	CPUs string // default "", list of "1,2"
+	CPUs []int // default nil, list of [1,2]
 	// Needed for device passthru
 	DeviceTree string // default ""; sets device_tree
 	// Example: device_tree="guest-gpio.dtb"

pkg/xen-tools/patches-4.19.0/13-qemu-Add-cpu-pin-and-cpumask-options.patch

@@ -149,4 +149,3 @@ index ea20b23..59ff7c7 100644
 base-commit: 9c55fdd5e54c5ea4bf238cee787f13a03eac1c86
 -- 
 2.35.1
-

pkg/xen-tools/patches-4.19.0/14-qemu-Init-CPU-mask-per-VCPU.patch

@@ -199,4 +199,3 @@ index ea20b23..98287f4 100644
 
 -- 
 2.35.1
-