pillar/hypervisor: add multifunc dev

add support for multifunction pci devices

Signed-off-by: Christoph Ostarek <christoph@zededa.com>

pkg/pillar/hypervisor/kvm.go

@@ -394,12 +394,19 @@ const qemuRootPortPciPassthruTemplate = `
   addr = "{{printf "0x%x" .PCIId}}"
 `
 
+const qemuPCIPassthruBridgeTemplate = `
+[device "pcie-bridge.{{.Bus}}"]
+  driver = "pcie-pci-bridge"
+  bus = "pci.{{.Bus}}"
+  addr = "{{printf "0x%x" .PCIId}}"
+`
+
 const qemuPciPassthruTemplate = `
 [device]
   driver = "vfio-pci"
   host = "{{.PciShortAddr}}"
-  bus = "pci.{{.PCIId}}"
-  addr = "0x0"
+  bus = "{{.Bus}}"
+  addr = "{{.Addr}}"
 {{- if .Xvga }}
   x-vga = "on"
 {{- end -}}
@@ -932,7 +939,102 @@ func getFmlCustomResolution(status *types.DomainStatus, globalConfig *types.Conf
 	return "", fmt.Errorf("invalid fml resolution %s", fmlResolutions)
 }
 
-func pciAssignmentsTemplateFill(file io.Writer, pciAssignments []pciDevice, pciID int) error {
+type pciDevicesWithBridge struct {
+	bridgeBus string
+	devs      []*pciDevice
+}
+
+type multifunctionDevs map[string]*pciDevicesWithBridge // key: pci long without function number
+
+func (md multifunctionDevs) isMultiFunction(p pciDevice) bool {
+	pciLongWOFunc, err := p.pciLongWOFunction()
+	if err != nil {
+		return false
+	}
+	devs, found := md[pciLongWOFunc]
+	return found && len(devs.devs) > 1
+}
+
+func (md multifunctionDevs) index(p pciDevice) int {
+	pciLongWOFunc, err := p.pciLongWOFunction()
+	if err != nil {
+		logrus.Warnf("retrieving pci address without function failed: %v", err)
+		return -1
+	}
+	pciDevices, found := md[pciLongWOFunc]
+	if !found {
+		return -1
+	}
+
+	for i, dev := range pciDevices.devs {
+		if p.ioType == dev.ioType && p.pciLong == dev.pciLong {
+			return i
+		}
+	}
+
+	return -1
+}
+
+func multifunctionDevGroup(pcis []pciDevice) multifunctionDevs {
+	mds := make(multifunctionDevs)
+
+	for i, pa := range pcis {
+		pciWithoutFunction, err := pa.pciLongWOFunction()
+		if err != nil {
+			logrus.Warnf("retrieving pci address without function failed: %v", err)
+			continue
+		}
+
+		_, found := mds[pciWithoutFunction]
+		if !found {
+			mds[pciWithoutFunction] = &pciDevicesWithBridge{
+				bridgeBus: "",
+				devs:      []*pciDevice{},
+			}
+		}
+		mds[pciWithoutFunction].devs = append(mds[pciWithoutFunction].devs, &pcis[i])
+	}
+
+	return mds
+}
+
+func (p pciDevice) pciLongWOFunction() (string, error) {
+	pciLongSplit := strings.Split(p.pciLong, ".")
+	if len(pciLongSplit) == 0 {
+		return "", fmt.Errorf("could not split %s", p.pciLong)
+	}
+	pciWithoutFunction := strings.Join(pciLongSplit[0:len(pciLongSplit)-1], ".")
+
+	return pciWithoutFunction, nil
+}
+
+type pciAssignmentsTemplateFiller struct {
+	multifunctionDevices multifunctionDevs
+	file                 io.Writer
+}
+
+func (f *pciAssignmentsTemplateFiller) pciEBridge(pciID int, pciWOFunction string) error {
+	pciChildTemplateVars := struct {
+		PCIId int
+		Bus   int
+	}{
+		PCIId: 0,
+		Bus:   pciID,
+	}
+
+	tPCIeBridge, err := template.New("qemuPCIeBridge").Parse(qemuPCIPassthruBridgeTemplate)
+	if err != nil {
+		return fmt.Errorf("parsing qemuPCIPassthruBridgeTemplate failed: %w", err)
+	}
+	if err := tPCIeBridge.Execute(f.file, pciChildTemplateVars); err != nil {
+		return fmt.Errorf("can't write PCIe bridge Passthrough to config file (%w)", err)
+	}
+	f.multifunctionDevices[pciWOFunction].bridgeBus = fmt.Sprintf("pcie-bridge.%d", pciID)
+
+	return nil
+}
+
+func (f *pciAssignmentsTemplateFiller) do(file io.Writer, pciAssignments []pciDevice, pciID int) error {
 	if len(pciAssignments) == 0 {
 		return nil
 	}
@@ -942,13 +1044,18 @@ func pciAssignmentsTemplateFill(file io.Writer, pciAssignments []pciDevice, pciI
 		PciShortAddr string
 		Xvga         bool
 		Xopregion    bool
+		Bus          string
+		Addr         string
 	}{PCIId: pciID, PciShortAddr: "", Xvga: false, Xopregion: false}
 
-	tRootPortPCI, _ := template.New("qemuPciPT").Parse(qemuRootPortPciPassthruTemplate)
-	tPCI, _ := template.New("qemuPciPT").Parse(qemuPciPassthruTemplate)
-	for _, pa := range pciAssignments {
-		short := types.PCILongToShort(pa.pciLong)
+	tPCI, _ := template.New("qemuPCI").Parse(qemuPciPassthruTemplate)
+
+	pciEBridgeForMultiFuncDevCreated := make(map[string]struct{}) // key: pci long without function number
+	for i, pa := range pciAssignments {
+		pciPTContext.Xopregion = false
 		pciPTContext.Xvga = pa.isVGA()
+		pciPTContext.Bus = fmt.Sprintf("pci.%d", pciPTContext.PCIId)
+		pciPTContext.PciShortAddr = types.PCILongToShort(pa.pciLong)
 
 		if vendor, err := pa.vid(); err == nil {
 			// check for Intel vendor
@@ -961,16 +1068,43 @@ func pciAssignmentsTemplateFill(file io.Writer, pciAssignments []pciDevice, pciI
 			}
 		}
 
-		pciPTContext.PciShortAddr = short
-		if err := tRootPortPCI.Execute(file, pciPTContext); err != nil {
-			return logError("can't write Root Port PCI Passthrough to config file (%v)", err)
+		pciLongWoFunc, err := pa.pciLongWOFunction()
+		if err != nil {
+			logrus.Warnf("retrieving pci address without function failed: %v", err)
+			continue
+		}
+		_, pciEBridgeCreated := pciEBridgeForMultiFuncDevCreated[pciLongWoFunc]
+		pciEBridgeForMultiFuncDevCreated[pciLongWoFunc] = struct{}{}
+
+		// for non-multifunction devices, every pci device gets a "pcie-root-port"
+		if !f.multifunctionDevices.isMultiFunction(pa) || !pciEBridgeCreated {
+			tRootPortPCI, _ := template.New("qemuRootPortPCI").Parse(qemuRootPortPciPassthruTemplate)
+			if err := tRootPortPCI.Execute(file, pciPTContext); err != nil {
+				return logError("can't write Root Port PCI Passthrough to config file (%v)", err)
+			}
+		}
+		if f.multifunctionDevices.isMultiFunction(pa) {
+			if !pciEBridgeCreated {
+				err := f.pciEBridge(pciPTContext.PCIId, pciLongWoFunc)
+				if err != nil {
+					logrus.Warnf("could not write template: %v, skipping", err)
+				}
+			}
+
+			pciPTContext.Bus = f.multifunctionDevices[pciLongWoFunc].bridgeBus
+			pciPTContext.PCIId = f.multifunctionDevices.index(pa)
+			if pciPTContext.PCIId < 0 {
+				logrus.Warn("PCIId is less than 0 - skipping")
+			}
+			pciPTContext.Addr = fmt.Sprintf("0x%x", pciPTContext.PCIId+1) // Unsupported PCI slot 0 for standard hotplug controller
+		} else {
+			pciPTContext.Bus = fmt.Sprintf("pci.%d", pciPTContext.PCIId)
+			pciPTContext.Addr = "0x0"
 		}
 		if err := tPCI.Execute(file, pciPTContext); err != nil {
 			return logError("can't write PCI Passthrough to config file (%v)", err)
 		}
-		pciPTContext.Xvga = false
-		pciPTContext.Xopregion = false
-		pciPTContext.PCIId = pciPTContext.PCIId + 1
+		pciPTContext.PCIId = pciID + i + 1
 	}
 
 	return nil
@@ -982,6 +1116,7 @@ func (ctx KvmContext) CreateDomConfig(domainName string,
 	config types.DomainConfig, status types.DomainStatus,
 	diskStatusList []types.DiskStatus, aa *types.AssignableAdapters,
 	globalConfig *types.ConfigItemValueMap, swtpmCtrlSock string, file *os.File) error {
+
 	virtualizationMode := ""
 	bootLoaderSettingsFile, err := getOVMFSettingsFilename(domainName)
 	if err != nil {
@@ -1126,7 +1261,13 @@ func (ctx KvmContext) CreateDomConfig(domainName string,
 			}
 		}
 	}
-	err = pciAssignmentsTemplateFill(file, pciAssignments, netContext.PCIId)
+
+	pciAssignmentsFiller := pciAssignmentsTemplateFiller{
+		multifunctionDevices: multifunctionDevGroup(pciAssignments),
+		file:                 file,
+	}
+
+	err = pciAssignmentsFiller.do(file, pciAssignments, netContext.PCIId)
 	if err != nil {
 		return fmt.Errorf("writing to template file %s failed: %w", file.Name(), err)
 	}

pkg/pillar/hypervisor/kvm_test.go

@@ -995,7 +995,7 @@ func TestCreateDomConfigAmd64Fml(t *testing.T) {
 
 	result = setStaticVsockCid(result)
 	if string(result) != domConfigAmd64FML() {
-		t.Errorf("got an unexpected resulting config %s", string(result))
+		t.Errorf("got an unexpected resulting config %s", cmp.Diff(string(result), domConfigAmd64FML()))
 	}
 }
 
@@ -2771,7 +2771,7 @@ func expectedMultifunctionDevice() string {
 
 [device]
   driver = "vfio-pci"
-  host = "0d.0"
+  host = "00:0a.0"
   bus = "pci.0"
   addr = "0x0"
 [device "pci.1"]
@@ -2782,29 +2782,104 @@ func expectedMultifunctionDevice() string {
   multifunction = "on"
   addr = "0x1"
 
+[device "pcie-bridge.1"]
+  driver = "pcie-pci-bridge"
+  bus = "pci.1"
+  addr = "0x0"
+
 [device]
   driver = "vfio-pci"
-  host = "0d.2"
-  bus = "pci.1"
-  addr = "0x0"`
+  host = "00:0d.0"
+  bus = "pcie-bridge.1"
+  addr = "0x1"
+[device "pci.2"]
+  driver = "pcie-root-port"
+  port = "12"
+  chassis = "2"
+  bus = "pcie.0"
+  multifunction = "on"
+  addr = "0x2"
+
+[device]
+  driver = "vfio-pci"
+  host = "00:0b.0"
+  bus = "pci.2"
+  addr = "0x0"
+[device]
+  driver = "vfio-pci"
+  host = "00:0d.2"
+  bus = "pcie-bridge.1"
+  addr = "0x2"`
 }
 
 func TestPCIAssignmentsTemplateFillMultifunctionDevice(t *testing.T) {
 	pciAssignments := []pciDevice{
 		{
-			pciLong: "00:0d.0",
+			pciLong: "0000:00:0a.0",
+			ioType:  0,
+		},
+		{
+			pciLong: "0000:00:0d.0",
+			ioType:  0,
+		},
+		{
+			pciLong: "0000:00:0b.0",
 			ioType:  0,
 		},
 		{
-			pciLong: "00:0d.2",
+			pciLong: "0000:00:0d.2",
 			ioType:  0,
 		},
 	}
 
 	wr := bytes.Buffer{}
-	pciAssignmentsTemplateFill(&wr, pciAssignments, 0)
+	p := pciAssignmentsTemplateFiller{
+		multifunctionDevices: multifunctionDevGroup(pciAssignments),
+		file:                 &wr,
+	}
+	p.do(&wr, pciAssignments, 0)
 
 	if wr.String() != expectedMultifunctionDevice() {
-		t.Fatalf("not equal, diff: \n%s\n", cmp.Diff(wr.String(), expectedMultifunctionDevice()))
+		t.Fatalf("not equal, diff: \n%s\ncomplete:\n%s", cmp.Diff(wr.String(), expectedMultifunctionDevice()), wr.String())
+	}
+}
+
+func TestConvertToMultifunctionPCIDevices(t *testing.T) {
+	pciAssignments := []pciDevice{
+		{
+			pciLong: "0000:00:0d.0",
+			ioType:  0,
+		},
+		{
+			pciLong: "0000:00:aa.8",
+			ioType:  0,
+		},
+		{
+			pciLong: "0000:00:0d.2",
+			ioType:  0,
+		},
+		{
+			pciLong: "0000:00:0d.f",
+			ioType:  0,
+		},
+	}
+
+	mds := multifunctionDevGroup(pciAssignments)
+
+	if len(mds) != 2 {
+		t.Fatalf("expected two multifunction pci assignments, but got %d", len(mds))
+	}
+
+	t.Log(mds)
+	for i, pci := range []string{"0000:00:0d.0", "0000:00:0d.2", "0000:00:0d.f"} {
+		functionPCIDev := mds["0000:00:0d"].devs[i].pciLong
+		if functionPCIDev != pci {
+			t.Logf("expected %s got %s", pci, functionPCIDev)
+			t.Fail()
+		}
+	}
+
+	if len(mds["0000:00:aa"].devs) != 1 {
+		t.Fatal("expected one device")
 	}
 }