[PATCH v3 01/21] qemu: address: Separate the slots into multiple aggregates
Daniel Henrique Barboza
danielhb413 at gmail.com
Wed May 20 21:11:23 UTC 2020
From: Shivaprasad G Bhat <sbhat at linux.vnet.ibm.com>
Today's aggregate flag with the slot being true for pcie-root-ports
is not enough as there will more number of aggregates depending on
the number of Multifunction PCI cards assigned to the domain.
The aggregate is changed to unsigned int. Zero means Not Applicable,
1 is reserved for the pcie-root-ports and >= 2 for the the PCI
Multifunction cards(coming..).
Signed-off-by: Shivaprasad G Bhat <sbhat at linux.vnet.ibm.com>
Signed-off-by: Daniel Henrique Barboza <danielhb413 at gmail.com>
---
src/conf/device_conf.h | 1 +
src/conf/domain_addr.c | 46 +++++++++++++------
src/conf/domain_addr.h | 38 +++++++--------
src/qemu/qemu_domain_address.c | 84 ++++++++++++++++++++++++++++------
src/qemu/qemu_domain_address.h | 9 ++++
5 files changed, 128 insertions(+), 50 deletions(-)
diff --git a/src/conf/device_conf.h b/src/conf/device_conf.h
index d7395f2201..cc1707c1d1 100644
--- a/src/conf/device_conf.h
+++ b/src/conf/device_conf.h
@@ -169,6 +169,7 @@ struct _virDomainDeviceInfo {
*/
int pciAddrExtFlags; /* enum virDomainPCIAddressExtensionFlags */
char *loadparm;
+ unsigned int aggregateSlotIdx; /* Used when the aggregate flag is set */
/* PCI devices will only be automatically placed on a PCI bus
* that shares the same isolation group */
diff --git a/src/conf/domain_addr.c b/src/conf/domain_addr.c
index 8623e79daf..5c8fc62725 100644
--- a/src/conf/domain_addr.c
+++ b/src/conf/domain_addr.c
@@ -297,7 +297,7 @@ virDomainPCIControllerModelToConnectType(virDomainControllerModelPCI model)
return VIR_PCI_CONNECT_TYPE_PCIE_TO_PCI_BRIDGE;
case VIR_DOMAIN_CONTROLLER_MODEL_PCIE_ROOT_PORT:
- return VIR_PCI_CONNECT_TYPE_PCIE_ROOT_PORT | VIR_PCI_CONNECT_AGGREGATE_SLOT;
+ return VIR_PCI_CONNECT_TYPE_PCIE_ROOT_PORT;
case VIR_DOMAIN_CONTROLLER_MODEL_PCIE_SWITCH_UPSTREAM_PORT:
return VIR_PCI_CONNECT_TYPE_PCIE_SWITCH_UPSTREAM_PORT;
@@ -843,6 +843,7 @@ virDomainPCIAddressReserveAddrInternal(virDomainPCIAddressSetPtr addrs,
virPCIDeviceAddressPtr addr,
virDomainPCIConnectFlags flags,
unsigned int isolationGroup,
+ unsigned int aggregateSlotIdx,
bool fromConfig)
{
g_autofree char *addrStr = NULL;
@@ -874,9 +875,14 @@ virDomainPCIAddressReserveAddrInternal(virDomainPCIAddressSetPtr addrs,
* the device it's being reserved for can aggregate multiples on a
* slot, set the slot's aggregate flag.
*/
- if (!bus->slot[addr->slot].functions &&
- flags & VIR_PCI_CONNECT_AGGREGATE_SLOT) {
- bus->slot[addr->slot].aggregate = true;
+ if (!bus->slot[addr->slot].functions && aggregateSlotIdx > 0) {
+ bus->slot[addr->slot].aggregateSlotIdx = aggregateSlotIdx;
+ } else if (bus->slot[addr->slot].aggregateSlotIdx !=
+ aggregateSlotIdx && fromConfig) {
+ bus->slot[addr->slot].aggregateSlotIdx = 0;
+ VIR_DEBUG("PCI functions of %.4x:%.2x is aggregated to slot %u"
+ "because of user assigned address %s",
+ addr->domain, addr->bus, aggregateSlotIdx, addrStr);
}
if (virDomainPCIAddressBusIsEmpty(bus) && !bus->isolationGroupLocked) {
@@ -901,8 +907,8 @@ virDomainPCIAddressReserveAddrInternal(virDomainPCIAddressSetPtr addrs,
/* mark the requested function as reserved */
bus->slot[addr->slot].functions |= (1 << addr->function);
- VIR_DEBUG("Reserving PCI address %s (aggregate='%s')", addrStr,
- bus->slot[addr->slot].aggregate ? "true" : "false");
+ VIR_DEBUG("Reserving PCI address %s (aggregateSlotIdx='%d')",
+ addrStr, bus->slot[addr->slot].aggregateSlotIdx);
return 0;
}
@@ -912,10 +918,12 @@ int
virDomainPCIAddressReserveAddr(virDomainPCIAddressSetPtr addrs,
virPCIDeviceAddressPtr addr,
virDomainPCIConnectFlags flags,
- unsigned int isolationGroup)
+ unsigned int isolationGroup,
+ unsigned int aggregateSlotIdx)
{
return virDomainPCIAddressReserveAddrInternal(addrs, addr, flags,
- isolationGroup, true);
+ isolationGroup,
+ aggregateSlotIdx, true);
}
int
@@ -956,6 +964,7 @@ virDomainPCIAddressEnsureAddr(virDomainPCIAddressSetPtr addrs,
if (virDomainPCIAddressReserveAddrInternal(addrs, &dev->addr.pci,
flags, dev->isolationGroup,
+ dev->aggregateSlotIdx,
true) < 0) {
return -1;
}
@@ -1112,6 +1121,7 @@ virDomainPCIAddressSetFree(virDomainPCIAddressSetPtr addrs)
static int
virDomainPCIAddressFindUnusedFunctionOnBus(virDomainPCIAddressBusPtr bus,
virPCIDeviceAddressPtr searchAddr,
+ unsigned int aggregateSlotIdx,
int function,
virDomainPCIConnectFlags flags,
bool *found)
@@ -1134,8 +1144,8 @@ virDomainPCIAddressFindUnusedFunctionOnBus(virDomainPCIAddressBusPtr bus,
break;
}
- if (flags & VIR_PCI_CONNECT_AGGREGATE_SLOT &&
- bus->slot[searchAddr->slot].aggregate) {
+ if (bus->slot[searchAddr->slot].aggregateSlotIdx > 0 &&
+ bus->slot[searchAddr->slot].aggregateSlotIdx == aggregateSlotIdx) {
/* slot and device are okay with aggregating devices */
if ((bus->slot[searchAddr->slot].functions &
(1 << searchAddr->function)) == 0) {
@@ -1176,6 +1186,7 @@ virDomainPCIAddressGetNextAddr(virDomainPCIAddressSetPtr addrs,
virPCIDeviceAddressPtr next_addr,
virDomainPCIConnectFlags flags,
unsigned int isolationGroup,
+ unsigned int aggregateSlotIdx,
int function)
{
virPCIDeviceAddress a = { 0 };
@@ -1204,7 +1215,9 @@ virDomainPCIAddressGetNextAddr(virDomainPCIAddressSetPtr addrs,
a.slot = bus->minSlot;
- if (virDomainPCIAddressFindUnusedFunctionOnBus(bus, &a, function,
+ if (virDomainPCIAddressFindUnusedFunctionOnBus(bus, &a,
+ aggregateSlotIdx,
+ function,
flags, &found) < 0) {
return -1;
}
@@ -1228,7 +1241,9 @@ virDomainPCIAddressGetNextAddr(virDomainPCIAddressSetPtr addrs,
a.slot = bus->minSlot;
- if (virDomainPCIAddressFindUnusedFunctionOnBus(bus, &a, function,
+ if (virDomainPCIAddressFindUnusedFunctionOnBus(bus, &a,
+ aggregateSlotIdx,
+ function,
flags, &found) < 0) {
return -1;
}
@@ -1286,12 +1301,13 @@ virDomainPCIAddressReserveNextAddr(virDomainPCIAddressSetPtr addrs,
{
virPCIDeviceAddress addr;
- if (virDomainPCIAddressGetNextAddr(addrs, &addr, flags,
- dev->isolationGroup, function) < 0)
+ if (virDomainPCIAddressGetNextAddr(addrs, &addr, flags, dev->isolationGroup,
+ dev->aggregateSlotIdx, function) < 0)
return -1;
if (virDomainPCIAddressReserveAddrInternal(addrs, &addr, flags,
- dev->isolationGroup, false) < 0)
+ dev->isolationGroup,
+ dev->aggregateSlotIdx, false) < 0)
return -1;
addr.extFlags = dev->addr.pci.extFlags;
diff --git a/src/conf/domain_addr.h b/src/conf/domain_addr.h
index c1363c1490..ddcff36e51 100644
--- a/src/conf/domain_addr.h
+++ b/src/conf/domain_addr.h
@@ -35,24 +35,19 @@ typedef enum {
VIR_PCI_CONNECT_AUTOASSIGN = 1 << 0, /* okay to autoassign a device to this controller */
VIR_PCI_CONNECT_HOTPLUGGABLE = 1 << 1, /* is hotplug needed/supported */
- /* set for devices that can share a single slot in auto-assignment
- * (by assigning one device to each of the 8 functions on the slot)
- */
- VIR_PCI_CONNECT_AGGREGATE_SLOT = 1 << 2,
-
/* kinds of devices as a bitmap so they can be combined (some PCI
* controllers permit connecting multiple types of devices)
*/
- VIR_PCI_CONNECT_TYPE_PCI_DEVICE = 1 << 3,
- VIR_PCI_CONNECT_TYPE_PCIE_DEVICE = 1 << 4,
- VIR_PCI_CONNECT_TYPE_PCIE_ROOT_PORT = 1 << 5,
- VIR_PCI_CONNECT_TYPE_PCIE_SWITCH_UPSTREAM_PORT = 1 << 6,
- VIR_PCI_CONNECT_TYPE_PCIE_SWITCH_DOWNSTREAM_PORT = 1 << 7,
- VIR_PCI_CONNECT_TYPE_DMI_TO_PCI_BRIDGE = 1 << 8,
- VIR_PCI_CONNECT_TYPE_PCI_EXPANDER_BUS = 1 << 9,
- VIR_PCI_CONNECT_TYPE_PCIE_EXPANDER_BUS = 1 << 10,
- VIR_PCI_CONNECT_TYPE_PCI_BRIDGE = 1 << 11,
- VIR_PCI_CONNECT_TYPE_PCIE_TO_PCI_BRIDGE = 1 << 12,
+ VIR_PCI_CONNECT_TYPE_PCI_DEVICE = 1 << 2,
+ VIR_PCI_CONNECT_TYPE_PCIE_DEVICE = 1 << 3,
+ VIR_PCI_CONNECT_TYPE_PCIE_ROOT_PORT = 1 << 4,
+ VIR_PCI_CONNECT_TYPE_PCIE_SWITCH_UPSTREAM_PORT = 1 << 5,
+ VIR_PCI_CONNECT_TYPE_PCIE_SWITCH_DOWNSTREAM_PORT = 1 << 6,
+ VIR_PCI_CONNECT_TYPE_DMI_TO_PCI_BRIDGE = 1 << 7,
+ VIR_PCI_CONNECT_TYPE_PCI_EXPANDER_BUS = 1 << 8,
+ VIR_PCI_CONNECT_TYPE_PCIE_EXPANDER_BUS = 1 << 9,
+ VIR_PCI_CONNECT_TYPE_PCI_BRIDGE = 1 << 10,
+ VIR_PCI_CONNECT_TYPE_PCIE_TO_PCI_BRIDGE = 1 << 11,
} virDomainPCIConnectFlags;
/* a combination of all bits that describe the type of connections
@@ -86,12 +81,12 @@ typedef struct {
*/
uint8_t functions;
- /* aggregate is true if this slot has only devices with
- * VIR_PCI_CONNECT_AGGREGATE assigned to its functions (meaning
- * that other devices with the same flags could also be
- * auto-assigned to the other functions)
+ /* aggregate is greater than zero if this slot has only devices with
+ * VIR_PCI_CONNECT_AGGREGATE assigned to its functions and
+ * that other devices with the same aggregateSlotIdx could also be
+ * auto-assigned to the other functions on this slot)
*/
- bool aggregate;
+ unsigned int aggregateSlotIdx;
} virDomainPCIAddressSlot;
typedef struct {
@@ -170,7 +165,8 @@ int virDomainPCIAddressExtensionReserveNextAddr(virDomainPCIAddressSetPtr addrs,
int virDomainPCIAddressReserveAddr(virDomainPCIAddressSetPtr addrs,
virPCIDeviceAddressPtr addr,
virDomainPCIConnectFlags flags,
- unsigned int isolationGroup)
+ unsigned int isolationGroup,
+ unsigned int agregateSlotIdx)
ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2);
int virDomainPCIAddressReserveNextAddr(virDomainPCIAddressSetPtr addrs,
diff --git a/src/qemu/qemu_domain_address.c b/src/qemu/qemu_domain_address.c
index 07431343ed..f86c85eaa9 100644
--- a/src/qemu/qemu_domain_address.c
+++ b/src/qemu/qemu_domain_address.c
@@ -24,6 +24,7 @@
#include "qemu_domain_address.h"
#include "qemu_domain.h"
#include "viralloc.h"
+#include "virhostdev.h"
#include "virerror.h"
#include "virlog.h"
@@ -1419,6 +1420,54 @@ qemuDomainSetupIsolationGroups(virDomainDefPtr def)
}
+void
+qemuDomainSetDeviceSlotAggregateIdx(virDomainDefPtr def G_GNUC_UNUSED,
+ virDomainDeviceDefPtr dev)
+{
+ virDomainDeviceInfoPtr info = virDomainDeviceGetInfo(dev);
+
+ if (!info)
+ return;
+
+ info->aggregateSlotIdx = 0;
+
+ if (dev->type == VIR_DOMAIN_DEVICE_CONTROLLER) {
+ virDomainControllerDefPtr cont = dev->data.controller;
+ if (cont->type == VIR_DOMAIN_CONTROLLER_TYPE_PCI &&
+ cont->model == VIR_DOMAIN_CONTROLLER_MODEL_PCIE_ROOT_PORT) {
+ info->aggregateSlotIdx = 1;
+ }
+ }
+
+ return;
+}
+
+
+static int
+qemuDomainFillDeviceSlotAggregationIter(virDomainDefPtr def,
+ virDomainDeviceDefPtr dev,
+ virDomainDeviceInfoPtr info G_GNUC_UNUSED,
+ void *opaque G_GNUC_UNUSED)
+{
+ qemuDomainSetDeviceSlotAggregateIdx(def, dev);
+
+ return 0;
+}
+
+
+static int
+qemuDomainSetupSlotAggregation(virDomainDefPtr def)
+{
+ if (virDomainDeviceInfoIterate(def,
+ qemuDomainFillDeviceSlotAggregationIter,
+ NULL) < 0) {
+ return -1;
+ }
+
+ return 0;
+}
+
+
/**
* qemuDomainFillDevicePCIConnectFlags:
*
@@ -1590,7 +1639,8 @@ qemuDomainCollectPCIAddress(virDomainDefPtr def G_GNUC_UNUSED,
if (virDomainPCIAddressReserveAddr(addrs, addr,
info->pciConnectFlags,
- info->isolationGroup) < 0) {
+ info->isolationGroup,
+ info->aggregateSlotIdx) < 0) {
return -1;
}
@@ -1787,7 +1837,7 @@ qemuDomainValidateDevicePCISlotsPIIX3(virDomainDefPtr def,
continue;
}
if (addrs->nbuses &&
- virDomainPCIAddressReserveAddr(addrs, &cont->info.addr.pci, flags, 0) < 0)
+ virDomainPCIAddressReserveAddr(addrs, &cont->info.addr.pci, flags, 0, 0) < 0)
return -1;
}
@@ -1796,11 +1846,11 @@ qemuDomainValidateDevicePCISlotsPIIX3(virDomainDefPtr def,
memset(&tmp_addr, 0, sizeof(tmp_addr));
tmp_addr.slot = 1;
/* ISA Bridge at 00:01.0 */
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
return -1;
/* Bridge at 00:01.3 */
tmp_addr.function = 3;
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
return -1;
}
@@ -1841,7 +1891,7 @@ qemuDomainValidateDevicePCISlotsPIIX3(virDomainDefPtr def,
return -1;
}
} else {
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
return -1;
primaryVideo->info.addr.pci = tmp_addr;
primaryVideo->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI;
@@ -1864,7 +1914,7 @@ qemuDomainValidateDevicePCISlotsPIIX3(virDomainDefPtr def,
VIR_DEBUG("PCI address 0:0:2.0 in use, future addition of a video"
" device will not be possible without manual"
" intervention");
- } else if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0) {
+ } else if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0) {
return -1;
}
}
@@ -1934,7 +1984,7 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def,
assign = true;
}
if (assign) {
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
return -1;
cont->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI;
@@ -1957,7 +2007,7 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def,
memset(&tmp_addr, 0, sizeof(tmp_addr));
tmp_addr.slot = 0x1E;
if (!virDomainPCIAddressSlotInUse(addrs, &tmp_addr)) {
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
return -1;
cont->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI;
@@ -1981,12 +2031,12 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def,
tmp_addr.slot = 0x1F;
tmp_addr.function = 0;
tmp_addr.multi = VIR_TRISTATE_SWITCH_ON;
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
return -1;
tmp_addr.function = 3;
tmp_addr.multi = VIR_TRISTATE_SWITCH_ABSENT;
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
return -1;
}
@@ -2023,7 +2073,7 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def,
return -1;
}
} else {
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
return -1;
primaryVideo->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI;
primaryVideo->info.addr.pci = tmp_addr;
@@ -2049,7 +2099,7 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def,
" device will not be possible without manual"
" intervention");
virResetLastError();
- } else if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0) {
+ } else if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0) {
return -1;
}
}
@@ -2070,7 +2120,7 @@ qemuDomainValidateDevicePCISlotsQ35(virDomainDefPtr def,
!virDeviceInfoPCIAddressIsWanted(&sound->info)) {
continue;
}
- if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0) < 0)
+ if (virDomainPCIAddressReserveAddr(addrs, &tmp_addr, flags, 0, 0) < 0)
return -1;
sound->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI;
@@ -2272,7 +2322,8 @@ qemuDomainAssignDevicePCISlots(virDomainDefPtr def,
/* Reserve this function on the slot we found */
if (virDomainPCIAddressReserveAddr(addrs, &addr,
cont->info.pciConnectFlags,
- cont->info.isolationGroup) < 0) {
+ cont->info.isolationGroup,
+ cont->info.aggregateSlotIdx) < 0) {
return -1;
}
@@ -2646,6 +2697,9 @@ qemuDomainAssignPCIAddresses(virDomainDefPtr def,
if (qemuDomainSetupIsolationGroups(def) < 0)
goto cleanup;
+ if (qemuDomainSetupSlotAggregation(def) < 0)
+ goto cleanup;
+
if (nbuses > 0) {
/* 1st pass to figure out how many PCI bridges we need */
if (!(addrs = qemuDomainPCIAddressSetCreate(def, qemuCaps, nbuses, true)))
@@ -2766,6 +2820,7 @@ qemuDomainAssignPCIAddresses(virDomainDefPtr def,
dev.data.controller = def->controllers[contIndex];
/* set connect flags so it will be properly addressed */
qemuDomainFillDevicePCIConnectFlags(def, &dev, qemuCaps, driver);
+ qemuDomainSetDeviceSlotAggregateIdx(def, &dev);
/* Reserve an address for the controller. pci-root and pcie-root
* controllers don't plug into any other PCI controller, hence
@@ -3239,6 +3294,7 @@ qemuDomainEnsurePCIAddress(virDomainObjPtr obj,
return 0;
qemuDomainFillDevicePCIConnectFlags(obj->def, dev, priv->qemuCaps, driver);
+ qemuDomainSetDeviceSlotAggregateIdx(obj->def, dev);
qemuDomainFillDevicePCIExtensionFlags(dev, info, priv->qemuCaps);
diff --git a/src/qemu/qemu_domain_address.h b/src/qemu/qemu_domain_address.h
index 7ef3308246..198f813595 100644
--- a/src/qemu/qemu_domain_address.h
+++ b/src/qemu/qemu_domain_address.h
@@ -53,6 +53,15 @@ void qemuDomainFillDeviceIsolationGroup(virDomainDefPtr def,
virDomainDeviceDefPtr dev)
ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2);
+void
+qemuDomainSetDeviceSlotAggregateIdx(virDomainDefPtr def,
+ virDomainDeviceDefPtr dev);
+
+int
+qemuDomainDefDeviceFindSlotAggregateIdx(virDomainDefPtr def,
+ virDomainDeviceDefPtr dev);
+
+
void qemuDomainReleaseDeviceAddress(virDomainObjPtr vm,
virDomainDeviceInfoPtr info);
--
2.26.2
More information about the libvir-list
mailing list