[PATCH 12/13] qemu: Build HMAT command line
Michal Privoznik
mprivozn at redhat.com
Wed Jun 24 13:49:10 UTC 2020
Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1786303
Signed-off-by: Michal Privoznik <mprivozn at redhat.com>
---
src/conf/numa_conf.c | 7 +
src/qemu/qemu_command.c | 183 ++++++++++++++++++
.../numatune-hmat.x86_64-latest.args | 53 +++++
tests/qemuxml2argvtest.c | 1 +
4 files changed, 244 insertions(+)
create mode 100644 tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args
diff --git a/src/conf/numa_conf.c b/src/conf/numa_conf.c
index b35e5040fc..7fbbe0f52e 100644
--- a/src/conf/numa_conf.c
+++ b/src/conf/numa_conf.c
@@ -1904,6 +1904,13 @@ virDomainNumaGetNodeInitiator(const virDomainNuma *numa,
if (!numa || node >= numa->nmem_nodes)
return -1;
+ /* A NUMA node which has at least one vCPU is initiator to itself by
+ * definition. */
+ if (numa->mem_nodes[node].cpumask)
+ return node;
+
+ /* For the rest, "NUMA node that has best performance (the lowest
+ * latency or largest bandwidth) to this NUMA node." */
for (i = 0; i < numa->nlatencies; i++) {
const virDomainNumaLatency *l = &numa->latencies[i];
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index 628ff970bd..b28c43f110 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -6932,6 +6932,16 @@ qemuBuildMachineCommandLine(virCommandPtr cmd,
virBufferAsprintf(&buf, ",pflash1=%s", priv->pflash1->nodeformat);
}
+ if (virDomainNumaHasHMAT(def->numa)) {
+ if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA_HMAT)) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("HMAT is not supported with this QEMU"));
+ return -1;
+ }
+
+ virBufferAddLit(&buf, ",hmat=on");
+ }
+
virCommandAddArgBuffer(cmd, &buf);
return 0;
@@ -7114,6 +7124,134 @@ qemuBuildIOThreadCommandLine(virCommandPtr cmd,
}
+static int
+qemuBuilNumaCellCache(virCommandPtr cmd,
+ const virDomainDef *def,
+ size_t cell)
+{
+ size_t ncaches = virDomainNumaGetNodeCacheCount(def->numa, cell);
+ size_t i;
+
+ if (ncaches == 0)
+ return 0;
+
+ for (i = 0; i < ncaches; i++) {
+ g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
+ unsigned int level;
+ unsigned int size;
+ unsigned int line;
+ virDomainCacheAssociativity associativity;
+ virDomainCachePolicy policy;
+
+ if (virDomainNumaGetNodeCache(def->numa, cell, i,
+ &level, &size, &line,
+ &associativity, &policy) < 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Unable to format NUMA node cache"));
+ return -1;
+ }
+
+ virBufferAsprintf(&buf,
+ "hmat-cache,node-id=%zu,size=%uK,level=%u",
+ cell, size, level);
+
+ switch (associativity) {
+ case VIR_DOMAIN_CACHE_ASSOCIATIVITY_NONE:
+ virBufferAddLit(&buf, ",associativity=none");
+ break;
+ case VIR_DOMAIN_CACHE_ASSOCIATIVITY_DIRECT:
+ virBufferAddLit(&buf, ",associativity=direct");
+ break;
+ case VIR_DOMAIN_CACHE_ASSOCIATIVITY_FULL:
+ virBufferAddLit(&buf, ",associativity=complex");
+ break;
+ case VIR_DOMAIN_CACHE_ASSOCIATIVITY_LAST:
+ break;
+ }
+
+ switch (policy) {
+ case VIR_DOMAIN_CACHE_POLICY_NONE:
+ virBufferAddLit(&buf, ",policy=none");
+ break;
+ case VIR_DOMAIN_CACHE_POLICY_WRITEBACK:
+ virBufferAddLit(&buf, ",policy=write-back");
+ break;
+ case VIR_DOMAIN_CACHE_POLICY_WRITETHROUGH:
+ virBufferAddLit(&buf, ",policy=write-through");
+ break;
+ case VIR_DOMAIN_CACHE_POLICY_LAST:
+ break;
+ }
+
+ if (line > 0)
+ virBufferAsprintf(&buf, ",line=%u", line);
+
+ virCommandAddArg(cmd, "-numa");
+ virCommandAddArgBuffer(cmd, &buf);
+ }
+
+ return 0;
+}
+
+
+VIR_ENUM_DECL(qemuDomainMemoryHierarchy);
+VIR_ENUM_IMPL(qemuDomainMemoryHierarchy,
+ 4, /* Maximum level of cache */
+ "memory", /* Special case, whole memory not specific cache */
+ "first-level",
+ "second-level",
+ "third-level");
+
+static int
+qemuBuildNumaHMATCommandLine(virCommandPtr cmd,
+ const virDomainDef *def)
+{
+ size_t nlatencies;
+ size_t i;
+
+ if (!def->numa)
+ return 0;
+
+ nlatencies = virDomainNumaGetLatenciesCount(def->numa);
+ for (i = 0; i < nlatencies; i++) {
+ g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
+ virDomainNumaLatencyType type;
+ unsigned int initiator;
+ unsigned int target;
+ unsigned int cache;
+ virDomainMemoryLatency accessType;
+ unsigned long value;
+ const char *hierarchyStr;
+ const char *accessStr;
+
+ if (virDomainNumaGetLatency(def->numa, i,
+ &type, &initiator, &target,
+ &cache, &accessType, &value) < 0)
+ return -1;
+
+ hierarchyStr = qemuDomainMemoryHierarchyTypeToString(cache);
+ accessStr = virDomainMemoryLatencyTypeToString(accessType);
+ virBufferAsprintf(&buf,
+ "hmat-lb,initiator=%u,target=%u,hierarchy=%s,data-type=%s-",
+ initiator, target, hierarchyStr, accessStr);
+
+ switch (type) {
+ case VIR_DOMAIN_NUMA_LATENCY_TYPE_LATENCY:
+ virBufferAsprintf(&buf, "latency,latency=%lu", value);
+ break;
+ case VIR_DOMAIN_NUMA_LATENCY_TYPE_BANDWIDTH:
+ virBufferAsprintf(&buf, "bandwidth,bandwidth=%luK", value);
+ break;
+ }
+
+ virCommandAddArg(cmd, "-numa");
+ virCommandAddArgBuffer(cmd, &buf);
+ }
+
+ return 0;
+}
+
+
static int
qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
virDomainDefPtr def,
@@ -7126,9 +7264,11 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
char *next = NULL;
virBufferPtr nodeBackends = NULL;
bool needBackend = false;
+ bool hmat = false;
int rc;
int ret = -1;
size_t ncells = virDomainNumaGetNodeCount(def->numa);
+ ssize_t masterInitiator = -1;
if (!virDomainNumatuneNodesetIsAvailable(def->numa, priv->autoNodeset))
goto cleanup;
@@ -7138,6 +7278,16 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
def->os.machine))
needBackend = true;
+ if (virDomainNumaHasHMAT(def->numa)) {
+ if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA_HMAT)) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("HMAT is not supported with this QEMU"));
+ goto cleanup;
+ }
+ needBackend = true;
+ hmat = true;
+ }
+
if (VIR_ALLOC_N(nodeBackends, ncells) < 0)
goto cleanup;
@@ -7166,8 +7316,22 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
qemuBuildMemPathStr(def, cmd, priv) < 0)
goto cleanup;
+ for (i = 0; i < ncells; i++) {
+ if (virDomainNumaGetNodeCpumask(def->numa, i)) {
+ masterInitiator = i;
+ break;
+ }
+ }
+
+ if (masterInitiator) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("At least one NUMA node has to have CPUs"));
+ goto cleanup;
+ }
+
for (i = 0; i < ncells; i++) {
virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def->numa, i);
+ ssize_t initiator = virDomainNumaGetNodeInitiator(def->numa, i);
if (needBackend) {
virCommandAddArg(cmd, "-object");
@@ -7192,6 +7356,13 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
}
}
+ if (hmat) {
+ if (initiator < 0)
+ initiator = masterInitiator;
+
+ virBufferAsprintf(&buf, ",initiator=%zd", initiator);
+ }
+
if (needBackend)
virBufferAsprintf(&buf, ",memdev=ram-node%zu", i);
else
@@ -7217,6 +7388,18 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
}
}
+ if (hmat) {
+ if (qemuBuildNumaHMATCommandLine(cmd, def) < 0)
+ goto cleanup;
+
+ /* This can't be moved into any of the loops above,
+ * because hmat-cache can be specified only after hmat-lb. */
+ for (i = 0; i < ncells; i++) {
+ if (qemuBuilNumaCellCache(cmd, def, i) < 0)
+ goto cleanup;
+ }
+ }
+
ret = 0;
cleanup:
diff --git a/tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args b/tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args
new file mode 100644
index 0000000000..c52015caa8
--- /dev/null
+++ b/tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args
@@ -0,0 +1,53 @@
+LC_ALL=C \
+PATH=/bin \
+HOME=/tmp/lib/domain--1-QEMUGuest \
+USER=test \
+LOGNAME=test \
+XDG_DATA_HOME=/tmp/lib/domain--1-QEMUGuest/.local/share \
+XDG_CACHE_HOME=/tmp/lib/domain--1-QEMUGuest/.cache \
+XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest/.config \
+QEMU_AUDIO_DRV=none \
+/usr/bin/qemu-system-x86_64 \
+-name guest=QEMUGuest,debug-threads=on \
+-S \
+-object secret,id=masterKey0,format=raw,\
+file=/tmp/lib/domain--1-QEMUGuest/master-key.aes \
+-machine pc,accel=tcg,usb=off,dump-guest-core=off,hmat=on \
+-cpu qemu64 \
+-m 12288 \
+-overcommit mem-lock=off \
+-smp 12,sockets=12,cores=1,threads=1 \
+-object memory-backend-ram,id=ram-node0,size=2147483648 \
+-numa node,nodeid=0,cpus=0-3,initiator=0,memdev=ram-node0 \
+-object memory-backend-ram,id=ram-node1,size=2147483648 \
+-numa node,nodeid=1,cpus=4-7,initiator=1,memdev=ram-node1 \
+-object memory-backend-ram,id=ram-node2,size=2147483648 \
+-numa node,nodeid=2,cpus=8-11,initiator=2,memdev=ram-node2 \
+-object memory-backend-ram,id=ram-node3,size=2147483648 \
+-numa node,nodeid=3,initiator=0,memdev=ram-node3 \
+-object memory-backend-ram,id=ram-node4,size=2147483648 \
+-numa node,nodeid=4,initiator=0,memdev=ram-node4 \
+-object memory-backend-ram,id=ram-node5,size=2147483648 \
+-numa node,nodeid=5,initiator=0,memdev=ram-node5 \
+-numa hmat-lb,initiator=0,target=0,hierarchy=memory,data-type=access-latency,\
+latency=5 \
+-numa hmat-lb,initiator=0,target=0,hierarchy=first-level,\
+data-type=access-latency,latency=10 \
+-numa hmat-lb,initiator=0,target=0,hierarchy=memory,data-type=access-bandwidth,\
+bandwidth=204800K \
+-numa hmat-cache,node-id=0,size=10K,level=1,associativity=direct,\
+policy=write-back,line=8 \
+-uuid c7a5fdb2-cdaf-9455-926a-d65c16db1809 \
+-display none \
+-no-user-config \
+-nodefaults \
+-chardev socket,id=charmonitor,fd=1729,server,nowait \
+-mon chardev=charmonitor,id=monitor,mode=control \
+-rtc base=utc \
+-no-shutdown \
+-boot strict=on \
+-device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 \
+-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3 \
+-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,\
+resourcecontrol=deny \
+-msg timestamp=on
diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c
index 98d9a7b5b2..3fdcd41317 100644
--- a/tests/qemuxml2argvtest.c
+++ b/tests/qemuxml2argvtest.c
@@ -1913,6 +1913,7 @@ mymain(void)
DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST);
DO_TEST("numatune-no-vcpu", NONE);
+ DO_TEST_CAPS_LATEST("numatune-hmat");
DO_TEST("numatune-auto-nodeset-invalid", NONE);
DO_TEST("numatune-auto-prefer", QEMU_CAPS_OBJECT_MEMORY_RAM,
--
2.26.2
More information about the libvir-list
mailing list