[PATCH RFC 20/21] conf: Add possibility to configure multiple iothreads per disk
Peter Krempa
pkrempa at redhat.com
Thu Mar 2 14:38:02 UTC 2023
Introduce a new <iothreads> sub-element of disk's <driver> which will
allow configuring multiple iothreads and also map them to specific
virt-queues of virtio devices.
Signed-off-by: Peter Krempa <pkrempa at redhat.com>
---
docs/formatdomain.rst | 23 +++++-
src/conf/domain_conf.c | 76 +++++++++++++++++++
src/conf/domain_conf.h | 14 ++++
src/conf/domain_validate.c | 8 ++
src/conf/schemas/domaincommon.rng | 47 +++++++++---
.../iothreads-disk.x86_64-latest.args | 13 +++-
tests/qemuxml2argvdata/iothreads-disk.xml | 25 +++++-
.../iothreads-disk.x86_64-latest.xml | 26 ++++++-
8 files changed, 211 insertions(+), 21 deletions(-)
diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
index d831c1e527..4940f3d857 100644
--- a/docs/formatdomain.rst
+++ b/docs/formatdomain.rst
@@ -3233,7 +3233,28 @@ paravirtualized driver is specified via the ``disk`` element.
assigned to the same IOThread and are numbered from 1 to the domain
iothreads value. Available for a disk device ``target`` configured to use
"virtio" ``bus`` and "pci" or "ccw" ``address`` types. :since:`Since 1.2.8
- (QEMU 2.1)`
+ (QEMU 2.1)` *Note:* ``iothread`` is mutually exclusive with ``iothreads``.
+ - The optional ``iothreads`` sub-element allows specifying multiple IOThreads
+ via the ``iothread`` sub-element with attribute ``id`` the disk will use
+ for I/O operations. Optionally the ``iothread`` element can have multiple
+ ``queue`` subelements specifying that given iothread should be used to
+ handle given queues. :since:`Since XXXXXX`.
+ Example::
+
+ <driver name='qemu' queues='2'>
+ <iothreads>
+ <iothread id='1'>
+ <queue id='1'/>
+ </iothread>
+ <iothread id='2'>
+ <queue id='1'/>
+ </iothread>
+ <iothread id='3'>
+ <queue id='2'/>
+ </iothread>
+ </iothreads>
+ </driver>
+
- The optional ``queues`` attribute specifies the number of virt queues for
virtio-blk. ( :since:`Since 3.9.0` )
- The optional ``queue_size`` attribute specifies the size of each virt
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 9426d55f8d..6904a71c80 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -2333,6 +2333,17 @@ virDomainDefGetVcpusTopology(const virDomainDef *def,
}
+void
+virDomainDiskIothreadDefFree(virDomainDiskIothreadDef *def)
+{
+ if (!def)
+ return;
+
+ g_free(def->queues);
+ g_free(def);
+}
+
+
static virDomainDiskDef *
virDomainDiskDefNewSource(virDomainXMLOption *xmlopt,
virStorageSource **src)
@@ -2381,6 +2392,7 @@ virDomainDiskDefFree(virDomainDiskDef *def)
g_free(def->virtio);
virDomainDeviceInfoClear(&def->info);
virObjectUnref(def->privateData);
+ g_slist_free_full(def->iothreads, (GDestroyNotify) virDomainDiskIothreadDefFree);
g_free(def);
}
@@ -7745,6 +7757,8 @@ static int
virDomainDiskDefDriverParseXML(virDomainDiskDef *def,
xmlNodePtr cur)
{
+ xmlNodePtr iothreadsNode;
+
def->driverName = virXMLPropString(cur, "name");
if (virXMLPropEnum(cur, "cache", virDomainDiskCacheTypeFromString,
@@ -7791,6 +7805,44 @@ virDomainDiskDefDriverParseXML(virDomainDiskDef *def,
if (virXMLPropUInt(cur, "iothread", 10, VIR_XML_PROP_NONZERO, &def->iothread) < 0)
return -1;
+ if ((iothreadsNode = virXMLNodeGetSubelement(cur, "iothreads"))) {
+ g_autoslist(virDomainDiskIothreadDef) ioth = NULL;
+ g_autoptr(GPtrArray) iothreadNodes = NULL;
+
+ if ((iothreadNodes = virXMLNodeGetSubelementList(iothreadsNode, "iothread"))) {
+ size_t i;
+
+ for (i = 0; i < iothreadNodes->len; i++) {
+ xmlNodePtr iothNode = g_ptr_array_index(iothreadNodes, i);
+ g_autoptr(virDomainDiskIothreadDef) iothdef = g_new0(virDomainDiskIothreadDef, 1);
+ g_autoptr(GPtrArray) queueNodes = NULL;
+
+ if (virXMLPropUInt(iothNode, "id", 10, VIR_XML_PROP_REQUIRED,
+ &iothdef->id) < 0)
+ return -1;
+
+ if ((queueNodes = virXMLNodeGetSubelementList(iothNode, "queue"))) {
+ size_t q;
+
+ iothdef->queues = g_new0(unsigned int, queueNodes->len);
+ iothdef->nqueues = queueNodes->len;
+
+ for (q = 0; q < queueNodes->len; q++) {
+ xmlNodePtr queueNode = g_ptr_array_index(queueNodes, q);
+
+ if (virXMLPropUInt(queueNode, "id", 10, VIR_XML_PROP_REQUIRED,
+ &(iothdef->queues[q])) < 0)
+ return -1;
+ }
+ }
+
+ ioth = g_slist_prepend(ioth, g_steal_pointer(&iothdef));
+ }
+
+ def->iothreads = g_slist_reverse(g_steal_pointer(&ioth));
+ }
+ }
+
if (virXMLPropEnum(cur, "detect_zeroes",
virDomainDiskDetectZeroesTypeFromString,
VIR_XML_PROP_NONZERO, &def->detect_zeroes) < 0)
@@ -22513,6 +22565,30 @@ virDomainDiskDefFormatDriver(virBuffer *buf,
virXMLFormatElement(&childBuf, "metadata_cache", NULL, &metadataCacheChildBuf);
}
+ if (disk->iothreads) {
+ g_auto(virBuffer) iothreadsChildBuf = VIR_BUFFER_INIT_CHILD(&childBuf);
+ GSList *n;
+
+ for (n = disk->iothreads; n; n = n->next) {
+ virDomainDiskIothreadDef *iothDef = n->data;
+ g_auto(virBuffer) iothreadAttrBuf = VIR_BUFFER_INITIALIZER;
+ g_auto(virBuffer) iothreadChildBuf = VIR_BUFFER_INIT_CHILD(&iothreadsChildBuf);
+
+ virBufferAsprintf(&iothreadAttrBuf, " id='%u'", iothDef->id);
+
+ if (iothDef->queues) {
+ size_t q;
+
+ for (q = 0; q < iothDef->nqueues; q++)
+ virBufferAsprintf(&iothreadChildBuf, "<queue id='%u'/>\n", iothDef->queues[q]);
+ }
+
+ virXMLFormatElement(&iothreadsChildBuf, "iothread", &iothreadAttrBuf, &iothreadChildBuf);
+ }
+
+ virXMLFormatElement(&childBuf, "iothreads", NULL, &iothreadsChildBuf);
+ }
+
virXMLFormatElement(buf, "driver", &attrBuf, &childBuf);
}
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index f6dade62fc..708993174a 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -552,6 +552,19 @@ typedef enum {
VIR_ENUM_DECL(virDomainSnapshotLocation);
+struct _virDomainDiskIothreadDef {
+ unsigned int id;
+
+ /* optional list of virtqueues the iothread should handle */
+ unsigned int *queues;
+ size_t nqueues;
+};
+
+typedef struct _virDomainDiskIothreadDef virDomainDiskIothreadDef;
+void virDomainDiskIothreadDefFree(virDomainDiskIothreadDef *def);
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(virDomainDiskIothreadDef, virDomainDiskIothreadDefFree);
+
+
/* Stores the virtual disk configuration */
struct _virDomainDiskDef {
virStorageSource *src; /* non-NULL. XXX Allow NULL for empty cdrom? */
@@ -605,6 +618,7 @@ struct _virDomainDiskDef {
virDomainDeviceSGIO sgio;
virDomainDiskDiscard discard;
unsigned int iothread; /* unused = 0, > 0 specific thread # */
+ GSList *iothreads; /* List of virDomainDiskIothreadsDef */
virDomainDiskDetectZeroes detect_zeroes;
char *domain_name; /* backend domain name */
unsigned int queues;
diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
index 5559a71e14..2aba645279 100644
--- a/src/conf/domain_validate.c
+++ b/src/conf/domain_validate.c
@@ -914,6 +914,14 @@ virDomainDiskDefValidate(const virDomainDef *def,
return -1;
}
+ /* configuring both <driver iothread='n'> and it's <iothreads> sub-element
+ * isn't supported */
+ if (disk->iothread && disk->iothreads) {
+ virReportError(VIR_ERR_XML_ERROR, "%s",
+ _("disk driver 'iothread' attribute can't be used together with 'iothreads' subelement"));
+ return -1;
+ }
+
return 0;
}
diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
index 5a1d79672f..3e54091da2 100644
--- a/src/conf/schemas/domaincommon.rng
+++ b/src/conf/schemas/domaincommon.rng
@@ -2467,9 +2467,26 @@
</optional>
</element>
</define>
- <!--
- Disk may use a special driver for access.
- -->
+
+ <define name="diskDriverIothreads">
+ <element name="iothreads">
+ <oneOrMore>
+ <element name="iothread">
+ <attribute name="id">
+ <ref name="unsignedInt"/>
+ </attribute>
+ <zeroOrMore>
+ <element name="queue">
+ <attribute name="id">
+ <ref name="unsignedInt"/>
+ </attribute>
+ </element>
+ </zeroOrMore>
+ </element>
+ </oneOrMore>
+ </element>
+ </define>
+
<define name="diskDriver">
<element name="driver">
<optional>
@@ -2516,17 +2533,23 @@
</attribute>
</optional>
<ref name="virtioOptions"/>
- <optional>
- <element name="metadata_cache">
- <optional>
- <element name="max_size">
- <ref name="scaledInteger"/>
- </element>
- </optional>
- </element>
- </optional>
+ <interleave>
+ <optional>
+ <element name="metadata_cache">
+ <optional>
+ <element name="max_size">
+ <ref name="scaledInteger"/>
+ </element>
+ </optional>
+ </element>
+ </optional>
+ <optional>
+ <ref name="diskDriverIothreads"/>
+ </optional>
+ </interleave>
</element>
</define>
+
<define name="driverFormat">
<optional>
<attribute name="name">
diff --git a/tests/qemuxml2argvdata/iothreads-disk.x86_64-latest.args b/tests/qemuxml2argvdata/iothreads-disk.x86_64-latest.args
index 02fb74d945..d1953327a7 100644
--- a/tests/qemuxml2argvdata/iothreads-disk.x86_64-latest.args
+++ b/tests/qemuxml2argvdata/iothreads-disk.x86_64-latest.args
@@ -19,6 +19,8 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
-smp 2,sockets=2,cores=1,threads=1 \
-object '{"qom-type":"iothread","id":"iothread1"}' \
-object '{"qom-type":"iothread","id":"iothread2"}' \
+-object '{"qom-type":"iothread","id":"iothread3"}' \
+-object '{"qom-type":"iothread","id":"iothread4"}' \
-uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
-display none \
-no-user-config \
@@ -30,12 +32,15 @@ XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
-no-acpi \
-boot strict=on \
-device '{"driver":"piix3-usb-uhci","id":"usb","bus":"pci.0","addr":"0x1.0x2"}' \
--blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/iothrtest1.img","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"}' \
+-blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/iothrtest1.img","node-name":"libvirt-3-storage","auto-read-only":true,"discard":"unmap"}' \
+-blockdev '{"node-name":"libvirt-3-format","read-only":false,"driver":"raw","file":"libvirt-3-storage"}' \
+-device '{"driver":"virtio-blk-pci","iothread":"iothread1","bus":"pci.0","addr":"0x4","drive":"libvirt-3-format","id":"virtio-disk1","bootindex":1}' \
+-blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/iothrtest2.img","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-2-format","read-only":false,"driver":"raw","file":"libvirt-2-storage"}' \
--device '{"driver":"virtio-blk-pci","iothread":"iothread1","bus":"pci.0","addr":"0x4","drive":"libvirt-2-format","id":"virtio-disk1","bootindex":1}' \
--blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/iothrtest2.img","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
+-device '{"driver":"virtio-blk-pci","num-queues":4,"bus":"pci.0","addr":"0x2","drive":"libvirt-2-format","id":"virtio-disk2"}' \
+-blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/iothrtest3.img","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"raw","file":"libvirt-1-storage"}' \
--device '{"driver":"virtio-blk-pci","iothread":"iothread2","bus":"pci.0","addr":"0x2","drive":"libvirt-1-format","id":"virtio-disk2"}' \
+-device '{"driver":"virtio-blk-pci","bus":"pci.0","addr":"0x3","drive":"libvirt-1-format","id":"virtio-disk3"}' \
-audiodev '{"id":"audio1","driver":"none"}' \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
diff --git a/tests/qemuxml2argvdata/iothreads-disk.xml b/tests/qemuxml2argvdata/iothreads-disk.xml
index ad0731c79c..7ce25f559e 100644
--- a/tests/qemuxml2argvdata/iothreads-disk.xml
+++ b/tests/qemuxml2argvdata/iothreads-disk.xml
@@ -4,7 +4,7 @@
<memory unit='KiB'>219136</memory>
<currentMemory unit='KiB'>219136</currentMemory>
<vcpu placement='static'>2</vcpu>
- <iothreads>2</iothreads>
+ <iothreads>4</iothreads>
<os>
<type arch='x86_64' machine='pc'>hvm</type>
<boot dev='hd'/>
@@ -22,10 +22,31 @@
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
</disk>
<disk type='file' device='disk'>
- <driver name='qemu' type='raw' iothread='2'/>
+ <driver name='qemu' type='raw' queues='4'>
+ <iothreads>
+ <iothread id='2'>
+ <queue id='1'/>
+ <queue id='3'/>
+ </iothread>
+ <iothread id='3'>
+ <queue id='0'/>
+ <queue id='2'/>
+ </iothread>
+ </iothreads>
+ </driver>
<source file='/var/lib/libvirt/images/iothrtest2.img'/>
<target dev='vdc' bus='virtio'/>
</disk>
+ <disk type='file' device='disk'>
+ <driver name='qemu' type='raw'>
+ <iothreads>
+ <iothread id='4'/>
+ <iothread id='1'/>
+ </iothreads>
+ </driver>
+ <source file='/var/lib/libvirt/images/iothrtest3.img'/>
+ <target dev='vdd' bus='virtio'/>
+ </disk>
<controller type='usb' index='0'/>
<controller type='ide' index='0'/>
<controller type='pci' index='0' model='pci-root'/>
diff --git a/tests/qemuxml2xmloutdata/iothreads-disk.x86_64-latest.xml b/tests/qemuxml2xmloutdata/iothreads-disk.x86_64-latest.xml
index ae1da9ec2a..94864feb85 100644
--- a/tests/qemuxml2xmloutdata/iothreads-disk.x86_64-latest.xml
+++ b/tests/qemuxml2xmloutdata/iothreads-disk.x86_64-latest.xml
@@ -4,7 +4,7 @@
<memory unit='KiB'>219136</memory>
<currentMemory unit='KiB'>219136</currentMemory>
<vcpu placement='static'>2</vcpu>
- <iothreads>2</iothreads>
+ <iothreads>4</iothreads>
<os>
<type arch='x86_64' machine='pc'>hvm</type>
<boot dev='hd'/>
@@ -25,11 +25,33 @@
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
</disk>
<disk type='file' device='disk'>
- <driver name='qemu' type='raw' iothread='2'/>
+ <driver name='qemu' type='raw' queues='4'>
+ <iothreads>
+ <iothread id='2'>
+ <queue id='1'/>
+ <queue id='3'/>
+ </iothread>
+ <iothread id='3'>
+ <queue id='0'/>
+ <queue id='2'/>
+ </iothread>
+ </iothreads>
+ </driver>
<source file='/var/lib/libvirt/images/iothrtest2.img'/>
<target dev='vdc' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
</disk>
+ <disk type='file' device='disk'>
+ <driver name='qemu' type='raw'>
+ <iothreads>
+ <iothread id='4'/>
+ <iothread id='1'/>
+ </iothreads>
+ </driver>
+ <source file='/var/lib/libvirt/images/iothrtest3.img'/>
+ <target dev='vdd' bus='virtio'/>
+ <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+ </disk>
<controller type='usb' index='0' model='piix3-uhci'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
</controller>
--
2.39.2
More information about the libvir-list
mailing list