[Libguestfs] [PATCH nbdkit v5 FINAL 14/19] data, memory: Implement extents.
Richard W.M. Jones
rjones at redhat.com
Thu Mar 28 16:18:41 UTC 2019
These plugins are both based on the same sparse array structure which
supports a simple implementation of extents.
---
common/sparse/sparse.h | 7 +-
common/sparse/sparse.c | 37 ++++++++++-
plugins/data/data.c | 16 ++++-
plugins/memory/memory.c | 16 ++++-
README | 2 +
tests/Makefile.am | 2 +
tests/test-data-extents.sh | 131 +++++++++++++++++++++++++++++++++++++
7 files changed, 207 insertions(+), 4 deletions(-)
diff --git a/common/sparse/sparse.h b/common/sparse/sparse.h
index 818d804..eb24a0b 100644
--- a/common/sparse/sparse.h
+++ b/common/sparse/sparse.h
@@ -1,5 +1,5 @@
/* nbdkit
- * Copyright (C) 2017-2018 Red Hat Inc.
+ * Copyright (C) 2017-2019 Red Hat Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -87,4 +87,9 @@ extern void sparse_array_zero (struct sparse_array *sa,
uint32_t count, uint64_t offset)
__attribute__((__nonnull__ (1)));
+/* Return information about allocated pages and holes. */
+extern int sparse_array_extents (struct sparse_array *sa,
+ uint32_t count, uint64_t offset,
+ struct nbdkit_extents *extents);
+
#endif /* NBDKIT_SPARSE_H */
diff --git a/common/sparse/sparse.c b/common/sparse/sparse.c
index a5ace48..d5654c2 100644
--- a/common/sparse/sparse.c
+++ b/common/sparse/sparse.c
@@ -1,5 +1,5 @@
/* nbdkit
- * Copyright (C) 2017-2018 Red Hat Inc.
+ * Copyright (C) 2017-2019 Red Hat Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -360,3 +360,38 @@ sparse_array_zero (struct sparse_array *sa, uint32_t count, uint64_t offset)
offset += n;
}
}
+
+int
+sparse_array_extents (struct sparse_array *sa,
+ uint32_t count, uint64_t offset,
+ struct nbdkit_extents *extents)
+{
+ uint32_t n, type;
+ void *p;
+
+ while (count > 0) {
+ p = lookup (sa, offset, false, &n, NULL);
+ if (n > count)
+ n = count;
+
+ /* Work out the type of this extent. */
+ if (p == NULL)
+ /* No backing page, so it's a hole. */
+ type = NBDKIT_EXTENT_HOLE | NBDKIT_EXTENT_ZERO;
+ else {
+ if (is_zero (p, n))
+ /* A backing page and it's all zero, it's a zero extent. */
+ type = NBDKIT_EXTENT_ZERO;
+ else
+ /* Normal allocated data. */
+ type = 0;
+ }
+ if (nbdkit_add_extent (extents, offset, n, type) == -1)
+ return -1;
+
+ count -= n;
+ offset += n;
+ }
+
+ return 0;
+}
diff --git a/plugins/data/data.c b/plugins/data/data.c
index f9d3881..c5540f6 100644
--- a/plugins/data/data.c
+++ b/plugins/data/data.c
@@ -1,5 +1,5 @@
/* nbdkit
- * Copyright (C) 2018 Red Hat Inc.
+ * Copyright (C) 2018-2019 Red Hat Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -378,6 +378,19 @@ data_trim (void *handle, uint32_t count, uint64_t offset)
return 0;
}
+/* Extents. */
+static int
+data_extents (void *handle, uint32_t count, uint64_t offset,
+ uint32_t flags, struct nbdkit_extents *extents)
+{
+ int r;
+
+ pthread_mutex_lock (&lock);
+ r = sparse_array_extents (sa, count, offset, extents);
+ pthread_mutex_unlock (&lock);
+ return r;
+}
+
static struct nbdkit_plugin plugin = {
.name = "data",
.version = PACKAGE_VERSION,
@@ -394,6 +407,7 @@ static struct nbdkit_plugin plugin = {
.pwrite = data_pwrite,
.zero = data_zero,
.trim = data_trim,
+ .extents = data_extents,
/* In this plugin, errno is preserved properly along error return
* paths from failed system calls.
*/
diff --git a/plugins/memory/memory.c b/plugins/memory/memory.c
index e27e127..741eaad 100644
--- a/plugins/memory/memory.c
+++ b/plugins/memory/memory.c
@@ -1,5 +1,5 @@
/* nbdkit
- * Copyright (C) 2017-2018 Red Hat Inc.
+ * Copyright (C) 2017-2019 Red Hat Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -170,6 +170,19 @@ memory_trim (void *handle, uint32_t count, uint64_t offset)
return 0;
}
+/* Extents. */
+static int
+memory_extents (void *handle, uint32_t count, uint64_t offset,
+ uint32_t flags, struct nbdkit_extents *extents)
+{
+ int r;
+
+ pthread_mutex_lock (&lock);
+ r = sparse_array_extents (sa, count, offset, extents);
+ pthread_mutex_unlock (&lock);
+ return r;
+}
+
static struct nbdkit_plugin plugin = {
.name = "memory",
.version = PACKAGE_VERSION,
@@ -185,6 +198,7 @@ static struct nbdkit_plugin plugin = {
.pwrite = memory_pwrite,
.zero = memory_zero,
.trim = memory_trim,
+ .extents = memory_extents,
/* In this plugin, errno is preserved properly along error return
* paths from failed system calls.
*/
diff --git a/README b/README
index 75230c9..497f4fd 100644
--- a/README
+++ b/README
@@ -152,6 +152,8 @@ To test for memory leaks (‘make check-valgrind’):
For non-essential enhancements to the test suite:
+ - jq
+
- ip, ss (from iproute package)
- losetup (from util-linux package)
diff --git a/tests/Makefile.am b/tests/Makefile.am
index dec44b5..174da29 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -53,6 +53,7 @@ EXTRA_DIST = \
test-cxx.sh \
test-data-7E.sh \
test-data-base64.sh \
+ test-data-extents.sh \
test-data-file.sh \
test-data-raw.sh \
test-debug-flags.sh \
@@ -372,6 +373,7 @@ LIBGUESTFS_TESTS += test-data
TESTS += \
test-data-7E.sh \
test-data-base64.sh \
+ test-data-extents.sh \
test-data-file.sh \
test-data-raw.sh
diff --git a/tests/test-data-extents.sh b/tests/test-data-extents.sh
new file mode 100755
index 0000000..2b25bde
--- /dev/null
+++ b/tests/test-data-extents.sh
@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2018-2019 Red Hat Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+# Unfortunately the output of this test depends on the PAGE_SIZE
+# defined in common/sparse/sparse.c and would change (breaking the
+# test) if we ever changed that definition.
+
+source ./functions.sh
+set -e
+set -x
+
+requires jq --version
+requires qemu-img --version
+requires qemu-img map --help
+
+out="test-data-extents.out"
+expected="test-data-extents.expected"
+files="$out $expected"
+rm -f $files
+cleanup_fn rm -f $files
+
+do_test ()
+{
+ # We use jq to normalize the output and convert it to plain text.
+ nbdkit -U - data data="$1" size="$2" \
+ --run 'qemu-img map -f raw --output=json $nbd' |
+ jq -c '.[] | {start:.start, length:.length, data:.data, zero:.zero}' \
+ > $out
+ if ! cmp $out $expected; then
+ echo "$0: output did not match expected data"
+ echo "expected:"
+ cat $expected
+ echo "output:"
+ cat $out
+ exit 1
+ fi
+}
+
+# Completely sparse disk.
+cat > $expected <<'EOF'
+{"start":0,"length":1048576,"data":false,"zero":true}
+EOF
+do_test "" 1M
+
+# Completely allocated disk.
+cat > $expected <<'EOF'
+{"start":0,"length":32768,"data":true,"zero":false}
+EOF
+do_test "1" 32K
+
+# Allocated data at the start of a 1M disk.
+cat > $expected <<'EOF'
+{"start":0,"length":32768,"data":true,"zero":false}
+{"start":32768,"length":1015808,"data":false,"zero":true}
+EOF
+do_test "1" 1M
+
+# Allocated zeroes at the start of the disk. This should create a
+# zero extent (data=true zero=true).
+cat > $expected <<'EOF'
+{"start":0,"length":32768,"data":true,"zero":true}
+{"start":32768,"length":1015808,"data":false,"zero":true}
+EOF
+do_test "0" 1M
+
+# Completely zero disk.
+cat > $expected <<'EOF'
+{"start":0,"length":32768,"data":true,"zero":true}
+EOF
+do_test "0 0 0" 32K
+
+# Allocated data and zero extents in a few places in the middle.
+cat > $expected <<'EOF'
+{"start":0,"length":32768,"data":false,"zero":true}
+{"start":32768,"length":32768,"data":true,"zero":false}
+{"start":65536,"length":32768,"data":true,"zero":true}
+{"start":98304,"length":32768,"data":false,"zero":true}
+{"start":131072,"length":32768,"data":true,"zero":false}
+{"start":163840,"length":98304,"data":false,"zero":true}
+{"start":262144,"length":32768,"data":true,"zero":true}
+{"start":294912,"length":32768,"data":true,"zero":false}
+{"start":327680,"length":720896,"data":false,"zero":true}
+EOF
+do_test "@32768 1 @65536 0 @131072 1 @262144 0 @294912 1" 1M
+
+# This should test coalescing in the extents code.
+cat > $expected <<'EOF'
+{"start":0,"length":32768,"data":false,"zero":true}
+{"start":32768,"length":65536,"data":true,"zero":false}
+{"start":98304,"length":32768,"data":false,"zero":true}
+{"start":131072,"length":65536,"data":true,"zero":true}
+{"start":196608,"length":851968,"data":false,"zero":true}
+EOF
+do_test "@32768 1 @65536 1 @131072 0 @163840 0" 1M
+
+# Zero-length plugin. Unlike nbdkit-zero-plugin, the data plugin
+# advertises extents and so will behave differently.
+cat > $expected <<'EOF'
+{"start":0,"length":0,"data":false,"zero":false}
+EOF
+do_test "" 0
--
2.20.1
More information about the Libguestfs
mailing list