[Libguestfs] [nbdkit PATCH] split: Add support for .extents

Eric Blake eblake at redhat.com
Mon Feb 10 15:00:08 UTC 2020


Copies somewhat from the file plugin, with the difference that we
always provide an .extents even if one or more of the split files does
not support SEEK_HOLE.

Testing is possible on a file system that supports sparse files when
using nbdsh.

Signed-off-by: Eric Blake <eblake at redhat.com>
---

I'm going ahead and committing this, but am also posting this for
review in case we later find any problems with what I just pushed.

 plugins/split/Makefile.am   |   6 +-
 plugins/split/split.c       | 125 +++++++++++++++++++++++++++++++++++-
 tests/Makefile.am           |   3 +
 tests/test-split-extents.sh |  74 +++++++++++++++++++++
 4 files changed, 206 insertions(+), 2 deletions(-)
 create mode 100755 tests/test-split-extents.sh

diff --git a/plugins/split/Makefile.am b/plugins/split/Makefile.am
index 003f6d9..ceff349 100644
--- a/plugins/split/Makefile.am
+++ b/plugins/split/Makefile.am
@@ -1,5 +1,5 @@
 # nbdkit
-# Copyright (C) 2017 Red Hat Inc.
+# Copyright (C) 2017-2020 Red Hat Inc.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
@@ -42,12 +42,16 @@ nbdkit_split_plugin_la_SOURCES = \

 nbdkit_split_plugin_la_CPPFLAGS = \
 	-I$(top_srcdir)/include \
+	-I$(top_srcdir)/common/utils \
 	$(NULL)
 nbdkit_split_plugin_la_CFLAGS = $(WARNINGS_CFLAGS)
 nbdkit_split_plugin_la_LDFLAGS = \
 	-module -avoid-version -shared \
 	-Wl,--version-script=$(top_srcdir)/plugins/plugins.syms \
 	$(NULL)
+nbdkit_split_plugin_la_LIBADD = \
+	$(top_builddir)/common/utils/libutils.la \
+	$(NULL)

 if HAVE_POD

diff --git a/plugins/split/split.c b/plugins/split/split.c
index 68682b2..70fd4d4 100644
--- a/plugins/split/split.c
+++ b/plugins/split/split.c
@@ -1,5 +1,5 @@
 /* nbdkit
- * Copyright (C) 2017-2019 Red Hat Inc.
+ * Copyright (C) 2017-2020 Red Hat Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
@@ -42,13 +42,19 @@
 #include <errno.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <stdbool.h>

 #include <nbdkit-plugin.h>

+#include "cleanup.h"
+
 /* The files. */
 static char **filenames = NULL;
 static size_t nr_files = 0;

+/* Any callbacks using lseek must be protected by this lock. */
+static pthread_mutex_t lseek_lock = PTHREAD_MUTEX_INITIALIZER;
+
 static void
 split_unload (void)
 {
@@ -96,6 +102,7 @@ struct handle {
 struct file {
   uint64_t offset, size;
   int fd;
+  bool can_extents;
 };

 /* Create the per-connection handle. */
@@ -107,6 +114,7 @@ split_open (int readonly)
   size_t i;
   uint64_t offset;
   struct stat statbuf;
+  off_t r;

   h = malloc (sizeof *h);
   if (h == NULL) {
@@ -151,6 +159,20 @@ split_open (int readonly)

     nbdkit_debug ("file[%zu]=%s: offset=%" PRIu64 ", size=%" PRIu64,
                   i, filenames[i], h->files[i].offset, h->files[i].size);
+
+#ifdef SEEK_HOLE
+    /* Test if this file supports extents. */
+    ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lseek_lock);
+    r = lseek (h->files[i].fd, 0, SEEK_DATA);
+    if (r == -1 && errno != ENXIO) {
+      nbdkit_debug ("disabling extents: lseek on %s: %m", filenames[i]);
+      h->files[i].can_extents = false;
+    }
+    else
+      h->files[i].can_extents = true;
+#else
+    h->files[i].can_extents = false;
+#endif
   }
   h->size = offset;
   nbdkit_debug ("total size=%" PRIu64, h->size);
@@ -319,6 +341,104 @@ split_cache (void *handle, uint32_t count, uint64_t offset, uint32_t flags)
 }
 #endif /* HAVE_POSIX_FADVISE */

+#ifdef SEEK_HOLE
+static int64_t
+do_extents (struct file *file, uint32_t count, uint64_t offset,
+            bool req_one, struct nbdkit_extents *extents)
+{
+  int64_t r = 0;
+  uint64_t end = offset + count;
+
+  do {
+    off_t pos;
+
+    pos = lseek (file->fd, offset, SEEK_DATA);
+    if (pos == -1) {
+      if (errno == ENXIO) {
+        /* The current man page does not describe this situation well,
+         * but a proposed change to POSIX adds these words for ENXIO:
+         * "or the whence argument is SEEK_DATA and the offset falls
+         * within the final hole of the file."
+         */
+        pos = end;
+      }
+      else {
+        nbdkit_error ("lseek: SEEK_DATA: %" PRIu64 ": %m", offset);
+        return -1;
+      }
+    }
+
+    /* We know there is a hole from offset to pos-1. */
+    if (pos > offset) {
+      if (nbdkit_add_extent (extents, offset + file->offset, pos - offset,
+                             NBDKIT_EXTENT_HOLE | NBDKIT_EXTENT_ZERO) == -1)
+        return -1;
+      r += pos - offset;
+      if (req_one)
+        break;
+    }
+
+    offset = pos;
+    if (offset >= end)
+      break;
+
+    pos = lseek (file->fd, offset, SEEK_HOLE);
+    if (pos == -1) {
+      nbdkit_error ("lseek: SEEK_HOLE: %" PRIu64 ": %m", offset);
+      return -1;
+    }
+
+    /* We know there is data from offset to pos-1. */
+    if (pos > offset) {
+      if (nbdkit_add_extent (extents, offset + file->offset, pos - offset,
+                             0 /* allocated data */) == -1)
+        return -1;
+      r += pos - offset;
+      if (req_one)
+        break;
+    }
+
+    offset = pos;
+  } while (offset < end);
+
+  return r;
+}
+
+static int
+split_extents (void *handle, uint32_t count, uint64_t offset,
+               uint32_t flags, struct nbdkit_extents *extents)
+{
+  struct handle *h = handle;
+  const bool req_one = flags & NBDKIT_FLAG_REQ_ONE;
+
+  while (count > 0) {
+    struct file *file = get_file (h, offset);
+    uint64_t foffs = offset - file->offset;
+    uint64_t max;
+    int64_t r;
+
+    max = file->size - foffs;
+    if (max > count)
+      max = count;
+
+    if (file->can_extents) {
+      ACQUIRE_LOCK_FOR_CURRENT_SCOPE (&lseek_lock);
+      max = r = do_extents (file, max, foffs, req_one, extents);
+    }
+    else
+      r = nbdkit_add_extent (extents, offset, max, 0 /* allocated data */);
+    if (r == -1)
+      return -1;
+    count -= max;
+    offset += max;
+    if (req_one)
+      break;
+  }
+
+  return 0;
+}
+#endif /* SEEK_HOLE */
+
 static struct nbdkit_plugin plugin = {
   .name              = "split",
   .version           = PACKAGE_VERSION,
@@ -334,6 +454,9 @@ static struct nbdkit_plugin plugin = {
   .pwrite            = split_pwrite,
 #if HAVE_POSIX_FADVISE
   .cache             = split_cache,
+#endif
+#ifdef SEEK_HOLE
+  .extents           = split_extents,
 #endif
   /* In this plugin, errno is preserved properly along error return
    * paths from failed system calls.
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 60583a0..ea6b147 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -193,6 +193,7 @@ EXTRA_DIST = \
 	test-sh-extents.sh \
 	test-single.sh \
 	test-single-from-file.sh \
+	test-split-extents.sh \
 	test-start.sh \
 	test-random-sock.sh \
 	test-tls.sh \
@@ -672,6 +673,8 @@ test_split_SOURCES = test-split.c
 test_split_CFLAGS = $(WARNINGS_CFLAGS) $(LIBNBD_CFLAGS)
 test_split_LDADD = $(LIBNBD_LIBS)

+TESTS += test-split-extents.sh
+
 # ssh plugin test.
 if HAVE_SSH
 # Uses ‘disk’ so we have to make it conditional on guestfish.
diff --git a/tests/test-split-extents.sh b/tests/test-split-extents.sh
new file mode 100755
index 0000000..da385a4
--- /dev/null
+++ b/tests/test-split-extents.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2018-2020 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+# Concatenating sparse and data files should have observable extents.
+
+source ./functions.sh
+set -e
+set -x
+
+requires nbdsh --base-allocation -c 'exit(not h.supports_uri())'
+requires truncate --help
+requires stat --help
+
+files="test-split-extents.1 test-split-extents.2"
+rm -f $files
+cleanup_fn rm -f $files
+
+# Create two files, each half data and half sparse
+truncate --size=512k test-split-extents.1
+if test "$(stat -c %b test-split-extents.1)" != 0; then
+    echo "$0: unable to create sparse file, skipping this test"
+    exit 77
+fi
+printf %$((512*1024))d 1 >> test-split-extents.1
+printf %$((512*1024))d 1 >> test-split-extents.2
+truncate --size=1M test-split-extents.2
+
+# Test the split plugin
+nbdkit -v -U - split test-split-extents.1 test-split-extents.2 \
+       --run 'nbdsh --base-allocation --uri $uri -c "
+entries = []
+def f (metacontext, offset, e, err):
+    global entries
+    assert err.value == 0
+    assert metacontext == nbd.CONTEXT_BASE_ALLOCATION
+    entries = e
+h.block_status (2 * 1024 * 1024, 0, f)
+assert entries == [ 512 * 1024, 3,
+                    1024 * 1024, 0,
+                    512 * 1024, 3 ]
+entries = []
+# With req one, extents stop at file boundaries
+h.block_status (1024 * 1024, 768 * 1024, f, nbd.CMD_FLAG_REQ_ONE)
+assert entries == [ 256 * 1024, 0 ]
+       "'
-- 
2.24.1




More information about the Libguestfs mailing list