[libvirt] [PATCH] add compress stream support
Vasiliy Tolstov
v.tolstov at selfip.ru
Tue Sep 22 22:26:19 UTC 2015
Some libvirt functions use streams, this patch add
compress stream support.
So VolumeDownload/VolumeUpload can greatly speedup by using
compressed streams to save network bandtwidth and don't transfer
zero bytes (in case of raw disk format)
Signed-off-by: Vasiliy Tolstov <v.tolstov at selfip.ru>
---
configure.ac | 2 ++
daemon/Makefile.am | 3 +++
include/libvirt/libvirt-stream.h | 3 +++
m4/virt-archive.m4 | 29 ++++++++++++++++++++++++++
src/Makefile.am | 17 +++++++++++-----
src/datatypes.h | 7 +++++++
src/fdstream.c | 44 ++++++++++++++++++++++++++++++++++++++++
src/libvirt-stream.c | 15 ++++++++++++++
tools/Makefile.am | 17 ++++++++++++++--
9 files changed, 130 insertions(+), 7 deletions(-)
create mode 100644 m4/virt-archive.m4
diff --git a/configure.ac b/configure.ac
index 03463b0..58a15bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -250,6 +250,7 @@ LIBVIRT_CHECK_SANLOCK
LIBVIRT_CHECK_SASL
LIBVIRT_CHECK_SELINUX
LIBVIRT_CHECK_SSH2
+LIBVIRT_CHECK_LIBARCHIVE
LIBVIRT_CHECK_SYSTEMD_DAEMON
LIBVIRT_CHECK_UDEV
LIBVIRT_CHECK_WIRESHARK
@@ -2892,6 +2893,7 @@ LIBVIRT_RESULT_SANLOCK
LIBVIRT_RESULT_SASL
LIBVIRT_RESULT_SELINUX
LIBVIRT_RESULT_SSH2
+LIBVIRT_RESULT_LIBARCHIVE
LIBVIRT_RESULT_SYSTEMD_DAEMON
LIBVIRT_RESULT_UDEV
LIBVIRT_RESULT_WIRESHARK
diff --git a/daemon/Makefile.am b/daemon/Makefile.am
index be1b5a9..a4d63eb 100644
--- a/daemon/Makefile.am
+++ b/daemon/Makefile.am
@@ -177,17 +177,20 @@ libvirtd_CFLAGS = \
$(XDR_CFLAGS) $(DBUS_CFLAGS) $(LIBNL_CFLAGS) \
$(WARN_CFLAGS) $(PIE_CFLAGS) \
$(COVERAGE_CFLAGS) \
+ $(LIBARCHIVE_CFLAGS) \
-DQEMUD_PID_FILE="\"$(QEMUD_PID_FILE)\""
libvirtd_LDFLAGS = \
$(RELRO_LDFLAGS) \
$(PIE_LDFLAGS) \
$(COVERAGE_LDFLAGS) \
+ $(LIBARCHIVE_LDFLAGS) \
$(NO_INDIRECT_LDFLAGS) \
$(NULL)
libvirtd_LDADD = \
$(LIBXML_LIBS) \
+ $(LIBARCHIVE_LIBS) \
$(GNUTLS_LIBS) \
$(SASL_LIBS) \
$(DBUS_LIBS) \
diff --git a/include/libvirt/libvirt-stream.h b/include/libvirt/libvirt-stream.h
index 831640d..c75f03e 100644
--- a/include/libvirt/libvirt-stream.h
+++ b/include/libvirt/libvirt-stream.h
@@ -31,10 +31,13 @@
typedef enum {
VIR_STREAM_NONBLOCK = (1 << 0),
+ VIR_STREAM_COMPRESS_GZIP = (1 << 1),
+ VIR_STREAM_COMPRESS_XZ = (1 << 2),
} virStreamFlags;
virStreamPtr virStreamNew(virConnectPtr conn,
unsigned int flags);
+
int virStreamRef(virStreamPtr st);
int virStreamSend(virStreamPtr st,
diff --git a/m4/virt-archive.m4 b/m4/virt-archive.m4
new file mode 100644
index 0000000..9770732
--- /dev/null
+++ b/m4/virt-archive.m4
@@ -0,0 +1,29 @@
+dnl The libarchive.so library
+dnl
+dnl Copyright (C) 2012-2013 Red Hat, Inc.
+dnl
+dnl This library is free software; you can redistribute it and/or
+dnl modify it under the terms of the GNU Lesser General Public
+dnl License as published by the Free Software Foundation; either
+dnl version 2.1 of the License, or (at your option) any later version.
+dnl
+dnl This library is distributed in the hope that it will be useful,
+dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
+dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+dnl Lesser General Public License for more details.
+dnl
+dnl You should have received a copy of the GNU Lesser General Public
+dnl License along with this library. If not, see
+dnl <http://www.gnu.org/licenses/>.
+dnl
+
+AC_DEFUN([LIBVIRT_CHECK_LIBARCHIVE],[
+ LIBVIRT_CHECK_PKG([LIBARCHIVE], [libarchive], [3.1.2])
+ AC_SUBST(LIBARCHIVE_CFLAGS)
+ AC_SUBST(LIBARCHIVE_LIBS)
+ AC_SUBST(LIBARCHIVE_LDFLAGS)
+])
+
+AC_DEFUN([LIBVIRT_RESULT_LIBARCHIVE],[
+ LIBVIRT_RESULT_LIB([LIBARCHIVE])
+])
diff --git a/src/Makefile.am b/src/Makefile.am
index 060abe8..429c2c7 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -25,11 +25,11 @@ abs_topsrcdir = $(shell cd $(top_srcdir) && pwd)
# No libraries with the exception of LIBXML should be listed
# here. List them against the individual XXX_la_CFLAGS targets
# that actually use them. Also keep GETTEXT_CPPFLAGS at the end.
-INCLUDES = -I../gnulib/lib \
+INCLUDES = -I../gnulib/lib \
-I$(top_srcdir)/gnulib/lib \
- -I$(top_srcdir) \
+ -I$(top_srcdir) \
-I../include \
- -I$(top_srcdir)/include \
+ -I$(top_srcdir)/include \
-I$(srcdir)/util \
-DIN_LIBVIRT \
-Dabs_topbuilddir="\"$(abs_topbuilddir)\"" \
@@ -37,14 +37,16 @@ INCLUDES = -I../gnulib/lib \
$(GETTEXT_CPPFLAGS)
AM_CFLAGS = $(LIBXML_CFLAGS) \
+ $(LIBARCHIVE_CFLAGS) \
$(WARN_CFLAGS) \
- $(LOCK_CHECKING_CFLAGS) \
+ $(LOCK_CHECKING_CFLAGS) \
$(WIN32_EXTRA_CFLAGS) \
$(COVERAGE_CFLAGS)
AM_LDFLAGS = $(DRIVER_MODULE_LDFLAGS) \
$(COVERAGE_LDFLAGS) \
$(RELRO_LDFLAGS) \
$(NO_INDIRECT_LDFLAGS) \
+ $(LIBARCHIVE_LDFLAGS) \
$(NULL)
EXTRA_DIST = $(conf_DATA) util/keymaps.csv
@@ -1051,11 +1053,13 @@ libvirt_util_la_CFLAGS = $(CAPNG_CFLAGS) $(YAJL_CFLAGS) $(LIBNL_CFLAGS) \
$(AM_CFLAGS) $(AUDIT_CFLAGS) $(DEVMAPPER_CFLAGS) \
$(DBUS_CFLAGS) $(LDEXP_LIBM) $(NUMACTL_CFLAGS) \
$(SYSTEMD_DAEMON_CFLAGS) $(POLKIT_CFLAGS) \
+ $(LIBARCHIVE_CFLAGS) \
-I$(srcdir)/conf
libvirt_util_la_LIBADD = $(CAPNG_LIBS) $(YAJL_LIBS) $(LIBNL_LIBS) \
$(THREAD_LIBS) $(AUDIT_LIBS) $(DEVMAPPER_LIBS) \
$(LIB_CLOCK_GETTIME) $(DBUS_LIBS) $(MSCOM_LIBS) $(LIBXML_LIBS) \
$(SECDRIVER_LIBS) $(NUMACTL_LIBS) $(SYSTEMD_DAEMON_LIBS) \
+ $(LIBARCHIVE_LIBS) \
$(POLKIT_LIBS)
@@ -2214,6 +2218,7 @@ libvirt_lxc_la_LDFLAGS = \
$(AM_LDFLAGS) \
$(CYGWIN_EXTRA_LDFLAGS) \
$(MINGW_EXTRA_LDFLAGS) \
+ $(LIBARCHIVE_LDFLAGS) \
$(NULL)
libvirt_lxc_la_CFLAGS = $(AM_CFLAGS)
libvirt_lxc_la_LIBADD = libvirt.la $(CYGWIN_EXTRA_LIBADD)
@@ -2292,13 +2297,15 @@ libvirt_setuid_rpc_client_la_SOURCES = \
libvirt_setuid_rpc_client_la_LDFLAGS = \
$(AM_LDFLAGS) \
$(LIBXML_LIBS) \
+ $(LIBARCHIVE_LIBS) \
$(SECDRIVER_LIBS) \
$(NULL)
libvirt_setuid_rpc_client_la_CFLAGS = \
-DLIBVIRT_SETUID_RPC_CLIENT \
-I$(srcdir)/conf \
- -I$(srcdir)/rpc \
+ -I$(srcdir)/rpc \
$(AM_CFLAGS) \
+ $(LIBARCHIVE_CFLAGS) \
$(SECDRIVER_CFLAGS) \
$(XDR_CFLAGS) \
$(NULL)
diff --git a/src/datatypes.h b/src/datatypes.h
index be108fe..4c0c10d 100644
--- a/src/datatypes.h
+++ b/src/datatypes.h
@@ -29,6 +29,10 @@
# include "virobject.h"
# include "viruuid.h"
+#ifdef WITH_LIBARCHIVE
+# include <archive.h>
+#endif
+
extern virClassPtr virConnectClass;
extern virClassPtr virDomainClass;
extern virClassPtr virDomainSnapshotClass;
@@ -520,6 +524,9 @@ struct _virStream {
virStreamDriverPtr driver;
void *privateData;
+#ifdef WITH_LIBARCHIVE
+ struct archive *archive;
+#endif
};
/**
diff --git a/src/fdstream.c b/src/fdstream.c
index b8ea86e..9a71a9f 100644
--- a/src/fdstream.c
+++ b/src/fdstream.c
@@ -33,6 +33,10 @@
#include <netinet/in.h>
#include <termios.h>
+#ifdef WITH_LIBARCHIVE
+# include <archive.h>
+#endif
+
#include "fdstream.h"
#include "virerror.h"
#include "datatypes.h"
@@ -319,6 +323,12 @@ virFDStreamCloseInt(virStreamPtr st, bool streamAbort)
if (VIR_CLOSE(fdst->errfd) < 0)
VIR_DEBUG("ignoring failed close on fd %d", fdst->errfd);
+#ifdef WITH_LIBARCHIVE
+ if (st->archive != NULL) {
+ archive_write_free(st->archive);
+ archive_read_free(st->archive);
+ }
+#endif
st->privateData = NULL;
/* call the internal stream closing callback */
@@ -385,7 +395,16 @@ static int virFDStreamWrite(virStreamPtr st, const char *bytes, size_t nbytes)
}
retry:
+#ifdef WITH_LIBARCHIVE
+ if (st->archive != NULL) {
+ archive_write_open_fd(st->archive, fdst->fd);
+ ret = archive_write_data(st->archive, bytes, nbytes);
+ } else {
+ ret = write(fdst->fd, bytes, nbytes);
+ }
+#else
ret = write(fdst->fd, bytes, nbytes);
+#endif
if (ret < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK) {
ret = -2;
@@ -397,7 +416,15 @@ static int virFDStreamWrite(virStreamPtr st, const char *bytes, size_t nbytes)
_("cannot write to stream"));
}
} else if (fdst->length) {
+#ifdef WITH_LIBARCHIVE
+ if (st->archive != NULL) {
+ fdst->offset += nbytes;
+ } else {
+ fdst->offset += ret;
+ }
+#else
fdst->offset += ret;
+#endif
}
virMutexUnlock(&fdst->lock);
@@ -435,7 +462,16 @@ static int virFDStreamRead(virStreamPtr st, char *bytes, size_t nbytes)
}
retry:
+#ifdef WITH_LIBARCHIVE
+ if (st->archive != NULL) {
+ archive_read_open_fd(st->archive, fdst->fd, 4096);
+ ret = archive_read_data(st->archive, bytes, nbytes);
+ } else {
+ ret = read(fdst->fd, bytes, nbytes);
+ }
+#else
ret = read(fdst->fd, bytes, nbytes);
+#endif
if (ret < 0) {
if (errno == EAGAIN || errno == EWOULDBLOCK) {
ret = -2;
@@ -447,7 +483,15 @@ static int virFDStreamRead(virStreamPtr st, char *bytes, size_t nbytes)
_("cannot read from stream"));
}
} else if (fdst->length) {
+#ifdef WITH_LIBARCHIVE
+ if (st->archive != NULL) {
+ fdst->offset += nbytes;
+ } else {
+ fdst->offset += ret;
+ }
+#else
fdst->offset += ret;
+#endif
}
virMutexUnlock(&fdst->lock);
diff --git a/src/libvirt-stream.c b/src/libvirt-stream.c
index c16f586..0f09d9d 100644
--- a/src/libvirt-stream.c
+++ b/src/libvirt-stream.c
@@ -24,6 +24,10 @@
#include "viralloc.h"
#include "virlog.h"
+#ifdef WITH_LIBARCHIVE
+# include <archive.h>
+#endif
+
VIR_LOG_INIT("libvirt.stream");
#define VIR_FROM_THIS VIR_FROM_STREAMS
@@ -45,6 +49,7 @@ VIR_LOG_INIT("libvirt.stream");
*
* If a non-blocking data stream is required passed
* VIR_STREAM_NONBLOCK for flags, otherwise pass 0.
+ * VIR_STREAM_COMPRESS_GZIP or VIR_STREAM_COMPRESS_XZ
*
* Returns the new stream, or NULL upon error
*/
@@ -66,6 +71,16 @@ virStreamNew(virConnectPtr conn,
else
virDispatchError(conn);
+ if ((flags & VIR_STREAM_COMPRESS_GZIP) || (flags & VIR_STREAM_COMPRESS_XZ)) {
+ st->archive = archive_write_new();
+
+ if (flags & VIR_STREAM_COMPRESS_GZIP)
+ archive_write_add_filter(st->archive, ARCHIVE_FILTER_GZIP);
+
+ if (flags & VIR_STREAM_COMPRESS_XZ)
+ archive_write_add_filter(st->archive, ARCHIVE_FILTER_XZ);
+ }
+
return st;
}
diff --git a/tools/Makefile.am b/tools/Makefile.am
index 03e9339..fa19a20 100644
--- a/tools/Makefile.am
+++ b/tools/Makefile.am
@@ -25,8 +25,12 @@ INCLUDES = \
AM_LDFLAGS = \
$(RELRO_LDFLAGS) \
$(NO_INDIRECT_LDFLAGS) \
+ $(LIBARCHIVE_LDFLAGS) \
$(NULL)
+AM_CFLAGS = \
+ $(LIBARCHIVE_CFLAGS)
+
POD2MAN = pod2man -c "Virtualization Support" -r "$(PACKAGE)-$(VERSION)"
ICON_FILES = \
@@ -140,15 +144,18 @@ virt_host_validate_LDFLAGS = \
$(AM_LDFLAGS) \
$(PIE_LDFLAGS) \
$(COVERAGE_LDFLAGS) \
+ $(LIBARCHIVE_LDFLAGS) \
$(NULL)
virt_host_validate_LDADD = \
../src/libvirt.la \
- ../gnulib/lib/libgnu.la \
+ ../gnulib/lib/libgnu.la \
+ $(LIBARCHIVE_LDFLAGS) \
$(NULL)
virt_host_validate_CFLAGS = \
$(LIBXML_CFLAGS) \
+ $(LIBARCHIVE_CFLAGS) \
$(WARN_CFLAGS) \
$(PIE_CFLAGS) \
$(COVERAGE_CFLAGS) \
@@ -165,15 +172,18 @@ virt_login_shell_SOURCES = \
virt_login_shell_LDFLAGS = \
$(AM_LDFLAGS) \
$(COVERAGE_LDFLAGS) \
+ $(LIBARHCIVE_LDFLAGS) \
$(NULL)
virt_login_shell_LDADD = \
$(STATIC_BINARIES) \
$(PIE_LDFLAGS) \
../src/libvirt-setuid-rpc-client.la \
+ $(LIBARCHIVE_LDFLAGS) \
../gnulib/lib/libgnu.la
virt_login_shell_CFLAGS = \
-DLIBVIRT_SETUID_RPC_CLIENT \
+ $(LIBARCHIVE_CFLAGS) \
$(LIBXML_CFLAGS) \
$(WARN_CFLAGS) \
$(PIE_CFLAGS) \
@@ -202,6 +212,7 @@ virsh_SOURCES = \
virsh_LDFLAGS = \
$(AM_LDFLAGS) \
$(COVERAGE_LDFLAGS) \
+ $(LIBARCHIVE_LDFLAGS) \
$(NULL)
virsh_LDADD = \
$(STATIC_BINARIES) \
@@ -209,14 +220,16 @@ virsh_LDADD = \
../src/libvirt.la \
../src/libvirt-lxc.la \
../src/libvirt-qemu.la \
- ../gnulib/lib/libgnu.la \
+ ../gnulib/lib/libgnu.la \
$(LIBXML_LIBS) \
+ $(LIBARCHIVE_LIBS) \
$(VIRSH_LIBS)
virsh_CFLAGS = \
$(WARN_CFLAGS) \
$(PIE_CFLAGS) \
$(COVERAGE_CFLAGS) \
$(LIBXML_CFLAGS) \
+ $(LIBARCHIVE_CFLAGS) \
$(READLINE_CFLAGS)
BUILT_SOURCES =
--
2.5.0
More information about the libvir-list
mailing list