[libvirt] [PATCH 11/11] Add a plugin for the 'sanlock' project
Daniel P. Berrange
berrange at redhat.com
Mon Jan 24 15:13:10 UTC 2011
Sanlock is a project that implements a disk-paxos locking
algorithm. This is suitable for cluster deployments with
shared storage.
* src/Makefile.am: Add dlopen plugin for sanlock
* src/locking/lock_driver_sanlock.c: Sanlock driver
---
po/POTFILES.in | 1 +
src/Makefile.am | 12 +
src/libvirt_private.syms | 1 +
src/locking/lock_driver_sanlock.c | 452 +++++++++++++++++++++++++++++++++++++
4 files changed, 466 insertions(+), 0 deletions(-)
create mode 100644 src/locking/lock_driver_sanlock.c
diff --git a/po/POTFILES.in b/po/POTFILES.in
index 47f2f20..302b9c0 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -30,6 +30,7 @@ src/interface/netcf_driver.c
src/internal.h
src/libvirt.c
src/locking/lock_manager.c
+src/locking/lock_driver_sanlock.c
src/lxc/lxc_container.c
src/lxc/lxc_conf.c
src/lxc/lxc_controller.c
diff --git a/src/Makefile.am b/src/Makefile.am
index b68a9b4..f56ff17 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -97,6 +97,9 @@ DRIVER_SOURCES = \
locking/lock_driver_nop.h locking/lock_driver_nop.c \
locking/domain_lock.h locking/domain_lock.c
+LOCK_DRIVER_SANLOCK_SOURCES = \
+ locking/lock_driver_sanlock.c
+
# XML configuration format handling sources
# Domain driver generic impl APIs
@@ -1148,6 +1151,15 @@ libvirt_qemu_la_CFLAGS = $(AM_CFLAGS)
libvirt_qemu_la_LIBADD = libvirt.la $(CYGWIN_EXTRA_LIBADD)
EXTRA_DIST += $(LIBVIRT_QEMU_SYMBOL_FILE)
+
+lockdriverdir = $(libdir)/libvirt/lock-driver
+lockdriver_LTLIBRARIES = sanlock.la
+
+sanlock_la_SOURCES = $(LOCK_DRIVER_SANLOCK_SOURCES)
+sanlock_la_CFLAGS = $(AM_CLFAGS)
+sanlock_la_LDFLAGS = -no-version -module
+sanlock_la_LIBADD = -lsanlock
+
libexec_PROGRAMS =
if WITH_STORAGE_DISK
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index 8005f20..b2f11b8 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -585,6 +585,7 @@ virVMOperationTypeToString;
# memory.h
virAlloc;
virAllocN;
+virAllocVar;
virExpandN;
virFree;
virReallocN;
diff --git a/src/locking/lock_driver_sanlock.c b/src/locking/lock_driver_sanlock.c
new file mode 100644
index 0000000..90afe18
--- /dev/null
+++ b/src/locking/lock_driver_sanlock.c
@@ -0,0 +1,452 @@
+/*
+ * lock_driver_sanlock.c: A lock driver for Sanlock
+ *
+ * Copyright (C) 2010-2011 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *
+ */
+
+#include <config.h>
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <string.h>
+#include <stdio.h>
+#include <errno.h>
+#include <sys/types.h>
+
+#include <sanlock.h>
+#include <sanlock_resource.h>
+
+#include "lock_driver.h"
+#include "logging.h"
+#include "virterror_internal.h"
+#include "memory.h"
+#include "util.h"
+#include "files.h"
+
+#define VIR_FROM_THIS VIR_FROM_LOCKING
+
+#define virLockError(code, ...) \
+ virReportErrorHelper(NULL, VIR_FROM_THIS, code, __FILE__, \
+ __FUNCTION__, __LINE__, __VA_ARGS__)
+
+struct snlk_con {
+ char vm_name[SANLK_NAME_LEN];
+ char vm_uuid[VIR_UUID_BUFLEN];
+ unsigned int vm_id;
+ unsigned int vm_pid;
+ unsigned int flags;
+ int sock;
+ int res_count;
+ struct sanlk_resource *res_args[SANLK_MAX_RESOURCES];
+};
+
+/*
+ * sanlock plugin for the libvirt virLockManager API
+ */
+
+static int drv_snlk_init(unsigned int version ATTRIBUTE_UNUSED,
+ unsigned int flags)
+{
+ virCheckFlags(VIR_LOCK_MANAGER_MODE_CONTENT, -1);
+ return 0;
+}
+
+static int drv_snlk_deinit(void)
+{
+ virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Unloading sanlock plugin is forbidden"));
+ return -1;
+}
+
+static int drv_snlk_new(virLockManagerPtr man,
+ unsigned int type,
+ size_t nparams,
+ virLockManagerParamPtr params,
+ unsigned int flags)
+{
+ virLockManagerParamPtr param;
+ struct snlk_con *con;
+ int i;
+
+ virCheckFlags(VIR_LOCK_MANAGER_MODE_CONTENT, -1);
+
+ if (type != VIR_LOCK_MANAGER_OBJECT_TYPE_DOMAIN) {
+ virLockError(VIR_ERR_INTERNAL_ERROR,
+ _("Unsupported object type %d"), type);
+ return -1;
+ }
+
+ if (VIR_ALLOC(con) < 0) {
+ virReportOOMError();
+ return -1;
+ }
+
+ con->flags = flags;
+ con->sock = -1;
+
+ for (i = 0; i < nparams; i++) {
+ param = ¶ms[i];
+
+ if (STREQ(param->key, "uuid")) {
+ memcpy(con->vm_uuid, param->value.uuid, 16);
+ } else if (STREQ(param->key, "name")) {
+ if (!virStrcpy(con->vm_name, param->value.str, SANLK_NAME_LEN)) {
+ virLockError(VIR_ERR_INTERNAL_ERROR,
+ _("Domain name '%s' exceeded %d characters"),
+ param->value.str, SANLK_NAME_LEN);
+ goto error;
+ }
+ } else if (STREQ(param->key, "pid")) {
+ con->vm_pid = param->value.ui;
+ } else if (STREQ(param->key, "id")) {
+ con->vm_id = param->value.ui;
+ }
+ }
+
+ man->privateData = con;
+ return 0;
+
+error:
+ VIR_FREE(con);
+ return -1;
+}
+
+static void drv_snlk_free(virLockManagerPtr man)
+{
+ struct snlk_con *con = man->privateData;
+
+ DEBUG("man=%p sock=%d", man, con->sock);
+#if 0
+ /* We do *not* want to close the socket here. We need the
+ * socket to keep alive other sanlock will fence the
+ * process. The socket will be explicitly closed before
+ * free, in the release_object method, if neccessary.
+ */
+ VIR_FORCE_CLOSE(con->sock);
+#endif
+ VIR_FREE(con);
+ man->privateData = NULL;
+}
+
+static int add_con_resource(struct snlk_con *con,
+ const char *name,
+ size_t nparams,
+ virLockManagerParamPtr params)
+{
+ virLockManagerParamPtr param;
+ struct sanlk_resource *res;
+ int i;
+
+ if (VIR_ALLOC_VAR(res, struct sanlk_disk, 1) < 0) {
+ virReportOOMError();
+ return -1;
+ }
+
+ res->num_disks = 1;
+ if (!virStrcpy(res->name, name, SANLK_NAME_LEN)) {
+ virLockError(VIR_ERR_INTERNAL_ERROR,
+ _("Resource name '%s' exceeds %d characters"),
+ name, SANLK_NAME_LEN);
+ goto error;
+ }
+
+ for (i = 0; i < nparams; i++) {
+ param = ¶ms[i];
+
+ if (STREQ(param->key, "path")) {
+ if (!virStrcpy(res->disks[0].path, param->value.str, SANLK_PATH_LEN)) {
+ virLockError(VIR_ERR_INTERNAL_ERROR,
+ _("Lease path '%s' exceeds %d characters"),
+ param->value.str, SANLK_PATH_LEN);
+ goto error;
+ }
+ } else if (STREQ(param->key, "offset")) {
+ res->disks[0].offset = param->value.ul;
+ }
+ }
+
+ con->res_args[con->res_count] = res;
+ con->res_count++;
+ return 0;
+
+error:
+ VIR_FREE(res);
+ return -1;
+}
+
+static int drv_snlk_add_resource(virLockManagerPtr man,
+ unsigned int type,
+ const char *name,
+ size_t nparams,
+ virLockManagerParamPtr params,
+ unsigned int flags ATTRIBUTE_UNUSED)
+{
+ struct snlk_con *con = man->privateData;
+ /* must be called before acquire_object */
+ if (con->sock != -1) {
+ virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Cannot add resources to an existing lock"));
+ return -1;
+ }
+
+ if (con->res_count == SANLK_MAX_RESOURCES) {
+ virLockError(VIR_ERR_INTERNAL_ERROR,
+ _("Too many resources %d for object"),
+ SANLK_MAX_RESOURCES);
+ return -1;
+ }
+
+ if (type != VIR_LOCK_MANAGER_RESOURCE_TYPE_LEASE)
+ return 0;
+
+ if (add_con_resource(con, name, nparams, params) < 0)
+ return -1;
+
+ return 0;
+}
+
+static int drv_snlk_acquire_object(virLockManagerPtr man,
+ const char *state,
+ unsigned int flags ATTRIBUTE_UNUSED)
+{
+ struct snlk_con *con = man->privateData;
+ struct sanlk_options *opt = NULL;
+ int i, rv, sock;
+ int pid = getpid();
+
+ /* acquire_object can be called only once */
+ if (con->sock != -1) {
+ virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Object lock is already held"));
+ return -1;
+ }
+
+ if (con->vm_pid != pid) {
+ virLockError(VIR_ERR_INTERNAL_ERROR,
+ _("Object lock attempt from pid %d, expected %d"),
+ pid, con->vm_pid);
+ return -1;
+ }
+
+ if (VIR_ALLOC_VAR(opt, char, state ? strlen(state) : 0) < 0) {
+ virReportOOMError();
+ return -1;
+ }
+
+ if (!virStrcpy(opt->owner_name, con->vm_name, SANLK_NAME_LEN)) {
+ virLockError(VIR_ERR_INTERNAL_ERROR,
+ _("Domain name '%s' exceeded %d characters"),
+ con->vm_name, SANLK_NAME_LEN);
+ goto error;
+ }
+
+ if (state) {
+ opt->flags = SANLK_FLG_INCOMING;
+ opt->len = strlen(state);
+ strcpy(opt->str, state);
+ }
+
+ VIR_DEBUG0("Register sanlock");
+ sock = sanlock_register();
+ if (sock < 0) {
+ virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Failed to open socket to sanlock daemon"));
+ goto error;
+ }
+ VIR_DEBUG("Acquiring object %u", con->res_count);
+ rv = sanlock_acquire(sock, -1, con->res_count, con->res_args, opt);
+ VIR_DEBUG("Acquire result %d", rv);
+ if (rv < 0) {
+ virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Failed to acquire lock"));
+ goto error;
+ }
+ VIR_FREE(opt);
+
+ for (i = 0; i < con->res_count; i++)
+ VIR_FREE(con->res_args[i]);
+
+ con->sock = sock;
+
+ return 0;
+
+error:
+ VIR_FORCE_CLOSE(sock);
+ VIR_FREE(opt);
+ return -1;
+}
+
+static int drv_snlk_attach_object(virLockManagerPtr man ATTRIBUTE_UNUSED,
+ unsigned int flags ATTRIBUTE_UNUSED)
+{
+ return 0;
+}
+
+static int drv_snlk_detach_object(virLockManagerPtr man ATTRIBUTE_UNUSED,
+ unsigned int flags ATTRIBUTE_UNUSED)
+{
+ return 0;
+}
+
+static int drv_snlk_release_object(virLockManagerPtr man ATTRIBUTE_UNUSED,
+ unsigned int flags ATTRIBUTE_UNUSED)
+{
+ struct snlk_con *con = man->privateData;
+
+ if (con->sock == -1) {
+ virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Cannot release object that is not locked"));
+ return -1;
+ }
+ VIR_FORCE_CLOSE(con->sock);
+ return 0;
+}
+
+static int drv_snlk_get_state(virLockManagerPtr man ATTRIBUTE_UNUSED,
+ char **state,
+ unsigned int flags ATTRIBUTE_UNUSED)
+{
+ *state = NULL;
+
+ return 0;
+}
+
+
+static int drv_snlk_acquire_resource(virLockManagerPtr man,
+ unsigned int type,
+ const char *name,
+ size_t nparams,
+ virLockManagerParamPtr params,
+ unsigned int flags ATTRIBUTE_UNUSED)
+{
+ struct snlk_con *con = man->privateData;
+ struct sanlk_options opt;
+ int rv;
+
+ if (con->sock != -1) {
+ virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Cannot acquire resource on unlocked object"));
+ return -1;
+ }
+
+ if (!con->vm_pid) {
+ virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Cannot acquire resource on unlocked object"));
+ return -1;
+ }
+
+ if (type != VIR_LOCK_MANAGER_RESOURCE_TYPE_LEASE)
+ return 0;
+
+ if (add_con_resource(con, name, nparams, params) < 0)
+ return -1;
+
+ /* Setting REACQUIRE tells sanlock that if con->vm_pid previously held
+ and released the resource, we need to ensure no other host has
+ acquired a lease on it in the mean time. If this is a new resource
+ that the pid hasn't held before, then REACQUIRE will have no effect
+ since sanlock will have no memory of a previous version. */
+
+ memset(&opt, 0, sizeof(struct sanlk_options));
+ if (!virStrcpy(opt.owner_name, con->vm_name, SANLK_NAME_LEN)) {
+ virLockError(VIR_ERR_INTERNAL_ERROR,
+ _("Domain name '%s' exceeds %d characters"),
+ con->vm_name, SANLK_NAME_LEN);
+ return -1;
+ }
+ opt.flags = SANLK_FLG_REACQUIRE;
+ opt.len = 0;
+
+ rv = sanlock_acquire(-1, con->vm_pid, con->res_count, con->res_args, &opt);
+
+ VIR_FREE(con->res_args[0]);
+
+ if (rv < 0) {
+ virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Unable to acquire resource"));
+ return -1;
+ }
+
+ return 0;
+}
+
+static int drv_snlk_release_resource(virLockManagerPtr man,
+ unsigned int type,
+ const char *name,
+ size_t nparams,
+ virLockManagerParamPtr params,
+ unsigned int flags ATTRIBUTE_UNUSED)
+{
+ struct snlk_con *con = man->privateData;
+ int rv;
+
+ if (con->sock != -1) {
+ virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Cannot acquire resource on unlocked object"));
+ return -1;
+ }
+
+ if (!con->vm_pid) {
+ virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Cannot acquire resource on unlocked object"));
+ return -1;
+ }
+
+ if (type != VIR_LOCK_MANAGER_RESOURCE_TYPE_LEASE)
+ return 0;
+
+ if (add_con_resource(con, name, nparams, params) < 0)
+ return -1;
+
+ rv = sanlock_release(-1, con->vm_pid, con->res_count, con->res_args);
+
+ VIR_FREE(con->res_args[0]);
+
+ if (rv < 0) {
+ virLockError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("Unable to release resource"));
+ return -1;
+ }
+
+ return 0;
+}
+
+virLockDriver virLockDriverImpl =
+{
+ .version = VIR_LOCK_MANAGER_VERSION,
+ .flags = VIR_LOCK_MANAGER_MODE_CONTENT,
+
+ .drvInit = drv_snlk_init,
+ .drvDeinit = drv_snlk_deinit,
+
+ .drvNew = drv_snlk_new,
+ .drvFree = drv_snlk_free,
+
+ .drvAddResource = drv_snlk_add_resource,
+
+ .drvAcquireObject = drv_snlk_acquire_object,
+ .drvAttachObject = drv_snlk_attach_object,
+ .drvDetachObject = drv_snlk_detach_object,
+ .drvReleaseObject = drv_snlk_release_object,
+
+ .drvGetState = drv_snlk_get_state,
+
+ .drvAcquireResource = drv_snlk_acquire_resource,
+ .drvReleaseResource = drv_snlk_release_resource,
+};
--
1.7.3.4
More information about the libvir-list
mailing list