[libvirt] [PATCHv2] unlock eventLoop before calling callback function

Daniel P. Berrange berrange at redhat.com
Mon Mar 7 17:18:15 UTC 2011


On Mon, Mar 07, 2011 at 10:09:36AM -0700, Eric Blake wrote:
> From: Wen Congyang <wency at cn.fujitsu.com>
> 
> When I use newest libvirt to save a domain, libvirtd will be deadlock.
> Here is the output of gdb:
> (gdb) thread 3
> [Switching to thread 3 (Thread 0x7f972a1fc710 (LWP 30265))]#0  0x000000351fe0e034 in __lll_lock_wait () from /lib64/libpthread.so.0
> (gdb) bt
>     at qemu/qemu_driver.c:2074
>     ret=0x7f972a1fbbe0) at remote.c:2273
> (gdb) thread 7
> [Switching to thread 7 (Thread 0x7f9730bcd710 (LWP 30261))]#0  0x000000351fe0e034 in __lll_lock_wait () from /lib64/libpthread.so.0
> (gdb) bt
> (gdb) p *(virMutexPtr)0x6fdd60
> $2 = {lock = {__data = {__lock = 2, __count = 0, __owner = 30261, __nusers = 1, __kind = 0, __spins = 0, __list = {__prev = 0x0, __next = 0x0}},
>     __size = "\002\000\000\000\000\000\000\000\065v\000\000\001", '\000' <repeats 26 times>, __align = 2}}
> (gdb) p *(virMutexPtr)0x1a63ac0
> $3 = {lock = {__data = {__lock = 2, __count = 0, __owner = 30265, __nusers = 1, __kind = 0, __spins = 0, __list = {__prev = 0x0, __next = 0x0}},
>     __size = "\002\000\000\000\000\000\000\000\071v\000\000\001", '\000' <repeats 26 times>, __align = 2}}
> (gdb) info threads
>   7 Thread 0x7f9730bcd710 (LWP 30261)  0x000000351fe0e034 in __lll_lock_wait () from /lib64/libpthread.so.0
>   6 Thread 0x7f972bfff710 (LWP 30262)  0x000000351fe0b43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
>   5 Thread 0x7f972b5fe710 (LWP 30263)  0x000000351fe0b43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
>   4 Thread 0x7f972abfd710 (LWP 30264)  0x000000351fe0b43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
> * 3 Thread 0x7f972a1fc710 (LWP 30265)  0x000000351fe0e034 in __lll_lock_wait () from /lib64/libpthread.so.0
>   2 Thread 0x7f97297fb710 (LWP 30266)  0x000000351fe0b43c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
>   1 Thread 0x7f9737aac800 (LWP 30260)  0x000000351fe0803d in pthread_join () from /lib64/libpthread.so.0
> 
> The reason is that we will try to lock some object in callback function, and we may call event API with locking the same object.
> In the function virEventDispatchHandles(), we unlock eventLoop before calling callback function. I think we should
> do the same thing in the function virEventCleanupTimeouts() and virEventCleanupHandles().
> 
> Signed-off-by: Wen Congyang <wency at cn.fujitsu.com>
> Signed-off-by: Eric Blake <eblake at redhat.com>
> ---
> 
> v2: incorporate comments from reviewers, and rebase on top of file move
> 
> I tested that this avoided deadlock for my 'virsh save' case
> where I was reporting failure last week.
> 
>  src/util/event_poll.c |   27 +++++++++++++++++++--------
>  1 files changed, 19 insertions(+), 8 deletions(-)
> 
> diff --git a/src/util/event_poll.c b/src/util/event_poll.c
> index dd83fc3..91000e2 100644
> --- a/src/util/event_poll.c
> +++ b/src/util/event_poll.c
> @@ -354,7 +354,7 @@ static struct pollfd *virEventPollMakePollFDs(int *nfds) {
> 
>      *nfds = 0;
>      for (i = 0 ; i < eventLoop.handlesCount ; i++) {
> -        if (eventLoop.handles[i].events)
> +        if (eventLoop.handles[i].events && !eventLoop.handles[i].deleted)
>              (*nfds)++;
>      }
> 
> @@ -366,11 +366,12 @@ static struct pollfd *virEventPollMakePollFDs(int *nfds) {
> 
>      *nfds = 0;
>      for (i = 0 ; i < eventLoop.handlesCount ; i++) {
> -        EVENT_DEBUG("Prepare n=%d w=%d, f=%d e=%d", i,
> +        EVENT_DEBUG("Prepare n=%d w=%d, f=%d e=%d d=%d", i,
>                      eventLoop.handles[i].watch,
>                      eventLoop.handles[i].fd,
> -                    eventLoop.handles[i].events);
> -        if (!eventLoop.handles[i].events)
> +                    eventLoop.handles[i].events,
> +                    eventLoop.handles[i].deleted);
> +        if (!eventLoop.handles[i].events || eventLoop.handles[i].deleted)
>              continue;
>          fds[*nfds].fd = eventLoop.handles[i].fd;
>          fds[*nfds].events = eventLoop.handles[i].events;
> @@ -506,8 +507,13 @@ static void virEventPollCleanupTimeouts(void) {
> 
>          EVENT_DEBUG("Purging timeout %d with id %d", i,
>                      eventLoop.timeouts[i].timer);
> -        if (eventLoop.timeouts[i].ff)
> -            (eventLoop.timeouts[i].ff)(eventLoop.timeouts[i].opaque);
> +        if (eventLoop.timeouts[i].ff) {
> +            virFreeCallback ff = eventLoop.timeouts[i].ff;
> +            void *opaque = eventLoop.timeouts[i].opaque;
> +            virMutexUnlock(&eventLoop.lock);
> +            ff(opaque);
> +            virMutexLock(&eventLoop.lock);
> +        }
> 
>          if ((i+1) < eventLoop.timeoutsCount) {
>              memmove(eventLoop.timeouts+i,
> @@ -546,8 +552,13 @@ static void virEventPollCleanupHandles(void) {
>              continue;
>          }
> 
> -        if (eventLoop.handles[i].ff)
> -            (eventLoop.handles[i].ff)(eventLoop.handles[i].opaque);
> +        if (eventLoop.handles[i].ff) {
> +            virFreeCallback ff = eventLoop.handles[i].ff;
> +            void *opaque = eventLoop.handles[i].opaque;
> +            virMutexUnlock(&eventLoop.lock);
> +            ff(opaque);
> +            virMutexLock(&eventLoop.lock);
> +        }
> 
>          if ((i+1) < eventLoop.handlesCount) {
>              memmove(eventLoop.handles+i,

ACK

Daniel
-- 
|: http://berrange.com      -o-    http://www.flickr.com/photos/dberrange/ :|
|: http://libvirt.org              -o-             http://virt-manager.org :|
|: http://autobuild.org       -o-         http://search.cpan.org/~danberr/ :|
|: http://entangle-photo.org       -o-       http://live.gnome.org/gtk-vnc :|




More information about the libvir-list mailing list