[libvirt] [PATCH] MacOS: Handle changes to args in xdrproc_t

[Doug Goldstein]

With Mac OS X 10.9, xdrproc_t is no longer defined as:

typedef bool_t (*xdrproc_t) (XDR *, void *, ...);

but instead as

typedef bool-t (*xdrproc_t) (XDR *, void *, unsigned int);

The rationale explained in the header is that using a vararg is
incorrect and has a potential to change the ABI slightly. They decided
to specify the exact number of parameters and for compatibility with old
code decided to make the signature require 3 arguments. The third
argument is ignored for cases that its not used and its recommended to
supply a 0.
---
 configure.ac            | 41 +++++++++++++++++++++++++++++++++++++++++
 src/rpc/virnetmessage.c | 10 ++++++++--
 2 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index 1c5b168..f2bae88 100644
--- a/configure.ac
+++ b/configure.ac
@@ -697,6 +697,47 @@ if test x"$with_remote" = x"yes" || test x"$with_libvirtd" = x"yes"; then
       *) XDR_CFLAGS=$lv_cv_xdr_cflags ;;
     esac
     AC_SUBST([XDR_CFLAGS])
+
+    dnl Mac OS X Mavericks (10.9) dropped the varargs definition that
+    dnl allowed 2 or 3 parameters to xdrproc_t callbacks and decided on
+    dnl 3 arguments being a must.
+    old_CFLAGS=$CFLAGS
+    AC_CACHE_CHECK([whether xdrproc_t callbacks take 2 or 3 args],
+                   [lv_cv_xdrproc_t_args], [
+      CFLAGS="$old_CFLAGS $XDR_CFLAGS"
+      AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
+        [[
+          #include <rpc/rpc.h>
+        ]],
+        [[
+          XDR xdr;
+          xdrproc_t filter = NULL;
+
+          return (filter)(&xdr, NULL);
+        ]])],
+        [lv_cv_xdrproc_t_args=2], [
+          AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
+            [[
+              #include <rpc/rpc.h>
+            ]],
+            [[
+              XDR xdr;
+              xdrproc_t filter = NULL;
+
+              return (filter)(&xdr, NULL, 0);
+            ]])],
+            [lv_cv_xdrproc_t_args=3],
+            [lv_cv_xdrproc_t_args=unknown])
+        ])
+    CFLAGS="$old_CFLAGS"
+    ])
+    case $lv_cv_xdrproc_t_args in
+      unknown) AC_MSG_ERROR([cannot determine arg count for xdrproc_t]) ;;
+      *)
+      AC_DEFINE_UNQUOTED([XDRPROC_T_ARG_COUNT], [$lv_cv_xdrproc_t_args],
+                         [number of arguments that xdrproc_t func ptr takes])
+      ;;
+    esac
 fi
 
 
diff --git a/src/rpc/virnetmessage.c b/src/rpc/virnetmessage.c
index d60366b..79e496f 100644
--- a/src/rpc/virnetmessage.c
+++ b/src/rpc/virnetmessage.c
@@ -33,6 +33,12 @@
 
 #define VIR_FROM_THIS VIR_FROM_RPC
 
+#if XDRPROC_T_ARG_COUNT == 3
+# define VIR_XDRPROC(proc, xdr, data) ((proc)((xdr), (data), 0))
+#else
+# define VIR_XDRPROC(proc, xdr, data) ((proc)((xdr), (data)))
+#endif
+
 virNetMessagePtr virNetMessageNew(bool tracked)
 {
     virNetMessagePtr msg;
@@ -345,7 +351,7 @@ int virNetMessageEncodePayload(virNetMessagePtr msg,
                   msg->bufferLength - msg->bufferOffset, XDR_ENCODE);
 
     /* Try to encode the payload. If the buffer is too small increase it. */
-    while (!(*filter)(&xdr, data)) {
+    while (!VIR_XDRPROC(*filter, &xdr, data)) {
         unsigned int newlen = (msg->bufferLength - VIR_NET_MESSAGE_LEN_MAX) * 4;
 
         if (newlen > VIR_NET_MESSAGE_MAX) {
@@ -402,7 +408,7 @@ int virNetMessageDecodePayload(virNetMessagePtr msg,
     xdrmem_create(&xdr, msg->buffer + msg->bufferOffset,
                   msg->bufferLength - msg->bufferOffset, XDR_DECODE);
 
-    if (!(*filter)(&xdr, data)) {
+    if (!VIR_XDRPROC(*filter, &xdr, data)) {
         virReportError(VIR_ERR_RPC, "%s", _("Unable to decode message payload"));
         goto error;
     }
-- 
1.8.1.5