[Libguestfs] [nbdkit PATCH 2/2] server: Cork around grouped transmission send()s

[Eric Blake]

As mentioned in the previous patch, sending small packets as soon as
possible leads to network packet overhead.  Grouping related writes
under corking appears to help everything but unencrypted Unix plain
sockets. I tested with appropriate combinations from:

$ nbdkit {-p 10810,-U -} \
  {--tls=require --tls-verify-peer --tls-psk=./keys.psk,} memory size=64m \
  --run './aio-parallel-load{-tls,} {$unixsocket,nbd://localhost:10810}'

with the following IOPS measurements:

               pre       post
unix plain:    81003.5   78538.8
unix tls:      11678.0   12256.1
tcp plain:     65702.6   68010.9
tcp tls:       11058.5   11762.0

I intentionally did not bother with using corking during handshake
phase - that part of the protocol may benefit, but it is not the hot
spot in overall execution.

Signed-off-by: Eric Blake <eblake at redhat.com>

---
I'm not sure why Unix sockets degraded...
I also probably out to repeat my timings more than a single run...
---
 server/connections.c |  2 ++
 server/protocol.c    | 52 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)

diff --git a/server/connections.c b/server/connections.c
index 9b0b75c..8c92dc5 100644
--- a/server/connections.c
+++ b/server/connections.c
@@ -356,6 +356,7 @@ raw_send (struct connection *conn, const void *vbuf, size_t len)
   return 0;
 }

+#ifdef TCP_CORK
 /* Change the cork status to batch a group of send calls, and either succeed
  * completely (returns 0) or fail (returns -1).
  */
@@ -368,6 +369,7 @@ raw_cork (struct connection *conn, bool cork)
   setsockopt (conn->sockout, IPPROTO_TCP, TCP_CORK, &opt, sizeof opt);
   return 0;
 }
+#endif /* TCP_CORK */

 /* Read buffer from conn->sockin and either succeed completely
  * (returns > 0), read an EOF (returns 0), or fail (returns -1).
diff --git a/server/protocol.c b/server/protocol.c
index 6d519e7..0057aed 100644
--- a/server/protocol.c
+++ b/server/protocol.c
@@ -398,6 +398,11 @@ send_simple_reply (struct connection *conn,
   reply.handle = handle;
   reply.error = htobe32 (nbd_errno (error, false));

+  if (conn->cork) {
+    r = conn->cork (conn, true);
+    assert (r == 0); /* For now, only uncorking can fail */
+  }
+
   r = conn->send (conn, &reply, sizeof reply);
   if (r == -1) {
     nbdkit_error ("write reply: %s: %m", name_of_nbd_cmd (cmd));
@@ -413,6 +418,14 @@ send_simple_reply (struct connection *conn,
     }
   }

+  if (conn->cork) {
+    r = conn->cork (conn, false);
+    if (r == -1) {
+      nbdkit_error ("write uncork: %s: %m", name_of_nbd_cmd (cmd));
+      return connection_set_status (conn, -1);
+    }
+  }
+
   return 1;                     /* command processed ok */
 }

@@ -433,6 +446,11 @@ send_structured_reply_read (struct connection *conn,

   assert (cmd == NBD_CMD_READ);

+  if (conn->cork) {
+    r = conn->cork (conn, true);
+    assert (r == 0); /* For now, only uncorking can fail */
+  }
+
   reply.magic = htobe32 (NBD_STRUCTURED_REPLY_MAGIC);
   reply.handle = handle;
   reply.flags = htobe16 (NBD_REPLY_FLAG_DONE);
@@ -459,6 +477,14 @@ send_structured_reply_read (struct connection *conn,
     return connection_set_status (conn, -1);
   }

+  if (conn->cork) {
+    r = conn->cork (conn, false);
+    if (r == -1) {
+      nbdkit_error ("write uncork: %s: %m", name_of_nbd_cmd (cmd));
+      return connection_set_status (conn, -1);
+    }
+  }
+
   return 1;                     /* command processed ok */
 }

@@ -566,6 +592,11 @@ send_structured_reply_block_status (struct connection *conn,
   if (blocks == NULL)
     return connection_set_status (conn, -1);

+  if (conn->cork) {
+    r = conn->cork (conn, true);
+    assert (r == 0); /* For now, only uncorking can fail */
+  }
+
   reply.magic = htobe32 (NBD_STRUCTURED_REPLY_MAGIC);
   reply.handle = handle;
   reply.flags = htobe16 (NBD_REPLY_FLAG_DONE);
@@ -596,6 +627,14 @@ send_structured_reply_block_status (struct connection *conn,
     }
   }

+  if (conn->cork) {
+    r = conn->cork (conn, false);
+    if (r == -1) {
+      nbdkit_error ("write uncork: %s: %m", name_of_nbd_cmd (cmd));
+      return connection_set_status (conn, -1);
+    }
+  }
+
   return 1;                     /* command processed ok */
 }

@@ -609,6 +648,11 @@ send_structured_reply_error (struct connection *conn,
   struct structured_reply_error error_data;
   int r;

+  if (conn->cork) {
+    r = conn->cork (conn, true);
+    assert (r == 0); /* For now, only uncorking can fail */
+  }
+
   reply.magic = htobe32 (NBD_STRUCTURED_REPLY_MAGIC);
   reply.handle = handle;
   reply.flags = htobe16 (NBD_REPLY_FLAG_DONE);
@@ -631,6 +675,14 @@ send_structured_reply_error (struct connection *conn,
   }
   /* No human readable error message at the moment. */

+  if (conn->cork) {
+    r = conn->cork (conn, false);
+    if (r == -1) {
+      nbdkit_error ("write uncork: %s: %m", name_of_nbd_cmd (cmd));
+      return connection_set_status (conn, -1);
+    }
+  }
+
   return 1;                     /* command processed ok */
 }

-- 
2.20.1