commit 5e36776d7d2a85888caea447a083d417c161c27d
Author: Mikolaj Golub <trociny@freebsd.org>
Date:   Sat Aug 31 17:05:07 2013 +0300

    If the secondary disconnects, the primary's remote_recv_thread() moves
    pending requests to done queue. But it does not work correctly for
    write requests in memsync mode, and as a result they may leak.
    
    Fix this by adding hio_memsyncacked flag to hio and using it to
    properly handle pending memsync requests after disconnect.

diff --git a/sbin/hastd/primary.c b/sbin/hastd/primary.c
index 17ce583..fd62e0b 100644
--- a/sbin/hastd/primary.c
+++ b/sbin/hastd/primary.c
@@ -98,6 +98,10 @@ struct hio {
 	 * so we won't get confused when replication changes on reload.
 	 */
 	int			 hio_replication;
+	/*
+	 * Memsync request was acknowleged by remote.
+	 */
+	bool			 hio_memsyncacked;
 	TAILQ_ENTRY(hio)	*hio_next;
 };
 #define	hio_free_next	hio_next[0]
@@ -1176,6 +1180,7 @@ ggate_recv_thread(void *arg)
 		ggio->gctl_error = 0;
 		hio->hio_done = false;
 		hio->hio_replication = res->hr_replication;
+		hio->hio_memsyncacked = false;
 		pjdlog_debug(2,
 		    "ggate_recv: (%p) Waiting for request from the kernel.",
 		    hio);
@@ -1710,6 +1715,7 @@ remote_recv_thread(void *arg)
 			TAILQ_REMOVE(&hio_recv_list[ncomp], hio,
 			    hio_next[ncomp]);
 			mtx_unlock(&hio_recv_list_lock[ncomp]);
+			hio->hio_errors[ncomp] = ENOTCONN;
 			goto done_queue;
 		}
 		if (hast_proto_recv_hdr(res->hr_remotein, &nv) == -1) {
@@ -1811,6 +1817,12 @@ done_queue:
 				PJDLOG_ASSERT(!memsyncack);
 				break;
 			case 1:
+				if (hio->hio_errors[ncomp] == ENOTCONN) {
+					if (hio->hio_memsyncacked)
+						continue;
+					refcnt_release(&hio->hio_countdown);
+					break;
+				}
 				if (memsyncack) {
 					/*
 					 * Local request already finished, so we
@@ -1825,6 +1837,7 @@ done_queue:
 					pjdlog_debug(2,
 					    "remote_recv: (%p) Moving request back to the recv queue.",
 					    hio);
+					hio->hio_memsyncacked = true;
 					mtx_lock(&hio_recv_list_lock[ncomp]);
 					TAILQ_INSERT_TAIL(&hio_recv_list[ncomp],
 					    hio, hio_next[ncomp]);
@@ -1838,6 +1851,11 @@ done_queue:
 				}
 				continue;
 			case 2:
+				if (hio->hio_errors[ncomp] == ENOTCONN) {
+					PJDLOG_ASSERT(!hio->hio_memsyncacked);
+					refcnt_release(&hio->hio_countdown);
+					continue;
+				}
 				/*
 				 * We received remote memsync reply even before
 				 * local write finished.
@@ -1847,6 +1865,7 @@ done_queue:
 				pjdlog_debug(2,
 				    "remote_recv: (%p) Moving request back to the recv queue.",
 				    hio);
+				hio->hio_memsyncacked = true;
 				mtx_lock(&hio_recv_list_lock[ncomp]);
 				TAILQ_INSERT_TAIL(&hio_recv_list[ncomp], hio,
 				    hio_next[ncomp]);
@@ -2116,6 +2135,7 @@ sync_thread(void *arg __unused)
 		ggio->gctl_error = 0;
 		hio->hio_done = false;
 		hio->hio_replication = res->hr_replication;
+		hio->hio_memsyncacked = false;
 		for (ii = 0; ii < ncomps; ii++)
 			hio->hio_errors[ii] = EINVAL;
 		reqlog(LOG_DEBUG, 2, ggio, "sync: (%p) Sending sync request: ",