!
! Combined version of the psync/locking changes.  Does not include
! the patch posted earlier today to fix state removal.  Apply both
! if needed.
!
Index: sys/contrib/pf/net/pf_ioctl.c
===================================================================
--- sys/contrib/pf/net/pf_ioctl.c	(revision 226527)
+++ sys/contrib/pf/net/pf_ioctl.c	(revision 226544)
@@ -266,7 +266,7 @@
 static volatile VNET_DEFINE(int, pf_pfil_hooked);
 #define V_pf_pfil_hooked	VNET(pf_pfil_hooked)
 VNET_DEFINE(int,		pf_end_threads);
-VNET_DEFINE(struct mtx,		pf_task_mtx);
+struct mtx			pf_task_mtx;
 
 /* pfsync */
 pfsync_state_import_t 		*pfsync_state_import_ptr = NULL;
@@ -287,18 +287,18 @@
 	&VNET_NAME(debug_pfugidhack), 0,
 	"Enable/disable pf user/group rules mpsafe hack");
 
-void
+static void
 init_pf_mutex(void)
 {
 
-	mtx_init(&V_pf_task_mtx, "pf task mtx", NULL, MTX_DEF);
+	mtx_init(&pf_task_mtx, "pf task mtx", NULL, MTX_DEF);
 }
 
-void
+static void
 destroy_pf_mutex(void)
 {
 
-	mtx_destroy(&V_pf_task_mtx);
+	mtx_destroy(&pf_task_mtx);
 }
 void
 init_zone_var(void)
@@ -4259,7 +4259,7 @@
 	struct pfil_head *pfh_inet6;
 #endif
 
-	PF_ASSERT(MA_NOTOWNED);
+	PF_UNLOCK_ASSERT();
 
 	if (V_pf_pfil_hooked)
 		return (0); 
@@ -4300,7 +4300,7 @@
 	struct pfil_head *pfh_inet6;
 #endif
 
-	PF_ASSERT(MA_NOTOWNED);
+	PF_UNLOCK_ASSERT();
 
 	if (V_pf_pfil_hooked == 0)
 		return (0);
@@ -4381,11 +4381,8 @@
 
 	init_zone_var();
 	sx_init(&V_pf_consistency_lock, "pf_statetbl_lock");
-	init_pf_mutex();
-	if (pfattach() < 0) {
-		destroy_pf_mutex();
+	if (pfattach() < 0)
 		return (ENOMEM);
-	}
 
 	return (0);
 }
@@ -4413,14 +4410,13 @@
 	V_pf_end_threads = 1;
 	while (V_pf_end_threads < 2) {
 		wakeup_one(pf_purge_thread);
-		msleep(pf_purge_thread, &V_pf_task_mtx, 0, "pftmo", hz);
+		msleep(pf_purge_thread, &pf_task_mtx, 0, "pftmo", hz);
 	}
 	pfi_cleanup();
 	pf_osfp_flush();
 	pf_osfp_cleanup();
 	cleanup_pf_zone();
 	PF_UNLOCK();
-	destroy_pf_mutex();
 	sx_destroy(&V_pf_consistency_lock);
 	return error;
 }
@@ -4432,10 +4428,12 @@
 
 	switch(type) {
 	case MOD_LOAD:
+		init_pf_mutex();
 		pf_dev = make_dev(&pf_cdevsw, 0, 0, 0, 0600, PF_NAME);
 		break;
 	case MOD_UNLOAD:
 		destroy_dev(pf_dev);
+		destroy_pf_mutex();
 		break;
 	default:
 		error = EINVAL;
@@ -4450,6 +4448,6 @@
 	0
 };
 
-DECLARE_MODULE(pf, pf_mod, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_FIRST);
+DECLARE_MODULE(pf, pf_mod, SI_SUB_PSEUDO, SI_ORDER_FIRST);
 MODULE_VERSION(pf, PF_MODVER);
 #endif /* __FreeBSD__ */
Index: sys/contrib/pf/net/if_pfsync.c
===================================================================
--- sys/contrib/pf/net/if_pfsync.c	(revision 226527)
+++ sys/contrib/pf/net/if_pfsync.c	(revision 226544)
@@ -714,6 +714,8 @@
 	int pool_flags;
 	int error;
 
+	PF_LOCK_ASSERT();
+
 #ifdef __FreeBSD__
 	if (sp->creatorid == 0 && V_pf_status.debug >= PF_DEBUG_MISC) {
 #else
@@ -1469,7 +1471,9 @@
 			if (ISSET(st->state_flags, PFSTATE_NOSYNC))
 				continue;
 
+			PF_LOCK();
 			pfsync_update_state_req(st);
+			PF_UNLOCK();
 		}
 	}
 
@@ -1558,7 +1562,7 @@
 		pf_unlink_state(st);
 	}
 #ifdef __FreeBSD__
-	PF_LOCK();
+	PF_UNLOCK();
 #endif
 	splx(s);
 
@@ -2143,7 +2147,7 @@
 	int q, count = 0;
 
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #else
 	splassert(IPL_NET);
 #endif
@@ -2378,7 +2382,7 @@
 #endif
 
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #else
 	splassert(IPL_SOFTNET);
 #endif
@@ -2430,7 +2434,7 @@
 	struct pfsync_deferral *pd;
 
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #else
 	splassert(IPL_SOFTNET);
 #endif
@@ -2477,7 +2481,7 @@
 	int s;
 
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #else
 	splassert(IPL_SOFTNET);
 #endif
@@ -2560,7 +2564,7 @@
 	int sync = 0;
 
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #else
 	splassert(IPL_SOFTNET);
 #endif
@@ -2625,6 +2629,8 @@
 	size_t nlen = sizeof(struct pfsync_upd_req);
 	int s;
 
+	PF_LOCK_ASSERT();
+
 	/*
 	 * this code does nothing to prevent multiple update requests for the
 	 * same state being generated.
@@ -2670,6 +2676,8 @@
 	struct pfsync_softc *sc = pfsyncif;
 #endif
 
+	PF_LOCK_ASSERT();
+
 	if (sc == NULL)
 		panic("pfsync_update_state_req: nonexistant instance");
 
@@ -2710,7 +2718,7 @@
 #endif
 
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #else
 	splassert(IPL_SOFTNET);
 #endif
@@ -2771,7 +2779,7 @@
 #endif
 
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #else
 	splassert(IPL_SOFTNET);
 #endif
@@ -2801,6 +2809,8 @@
 	size_t nlen = pfsync_qs[q].len;
 	int s;
 
+	PF_LOCK_ASSERT();
+
 #ifdef __FreeBSD__
 	KASSERT(st->sync_state == PFSYNC_S_NONE,
 		("%s: st->sync_state == PFSYNC_S_NONE", __FUNCTION__));
@@ -2825,13 +2835,7 @@
 	if (sc->sc_len + nlen > sc->sc_if.if_mtu) {
 #endif
 		s = splnet();
-#ifdef __FreeBSD__
-		PF_LOCK();
-#endif
 		pfsync_sendout();
-#ifdef __FreeBSD__
-		PF_UNLOCK();
-#endif
 		splx(s);
 
 		nlen = sizeof(struct pfsync_subheader) + pfsync_qs[q].len;
@@ -2888,7 +2892,9 @@
 
 		if (sc->sc_len + nlen > sc->sc_if.if_mtu) {
 			s = splnet();
+			PF_LOCK();
 			pfsync_sendout();
+			PF_UNLOCK();
 			splx(s);
 
 			nlen = sizeof(struct pfsync_subheader) +
@@ -2991,8 +2997,10 @@
 #endif
 		printf("pfsync: received bulk update request\n");
 
+	PF_LOCK();
 	pfsync_bulk_status(PFSYNC_BUS_START);
 	pfsync_bulk_update(sc);
+	PF_UNLOCK();
 }
 
 void
@@ -3003,10 +3011,11 @@
 	int i = 0;
 	int s;
 
+	PF_LOCK_ASSERT();
+
 	s = splsoftnet();
 #ifdef __FreeBSD__
 	CURVNET_SET(sc->sc_ifp->if_vnet);
-	PF_LOCK();
 #endif
 	do {
 		if (st->sync_state == PFSYNC_S_NONE &&
@@ -3043,7 +3052,6 @@
 
 out:
 #ifdef __FreeBSD__
-	PF_UNLOCK();
 	CURVNET_RESTORE();
 #endif
 	splx(s);
@@ -3063,6 +3071,8 @@
 	struct pfsync_softc *sc = pfsyncif;
 #endif
 
+	PF_LOCK_ASSERT();
+
 	bzero(&r, sizeof(r));
 
 	r.subh.action = PFSYNC_ACT_BUS;
@@ -3096,7 +3106,9 @@
 #else
 		timeout_add_sec(&sc->sc_bulkfail_tmo, 5);
 #endif
+		PF_LOCK();
 		pfsync_request_update(0, 0);
+		PF_UNLOCK();
 	} else {
 		/* Pretend like the transfer was ok */
 		sc->sc_ureq_sent = 0;
@@ -3139,19 +3151,15 @@
 #endif
 	int s;
 
+	PF_LOCK_ASSERT();
+
 #ifdef __FreeBSD__
 	if (sc->sc_len + pluslen > sc->sc_ifp->if_mtu) {
 #else
 	if (sc->sc_len + pluslen > sc->sc_if.if_mtu) {
 #endif
 		s = splnet();
-#ifdef __FreeBSD__
-		PF_LOCK();
-#endif
 		pfsync_sendout();
-#ifdef __FreeBSD__
-		PF_UNLOCK();
-#endif
 		splx(s);
 	}
 
@@ -3159,13 +3167,7 @@
 	sc->sc_len += (sc->sc_pluslen = pluslen);
 
 	s = splnet();
-#ifdef __FreeBSD__
-	PF_LOCK();
-#endif
 	pfsync_sendout();
-#ifdef __FreeBSD__
-	PF_UNLOCK();
-#endif
 	splx(s);
 }
 
@@ -3380,7 +3382,7 @@
 }
 
 /* Define startup order. */
-#define	PFSYNC_SYSINIT_ORDER	SI_SUB_PROTO_BEGIN
+#define	PFSYNC_SYSINIT_ORDER	SI_SUB_PROTO_IF
 #define	PFSYNC_MODEVENT_ORDER	(SI_ORDER_FIRST) /* On boot slot in here. */
 #define	PFSYNC_VNET_ORDER	(PFSYNC_MODEVENT_ORDER + 2) /* Later still. */
 
@@ -3430,7 +3432,7 @@
 
 #define PFSYNC_MODVER 1
 
-DECLARE_MODULE(pfsync, pfsync_mod, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY);
+DECLARE_MODULE(pfsync, pfsync_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
 MODULE_VERSION(pfsync, PFSYNC_MODVER);
 MODULE_DEPEND(pfsync, pf, PF_MODVER, PF_MODVER, PF_MODVER);
 #endif /* __FreeBSD__ */
Index: sys/contrib/pf/net/pfvar.h
===================================================================
--- sys/contrib/pf/net/pfvar.h	(revision 226527)
+++ sys/contrib/pf/net/pfvar.h	(revision 226544)
@@ -237,33 +237,25 @@
 		uma_zdestroy(var)
 
 #ifdef __FreeBSD__
-VNET_DECLARE(struct mtx,	 pf_task_mtx);
-#define	V_pf_task_mtx		 VNET(pf_task_mtx)
-
-#define	PF_ASSERT(h)	mtx_assert(&V_pf_task_mtx, (h))
-
-#define	PF_LOCK()	do {				\
-	PF_ASSERT(MA_NOTOWNED);				\
-	mtx_lock(&V_pf_task_mtx);			\
-} while(0)
-#define	PF_UNLOCK()	do {				\
-	PF_ASSERT(MA_OWNED);				\
-	mtx_unlock(&V_pf_task_mtx);			\
-} while(0)
-#else
 extern struct mtx pf_task_mtx;
 
-#define	PF_ASSERT(h)	mtx_assert(&pf_task_mtx, (h))
+#define	PF_LOCK_ASSERT()	mtx_assert(&pf_task_mtx, MA_OWNED)
+#define	PF_UNLOCK_ASSERT()	mtx_assert(&pf_task_mtx, MA_NOTOWNED)
 
 #define	PF_LOCK()	do {				\
-	PF_ASSERT(MA_NOTOWNED);				\
+	PF_UNLOCK_ASSERT();				\
 	mtx_lock(&pf_task_mtx);				\
 } while(0)
 #define	PF_UNLOCK()	do {				\
-	PF_ASSERT(MA_OWNED);				\
+	PF_LOCK_ASSERT();				\
 	mtx_unlock(&pf_task_mtx);			\
 } while(0)
-#endif
+#else
+#define	PF_LOCK_ASSERT()
+#define	PF_UNLOCK_ASSERT()
+#define	PF_LOCK()
+#define	PF_UNLOCK()
+#endif /* __FreeBSD__ */
 
 #define	PF_COPYIN(uaddr, kaddr, len, r)		do {	\
 	PF_UNLOCK();					\
@@ -277,9 +269,6 @@
 	PF_LOCK();					\
 } while(0)
 
-extern void init_pf_mutex(void);
-extern void destroy_pf_mutex(void);
-
 #define	PF_MODVER	1
 #define	PFLOG_MODVER	1
 #define	PFSYNC_MODVER	1
Index: sys/contrib/pf/net/if_pflog.c
===================================================================
--- sys/contrib/pf/net/if_pflog.c	(revision 226527)
+++ sys/contrib/pf/net/if_pflog.c	(revision 226544)
@@ -429,7 +429,7 @@
 
 #define PFLOG_MODVER 1
 
-DECLARE_MODULE(pflog, pflog_mod, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY);
+DECLARE_MODULE(pflog, pflog_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
 MODULE_VERSION(pflog, PFLOG_MODVER);
 MODULE_DEPEND(pflog, pf, PF_MODVER, PF_MODVER, PF_MODVER);
 #endif /* __FreeBSD__ */
Index: sys/contrib/pf/net/pf_table.c
===================================================================
--- sys/contrib/pf/net/pf_table.c	(revision 226527)
+++ sys/contrib/pf/net/pf_table.c	(revision 226544)
@@ -906,7 +906,7 @@
 		pfr_prepare_network(&mask, ad->pfra_af, ad->pfra_net);
 		s = splsoftnet(); /* rn_lookup makes use of globals */
 #ifdef __FreeBSD__
-		PF_ASSERT(MA_OWNED);
+		PF_LOCK_ASSERT();
 #endif
 		ke = (struct pfr_kentry *)rn_lookup(&sa, &mask, head);
 		splx(s);
@@ -1127,7 +1127,7 @@
 
 	s = splsoftnet();
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #endif
 	if (KENTRY_NETWORK(ke)) {
 		pfr_prepare_network(&mask, ke->pfrke_af, ke->pfrke_net);
@@ -1166,7 +1166,7 @@
 
 	s = splsoftnet();
 #ifdef __FreeBSD__
-	PF_ASSERT(MA_OWNED);
+	PF_LOCK_ASSERT();
 #endif
 	if (KENTRY_NETWORK(ke)) {
 		pfr_prepare_network(&mask, ke->pfrke_af, ke->pfrke_net);