diff -ruNp netgraph.prev/ng_base.c netgraph/ng_base.c --- netgraph.prev/ng_base.c 2008-03-28 00:31:36.000000000 +0200 +++ netgraph/ng_base.c 2008-03-29 19:08:34.000000000 +0200 @@ -61,6 +61,8 @@ #include #include #include +#include +#include #include @@ -201,11 +203,10 @@ static int ng_add_hook(node_p node, cons static int ng_generic_msg(node_p here, item_p item, hook_p lasthook); static ng_ID_t ng_decodeidname(const char *name); static int ngb_mod_event(module_t mod, int event, void *data); -static void ng_worklist_remove(node_p node); -static void ngintr(void); +static void ng_worklist_add(node_p node); +static void ngthread(void *); static int ng_apply_item(node_p node, item_p item, int rw); static void ng_flush_input_queue(struct ng_queue * ngq); -static void ng_setisr(node_p node); static node_p ng_ID2noderef(ng_ID_t ID); static int ng_con_nodes(item_p item, node_p node, const char *name, node_p node2, const char *name2); @@ -251,6 +252,10 @@ MALLOC_DEFINE(M_NETGRAPH_MSG, "netgraph_ mtx_lock(&ng_worklist_mtx) #define NG_WORKLIST_UNLOCK() \ mtx_unlock(&ng_worklist_mtx) +#define NG_WORKLIST_SLEEP() \ + mtx_sleep(&ng_worklist, &ng_worklist_mtx, 0, "sleep", 0) +#define NG_WORKLIST_WAKEUP() \ + wakeup_one(&ng_worklist) #ifdef NETGRAPH_DEBUG /*----------------------------------------------*/ /* @@ -1884,12 +1889,7 @@ ng_dequeue(struct ng_queue *ngq, int *rw */ if (HEAD_IS_READER(ngq)) { if (!QUEUED_READER_CAN_PROCEED(ngq)) { - /* - * It's a reader but we can't use it. - * We are stalled so make sure we don't - * get called again until something changes. - */ - ng_worklist_remove(ngq->q_node); + /* It's a reader but we can't use it. */ CTR4(KTR_NET, "%20s: node [%x] (%p) queued reader " "can't proceed; queue flags 0x%lx", __func__, ngq->q_node->nd_ID, ngq->q_node, ngq->q_flags); @@ -1955,10 +1955,7 @@ ng_dequeue(struct ng_queue *ngq, int *rw * state, then it will be coming through in just a moment, * (just as soon as we release the mutex) and keep things * moving. - * Make sure we remove ourselves from the work queue. It - * would be a waste of effort to do all this again. */ - ng_worklist_remove(ngq->q_node); CTR4(KTR_NET, "%20s: node [%x] (%p) can't dequeue anything; " "queue flags 0x%lx", __func__, ngq->q_node->nd_ID, ngq->q_node, ngq->q_flags); @@ -1989,7 +1986,6 @@ ng_dequeue(struct ng_queue *ngq, int *rw * But we know we don't need to be on the work list. */ atomic_add_long(&ngq->q_flags, add_arg); - ng_worklist_remove(ngq->q_node); } else { /* * Since there is still something on the queue @@ -2039,7 +2035,7 @@ ng_queue_rw(struct ng_queue * ngq, item_ * BUT NOT THE REVERSE! */ if (NEXT_QUEUED_ITEM_CAN_PROCEED(ngq)) - ng_setisr(ngq->q_node); + ng_worklist_add(ngq->q_node); } @@ -2207,7 +2203,7 @@ ng_upgrade_write(struct ng_queue *ngq, i /* Reverse what we did above. That downgrades us back to reader */ atomic_add_long(&ngq->q_flags, READER_INCREMENT - WRITER_ACTIVE); if (NEXT_QUEUED_ITEM_CAN_PROCEED(ngq)) - ng_setisr(ngq->q_node); + ng_worklist_add(ngq->q_node); mtx_unlock_spin(&(ngq->q_mtx)); return; @@ -2254,11 +2250,6 @@ ng_flush_input_queue(struct ng_queue * n NG_FREE_ITEM(item); NG_QUEUE_LOCK(ngq); } - /* - * Take us off the work queue if we are there. - * We definately have no work to be done. - */ - ng_worklist_remove(ngq->q_node); NG_QUEUE_UNLOCK(ngq); } @@ -2405,7 +2396,7 @@ ng_snd_item(item_p item, int flags) NG_QUEUE_LOCK(ngq); if (NEXT_QUEUED_ITEM_CAN_PROCEED(ngq)) - ng_setisr(ngq->q_node); + ng_worklist_add(ngq->q_node); NG_QUEUE_UNLOCK(ngq); return (error); @@ -3003,10 +2994,14 @@ out: ************************************************************************/ uma_zone_t ng_qzone; +static int numthreads = 0; /* number of queue threads */ static int maxalloc = 4096;/* limit the damage of a leak */ static int maxdata = 512; /* limit the damage of a DoS */ static int useddata = 0; +TUNABLE_INT("net.graph.threads", &numthreads); +SYSCTL_INT(_net_graph, OID_AUTO, threads, CTLFLAG_RDTUN, &numthreads, + 0, "Number of queue processing threads to create"); TUNABLE_INT("net.graph.maxalloc", &maxalloc); SYSCTL_INT(_net_graph, OID_AUTO, maxalloc, CTLFLAG_RDTUN, &maxalloc, 0, "Maximum number of queue items to allocate"); @@ -3167,7 +3162,9 @@ ng_mod_event(module_t mod, int event, vo static int ngb_mod_event(module_t mod, int event, void *data) { + struct proc *ngp; int error = 0; + int i; switch (event) { case MOD_LOAD: @@ -3190,8 +3187,20 @@ ngb_mod_event(module_t mod, int event, v ng_qzone = uma_zcreate("NetGraph items", sizeof(struct ng_item), NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); uma_zone_set_max(ng_qzone, maxalloc); - netisr_register(NETISR_NETGRAPH, (netisr_t *)ngintr, NULL, - NETISR_MPSAFE); + /* Autoconfigure number of threads. */ + if (numthreads <= 0) + numthreads = mp_ncpus; + /* Create threads. */ + if ((error = kproc_create(ngthread, NULL, &ngp, + 0, 0, "ng_queue")) != 0) + break; + for (i = 1; i < numthreads; i++) { + if (kthread_add(ngthread, NULL, ngp, NULL, + 0, 0, "ng_queue%d", i)) { + numthreads = i - 1; + break; + } + } break; case MOD_UNLOAD: /* You can't unload it because an interface may be using it. */ @@ -3346,30 +3355,25 @@ SYSCTL_PROC(_debug, OID_AUTO, ng_dump_it /*********************************************************************** * Worklist routines **********************************************************************/ -/* NETISR thread enters here */ /* * Pick a node off the list of nodes with work, - * try get an item to process off it. - * If there are no more, remove the node from the list. + * try get an item to process off it. Remove the node from the list. */ static void -ngintr(void) +ngthread(void *arg) { - item_p item; - node_p node = NULL; - for (;;) { + node_p node; + + /* Get node from the worklist. Sleep if none. */ NG_WORKLIST_LOCK(); - node = TAILQ_FIRST(&ng_worklist); - if (!node) { - NG_WORKLIST_UNLOCK(); - break; - } - node->nd_flags &= ~NGF_WORKQ; + while ((node = TAILQ_FIRST(&ng_worklist)) == NULL) + NG_WORKLIST_SLEEP(); TAILQ_REMOVE(&ng_worklist, node, nd_work); NG_WORKLIST_UNLOCK(); CTR3(KTR_NET, "%20s: node [%x] (%p) taken off worklist", __func__, node->nd_ID, node); + /* * We have the node. We also take over the reference * that the list had on it. @@ -3380,11 +3384,13 @@ ngintr(void) * Let the reference go at the last minute. */ for (;;) { + item_p item; int rw; NG_QUEUE_LOCK(&node->nd_input_queue); item = ng_dequeue(&node->nd_input_queue, &rw); if (item == NULL) { + node->nd_flags &= ~NGF_WORKQ; NG_QUEUE_UNLOCK(&node->nd_input_queue); break; /* go look for another node */ } else { @@ -3398,31 +3404,13 @@ ngintr(void) } } -static void -ng_worklist_remove(node_p node) -{ - mtx_assert(&node->nd_input_queue.q_mtx, MA_OWNED); - - NG_WORKLIST_LOCK(); - if (node->nd_flags & NGF_WORKQ) { - node->nd_flags &= ~NGF_WORKQ; - TAILQ_REMOVE(&ng_worklist, node, nd_work); - NG_WORKLIST_UNLOCK(); - NG_NODE_UNREF(node); - CTR3(KTR_NET, "%20s: node [%x] (%p) removed from worklist", - __func__, node->nd_ID, node); - } else { - NG_WORKLIST_UNLOCK(); - } -} - /* * XXX * It's posible that a debugging NG_NODE_REF may need * to be outside the mutex zone */ static void -ng_setisr(node_p node) +ng_worklist_add(node_p node) { mtx_assert(&node->nd_input_queue.q_mtx, MA_OWNED); @@ -3433,16 +3421,16 @@ ng_setisr(node_p node) * then put us on. */ node->nd_flags |= NGF_WORKQ; + NG_NODE_REF(node); /* XXX fafe in mutex? */ NG_WORKLIST_LOCK(); TAILQ_INSERT_TAIL(&ng_worklist, node, nd_work); NG_WORKLIST_UNLOCK(); - NG_NODE_REF(node); /* XXX fafe in mutex? */ CTR3(KTR_NET, "%20s: node [%x] (%p) put on worklist", __func__, node->nd_ID, node); + NG_WORKLIST_WAKEUP(); } else CTR3(KTR_NET, "%20s: node [%x] (%p) already on worklist", __func__, node->nd_ID, node); - schednetisr(NETISR_NETGRAPH); }