diff --git a/sys/kern/subr_gtaskqueue.c b/sys/kern/subr_gtaskqueue.c index 9bf4b0ca3ad3..fe5b5f243621 100644 --- a/sys/kern/subr_gtaskqueue.c +++ b/sys/kern/subr_gtaskqueue.c @@ -591,18 +591,16 @@ gtaskqueue_create_fast(const char *name, int mflags, } struct taskqgroup_cpu { - LIST_HEAD(, grouptask) tgc_tasks; - struct gtaskqueue *tgc_taskq; - int tgc_cnt; - int tgc_cpu; + LIST_HEAD(, grouptask) tgc_tasks; + struct gtaskqueue *tgc_taskq; + int tgc_cnt; + int tgc_cpu; }; struct taskqgroup { struct taskqgroup_cpu tqg_queue[MAXCPU]; struct mtx tqg_lock; const char * tqg_name; - int tqg_adjusting; - int tqg_stride; int tqg_cnt; }; @@ -625,13 +623,6 @@ taskqgroup_cpu_create(struct taskqgroup *qgroup, int idx, int cpu) qcpu->tgc_cpu = cpu; } -static void -taskqgroup_cpu_remove(struct taskqgroup *qgroup, int idx) -{ - - gtaskqueue_free(qgroup->tqg_queue[idx].tgc_taskq); -} - /* * Find the taskq with least # of tasks that doesn't currently have any * other queues from the uniq identifier. @@ -644,8 +635,9 @@ taskqgroup_find(struct taskqgroup *qgroup, void *uniq) int strict; mtx_assert(&qgroup->tqg_lock, MA_OWNED); - if (qgroup->tqg_cnt == 0) - return (0); + KASSERT(qgroup->tqg_cnt != 0, + ("qgroup %s has no queues", qgroup->tqg_name)); + idx = -1; mincnt = INT_MAX; /* @@ -706,6 +698,9 @@ taskqgroup_attach(struct taskqgroup *qgroup, struct grouptask *gtask, { int cpu, qid, error; + KASSERT(qgroup->tqg_cnt > 0, + ("qgroup %s has no queues", qgroup->tqg_name)); + gtask->gt_uniq = uniq; snprintf(gtask->gt_name, GROUPTASK_NAMELEN, "%s", name ? name : "grouptask"); gtask->gt_dev = dev; @@ -728,30 +723,6 @@ taskqgroup_attach(struct taskqgroup *qgroup, struct grouptask *gtask, mtx_unlock(&qgroup->tqg_lock); } -static void -taskqgroup_attach_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) -{ - int qid, cpu, error; - - mtx_lock(&qgroup->tqg_lock); - qid = taskqgroup_find(qgroup, gtask->gt_uniq); - cpu = qgroup->tqg_queue[qid].tgc_cpu; - if (gtask->gt_dev != NULL && gtask->gt_irq != NULL) { - mtx_unlock(&qgroup->tqg_lock); - error = bus_bind_intr(gtask->gt_dev, gtask->gt_irq, cpu); - mtx_lock(&qgroup->tqg_lock); - if (error) - printf("%s: binding interrupt failed for %s: %d\n", - __func__, gtask->gt_name, error); - - } - qgroup->tqg_queue[qid].tgc_cnt++; - LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); - MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); - gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; - mtx_unlock(&qgroup->tqg_lock); -} - int taskqgroup_attach_cpu(struct taskqgroup *qgroup, struct grouptask *gtask, void *uniq, int cpu, device_t dev, struct resource *irq, const char *name) @@ -793,44 +764,6 @@ taskqgroup_attach_cpu(struct taskqgroup *qgroup, struct grouptask *gtask, return (0); } -static int -taskqgroup_attach_cpu_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) -{ - device_t dev; - struct resource *irq; - int cpu, error, i, qid; - - qid = -1; - dev = gtask->gt_dev; - irq = gtask->gt_irq; - cpu = gtask->gt_cpu; - MPASS(tqg_smp_started); - mtx_lock(&qgroup->tqg_lock); - for (i = 0; i < qgroup->tqg_cnt; i++) - if (qgroup->tqg_queue[i].tgc_cpu == cpu) { - qid = i; - break; - } - if (qid == -1) { - mtx_unlock(&qgroup->tqg_lock); - printf("%s: qid not found for %s cpu=%d\n", __func__, gtask->gt_name, cpu); - return (EINVAL); - } - qgroup->tqg_queue[qid].tgc_cnt++; - LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); - MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); - gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; - mtx_unlock(&qgroup->tqg_lock); - - if (dev != NULL && irq != NULL) { - error = bus_bind_intr(dev, irq, cpu); - if (error) - printf("%s: binding interrupt failed for %s: %d\n", - __func__, gtask->gt_name, error); - } - return (0); -} - void taskqgroup_detach(struct taskqgroup *qgroup, struct grouptask *gtask) { @@ -853,10 +786,11 @@ taskqgroup_detach(struct taskqgroup *qgroup, struct grouptask *gtask) static void taskqgroup_binder(void *ctx) { - struct taskq_bind_task *gtask = (struct taskq_bind_task *)ctx; + struct taskq_bind_task *gtask; cpuset_t mask; int error; + gtask = ctx; CPU_ZERO(&mask); CPU_SET(gtask->bt_cpuid, &mask); error = cpuset_setthread(curthread->td_tid, &mask); @@ -869,7 +803,7 @@ taskqgroup_binder(void *ctx) free(gtask, M_DEVBUF); } -static void +void taskqgroup_bind(struct taskqgroup *qgroup) { struct taskq_bind_task *gtask; @@ -883,7 +817,7 @@ taskqgroup_bind(struct taskqgroup *qgroup) return; for (i = 0; i < qgroup->tqg_cnt; i++) { - gtask = malloc(sizeof (*gtask), M_DEVBUF, M_WAITOK); + gtask = malloc(sizeof(*gtask), M_DEVBUF, M_WAITOK); GTASK_INIT(>ask->bt_task, 0, 0, taskqgroup_binder, gtask); gtask->bt_cpuid = qgroup->tqg_queue[i].tgc_cpu; grouptaskqueue_enqueue(qgroup->tqg_queue[i].tgc_taskq, @@ -891,137 +825,22 @@ taskqgroup_bind(struct taskqgroup *qgroup) } } -static void -taskqgroup_config_init(void *arg) -{ - struct taskqgroup *qgroup = qgroup_config; - LIST_HEAD(, grouptask) gtask_head = LIST_HEAD_INITIALIZER(NULL); - - LIST_SWAP(>ask_head, &qgroup->tqg_queue[0].tgc_tasks, - grouptask, gt_list); - qgroup->tqg_queue[0].tgc_cnt = 0; - taskqgroup_cpu_create(qgroup, 0, 0); - - qgroup->tqg_cnt = 1; - qgroup->tqg_stride = 1; -} - -SYSINIT(taskqgroup_config_init, SI_SUB_TASKQ, SI_ORDER_SECOND, - taskqgroup_config_init, NULL); - -static int -_taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride) -{ - LIST_HEAD(, grouptask) gtask_head = LIST_HEAD_INITIALIZER(NULL); - struct grouptask *gtask; - int i, k, old_cnt, old_cpu, cpu; - - mtx_assert(&qgroup->tqg_lock, MA_OWNED); - - if (cnt < 1 || cnt * stride > mp_ncpus || !tqg_smp_started) { - printf("%s: failed cnt: %d stride: %d " - "mp_ncpus: %d tqg_smp_started: %d\n", - __func__, cnt, stride, mp_ncpus, tqg_smp_started); - return (EINVAL); - } - if (qgroup->tqg_adjusting) { - printf("%s failed: adjusting\n", __func__); - return (EBUSY); - } - qgroup->tqg_adjusting = 1; - old_cnt = qgroup->tqg_cnt; - old_cpu = 0; - if (old_cnt < cnt) - old_cpu = qgroup->tqg_queue[old_cnt].tgc_cpu; - mtx_unlock(&qgroup->tqg_lock); - /* - * Set up queue for tasks added before boot. - */ - if (old_cnt == 0) { - LIST_SWAP(>ask_head, &qgroup->tqg_queue[0].tgc_tasks, - grouptask, gt_list); - qgroup->tqg_queue[0].tgc_cnt = 0; - } - - /* - * If new taskq threads have been added. - */ - cpu = old_cpu; - for (i = old_cnt; i < cnt; i++) { - taskqgroup_cpu_create(qgroup, i, cpu); - - for (k = 0; k < stride; k++) - cpu = CPU_NEXT(cpu); - } - mtx_lock(&qgroup->tqg_lock); - qgroup->tqg_cnt = cnt; - qgroup->tqg_stride = stride; - - /* - * Adjust drivers to use new taskqs. - */ - for (i = 0; i < old_cnt; i++) { - while ((gtask = LIST_FIRST(&qgroup->tqg_queue[i].tgc_tasks))) { - LIST_REMOVE(gtask, gt_list); - qgroup->tqg_queue[i].tgc_cnt--; - LIST_INSERT_HEAD(>ask_head, gtask, gt_list); - } - } - mtx_unlock(&qgroup->tqg_lock); - - while ((gtask = LIST_FIRST(>ask_head))) { - LIST_REMOVE(gtask, gt_list); - if (gtask->gt_cpu == -1) - taskqgroup_attach_deferred(qgroup, gtask); - else if (taskqgroup_attach_cpu_deferred(qgroup, gtask)) - taskqgroup_attach_deferred(qgroup, gtask); - } - -#ifdef INVARIANTS - mtx_lock(&qgroup->tqg_lock); - for (i = 0; i < qgroup->tqg_cnt; i++) { - MPASS(qgroup->tqg_queue[i].tgc_taskq != NULL); - LIST_FOREACH(gtask, &qgroup->tqg_queue[i].tgc_tasks, gt_list) - MPASS(gtask->gt_taskqueue != NULL); - } - mtx_unlock(&qgroup->tqg_lock); -#endif - /* - * If taskq thread count has been reduced. - */ - for (i = cnt; i < old_cnt; i++) - taskqgroup_cpu_remove(qgroup, i); - - taskqgroup_bind(qgroup); - - mtx_lock(&qgroup->tqg_lock); - qgroup->tqg_adjusting = 0; - - return (0); -} - -int -taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride) -{ - int error; - - mtx_lock(&qgroup->tqg_lock); - error = _taskqgroup_adjust(qgroup, cnt, stride); - mtx_unlock(&qgroup->tqg_lock); - - return (error); -} - struct taskqgroup * -taskqgroup_create(const char *name) +taskqgroup_create(const char *name, int cnt, int stride) { struct taskqgroup *qgroup; + int cpu, i, j; qgroup = malloc(sizeof(*qgroup), M_GTASKQUEUE, M_WAITOK | M_ZERO); mtx_init(&qgroup->tqg_lock, "taskqgroup", NULL, MTX_DEF); qgroup->tqg_name = name; - LIST_INIT(&qgroup->tqg_queue[0].tgc_tasks); + qgroup->tqg_cnt = cnt; + for (cpu = i = 0; i < cnt; i++) { + taskqgroup_cpu_create(qgroup, i, cpu); + for (j = 0; j < stride; j++) + cpu = CPU_NEXT(cpu); + } return (qgroup); } diff --git a/sys/sys/gtaskqueue.h b/sys/sys/gtaskqueue.h index 96fd57dfb76d..824e6215fe4f 100644 --- a/sys/sys/gtaskqueue.h +++ b/sys/sys/gtaskqueue.h @@ -77,9 +77,9 @@ int taskqgroup_attach_cpu(struct taskqgroup *qgroup, struct grouptask *grptask, void *uniq, int cpu, device_t dev, struct resource *irq, const char *name); void taskqgroup_detach(struct taskqgroup *qgroup, struct grouptask *gtask); -struct taskqgroup *taskqgroup_create(const char *name); +struct taskqgroup *taskqgroup_create(const char *name, int cnt, int stride); void taskqgroup_destroy(struct taskqgroup *qgroup); -int taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride); +void taskqgroup_bind(struct taskqgroup *qgroup); void taskqgroup_config_gtask_init(void *ctx, struct grouptask *gtask, gtask_fn_t *fn, const char *name); void taskqgroup_config_gtask_deinit(struct grouptask *gtask); @@ -107,22 +107,19 @@ struct taskqgroup *qgroup_##name; \ static void \ taskqgroup_define_##name(void *arg) \ { \ - qgroup_##name = taskqgroup_create(#name); \ + qgroup_##name = taskqgroup_create(#name, (cnt), (stride)); \ } \ - \ SYSINIT(taskqgroup_##name, SI_SUB_TASKQ, SI_ORDER_FIRST, \ - taskqgroup_define_##name, NULL); \ + taskqgroup_define_##name, NULL); \ \ static void \ -taskqgroup_adjust_##name(void *arg) \ +taskqgroup_bind_##name(void *arg) \ { \ - taskqgroup_adjust(qgroup_##name, (cnt), (stride)); \ + taskqgroup_bind(qgroup_##name); \ } \ - \ -SYSINIT(taskqgroup_adj_##name, SI_SUB_SMP, SI_ORDER_ANY, \ - taskqgroup_adjust_##name, NULL) +SYSINIT(taskqgroup_bind_##name, SI_SUB_SMP, SI_ORDER_ANY, \ + taskqgroup_bind_##name, NULL) -TASKQGROUP_DECLARE(net); TASKQGROUP_DECLARE(softirq); #endif /* !_SYS_GTASKQUEUE_H_ */