Index: opencrypto/crypto.c =================================================================== RCS file: /usr/repo/src/sys/opencrypto/crypto.c,v retrieving revision 1.27 diff -u -p -r1.27 crypto.c --- opencrypto/crypto.c 21 Mar 2007 03:42:51 -0000 1.27 +++ opencrypto/crypto.c 30 Aug 2007 12:56:06 -0000 @@ -68,6 +68,8 @@ __FBSDID("$FreeBSD: src/sys/opencrypto/c #include #include #include +#include +#include #include #include @@ -125,9 +127,9 @@ static int crypto_drivers_num = 0; * have one per-queue but having one simplifies handling of block/unblock * operations. */ -static int crp_sleep = 0; static TAILQ_HEAD(,cryptop) crp_q; /* request queues */ static TAILQ_HEAD(,cryptkop) crp_kq; +static uint64_t crp_q_seq; static struct mtx crypto_q_mtx; #define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx) #define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx) @@ -142,6 +144,7 @@ static struct mtx crypto_q_mtx; */ static TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */ static TAILQ_HEAD(,cryptkop) crp_ret_kq; +static uint64_t crp_ret_q_seq; static struct mtx crypto_ret_q_mtx; #define CRYPTO_RETQ_LOCK() mtx_lock(&crypto_ret_q_mtx) #define CRYPTO_RETQ_UNLOCK() mtx_unlock(&crypto_ret_q_mtx) @@ -160,9 +163,9 @@ SYSCTL_INT(_kern, OID_AUTO, cryptodevall "Enable/disable use of software asym crypto support"); MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); - -static void crypto_proc(void); -static struct proc *cryptoproc; + +static void crypto_proc(void *arg); +static struct proc *cryptoproc[MAXCPU]; static void crypto_ret_proc(void); static struct proc *cryptoretproc; static void crypto_destroy(void); @@ -182,7 +185,7 @@ SYSCTL_INT(_debug, OID_AUTO, crypto_timi static int crypto_init(void) { - int error; + int error, i; mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table", MTX_DEF|MTX_QUIET); @@ -216,12 +219,14 @@ crypto_init(void) goto bad; } - error = kthread_create((void (*)(void *)) crypto_proc, NULL, - &cryptoproc, 0, 0, "crypto"); - if (error) { - printf("crypto_init: cannot start crypto thread; error %d", - error); - goto bad; + for (i = 0; i < mp_ncpus; i++) { + error = kthread_create(crypto_proc, (void *)(uintptr_t)i, + &cryptoproc[i], 0, 0, "crypto %d", i); + if (error) { + printf("crypto_init: cannot start crypto thread; error %d", + error); + goto bad; + } } error = kthread_create((void (*)(void *)) crypto_ret_proc, NULL, @@ -253,7 +258,7 @@ crypto_terminate(struct proc **pp, void p = *pp; *pp = NULL; if (p) { - wakeup_one(q); + wakeup(q); PROC_LOCK(p); /* NB: insure we don't miss wakeup */ CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */ msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0); @@ -265,11 +270,14 @@ crypto_terminate(struct proc **pp, void static void crypto_destroy(void) { + int i; + /* * Terminate any crypto threads. */ CRYPTO_DRIVER_LOCK(); - crypto_terminate(&cryptoproc, &crp_q); + for (i = 0; i < mp_ncpus; i++) + crypto_terminate(&cryptoproc[i], &crp_q); crypto_terminate(&cryptoretproc, &crp_ret_q); CRYPTO_DRIVER_UNLOCK(); @@ -756,8 +764,7 @@ crypto_unblock(u_int32_t driverid, int w cap->cc_qblocked = 0; if (what & CRYPTO_ASYMQ) cap->cc_kqblocked = 0; - if (crp_sleep) - wakeup_one(&crp_q); + wakeup(&crp_q); err = 0; } else err = EINVAL; @@ -785,7 +792,7 @@ crypto_dispatch(struct cryptop *crp) hid = CRYPTO_SESID2HID(crp->crp_sid); - if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { + if (0 && (crp->crp_flags & CRYPTO_F_BATCH) == 0) { /* * Caller marked the request to be processed * immediately; dispatch it directly to the @@ -805,9 +812,9 @@ crypto_dispatch(struct cryptop *crp) } } CRYPTO_Q_LOCK(); + crp->crp_seq = crp_q_seq++; TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); - if (crp_sleep) - wakeup_one(&crp_q); + wakeup_one(&crp_q); CRYPTO_Q_UNLOCK(); return 0; } @@ -827,8 +834,7 @@ crypto_kdispatch(struct cryptkop *krp) if (error == ERESTART) { CRYPTO_Q_LOCK(); TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); - if (crp_sleep) - wakeup_one(&crp_q); + wakeup_one(&crp_q); CRYPTO_Q_UNLOCK(); error = 0; } @@ -1117,6 +1123,7 @@ crypto_done(struct cryptop *crp) if (crypto_timing) crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); #endif +goto rettd; /* * CBIMM means unconditionally do the callback immediately; * CBIFSYNC means do the callback immediately only if the @@ -1147,13 +1154,22 @@ crypto_done(struct cryptop *crp) #endif crp->crp_callback(crp); } else { + struct cryptop *tmp; +rettd: /* * Normal case; queue the callback for the thread. */ CRYPTO_RETQ_LOCK(); - if (CRYPTO_RETQ_EMPTY()) + TAILQ_FOREACH(tmp, &crp_ret_q, crp_next) { + if (crp->crp_seq < tmp->crp_seq) { + TAILQ_INSERT_BEFORE(tmp, crp, crp_next); + break; + } + } + if (tmp == NULL) + TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); + if (crp->crp_seq == crp_ret_q_seq) wakeup_one(&crp_ret_q); /* shared wait channel */ - TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); CRYPTO_RETQ_UNLOCK(); } } @@ -1229,7 +1245,7 @@ crypto_finis(void *chan) * Crypto thread, dispatches crypto requests. */ static void -crypto_proc(void) +crypto_proc(void *arg) { struct cryptop *crp, *submit; struct cryptkop *krp; @@ -1237,6 +1253,18 @@ crypto_proc(void) u_int32_t hid; int result, hint; + printf("%s: Thread %d started.\n", __func__, (int)(uintptr_t)arg); +#ifdef SMP + /* Before sched_bind() to a CPU, wait for all CPUs to go on-line. */ + if (mp_ncpus > 1) { + while (!smp_started) + tsleep(curthread, 0, "crp:smp", hz / 4); + } +#endif + thread_lock(curthread); + sched_bind(curthread, (int)(uintptr_t)arg); + thread_unlock(curthread); + CRYPTO_Q_LOCK(); for (;;) { /* @@ -1276,7 +1304,7 @@ crypto_proc(void) break; } else { submit = crp; - if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) + if (0 && (submit->crp_flags & CRYPTO_F_BATCH) == 0) break; /* keep scanning for more are q'd */ } @@ -1284,11 +1312,13 @@ crypto_proc(void) } if (submit != NULL) { TAILQ_REMOVE(&crp_q, submit, crp_next); + CRYPTO_Q_UNLOCK(); hid = CRYPTO_SESID2HID(submit->crp_sid); cap = crypto_checkdriver(hid); KASSERT(cap != NULL, ("%s:%u Driver disappeared.", __func__, __LINE__)); result = crypto_invoke(cap, submit, hint); + CRYPTO_Q_LOCK(); if (result == ERESTART) { /* * The driver ran out of resources, mark the @@ -1328,7 +1358,9 @@ crypto_proc(void) } if (krp != NULL) { TAILQ_REMOVE(&crp_kq, krp, krp_next); + CRYPTO_Q_UNLOCK(); result = crypto_kinvoke(krp, krp->krp_hid); + CRYPTO_Q_LOCK(); if (result == ERESTART) { /* * The driver ran out of resources, mark the @@ -1359,10 +1391,8 @@ crypto_proc(void) * out of order if dispatched to different devices * and some become blocked while others do not. */ - crp_sleep = 1; msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0); - crp_sleep = 0; - if (cryptoproc == NULL) + if (cryptoproc[(int)(uintptr_t)arg] == NULL) break; cryptostats.cs_intrs++; } @@ -1387,8 +1417,14 @@ crypto_ret_proc(void) for (;;) { /* Harvest return q's for completed ops */ crpt = TAILQ_FIRST(&crp_ret_q); - if (crpt != NULL) - TAILQ_REMOVE(&crp_ret_q, crpt, crp_next); + if (crpt != NULL) { + if (crpt->crp_seq == crp_ret_q_seq) { + TAILQ_REMOVE(&crp_ret_q, crpt, crp_next); + crp_ret_q_seq++; + } else { + crpt = NULL; + } + } krpt = TAILQ_FIRST(&crp_ret_kq); if (krpt != NULL) Index: opencrypto/cryptodev.h =================================================================== RCS file: /usr/repo/src/sys/opencrypto/cryptodev.h,v retrieving revision 1.25 diff -u -p -r1.25 cryptodev.h --- opencrypto/cryptodev.h 9 May 2007 19:37:02 -0000 1.25 +++ opencrypto/cryptodev.h 30 Aug 2007 12:56:11 -0000 @@ -338,6 +338,7 @@ struct cryptop { int (*crp_callback)(struct cryptop *); /* Callback function */ struct bintime crp_tstamp; /* performance time stamp */ + uint64_t crp_seq; }; #define CRYPTO_BUF_CONTIG 0x0