diff --git a/sys/cam/ata/ata_da.c b/sys/cam/ata/ata_da.c index 6ea25c2..622b6e6 100644 --- a/sys/cam/ata/ata_da.c +++ b/sys/cam/ata/ata_da.c @@ -805,7 +805,8 @@ adadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t len } if (length > 0) { - xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL); + xpt_setup_ccb_flags(&ccb.ccb_h, periph->path, + CAM_PRIORITY_NORMAL, 0, CAM_CCB_DUMP); ccb.ccb_h.ccb_state = ADA_CCB_DUMP; cam_fill_ataio(&ccb.ataio, 0, @@ -839,7 +840,8 @@ adadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t len } if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) { - xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL); + xpt_setup_ccb_flags(&ccb.ccb_h, periph->path, + CAM_PRIORITY_NORMAL, 0, CAM_CCB_DUMP); ccb.ccb_h.ccb_state = ADA_CCB_DUMP; cam_fill_ataio(&ccb.ataio, diff --git a/sys/cam/cam_ccb.h b/sys/cam/cam_ccb.h index 28abc30..fd600da0 100644 --- a/sys/cam/cam_ccb.h +++ b/sys/cam/cam_ccb.h @@ -112,7 +112,8 @@ typedef enum { typedef enum { CAM_USER_DATA_ADDR = 0x00000002,/* Userspace data pointers */ CAM_SG_FORMAT_IOVEC = 0x00000004,/* iovec instead of busdma S/G*/ - CAM_UNMAPPED_BUF = 0x00000008 /* use unmapped I/O */ + CAM_UNMAPPED_BUF = 0x00000008,/* use unmapped I/O */ + CAM_CCB_DUMP = 0x00000010,/* used for dump I/O */ } ccb_xflags; /* XPT Opcodes for xpt_action */ @@ -1253,7 +1254,6 @@ cam_fill_csio(struct ccb_scsiio *csio, u_int32_t retries, { csio->ccb_h.func_code = XPT_SCSI_IO; csio->ccb_h.flags = flags; - csio->ccb_h.xflags = 0; csio->ccb_h.retry_count = retries; csio->ccb_h.cbfcnp = cbfcnp; csio->ccb_h.timeout = timeout; @@ -1273,7 +1273,6 @@ cam_fill_ctio(struct ccb_scsiio *csio, u_int32_t retries, { csio->ccb_h.func_code = XPT_CONT_TARGET_IO; csio->ccb_h.flags = flags; - csio->ccb_h.xflags = 0; csio->ccb_h.retry_count = retries; csio->ccb_h.cbfcnp = cbfcnp; csio->ccb_h.timeout = timeout; diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c index c2dd6a5..7f22cd1 100644 --- a/sys/cam/cam_xpt.c +++ b/sys/cam/cam_xpt.c @@ -165,6 +165,7 @@ struct cam_doneq { }; static struct cam_doneq cam_doneqs[MAXCPU]; +static struct cam_doneq cam_dumpq; static int cam_num_doneqs; static struct proc *cam_proc; @@ -256,7 +257,8 @@ static int xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo, static xpt_devicefunc_t xptpassannouncefunc; static void xptaction(struct cam_sim *sim, union ccb *work_ccb); static void xptpoll(struct cam_sim *sim); -static void camisr_runqueue(void); +static void camisr_runqueue(struct cam_doneq *queue); +static void camisr_runqueues(void); static void xpt_done_process(struct ccb_hdr *ccb_h); static void xpt_done_td(void *); static dev_match_ret xptbusmatch(struct dev_match_pattern *patterns, @@ -928,6 +930,8 @@ xpt_init(void *dummy) "- failing attach\n"); return (ENOMEM); } + mtx_init(&cam_dumpq.cam_doneq_mtx, "CAM dumpq", NULL, MTX_DEF); + STAILQ_INIT(&cam_dumpq.cam_doneq); /* * Register a callback for when interrupts are enabled. */ @@ -3036,7 +3040,7 @@ xpt_polled_action(union ccb *start_ccb) CAM_SIM_LOCK(sim); (*(sim->sim_poll))(sim); CAM_SIM_UNLOCK(sim); - camisr_runqueue(); + camisr_runqueues(); mtx_lock(&devq->send_mtx); } dev->ccbq.dev_openings++; @@ -3048,9 +3052,9 @@ xpt_polled_action(union ccb *start_ccb) CAM_SIM_LOCK(sim); (*(sim->sim_poll))(sim); CAM_SIM_UNLOCK(sim); - camisr_runqueue(); - if ((start_ccb->ccb_h.status & CAM_STATUS_MASK) - != CAM_REQ_INPROG) + camisr_runqueues(); + if ((start_ccb->ccb_h.status & CAM_STATUS_MASK) != + CAM_REQ_INPROG) break; DELAY(100); } @@ -3340,7 +3344,7 @@ xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb) void xpt_setup_ccb_flags(struct ccb_hdr *ccb_h, struct cam_path *path, - u_int32_t priority, u_int32_t flags) + uint32_t priority, uint32_t flags, uint32_t xflags) { CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n")); @@ -3359,13 +3363,13 @@ xpt_setup_ccb_flags(struct ccb_hdr *ccb_h, struct cam_path *path, } ccb_h->pinfo.index = CAM_UNQUEUED_INDEX; ccb_h->flags = flags; - ccb_h->xflags = 0; + ccb_h->xflags = xflags; } void xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority) { - xpt_setup_ccb_flags(ccb_h, path, priority, /*flags*/ 0); + xpt_setup_ccb_flags(ccb_h, path, priority, /*flags*/ 0, /*xflags*/ 0); } /* Path manipulation functions */ @@ -4438,9 +4442,13 @@ xpt_done(union ccb *done_ccb) if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0) return; - hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id + - done_ccb->ccb_h.target_lun) % cam_num_doneqs; - queue = &cam_doneqs[hash]; + if (__predict_false((done_ccb->ccb_h.xflags & CAM_CCB_DUMP) != 0)) + queue = &cam_dumpq; + else { + hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id + + done_ccb->ccb_h.target_lun) % cam_num_doneqs; + queue = &cam_doneqs[hash]; + } mtx_lock(&queue->cam_doneq_mtx); run = (queue->cam_doneq_sleep && STAILQ_EMPTY(&queue->cam_doneq)); STAILQ_INSERT_TAIL(&queue->cam_doneq, &done_ccb->ccb_h, sim_links.stqe); @@ -5271,22 +5279,31 @@ xpt_done_td(void *arg) } static void -camisr_runqueue(void) +camisr_runqueue(struct cam_doneq *queue) { - struct ccb_hdr *ccb_h; - struct cam_doneq *queue; - int i; + struct ccb_hdr *ccb_h; - /* Process global queues. */ - for (i = 0; i < cam_num_doneqs; i++) { - queue = &cam_doneqs[i]; - mtx_lock(&queue->cam_doneq_mtx); - while ((ccb_h = STAILQ_FIRST(&queue->cam_doneq)) != NULL) { - STAILQ_REMOVE_HEAD(&queue->cam_doneq, sim_links.stqe); - mtx_unlock(&queue->cam_doneq_mtx); - xpt_done_process(ccb_h); - mtx_lock(&queue->cam_doneq_mtx); - } + mtx_lock(&queue->cam_doneq_mtx); + while ((ccb_h = STAILQ_FIRST(&queue->cam_doneq)) != NULL) { + STAILQ_REMOVE_HEAD(&queue->cam_doneq, sim_links.stqe); mtx_unlock(&queue->cam_doneq_mtx); + xpt_done_process(ccb_h); + mtx_lock(&queue->cam_doneq_mtx); } + mtx_unlock(&queue->cam_doneq_mtx); +} + +static void +camisr_runqueues(void) +{ + int i; + + if (dumping) { + camisr_runqueue(&cam_dumpq); + return; + } + + /* Process global queues. */ + for (i = 0; i < cam_num_doneqs; i++) + camisr_runqueue(&cam_doneqs[i]); } diff --git a/sys/cam/cam_xpt.h b/sys/cam/cam_xpt.h index ba5c924..8a8d63a 100644 --- a/sys/cam/cam_xpt.h +++ b/sys/cam/cam_xpt.h @@ -72,8 +72,8 @@ union ccb *xpt_alloc_ccb_nowait(void); void xpt_free_ccb(union ccb *free_ccb); void xpt_setup_ccb_flags(struct ccb_hdr *ccb_h, struct cam_path *path, - u_int32_t priority, - u_int32_t flags); + uint32_t priority, + uint32_t flags, uint32_t xflags); void xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority); diff --git a/sys/cam/scsi/scsi_da.c b/sys/cam/scsi/scsi_da.c index 0a7f627..04486ba 100644 --- a/sys/cam/scsi/scsi_da.c +++ b/sys/cam/scsi/scsi_da.c @@ -1482,7 +1482,8 @@ dadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t leng } if (length > 0) { - xpt_setup_ccb(&csio.ccb_h, periph->path, CAM_PRIORITY_NORMAL); + xpt_setup_ccb_flags(&csio.ccb_h, periph->path, + CAM_PRIORITY_NORMAL, 0, CAM_CCB_DUMP); csio.ccb_h.ccb_state = DA_CCB_DUMP; scsi_read_write(&csio, /*retries*/0, @@ -1515,7 +1516,8 @@ dadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t leng */ if ((softc->quirks & DA_Q_NO_SYNC_CACHE) == 0) { - xpt_setup_ccb(&csio.ccb_h, periph->path, CAM_PRIORITY_NORMAL); + xpt_setup_ccb_flags(&csio.ccb_h, periph->path, + CAM_PRIORITY_NORMAL, 0, CAM_CCB_DUMP); csio.ccb_h.ccb_state = DA_CCB_DUMP; scsi_synchronize_cache(&csio, /*retries*/0, diff --git a/sys/cam/scsi/scsi_pass.c b/sys/cam/scsi/scsi_pass.c index 09cda5b..b63bbeb 100644 --- a/sys/cam/scsi/scsi_pass.c +++ b/sys/cam/scsi/scsi_pass.c @@ -1915,7 +1915,7 @@ passdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread * it is a pointer to a kernel data structure. */ xpt_setup_ccb_flags(&ccb->ccb_h, periph->path, priority, - ccb->ccb_h.flags); + ccb->ccb_h.flags, 0); /* * Setup our done routine. There is no way for the user to