--- //depot/users/kenm/FreeBSD-test2/sys/cam/cam_periph.c 2016-10-06 19:23:16.000000000 -0600 +++ /usr/home/kenm/perforce4/kenm/FreeBSD-test2/sys/cam/cam_periph.c 2016-10-06 19:23:16.000000000 -0600 --- /tmp/tmp.6866.23 2016-10-06 13:31:26.000000000 -0600 +++ /usr/home/kenm/perforce4/kenm/FreeBSD-test2/sys/cam/cam_periph.c 2016-10-06 13:25:30.773080295 -0600 @@ -179,19 +179,31 @@ */ if ((periph = cam_periph_find(path, name)) != NULL) { + if (periph->periph_status != NULL) + periph->periph_status(periph); + if ((periph->flags & CAM_PERIPH_INVALID) != 0 && (periph->flags & CAM_PERIPH_NEW_DEV_FOUND) == 0) { periph->flags |= CAM_PERIPH_NEW_DEV_FOUND; periph->deferred_callback = ac_callback; periph->deferred_ac = code; return (CAM_REQ_INPROG); + } else if ((periph->flags & CAM_PERIPH_INVALID) + && (periph->flags & CAM_PERIPH_NEW_DEV_FOUND)) { + printf("%s: attempt to re-allocate " + "invalid device %s%d with new device already " + "found rejected flags %#x refcount %d\n", + __func__, periph->periph_name, + periph->unit_number, periph->flags, + periph->refcount); } else { - printf("cam_periph_alloc: attempt to re-allocate " + printf("%s: attempt to re-allocate " "valid device %s%d rejected flags %#x " - "refcount %d\n", periph->periph_name, + "refcount %d\n", __func__, periph->periph_name, periph->unit_number, periph->flags, periph->refcount); } + return (CAM_REQ_INVALID); } --- //depot/users/kenm/FreeBSD-test2/sys/cam/cam_periph.h 2016-10-06 19:23:16.000000000 -0600 +++ /usr/home/kenm/perforce4/kenm/FreeBSD-test2/sys/cam/cam_periph.h 2016-10-06 19:23:16.000000000 -0600 --- /tmp/tmp.6866.29 2016-10-06 13:31:26.000000000 -0600 +++ /usr/home/kenm/perforce4/kenm/FreeBSD-test2/sys/cam/cam_periph.h 2016-10-06 13:25:48.372078835 -0600 @@ -103,10 +103,14 @@ void *arg); typedef void periph_oninv_t (struct cam_periph *periph); typedef void periph_dtor_t (struct cam_periph *periph); +typedef void periph_alloc_rel_t (struct cam_periph *periph); +typedef void periph_status_t (struct cam_periph *periph); struct cam_periph { periph_start_t *periph_start; periph_oninv_t *periph_oninval; periph_dtor_t *periph_dtor; + periph_alloc_rel_t *periph_alloc_rel; + periph_status_t *periph_status; char *periph_name; struct cam_path *path; /* Compiled path to device */ void *softc; @@ -123,6 +127,7 @@ #define CAM_PERIPH_RUN_TASK 0x40 #define CAM_PERIPH_FREE 0x80 #define CAM_PERIPH_ANNOUNCED 0x100 +#define CAM_PERIPH_ALLOC_REF 0x200 uint32_t scheduled_priority; uint32_t immediate_priority; int periph_allocating; --- //depot/users/kenm/FreeBSD-test2/sys/cam/cam_xpt.c 2016-10-06 19:23:16.000000000 -0600 +++ /usr/home/kenm/perforce4/kenm/FreeBSD-test2/sys/cam/cam_xpt.c 2016-10-06 19:23:16.000000000 -0600 --- /tmp/tmp.6866.35 2016-10-06 13:31:26.000000000 -0600 +++ /usr/home/kenm/perforce4/kenm/FreeBSD-test2/sys/cam/cam_xpt.c 2016-10-06 13:25:51.260078627 -0600 @@ -3193,9 +3193,26 @@ uint32_t prio; cam_periph_assert(periph, MA_OWNED); + /* + * If the peripheral has been invalidated, no need to continue, + * the device is gone. + */ + if (periph->flags & CAM_PERIPH_INVALID) + return; + if (periph->periph_allocating) return; periph->periph_allocating = 1; + + /* + * This flag is set by the peripheral on the initial probe, before + * any allocation happens. If it isn't set, that means that the + * peripheral doesn't acquire a reference for allocation, and we + * need to do it here. + */ + if ((periph->flags & CAM_PERIPH_ALLOC_REF) == 0) + cam_periph_doacquire(periph); + CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_allocq(%p)\n", periph)); device = periph->path->device; ccb = NULL; @@ -3238,6 +3255,24 @@ if (ccb != NULL) xpt_release_ccb(ccb); periph->periph_allocating = 0; + + /* + * If the peripheral doesn't have an allocation reference, we need + * to release here every time. If it does, and the peripheral was + * invalidated while we were allocating, we are responsible for + * releasing the allocation reference here. + */ + if (((periph->flags & CAM_PERIPH_ALLOC_REF) == 0) + || (periph->flags & CAM_PERIPH_INVALID)) { + /* + * If the peripheral has a routine for releasing its + * allocation reference, call it here. + */ + if (periph->periph_alloc_rel != NULL) + periph->periph_alloc_rel(periph); + else + cam_periph_release_locked(periph); + } } static void --- //depot/users/kenm/FreeBSD-test2/sys/cam/scsi/scsi_da.c 2016-08-15 15:35:12.000000000 -0600 +++ /usr/home/kenm/perforce4/kenm/FreeBSD-test2/sys/cam/scsi/scsi_da.c 2016-08-15 15:35:12.000000000 -0600 --- /tmp/tmp.6866.125 2016-10-06 13:31:26.000000000 -0600 +++ /usr/home/kenm/perforce4/kenm/FreeBSD-test2/sys/cam/scsi/scsi_da.c 2016-10-06 07:44:11.968491268 -0600 @@ -94,6 +94,18 @@ } da_state; typedef enum { + DA_REF_NONE = 0x000, + DA_REF_OPEN = 0x001, + DA_REF_ALLOC = 0x002, + DA_REF_OTAG = 0x004, + DA_REF_GEOM = 0x008, + DA_REF_MEDIA = 0x010, + DA_REF_TUR = 0x020, + DA_REF_SYSCTL = 0x040, + DA_REF_REPROBE = 0x080, +} da_ref_src; + +typedef enum { DA_FLAG_PACK_INVALID = 0x000001, DA_FLAG_NEW_PACK = 0x000002, DA_FLAG_PACK_LOCKED = 0x000004, @@ -293,6 +305,7 @@ da_state state; da_flags flags; da_quirks quirks; + da_ref_src ref_src; int minimum_cmd_size; int error_inject; int trim_max_ranges; @@ -1305,6 +1318,11 @@ }, }; +static cam_status daacquire(struct cam_periph *periph, da_ref_src src); +static void darelease(struct cam_periph *periph, da_ref_src src, + int locked); +static void daallocrel(struct cam_periph *periph); +static void dastatus(struct cam_periph *periph); static disk_strategy_t dastrategy; static dumper_t dadump; static periph_init_t dainit; @@ -1412,6 +1430,69 @@ static MALLOC_DEFINE(M_SCSIDA, "scsi_da", "scsi_da buffers"); +static cam_status +daacquire(struct cam_periph *periph, da_ref_src src) +{ + struct da_softc *softc; + cam_status status; + + softc = (struct da_softc *)periph->softc; + + KASSERT((softc->ref_src & src) == 0, + ("daacquire: src %#x is already held refcnt %d, mask %#x", + src, periph->refcount, softc->ref_src)); + status = cam_periph_acquire(periph); + if (status == CAM_REQ_CMP) + softc->ref_src |= src; + return (status); +} + +static void +darelease(struct cam_periph *periph, da_ref_src src, int locked) +{ + struct da_softc *softc; + struct mtx *mtx; + + softc = (struct da_softc *)periph->softc; + + mtx = cam_periph_mtx(periph); + + if (locked == 0) + mtx_lock(mtx); + + KASSERT((softc->ref_src & src) != 0, + ("darelease: src %#x is NOT held", src)); + + softc->ref_src &= ~src; + + cam_periph_release_locked(periph); + + if (locked == 0) + mtx_unlock(mtx); +} + +static void +daallocrel(struct cam_periph *periph) +{ + darelease(periph, DA_REF_ALLOC, /*locked*/ 1); +} + +static void +dastatus(struct cam_periph *periph) +{ + struct da_softc *softc; + + + cam_periph_assert(periph, MA_OWNED); + + softc = (struct da_softc *)periph->softc; + + printf("%s%u: ref src %#x, refcount %d, softc refcount %d\n", + periph->periph_name, periph->unit_number, softc->ref_src, + periph->refcount, softc->refcount); + +} + static int daopen(struct disk *dp) { @@ -1420,14 +1501,14 @@ int error; periph = (struct cam_periph *)dp->d_drv1; - if (cam_periph_acquire(periph) != CAM_REQ_CMP) { + if (daacquire(periph, DA_REF_OPEN) != CAM_REQ_CMP) { return (ENXIO); } cam_periph_lock(periph); if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) { cam_periph_unlock(periph); - cam_periph_release(periph); + darelease(periph, DA_REF_OPEN, /*locked*/ 0); return (error); } @@ -1459,7 +1540,7 @@ cam_periph_unlock(periph); if (error != 0) - cam_periph_release(periph); + darelease(periph, DA_REF_OPEN, /*locked*/ 0); return (error); } @@ -1517,7 +1598,7 @@ while (softc->refcount != 0) cam_periph_sleep(periph, &softc->refcount, PRIBIO, "daclose", 1); cam_periph_unlock(periph); - cam_periph_release(periph); + darelease(periph, DA_REF_OPEN, /*locked*/ 0); return (0); } @@ -1709,7 +1790,16 @@ struct cam_periph *periph; periph = (struct cam_periph *)dp->d_drv1; - cam_periph_release(periph); + + printf("(%s%d:%s%d:%d:%d:%jx): got GEOM disk gone callback\n", + periph->periph_name, periph->unit_number, + xpt_path_sim(periph->path)->sim_name, + xpt_path_sim(periph->path)->unit_number, + xpt_path_sim(periph->path)->bus_id, + xpt_path_target_id(periph->path), + (uintmax_t)xpt_path_lun_id(periph->path)); + + darelease(periph, DA_REF_GEOM, /*locked*/ 0); } static void @@ -1719,6 +1809,16 @@ softc = (struct da_softc *)periph->softc; + printf("(%s%d:%s%d:%d:%d:%jx): lost device\n", + periph->periph_name, periph->unit_number, + xpt_path_sim(periph->path)->sim_name, + xpt_path_sim(periph->path)->unit_number, + xpt_path_sim(periph->path)->bus_id, + xpt_path_target_id(periph->path), + (uintmax_t)xpt_path_lun_id(periph->path)); + + cam_periph_assert(periph, MA_OWNED); + /* * De-register any async callbacks. */ @@ -1737,6 +1837,45 @@ cam_iosched_flush(softc->cam_iosched, NULL, ENXIO); /* + * Stop the ordered tag callout. If it isn't pending or executing + * we can go ahead and release the reference. If it couldn't be + * cancelled, the callout will + */ + if (callout_stop(&softc->sendordered_c) != 0) + darelease(periph, DA_REF_OTAG, /*locked*/ 1); + + /* + * Drain the media poll callout. If it isn't pending or executing + * we can go ahead and release the reference. If it couldn't be + * cancelled, we'll get a callback when it is done. + */ + if (callout_stop(&softc->mediapoll_c) != 0) + darelease(periph, DA_REF_MEDIA, /*locked*/ 1); + + /* + * If we are not allocating, we have the responsibility to release + * the reference held for the allocation code (xpt_run_allocq()). + * If the allocation code is active, it will release the reference + * once it completes. + */ + if (periph->periph_allocating == 0) + darelease(periph, DA_REF_ALLOC, /*locked*/ 1); + + /* + * If this device went away during the initial probe, it still has + * an active hold. Release the hold now that it has been + * invalidated. + * + * XXX KDM make sure this doesn't open up any holes in the probe. + */ +#if 0 + if ((softc->flags & DA_FLAG_ANNOUNCED) == 0) { + softc->flags |= DA_FLAG_ANNOUNCED; + cam_periph_unhold(periph); + } +#endif + + /* * Tell GEOM that we've gone away, we'll get a callback when it is * done cleaning up its resources. */ @@ -1750,6 +1889,14 @@ softc = (struct da_softc *)periph->softc; + printf("(%s%d:%s%d:%d:%d:%jx): removing device entry\n", + periph->periph_name, periph->unit_number, + xpt_path_sim(periph->path)->sim_name, + xpt_path_sim(periph->path)->unit_number, + xpt_path_sim(periph->path)->bus_id, + xpt_path_target_id(periph->path), + (uintmax_t)xpt_path_lun_id(periph->path)); + cam_periph_unlock(periph); cam_iosched_fini(softc->cam_iosched); @@ -1768,9 +1915,7 @@ "can't remove sysctl context\n"); } - callout_drain(&softc->mediapoll_c); disk_destroy(softc->disk); - callout_drain(&softc->sendordered_c); free(softc, M_DEVBUF); cam_periph_lock(periph); } @@ -1869,11 +2014,11 @@ } case AC_SCSI_AEN: softc = (struct da_softc *)periph->softc; - if (!cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR)) { - if (cam_periph_acquire(periph) == CAM_REQ_CMP) { - cam_iosched_set_work_flags(softc->cam_iosched, DA_WORK_TUR); - daschedule(periph); - } + if (!cam_iosched_has_work_flags(softc->cam_iosched, + DA_WORK_TUR)) { + cam_iosched_set_work_flags(softc->cam_iosched, + DA_WORK_TUR); + daschedule(periph); } /* FALLTHROUGH */ case AC_SENT_BDR: @@ -1915,7 +2060,7 @@ * periph was held for us when this task was enqueued */ if (periph->flags & CAM_PERIPH_INVALID) { - cam_periph_release(periph); + darelease(periph, DA_REF_SYSCTL, /*locked*/ 0); return; } @@ -1930,7 +2075,7 @@ CTLFLAG_RD, 0, tmpstr); if (softc->sysctl_tree == NULL) { printf("dasysctlinit: unable to allocate sysctl tree\n"); - cam_periph_release(periph); + darelease(periph, DA_REF_SYSCTL, /*locked*/ 0); return; } @@ -2012,7 +2157,7 @@ xpt_action((union ccb *)&cts); cam_periph_unlock(periph); if (cts.ccb_h.status != CAM_REQ_CMP) { - cam_periph_release(periph); + darelease(periph, DA_REF_SYSCTL, /*locked*/ 0); return; } if (cts.protocol == PROTO_SCSI && cts.transport == XPORT_FC) { @@ -2063,7 +2208,7 @@ cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx, softc->sysctl_tree); - cam_periph_release(periph); + darelease(periph, DA_REF_SYSCTL, /*locked*/ 0); } static int @@ -2231,7 +2376,7 @@ softc->flags |= DA_FLAG_ANNOUNCED; cam_periph_unhold(periph); } else - cam_periph_release_locked(periph); + darelease(periph, DA_REF_REPROBE, /*locked*/ 1); } static void @@ -2401,6 +2546,8 @@ softc->rotating = 1; periph->softc = softc; + periph->periph_alloc_rel = daallocrel; + periph->periph_status = dastatus; /* * See if this device has any quirks. @@ -2450,13 +2597,44 @@ (void)cam_periph_hold(periph, PRIBIO); /* + * Acquire a reference for allocating. This will be released by + * the allocation code (xpt_run_allocq()) if the device goes away + * while we're allocating. Otherwise, it'll be released by + * daoninvalidate(). + */ + if (daacquire(periph, DA_REF_ALLOC) != CAM_REQ_CMP) { + cam_periph_unhold(periph); + xpt_print(periph->path, "%s: lost periph during " + "registration!\n", __func__); + return (CAM_REQ_CMP_ERR); + } + /* + * Tell the allocation code that we have acquired a reference for + * it that we will release if it is not active. + */ + periph->flags |= CAM_PERIPH_ALLOC_REF; + + /* + * Acquire a reference to the periph before we start the ordered + * tag callout. We'll release this reference once we have shut down + * the callout. + */ + if (daacquire(periph, DA_REF_OTAG) != CAM_REQ_CMP) { + cam_periph_unhold(periph); + xpt_print(periph->path, "%s: lost periph during " + "registration!\n", __func__); + return (CAM_REQ_CMP_ERR); + } + + /* * Schedule a periodic event to occasionally send an * ordered tag to a device. */ - callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0); + callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), + CALLOUT_RETURNUNLOCKED); callout_reset(&softc->sendordered_c, (da_default_timeout * hz) / DA_ORDEREDTAG_INTERVAL, - dasendorderedtag, softc); + dasendorderedtag, periph); cam_periph_unlock(periph); /* @@ -2545,10 +2723,11 @@ * We'll release this reference once GEOM calls us back (via * dadiskgonecb()) telling us that our provider has been freed. */ - if (cam_periph_acquire(periph) != CAM_REQ_CMP) { + if (daacquire(periph, DA_REF_GEOM) != CAM_REQ_CMP) { xpt_print(periph->path, "%s: lost periph during " "registration!\n", __func__); cam_periph_lock(periph); + cam_periph_unhold(periph); return (CAM_REQ_CMP_ERR); } @@ -2575,14 +2754,28 @@ disk_attr_changed(softc->disk, "GEOM::physpath", M_NOWAIT); /* - * Schedule a periodic media polling events. + * Acquire a reference to the periph before we start the media poll + * callout. We'll release this reference once we have shut down + * the callout. + */ + if (daacquire(periph, DA_REF_MEDIA) != CAM_REQ_CMP) { + xpt_print(periph->path, "%s: lost periph during " + "registration!\n", __func__); + cam_periph_unhold(periph); + return (CAM_REQ_CMP_ERR); + } + + /* + * Schedule a periodic media polling event. */ - callout_init_mtx(&softc->mediapoll_c, cam_periph_mtx(periph), 0); + callout_init_mtx(&softc->mediapoll_c, cam_periph_mtx(periph), + CALLOUT_RETURNUNLOCKED); if ((softc->flags & DA_FLAG_PACK_REMOVABLE) && (cgd->inq_flags & SID_AEN) == 0 && - da_poll_period != 0) + da_poll_period != 0) { callout_reset(&softc->mediapoll_c, da_poll_period * hz, damediapoll, periph); + } xpt_schedule(periph, CAM_PRIORITY_DEV); @@ -2902,8 +3095,23 @@ more: bp = cam_iosched_next_bio(softc->cam_iosched); if (bp == NULL) { - if (cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR)) { - cam_iosched_clr_work_flags(softc->cam_iosched, DA_WORK_TUR); + /* + * Note that we're acquiring a reference to the + * peripheral if we need to send a media poll TUR. + * This keeps the device from going away while + * we have I/O outstanding. + * + * In the general I/O case, we're protected from + * the peripheral going away while I/O is + * outstanding by the acquire in daopen() and + * release in daclose(). GEOM shouldn't send a close + * while I/O is outstanding. + */ + if ((cam_iosched_has_work_flags(softc->cam_iosched, + DA_WORK_TUR)) + && (daacquire(periph, DA_REF_TUR) == CAM_REQ_CMP)) { + cam_iosched_clr_work_flags(softc->cam_iosched, + DA_WORK_TUR); scsi_test_unit_ready(&start_ccb->csio, /*retries*/ da_retry_count, dadone, @@ -2929,10 +3137,10 @@ } } - if (cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR)) { - cam_iosched_clr_work_flags(softc->cam_iosched, DA_WORK_TUR); - cam_periph_release_locked(periph); /* XXX is this still valid? I think so but unverified */ - } + if (cam_iosched_has_work_flags(softc->cam_iosched, + DA_WORK_TUR)) + cam_iosched_clr_work_flags(softc->cam_iosched, + DA_WORK_TUR); if ((bp->bio_flags & BIO_ORDERED) != 0 || (softc->flags & DA_FLAG_NEED_OTAG) != 0) { @@ -4346,7 +4554,7 @@ * we have successfully attached. */ /* increase the refcount */ - if (cam_periph_acquire(periph) == CAM_REQ_CMP) { + if (daacquire(periph, DA_REF_SYSCTL) == CAM_REQ_CMP) { taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task); xpt_announce_periph(periph, announce_buf); @@ -5180,7 +5388,11 @@ /*getcount_only*/0); } xpt_release_ccb(done_ccb); - cam_periph_release_locked(periph); + /* + * Release the reference we acquired when we queued this + * CCB. + */ + darelease(periph, DA_REF_TUR, /*locked*/ 1); return; } default: @@ -5201,7 +5413,7 @@ if (softc->state != DA_STATE_NORMAL) return; - status = cam_periph_acquire(periph); + status = daacquire(periph, DA_REF_REPROBE); KASSERT(status == CAM_REQ_CMP, ("dareprobe: cam_periph_acquire failed")); @@ -5299,16 +5511,25 @@ struct cam_periph *periph = arg; struct da_softc *softc = periph->softc; + if (periph->flags & CAM_PERIPH_INVALID) { + callout_deactivate(&softc->mediapoll_c); + cam_periph_unlock(periph); + darelease(periph, DA_REF_MEDIA, /*locked*/ 0); + return; + } + if (!cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR) && LIST_EMPTY(&softc->pending_ccbs)) { - if (cam_periph_acquire(periph) == CAM_REQ_CMP) { - cam_iosched_set_work_flags(softc->cam_iosched, DA_WORK_TUR); - daschedule(periph); - } + cam_iosched_set_work_flags(softc->cam_iosched, DA_WORK_TUR); + daschedule(periph); } + /* Queue us up again */ if (da_poll_period != 0) - callout_schedule(&softc->mediapoll_c, da_poll_period * hz); + callout_schedule(&softc->mediapoll_c, + da_poll_period * hz); + + cam_periph_unlock(periph); } static void @@ -5469,7 +5690,18 @@ static void dasendorderedtag(void *arg) { - struct da_softc *softc = arg; + struct cam_periph *periph = arg; + struct da_softc *softc = periph->softc; + + /* + * If the callout is pending, that means that it was rescheduled + * just before this instance was due to run. Normally the pending + * bit should be clear when we enter. + */ + if (callout_pending(&softc->sendordered_c)) { + cam_periph_unlock(periph); + return; + } if (da_send_ordered) { if (!LIST_EMPTY(&softc->pending_ccbs)) { @@ -5478,10 +5710,32 @@ softc->flags &= ~DA_FLAG_WAS_OTAG; } } - /* Queue us up again */ - callout_reset(&softc->sendordered_c, - (da_default_timeout * hz) / DA_ORDEREDTAG_INTERVAL, - dasendorderedtag, softc); + + /* + * Check to see whether this peripheral has been invalidated. If + * it has, and the callout has fired, it means that this callout + * was pending and could not be stopped. So, in that case, we need + * to release the reference held on the peripheral for this callout. + * + * This callout was initialized with the CALLOUT_RETURNUNLOCKED + * flag, because in the case where we're the last reference held + * on the peripheral, our call to cam_periph_release() may trigger + * a free of the peripheral driver. If we didn't return unlocked, + * the callout code would have to unlock the mutex after this call + * finishes. If this is the last reference, though, the mutex will + * have already been freed. + */ + if (periph->flags & CAM_PERIPH_INVALID) { + callout_deactivate(&softc->sendordered_c); + cam_periph_unlock(periph); + darelease(periph, DA_REF_OTAG, /*locked*/ 0); + } else { + /* Queue us up again */ + callout_reset(&softc->sendordered_c, + (da_default_timeout * hz) / DA_ORDEREDTAG_INTERVAL, + dasendorderedtag, periph); + cam_periph_unlock(periph); + } } /* --- //depot/users/kenm/FreeBSD-test2/sys/geom/geom_dev.c 2016-06-15 12:35:37.000000000 -0600 +++ /usr/home/kenm/perforce4/kenm/FreeBSD-test2/sys/geom/geom_dev.c 2016-06-15 12:35:37.000000000 -0600 --- /tmp/tmp.6866.162 2016-10-06 13:31:26.000000000 -0600 +++ /usr/home/kenm/perforce4/kenm/FreeBSD-test2/sys/geom/geom_dev.c 2016-10-06 07:44:57.171853029 -0600 @@ -723,6 +723,8 @@ sc = cp->private; g_trace(G_T_TOPOLOGY, "g_dev_callback(%p(%s))", cp, cp->geom->name); + printf("%s: got DEVFS callback\n", cp->geom->name); + mtx_lock(&sc->sc_mtx); sc->sc_dev = NULL; sc->sc_alias = NULL; @@ -757,6 +759,8 @@ if (dev->si_flags & SI_DUMPDEV) (void)set_dumper(NULL, NULL, curthread); + printf("%s: calling destroy_dev_sched_cb\n", cp->geom->name); + /* Destroy the struct cdev *so we get no more requests */ destroy_dev_sched_cb(dev, g_dev_callback, cp); }