--- sys/geom/mirror/g_mirror.c +++ sys/geom/mirror/g_mirror.c @@ -78,7 +78,7 @@ G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, (ident)); \ } while (0) -static eventhandler_tag g_mirror_pre_sync = NULL, g_mirror_post_sync = NULL; +static eventhandler_tag g_mirror_pre_sync = NULL; static int g_mirror_destroy_geom(struct gctl_req *req, struct g_class *mp, struct g_geom *gp); @@ -2697,11 +2697,37 @@ return (0); } +static void +g_mirror_destroy_delayed(void *arg, int flag) +{ + struct g_mirror_softc *sc; + int error; + + if (flag == EV_CANCEL) { + G_MIRROR_DEBUG(1, "Destroying canceled."); + return; + } + sc = arg; + g_topology_unlock(); + sx_xlock(&sc->sc_lock); + KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) == 0, + ("DESTROY flag set on %s.", sc->sc_name)); + KASSERT((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0, + ("DESTROYING flag not set on %s.", sc->sc_name)); + G_MIRROR_DEBUG(1, "Destroying %s (delayed).", sc->sc_name); + error = g_mirror_destroy(sc, G_MIRROR_DESTROY_SOFT); + if (error != 0) { + G_MIRROR_DEBUG(0, "Cannot destroy %s.", sc->sc_name); + sx_xunlock(&sc->sc_lock); + } + g_topology_lock(); +} + static int g_mirror_access(struct g_provider *pp, int acr, int acw, int ace) { struct g_mirror_softc *sc; - int dcr, dcw, dce; + int dcr, dcw, dce, error = 0; g_topology_assert(); G_MIRROR_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name, acr, @@ -2712,21 +2738,32 @@ dce = pp->ace + ace; sc = pp->geom->softc; - if (sc == NULL || LIST_EMPTY(&sc->sc_disks) || - (sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0) { - if (acr <= 0 && acw <= 0 && ace <= 0) - return (0); - else - return (ENXIO); + KASSERT(sc != NULL, ("NULL softc (provider=%s).", pp->name)); + + g_topology_unlock(); + sx_xlock(&sc->sc_lock); + if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROY) != 0 || + LIST_EMPTY(&sc->sc_disks)) { + if (acr > 0 || acw > 0 || ace > 0) + error = ENXIO; + goto end; } - if (dcw == 0 && !sc->sc_idle) { - g_topology_unlock(); - sx_xlock(&sc->sc_lock); + if (dcw == 0 && !sc->sc_idle) g_mirror_idle(sc, dcw); - sx_xunlock(&sc->sc_lock); - g_topology_lock(); + if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0) { + if (acr > 0 || acw > 0 || ace > 0) { + error = ENXIO; + goto end; + } + if (dcr == 0 && dcw == 0 && dce == 0) { + g_post_event(g_mirror_destroy_delayed, sc, M_WAITOK, + sc, NULL); + } } - return (0); +end: + sx_xunlock(&sc->sc_lock); + g_topology_lock(); + return (error); } static struct g_geom * @@ -2813,8 +2850,9 @@ } int -g_mirror_destroy(struct g_mirror_softc *sc, boolean_t force) +g_mirror_destroy(struct g_mirror_softc *sc, int how) { + struct g_mirror_disk *disk; struct g_provider *pp; g_topology_assert_not(); @@ -2824,14 +2862,27 @@ pp = sc->sc_provider; if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) { - if (force) { - G_MIRROR_DEBUG(1, "Device %s is still open, so it " - "can't be definitely removed.", pp->name); - } else { + switch (how) { + case G_MIRROR_DESTROY_SOFT: G_MIRROR_DEBUG(1, "Device %s is still open (r%dw%de%d).", pp->name, pp->acr, pp->acw, pp->ace); return (EBUSY); + case G_MIRROR_DESTROY_DELAYED: + G_MIRROR_DEBUG(1, + "Device %s will be destroyed on last close.", + pp->name); + LIST_FOREACH(disk, &sc->sc_disks, d_next) { + if (disk->d_state == + G_MIRROR_DISK_STATE_SYNCHRONIZING) { + g_mirror_sync_stop(disk, 1); + } + } + sc->sc_flags |= G_MIRROR_DEVICE_FLAG_DESTROYING; + return (EBUSY); + case G_MIRROR_DESTROY_HARD: + G_MIRROR_DEBUG(1, "Device %s is still open, so it " + "can't be definitely removed.", pp->name); } } @@ -2937,6 +2988,7 @@ G_MIRROR_DEBUG(0, "Cannot add disk %s to %s (error=%d).", pp->name, gp->name, error); if (LIST_EMPTY(&sc->sc_disks)) { + g_cancel_event(sc); g_mirror_destroy(sc, 1); g_topology_lock(); return (NULL); @@ -2958,6 +3010,7 @@ g_topology_unlock(); sc = gp->softc; sx_xlock(&sc->sc_lock); + g_cancel_event(sc); error = g_mirror_destroy(gp->softc, 0); if (error != 0) sx_xunlock(&sc->sc_lock); @@ -3087,7 +3140,7 @@ struct g_class *mp; struct g_geom *gp, *gp2; struct g_mirror_softc *sc; - struct g_mirror_disk *disk; + int error; mp = arg; DROP_GIANT(); @@ -3095,42 +3148,19 @@ LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) { if ((sc = gp->softc) == NULL) continue; - g_topology_unlock(); - sx_xlock(&sc->sc_lock); - LIST_FOREACH(disk, &sc->sc_disks, d_next) { - if (disk->d_state == G_MIRROR_DISK_STATE_SYNCHRONIZING) - g_mirror_sync_stop(disk, 1); - } - sx_xunlock(&sc->sc_lock); - g_topology_lock(); - } - g_topology_unlock(); - PICKUP_GIANT(); -} - -static void -g_mirror_shutdown_post_sync(void *arg, int howto) -{ - struct g_class *mp; - struct g_geom *gp, *gp2; - struct g_mirror_softc *sc; - - mp = arg; - DROP_GIANT(); - g_topology_lock(); - LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) { - if ((sc = gp->softc) == NULL) + /* Skip synchronization geom. */ + if (gp == sc->sc_sync.ds_geom) continue; g_topology_unlock(); sx_xlock(&sc->sc_lock); - g_mirror_destroy(sc, 1); + g_cancel_event(sc); + error = g_mirror_destroy(sc, G_MIRROR_DESTROY_DELAYED); + if (error != 0) + sx_xunlock(&sc->sc_lock); g_topology_lock(); } g_topology_unlock(); PICKUP_GIANT(); -#if 0 - tsleep(&gp, PRIBIO, "m:shutdown", hz * 20); -#endif } static void @@ -3139,9 +3169,7 @@ g_mirror_pre_sync = EVENTHANDLER_REGISTER(shutdown_pre_sync, g_mirror_shutdown_pre_sync, mp, SHUTDOWN_PRI_FIRST); - g_mirror_post_sync = EVENTHANDLER_REGISTER(shutdown_post_sync, - g_mirror_shutdown_post_sync, mp, SHUTDOWN_PRI_FIRST); - if (g_mirror_pre_sync == NULL || g_mirror_post_sync == NULL) + if (g_mirror_pre_sync == NULL) G_MIRROR_DEBUG(0, "Warning! Cannot register shutdown event."); } @@ -3151,8 +3179,6 @@ if (g_mirror_pre_sync != NULL) EVENTHANDLER_DEREGISTER(shutdown_pre_sync, g_mirror_pre_sync); - if (g_mirror_post_sync != NULL) - EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_mirror_post_sync); } DECLARE_GEOM_CLASS(g_mirror_class, g_mirror); --- sys/geom/mirror/g_mirror.h +++ sys/geom/mirror/g_mirror.h @@ -153,6 +153,7 @@ #define G_MIRROR_DEVICE_FLAG_DESTROY 0x0100000000000000ULL #define G_MIRROR_DEVICE_FLAG_WAIT 0x0200000000000000ULL +#define G_MIRROR_DEVICE_FLAG_DESTROYING 0x0400000000000000ULL #define G_MIRROR_DEVICE_STATE_STARTING 0 #define G_MIRROR_DEVICE_STATE_RUNNING 1 @@ -209,7 +210,10 @@ #define sc_name sc_geom->name u_int g_mirror_ndisks(struct g_mirror_softc *sc, int state); -int g_mirror_destroy(struct g_mirror_softc *sc, boolean_t force); +#define G_MIRROR_DESTROY_SOFT 0 +#define G_MIRROR_DESTROY_DELAYED 1 +#define G_MIRROR_DESTROY_HARD 2 +int g_mirror_destroy(struct g_mirror_softc *sc, int how); int g_mirror_event_send(void *arg, int state, int flags); struct g_mirror_metadata; int g_mirror_add_disk(struct g_mirror_softc *sc, struct g_provider *pp, --- sys/geom/mirror/g_mirror_ctl.c +++ sys/geom/mirror/g_mirror_ctl.c @@ -628,6 +628,7 @@ const char *name; char param[16]; u_int i; + int how; nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs)); if (nargs == NULL) { @@ -643,6 +644,10 @@ gctl_error(req, "No '%s' argument.", "force"); return; } + if (*force) + how = G_MIRROR_DESTROY_HARD; + else + how = G_MIRROR_DESTROY_SOFT; for (i = 0; i < (u_int)*nargs; i++) { snprintf(param, sizeof(param), "arg%u", i); @@ -656,7 +661,8 @@ gctl_error(req, "No such device: %s.", name); return; } - error = g_mirror_destroy(sc, *force); + g_cancel_event(sc); + error = g_mirror_destroy(sc, how); if (error != 0) { gctl_error(req, "Cannot destroy device %s (error=%d).", sc->sc_geom->name, error); --- sys/geom/raid3/g_raid3.c +++ sys/geom/raid3/g_raid3.c @@ -97,7 +97,7 @@ G_RAID3_DEBUG(4, "%s: Woken up %p.", __func__, (ident)); \ } while (0) -static eventhandler_tag g_raid3_pre_sync = NULL, g_raid3_post_sync = NULL; +static eventhandler_tag g_raid3_pre_sync = NULL; static int g_raid3_destroy_geom(struct gctl_req *req, struct g_class *mp, struct g_geom *gp); @@ -2896,11 +2896,37 @@ return (0); } +static void +g_raid3_destroy_delayed(void *arg, int flag) +{ + struct g_raid3_softc *sc; + int error; + + if (flag == EV_CANCEL) { + G_RAID3_DEBUG(1, "Destroying canceled."); + return; + } + sc = arg; + g_topology_unlock(); + sx_xlock(&sc->sc_lock); + KASSERT((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROY) == 0, + ("DESTROY flag set on %s.", sc->sc_name)); + KASSERT((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROYING) != 0, + ("DESTROYING flag not set on %s.", sc->sc_name)); + G_RAID3_DEBUG(0, "Destroying %s (delayed).", sc->sc_name); + error = g_raid3_destroy(sc, G_RAID3_DESTROY_SOFT); + if (error != 0) { + G_RAID3_DEBUG(0, "Cannot destroy %s.", sc->sc_name); + sx_xunlock(&sc->sc_lock); + } + g_topology_lock(); +} + static int g_raid3_access(struct g_provider *pp, int acr, int acw, int ace) { struct g_raid3_softc *sc; - int dcr, dcw, dce, error; + int dcr, dcw, dce, error = 0; g_topology_assert(); G_RAID3_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name, acr, @@ -2910,17 +2936,12 @@ dcw = pp->acw + acw; dce = pp->ace + ace; - error = 0; sc = pp->geom->softc; - if (sc != NULL) { - if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROY) != 0) - sc = NULL; - else { - g_topology_unlock(); - sx_xlock(&sc->sc_lock); - } - } - if (sc == NULL || + KASSERT(sc != NULL, ("NULL softc (provider=%s).", pp->name)); + + g_topology_unlock(); + sx_xlock(&sc->sc_lock); + if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROY) != 0 || g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) < sc->sc_ndisks - 1) { if (acr > 0 || acw > 0 || ace > 0) error = ENXIO; @@ -2928,11 +2949,19 @@ } if (dcw == 0 && !sc->sc_idle) g_raid3_idle(sc, dcw); + if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROYING) != 0) { + if (acr > 0 || acw > 0 || ace > 0) { + error = ENXIO; + goto end; + } + if (dcr == 0 && dcw == 0 && dce == 0) { + g_post_event(g_raid3_destroy_delayed, sc, M_WAITOK, + sc, NULL); + } + } end: - if (sc != NULL) { - sx_xunlock(&sc->sc_lock); - g_topology_lock(); - } + sx_xunlock(&sc->sc_lock); + g_topology_lock(); return (error); } @@ -3049,7 +3078,7 @@ } int -g_raid3_destroy(struct g_raid3_softc *sc, boolean_t force) +g_raid3_destroy(struct g_raid3_softc *sc, int how) { struct g_provider *pp; @@ -3060,14 +3089,24 @@ pp = sc->sc_provider; if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) { - if (force) { - G_RAID3_DEBUG(1, "Device %s is still open, so it " - "can't be definitely removed.", pp->name); - } else { + switch (how) { + case G_RAID3_DESTROY_SOFT: G_RAID3_DEBUG(1, "Device %s is still open (r%dw%de%d).", pp->name, pp->acr, pp->acw, pp->ace); return (EBUSY); + case G_RAID3_DESTROY_DELAYED: + G_RAID3_DEBUG(1, + "Device %s will be destroyed on last close.", + pp->name); + if (sc->sc_syncdisk != NULL) + g_raid3_sync_stop(sc, 1); + sc->sc_flags |= G_RAID3_DEVICE_FLAG_DESTROYING; + return (EBUSY); + case G_RAID3_DESTROY_HARD: + G_RAID3_DEBUG(1, "Device %s is still open, so it " + "can't be definitely removed.", pp->name); + break; } } @@ -3168,6 +3207,7 @@ pp->name, gp->name, error); if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_NODISK) == sc->sc_ndisks) { + g_cancel_event(sc); g_raid3_destroy(sc, 1); g_topology_lock(); return (NULL); @@ -3189,6 +3229,7 @@ g_topology_unlock(); sc = gp->softc; sx_xlock(&sc->sc_lock); + g_cancel_event(sc); error = g_raid3_destroy(gp->softc, 0); if (error != 0) sx_xunlock(&sc->sc_lock); @@ -3325,6 +3366,7 @@ struct g_class *mp; struct g_geom *gp, *gp2; struct g_raid3_softc *sc; + int error; mp = arg; DROP_GIANT(); @@ -3332,40 +3374,19 @@ LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) { if ((sc = gp->softc) == NULL) continue; - g_topology_unlock(); - sx_xlock(&sc->sc_lock); - if (sc->sc_syncdisk != NULL) - g_raid3_sync_stop(sc, 1); - sx_xunlock(&sc->sc_lock); - g_topology_lock(); - } - g_topology_unlock(); - PICKUP_GIANT(); -} - -static void -g_raid3_shutdown_post_sync(void *arg, int howto) -{ - struct g_class *mp; - struct g_geom *gp, *gp2; - struct g_raid3_softc *sc; - - mp = arg; - DROP_GIANT(); - g_topology_lock(); - LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) { - if ((sc = gp->softc) == NULL) + /* Skip synchronization geom. */ + if (gp == sc->sc_sync.ds_geom) continue; g_topology_unlock(); sx_xlock(&sc->sc_lock); - g_raid3_destroy(sc, 1); + g_cancel_event(sc); + error = g_raid3_destroy(sc, G_RAID3_DESTROY_DELAYED); + if (error != 0) + sx_xunlock(&sc->sc_lock); g_topology_lock(); } g_topology_unlock(); PICKUP_GIANT(); -#if 0 - tsleep(&gp, PRIBIO, "r3:shutdown", hz * 20); -#endif } static void @@ -3374,9 +3395,7 @@ g_raid3_pre_sync = EVENTHANDLER_REGISTER(shutdown_pre_sync, g_raid3_shutdown_pre_sync, mp, SHUTDOWN_PRI_FIRST); - g_raid3_post_sync = EVENTHANDLER_REGISTER(shutdown_post_sync, - g_raid3_shutdown_post_sync, mp, SHUTDOWN_PRI_FIRST); - if (g_raid3_pre_sync == NULL || g_raid3_post_sync == NULL) + if (g_raid3_pre_sync == NULL) G_RAID3_DEBUG(0, "Warning! Cannot register shutdown event."); } @@ -3386,8 +3405,6 @@ if (g_raid3_pre_sync != NULL) EVENTHANDLER_DEREGISTER(shutdown_pre_sync, g_raid3_pre_sync); - if (g_raid3_post_sync != NULL) - EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_raid3_post_sync); } DECLARE_GEOM_CLASS(g_raid3_class, g_raid3); --- sys/geom/raid3/g_raid3.h +++ sys/geom/raid3/g_raid3.h @@ -160,6 +160,7 @@ #define G_RAID3_DEVICE_FLAG_DESTROY 0x0100000000000000ULL #define G_RAID3_DEVICE_FLAG_WAIT 0x0200000000000000ULL +#define G_RAID3_DEVICE_FLAG_DESTROYING 0x0400000000000000ULL #define G_RAID3_DEVICE_STATE_STARTING 0 #define G_RAID3_DEVICE_STATE_DEGRADED 1 @@ -243,7 +244,10 @@ const char *g_raid3_get_diskname(struct g_raid3_disk *disk); u_int g_raid3_ndisks(struct g_raid3_softc *sc, int state); -int g_raid3_destroy(struct g_raid3_softc *sc, boolean_t force); +#define G_RAID3_DESTROY_SOFT 0 +#define G_RAID3_DESTROY_DELAYED 1 +#define G_RAID3_DESTROY_HARD 2 +int g_raid3_destroy(struct g_raid3_softc *sc, int how); int g_raid3_event_send(void *arg, int state, int flags); struct g_raid3_metadata; int g_raid3_add_disk(struct g_raid3_softc *sc, struct g_provider *pp, --- sys/geom/raid3/g_raid3_ctl.c +++ sys/geom/raid3/g_raid3_ctl.c @@ -310,6 +310,7 @@ const char *name; char param[16]; u_int i; + int how; nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs)); if (nargs == NULL) { @@ -325,6 +326,10 @@ gctl_error(req, "No '%s' argument.", "force"); return; } + if (*force) + how = G_RAID3_DESTROY_HARD; + else + how = G_RAID3_DESTROY_SOFT; for (i = 0; i < (u_int)*nargs; i++) { snprintf(param, sizeof(param), "arg%u", i); @@ -338,7 +343,8 @@ gctl_error(req, "No such device: %s.", name); return; } - error = g_raid3_destroy(sc, *force); + g_cancel_event(sc); + error = g_raid3_destroy(sc, how); if (error != 0) { gctl_error(req, "Cannot destroy device %s (error=%d).", sc->sc_geom->name, error);