Using FreeBSD's re-root capability for fun and profit

#
# Task: I wanted to be able to reinstall a virtual machine that
# was installed with FreeBSD on UFS.  I wanted to have the machine
# run using ZFS filesystems.
#
# FreeBSD 11.0-RELEASE contains kernel and userland support allowing
# re-rooting of a running system.  The user component of the re-root
# support is provided by the reboot program.
#
# The basic strategy is to create a ram disk big enough to copy the
# entire UFS filesystem into, re-root into that filesystem.  Then
# repartition the disk, create ZFS filesystems, and copy all the data
# from the ramdisk into the ZFS filesystems.
#
# Original work by: Kurt Lidl  November, 2017

# '
# ** WARNING ** WARNING ** WARNING ** WARNING ** WARNING ** WARNING **
#
# This procedure is inherently dangerous.  Any data on the machine could
# disappear without chance of recovery if something goes wrong during
# this procedure.  BACKUP ALL DATA BEFORE TRYING THIS PROCEDURE!
# This procedure could easily render your machine unbootable,
# as it destroys the on-disk configuration while running from the
# created ramdisk.  A reboot, crash of the machine, or power outage
# during this procedure is likely to result in the machine being unable
# to reboot and total loss of all data.
#
# ** WARNING ** WARNING ** WARNING ** WARNING ** WARNING ** WARNING **
# '

# First, cleanup the running machine, so that less data has to be
# copied into the ram disk.

# Note: It is assumed this is run inside a /bin/sh shell as the root user
rm -f /var/crash/*
rm -f /var/tmp/*
find /var/db/freebsd-update/files -type f -print | xargs rm -f

# Create 2GB ram disk - this needs to be big enough to hold all the data.
md=$(mdconfig -s 2g)

# Put a UFS filesystem on it, mount it:
newfs /dev/$md
mount /dev/$md /mnt

# This assumes there is a single filesystem to preserve.
# Dump the entire live filesystem, restore the dump into the ramdisk.
dump -0f - / | (cd /mnt; restore -rf -)

# Re-root the system.
kenv vfs.root.mountfrom=ufs:/dev/$md
reboot -r

# After re-rooting to the MFS device and rebooting, everything should
# come back up on the network, so login to the machine again and
# start a new /bin/sh session

exec /bin/sh

# On my host, there was an unforseen problem.  There was swap space
# added to the system via the /etc/fstab file, so even though the root
# filesystem was now entirely mfs-based, the swap space was added
# back to the running system, making the geom of the hard disk busy...

swapoff -a
gpart delete -i 2 ada0s1
gpart delete -i 1 ada0s1
gpart destroy -F ada0

# At this point, the hard disk (ada0) is completely unused
# and the system is entirely running from the ramdisk.
# A reboot or panic of the machine at this point would mean the
# system would not reboot, as there is no longer bootable media...

# Create ZFS partitions and make them bootable,
# Copy over data from the ramdisk.

sysctl kern.geom.label.disk_ident.enable=0
gpart create -s gpt ada0
gpart add -b 128 -s 448K -t freebsd-boot -l boot0 ada0
gpart add -s 4096M -t freebsd-swap -l swap0 ada0
gpart add -t freebsd-zfs -l sys0 ada0
gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot -i 1 ada0

# Load zfs kernel module, make default blocksize be 4K.
kldload zfs
sysctl vfs.zfs.min_auto_ashift=12

# Set a few variables to make the following easier to update.
P=sys
mnt=/mnt
zpool create -f -o cachefile=none -O checksum=fletcher4 \
         -R $mnt -m none $P /dev/gpt/sys0
zfs create $P/ROOT
zfs create $P/ROOT/default
zpool set bootfs=$P/ROOT/default $P
zfs set mountpoint=/ $P/ROOT/default
zfs create $P/vartmp ; zfs set mountpoint=/var/tmp $P/vartmp
zfs create $P/home ; zfs set mountpoint=/usr/home $P/home
zfs mount -a

# Copy the mfs filesystem back to the ZFS pool.
dump -0f - / | (cd /mnt; restore -rf -)

# Fix mountpoint of the /home directory.
zfs set mountpoint=/home $P/home

# Fixup the system to be able to boot via zfs.
echo 'zfs_load="YES"' >> /mnt/boot/loader.conf
echo 'zfs_enable="YES"' >> /mnt/etc/rc.conf

# Site-specific settings you might want.  I find them useful.
echo 'clear_tmp_enable="YES"' >> /mnt/etc/rc.conf
echo 'ntpdate_enable="YES"' >> /mnt/etc/rc.conf
echo 'ntpd_enable="YES"' >> /mnt/etc/rc.conf

mv /mnt/etc/fstab /mnt/etc/fstab.hold

echo "/dev/gpt/swap0	none	swap	sw	0	0" > /mnt/etc/fstab
echo "tmpfs		/tmp	tmpfs	rw,mode=1777 0	0" >> /mnt/etc/fstab

sync
sync
reboot

# When the machine reboots, it should be running from ZFS.