No panics, No vmcore

KDB: debugger backends: ddb
KDB: current backend: ddb
---<<BOOT>>---
Copyright (c) 1992-2022 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
	The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 14.0-CURRENT #1 ufs-n252521-bac0bd31f7b-dirty: Wed Jan 19 10:28:56 CET 2022
    pho@mercat1.netperf.freebsd.org:/var/tmp/deviant3/sys/amd64/compile/PHO amd64
FreeBSD clang version 13.0.0 (git@github.com:llvm/llvm-project.git llvmorg-13.0.0-0-gd7b669b3a303)
WARNING: WITNESS option enabled, expect reduced performance.
WARNING: DIAGNOSTIC option enabled, expect reduced performance.
VT(vga): resolution 640x480
CPU: Intel(R) Xeon(R) CPU E5-1650 v3 @ 3.50GHz (3500.22-MHz K8-class CPU)
  Origin="GenuineIntel"  Id=0x306f2  Family=0x6  Model=0x3f  Stepping=2
  Features=0xbfebfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM,PBE>
  Features2=0x7ffefbff<SSE3,PCLMULQDQ,DTES64,MON,DS_CPL,VMX,SMX,EST,TM2,SSSE3,SDBG,FMA,CX16,xTPR,PDCM,PCID,DCA,SSE4.1,SSE4.2,x2APIC,MOVBE,POPCNT,TSCDLT,AESNI,XSAVE,OSXSAVE,AVX,F16C,RDRAND>
  AMD Features=0x2c100800<SYSCALL,NX,Page1GB,RDTSCP,LM>
  AMD Features2=0x21<LAHF,ABM>
  Structured Extended Features=0x37ab<FSGSBASE,TSCADJ,BMI1,AVX2,SMEP,BMI2,ERMS,INVPCID,PQM,NFPUSG>
  XSAVE Features=0x1<XSAVEOPT>
  VT-x: PAT,HLT,MTF,PAUSE,EPT,UG,VPID,VID,PostIntr
  TSC: P-state invariant, performance statistics
real memory  = 34359738368 (32768 MB)
avail memory = 33176580096 (31639 MB)
Event timer "LAPIC" quality 600
ACPI APIC Table: <SUPERM SMCI--MB>
FreeBSD/SMP: Multiprocessor System Detected: 12 CPUs
FreeBSD/SMP: 1 package(s) x 6 core(s) x 2 hardware threads
random: registering fast source Intel Secure Key RNG
random: fast provider: "Intel Secure Key RNG"
random: unblocking device.
ioapic0 <Version 2.0> irqs 0-23
ioapic1 <Version 2.0> irqs 24-47
Launching APs: 1 11 3 8 10 9 5 2 4 6 7
random: entropy device external interface
kbd1 at kbdmux0
vtvga0: <VT VGA driver>
smbios0: <System Management BIOS> at iomem 0xf0560-0xf057e
smbios0: Version: 2.8, BCD Revision: 2.7
aesni0: <AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS>
acpi0: <SUPERM SMCI--MB>
acpi0: Power Button (fixed)
cpu0: <ACPI CPU> on acpi0
atrtc0: <AT realtime clock> port 0x70-0x71,0x74-0x77 irq 8 on acpi0
atrtc0: registered as a time-of-day clock, resolution 1.000000s
Event timer "RTC" frequency 32768 Hz quality 0
attimer0: <AT timer> port 0x40-0x43,0x50-0x53 irq 0 on acpi0
Timecounter "i8254" frequency 1193182 Hz quality 0
Event timer "i8254" frequency 1193182 Hz quality 100
hpet0: <High Precision Event Timer> iomem 0xfed00000-0xfed003ff on acpi0
Timecounter "HPET" frequency 14318180 Hz quality 950
Event timer "HPET" frequency 14318180 Hz quality 350
Event timer "HPET1" frequency 14318180 Hz quality 340
Event timer "HPET2" frequency 14318180 Hz quality 340
Event timer "HPET3" frequency 14318180 Hz quality 340
Event timer "HPET4" frequency 14318180 Hz quality 340
Event timer "HPET5" frequency 14318180 Hz quality 340
Event timer "HPET6" frequency 14318180 Hz quality 340
Event timer "HPET7" frequency 14318180 Hz quality 340
Timecounter "ACPI-fast" frequency 3579545 Hz quality 900
acpi_timer0: <24-bit timer at 3.579545MHz> port 0x408-0x40b on acpi0
pcib0: <ACPI Host-PCI bridge> on acpi0
pci0: <ACPI PCI bus> on pcib0
pci0: <dasp, performance counters> at device 11.1 (no driver attached)
pci0: <dasp, performance counters> at device 11.2 (no driver attached)
pci0: <dasp, performance counters> at device 16.1 (no driver attached)
pci0: <dasp, performance counters> at device 16.6 (no driver attached)
pci0: <dasp, performance counters> at device 18.1 (no driver attached)
acpi_syscontainer0: <System Container> on acpi0
acpi_syscontainer1: <System Container> on acpi0
acpi_syscontainer2: <System Container> on acpi0
acpi_syscontainer3: <System Container> on acpi0
apei0: <ACPI Platform Error Interface> on acpi0
pcib1: <ACPI Host-PCI bridge> port 0xcf8-0xcff on acpi0
pci1: <ACPI PCI bus> on pcib1
pcib2: <ACPI PCI-PCI bridge> irq 26 at device 1.0 on pci1
pci2: <ACPI PCI bus> on pcib2
pcib3: <ACPI PCI-PCI bridge> irq 32 at device 2.0 on pci1
pci3: <ACPI PCI bus> on pcib3
pci3: <network, ethernet> at device 0.0 (no driver attached)
pci3: <network, ethernet> at device 0.1 (no driver attached)
pcib4: <ACPI PCI-PCI bridge> irq 40 at device 3.0 on pci1
pci4: <ACPI PCI bus> on pcib4
pci1: <unknown> at device 17.0 (no driver attached)
ahci0: <Intel Wellsburg AHCI SATA controller> port 0xf110-0xf117,0xf100-0xf103,0xf0f0-0xf0f7,0xf0e0-0xf0e3,0xf020-0xf03f mem 0xfb438000-0xfb4387ff irq 16 at device 17.4 on pci1
ahci0: AHCI v1.30 with 4 6Gbps ports, Port Multiplier not supported
ahcich0: <AHCI channel> at channel 0 on ahci0
ahcich1: <AHCI channel> at channel 1 on ahci0
ahcich2: <AHCI channel> at channel 2 on ahci0
ahcich3: <AHCI channel> at channel 3 on ahci0
ahciem0: <AHCI enclosure management bridge> on ahci0
xhci0: <Intel Wellsburg USB 3.0 controller> mem 0xfb400000-0xfb40ffff irq 19 at device 20.0 on pci1
xhci0: 32 bytes context size, 64-bit DMA
usbus0 on xhci0
usbus0: 5.0Gbps Super Speed USB v3.0
pci1: <simple comms> at device 22.0 (no driver attached)
pci1: <simple comms> at device 22.1 (no driver attached)
ehci0: <Intel Wellsburg USB 2.0 controller> mem 0xfb434000-0xfb4343ff irq 18 at device 26.0 on pci1
usbus1: EHCI version 1.0
usbus1 on ehci0
usbus1: 480Mbps High Speed USB v2.0
pcib5: <ACPI PCI-PCI bridge> irq 16 at device 28.0 on pci1
pci5: <ACPI PCI bus> on pcib5
pcib6: <ACPI PCI-PCI bridge> irq 18 at device 28.2 on pci1
pci6: <ACPI PCI bus> on pcib6
pcib7: <ACPI PCI-PCI bridge> at device 0.0 on pci6
pci7: <ACPI PCI bus> on pcib7
vgapci0: <VGA-compatible display> port 0xe000-0xe07f mem 0xfa000000-0xfaffffff,0xfb000000-0xfb01ffff irq 18 at device 0.0 on pci7
vgapci0: Boot video device
pcib8: <ACPI PCI-PCI bridge> irq 16 at device 28.4 on pci1
pci8: <ACPI PCI bus> on pcib8
igb0: <Intel(R) I350 (Copper)> port 0xd020-0xd03f mem 0xfb320000-0xfb33ffff,0xfb344000-0xfb347fff irq 16 at device 0.0 on pci8
igb0: EEPROM V1.63-0 eTrack 0x800009fa
igb0: Using 1024 TX descriptors and 1024 RX descriptors
igb0: queue equality override not set, capping rx_queues at 6 and tx_queues at 6
igb0: Using 6 ugen0.3: <vendor 0x0557 product 0x2419> at usbus0
ukbd0 on uhub3
ukbd0: <vendor 0x0557 product 0x2419, class 0/0, rev 1.10/1.00, addr 2> on usbus0
kbd2 at ukbd0
 fsck** /dev/gpt/tmp
** Last Mounted on /tmp
** Phase 1 - Check Blocks and Sizes
** Phase 2 - Check Pathnames
** Phase 3 - Check Connectivity
** Phase 4 - Check Reference Counts
** Phase 5 - Check Cyl groups
418 files, 22144 used, 1243263 free (71 frags, 155399 blocks, 0.0% fragmentation)

***** FILE SYSTEM IS CLEAN *****
Updating CPU Microcode...
CPU: Intel(R) Xeon(R) CPU E5-1650 v3 @ 3.50GHz (3500.00-MHz K8-class CPU)
  Origin="GenuineIntel"  Id=0x306f2  Family=0x6  Model=0x3f  Stepping=2
  Features=0xbfebfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM,PBE>
  Features2=0x7ffefbff<SSE3,PCLMULQDQ,DTES64,MON,DS_CPL,VMX,SMX,EST,TM2,SSSE3,SDBG,FMA,CX16,xTPR,PDCM,PCID,DCA,SSE4.1,SSE4.2,x2APIC,MOVBE,POPCNT,TSCDLT,AESNI,XSAVE,OSXSAVE,AVX,F16C,RDRAND>
  AMD Features=0x2c100800<SYSCALL,NX,Page1GB,RDTSCP,LM>
  AMD Features2=0x21<LAHF,ABM>
  Structured Extended Features=0x37ab<FSGSBASE,TSCADJ,BMI1,AVX2,SMEP,BMI2,ERMS,INVPCID,PQM,NFPUSG>
  Structured Extended Features3=0x9c000400<MD_CLEAR,IBPB,STIBP,L1DFL,SSBD>
  XSAVE Features=0x1<XSAVEOPT>
  VT-x: PAT,HLT,MTF,PAUSE,EPT,UG,VPID,VID,PostIntr
  TSC: P-state invariant, performance statistics
Done.
ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib /usr/local/lib/compat/pkg /usr/local/lib/compat /usr/local/lib/compat/pkg /usr/local/lib/e2fsprogs /usr/local/lib/gcc10 /usr/local/lib/gcc8 /usr/local/lib/gcc9 /usr/local/lib/perl5/5.32/mach/CORE
32-bit compatibility ldconfig path: /usr/lib32 /usr/local/lib32/compat
Setting hostname: mercat1.netperf.freebsd.org.
Setting up harvesting: PURE_RDRAND,[UMA],[FS_ATIME],SWI,INTERRUPT,NET_NG,[NET_ETHER],NET_TUN,MOUSE,KEYBOARD,ATTACH,CACHED
Feeding entropy: .
Autoloading module: acpi_wmi.ko
Autoloading module: ichsmb.ko
Autoloading module: ioat.ko
acpi_wmi0: <ACPI-WMI mapping> on acpi0
acpi_wmi0: cannot find EC device
ioat0: <HSW IOAT Ch0> mem 0xfb42c000-0xfb42ffff irq 31 at device 4.0 on pci1
ioat0: Capabilities: 2f7<PQ,Extended_APIC_ID,Block_Fill,Move_CRC,DCA,Marker_Skipping,CRC,Page_Break>
ioat1: <HSW IOAT Ch1> mem 0xfb428000-0xfb42bfff irq 39 at device 4.1 on pci1
ioat1: Capabilities: 2f7<PQ,Extended_APIC_ID,Block_Fill,Move_CRC,DCA,Marker_Skipping,CRC,Page_Break>
ioat2: <HSW IOAT Ch2> mem 0xfb424000-0xfb427fff irq 31 at device 4.2 on pci1
ioat2: Capabilities: f7<Extended_APIC_ID,Block_Fill,Move_CRC,DCA,Marker_Skipping,CRC,Page_Break>
ioat3: <HSW IOAT Ch3> mem 0xfb420000-0xfb423fff irq 39 at device 4.3 on pci1
ioat3: Capabilities: f7<Extended_APIC_ID,Block_Fill,Move_CRC,DCA,Marker_Skipping,CRC,Page_Break>
ioat4: <HSW IOAT Ch4> mem 0xfb41c000-0xfb41ffff irq 31 at device 4.4 on pci1
ioat4: Capabilities: f7<Extended_APIC_ID,Block_Fill,Move_CRC,DCA,Marker_Skipping,CRC,Page_Break>
ioat5: <HSW IOAT Ch5> mem 0xfb418000-0xfb41bfff irq 39 at device 4.5 on pci1
ioat5: Capabilities: f7<Extended_APIC_ID,Block_Fill,Move_CRC,DCA,Marker_Skipping,CRC,Page_Break>
ioat6: <HSW IOAT Ch6> mem 0xfb414000-0xfb417fff irq 31 at device 4.6 on pci1
ioat6: Capabilities: f7<Extended_APIC_ID,Block_Fill,Move_CRC,DCA,Marker_Skipping,CRC,Page_Break>
ioat7: <HSW IOAT Ch7> mem 0xfb410000-0xfb413fff irq 39 at device 4.7 on pci1
ioat7: Capabilities: f7<Extended_APIC_ID,Block_Fill,Move_CRC,DCA,Marker_Skipping,CRC,Page_Break>
ichsmb0: <Intel Wellsburg SMBus controller> port 0x580-0x59f mem 0xfb431000-0xfb4310ff irq 18 at device 31.3 on pci1
smbus0: <System Management Bus> on ichsmb0
lo0: link state changed to UP
Autoloading module: acpi_wmi.ko
Autoloading module: ioat.ko
Autoloading module: ioat.ko
Autoloading module: ioat.ko
Autoloading module: ioat.ko
Autoloading module: ioat.ko
Autoloading module: ioat.ko
Autoloading module: ioat.ko
Autoloading module: ioat.ko
Starting Network: igb1.
igb1: flags=8822<BROADCAST,SIMPLEX,MULTICAST> metric 0 mtu 1500
	options=4e507bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6,TXCSUM_IPV6,NOMAP>
	ether 0c:c4:7a:a8:cd:db
	media: Ethernet autoselect
	status: no carrier
	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
Autoloading module: ichsmb.ko
Starting Network: mce0.
mce0: flags=8822<BROADCAST,SIMPLEX,MULTICAST> metric 0 mtu 1500
	options=7eed07bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,VLAN_HWFILTER,VLAN_HWTSO,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6,HWRXTSTMP,NOMAP,TXTLS4,TXTLS6,VXLAN_HWCSUM,VXLAN_HWTSO>
	ether 7c:fe:90:30:ee:38
	media: Ethernet autoselect <full-duplex,rxpause,txpause>
	status: no carrier
	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
Starting Network: mce1.
mce1: flags=8822<BROADCAST,SIMPLEX,MULTICAST> metric 0 mtu 1500
	options=7eed07bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,VLAN_HWFILTER,VLAN_HWTSO,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6,HWRXTSTMP,NOMAP,TXTLS4,TXTLS6,VXLAN_HWCSUM,VXLAN_HWTSO>
	ether 7c:fe:90:30:ee:39
	media: Ethernet autoselect <full-duplex,rxpause,txpause>
	status: no carrier
	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
Autoloading module: uhid.ko
Autoloading module: ums.ko
Autoloading module: usbhid.ko
ums0 on uhub3
ums0: <vendor 0x0557 product 0x2419, class 0/0, rev 1.10/1.00, addr 2> on usbus0
ums0: 3 buttons and [Z] coordinates ID=0
Starting dhclient.
DHCPREQUEST on igb0 to 255.255.255.255 port 67
DHCPACK from 192if_delmulti_locked: detaching ifnet instance 0xfffff80004984800
.168.5.1
bound to 192.168.5.171 -- renewal in 300 seconds.
add host 127.0.0.1: gateway lo0 fib 0: route already in table
add host ::1: gateway lo0 fib 0: route already in table
add net fe80::: gateway ::1
add net ff02::: gateway ::1
add net ::ffff:0.0.0.0: gateway ::1
add net ::0.0.0.0: gateway ::1
Creating and/or trimming log files.
Recovering vi editor sessions:.
Updating motd:.
Clearing /tmp (X related).
Starting syslogd.
Starting watchdogd.
No core dumps found.
Setting date via ntp.
19 Jan 11:01:29 ntpdate[2587]: step time server 192.168.5.1 offset +0.433363 sec
Starting rpcbind.
NFS access cache time=60
Starting statd.
Starting lockd.
Starting powerd.
Security policy loaded: MAC/ntpd (mac_ntpd)
Starting ntpd.
Mounting late filesystems:.
Starting inetd.
Configuring vt: blanktime.
Performing sanity check on sshd configuration.
Starting sshd.
Starting cron.
sysctl: unknown oid 'debug.fail_point.pmap_allocpte_page_alloc' at line 12

Wed Jan 19 11:01
FreeBSD/amd64 (mercat1.netperf.freebsd.org) (ttyu1)

login: Jan 19 11:01:34 mercat1 su[2739]: pho to root on /dev/pts/0

20220119 11:02:01 all (1/20): altbufferflushes.sh
stress2: newblk leak: 135157/135193.
20220119 11:15:28 all (2/20): flock_open_close.sh
20220119 11:18:57 all (3/20): quota6.sh
lock order reversal:
 1st 0xfffffe014feb90d8 ufs (ufs, lockmgr) @ kern/vfs_subr.c:2993
 2nd 0xfffff8003898c230 snaplk (snaplk, lockmgr) @ kern/vfs_subr.c:2993
 3rd 0xfffffe014fd1e0d8 ufs (ufs, lockmgr) @ ufs/ufs/ufs_quota.c:1591
lock order ufs -> snaplk established at:
#0 0xffffffff80c7659e at witness_checkorder+0x31e
#1 0xffffffff80bd481c at lockmgr_xlock_hard+0x6c
#2 0xffffffff80bd5129 at __lockmgr_args+0x1f9
#3 0xffffffff80f1eff5 at _lockmgr_args+0x85
#4 0xffffffff80f20930 at ffs_snapblkfree+0x120
#5 0xffffffff80f0e3c2 at ffs_blkfree+0x62
#6 0xffffffff80f3e334 at freework_freeblock+0x714
#7 0xffffffff80f303ab at handle_workitem_freeblocks+0x1eb
#8 0xffffffff80f313fc at softdep_setup_freeblocks+0x97c
#9 0xffffffff80f1844e at ffs_truncate+0xd8e
#10 0xffffffff80f5390a at ufs_inactive+0x1aa
#11 0xffffffff811ad2d9 at VOP_INACTIVE_APV+0x59
#12 0xffffffff80cfb10a at vinactivef+0xda
#13 0xffffffff80cfa75a at vput_final+0x2aa
#14 0xffffffff80d0eb90 at kern_frmdirat+0x2d0
#15 0xffffffff810e1455 at amd64_syscall+0x145
#16 0xffffffff810b297b at fast_syscall_common+0xf8
lock order snaplk -> ufs attempted at:
#0 0xffffffff80c76e5d at witness_checkorder+0xbdd
#1 0xffffffff80bd40fc at lockmgr_lock_flags+0x16c
#2 0xffffffff80f4887b at ffs_lock+0x14b
#3 0xffffffff811ad74c at VOP_LOCK1_APV+0x3c
#4 0xffffffff80d126b4 at _vn_lock+0x54
#5 0xffffffff80f59bde at dqsync+0x15e
#6 0xffffffff80f59eed at qsyncvp+0x3d
#7 0xffffffff80f5379e at ufs_inactive+0x3e
#8 0xffffffff811ad2d9 at VOP_INACTIVE_APV+0x59
#9 0xffffffff80cfb10a at vinactivef+0xda
#10 0xffffffff80cfa75a at vput_final+0x2aa
#11 0xffffffff80ceb58a at lookup+0xfea
#12 0xffffffff80ce98f8 at namei+0x3f8
#13 0xffffffff80d11e10 at vn_open_cred+0x4b0
#14 0xffffffff80d087d6 at kern_openat+0x286
#15 0xffffffff810e1455 at amd64_syscall+0x145
#16 0xffffffff810b297b at fast_syscall_common+0xf8
Jan 19 11:19:01 mercat1 kernel: pid 64730 (rw), uid 0 inumber 8 on /mnt: filesystem full
Jan 19 11:19:03 mercat1 kernel: pid 64735 (rw), uid 0 inumber 6 on /mnt: filesystem full
lock order reversal:
 1st 0xfffff8003898c230 snaplk (snaplk, lockmgr) @ ufs/ffs/ffs_snapshot.c:2760
 2nd 0xfffffe0037c6c040 bufwait (bufwait, lockmgr) @ ufs/ffs/ffs_softdep.c:14724
lock order bufwait -> snaplk established at:
#0 0xffffffff80c7659e at witness_checkorder+0x31e
#1 0xffffffff80bd481c at lockmgr_xlock_hard+0x6c
#2 0xffffffff80bd5129 at __lockmgr_args+0x1f9
#3 0xffffffff80f1eff5 at _lockmgr_args+0x85
#4 0xffffffff80f22606 at ffs_copyonwrite+0x2e6
#5 0xffffffff80f46e1f at ffs_geom_strategy+0xcf
#6 0xffffffff80cca74d at bstrategy+0xdd
#7 0xffffffff80cc7551 at bufwrite+0x291
#8 0xffffffff80f17646 at bwrite+0xd6
#9 0xffffffff80f17438 at ffs_update+0x3f8
#10 0xffffffff80f38570 at softdep_sync_buf+0xa20
#11 0xffffffff80f4a72d at ffs_syncvnode+0x35d
#12 0xffffffff80f183db at ffs_truncate+0xd1b
#13 0xffffffff80f491cf at ffs_vput_pair+0x2ef
#14 0xffffffff80d11ce2 at vn_open_cred+0x382
#15 0xffffffff80d087d6 at kern_openat+0x286
#16 0xffffffff810e1455 at amd64_syscall+0x145
#17 0xffffffff810b297b at fast_syscall_common+0xf8
lock order snaplk -> bufwait attempted at:
#0 0xffffffff80c76e5d at witness_checkorder+0xbdd
#1 0xffffffff80bd481c at lockmgr_xlock_hard+0x6c
#2 0xffffffff80bd5129 at __lockmgr_args+0x1f9
#3 0xffffffff80f36a27 at getdirtybuf+0x77
#4 0xffffffff80f40c17 at flush_deplist+0x97
#5 0xffffffff80f37a82 at softdep_sync_metadata+0x2e2
#6 0xffffffff80f4a43e at ffs_syncvnode+0x6e
#7 0xffffffff80f1d526 at ffs_snapshot+0x3096
#8 0xffffffff80f445ca at ffs_mount+0x60a
#9 0xffffffff80cf2be1 at vfs_domount_update+0x291
#10 0xffffffff80cee91b at vfs_domount+0x26b
#11 0xffffffff80ced1d8 at vfs_donmount+0x878
#12 0xffffffff80cec929 at sys_nmount+0x69
#13 0xffffffff810e1455 at amd64_syscall+0x145
#14 0xffffffff810b297b at fast_syscall_common+0xf8
Jan 19 11:19:13 mercat1 kernel: pid 64795 (rw), uid 0 inumber 9 on /mnt: filesystem full
lock order reversal:
 1st 0xfffff802da808a98 FFS (FFS Lock, sleep mutex) @ ufs/ufs/ufs_quota.c:755
 2nd 0xfffffe014fec42c0 struct mount mtx (struct mount mtx, sleep mutex) @ ufs/ufs/ufs_quota.c:761
lock order struct mount mtx -> FFS Lock established at:
#0 0xffffffff80c7659e at witness_checkorder+0x31e
#1 0xffffffff80be0dc4 at __mtx_lock_flags+0x94
#2 0xffffffff80f242a5 at softdep_flushfiles+0x395
#3 0xffffffff80f44d7b at ffs_unmount+0xab
#4 0xffffffff80cef92a at dounmount+0x7ea
#5 0xffffffff80cef0f7 at kern_unmount+0x2d7
#6 0xffffffff810e1455 at amd64_syscall+0x145
#7 0xffffffff810b297b at fast_syscall_common+0xf8
lock order FFS Lock -> struct mount mtx attempted at:
#0 0xffffffff80c76e5d at witness_checkorder+0xbdd
#1 0xffffffff80be0dc4 at __mtx_lock_flags+0x94
#2 0xffffffff80f58fad at quotaoff_inchange+0xbd
#3 0xffffffff80f434da at ffs_flushfiles+0x10a
#4 0xffffffff80f23fbd at softdep_flushfiles+0xad
#5 0xffffffff80f44d7b at ffs_unmount+0xab
#6 0xffffffff80cef92a at dounmount+0x7ea
#7 0xffffffff80cef0f7 at kern_unmount+0x2d7
#8 0xffffffff810e1455 at amd64_syscall+0x145
#9 0xffffffff810b297b at fast_syscall_common+0xf8
stress2: newblk leak: 7544/7548.
20220119 11:19:17 all (4/20): quota8.sh
stress2: newblk leak: 21124/28674.
20220119 11:20:01 all (5/20): recursiveflushes.sh
20220119 11:20:04 all (6/20): snap.sh
20220119 11:20:06 all (7/20): snap10.sh
20220119 11:20:09 all (8/20): snap11.sh
20220119 11:22:12 all (9/20): snap12.sh
20220119 11:22:14 all (10/20): snap2-1.sh
20220119 11:22:15 all (11/20): snap2.sh
20220119 11:22:17 all (12/20): snap3.sh
stress2: pts leak: 1/2
20220119 11:42:20 all (13/20): snap4.sh
20220119 12:02:22 all (14/20): snap5-1.sh
20220119 12:12:53 all (15/20): snap5.sh
20220119 12:33:52 all (16/20): snap6.sh
20220119 12:44:22 all (17/20): snap7.sh
20220119 13:04:23 all (18/20): snap8.sh
20220119 13:26:12 all (19/20): snap9.sh
Jan 19 13:26:15 mercat1 kernel: pid 18684 (dd), uid 0 inumber 8 on /mnt: filesystem full
20220119 13:26:15 all (20/20): trim5.sh
20220119 13:26:17 all.sh done, elapsed 0 day(s), 02:24.26
Expensive timeout(9) function: 0xffffffff80f70130(0) 0.012726405 s

20220119 13:43:30 all (1/1): quota8.sh
stress2: newblk leak: 28657/28671.
20220119 13:44:32 all (1/1): quota8.sh
20220119 13:45:30 all (1/1): quota8.sh
20220119 13:46:25 all (1/1): quota8.sh
20220119 13:47:18 all (1/1): quota8.sh
20220119 13:48:17 all (1/1): quota8.sh
20220119 13:49:04 all (1/1): quota8.sh
20220119 13:49:57 all (1/1): quota8.sh
20220119 13:50:54 all (1/1): quota8.sh
20220119 13:51:46 all (1/1): quota8.sh
20220119 13:52:42 all (1/1): quota8.sh
20220119 13:53:32 all (1/1): quota8.sh
20220119 13:54:26 all (1/1): quota8.sh
20220119 13:55:24 all (1/1): quota8.sh
20220119 13:56:21 all (1/1): quota8.sh
20220119 13:57:21 all (1/1): quota8.sh
20220119 13:58:05 all (1/1): quota8.sh
20220119 13:59:00 all (1/1): quota8.sh
20220119 14:00:04 all (1/1): quota8.sh
20220119 14:00:57 all (1/1): quota8.sh
20220119 14:01:53 all (1/1): quota8.sh
20220119 14:02:35 all (1/1): quota8.sh
20220119 14:03:28 all (1/1): quota8.sh
20220119 14:04:19 all (1/1): quota8.sh
20220119 14:05:16 all (1/1): quota8.sh
20220119 14:06:08 all (1/1): quota8.sh
20220119 14:06:50 all (1/1): quota8.sh
20220119 14:07:42 all (1/1): quota8.sh
20220119 14:08:36 all (1/1): quota8.sh
20220119 14:09:31 all (1/1): quota8.sh
20220119 14:10:39 all (1/1): quota8.sh
20220119 14:11:34 all (1/1): quota8.sh
20220119 14:12:21 all (1/1): quota8.sh
20220119 14:13:18 all (1/1): quota8.sh
20220119 14:14:12 all (1/1): quota8.sh
20220119 14:15:02 all (1/1): quota8.sh
20220119 14:15:45 all (1/1): quota8.sh
20220119 14:16:51 all (1/1): quota8.sh
20220119 14:18:00 all (1/1): quota8.sh
stress2: newblk leak: 24583/28674.
20220119 14:18:47 all (1/1): quota8.sh
20220119 14:19:44 all (1/1): quota8.sh
20220119 14:20:37 all (1/1): quota8.sh
20220119 14:21:15 all (1/1): quota8.sh
20220119 14:22:14 all (1/1): quota8.sh
20220119 14:23:07 all (1/1): quota8.sh
20220119 14:24:00 all (1/1): quota8.sh
20220119 14:25:05 all (1/1): quota8.sh
20220119 14:26:05 all (1/1): quota8.sh
20220119 14:26:59 all (1/1): quota8.sh
20220119 14:27:58 all (1/1): quota8.sh
stress2: newblk leak: 28667/28674.
20220119 14:28:51 all (1/1): quota8.sh
Jan 19 14:29:38 mercat1 kernel: pid 27018 (rw), uid 2007 inumber 65850 on /mnt: filesystem full
20220119 14:29:49 all (1/1): quota8.sh
20220119 14:30:41 all (1/1): quota8.sh
20220119 14:31:38 all (1/1): quota8.sh
20220119 14:32:27 all (1/1): quota8.sh
20220119 14:33:20 all (1/1): quota8.sh
20220119 14:34:18 all (1/1): quota8.sh
20220119 14:35:16 all (1/1): quota8.sh
20220119 14:36:09 all (1/1): quota8.sh
20220119 14:37:03 all (1/1): quota8.sh
20220119 14:38:04 all (1/1): quota8.sh
20220119 14:38:56 all (1/1): quota8.sh
20220119 14:39:55 all (1/1): quota8.sh
20220119 14:41:00 all (1/1): quota8.sh
20220119 14:41:53 all (1/1): quota8.sh
20220119 14:42:37 all (1/1): quota8.sh
20220119 14:43:35 all.sh done, elapsed 0 day(s), 01:00.05

20220119 14:54:03 all (1/20): altbufferflushes.sh
stress2: newblk leak: 176139/176153.
20220119 15:06:54 all (2/20): flock_open_close.sh
20220119 15:09:33 all (3/20): quota6.sh
stress2: newblk leak: 1460/1464.
20220119 15:09:37 all (4/20): quota8.sh
stress2: newblk leak: 27208/28674.
20220119 15:10:28 all (5/20): recursiveflushes.sh
20220119 15:10:31 all (6/20): snap.sh
20220119 15:10:33 all (7/20): snap10.sh
20220119 15:10:36 all (8/20): snap11.sh
Jan 19 15:11:37 mercat1 kernel: pid 60761 (rw), uid 0 inumber 5 on /mnt: filesystem full
20220119 15:12:44 all (9/20): snap12.sh
20220119 15:12:46 all (10/20): snap2-1.sh
20220119 15:12:48 all (11/20): snap2.sh
20220119 15:12:50 all (12/20): snap3.sh
20220119 15:32:52 all (13/20): snap4.sh
20220119 15:52:54 all (14/20): snap5-1.sh
20220119 16:03:04 all (15/20): snap5.sh
20220119 16:23:29 all (16/20): snap6.sh
20220119 16:33:59 all (17/20): snap7.sh
Jan 19 16:52:04 mercat1 su[26170]: pho to root on /dev/pts/1
20220119 16:54:01 all (18/20): snap8.sh
20220119 17:16:48 all (19/20): snap9.sh
Jan 19 17:16:50 mercat1 kernel: pid 32345 (dd), uid 0 inumber 8 on /mnt: filesystem full
20220119 17:16:51 all (20/20): trim5.sh
20220119 17:16:53 all.sh done, elapsed 0 day(s), 02:22.50


diff --git a/sys/ufs/ffs/ffs_snapshot.c b/sys/ufs/ffs/ffs_snapshot.c
index 76b377da3cd..0ff0b0e2b3a 100644
--- a/sys/ufs/ffs/ffs_snapshot.c
+++ b/sys/ufs/ffs/ffs_snapshot.c
@@ -181,6 +181,7 @@ static int mapacct_ufs2(struct vnode *, ufs2_daddr_t *, ufs2_daddr_t *,
     struct fs *, ufs_lbn_t, int);
 static int readblock(struct vnode *vp, struct buf *, ufs2_daddr_t);
 static void try_free_snapdata(struct vnode *devvp);
+static void revert_snaplock(struct vnode *, struct vnode *, struct snapdata *);
 static struct snapdata *ffs_snapdata_acquire(struct vnode *devvp);
 static int ffs_bp_snapblk(struct vnode *, struct buf *);
 
@@ -1657,7 +1658,7 @@ ffs_snapremove(vp)
 	struct buf *ibp;
 	struct fs *fs;
 	ufs2_daddr_t numblks, blkno, dblk;
-	int error, i, last, loc;
+	int error, last, loc;
 	struct snapdata *sn;
 
 	ip = VTOI(vp);
@@ -1675,20 +1676,10 @@ ffs_snapremove(vp)
 		sn = devvp->v_rdev->si_snapdata;
 		TAILQ_REMOVE(&sn->sn_head, ip, i_nextsnap);
 		ip->i_nextsnap.tqe_prev = 0;
-		VI_UNLOCK(devvp);
-		lockmgr(&vp->v_lock, LK_EXCLUSIVE, NULL);
-		for (i = 0; i < sn->sn_lock.lk_recurse; i++)
-			lockmgr(&vp->v_lock, LK_EXCLUSIVE, NULL);
-		KASSERT(vp->v_vnlock == &sn->sn_lock,
-			("ffs_snapremove: lost lock mutation")); 
-		vp->v_vnlock = &vp->v_lock;
-		VI_LOCK(devvp);
-		while (sn->sn_lock.lk_recurse > 0)
-			lockmgr(&sn->sn_lock, LK_RELEASE, NULL);
-		lockmgr(&sn->sn_lock, LK_RELEASE, NULL);
+		revert_snaplock(vp, devvp, sn);
 		try_free_snapdata(devvp);
-	} else
-		VI_UNLOCK(devvp);
+	}
+	VI_UNLOCK(devvp);
 	/*
 	 * Clear all BLK_NOCOPY fields. Pass any block claims to other
 	 * snapshots that want them (see ffs_snapblkfree below).
@@ -2158,27 +2149,18 @@ ffs_snapshot_unmount(mp)
 		xp->i_nextsnap.tqe_prev = 0;
 		lockmgr(&sn->sn_lock, LK_INTERLOCK | LK_EXCLUSIVE,
 		    VI_MTX(devvp));
-		/*
-		 * Avoid LOR with above snapshot lock. The LK_NOWAIT should
-		 * never fail as the lock is currently unused. Rather than
-		 * panic, we recover by doing the blocking lock.
-		 */
-		if (lockmgr(&vp->v_lock, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0) {
-			printf("ffs_snapshot_unmount: Unexpected LK_NOWAIT "
-			    "failure\n");
-			lockmgr(&vp->v_lock, LK_EXCLUSIVE, NULL);
-		}
-		KASSERT(vp->v_vnlock == &sn->sn_lock,
-		("ffs_snapshot_unmount: lost lock mutation")); 
-		vp->v_vnlock = &vp->v_lock;
+		VI_LOCK(devvp);
+		revert_snaplock(vp, devvp, sn);
 		lockmgr(&vp->v_lock, LK_RELEASE, NULL);
-		lockmgr(&sn->sn_lock, LK_RELEASE, NULL);
-		if (xp->i_effnlink > 0)
+		if (xp->i_effnlink > 0) {
+			VI_UNLOCK(devvp);
 			vrele(vp);
-		VI_LOCK(devvp);
+			VI_LOCK(devvp);
+		}
 		sn = devvp->v_rdev->si_snapdata;
 	}
 	try_free_snapdata(devvp);
+	VI_UNLOCK(devvp);
 }
 
 /*
@@ -2682,10 +2664,8 @@ try_free_snapdata(struct vnode *devvp)
 	sn = devvp->v_rdev->si_snapdata;
 
 	if (sn == NULL || TAILQ_FIRST(&sn->sn_head) != NULL ||
-	    (devvp->v_vflag & VV_COPYONWRITE) == 0) {
-		VI_UNLOCK(devvp);
+	    (devvp->v_vflag & VV_COPYONWRITE) == 0)
 		return;
-	}
 
 	devvp->v_rdev->si_snapdata = NULL;
 	devvp->v_vflag &= ~VV_COPYONWRITE;
@@ -2697,6 +2677,46 @@ try_free_snapdata(struct vnode *devvp)
 	if (snapblklist != NULL)
 		free(snapblklist, M_UFSMNT);
 	ffs_snapdata_free(sn);
+	VI_LOCK(devvp);
+}
+
+/*
+ * Revert a vnode lock from using the snapshot lock back to its own lock.
+ *
+ * Aquire a lock on the vnode's own lock and release the lock on the
+ * snapshot lock. If there are any recursions on the snapshot lock
+ * get the same number of recursions on the vnode's own lock.
+ */
+static void
+revert_snaplock(vp, devvp, sn)
+	struct vnode *vp;
+	struct vnode *devvp;
+	struct snapdata *sn;
+{
+	int i;
+
+	ASSERT_VI_LOCKED(devvp, "revert_snaplock");
+	/*
+	 * Avoid LOR with snapshot lock. The LK_NOWAIT should
+	 * never fail as the lock is currently unused. Rather than
+	 * panic, we recover by doing the blocking lock.
+	 */
+	for (i = 0; i <= sn->sn_lock.lk_recurse; i++) {
+		if (lockmgr(&vp->v_lock, LK_EXCLUSIVE | LK_NOWAIT |
+		    LK_INTERLOCK, VI_MTX(devvp)) != 0) {
+			printf("revert_snaplock: Unexpected LK_NOWAIT "
+			    "failure\n");
+			lockmgr(&vp->v_lock, LK_EXCLUSIVE | LK_INTERLOCK,
+			    VI_MTX(devvp));
+		}
+		VI_LOCK(devvp);
+	}
+	KASSERT(vp->v_vnlock == &sn->sn_lock,
+	    ("revert_snaplock: lost lock mutation")); 
+	vp->v_vnlock = &vp->v_lock;
+	while (sn->sn_lock.lk_recurse > 0)
+		lockmgr(&sn->sn_lock, LK_RELEASE, NULL);
+	lockmgr(&sn->sn_lock, LK_RELEASE, NULL);
 }
 
 static struct snapdata *

--
Test scenario: all of the mksnap_ffs tests