panic: condition !VN_IS_DOOMED(dvp) not met at ../../../kern/vfs_cache.c:2481 (cache_enter_time), vmcore.833 KDB: debugger backends: ddb KDB: current backend: ddb ---<>--- Copyright (c) 1992-2024 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 15.0-CURRENT #0 main-n271039-b81424adf718: Fri Jul 5 19:55:53 CEST 2024 pho@mercat1.netperf.freebsd.org:/usr/src/sys/amd64/compile/PHO amd64 FreeBSD clang version 18.1.6 (https://github.com/llvm/llvm-project.git llvmorg-18.1.6-0-g1118c2e05e67) WARNING: WITNESS option enabled, expect reduced performance. WARNING: DIAGNOSTIC option enabled, expect reduced performance. VT(vga): resolution 640x480 CPU: Intel(R) Xeon(R) CPU E5-1650 v3 @ 3.50GHz (3500.13-MHz K8-class CPU) Origin="GenuineIntel" Id=0x306f2 Family=0x6 Model=0x3f Stepping=2 Features=0xbfebfbff Features2=0x7ffefbff AMD Features=0x2c100800 AMD Features2=0x21 Structured Extended Features=0x37ab Structured Extended Features3=0x9c000400 XSAVE Features=0x1 VT-x: PAT,HLT,MTF,PAUSE,EPT,UG,VPID,VID,PostIntr TSC: P-state invariant, performance statistics real memory = 34359738368 (32768 MB) avail memory = 33171128320 (31634 MB) Event timer "LAPIC" quality 600 ACPI APIC Table: FreeBSD/SMP: Multiprocessor System Detected: 12 CPUs FreeBSD/SMP: 1 package(s) x 6 core(s) x 2 hardware threads random: registering fast source Intel Secure Key RNG random: fast provider: "Intel Secure Key RNG" random: unblocking device. ioapic0 irqs 0-23 ioapic1 irqs 24-47 Launching APs: 1 10 6 4 2 5 9 8 11 7 3 random: entropy device external interface kbd1 at kbdmux0 vtvga0: smbios0: at iomem 0xf05e0-0xf05fe smbios0: Version: 3.0, BCD Revision: 3.0 aesni0: acpi0: <> acpi0: Power Button (fixed) cpu0: on acpi0 atrtc0: port 0x70-0x71,0x74-0x77 irq 8 on acpi0 atrtc0: registered as a time-of-day clock, resolution 1.000000s Event timer "RTC" frequency 32768 Hz quality 0 attimer0: port 0x40-0x43,0x50-0x53 irq 0 on acpi0 Timecounter "i8254" frequency 1193182 Hz quality 0 Event timer "i8254" frequency 1193182 Hz quality 100 hpet0: iomem 0xfed00000-0xfed003ff on acpi0 Timecounter "HPET" frequency 14318180 Hz quality 950 Event timer "HPET" frequency 14318180 Hz quality 350 Event timer "HPET1" frequency 14318180 Hz quality 340 Event timer "HPET2" frequency 14318180 Hz quality 340 Event timer "HPET3" frequency 14318180 Hz quality 340 Event timer "HPET4" frequency 14318180 Hz quality 340 Event timer "HPET5" frequency 14318180 Hz quality 340 Event timer "HPET6" frequency 14318180 Hz quality 340 Event timer "HPET7" frequency 14318180 Hz quality 340 Timecounter "ACPI-fast" frequency 3579545 Hz quality 900 acpi_timer0: <24-bit timer at 3.579545MHz> port 0x408-0x40b on acpi0 pcib0: on acpi0 pci0: on pcib0 pci0: at device 11.1 (no driver attached) pci0: at device 11.2 (no driver attached) pci0: at device 16.1 (no driver attached) pci0: at device 16.6 (no driver attached) pci0: at device 18.1 (no driver attached) acpi_syscontainer0: on acpi0 pcib1: port 0xcf8-0xcff on acpi0 pci1: on pcib1 pcib2: irq 26 at device 1.0 on pci1 pci2: on pcib2 pcib3: irq 32 at device 2.0 on pci1 pci3: on pcib3 pci3: at device 0.0 (no driver attached) pci3: at device 0.1 (no driver attached) pcib4: irq 40 at device 3.0 on pci1 pci4: on pcib4 pci1: at device 17.0 (no driver attached) ahci0: port 0xf110-0xf117,0xf100-0xf103,0xf0f0-0xf0f7,0xf0e0-0xf0e3,0xf020-0xf03f mem 0xfb416000-0xfb4167ff irq 16 at device 17.4 on pci1 ahci0: AHCI v1.30 with 4 6Gbps ports, Port Multiplier not supported ahcich0: at channel 0 on ahci0 ahcich1: at channel 1 on ahci0 ahcich2: at channel 2 on ahci0 ahcich3: at channel 3 on ahci0 ahciem0: on ahci0 xhci0: mem 0xfb400000-0xfb40ffff irq 19 at device 20.0 on pci1 xhci0: 32 bytes context size, 64-bit DMA usbus0 on xhci0 usbus0: 5.0Gbps Super Speed USB v3.0 pci1: at device 22.0 (no driver attached) pci1: at device 22.1 (no driver attached) ehci0: mem 0xfb412000-0xfb4123ff irq 18 at device 26.0 on pci1 usbus1: EHCI version 1.0 usbus1 on ehci0 usbus1: 480Mbps High Speed USB v2.0 pcib5: irq 16 at device 28.0 on pci1 pci5: on pcib5 pcib6: irq 18 at device 28.2 on pci1 pci6: on pcib6 pcib7: at device 0.0 on pci6 pci7: on pcib7 vgapci0: port 0xe000-0xe07f mem 0xfa000000-0xfaffffff,0xfb000000-0xfb01ffff irq 18 at device 0.0 on pci7 vgapci0: Boot video device pcib8: irq 16 at device 28.4 on pci1 pci8: on pcib8 igb0: port 0xd020-0xd03f mem 0xfb320000-0xfb33ffff,0xfb344000-0xfb347fff irq 16 at device 0.0 on pci8 igb0: EEPROM V1.63-0 eTrack 0x800009fa igb0: Using 1024 TX descriptors and 1024 RX descriptors igb0: queue equality override not set, capping rx_queues at 6 and tx_queues at Setting hostuuid: 00000000-0000-0000-0000-002590c99976. Setting hostid: 0x90b83241. Starting file system checks: WARNING: / was not properly dismounted Mounting local filesystems:. Autoloading module: acpi_wmi Autoloading module: ichsmb Autoloading module: ioat ioat0: mem 0x383ffff1c000-0x383ffff1ffff irq 31 at device 4.0 on pci1 ioat0: Capabilities: 2f7 ioat1: mem 0x383ffff18000-0x383ffff1bfff irq 39 at device 4.1 on pci1 ioat1: Capabilities: 2f7 ioat2: mem 0x383ffff14000-0x383ffff17fff irq 31 at device 4.2 on pci1 ioat2: Capabilities: f7 ioat3: mem 0x383ffff10000-0x383ffff13fff irq 39 at device 4.3 on pci1 ioat3: Capabilities: f7 ioat4: mem 0x383ffff0c000-0x383ffff0ffff irq 31 at device 4.4 on pci1 ioat4: Capabilities: f7 ioat5: mem 0x383ffff08000-0x383ffff0bfff irq 39 at device 4.5 on pci1 ioat5: Capabilities: f7 ioat6: mem 0x383ffff04000-0x383ffff07fff irq 31 at device 4.6 on pci1 ioat6: Capabilities: f7 ioat7: mem 0x383ffff00000-0x383ffff03fff irq 39 at device 4.7 on pci1 ioat7: Capabilities: f7 ichsmb0: port 0x580-0x59f mem 0x383ffff21000-0x383ffff210ff irq 18 at device 31.3 on pci1 smbus0: on ichsmb0 acpi_wmi0: on acpi0 acpi_wmi0: cannot find EC device fsck2+ fsck -fy /tmp ** /dev/gpt/tmp ** Last Mounted on /tmp ** Phase 1 - Check Blocks and Sizes ** Phase 2 - Check Pathnames ** Phase 3 - Check Connectivity ** Phase 4 - Check Reference Counts ** Phase 5 - Check Cyl groups 1165 files, 65041 used, 1200366 free (102 frags, 150033 blocks, 0.0% fragmentation) ***** FILE SYSTEM IS CLEAN ***** + set +x Updating CPU Microcode... CPU: Intel(R) Xeon(R) CPU E5-1650 v3 @ 3.50GHz (3500.00-MHz K8-class CPU) Origin="GenuineIntel" Id=0x306f2 Family=0x6 Model=0x3f Stepping=2 Features=0xbfebfbff Features2=0x7ffefbff AMD Features=0x2c100800 AMD Features2=0x21 Structured Extended Features=0x37ab Structured Extended Features3=0x9c000400 XSAVE Features=0x1 VT-x: PAT,HLT,MTF,PAUSE,EPT,UG,VPID,VID,PostIntr TSC: P-state invariant, performance statistics Done. ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib /usr/local/lib/e2fsprogs /usr/local/lib/perl5/5.36/mach/CORE 32-bit compatibility ldconfig path: /usr/lib32 /usr/local/lib32/compat Setting hostname: mercat1.netperf.freebsd.org. Setting up harvesting: PURE_RDRAND,[CALLOUT],[UMA],[FS_ATIME],SWI,INTERRUPT,NET_NG,[NET_ETHER],NET_TUN,MOUSE,KEYBOARD,ATTACH,CACHED Feeding entropy: . igb0: link state changed to UP debugnet_any_ifnet_update: Bad dn_init result from igb0 (ifp 0xfffff80005962800), ignoring. Starting Network: igb1. igb1: flags=8802 metric 0 mtu 1500 options=4e507bb ether 0c:c4:7a:a8:cd:db media: Ethernet autoselect status: no carrier nd6 options=29 Starting Network: mce0. mce0: flags=8802 metric 0 mtu 1500 options=7eef07bb ether 7c:fe:90:30:ee:38 media: Ethernet autoselect status: no carrier nd6 options=29 Starting Network: mce1. mce1: flags=8802 metric 0 mtu 1500 options=7eef07bb ether 7c:fe:90:30:ee:39 media: Ethernet autoselect status: no carrier nd6 options=29 Autoloading module: uhid Autoloading module: ums Autoloading module: usbhid ums0 on uhub3 ums0: on usbus0 ums0: 3 buttons and [Z] coordinates ID=0 Starting dhclient. DHCPREQUEST on igb0 to 255.255.255.255 port 67 DHCPACK from 192.168.5.1 bound to 192.168.5.171 -- renewal in 300 seconds. route: message indicates error: File exists add host 127.0.0.1: gateway lo0 fib 0: route already in table route: message indicates error: File exists add host ::1: gateway lo0 fib 0: route already in table add net fe80::: gateway ::1 add net ff02::: gateway ::1 add net ::ffff:0.0.0.0: gateway ::1 add net ::0.0.0.0: gateway ::1 Creating and/or trimming log files. Starting syslogd. No core dumps found. Starting watchdogd. Setting date via ntp. 5 Jul 20:11:08 ntpdate[3459]: step time server 192.168.5.1 offset +0.319541 sec Clearing /tmp (X related). Recovering vi editor sessions:. Updating motd:. Updating /var/run/os-release done. Security policy loaded: MAC/ntpd (mac_ntpd) Starting ntpd. Mounting late filesystems:. Starting powerd. Starting inetd. Starting cron. Performing sanity check on sshd configuration. Starting sshd. Fri Jul 5 20:11 FreeBSD/amd64 (mercat1.netperf.freebsd.org) (ttyu1) login: Jul 5 20:11:24 mercat1 su[3645]: pho to root on /dev/pts/0 20240705 20:11:54 all (1/1): unionfs19.sh stress2: pts leak: 1/2 20240705 20:12:08 all (1/1): unionfs19.sh 20240705 20:12:22 all (1/1): unionfs19.sh 20240705 20:12:36 all (1/1): unionfs19.sh 20240705 20:12:51 all (1/1): unionfs19.sh 20240705 20:13:05 all (1/1): unionfs19.sh 20240705 20:13:18 all (1/1): unionfs19.sh 20240705 20:13:33 all (1/1): unionfs19.sh 20240705 20:13:56 all (1/1): unionfs19.sh 20240705 20:14:13 all (1/1): unionfs19.sh 20240705 20:14:36 all (1/1): unionfs19.sh 20240705 20:14:54 all (1/1): unionfs19.sh 20240705 20:15:08 all (1/1): unionfs19.sh 20240705 20:15:27 all (1/1): unionfs19.sh 20240705 20:15:41 all (1/1): unionfs19.sh 20240705 20:15:59 all (1/1): unionfs19.sh 20240705 20:16:17 all (1/1): unionfs19.sh 20240705 20:17:01 all (1/1): unionfs19.sh 20240705 20:17:39 all (1/1): unionfs19.sh Jul 5 20:18:22 mercat1 kernel: pid 52803 (swap), jid 0, uid 0, was killed: a thread waited too long to allocate a page 20240705 20:18:31 all (1/1): unionfs19.sh 20240705 20:18:45 all (1/1): unionfs19.sh 20240705 20:19:00 all (1/1): unionfs19.sh 20240705 20:19:14 all (1/1): unionfs19.sh 20240705 20:20:03 all (1/1): unionfs19.sh 20240705 20:20:18 all (1/1): unionfs19.sh 20240705 20:20:36 all (1/1): unionfs19.sh 20240705 20:20:51 all (1/1): unionfs19.sh 20240705 20:21:05 all (1/1): unionfs19.sh 20240705 20:21:25 all (1/1): unionfs19.sh 20240705 20:21:39 all (1/1): unionfs19.sh 20240705 20:21:59 all (1/1): unionfs19.sh 20240705 20:22:18 all (1/1): unionfs19.sh 20240705 20:22:32 all (1/1): unionfs19.sh 20240705 20:22:46 all (1/1): unionfs19.sh 20240705 20:23:20 all (1/1): unionfs19.sh 20240705 20:24:03 all (1/1): unionfs19.sh 20240705 20:24:17 all (1/1): unionfs19.sh 20240705 20:24:37 all (1/1): unionfs19.sh 20240705 20:25:38 all (1/1): unionfs19.sh VNASSERT failed: !__builtin_expect(((_Generic(*(&(dvp)->v_irflag), short: (*(volatile u_short *)(&(dvp)->v_irflag)), u_short: VNASSERT failed: !__builtin_expect(((_Generic(*(&(dvp)->v_irflag), short: (*(volatile u_short *)(&(dvp)->v_irflag)), u_short: (*(volatile u_short *)(&(dvp)->v_irflag))) & 0x0001) != 0), 0) not true at ../../../kern/vfs_cache.c:2481 (cache_enter_time) VNASSERT failed: !__builtin_expect(((_Generic(*(&(dvp)->v_irflag), short: (*(volatile u_short *)(&(dvp)->v_irflag)), u_short: VNASSERT failed: !__builtin_expect(((_Generic(*(&(dvp)->v_irflag), short: (*(volatile u_short *)(&(dvp)->v_irflag)), u_short: (*(volatile u_short *)(&(dvp)->v_irflag))) & 0x0001) != 0), 0) not true at ../../../kern/vfs_cache.c:2481 (cache_enter_time) VNASSERT failed: !__builtin_expect(((_Generic(*(&(dvp)->v_irflag), short: (*(volatile u_short *)(&(dvp)->v_irflag)), u_short: (*(volatile u_short *)(&(dvp)->v_irflag))) & 0x0001) != 0), 0) not true at ../../../kern/vfs_cache.c:2481 (cache_enter_time) 0xfffffe016f682bb8: (*(volatile u_short *)(&(dvp)->v_irflag))) & 0x0001) != 0), 0) not true at ../../../kern/vfs_cache.c:2481 (cache_enter_time) (*(volatile u_short *)(&(dvp)->v_irflag))) & 0x0001) != 0), 0) not true at ../../../kern/vfs_cache.c:2481 (cache_enter_time) 0xfffffe016f6d04b0: 0xfffffe016f1aa068: type VBAD state VSTATE_DEAD op 0xffffffff818ac760 usecount 2, writecount 0, refcount 1 seqc users 10xfffffe016ffba068: type VBAD state VSTATE_DEAD op 0xffffffff818ac760 hold count flags () 0xfffffe016fed0000: flags (VIRF_DOOMED)type VBAD state VSTATE_DEAD op 0xffffffff818ac760 usecount 2, writecount 0, refcount 1 seqc users 1type VBAD state VSTATE_DEAD op 0xffffffff818ac760 type VBAD state VSTATE_DEAD op 0xffffffff818ac760 usecount 2, writecount 0, refcount 1 seqc users 1 hold count flags () lock type unionfs: EXCL by thread 0xfffff802f76c7740 (pid 92551, mmap, tid 100429) usecount 2, writecount 0, refcount 1 seqc users 1 usecount 2, writecount 0, refcount 1 seqc users 1 hold count flags () flags (VIRF_DOOMED) hold count flags () flags (VIRF_DOOMED) flags (VIRF_DOOMED)#0 0xffffffff80b14d98 at lockmgr_lock_flags+0x1b8 #1 0xffffffff8113416a at VOP_LOCK1_APV+0x3a lock type unionfs: EXCL by thread 0xfffff802f72a0740 (pid 92547, mmap, tid 100442) #2 0xffffffff80c5ae03 at _vn_lock+0x53 hold count flags () flags (VIRF_DOOMED) #3 0xffffffff80c60017 at vn_lock_pair+0x3d7 lock type unionfs: EXCL by thread 0xfffff806a0787740 (pid 92565, mmap, tid 100465) #0 0xffffffff80b14d98 at lockmgr_lock_flags+0x1b8 #0 0xffffffff80b14d98 at lockmgr_lock_flags+0x1b8 #1 0xffffffff8113416a at VOP_LOCK1_APV+0x3a lock type unionfs: EXCL by thread 0xfffff802132f4740 (pid 92567, mmap, tid 100436) #1 0xffffffff8113416a at VOP_LOCK1_APV+0x3a #0 0xffffffff80b14d98 at lockmgr_lock_flags+0x1b8 #2 0xffffffff80c5ae03 at _vn_lock+0x53 #4 0xffffffff8284e086 at unionfs_forward_vop_finish_pair+0x176 #1 0xffffffff8113416a at VOP_LOCK1_APV+0x3a #3 0xffffffff80c60017 at vn_lock_pair+0x3d7 #2 0xffffffff80c5ae03 at _vn_lock+0x53 #5 0xffffffff8285014d at unionfs_lookup+0x22d #2 0xffffffff80c5ae03 at _vn_lock+0x53 #3 0xffffffff80c60017 at vn_lock_pair+0x3d7 #6 0xffffffff81130c0f at VOP_CACHEDLOOKUP_APV+0x5f lock type unionfs: EXCL by thread 0xfffff8021314b740 (pid 92569, mmap, tid 100415) #3 0xffffffff80c60017 at vn_lock_pair+0x3d7 #7 0xffffffff80c20c36 at vfs_cache_lookup+0xa6 #0 0xffffffff80b14d98 at lockmgr_lock_flags+0x1b8 #4 0xffffffff8284e086 at unionfs_forward_vop_finish_pair+0x176 #8 0xffffffff81130a4f at VOP_LOOKUP_APV+0x5f #1 0xffffffff8113416a at VOP_LOCK1_APV+0x3a #9 0xffffffff80c321c7 at vfs_lookup+0x487 #2 0xffffffff80c5ae03 at _vn_lock+0x53 #10 0xffffffff80c312e1 at namei+0x2d1 #4 0xffffffff8284e086 at unionfs_forward_vop_finish_pair+0x176 #11 0xffffffff80c50223 at kern_chdir+0xc3 #4 0xffffffff8284e086 at unionfs_forward_vop_finish_pair+0x176 #12 0xffffffff8106a8a8 at amd64_syscall+0x158 #5 0xffffffff8285014d at unionfs_lookup+0x22d #13 0xffffffff8103bd0b at fast_syscall_common+0xf8 panic: condition !VN_IS_DOOMED(dvp) not met at ../../../kern/vfs_cache.c:2481 (cache_enter_time) cpuid = 2 time = 1720203945 KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe010874e780 vpanic() at vpanic+0x13f/frame 0xfffffe010874e8b0 panic() at panic+0x43/frame 0xfffffe010874e910 cache_enter_time() at cache_enter_time+0x145b/frame 0xfffffe010874e9f0 unionfs_lookup() at unionfs_lookup+0x7fa/frame 0xfffffe010874eb50 VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0x5f/frame 0xfffffe010874eb80 vfs_cache_lookup() at vfs_cache_lookup+0xa6/frame 0xfffffe010874ebd0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0x5f/frame 0xfffffe010874ec00 vfs_lookup() at vfs_lookup+0x487/frame 0xfffffe010874ec80 namei() at namei+0x2d1/frame 0xfffffe010874ece0 kern_chdir() at kern_chdir+0xc3/frame 0xfffffe010874ee00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010874ef30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010874ef30 --- syscall (12, FreeBSDdf55a, rsp = 0x3a114032c808, rbp = 0x3a114032c810 --- KDB: enter: panic [ thread pid 92551 tid 100429 ] Stopped at kdb_enter+0x33: movq $0,0x105cf72(%rip) db> run pho db:0:pho> set $lines 20000 db:0:pho> run pho1 db:1:pho1> dump Dumping 1511 out of 32620 MB:..2%..11%..21%..31%..41%..51%..61%..71%..81%..92% Dump complete db:1:pho1> bt Tracing pid 92551 tid 100429 td 0xfffff802f76c7740 kdb_enter() at kdb_enter+0x33/frame 0xfffffe010874e8b0 panic() at panic+0x43/frame 0xfffffe010874e910 cache_enter_time() at cache_enter_time+0x145b/frame 0xfffffe010874e9f0 unionfs_lookup() at unionfs_lookup+0x7fa/frame 0xfffffe010874eb50 VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0x5f/frame 0xfffffe010874eb80 vfs_cache_lookup() at vfs_cache_lookup+0xa6/frame 0xfffffe010874ebd0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0x5f/frame 0xfffffe010874ec00 vfs_lookup() at vfs_lookup+0x487/frame 0xfffffe010874ec80 namei() at namei+0x2d1/frame 0xfffffe010874ece0 kern_chdir() at kern_chdir+0xc3/frame 0xfffffe010874ee00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010874ef30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010874ef30 --- syscall (12, FreeBSD ELF64, chdir), rip = 0x3a11423df55a, rsp = 0x3a114032c808, rbp = 0x3a114032c810 --- db:1:pho1> show allpcpu Current CPU: 2 cpuid = 0 dynamic pcpu = 0x9f5e80 curthread = 0xfffff8021314b740: pid 92569 tid 100415 critnest 0 "mmap" curpcb = 0xfffff8021314bc60 fpcurthread = 0xfffff8021314b740: pid 92569 "mmap" idlethread = 0xfffff80003bf4740: tid 100003 "idle: cpu0" self = 0xffffffff82410000 curpmap = 0xfffff800092f2130 tssp = 0xffffffff82410384 rsp0 = 0xfffffe01086ff000 kcr3 = 0x80000002af12a1dc ucr3 = 0x80000002afbb19dc scr3 = 0x2afbb19dc gs32p = 0xffffffff82410404 ldt = 0xffffffff82410444 tss = 0xffffffff82410434 curvnet = 0 spin locks held: cpuid = 1 dynamic pcpu = 0xfffffe00bb4b5e80 curthread = 0xfffff802f7650000: pid 92573 tid 100449 critnest 0 "swap" curpcb = 0xfffff802f7650520 fpcurthread = 0xfffff802f7650000: pid 92573 "swap" idlethread = 0xfffff80003bf4000: tid 100004 "idle: cpu1" self = 0xffffffff82411000 curpmap = 0xfffff80005910868 tssp = 0xffffffff82411384 rsp0 = 0xfffffe010872c000 kcr3 = 0x80000000330f5095 ucr3 = 0x80000000330f4895 scr3 = 0x2f7989896 gs32p = 0xffffffff82411404 ldt = 0xffffffff82411444 tss = 0xffffffff82411434 curvnet = 0 spin locks held: cpuid = 2 dynamic pcpu = 0xfffffe00bb4c4e80 curthread = 0xfffff802f76c7740: pid 92551 tid 100429 critnest 1 "mmap" curpcb = 0xfffff802f76c7c60 fpcurthread = 0xfffff802f76c7740: pid 92551 "mmap" idlethread = 0xfffff80003bf5740: tid 100005 "idle: cpu2" self = 0xffffffff82412000 curpmap = 0xfffff802f782c868 tssp = 0xffffffff82412384 rsp0 = 0xfffffe010874f000 kcr3 = 0x800000034cb02788 ucr3 = 0x80000004d3315f88 scr3 = 0x4d3315f88 gs32p = 0xffffffff82412404 ldt = 0xffffffff82412444 tss = 0xffffffff82412434 curvnet = 0 spin locks held: cpuid = 3 dynamic pcpu = 0xfffffe00bb4d3e80 curthread = 0xfffff80213886000: pid 92583 tid 100408 critnest 0 "swap" curpcb = 0xfffff80213886520 fpcurthread = 0xfffff80213886000: pid 92583 "swap" idlethread = 0xfffff80003bf5000: tid 100006 "idle: cpu3" self = 0xffffffff82413000 curpmap = 0xfffff80040b6bad0 tssp = 0xffffffff82413384 rsp0 = 0xfffffe01086d2000 kcr3 = 0x80000005a0dfc42b ucr3 = 0x80000005a0073c2b scr3 = 0x5a0073c2b gs32p = 0xffffffff82413404 ldt = 0xffffffff82413444 tss = 0xffffffff82413434 curvnet = 0 spin locks held: cpuid = 4 dynamic pcpu = 0xfffffe00bb4e2e80 curthread = 0xfffff806a0787740: pid 92565 tid 100465 critnest 0 "mmap" curpcb = 0xfffff806a0787c60 fpcurthread = 0xfffff806a0787740: pid 92565 "mmap" idlethread = 0xfffff80003bf6740: tid 100007 "idle: cpu4" self = 0xffffffff82414000 curpmap = 0xfffff80005910600 tssp = 0xffffffff82414384 rsp0 = 0xfffffe01087f9000 kcr3 = 0x80000002af35672b ucr3 = 0x800000036e427f2b scr3 = 0x602501f2f gs32p = 0xffffffff82414404 ldt = 0xffffffff82414444 tss = 0xffffffff82414434 curvnet = 0 spin locks held: cpuid = 5 dynamic pcpu = 0xfffffe00bb4f1e80 curthread = 0xfffff800331cd000: pid 92558 tid 100534 critnest 0 "swap" curpcb = 0xfffff800331cd520 fpcurthread = 0xfffff800331cd000: pid 92558 "swap" idlethread = 0xfffff80003bf6000: tid 100008 "idle: cpu5" self = 0xffffffff82415000 curpmap = 0xfffff80015a16600 tssp = 0xffffffff82415384 rsp0 = 0xfffffe01087cc000 kcr3 = 0x80000003f59630fe ucr3 = 0x80000003f59628fe scr3 = 0x4213c4a17 gs32p = 0xffffffff82415404 ldt = 0xffffffff82415444 tss = 0xffffffff82415434 curvnet = 0 spin locks held: cpuid = 6 dynamic pcpu = 0xfffffe00bb500e80 curthread = 0xfffff802f7814740: pid 92548 tid 100403 critnest 0 "swap" curpcb = 0xfffff802f7814c60 fpcurthread = 0xfffff802f7814740: pid 92548 "swap" idlethread = 0xfffff80003bdf740: tid 100009 "idle: cpu6" self = 0xffffffff82416000 curpmap = 0xfffff80040155d38 tssp = 0xffffffff82416384 rsp0 = 0xfffffe01086c3000 kcr3 = 0x80000007899a0457 ucr3 = 0x80000005a0e08c57 scr3 = 0x2f7bd4c9e gs32p = 0xffffffff82416404 ldt = 0xffffffff82416444 tss = 0xffffffff82416434 curvnet = 0 spin locks held: cpuid = 7 dynamic pcpu = 0xfffffe00bb50fe80 curthread = 0xfffff802f72a0740: pid 92547 tid 100442 critnest 1 "mmap" curpcb = 0xfffff802f72a0c60 fpcurthread = 0xfffff802f72a0740: pid 92547 "mmap" idlethread = 0xfffff80003bdf000: tid 100010 "idle: cpu7" self = 0xffffffff82417000 curpmap = 0xfffff80040c28130 tssp = 0xffffffff82417384 rsp0 = 0xfffffe010867d000 kcr3 = 0x80000003c423d013 ucr3 = 0x80000003c423c813 scr3 = 0x14166b80c gs32p = 0xffffffff82417404 ldt = 0xffffffff82417444 tss = 0xffffffff82417434 curvnet = 0 spin locks held: cpuid = 8 dynamic pcpu = 0xfffffe00bb51ee80 curthread = 0xfffff802f7303740: pid 92589 tid 100391 critnest 0 "swap" curpcb = 0xfffff802f7303c60 fpcurthread = 0xfffff802f7303740: pid 92589 "swap" idlethread = 0xfffff80003be0740: tid 100011 "idle: cpu8" self = 0xffffffff82418000 curpmap = 0xfffff801a5fead38 tssp = 0xffffffff82418384 rsp0 = 0xfffffe0108691000 kcr3 = 0x80000002af8b149f ucr3 = 0x80000002af8b0c9f scr3 = 0x2af8b0c9f gs32p = 0xffffffff82418404 ldt = 0xffffffff82418444 tss = 0xffffffff82418434 curvnet = 0 spin locks held: cpuid = 9 dynamic pcpu = 0xfffffe00bb52de80 curthread = 0xfffff802132f4740: pid 92567 tid 100436 critnest 0 "mmap" curpcb = 0xfffff802132f4c60 fpcurthread = 0xfffff802132f4740: pid 92567 "mmap" idlethread = 0xfffff80003be0000: tid 100012 "idle: cpu9" self = 0xffffffff82419000 curpmap = 0xfffff8004014e868 tssp = 0xffffffff82419384 rsp0 = 0xfffffe0108777000 kcr3 = 0x800000028cff15a3 ucr3 = 0x800000028cff0da3 scr3 = 0x60e41eda1 gs32p = 0xffffffff82419404 ldt = 0xffffffff82419444 tss = 0xffffffff82419434 curvnet = 0 spin locks held: cpuid = 10 dynamic pcpu = 0xfffffe00bb53ce80 curthread = 0xfffff80015ebd000: pid 92576 tid 100373 critnest 0 "swap" curpcb = 0xfffff80015ebd520 fpcurthread = 0xfffff80015ebd000: pid 92576 "swap" idlethread = 0xfffff80003be1740: tid 100013 "idle: cpu10" self = 0xffffffff8241a000 curpmap = 0xfffff80015b74130 tssp = 0xffffffff8241a384 rsp0 = 0xfffffe0108506000 kcr3 = 0x8000000789c94677 ucr3 = 0x80000004a9417e77 scr3 = 0x4a9417e77 gs32p = 0xffffffff8241a404 ldt = 0xffffffff8241a444 tss = 0xffffffff8241a434 curvnet = 0 spin locks held: cpuid = 11 dynamic pcpu = 0xfffffe00bb54be80 curthread = 0xfffff802f7638000: pid 92564 tid 100536 critnest 0 "swap" curpcb = 0xfffff802f7638520 fpcurthread = 0xfffff802f7638000: pid 92564 "swap" idlethread = 0xfffff80003be1000: tid 100014 "idle: cpu11" self = 0xffffffff8241b000 curpmap = 0xfffff80003fce600 tssp = 0xffffffff8241b384 rsp0 = 0xfffffe01087b3000 kcr3 = 0x80000003e485473d ucr3 = 0x800000030f103f3d scr3 = 0x4213c4f3b gs32p = 0xffffffff8241b404 ldt = 0xffffffff8241b444 tss = 0xffffffff8241b434 curvnet = 0 spin locks held: db:1:pho1> show alllocks Process 92569 (mmap) thread 0xfffff8021314b740 (100415) exclusive lockmgr unionfs (unionfs) r = 0 (0xfffffe016fed0070) locked @ kern/vfs_vnops.c:4237 Process 92567 (mmap) thread 0xfffff802132f4740 (100436) exclusive lockmgr unionfs (unionfs) r = 0 (0xfffffe016f6d0520) locked @ kern/vfs_vnops.c:4237 Process 92565 (mmap) thread 0xfffff806a0787740 (100465) exclusive lockmgr unionfs (unionfs) r = 0 (0xfffffe016ffba0d8) locked @ kern/vfs_vnops.c:4237 Process 92551 (mmap) thread 0xfffff802f76c7740 (100429) exclusive lockmgr unionfs (unionfs) r = 0 (0xfffffe016f682c28) locked @ kern/vfs_vnops.c:4237 Process 92547 (mmap) thread 0xfffff802f72a0740 (100442) exclusive lockmgr unionfs (unionfs) r = 0 (0xfffffe016f1aa0d8) locked @ kern/vfs_vnops.c:4237 Process 3420 (syslogd) thread 0xfffff80003fe2740 (100261) exclusive lockmgr ufs (ufs) r = 0 (0xfffffe016d4c6070) locked @ kern/vfs_syscalls.c:3581 db:1:pho1> show lockedvnods Locked vnodes vnode 0xfffffe016d4c6000: type VREG state VSTATE_CONSTRUCTED op 0xffffffff81942638 usecount 2, writecount 1, refcount 6 seqc users 0 hold count flags () flags (VIRF_PGREAD|VMP_LAZYLIST) v_object 0xfffff80003f10210 ref 0 pages 12 cleanbuf 1 dirtybuf 1 lock type ufs: EXCL by thread 0xfffff80003fe2740 (pid 3420, syslogd, tid 100261) with exclusive waiters pending #0 0xffffffff80b14d98 at lockmgr_lock_flags+0x1b8 #1 0xffffffff80ec41ed at ffs_lock+0x9d #2 0xffffffff8113416a at VOP_LOCK1_APV+0x3a #3 0xffffffff80c5ae03 at _vn_lock+0x53 #4 0xffffffff80c56018 at kern_fsync+0xe8 #5 0xffffffff8106a8a8 at amd64_syscall+0x158 #6 0xffffffff8103bd0b at fast_syscall_common+0xf8 nlink=1, effnlink=1, size=111862, extsize 0 generation=36c16c00, uid=0, gid=0, flags=0x0 ino 83145217, on dev gpt/work db:1:pho1> show mount 0xfffffe010ebfb700 /dev/gpt/work on / (ufs) 0xfffffe010ebfc280 devfs on /dev (devfs) 0xfffffe016d290280 /dev/gpt/tmp on /tmp (ufs) 0xfffffe010ebfa000 /dev/md10 on /mnt10 (ufs) 0xfffffe016f6c2700 /dev/md11 on /mnt11 (ufs) 0xfffffe016f666700 :/mnt10 on /mnt11 (unionfs) More info: show mount db:1:pho1> show bufqueues bqempty: 100437 Buf domain 0 freebufs 14738 lofreebufs 219 hifreebufs 329 bufspace 5640192 maxbufspace 247939072 hibufspace 247845449 lobufspace 235453176 bufspacethresh 241649312 numdirtybuffers 3 lodirtybuffers 1893 hidirtybuffers 3786 dirtybufthresh 3407 cleanq count 297 (3903488) dirtyq count 3 (57344) wakeup 0 lim 25 CPU 11, 10, 0, 7, 8, 12, 4, 3, 9, 9, 4, 18, Locked buffers: 0 space 0 Total buffers: 14860 space 5640192 Buf domain 1 freebufs 14628 lofreebufs 219 hifreebufs 329 bufspace 8994816 maxbufspace 247939072 hibufspace 247845449 lobufspace 235453176 bufspacethresh 241649312 numdirtybuffers 11 lodirtybuffers 1893 hidirtybuffers 3786 dirtybufthresh 3407 cleanq count 408 (6991872) dirtyq count 11 (245760) wakeup 0 lim 25 CPU 0, 4, 6, 0, 13, 10, 14, 5, 10, 18, 0, 6, Locked buffers: 0 space 0 Total buffers: 14786 space 8994816 Buf domain 2 freebufs 14772 lofreebufs 219 hifreebufs 329 bufspace 4784128 maxbufspace 247939072 hibufspace 247845449 lobufspace 235453176 bufspacethresh 241649312 numdirtybuffers 2 lodirtybuffers 1893 hidirtybuffers 3786 dirtybufthresh 3407 cleanq count 272 (3411968) dirtyq count 2 (36864) wakeup 0 lim 25 CPU 16, 14, 0, 0, 3, 2, 0, 1, 21, 15, 7, 8, Locked buffers: 0 space 0 Total buffers: 15563 space 4784128 Buf domain 3 freebufs 13764 lofreebufs 219 hifreebufs 329 bufspace 37756928 maxbufspace 247939072 hibufspace 247845449 lobufspace 235453176 bufspacethresh 241649312 numdirtybuffers 3 lodirtybuffers 1893 hidirtybuffers 3786 dirtybufthresh 3407 cleanq count 1274 (36159488) dirtyq count 3 (12288) wakeup 0 lim 25 CPU 1, 5, 23, 8, 4, 2, 4, 1, 7, 11, 7, 19, Locked buffers: 0 space 0 Total buffers: 15448 space 37756928 Buf domain 4 freebufs 13802 lofreebufs 219 hifreebufs 329 bufspace 36667392 maxbufspace 247939072 hibufspace 247845449 lobufspace 235453176 bufspacethresh 241649312 numdirtybuffers 3 lodirtybuffers 1893 hidirtybuffers 3786 dirtybufthresh 3407 cleanq count 1289 (35819520) dirtyq count 3 (28672) wakeup 0 lim 25 CPU 0, 6, 0, 9, 4, 2, 0, 6, 0, 2, 4, 6, Locked buffers: 0 space 0 Total buffers: 15713 space 36667392 Buf domain 5 freebufs 14681 lofreebufs 219 hifreebufs 329 bufspace 7979008 maxbufspace 247939072 hibufspace 247845449 lobufspace 235453176 bufspacethresh 241649312 numdirtybuffers 7 lodirtybuffers 1893 hidirtybuffers 3786 dirtybufthresh 3407 cleanq count 399 (7069696) dirtyq count 7 (45056) wakeup 0 lim 25 CPU 0, 7, 0, 12, 7, 5, 0, 1, 0, 9, 4, 1, Locked buffers: 0 space 0 Total buffers: 14859 space 7979008 Buf domain 6 freebufs 14213 lofreebufs 219 hifreebufs 329 bufspace 22581248 maxbufspace 247939072 hibufspace 247845449 lobufspace 235453176 bufspacethresh 241649312 numdirtybuffers 29 lodirtybuffers 1893 hidirtybuffers 3786 dirtybufthresh 3407 cleanq count 851 (20594688) dirtyq count 29 (864256) wakeup 0 lim 25 CPU 1, 0, 2, 0, 2, 11, 1, 0, 1, 0, 2, 20, Locked buffers: 0 space 0 Total buffers: 14702 space 22581248 db:1:pho1> show page vm_cnt.v_free_count: 4762989 vm_cnt.v_inactive_count: 863 vm_cnt.v_active_count: 3174474 vm_cnt.v_laundry_count: 0 vm_cnt.v_wire_count: 183932 vm_cnt.v_free_reserved: 10642 vm_cnt.v_free_min: 51257 vm_cnt.v_free_target: 173102 vm_cnt.v_inactive_target: 259653 db:1:pho1> show pageq pq_free 4762989 dom 0 page_cnt 8123039 free 4762989 pq_act 3174474 pq_inact 863 pq_laund 0 pq_unsw 0 db:0:pho> show freepages DOMAIN: 0 FREE LIST 0: ORDER (SIZE) | NUMBER | POOL 0 | POOL 1 | POOL 2 -- -- -- -- -- -- -- -- 12 (016384K) | 000000 | 000741 | 000257 11 (008192K) | 000000 | 000001 | 000053 10 (004096K) | 000000 | 000001 | 000031 09 (002048K) | 000000 | 000000 | 000037 08 (001024K) | 000000 | 000492 | 000030 07 (000512K) | 000000 | 000545 | 000034 06 (000256K) | 000000 | 000603 | 000066 05 (000128K) | 000000 | 000737 | 000213 04 (000064K) | 000000 | 000911 | 000519 03 (000032K) | 000000 | 001081 | 000367 02 (000016K) | 000000 | 001387 | 000000 01 (000008K) | 000000 | 002378 | 000000 00 (000004K) | 000000 | 002450 | 000001 FREE LIST 1: ORDER (SIZE) | NUMBER | POOL 0 | POOL 1 | POOL 2 -- -- -- -- -- -- -- -- 12 (016384K) | 000000 | 000000 | 000000 11 (008192K) | 000000 | 000000 | 000000 10 (004096K) | 000000 | 000000 | 000000 09 (002048K) | 000000 | 000000 | 000000 08 (001024K) | 000000 | 000001 | 000000 07 (000512K) | 000000 | 000000 | 000000 06 (000256K) | 000000 | 000001 | 000000 05 (000128K) | 000000 | 000001 | 000000 04 (000064K) | 000000 | 000002 | 000000 03 (000032K) | 000000 | 000000 | 000000 02 (000016K) | 000000 | 000001 | 000000 01 (000008K) | 000000 | 000001 | 000000 00 (000004K) | 000000 | 000001 | 000000 db:0:pho> show uma Zone Size Used Free Requests Sleeps Bucket Total Mem XFree malloc-4096 4096 34124 2 162408 0 2 139780096 0 RADIX NODE 144 220673 1224 26626065 0 63 31953168 0 mbuf_cluster 2048 13310 20 24628 0 8 27299840 0 BUF TRIE 144 627 105325 5526 0 62 15257088 0 FFS inode 1168 9791 415 2182289 0 15 11920608 0 swblk 136 1576 58483 216441 0 62 8168024 0 VNODE 600 9936 1049 6562539 0 22 6591000 0 malloc-128 128 44549 401 1638069 0 126 5753600 0 UMA Slabs 0 112 44038 884 57214 0 126 5031264 0 mbuf 256 16431 1789 35827 0 254 4664320 0 pbuf 2776 0 1369 0 0 0 3800344 0 FFS2 dinode 256 9791 724 2182289 0 62 2691840 0 malloc-128 128 12666 1501 2185074 0 126 1813376 0 malloc-4096 4096 332 2 4049 0 2 1368064 0 S VFS Cache 104 10183 2102 47859 0 126 1277640 0 VM OBJECT 264 3530 430 6556887 0 30 1045440 0 vmem btag 56 16196 1144 16446 0 254 971040 0 THREAD 1824 426 62 568 0 8 890112 0 swpctrie 144 253 5543 21066 0 62 834624 0 malloc-8192 8192 99 0 111 0 1 811008 0 malloc-32768 32768 24 0 24 0 1 786432 0 malloc-64 64 11697 588 21425 0 254 786240 0 256 Bucket 2048 332 46 1316110 0 8 774144 0 malloc-64 64 9004 824 11923 0 254 628992 0 malloc-32768 32768 14 5 109 0 1 622592 0 malloc-4096 4096 128 21 293239 0 2 610304 0 lkpicurr 168 13 3083 13 0 62 520128 0 malloc-128 128 1039 2898 4588434 0 126 503936 0 malloc-64 64 4717 3032 297294 0 254 495936 0 UMA Zones 2048 228 7 228 0 8 481280 0 MAP ENTRY 96 3431 1399 20993578 0 126 463680 0 FPU_save_area 832 428 112 1520 0 16 449280 0 malloc-256 256 892 713 6538024 0 62 410880 0 128 Bucket 1024 251 148 340347 0 16 408576 0 malloc-1024 1024 256 80 12544 0 16 344064 0 malloc-32 32 8145 2313 10652 0 254 334656 0 PROC 1376 120 111 292493 0 8 317856 0 malloc-16 16 15768 1982 15952 0 254 284000 0 malloc-128 128 1018 1152 2177823 0 126 277760 0 mbuf_packet 256 12 1013 6612 0 254 262400 0 malloc-16384 16384 16 0 4786 0 1 262144 0 64 Bucket 512 184 312 207761 0 30 253952 0 malloc-128 128 392 1530 6842 0 126 246016 0 L VFS Cache 320 730 38 731 0 30 245760 0 filedesc0 1072 119 105 292494 0 8 240128 0 VMSPACE 616 98 286 292398 0 16 236544 0 malloc-2048 2048 107 5 198 0 8 229376 0 g_bio 408 0 560 535904 0 30 228480 0 malloc-64 64 158 2992 329886 0 254 201600 0 malloc-2048 2048 89 9 1685 0 8 200704 0 malloc-1024 1024 4 192 341357 0 16 200704 0 malloc-64 64 29 3058 831260 0 254 197568 0 NAMEI 1024 6 186 9392151 0 16 196608 0 malloc-65536 65536 3 0 3270 0 1 196608 0 malloc-65536 65536 3 0 3 0 1 196608 0 malloc-65536 65536 2 1 18 0 1 196608 0 malloc-256 256 544 221 686 0 62 195840 0 malloc-64 64 196 2765 318124 0 254 189504 0 malloc-8192 8192 22 1 3956 0 1 188416 0 malloc-256 256 53 682 4344521 0 62 188160 0 32 Bucket 256 168 567 570988 0 62 188160 0 PWD 40 30 4515 149454 0 254 181800 0 malloc-64 64 14 2821 2171493 0 254 181440 0 DIRHASH 1024 100 76 1380 0 16 180224 0 lkpimm 56 3 3093 3 0 254 173376 0 PGRP 120 31 1355 114 0 126 166320 0 swrbuf 2776 0 59 2792 0 4 163784 0 pipe 728 5 215 152476 0 16 160160 0 fakepg 104 0 1521 468 0 126 158184 0 Mountpoints 2944 6 46 181 0 4 153088 0 Files 80 133 1767 6133193 0 126 152000 0 malloc-128 128 48 1130 1097 0 126 150784 0 malloc-128 128 138 978 1313 0 126 142848 0 malloc-1024 1024 121 11 218 0 16 135168 0 malloc-384 384 44 306 2173006 0 30 134400 0 malloc-65536 65536 2 0 2 0 1 131072 0 malloc-65536 65536 1 1 494 0 1 131072 0 malloc-32768 32768 4 0 3574 0 1 131072 0 malloc-512 512 54 202 404 0 30 131072 0 malloc-256 256 82 428 3848 0 62 130560 0 8 Bucket 80 79 1521 293381 0 126 128000 0 ksiginfo 112 210 906 13931 0 126 124992 0 DEVCTL 1024 0 120 2212 0 0 122880 0 malloc-1024 1024 27 93 491 0 16 122880 0 malloc-384 384 232 88 232 0 30 122880 0 16 Bucket 144 240 600 302166 0 62 120960 0 TURNSTILE 136 489 393 489 0 62 119952 0 socket 960 41 83 6843 0 16 119040 0 malloc-32 32 549 3105 26234 0 254 116928 0 malloc-128 128 518 381 43066 0 126 115072 0 malloc-8192 8192 14 0 94 0 1 114688 0 malloc-256 256 0 435 1680 0 62 111360 0 4 Bucket 48 25 2075 167757 0 254 100800 0 ada_ccb 272 0 364 128499 0 30 99008 0 malloc-32768 32768 3 0 4 0 1 98304 0 malloc-4096 4096 14 10 223 0 2 98304 0 udp_inpcb 416 11 223 157 0 30 97344 0 malloc-32 32 114 2910 98288 0 254 96768 0 ttyinq 160 330 245 825 0 62 92000 0 malloc-4096 4096 22 0 39 0 2 90112 0 unpcb 256 21 324 6609 0 62 88320 0 malloc-32 32 1151 1495 552987 0 254 84672 0 ttyoutq 256 173 157 431 0 62 84480 0 malloc-384 384 164 56 8814 0 30 84480 0 malloc-256 256 177 153 1563 0 62 84480 0 malloc-16384 16384 5 0 5 0 1 81920 0 malloc-16384 16384 5 0 10 0 1 81920 0 UMA Kegs 384 198 5 198 0 30 77952 0 malloc-4096 4096 19 0 21 0 2 77824 0 malloc-2048 2048 29 9 79 0 8 77824 0 swwbuf 2776 0 28 113709 0 2 77728 0 SLEEPQUEUE 88 489 375 489 0 126 76032 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-65536 65536 1 0 1 0 1 65536 0 malloc-32768 32768 2 0 6 0 1 65536 0 malloc-8192 8192 8 0 162 0 1 65536 0 malloc-8192 8192 8 0 50 0 1 65536 0 malloc-4096 4096 4 12 718 0 2 65536 0 malloc-2048 2048 16 16 536 0 8 65536 0 rl_entry 40 206 1410 210 0 254 64640 0 malloc-64 64 189 819 265 0 254 64512 0 malloc-32 32 378 1638 2108 0 254 64512 0 2 Bucket 32 92 1924 55692 0 254 64512 0 malloc-32 32 83 1555 291 0 254 52416 0 malloc-16 16 778 2472 88320 0 254 52000 0 malloc-16384 16384 0 3 118 0 1 49152 0 malloc-16384 16384 3 0 7 0 1 49152 0 pcpu-8 8 3912 2232 3914 0 254 49152 0 malloc-16 16 103 2897 471873 0 254 48000 0 malloc-16 16 862 2138 1686222 0 254 48000 0 malloc-16 16 79 2921 4380027 0 254 48000 0 pcpu-64 64 454 250 454 0 254 45056 0 pcpu-16 16 52 2764 52 0 254 45056 0 malloc-16 16 63 2437 1995 0 254 40000 0 malloc-384 384 27 73 50 0 30 38400 0 malloc-1024 1024 17 19 34 0 16 36864 0 malloc-32 32 41 1093 477 0 254 36288 0 malloc-32768 32768 1 0 1 0 1 32768 0 malloc-16384 16384 2 0 9313 0 1 32768 0 malloc-512 512 4 60 82 0 30 32768 0 malloc-512 512 28 36 5959 0 30 32768 0 udp_inpcb ports 32 3 1005 17 0 254 32256 0 KNOTE 160 13 187 7122 0 62 32000 0 malloc-384 384 27 53 490 0 30 30720 0 cpuset 200 27 117 27 0 62 28800 0 malloc-2048 2048 11 3 74197 0 8 28672 0 malloc-32 32 38 844 539 0 254 28224 0 tcp_inpcb 1304 6 15 11 0 8 27384 0 malloc-256 256 16 89 683 0 62 26880 0 malloc-8192 8192 3 0 3 0 1 24576 0 ertt_txseginfo 40 0 606 1994 0 254 24240 0 malloc-16 16 26 1474 4684 0 254 24000 0 malloc-256 256 13 77 95 0 62 23040 0 rtentry 168 12 108 14 0 62 20160 0 routing nhops 256 9 66 14 0 62 19200 0 malloc-16 16 4 996 4 0 254 16000 0 malloc-2048 2048 3 3 3 0 8 12288 0 malloc-1024 1024 5 7 5 0 16 12288 0 malloc-512 512 12 12 12 0 30 12288 0 malloc-512 512 3 21 3 0 30 12288 0 ertt 72 6 162 11 0 126 12096 0 malloc-384 384 10 20 10 0 30 11520 0 malloc-384 384 13 17 13 0 30 11520 0 malloc-8192 8192 1 0 1 0 1 8192 0 malloc-8192 8192 1 0 1 0 1 8192 0 tcp_inpcb ports 32 2 250 2 0 254 8064 0 kenv 258 18 12 27409 0 30 7740 0 ripcb 384 1 19 3 0 30 7680 0 epoch_record pcpu 256 4 12 4 0 62 4096 0 malloc-2048 2048 1 1 1 0 8 4096 0 malloc-1024 1024 2 2 2 0 16 4096 0 malloc-512 512 1 7 57 0 30 4096 0 malloc-512 512 2 6 22 0 30 4096 0 SMR CPU 32 7 120 7 0 254 4064 0 vtnet_tx_hdr 24 0 167 0 0 254 4008 0 UMA Slabs 1 176 11 11 11 0 62 3872 0 KMAP ENTRY 96 36 3 36 0 0 3744 0 vmem 1856 2 0 2 0 8 3712 0 SMR SHARED 24 7 120 7 0 254 3048 0 FFS1 dinode 128 0 0 0 0 126 0 0 da_ccb 544 0 0 0 0 16 0 0 tfo_ccache_entries 80 0 0 0 0 126 0 0 tfo 4 0 0 0 0 254 0 0 sackhole 32 0 0 0 0 254 0 0 hostcache 64 0 0 0 0 254 0 0 syncache 168 0 0 2 0 62 0 0 ipq 56 0 0 0 0 254 0 0 tcp_log_id_node 120 0 0 0 0 126 0 0 tcp_log_id_bucket 176 0 0 0 0 62 0 0 tcp_log 416 0 0 0 0 30 0 0 tcpreass 48 0 0 0 0 254 0 0 udplite_inpcb ports 32 0 0 0 0 254 0 0 udplite_inpcb 416 0 0 0 0 30 0 0 ripcb ports 32 0 0 0 0 254 0 0 IPsec SA lft_c 16 0 0 0 0 254 0 0 itimer 352 0 0 0 0 30 0 0 AIOLIO 272 0 0 0 0 30 0 0 AIOCB 552 0 0 0 0 16 0 0 AIO 208 0 0 0 0 62 0 0 TMPFS node 232 0 0 0 0 62 0 0 NCLNODE 760 0 0 0 0 16 0 0 LTS VFS Cache 360 0 0 0 0 30 0 0 STS VFS Cache 144 0 0 0 0 62 0 0 cryptop 280 0 0 0 0 30 0 0 linux_dma_object 32 0 0 0 0 254 0 0 linux_dma_pctrie 144 0 0 0 0 62 0 0 IOMMU_MAP_ENTRY 104 0 0 0 0 126 0 0 mbuf_jumbo_16k 16384 0 0 0 0 1 0 0 mbuf_jumbo_9k 9216 0 0 0 0 1 0 0 mbuf_jumbo_page 4096 0 0 4 0 2 0 0 audit_record 1280 0 0 0 0 8 0 0 domainset 40 0 0 0 0 254 0 0 MAC labels 40 0 0 0 0 254 0 0 clpbuf 2776 0 0 12 0 4 0 0 vnpbuf 2776 0 0 0 0 4 0 0 nfspbuf 2776 0 0 0 0 4 0 0 umtx_shm 88 0 0 0 0 126 0 0 umtx pi 96 0 0 0 0 126 0 0 rangeset pctrie nodes 144 0 0 0 0 62 0 0 malloc-65536 65536 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-32768 32768 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-16384 16384 0 0 0 0 1 0 0 malloc-4096 4096 0 0 0 0 2 0 0 malloc-2048 2048 0 0 0 0 8 0 0 malloc-1024 1024 0 0 0 0 16 0 0 malloc-512 512 0 0 0 0 30 0 0 malloc-384 384 0 0 1 0 30 0 0 pcpu-32 32 0 0 0 0 254 0 0 pcpu-4 4 0 0 0 0 254 0 0 UMA Hash 256 0 0 0 0 62 0 0 db:0:pho> show umacache Zone Size Used Free Requests Bucket debugnet_mbuf_packet 2048 0 0 0 0 debugnet_mbuf_cluster 2048 -6144 0 0 0 debugnet_mbuf 256 -10240 0 0 0 vm pgcache 4096 72640 141 18402630 9 vm pgcache 4096 21792 191 27109308 9 vm pgcache 4096 0 0 0 2 buf free cache 848 5333 161 2178814 16 buffer arena-65 65536 0 0 0 1 buffer arena-61 61440 0 0 0 1 buffer arena-57 57344 0 0 0 1 buffer arena-53 53248 0 0 0 1 buffer arena-49 49152 0 0 0 1 buffer arena-45 45056 0 0 0 1 buffer arena-40 40960 0 0 0 1 buffer arena-36 36864 0 0 0 1 buffer arena-32 32768 1528 5 3045 1 buffer arena-28 28672 0 0 0 1 buffer arena-24 24576 0 0 0 1 buffer arena-20 20480 0 0 0 1 buffer arena-16 16384 7 0 322 1 buffer arena-12 12288 0 0 0 1 buffer arena-81 8192 0 0 0 1 buffer arena-40 4096 0 0 0 2 kstack_cache 16384 426 0 568 1 db:0:pho> acttrace Tracing command mmap pid 92547 tid 100442 td 0xfffff802f72a0740 (CPU 7) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ce96df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ce96e00 trap() at trap+0x56/frame 0xfffffe003ce96f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ce96f20 --- trap 0x13, rip = 0xffffffff80917af8, rsp = 0xfffffe010867c380, rbp = 0xfffffe010867c3a0 --- ns8250_putc() at ns8250_putc+0x108/frame 0xfffffe010867c3a0 uart_cnputc() at uart_cnputc+0x45/frame 0xfffffe010867c3c0 cnputc() at cnputc+0x4b/frame 0xfffffe010867c3f0 cnputs() at cnputs+0x7a/frame 0xfffffe010867c420 putchar() at putchar+0x165/frame 0xfffffe010867c4b0 kvprintf() at kvprintf+0xe6/frame 0xfffffe010867c5c0 _vprintf() at _vprintf+0x8b/frame 0xfffffe010867c6a0 printf() at printf+0x53/frame 0xfffffe010867c700 stack_print_ddb() at stack_print_ddb+0x68/frame 0xfffffe010867c760 vn_printf() at vn_printf+0x42b/frame 0xfffffe010867c910 cache_enter_time() at cache_enter_time+0x12b0/frame 0xfffffe010867c9f0 unionfs_lookup() at unionfs_lookup+0x7fa/frame 0xfffffe010867cb50 VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0x5f/frame 0xfffffe010867cb80 vfs_cache_lookup() at vfs_cache_lookup+0xa6/frame 0xfffffe010867cbd0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0x5f/frame 0xfffffe010867cc00 vfs_lookup() at vfs_lookup+0x487/frame 0xfffffe010867cc80 namei() at namei+0x2d1/frame 0xfffffe010867cce0 kern_chdir() at kern_chdir+0xc3/frame 0xfffffe010867ce00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010867cf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010867cf30 --- syscall (12, FreeBSD ELF64, chdir), rip = 0x3a11423df55a, rsp = 0x3a114032c808, rbp = 0x3a114032c810 --- Tracing command swap pid 92548 tid 100403 td 0xfffff802f7814740 (CPU 6) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ce87df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ce87e00 trap() at trap+0x56/frame 0xfffffe003ce87f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ce87f20 --- trap 0x13, rip = 0xf88a4d0e574, rsp = 0xf90c52dd100, rbp = 0xf90c52dd130 --- Tracing command mmap pid 92551 tid 100429 td 0xfffff802f76c7740 (CPU 2) kdb_enter() at kdb_enter+0x33/frame 0xfffffe010874e8b0 panic() at panic+0x43/frame 0xfffffe010874e910 cache_enter_time() at cache_enter_time+0x145b/frame 0xfffffe010874e9f0 unionfs_lookup() at unionfs_lookup+0x7fa/frame 0xfffffe010874eb50 VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0x5f/frame 0xfffffe010874eb80 vfs_cache_lookup() at vfs_cache_lookup+0xa6/frame 0xfffffe010874ebd0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0x5f/frame 0xfffffe010874ec00 vfs_lookup() at vfs_lookup+0x487/frame 0xfffffe010874ec80 namei() at namei+0x2d1/frame 0xfffffe010874ece0 kern_chdir() at kern_chdir+0xc3/frame 0xfffffe010874ee00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010874ef30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010874ef30 --- syscall (12, FreeBSD ELF64, chdir), rip = 0x3a11423df55a, rsp = 0x3a114032c808, rbp = 0x3a114032c810 --- Tracing command swap pid 92558 tid 100534 td 0xfffff800331cd000 (CPU 5) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ce78df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ce78e00 trap() at trap+0x56/frame 0xfffffe003ce78f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ce78f20 --- trap 0x13, rip = 0xf88a4d0e574, rsp = 0xf90c52dd100, rbp = 0xf90c52dd130 --- Tracing command swap pid 92564 tid 100536 td 0xfffff802f7638000 (CPU 11) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ced2df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ced2e00 trap() at trap+0x56/frame 0xfffffe003ced2f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ced2f20 --- trap 0x13, rip = 0xf88a4d0e574, rsp = 0xf90c52dd100, rbp = 0xf90c52dd130 --- Tracing command mmap pid 92565 tid 100465 td 0xfffff806a0787740 (CPU 4) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ce69df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ce69e00 trap() at trap+0x56/frame 0xfffffe003ce69f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ce69f20 --- trap 0x13, rip = 0xffffffff80b985a2, rsp = 0xfffffe01087f8348, rbp = 0xfffffe01087f83b0 --- lock_delay() at lock_delay+0x12/frame 0xfffffe01087f83b0 __mtx_lock_spin_flags() at __mtx_lock_spin_flags+0xc3/frame 0xfffffe01087f83f0 cnputs() at cnputs+0x59/frame 0xfffffe01087f8420 putchar() at putchar+0x165/frame 0xfffffe01087f84b0 kvprintf() at kvprintf+0xe6/frame 0xfffffe01087f85c0 _vprintf() at _vprintf+0x8b/frame 0xfffffe01087f86a0 printf() at printf+0x53/frame 0xfffffe01087f8700 stack_print_ddb() at stack_print_ddb+0x68/frame 0xfffffe01087f8760 vn_printf() at vn_printf+0x42b/frame 0xfffffe01087f8910 cache_enter_time() at cache_enter_time+0x12b0/frame 0xfffffe01087f89f0 unionfs_lookup() at unionfs_lookup+0x7fa/frame 0xfffffe01087f8b50 VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0x5f/frame 0xfffffe01087f8b80 vfs_cache_lookup() at vfs_cache_lookup+0xa6/frame 0xfffffe01087f8bd0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0x5f/frame 0xfffffe01087f8c00 vfs_lookup() at vfs_lookup+0x487/frame 0xfffffe01087f8c80 namei() at namei+0x2d1/frame 0xfffffe01087f8ce0 kern_chdir() at kern_chdir+0xc3/frame 0xfffffe01087f8e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01087f8f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01087f8f30 --- syscall (12, FreeBSD ELF64, chdir), rip = 0x3a11423df55a, rsp = 0x3a114032c808, rbp = 0x3a114032c810 --- Tracing command mmap pid 92567 tid 100436 td 0xfffff802132f4740 (CPU 9) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ceb4df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ceb4e00 trap() at trap+0x56/frame 0xfffffe003ceb4f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ceb4f20 --- trap 0x13, rip = 0xffffffff80b985a2, rsp = 0xfffffe0108776348, rbp = 0xfffffe01087763b0 --- lock_delay() at lock_delay+0x12/frame 0xfffffe01087763b0 __mtx_lock_spin_flags() at __mtx_lock_spin_flags+0xc3/frame 0xfffffe01087763f0 cnputs() at cnputs+0x59/frame 0xfffffe0108776420 putchar() at putchar+0x165/frame 0xfffffe01087764b0 kvprintf() at kvprintf+0xe6/frame 0xfffffe01087765c0 _vprintf() at _vprintf+0x8b/frame 0xfffffe01087766a0 printf() at printf+0x53/frame 0xfffffe0108776700 stack_print_ddb() at stack_print_ddb+0x68/frame 0xfffffe0108776760 vn_printf() at vn_printf+0x42b/frame 0xfffffe0108776910 cache_enter_time() at cache_enter_time+0x12b0/frame 0xfffffe01087769f0 unionfs_lookup() at unionfs_lookup+0x7fa/frame 0xfffffe0108776b50 VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0x5f/frame 0xfffffe0108776b80 vfs_cache_lookup() at vfs_cache_lookup+0xa6/frame 0xfffffe0108776bd0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0x5f/frame 0xfffffe0108776c00 vfs_lookup() at vfs_lookup+0x487/frame 0xfffffe0108776c80 namei() at namei+0x2d1/frame 0xfffffe0108776ce0 kern_chdir() at kern_chdir+0xc3/frame 0xfffffe0108776e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108776f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108776f30 --- syscall (12, FreeBSD ELF64, chdir), rip = 0x3a11423df55a, rsp = 0x3a114032c808, rbp = 0x3a114032c810 --- Tracing command mmap pid 92569 tid 100415 td 0xfffff8021314b740 (CPU 0) cpustop_handler() at cpustop_handler+0x27/frame 0xffffffff81d09f80 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xffffffff81d09f90 trap() at trap+0x56/frame 0xffffffff81d0a0b0 nmi_calltrap() at nmi_calltrap+0x8/frame 0xffffffff81d0a0b0 --- trap 0x13, rip = 0xffffffff80b985a2, rsp = 0xfffffe01086fe348, rbp = 0xfffffe01086fe3b0 --- lock_delay() at lock_delay+0x12/frame 0xfffffe01086fe3b0 __mtx_lock_spin_flags() at __mtx_lock_spin_flags+0xc3/frame 0xfffffe01086fe3f0 cnputs() at cnputs+0x59/frame 0xfffffe01086fe420 putchar() at putchar+0x165/frame 0xfffffe01086fe4b0 kvprintf() at kvprintf+0xe6/frame 0xfffffe01086fe5c0 _vprintf() at _vprintf+0x8b/frame 0xfffffe01086fe6a0 printf() at printf+0x53/frame 0xfffffe01086fe700 stack_print_ddb() at stack_print_ddb+0x68/frame 0xfffffe01086fe760 vn_printf() at vn_printf+0x42b/frame 0xfffffe01086fe910 cache_enter_time() at cache_enter_time+0x12b0/frame 0xfffffe01086fe9f0 unionfs_lookup() at unionfs_lookup+0x7fa/frame 0xfffffe01086feb50 VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0x5f/frame 0xfffffe01086feb80 vfs_cache_lookup() at vfs_cache_lookup+0xa6/frame 0xfffffe01086febd0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0x5f/frame 0xfffffe01086fec00 vfs_lookup() at vfs_lookup+0x487/frame 0xfffffe01086fec80 namei() at namei+0x2d1/frame 0xfffffe01086fece0 kern_chdir() at kern_chdir+0xc3/frame 0xfffffe01086fee00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01086fef30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01086fef30 --- syscall (12, FreeBSD ELF64, chdir), rip = 0x3a11423df55a, rsp = 0x3a114032c808, rbp = 0x3a114032c810 --- Tracing command swap pid 92573 tid 100449 td 0xfffff802f7650000 (CPU 1) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ce3cdf0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ce3ce00 trap() at trap+0x56/frame 0xfffffe003ce3cf20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ce3cf20 --- trap 0x13, rip = 0xf88a4d0e57a, rsp = 0xf90c52dd100, rbp = 0xf90c52dd130 --- Tracing command swap pid 92576 tid 100373 td 0xfffff80015ebd000 (CPU 10) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003cec3df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003cec3e00 trap() at trap+0x56/frame 0xfffffe003cec3f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003cec3f20 --- trap 0x13, rip = 0xffffffff80b53ca8, rsp = 0xfffffe0108505b20, rbp = 0xfffffe0108505b30 --- _sx_slock_int() at _sx_slock_int+0x8/frame 0xfffffe0108505b30 vm_map_lookup() at vm_map_lookup+0xa7/frame 0xfffffe0108505c20 vm_fault() at vm_fault+0x11d/frame 0xfffffe0108505d60 vm_fault_trap() at vm_fault_trap+0x5d/frame 0xfffffe0108505da0 trap_pfault() at trap_pfault+0x21d/frame 0xfffffe0108505e10 trap() at trap+0x51d/frame 0xfffffe0108505f30 calltrap() at calltrap+0x8/frame 0xfffffe0108505f30 --- trap 0xc, rip = 0xf90c7212325, rsp = 0xf90c52dcd78, rbp = 0xf90c52dcdc0 --- Tracing command swap pid 92583 tid 100408 td 0xfffff80213886000 (CPU 3) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ce5adf0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ce5ae00 trap() at trap+0x56/frame 0xfffffe003ce5af20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ce5af20 --- trap 0x13, rip = 0xf88a4d0e57a, rsp = 0xf90c52dd100, rbp = 0xf90c52dd130 --- Tracing command swap pid 92589 tid 100391 td 0xfffff802f7303740 (CPU 8) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003cea5df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003cea5e00 trap() at trap+0x56/frame 0xfffffe003cea5f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003cea5f20 --- trap 0x13, rip = 0xffffffff80b545af, rsp = 0xfffffe0108690bf0, rbp = 0xfffffe0108690c00 --- _sx_sunlock_int() at _sx_sunlock_int+0xbf/frame 0xfffffe0108690c00 vm_map_lookup_done() at vm_map_lookup_done+0x46/frame 0xfffffe0108690c10 vm_fault_deallocate() at vm_fault_deallocate+0x161/frame 0xfffffe0108690c30 vm_fault() at vm_fault+0x14da/frame 0xfffffe0108690d60 vm_fault_trap() at vm_fault_trap+0x5d/frame 0xfffffe0108690da0 trap_pfault() at trap_pfault+0x21d/frame 0xfffffe0108690e10 trap() at trap+0x51d/frame 0xfffffe0108690f30 calltrap() at calltrap+0x8/frame 0xfffffe0108690f30 --- trap 0xc, rip = 0xf90c7212325, rsp = 0xf90c52dcd78, rbp = 0xf90c52dcdc0 --- db:0:pho> ps pid ppid pgrp uid state wmesg wchan cmd 92605 92516 3706 0 S+ nanslp 0xffffffff81bdb1a6 sleep 92602 1 3706 0 S+ select 0xfffff801a54b2340 tail 92596 92516 3706 0 D+ ufs 0xfffffe016d4c6070 tail 92595 92546 3706 0 S+ nanslp 0xffffffff81bdb1a6 swap 92594 92546 3706 0 S+ nanslp 0xffffffff81bdb1a5 swap 92593 92546 3706 0 R+ swap 92592 92546 3706 0 S+ nanslp 0xffffffff81bdb1a7 swap 92591 92546 3706 0 R+ swap 92590 92546 3706 0 S+ nanslp 0xffffffff81bdb1a1 swap 92589 92546 3706 0 R+ CPU 8 swap 92588 92546 3706 0 S+ nanslp 0xffffffff81bdb1a4 swap 92587 92546 3706 0 S+ nanslp 0xffffffff81bdb1aa swap 92586 92546 3706 0 S+ nanslp 0xffffffff81bdb1a6 swap 92585 92546 3706 0 R+ swap 92584 92546 3706 0 S+ nanslp 0xffffffff81bdb1a5 swap 92583 92546 3706 0 R+ CPU 3 swap 92582 92546 3706 0 S+ nanslp 0xffffffff81bdb1a3 swap 92581 92546 3706 0 S+ nanslp 0xffffffff81bdb1a1 swap 92580 92546 3706 0 S+ nanslp 0xffffffff81bdb1a5 swap 92579 92546 3706 0 S+ nanslp 0xffffffff81bdb1ab swap 92578 92546 3706 0 S+ nanslp 0xffffffff81bdb1a0 swap 92577 92546 3706 0 S+ nanslp 0xffffffff81bdb1a0 swap 92576 92546 3706 0 R+ CPU 10 swap 92575 92546 3706 0 R+ swap 92574 92546 3706 0 S+ nanslp 0xffffffff81bdb1a6 swap 92573 92546 3706 0 R+ CPU 1 swap 92572 92546 3706 0 S+ nanslp 0xffffffff81bdb1a3 swap 92571 92545 3706 0 R+ mmap 92570 92546 3706 0 R+ swap 92569 92545 3706 0 R+ CPU 0 mmap 92568 92546 3706 0 S+ nanslp 0xffffffff81bdb1ab swap 92567 92545 3706 0 R+ CPU 9 mmap 92566 92546 3706 0 S+ nanslp 0xffffffff81bdb1a1 swap 92565 92545 3706 0 R+ CPU 4 mmap 92564 92546 3706 0 R+ CPU 11 swap 92563 92545 3706 0 S+ nanslp 0xffffffff81bdb1a0 mmap 92562 92546 3706 0 S+ nanslp 0xffffffff81bdb1aa swap 92561 92545 3706 0 S+ nanslp 0xffffffff81bdb1a2 mmap 92560 92546 3706 0 S+ nanslp 0xffffffff81bdb1a0 swap 92558 92546 3706 0 R+ CPU 5 swap 92557 92545 3706 0 S+ nanslp 0xffffffff81bdb1a3 mmap 92556 92546 3706 0 S+ nanslp 0xffffffff81bdb1aa swap 92555 92545 3706 0 S+ nanslp 0xffffffff81bdb1a0 mmap 92554 92546 3706 0 S+ nanslp 0xffffffff81bdb1a7 swap 92552 92546 3706 0 R+ swap 92551 92545 3706 0 R+ CPU 2 mmap 92550 92546 3706 0 S+ nanslp 0xffffffff81bdb1a0 swap 92549 92545 3706 0 S+ nanslp 0xffffffff81bdb1a3 mmap 92548 92546 3706 0 R+ CPU 6 swap 92547 92545 3706 0 R+ CPU 7 mmap 92546 92544 3706 0 S+ wait 0xfffffe016ad315a0 swap 92545 92543 3706 0 S+ wait 0xfffffe016e56cac0 mmap 92544 92542 3706 0 S+ nanslp 0xffffffff81bdb1a6 swap 92543 92542 3706 0 S+ nanslp 0xffffffff81bdb1a0 mmap 92542 92541 3706 0 S+ wait 0xfffffe016f191040 run 92541 92539 3706 0 S+ wait 0xfffffe016d35b040 run 92539 92516 3706 0 S+ nanslp 0xffffffff81bdb1aa run 92530 0 0 0 DL mdwait 0xfffff80213894000 [md11] 92527 0 0 0 DL mdwait 0xfffff80040977000 [md10] 92516 92514 3706 0 S+ wait 0xfffffe016e4f1580 sh 92515 3706 3706 0 S+ piperd 0xfffffe016ad90c70 tee 92514 3706 3706 0 S+ wait 0xfffffe01623735a0 sh 92509 3821 3789 0 S nanslp 0xffffffff81bdb1a3 sleep 87509 8883 8883 2006 S+ nanslp 0xffffffff81bdb1a5 sleep 8883 6165 8883 2006 S+ wait 0xfffffe016e4f0000 sh 6165 6082 6165 2006 SWs+ wait 0xfffffe0162371560 sh 6082 3956 3956 2006 S select 0xfffff800050084c0 sshd 3956 3595 3956 0 Ss select 0xfffff80009410e40 sshd 3824 3789 3789 0 S piperd 0xfffffe016caa3c70 sh 3822 3789 3789 0 S piperd 0xfffffe0162361e38 awk 3821 3789 3789 0 S wait 0xfffffe016ad31040 sh 3789 1 3789 0 SWs wait 0xfffffe003ce475a0 sh 3706 3646 3706 0 S+ wait 0xfffffe016d35c060 sh 3646 3645 3646 0 SW+ wait 0xfffffe016ad30580 bash 3645 3644 3645 0 SW+ wait 0xfffffe003ce47040 sh 3644 3622 3644 2006 SW+ wait 0xfffffe016d35c5c0 su 3622 3621 3622 2006 SWs+ wait 0xfffffe01623685c0 sh 3621 3618 3618 2006 S select 0xfffff8000b60d040 sshd 3618 3595 3618 0 Ss select 0xfffff80005008c40 sshd 3617 1 3617 0 Ss+ ttyin 0xfffff800063238b0 getty 3616 1 3616 0 Ss+ ttyin 0xfffff800094748b0 getty 3615 1 3615 0 Ss+ ttyin 0xfffff80009474cb0 getty 3614 1 3614 0 Ss+ ttyin 0xfffff800094730b0 getty 3613 1 3613 0 Ss+ ttyin 0xfffff800094734b0 getty 3612 1 3612 0 Ss+ ttyin 0xfffff800094738b0 getty 3611 1 3611 0 Ss+ ttyin 0xfffff80009473cb0 getty 3610 1 3610 0 Ss+ ttyin 0xfffff800094720b0 getty 3609 1 3609 0 Ss+ ttyin 0xfffff800094724b0 getty 3595 1 3595 0 Ss select 0xfffff800094f5ac0 sshd 3578 1 3578 0 SWs nanslp 0xffffffff81bdb1a1 cron 3573 1 3573 0 Ss select 0xfffff80021017bc0 inetd 3552 1 3552 0 Ss select 0xfffff80006269a40 powerd 3535 1 3535 123 Ss (threaded) ntpd 100339 S select 0xfffff800057a8e40 ntpd 100561 S usem 0xfffff80003b8d780 ntpd 3437 1 3437 0 Ss nanslp 0xffffffff81bdb1ab watchdogd 3420 1 3420 0 Rs syslogd 3272 1 3272 0 Ss select 0xfffff80003d2f940 devd 3257 1 3257 65 Ss select 0xfffff80006269940 dhclient 3198 1 3198 0 Ss select 0xfffff8000b60d140 dhclient 3195 1 3195 0 Ss select 0xfffff800050b6c40 dhclient 19 0 0 0 DL syncer 0xffffffff81cc19a8 [syncer] 18 0 0 0 DL vlruwt 0xfffffe010ebf7020 [vnlru] 17 0 0 0 DL (threaded) [bufdaemon] 100201 D psleep 0xffffffff81cc09c0 [bufdaemon] 100203 D - 0xffffffff81802100 [bufspacedaemon-0] 100204 D - 0xffffffff81802380 [bufspacedaemon-1] 100205 D - 0xffffffff81802600 [bufspacedaemon-2] 100206 D - 0xffffffff81802880 [bufspacedaemon-3] 100207 D - 0xffffffff81802b00 [bufspacedaemon-4] 100208 D - 0xffffffff81802d80 [bufspacedaemon-5] 100209 D - 0xffffffff81803000 [bufspacedaemon-6] 100235 D sdflush 0xfffff80015de4ce8 [/ worker] 100252 D sdflush 0xfffff8002124ace8 [/tmp worker] 100566 D sdflush 0xfffff80125f0ece8 [/mnt10 worker] 100567 D sdflush 0xfffff804a985fce8 [/mnt11 worker] 16 0 0 0 DL psleep 0xffffffff81cf6450 [vmdaemon] 9 0 0 0 DL (threaded) [pagedaemon] 100198 D psleep 0xffffffff81cdc978 [dom0] 100200 D laundp 0xffffffff813c49b7 [laundry: dom0] 100202 D umarcl 0xffffffff80ee62b0 [uma] 8 0 0 0 DL idle 0xfffffe003ce75060 [enc_daemon1] 7 0 0 0 DL idle 0xfffffe003ce755c0 [enc_daemon0] 6 0 0 0 DL - 0xffffffff819dc9b0 [rand_harvestq] 15 0 0 0 DL (threaded) [usb] 100128 D - 0xfffffe010a9cf440 [usbus0] 100129 D - 0xfffffe010a9cf498 [usbus0] 100130 D - 0xfffffe010a9cf4f0 [usbus0] 100131 D - 0xfffffe010a9cf548 [usbus0] 100132 D - 0xfffffe010a9cf5a0 [usbus0] 100134 D - 0xfffffe00e4d9ccf0 [usbus1] 100135 D - 0xfffffe00e4d9cd48 [usbus1] 100136 D - 0xfffffe00e4d9cda0 [usbus1] 100137 D - 0xfffffe00e4d9cdf8 [usbus1] 100138 D - 0xfffffe00e4d9ce50 [usbus1] 100139 D - 0xfffffe010ad7ccf0 [usbus2] 100140 D - 0xfffffe010ad7cd48 [usbus2] 100141 D - 0xfffffe010ad7cda0 [usbus2] 100142 D - 0xfffffe010ad7cdf8 [usbus2] 100143 D - 0xfffffe010ad7ce50 [usbus2] 5 0 0 0 DL - 0xffffffff81d009e8 [busdma] 4 0 0 0 DL (threaded) [cam] 100121 D - 0xffffffff819ae240 [doneq0] 100122 D - 0xffffffff819ae2c0 [doneq1] 100123 D - 0xffffffff819ae340 [doneq2] 100124 D - 0xffffffff819ae1c0 [async] 100153 D - 0xffffffff819ae090 [scanner] 3 0 0 0 DL (threaded) [crypto] 100108 D crypto_ 0xffffffff81cd9db0 [crypto] 100109 D crypto_ 0xfffff8000514b830 [crypto returns 0] 100110 D crypto_ 0xfffff8000514b880 [crypto returns 1] 100111 D crypto_ 0xfffff8000514b8d0 [crypto returns 2] 100112 D crypto_ 0xfffff8000514b920 [crypto returns 3] 100113 D crypto_ 0xfffff8000514b970 [crypto returns 4] 100114 D crypto_ 0xfffff8000514b9c0 [crypto returns 5] 100115 D crypto_ 0xfffff8000514ba10 [crypto returns 6] 100116 D crypto_ 0xfffff8000514ba60 [crypto returns 7] 100117 D crypto_ 0xfffff8000514bab0 [crypto returns 8] 100118 D crypto_ 0xfffff8000514bb00 [crypto returns 9] 100119 D crypto_ 0xfffff8000514bb50 [crypto returns 10] 100120 D crypto_ 0xfffff8000514bba0 [crypto returns 11] 14 0 0 0 DL seqstat 0xfffff80005103488 [sequencer 00] 13 0 0 0 DL (threaded) [geom] 100092 D - 0xffffffff81b96bb0 [g_event] 100093 D - 0xffffffff81b96bb8 [g_up] 100094 D - 0xffffffff81b96bc0 [g_down] 2 0 0 0 WL (threaded) [clock] 100078 I [clock (0)] 100079 I [clock (1)] 100080 I [clock (2)] 100081 I [clock (3)] 100082 I [clock (4)] 100083 I [clock (5)] 100084 I [clock (6)] 100085 I [clock (7)] 100086 I [clock (8)] 100087 I [clock (9)] 100088 I [clock (10)] 100089 I [clock (11)] 12 0 0 0 WL (threaded) [intr] 100042 I [swi6: task queue] 100043 I [swi6: Giant taskq] 100045 I [swi5: fast taskq] 100090 I [swi1: netisr 0] 100125 I [irq56: ahci0] 100127 I [irq19: xhci0] 100133 I [irq18: ehci0 ehci1+] 100144 I [irq71: ahci1] 100145 I [swi4: apei] 100146 I [swi0: uart uart] 100160 I [irq73: mlx5_core0] 100161 I [irq74: mlx5_core0] 100162 I [irq72: mlx5_core0] 100163 I [irq75: mlx5_core0] 100164 I [irq76: mlx5_core0] 100165 I [irq77: mlx5_core0] 100166 I [irq78: mlx5_core0] 100167 I [irq79: mlx5_core0] 100168 I [irq80: mlx5_core0] 100169 I [irq81: mlx5_core0] 100170 I [irq82: mlx5_core0] 100171 I [irq83: mlx5_core0] 100172 I [irq84: mlx5_core0] 100173 I [irq85: mlx5_core0] 100174 I [irq86: mlx5_core0] 100181 I [irq88: mlx5_core1] 100182 I [irq89: mlx5_core1] 100183 I [irq87: mlx5_core1] 100184 I [irq90: mlx5_core1] 100185 I [irq91: mlx5_core1] 100186 I [irq92: mlx5_core1] 100187 I [irq93: mlx5_core1] 100188 I [irq94: mlx5_core1] 100189 I [irq95: mlx5_core1] 100190 I [irq96: mlx5_core1] 100191 I [irq97: mlx5_core1] 100192 I [irq98: mlx5_core1] 100193 I [irq99: mlx5_core1] 100194 I [irq100: mlx5_core1] 100195 I [irq101: mlx5_core1] 100244 I [irq102: ioat0] 100245 I [irq103: ioat1] 100246 I [irq104: ioat2] 100247 I [irq105: ioat3] 100248 I [irq106: ioat4] 100249 I [irq107: ioat5] 100250 I [irq108: ioat6] 100251 I [irq109: ioat7] 11 0 0 0 RL (threaded) [idle] 100003 CanRun [idle: cpu0] 100004 CanRun [idle: cpu1] 100005 CanRun [idle: cpu2] 100006 CanRun [idle: cpu3] 100007 CanRun [idle: cpu4] 100008 CanRun [idle: cpu5] 100009 CanRun [idle: cpu6] 100010 CanRun [idle: cpu7] 100011 CanRun [idle: cpu8] 100012 CanRun [idle: cpu9] 100013 CanRun [idle: cpu10] 100014 CanRun [idle: cpu11] 1 0 1 0 SLs wait 0xfffffe00e4c84040 [init] 10 0 0 0 DL audit_w 0xffffffff81cda2b0 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff81b97120 [swapper] 100015 D - 0xfffff800039edd00 [softirq_0] 100016 D - 0xfffff800039edc00 [softirq_1] 100017 D - 0xfffff800039edb00 [softirq_2] 100018 D - 0xfffff800039eda00 [softirq_3] 100019 D - 0xfffff800039ed900 [softirq_4] 100020 D - 0xfffff800039ed800 [softirq_5] 100021 D - 0xfffff800039ed700 [softirq_6] 100022 D - 0xfffff800039ed600 [softirq_7] 100023 D - 0xfffff800039ed500 [softirq_8] 100024 D - 0xfffff800039ed400 [softirq_9] 100025 D - 0xfffff800039ed300 [softirq_10] 100026 D - 0xfffff800039ed200 [softirq_11] 100027 D - 0xfffff800039ed100 [if_io_tqg_0] 100028 D - 0xfffff800039ed000 [if_io_tqg_1] 100029 D - 0xfffff800039ece00 [if_io_tqg_2] 100030 D - 0xfffff800039ecd00 [if_io_tqg_3] 100031 D - 0xfffff800039ecc00 [if_io_tqg_4] 100032 D - 0xfffff800039ecb00 [if_io_tqg_5] 100033 D - 0xfffff800039eca00 [if_io_tqg_6] 100034 D - 0xfffff800039ec900 [if_io_tqg_7] 100035 D - 0xfffff800039ec800 [if_io_tqg_8] 100036 D - 0xfffff800039ec700 [if_io_tqg_9] 100037 D - 0xfffff800039ec600 [if_io_tqg_10] 100038 D - 0xfffff800039ec500 [if_io_tqg_11] 100039 D - 0xfffff800039ec400 [if_config_tqg_0] 100040 D - 0xfffff800039ec300 [pci_hp taskq] 100041 D - 0xfffff800039ec200 [kqueue_ctx taskq] 100044 D - 0xfffff80003b9c300 [thread taskq] 100046 D - 0xfffff80003b9c100 [aiod_kick taskq] 100047 D - 0xfffff80003b9c000 [deferred_unmount ta] 100048 D - 0xfffff80003b9de00 [inm_free taskq] 100049 D - 0xfffff80003b9dd00 [in6m_free taskq] 100050 D - 0xfffff80003b9dc00 [linuxkpi_irq_wq] 100051 D - 0xfffff80003b9db00 [linuxkpi_short_wq_0] 100052 D - 0xfffff80003b9db00 [linuxkpi_short_wq_1] 100053 D - 0xfffff80003b9db00 [linuxkpi_short_wq_2] 100054 D - 0xfffff80003b9db00 [linuxkpi_short_wq_3] 100055 D - 0xfffff80003b9db00 [linuxkpi_short_wq_4] 100056 D - 0xfffff80003b9db00 [linuxkpi_short_wq_5] 100057 D - 0xfffff80003b9db00 [linuxkpi_short_wq_6] 100058 D - 0xfffff80003b9db00 [linuxkpi_short_wq_7] 100059 D - 0xfffff80003b9db00 [linuxkpi_short_wq_8] 100060 D - 0xfffff80003b9db00 [linuxkpi_short_wq_9] 100061 D - 0xfffff80003b9db00 [linuxkpi_short_wq_1] 100062 D - 0xfffff80003b9db00 [linuxkpi_short_wq_1] 100063 D - 0xfffff80003b9db00 [linuxkpi_short_wq_1] 100064 D - 0xfffff80003b9da00 [linuxkpi_long_wq_0] 100065 D - 0xfffff80003b9da00 [linuxkpi_long_wq_1] 100066 D - 0xfffff80003b9da00 [linuxkpi_long_wq_2] 100067 D - 0xfffff80003b9da00 [linuxkpi_long_wq_3] 100068 D - 0xfffff80003b9da00 [linuxkpi_long_wq_4] 100069 D - 0xfffff80003b9da00 [linuxkpi_long_wq_5] 100070 D - 0xfffff80003b9da00 [linuxkpi_long_wq_6] 100071 D - 0xfffff80003b9da00 [linuxkpi_long_wq_7] 100072 D - 0xfffff80003b9da00 [linuxkpi_long_wq_8] 100073 D - 0xfffff80003b9da00 [linuxkpi_long_wq_9] 100074 D - 0xfffff80003b9da00 [linuxkpi_long_wq_10] 100075 D - 0xfffff80003b9da00 [linuxkpi_long_wq_11] 100076 D - 0xfffff80003b9da00 [linuxkpi_long_wq_12] 100091 D - 0xfffff80003b9d800 [firmware taskq] 100096 D - 0xfffff80003b9ec00 [crypto_0] 100097 D - 0xfffff80003b9ec00 [crypto_1] 100098 D - 0xfffff80003b9ec00 [crypto_2] 100099 D - 0xfffff80003b9ec00 [crypto_3] 100100 D - 0xfffff80003b9ec00 [crypto_4] 100101 D - 0xfffff80003b9ec00 [crypto_5] 100102 D - 0xfffff80003b9ec00 [crypto_6] 100103 D - 0xfffff80003b9ec00 [crypto_7] 100104 D - 0xfffff80003b9ec00 [crypto_8] 100105 D - 0xfffff80003b9ec00 [crypto_9] 100106 D - 0xfffff80003b9ec00 [crypto_10] 100107 D - 0xfffff80003b9ec00 [crypto_11] 100147 D - 0xfffff80003f35100 [acpi_task_0] 100148 D - 0xfffff80003f35100 [acpi_task_1] 100149 D - 0xfffff80003f35100 [acpi_task_2] 100151 D - 0xfffff800039ede00 [mca taskq] 100152 D - 0xfffff80003b9e500 [CAM taskq] 100156 D - 0xfffff80003f36400 [mlx5_core0-rec] 100157 D - 0xfffff80003f36300 [mlx5_core0-wdg] 100158 D - 0xfffff80003f36200 [mlx5_core0-cmd] 100159 D - 0xfffff80005873300 [mlx5_page_allocator] 100175 D - 0xfffff80005873100 [mlx5_fc] 100176 D - 0xfffff80005873000 [mlx5_esw_wq] 100177 D - 0xfffff80005874e00 [mlx5_core1-rec] 100178 D - 0xfffff80005874d00 [mlx5_core1-wdg] 100179 D - 0xfffff80005874c00 [mlx5_core1-cmd] 100180 D - 0xfffff80005874b00 [mlx5_page_allocator] 100196 D - 0xfffff80005874900 [mlx5_fc] 100197 D - 0xfffff80005874800 [mlx5_esw_wq] 100388 D - 0xfffff80003f38a00 [unionfs_rele taskq] 92553 92545 3706 0 Z+ mmap 92559 92545 3706 0 Z+ mmap db:0:pho> allt Tracing command kernel pid 0 tid 100000 td 0xffffffff81b97680 sched_switch() at sched_switch+0x5c6/frame 0xffffffff8237de10 mi_switch() at mi_switch+0x170/frame 0xffffffff8237de30 sleepq_switch() at sleepq_switch+0x101/frame 0xffffffff8237de70 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xffffffff8237deb0 _sleep() at _sleep+0x27e/frame 0xffffffff8237df50 swapper() at swapper+0x75/frame 0xffffffff8237dfa0 mi_startup() at mi_startup+0x226/frame 0xffffffff8237dff0 Tracing command kernel pid 0 tid 100015 td 0xfffff80003be2740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108050df0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108050e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108050e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe0108050ec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe0108050ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108050f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108050f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100016 td 0xfffff80003be2000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010804bdf0 mi_switch() at mi_switch+0x170/frame 0xfffffe010804be10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010804be50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe010804bec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe010804bef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010804bf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010804bf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100017 td 0xfffff80003bb5740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010805adf0 mi_switch() at mi_switch+0x170/frame 0xfffffe010805ae10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010805ae50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe010805aec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe010805aef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010805af30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010805af30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100018 td 0xfffff80003bb5000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108055df0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108055e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108055e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe0108055ec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe0108055ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108055f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108055f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100019 td 0xfffff80003bb6740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108064df0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108064e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108064e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe0108064ec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe0108064ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108064f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108064f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100020 td 0xfffff80003bb6000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010805fdf0 mi_switch() at mi_switch+0x170/frame 0xfffffe010805fe10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010805fe50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe010805fec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe010805fef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010805ff30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010805ff30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100021 td 0xfffff80003bb7740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010806edf0 mi_switch() at mi_switch+0x170/frame 0xfffffe010806ee10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010806ee50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe010806eec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe010806eef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010806ef30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010806ef30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100022 td 0xfffff80003bb7000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108069df0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108069e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108069e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe0108069ec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe0108069ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108069f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108069f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100023 td 0xfffff80003bb8740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108078df0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108078e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108078e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe0108078ec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe0108078ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108078f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108078f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100024 td 0xfffff80003bb8000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108073df0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108073e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108073e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe0108073ec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe0108073ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108073f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108073f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100025 td 0xfffff80003bb1740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108082df0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108082e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108082e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe0108082ec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe0108082ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108082f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108082f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100026 td 0xfffff80003bb1000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010807ddf0 mi_switch() at mi_switch+0x170/frame 0xfffffe010807de10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010807de50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe010807dec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe010807def0 fork_exit() at fork_exit+0x82/frame 0xfffffe010807df30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010807df30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100027 td 0xfffff80003bb2740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010808cdf0 mi_switch() at mi_switch+0x170/frame 0xfffffe010808ce10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010808ce50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe010808cec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe010808cef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010808cf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010808cf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100028 td 0xfffff80003bb2000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108087df0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108087e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108087e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe0108087ec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe0108087ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108087f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108087f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100029 td 0xfffff80003bb3740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108096df0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108096e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108096e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe0108096ec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe0108096ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108096f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108096f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100030 td 0xfffff80003bb3000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108091df0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108091e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108091e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe0108091ec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe0108091ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108091f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108091f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100031 td 0xfffff80003bb4740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080a0df0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080a0e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080a0e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe01080a0ec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe01080a0ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080a0f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080a0f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100032 td 0xfffff80003bb4000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010809bdf0 mi_switch() at mi_switch+0x170/frame 0xfffffe010809be10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010809be50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe010809bec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe010809bef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010809bf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010809bf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100033 td 0xfffff80003bab740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080aadf0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080aae10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080aae50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe01080aaec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe01080aaef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080aaf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080aaf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100034 td 0xfffff80003bab000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080a5df0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080a5e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080a5e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe01080a5ec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe01080a5ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080a5f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080a5f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100035 td 0xfffff80003bac740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080b4df0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080b4e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080b4e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe01080b4ec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe01080b4ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080b4f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080b4f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100036 td 0xfffff80003bac000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080afdf0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080afe10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080afe50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe01080afec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe01080afef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080aff30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080aff30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100037 td 0xfffff80003bad740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080bedf0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080bee10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080bee50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe01080beec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe01080beef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080bef30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080bef30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100038 td 0xfffff80003bad000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080b9df0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080b9e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080b9e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe01080b9ec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe01080b9ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080b9f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080b9f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100039 td 0xfffff80003bae740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080c8df0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080c8e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080c8e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe01080c8ec0 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0x104/frame 0xfffffe01080c8ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080c8f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080c8f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100040 td 0xfffff80003bae000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080c3dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080c3de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080c3e20 _sleep() at _sleep+0x2d8/frame 0xfffffe01080c3ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01080c3ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080c3f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080c3f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100041 td 0xfffff80003ba7740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080d2dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080d2de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080d2e20 _sleep() at _sleep+0x2d8/frame 0xfffffe01080d2ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01080d2ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080d2f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080d2f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100044 td 0xfffff80003ba8000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080d7dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080d7de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080d7e20 _sleep() at _sleep+0x2d8/frame 0xfffffe01080d7ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01080d7ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080d7f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080d7f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100046 td 0xfffff80003ba9000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080e1dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080e1de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080e1e20 _sleep() at _sleep+0x2d8/frame 0xfffffe01080e1ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01080e1ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080e1f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080e1f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100047 td 0xfffff80003baa740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080f0dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080f0de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080f0e20 _sleep() at _sleep+0x2d8/frame 0xfffffe01080f0ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01080f0ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080f0f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080f0f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100048 td 0xfffff80003baa000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080ebdc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080ebde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080ebe20 _sleep() at _sleep+0x2d8/frame 0xfffffe01080ebec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01080ebef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080ebf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080ebf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100049 td 0xfffff80003b98740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080fadc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080fade0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080fae20 _sleep() at _sleep+0x2d8/frame 0xfffffe01080faec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01080faef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080faf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080faf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100050 td 0xfffff80003b98000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080f5df0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080f5e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080f5e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe01080f5ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0x104/frame 0xfffffe01080f5ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080f5f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080f5f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100051 td 0xfffff80003b99740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108104dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108104de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108104e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108104ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108104ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108104f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108104f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100052 td 0xfffff80003b99000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080ffdc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01080ffde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01080ffe20 _sleep() at _sleep+0x2d8/frame 0xfffffe01080ffec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01080ffef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080fff30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080fff30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100053 td 0xfffff80003b9a740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010810edc0 mi_switch() at mi_switch+0x170/frame 0xfffffe010810ede0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010810ee20 _sleep() at _sleep+0x2d8/frame 0xfffffe010810eec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe010810eef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010810ef30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010810ef30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100054 td 0xfffff80003b9a000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108109dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108109de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108109e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108109ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108109ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108109f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108109f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100055 td 0xfffff80003b9b740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108118dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108118de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108118e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108118ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108118ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108118f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108118f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100056 td 0xfffff80003b9b000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108113dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108113de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108113e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108113ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108113ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108113f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108113f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100057 td 0xfffff80003b91740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108122dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108122de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108122e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108122ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108122ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108122f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108122f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100058 td 0xfffff80003b91000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010811ddc0 mi_switch() at mi_switch+0x170/frame 0xfffffe010811dde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010811de20 _sleep() at _sleep+0x2d8/frame 0xfffffe010811dec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe010811def0 fork_exit() at fork_exit+0x82/frame 0xfffffe010811df30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010811df30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100059 td 0xfffff80003b95740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010812cdc0 mi_switch() at mi_switch+0x170/frame 0xfffffe010812cde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010812ce20 _sleep() at _sleep+0x2d8/frame 0xfffffe010812cec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe010812cef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010812cf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010812cf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100060 td 0xfffff80003b95000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108127dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108127de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108127e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108127ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108127ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108127f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108127f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100061 td 0xfffff80003b96740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108136dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108136de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108136e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108136ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108136ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108136f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108136f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100062 td 0xfffff80003b96000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108131dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108131de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108131e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108131ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108131ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108131f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108131f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100063 td 0xfffff80003b97740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108140dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108140de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108140e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108140ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108140ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108140f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108140f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100064 td 0xfffff80003b97000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010813bdc0 mi_switch() at mi_switch+0x170/frame 0xfffffe010813bde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010813be20 _sleep() at _sleep+0x2d8/frame 0xfffffe010813bec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe010813bef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010813bf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010813bf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100065 td 0xfffff80003b89740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010814adc0 mi_switch() at mi_switch+0x170/frame 0xfffffe010814ade0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010814ae20 _sleep() at _sleep+0x2d8/frame 0xfffffe010814aec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe010814aef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010814af30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010814af30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100066 td 0xfffff80003b89000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108145dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108145de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108145e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108145ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108145ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108145f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108145f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100067 td 0xfffff80003b8a740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108154dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108154de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108154e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108154ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108154ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108154f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108154f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100068 td 0xfffff80003b8a000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010814fdc0 mi_switch() at mi_switch+0x170/frame 0xfffffe010814fde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010814fe20 _sleep() at _sleep+0x2d8/frame 0xfffffe010814fec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe010814fef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010814ff30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010814ff30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100069 td 0xfffff80003b8b740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010815edc0 mi_switch() at mi_switch+0x170/frame 0xfffffe010815ede0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010815ee20 _sleep() at _sleep+0x2d8/frame 0xfffffe010815eec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe010815eef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010815ef30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010815ef30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100070 td 0xfffff80003b8b000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108159dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108159de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108159e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108159ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108159ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108159f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108159f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100071 td 0xfffff80003b8c740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108168dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108168de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108168e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108168ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108168ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108168f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108168f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100072 td 0xfffff80003b8c000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108163dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108163de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108163e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108163ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108163ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108163f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108163f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100073 td 0xfffff80003b85740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108172dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108172de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108172e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108172ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108172ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108172f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108172f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100074 td 0xfffff80003b85000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010816ddc0 mi_switch() at mi_switch+0x170/frame 0xfffffe010816dde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010816de20 _sleep() at _sleep+0x2d8/frame 0xfffffe010816dec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe010816def0 fork_exit() at fork_exit+0x82/frame 0xfffffe010816df30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010816df30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100075 td 0xfffff80003b86740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010817cdc0 mi_switch() at mi_switch+0x170/frame 0xfffffe010817cde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010817ce20 _sleep() at _sleep+0x2d8/frame 0xfffffe010817cec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe010817cef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010817cf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010817cf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100076 td 0xfffff80003b86000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108177dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108177de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108177e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108177ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108177ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108177f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108177f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100091 td 0xfffff80003c93740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01081ccdc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01081ccde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01081cce20 _sleep() at _sleep+0x2d8/frame 0xfffffe01081ccec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01081ccef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01081ccf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01081ccf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100096 td 0xfffff80003c95000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01081dbdc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01081dbde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01081dbe20 _sleep() at _sleep+0x2d8/frame 0xfffffe01081dbec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01081dbef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01081dbf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01081dbf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100097 td 0xfffff800051d1740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01081eadc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01081eade0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01081eae20 _sleep() at _sleep+0x2d8/frame 0xfffffe01081eaec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01081eaef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01081eaf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01081eaf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100098 td 0xfffff800051d1000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01081e5dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01081e5de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01081e5e20 _sleep() at _sleep+0x2d8/frame 0xfffffe01081e5ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01081e5ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01081e5f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01081e5f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100099 td 0xfffff800051d2740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01081f4dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01081f4de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01081f4e20 _sleep() at _sleep+0x2d8/frame 0xfffffe01081f4ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01081f4ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01081f4f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01081f4f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100100 td 0xfffff800051d2000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01081efdc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01081efde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01081efe20 _sleep() at _sleep+0x2d8/frame 0xfffffe01081efec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01081efef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01081eff30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01081eff30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100101 td 0xfffff800051d3740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01081fedc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01081fede0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01081fee20 _sleep() at _sleep+0x2d8/frame 0xfffffe01081feec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01081feef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01081fef30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01081fef30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100102 td 0xfffff800051d3000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01081f9dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01081f9de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01081f9e20 _sleep() at _sleep+0x2d8/frame 0xfffffe01081f9ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01081f9ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01081f9f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01081f9f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100103 td 0xfffff800051d4740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108208dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108208de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108208e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108208ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108208ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108208f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108208f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100104 td 0xfffff800051d4000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108203dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108203de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108203e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108203ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108203ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108203f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108203f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100105 td 0xfffff800051cd740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108212dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108212de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108212e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108212ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108212ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108212f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108212f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100106 td 0xfffff800051cd000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010820ddc0 mi_switch() at mi_switch+0x170/frame 0xfffffe010820dde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010820de20 _sleep() at _sleep+0x2d8/frame 0xfffffe010820dec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe010820def0 fork_exit() at fork_exit+0x82/frame 0xfffffe010820df30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010820df30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100107 td 0xfffff800051ce740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010821cdc0 mi_switch() at mi_switch+0x170/frame 0xfffffe010821cde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010821ce20 _sleep() at _sleep+0x2d8/frame 0xfffffe010821cec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe010821cef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010821cf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010821cf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100147 td 0xfffff80009439740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082e4df0 mi_switch() at mi_switch+0x170/frame 0xfffffe01082e4e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082e4e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe01082e4ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0x104/frame 0xfffffe01082e4ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082e4f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082e4f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100148 td 0xfffff80009439000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082dfdf0 mi_switch() at mi_switch+0x170/frame 0xfffffe01082dfe10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082dfe50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe01082dfec0 taskqueue_thread_loop() at taskqueue_thread_loop+0x104/frame 0xfffffe01082dfef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082dff30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082dff30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100149 td 0xfffff8000943a740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082eedf0 mi_switch() at mi_switch+0x170/frame 0xfffffe01082eee10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082eee50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe01082eeec0 taskqueue_thread_loop() at taskqueue_thread_loop+0x104/frame 0xfffffe01082eeef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082eef30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082eef30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100151 td 0xfffff8000943b740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082f8df0 mi_switch() at mi_switch+0x170/frame 0xfffffe01082f8e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082f8e50 msleep_spin_sbt() at msleep_spin_sbt+0x154/frame 0xfffffe01082f8ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0x104/frame 0xfffffe01082f8ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082f8f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082f8f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100152 td 0xfffff8000943b000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082f3dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01082f3de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082f3e20 _sleep() at _sleep+0x2d8/frame 0xfffffe01082f3ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01082f3ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082f3f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082f3f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100156 td 0xfffff80009443000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082fddc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01082fdde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082fde20 _sleep() at _sleep+0x2d8/frame 0xfffffe01082fdec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01082fdef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082fdf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082fdf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100157 td 0xfffff800093f3740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108316dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108316de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108316e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108316ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108316ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108316f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108316f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100158 td 0xfffff800093f3000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108311dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108311de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108311e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108311ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108311ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108311f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108311f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100159 td 0xfffff80005868740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108320dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108320de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108320e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108320ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108320ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108320f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108320f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100175 td 0xfffff8000eb3d740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108370dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108370de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108370e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108370ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108370ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108370f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108370f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100176 td 0xfffff8000eb3d000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010836bdc0 mi_switch() at mi_switch+0x170/frame 0xfffffe010836bde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010836be20 _sleep() at _sleep+0x2d8/frame 0xfffffe010836bec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe010836bef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010836bf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010836bf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100177 td 0xfffff8000eb3e740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010837adc0 mi_switch() at mi_switch+0x170/frame 0xfffffe010837ade0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010837ae20 _sleep() at _sleep+0x2d8/frame 0xfffffe010837aec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe010837aef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010837af30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010837af30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100178 td 0xfffff8000eb3e000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108375dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108375de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108375e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108375ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108375ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108375f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108375f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100179 td 0xfffff8000eb3f740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108384dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108384de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108384e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108384ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108384ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108384f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108384f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100180 td 0xfffff8000eb3f000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010837fdc0 mi_switch() at mi_switch+0x170/frame 0xfffffe010837fde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010837fe20 _sleep() at _sleep+0x2d8/frame 0xfffffe010837fec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe010837fef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010837ff30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010837ff30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100196 td 0xfffff80015257000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01083cfdc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01083cfde0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01083cfe20 _sleep() at _sleep+0x2d8/frame 0xfffffe01083cfec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01083cfef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01083cff30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01083cff30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100197 td 0xfffff80015258740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01083dedc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01083dede0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01083dee20 _sleep() at _sleep+0x2d8/frame 0xfffffe01083deec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe01083deef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01083def30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01083def30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100388 td 0xfffff802f72a0000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108677dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108677de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108677e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108677ec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc1/frame 0xfffffe0108677ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108677f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108677f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command init pid 1 tid 100002 td 0xfffff80003bf3000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108005a10 mi_switch() at mi_switch+0x170/frame 0xfffffe0108005a30 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108005a70 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108005ad0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe0108005ae0 _sleep() at _sleep+0x2ce/frame 0xfffffe0108005b80 kern_wait6() at kern_wait6+0x471/frame 0xfffffe0108005c10 sys_wait4() at sys_wait4+0x6b/frame 0xfffffe0108005e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108005f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108005f30 --- syscall (7, FreeBSD ELF64, wait4), rip = 0x2dd08a, rsp = 0x820777748, rbp = 0x8207777b0 --- Tracing command clock pid 2 tid 100078 td 0xfffff80003b87000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108181ea0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108181ec0 softclock_thread() at softclock_thread+0x6e/frame 0xfffffe0108181ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108181f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108181f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command clock pid 2 tid 100079 td 0xfffff80003b88740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108190ea0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108190ec0 softclock_thread() at softclock_thread+0x6e/frame 0xfffffe0108190ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108190f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108190f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command clock pid 2 tid 100080 td 0xfffff80003b88000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010818bea0 mi_switch() at mi_switch+0x170/frame 0xfffffe010818bec0 softclock_thread() at softclock_thread+0x6e/frame 0xfffffe010818bef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010818bf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010818bf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command clock pid 2 tid 100081 td 0xfffff80003c96740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010819aea0 mi_switch() at mi_switch+0x170/frame 0xfffffe010819aec0 softclock_thread() at softclock_thread+0x6e/frame 0xfffffe010819aef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010819af30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010819af30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command clock pid 2 tid 100082 td 0xfffff80003c96000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108195ea0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108195ec0 softclock_thread() at softclock_thread+0x6e/frame 0xfffffe0108195ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108195f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108195f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command clock pid 2 tid 100083 td 0xfffff80003c97740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01081a4ea0 mi_switch() at mi_switch+0x170/frame 0xfffffe01081a4ec0 softclock_thread() at softclock_thread+0x6e/frame 0xfffffe01081a4ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01081a4f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01081a4f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command clock pid 2 tid 100084 td 0xfffff80003c97000 fork_trampoline() at fork_trampoline Tracing command clock pid 2 tid 100085 td 0xfffff80003c98740 fork_trampoline() at fork_trampoline Tracing command clock pid 2 tid 100086 td 0xfffff80003c98000 fork_trampoline() at fork_trampoline Tracing command clock pid 2 tid 100087 td 0xfffff80003c99740 fork_trampoline() at fork_trampoline Tracing command clock pid 2 tid 100088 td 0xfffff80003c99000 fork_trampoline() at fork_trampoline Tracing command clock pid 2 tid 100089 td 0xfffff80003c92740 fork_trampoline() at fork_trampoline Tracing command crypto pid 3 tid 100108 td 0xfffff800051ce000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108217dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108217de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108217e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108217ec0 crypto_dispatch_thread() at crypto_dispatch_thread+0x193/frame 0xfffffe0108217ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108217f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108217f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto pid 3 tid 100109 td 0xfffff800051cf740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108226db0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108226dd0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108226e10 _sleep() at _sleep+0x2d8/frame 0xfffffe0108226eb0 crypto_ret_thread() at crypto_ret_thread+0x144/frame 0xfffffe0108226ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108226f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108226f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto pid 3 tid 100110 td 0xfffff800051cf000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108221db0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108221dd0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108221e10 _sleep() at _sleep+0x2d8/frame 0xfffffe0108221eb0 crypto_ret_thread() at crypto_ret_thread+0x144/frame 0xfffffe0108221ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108221f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108221f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto pid 3 tid 100111 td 0xfffff800051d0740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108230db0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108230dd0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108230e10 _sleep() at _sleep+0x2d8/frame 0xfffffe0108230eb0 crypto_ret_thread() at crypto_ret_thread+0x144/frame 0xfffffe0108230ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108230f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108230f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto pid 3 tid 100112 td 0xfffff800051d0000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010822bdb0 mi_switch() at mi_switch+0x170/frame 0xfffffe010822bdd0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010822be10 _sleep() at _sleep+0x2d8/frame 0xfffffe010822beb0 crypto_ret_thread() at crypto_ret_thread+0x144/frame 0xfffffe010822bef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010822bf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010822bf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto pid 3 tid 100113 td 0xfffff800051c7740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010823adb0 mi_switch() at mi_switch+0x170/frame 0xfffffe010823add0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010823ae10 _sleep() at _sleep+0x2d8/frame 0xfffffe010823aeb0 crypto_ret_thread() at crypto_ret_thread+0x144/frame 0xfffffe010823aef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010823af30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010823af30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto pid 3 tid 100114 td 0xfffff800051c7000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108235db0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108235dd0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108235e10 _sleep() at _sleep+0x2d8/frame 0xfffffe0108235eb0 crypto_ret_thread() at crypto_ret_thread+0x144/frame 0xfffffe0108235ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108235f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108235f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto pid 3 tid 100115 td 0xfffff800051c8740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108244db0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108244dd0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108244e10 _sleep() at _sleep+0x2d8/frame 0xfffffe0108244eb0 crypto_ret_thread() at crypto_ret_thread+0x144/frame 0xfffffe0108244ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108244f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108244f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto pid 3 tid 100116 td 0xfffff800051c8000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010823fdb0 mi_switch() at mi_switch+0x170/frame 0xfffffe010823fdd0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010823fe10 _sleep() at _sleep+0x2d8/frame 0xfffffe010823feb0 crypto_ret_thread() at crypto_ret_thread+0x144/frame 0xfffffe010823fef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010823ff30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010823ff30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto pid 3 tid 100117 td 0xfffff800051c9740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010824edb0 mi_switch() at mi_switch+0x170/frame 0xfffffe010824edd0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010824ee10 _sleep() at _sleep+0x2d8/frame 0xfffffe010824eeb0 crypto_ret_thread() at crypto_ret_thread+0x144/frame 0xfffffe010824eef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010824ef30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010824ef30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto pid 3 tid 100118 td 0xfffff800051c9000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108249db0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108249dd0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108249e10 _sleep() at _sleep+0x2d8/frame 0xfffffe0108249eb0 crypto_ret_thread() at crypto_ret_thread+0x144/frame 0xfffffe0108249ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108249f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108249f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto pid 3 tid 100119 td 0xfffff800051ca740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108258db0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108258dd0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108258e10 _sleep() at _sleep+0x2d8/frame 0xfffffe0108258eb0 crypto_ret_thread() at crypto_ret_thread+0x144/frame 0xfffffe0108258ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108258f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108258f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto pid 3 tid 100120 td 0xfffff800051ca000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108253db0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108253dd0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108253e10 _sleep() at _sleep+0x2d8/frame 0xfffffe0108253eb0 crypto_ret_thread() at crypto_ret_thread+0x144/frame 0xfffffe0108253ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108253f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108253f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command cam pid 4 tid 100121 td 0xfffff800052a8740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108262da0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108262dc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108262e00 _sleep() at _sleep+0x2d8/frame 0xfffffe0108262ea0 xpt_done_td() at xpt_done_td+0x7b/frame 0xfffffe0108262ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108262f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108262f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command cam pid 4 tid 100122 td 0xfffff800052a8000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010825dda0 mi_switch() at mi_switch+0x170/frame 0xfffffe010825ddc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010825de00 _sleep() at _sleep+0x2d8/frame 0xfffffe010825dea0 xpt_done_td() at xpt_done_td+0x7b/frame 0xfffffe010825def0 fork_exit() at fork_exit+0x82/frame 0xfffffe010825df30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010825df30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command cam pid 4 tid 100123 td 0xfffff800052ad740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010826cda0 mi_switch() at mi_switch+0x170/frame 0xfffffe010826cdc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010826ce00 _sleep() at _sleep+0x2d8/frame 0xfffffe010826cea0 xpt_done_td() at xpt_done_td+0x7b/frame 0xfffffe010826cef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010826cf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010826cf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command cam pid 4 tid 100124 td 0xfffff800052ad000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108267da0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108267dc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108267e00 _sleep() at _sleep+0x2d8/frame 0xfffffe0108267ea0 xpt_async_td() at xpt_async_td+0x74/frame 0xfffffe0108267ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108267f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108267f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command cam pid 4 tid 100153 td 0xfffff80009443740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108302dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108302de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108302e20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108302ec0 xpt_scanner_thread() at xpt_scanner_thread+0x138/frame 0xfffffe0108302ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108302f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108302f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command busdma pid 5 tid 100126 td 0xfffff8000577b740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108276da0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108276dc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108276e00 _sleep() at _sleep+0x2d8/frame 0xfffffe0108276ea0 busdma_thread() at busdma_thread+0xc5/frame 0xfffffe0108276ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108276f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108276f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command rand_harvestq pid 6 tid 100150 td 0xfffff8000943a000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082e9d20 mi_switch() at mi_switch+0x170/frame 0xfffffe01082e9d40 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082e9d80 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe01082e9dc0 _sleep() at _sleep+0x27e/frame 0xfffffe01082e9e60 random_kthread() at random_kthread+0xb7/frame 0xfffffe01082e9ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082e9f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082e9f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command enc_daemon0 pid 7 tid 100154 td 0xfffff80005975000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082add80 mi_switch() at mi_switch+0x170/frame 0xfffffe01082adda0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082adde0 _sleep() at _sleep+0x2d8/frame 0xfffffe01082ade80 enc_daemon() at enc_daemon+0xdd/frame 0xfffffe01082adef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082adf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082adf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command enc_daemon1 pid 8 tid 100155 td 0xfffff80005976740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010830cd80 mi_switch() at mi_switch+0x170/frame 0xfffffe010830cda0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010830cde0 _sleep() at _sleep+0x2d8/frame 0xfffffe010830ce80 enc_daemon() at enc_daemon+0xdd/frame 0xfffffe010830cef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010830cf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010830cf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command pagedaemon pid 9 tid 100198 td 0xfffff80015258000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01083d99b0 mi_switch() at mi_switch+0x170/frame 0xfffffe01083d99d0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01083d9a10 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe01083d9a50 _sleep() at _sleep+0x27e/frame 0xfffffe01083d9af0 vm_pageout_worker() at vm_pageout_worker+0x28c/frame 0xfffffe01083d9eb0 vm_pageout() at vm_pageout+0x1d7/frame 0xfffffe01083d9ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01083d9f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01083d9f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command pagedaemon pid 9 tid 100200 td 0xfffff80003fe0740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01083f2a00 mi_switch() at mi_switch+0x170/frame 0xfffffe01083f2a20 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01083f2a60 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe01083f2aa0 _sleep() at _sleep+0x27e/frame 0xfffffe01083f2b40 pause_sbt() at pause_sbt+0x7d/frame 0xfffffe01083f2b70 vm_pageout_laundry_worker() at vm_pageout_laundry_worker+0xa00/frame 0xfffffe01083f2ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01083f2f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01083f2f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command pagedaemon pid 9 tid 100202 td 0xfffff80003fe0000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01083e8d80 mi_switch() at mi_switch+0x170/frame 0xfffffe01083e8da0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01083e8de0 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe01083e8e20 _sleep() at _sleep+0x27e/frame 0xfffffe01083e8ec0 uma_reclaim_worker() at uma_reclaim_worker+0xc8/frame 0xfffffe01083e8ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01083e8f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01083e8f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command audit pid 10 tid 100001 td 0xfffff80003bf3740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010800adc0 mi_switch() at mi_switch+0x170/frame 0xfffffe010800ade0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010800ae20 _cv_wait() at _cv_wait+0x160/frame 0xfffffe010800ae90 audit_worker() at audit_worker+0x8f/frame 0xfffffe010800aef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010800af30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010800af30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100003 td 0xfffff80003bf4740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108014dd0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108014df0 sched_idletd() at sched_idletd+0x57c/frame 0xfffffe0108014ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108014f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108014f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100004 td 0xfffff80003bf4000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010800fdd0 mi_switch() at mi_switch+0x170/frame 0xfffffe010800fdf0 sched_idletd() at sched_idletd+0x57c/frame 0xfffffe010800fef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010800ff30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010800ff30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100005 td 0xfffff80003bf5740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010801edd0 mi_switch() at mi_switch+0x170/frame 0xfffffe010801edf0 sched_idletd() at sched_idletd+0x90/frame 0xfffffe010801eef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010801ef30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010801ef30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100006 td 0xfffff80003bf5000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108019dd0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108019df0 sched_idletd() at sched_idletd+0x90/frame 0xfffffe0108019ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108019f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108019f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100007 td 0xfffff80003bf6740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108028dd0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108028df0 sched_idletd() at sched_idletd+0x57c/frame 0xfffffe0108028ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108028f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108028f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100008 td 0xfffff80003bf6000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108023dd0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108023df0 sched_idletd() at sched_idletd+0x90/frame 0xfffffe0108023ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108023f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108023f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100009 td 0xfffff80003bdf740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108032dd0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108032df0 sched_idletd() at sched_idletd+0x90/frame 0xfffffe0108032ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108032f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108032f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100010 td 0xfffff80003bdf000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010802ddd0 mi_switch() at mi_switch+0x170/frame 0xfffffe010802ddf0 sched_idletd() at sched_idletd+0x534/frame 0xfffffe010802def0 fork_exit() at fork_exit+0x82/frame 0xfffffe010802df30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010802df30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100011 td 0xfffff80003be0740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010803cdd0 mi_switch() at mi_switch+0x170/frame 0xfffffe010803cdf0 sched_idletd() at sched_idletd+0x57c/frame 0xfffffe010803cef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010803cf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010803cf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100012 td 0xfffff80003be0000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108037dd0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108037df0 sched_idletd() at sched_idletd+0x57c/frame 0xfffffe0108037ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108037f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108037f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100013 td 0xfffff80003be1740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108046dd0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108046df0 sched_idletd() at sched_idletd+0x90/frame 0xfffffe0108046ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108046f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108046f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100014 td 0xfffff80003be1000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108041dd0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108041df0 sched_idletd() at sched_idletd+0x90/frame 0xfffffe0108041ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108041f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108041f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100042 td 0xfffff80003ba7000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01080cde40 mi_switch() at mi_switch+0x170/frame 0xfffffe01080cde60 ithread_loop() at ithread_loop+0x99/frame 0xfffffe01080cdef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01080cdf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01080cdf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100043 td 0xfffff80003ba8740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100045 td 0xfffff80003ba9740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100090 td 0xfffff80003c92000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01081bde40 mi_switch() at mi_switch+0x170/frame 0xfffffe01081bde60 ithread_loop() at ithread_loop+0x99/frame 0xfffffe01081bdef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01081bdf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01081bdf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100125 td 0xfffff80003b87740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108186e40 mi_switch() at mi_switch+0x170/frame 0xfffffe0108186e60 ithread_loop() at ithread_loop+0x99/frame 0xfffffe0108186ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108186f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108186f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100127 td 0xfffff8000577b000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108271e40 mi_switch() at mi_switch+0x170/frame 0xfffffe0108271e60 ithread_loop() at ithread_loop+0x99/frame 0xfffffe0108271ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108271f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108271f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100133 td 0xfffff80005789000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010828fe40 mi_switch() at mi_switch+0x170/frame 0xfffffe010828fe60 ithread_loop() at ithread_loop+0x99/frame 0xfffffe010828fef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010828ff30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010828ff30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100144 td 0xfffff80003d1a000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100145 td 0xfffff80003d22740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100146 td 0xfffff80003d22000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082d5e40 mi_switch() at mi_switch+0x170/frame 0xfffffe01082d5e60 ithread_loop() at ithread_loop+0x99/frame 0xfffffe01082d5ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082d5f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082d5f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100160 td 0xfffff80005868000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010831be40 mi_switch() at mi_switch+0x170/frame 0xfffffe010831be60 ithread_loop() at ithread_loop+0x99/frame 0xfffffe010831bef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010831bf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010831bf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100161 td 0xfffff80005869740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010832ae40 mi_switch() at mi_switch+0x170/frame 0xfffffe010832ae60 ithread_loop() at ithread_loop+0x99/frame 0xfffffe010832aef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010832af30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010832af30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100162 td 0xfffff80005869000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100163 td 0xfffff8000586a740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100164 td 0xfffff8000586a000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100165 td 0xfffff80005872740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100166 td 0xfffff80005872000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100167 td 0xfffff8000eb41740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100168 td 0xfffff8000eb41000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100169 td 0xfffff8000eb42740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100170 td 0xfffff8000eb42000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100171 td 0xfffff8000eb43740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100172 td 0xfffff8000eb43000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100173 td 0xfffff8000eb44740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100174 td 0xfffff8000eb44000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100181 td 0xfffff8000eb40740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010838ee40 mi_switch() at mi_switch+0x170/frame 0xfffffe010838ee60 ithread_loop() at ithread_loop+0x99/frame 0xfffffe010838eef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010838ef30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010838ef30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100182 td 0xfffff8000eb40000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108389e40 mi_switch() at mi_switch+0x170/frame 0xfffffe0108389e60 ithread_loop() at ithread_loop+0x99/frame 0xfffffe0108389ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108389f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108389f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100183 td 0xfffff80015259740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100184 td 0xfffff80015259000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100185 td 0xfffff8001525a740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100186 td 0xfffff8001525a000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100187 td 0xfffff8001525b740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100188 td 0xfffff8001525b000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100189 td 0xfffff8001525c740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100190 td 0xfffff8001525c000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100191 td 0xfffff80015255740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100192 td 0xfffff80015255000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100193 td 0xfffff80015256740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100194 td 0xfffff80015256000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100195 td 0xfffff80015257740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100244 td 0xfffff800052b6000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100245 td 0xfffff8002135e740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100246 td 0xfffff8002135e000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100247 td 0xfffff80021370740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100248 td 0xfffff80021370000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100249 td 0xfffff80021360740 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100250 td 0xfffff80021360000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100251 td 0xfffff80021357740 fork_trampoline() at fork_trampoline Tracing command geom pid 13 tid 100092 td 0xfffff80003c93000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01081c7dc0 mi_switch() at mi_switch+0x170/frame 0xfffffe01081c7de0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01081c7e20 _sleep() at _sleep+0x2d8/frame 0xfffffe01081c7ec0 g_run_events() at g_run_events+0x434/frame 0xfffffe01081c7ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01081c7f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01081c7f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command geom pid 13 tid 100093 td 0xfffff80003c94740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01081d6db0 mi_switch() at mi_switch+0x170/frame 0xfffffe01081d6dd0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01081d6e10 _sleep() at _sleep+0x2d8/frame 0xfffffe01081d6eb0 g_io_schedule_up() at g_io_schedule_up+0x3c/frame 0xfffffe01081d6ee0 g_up_procbody() at g_up_procbody+0x5c/frame 0xfffffe01081d6ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01081d6f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01081d6f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command geom pid 13 tid 100094 td 0xfffff80003c94000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01081d1db0 mi_switch() at mi_switch+0x170/frame 0xfffffe01081d1dd0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01081d1e10 _sleep() at _sleep+0x2d8/frame 0xfffffe01081d1eb0 g_io_schedule_down() at g_io_schedule_down+0x3c/frame 0xfffffe01081d1ee0 g_down_procbody() at g_down_procbody+0x5c/frame 0xfffffe01081d1ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01081d1f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01081d1f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command sequencer 00 pid 14 tid 100095 td 0xfffff80003c95740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01081e0da0 mi_switch() at mi_switch+0x170/frame 0xfffffe01081e0dc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01081e0e00 _cv_wait() at _cv_wait+0x160/frame 0xfffffe01081e0e70 seq_eventthread() at seq_eventthread+0x8c/frame 0xfffffe01081e0ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01081e0f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01081e0f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 15 tid 100128 td 0xfffff8000577c740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108280df0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108280e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108280e50 _cv_wait() at _cv_wait+0x160/frame 0xfffffe0108280ec0 usb_process() at usb_process+0xb0/frame 0xfffffe0108280ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108280f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108280f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 15 tid 100129 td 0xfffff8000577c000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010827bdf0 mi_switch() at mi_switch+0x170/frame 0xfffffe010827be10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010827be50 _cv_wait() at _cv_wait+0x160/frame 0xfffffe010827bec0 usb_process() at usb_process+0xb0/frame 0xfffffe010827bef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010827bf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010827bf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 15 tid 100130 td 0xfffff8000577d740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010828adf0 mi_switch() at mi_switch+0x170/frame 0xfffffe010828ae10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010828ae50 _cv_wait() at _cv_wait+0x160/frame 0xfffffe010828aec0 usb_process() at usb_process+0xb0/frame 0xfffffe010828aef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010828af30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010828af30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 15 tid 100131 td 0xfffff8000577d000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108285df0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108285e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108285e50 _cv_wait() at _cv_wait+0x160/frame 0xfffffe0108285ec0 usb_process() at usb_process+0xb0/frame 0xfffffe0108285ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108285f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108285f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 15 tid 100132 td 0xfffff80005789740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108294df0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108294e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108294e50 _cv_wait() at _cv_wait+0x160/frame 0xfffffe0108294ec0 usb_process() at usb_process+0xb0/frame 0xfffffe0108294ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108294f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108294f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 15 tid 100134 td 0xfffff80005973740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010829edf0 mi_switch() at mi_switch+0x170/frame 0xfffffe010829ee10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010829ee50 _cv_wait() at _cv_wait+0x160/frame 0xfffffe010829eec0 usb_process() at usb_process+0xb0/frame 0xfffffe010829eef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010829ef30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010829ef30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 15 tid 100135 td 0xfffff80005973000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108299df0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108299e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108299e50 _cv_wait() at _cv_wait+0x160/frame 0xfffffe0108299ec0 usb_process() at usb_process+0xb0/frame 0xfffffe0108299ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108299f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108299f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 15 tid 100136 td 0xfffff80005974740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082a8df0 mi_switch() at mi_switch+0x170/frame 0xfffffe01082a8e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082a8e50 _cv_wait() at _cv_wait+0x160/frame 0xfffffe01082a8ec0 usb_process() at usb_process+0xb0/frame 0xfffffe01082a8ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082a8f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082a8f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 15 tid 100137 td 0xfffff80005974000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082a3df0 mi_switch() at mi_switch+0x170/frame 0xfffffe01082a3e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082a3e50 _cv_wait() at _cv_wait+0x160/frame 0xfffffe01082a3ec0 usb_process() at usb_process+0xb0/frame 0xfffffe01082a3ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082a3f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082a3f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 15 tid 100138 td 0xfffff80005975740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082b2df0 mi_switch() at mi_switch+0x170/frame 0xfffffe01082b2e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082b2e50 _cv_wait() at _cv_wait+0x160/frame 0xfffffe01082b2ec0 usb_process() at usb_process+0xb0/frame 0xfffffe01082b2ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082b2f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082b2f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 15 tid 100139 td 0xfffff80003d18740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082bcdf0 mi_switch() at mi_switch+0x170/frame 0xfffffe01082bce10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082bce50 _cv_wait() at _cv_wait+0x160/frame 0xfffffe01082bcec0 usb_process() at usb_process+0xb0/frame 0xfffffe01082bcef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082bcf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082bcf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 15 tid 100140 td 0xfffff80003d18000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082b7df0 mi_switch() at mi_switch+0x170/frame 0xfffffe01082b7e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082b7e50 _cv_wait() at _cv_wait+0x160/frame 0xfffffe01082b7ec0 usb_process() at usb_process+0xb0/frame 0xfffffe01082b7ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082b7f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082b7f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 15 tid 100141 td 0xfffff80003d19740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082c6df0 mi_switch() at mi_switch+0x170/frame 0xfffffe01082c6e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082c6e50 _cv_wait() at _cv_wait+0x160/frame 0xfffffe01082c6ec0 usb_process() at usb_process+0xb0/frame 0xfffffe01082c6ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082c6f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082c6f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 15 tid 100142 td 0xfffff80003d19000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082c1df0 mi_switch() at mi_switch+0x170/frame 0xfffffe01082c1e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082c1e50 _cv_wait() at _cv_wait+0x160/frame 0xfffffe01082c1ec0 usb_process() at usb_process+0xb0/frame 0xfffffe01082c1ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082c1f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082c1f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 15 tid 100143 td 0xfffff80003d1a740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01082d0df0 mi_switch() at mi_switch+0x170/frame 0xfffffe01082d0e10 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01082d0e50 _cv_wait() at _cv_wait+0x160/frame 0xfffffe01082d0ec0 usb_process() at usb_process+0xb0/frame 0xfffffe01082d0ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01082d0f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01082d0f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command vmdaemon pid 16 tid 100199 td 0xfffff800151de740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01083edd80 mi_switch() at mi_switch+0x170/frame 0xfffffe01083edda0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01083edde0 _sleep() at _sleep+0x2d8/frame 0xfffffe01083ede80 vm_daemon() at vm_daemon+0x82/frame 0xfffffe01083edef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01083edf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01083edf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 17 tid 100201 td 0xfffff800151de000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01083e3d60 mi_switch() at mi_switch+0x170/frame 0xfffffe01083e3d80 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01083e3dc0 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe01083e3e00 _sleep() at _sleep+0x27e/frame 0xfffffe01083e3ea0 buf_daemon() at buf_daemon+0x121/frame 0xfffffe01083e3ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01083e3f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01083e3f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 17 tid 100203 td 0xfffff800094ae740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01083fcd40 mi_switch() at mi_switch+0x170/frame 0xfffffe01083fcd60 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01083fcda0 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe01083fcde0 _sleep() at _sleep+0x27e/frame 0xfffffe01083fce80 bufspace_daemon() at bufspace_daemon+0x322/frame 0xfffffe01083fcef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01083fcf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01083fcf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 17 tid 100204 td 0xfffff800094ae000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01083f7d40 mi_switch() at mi_switch+0x170/frame 0xfffffe01083f7d60 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01083f7da0 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe01083f7de0 _sleep() at _sleep+0x27e/frame 0xfffffe01083f7e80 bufspace_daemon() at bufspace_daemon+0x322/frame 0xfffffe01083f7ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01083f7f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01083f7f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 17 tid 100205 td 0xfffff800094b3740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108406d40 mi_switch() at mi_switch+0x170/frame 0xfffffe0108406d60 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108406da0 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe0108406de0 _sleep() at _sleep+0x27e/frame 0xfffffe0108406e80 bufspace_daemon() at bufspace_daemon+0x322/frame 0xfffffe0108406ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108406f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108406f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 17 tid 100206 td 0xfffff800094b3000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108401d40 mi_switch() at mi_switch+0x170/frame 0xfffffe0108401d60 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108401da0 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe0108401de0 _sleep() at _sleep+0x27e/frame 0xfffffe0108401e80 bufspace_daemon() at bufspace_daemon+0x322/frame 0xfffffe0108401ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108401f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108401f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 17 tid 100207 td 0xfffff800094b4740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108410d40 mi_switch() at mi_switch+0x170/frame 0xfffffe0108410d60 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108410da0 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe0108410de0 _sleep() at _sleep+0x27e/frame 0xfffffe0108410e80 bufspace_daemon() at bufspace_daemon+0x322/frame 0xfffffe0108410ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108410f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108410f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 17 tid 100208 td 0xfffff800094b4000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010840bd40 mi_switch() at mi_switch+0x170/frame 0xfffffe010840bd60 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010840bda0 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe010840bde0 _sleep() at _sleep+0x27e/frame 0xfffffe010840be80 bufspace_daemon() at bufspace_daemon+0x322/frame 0xfffffe010840bef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010840bf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010840bf30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 17 tid 100209 td 0xfffff800094bc740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010841ad40 mi_switch() at mi_switch+0x170/frame 0xfffffe010841ad60 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010841ada0 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe010841ade0 _sleep() at _sleep+0x27e/frame 0xfffffe010841ae80 bufspace_daemon() at bufspace_daemon+0x322/frame 0xfffffe010841aef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010841af30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010841af30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 17 tid 100235 td 0xfffff80015de9740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108488d70 mi_switch() at mi_switch+0x170/frame 0xfffffe0108488d90 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108488dd0 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe0108488e10 _sleep() at _sleep+0x27e/frame 0xfffffe0108488eb0 softdep_flush() at softdep_flush+0x188/frame 0xfffffe0108488ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108488f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108488f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 17 tid 100252 td 0xfffff80005095000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108460d70 mi_switch() at mi_switch+0x170/frame 0xfffffe0108460d90 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108460dd0 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe0108460e10 _sleep() at _sleep+0x27e/frame 0xfffffe0108460eb0 softdep_flush() at softdep_flush+0x188/frame 0xfffffe0108460ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108460f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108460f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 17 tid 100566 td 0xfffff800331cd740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010875dd70 mi_switch() at mi_switch+0x170/frame 0xfffffe010875dd90 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010875ddd0 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe010875de10 _sleep() at _sleep+0x27e/frame 0xfffffe010875deb0 softdep_flush() at softdep_flush+0x188/frame 0xfffffe010875def0 fork_exit() at fork_exit+0x82/frame 0xfffffe010875df30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010875df30 --- trap 0xa5a5a5a5, rip = 0xa5a5a5a5a5a5a5a5, rsp = 0xa5a5a5a5a5a5a5a5, rbp = 0xa5a5a5a5a5a5a5a5 --- Tracing command bufdaemon pid 17 tid 100567 td 0xfffff800403b5000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01086dbd70 mi_switch() at mi_switch+0x170/frame 0xfffffe01086dbd90 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01086dbdd0 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe01086dbe10 _sleep() at _sleep+0x27e/frame 0xfffffe01086dbeb0 softdep_flush() at softdep_flush+0x188/frame 0xfffffe01086dbef0 fork_exit() at fork_exit+0x82/frame 0xfffffe01086dbf30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe01086dbf30 --- trap 0xa5a5a5a5, rip = 0xa5a5a5a5a5a5a5a5, rsp = 0xa5a5a5a5a5a5a5a5, rbp = 0xa5a5a5a5a5a5a5a5 --- Tracing command vnlru pid 18 tid 100210 td 0xfffff800151df740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108424d10 mi_switch() at mi_switch+0x170/frame 0xfffffe0108424d30 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108424d70 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe0108424db0 _sleep() at _sleep+0x27e/frame 0xfffffe0108424e50 vnlru_proc() at vnlru_proc+0x18b/frame 0xfffffe0108424ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108424f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108424f30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command syncer pid 19 tid 100211 td 0xfffff800151df000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010841fd30 mi_switch() at mi_switch+0x170/frame 0xfffffe010841fd50 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010841fd90 sleepq_timedwait() at sleepq_timedwait+0x4b/frame 0xfffffe010841fdd0 _cv_timedwait_sbt() at _cv_timedwait_sbt+0x187/frame 0xfffffe010841fe60 sched_sync() at sched_sync+0x6ae/frame 0xfffffe010841fef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010841ff30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010841ff30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command sh pid 8883 tid 100379 td 0xfffff802f7304000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010864fa10 mi_switch() at mi_switch+0x170/frame 0xfffffe010864fa30 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010864fa70 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe010864fad0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe010864fae0 _sleep() at _sleep+0x2ce/frame 0xfffffe010864fb80 kern_wait6() at kern_wait6+0x471/frame 0xfffffe010864fc10 sys_wait4() at sys_wait4+0x6b/frame 0xfffffe010864fe00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010864ff30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010864ff30 --- syscall (7, FreeBSD ELF64, wait4), rip = 0xeae5077099a, rsp = 0xeae4cc55498, rbp = 0xeae4cc55540 --- Tracing command sleep pid 92509 tid 100443 td 0xfffff80003fb4000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010859bba0 mi_switch() at mi_switch+0x170/frame 0xfffffe010859bbc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010859bc00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe010859bc60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe010859bca0 _sleep() at _sleep+0x26c/frame 0xfffffe010859bd40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe010859bdc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe010859be00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010859bf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010859bf30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0x3a815feaa6ba, rsp = 0x3a815e9df0f8, rbp = 0x3a815e9df150 --- Tracing command sh pid 92514 tid 100233 td 0xfffff80003fe1740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010847ea10 mi_switch() at mi_switch+0x170/frame 0xfffffe010847ea30 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010847ea70 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe010847ead0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe010847eae0 _sleep() at _sleep+0x2ce/frame 0xfffffe010847eb80 kern_wait6() at kern_wait6+0x471/frame 0xfffffe010847ec10 sys_wait4() at sys_wait4+0x6b/frame 0xfffffe010847ee00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010847ef30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010847ef30 --- syscall (7, FreeBSD ELF64, wait4), rip = 0x876b62a899a, rsp = 0x876b23731e8, rbp = 0x876b2373290 --- Tracing command tee pid 92515 tid 100437 td 0xfffff802132f4000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108771b60 mi_switch() at mi_switch+0x170/frame 0xfffffe0108771b80 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108771bc0 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108771c20 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe0108771c30 _sleep() at _sleep+0x2ce/frame 0xfffffe0108771cd0 pipe_read() at pipe_read+0x450/frame 0xfffffe0108771d40 dofileread() at dofileread+0x80/frame 0xfffffe0108771d90 sys_read() at sys_read+0xb7/frame 0xfffffe0108771e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108771f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108771f30 --- syscall (3, FreeBSD ELF64, read), rip = 0x2ef6f3c357ba, rsp = 0x2ef6f2598678, rbp = 0x2ef6f25986d0 --- Tracing command sh pid 92516 tid 100498 td 0xfffff80015de9000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01086e5a10 mi_switch() at mi_switch+0x170/frame 0xfffffe01086e5a30 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01086e5a70 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01086e5ad0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe01086e5ae0 _sleep() at _sleep+0x2ce/frame 0xfffffe01086e5b80 kern_wait6() at kern_wait6+0x471/frame 0xfffffe01086e5c10 sys_wait4() at sys_wait4+0x6b/frame 0xfffffe01086e5e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01086e5f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01086e5f30 --- syscall (7, FreeBSD ELF64, wait4), rip = 0x36e3103be99a, rsp = 0x36e30dd429a8, rbp = 0x36e30dd42a50 --- Tracing command md10 pid 92527 tid 100360 td 0xfffff802f7079000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108609da0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108609dc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108609e00 _sleep() at _sleep+0x2d8/frame 0xfffffe0108609ea0 md_kthread() at md_kthread+0xd5/frame 0xfffffe0108609ef0 fork_exit() at fork_exit+0x82/frame 0xfffffe0108609f30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0108609f30 --- trap 0xc, rip = 0x3588f9ebd4ba, rsp = 0x3588f8e88378, rbp = 0x3588f8e88390 --- Tracing command md11 pid 92530 tid 100475 td 0xfffff806a0be7740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010882ada0 mi_switch() at mi_switch+0x170/frame 0xfffffe010882adc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010882ae00 _sleep() at _sleep+0x2d8/frame 0xfffffe010882aea0 md_kthread() at md_kthread+0xd5/frame 0xfffffe010882aef0 fork_exit() at fork_exit+0x82/frame 0xfffffe010882af30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe010882af30 --- trap 0xc, rip = 0xcab777634ba, rsp = 0xcab75ef9a48, rbp = 0xcab75ef9a60 --- Tracing command run pid 92539 tid 100383 td 0xfffff80015af0000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108659ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108659bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108659c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108659c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe0108659ca0 _sleep() at _sleep+0x26c/frame 0xfffffe0108659d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe0108659dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe0108659e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108659f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108659f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0x2ee11ae4c6ba, rsp = 0x2ee11938aee8, rbp = 0x2ee11938af30 --- Tracing command run pid 92541 tid 100452 td 0xfffff80021127740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01086eaa10 mi_switch() at mi_switch+0x170/frame 0xfffffe01086eaa30 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01086eaa70 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01086eaad0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe01086eaae0 _sleep() at _sleep+0x2ce/frame 0xfffffe01086eab80 kern_wait6() at kern_wait6+0x471/frame 0xfffffe01086eac10 sys_wait4() at sys_wait4+0x6b/frame 0xfffffe01086eae00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01086eaf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01086eaf30 --- syscall (7, FreeBSD ELF64, wait4), rip = 0x2ee11ae4c99a, rsp = 0x2ee11938af38, rbp = 0x2ee11938b060 --- Tracing command run pid 92542 tid 100423 td 0xfffff802f7504740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108726a10 mi_switch() at mi_switch+0x170/frame 0xfffffe0108726a30 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108726a70 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108726ad0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe0108726ae0 _sleep() at _sleep+0x2ce/frame 0xfffffe0108726b80 kern_wait6() at kern_wait6+0x471/frame 0xfffffe0108726c10 sys_wait4() at sys_wait4+0x6b/frame 0xfffffe0108726e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108726f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108726f30 --- syscall (7, FreeBSD ELF64, wait4), rip = 0x2ee11ae4c99a, rsp = 0x2ee11938a9f8, rbp = 0x2ee11938af30 --- Tracing command mmap pid 92543 tid 100253 td 0xfffff80015b7a740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01084e2ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01084e2bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01084e2c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01084e2c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01084e2ca0 _sleep() at _sleep+0x26c/frame 0xfffffe01084e2d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01084e2dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01084e2e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01084e2f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01084e2f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0x3a11423e26ba, rsp = 0x3a114032cb78, rbp = 0x3a114032cbc0 --- Tracing command swap pid 92544 tid 100393 td 0xfffff800401ff740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010869aba0 mi_switch() at mi_switch+0x170/frame 0xfffffe010869abc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010869ac00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe010869ac60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe010869aca0 _sleep() at _sleep+0x26c/frame 0xfffffe010869ad40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe010869adc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe010869ae00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010869af30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010869af30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command mmap pid 92545 tid 100392 td 0xfffff8004020a000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108681a10 mi_switch() at mi_switch+0x170/frame 0xfffffe0108681a30 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108681a70 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108681ad0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe0108681ae0 _sleep() at _sleep+0x2ce/frame 0xfffffe0108681b80 kern_wait6() at kern_wait6+0x471/frame 0xfffffe0108681c10 sys_wait4() at sys_wait4+0x6b/frame 0xfffffe0108681e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108681f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108681f30 --- syscall (7, FreeBSD ELF64, wait4), rip = 0x3a11423e299a, rsp = 0x3a114032cbc8, rbp = 0x3a114032ccf0 --- Tracing command swap pid 92546 tid 100271 td 0xfffff80015b7b740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010853ca10 mi_switch() at mi_switch+0x170/frame 0xfffffe010853ca30 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010853ca70 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe010853cad0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe010853cae0 _sleep() at _sleep+0x2ce/frame 0xfffffe010853cb80 kern_wait6() at kern_wait6+0x471/frame 0xfffffe010853cc10 sys_wait4() at sys_wait4+0x6b/frame 0xfffffe010853ce00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010853cf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010853cf30 --- syscall (7, FreeBSD ELF64, wait4), rip = 0xf90c624b99a, rsp = 0xf90c52dd138, rbp = 0xf90c52dd260 --- Tracing command mmap pid 92547 tid 100442 td 0xfffff802f72a0740 (CPU 7) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ce96df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ce96e00 trap() at trap+0x56/frame 0xfffffe003ce96f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ce96f20 --- trap 0x13, rip = 0xffffffff80917af8, rsp = 0xfffffe010867c380, rbp = 0xfffffe010867c3a0 --- ns8250_putc() at ns8250_putc+0x108/frame 0xfffffe010867c3a0 uart_cnputc() at uart_cnputc+0x45/frame 0xfffffe010867c3c0 cnputc() at cnputc+0x4b/frame 0xfffffe010867c3f0 cnputs() at cnputs+0x7a/frame 0xfffffe010867c420 putchar() at putchar+0x165/frame 0xfffffe010867c4b0 kvprintf() at kvprintf+0xe6/frame 0xfffffe010867c5c0 _vprintf() at _vprintf+0x8b/frame 0xfffffe010867c6a0 printf() at printf+0x53/frame 0xfffffe010867c700 stack_print_ddb() at stack_print_ddb+0x68/frame 0xfffffe010867c760 vn_printf() at vn_printf+0x42b/frame 0xfffffe010867c910 cache_enter_time() at cache_enter_time+0x12b0/frame 0xfffffe010867c9f0 unionfs_lookup() at unionfs_lookup+0x7fa/frame 0xfffffe010867cb50 VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0x5f/frame 0xfffffe010867cb80 vfs_cache_lookup() at vfs_cache_lookup+0xa6/frame 0xfffffe010867cbd0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0x5f/frame 0xfffffe010867cc00 vfs_lookup() at vfs_lookup+0x487/frame 0xfffffe010867cc80 namei() at namei+0x2d1/frame 0xfffffe010867cce0 kern_chdir() at kern_chdir+0xc3/frame 0xfffffe010867ce00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010867cf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010867cf30 --- syscall (12, FreeBSD ELF64, chdir), rip = 0x3a11423df55a, rsp = 0x3a114032c808, rbp = 0x3a114032c810 --- Tracing command swap pid 92548 tid 100403 td 0xfffff802f7814740 (CPU 6) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ce87df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ce87e00 trap() at trap+0x56/frame 0xfffffe003ce87f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ce87f20 --- trap 0x13, rip = 0xf88a4d0e574, rsp = 0xf90c52dd100, rbp = 0xf90c52dd130 --- Tracing command mmap pid 92549 tid 100424 td 0xfffff802f7504000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108721ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108721bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108721c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108721c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe0108721ca0 _sleep() at _sleep+0x26c/frame 0xfffffe0108721d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe0108721dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe0108721e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108721f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108721f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0x3a11423e26ba, rsp = 0x3a114032cb78, rbp = 0x3a114032cbc0 --- Tracing command swap pid 92550 tid 100414 td 0xfffff80040896000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01086efba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01086efbc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01086efc00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01086efc60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01086efca0 _sleep() at _sleep+0x26c/frame 0xfffffe01086efd40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01086efdc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01086efe00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01086eff30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01086eff30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command mmap pid 92551 tid 100429 td 0xfffff802f76c7740 (CPU 2) kdb_enter() at kdb_enter+0x33/frame 0xfffffe010874e8b0 panic() at panic+0x43/frame 0xfffffe010874e910 cache_enter_time() at cache_enter_time+0x145b/frame 0xfffffe010874e9f0 unionfs_lookup() at unionfs_lookup+0x7fa/frame 0xfffffe010874eb50 VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0x5f/frame 0xfffffe010874eb80 vfs_cache_lookup() at vfs_cache_lookup+0xa6/frame 0xfffffe010874ebd0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0x5f/frame 0xfffffe010874ec00 vfs_lookup() at vfs_lookup+0x487/frame 0xfffffe010874ec80 namei() at namei+0x2d1/frame 0xfffffe010874ece0 kern_chdir() at kern_chdir+0xc3/frame 0xfffffe010874ee00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010874ef30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010874ef30 --- syscall (12, FreeBSD ELF64, chdir), rip = 0x3a11423df55a, rsp = 0x3a114032c808, rbp = 0x3a114032c810 --- Tracing command swap pid 92552 tid 100306 td 0xfffff80005097000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108569ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108569bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108569c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108569c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe0108569ca0 _sleep() at _sleep+0x26c/frame 0xfffffe0108569d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe0108569dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe0108569e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108569f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108569f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0c8, rbp = 0xf90c52dd0f0 --- Tracing command mmap pid 92553 tid 100358 td 0xfffff802f7305000 thread_exit() at thread_exit+0x1ea/frame 0xfffffe01085ffd80 exit1() at exit1+0xe0f/frame 0xfffffe01085ffdf0 sys_exit() at sys_exit+0xd/frame 0xfffffe01085ffe00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01085fff30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01085fff30 --- syscall (1, FreeBSD ELF64, exit), rip = 0x3a11423e24ba, rsp = 0x3a114032c8c8, rbp = 0x3a114032c8e0 --- Tracing command swap pid 92554 tid 100234 td 0xfffff800151e1000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108465ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108465bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108465c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108465c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe0108465ca0 _sleep() at _sleep+0x26c/frame 0xfffffe0108465d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe0108465dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe0108465e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108465f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108465f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command mmap pid 92555 tid 100378 td 0xfffff802f7304740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108654ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108654bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108654c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108654c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe0108654ca0 _sleep() at _sleep+0x26c/frame 0xfffffe0108654d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe0108654dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe0108654e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108654f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108654f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0x3a11423e26ba, rsp = 0x3a114032cb78, rbp = 0x3a114032cbc0 --- Tracing command swap pid 92556 tid 100356 td 0xfffff80125eb8000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01085f5ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01085f5bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01085f5c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01085f5c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01085f5ca0 _sleep() at _sleep+0x26c/frame 0xfffffe01085f5d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01085f5dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01085f5e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01085f5f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01085f5f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command mmap pid 92557 tid 100460 td 0xfffff803c43b4000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01087d5ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01087d5bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01087d5c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01087d5c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01087d5ca0 _sleep() at _sleep+0x26c/frame 0xfffffe01087d5d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01087d5dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01087d5e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01087d5f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01087d5f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0x3a11423e26ba, rsp = 0x3a114032cb78, rbp = 0x3a114032cbc0 --- Tracing command swap pid 92558 tid 100534 td 0xfffff800331cd000 (CPU 5) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ce78df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ce78e00 trap() at trap+0x56/frame 0xfffffe003ce78f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ce78f20 --- trap 0x13, rip = 0xf88a4d0e574, rsp = 0xf90c52dd100, rbp = 0xf90c52dd130 --- Tracing command mmap pid 92559 tid 100462 td 0xfffff804a9e97000 thread_exit() at thread_exit+0x1ea/frame 0xfffffe01087dfd80 exit1() at exit1+0xe0f/frame 0xfffffe01087dfdf0 sys_exit() at sys_exit+0xd/frame 0xfffffe01087dfe00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01087dff30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01087dff30 --- syscall (1, FreeBSD ELF64, exit), rip = 0x3a11423e24ba, rsp = 0x3a114032c8c8, rbp = 0x3a114032c8e0 --- Tracing command swap pid 92560 tid 100297 td 0xfffff80003e77000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108555ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108555bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108555c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108555c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe0108555ca0 _sleep() at _sleep+0x26c/frame 0xfffffe0108555d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe0108555dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe0108555e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108555f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108555f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command mmap pid 92561 tid 100463 td 0xfffff805a09c5740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01087eeba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01087eebc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01087eec00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01087eec60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01087eeca0 _sleep() at _sleep+0x26c/frame 0xfffffe01087eed40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01087eedc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01087eee00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01087eef30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01087eef30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0x3a11423e26ba, rsp = 0x3a114032cb78, rbp = 0x3a114032cbc0 --- Tracing command swap pid 92562 tid 100535 td 0xfffff802f7638740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01087adba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01087adbc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01087adc00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01087adc60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01087adca0 _sleep() at _sleep+0x26c/frame 0xfffffe01087add40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01087addc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01087ade00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01087adf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01087adf30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command mmap pid 92563 tid 100464 td 0xfffff805a09c5000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01087e9ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01087e9bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01087e9c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01087e9c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01087e9ca0 _sleep() at _sleep+0x26c/frame 0xfffffe01087e9d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01087e9dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01087e9e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01087e9f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01087e9f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0x3a11423e26ba, rsp = 0x3a114032cb78, rbp = 0x3a114032cbc0 --- Tracing command swap pid 92564 tid 100536 td 0xfffff802f7638000 (CPU 11) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ced2df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ced2e00 trap() at trap+0x56/frame 0xfffffe003ced2f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ced2f20 --- trap 0x13, rip = 0xf88a4d0e574, rsp = 0xf90c52dd100, rbp = 0xf90c52dd130 --- Tracing command mmap pid 92565 tid 100465 td 0xfffff806a0787740 (CPU 4) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ce69df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ce69e00 trap() at trap+0x56/frame 0xfffffe003ce69f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ce69f20 --- trap 0x13, rip = 0xffffffff80b985a2, rsp = 0xfffffe01087f8348, rbp = 0xfffffe01087f83b0 --- lock_delay() at lock_delay+0x12/frame 0xfffffe01087f83b0 __mtx_lock_spin_flags() at __mtx_lock_spin_flags+0xc3/frame 0xfffffe01087f83f0 cnputs() at cnputs+0x59/frame 0xfffffe01087f8420 putchar() at putchar+0x165/frame 0xfffffe01087f84b0 kvprintf() at kvprintf+0xe6/frame 0xfffffe01087f85c0 _vprintf() at _vprintf+0x8b/frame 0xfffffe01087f86a0 printf() at printf+0x53/frame 0xfffffe01087f8700 stack_print_ddb() at stack_print_ddb+0x68/frame 0xfffffe01087f8760 vn_printf() at vn_printf+0x42b/frame 0xfffffe01087f8910 cache_enter_time() at cache_enter_time+0x12b0/frame 0xfffffe01087f89f0 unionfs_lookup() at unionfs_lookup+0x7fa/frame 0xfffffe01087f8b50 VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0x5f/frame 0xfffffe01087f8b80 vfs_cache_lookup() at vfs_cache_lookup+0xa6/frame 0xfffffe01087f8bd0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0x5f/frame 0xfffffe01087f8c00 vfs_lookup() at vfs_lookup+0x487/frame 0xfffffe01087f8c80 namei() at namei+0x2d1/frame 0xfffffe01087f8ce0 kern_chdir() at kern_chdir+0xc3/frame 0xfffffe01087f8e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01087f8f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01087f8f30 --- syscall (12, FreeBSD ELF64, chdir), rip = 0x3a11423df55a, rsp = 0x3a114032c808, rbp = 0x3a114032c810 --- Tracing command swap pid 92566 tid 100375 td 0xfffff802137b8740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010864aba0 mi_switch() at mi_switch+0x170/frame 0xfffffe010864abc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010864ac00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe010864ac60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe010864aca0 _sleep() at _sleep+0x26c/frame 0xfffffe010864ad40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe010864adc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe010864ae00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010864af30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010864af30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command mmap pid 92567 tid 100436 td 0xfffff802132f4740 (CPU 9) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ceb4df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ceb4e00 trap() at trap+0x56/frame 0xfffffe003ceb4f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ceb4f20 --- trap 0x13, rip = 0xffffffff80b985a2, rsp = 0xfffffe0108776348, rbp = 0xfffffe01087763b0 --- lock_delay() at lock_delay+0x12/frame 0xfffffe01087763b0 __mtx_lock_spin_flags() at __mtx_lock_spin_flags+0xc3/frame 0xfffffe01087763f0 cnputs() at cnputs+0x59/frame 0xfffffe0108776420 putchar() at putchar+0x165/frame 0xfffffe01087764b0 kvprintf() at kvprintf+0xe6/frame 0xfffffe01087765c0 _vprintf() at _vprintf+0x8b/frame 0xfffffe01087766a0 printf() at printf+0x53/frame 0xfffffe0108776700 stack_print_ddb() at stack_print_ddb+0x68/frame 0xfffffe0108776760 vn_printf() at vn_printf+0x42b/frame 0xfffffe0108776910 cache_enter_time() at cache_enter_time+0x12b0/frame 0xfffffe01087769f0 unionfs_lookup() at unionfs_lookup+0x7fa/frame 0xfffffe0108776b50 VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0x5f/frame 0xfffffe0108776b80 vfs_cache_lookup() at vfs_cache_lookup+0xa6/frame 0xfffffe0108776bd0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0x5f/frame 0xfffffe0108776c00 vfs_lookup() at vfs_lookup+0x487/frame 0xfffffe0108776c80 namei() at namei+0x2d1/frame 0xfffffe0108776ce0 kern_chdir() at kern_chdir+0xc3/frame 0xfffffe0108776e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108776f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108776f30 --- syscall (12, FreeBSD ELF64, chdir), rip = 0x3a11423df55a, rsp = 0x3a114032c808, rbp = 0x3a114032c810 --- Tracing command swap pid 92568 tid 100376 td 0xfffff802137b8000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108645ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108645bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108645c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108645c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe0108645ca0 _sleep() at _sleep+0x26c/frame 0xfffffe0108645d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe0108645dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe0108645e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108645f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108645f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command mmap pid 92569 tid 100415 td 0xfffff8021314b740 (CPU 0) cpustop_handler() at cpustop_handler+0x27/frame 0xffffffff81d09f80 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xffffffff81d09f90 trap() at trap+0x56/frame 0xffffffff81d0a0b0 nmi_calltrap() at nmi_calltrap+0x8/frame 0xffffffff81d0a0b0 --- trap 0x13, rip = 0xffffffff80b985a2, rsp = 0xfffffe01086fe348, rbp = 0xfffffe01086fe3b0 --- lock_delay() at lock_delay+0x12/frame 0xfffffe01086fe3b0 __mtx_lock_spin_flags() at __mtx_lock_spin_flags+0xc3/frame 0xfffffe01086fe3f0 cnputs() at cnputs+0x59/frame 0xfffffe01086fe420 putchar() at putchar+0x165/frame 0xfffffe01086fe4b0 kvprintf() at kvprintf+0xe6/frame 0xfffffe01086fe5c0 _vprintf() at _vprintf+0x8b/frame 0xfffffe01086fe6a0 printf() at printf+0x53/frame 0xfffffe01086fe700 stack_print_ddb() at stack_print_ddb+0x68/frame 0xfffffe01086fe760 vn_printf() at vn_printf+0x42b/frame 0xfffffe01086fe910 cache_enter_time() at cache_enter_time+0x12b0/frame 0xfffffe01086fe9f0 unionfs_lookup() at unionfs_lookup+0x7fa/frame 0xfffffe01086feb50 VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0x5f/frame 0xfffffe01086feb80 vfs_cache_lookup() at vfs_cache_lookup+0xa6/frame 0xfffffe01086febd0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0x5f/frame 0xfffffe01086fec00 vfs_lookup() at vfs_lookup+0x487/frame 0xfffffe01086fec80 namei() at namei+0x2d1/frame 0xfffffe01086fece0 kern_chdir() at kern_chdir+0xc3/frame 0xfffffe01086fee00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01086fef30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01086fef30 --- syscall (12, FreeBSD ELF64, chdir), rip = 0x3a11423df55a, rsp = 0x3a114032c808, rbp = 0x3a114032c810 --- Tracing command swap pid 92570 tid 100381 td 0xfffff800403b2740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108668e90 mi_switch() at mi_switch+0x170/frame 0xfffffe0108668eb0 ast_scheduler() at ast_scheduler+0x45/frame 0xfffffe0108668ed0 ast_handler() at ast_handler+0xe8/frame 0xfffffe0108668f10 ast() at ast+0x20/frame 0xfffffe0108668f30 doreti_ast() at doreti_ast+0x1c/frame 0xf90c52dd130 Tracing command mmap pid 92571 tid 100416 td 0xfffff8021314b000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01086f96a0 mi_switch() at mi_switch+0x170/frame 0xfffffe01086f96c0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01086f9700 sleeplk() at sleeplk+0xfe/frame 0xfffffe01086f9760 lockmgr_xlock_hard() at lockmgr_xlock_hard+0x3d0/frame 0xfffffe01086f9800 ffs_lock() at ffs_lock+0x9d/frame 0xfffffe01086f9840 VOP_LOCK1_APV() at VOP_LOCK1_APV+0x3a/frame 0xfffffe01086f9860 _vn_lock() at _vn_lock+0x53/frame 0xfffffe01086f98c0 vget_finish() at vget_finish+0x75/frame 0xfffffe01086f98f0 cache_lookup_dotdot() at cache_lookup_dotdot+0x1ce/frame 0xfffffe01086f9970 vfs_cache_lookup() at vfs_cache_lookup+0x7c/frame 0xfffffe01086f99c0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0x5f/frame 0xfffffe01086f99f0 unionfs_lookup() at unionfs_lookup+0x214/frame 0xfffffe01086f9b50 VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0x5f/frame 0xfffffe01086f9b80 vfs_cache_lookup() at vfs_cache_lookup+0xa6/frame 0xfffffe01086f9bd0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0x5f/frame 0xfffffe01086f9c00 vfs_lookup() at vfs_lookup+0x487/frame 0xfffffe01086f9c80 namei() at namei+0x2d1/frame 0xfffffe01086f9ce0 kern_chdir() at kern_chdir+0xc3/frame 0xfffffe01086f9e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01086f9f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01086f9f30 --- syscall (12, FreeBSD ELF64, chdir), rip = 0x3a11423df55a, rsp = 0x3a114032c808, rbp = 0x3a114032c810 --- Tracing command swap pid 92572 tid 100382 td 0xfffff800403b2000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108663ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108663bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108663c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108663c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe0108663ca0 _sleep() at _sleep+0x26c/frame 0xfffffe0108663d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe0108663dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe0108663e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108663f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108663f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command swap pid 92573 tid 100449 td 0xfffff802f7650000 (CPU 1) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ce3cdf0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ce3ce00 trap() at trap+0x56/frame 0xfffffe003ce3cf20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ce3cf20 --- trap 0x13, rip = 0xf88a4d0e57a, rsp = 0xf90c52dd100, rbp = 0xf90c52dd130 --- Tracing command swap pid 92574 tid 100492 td 0xfffff805a0315740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01087a8ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01087a8bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01087a8c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01087a8c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01087a8ca0 _sleep() at _sleep+0x26c/frame 0xfffffe01087a8d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01087a8dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01087a8e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01087a8f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01087a8f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command swap pid 92575 tid 100493 td 0xfffff800403b5740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010887ae90 mi_switch() at mi_switch+0x170/frame 0xfffffe010887aeb0 ast_scheduler() at ast_scheduler+0x45/frame 0xfffffe010887aed0 ast_handler() at ast_handler+0xe8/frame 0xfffffe010887af10 ast() at ast+0x20/frame 0xfffffe010887af30 doreti_ast() at doreti_ast+0x1c/frame 0xf90c52dd130 Tracing command swap pid 92576 tid 100373 td 0xfffff80015ebd000 (CPU 10) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003cec3df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003cec3e00 trap() at trap+0x56/frame 0xfffffe003cec3f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003cec3f20 --- trap 0x13, rip = 0xffffffff80b53ca8, rsp = 0xfffffe0108505b20, rbp = 0xfffffe0108505b30 --- _sx_slock_int() at _sx_slock_int+0x8/frame 0xfffffe0108505b30 vm_map_lookup() at vm_map_lookup+0xa7/frame 0xfffffe0108505c20 vm_fault() at vm_fault+0x11d/frame 0xfffffe0108505d60 vm_fault_trap() at vm_fault_trap+0x5d/frame 0xfffffe0108505da0 trap_pfault() at trap_pfault+0x21d/frame 0xfffffe0108505e10 trap() at trap+0x51d/frame 0xfffffe0108505f30 calltrap() at calltrap+0x8/frame 0xfffffe0108505f30 --- trap 0xc, rip = 0xf90c7212325, rsp = 0xf90c52dcd78, rbp = 0xf90c52dcdc0 --- Tracing command swap pid 92577 tid 100352 td 0xfffff802132fd000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108519ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108519bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108519c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108519c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe0108519ca0 _sleep() at _sleep+0x26c/frame 0xfffffe0108519d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe0108519dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe0108519e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108519f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108519f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command swap pid 92578 tid 100353 td 0xfffff802132f1740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01085f0ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01085f0bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01085f0c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01085f0c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01085f0ca0 _sleep() at _sleep+0x26c/frame 0xfffffe01085f0d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01085f0dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01085f0e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01085f0f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01085f0f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command swap pid 92579 tid 100367 td 0xfffff802132f1000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01085ebba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01085ebbc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01085ebc00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01085ebc60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01085ebca0 _sleep() at _sleep+0x26c/frame 0xfffffe01085ebd40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01085ebdc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01085ebe00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01085ebf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01085ebf30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command swap pid 92580 tid 100405 td 0xfffff802f7c89740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01086ccba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01086ccbc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01086ccc00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01086ccc60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01086ccca0 _sleep() at _sleep+0x26c/frame 0xfffffe01086ccd40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01086ccdc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01086cce00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01086ccf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01086ccf30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command swap pid 92581 tid 100406 td 0xfffff802f7c89000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01086c7ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01086c7bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01086c7c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01086c7c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01086c7ca0 _sleep() at _sleep+0x26c/frame 0xfffffe01086c7d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01086c7dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01086c7e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01086c7f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01086c7f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command swap pid 92582 tid 100407 td 0xfffff80213886740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01086d6ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01086d6bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01086d6c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01086d6c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01086d6ca0 _sleep() at _sleep+0x26c/frame 0xfffffe01086d6d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01086d6dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01086d6e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01086d6f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01086d6f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command swap pid 92583 tid 100408 td 0xfffff80213886000 (CPU 3) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003ce5adf0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003ce5ae00 trap() at trap+0x56/frame 0xfffffe003ce5af20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003ce5af20 --- trap 0x13, rip = 0xf88a4d0e57a, rsp = 0xf90c52dd100, rbp = 0xf90c52dd130 --- Tracing command swap pid 92584 tid 100413 td 0xfffff802f707d000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108613ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108613bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108613c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108613c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe0108613ca0 _sleep() at _sleep+0x26c/frame 0xfffffe0108613d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe0108613dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe0108613e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108613f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108613f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command swap pid 92585 tid 100472 td 0xfffff806a0786000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108811e90 mi_switch() at mi_switch+0x170/frame 0xfffffe0108811eb0 ast_scheduler() at ast_scheduler+0x45/frame 0xfffffe0108811ed0 ast_handler() at ast_handler+0xe8/frame 0xfffffe0108811f10 ast() at ast+0x20/frame 0xfffffe0108811f30 doreti_ast() at doreti_ast+0x1c/frame 0xf90c52dd130 Tracing command swap pid 92586 tid 100473 td 0xfffff806a0be6740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108820ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108820bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108820c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108820c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe0108820ca0 _sleep() at _sleep+0x26c/frame 0xfffffe0108820d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe0108820dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe0108820e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108820f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108820f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command swap pid 92587 tid 100458 td 0xfffff805a0315000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01087a3ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01087a3bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01087a3c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01087a3c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01087a3ca0 _sleep() at _sleep+0x26c/frame 0xfffffe01087a3d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01087a3dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01087a3e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01087a3f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01087a3f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command swap pid 92588 tid 100459 td 0xfffff803c43b4740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01087daba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01087dabc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01087dac00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01087dac60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01087daca0 _sleep() at _sleep+0x26c/frame 0xfffffe01087dad40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01087dadc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01087dae00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01087daf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01087daf30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command swap pid 92589 tid 100391 td 0xfffff802f7303740 (CPU 8) cpustop_handler() at cpustop_handler+0x27/frame 0xfffffe003cea5df0 ipi_nmi_handler() at ipi_nmi_handler+0x38/frame 0xfffffe003cea5e00 trap() at trap+0x56/frame 0xfffffe003cea5f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe003cea5f20 --- trap 0x13, rip = 0xffffffff80b545af, rsp = 0xfffffe0108690bf0, rbp = 0xfffffe0108690c00 --- _sx_sunlock_int() at _sx_sunlock_int+0xbf/frame 0xfffffe0108690c00 vm_map_lookup_done() at vm_map_lookup_done+0x46/frame 0xfffffe0108690c10 vm_fault_deallocate() at vm_fault_deallocate+0x161/frame 0xfffffe0108690c30 vm_fault() at vm_fault+0x14da/frame 0xfffffe0108690d60 vm_fault_trap() at vm_fault_trap+0x5d/frame 0xfffffe0108690da0 trap_pfault() at trap_pfault+0x21d/frame 0xfffffe0108690e10 trap() at trap+0x51d/frame 0xfffffe0108690f30 calltrap() at calltrap+0x8/frame 0xfffffe0108690f30 --- trap 0xc, rip = 0xf90c7212325, rsp = 0xf90c52dcd78, rbp = 0xf90c52dcdc0 --- Tracing command swap pid 92590 tid 100399 td 0xfffff802f7303000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010868bba0 mi_switch() at mi_switch+0x170/frame 0xfffffe010868bbc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010868bc00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe010868bc60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe010868bca0 _sleep() at _sleep+0x26c/frame 0xfffffe010868bd40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe010868bdc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe010868be00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010868bf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010868bf30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0c8, rbp = 0xf90c52dd0f0 --- Tracing command swap pid 92591 tid 100401 td 0xfffff802f72fd740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01086b8e90 mi_switch() at mi_switch+0x170/frame 0xfffffe01086b8eb0 ast_scheduler() at ast_scheduler+0x45/frame 0xfffffe01086b8ed0 ast_handler() at ast_handler+0xe8/frame 0xfffffe01086b8f10 ast() at ast+0x20/frame 0xfffffe01086b8f30 doreti_ast() at doreti_ast+0x1c/frame 0xf90c52dd130 Tracing command swap pid 92592 tid 100402 td 0xfffff802f72fd000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01086b3ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01086b3bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01086b3c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01086b3c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01086b3ca0 _sleep() at _sleep+0x26c/frame 0xfffffe01086b3d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01086b3dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01086b3e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01086b3f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01086b3f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command swap pid 92593 tid 100568 td 0xfffff802f7704000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01085a5e90 mi_switch() at mi_switch+0x170/frame 0xfffffe01085a5eb0 ast_scheduler() at ast_scheduler+0x45/frame 0xfffffe01085a5ed0 ast_handler() at ast_handler+0xe8/frame 0xfffffe01085a5f10 ast() at ast+0x20/frame 0xfffffe01085a5f30 doreti_ast() at doreti_ast+0x1c/frame 0xf90c52dd130 Tracing command swap pid 92594 tid 100488 td 0xfffff80033151740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010885cba0 mi_switch() at mi_switch+0x170/frame 0xfffffe010885cbc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010885cc00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe010885cc60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe010885cca0 _sleep() at _sleep+0x26c/frame 0xfffffe010885cd40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe010885cdc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe010885ce00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010885cf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010885cf30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0c8, rbp = 0xf90c52dd0f0 --- Tracing command swap pid 92595 tid 100466 td 0xfffff806a0787000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01087f3ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01087f3bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01087f3c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01087f3c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01087f3ca0 _sleep() at _sleep+0x26c/frame 0xfffffe01087f3d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01087f3dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01087f3e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01087f3f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01087f3f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0xf90c624b6ba, rsp = 0xf90c52dd0e8, rbp = 0xf90c52dd130 --- Tracing command tail pid 92596 tid 100457 td 0xfffff8021326b740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01087d0750 mi_switch() at mi_switch+0x170/frame 0xfffffe01087d0770 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01087d07b0 sleeplk() at sleeplk+0xfe/frame 0xfffffe01087d0810 lockmgr_xlock_hard() at lockmgr_xlock_hard+0x3d0/frame 0xfffffe01087d08b0 ffs_lock() at ffs_lock+0x9d/frame 0xfffffe01087d08f0 VOP_LOCK1_APV() at VOP_LOCK1_APV+0x3a/frame 0xfffffe01087d0910 _vn_lock() at _vn_lock+0x53/frame 0xfffffe01087d0970 kqueue_scan() at kqueue_scan+0x4e5/frame 0xfffffe01087d0a30 kqueue_kevent() at kqueue_kevent+0x13b/frame 0xfffffe01087d0c90 kern_kevent_fp() at kern_kevent_fp+0x97/frame 0xfffffe01087d0ce0 kern_kevent() at kern_kevent+0x82/frame 0xfffffe01087d0d40 kern_kevent_generic() at kern_kevent_generic+0x73/frame 0xfffffe01087d0da0 sys_kevent() at sys_kevent+0x61/frame 0xfffffe01087d0e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01087d0f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01087d0f30 --- syscall (560, FreeBSD ELF64, kevent), rip = 0x3d2451e8a67a, rsp = 0x3d244ea70f88, rbp = 0x3d244ea710c0 --- Tracing command tail pid 92602 tid 100269 td 0xfffff80005097740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108528a90 mi_switch() at mi_switch+0x170/frame 0xfffffe0108528ab0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108528af0 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108528b50 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe0108528b60 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe0108528bd0 kern_select() at kern_select+0xa29/frame 0xfffffe0108528dc0 sys_select() at sys_select+0x57/frame 0xfffffe0108528e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108528f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108528f30 --- syscall (93, FreeBSD ELF64, select), rip = 0x3d2451e8a83a, rsp = 0x3d244ea70e28, rbp = 0x3d244ea70ee0 --- Tracing command sleep pid 92605 tid 100417 td 0xfffff80015e9c740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108708ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108708bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108708c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108708c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe0108708ca0 _sleep() at _sleep+0x26c/frame 0xfffffe0108708d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe0108708dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe0108708e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108708f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108708f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0x218b964856ba, rsp = 0x218b9520d568, rbp = 0x218b9520d5c0 --- Tracing command dhclient pid 3195 tid 100268 td 0xfffff80021129740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010851ea90 mi_switch() at mi_switch+0x170/frame 0xfffffe010851eab0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010851eaf0 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe010851eb50 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe010851eb60 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe010851ebd0 kern_select() at kern_select+0xa29/frame 0xfffffe010851edc0 sys_select() at sys_select+0x57/frame 0xfffffe010851ee00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010851ef30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010851ef30 --- syscall (93, FreeBSD ELF64, select), rip = 0xb9ce51ed83a, rsp = 0xb9cdfe03238, rbp = 0xb9cdfe032f0 --- Tracing command dhclient pid 3198 tid 100242 td 0xfffff80015ae6000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108429ac0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108429ae0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108429b20 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108429b80 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe0108429b90 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe0108429c00 kern_poll_kfds() at kern_poll_kfds+0x4cc/frame 0xfffffe0108429c90 kern_poll() at kern_poll+0x9d/frame 0xfffffe0108429de0 sys_poll() at sys_poll+0x50/frame 0xfffffe0108429e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108429f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108429f30 --- syscall (209, FreeBSD ELF64, poll), rip = 0xb9ce51ed73a, rsp = 0xb9cdfe03478, rbp = 0xb9cdfe034b0 --- Tracing command dhclient pid 3257 tid 100325 td 0xfffff80040b1b000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108591a70 mi_switch() at mi_switch+0x170/frame 0xfffffe0108591a90 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108591ad0 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108591b30 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe0108591b70 _cv_timedwait_sig_sbt() at _cv_timedwait_sig_sbt+0x187/frame 0xfffffe0108591c00 kern_poll_kfds() at kern_poll_kfds+0x493/frame 0xfffffe0108591c90 kern_poll() at kern_poll+0x9d/frame 0xfffffe0108591de0 sys_poll() at sys_poll+0x50/frame 0xfffffe0108591e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108591f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108591f30 --- syscall (209, FreeBSD ELF64, poll), rip = 0xb9ce51ed73a, rsp = 0xb9cdfe03478, rbp = 0xb9cdfe034b0 --- Tracing command devd pid 3272 tid 100238 td 0xfffff800093f5000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108456a40 mi_switch() at mi_switch+0x170/frame 0xfffffe0108456a60 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108456aa0 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108456b00 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe0108456b40 _cv_timedwait_sig_sbt() at _cv_timedwait_sig_sbt+0x187/frame 0xfffffe0108456bd0 kern_select() at kern_select+0xa03/frame 0xfffffe0108456dc0 sys_select() at sys_select+0x57/frame 0xfffffe0108456e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108456f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108456f30 --- syscall (93, FreeBSD ELF64, select), rip = 0x27afcc33c83a, rsp = 0x27afc93a7ed8, rbp = 0x27afc93aa070 --- Tracing command syslogd pid 3420 tid 100261 td 0xfffff80003fe2740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108500ac0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108500ae0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108500b20 _sleep() at _sleep+0x2d8/frame 0xfffffe0108500bc0 getdirtybuf() at getdirtybuf+0x284/frame 0xfffffe0108500c00 flush_deplist() at flush_deplist+0x98/frame 0xfffffe0108500c40 softdep_sync_metadata() at softdep_sync_metadata+0x2b2/frame 0xfffffe0108500ca0 ffs_syncvnode() at ffs_syncvnode+0x50/frame 0xfffffe0108500d30 ffs_fsync() at ffs_fsync+0x1e/frame 0xfffffe0108500d60 VOP_FSYNC_APV() at VOP_FSYNC_APV+0x3a/frame 0xfffffe0108500d80 kern_fsync() at kern_fsync+0x128/frame 0xfffffe0108500e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108500f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108500f30 --- syscall (95, FreeBSD ELF64, fsync), rip = 0xe372773a63a, rsp = 0xe37260d9888, rbp = 0xe37260d9d70 --- Tracing command watchdogd pid 3437 tid 100239 td 0xfffff80015de8740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010844cba0 mi_switch() at mi_switch+0x170/frame 0xfffffe010844cbc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010844cc00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe010844cc60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe010844cca0 _sleep() at _sleep+0x26c/frame 0xfffffe010844cd40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe010844cdc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe010844ce00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010844cf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010844cf30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0x3be5000b76ba, rsp = 0x3be4fd8cd4b8, rbp = 0x3be4fd8cd500 --- Tracing command ntpd pid 3535 tid 100339 td 0xfffff80040143000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108497a90 mi_switch() at mi_switch+0x170/frame 0xfffffe0108497ab0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108497af0 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108497b50 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe0108497b60 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe0108497bd0 kern_select() at kern_select+0xa29/frame 0xfffffe0108497dc0 sys_select() at sys_select+0x57/frame 0xfffffe0108497e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108497f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108497f30 --- syscall (93, FreeBSD ELF64, select), rip = 0x21b5633d83a, rsp = 0x21b52f06608, rbp = 0x21b52f06640 --- Tracing command ntpd pid 3535 tid 100561 td 0xfffff801a528b740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108767b10 mi_switch() at mi_switch+0x170/frame 0xfffffe0108767b30 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108767b70 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108767bd0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe0108767be0 _sleep() at _sleep+0x2ce/frame 0xfffffe0108767c80 umtxq_sleep() at umtxq_sleep+0x302/frame 0xfffffe0108767d00 __umtx_op_sem2_wait() at __umtx_op_sem2_wait+0x3fb/frame 0xfffffe0108767dc0 sys__umtx_op() at sys__umtx_op+0x7e/frame 0xfffffe0108767e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108767f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108767f30 --- syscall (454, FreeBSD ELF64, _umtx_op), rip = 0x21b5633c5da, rsp = 0x21b59f76ee8, rbp = 0x21b59f76f60 --- Tracing command powerd pid 3552 tid 100240 td 0xfffff80005976000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108307a40 mi_switch() at mi_switch+0x170/frame 0xfffffe0108307a60 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108307aa0 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108307b00 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe0108307b40 _cv_timedwait_sig_sbt() at _cv_timedwait_sig_sbt+0x187/frame 0xfffffe0108307bd0 kern_select() at kern_select+0xa03/frame 0xfffffe0108307dc0 sys_select() at sys_select+0x57/frame 0xfffffe0108307e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108307f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108307f30 --- syscall (93, FreeBSD ELF64, select), rip = 0x3fb5672fc83a, rsp = 0x3fb564464df8, rbp = 0x3fb564464f30 --- Tracing command inetd pid 3573 tid 100326 td 0xfffff80040043740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01085b4a90 mi_switch() at mi_switch+0x170/frame 0xfffffe01085b4ab0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01085b4af0 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01085b4b50 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe01085b4b60 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe01085b4bd0 kern_select() at kern_select+0xa29/frame 0xfffffe01085b4dc0 sys_select() at sys_select+0x57/frame 0xfffffe01085b4e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01085b4f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01085b4f30 --- syscall (93, FreeBSD ELF64, select), rip = 0x154439ae083a, rsp = 0x154436e132d8, rbp = 0x154436e14160 --- Tracing command cron pid 3578 tid 100295 td 0xfffff80003e79740 --- swapped out Tracing command sshd pid 3595 tid 100318 td 0xfffff80015deb740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108578a90 mi_switch() at mi_switch+0x170/frame 0xfffffe0108578ab0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108578af0 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108578b50 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe0108578b60 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe0108578bd0 kern_poll_kfds() at kern_poll_kfds+0x4cc/frame 0xfffffe0108578c60 kern_poll() at kern_poll+0x9d/frame 0xfffffe0108578db0 sys_ppoll() at sys_ppoll+0x70/frame 0xfffffe0108578e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108578f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108578f30 --- syscall (545, FreeBSD ELF64, ppoll), rip = 0x34ba6085e75a, rsp = 0x34ba52cd07d8, rbp = 0x34ba52cd0810 --- Tracing command getty pid 3609 tid 100212 td 0xfffff80015ae6740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010842eae0 mi_switch() at mi_switch+0x170/frame 0xfffffe010842eb00 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010842eb40 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe010842eba0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe010842ebb0 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe010842ec20 tty_wait() at tty_wait+0x42/frame 0xfffffe010842ec50 ttydisc_read() at ttydisc_read+0x1f5/frame 0xfffffe010842ecb0 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe010842ece0 devfs_read_f() at devfs_read_f+0xe4/frame 0xfffffe010842ed40 dofileread() at dofileread+0x80/frame 0xfffffe010842ed90 sys_read() at sys_read+0xb7/frame 0xfffffe010842ee00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010842ef30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010842ef30 --- syscall (3, FreeBSD ELF64, read), rip = 0x2b27235787ba, rsp = 0x2b2720c62718, rbp = 0x2b2720c62760 --- Tracing command getty pid 3610 tid 100241 td 0xfffff800052b6740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01084a6ae0 mi_switch() at mi_switch+0x170/frame 0xfffffe01084a6b00 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01084a6b40 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01084a6ba0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe01084a6bb0 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe01084a6c20 tty_wait() at tty_wait+0x42/frame 0xfffffe01084a6c50 ttydisc_read() at ttydisc_read+0x1f5/frame 0xfffffe01084a6cb0 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe01084a6ce0 devfs_read_f() at devfs_read_f+0xe4/frame 0xfffffe01084a6d40 dofileread() at dofileread+0x80/frame 0xfffffe01084a6d90 sys_read() at sys_read+0xb7/frame 0xfffffe01084a6e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01084a6f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01084a6f30 --- syscall (3, FreeBSD ELF64, read), rip = 0x29b5f8d157ba, rsp = 0x29b5f6122248, rbp = 0x29b5f6122290 --- Tracing command getty pid 3611 tid 100265 td 0xfffff800151e0740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010850fae0 mi_switch() at mi_switch+0x170/frame 0xfffffe010850fb00 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010850fb40 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe010850fba0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe010850fbb0 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe010850fc20 tty_wait() at tty_wait+0x42/frame 0xfffffe010850fc50 ttydisc_read() at ttydisc_read+0x1f5/frame 0xfffffe010850fcb0 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe010850fce0 devfs_read_f() at devfs_read_f+0xe4/frame 0xfffffe010850fd40 dofileread() at dofileread+0x80/frame 0xfffffe010850fd90 sys_read() at sys_read+0xb7/frame 0xfffffe010850fe00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010850ff30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010850ff30 --- syscall (3, FreeBSD ELF64, read), rip = 0x3dc4a00b77ba, rsp = 0x3dc49ec118e8, rbp = 0x3dc49ec11930 --- Tracing command getty pid 3612 tid 100343 td 0xfffff800402f9000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108596ae0 mi_switch() at mi_switch+0x170/frame 0xfffffe0108596b00 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108596b40 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108596ba0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe0108596bb0 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe0108596c20 tty_wait() at tty_wait+0x42/frame 0xfffffe0108596c50 ttydisc_read() at ttydisc_read+0x1f5/frame 0xfffffe0108596cb0 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe0108596ce0 devfs_read_f() at devfs_read_f+0xe4/frame 0xfffffe0108596d40 dofileread() at dofileread+0x80/frame 0xfffffe0108596d90 sys_read() at sys_read+0xb7/frame 0xfffffe0108596e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108596f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108596f30 --- syscall (3, FreeBSD ELF64, read), rip = 0x1ce28e41e7ba, rsp = 0x1ce28b6af458, rbp = 0x1ce28b6af4a0 --- Tracing command getty pid 3613 tid 100344 td 0xfffff800402fa740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01085d2ae0 mi_switch() at mi_switch+0x170/frame 0xfffffe01085d2b00 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01085d2b40 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01085d2ba0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe01085d2bb0 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe01085d2c20 tty_wait() at tty_wait+0x42/frame 0xfffffe01085d2c50 ttydisc_read() at ttydisc_read+0x1f5/frame 0xfffffe01085d2cb0 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe01085d2ce0 devfs_read_f() at devfs_read_f+0xe4/frame 0xfffffe01085d2d40 dofileread() at dofileread+0x80/frame 0xfffffe01085d2d90 sys_read() at sys_read+0xb7/frame 0xfffffe01085d2e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01085d2f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01085d2f30 --- syscall (3, FreeBSD ELF64, read), rip = 0x3b3e39a307ba, rsp = 0x3b3e37ccc438, rbp = 0x3b3e37ccc480 --- Tracing command getty pid 3614 tid 100345 td 0xfffff800402fa000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01085cdae0 mi_switch() at mi_switch+0x170/frame 0xfffffe01085cdb00 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01085cdb40 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01085cdba0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe01085cdbb0 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe01085cdc20 tty_wait() at tty_wait+0x42/frame 0xfffffe01085cdc50 ttydisc_read() at ttydisc_read+0x1f5/frame 0xfffffe01085cdcb0 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe01085cdce0 devfs_read_f() at devfs_read_f+0xe4/frame 0xfffffe01085cdd40 dofileread() at dofileread+0x80/frame 0xfffffe01085cdd90 sys_read() at sys_read+0xb7/frame 0xfffffe01085cde00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01085cdf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01085cdf30 --- syscall (3, FreeBSD ELF64, read), rip = 0x3ea10e9f47ba, rsp = 0x3ea10b8aa858, rbp = 0x3ea10b8aa8a0 --- Tracing command getty pid 3615 tid 100346 td 0xfffff800402fc740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01085dcae0 mi_switch() at mi_switch+0x170/frame 0xfffffe01085dcb00 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01085dcb40 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01085dcba0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe01085dcbb0 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe01085dcc20 tty_wait() at tty_wait+0x42/frame 0xfffffe01085dcc50 ttydisc_read() at ttydisc_read+0x1f5/frame 0xfffffe01085dccb0 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe01085dcce0 devfs_read_f() at devfs_read_f+0xe4/frame 0xfffffe01085dcd40 dofileread() at dofileread+0x80/frame 0xfffffe01085dcd90 sys_read() at sys_read+0xb7/frame 0xfffffe01085dce00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01085dcf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01085dcf30 --- syscall (3, FreeBSD ELF64, read), rip = 0x3b43eddf87ba, rsp = 0x3b43ed3eb068, rbp = 0x3b43ed3eb0b0 --- Tracing command getty pid 3616 tid 100347 td 0xfffff800402fc000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01085d7ae0 mi_switch() at mi_switch+0x170/frame 0xfffffe01085d7b00 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01085d7b40 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01085d7ba0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe01085d7bb0 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe01085d7c20 tty_wait() at tty_wait+0x42/frame 0xfffffe01085d7c50 ttydisc_read() at ttydisc_read+0x1f5/frame 0xfffffe01085d7cb0 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe01085d7ce0 devfs_read_f() at devfs_read_f+0xe4/frame 0xfffffe01085d7d40 dofileread() at dofileread+0x80/frame 0xfffffe01085d7d90 sys_read() at sys_read+0xb7/frame 0xfffffe01085d7e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01085d7f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01085d7f30 --- syscall (3, FreeBSD ELF64, read), rip = 0x2c2c1967f7ba, rsp = 0x2c2c16b6a7c8, rbp = 0x2c2c16b6a810 --- Tracing command getty pid 3617 tid 100348 td 0xfffff80213306740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01085e6ae0 mi_switch() at mi_switch+0x170/frame 0xfffffe01085e6b00 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01085e6b40 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01085e6ba0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe01085e6bb0 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe01085e6c20 tty_wait() at tty_wait+0x42/frame 0xfffffe01085e6c50 ttydisc_read() at ttydisc_read+0x1f5/frame 0xfffffe01085e6cb0 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe01085e6ce0 devfs_read_f() at devfs_read_f+0xe4/frame 0xfffffe01085e6d40 dofileread() at dofileread+0x80/frame 0xfffffe01085e6d90 sys_read() at sys_read+0xb7/frame 0xfffffe01085e6e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01085e6f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01085e6f30 --- syscall (3, FreeBSD ELF64, read), rip = 0x2896f35427ba, rsp = 0x2896f076a9f8, rbp = 0x2896f076aa40 --- Tracing command sshd pid 3618 tid 100255 td 0xfffff80015b7a000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01084ddac0 mi_switch() at mi_switch+0x170/frame 0xfffffe01084ddae0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01084ddb20 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01084ddb80 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe01084ddb90 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe01084ddc00 kern_poll_kfds() at kern_poll_kfds+0x4cc/frame 0xfffffe01084ddc90 kern_poll() at kern_poll+0x9d/frame 0xfffffe01084ddde0 sys_poll() at sys_poll+0x50/frame 0xfffffe01084dde00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01084ddf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01084ddf30 --- syscall (209, FreeBSD ELF64, poll), rip = 0x3209496af73a, rsp = 0x32093e937a08, rbp = 0x32093e937a30 --- Tracing command sshd pid 3621 tid 100213 td 0xfffff80015b79740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108438a90 mi_switch() at mi_switch+0x170/frame 0xfffffe0108438ab0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108438af0 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108438b50 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe0108438b60 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe0108438bd0 kern_poll_kfds() at kern_poll_kfds+0x4cc/frame 0xfffffe0108438c60 kern_poll() at kern_poll+0x9d/frame 0xfffffe0108438db0 sys_ppoll() at sys_ppoll+0x70/frame 0xfffffe0108438e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108438f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108438f30 --- syscall (545, FreeBSD ELF64, ppoll), rip = 0x3209496af75a, rsp = 0x32093e9377b8, rbp = 0x32093e9377f0 --- Tracing command sh pid 3622 tid 100231 td 0xfffff800094bc000 --- swapped out Tracing command su pid 3644 tid 100329 td 0xfffff80003fb4740 --- swapped out Tracing command sh pid 3645 tid 100256 td 0xfffff800093f6000 --- swapped out Tracing command bash pid 3646 tid 100342 td 0xfffff80015b83000 --- swapped out Tracing command sh pid 3706 tid 100330 td 0xfffff80015dec000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108546a10 mi_switch() at mi_switch+0x170/frame 0xfffffe0108546a30 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108546a70 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108546ad0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe0108546ae0 _sleep() at _sleep+0x2ce/frame 0xfffffe0108546b80 kern_wait6() at kern_wait6+0x471/frame 0xfffffe0108546c10 sys_wait4() at sys_wait4+0x6b/frame 0xfffffe0108546e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108546f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108546f30 --- syscall (7, FreeBSD ELF64, wait4), rip = 0x876b62a899a, rsp = 0x876b2373538, rbp = 0x876b23735e0 --- Tracing command sh pid 3789 tid 100254 td 0xfffff800093f6740 --- swapped out Tracing command sh pid 3821 tid 100277 td 0xfffff80015b7b000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108537a10 mi_switch() at mi_switch+0x170/frame 0xfffffe0108537a30 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108537a70 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108537ad0 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe0108537ae0 _sleep() at _sleep+0x2ce/frame 0xfffffe0108537b80 kern_wait6() at kern_wait6+0x471/frame 0xfffffe0108537c10 sys_wait4() at sys_wait4+0x6b/frame 0xfffffe0108537e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108537f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108537f30 --- syscall (7, FreeBSD ELF64, wait4), rip = 0x3a0d721ce99a, rsp = 0x3a0d6f0e08c8, rbp = 0x3a0d6f0e0970 --- Tracing command awk pid 3822 tid 100262 td 0xfffff80015ebd740 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010850ab60 mi_switch() at mi_switch+0x170/frame 0xfffffe010850ab80 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010850abc0 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe010850ac20 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe010850ac30 _sleep() at _sleep+0x2ce/frame 0xfffffe010850acd0 pipe_read() at pipe_read+0x450/frame 0xfffffe010850ad40 dofileread() at dofileread+0x80/frame 0xfffffe010850ad90 sys_read() at sys_read+0xb7/frame 0xfffffe010850ae00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010850af30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010850af30 --- syscall (3, FreeBSD ELF64, read), rip = 0x185e1e29e7ba, rsp = 0x185e1de1ce48, rbp = 0x185e1de1ce60 --- Tracing command sh pid 3824 tid 100226 td 0xfffff80015de8000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010843db60 mi_switch() at mi_switch+0x170/frame 0xfffffe010843db80 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010843dbc0 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe010843dc20 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe010843dc30 _sleep() at _sleep+0x2ce/frame 0xfffffe010843dcd0 pipe_read() at pipe_read+0x450/frame 0xfffffe010843dd40 dofileread() at dofileread+0x80/frame 0xfffffe010843dd90 sys_read() at sys_read+0xb7/frame 0xfffffe010843de00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010843df30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010843df30 --- syscall (3, FreeBSD ELF64, read), rip = 0x3a0d721ce7ba, rsp = 0x3a0d6f0e0498, rbp = 0x3a0d6f0e09b0 --- Tracing command sshd pid 3956 tid 100230 td 0xfffff800092c5000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe010846fac0 mi_switch() at mi_switch+0x170/frame 0xfffffe010846fae0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe010846fb20 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe010846fb80 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe010846fb90 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe010846fc00 kern_poll_kfds() at kern_poll_kfds+0x4cc/frame 0xfffffe010846fc90 kern_poll() at kern_poll+0x9d/frame 0xfffffe010846fde0 sys_poll() at sys_poll+0x50/frame 0xfffffe010846fe00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe010846ff30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe010846ff30 --- syscall (209, FreeBSD ELF64, poll), rip = 0x31b54c95a73a, rsp = 0x31b53e6268a8, rbp = 0x31b53e6268d0 --- Tracing command sleep pid 87509 tid 100454 td 0xfffff802f76e1000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe01087b7ba0 mi_switch() at mi_switch+0x170/frame 0xfffffe01087b7bc0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe01087b7c00 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe01087b7c60 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x12/frame 0xfffffe01087b7ca0 _sleep() at _sleep+0x26c/frame 0xfffffe01087b7d40 kern_clock_nanosleep() at kern_clock_nanosleep+0x1cf/frame 0xfffffe01087b7dc0 sys_nanosleep() at sys_nanosleep+0x3b/frame 0xfffffe01087b7e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe01087b7f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe01087b7f30 --- syscall (240, FreeBSD ELF64, nanosleep), rip = 0x21086fd0b6ba, rsp = 0x21086def8958, rbp = 0x21086def89b0 --- Tracing command sshd pid 6082 tid 100263 td 0xfffff800151e0000 sched_switch() at sched_switch+0x5c6/frame 0xfffffe0108514a90 mi_switch() at mi_switch+0x170/frame 0xfffffe0108514ab0 sleepq_switch() at sleepq_switch+0x101/frame 0xfffffe0108514af0 sleepq_catch_signals() at sleepq_catch_signals+0x282/frame 0xfffffe0108514b50 sleepq_wait_sig() at sleepq_wait_sig+0x9/frame 0xfffffe0108514b60 _cv_wait_sig() at _cv_wait_sig+0x165/frame 0xfffffe0108514bd0 kern_poll_kfds() at kern_poll_kfds+0x4cc/frame 0xfffffe0108514c60 kern_poll() at kern_poll+0x9d/frame 0xfffffe0108514db0 sys_ppoll() at sys_ppoll+0x70/frame 0xfffffe0108514e00 amd64_syscall() at amd64_syscall+0x158/frame 0xfffffe0108514f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0108514f30 --- syscall (545, FreeBSD ELF64, ppoll), rip = 0x31b54c95a75a, rsp = 0x31b53e626658, rbp = 0x31b53e626690 --- Tracing command sh pid 6165 tid 100362 td 0xfffff800402f9740 --- swapped out db> reset Uptime: 15m2s (kgdb) bt #0 __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:57 #1 doadump (textdump=textdump@entry=0x0) at ../../../kern/kern_shutdown.c:404 #2 0xffffffff8049d09a in db_dump (dummy=, dummy2=, dummy3=, dummy4=) at ../../../ddb/db_command.c:595 #3 0xffffffff8049ce9d in db_command (last_cmdp=, cmd_table=, dopager=0x0) at ../../../ddb/db_command.c:508 #4 0xffffffff8049cfe6 in db_command_script (command=command@entry=0xffffffff819d1790 "dump") at ../../../ddb/db_command.c:573 #5 0xffffffff804a2e98 in db_script_exec (scriptname=, warnifnotfound=) at ../../../ddb/db_script.c:301 #6 0xffffffff8049ce9d in db_command (last_cmdp=, cmd_table=, dopager=0x0) at ../../../ddb/db_command.c:508 #7 0xffffffff8049cfe6 in db_command_script (command=command@entry=0xffffffff819d1722 "run pho1") at ../../../ddb/db_command.c:573 #8 0xffffffff804a2e98 in db_script_exec (scriptname=, warnifnotfound=) at ../../../ddb/db_script.c:301 #9 0xffffffff8049ce9d in db_command (last_cmdp=, cmd_table=, dopager=0x1) at ../../../ddb/db_command.c:508 #10 0xffffffff8049cb5d in db_command_loop () at ../../../ddb/db_command.c:555 #11 0xffffffff804a0616 in db_trap (type=, code=) at ../../../ddb/db_main.c:267 #12 0xffffffff80b9778f in kdb_trap (type=type@entry=0x3, code=code@entry=0x0, tf=tf@entry=0xfffffe010874e6c0) at ../../../kern/subr_kdb.c:790 #13 0xffffffff81069499 in trap (frame=0xfffffe010874e6c0) at ../../../amd64/amd64/trap.c:606 #14 #15 kdb_enter (why=, msg=) at ../../../kern/subr_kdb.c:556 #16 0xffffffff80b47ca0 in vpanic (fmt=0xffffffff8120926f "condition %s not met at %s:%d (%s)", ap=ap@entry=0xfffffe010874e8f0) at ../../../kern/kern_shutdown.c:967 #17 0xffffffff80b47b23 in panic (fmt=0xffffffff81b98480 ";\310\024\201\377\377\377\377") at ../../../kern/kern_shutdown.c:892 #18 0xffffffff80c1e8eb in cache_enter_time (dvp=dvp@entry=0xfffffe016f682bb8, vp=vp@entry=0x0, cnp=cnp@entry=0xfffffe010874ed78, tsp=tsp@entry=0x0, dtsp=dtsp@entry=0x0) at ../../../kern/vfs_cache.c:2485 #19 0xffffffff8285071a in unionfs_lookup (ap=0xfffffe010874eb90) at /usr/src/sys/fs/unionfs/union_vnops.c:398 #20 0xffffffff81130c0f in VOP_CACHEDLOOKUP_APV (vop=0xffffffff828574f8 , a=a@entry=0xfffffe010874eb90) at vnode_if.c:178 #21 0xffffffff80c20c36 in VOP_CACHEDLOOKUP (dvp=0xfffffe016f682bb8, vpp=0xfffffe010874ed50, cnp=0xfffffe010874ed78) at ./vnode_if.h:101 #22 vfs_cache_lookup (ap=) at ../../../kern/vfs_cache.c:3167 #23 0xffffffff81130a4f in VOP_LOOKUP_APV (vop=0xffffffff828574f8 , a=a@entry=0xfffffe010874ec30) at vnode_if.c:115 #24 0xffffffff80c321c7 in VOP_LOOKUP (dvp=0xfffffe016f682bb8, vpp=0xfffffe010874ed50, cnp=0xfffffe010874ed78) at ./vnode_if.h:67 #25 vfs_lookup (ndp=ndp@entry=0xfffffe010874ecf8) at ../../../kern/vfs_lookup.c:1283 #26 0xffffffff80c312e1 in namei (ndp=ndp@entry=0xfffffe010874ecf8) at ../../../kern/vfs_lookup.c:705 #27 0xffffffff80c50223 in kern_chdir (td=0xfffff802f76c7740, path=0x3a09201bf9d7 , pathseg=UIO_USERSPACE) at ../../../kern/vfs_syscalls.c:953 #28 0xffffffff8106a8a8 in syscallenter (td=0xfffff802f76c7740) at ../../../amd64/amd64/../../kern/subr_syscall.c:189 #29 amd64_syscall (td=0xfffff802f76c7740, traced=0x0) at ../../../amd64/amd64/trap.c:1192 #30 #31 0x00003a11423df55a in ?? () Backtrace stopped: Cannot access memory at address 0x3a114032c808 (kgdb) f 19 #19 0xffffffff8285071a in unionfs_lookup (ap=0xfffffe010874eb90) at /usr/src/sys/fs/unionfs/union_vnops.c:398 398 if (error == ENOENT && (cnflags & MAKEENTRY) != 0 && (kgdb) l 393 if (uvp != NULLVP) 394 vrele(uvp); 395 if (lvp != NULLVP) 396 vrele(lvp); 397 398 if (error == ENOENT && (cnflags & MAKEENTRY) != 0 && 399 !VN_IS_DOOMED(dvp)) 400 cache_enter(dvp, NULLVP, cnp); 401 402 unionfs_lookup_return: (kgdb) info loc vp = 0xfffffe016f6e4e10 uvp = 0x0 lvp = 0x0 va = {va_type = 64, va_mode = 0x8196, va_padding0 = 0xffff, va_uid = 0x6f6832c0, va_gid = 0xfffffe01, va_nlink = 0xfffffe010874eb08, va_fsid = 0xfffffe010874ed78, va_fileid = 0xffffffff00000009, va_size = 0xfffffe010d6a3560, va_blocksize = 0xfffff802f76c7740, va_atime = { tv_sec = 0x88f, tv_nsec = 0xfffff802f76c7740}, va_mtime = {tv_sec = 0x86400, tv_nsec = 0xfffffe016f6833f0}, va_ctime = { tv_sec = 0xfffffe010874eaa0, tv_nsec = 0xffffffff80b14d98}, va_birthtime = {tv_sec = 0x80000, tv_nsec = 0xfffffe016f6832c0}, va_gen = 0x0, va_flags = 0xfffffe010d6a3578, va_rdev = 0x8, va_bytes = 0xffffffff8117423e, va_filerev = 0xfffff802f76c7740, va_vaflags = 0x246, va_spare = 0x1} lkflags = 0x80000 dvp = 0xfffffe016f682bb8 lockflag = lerror = uerror = error = cnp = 0xfffffe010874ed78 nameiop = cnflags = 0x34040144 dunp = ldvp = udvp = 0xfffffe016f6832c0 td = 0xfffff802f76c7740 dtmpvp = unp = (kgdb) f 24 #24 0xffffffff80c321c7 in VOP_LOOKUP (dvp=0xfffffe016f682bb8, vpp=0xfffffe010874ed50, cnp=0xfffffe010874ed78) at ./vnode_if.h:67 67 return (VOP_LOOKUP_APV(dvp->v_op, &a)); (kgdb) l 62 if (!SDT_PROBES_ENABLED()) 63 return (dvp->v_op->vop_lookup(&a)); 64 else 65 return (VOP_LOOKUP_APV(dvp->v_op, &a)); 66 #else 67 return (VOP_LOOKUP_APV(dvp->v_op, &a)); 68 #endif 69 } 70 71 struct vop_cachedlookup_args { (kgdb) info loc a = {a_gen = {a_desc = 0xffffffff8196ce40 }, a_dvp = 0xfffffe016f682bb8, a_vpp = 0xfffffe010874ed50, a_cnp = 0xfffffe010874ed78} (kgdb) p *dvp $1 = {v_type = VBAD, v_state = VSTATE_DEAD, v_irflag = 0x1, v_seqc = 0x5183, v_nchash = 0x81268a4c, v_hash = 0x16f682b, v_op = 0xffffffff818ac760 , v_data = 0x0, v_mount = 0x0, v_nmntvnodes = {tqe_next = 0xffffffffffffffff, tqe_prev = 0xffffffffffffffff}, {v_mountedhere = 0x0, v_unpcb = 0x0, v_rdev = 0x0, v_fifoinfo = 0x0}, v_hashlist = { le_next = 0xffffffffffffffff, le_prev = 0xffffffffffffffff}, v_cache_src = {lh_first = 0x0}, v_cache_dst = {tqh_first = 0x0, tqh_last = 0xfffffe016f682c10}, v_cache_dd = 0x0, v_lock = {lock_object = {lo_name = 0xffffffff82856d6a "unionfs", lo_flags = 0x6fb0008, lo_data = 0x0, lo_witness = 0xfffff8087fd85400}, lk_lock = 0xfffff802f76c7740, lk_exslpfail = 0x0, lk_pri = 0x40, lk_timo = 0x33, lk_stack = {depth = 0xe, pcs = {0xffffffff80b14d98, 0xffffffff8113416a, 0xffffffff80c5ae03, 0xffffffff80c60017, 0xffffffff8284e086, 0xffffffff8285014d, 0xffffffff81130c0f, 0xffffffff80c20c36, 0xffffffff81130a4f, 0xffffffff80c321c7, 0xffffffff80c312e1, 0xffffffff80c50223, 0xffffffff8106a8a8, 0xffffffff8103bd0b, 0x0, 0x0, 0x0, 0x0}}}, v_interlock = {lock_object = { lo_name = 0xffffffff811e5140 "vnode interlock", lo_flags = 0x1030000, lo_data = 0x0, lo_witness = 0xfffff8087fd75500}, mtx_lock = 0x0}, v_vnlock = 0xfffffe016f682c28, v_vnodelist = {tqe_next = 0xfffffe016f682e10, tqe_prev = 0xfffffe016f682860}, v_lazylist = { tqe_next = 0xffffffffffffffff, tqe_prev = 0xffffffffffffffff}, v_bufobj = {bo_lock = {lock_object = { lo_name = 0xffffffff81230c95 "bufobj interlock", lo_flags = 0x5230000, lo_data = 0x0, lo_witness = 0xfffff8087fd80c00}, rw_lock = 0x1}, bo_ops = 0xffffffff818ed8e0 , bo_object = 0x0, bo_synclist = {le_next = 0xffffffffffffffff, le_prev = 0xffffffffffffffff}, bo_private = 0xfffffe016f682bb8, bo_clean = {bv_hd = {tqh_first = 0x0, tqh_last = 0xfffffe016f682d78}, bv_root = {pt_root = 0x1}, bv_cnt = 0x0}, bo_dirty = {bv_hd = {tqh_first = 0x0, tqh_last = 0xfffffe016f682d98}, bv_root = { pt_root = 0x1}, bv_cnt = 0x0}, bo_numoutput = 0x0, bo_flag = 0x4, bo_domain = 0x4, bo_bsize = 0x8000}, v_pollinfo = 0x0, v_label = 0x0, v_lockf = 0x0, v_rl = {rl_waiters = {tqh_first = 0x0, tqh_last = 0xfffffe016f682de0}, rl_currdep = 0x0}, v_holdcnt = 0x1, v_usecount = 0x2, v_iflag = 0x0, v_vflag = 0x0, v_mflag = 0x0, v_dbatchcpu = 0xffff, v_writecount = 0x0, v_seqc_users = 0x1} (kgdb) diff --git a/sys/fs/unionfs/union.h b/sys/fs/unionfs/union.h index 467db3b29ff8..0bd1894a2195 100644 --- a/sys/fs/unionfs/union.h +++ b/sys/fs/unionfs/union.h @@ -97,15 +97,17 @@ struct unionfs_node { char *un_path; /* path */ int un_pathlen; /* strlen of path */ - int un_flag; /* unionfs node flag */ -}; -/* - * unionfs node flags - * It needs the vnode with exclusive lock, when changing the un_flag variable. - */ -#define UNIONFS_OPENEXTL 0x01 /* openextattr (lower) */ -#define UNIONFS_OPENEXTU 0x02 /* openextattr (upper) */ + /* + * unionfs node flags + * Changing these flags requires the vnode to be locked exclusive. + */ + #define UNIONFS_OPENEXTL 0x01 /* openextattr (lower) */ + #define UNIONFS_OPENEXTU 0x02 /* openextattr (upper) */ + #define UNIONFS_COPY_IN_PROGRESS 0x04 /* copy/dir shadow in progres */ + #define UNIONFS_LOOKUP_IN_PROGRESS 0x08 + unsigned int un_flag; /* unionfs node flag */ +}; extern struct vop_vector unionfs_vnodeops; @@ -131,34 +133,32 @@ int unionfs_uninit(struct vfsconf *); int unionfs_nodeget(struct mount *, struct vnode *, struct vnode *, struct vnode *, struct vnode **, struct componentname *); void unionfs_noderem(struct vnode *); +struct unionfs_node_status * unionfs_find_node_status(struct unionfs_node *, + struct thread *td); void unionfs_get_node_status(struct unionfs_node *, struct thread *, struct unionfs_node_status **); void unionfs_tryrem_node_status(struct unionfs_node *, struct unionfs_node_status *); int unionfs_check_rmdir(struct vnode *, struct ucred *, struct thread *td); -int unionfs_copyfile(struct unionfs_node *, int, struct ucred *, +int unionfs_copyfile(struct vnode *, int, struct ucred *, struct thread *); void unionfs_create_uppervattr_core(struct unionfs_mount *, struct vattr *, struct vattr *, struct thread *); int unionfs_create_uppervattr(struct unionfs_mount *, struct vnode *, struct vattr *, struct ucred *, struct thread *); -int unionfs_mkshadowdir(struct unionfs_mount *, struct vnode *, - struct unionfs_node *, struct componentname *, struct thread *); +int unionfs_mkshadowdir(struct vnode *, struct vnode *, + struct componentname *, struct thread *); int unionfs_mkwhiteout(struct vnode *, struct vnode *, struct componentname *, struct thread *, char *, int); int unionfs_relookup(struct vnode *, struct vnode **, struct componentname *, struct componentname *, struct thread *, char *, int, u_long); -int unionfs_relookup_for_create(struct vnode *, struct componentname *, - struct thread *); -int unionfs_relookup_for_delete(struct vnode *, struct componentname *, - struct thread *); -int unionfs_relookup_for_rename(struct vnode *, struct componentname *, - struct thread *); void unionfs_forward_vop_start_pair(struct vnode *, int *, struct vnode *, int *); bool unionfs_forward_vop_finish_pair(struct vnode *, struct vnode *, int, struct vnode *, struct vnode *, int); +int unionfs_set_in_progress_flag(struct vnode *, unsigned int); +void unionfs_clear_in_progress_flag(struct vnode *, unsigned int); static inline void unionfs_forward_vop_start(struct vnode *basevp, int *lkflags) diff --git a/sys/fs/unionfs/union_subr.c b/sys/fs/unionfs/union_subr.c index bb57f3d56ade..e02dd547f249 100644 --- a/sys/fs/unionfs/union_subr.c +++ b/sys/fs/unionfs/union_subr.c @@ -203,19 +203,19 @@ unionfs_ins_cached_vnode(struct unionfs_node *uncp, struct unionfs_node_hashhead *hd; struct vnode *vp; - ASSERT_VOP_ELOCKED(uncp->un_uppervp, __func__); - ASSERT_VOP_ELOCKED(uncp->un_lowervp, __func__); - KASSERT(uncp->un_uppervp == NULLVP || uncp->un_uppervp->v_type == VDIR, - ("%s: v_type != VDIR", __func__)); - KASSERT(uncp->un_lowervp == NULLVP || uncp->un_lowervp->v_type == VDIR, - ("%s: v_type != VDIR", __func__)); - vp = NULLVP; VI_LOCK(dvp); - if (uncp->un_uppervp != NULL) + if (uncp->un_uppervp != NULLVP) { + ASSERT_VOP_ELOCKED(uncp->un_uppervp, __func__); + KASSERT(uncp->un_uppervp->v_type == VDIR, + ("%s: v_type != VDIR", __func__)); vp = unionfs_get_cached_vnode_locked(uncp->un_uppervp, dvp); - else if (uncp->un_lowervp != NULL) + } else if (uncp->un_lowervp != NULLVP) { + ASSERT_VOP_ELOCKED(uncp->un_lowervp, __func__); + KASSERT(uncp->un_lowervp->v_type == VDIR, + ("%s: v_type != VDIR", __func__)); vp = unionfs_get_cached_vnode_locked(uncp->un_lowervp, dvp); + } if (vp == NULLVP) { hd = unionfs_get_hashhead(dvp, (uncp->un_uppervp != NULLVP ? uncp->un_uppervp : uncp->un_lowervp)); @@ -276,9 +276,11 @@ unionfs_nodeget_cleanup(struct vnode *vp, struct unionfs_node *unp) if (unp->un_dvp != NULLVP) vrele(unp->un_dvp); - if (unp->un_uppervp != NULLVP) + if (unp->un_uppervp != NULLVP) { vput(unp->un_uppervp); - if (unp->un_lowervp != NULLVP) + if (unp->un_lowervp != NULLVP) + vrele(unp->un_lowervp); + } else if (unp->un_lowervp != NULLVP) vput(unp->un_lowervp); if (unp->un_hashtbl != NULL) hashdestroy(unp->un_hashtbl, M_UNIONFSHASH, UNIONFSHASHMASK); @@ -314,7 +316,7 @@ unionfs_nodeget(struct mount *mp, struct vnode *uppervp, *vpp = NULLVP; if (uppervp == NULLVP && lowervp == NULLVP) - panic("%s: upper and lower is null", __func__); + panic("%s: upper and lower are both null", __func__); vt = (uppervp != NULLVP ? uppervp->v_type : lowervp->v_type); @@ -327,7 +329,9 @@ unionfs_nodeget(struct mount *mp, struct vnode *uppervp, vp = unionfs_get_cached_vnode(uppervp, lowervp, dvp); if (vp != NULLVP) { *vpp = vp; - goto unionfs_nodeget_out; + if (lkflags != 0) + vn_lock(*vpp, lkflags | LK_RETRY); + return (0); } } @@ -385,27 +389,47 @@ unionfs_nodeget(struct mount *mp, struct vnode *uppervp, KASSERT(dvp != NULL || (vp->v_vflag & VV_ROOT) != 0, ("%s: NULL dvp for non-root vp %p", __func__, vp)); - vn_lock_pair(lowervp, false, LK_EXCLUSIVE, uppervp, false, - LK_EXCLUSIVE); + + /* + * NOTE: There is still a possibility for cross-filesystem locking here. + * If dvp has an upper FS component and is locked, while the new vnode + * created here only has a lower-layer FS component, then we will end + * up taking a lower-FS lock while holding an upper-FS lock. + * That situation could be dealt with here using vn_lock_pair(). + * However, that would only address one instance out of many in which + * a child vnode lock is taken while holding a lock on its parent + * directory. This is done in many places in common VFS code, as well as + * a few places within unionfs (which could lead to the same cross-FS + * locking issue if, for example, the upper FS is another nested unionfs + * instance). Additionally, it is unclear under what circumstances this + * specific lock sequence (a directory on one FS followed by a child of + * its 'peer' directory on another FS) would present the practical + * possibility of deadlock due to some other agent on the system + * attempting to lock those two specific vnodes in the opposite order. + */ + if (uppervp != NULLVP) + vn_lock(uppervp, LK_EXCLUSIVE | LK_RETRY); + else + vn_lock(lowervp, LK_EXCLUSIVE | LK_RETRY); error = insmntque1(vp, mp); if (error != 0) { unionfs_nodeget_cleanup(vp, unp); return (error); } - if (lowervp != NULL && VN_IS_DOOMED(lowervp)) { - vput(lowervp); - unp->un_lowervp = lowervp = NULL; - } - if (uppervp != NULL && VN_IS_DOOMED(uppervp)) { - vput(uppervp); - unp->un_uppervp = uppervp = NULL; - if (lowervp != NULLVP) - vp->v_vnlock = lowervp->v_vnlock; - } - if (lowervp == NULL && uppervp == NULL) { - unionfs_nodeget_cleanup(vp, unp); - return (ENOENT); - } + /* + * lowervp and uppervp should only be doomed by a forced unmount of + * their respective filesystems, but that can only happen if the + * unionfs instance is first unmounted. We also effectively hold the + * lock on the new unionfs vnode at this point. Therefore, if a + * unionfs umount has not yet reached the point at which the above + * insmntque1() would fail, then its vflush() call will end up + * blocked on our vnode lock, effectively also preventing unmount + * of the underlying filesystems. + */ + VNASSERT(lowervp == NULLVP || !VN_IS_DOOMED(lowervp), vp, + ("%s: doomed lowervp %p", __func__, lowervp)); + VNASSERT(uppervp == NULLVP || !VN_IS_DOOMED(uppervp), vp, + ("%s: doomed lowervp %p", __func__, uppervp)); vn_set_state(vp, VSTATE_CONSTRUCTED); @@ -413,18 +437,16 @@ unionfs_nodeget(struct mount *mp, struct vnode *uppervp, *vpp = unionfs_ins_cached_vnode(unp, dvp); if (*vpp != NULLVP) { unionfs_nodeget_cleanup(vp, unp); - vp = *vpp; - } else { - if (uppervp != NULL) - VOP_UNLOCK(uppervp); - if (lowervp != NULL) - VOP_UNLOCK(lowervp); + if (lkflags != 0) + vn_lock(*vpp, lkflags | LK_RETRY); + return (0); + } else *vpp = vp; - } -unionfs_nodeget_out: - if (lkflags & LK_TYPE_MASK) - vn_lock(vp, lkflags | LK_RETRY); + if ((lkflags & LK_SHARED) != 0) + vn_lock(vp, LK_DOWNGRADE); + else if ((lkflags & LK_EXCLUSIVE) == 0) + VOP_UNLOCK(vp); return (0); } @@ -443,6 +465,7 @@ unionfs_noderem(struct vnode *vp) struct vnode *dvp; int count; int writerefs; + bool unlock_lvp; /* * The root vnode lock may be recursed during unmount, because @@ -455,18 +478,36 @@ unionfs_noderem(struct vnode *vp) */ KASSERT(vp->v_vnlock->lk_recurse == 0 || (vp->v_vflag & VV_ROOT) != 0, ("%s: vnode %p locked recursively", __func__, vp)); + + unp = VTOUNIONFS(vp); + VNASSERT(unp != NULL, vp, ("%s: already reclaimed", __func__)); + lvp = unp->un_lowervp; + uvp = unp->un_uppervp; + dvp = unp->un_dvp; + unlock_lvp = (uvp == NULLVP); + + /* + * Lock the lower vnode in addition to the upper vnode lock in order + * to synchronize against any unionfs_lock() operation which may still + * hold the lower vnode lock. We do not need to do this for the root + * vnode, as the root vnode should always have both upper and lower + * base vnodes for its entire lifecycled, so unionfs_lock() should + * never attempt to lock its lower vnode in the first place. + * Moreover, during unmount of a non-"below" unionfs mount, the lower + * root vnode will already be locked as it is the covered vnode. + */ + if (uvp != NULLVP && lvp != NULLVP && (vp->v_vflag & VV_ROOT) == 0) { + vn_lock_pair(uvp, true, LK_EXCLUSIVE, lvp, false, LK_EXCLUSIVE); + unlock_lvp = true; + } + if (lockmgr(&vp->v_lock, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0) panic("%s: failed to acquire lock for vnode lock", __func__); - /* * Use the interlock to protect the clearing of v_data to * prevent faults in unionfs_lock(). */ VI_LOCK(vp); - unp = VTOUNIONFS(vp); - lvp = unp->un_lowervp; - uvp = unp->un_uppervp; - dvp = unp->un_dvp; unp->un_lowervp = unp->un_uppervp = NULLVP; vp->v_vnlock = &(vp->v_lock); vp->v_data = NULL; @@ -502,18 +543,16 @@ unionfs_noderem(struct vnode *vp) ("%s: write reference without upper vnode", __func__)); VOP_ADD_WRITECOUNT(uvp, -writerefs); } - if (lvp != NULLVP) - VOP_UNLOCK(lvp); if (uvp != NULLVP) - VOP_UNLOCK(uvp); + vput(uvp); + if (unlock_lvp) + vput(lvp); + else if (lvp != NULLVP) + vrele(lvp); if (dvp != NULLVP) unionfs_rem_cached_vnode(unp, dvp); - if (lvp != NULLVP) - vrele(lvp); - if (uvp != NULLVP) - vrele(uvp); if (unp->un_path != NULL) { free(unp->un_path, M_UNIONFSPATH); unp->un_path = NULL; @@ -539,35 +578,52 @@ unionfs_noderem(struct vnode *vp) } /* - * Get the unionfs node status object for the vnode corresponding to unp, - * for the process that owns td. Allocate a new status object if one - * does not already exist. + * Find the unionfs node status object for the vnode corresponding to unp, + * for the process that owns td. Return NULL if no such object exists. */ -void -unionfs_get_node_status(struct unionfs_node *unp, struct thread *td, - struct unionfs_node_status **unspp) +struct unionfs_node_status * +unionfs_find_node_status(struct unionfs_node *unp, struct thread *td) { struct unionfs_node_status *unsp; pid_t pid; pid = td->td_proc->p_pid; - KASSERT(NULL != unspp, ("%s: NULL status", __func__)); ASSERT_VOP_ELOCKED(UNIONFSTOV(unp), __func__); LIST_FOREACH(unsp, &(unp->un_unshead), uns_list) { if (unsp->uns_pid == pid) { - *unspp = unsp; - return; + return (unsp); } } - /* create a new unionfs node status */ - unsp = malloc(sizeof(struct unionfs_node_status), - M_TEMP, M_WAITOK | M_ZERO); + return (NULL); +} + +/* + * Get the unionfs node status object for the vnode corresponding to unp, + * for the process that owns td. Allocate a new status object if one + * does not already exist. + */ +void +unionfs_get_node_status(struct unionfs_node *unp, struct thread *td, + struct unionfs_node_status **unspp) +{ + struct unionfs_node_status *unsp; + pid_t pid; + + pid = td->td_proc->p_pid; - unsp->uns_pid = pid; - LIST_INSERT_HEAD(&(unp->un_unshead), unsp, uns_list); + KASSERT(NULL != unspp, ("%s: NULL status", __func__)); + unsp = unionfs_find_node_status(unp, td); + if (unsp == NULL) { + /* create a new unionfs node status */ + unsp = malloc(sizeof(struct unionfs_node_status), + M_TEMP, M_WAITOK | M_ZERO); + + unsp->uns_pid = pid; + LIST_INSERT_HEAD(&(unp->un_unshead), unsp, uns_list); + } *unspp = unsp; } @@ -696,110 +752,6 @@ unionfs_relookup(struct vnode *dvp, struct vnode **vpp, return (error); } -/* - * relookup for CREATE namei operation. - * - * dvp is unionfs vnode. dvp should be locked. - * - * If it called 'unionfs_copyfile' function by unionfs_link etc, - * VOP_LOOKUP information is broken. - * So it need relookup in order to create link etc. - */ -int -unionfs_relookup_for_create(struct vnode *dvp, struct componentname *cnp, - struct thread *td) -{ - struct vnode *udvp; - struct vnode *vp; - struct componentname cn; - int error; - - udvp = UNIONFSVPTOUPPERVP(dvp); - vp = NULLVP; - - error = unionfs_relookup(udvp, &vp, cnp, &cn, td, cnp->cn_nameptr, - cnp->cn_namelen, CREATE); - if (error) - return (error); - - if (vp != NULLVP) { - if (udvp == vp) - vrele(vp); - else - vput(vp); - - error = EEXIST; - } - - return (error); -} - -/* - * relookup for DELETE namei operation. - * - * dvp is unionfs vnode. dvp should be locked. - */ -int -unionfs_relookup_for_delete(struct vnode *dvp, struct componentname *cnp, - struct thread *td) -{ - struct vnode *udvp; - struct vnode *vp; - struct componentname cn; - int error; - - udvp = UNIONFSVPTOUPPERVP(dvp); - vp = NULLVP; - - error = unionfs_relookup(udvp, &vp, cnp, &cn, td, cnp->cn_nameptr, - cnp->cn_namelen, DELETE); - if (error) - return (error); - - if (vp == NULLVP) - error = ENOENT; - else { - if (udvp == vp) - vrele(vp); - else - vput(vp); - } - - return (error); -} - -/* - * relookup for RENAME namei operation. - * - * dvp is unionfs vnode. dvp should be locked. - */ -int -unionfs_relookup_for_rename(struct vnode *dvp, struct componentname *cnp, - struct thread *td) -{ - struct vnode *udvp; - struct vnode *vp; - struct componentname cn; - int error; - - udvp = UNIONFSVPTOUPPERVP(dvp); - vp = NULLVP; - - error = unionfs_relookup(udvp, &vp, cnp, &cn, td, cnp->cn_nameptr, - cnp->cn_namelen, RENAME); - if (error) - return (error); - - if (vp != NULLVP) { - if (udvp == vp) - vrele(vp); - else - vput(vp); - } - - return (error); -} - /* * Update the unionfs_node. * @@ -836,6 +788,8 @@ unionfs_node_update(struct unionfs_node *unp, struct vnode *uvp, vp->v_vnlock = uvp->v_vnlock; VI_UNLOCK(vp); + for (count = 0; count < lockrec + 1; count++) + VOP_UNLOCK(lvp); /* * Re-cache the unionfs vnode against the upper vnode */ @@ -850,19 +804,88 @@ unionfs_node_update(struct unionfs_node *unp, struct vnode *uvp, } } +/* + * Mark a unionfs operation as being in progress, sleeping if the + * same operation is already in progress. + * This is useful, for example, during copy-up operations in which + * we may drop the target vnode lock, but we want to avoid the + * possibility of a concurrent copy-up on the same vnode triggering + * a spurious failure. + */ +int +unionfs_set_in_progress_flag(struct vnode *vp, unsigned int flag) +{ + struct unionfs_node *unp; + int error; + + error = 0; + ASSERT_VOP_ELOCKED(vp, __func__); + VI_LOCK(vp); + unp = VTOUNIONFS(vp); + while (error == 0 && (unp->un_flag & flag) != 0) { + VOP_UNLOCK(vp); + error = msleep(vp, VI_MTX(vp), PCATCH | PDROP, "unioncp", 0); + vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); + VI_LOCK(vp); + if (error == 0) { + /* + * If we waited on a concurrent copy-up and that + * copy-up was successful, return a non-fatal + * indication that the desired operation is already + * complete. If we waited on a concurrent lookup, + * return ERELOOKUP to indicate the VFS cache should + * be re-queried to avoid creating a duplicate unionfs + * vnode. + */ + unp = VTOUNIONFS(vp); + if (unp == NULL) + error = ENOENT; + else if (flag == UNIONFS_COPY_IN_PROGRESS && + unp->un_uppervp != NULLVP) + error = EJUSTRETURN; + else if (flag == UNIONFS_LOOKUP_IN_PROGRESS) + error = ERELOOKUP; + } + } + if (error == 0) + unp->un_flag |= flag; + VI_UNLOCK(vp); + + return (error); +} + +void +unionfs_clear_in_progress_flag(struct vnode *vp, unsigned int flag) +{ + struct unionfs_node *unp; + + ASSERT_VOP_ELOCKED(vp, __func__); + unp = VTOUNIONFS(vp); + VI_LOCK(vp); + if (unp != NULL) { + VNASSERT((unp->un_flag & flag) != 0, vp, + ("%s: copy not in progress", __func__)); + unp->un_flag &= ~flag; + } + wakeup(vp); + VI_UNLOCK(vp); +} + /* * Create a new shadow dir. * - * udvp should be locked on entry and will be locked on return. + * dvp and vp are unionfs vnodes representing a parent directory and + * child file, should be locked on entry, and will be locked on return. * * If no error returned, unp will be updated. */ int -unionfs_mkshadowdir(struct unionfs_mount *ump, struct vnode *udvp, - struct unionfs_node *unp, struct componentname *cnp, struct thread *td) +unionfs_mkshadowdir(struct vnode *dvp, struct vnode *vp, + struct componentname *cnp, struct thread *td) { struct vnode *lvp; struct vnode *uvp; + struct vnode *udvp; struct vattr va; struct vattr lva; struct nameidata nd; @@ -870,10 +893,25 @@ unionfs_mkshadowdir(struct unionfs_mount *ump, struct vnode *udvp, struct ucred *cred; struct ucred *credbk; struct uidinfo *rootinfo; + struct unionfs_mount *ump; + struct unionfs_node *dunp; + struct unionfs_node *unp; int error; + ASSERT_VOP_ELOCKED(dvp, __func__); + ASSERT_VOP_ELOCKED(vp, __func__); + ump = MOUNTTOUNIONFSMOUNT(vp->v_mount); + unp = VTOUNIONFS(vp); if (unp->un_uppervp != NULLVP) return (EEXIST); + dunp = VTOUNIONFS(dvp); + udvp = dunp->un_uppervp; + + error = unionfs_set_in_progress_flag(vp, UNIONFS_COPY_IN_PROGRESS); + if (error == EJUSTRETURN) + return (0); + else if (error != 0) + return (error); lvp = unp->un_lowervp; uvp = NULLVP; @@ -897,11 +935,29 @@ unionfs_mkshadowdir(struct unionfs_mount *ump, struct vnode *udvp, NDPREINIT(&nd); if ((error = VOP_GETATTR(lvp, &lva, cnp->cn_cred))) - goto unionfs_mkshadowdir_abort; + goto unionfs_mkshadowdir_finish; + vref(udvp); + VOP_UNLOCK(vp); if ((error = unionfs_relookup(udvp, &uvp, cnp, &nd.ni_cnd, td, - cnp->cn_nameptr, cnp->cn_namelen, CREATE))) - goto unionfs_mkshadowdir_abort; + cnp->cn_nameptr, cnp->cn_namelen, CREATE))) { + /* + * When handling error cases here, we drop udvp's lock and + * then jump to exit code that relocks dvp, which in most + * cases will effectively relock udvp. However, this is + * not guaranteed to be the case, as various calls made + * here (such as unionfs_relookup() above and VOP_MKDIR() + * below) may unlock and then relock udvp, allowing dvp to + * be reclaimed in the meantime. In such a situation dvp + * will no longer share its lock with udvp. Since + * performance isn't a concern for these error cases, it + * makes more sense to reuse the common code that locks + * dvp on exit than to explicitly check for reclamation + * of dvp. + */ + vput(udvp); + goto unionfs_mkshadowdir_relock; + } if (uvp != NULLVP) { if (udvp == uvp) vrele(uvp); @@ -909,11 +965,14 @@ unionfs_mkshadowdir(struct unionfs_mount *ump, struct vnode *udvp, vput(uvp); error = EEXIST; - goto unionfs_mkshadowdir_abort; + vput(udvp); + goto unionfs_mkshadowdir_relock; } - if ((error = vn_start_write(udvp, &mp, V_WAIT | V_PCATCH))) - goto unionfs_mkshadowdir_abort; + if ((error = vn_start_write(udvp, &mp, V_WAIT | V_PCATCH))) { + vput(udvp); + goto unionfs_mkshadowdir_relock; + } unionfs_create_uppervattr_core(ump, &lva, &va, td); /* @@ -924,7 +983,7 @@ unionfs_mkshadowdir(struct unionfs_mount *ump, struct vnode *udvp, * component. This *should* be fine, as cn_namelen will still * correctly indicate the length of only the current component, * but ZFS in particular does not respect cn_namelen in its VOP_MKDIR - * implementation + * implementation. * Note that this assumes nd.ni_cnd.cn_pnbuf was allocated by * something like a local namei() operation and the temporary * NUL-termination will not have an effect on other threads. @@ -934,27 +993,58 @@ unionfs_mkshadowdir(struct unionfs_mount *ump, struct vnode *udvp, *pathend = '\0'; error = VOP_MKDIR(udvp, &uvp, &nd.ni_cnd, &va); *pathend = pathterm; - - if (!error) { - /* - * XXX The bug which cannot set uid/gid was corrected. - * Ignore errors. - */ - va.va_type = VNON; - VOP_SETATTR(uvp, &va, nd.ni_cnd.cn_cred); - + if (error != 0) { /* - * VOP_SETATTR() may transiently drop uvp's lock, so it's - * important to call it before unionfs_node_update() transfers - * the unionfs vnode's lock from lvp to uvp; otherwise the - * unionfs vnode itself would be transiently unlocked and - * potentially doomed. + * See the comment after unionfs_relookup() above for an + * explanation of why we unlock udvp here only to relock + * dvp on exit. */ - unionfs_node_update(unp, uvp, td); + vput(udvp); + vn_finished_write(mp); + goto unionfs_mkshadowdir_relock; } + + /* + * XXX The bug which cannot set uid/gid was corrected. + * Ignore errors. + */ + va.va_type = VNON; + /* + * VOP_SETATTR() may transiently drop uvp's lock, so it's + * important to call it before unionfs_node_update() transfers + * the unionfs vnode's lock from lvp to uvp; otherwise the + * unionfs vnode itself would be transiently unlocked and + * potentially doomed. + */ + VOP_SETATTR(uvp, &va, nd.ni_cnd.cn_cred); + + /* + * uvp may become doomed during VOP_VPUT_PAIR() if the implementation + * must temporarily drop uvp's lock. However, since we hold a + * reference to uvp from the VOP_MKDIR() call above, this would require + * a forcible unmount of uvp's filesystem, which in turn can only + * happen if our unionfs instance is first forcibly unmounted. We'll + * therefore catch this case in the NULL check of unp below. + */ + VOP_VPUT_PAIR(udvp, &uvp, false); vn_finished_write(mp); + vn_lock_pair(vp, false, LK_EXCLUSIVE, uvp, true, LK_EXCLUSIVE); + unp = VTOUNIONFS(vp); + if (unp == NULL) { + vput(uvp); + error = ENOENT; + } else + unionfs_node_update(unp, uvp, td); + VOP_UNLOCK(vp); + +unionfs_mkshadowdir_relock: + vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY); + vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); + if (error == 0 && (VN_IS_DOOMED(dvp) || VN_IS_DOOMED(vp))) + error = ENOENT; -unionfs_mkshadowdir_abort: +unionfs_mkshadowdir_finish: + unionfs_clear_in_progress_flag(vp, UNIONFS_COPY_IN_PROGRESS); cnp->cn_cred = credbk; chgproccnt(cred->cr_ruidinfo, -1, 0); crfree(cred); @@ -1116,23 +1206,31 @@ unionfs_forward_vop_finish_pair( /* * Create a new whiteout. * - * udvp and dvp should be locked on entry and will be locked on return. + * dvp and vp are unionfs vnodes representing a parent directory and + * child file, should be locked on entry, and will be locked on return. */ int -unionfs_mkwhiteout(struct vnode *dvp, struct vnode *udvp, +unionfs_mkwhiteout(struct vnode *dvp, struct vnode *vp, struct componentname *cnp, struct thread *td, char *path, int pathlen) { + struct vnode *udvp; struct vnode *wvp; struct nameidata nd; struct mount *mp; int error; - int lkflags; + bool dvp_locked; + + ASSERT_VOP_ELOCKED(dvp, __func__); + ASSERT_VOP_ELOCKED(vp, __func__); + udvp = VTOUNIONFS(dvp)->un_uppervp; wvp = NULLVP; NDPREINIT(&nd); + vref(udvp); + VOP_UNLOCK(vp); if ((error = unionfs_relookup(udvp, &wvp, cnp, &nd.ni_cnd, td, path, pathlen, CREATE))) { - return (error); + goto unionfs_mkwhiteout_cleanup; } if (wvp != NULLVP) { if (udvp == wvp) @@ -1140,18 +1238,27 @@ unionfs_mkwhiteout(struct vnode *dvp, struct vnode *udvp, else vput(wvp); - return (EEXIST); + if (nd.ni_cnd.cn_flags & ISWHITEOUT) + error = 0; + else + error = EEXIST; + goto unionfs_mkwhiteout_cleanup; } if ((error = vn_start_write(udvp, &mp, V_WAIT | V_PCATCH))) - goto unionfs_mkwhiteout_free_out; - unionfs_forward_vop_start(udvp, &lkflags); + goto unionfs_mkwhiteout_cleanup; error = VOP_WHITEOUT(udvp, &nd.ni_cnd, CREATE); - unionfs_forward_vop_finish(dvp, udvp, lkflags); - vn_finished_write(mp); -unionfs_mkwhiteout_free_out: +unionfs_mkwhiteout_cleanup: + if (VTOUNIONFS(dvp) == NULL) { + vput(udvp); + dvp_locked = false; + } else { + vrele(udvp); + dvp_locked = true; + } + vn_lock_pair(dvp, dvp_locked, LK_EXCLUSIVE, vp, false, LK_EXCLUSIVE); return (error); } @@ -1165,10 +1272,11 @@ unionfs_mkwhiteout(struct vnode *dvp, struct vnode *udvp, */ static int unionfs_vn_create_on_upper(struct vnode **vpp, struct vnode *udvp, - struct unionfs_node *unp, struct vattr *uvap, struct thread *td) + struct vnode *vp, struct vattr *uvap, struct thread *td) { struct unionfs_mount *ump; - struct vnode *vp; + struct unionfs_node *unp; + struct vnode *uvp; struct vnode *lvp; struct ucred *cred; struct vattr lva; @@ -1176,8 +1284,10 @@ unionfs_vn_create_on_upper(struct vnode **vpp, struct vnode *udvp, int fmode; int error; + ASSERT_VOP_ELOCKED(vp, __func__); + unp = VTOUNIONFS(vp); ump = MOUNTTOUNIONFSMOUNT(UNIONFSTOV(unp)->v_mount); - vp = NULLVP; + uvp = NULLVP; lvp = unp->un_lowervp; cred = td->td_ucred; fmode = FFLAGS(O_WRONLY | O_CREAT | O_TRUNC | O_EXCL); @@ -1200,42 +1310,39 @@ unionfs_vn_create_on_upper(struct vnode **vpp, struct vnode *udvp, NDPREINIT(&nd); vref(udvp); - if ((error = vfs_relookup(udvp, &vp, &nd.ni_cnd, false)) != 0) - goto unionfs_vn_create_on_upper_free_out2; - vrele(udvp); + VOP_UNLOCK(vp); + if ((error = vfs_relookup(udvp, &uvp, &nd.ni_cnd, false)) != 0) { + vrele(udvp); + return (error); + } - if (vp != NULLVP) { - if (vp == udvp) - vrele(vp); + if (uvp != NULLVP) { + if (uvp == udvp) + vrele(uvp); else - vput(vp); + vput(uvp); error = EEXIST; - goto unionfs_vn_create_on_upper_free_out1; + goto unionfs_vn_create_on_upper_cleanup; } - if ((error = VOP_CREATE(udvp, &vp, &nd.ni_cnd, uvap)) != 0) - goto unionfs_vn_create_on_upper_free_out1; + if ((error = VOP_CREATE(udvp, &uvp, &nd.ni_cnd, uvap)) != 0) + goto unionfs_vn_create_on_upper_cleanup; - if ((error = VOP_OPEN(vp, fmode, cred, td, NULL)) != 0) { - vput(vp); - goto unionfs_vn_create_on_upper_free_out1; + if ((error = VOP_OPEN(uvp, fmode, cred, td, NULL)) != 0) { + vput(uvp); + goto unionfs_vn_create_on_upper_cleanup; } - error = VOP_ADD_WRITECOUNT(vp, 1); - CTR3(KTR_VFS, "%s: vp %p v_writecount increased to %d", - __func__, vp, vp->v_writecount); + error = VOP_ADD_WRITECOUNT(uvp, 1); + CTR3(KTR_VFS, "%s: newvp %p v_writecount increased to %d", + __func__, newvp, newvp->v_writecount); if (error == 0) { - *vpp = vp; + *vpp = uvp; } else { - VOP_CLOSE(vp, fmode, cred, td); + VOP_CLOSE(uvp, fmode, cred, td); } -unionfs_vn_create_on_upper_free_out1: - VOP_UNLOCK(udvp); - -unionfs_vn_create_on_upper_free_out2: - KASSERT(nd.ni_cnd.cn_pnbuf == unp->un_path, - ("%s: cn_pnbuf changed", __func__)); - +unionfs_vn_create_on_upper_cleanup: + vput(udvp); return (error); } @@ -1310,13 +1417,18 @@ unionfs_copyfile_core(struct vnode *lvp, struct vnode *uvp, * * If you need copy of the contents, set 1 to docopy. Otherwise, set 0 to * docopy. + * + * vp is a unionfs vnode that should be locked on entry and will be + * locked on return. * * If no error returned, unp will be updated. */ int -unionfs_copyfile(struct unionfs_node *unp, int docopy, struct ucred *cred, +unionfs_copyfile(struct vnode *vp, int docopy, struct ucred *cred, struct thread *td) { + struct unionfs_node *unp; + struct unionfs_node *dunp; struct mount *mp; struct vnode *udvp; struct vnode *lvp; @@ -1324,6 +1436,8 @@ unionfs_copyfile(struct unionfs_node *unp, int docopy, struct ucred *cred, struct vattr uva; int error; + ASSERT_VOP_ELOCKED(vp, __func__); + unp = VTOUNIONFS(vp); lvp = unp->un_lowervp; uvp = NULLVP; @@ -1333,22 +1447,51 @@ unionfs_copyfile(struct unionfs_node *unp, int docopy, struct ucred *cred, return (EINVAL); if (unp->un_uppervp != NULLVP) return (EEXIST); - udvp = VTOUNIONFS(unp->un_dvp)->un_uppervp; + + udvp = NULLVP; + VI_LOCK(unp->un_dvp); + dunp = VTOUNIONFS(unp->un_dvp); + if (dunp != NULL) + udvp = dunp->un_uppervp; + VI_UNLOCK(unp->un_dvp); + if (udvp == NULLVP) return (EROFS); if ((udvp->v_mount->mnt_flag & MNT_RDONLY)) return (EROFS); + ASSERT_VOP_UNLOCKED(udvp, __func__); + + error = unionfs_set_in_progress_flag(vp, UNIONFS_COPY_IN_PROGRESS); + if (error == EJUSTRETURN) + return (0); + else if (error != 0) + return (error); error = VOP_ACCESS(lvp, VREAD, cred, td); if (error != 0) - return (error); + goto unionfs_copyfile_cleanup; if ((error = vn_start_write(udvp, &mp, V_WAIT | V_PCATCH)) != 0) - return (error); - error = unionfs_vn_create_on_upper(&uvp, udvp, unp, &uva, td); + goto unionfs_copyfile_cleanup; + error = unionfs_vn_create_on_upper(&uvp, udvp, vp, &uva, td); if (error != 0) { vn_finished_write(mp); - return (error); + vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); + goto unionfs_copyfile_cleanup; + } + + /* + * Note that it's still possible for e.g. VOP_WRITE to relock + * uvp below while holding vp[=lvp] locked. Replacing + * unionfs_copyfile_core with vn_generic_copy_file_range() will + * allow us to avoid the problem by moving this vn_lock_pair() + * call much later. + */ + vn_lock_pair(vp, false, LK_EXCLUSIVE, uvp, true, LK_EXCLUSIVE); + unp = VTOUNIONFS(vp); + if (unp == NULL) { + error = ENOENT; + goto unionfs_copyfile_cleanup; } if (docopy != 0) { @@ -1369,18 +1512,30 @@ unionfs_copyfile(struct unionfs_node *unp, int docopy, struct ucred *cred, /* Reset the attributes. Ignore errors. */ uva.va_type = VNON; VOP_SETATTR(uvp, &uva, cred); + unionfs_node_update(unp, uvp, td); } - unionfs_node_update(unp, uvp, td); - +unionfs_copyfile_cleanup: + unionfs_clear_in_progress_flag(vp, UNIONFS_COPY_IN_PROGRESS); return (error); } /* - * It checks whether vp can rmdir. (check empty) + * Determine if the unionfs view of a directory is empty such that + * an rmdir operation can be permitted. + * + * We assume the VOP_RMDIR() against the upper layer vnode will take + * care of this check for us where the upper FS is concerned, so here + * we concentrate on the lower FS. We need to check for the presence + * of files other than "." and ".." in the lower FS directory and + * then cross-check any files we find against the upper FS to see if + * a whiteout is present (in which case we treat the lower file as + * non-present). * - * vp is unionfs vnode. - * vp should be locked. + * The logic here is based heavily on vn_dir_check_empty(). + * + * vp should be a locked unionfs node, and vp's lowervp should also be + * locked. */ int unionfs_check_rmdir(struct vnode *vp, struct ucred *cred, struct thread *td) @@ -1388,115 +1543,127 @@ unionfs_check_rmdir(struct vnode *vp, struct ucred *cred, struct thread *td) struct vnode *uvp; struct vnode *lvp; struct vnode *tvp; + char *dirbuf; + size_t dirbuflen, len; + off_t off; struct dirent *dp; - struct dirent *edp; struct componentname cn; - struct iovec iov; - struct uio uio; struct vattr va; int error; int eofflag; - int lookuperr; - - /* - * The size of buf needs to be larger than DIRBLKSIZ. - */ - char buf[256 * 6]; - - ASSERT_VOP_ELOCKED(vp, __func__); eofflag = 0; - uvp = UNIONFSVPTOUPPERVP(vp); lvp = UNIONFSVPTOLOWERVP(vp); + uvp = UNIONFSVPTOUPPERVP(vp); + + /* + * Note that the locking here still isn't ideal: We expect the caller + * to hold both the upper and lower layer locks as well as the upper + * parent directory lock, which it can do in a manner that avoids + * deadlock. However, if the cross-check logic below needs to call + * VOP_LOOKUP(), that may relock the upper vnode and lock any found + * child vnode in a way that doesn't protect against deadlock given + * the other held locks. Beyond that, the various other VOPs we issue + * below, such as VOP_OPEN() and VOP_READDIR(), may also re-lock the + * lower vnode. + * We might instead just handoff between the upper vnode lock + * (and its parent directory lock) and the lower vnode lock as needed, + * so that the lower lock is never held at the same time as the upper + * locks, but that opens up a wider window in which the upper + * directory (and also the lower directory if it isn't truly + * read-only) may change while the relevant lock is dropped. But + * since re-locking may happen here and open up such a window anyway, + * perhaps that is a worthwile tradeoff? Or perhaps we can ultimately + * do sufficient tracking of empty state within the unionfs vnode + * (in conjunction with upcalls from the lower FSes to notify us + * of out-of-band state changes) that we can avoid these costly checks + * altogether. + */ + ASSERT_VOP_LOCKED(lvp, __func__); + ASSERT_VOP_ELOCKED(uvp, __func__); - /* check opaque */ if ((error = VOP_GETATTR(uvp, &va, cred)) != 0) return (error); if (va.va_flags & OPAQUE) return (0); - /* open vnode */ #ifdef MAC - if ((error = mac_vnode_check_open(cred, vp, VEXEC|VREAD)) != 0) + if ((error = mac_vnode_check_open(cred, lvp, VEXEC | VREAD)) != 0) return (error); #endif - if ((error = VOP_ACCESS(vp, VEXEC|VREAD, cred, td)) != 0) + if ((error = VOP_ACCESS(lvp, VEXEC | VREAD, cred, td)) != 0) return (error); - if ((error = VOP_OPEN(vp, FREAD, cred, td, NULL)) != 0) + if ((error = VOP_OPEN(lvp, FREAD, cred, td, NULL)) != 0) + return (error); + if ((error = VOP_GETATTR(lvp, &va, cred)) != 0) return (error); - uio.uio_rw = UIO_READ; - uio.uio_segflg = UIO_SYSSPACE; - uio.uio_td = td; - uio.uio_offset = 0; + dirbuflen = max(DEV_BSIZE, GENERIC_MAXDIRSIZ); + if (dirbuflen < va.va_blocksize) + dirbuflen = va.va_blocksize; + dirbuf = malloc(dirbuflen, M_TEMP, M_WAITOK); -#ifdef MAC - error = mac_vnode_check_readdir(td->td_ucred, lvp); -#endif - while (!error && !eofflag) { - iov.iov_base = buf; - iov.iov_len = sizeof(buf); - uio.uio_iov = &iov; - uio.uio_iovcnt = 1; - uio.uio_resid = iov.iov_len; + len = 0; + off = 0; + eofflag = 0; - error = VOP_READDIR(lvp, &uio, cred, &eofflag, NULL, NULL); + for (;;) { + error = vn_dir_next_dirent(lvp, td, dirbuf, dirbuflen, + &dp, &len, &off, &eofflag); if (error != 0) break; - KASSERT(eofflag != 0 || uio.uio_resid < sizeof(buf), - ("%s: empty read from lower FS", __func__)); - - edp = (struct dirent*)&buf[sizeof(buf) - uio.uio_resid]; - for (dp = (struct dirent*)buf; !error && dp < edp; - dp = (struct dirent*)((caddr_t)dp + dp->d_reclen)) { - if (dp->d_type == DT_WHT || dp->d_fileno == 0 || - (dp->d_namlen == 1 && dp->d_name[0] == '.') || - (dp->d_namlen == 2 && !bcmp(dp->d_name, "..", 2))) - continue; - - cn.cn_namelen = dp->d_namlen; - cn.cn_pnbuf = NULL; - cn.cn_nameptr = dp->d_name; - cn.cn_nameiop = LOOKUP; - cn.cn_flags = LOCKPARENT | LOCKLEAF | RDONLY | ISLASTCN; - cn.cn_lkflags = LK_EXCLUSIVE; - cn.cn_cred = cred; - - /* - * check entry in lower. - * Sometimes, readdir function returns - * wrong entry. - */ - lookuperr = VOP_LOOKUP(lvp, &tvp, &cn); - - if (!lookuperr) - vput(tvp); - else - continue; /* skip entry */ - /* - * check entry - * If it has no exist/whiteout entry in upper, - * directory is not empty. - */ - cn.cn_flags = LOCKPARENT | LOCKLEAF | RDONLY | ISLASTCN; - lookuperr = VOP_LOOKUP(uvp, &tvp, &cn); + if (len == 0) { + /* EOF */ + error = 0; + break; + } - if (!lookuperr) - vput(tvp); + if (dp->d_type == DT_WHT) + continue; - /* ignore exist or whiteout entry */ - if (!lookuperr || - (lookuperr == ENOENT && (cn.cn_flags & ISWHITEOUT))) - continue; + /* + * Any file in the directory which is not '.' or '..' indicates + * the directory is not empty. + */ + switch (dp->d_namlen) { + case 2: + if (dp->d_name[1] != '.') { + /* Can't be '..' (nor '.') */ + break; + } + /* FALLTHROUGH */ + case 1: + if (dp->d_name[0] != '.') { + /* Can't be '..' nor '.' */ + break; + } + continue; + default: + break; + } + cn.cn_namelen = dp->d_namlen; + cn.cn_pnbuf = NULL; + cn.cn_nameptr = dp->d_name; + cn.cn_nameiop = LOOKUP; + cn.cn_flags = LOCKPARENT | LOCKLEAF | RDONLY | ISLASTCN; + cn.cn_lkflags = LK_EXCLUSIVE; + cn.cn_cred = cred; + + error = VOP_LOOKUP(uvp, &tvp, &cn); + if (tvp != NULLVP) + vput(tvp); + if (error != 0 && error != ENOENT && error != EJUSTRETURN) + break; + else if ((cn.cn_flags & ISWHITEOUT) == 0) { error = ENOTEMPTY; - } + break; + } else + error = 0; } - /* close vnode */ - VOP_CLOSE(vp, FREAD, cred, td); - + VOP_CLOSE(lvp, FREAD, cred, td); + free(dirbuf, M_TEMP); return (error); } - diff --git a/sys/fs/unionfs/union_vfsops.c b/sys/fs/unionfs/union_vfsops.c index cb55c2dd6474..9342317ad08e 100644 --- a/sys/fs/unionfs/union_vfsops.c +++ b/sys/fs/unionfs/union_vfsops.c @@ -327,18 +327,15 @@ unionfs_domount(struct mount *mp) * unionfs_lock()) and the mountpoint's busy count. Without this, * unmount will lock the covered vnode lock (directly through the * covered vnode) and wait for the busy count to drain, while a - * concurrent lookup will increment the busy count and then lock + * concurrent lookup will increment the busy count and then may lock * the covered vnode lock (indirectly through unionfs_lock()). * - * Note that we can't yet use this facility for the 'below' case - * in which the upper vnode is the covered vnode, because that would - * introduce a different LOR in which the cross-mount lookup would - * effectively hold the upper vnode lock before acquiring the lower - * vnode lock, while an unrelated lock operation would still acquire - * the lower vnode lock before the upper vnode lock, which is the - * order unionfs currently requires. + * Note that this is only needed for the 'below' case in which the + * upper vnode is also the covered vnode, because unionfs_lock() + * only locks the upper vnode as long as both lower and upper vnodes + * are present (which they will always be for the unionfs mount root). */ - if (!below) { + if (below) { vn_lock(mp->mnt_vnodecovered, LK_EXCLUSIVE | LK_RETRY | LK_CANRECURSE); mp->mnt_vnodecovered->v_vflag |= VV_CROSSLOCK; VOP_UNLOCK(mp->mnt_vnodecovered); diff --git a/sys/fs/unionfs/union_vnops.c b/sys/fs/unionfs/union_vnops.c index aa2a7273825a..7a64baebd3f3 100644 --- a/sys/fs/unionfs/union_vnops.c +++ b/sys/fs/unionfs/union_vnops.c @@ -85,12 +85,11 @@ unionfs_lookup(struct vop_cachedlookup_args *ap) struct componentname *cnp; struct thread *td; u_long nameiop; - u_long cnflags, cnflagsbk; - int iswhiteout; + u_long cnflags; int lockflag; - int error , uerror, lerror; + int lkflags; + int error, uerror, lerror; - iswhiteout = 0; lockflag = 0; error = uerror = lerror = ENOENT; cnp = ap->a_cnp; @@ -119,88 +118,182 @@ unionfs_lookup(struct vop_cachedlookup_args *ap) LOOKUP != nameiop) return (EROFS); + /* + * Note that a lookup is in-flight, and block if another lookup + * is already in-flight against dvp. This is done because we may + * end up dropping dvp's lock to look up a lower vnode or to create + * a shadow directory, opening up the possibility of parallel lookups + * against the same directory creating duplicate unionfs vnodes for + * the same file(s). Note that if this function encounters an + * in-progress lookup for the directory, it will block until the + * lookup is complete and then return ERELOOKUP to allow any + * existing unionfs vnode to be loaded from the VFS cache. + * This is really a hack; filesystems that support MNTK_LOOKUP_SHARED + * (which unionfs currently doesn't) seem to deal with this by using + * the vfs_hash_* functions to manage a per-mount vnode cache keyed + * by the inode number (or some roughly equivalent unique ID + * usually assocated with the storage medium). It may make sense + * for unionfs to adopt something similar as a replacement for its + * current half-baked directory-only cache implementation, particularly + * if we want to support MNTK_LOOKUP_SHARED here. + */ + error = unionfs_set_in_progress_flag(dvp, UNIONFS_LOOKUP_IN_PROGRESS); + if (error != 0) + return (error); /* * lookup dotdot */ if (cnflags & ISDOTDOT) { - if (LOOKUP != nameiop && udvp == NULLVP) - return (EROFS); + if (LOOKUP != nameiop && udvp == NULLVP) { + error = EROFS; + goto unionfs_lookup_return; + } - if (udvp != NULLVP) { + if (udvp != NULLVP) dtmpvp = udvp; - if (ldvp != NULLVP) - VOP_UNLOCK(ldvp); - } else dtmpvp = ldvp; + unionfs_forward_vop_start(dtmpvp, &lkflags); error = VOP_LOOKUP(dtmpvp, &vp, cnp); + unionfs_forward_vop_finish(dvp, dtmpvp, lkflags); - if (dtmpvp == udvp && ldvp != NULLVP) { - VOP_UNLOCK(udvp); - vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY); - dunp = VTOUNIONFS(dvp); - if (error == 0 && dunp == NULL) - error = ENOENT; - } + /* + * Drop the lock and reference on vp. If the lookup was + * successful, we'll either need to exchange vp's lock and + * reference for the unionfs parent vnode's lock and + * reference, or (if dvp was reclaimed) we'll need to drop + * vp's lock and reference to return early. + */ + if (vp != NULLVP) + vput(vp); + dunp = VTOUNIONFS(dvp); + if (error == 0 && dunp == NULL) + error = ENOENT; if (error == 0) { - /* - * Exchange lock and reference from vp to - * dunp->un_dvp. vp is upper/lower vnode, but it - * will need to return the unionfs vnode. - */ - if (nameiop == DELETE || nameiop == RENAME || - (cnp->cn_lkflags & LK_TYPE_MASK)) - VOP_UNLOCK(vp); - vrele(vp); - dtmpvp = dunp->un_dvp; vref(dtmpvp); VOP_UNLOCK(dvp); *(ap->a_vpp) = dtmpvp; - if (nameiop == DELETE || nameiop == RENAME) - vn_lock(dtmpvp, LK_EXCLUSIVE | LK_RETRY); - else if (cnp->cn_lkflags & LK_TYPE_MASK) - vn_lock(dtmpvp, cnp->cn_lkflags | - LK_RETRY); + vn_lock(dtmpvp, cnp->cn_lkflags | LK_RETRY); + if (VN_IS_DOOMED(dtmpvp)) { + vput(dtmpvp); + *(ap->a_vpp) = NULLVP; + error = ENOENT; + } vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY); - } else if (error == ENOENT && (cnflags & MAKEENTRY) != 0) + } + if (error == ENOENT && (cnflags & MAKEENTRY) != 0) cache_enter(dvp, NULLVP, cnp); goto unionfs_lookup_return; } + /* + * Lookup lower layer. We do this before looking up the the upper + * layer, as we may drop the upper parent directory's lock, and we + * want to ensure the upper parent remains locked from the point of + * lookup through any ensuing VOP that may require it to be locked. + * The cost of this is that we may end up performing an unnecessary + * lower layer lookup if a whiteout is present in the upper layer. + */ + if (ldvp != NULLVP && !(cnflags & DOWHITEOUT)) { + struct componentname lcn; + bool is_dot; + + if (udvp != NULLVP) { + vref(ldvp); + VOP_UNLOCK(dvp); + vn_lock(ldvp, LK_EXCLUSIVE | LK_RETRY); + } + + lcn = *cnp; + /* always op is LOOKUP */ + lcn.cn_nameiop = LOOKUP; + lcn.cn_flags = cnflags; + is_dot = false; + + if (udvp == NULLVP) + unionfs_forward_vop_start(ldvp, &lkflags); + lerror = VOP_LOOKUP(ldvp, &lvp, &lcn); + if (udvp == NULLVP && + unionfs_forward_vop_finish(dvp, ldvp, lkflags)) { + if (lvp != NULLVP) + VOP_UNLOCK(lvp); + error = ENOENT; + goto unionfs_lookup_cleanup; + } + + if (udvp == NULLVP) + cnp->cn_flags = lcn.cn_flags; + + if (lerror == 0) { + if (ldvp == lvp) { /* is dot */ + vrele(lvp); + *(ap->a_vpp) = dvp; + vref(dvp); + is_dot = true; + error = lerror; + } else if (lvp != NULLVP) + VOP_UNLOCK(lvp); + } + + if (udvp != NULLVP) { + vput(ldvp); + vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY); + if (VN_IS_DOOMED(dvp)) + error = ENOENT; + } + if (is_dot) + goto unionfs_lookup_return; + else if (error != 0) + goto unionfs_lookup_cleanup; + } /* * lookup upper layer */ if (udvp != NULLVP) { + bool iswhiteout = false; + + unionfs_forward_vop_start(udvp, &lkflags); uerror = VOP_LOOKUP(udvp, &uvp, cnp); + if (unionfs_forward_vop_finish(dvp, udvp, lkflags)) { + if (uvp != NULLVP) + VOP_UNLOCK(uvp); + error = ENOENT; + goto unionfs_lookup_cleanup; + } if (uerror == 0) { if (udvp == uvp) { /* is dot */ + if (lvp != NULLVP) + vrele(lvp); vrele(uvp); *(ap->a_vpp) = dvp; vref(dvp); error = uerror; goto unionfs_lookup_return; - } - if (nameiop == DELETE || nameiop == RENAME || - (cnp->cn_lkflags & LK_TYPE_MASK)) + } else if (uvp != NULLVP) VOP_UNLOCK(uvp); } /* check whiteout */ - if (uerror == ENOENT || uerror == EJUSTRETURN) - if (cnp->cn_flags & ISWHITEOUT) - iswhiteout = 1; /* don't lookup lower */ - if (iswhiteout == 0 && ldvp != NULLVP) - if (!VOP_GETATTR(udvp, &va, cnp->cn_cred) && - (va.va_flags & OPAQUE)) - iswhiteout = 1; /* don't lookup lower */ + if ((uerror == ENOENT || uerror == EJUSTRETURN) && + (cnp->cn_flags & ISWHITEOUT)) + iswhiteout = true; + else if (VOP_GETATTR(udvp, &va, cnp->cn_cred) == 0 && + (va.va_flags & OPAQUE)) + iswhiteout = true; + + if (iswhiteout && lvp != NULLVP) { + vrele(lvp); + lvp = NULLVP; + } + #if 0 UNIONFS_INTERNAL_DEBUG( "unionfs_lookup: debug: whiteout=%d, path=%s\n", @@ -208,39 +301,6 @@ unionfs_lookup(struct vop_cachedlookup_args *ap) #endif } - /* - * lookup lower layer - */ - if (ldvp != NULLVP && !(cnflags & DOWHITEOUT) && iswhiteout == 0) { - /* always op is LOOKUP */ - cnp->cn_nameiop = LOOKUP; - cnflagsbk = cnp->cn_flags; - cnp->cn_flags = cnflags; - - lerror = VOP_LOOKUP(ldvp, &lvp, cnp); - - cnp->cn_nameiop = nameiop; - if (udvp != NULLVP && (uerror == 0 || uerror == EJUSTRETURN)) - cnp->cn_flags = cnflagsbk; - - if (lerror == 0) { - if (ldvp == lvp) { /* is dot */ - if (uvp != NULLVP) - vrele(uvp); /* no need? */ - vrele(lvp); - *(ap->a_vpp) = dvp; - vref(dvp); - - UNIONFS_INTERNAL_DEBUG( - "unionfs_lookup: leave (%d)\n", lerror); - - return (lerror); - } - if (cnp->cn_lkflags & LK_TYPE_MASK) - VOP_UNLOCK(lvp); - } - } - /* * check lookup result */ @@ -280,8 +340,7 @@ unionfs_lookup(struct vop_cachedlookup_args *ap) if (unp == NULL) error = ENOENT; else - error = unionfs_mkshadowdir(MOUNTTOUNIONFSMOUNT(dvp->v_mount), - udvp, unp, cnp, td); + error = unionfs_mkshadowdir(dvp, vp, cnp, td); if (lockflag != 0) VOP_UNLOCK(vp); if (error != 0) { @@ -293,6 +352,10 @@ unionfs_lookup(struct vop_cachedlookup_args *ap) vrele(vp); goto unionfs_lookup_cleanup; } + /* + * TODO: Since unionfs_mkshadowdir() relocks udvp after + * creating the new directory, return ERELOOKUP here? + */ if ((cnp->cn_lkflags & LK_TYPE_MASK) == LK_SHARED) vn_lock(vp, LK_SHARED | LK_RETRY); } @@ -313,9 +376,12 @@ unionfs_lookup(struct vop_cachedlookup_args *ap) "unionfs_lookup: Unable to create unionfs vnode."); goto unionfs_lookup_cleanup; } - if ((nameiop == DELETE || nameiop == RENAME) && - (cnp->cn_lkflags & LK_TYPE_MASK) == 0) - vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); + } + + if (VN_IS_DOOMED(dvp) || VN_IS_DOOMED(vp)) { + error = ENOENT; + vput(vp); + goto unionfs_lookup_cleanup; } *(ap->a_vpp) = vp; @@ -329,10 +395,12 @@ unionfs_lookup(struct vop_cachedlookup_args *ap) if (lvp != NULLVP) vrele(lvp); - if (error == ENOENT && (cnflags & MAKEENTRY) != 0) + if (error == ENOENT && (cnflags & MAKEENTRY) != 0 && + !VN_IS_DOOMED(dvp)) cache_enter(dvp, NULLVP, cnp); unionfs_lookup_return: + unionfs_clear_in_progress_flag(dvp, UNIONFS_LOOKUP_IN_PROGRESS); UNIONFS_INTERNAL_DEBUG("unionfs_lookup: leave (%d)\n", error); @@ -492,6 +560,61 @@ unionfs_downgrade_lock(struct vnode *vp, enum unionfs_lkupgrade status) vn_lock(vp, LK_DOWNGRADE | LK_RETRY); } +/* + * Exchange the default (upper vnode) lock on a unionfs vnode for the lower + * vnode lock, in support of operations that require access to the lower vnode + * even when an upper vnode is present. We don't use vn_lock_pair() to hold + * both vnodes at the same time, primarily because the caller may proceed + * to issue VOPs to the lower layer which re-lock or perform other operations + * which may not be safe in the presence of a locked vnode from another FS. + * Moreover, vn_lock_pair()'s deadlock resolution approach can introduce + * additional overhead that isn't necessary on these paths. + * + * vp must be a locked unionfs vnode; the lock state of this vnode is + * returned through *lkflags for later use in unionfs_unlock_lvp(). + * + * Returns the locked lower vnode, or NULL if the lower vnode (and therefore + * also the unionfs vnode above it) has been doomed. + */ +static struct vnode * +unionfs_lock_lvp(struct vnode *vp, int *lkflags) +{ + struct unionfs_node *unp; + struct vnode *lvp; + + unp = VTOUNIONFS(vp); + lvp = unp->un_lowervp; + ASSERT_VOP_LOCKED(vp, __func__); + ASSERT_VOP_UNLOCKED(lvp, __func__); + *lkflags = VOP_ISLOCKED(vp); + vref(lvp); + VOP_UNLOCK(vp); + vn_lock(lvp, *lkflags | LK_RETRY); + if (VN_IS_DOOMED(lvp)) { + vput(lvp); + lvp = NULLVP; + vn_lock(vp, *lkflags | LK_RETRY); + } + return (lvp); +} + +/* + * Undo a previous call to unionfs_lock_lvp(), restoring the default lock + * on the unionfs vnode. This function reloads and returns the vnode + * private data for the unionfs vnode, which will be NULL if the unionfs + * vnode became doomed while its lock was dropped. The caller must check + * for this case. + */ +static struct unionfs_node * +unionfs_unlock_lvp(struct vnode *vp, struct vnode *lvp, int lkflags) +{ + ASSERT_VOP_LOCKED(lvp, __func__); + ASSERT_VOP_UNLOCKED(vp, __func__); + vput(lvp); + vn_lock(vp, lkflags | LK_RETRY); + return (VTOUNIONFS(vp)); +} + static int unionfs_open(struct vop_open_args *ap) { @@ -504,7 +627,9 @@ unionfs_open(struct vop_open_args *ap) struct ucred *cred; struct thread *td; int error; + int lkflags; enum unionfs_lkupgrade lkstatus; + bool lock_lvp, open_lvp; UNIONFS_INTERNAL_DEBUG("unionfs_open: enter\n"); @@ -515,6 +640,7 @@ unionfs_open(struct vop_open_args *ap) targetvp = NULLVP; cred = ap->a_cred; td = ap->a_td; + open_lvp = lock_lvp = false; /* * The executable loader path may call this function with vp locked @@ -546,10 +672,12 @@ unionfs_open(struct vop_open_args *ap) if (targetvp == NULLVP) { if (uvp == NULLVP) { if ((ap->a_mode & FWRITE) && lvp->v_type == VREG) { - error = unionfs_copyfile(unp, + error = unionfs_copyfile(vp, !(ap->a_mode & O_TRUNC), cred, td); - if (error != 0) + if (error != 0) { + unp = VTOUNIONFS(vp); goto unionfs_open_abort; + } targetvp = uvp = unp->un_uppervp; } else targetvp = lvp; @@ -557,30 +685,69 @@ unionfs_open(struct vop_open_args *ap) targetvp = uvp; } + if (targetvp == uvp && uvp->v_type == VDIR && lvp != NULLVP && + unsp->uns_lower_opencnt <= 0) + open_lvp = true; + else if (targetvp == lvp && uvp != NULLVP) + lock_lvp = true; + + if (lock_lvp) { + unp = NULL; + lvp = unionfs_lock_lvp(vp, &lkflags); + if (lvp == NULLVP) { + error = ENOENT; + goto unionfs_open_abort; + } + } else + unionfs_forward_vop_start(targetvp, &lkflags); + error = VOP_OPEN(targetvp, ap->a_mode, cred, td, ap->a_fp); - if (error == 0) { - if (targetvp == uvp) { - if (uvp->v_type == VDIR && lvp != NULLVP && - unsp->uns_lower_opencnt <= 0) { - /* open lower for readdir */ - error = VOP_OPEN(lvp, FREAD, cred, td, NULL); - if (error != 0) { - VOP_CLOSE(uvp, ap->a_mode, cred, td); - goto unionfs_open_abort; - } - unsp->uns_node_flag |= UNS_OPENL_4_READDIR; - unsp->uns_lower_opencnt++; + + if (lock_lvp) { + unp = unionfs_unlock_lvp(vp, lvp, lkflags); + if (unp == NULL && error == 0) + error = ENOENT; + } else if (unionfs_forward_vop_finish(vp, targetvp, lkflags)) + error = error ? error : ENOENT; + + if (error != 0) + goto unionfs_open_abort; + + if (targetvp == uvp) { + if (open_lvp) { + unp = NULL; + lvp = unionfs_lock_lvp(vp, &lkflags); + if (lvp == NULLVP) { + error = ENOENT; + goto unionfs_open_abort; } - unsp->uns_upper_opencnt++; - } else { + /* open lower for readdir */ + error = VOP_OPEN(lvp, FREAD, cred, td, NULL); + unp = unionfs_unlock_lvp(vp, lvp, lkflags); + if (unp == NULL) { + error = error ? error : ENOENT; + goto unionfs_open_abort; + } + if (error != 0) { + unionfs_forward_vop_start(uvp, &lkflags); + VOP_CLOSE(uvp, ap->a_mode, cred, td); + if (unionfs_forward_vop_finish(vp, uvp, lkflags)) + unp = NULL; + goto unionfs_open_abort; + } + unsp->uns_node_flag |= UNS_OPENL_4_READDIR; unsp->uns_lower_opencnt++; - unsp->uns_lower_openmode = ap->a_mode; } - vp->v_object = targetvp->v_object; + unsp->uns_upper_opencnt++; + } else { + unsp->uns_lower_opencnt++; + unsp->uns_lower_openmode = ap->a_mode; } + vp->v_object = targetvp->v_object; unionfs_open_abort: - if (error != 0) + + if (error != 0 && unp != NULL) unionfs_tryrem_node_status(unp, unsp); unionfs_open_cleanup: @@ -599,9 +766,13 @@ unionfs_close(struct vop_close_args *ap) struct ucred *cred; struct thread *td; struct vnode *vp; + struct vnode *uvp; + struct vnode *lvp; struct vnode *ovp; int error; + int lkflags; enum unionfs_lkupgrade lkstatus; + bool lock_lvp; UNIONFS_INTERNAL_DEBUG("unionfs_close: enter\n"); @@ -611,6 +782,7 @@ unionfs_close(struct vop_close_args *ap) cred = ap->a_cred; td = ap->a_td; error = 0; + lock_lvp = false; /* * If the vnode is reclaimed while upgrading, we can't safely use unp @@ -621,44 +793,77 @@ unionfs_close(struct vop_close_args *ap) goto unionfs_close_cleanup; unp = VTOUNIONFS(vp); - unionfs_get_node_status(unp, td, &unsp); + lvp = unp->un_lowervp; + uvp = unp->un_uppervp; + unsp = unionfs_find_node_status(unp, td); - if (unsp->uns_lower_opencnt <= 0 && unsp->uns_upper_opencnt <= 0) { + if (unsp == NULL || + (unsp->uns_lower_opencnt <= 0 && unsp->uns_upper_opencnt <= 0)) { #ifdef DIAGNOSTIC - printf("unionfs_close: warning: open count is 0\n"); + if (unsp != NULL) + printf("unionfs_close: warning: open count is 0\n"); #endif - if (unp->un_uppervp != NULLVP) - ovp = unp->un_uppervp; + if (uvp != NULLVP) + ovp = uvp; else - ovp = unp->un_lowervp; + ovp = lvp; } else if (unsp->uns_upper_opencnt > 0) - ovp = unp->un_uppervp; + ovp = uvp; else - ovp = unp->un_lowervp; + ovp = lvp; + + if (ovp == lvp && uvp != NULLVP) { + lock_lvp = true; + unp = NULL; + lvp = unionfs_lock_lvp(vp, &lkflags); + if (lvp == NULLVP) { + error = ENOENT; + goto unionfs_close_abort; + } + } else + unionfs_forward_vop_start(ovp, &lkflags); error = VOP_CLOSE(ovp, ap->a_fflag, cred, td); + if (lock_lvp) { + unp = unionfs_unlock_lvp(vp, lvp, lkflags); + if (unp == NULL && error == 0) + error = ENOENT; + } else if (unionfs_forward_vop_finish(vp, ovp, lkflags)) + error = error ? error : ENOENT; + if (error != 0) goto unionfs_close_abort; vp->v_object = ovp->v_object; - if (ovp == unp->un_uppervp) { - unsp->uns_upper_opencnt--; - if (unsp->uns_upper_opencnt == 0) { + if (ovp == uvp) { + if (unsp != NULL && ((--unsp->uns_upper_opencnt) == 0)) { if (unsp->uns_node_flag & UNS_OPENL_4_READDIR) { - VOP_CLOSE(unp->un_lowervp, FREAD, cred, td); + unp = NULL; + lvp = unionfs_lock_lvp(vp, &lkflags); + if (lvp == NULLVP) { + error = ENOENT; + goto unionfs_close_abort; + } + VOP_CLOSE(lvp, FREAD, cred, td); + unp = unionfs_unlock_lvp(vp, lvp, lkflags); + if (unp == NULL) { + error = ENOENT; + goto unionfs_close_abort; + } unsp->uns_node_flag &= ~UNS_OPENL_4_READDIR; unsp->uns_lower_opencnt--; } if (unsp->uns_lower_opencnt > 0) - vp->v_object = unp->un_lowervp->v_object; + vp->v_object = lvp->v_object; } - } else + } else if (unsp != NULL) unsp->uns_lower_opencnt--; unionfs_close_abort: - unionfs_tryrem_node_status(unp, unsp); + if (unp != NULL && unsp != NULL) + unionfs_tryrem_node_status(unp, unsp); unionfs_close_cleanup: unionfs_downgrade_lock(vp, lkstatus); @@ -883,7 +1088,7 @@ unionfs_setattr(struct vop_setattr_args *ap) return (EROFS); if (uvp == NULLVP && lvp->v_type == VREG) { - error = unionfs_copyfile(unp, (vap->va_size != 0), + error = unionfs_copyfile(ap->a_vp, (vap->va_size != 0), ap->a_cred, td); if (error != 0) return (error); @@ -1078,8 +1283,10 @@ unionfs_remove(struct vop_remove_args *ap) error = VOP_REMOVE(udvp, uvp, cnp); unionfs_forward_vop_finish_pair(ap->a_dvp, udvp, udvp_lkflags, ap->a_vp, uvp, uvp_lkflags); - } else if (lvp != NULLVP) - error = unionfs_mkwhiteout(ap->a_dvp, udvp, cnp, td, path, pathlen); + } else if (lvp != NULLVP) { + error = unionfs_mkwhiteout(ap->a_dvp, ap->a_vp, cnp, td, + path, pathlen); + } UNIONFS_INTERNAL_DEBUG("unionfs_remove: leave (%d)\n", error); @@ -1096,7 +1303,6 @@ unionfs_link(struct vop_link_args *ap) struct componentname *cnp; struct thread *td; int error; - int needrelookup; UNIONFS_INTERNAL_DEBUG("unionfs_link: enter\n"); @@ -1104,7 +1310,6 @@ unionfs_link(struct vop_link_args *ap) KASSERT_UNIONFS_VNODE(ap->a_vp); error = 0; - needrelookup = 0; dunp = VTOUNIONFS(ap->a_tdvp); unp = NULL; udvp = dunp->un_uppervp; @@ -1121,16 +1326,15 @@ unionfs_link(struct vop_link_args *ap) if (ap->a_vp->v_type != VREG) return (EOPNOTSUPP); - error = unionfs_copyfile(unp, 1, cnp->cn_cred, td); - if (error != 0) - return (error); - needrelookup = 1; + VOP_UNLOCK(ap->a_tdvp); + error = unionfs_copyfile(ap->a_vp, 1, cnp->cn_cred, td); + vn_lock(ap->a_tdvp, LK_EXCLUSIVE | LK_RETRY); + if (error == 0) + error = ERELOOKUP; + return (error); } uvp = unp->un_uppervp; - if (needrelookup != 0) - error = unionfs_relookup_for_create(ap->a_tdvp, cnp, td); - if (error == 0) { int udvp_lkflags, uvp_lkflags; unionfs_forward_vop_start_pair(udvp, &udvp_lkflags, @@ -1154,8 +1358,6 @@ unionfs_rename(struct vop_rename_args *ap) struct vnode *tdvp; struct vnode *tvp; struct componentname *tcnp; - struct vnode *ltdvp; - struct vnode *ltvp; struct thread *td; /* rename target vnodes */ @@ -1164,7 +1366,6 @@ unionfs_rename(struct vop_rename_args *ap) struct vnode *rtdvp; struct vnode *rtvp; - struct unionfs_mount *ump; struct unionfs_node *unp; int error; @@ -1177,8 +1378,6 @@ unionfs_rename(struct vop_rename_args *ap) tdvp = ap->a_tdvp; tvp = ap->a_tvp; tcnp = ap->a_tcnp; - ltdvp = NULLVP; - ltvp = NULLVP; td = curthread; rfdvp = fdvp; rfvp = fvp; @@ -1238,7 +1437,6 @@ unionfs_rename(struct vop_rename_args *ap) UNIONFS_INTERNAL_DEBUG("fvp=%p, ufvp=%p, lfvp=%p\n", fvp, unp->un_uppervp, unp->un_lowervp); #endif - ump = MOUNTTOUNIONFSMOUNT(fvp->v_mount); /* * If we only have a lower vnode, copy the source file to the upper * FS so that the rename operation can be issued against the upper FS. @@ -1282,10 +1480,10 @@ unionfs_rename(struct vop_rename_args *ap) else if (unp->un_uppervp == NULLVP) { switch (fvp->v_type) { case VREG: - error = unionfs_copyfile(unp, 1, fcnp->cn_cred, td); + error = unionfs_copyfile(fvp, 1, fcnp->cn_cred, td); break; case VDIR: - error = unionfs_mkshadowdir(ump, rfdvp, unp, fcnp, td); + error = unionfs_mkshadowdir(fdvp, fvp, fcnp, td); break; default: error = ENODEV; @@ -1327,7 +1525,6 @@ unionfs_rename(struct vop_rename_args *ap) goto unionfs_rename_abort; } rtdvp = unp->un_uppervp; - ltdvp = unp->un_lowervp; vref(rtdvp); if (tvp != NULLVP) { @@ -1348,7 +1545,6 @@ unionfs_rename(struct vop_rename_args *ap) goto unionfs_rename_abort; } rtvp = unp->un_uppervp; - ltvp = unp->un_lowervp; vref(rtvp); } } @@ -1365,12 +1561,8 @@ unionfs_rename(struct vop_rename_args *ap) cache_purge(fdvp); } - if (ltdvp != NULLVP) - VOP_UNLOCK(ltdvp); if (tdvp != rtdvp) vrele(tdvp); - if (ltvp != NULLVP) - VOP_UNLOCK(ltvp); if (tvp != rtvp && tvp != NULLVP) { if (rtvp == NULLVP) vput(tvp); @@ -1504,43 +1696,55 @@ unionfs_rmdir(struct vop_rmdir_args *ap) if (uvp != NULLVP) { if (lvp != NULLVP) { + /* + * We need to keep dvp and vp's upper vnodes locked + * going into the VOP_RMDIR() call, but the empty + * directory check also requires the lower vnode lock. + * For this third, cross-filesystem lock we use a + * similar approach taken by various FS' VOP_RENAME + * implementations (which require 2-4 vnode locks). + * First we attempt a NOWAIT acquisition, then if + * that fails we drops the other two vnode locks, + * acquire lvp's lock in the normal fashion to reduce + * the likelihood of spinning on it in the future, + * then drop, reacquire the other locks, and return + * ERELOOKUP to re-drive the lookup in case the dvp-> + * vp relationship has changed. + */ + if (vn_lock(lvp, LK_SHARED | LK_NOWAIT) != 0) { + VOP_UNLOCK(ap->a_vp); + VOP_UNLOCK(ap->a_dvp); + vn_lock(lvp, LK_SHARED | LK_RETRY); + VOP_UNLOCK(lvp); + vn_lock(ap->a_dvp, LK_EXCLUSIVE | LK_RETRY); + vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY); + return (ERELOOKUP); + } error = unionfs_check_rmdir(ap->a_vp, cnp->cn_cred, td); + /* + * It's possible for a direct operation on the lower FS + * to make the lower directory non-empty after we drop + * the lock, but it's also possible for the upper-layer + * VOP_RMDIR to relock udvp/uvp which would lead to + * LOR if we kept lvp locked across that call. + */ + VOP_UNLOCK(lvp); if (error != 0) return (error); } ump = MOUNTTOUNIONFSMOUNT(ap->a_vp->v_mount); if (ump->um_whitemode == UNIONFS_WHITE_ALWAYS || lvp != NULLVP) cnp->cn_flags |= DOWHITEOUT; - /* - * The relookup path will need to relock the parent dvp and - * possibly the vp as well. Locking is expected to be done - * in parent->child order; drop the lock on vp to avoid LOR - * and potential recursion on vp's lock. - * vp is expected to remain referenced during VOP_RMDIR(), - * so vref/vrele should not be necessary here. - */ - VOP_UNLOCK(ap->a_vp); - VNPASS(vrefcnt(ap->a_vp) > 0, ap->a_vp); - error = unionfs_relookup_for_delete(ap->a_dvp, cnp, td); - vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY); - /* - * VOP_RMDIR is dispatched against udvp, so if uvp became - * doomed while the lock was dropped above the target - * filesystem may not be able to cope. - */ - if (error == 0 && VN_IS_DOOMED(uvp)) - error = ENOENT; - if (error == 0) { - int udvp_lkflags, uvp_lkflags; - unionfs_forward_vop_start_pair(udvp, &udvp_lkflags, - uvp, &uvp_lkflags); - error = VOP_RMDIR(udvp, uvp, cnp); - unionfs_forward_vop_finish_pair(ap->a_dvp, udvp, udvp_lkflags, - ap->a_vp, uvp, uvp_lkflags); - } - } else if (lvp != NULLVP) - error = unionfs_mkwhiteout(ap->a_dvp, udvp, cnp, td, + int udvp_lkflags, uvp_lkflags; + unionfs_forward_vop_start_pair(udvp, &udvp_lkflags, + uvp, &uvp_lkflags); + error = VOP_RMDIR(udvp, uvp, cnp); + unionfs_forward_vop_finish_pair(ap->a_dvp, udvp, udvp_lkflags, + ap->a_vp, uvp, uvp_lkflags); + } else if (lvp != NULLVP) { + error = unionfs_mkwhiteout(ap->a_dvp, ap->a_vp, cnp, td, unp->un_path, unp->un_pathlen); + } if (error == 0) { cache_purge(ap->a_dvp); @@ -1613,6 +1817,7 @@ unionfs_readdir(struct vop_readdir_args *ap) uint64_t *cookies_bk; int error; int eofflag; + int lkflags; int ncookies_bk; int uio_offset_bk; enum unionfs_lkupgrade lkstatus; @@ -1668,18 +1873,26 @@ unionfs_readdir(struct vop_readdir_args *ap) /* upper only */ if (uvp != NULLVP && lvp == NULLVP) { + unionfs_forward_vop_start(uvp, &lkflags); error = VOP_READDIR(uvp, uio, ap->a_cred, ap->a_eofflag, ap->a_ncookies, ap->a_cookies); - unsp->uns_readdir_status = 0; + if (unionfs_forward_vop_finish(vp, uvp, lkflags)) + error = error ? error : ENOENT; + else + unsp->uns_readdir_status = 0; goto unionfs_readdir_exit; } /* lower only */ if (uvp == NULLVP && lvp != NULLVP) { + unionfs_forward_vop_start(lvp, &lkflags); error = VOP_READDIR(lvp, uio, ap->a_cred, ap->a_eofflag, ap->a_ncookies, ap->a_cookies); - unsp->uns_readdir_status = 2; + if (unionfs_forward_vop_finish(vp, lvp, lkflags)) + error = error ? error : ENOENT; + else + unsp->uns_readdir_status = 2; goto unionfs_readdir_exit; } @@ -1689,14 +1902,17 @@ unionfs_readdir(struct vop_readdir_args *ap) */ KASSERT(uvp != NULLVP, ("unionfs_readdir: null upper vp")); KASSERT(lvp != NULLVP, ("unionfs_readdir: null lower vp")); + if (uio->uio_offset == 0) unsp->uns_readdir_status = 0; if (unsp->uns_readdir_status == 0) { /* read upper */ + unionfs_forward_vop_start(uvp, &lkflags); error = VOP_READDIR(uvp, uio, ap->a_cred, &eofflag, ap->a_ncookies, ap->a_cookies); - + if (unionfs_forward_vop_finish(vp, uvp, lkflags) && error == 0) + error = ENOENT; if (error != 0 || eofflag == 0) goto unionfs_readdir_exit; unsp->uns_readdir_status = 1; @@ -1735,14 +1951,22 @@ unionfs_readdir(struct vop_readdir_args *ap) uio->uio_offset = 0; } - if (lvp == NULLVP) { - error = EBADF; + lvp = unionfs_lock_lvp(vp, &lkflags); + if (lvp == NULL) { + error = ENOENT; goto unionfs_readdir_exit; } + /* read lower */ error = VOP_READDIR(lvp, uio, ap->a_cred, ap->a_eofflag, ap->a_ncookies, ap->a_cookies); + + unp = unionfs_unlock_lvp(vp, lvp, lkflags); + if (unp == NULL && error == 0) + error = ENOENT; + + /* * We can't return an uio_offset of 0: this would trigger an * infinite loop, because the next call to unionfs_readdir would @@ -1906,97 +2130,50 @@ unionfs_print(struct vop_print_args *ap) return (0); } -static int -unionfs_get_llt_revlock(struct vnode *vp, int flags) -{ - int revlock; - - revlock = 0; - - switch (flags & LK_TYPE_MASK) { - case LK_SHARED: - if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE) - revlock = LK_UPGRADE; - else - revlock = LK_RELEASE; - break; - case LK_EXCLUSIVE: - case LK_UPGRADE: - revlock = LK_RELEASE; - break; - case LK_DOWNGRADE: - revlock = LK_UPGRADE; - break; - default: - break; - } - - return (revlock); -} - -/* - * The state of an acquired lock is adjusted similarly to - * the time of error generating. - * flags: LK_RELEASE or LK_UPGRADE - */ -static void -unionfs_revlock(struct vnode *vp, int flags) -{ - if (flags & LK_RELEASE) - VOP_UNLOCK_FLAGS(vp, flags); - else { - /* UPGRADE */ - if (vn_lock(vp, flags) != 0) - vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); - } -} - static int unionfs_lock(struct vop_lock1_args *ap) { struct unionfs_node *unp; struct vnode *vp; - struct vnode *uvp; - struct vnode *lvp; + struct vnode *tvp; int error; int flags; - int revlock; - int interlock; - int uhold; + bool lvp_locked; - /* - * TODO: rework the unionfs locking scheme. - * It's not guaranteed to be safe to blindly lock two vnodes on - * different mounts as is done here. Further, the entanglement - * of locking both vnodes with the various options that can be - * passed to VOP_LOCK() makes this code hard to reason about. - * Instead, consider locking only the upper vnode, or the lower - * vnode is the upper is not present, and taking separate measures - * to lock both vnodes in the few cases when that is needed. - */ error = 0; - interlock = 1; - uhold = 0; flags = ap->a_flags; vp = ap->a_vp; if (LK_RELEASE == (flags & LK_TYPE_MASK) || !(flags & LK_TYPE_MASK)) return (VOP_UNLOCK_FLAGS(vp, flags | LK_RELEASE)); +unionfs_lock_restart: + /* + * We currently need the interlock here to ensure we can safely + * access the unionfs vnode's private data. We may be able to + * eliminate this extra locking by instead using vfs_smr_enter() + * and vn_load_v_data_smr() here in conjunction with an SMR UMA + * zone for unionfs nodes. + */ if ((flags & LK_INTERLOCK) == 0) VI_LOCK(vp); + else + flags &= ~LK_INTERLOCK; unp = VTOUNIONFS(vp); - if (unp == NULL) - goto unionfs_lock_null_vnode; - - KASSERT_UNIONFS_VNODE(ap->a_vp); - - lvp = unp->un_lowervp; - uvp = unp->un_uppervp; + if (unp == NULL) { + VI_UNLOCK(vp); + ap->a_flags = flags; + return (vop_stdlock(ap)); + } - if ((revlock = unionfs_get_llt_revlock(vp, flags)) == 0) - panic("unknown lock type: 0x%x", flags & LK_TYPE_MASK); + if (unp->un_uppervp != NULL) { + tvp = unp->un_uppervp; + lvp_locked = false; + } else { + tvp = unp->un_lowervp; + lvp_locked = true; + } /* * During unmount, the root vnode lock may be taken recursively, @@ -2009,150 +2186,77 @@ unionfs_lock(struct vop_lock1_args *ap) (vp->v_vflag & VV_ROOT) != 0) flags |= LK_CANRECURSE; - if (lvp != NULLVP) { - if (uvp != NULLVP && flags & LK_UPGRADE) { + vholdnz(tvp); + VI_UNLOCK(vp); + error = VOP_LOCK(tvp, flags); + vdrop(tvp); + if (error == 0 && (lvp_locked || VTOUNIONFS(vp) == NULL)) { + /* + * After dropping the interlock above, there exists a window + * in which another thread may acquire the lower vnode lock + * and then either doom the unionfs vnode or create an upper + * vnode. In either case, we will effectively be holding the + * wrong lock, so we must drop the lower vnode lock and + * restart the lock operation. + * + * If unp is not already NULL, we assume that we can safely + * access it because we currently hold lvp's lock. + * unionfs_noderem() acquires lvp's lock before freeing + * the vnode private data, ensuring it can't be concurrently + * freed while we are using it here. Likewise, + * unionfs_node_update() acquires lvp's lock before installing + * an upper vnode. Without those guarantees, we would need to + * reacquire the vnode interlock here. + * Note that unionfs_noderem() doesn't acquire lvp's lock if + * this is the root vnode, but the root vnode should always + * have an upper vnode and therefore we should never use its + * lower vnode lock here. + */ + unp = VTOUNIONFS(vp); + if (unp == NULL || unp->un_uppervp != NULLVP) { + VOP_UNLOCK(tvp); /* - * Share Lock is once released and a deadlock is - * avoided. + * If we previously held the lock, the upgrade may + * have temporarily dropped the lock, in which case + * concurrent dooming or copy-up will necessitate + * acquiring a different lock. Since we never held + * the new lock, LK_UPGRADE must be cleared here to + * avoid triggering a lockmgr panic. */ - vholdnz(uvp); - uhold = 1; - VOP_UNLOCK(uvp); - } - VI_LOCK_FLAGS(lvp, MTX_DUPOK); - flags |= LK_INTERLOCK; - vholdl(lvp); - - VI_UNLOCK(vp); - ap->a_flags &= ~LK_INTERLOCK; - - error = VOP_LOCK(lvp, flags); - - VI_LOCK(vp); - unp = VTOUNIONFS(vp); - if (unp == NULL) { - /* vnode is released. */ - VI_UNLOCK(vp); - if (error == 0) - VOP_UNLOCK(lvp); - vdrop(lvp); - if (uhold != 0) - vdrop(uvp); - goto unionfs_lock_fallback; + if (flags & LK_UPGRADE) + flags = (flags & ~LK_TYPE_MASK) | LK_EXCLUSIVE; + VNASSERT((flags & LK_DOWNGRADE) == 0, vp, + ("%s: vnode doomed during downgrade", __func__)); + goto unionfs_lock_restart; } } - if (error == 0 && uvp != NULLVP) { - if (uhold && flags & LK_UPGRADE) { - flags &= ~LK_TYPE_MASK; - flags |= LK_EXCLUSIVE; - } - VI_LOCK_FLAGS(uvp, MTX_DUPOK); - flags |= LK_INTERLOCK; - if (uhold == 0) { - vholdl(uvp); - uhold = 1; - } - - VI_UNLOCK(vp); - ap->a_flags &= ~LK_INTERLOCK; - - error = VOP_LOCK(uvp, flags); - - VI_LOCK(vp); - unp = VTOUNIONFS(vp); - if (unp == NULL) { - /* vnode is released. */ - VI_UNLOCK(vp); - if (error == 0) - VOP_UNLOCK(uvp); - vdrop(uvp); - if (lvp != NULLVP) { - VOP_UNLOCK(lvp); - vdrop(lvp); - } - goto unionfs_lock_fallback; - } - if (error != 0 && lvp != NULLVP) { - /* rollback */ - VI_UNLOCK(vp); - unionfs_revlock(lvp, revlock); - interlock = 0; - } - } - - if (interlock) - VI_UNLOCK(vp); - if (lvp != NULLVP) - vdrop(lvp); - if (uhold != 0) - vdrop(uvp); - return (error); - -unionfs_lock_null_vnode: - ap->a_flags |= LK_INTERLOCK; - return (vop_stdlock(ap)); - -unionfs_lock_fallback: - /* - * If we reach this point, we've discovered the unionfs vnode - * has been reclaimed while the upper/lower vnode locks were - * temporarily dropped. Such temporary droppage may happen - * during the course of an LK_UPGRADE operation itself, and in - * that case LK_UPGRADE must be cleared as the unionfs vnode's - * lock has been reset to point to the standard v_lock field, - * which has not previously been held. - */ - if (flags & LK_UPGRADE) { - ap->a_flags &= ~LK_TYPE_MASK; - ap->a_flags |= LK_EXCLUSIVE; - } - return (vop_stdlock(ap)); } static int unionfs_unlock(struct vop_unlock_args *ap) { struct vnode *vp; - struct vnode *lvp; - struct vnode *uvp; + struct vnode *tvp; struct unionfs_node *unp; int error; - int uhold; KASSERT_UNIONFS_VNODE(ap->a_vp); - error = 0; - uhold = 0; vp = ap->a_vp; unp = VTOUNIONFS(vp); if (unp == NULL) - goto unionfs_unlock_null_vnode; - lvp = unp->un_lowervp; - uvp = unp->un_uppervp; - - if (lvp != NULLVP) { - vholdnz(lvp); - error = VOP_UNLOCK(lvp); - } - - if (error == 0 && uvp != NULLVP) { - vholdnz(uvp); - uhold = 1; - error = VOP_UNLOCK(uvp); - } + return (vop_stdunlock(ap)); - if (lvp != NULLVP) - vdrop(lvp); - if (uhold != 0) - vdrop(uvp); + tvp = (unp->un_uppervp != NULL ? unp->un_uppervp : unp->un_lowervp); - return error; + vholdnz(tvp); + error = VOP_UNLOCK(tvp); + vdrop(tvp); -unionfs_unlock_null_vnode: - return (vop_stdunlock(ap)); + return (error); } static int @@ -2192,7 +2296,7 @@ unionfs_advlock(struct vop_advlock_args *ap) uvp = unp->un_uppervp; if (uvp == NULLVP) { - error = unionfs_copyfile(unp, 1, td->td_ucred, td); + error = unionfs_copyfile(ap->a_vp, 1, td->td_ucred, td); if (error != 0) goto unionfs_advlock_abort; uvp = unp->un_uppervp; @@ -2294,7 +2398,7 @@ unionfs_setacl(struct vop_setacl_args *ap) return (EROFS); if (uvp == NULLVP && lvp->v_type == VREG) { - if ((error = unionfs_copyfile(unp, 1, ap->a_cred, td)) != 0) + if ((error = unionfs_copyfile(ap->a_vp, 1, ap->a_cred, td)) != 0) return (error); uvp = unp->un_uppervp; } @@ -2467,9 +2571,10 @@ unionfs_setextattr(struct vop_setextattr_args *ap) if (ovp == lvp && lvp->v_type == VREG) { VOP_CLOSEEXTATTR(lvp, 0, cred, td); if (uvp == NULLVP && - (error = unionfs_copyfile(unp, 1, cred, td)) != 0) { + (error = unionfs_copyfile(ap->a_vp, 1, cred, td)) != 0) { unionfs_setextattr_reopen: - if ((unp->un_flag & UNIONFS_OPENEXTL) && + unp = VTOUNIONFS(ap->a_vp); + if (unp != NULL && (unp->un_flag & UNIONFS_OPENEXTL) && VOP_OPENEXTATTR(lvp, cred, td)) { #ifdef DIAGNOSTIC panic("unionfs: VOP_OPENEXTATTR failed"); @@ -2561,9 +2666,10 @@ unionfs_deleteextattr(struct vop_deleteextattr_args *ap) if (ovp == lvp && lvp->v_type == VREG) { VOP_CLOSEEXTATTR(lvp, 0, cred, td); if (uvp == NULLVP && - (error = unionfs_copyfile(unp, 1, cred, td)) != 0) { + (error = unionfs_copyfile(ap->a_vp, 1, cred, td)) != 0) { unionfs_deleteextattr_reopen: - if ((unp->un_flag & UNIONFS_OPENEXTL) && + unp = VTOUNIONFS(ap->a_vp); + if (unp != NULL && (unp->un_flag & UNIONFS_OPENEXTL) && VOP_OPENEXTATTR(lvp, cred, td)) { #ifdef DIAGNOSTIC panic("unionfs: VOP_OPENEXTATTR failed"); @@ -2613,7 +2719,7 @@ unionfs_setlabel(struct vop_setlabel_args *ap) return (EROFS); if (uvp == NULLVP && lvp->v_type == VREG) { - if ((error = unionfs_copyfile(unp, 1, ap->a_cred, td)) != 0) + if ((error = unionfs_copyfile(ap->a_vp, 1, ap->a_cred, td)) != 0) return (error); uvp = unp->un_uppervp; } @@ -2665,7 +2771,7 @@ static int unionfs_vput_pair(struct vop_vput_pair_args *ap) { struct mount *mp; - struct vnode *dvp, *vp, **vpp, *lvp, *ldvp, *uvp, *udvp, *tempvp; + struct vnode *dvp, *vp, **vpp, *lvp, *uvp, *tvp, *tdvp, *tempvp; struct unionfs_node *dunp, *unp; int error, res; @@ -2674,11 +2780,14 @@ unionfs_vput_pair(struct vop_vput_pair_args *ap) vp = NULLVP; lvp = NULLVP; uvp = NULLVP; + tvp = NULLVP; unp = NULL; dunp = VTOUNIONFS(dvp); - udvp = dunp->un_uppervp; - ldvp = dunp->un_lowervp; + if (dunp->un_uppervp != NULL) + tdvp = dunp->un_uppervp; + else + tdvp = dunp->un_lowervp; /* * Underlying vnodes should be locked because the encompassing unionfs @@ -2686,10 +2795,7 @@ unionfs_vput_pair(struct vop_vput_pair_args *ap) * only be on the unionfs node. Reference them now so that the vput()s * performed by VOP_VPUT_PAIR() will have a reference to drop. */ - if (udvp != NULLVP) - vref(udvp); - if (ldvp != NULLVP) - vref(ldvp); + vref(tdvp); if (vpp != NULL) vp = *vpp; @@ -2699,9 +2805,10 @@ unionfs_vput_pair(struct vop_vput_pair_args *ap) uvp = unp->un_uppervp; lvp = unp->un_lowervp; if (uvp != NULLVP) - vref(uvp); - if (lvp != NULLVP) - vref(lvp); + tvp = uvp; + else + tvp = lvp; + vref(tvp); /* * If we're being asked to return a locked child vnode, then @@ -2721,31 +2828,19 @@ unionfs_vput_pair(struct vop_vput_pair_args *ap) } } - /* - * TODO: Because unionfs_lock() locks both the lower and upper vnodes - * (if available), we must also call VOP_VPUT_PAIR() on both the lower - * and upper parent/child pairs. If unionfs_lock() is reworked to lock - * only a single vnode, this code will need to change to also only - * operate on one vnode pair. - */ - ASSERT_VOP_LOCKED(ldvp, __func__); - ASSERT_VOP_LOCKED(udvp, __func__); - ASSERT_VOP_LOCKED(lvp, __func__); - ASSERT_VOP_LOCKED(uvp, __func__); - - KASSERT(lvp == NULLVP || ldvp != NULLVP, - ("%s: NULL ldvp with non-NULL lvp", __func__)); - if (ldvp != NULLVP) - res = VOP_VPUT_PAIR(ldvp, lvp != NULLVP ? &lvp : NULL, true); - KASSERT(uvp == NULLVP || udvp != NULLVP, - ("%s: NULL udvp with non-NULL uvp", __func__)); - if (udvp != NULLVP) - res = VOP_VPUT_PAIR(udvp, uvp != NULLVP ? &uvp : NULL, true); - - ASSERT_VOP_UNLOCKED(ldvp, __func__); - ASSERT_VOP_UNLOCKED(udvp, __func__); - ASSERT_VOP_UNLOCKED(lvp, __func__); - ASSERT_VOP_UNLOCKED(uvp, __func__); + ASSERT_VOP_LOCKED(tdvp, __func__); + ASSERT_VOP_LOCKED(tvp, __func__); + + if (tdvp == dunp->un_uppervp && tvp != NULLVP && tvp == lvp) { + vput(tvp); + vput(tdvp); + res = 0; + } else { + res = VOP_VPUT_PAIR(tdvp, tvp != NULLVP ? &tvp : NULL, true); + } + + ASSERT_VOP_UNLOCKED(tdvp, __func__); + ASSERT_VOP_UNLOCKED(tvp, __func__); /* * VOP_VPUT_PAIR() dropped the references we added to the underlying -- Test scenario: src/tools/test/stress2/misc/unionfs19.sh