vmcore.62 KDB: debugger backends: ddb KDB: current backend: ddb Copyright (c) 1992-2018 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 12.0-CURRENT #2 r330766M: Mon Mar 12 07:29:22 CET 2018 pho@mercat1.netperf.freebsd.org:/usr/src/sys/amd64/compile/PHO amd64 FreeBSD clang version 6.0.0 (tags/RELEASE_600/final 326565) (based on LLVM 6.0.0) WARNING: WITNESS option enabled, expect reduced performance. WARNING: DIAGNOSTIC option enabled, expect reduced performance. Entering uma_startup with 7 boot pages configured startup_alloc from "UMA Kegs", 6 boot pages left startup_alloc from "UMA Zones", 5 boot pages left startup_alloc from "UMA Zones", 4 boot pages left startup_alloc from "UMA Zones", 3 boot pages left startup_alloc from "UMA Zones", 2 boot pages left startup_alloc from "UMA Hash", 1 boot pages left Entering uma_startup1 with 0 boot pages left Entering uma_startup2 with 0 boot pages left VT(vga): resolution 640x480 CPU: Intel(R) Xeon(R) CPU E5-1650 v3 @ 3.50GHz (3500.08-MHz K8-class CPU) Origin="GenuineIntel" Id=0x306f2 Family=0x6 Model=0x3f Stepping=2 Features=0xbfebfbff Features2=0x7ffefbff AMD Features=0x2c100800 AMD Features2=0x21 Structured Extended Features=0x37ab XSAVE Features=0x1 VT-x: PAT,HLT,MTF,PAUSE,EPT,UG,VPID,VID,PostIntr TSC: P-state invariant, performance statistics real memory = 34359738368 (32768 MB) avail memory = 33179262976 (31642 MB) Event timer "LAPIC" quality 600 ACPI APIC Table: FreeBSD/SMP: Multiprocessor System Detected: 12 CPUs FreeBSD/SMP: 1 package(s) x 6 core(s) x 2 hardware threads arc4random: no preloaded entropy cache ioapic0 irqs 0-23 on motherboard ioapic1 irqs 24-47 on motherboard SMP: AP CPU #1 Launched! SMP: AP CPU #5 Launched! SMP: AP CPU #2 Launched! SMP: AP CPU #7 Launched! SMP: AP CPU #10 Launched! SMP: AP CPU #11 Launched! SMP: AP CPU #9 Launched! SMP: AP CPU #8 Launched! SMP: AP CPU #3 Launched! SMP: AP CPU #6 Launched! SMP: AP CPU #4 Launched! Timecounter "TSC-low" frequency 1750038590 Hz quality 1000 random: entropy device external interface module_register_init: MOD_LOAD (vesa, 0xffffffff80fec5d0, 0) error 19 random: registering fast source Intel Secure Key RNG random: fast provider: "Intel Secure Key RNG" kbd1 at kbdmux0 netmap: loaded module [ath_hal] loaded nexus0 vtvga0: on motherboard cryptosoft0: on motherboard acpi0: on motherboard acpi0: Power Button (fixed) cpu0: on acpi0 cpu1: on acpi0 cpu2: on acpi0 cpu3: on acpi0 cpu4: on acpi0 cpu5: on acpi0 cpu6: on acpi0 cpu7: on acpi0 cpu8: on acpi0 cpu9: on acpi0 cpu10: on acpi0 cpu11: on acpi0 atrtc0: port 0x70-0x71,0x74-0x77 irq 8 on acpi0 atrtc0: registered as a time-of-day clock, resolution 1.000000s Event timer "RTC" frequency 32768 Hz quality 0 attimer0: port 0x40-0x43,0x50-0x53 irq 0 on acpi0 Timecounter "i8254" frequency 1193182 Hz quality 0 Event timer "i8254" frequency 1193182 Hz quality 100 hpet0: iomem 0xfed00000-0xfed003ff on acpi0 Timecounter "HPET" frequency 14318180 Hz quality 950 Event timer "HPET" frequency 14318180 Hz quality 350 Event timer "HPET1" frequency 14318180 Hz quality 340 Event timer "HPET2" frequency 14318180 Hz quality 340 Event timer "HPET3" frequency 14318180 Hz quality 340 Event timer "HPET4" frequency 14318180 Hz quality 340 Event timer "HPET5" frequency 14318180 Hz quality 340 Event timer "HPET6" frequency 14318180 Hz quality 340 Event timer "HPET7" frequency 14318180 Hz quality 340 Timecounter "ACPI-fast" frequency 3579545 Hz quality 900 acpi_timer0: <24-bit timer at 3.579545MHz> port 0x408-0x40b on acpi0 pcib0: on acpi0 pci0: on pcib0 pci0: at device 11.1 (no driver attached) pci0: at device 11.2 (no driver attached) pci0: at device 16.1 (no driver attached) pci0: at device 16.6 (no driver attached) pci0: at device 18.1 (no driver attached) acpi_syscontainer0: on acpi0 acpi_syscontainer1: on acpi0 acpi_syscontainer2: on acpi0 acpi_syscontainer3: on acpi0 pcib1: port 0xcf8-0xcff on acpi0 pci1: on pcib1 pcib2: irq 26 at device 1.0 on pci1 pci2: on pcib2 pcib3: irq 32 at device 2.0 on pci1 pci3: on pcib3 pci3: at device 0.0 (no driver attached) pci3: at device 0.1 (no driver attached) pcib4: irq 40 at device 3.0 on pci1 pci4: on pcib4 pci1: at device 17.0 (no driver attached) ahci0: port 0xf110-0xf117,0xf100-0xf103,0xf0f0-0xf0f7,0xf0e0-0xf0e3,0xf020-0xf03f mem 0xfb438000-0xfb4387ff irq 16 at device 17.4 on pci1 ahci0: AHCI v1.30 with 4 6Gbps ports, Port Multiplier not supported ahcich0: at channel 0 on ahci0 ahcich1: at channel 1 on ahci0 ahcich2: at channel 2 on ahci0 ahcich3: at channel 3 on ahci0 ahciem0: on ahci0 xhci0: mem 0xfb400000-0xfb40ffff irq 19 at device 20.0 on pci1 xhci0: 32 bytes context size, 64-bit DMA usbus0 on xhci0 usbus0: 5.0Gbps Super Speed USB v3.0 pci1: at device 22.0 (no driver attached) pci1: at device 22.1 (no driver attached) ehci0: mem 0xfb434000-0xfb4343ff irq 18 at device 26.0 on pci1 usbus1: EHCI version 1.0 usbus1 on ehci0 usbus1: 480Mbps High Speed USB v2.0 pcib5: irq 16 at device 28.0 on pci1 pci5: on pcib5 pcib6: irq 18 at device 28.2 on pci1 pci6: on pcib6 pcib7: at device 0.0 on pci6 pci7: on pcib7 vgapci0: port 0xe000-0xe07f mem 0xfa000000-0xfaffffff,0xfb000000-0xfb01ffff irq 18 at device 0.0 on pci7 vgapci0: Boot video device pcib8: irq 16 at device 28.4 on pci1 pci8: on pcib8 igb0: port 0xd020-0xd03f mem 0xfb320000-0xfb33ffff,0xfb344000-0xfb347fff irq 16 at device 0.0 on pci8 igb0: attach_pre capping queues at 8 igb0: using 1024 tx descriptors and 1024 rx descriptors igb0: msix_init qsets capped at 8 igb0: pxm cpus: 6 queue msgs: 9 admincnt: 1 igb0: queue equality override not set, capping rx_queues at 6 and tx_queues at 6 igb0: using 6 rx queues 6 tx queues igb0: Using MSIX interrupts with 7 vectors igb0: allocated for 6 tx_queues igb0: allocated for 6 rx_queues igb0: Ethernet address: 0c:c4:7a:a8:cd:da igb0: netmap queues/slots: TX 6/1024, RX 6/1024 igb1: port 0xd000-0xd01f mem 0xfb300000-0xfb31ffff,0xfb340000-0xfb343fff irq 17 at device 0.1 on pci8 igb1: attach_pre capping queues at 8 igb1: using 1024 tx descriptors and 1024 rx descriptors igb1: msix_init qsets capped at 8 igb1: pxm cpus: 6 queue msgs: 9 admincnt: 1 igb1: queue equality override not set, capping rx_queues at 6 and tx_queues at 6 igb1: using 6 rx queues 6 tx queues igb1: Using MSIX interrupts with 7 vectors igb1: allocated for 6 tx_queues igb1: allocated for 6 rx_queues igb1: Ethernet address: 0c:c4:7a:a8:cd:db igb1: netmap queues/slots: TX 6/1024, RX 6/1024 ehci1: mem 0xfb433000-0xfb4333ff irq 18 at device 29.0 on pci1 usbus2: EHCI version 1.0 usbus2 on ehci1 usbus2: 480Mbps High Speed USB v2.0 isab0: at device 31.0 on pci1 isa0: on isab0 ahci1: port 0xf070-0xf077,0xf060-0xf063,0xf050-0xf057,0xf040-0xf043,0xf000-0xf01f mem 0xfb432000-0xfb4327ff irq 16 at device 31.2 on pci1 ahci1: AHCI v1.30 with 6 6Gbps ports, Port Multiplier not supported ahcich4: at channel 0 on ahci1 ahcich5: at channel 1 on ahci1 ahcich6: at channel 2 on ahci1 ahcich7: at channel 3 on ahci1 ahcich8: at channel 4 on ahci1 ahcich9: at channel 5 on ahci1 ahciem1: on ahci1 acpi_button0: on acpi0 uart0: <16550 or compatible> port 0x3f8-0x3ff irq 4 flags 0x10 on acpi0 uart1: <16550 or compatible> port 0x2f8-0x2ff irq 3 on acpi0 uart1: console (115200,n,8,1) orm0: at iomem 0xc0000-0xc7fff,0xcb000-0xcbfff pnpid ORM0000 on isa0 est0: on cpu0 p4tcc0: on cpu0 est1: on cpu1 p4tcc1: on cpu1 est2: on cpu2 p4tcc2: on cpu2 est3: on cpu3 p4tcc3: on cpu3 est4: on cpu4 p4tcc4: on cpu4 est5: on cpu5 p4tcc5: on cpu5 est6: on cpu6 p4tcc6: on cpu6 est7: on cpu7 p4tcc7: on cpu7 est8: on cpu8 p4tcc8: on cpu8 est9: on cpu9 p4tcc9: on cpu9 est10: on cpu10 p4tcc10: on cpu10 est11: on cpu11 p4tcc11: on cpu11 NULL mp in getnewvnode(9), tag crossmp Timecounters tick every 1.000 msec arc4random: no preloaded entropy cache ugen2.1: at usbus2 ugen1.1: at usbus1 uhub0: on usbus2 ugen0.1: <0x8086 XHCI root HUB> at usbus0 uhub1: on usbus1 uhub2: <0x8086 XHCI root HUB, class 9/0, rev 3.00t 0 lun 0 ses1: SEMB S-E-S 2.00 device ses1: SEMB SES Device ugen2.2: at usbus2 uhub3 on uhub0 uhub3: on usbus2 ugen1.2: at usbus1 uhub4 on uhub1 uhub4: on usbus1 ugen0.2: at usbus0 uhub5 on uhub2 uhub5: on usbus0 uhub3: 8 ports with 8 removable, self powered uhub4: 6 ports with 6 removable, self powered uhub5: 4 ports with 3 removable, self powered Writing entropy file:. Setting hostname: mercat1.netperf.freebsd.org. ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib 32-bit compatibility ldconfig path: /usr/lib32 NFS access cache time=60 Starting casperd. lock order reversal: 1st 0xfffffe000104f020 bufwait (bufwait) @ kern/vfs_bio.c:3828 2nd 0xfffff800041aba00 dirhash (dirhash) @ ufs/ufs/ufs_dirhash.c:289 stack backtrace: #0 0xffffffff80b14df3 at witness_debugger+0x73 #1 0xffffffff80b14c74 at witness_checkorder+0xe34 #2 0xffffffff80ab73d8 at _sx_xlock+0x68 #3 0xffffffff80db682d at ufsdirhash_move+0x4d #4 0xffffffff80dba90a at ufs_direnter+0x33a #5 0xffffffff80dc306c at ufs_mkdir+0x8dc #6 0xffffffff810e329d at VOP_MKDIR_APV+0xfd #7 0xffffffff80b8ede6 at kern_mkdirat+0x1d6 #8 0xffffffff80f6ce3b at amd64_syscall+0x7ab #9 0xffffffff80f47bbd at fast_syscall_common+0x101 Clearing /tmp (X related). Starting statd. Starting lockd. Updating motd:. Mounting late file systems:. Configuring syscons: blanktime. Performing sanity check on sshd configuration. Starting sshd. Starting cron. Mon Mar 12 13:13 FreeBSD/amd64 (mercat1.netperf.freebsd.org) (ttyu1) login: root Mar 12 13:14:07 mercat1 login: ROOT LOGIN (root) ON ttyu1 Last login: Mon Mar 12 07:48:34 on ttyu1 FreeBSD 12.0-CURRENT (PHO) #2 r330766M: Mon Mar 12 07:29:22 CET 2018 root@mercat1:~ # Mar 12 13:14:25 mercat1 su: pho to root on /dev/pts/0 lock order reversal: 1st 0xfffff80013a722e0 ufs (ufs) @ kern/vfs_mount.c:1280 2nd 0xfffff80013afc7d0 devfs (devfs) @ ufs/ffs/ffs_vfsops.c:1371 stack backtrace: #0 0xffffffff80b14df3 at witness_debugger+0x73 #1 0xffffffff80b14c74 at witness_checkorder+0xe34 #2 0xffffffff80a8387e at lockmgr_lock_fast_path+0x17e #3 0xffffffff810e3fb1 at VOP_LOCK1_APV+0xe1 #4 0xffffffff80b91cd6 at _vn_lock+0x66 #5 0xffffffff80dac474 at ffs_flushfiles+0xa4 #6 0xffffffff80d8f802 at softdep_flushfiles+0x82 #7 0xffffffff80daebb9 at ffs_unmount+0xd9 #8 0xffffffff80b77ee9 at dounmount+0x519 #9 0xffffffff80b77940 at sys_unmount+0x310 #10 0xffffffff80f6ce3b at amd64_syscall+0x7ab #11 0xffffffff80f47bbd at fast_syscall_common+0x101 20180312 13:45:12 all (1/1): fdatasync.sh Mar 12 13:45:13 mercat1 su: pho to root on /dev/pts/1 lock order reversal: 1st 0xfffff8029de29a48 ufs (ufs) @ kern/vfs_syscalls.c:3369 2nd 0xfffffe0000f58498 bufwait (bufwait) @ ufs/ffs/ffs_vnops.c:282 3rd 0xfffff802caca1558 ufs (ufs) @ kern/vfs_subr.c:2602 stack backtrace: #0 0xffffffff80b14df3 at witness_debugger+0x73 #1 0xffffffff80b14c74 at witness_checkorder+0xe34 #2 0xffffffff80a83d4b at lockmgr_xlock_hard+0x6b #3 0xffffffff80a8472a at __lockmgr_args+0x51a #4 0xffffffff80db0d0f at ffs_lock+0xaf #5 0xffffffff810e3fb1 at VOP_LOCK1_APV+0xe1 #6 0xffffffff80b91cd6 at _vn_lock+0x66 #7 0xffffffff80b80088 at vget+0x98 #8 0xffffffff80b71b91 at vfs_hash_get+0xd1 #9 0xffffffff80dac5bf at ffs_vgetf+0x3f #10 0xffffffff80da24c6 at softdep_sync_buf+0xd16 #11 0xffffffff80db1b14 at ffs_syncvnode+0x294 #12 0xffffffff810e7dcd at VOP_FDATASYNC_APV+0xfd #13 0xffffffff80b8e626 at kern_fsync+0x1d6 #14 0xffffffff80f6ce3b at amd64_syscall+0x7ab #15 0xffffffff80f47bbd at fast_syscall_common+0x101 newblk leak: 1912/1920. pts leak: 1/2 20180312 13:48:24 all (1/1): fdatasync.sh freework leak: 30/119. newblk leak: 65/1985. 20180312 13:51:36 all (1/1): fdatasync.sh KDB: enter: Break to debugger [ thread pid 11 tid 100003 ] Stopped at kdb_alt_break_internal+0x106: movq $0,kdb_why db> run pho db:0:pho> run pho1 db:1:pho1> bt Tracing pid 11 tid 100003 td 0xfffff80003938000 kdb_alt_break_internal() at kdb_alt_break_internal+0x106/frame 0xfffffe007895a700 kdb_alt_break() at kdb_alt_break+0xb/frame 0xfffffe007895a710 uart_intr_rxready() at uart_intr_rxready+0x9a/frame 0xfffffe007895a740 uart_intr() at uart_intr+0x138/frame 0xfffffe007895a780 intr_event_handle() at intr_event_handle+0xbc/frame 0xfffffe007895a7d0 intr_execute_handlers() at intr_execute_handlers+0x58/frame 0xfffffe007895a800 lapic_handle_intr() at lapic_handle_intr+0x6d/frame 0xfffffe007895a840 apic_isr1_u() at apic_isr1_u+0xc1/frame 0xfffffe007895a910 acpi_cpu_idle() at acpi_cpu_idle+0x2ee/frame 0xfffffe007895a960 cpu_idle_acpi() at cpu_idle_acpi+0x3f/frame 0xfffffe007895a980 cpu_idle() at cpu_idle+0x8f/frame 0xfffffe007895a9a0 sched_idletd() at sched_idletd+0x517/frame 0xfffffe007895aa70 fork_exit() at fork_exit+0x84/frame 0xfffffe007895aab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe007895aab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- db:1:pho1> show allpcpu Current CPU: 0 cpuid = 0 dynamic pcpu = 0xa0f1c0 curthread = 0xfffff80003938000: pid 11 tid 100003 "idle: cpu0" curpcb = 0xfffffe007895ab80 fpcurthread = none idlethread = 0xfffff80003938000: tid 100003 "idle: cpu0" curpmap = 0xffffffff81f40718 tssp = 0xffffffff81f6a810 commontssp = 0xffffffff81f6a810 rsp0 = 0xfffffe007895ab80 gs32p = 0xffffffff81f71058 ldt = 0xffffffff81f71098 tss = 0xffffffff81f71088 curvnet = 0 spin locks held: cpuid = 1 dynamic pcpu = 0xfffffe00856c51c0 curthread = 0xfffff80003937560: pid 11 tid 100004 "idle: cpu1" curpcb = 0xfffffe007895fb80 fpcurthread = none idlethread = 0xfffff80003937560: tid 100004 "idle: cpu1" curpmap = 0xffffffff81f40718 tssp = 0xffffffff81f6a878 commontssp = 0xffffffff81f6a878 rsp0 = 0xfffffe007895fb80 gs32p = 0xffffffff81f710c0 ldt = 0xffffffff81f71100 tss = 0xffffffff81f710f0 curvnet = 0 spin locks held: cpuid = 2 dynamic pcpu = 0xfffffe00856ce1c0 curthread = 0xfffff80003937000: pid 11 tid 100005 "idle: cpu2" curpcb = 0xfffffe0078964b80 fpcurthread = none idlethread = 0xfffff80003937000: tid 100005 "idle: cpu2" curpmap = 0xffffffff81f40718 tssp = 0xffffffff81f6a8e0 commontssp = 0xffffffff81f6a8e0 rsp0 = 0xfffffe0078964b80 gs32p = 0xffffffff81f71128 ldt = 0xffffffff81f71168 tss = 0xffffffff81f71158 curvnet = 0 spin locks held: cpuid = 3 dynamic pcpu = 0xfffffe00856d71c0 curthread = 0xfffff80003936560: pid 11 tid 100006 "idle: cpu3" curpcb = 0xfffffe0078969b80 fpcurthread = none idlethread = 0xfffff80003936560: tid 100006 "idle: cpu3" curpmap = 0xffffffff81f40718 tssp = 0xffffffff81f6a948 commontssp = 0xffffffff81f6a948 rsp0 = 0xfffffe0078969b80 gs32p = 0xffffffff81f71190 ldt = 0xffffffff81f711d0 tss = 0xffffffff81f711c0 curvnet = 0 spin locks held: cpuid = 4 dynamic pcpu = 0xfffffe00856e01c0 curthread = 0xfffff80003936000: pid 11 tid 100007 "idle: cpu4" curpcb = 0xfffffe007896eb80 fpcurthread = none idlethread = 0xfffff80003936000: tid 100007 "idle: cpu4" curpmap = 0xffffffff81f40718 tssp = 0xffffffff81f6a9b0 commontssp = 0xffffffff81f6a9b0 rsp0 = 0xfffffe007896eb80 gs32p = 0xffffffff81f711f8 ldt = 0xffffffff81f71238 tss = 0xffffffff81f71228 curvnet = 0 spin locks held: cpuid = 5 dynamic pcpu = 0xfffffe00856e91c0 curthread = 0xfffff80003935560: pid 11 tid 100008 "idle: cpu5" curpcb = 0xfffffe0078973b80 fpcurthread = none idlethread = 0xfffff80003935560: tid 100008 "idle: cpu5" curpmap = 0xffffffff81f40718 tssp = 0xffffffff81f6aa18 commontssp = 0xffffffff81f6aa18 rsp0 = 0xfffffe0078973b80 gs32p = 0xffffffff81f71260 ldt = 0xffffffff81f712a0 tss = 0xffffffff81f71290 curvnet = 0 spin locks held: cpuid = 6 dynamic pcpu = 0xfffffe00856f21c0 curthread = 0xfffff80003935000: pid 11 tid 100009 "idle: cpu6" curpcb = 0xfffffe0078978b80 fpcurthread = none idlethread = 0xfffff80003935000: tid 100009 "idle: cpu6" curpmap = 0xffffffff81f40718 tssp = 0xffffffff81f6aa80 commontssp = 0xffffffff81f6aa80 rsp0 = 0xfffffe0078978b80 gs32p = 0xffffffff81f712c8 ldt = 0xffffffff81f71308 tss = 0xffffffff81f712f8 curvnet = 0 spin locks held: cpuid = 7 dynamic pcpu = 0xfffffe00856fb1c0 curthread = 0xfffff80003949560: pid 11 tid 100010 "idle: cpu7" curpcb = 0xfffffe007897db80 fpcurthread = none idlethread = 0xfffff80003949560: tid 100010 "idle: cpu7" curpmap = 0xffffffff81f40718 tssp = 0xffffffff81f6aae8 commontssp = 0xffffffff81f6aae8 rsp0 = 0xfffffe007897db80 gs32p = 0xffffffff81f71330 ldt = 0xffffffff81f71370 tss = 0xffffffff81f71360 curvnet = 0 spin locks held: cpuid = 8 dynamic pcpu = 0xfffffe00857041c0 curthread = 0xfffff80003949000: pid 11 tid 100011 "idle: cpu8" curpcb = 0xfffffe0078982b80 fpcurthread = none idlethread = 0xfffff80003949000: tid 100011 "idle: cpu8" curpmap = 0xffffffff81f40718 tssp = 0xffffffff81f6ab50 commontssp = 0xffffffff81f6ab50 rsp0 = 0xfffffe0078982b80 gs32p = 0xffffffff81f71398 ldt = 0xffffffff81f713d8 tss = 0xffffffff81f713c8 curvnet = 0 spin locks held: cpuid = 9 dynamic pcpu = 0xfffffe008570d1c0 curthread = 0xfffff80003948560: pid 11 tid 100012 "idle: cpu9" curpcb = 0xfffffe0078987b80 fpcurthread = none idlethread = 0xfffff80003948560: tid 100012 "idle: cpu9" curpmap = 0xffffffff81f40718 tssp = 0xffffffff81f6abb8 commontssp = 0xffffffff81f6abb8 rsp0 = 0xfffffe0078987b80 gs32p = 0xffffffff81f71400 ldt = 0xffffffff81f71440 tss = 0xffffffff81f71430 curvnet = 0 spin locks held: cpuid = 10 dynamic pcpu = 0xfffffe00857161c0 curthread = 0xfffff80003948000: pid 11 tid 100013 "idle: cpu10" curpcb = 0xfffffe007898cb80 fpcurthread = none idlethread = 0xfffff80003948000: tid 100013 "idle: cpu10" curpmap = 0xffffffff81f40718 tssp = 0xffffffff81f6ac20 commontssp = 0xffffffff81f6ac20 rsp0 = 0xfffffe007898cb80 gs32p = 0xffffffff81f71468 ldt = 0xffffffff81f714a8 tss = 0xffffffff81f71498 curvnet = 0 spin locks held: cpuid = 11 dynamic pcpu = 0xfffffe007f1e71c0 curthread = 0xfffff80003947560: pid 11 tid 100014 "idle: cpu11" curpcb = 0xfffffe0078991b80 fpcurthread = none idlethread = 0xfffff80003947560: tid 100014 "idle: cpu11" curpmap = 0xffffffff81f40718 tssp = 0xffffffff81f6ac88 commontssp = 0xffffffff81f6ac88 rsp0 = 0xfffffe0078991b80 gs32p = 0xffffffff81f714d0 ldt = 0xffffffff81f71510 tss = 0xffffffff81f71500 curvnet = 0 spin locks held: db:1:pho1> show alllocks Process 1937 (sh) thread 0xfffff80003a25560 (100170) shared lockmgr ufs (ufs) r = 0 (0xfffff801ab924cc0) locked @ kern/vfs_subr.c:2602 Process 1931 (sh) thread 0xfffff8001396c560 (100182) shared lockmgr ufs (ufs) r = 0 (0xfffff801ab924cc0) locked @ kern/vfs_subr.c:2602 Process 1925 (sh) thread 0xfffff80013c02560 (100145) shared lockmgr ufs (ufs) r = 0 (0xfffff801ab924cc0) locked @ kern/vfs_subr.c:2602 Process 1923 (sshd) thread 0xfffff80013d3e000 (100187) shared lockmgr ufs (ufs) r = 0 (0xfffff801aba3d558) locked @ kern/vfs_vnops.c:803 Process 1917 (sh) thread 0xfffff802cab41560 (100233) shared lockmgr ufs (ufs) r = 0 (0xfffff801ab924cc0) locked @ kern/vfs_subr.c:2602 Process 1915 (newsyslog) thread 0xfffff8001398c000 (100165) shared sx dirhash (dirhash) r = 0 (0xfffff800033bda00) locked @ ufs/ufs/ufs_dirhash.c:240 shared lockmgr ufs (ufs) r = 0 (0xfffff800130722e0) locked @ kern/vfs_subr.c:2602 Process 1911 (sh) thread 0xfffff80003c78560 (100162) shared lockmgr ufs (ufs) r = 0 (0xfffff8026c5cdcc0) locked @ kern/vfs_vnops.c:803 Process 1908 (sh) thread 0xfffff8030f554560 (100247) shared lockmgr ufs (ufs) r = 0 (0xfffff8006640c068) locked @ kern/vfs_vnops.c:803 Process 1907 (fdatasync) thread 0xfffff80013c97560 (100256) shared lockmgr ufs (ufs) r = 0 (0xfffff802ca8b12e0) locked @ kern/vfs_lookup.c:671 Process 1907 (fdatasync) thread 0xfffff8030f715560 (100257) exclusive lockmgr bufwait (bufwait) r = 0 (0xfffffe0006b0de98) locked @ kern/vfs_bio.c:1610 exclusive lockmgr bufwait (bufwait) r = 0 (0xfffffe00036f50f8) locked @ kern/vfs_bio.c:3828 exclusive lockmgr ufs (ufs) r = 0 (0xfffff802ca90a7d0) locked @ kern/vfs_vnops.c:883 Process 1907 (fdatasync) thread 0xfffff8030f715000 (100258) exclusive lockmgr bufwait (bufwait) r = 0 (0xfffffe00056599f0) locked @ kern/vfs_bio.c:3828 exclusive lockmgr ufs (ufs) r = 0 (0xfffff802ca92f558) locked @ kern/vfs_vnops.c:883 Process 796 (csh) thread 0xfffff801aba71560 (100190) shared lockmgr ufs (ufs) r = 0 (0xfffff800665372e0) locked @ kern/vfs_subr.c:2602 Process 33 (syncer) thread 0xfffff80013975000 (100133) exclusive lockmgr ufs (ufs) r = 0 (0xfffff8030f7b7558) locked @ kern/vfs_subr.c:2602 exclusive lockmgr syncer (syncer) r = 0 (0xfffff80013de9068) locked @ kern/vfs_subr.c:2152 db:1:pho1> show lockedvnods Locked vnodes vnode 0xfffff80013072278: tag ufs, type VDIR usecount 1, writecount 0, refcount 3 mountedhere 0 flags (VI_ACTIVE) v_object 0xfffff800130b2100 ref 0 pages 2 cleanbuf 0 dirtybuf 0 lock type ufs: SHARED (count 1) #0 0xffffffff80a83c8b at lockmgr_slock_hard+0x32b #1 0xffffffff80a84757 at __lockmgr_args+0x547 #2 0xffffffff80db0d0f at ffs_lock+0xaf #3 0xffffffff810e3fb1 at VOP_LOCK1_APV+0xe1 #4 0xffffffff80b91cd6 at _vn_lock+0x66 #5 0xffffffff80b80088 at vget+0x98 #6 0xffffffff80b6547a at cache_lookup+0x6fa #7 0xffffffff80b6a39c at vfs_cache_lookup+0xac #8 0xffffffff810e09a7 at VOP_LOOKUP_APV+0xf7 #9 0xffffffff80b73b42 at lookup+0x692 #10 0xffffffff80b7308b at namei+0x53b #11 0xffffffff80b8bc68 at kern_statat+0x98 #12 0xffffffff80b8c39f at sys_fstatat+0x2f #13 0xffffffff80f6ce3b at amd64_syscall+0x7ab #14 0xffffffff80f47bbd at fast_syscall_common+0x101 ino 9951751, on dev gpt/work vnode 0xfffff8006640c000: tag ufs, type VREG usecount 1, writecount 0, refcount 2 flags (VI_ACTIVE) v_object 0xfffff80013dfb400 ref 0 pages 3 cleanbuf 0 dirtybuf 0 lock type ufs: SHARED (count 1) #0 0xffffffff80a83c8b at lockmgr_slock_hard+0x32b #1 0xffffffff80a84757 at __lockmgr_args+0x547 #2 0xffffffff80db0d0f at ffs_lock+0xaf #3 0xffffffff810e3fb1 at VOP_LOCK1_APV+0xe1 #4 0xffffffff80b91cd6 at _vn_lock+0x66 #5 0xffffffff80b944c8 at vn_read+0x118 #6 0xffffffff80b92379 at vn_io_fault1+0x179 #7 0xffffffff80b90735 at vn_io_fault+0x195 #8 0xffffffff80b19dba at dofileread+0xba #9 0xffffffff80b19998 at kern_readv+0x68 #10 0xffffffff80b19926 at sys_read+0x86 #11 0xffffffff80f6ce3b at amd64_syscall+0x7ab #12 0xffffffff80f47bbd at fast_syscall_common+0x101 ino 90528778, on dev gpt/work vnode 0xfffff801aba3d4f0: tag ufs, type VREG usecount 1, writecount 0, refcount 2 flags (VI_ACTIVE) v_object 0xfffff801ab97ee00 ref 0 pages 3 cleanbuf 0 dirtybuf 0 lock type ufs: SHARED (count 1) #0 0xffffffff80a83c8b at lockmgr_slock_hard+0x32b #1 0xffffffff80a84757 at __lockmgr_args+0x547 #2 0xffffffff80db0d0f at ffs_lock+0xaf #3 0xffffffff810e3fb1 at VOP_LOCK1_APV+0xe1 #4 0xffffffff80b91cd6 at _vn_lock+0x66 #5 0xffffffff80b944c8 at vn_read+0x118 #6 0xffffffff80b92379 at vn_io_fault1+0x179 #7 0xffffffff80b90735 at vn_io_fault+0x195 #8 0xffffffff80b19dba at dofileread+0xba #9 0xffffffff80b19998 at kern_readv+0x68 #10 0xffffffff80b19926 at sys_read+0x86 #11 0xffffffff80f6ce3b at amd64_syscall+0x7ab #12 0xffffffff80f47bbd at fast_syscall_common+0x101 ino 75601495, on dev gpt/work vnode 0xfffff801ab924c58: tag ufs, type VDIR usecount 4, writecount 0, refcount 7 mountedhere 0 flags (VI_ACTIVE) v_object 0xfffff80003a55e00 ref 0 pages 1 cleanbuf 1 dirtybuf 0 lock type ufs: SHARED (count 4) #0 0xffffffff80a83c8b at lockmgr_slock_hard+0x32b #1 0xffffffff80a84757 at __lockmgr_args+0x547 #2 0xffffffff80db0d0f at ffs_lock+0xaf #3 0xffffffff810e3fb1 at VOP_LOCK1_APV+0xe1 #4 0xffffffff80b91cd6 at _vn_lock+0x66 #5 0xffffffff80b80088 at vget+0x98 #6 0xffffffff80b6547a at cache_lookup+0x6fa #7 0xffffffff80b6a39c at vfs_cache_lookup+0xac #8 0xffffffff810e09a7 at VOP_LOOKUP_APV+0xf7 #9 0xffffffff80b73b42 at lookup+0x692 #10 0xffffffff80b7308b at namei+0x53b #11 0xffffffff80a66eac at kern_execve+0x3cc #12 0xffffffff80a6679c at sys_execve+0x4c #13 0xffffffff80f6ce3b at amd64_syscall+0x7ab #14 0xffffffff80f47bbd at fast_syscall_common+0x101 ino 49277189, on dev gpt/work vnode 0xfffff80066537278: tag ufs, type VDIR usecount 1, writecount 0, refcount 3 mountedhere 0 flags (VI_ACTIVE) v_object 0xfffff800130b1a00 ref 0 pages 1 cleanbuf 0 dirtybuf 0 lock type ufs: SHARED (count 1) #0 0xffffffff80a83c8b at lockmgr_slock_hard+0x32b #1 0xffffffff80a84757 at __lockmgr_args+0x547 #2 0xffffffff80db0d0f at ffs_lock+0xaf #3 0xffffffff810e3fb1 at VOP_LOCK1_APV+0xe1 #4 0xffffffff80b91cd6 at _vn_lock+0x66 #5 0xffffffff80b80088 at vget+0x98 #6 0xffffffff80b6547a at cache_lookup+0x6fa #7 0xffffffff80b6a39c at vfs_cache_lookup+0xac #8 0xffffffff810e09a7 at VOP_LOOKUP_APV+0xf7 #9 0xffffffff80b73b42 at lookup+0x692 #10 0xffffffff80b7308b at namei+0x53b #11 0xffffffff80b8bc68 at kern_statat+0x98 #12 0xffffffff80b8c39f at sys_fstatat+0x2f #13 0xffffffff80f6ce3b at amd64_syscall+0x7ab #14 0xffffffff80f47bbd at fast_syscall_common+0x101 ino 7704577, on dev gpt/work vnode 0xfffff8026c5cdc58: tag ufs, type VREG usecount 1, writecount 0, refcount 6 flags (VI_ACTIVE) v_object 0xfffff8027d93ec00 ref 0 pages 1 cleanbuf 0 dirtybuf 0 lock type ufs: SHARED (count 1) with exclusive waiters pending #0 0xffffffff80a83c8b at lockmgr_slock_hard+0x32b #1 0xffffffff80a84757 at __lockmgr_args+0x547 #2 0xffffffff80db0d0f at ffs_lock+0xaf #3 0xffffffff810e3fb1 at VOP_LOCK1_APV+0xe1 #4 0xffffffff80b91cd6 at _vn_lock+0x66 #5 0xffffffff80b944c8 at vn_read+0x118 #6 0xffffffff80b92379 at vn_io_fault1+0x179 #7 0xffffffff80b90735 at vn_io_fault+0x195 #8 0xffffffff80b19dba at dofileread+0xba #9 0xffffffff80b19998 at kern_readv+0x68 #10 0xffffffff80b19926 at sys_read+0x86 #11 0xffffffff80f6ce3b at amd64_syscall+0x7ab #12 0xffffffff80f47bbd at fast_syscall_common+0x101 ino 49281818, on dev gpt/work vnode 0xfffff80013de9000: tag syncer, type VNON usecount 1, writecount 0, refcount 2 flags (VI_ACTIVE) lock type syncer: EXCL by thread 0xfffff80013975000 (pid 33, syncer, tid 100133) #0 0xffffffff80a838e3 at lockmgr_lock_fast_path+0x1e3 #1 0xffffffff810e3fb1 at VOP_LOCK1_APV+0xe1 #2 0xffffffff80b91cd6 at _vn_lock+0x66 #3 0xffffffff80b85236 at sched_sync+0x256 #4 0xffffffff80a6f2a4 at fork_exit+0x84 #5 0xffffffff80f4808e at fork_trampoline+0xe vnode 0xfffff802ca8b1278: tag ufs, type VDIR usecount 3, writecount 0, refcount 5 mountedhere 0 flags (VI_ACTIVE) v_object 0xfffff802ca5ea500 ref 0 pages 0 cleanbuf 1 dirtybuf 0 lock type ufs: SHARED (count 1) #0 0xffffffff80a83c8b at lockmgr_slock_hard+0x32b #1 0xffffffff80a84757 at __lockmgr_args+0x547 #2 0xffffffff80db0d0f at ffs_lock+0xaf #3 0xffffffff810e3fb1 at VOP_LOCK1_APV+0xe1 #4 0xffffffff80b91cd6 at _vn_lock+0x66 #5 0xffffffff80b735c4 at lookup+0x114 #6 0xffffffff80b7308b at namei+0x53b #7 0xffffffff80b8bc68 at kern_statat+0x98 #8 0xffffffff80b8c39f at sys_fstatat+0x2f #9 0xffffffff80f6ce3b at amd64_syscall+0x7ab #10 0xffffffff80f47bbd at fast_syscall_common+0x101 ino 240768, on dev gpt/tmp vnode 0xfffff802ca90a768: tag ufs, type VREG usecount 1, writecount 1, refcount 39 flags (VI_ACTIVE) v_object 0xfffff802ca3f8300 ref 0 pages 303 cleanbuf 12 dirtybuf 25 lock type ufs: EXCL by thread 0xfffff8030f715560 (pid 1907, fdatasync, tid 100257) #0 0xffffffff80a8408c at lockmgr_xlock_hard+0x3ac #1 0xffffffff80a8472a at __lockmgr_args+0x51a #2 0xffffffff80db0d0f at ffs_lock+0xaf #3 0xffffffff810e3fb1 at VOP_LOCK1_APV+0xe1 #4 0xffffffff80b91cd6 at _vn_lock+0x66 #5 0xffffffff80b94864 at vn_write+0x194 #6 0xffffffff80b92379 at vn_io_fault1+0x179 #7 0xffffffff80b90735 at vn_io_fault+0x195 #8 0xffffffff80b1a397 at dofilewrite+0xa7 #9 0xffffffff80b19f88 at kern_writev+0x68 #10 0xffffffff80b19f16 at sys_write+0x86 #11 0xffffffff80f6ce3b at amd64_syscall+0x7ab #12 0xffffffff80f47bbd at fast_syscall_common+0x101 ino 240928, on dev gpt/tmp vnode 0xfffff802ca92f4f0: tag ufs, type VREG usecount 1, writecount 1, refcount 4 flags (VI_ACTIVE) v_object 0xfffff80013dfd400 ref 0 pages 61 cleanbuf 0 dirtybuf 1 lock type ufs: EXCL by thread 0xfffff8030f715000 (pid 1907, fdatasync, tid 100258) with shared waiters pending #0 0xffffffff80a8408c at lockmgr_xlock_hard+0x3ac #1 0xffffffff80a8472a at __lockmgr_args+0x51a #2 0xffffffff80db0d0f at ffs_lock+0xaf #3 0xffffffff810e3fb1 at VOP_LOCK1_APV+0xe1 #4 0xffffffff80b91cd6 at _vn_lock+0x66 #5 0xffffffff80b94864 at vn_write+0x194 #6 0xffffffff80b92379 at vn_io_fault1+0x179 #7 0xffffffff80b90735 at vn_io_fault+0x195 #8 0xffffffff80b1a397 at dofilewrite+0xa7 #9 0xffffffff80b19f88 at kern_writev+0x68 #10 0xffffffff80b19f16 at sys_write+0x86 #11 0xffffffff80f6ce3b at amd64_syscall+0x7ab #12 0xffffffff80f47bbd at fast_syscall_common+0x101 ino 241093, on dev gpt/tmp vnode 0xfffff8030f7b74f0: tag ufs, type VREG usecount 2, writecount 1, refcount 7 flags (VI_ACTIVE) v_object 0xfffff802caaa9e00 ref 0 pages 32 cleanbuf 1 dirtybuf 3 lock type ufs: EXCL by thread 0xfffff80013975000 (pid 33, syncer, tid 100133) #0 0xffffffff80a8408c at lockmgr_xlock_hard+0x3ac #1 0xffffffff80a8472a at __lockmgr_args+0x51a #2 0xffffffff80db0d0f at ffs_lock+0xaf #3 0xffffffff810e3fb1 at VOP_LOCK1_APV+0xe1 #4 0xffffffff80b91cd6 at _vn_lock+0x66 #5 0xffffffff80b80088 at vget+0x98 #6 0xffffffff80daf5a1 at ffs_sync+0x501 #7 0xffffffff80b8742f at sync_fsync+0xff #8 0xffffffff810e2b2d at VOP_FSYNC_APV+0xfd #9 0xffffffff80b85264 at sched_sync+0x284 #10 0xffffffff80a6f2a4 at fork_exit+0x84 #11 0xffffffff80f4808e at fork_trampoline+0xe ino 241268, on dev gpt/tmp db:1:pho1> show allchains chain 1: thread 100170 (pid 1937, sh) blocked on lockmgr ufsSHARED (count 1) chain 2: thread 100143 (pid 1936, cron) sleeping on 0xfffff80013edd000 "piperd" chain 3: thread 100182 (pid 1931, sh) blocked on lockmgr ufsSHARED (count 1) chain 4: thread 100168 (pid 1930, cron) sleeping on 0xfffff8006663fbe0 "piperd" chain 5: thread 100145 (pid 1925, sh) blocked on lockmgr ufsSHARED (count 1) chain 6: thread 100219 (pid 1924, cron) sleeping on 0xfffff802cab9c5f0 "piperd" chain 7: thread 100187 (pid 1923, sshd) sleeping on 0xffffffff819947b8 "newbuf" chain 8: thread 100233 (pid 1917, sh) blocked on lockmgr ufsSHARED (count 1) chain 9: thread 100165 (pid 1915, newsyslog) sleeping on 0xffffffff819947b8 "newbuf" chain 10: thread 100218 (pid 1914, cron) sleeping on 0xfffff802cab358e8 "piperd" chain 11: thread 100220 (pid 1913, cron) sleeping on 0xfffff80013d092f8 "piperd" chain 12: thread 100162 (pid 1911, sh) sleeping on 0xffffffff819947b8 "newbuf" chain 13: thread 100234 (pid 1909, cron) sleeping on 0xfffff80013c835f0 "piperd" chain 14: thread 100247 (pid 1908, sh) sleeping on 0xffffffff819947b8 "newbuf" chain 15: thread 100226 (pid 1907, fdatasync) sleeping on 0xfffff80003fb6880 "uwait" chain 16: thread 100256 (pid 1907, fdatasync) blocked on lockmgr ufsEXCL thread 100258 (pid 1907, fdatasync) sleeping on 0xffffffff819947b8 "newbuf" chain 17: thread 100257 (pid 1907, fdatasync) sleeping on 0xffffffff819947b8 "newbuf" chain 18: thread 100258 (pid 1907, fdatasync) sleeping on 0xffffffff819947b8 "newbuf" chain 19: thread 100215 (pid 1884, sh) blocked on lockmgr EXCL thread 0 (pid -512, ) is inactive chain 20: thread 100225 (pid 1883, tee) sleeping on 0xfffff80013c7dbe0 "piperd" chain 21: thread 100167 (pid 1882, sh) sleeping on 0xfffff80013da3000 "wait" chain 22: thread 100211 (pid 1264, top) sleeping on 0xfffff800133e76c0 "select" chain 23: thread 100144 (pid 1252, bash) sleeping on 0xfffff80013c43a70 "wait" chain 24: thread 100146 (pid 1249, csh) sleeping on 0xfffff80013c8bb18 "pause" chain 25: thread 100204 (pid 1248, su) sleeping on 0xfffff80004194000 "wait" chain 26: thread 100221 (pid 1247, bash) sleeping on 0xfffff802cab0fa70 "wait" chain 27: thread 100206 (pid 1246, sshd) sleeping on 0xfffff80013251cc0 "select" chain 28: thread 100188 (pid 993, sshd) sleeping on 0xfffff800041c8a40 "select" chain 29: thread 100205 (pid 942, sh) sleeping on 0xfffff802b6249a70 "wait" chain 30: thread 100202 (pid 807, bash) blocked on lockmgr 8EXCL thread 0 (pid -512, ) ??? (0x6) chain 31: thread 100201 (pid 805, csh) sleeping on 0xfffff80013bd65e0 "pause" chain 32: thread 100200 (pid 804, su) sleeping on 0xfffff801abb0b538 "wait" chain 33: thread 100199 (pid 803, bash) sleeping on 0xfffff80013bd6a70 "wait" chain 34: thread 100183 (pid 802, sshd) sleeping on 0xfffff80013281ec0 "select" chain 35: thread 100189 (pid 799, sshd) sleeping on 0xfffff80013251ec0 "select" chain 36: thread 100190 (pid 796, csh) sleeping on 0xffffffff819947b8 "newbuf" chain 37: thread 100198 (pid 795, login) sleeping on 0xfffff801abb0ba70 "wait" chain 38: thread 100197 (pid 794, getty) sleeping on 0xfffff800060228b0 "ttyin" chain 39: thread 100196 (pid 793, getty) sleeping on 0xfffff80003eea4b0 "ttyin" chain 40: thread 100195 (pid 792, getty) sleeping on 0xfffff80003eea8b0 "ttyin" chain 41: thread 100194 (pid 791, getty) sleeping on 0xfffff80003eeacb0 "ttyin" chain 42: thread 100193 (pid 790, getty) sleeping on 0xfffff80003eeb0b0 "ttyin" chain 43: thread 100192 (pid 789, getty) sleeping on 0xfffff80003eeb4b0 "ttyin" chain 44: thread 100178 (pid 788, getty) sleeping on 0xfffff80003eeb8b0 "ttyin" chain 45: thread 100179 (pid 787, getty) sleeping on 0xfffff80003eebcb0 "ttyin" chain 46: thread 100140 (pid 786, getty) sleeping on 0xfffff80003eec0b0 "ttyin" chain 47: thread 100180 (pid 752, cron) sleeping on 0xffffffff81d417eb "nanslp" chain 48: thread 100171 (pid 748, sshd) sleeping on 0xfffff80013251dc0 "select" chain 49: thread 100185 (pid 683, NLM: master) sleeping on 0xfffff80013e6da60 "rpcsvc" chain 50: thread 100160 (pid 680, rpc.statd) sleeping on 0xfffff800133ab740 "select" chain 51: thread 100151 (pid 655, casperd) sleeping on 0xfffff800039302c0 "select" chain 52: thread 100163 (pid 654, casperd) sleeping on 0xfffff80013116c40 "select" chain 53: thread 100177 (pid 623, dhclient) sleeping on 0xfffff80013b03d48 "piperd" chain 54: thread 100148 (pid 562, dhclient) sleeping on 0xfffff80013cc8a70 "wait" chain 55: thread 100173 (pid 559, dhclient) sleeping on 0xfffff80006101940 "select" chain 56: thread 100158 (pid 537, rpcbind) sleeping on 0xfffff80013307b40 "select" chain 57: thread 100164 (pid 532, syslogd) sleeping on 0xfffff80003930340 "select" chain 58: thread 100159 (pid 457, devd) sleeping on 0xfffff80006101a40 "select" chain 59: thread 100133 (pid 33, syncer) sleeping on 0xffffffff819947b8 "newbuf" chain 60: thread 100130 (pid 32, vnlru) sleeping on 0xfffff80013189000 "vlruwt" chain 61: thread 100128 (pid 31, bufdaemon) sleeping on 0xffffffff81ee90a8 "qsleep" chain 62: thread 100131 (pid 31, bufspacedaemon-0) sleeping on 0xffffffff81963780 "-" chain 63: thread 100132 (pid 31, bufspacedaemon-1) sleeping on 0xffffffff8196ba40 "-" chain 64: thread 100134 (pid 31, bufspacedaemon-2) sleeping on 0xffffffff81973d00 "-" chain 65: thread 100136 (pid 31, bufspacedaemon-3) sleeping on 0xffffffff8197bfc0 "-" chain 66: thread 100137 (pid 31, bufspacedaemon-4) sleeping on 0xffffffff81984280 "-" chain 67: thread 100138 (pid 31, bufspacedaemon-5) sleeping on 0xffffffff8198c540 "-" chain 68: thread 100139 (pid 31, bufspacedaemon-6) sleeping on 0xffffffff819947b8 "bufspace" chain 69: thread 100150 (pid 31, / worker) sleeping on 0xfffff800133212e8 "sdflush" chain 70: thread 100210 (pid 31, /tmp worker) sleeping on 0xffffffff819947b8 "newbuf" chain 71: thread 100127 (pid 30, vmdaemon) sleeping on 0xffffffff81ef5618 "psleep" chain 72: thread 100126 (pid 29, dom0) sleeping on 0xffffffff81f645e0 "psleep" chain 73: thread 100129 (pid 29, laundry: dom0) sleeping on 0xffffffff81f645e8 "launds" chain 74: thread 100135 (pid 29, uma) sleeping on 0xffffffff80dcedb0 "umarcl" chain 75: thread 100125 (pid 28, enc_daemon1) sleeping on 0xfffff80013091000 "idle" chain 76: thread 100124 (pid 27, enc_daemon0) sleeping on 0xfffff80013091538 "idle" chain 77: thread 100118 (pid 26, rand_harvestq) sleeping on 0xffffffff81c147c8 "-" chain 78: thread 100117 (pid 25, sctp_iterator) sleeping on 0xffffffff81f63080 "waiting_for_work" chain 79: thread 100116 (pid 24, soaiod4) sleeping on 0xffffffff81ee8aa4 "-" chain 80: thread 100115 (pid 23, soaiod3) sleeping on 0xffffffff81ee8aa4 "-" chain 81: thread 100114 (pid 22, soaiod2) sleeping on 0xffffffff81ee8aa4 "-" chain 82: thread 100113 (pid 21, soaiod1) sleeping on 0xffffffff81ee8aa4 "-" chain 83: thread 100094 (pid 20, usbus0) sleeping on 0xfffffe009c7e1460 "-" chain 84: thread 100095 (pid 20, usbus0) sleeping on 0xfffffe009c7e14b8 "-" chain 85: thread 100096 (pid 20, usbus0) sleeping on 0xfffffe009c7e1510 "-" chain 86: thread 100097 (pid 20, usbus0) sleeping on 0xfffffe009c7e1568 "-" chain 87: thread 100098 (pid 20, usbus0) sleeping on 0xfffffe009c7e15c0 "-" chain 88: thread 100100 (pid 20, usbus1) sleeping on 0xfffffe009a5eed10 "-" chain 89: thread 100101 (pid 20, usbus1) sleeping on 0xfffffe009a5eed68 "-" chain 90: thread 100102 (pid 20, usbus1) sleeping on 0xfffffe009a5eedc0 "-" chain 91: thread 100103 (pid 20, usbus1) sleeping on 0xfffffe009a5eee18 "-" chain 92: thread 100104 (pid 20, usbus1) sleeping on 0xfffffe009a5eee70 "-" chain 93: thread 100105 (pid 20, usbus2) sleeping on 0xfffffe009ca62d10 "-" chain 94: thread 100106 (pid 20, usbus2) sleeping on 0xfffffe009ca62d68 "-" chain 95: thread 100107 (pid 20, usbus2) sleeping on 0xfffffe009ca62dc0 "-" chain 96: thread 100108 (pid 20, usbus2) sleeping on 0xfffffe009ca62e18 "-" chain 97: thread 100109 (pid 20, usbus2) sleeping on 0xfffffe009ca62e70 "-" chain 98: thread 100089 (pid 19, doneq0) sleeping on 0xffffffff81aefac0 "-" chain 99: thread 100090 (pid 19, doneq1) sleeping on 0xffffffff81aefb40 "-" chain 100: thread 100091 (pid 19, doneq2) sleeping on 0xffffffff81aefbc0 "-" chain 101: thread 100123 (pid 19, scanner) sleeping on 0xffffffff81aef988 "-" chain 102: thread 100088 (pid 18, crypto returns 11) sleeping on 0xfffff80003ae7450 "crypto_ret_wait" chain 103: thread 100087 (pid 17, crypto returns 10) sleeping on 0xfffff80003ae73f0 "crypto_ret_wait" chain 104: thread 100086 (pid 16, crypto returns 9) sleeping on 0xfffff80003ae7390 "crypto_ret_wait" chain 105: thread 100085 (pid 15, crypto returns 8) sleeping on 0xfffff80003ae7330 "crypto_ret_wait" chain 106: thread 100084 (pid 14, crypto returns 7) sleeping on 0xfffff80003ae72d0 "crypto_ret_wait" chain 107: thread 100083 (pid 9, crypto returns 6) sleeping on 0xfffff80003ae7270 "crypto_ret_wait" chain 108: thread 100082 (pid 8, crypto returns 5) sleeping on 0xfffff80003ae7210 "crypto_ret_wait" chain 109: thread 100081 (pid 7, crypto returns 4) sleeping on 0xfffff80003ae71b0 "crypto_ret_wait" chain 110: thread 100080 (pid 6, crypto returns 3) sleeping on 0xfffff80003ae7150 "crypto_ret_wait" chain 111: thread 100079 (pid 5, crypto returns 2) sleeping on 0xfffff80003ae70f0 "crypto_ret_wait" chain 112: thread 100078 (pid 4, crypto returns 1) sleeping on 0xfffff80003ae7090 "crypto_ret_wait" chain 113: thread 100077 (pid 3, crypto returns 0) sleeping on 0xfffff80003ae7030 "crypto_ret_wait" chain 114: thread 100076 (pid 2, crypto) sleeping on 0xffffffff81eefe58 "crypto_wait" chain 115: thread 100060 (pid 13, g_event) sleeping on 0xffffffff81f3fae0 "-" chain 116: thread 100061 (pid 13, g_up) sleeping on 0xffffffff81f3fae8 "-" chain 117: thread 100062 (pid 13, g_down) sleeping on 0xffffffff81f3faf0 "-" chain 118: thread 100002 (pid 1, init) sleeping on 0xfffff80003934538 "wait" chain 119: thread 100001 (pid 10, audit) sleeping on 0xffffffff81f63d68 "audit_worker_cv" chain 120: thread 100000 (pid 0, swapper) sleeping on 0xffffffff81f3fb28 "swapin" chain 121: thread 100015 (pid 0, aiod_kick taskq) sleeping on 0xfffff80003970c00 "-" chain 122: thread 100018 (pid 0, thread taskq) sleeping on 0xfffff80003970400 "-" chain 123: thread 100020 (pid 0, kqueue_ctx taskq) sleeping on 0xfffff8000396fe00 "-" chain 124: thread 100021 (pid 0, if_io_tqg_0) sleeping on 0xfffff8000397e000 "-" chain 125: thread 100022 (pid 0, if_io_tqg_1) sleeping on 0xfffff8000397de00 "-" chain 126: thread 100023 (pid 0, if_io_tqg_2) sleeping on 0xfffff8000397dd00 "-" chain 127: thread 100024 (pid 0, if_io_tqg_3) sleeping on 0xfffff8000397dc00 "-" chain 128: thread 100025 (pid 0, if_io_tqg_4) sleeping on 0xfffff8000397db00 "-" chain 129: thread 100026 (pid 0, if_io_tqg_5) sleeping on 0xfffff8000397da00 "-" chain 130: thread 100027 (pid 0, if_io_tqg_6) sleeping on 0xfffff8000397d900 "-" chain 131: thread 100028 (pid 0, if_io_tqg_7) sleeping on 0xfffff8000397d800 "-" chain 132: thread 100029 (pid 0, if_io_tqg_8) sleeping on 0xfffff8000397d700 "-" chain 133: thread 100030 (pid 0, if_io_tqg_9) sleeping on 0xfffff8000397d600 "-" chain 134: thread 100031 (pid 0, if_io_tqg_10) sleeping on 0xfffff8000397d500 "-" chain 135: thread 100032 (pid 0, if_io_tqg_11) sleeping on 0xfffff8000397d400 "-" chain 136: thread 100033 (pid 0, if_config_tqg_0) sleeping on 0xfffff8000397d300 "-" chain 137: thread 100034 (pid 0, softirq_0) sleeping on 0xfffff80003a07100 "-" chain 138: thread 100035 (pid 0, softirq_1) sleeping on 0xfffff80003a07000 "-" chain 139: thread 100036 (pid 0, softirq_2) sleeping on 0xfffff80003a06e00 "-" chain 140: thread 100037 (pid 0, softirq_3) sleeping on 0xfffff80003a06d00 "-" chain 141: thread 100038 (pid 0, softirq_4) sleeping on 0xfffff80003a06c00 "-" chain 142: thread 100039 (pid 0, softirq_5) sleeping on 0xfffff80003a06b00 "-" chain 143: thread 100040 (pid 0, softirq_6) sleeping on 0xfffff80003a06a00 "-" chain 144: thread 100041 (pid 0, softirq_7) sleeping on 0xfffff80003a06900 "-" chain 145: thread 100042 (pid 0, softirq_8) sleeping on 0xfffff80003a06800 "-" chain 146: thread 100043 (pid 0, softirq_9) sleeping on 0xfffff80003a06700 "-" chain 147: thread 100044 (pid 0, softirq_10) sleeping on 0xfffff80003a06600 "-" chain 148: thread 100045 (pid 0, softirq_11) sleeping on 0xfffff80003a06500 "-" chain 149: thread 100063 (pid 0, firmware taskq) sleeping on 0xfffff80003a36200 "-" chain 150: thread 100064 (pid 0, crypto_0) sleeping on 0xfffff80003a36000 "-" chain 151: thread 100065 (pid 0, crypto_1) sleeping on 0xfffff80003a36000 "-" chain 152: thread 100066 (pid 0, crypto_2) sleeping on 0xfffff80003a36000 "-" chain 153: thread 100067 (pid 0, crypto_3) sleeping on 0xfffff80003a36000 "-" chain 154: thread 100068 (pid 0, crypto_4) sleeping on 0xfffff80003a36000 "-" chain 155: thread 100069 (pid 0, crypto_5) sleeping on 0xfffff80003a36000 "-" chain 156: thread 100070 (pid 0, crypto_6) sleeping on 0xfffff80003a36000 "-" chain 157: thread 100071 (pid 0, crypto_7) sleeping on 0xfffff80003a36000 "-" chain 158: thread 100072 (pid 0, crypto_8) sleeping on 0xfffff80003a36000 "-" chain 159: thread 100073 (pid 0, crypto_9) sleeping on 0xfffff80003a36000 "-" chain 160: thread 100074 (pid 0, crypto_10) sleeping on 0xfffff80003a36000 "-" chain 161: thread 100075 (pid 0, crypto_11) sleeping on 0xfffff80003a36000 "-" chain 162: thread 100112 (pid 0, mca taskq) blocked on lockmgr (null)EXCL KDB: reentering KDB: stack backtrace: db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe0078959e00 kdb_reenter() at kdb_reenter+0x2f/frame 0xfffffe0078959e10 trap() at trap+0x5c/frame 0xfffffe0078959f20 calltrap() at calltrap+0x8/frame 0xfffffe0078959f20 --- trap 0xc, rip = 0xffffffff80b0c9aa, rsp = 0xfffffe0078959ff0, rbp = 0xfffffe007895a010 --- print_lockchain() at print_lockchain+0x3a/frame 0xfffffe007895a010 db_show_allchains() at db_show_allchains+0x80/frame 0xfffffe007895a040 db_command() at db_command+0x279/frame 0xfffffe007895a110 db_script_exec() at db_script_exec+0x188/frame 0xfffffe007895a1b0 db_command() at db_command+0x279/frame 0xfffffe007895a280 db_script_exec() at db_script_exec+0x188/frame 0xfffffe007895a320 db_command() at db_command+0x279/frame 0xfffffe007895a3f0 db_command_loop() at db_command_loop+0x64/frame 0xfffffe007895a400 db_trap() at db_trap+0xef/frame 0xfffffe007895a490 kdb_trap() at kdb_trap+0x1c3/frame 0xfffffe007895a500 trap() at trap+0x328/frame 0xfffffe007895a610 calltrap() at calltrap+0x8/frame 0xfffffe007895a610 --- trap 0x3, rip = 0xffffffff80af3be6, rsp = 0xfffffe007895a6e0, rbp = 0xfffffe007895a700 --- kdb_alt_break_internal() at kdb_alt_break_internal+0x106/frame 0xfffffe007895a700 kdb_alt_break() at kdb_alt_break+0xb/frame 0xfffffe007895a710 uart_intr_rxready() at uart_intr_rxready+0x9a/frame 0xfffffe007895a740 uart_intr() at uart_intr+0x138/frame 0xfffffe007895a780 intr_event_handle() at intr_event_handle+0xbc/frame 0xfffffe007895a7d0 intr_execute_handlers() at intr_execute_handlers+0x58/frame 0xfffffe007895a800 lapic_handle_intr() at lapic_handle_intr+0x6d/frame 0xfffffe007895a840 apic_isr1_u() at apic_isr1_u+0xc1/frame 0xfffffe007895a910 acpi_cpu_idle() at acpi_cpu_idle+0x2ee/frame 0xfffffe007895a960 cpu_idle_acpi() at cpu_idle_acpi+0x3f/frame 0xfffffe007895a980 cpu_idle() at cpu_idle+0x8f/frame 0xfffffe007895a9a0 sched_idletd() at sched_idletd+0x517/frame 0xfffffe007895aa70 fork_exit() at fork_exit+0x84/frame 0xfffffe007895aab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe007895aab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Script command 'show allchains' returned error db:1:pho1> show mount 0xfffff80013992000 /dev/gpt/work on / (ufs) 0xfffff80013993000 devfs on /dev (devfs) 0xfffff80013d83000 192.168.5.1:tank/users/pho/mercat1 on /media (nfs) 0xfffff80013d74000 /dev/gpt/tmp on /tmp (ufs) More info: show mount db:1:pho1> show page vm_cnt.v_free_count: 6394853 vm_cnt.v_inactive_count: 1270865 vm_cnt.v_active_count: 8364 vm_cnt.v_laundry_count: 0 vm_cnt.v_wire_count: 465079 vm_cnt.v_free_reserved: 10652 vm_cnt.v_free_min: 51303 vm_cnt.v_free_target: 173256 vm_cnt.v_inactive_target: 259884 db:1:pho1> show freepages DOMAIN: 0 FREE LIST 0: ORDER (SIZE) | NUMBER | POOL 0 | POOL 1 -- -- -- -- -- -- 12 (016384K) | 001522 | 000000 11 (008192K) | 000000 | 000000 10 (004096K) | 000000 | 000001 09 (002048K) | 000001 | 000001 08 (001024K) | 000002 | 000000 07 (000512K) | 000012 | 000000 06 (000256K) | 000128 | 000000 05 (000128K) | 000327 | 000001 04 (000064K) | 000000 | 000001 03 (000032K) | 000000 | 000001 02 (000016K) | 000001 | 000001 01 (000008K) | 000001 | 000000 00 (000004K) | 000001 | 000000 FREE LIST 1: ORDER (SIZE) | NUMBER | POOL 0 | POOL 1 -- -- -- -- -- -- 12 (016384K) | 000000 | 000000 11 (008192K) | 000000 | 000000 10 (004096K) | 000000 | 000000 09 (002048K) | 000000 | 000000 08 (001024K) | 000001 | 000000 07 (000512K) | 000000 | 000000 06 (000256K) | 000001 | 000000 05 (000128K) | 000001 | 000000 04 (000064K) | 000002 | 000000 03 (000032K) | 000000 | 000000 02 (000016K) | 000000 | 000000 01 (000008K) | 000001 | 000000 00 (000004K) | 000000 | 000000 db:1:pho1> show pageq pq_free 6394853 dom 0 page_cnt 8130451 free 6394853 pq_act 8364 pq_inact 1270865 pq_laund 0 pq_unsw 0 db:1:pho1> show bufqueues bqempty: 2707 Buf domain 0 freebufs 9028 lofreebufs 219 hifreebufs 329 bufspace 178237440 maxbufspace 247939072 hibufspace 247845449 lobufspace 235453176 bufspacethresh 241649312 numdirtybuffers 944 lodirtybuffers 1893 hidirtybufferss 3786 dirtybufthresh 3407 cleanq count 5034 dirtyq count 944 wakeup 0 lim 25 CPU 3, 15, 11, 5, 18, 11, 6, 10, 11, 21, 5, 10, Buf domain 1 freebufs 10066 lofreebufs 219 hifreebufs 329 bufspace 156794880 maxbufspace 247939072 hibufspace 247845449 lobufspace 235453176 bufspacethresh 241649312 numdirtybuffers 1040 lodirtybuffers 1893 hidirtybufferss 3786 dirtybufthresh 3407 cleanq count 3911 dirtyq count 1040 wakeup 0 lim 25 CPU 7, 11, 10, 8, 19, 5, 5, 10, 17, 1, 2, 21, Buf domain 2 freebufs 5882 lofreebufs 219 hifreebufs 329 bufspace 175742976 maxbufspace 247939072 hibufspace 247845449 lobufspace 235453176 bufspacethresh 241649312 numdirtybuffers 919 lodirtybuffers 1893 hidirtybufferss 3786 dirtybufthresh 3407 cleanq count 8245 dirtyq count 919 wakeup 0 lim 25 CPU 4, 11, 11, 2, 17, 0, 5, 0, 10, 8, 16, 3, Buf domain 3 freebufs 6305 lofreebufs 219 hifreebufs 329 bufspace 196466688 maxbufspace 247939072 hibufspace 247845449 lobufspace 235453176 bufspacethresh 241649312 numdirtybuffers 967 lodirtybuffers 1893 hidirtybufferss 3786 dirtybufthresh 3407 cleanq count 7731 dirtyq count 967 wakeup 0 lim 25 CPU 10, 17, 3, 16, 11, 21, 0, 4, 16, 8, 5, 19, Buf domain 4 freebufs 5890 lofreebufs 219 hifreebufs 329 bufspace 175132672 maxbufspace 247939072 hibufspace 247845449 lobufspace 235453176 bufspacethresh 241649312 numdirtybuffers 918 lodirtybuffers 1893 hidirtybufferss 3786 dirtybufthresh 3407 cleanq count 8190 dirtyq count 918 wakeup 0 lim 25 CPU 9, 24, 1, 12, 2, 19, 9, 2, 17, 3, 20, 17, Buf domain 5 freebufs 10253 lofreebufs 219 hifreebufs 329 bufspace 141303808 maxbufspace 247939072 hibufspace 247845449 lobufspace 235453176 bufspacethresh 241649312 numdirtybuffers 896 lodirtybuffers 1893 hidirtybufferss 3786 dirtybufthresh 3407 cleanq count 3896 dirtyq count 896 wakeup 0 lim 25 CPU 4, 2, 2, 4, 3, 20, 10, 1, 15, 2, 16, 9, Buf domain 6 freebufs 7567 lofreebufs 219 hifreebufs 329 bufspace 247922688 maxbufspace 247939072 hibufspace 247845449 lobufspace 235453176 bufspacethresh 241649312 numdirtybuffers 949 lodirtybuffers 1893 hidirtybufferss 3786 dirtybufthresh 3407 cleanq count 0 dirtyq count 949 wakeup 1 lim 25 CPU 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, db:1:pho1> ps pid ppid pgrp uid state wmesg wchan cmd 1937 1936 1937 2 Ds ufs 0xfffff8026c5cdcc0 sh 1936 752 752 0 S piperd 0xfffff80013edd000 cron 1931 1930 1931 2 Ds ufs 0xfffff8026c5cdcc0 sh 1930 752 752 0 S piperd 0xfffff8006663fbe0 cron 1925 1924 1925 2 Ds ufs 0xfffff8026c5cdcc0 sh 1924 752 752 0 S piperd 0xfffff802cab9c5f0 cron 1923 748 1923 0 Ds newbuf 0xffffffff819947b8 sshd 1917 1914 1917 2 Ds ufs 0xfffff8026c5cdcc0 sh 1915 1913 1915 0 Ds newbuf 0xffffffff819947b8 newsyslog 1914 752 752 0 S piperd 0xfffff802cab358e8 cron 1913 752 752 0 S piperd 0xfffff80013d092f8 cron 1911 1909 1911 2 Ds newbuf 0xffffffff819947b8 sh 1909 752 752 0 S piperd 0xfffff80013c835f0 cron 1908 562 562 0 D newbuf 0xffffffff819947b8 sh 1907 1884 942 65534 D+ (threaded) fdatasync 100226 S uwait 0xfffff80003fb6880 fdatasync 100256 D ufs 0xfffff802ca92f558 fdatasync 100257 D newbuf 0xffffffff819947b8 fdatasync 100258 D newbuf 0xffffffff819947b8 fdatasync 1884 1882 942 0 S+ wait 0xfffff80066213a70 sh 1883 942 942 0 S+ piperd 0xfffff80013c7dbe0 tee 1882 942 942 0 S+ wait 0xfffff80013da3000 sh 1264 1252 1264 0 S+ select 0xfffff800133e76c0 top 1252 1249 1252 0 S+ wait 0xfffff80013c43a70 bash 1249 1248 1249 0 S+ pause 0xfffff80013c8bb18 csh 1248 1247 1248 2006 S+ wait 0xfffff80004194000 su 1247 1246 1247 2006 Ss+ wait 0xfffff802cab0fa70 bash 1246 993 993 2006 S select 0xfffff80013251cc0 sshd 993 748 993 0 Ss select 0xfffff800041c8a40 sshd 942 807 942 0 S+ wait 0xfffff802b6249a70 sh 807 805 807 0 S+ wait 0xfffff801abb0b000 bash 805 804 805 0 S+ pause 0xfffff80013bd65e0 csh 804 803 804 2006 S+ wait 0xfffff801abb0b538 su 803 802 803 2006 Ss+ wait 0xfffff80013bd6a70 bash 802 799 799 2006 S select 0xfffff80013281ec0 sshd 799 748 799 0 Ss select 0xfffff80013251ec0 sshd 796 795 796 0 D+ newbuf 0xffffffff819947b8 csh 795 1 795 0 Ss+ wait 0xfffff801abb0ba70 login 794 1 794 0 Ss+ ttyin 0xfffff800060228b0 getty 793 1 793 0 Ss+ ttyin 0xfffff80003eea4b0 getty 792 1 792 0 Ss+ ttyin 0xfffff80003eea8b0 getty 791 1 791 0 Ss+ ttyin 0xfffff80003eeacb0 getty 790 1 790 0 Ss+ ttyin 0xfffff80003eeb0b0 getty 789 1 789 0 Ss+ ttyin 0xfffff80003eeb4b0 getty 788 1 788 0 Ss+ ttyin 0xfffff80003eeb8b0 getty 787 1 787 0 Ss+ ttyin 0xfffff80003eebcb0 getty 786 1 786 0 Ss+ ttyin 0xfffff80003eec0b0 getty 752 1 752 0 Ss nanslp 0xffffffff81d417eb cron 748 1 748 0 Ss select 0xfffff80013251dc0 sshd 683 1 683 0 Ss rpcsvc 0xfffff80013e6da60 NLM: master 680 1 680 0 Ss select 0xfffff800133ab740 rpc.statd 655 1 655 0 Ss select 0xfffff800039302c0 casperd 654 1 654 0 Ss select 0xfffff80013116c40 casperd 623 1 623 65 Ss piperd 0xfffff80013b03d48 dhclient 562 1 562 0 Ss wait 0xfffff80013cc8a70 dhclient 559 1 559 0 Ss select 0xfffff80006101940 dhclient 537 1 537 0 Ss select 0xfffff80013307b40 rpcbind 532 1 532 0 Ss select 0xfffff80003930340 syslogd 457 1 457 0 Ss select 0xfffff80006101a40 devd 33 0 0 0 DL newbuf 0xffffffff819947b8 [syncer] 32 0 0 0 DL vlruwt 0xfffff80013189000 [vnlru] 31 0 0 0 DL (threaded) [bufdaemon] 100128 D qsleep 0xffffffff81ee90a8 [bufdaemon] 100131 D - 0xffffffff81963780 [bufspacedaemon-0] 100132 D - 0xffffffff8196ba40 [bufspacedaemon-1] 100134 D - 0xffffffff81973d00 [bufspacedaemon-2] 100136 D - 0xffffffff8197bfc0 [bufspacedaemon-3] 100137 D - 0xffffffff81984280 [bufspacedaemon-4] 100138 D - 0xffffffff8198c540 [bufspacedaemon-5] 100139 D bufspac 0xffffffff819947b8 [bufspacedaemon-6] 100150 D sdflush 0xfffff800133212e8 [/ worker] 100210 D newbuf 0xffffffff819947b8 [/tmp worker] 30 0 0 0 DL psleep 0xffffffff81ef5618 [vmdaemon] 29 0 0 0 DL (threaded) [pagedaemon] 100126 D psleep 0xffffffff81f645e0 [dom0] 100129 D launds 0xffffffff81f645e8 [laundry: dom0] 100135 D umarcl 0xffffffff80dcedb0 [uma] 28 0 0 0 DL idle 0xfffff80013091000 [enc_daemon1] 27 0 0 0 DL idle 0xfffff80013091538 [enc_daemon0] 26 0 0 0 DL - 0xffffffff81c147c8 [rand_harvestq] 25 0 0 0 DL waiting 0xffffffff81f63080 [sctp_iterator] 24 0 0 0 DL - 0xffffffff81ee8aa4 [soaiod4] 23 0 0 0 DL - 0xffffffff81ee8aa4 [soaiod3] 22 0 0 0 DL - 0xffffffff81ee8aa4 [soaiod2] 21 0 0 0 DL - 0xffffffff81ee8aa4 [soaiod1] 20 0 0 0 DL (threaded) [usb] 100094 D - 0xfffffe009c7e1460 [usbus0] 100095 D - 0xfffffe009c7e14b8 [usbus0] 100096 D - 0xfffffe009c7e1510 [usbus0] 100097 D - 0xfffffe009c7e1568 [usbus0] 100098 D - 0xfffffe009c7e15c0 [usbus0] 100100 D - 0xfffffe009a5eed10 [usbus1] 100101 D - 0xfffffe009a5eed68 [usbus1] 100102 D - 0xfffffe009a5eedc0 [usbus1] 100103 D - 0xfffffe009a5eee18 [usbus1] 100104 D - 0xfffffe009a5eee70 [usbus1] 100105 D - 0xfffffe009ca62d10 [usbus2] 100106 D - 0xfffffe009ca62d68 [usbus2] 100107 D - 0xfffffe009ca62dc0 [usbus2] 100108 D - 0xfffffe009ca62e18 [usbus2] 100109 D - 0xfffffe009ca62e70 [usbus2] 19 0 0 0 DL (threaded) [cam] 100089 D - 0xffffffff81aefac0 [doneq0] 100090 D - 0xffffffff81aefb40 [doneq1] 100091 D - 0xffffffff81aefbc0 [doneq2] 100123 D - 0xffffffff81aef988 [scanner] 18 0 0 0 DL crypto_ 0xfffff80003ae7450 [crypto returns 11] 17 0 0 0 DL crypto_ 0xfffff80003ae73f0 [crypto returns 10] 16 0 0 0 DL crypto_ 0xfffff80003ae7390 [crypto returns 9] 15 0 0 0 DL crypto_ 0xfffff80003ae7330 [crypto returns 8] 14 0 0 0 DL crypto_ 0xfffff80003ae72d0 [crypto returns 7] 9 0 0 0 DL crypto_ 0xfffff80003ae7270 [crypto returns 6] 8 0 0 0 DL crypto_ 0xfffff80003ae7210 [crypto returns 5] 7 0 0 0 DL crypto_ 0xfffff80003ae71b0 [crypto returns 4] 6 0 0 0 DL crypto_ 0xfffff80003ae7150 [crypto returns 3] 5 0 0 0 DL crypto_ 0xfffff80003ae70f0 [crypto returns 2] 4 0 0 0 DL crypto_ 0xfffff80003ae7090 [crypto returns 1] 3 0 0 0 DL crypto_ 0xfffff80003ae7030 [crypto returns 0] 2 0 0 0 DL crypto_ 0xffffffff81eefe58 [crypto] 13 0 0 0 DL (threaded) [geom] 100060 D - 0xffffffff81f3fae0 [g_event] 100061 D - 0xffffffff81f3fae8 [g_up] 100062 D - 0xffffffff81f3faf0 [g_down] 12 0 0 0 WL (threaded) [intr] 100016 I [swi6: task queue] 100017 I [swi6: Giant taskq] 100019 I [swi5: fast taskq] 100046 I [swi3: vm] 100047 I [swi4: clock (0)] 100048 I [swi4: clock (1)] 100049 I [swi4: clock (2)] 100050 I [swi4: clock (3)] 100051 I [swi4: clock (4)] 100052 I [swi4: clock (5)] 100053 I [swi4: clock (6)] 100054 I [swi4: clock (7)] 100055 I [swi4: clock (8)] 100056 I [swi4: clock (9)] 100057 I [swi4: clock (10)] 100058 I [swi4: clock (11)] 100059 I [swi1: netisr 0] 100092 I [irq264: ahci0] 100093 I [irq19: xhci0] 100099 I [irq18: ehci0 ehci1] 100110 I [irq279: ahci1] 100111 I [swi0: uart uart] 11 0 0 0 RL (threaded) [idle] 100003 Run CPU 0 [idle: cpu0] 100004 Run CPU 1 [idle: cpu1] 100005 Run CPU 2 [idle: cpu2] 100006 Run CPU 3 [idle: cpu3] 100007 Run CPU 4 [idle: cpu4] 100008 Run CPU 5 [idle: cpu5] 100009 Run CPU 6 [idle: cpu6] 100010 Run CPU 7 [idle: cpu7] 100011 Run CPU 8 [idle: cpu8] 100012 Run CPU 9 [idle: cpu9] 100013 Run CPU 10 [idle: cpu10] 100014 Run CPU 11 [idle: cpu11] 1 0 1 0 SLs wait 0xfffff80003934538 [init] 10 0 0 0 DL audit_w 0xffffffff81f63d68 [audit] 0 0 0 0 DLs (threaded) [kernel] 100000 D swapin 0xffffffff81f3fb28 [swapper] 100015 D - 0xfffff80003970c00 [aiod_kick taskq] 100018 D - 0xfffff80003970400 [thread taskq] 100020 D - 0xfffff8000396fe00 [kqueue_ctx taskq] 100021 D - 0xfffff8000397e000 [if_io_tqg_0] 100022 D - 0xfffff8000397de00 [if_io_tqg_1] 100023 D - 0xfffff8000397dd00 [if_io_tqg_2] 100024 D - 0xfffff8000397dc00 [if_io_tqg_3] 100025 D - 0xfffff8000397db00 [if_io_tqg_4] 100026 D - 0xfffff8000397da00 [if_io_tqg_5] 100027 D - 0xfffff8000397d900 [if_io_tqg_6] 100028 D - 0xfffff8000397d800 [if_io_tqg_7] 100029 D - 0xfffff8000397d700 [if_io_tqg_8] 100030 D - 0xfffff8000397d600 [if_io_tqg_9] 100031 D - 0xfffff8000397d500 [if_io_tqg_10] 100032 D - 0xfffff8000397d400 [if_io_tqg_11] 100033 D - 0xfffff8000397d300 [if_config_tqg_0] 100034 D - 0xfffff80003a07100 [softirq_0] 100035 D - 0xfffff80003a07000 [softirq_1] 100036 D - 0xfffff80003a06e00 [softirq_2] 100037 D - 0xfffff80003a06d00 [softirq_3] 100038 D - 0xfffff80003a06c00 [softirq_4] 100039 D - 0xfffff80003a06b00 [softirq_5] 100040 D - 0xfffff80003a06a00 [softirq_6] 100041 D - 0xfffff80003a06900 [softirq_7] 100042 D - 0xfffff80003a06800 [softirq_8] 100043 D - 0xfffff80003a06700 [softirq_9] 100044 D - 0xfffff80003a06600 [softirq_10] 100045 D - 0xfffff80003a06500 [softirq_11] 100063 D - 0xfffff80003a36200 [firmware taskq] 100064 D - 0xfffff80003a36000 [crypto_0] 100065 D - 0xfffff80003a36000 [crypto_1] 100066 D - 0xfffff80003a36000 [crypto_2] 100067 D - 0xfffff80003a36000 [crypto_3] 100068 D - 0xfffff80003a36000 [crypto_4] 100069 D - 0xfffff80003a36000 [crypto_5] 100070 D - 0xfffff80003a36000 [crypto_6] 100071 D - 0xfffff80003a36000 [crypto_7] 100072 D - 0xfffff80003a36000 [crypto_8] 100073 D - 0xfffff80003a36000 [crypto_9] 100074 D - 0xfffff80003a36000 [crypto_10] 100075 D - 0xfffff80003a36000 [crypto_11] 100112 D - 0xfffff800060cbd00 [mca taskq] 100119 D - 0xfffff80013199700 [acpi_task_0] 100120 D - 0xfffff80013199700 [acpi_task_1] 100121 D - 0xfffff80013199700 [acpi_task_2] 100122 D - 0xfffff80003a1f400 [CAM taskq] db:0:pho> allt Tracing command sh pid 1937 tid 100170 td 0xfffff80003a25560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee9ef80 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee9efb0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee9eff0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe009ee9f020 sleeplk() at sleeplk+0x149/frame 0xfffffe009ee9f080 lockmgr_xlock_hard() at lockmgr_xlock_hard+0x2f8/frame 0xfffffe009ee9f100 __lockmgr_args() at __lockmgr_args+0x51a/frame 0xfffffe009ee9f1a0 ffs_lock() at ffs_lock+0xaf/frame 0xfffffe009ee9f1f0 VOP_LOCK1_APV() at VOP_LOCK1_APV+0xe1/frame 0xfffffe009ee9f220 _vn_lock() at _vn_lock+0x66/frame 0xfffffe009ee9f290 vget() at vget+0x98/frame 0xfffffe009ee9f2d0 cache_lookup() at cache_lookup+0x6fa/frame 0xfffffe009ee9f3b0 vfs_cache_lookup() at vfs_cache_lookup+0xac/frame 0xfffffe009ee9f410 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0xf7/frame 0xfffffe009ee9f440 lookup() at lookup+0x692/frame 0xfffffe009ee9f4e0 namei() at namei+0x53b/frame 0xfffffe009ee9f5a0 kern_execve() at kern_execve+0x3cc/frame 0xfffffe009ee9f900 sys_execve() at sys_execve+0x4c/frame 0xfffffe009ee9f980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee9fab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee9fab0 --- syscall (59, FreeBSD ELF64, sys_execve), rip = 0x8003ddb2a, rsp = 0x7fffffffe858, rbp = 0x7fffffffe9a0 --- Tracing command cron pid 1936 tid 100143 td 0xfffff8000608e000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee18690 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee186c0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee18700 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ee18760 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ee18790 _sleep() at _sleep+0x359/frame 0xfffffe009ee18840 pipe_read() at pipe_read+0x345/frame 0xfffffe009ee188a0 dofileread() at dofileread+0xba/frame 0xfffffe009ee188e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009ee18930 sys_read() at sys_read+0x86/frame 0xfffffe009ee18980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee18ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee18ab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x80043fcfa, rsp = 0x7fffffffe4d8, rbp = 0x7fffffffe4f0 --- Tracing command sh pid 1931 tid 100182 td 0xfffff8001396c560 sched_switch() at sched_switch+0x942/frame 0xfffffe009eedaf80 mi_switch() at mi_switch+0x18c/frame 0xfffffe009eedafb0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009eedaff0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe009eedb020 sleeplk() at sleeplk+0x149/frame 0xfffffe009eedb080 lockmgr_xlock_hard() at lockmgr_xlock_hard+0x2f8/frame 0xfffffe009eedb100 __lockmgr_args() at __lockmgr_args+0x51a/frame 0xfffffe009eedb1a0 ffs_lock() at ffs_lock+0xaf/frame 0xfffffe009eedb1f0 VOP_LOCK1_APV() at VOP_LOCK1_APV+0xe1/frame 0xfffffe009eedb220 _vn_lock() at _vn_lock+0x66/frame 0xfffffe009eedb290 vget() at vget+0x98/frame 0xfffffe009eedb2d0 cache_lookup() at cache_lookup+0x6fa/frame 0xfffffe009eedb3b0 vfs_cache_lookup() at vfs_cache_lookup+0xac/frame 0xfffffe009eedb410 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0xf7/frame 0xfffffe009eedb440 lookup() at lookup+0x692/frame 0xfffffe009eedb4e0 namei() at namei+0x53b/frame 0xfffffe009eedb5a0 kern_execve() at kern_execve+0x3cc/frame 0xfffffe009eedb900 sys_execve() at sys_execve+0x4c/frame 0xfffffe009eedb980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009eedbab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009eedbab0 --- syscall (59, FreeBSD ELF64, sys_execve), rip = 0x8003ddb2a, rsp = 0x7fffffffe858, rbp = 0x7fffffffe9a0 --- Tracing command cron pid 1930 tid 100168 td 0xfffff80013c52000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee95690 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee956c0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee95700 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ee95760 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ee95790 _sleep() at _sleep+0x359/frame 0xfffffe009ee95840 pipe_read() at pipe_read+0x345/frame 0xfffffe009ee958a0 dofileread() at dofileread+0xba/frame 0xfffffe009ee958e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009ee95930 sys_read() at sys_read+0x86/frame 0xfffffe009ee95980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee95ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee95ab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x80043fcfa, rsp = 0x7fffffffe4d8, rbp = 0x7fffffffe4f0 --- Tracing command sh pid 1925 tid 100145 td 0xfffff80013c02560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee21f80 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee21fb0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee21ff0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe009ee22020 sleeplk() at sleeplk+0x149/frame 0xfffffe009ee22080 lockmgr_xlock_hard() at lockmgr_xlock_hard+0x2f8/frame 0xfffffe009ee22100 __lockmgr_args() at __lockmgr_args+0x51a/frame 0xfffffe009ee221a0 ffs_lock() at ffs_lock+0xaf/frame 0xfffffe009ee221f0 VOP_LOCK1_APV() at VOP_LOCK1_APV+0xe1/frame 0xfffffe009ee22220 _vn_lock() at _vn_lock+0x66/frame 0xfffffe009ee22290 vget() at vget+0x98/frame 0xfffffe009ee222d0 cache_lookup() at cache_lookup+0x6fa/frame 0xfffffe009ee223b0 vfs_cache_lookup() at vfs_cache_lookup+0xac/frame 0xfffffe009ee22410 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0xf7/frame 0xfffffe009ee22440 lookup() at lookup+0x692/frame 0xfffffe009ee224e0 namei() at namei+0x53b/frame 0xfffffe009ee225a0 kern_execve() at kern_execve+0x3cc/frame 0xfffffe009ee22900 sys_execve() at sys_execve+0x4c/frame 0xfffffe009ee22980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee22ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee22ab0 --- syscall (59, FreeBSD ELF64, sys_execve), rip = 0x8003ddb2a, rsp = 0x7fffffffe858, rbp = 0x7fffffffe9a0 --- Tracing command cron pid 1924 tid 100219 td 0xfffff80013c50000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef8a690 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef8a6c0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef8a700 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef8a760 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef8a790 _sleep() at _sleep+0x359/frame 0xfffffe009ef8a840 pipe_read() at pipe_read+0x345/frame 0xfffffe009ef8a8a0 dofileread() at dofileread+0xba/frame 0xfffffe009ef8a8e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009ef8a930 sys_read() at sys_read+0x86/frame 0xfffffe009ef8a980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef8aab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef8aab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x80043fcfa, rsp = 0x7fffffffe4d8, rbp = 0x7fffffffe4f0 --- Tracing command sshd pid 1923 tid 100187 td 0xfffff80013d3e000 sched_switch() at sched_switch+0x942/frame 0xfffffe009eef4280 mi_switch() at mi_switch+0x18c/frame 0xfffffe009eef42b0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009eef42f0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe009eef4320 _sleep() at _sleep+0x367/frame 0xfffffe009eef43d0 bufspace_wait() at bufspace_wait+0x1df/frame 0xfffffe009eef4430 getnewbuf() at getnewbuf+0x28a/frame 0xfffffe009eef44a0 getblk() at getblk+0x640/frame 0xfffffe009eef4530 breadn_flags() at breadn_flags+0x34/frame 0xfffffe009eef4590 ffs_read() at ffs_read+0x23b/frame 0xfffffe009eef4640 VOP_READ_APV() at VOP_READ_APV+0xf7/frame 0xfffffe009eef4670 vn_read() at vn_read+0x1f1/frame 0xfffffe009eef46f0 vn_io_fault1() at vn_io_fault1+0x179/frame 0xfffffe009eef4830 vn_io_fault() at vn_io_fault+0x195/frame 0xfffffe009eef48a0 dofileread() at dofileread+0xba/frame 0xfffffe009eef48e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009eef4930 sys_read() at sys_read+0x86/frame 0xfffffe009eef4980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009eef4ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009eef4ab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800874cfa, rsp = 0x7fffffffe458, rbp = 0x7fffffffe470 --- Tracing command sh pid 1917 tid 100233 td 0xfffff802cab41560 sched_switch() at sched_switch+0x942/frame 0xfffffe009efc0f80 mi_switch() at mi_switch+0x18c/frame 0xfffffe009efc0fb0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009efc0ff0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe009efc1020 sleeplk() at sleeplk+0x149/frame 0xfffffe009efc1080 lockmgr_xlock_hard() at lockmgr_xlock_hard+0x2f8/frame 0xfffffe009efc1100 __lockmgr_args() at __lockmgr_args+0x51a/frame 0xfffffe009efc11a0 ffs_lock() at ffs_lock+0xaf/frame 0xfffffe009efc11f0 VOP_LOCK1_APV() at VOP_LOCK1_APV+0xe1/frame 0xfffffe009efc1220 _vn_lock() at _vn_lock+0x66/frame 0xfffffe009efc1290 vget() at vget+0x98/frame 0xfffffe009efc12d0 cache_lookup() at cache_lookup+0x6fa/frame 0xfffffe009efc13b0 vfs_cache_lookup() at vfs_cache_lookup+0xac/frame 0xfffffe009efc1410 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0xf7/frame 0xfffffe009efc1440 lookup() at lookup+0x692/frame 0xfffffe009efc14e0 namei() at namei+0x53b/frame 0xfffffe009efc15a0 kern_execve() at kern_execve+0x3cc/frame 0xfffffe009efc1900 sys_execve() at sys_execve+0x4c/frame 0xfffffe009efc1980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009efc1ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009efc1ab0 --- syscall (59, FreeBSD ELF64, sys_execve), rip = 0x8003ddb2a, rsp = 0x7fffffffe858, rbp = 0x7fffffffe9a0 --- Tracing command newsyslog pid 1915 tid 100165 td 0xfffff8001398c000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee85ef0 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee85f20 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee85f60 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe009ee85f90 _sleep() at _sleep+0x367/frame 0xfffffe009ee86040 bufspace_wait() at bufspace_wait+0x1df/frame 0xfffffe009ee860a0 getnewbuf() at getnewbuf+0x28a/frame 0xfffffe009ee86110 getblk() at getblk+0x640/frame 0xfffffe009ee861a0 breadn_flags() at breadn_flags+0x34/frame 0xfffffe009ee86200 ffs_blkatoff() at ffs_blkatoff+0x86/frame 0xfffffe009ee86260 ufsdirhash_lookup() at ufsdirhash_lookup+0x487/frame 0xfffffe009ee86340 ufs_lookup_ino() at ufs_lookup_ino+0x2c1/frame 0xfffffe009ee86450 VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0xf7/frame 0xfffffe009ee86480 vfs_cache_lookup() at vfs_cache_lookup+0xd6/frame 0xfffffe009ee864e0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0xf7/frame 0xfffffe009ee86510 lookup() at lookup+0x692/frame 0xfffffe009ee865b0 namei() at namei+0x53b/frame 0xfffffe009ee86670 kern_statat() at kern_statat+0x98/frame 0xfffffe009ee86880 sys_fstatat() at sys_fstatat+0x2f/frame 0xfffffe009ee86980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee86ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee86ab0 --- syscall (552, FreeBSD ELF64, sys_fstatat), rip = 0x80041829a, rsp = 0x7fffffffc8d8, rbp = 0x7fffffffc980 --- Tracing command cron pid 1914 tid 100218 td 0xfffff80013c50560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef85690 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef856c0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef85700 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef85760 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef85790 _sleep() at _sleep+0x359/frame 0xfffffe009ef85840 pipe_read() at pipe_read+0x345/frame 0xfffffe009ef858a0 dofileread() at dofileread+0xba/frame 0xfffffe009ef858e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009ef85930 sys_read() at sys_read+0x86/frame 0xfffffe009ef85980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef85ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef85ab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x80043fcfa, rsp = 0x7fffffffe4d8, rbp = 0x7fffffffe4f0 --- Tracing command cron pid 1913 tid 100220 td 0xfffff80013c44560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef8f690 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef8f6c0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef8f700 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef8f760 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef8f790 _sleep() at _sleep+0x359/frame 0xfffffe009ef8f840 pipe_read() at pipe_read+0x345/frame 0xfffffe009ef8f8a0 dofileread() at dofileread+0xba/frame 0xfffffe009ef8f8e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009ef8f930 sys_read() at sys_read+0x86/frame 0xfffffe009ef8f980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef8fab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef8fab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x80043fcfa, rsp = 0x7fffffffe4d8, rbp = 0x7fffffffe4f0 --- Tracing command sh pid 1911 tid 100162 td 0xfffff80003c78560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee77280 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee772b0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee772f0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe009ee77320 _sleep() at _sleep+0x367/frame 0xfffffe009ee773d0 bufspace_wait() at bufspace_wait+0x1df/frame 0xfffffe009ee77430 getnewbuf() at getnewbuf+0x28a/frame 0xfffffe009ee774a0 getblk() at getblk+0x640/frame 0xfffffe009ee77530 breadn_flags() at breadn_flags+0x34/frame 0xfffffe009ee77590 ffs_read() at ffs_read+0x23b/frame 0xfffffe009ee77640 VOP_READ_APV() at VOP_READ_APV+0xf7/frame 0xfffffe009ee77670 vn_read() at vn_read+0x1f1/frame 0xfffffe009ee776f0 vn_io_fault1() at vn_io_fault1+0x179/frame 0xfffffe009ee77830 vn_io_fault() at vn_io_fault+0x195/frame 0xfffffe009ee778a0 dofileread() at dofileread+0xba/frame 0xfffffe009ee778e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009ee77930 sys_read() at sys_read+0x86/frame 0xfffffe009ee77980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee77ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee77ab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800471cfa, rsp = 0x7fffffffeb58, rbp = 0x7fffffffeba0 --- Tracing command cron pid 1909 tid 100234 td 0xfffff80013db3560 sched_switch() at sched_switch+0x942/frame 0xfffffe009efc6690 mi_switch() at mi_switch+0x18c/frame 0xfffffe009efc66c0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009efc6700 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009efc6760 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009efc6790 _sleep() at _sleep+0x359/frame 0xfffffe009efc6840 pipe_read() at pipe_read+0x345/frame 0xfffffe009efc68a0 dofileread() at dofileread+0xba/frame 0xfffffe009efc68e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009efc6930 sys_read() at sys_read+0x86/frame 0xfffffe009efc6980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009efc6ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009efc6ab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x80043fcfa, rsp = 0x7fffffffe4d8, rbp = 0x7fffffffe4f0 --- Tracing command sh pid 1908 tid 100247 td 0xfffff8030f554560 sched_switch() at sched_switch+0x942/frame 0xfffffe00a1409280 mi_switch() at mi_switch+0x18c/frame 0xfffffe00a14092b0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00a14092f0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00a1409320 _sleep() at _sleep+0x367/frame 0xfffffe00a14093d0 bufspace_wait() at bufspace_wait+0x1df/frame 0xfffffe00a1409430 getnewbuf() at getnewbuf+0x28a/frame 0xfffffe00a14094a0 getblk() at getblk+0x640/frame 0xfffffe00a1409530 breadn_flags() at breadn_flags+0x34/frame 0xfffffe00a1409590 ffs_read() at ffs_read+0x23b/frame 0xfffffe00a1409640 VOP_READ_APV() at VOP_READ_APV+0xf7/frame 0xfffffe00a1409670 vn_read() at vn_read+0x1f1/frame 0xfffffe00a14096f0 vn_io_fault1() at vn_io_fault1+0x179/frame 0xfffffe00a1409830 vn_io_fault() at vn_io_fault+0x195/frame 0xfffffe00a14098a0 dofileread() at dofileread+0xba/frame 0xfffffe00a14098e0 kern_readv() at kern_readv+0x68/frame 0xfffffe00a1409930 sys_read() at sys_read+0x86/frame 0xfffffe00a1409980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe00a1409ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe00a1409ab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800471cfa, rsp = 0x7fffffffe788, rbp = 0x7fffffffe7d0 --- Tracing command fdatasync pid 1907 tid 100226 td 0xfffff801abacf000 sched_switch() at sched_switch+0x942/frame 0xfffffe009efad6c0 mi_switch() at mi_switch+0x18c/frame 0xfffffe009efad6f0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009efad730 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009efad790 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009efad7c0 _sleep() at _sleep+0x359/frame 0xfffffe009efad870 umtxq_sleep() at umtxq_sleep+0x163/frame 0xfffffe009efad8d0 do_wait() at do_wait+0x20d/frame 0xfffffe009efad940 __umtx_op_wait() at __umtx_op_wait+0x4d/frame 0xfffffe009efad980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009efadab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009efadab0 --- syscall (454, FreeBSD ELF64, sys__umtx_op), rip = 0x8006612ec, rsp = 0x7fffffffe688, rbp = 0x7fffffffe710 --- Tracing command fdatasync pid 1907 tid 100256 td 0xfffff80013c97560 sched_switch() at sched_switch+0x942/frame 0xfffffe00a1427050 mi_switch() at mi_switch+0x18c/frame 0xfffffe00a1427080 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00a14270c0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00a14270f0 sleeplk() at sleeplk+0x149/frame 0xfffffe00a1427150 lockmgr_slock_hard() at lockmgr_slock_hard+0x2d4/frame 0xfffffe00a14271d0 __lockmgr_args() at __lockmgr_args+0x547/frame 0xfffffe00a1427270 ffs_lock() at ffs_lock+0xaf/frame 0xfffffe00a14272c0 VOP_LOCK1_APV() at VOP_LOCK1_APV+0xe1/frame 0xfffffe00a14272f0 _vn_lock() at _vn_lock+0x66/frame 0xfffffe00a1427360 vget() at vget+0x98/frame 0xfffffe00a14273a0 cache_lookup() at cache_lookup+0x6fa/frame 0xfffffe00a1427480 vfs_cache_lookup() at vfs_cache_lookup+0xac/frame 0xfffffe00a14274e0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0xf7/frame 0xfffffe00a1427510 lookup() at lookup+0x692/frame 0xfffffe00a14275b0 namei() at namei+0x53b/frame 0xfffffe00a1427670 kern_statat() at kern_statat+0x98/frame 0xfffffe00a1427880 sys_fstatat() at sys_fstatat+0x2f/frame 0xfffffe00a1427980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe00a1427ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe00a1427ab0 --- syscall (552, FreeBSD ELF64, sys_fstatat), rip = 0x80083729a, rsp = 0x7fffdfffdca8, rbp = 0x7fffdfffdd60 --- Tracing command fdatasync pid 1907 tid 100257 td 0xfffff8030f715560 sched_switch() at sched_switch+0x942/frame 0xfffffe00a142bf50 mi_switch() at mi_switch+0x18c/frame 0xfffffe00a142bf80 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00a142bfc0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00a142bff0 _sleep() at _sleep+0x367/frame 0xfffffe00a142c0a0 bufspace_wait() at bufspace_wait+0x1df/frame 0xfffffe00a142c100 getnewbuf() at getnewbuf+0x28a/frame 0xfffffe00a142c170 getblk() at getblk+0x640/frame 0xfffffe00a142c200 indirdep_lookup() at indirdep_lookup+0x7e/frame 0xfffffe00a142c260 setup_allocindir_phase2() at setup_allocindir_phase2+0x80/frame 0xfffffe00a142c2a0 softdep_setup_allocindir_page() at softdep_setup_allocindir_page+0xf4/frame 0xfffffe00a142c310 ffs_balloc_ufs2() at ffs_balloc_ufs2+0x162b/frame 0xfffffe00a142c4b0 ffs_write() at ffs_write+0x20f/frame 0xfffffe00a142c550 VOP_WRITE_APV() at VOP_WRITE_APV+0x17a/frame 0xfffffe00a142c660 vn_write() at vn_write+0x26b/frame 0xfffffe00a142c6e0 vn_io_fault1() at vn_io_fault1+0x179/frame 0xfffffe00a142c820 vn_io_fault() at vn_io_fault+0x195/frame 0xfffffe00a142c890 dofilewrite() at dofilewrite+0xa7/frame 0xfffffe00a142c8e0 kern_writev() at kern_writev+0x68/frame 0xfffffe00a142c930 sys_write() at sys_write+0x86/frame 0xfffffe00a142c980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe00a142cab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe00a142cab0 --- syscall (4, FreeBSD ELF64, sys_write), rip = 0x800839cba, rsp = 0x7fffdfdfcf38, rbp = 0x7fffdfdfcf70 --- Tracing command fdatasync pid 1907 tid 100258 td 0xfffff8030f715000 sched_switch() at sched_switch+0x942/frame 0xfffffe00a1431070 mi_switch() at mi_switch+0x18c/frame 0xfffffe00a14310a0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00a14310e0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00a1431110 _sleep() at _sleep+0x367/frame 0xfffffe00a14311c0 bufspace_wait() at bufspace_wait+0x1df/frame 0xfffffe00a1431220 getnewbuf() at getnewbuf+0x28a/frame 0xfffffe00a1431290 getblk() at getblk+0x640/frame 0xfffffe00a1431320 ffs_balloc_ufs2() at ffs_balloc_ufs2+0x901/frame 0xfffffe00a14314b0 ffs_write() at ffs_write+0x20f/frame 0xfffffe00a1431550 VOP_WRITE_APV() at VOP_WRITE_APV+0x17a/frame 0xfffffe00a1431660 vn_write() at vn_write+0x26b/frame 0xfffffe00a14316e0 vn_io_fault1() at vn_io_fault1+0x179/frame 0xfffffe00a1431820 vn_io_fault() at vn_io_fault+0x195/frame 0xfffffe00a1431890 dofilewrite() at dofilewrite+0xa7/frame 0xfffffe00a14318e0 kern_writev() at kern_writev+0x68/frame 0xfffffe00a1431930 sys_write() at sys_write+0x86/frame 0xfffffe00a1431980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe00a1431ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe00a1431ab0 --- syscall (4, FreeBSD ELF64, sys_write), rip = 0x800839cba, rsp = 0x7fffdfbfbf38, rbp = 0x7fffdfbfbf70 --- Tracing command sh pid 1884 tid 100215 td 0xfffff802caa28560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef76550 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef76580 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef765c0 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef76620 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef76650 _sleep() at _sleep+0x359/frame 0xfffffe009ef76700 kern_wait6() at kern_wait6+0x4c4/frame 0xfffffe009ef76790 sys_wait4() at sys_wait4+0x78/frame 0xfffffe009ef76980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef76ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef76ab0 --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x8003fe84a, rsp = 0x7fffffffe4e8, rbp = 0x7fffffffe5a0 --- Tracing command tee pid 1883 tid 100225 td 0xfffff80013980560 sched_switch() at sched_switch+0x942/frame 0xfffffe009efa8690 mi_switch() at mi_switch+0x18c/frame 0xfffffe009efa86c0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009efa8700 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009efa8760 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009efa8790 _sleep() at _sleep+0x359/frame 0xfffffe009efa8840 pipe_read() at pipe_read+0x345/frame 0xfffffe009efa88a0 dofileread() at dofileread+0xba/frame 0xfffffe009efa88e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009efa8930 sys_read() at sys_read+0x86/frame 0xfffffe009efa8980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009efa8ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009efa8ab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800410cfa, rsp = 0x7fffffffe708, rbp = 0x7fffffffe750 --- Tracing command sh pid 1882 tid 100167 td 0xfffff80013db2000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee90550 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee90580 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee905c0 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ee90620 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ee90650 _sleep() at _sleep+0x359/frame 0xfffffe009ee90700 kern_wait6() at kern_wait6+0x4c4/frame 0xfffffe009ee90790 sys_wait4() at sys_wait4+0x78/frame 0xfffffe009ee90980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee90ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee90ab0 --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x8003fe84a, rsp = 0x7fffffffe2a8, rbp = 0x7fffffffe360 --- Tracing command top pid 1264 tid 100211 td 0xfffff80013d36560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef62590 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef625c0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef62600 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef62660 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x14/frame 0xfffffe009ef626a0 _cv_timedwait_sig_sbt() at _cv_timedwait_sig_sbt+0x221/frame 0xfffffe009ef62710 seltdwait() at seltdwait+0x6b/frame 0xfffffe009ef62750 kern_select() at kern_select+0x886/frame 0xfffffe009ef62940 sys_select() at sys_select+0x56/frame 0xfffffe009ef62980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef62ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef62ab0 --- syscall (93, FreeBSD ELF64, sys_select), rip = 0x8004c9cda, rsp = 0x7fffffffe8b8, rbp = 0x7fffffffea90 --- Tracing command bash pid 1252 tid 100144 td 0xfffff80013c53000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee1d550 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee1d580 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee1d5c0 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ee1d620 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ee1d650 _sleep() at _sleep+0x359/frame 0xfffffe009ee1d700 kern_wait6() at kern_wait6+0x4c4/frame 0xfffffe009ee1d790 sys_wait4() at sys_wait4+0x78/frame 0xfffffe009ee1d980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee1dab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee1dab0 --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x800afa84a, rsp = 0x7fffffffe678, rbp = 0x7fffffffe6c0 --- Tracing command csh pid 1249 tid 100146 td 0xfffff80013c97000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee27740 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee27770 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee277b0 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ee27810 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ee27840 _sleep() at _sleep+0x359/frame 0xfffffe009ee278f0 kern_sigsuspend() at kern_sigsuspend+0xb4/frame 0xfffffe009ee27950 sys_sigsuspend() at sys_sigsuspend+0x31/frame 0xfffffe009ee27980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee27ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee27ab0 --- syscall (4, FreeBSD ELF64, sys_write), rip = 0x8004818ca, rsp = 0x7fffffffe018, rbp = 0x7fffffffe0a0 --- Tracing command su pid 1248 tid 100204 td 0xfffff801abadb560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef49550 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef49580 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef495c0 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef49620 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef49650 _sleep() at _sleep+0x359/frame 0xfffffe009ef49700 kern_wait6() at kern_wait6+0x4c4/frame 0xfffffe009ef49790 sys_wait4() at sys_wait4+0x78/frame 0xfffffe009ef49980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef49ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef49ab0 --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x8003e284a, rsp = 0x7fffffffe628, rbp = 0x7fffffffeb70 --- Tracing command bash pid 1247 tid 100221 td 0xfffff802cab5b560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef94550 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef94580 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef945c0 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef94620 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef94650 _sleep() at _sleep+0x359/frame 0xfffffe009ef94700 kern_wait6() at kern_wait6+0x4c4/frame 0xfffffe009ef94790 sys_wait4() at sys_wait4+0x78/frame 0xfffffe009ef94980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef94ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef94ab0 --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x800afa84a, rsp = 0x7fffffffe7b8, rbp = 0x7fffffffe800 --- Tracing command sshd pid 1246 tid 100206 td 0xfffff801abacf560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef535b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef535e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef53620 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef53680 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef536b0 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009ef53710 seltdwait() at seltdwait+0x8d/frame 0xfffffe009ef53750 kern_select() at kern_select+0x886/frame 0xfffffe009ef53940 sys_select() at sys_select+0x56/frame 0xfffffe009ef53980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef53ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef53ab0 --- syscall (93, FreeBSD ELF64, sys_select), rip = 0x800874cda, rsp = 0x7fffffffa5e8, rbp = 0x7fffffffa620 --- Tracing command sshd pid 993 tid 100188 td 0xfffff800041b7000 sched_switch() at sched_switch+0x942/frame 0xfffffe009eef9620 mi_switch() at mi_switch+0x18c/frame 0xfffffe009eef9650 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009eef9690 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009eef96f0 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009eef9720 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009eef9780 seltdwait() at seltdwait+0x8d/frame 0xfffffe009eef97c0 kern_poll() at kern_poll+0x3f9/frame 0xfffffe009eef9960 sys_poll() at sys_poll+0x50/frame 0xfffffe009eef9980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009eef9ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009eef9ab0 --- syscall (209, FreeBSD ELF64, sys_poll), rip = 0x80081419a, rsp = 0x7fffffffe5c8, rbp = 0x7fffffffe600 --- Tracing command sh pid 942 tid 100205 td 0xfffff801abadb000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef4e550 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef4e580 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef4e5c0 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef4e620 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef4e650 _sleep() at _sleep+0x359/frame 0xfffffe009ef4e700 kern_wait6() at kern_wait6+0x4c4/frame 0xfffffe009ef4e790 sys_wait4() at sys_wait4+0x78/frame 0xfffffe009ef4e980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef4eab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef4eab0 --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x8003fe84a, rsp = 0x7fffffffe5c8, rbp = 0x7fffffffe680 --- Tracing command bash pid 807 tid 100202 td 0xfffff801abb0f000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef3f550 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef3f580 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef3f5c0 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef3f620 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef3f650 _sleep() at _sleep+0x359/frame 0xfffffe009ef3f700 kern_wait6() at kern_wait6+0x4c4/frame 0xfffffe009ef3f790 sys_wait4() at sys_wait4+0x78/frame 0xfffffe009ef3f980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef3fab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef3fab0 --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x800afa84a, rsp = 0x7fffffffe678, rbp = 0x7fffffffe6c0 --- Tracing command csh pid 805 tid 100201 td 0xfffff80013bcd000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef3a740 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef3a770 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef3a7b0 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef3a810 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef3a840 _sleep() at _sleep+0x359/frame 0xfffffe009ef3a8f0 kern_sigsuspend() at kern_sigsuspend+0xb4/frame 0xfffffe009ef3a950 sys_sigsuspend() at sys_sigsuspend+0x31/frame 0xfffffe009ef3a980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef3aab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef3aab0 --- syscall (4, FreeBSD ELF64, sys_write), rip = 0x8004818ca, rsp = 0x7fffffffe018, rbp = 0x7fffffffe0a0 --- Tracing command su pid 804 tid 100200 td 0xfffff801abb0f560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef35550 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef35580 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef355c0 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef35620 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef35650 _sleep() at _sleep+0x359/frame 0xfffffe009ef35700 kern_wait6() at kern_wait6+0x4c4/frame 0xfffffe009ef35790 sys_wait4() at sys_wait4+0x78/frame 0xfffffe009ef35980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef35ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef35ab0 --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x8003e284a, rsp = 0x7fffffffe628, rbp = 0x7fffffffeb70 --- Tracing command bash pid 803 tid 100199 td 0xfffff80013bcd560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef30550 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef30580 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef305c0 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef30620 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef30650 _sleep() at _sleep+0x359/frame 0xfffffe009ef30700 kern_wait6() at kern_wait6+0x4c4/frame 0xfffffe009ef30790 sys_wait4() at sys_wait4+0x78/frame 0xfffffe009ef30980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef30ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef30ab0 --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x800afa84a, rsp = 0x7fffffffe7b8, rbp = 0x7fffffffe800 --- Tracing command sshd pid 802 tid 100183 td 0xfffff80013bf0000 sched_switch() at sched_switch+0x942/frame 0xfffffe009eee05b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe009eee05e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009eee0620 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009eee0680 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009eee06b0 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009eee0710 seltdwait() at seltdwait+0x8d/frame 0xfffffe009eee0750 kern_select() at kern_select+0x886/frame 0xfffffe009eee0940 sys_select() at sys_select+0x56/frame 0xfffffe009eee0980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009eee0ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009eee0ab0 --- syscall (93, FreeBSD ELF64, sys_select), rip = 0x800874cda, rsp = 0x7fffffffa5e8, rbp = 0x7fffffffa620 --- Tracing command sshd pid 799 tid 100189 td 0xfffff80004038560 sched_switch() at sched_switch+0x942/frame 0xfffffe009eefe620 mi_switch() at mi_switch+0x18c/frame 0xfffffe009eefe650 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009eefe690 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009eefe6f0 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009eefe720 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009eefe780 seltdwait() at seltdwait+0x8d/frame 0xfffffe009eefe7c0 kern_poll() at kern_poll+0x3f9/frame 0xfffffe009eefe960 sys_poll() at sys_poll+0x50/frame 0xfffffe009eefe980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009eefeab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009eefeab0 --- syscall (209, FreeBSD ELF64, sys_poll), rip = 0x80081419a, rsp = 0x7fffffffe5c8, rbp = 0x7fffffffe600 --- Tracing command csh pid 796 tid 100190 td 0xfffff801aba71560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef02fd0 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef03000 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef03040 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe009ef03070 _sleep() at _sleep+0x367/frame 0xfffffe009ef03120 bufspace_wait() at bufspace_wait+0x1df/frame 0xfffffe009ef03180 getnewbuf() at getnewbuf+0x28a/frame 0xfffffe009ef031f0 getblk() at getblk+0x640/frame 0xfffffe009ef03280 breadn_flags() at breadn_flags+0x34/frame 0xfffffe009ef032e0 ffs_blkatoff() at ffs_blkatoff+0x86/frame 0xfffffe009ef03340 ufs_lookup_ino() at ufs_lookup_ino+0x605/frame 0xfffffe009ef03450 VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0xf7/frame 0xfffffe009ef03480 vfs_cache_lookup() at vfs_cache_lookup+0xd6/frame 0xfffffe009ef034e0 VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0xf7/frame 0xfffffe009ef03510 lookup() at lookup+0x692/frame 0xfffffe009ef035b0 namei() at namei+0x53b/frame 0xfffffe009ef03670 kern_statat() at kern_statat+0x98/frame 0xfffffe009ef03880 sys_fstatat() at sys_fstatat+0x2f/frame 0xfffffe009ef03980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef03ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef03ab0 --- syscall (552, FreeBSD ELF64, sys_fstatat), rip = 0x8004f229a, rsp = 0x7fffffffe3c8, rbp = 0x7fffffffe470 --- Tracing command login pid 795 tid 100198 td 0xfffff801abb10000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef2b550 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef2b580 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef2b5c0 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef2b620 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef2b650 _sleep() at _sleep+0x359/frame 0xfffffe009ef2b700 kern_wait6() at kern_wait6+0x4c4/frame 0xfffffe009ef2b790 sys_wait4() at sys_wait4+0x78/frame 0xfffffe009ef2b980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef2bab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef2bab0 --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x8003e484a, rsp = 0x7fffffffeb68, rbp = 0x7fffffffed20 --- Tracing command getty pid 794 tid 100197 td 0xfffff80013c8c000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef26610 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef26640 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef26680 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef266e0 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef26710 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009ef26770 tty_wait() at tty_wait+0x42/frame 0xfffffe009ef267a0 ttydisc_read() at ttydisc_read+0x22c/frame 0xfffffe009ef26810 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe009ef26840 devfs_read_f() at devfs_read_f+0xe0/frame 0xfffffe009ef268a0 dofileread() at dofileread+0xba/frame 0xfffffe009ef268e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009ef26930 sys_read() at sys_read+0x86/frame 0xfffffe009ef26980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef26ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef26ab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x80042bcfa, rsp = 0x7fffffffebe8, rbp = 0x7fffffffec30 --- Tracing command getty pid 793 tid 100196 td 0xfffff80013c8c560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef21610 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef21640 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef21680 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef216e0 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef21710 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009ef21770 tty_wait() at tty_wait+0x42/frame 0xfffffe009ef217a0 ttydisc_read() at ttydisc_read+0x22c/frame 0xfffffe009ef21810 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe009ef21840 devfs_read_f() at devfs_read_f+0xe0/frame 0xfffffe009ef218a0 dofileread() at dofileread+0xba/frame 0xfffffe009ef218e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009ef21930 sys_read() at sys_read+0x86/frame 0xfffffe009ef21980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef21ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef21ab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x80042bcfa, rsp = 0x7fffffffebe8, rbp = 0x7fffffffec30 --- Tracing command getty pid 792 tid 100195 td 0xfffff80013c94000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef1c610 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef1c640 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef1c680 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef1c6e0 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef1c710 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009ef1c770 tty_wait() at tty_wait+0x42/frame 0xfffffe009ef1c7a0 ttydisc_read() at ttydisc_read+0x22c/frame 0xfffffe009ef1c810 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe009ef1c840 devfs_read_f() at devfs_read_f+0xe0/frame 0xfffffe009ef1c8a0 dofileread() at dofileread+0xba/frame 0xfffffe009ef1c8e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009ef1c930 sys_read() at sys_read+0x86/frame 0xfffffe009ef1c980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef1cab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef1cab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x80042bcfa, rsp = 0x7fffffffebe8, rbp = 0x7fffffffec30 --- Tracing command getty pid 791 tid 100194 td 0xfffff80013c94560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef17610 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef17640 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef17680 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef176e0 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef17710 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009ef17770 tty_wait() at tty_wait+0x42/frame 0xfffffe009ef177a0 ttydisc_read() at ttydisc_read+0x22c/frame 0xfffffe009ef17810 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe009ef17840 devfs_read_f() at devfs_read_f+0xe0/frame 0xfffffe009ef178a0 dofileread() at dofileread+0xba/frame 0xfffffe009ef178e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009ef17930 sys_read() at sys_read+0x86/frame 0xfffffe009ef17980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef17ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef17ab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x80042bcfa, rsp = 0x7fffffffebe8, rbp = 0x7fffffffec30 --- Tracing command getty pid 790 tid 100193 td 0xfffff80013c95000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef12610 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef12640 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef12680 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef126e0 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef12710 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009ef12770 tty_wait() at tty_wait+0x42/frame 0xfffffe009ef127a0 ttydisc_read() at ttydisc_read+0x22c/frame 0xfffffe009ef12810 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe009ef12840 devfs_read_f() at devfs_read_f+0xe0/frame 0xfffffe009ef128a0 dofileread() at dofileread+0xba/frame 0xfffffe009ef128e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009ef12930 sys_read() at sys_read+0x86/frame 0xfffffe009ef12980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef12ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef12ab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x80042bcfa, rsp = 0x7fffffffebe8, rbp = 0x7fffffffec30 --- Tracing command getty pid 789 tid 100192 td 0xfffff80013c95560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ef0d610 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ef0d640 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ef0d680 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ef0d6e0 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ef0d710 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009ef0d770 tty_wait() at tty_wait+0x42/frame 0xfffffe009ef0d7a0 ttydisc_read() at ttydisc_read+0x22c/frame 0xfffffe009ef0d810 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe009ef0d840 devfs_read_f() at devfs_read_f+0xe0/frame 0xfffffe009ef0d8a0 dofileread() at dofileread+0xba/frame 0xfffffe009ef0d8e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009ef0d930 sys_read() at sys_read+0x86/frame 0xfffffe009ef0d980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ef0dab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ef0dab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x80042bcfa, rsp = 0x7fffffffebe8, rbp = 0x7fffffffec30 --- Tracing command getty pid 788 tid 100178 td 0xfffff8001396e560 sched_switch() at sched_switch+0x942/frame 0xfffffe009eec7610 mi_switch() at mi_switch+0x18c/frame 0xfffffe009eec7640 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009eec7680 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009eec76e0 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009eec7710 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009eec7770 tty_wait() at tty_wait+0x42/frame 0xfffffe009eec77a0 ttydisc_read() at ttydisc_read+0x22c/frame 0xfffffe009eec7810 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe009eec7840 devfs_read_f() at devfs_read_f+0xe0/frame 0xfffffe009eec78a0 dofileread() at dofileread+0xba/frame 0xfffffe009eec78e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009eec7930 sys_read() at sys_read+0x86/frame 0xfffffe009eec7980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009eec7ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009eec7ab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x80042bcfa, rsp = 0x7fffffffebe8, rbp = 0x7fffffffec30 --- Tracing command getty pid 787 tid 100179 td 0xfffff8001396e000 sched_switch() at sched_switch+0x942/frame 0xfffffe009eecc610 mi_switch() at mi_switch+0x18c/frame 0xfffffe009eecc640 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009eecc680 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009eecc6e0 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009eecc710 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009eecc770 tty_wait() at tty_wait+0x42/frame 0xfffffe009eecc7a0 ttydisc_read() at ttydisc_read+0x22c/frame 0xfffffe009eecc810 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe009eecc840 devfs_read_f() at devfs_read_f+0xe0/frame 0xfffffe009eecc8a0 dofileread() at dofileread+0xba/frame 0xfffffe009eecc8e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009eecc930 sys_read() at sys_read+0x86/frame 0xfffffe009eecc980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009eeccab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009eeccab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x80042bcfa, rsp = 0x7fffffffebe8, rbp = 0x7fffffffec30 --- Tracing command getty pid 786 tid 100140 td 0xfffff80003983560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee09610 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee09640 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee09680 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ee096e0 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ee09710 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009ee09770 tty_wait() at tty_wait+0x42/frame 0xfffffe009ee097a0 ttydisc_read() at ttydisc_read+0x22c/frame 0xfffffe009ee09810 ttydev_read() at ttydev_read+0x4b/frame 0xfffffe009ee09840 devfs_read_f() at devfs_read_f+0xe0/frame 0xfffffe009ee098a0 dofileread() at dofileread+0xba/frame 0xfffffe009ee098e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009ee09930 sys_read() at sys_read+0x86/frame 0xfffffe009ee09980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee09ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee09ab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x80042bcfa, rsp = 0x7fffffffebe8, rbp = 0x7fffffffec30 --- Tracing command cron pid 752 tid 100180 td 0xfffff80013bf0560 sched_switch() at sched_switch+0x942/frame 0xfffffe009eed16f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe009eed1720 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009eed1760 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009eed17c0 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x14/frame 0xfffffe009eed1800 _sleep() at _sleep+0x2f4/frame 0xfffffe009eed18b0 kern_clock_nanosleep() at kern_clock_nanosleep+0x1b7/frame 0xfffffe009eed1940 sys_nanosleep() at sys_nanosleep+0x5f/frame 0xfffffe009eed1980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009eed1ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009eed1ab0 --- syscall (240, FreeBSD ELF64, sys_nanosleep), rip = 0x8003cc9ca, rsp = 0x7fffffffeae8, rbp = 0x7fffffffece0 --- Tracing command sshd pid 748 tid 100171 td 0xfffff80003a25000 sched_switch() at sched_switch+0x942/frame 0xfffffe009eea45b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe009eea45e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009eea4620 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009eea4680 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009eea46b0 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009eea4710 seltdwait() at seltdwait+0x8d/frame 0xfffffe009eea4750 kern_select() at kern_select+0x886/frame 0xfffffe009eea4940 sys_select() at sys_select+0x56/frame 0xfffffe009eea4980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009eea4ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009eea4ab0 --- syscall (93, FreeBSD ELF64, sys_select), rip = 0x800874cda, rsp = 0x7fffffffe698, rbp = 0x7fffffffe6d0 --- Tracing command rpc.lockd pid 683 tid 100185 td 0xfffff80013d3f000 sched_switch() at sched_switch+0x942/frame 0xfffffe009eeea4d0 mi_switch() at mi_switch+0x18c/frame 0xfffffe009eeea500 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009eeea540 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009eeea5a0 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x14/frame 0xfffffe009eeea5e0 _cv_timedwait_sig_sbt() at _cv_timedwait_sig_sbt+0x221/frame 0xfffffe009eeea650 svc_run_internal() at svc_run_internal+0x38e/frame 0xfffffe009eeea790 svc_run() at svc_run+0x1fe/frame 0xfffffe009eeea7f0 sys_nlm_syscall() at sys_nlm_syscall+0x776/frame 0xfffffe009eeea980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009eeeaab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009eeeaab0 --- syscall (154, FreeBSD ELF64, sys_nlm_syscall), rip = 0x8002ee68a, rsp = 0x7fffffffec28, rbp = 0x7fffffffece0 --- Tracing command rpc.statd pid 680 tid 100160 td 0xfffff80013d3f560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee6d590 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee6d5c0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee6d600 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ee6d660 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x14/frame 0xfffffe009ee6d6a0 _cv_timedwait_sig_sbt() at _cv_timedwait_sig_sbt+0x221/frame 0xfffffe009ee6d710 seltdwait() at seltdwait+0x6b/frame 0xfffffe009ee6d750 kern_select() at kern_select+0x886/frame 0xfffffe009ee6d940 sys_select() at sys_select+0x56/frame 0xfffffe009ee6d980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee6dab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee6dab0 --- syscall (93, FreeBSD ELF64, sys_select), rip = 0x800420cda, rsp = 0x7fffffffeab8, rbp = 0x7fffffffec00 --- Tracing command casperd pid 655 tid 100151 td 0xfffff800041b8560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee405b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee405e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee40620 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ee40680 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ee406b0 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009ee40710 seltdwait() at seltdwait+0x8d/frame 0xfffffe009ee40750 kern_select() at kern_select+0x886/frame 0xfffffe009ee40940 sys_select() at sys_select+0x56/frame 0xfffffe009ee40980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee40ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee40ab0 --- syscall (93, FreeBSD ELF64, sys_select), rip = 0x800850cda, rsp = 0x7fffffffe7f8, rbp = 0x7fffffffece0 --- Tracing command casperd pid 654 tid 100163 td 0xfffff800041b8000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee7c5b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee7c5e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee7c620 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ee7c680 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ee7c6b0 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009ee7c710 seltdwait() at seltdwait+0x8d/frame 0xfffffe009ee7c750 kern_select() at kern_select+0x886/frame 0xfffffe009ee7c940 sys_select() at sys_select+0x56/frame 0xfffffe009ee7c980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee7cab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee7cab0 --- syscall (93, FreeBSD ELF64, sys_select), rip = 0x800850cda, rsp = 0x7fffffffe658, rbp = 0x7fffffffe720 --- Tracing command dhclient pid 623 tid 100177 td 0xfffff80013c96000 sched_switch() at sched_switch+0x942/frame 0xfffffe009eec2690 mi_switch() at mi_switch+0x18c/frame 0xfffffe009eec26c0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009eec2700 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009eec2760 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009eec2790 _sleep() at _sleep+0x359/frame 0xfffffe009eec2840 pipe_read() at pipe_read+0x345/frame 0xfffffe009eec28a0 dofileread() at dofileread+0xba/frame 0xfffffe009eec28e0 kern_readv() at kern_readv+0x68/frame 0xfffffe009eec2930 sys_read() at sys_read+0x86/frame 0xfffffe009eec2980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009eec2ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009eec2ab0 --- syscall (3, FreeBSD ELF64, sys_read), rip = 0x800447cfa, rsp = 0x7fffffffca48, rbp = 0x7fffffffca70 --- Tracing command dhclient pid 562 tid 100148 td 0xfffff8001397f000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee31550 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee31580 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee315c0 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ee31620 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ee31650 _sleep() at _sleep+0x359/frame 0xfffffe009ee31700 kern_wait6() at kern_wait6+0x4c4/frame 0xfffffe009ee31790 sys_wait4() at sys_wait4+0x78/frame 0xfffffe009ee31980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee31ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee31ab0 --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x8003d484a, rsp = 0x7fffffffdac8, rbp = 0x7fffffffdb50 --- Tracing command dhclient pid 559 tid 100173 td 0xfffff8001397e000 sched_switch() at sched_switch+0x942/frame 0xfffffe009eeae5b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe009eeae5e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009eeae620 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009eeae680 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009eeae6b0 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009eeae710 seltdwait() at seltdwait+0x8d/frame 0xfffffe009eeae750 kern_select() at kern_select+0x886/frame 0xfffffe009eeae940 sys_select() at sys_select+0x56/frame 0xfffffe009eeae980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009eeaeab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009eeaeab0 --- syscall (93, FreeBSD ELF64, sys_select), rip = 0x800447cda, rsp = 0x7fffffffea08, rbp = 0x7fffffffeac0 --- Tracing command rpcbind pid 537 tid 100158 td 0xfffff8001398c560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee63600 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee63630 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee63670 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ee636d0 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x14/frame 0xfffffe009ee63710 _cv_timedwait_sig_sbt() at _cv_timedwait_sig_sbt+0x221/frame 0xfffffe009ee63780 seltdwait() at seltdwait+0x6b/frame 0xfffffe009ee637c0 kern_poll() at kern_poll+0x3f9/frame 0xfffffe009ee63960 sys_poll() at sys_poll+0x50/frame 0xfffffe009ee63980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee63ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee63ab0 --- syscall (209, FreeBSD ELF64, sys_poll), rip = 0x8003c519a, rsp = 0x7fffffffca58, rbp = 0x7fffffffec70 --- Tracing command syslogd pid 532 tid 100164 td 0xfffff8001397e560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee815b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee815e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee81620 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ee81680 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe009ee816b0 _cv_wait_sig() at _cv_wait_sig+0x1fe/frame 0xfffffe009ee81710 seltdwait() at seltdwait+0x8d/frame 0xfffffe009ee81750 kern_select() at kern_select+0x886/frame 0xfffffe009ee81940 sys_select() at sys_select+0x56/frame 0xfffffe009ee81980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee81ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee81ab0 --- syscall (93, FreeBSD ELF64, sys_select), rip = 0x80042fcda, rsp = 0x7fffffffe6c8, rbp = 0x7fffffffece0 --- Tracing command devd pid 457 tid 100159 td 0xfffff80013c52560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee68590 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee685c0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee68600 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe009ee68660 sleepq_timedwait_sig() at sleepq_timedwait_sig+0x14/frame 0xfffffe009ee686a0 _cv_timedwait_sig_sbt() at _cv_timedwait_sig_sbt+0x221/frame 0xfffffe009ee68710 seltdwait() at seltdwait+0x6b/frame 0xfffffe009ee68750 kern_select() at kern_select+0x886/frame 0xfffffe009ee68940 sys_select() at sys_select+0x56/frame 0xfffffe009ee68980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe009ee68ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe009ee68ab0 --- syscall (93, FreeBSD ELF64, sys_select), rip = 0x2f5aea, rsp = 0x7fffffffcb58, rbp = 0x7fffffffed00 --- Tracing command syncer pid 33 tid 100133 td 0xfffff80013975000 sched_switch() at sched_switch+0x942/frame 0xfffffe00005e1530 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005e1560 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005e15a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005e15d0 _sleep() at _sleep+0x367/frame 0xfffffe00005e1680 bufspace_wait() at bufspace_wait+0x1df/frame 0xfffffe00005e16e0 getnewbuf() at getnewbuf+0x28a/frame 0xfffffe00005e1750 getblk() at getblk+0x640/frame 0xfffffe00005e17e0 breadn_flags() at breadn_flags+0x34/frame 0xfffffe00005e1840 ffs_update() at ffs_update+0x11f/frame 0xfffffe00005e18c0 ffs_sync() at ffs_sync+0x516/frame 0xfffffe00005e1970 sync_fsync() at sync_fsync+0xff/frame 0xfffffe00005e19b0 VOP_FSYNC_APV() at VOP_FSYNC_APV+0xfd/frame 0xfffffe00005e19e0 sched_sync() at sched_sync+0x284/frame 0xfffffe00005e1a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005e1ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005e1ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command vnlru pid 32 tid 100130 td 0xfffff80013975560 sched_switch() at sched_switch+0x942/frame 0xfffffe00005d2870 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005d28a0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005d28e0 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe00005d2920 _sleep() at _sleep+0x307/frame 0xfffffe00005d29d0 vnlru_proc() at vnlru_proc+0xa4/frame 0xfffffe00005d2a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005d2ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005d2ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 31 tid 100128 td 0xfffff80013977000 sched_switch() at sched_switch+0x942/frame 0xfffffe00005cd8c0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005cd8f0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005cd930 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe00005cd970 _sleep() at _sleep+0x307/frame 0xfffffe00005cda20 buf_daemon() at buf_daemon+0xe8/frame 0xfffffe00005cda70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005cdab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005cdab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 31 tid 100131 td 0xfffff80013098560 sched_switch() at sched_switch+0x942/frame 0xfffffe00005d78b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005d78e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005d7920 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe00005d7960 _sleep() at _sleep+0x307/frame 0xfffffe00005d7a10 bufspace_daemon() at bufspace_daemon+0x266/frame 0xfffffe00005d7a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005d7ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005d7ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 31 tid 100132 td 0xfffff80013098000 sched_switch() at sched_switch+0x942/frame 0xfffffe00005dc8b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005dc8e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005dc920 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe00005dc960 _sleep() at _sleep+0x307/frame 0xfffffe00005dca10 bufspace_daemon() at bufspace_daemon+0x266/frame 0xfffffe00005dca70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005dcab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005dcab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 31 tid 100134 td 0xfffff80006099560 sched_switch() at sched_switch+0x942/frame 0xfffffe00005eb8b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005eb8e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005eb920 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe00005eb960 _sleep() at _sleep+0x307/frame 0xfffffe00005eba10 bufspace_daemon() at bufspace_daemon+0x266/frame 0xfffffe00005eba70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005ebab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005ebab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 31 tid 100136 td 0xfffff8001398f560 sched_switch() at sched_switch+0x942/frame 0xfffffe00005f58b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005f58e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005f5920 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe00005f5960 _sleep() at _sleep+0x307/frame 0xfffffe00005f5a10 bufspace_daemon() at bufspace_daemon+0x266/frame 0xfffffe00005f5a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005f5ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005f5ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 31 tid 100137 td 0xfffff8001398f000 sched_switch() at sched_switch+0x942/frame 0xfffffe00005fa8b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005fa8e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005fa920 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe00005fa960 _sleep() at _sleep+0x307/frame 0xfffffe00005faa10 bufspace_daemon() at bufspace_daemon+0x266/frame 0xfffffe00005faa70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005faab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005faab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 31 tid 100138 td 0xfffff8001398d560 sched_switch() at sched_switch+0x942/frame 0xfffffe00005ff8b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005ff8e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005ff920 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe00005ff960 _sleep() at _sleep+0x307/frame 0xfffffe00005ffa10 bufspace_daemon() at bufspace_daemon+0x266/frame 0xfffffe00005ffa70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005ffab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005ffab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 31 tid 100139 td 0xfffff8001398d000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee048b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee048e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee04920 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe009ee04960 _sleep() at _sleep+0x307/frame 0xfffffe009ee04a10 bufspace_daemon() at bufspace_daemon+0x1a4/frame 0xfffffe009ee04a70 fork_exit() at fork_exit+0x84/frame 0xfffffe009ee04ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe009ee04ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 31 tid 100150 td 0xfffff80013d40560 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee3b8e0 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee3b910 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee3b950 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe009ee3b990 _sleep() at _sleep+0x307/frame 0xfffffe009ee3ba40 softdep_flush() at softdep_flush+0x17f/frame 0xfffffe009ee3ba70 fork_exit() at fork_exit+0x84/frame 0xfffffe009ee3bab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe009ee3bab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command bufdaemon pid 31 tid 100210 td 0xfffff8001396c000 sched_switch() at sched_switch+0x942/frame 0xfffffe009ee5e550 mi_switch() at mi_switch+0x18c/frame 0xfffffe009ee5e580 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe009ee5e5c0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe009ee5e5f0 _sleep() at _sleep+0x367/frame 0xfffffe009ee5e6a0 bufspace_wait() at bufspace_wait+0x1df/frame 0xfffffe009ee5e700 getnewbuf() at getnewbuf+0x28a/frame 0xfffffe009ee5e770 getblk() at getblk+0x640/frame 0xfffffe009ee5e800 breadn_flags() at breadn_flags+0x34/frame 0xfffffe009ee5e860 ffs_getcg() at ffs_getcg+0xd8/frame 0xfffffe009ee5e8d0 ffs_freefile() at ffs_freefile+0x91/frame 0xfffffe009ee5e950 handle_workitem_freefile() at handle_workitem_freefile+0x161/frame 0xfffffe009ee5e990 process_worklist_item() at process_worklist_item+0x280/frame 0xfffffe009ee5ea00 softdep_process_worklist() at softdep_process_worklist+0xb2/frame 0xfffffe009ee5ea40 softdep_flush() at softdep_flush+0xef/frame 0xfffffe009ee5ea70 fork_exit() at fork_exit+0x84/frame 0xfffffe009ee5eab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe009ee5eab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command vmdaemon pid 30 tid 100127 td 0xfffff8001318b000 sched_switch() at sched_switch+0x942/frame 0xfffffe00005c88a0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005c88d0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005c8910 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005c8940 _sleep() at _sleep+0x367/frame 0xfffffe00005c89f0 vm_daemon() at vm_daemon+0x85/frame 0xfffffe00005c8a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005c8ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005c8ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command pagedaemon pid 29 tid 100126 td 0xfffff8001318b560 sched_switch() at sched_switch+0x942/frame 0xfffffe00005c3810 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005c3840 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005c3880 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe00005c38c0 _sleep() at _sleep+0x307/frame 0xfffffe00005c3970 vm_pageout_worker() at vm_pageout_worker+0x294/frame 0xfffffe00005c3a50 vm_pageout() at vm_pageout+0x133/frame 0xfffffe00005c3a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005c3ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005c3ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command pagedaemon pid 29 tid 100129 td 0xfffff80013980000 sched_switch() at sched_switch+0x942/frame 0xfffffe00005e6840 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005e6870 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005e68b0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005e68e0 _sleep() at _sleep+0x367/frame 0xfffffe00005e6990 vm_pageout_laundry_worker() at vm_pageout_laundry_worker+0xc05/frame 0xfffffe00005e6a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005e6ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005e6ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command pagedaemon pid 29 tid 100135 td 0xfffff8001397f560 sched_switch() at sched_switch+0x942/frame 0xfffffe00005f08e0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005f0910 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005f0950 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe00005f0990 _sleep() at _sleep+0x307/frame 0xfffffe00005f0a40 uma_reclaim_worker() at uma_reclaim_worker+0xda/frame 0xfffffe00005f0a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005f0ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005f0ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command enc_daemon1 pid 28 tid 100125 td 0xfffff800130a8000 sched_switch() at sched_switch+0x942/frame 0xfffffe00005be8a0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005be8d0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005be910 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005be940 _sleep() at _sleep+0x367/frame 0xfffffe00005be9f0 enc_daemon() at enc_daemon+0x1ca/frame 0xfffffe00005bea70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005beab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005beab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command enc_daemon0 pid 27 tid 100124 td 0xfffff800130a8560 sched_switch() at sched_switch+0x942/frame 0xfffffe00005b98a0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005b98d0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005b9910 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005b9940 _sleep() at _sleep+0x367/frame 0xfffffe00005b99f0 enc_daemon() at enc_daemon+0x1ca/frame 0xfffffe00005b9a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005b9ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005b9ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command rand_harvestq pid 26 tid 100118 td 0xfffff80013195560 sched_switch() at sched_switch+0x942/frame 0xfffffe000059b8b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe000059b8e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000059b920 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe000059b960 _sleep() at _sleep+0x307/frame 0xfffffe000059ba10 random_kthread() at random_kthread+0x57/frame 0xfffffe000059ba70 fork_exit() at fork_exit+0x84/frame 0xfffffe000059bab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000059bab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command sctp_iterator pid 25 tid 100117 td 0xfffff80013196000 sched_switch() at sched_switch+0x942/frame 0xfffffe0000596900 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000596930 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000596970 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005969a0 _sleep() at _sleep+0x367/frame 0xfffffe0000596a50 sctp_iterator_thread() at sctp_iterator_thread+0x59/frame 0xfffffe0000596a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000596ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000596ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command soaiod4 pid 24 tid 100116 td 0xfffff800130a9000 sched_switch() at sched_switch+0x942/frame 0xfffffe00005908d0 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000590900 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000590940 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe0000590980 _sleep() at _sleep+0x307/frame 0xfffffe0000590a30 soaio_kproc_loop() at soaio_kproc_loop+0x196/frame 0xfffffe0000590a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000590ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000590ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command soaiod3 pid 23 tid 100115 td 0xfffff800130a9560 sched_switch() at sched_switch+0x942/frame 0xfffffe000058b8d0 mi_switch() at mi_switch+0x18c/frame 0xfffffe000058b900 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000058b940 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe000058b980 _sleep() at _sleep+0x307/frame 0xfffffe000058ba30 soaio_kproc_loop() at soaio_kproc_loop+0x196/frame 0xfffffe000058ba70 fork_exit() at fork_exit+0x84/frame 0xfffffe000058bab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000058bab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command soaiod2 pid 22 tid 100114 td 0xfffff800130aa000 sched_switch() at sched_switch+0x942/frame 0xfffffe00005868d0 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000586900 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000586940 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe0000586980 _sleep() at _sleep+0x307/frame 0xfffffe0000586a30 soaio_kproc_loop() at soaio_kproc_loop+0x196/frame 0xfffffe0000586a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000586ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000586ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command soaiod1 pid 21 tid 100113 td 0xfffff800130aa560 sched_switch() at sched_switch+0x942/frame 0xfffffe00005818d0 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000581900 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000581940 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xfffffe0000581980 _sleep() at _sleep+0x307/frame 0xfffffe0000581a30 soaio_kproc_loop() at soaio_kproc_loop+0x196/frame 0xfffffe0000581a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000581ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000581ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 20 tid 100094 td 0xfffff80004037000 sched_switch() at sched_switch+0x942/frame 0xfffffe0000522930 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000522960 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005229a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005229d0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe0000522a30 usb_process() at usb_process+0x12b/frame 0xfffffe0000522a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000522ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000522ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 20 tid 100095 td 0xfffff80004036560 sched_switch() at sched_switch+0x942/frame 0xfffffe0000527930 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000527960 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005279a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005279d0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe0000527a30 usb_process() at usb_process+0x12b/frame 0xfffffe0000527a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000527ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000527ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 20 tid 100096 td 0xfffff80004036000 sched_switch() at sched_switch+0x942/frame 0xfffffe000052c930 mi_switch() at mi_switch+0x18c/frame 0xfffffe000052c960 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000052c9a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe000052c9d0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe000052ca30 usb_process() at usb_process+0x12b/frame 0xfffffe000052ca70 fork_exit() at fork_exit+0x84/frame 0xfffffe000052cab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000052cab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 20 tid 100097 td 0xfffff80004035560 sched_switch() at sched_switch+0x942/frame 0xfffffe0000531930 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000531960 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005319a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005319d0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe0000531a30 usb_process() at usb_process+0x12b/frame 0xfffffe0000531a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000531ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000531ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 20 tid 100098 td 0xfffff80004035000 sched_switch() at sched_switch+0x942/frame 0xfffffe0000536930 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000536960 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005369a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005369d0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe0000536a30 usb_process() at usb_process+0x12b/frame 0xfffffe0000536a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000536ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000536ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 20 tid 100100 td 0xfffff8000402d000 sched_switch() at sched_switch+0x942/frame 0xfffffe0000540930 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000540960 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005409a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005409d0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe0000540a30 usb_process() at usb_process+0x12b/frame 0xfffffe0000540a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000540ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000540ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 20 tid 100101 td 0xfffff800041ba560 sched_switch() at sched_switch+0x942/frame 0xfffffe0000545930 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000545960 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005459a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005459d0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe0000545a30 usb_process() at usb_process+0x12b/frame 0xfffffe0000545a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000545ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000545ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 20 tid 100102 td 0xfffff800041ba000 sched_switch() at sched_switch+0x942/frame 0xfffffe000054a930 mi_switch() at mi_switch+0x18c/frame 0xfffffe000054a960 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000054a9a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe000054a9d0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe000054aa30 usb_process() at usb_process+0x12b/frame 0xfffffe000054aa70 fork_exit() at fork_exit+0x84/frame 0xfffffe000054aab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000054aab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 20 tid 100103 td 0xfffff800041b9560 sched_switch() at sched_switch+0x942/frame 0xfffffe000054f930 mi_switch() at mi_switch+0x18c/frame 0xfffffe000054f960 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000054f9a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe000054f9d0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe000054fa30 usb_process() at usb_process+0x12b/frame 0xfffffe000054fa70 fork_exit() at fork_exit+0x84/frame 0xfffffe000054fab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000054fab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 20 tid 100104 td 0xfffff800041b9000 sched_switch() at sched_switch+0x942/frame 0xfffffe0000554930 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000554960 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005549a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005549d0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe0000554a30 usb_process() at usb_process+0x12b/frame 0xfffffe0000554a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000554ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000554ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 20 tid 100105 td 0xfffff80006099000 sched_switch() at sched_switch+0x942/frame 0xfffffe0000559930 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000559960 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005599a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005599d0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe0000559a30 usb_process() at usb_process+0x12b/frame 0xfffffe0000559a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000559ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000559ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 20 tid 100106 td 0xfffff80006098560 sched_switch() at sched_switch+0x942/frame 0xfffffe000055e930 mi_switch() at mi_switch+0x18c/frame 0xfffffe000055e960 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000055e9a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe000055e9d0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe000055ea30 usb_process() at usb_process+0x12b/frame 0xfffffe000055ea70 fork_exit() at fork_exit+0x84/frame 0xfffffe000055eab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000055eab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 20 tid 100107 td 0xfffff80006098000 sched_switch() at sched_switch+0x942/frame 0xfffffe0000563930 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000563960 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005639a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005639d0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe0000563a30 usb_process() at usb_process+0x12b/frame 0xfffffe0000563a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000563ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000563ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 20 tid 100108 td 0xfffff80006097560 sched_switch() at sched_switch+0x942/frame 0xfffffe0000568930 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000568960 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005689a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005689d0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe0000568a30 usb_process() at usb_process+0x12b/frame 0xfffffe0000568a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000568ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000568ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command usb pid 20 tid 100109 td 0xfffff80006097000 sched_switch() at sched_switch+0x942/frame 0xfffffe000056d930 mi_switch() at mi_switch+0x18c/frame 0xfffffe000056d960 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000056d9a0 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe000056d9d0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe000056da30 usb_process() at usb_process+0x12b/frame 0xfffffe000056da70 fork_exit() at fork_exit+0x84/frame 0xfffffe000056dab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000056dab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command cam pid 19 tid 100089 td 0xfffff80003aed560 sched_switch() at sched_switch+0x942/frame 0xfffffe00005098d0 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000509900 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000509940 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe0000509970 _sleep() at _sleep+0x367/frame 0xfffffe0000509a20 xpt_done_td() at xpt_done_td+0x7b/frame 0xfffffe0000509a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000509ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000509ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command cam pid 19 tid 100090 td 0xfffff80003aed000 sched_switch() at sched_switch+0x942/frame 0xfffffe000050e8d0 mi_switch() at mi_switch+0x18c/frame 0xfffffe000050e900 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000050e940 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe000050e970 _sleep() at _sleep+0x367/frame 0xfffffe000050ea20 xpt_done_td() at xpt_done_td+0x7b/frame 0xfffffe000050ea70 fork_exit() at fork_exit+0x84/frame 0xfffffe000050eab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000050eab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command cam pid 19 tid 100091 td 0xfffff80003c79560 sched_switch() at sched_switch+0x942/frame 0xfffffe00005138d0 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000513900 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000513940 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe0000513970 _sleep() at _sleep+0x367/frame 0xfffffe0000513a20 xpt_done_td() at xpt_done_td+0x7b/frame 0xfffffe0000513a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000513ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000513ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command cam pid 19 tid 100123 td 0xfffff80013193000 sched_switch() at sched_switch+0x942/frame 0xfffffe00005b48c0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005b48f0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005b4930 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005b4960 _sleep() at _sleep+0x367/frame 0xfffffe00005b4a10 xpt_scanner_thread() at xpt_scanner_thread+0x93/frame 0xfffffe00005b4a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005b4ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005b4ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto returns 11 pid 18 tid 100088 td 0xfffff80003aee000 sched_switch() at sched_switch+0x942/frame 0xfffffe00005048b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005048e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000504920 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe0000504950 _sleep() at _sleep+0x367/frame 0xfffffe0000504a00 crypto_ret_proc() at crypto_ret_proc+0x238/frame 0xfffffe0000504a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000504ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000504ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto returns 10 pid 17 tid 100087 td 0xfffff80003aee560 sched_switch() at sched_switch+0x942/frame 0xfffffe00004ff8b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004ff8e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004ff920 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004ff950 _sleep() at _sleep+0x367/frame 0xfffffe00004ffa00 crypto_ret_proc() at crypto_ret_proc+0x238/frame 0xfffffe00004ffa70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004ffab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004ffab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto returns 9 pid 16 tid 100086 td 0xfffff80003aef000 sched_switch() at sched_switch+0x942/frame 0xfffffe00004fa8b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004fa8e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004fa920 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004fa950 _sleep() at _sleep+0x367/frame 0xfffffe00004faa00 crypto_ret_proc() at crypto_ret_proc+0x238/frame 0xfffffe00004faa70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004faab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004faab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto returns 8 pid 15 tid 100085 td 0xfffff80003aef560 sched_switch() at sched_switch+0x942/frame 0xfffffe00004f58b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004f58e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004f5920 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004f5950 _sleep() at _sleep+0x367/frame 0xfffffe00004f5a00 crypto_ret_proc() at crypto_ret_proc+0x238/frame 0xfffffe00004f5a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004f5ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004f5ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto returns 7 pid 14 tid 100084 td 0xfffff80003af0000 sched_switch() at sched_switch+0x942/frame 0xfffffe00004f08b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004f08e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004f0920 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004f0950 _sleep() at _sleep+0x367/frame 0xfffffe00004f0a00 crypto_ret_proc() at crypto_ret_proc+0x238/frame 0xfffffe00004f0a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004f0ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004f0ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto returns 6 pid 9 tid 100083 td 0xfffff80003af0560 sched_switch() at sched_switch+0x942/frame 0xfffffe00004eb8b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004eb8e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004eb920 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004eb950 _sleep() at _sleep+0x367/frame 0xfffffe00004eba00 crypto_ret_proc() at crypto_ret_proc+0x238/frame 0xfffffe00004eba70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004ebab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004ebab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto returns 5 pid 8 tid 100082 td 0xfffff80003af1000 sched_switch() at sched_switch+0x942/frame 0xfffffe00004e68b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004e68e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004e6920 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004e6950 _sleep() at _sleep+0x367/frame 0xfffffe00004e6a00 crypto_ret_proc() at crypto_ret_proc+0x238/frame 0xfffffe00004e6a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004e6ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004e6ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto returns 4 pid 7 tid 100081 td 0xfffff80003adf560 sched_switch() at sched_switch+0x942/frame 0xfffffe00004e18b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004e18e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004e1920 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004e1950 _sleep() at _sleep+0x367/frame 0xfffffe00004e1a00 crypto_ret_proc() at crypto_ret_proc+0x238/frame 0xfffffe00004e1a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004e1ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004e1ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto returns 3 pid 6 tid 100080 td 0xfffff80003ae0000 sched_switch() at sched_switch+0x942/frame 0xfffffe00004dc8b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004dc8e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004dc920 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004dc950 _sleep() at _sleep+0x367/frame 0xfffffe00004dca00 crypto_ret_proc() at crypto_ret_proc+0x238/frame 0xfffffe00004dca70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004dcab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004dcab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto returns 2 pid 5 tid 100079 td 0xfffff80003ae0560 sched_switch() at sched_switch+0x942/frame 0xfffffe00004d78b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004d78e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004d7920 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004d7950 _sleep() at _sleep+0x367/frame 0xfffffe00004d7a00 crypto_ret_proc() at crypto_ret_proc+0x238/frame 0xfffffe00004d7a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004d7ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004d7ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto returns 1 pid 4 tid 100078 td 0xfffff80003ae1000 sched_switch() at sched_switch+0x942/frame 0xfffffe00004d28b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004d28e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004d2920 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004d2950 _sleep() at _sleep+0x367/frame 0xfffffe00004d2a00 crypto_ret_proc() at crypto_ret_proc+0x238/frame 0xfffffe00004d2a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004d2ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004d2ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto returns 0 pid 3 tid 100077 td 0xfffff80003ae1560 sched_switch() at sched_switch+0x942/frame 0xfffffe00004cd8b0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004cd8e0 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004cd920 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004cd950 _sleep() at _sleep+0x367/frame 0xfffffe00004cda00 crypto_ret_proc() at crypto_ret_proc+0x238/frame 0xfffffe00004cda70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004cdab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004cdab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command crypto pid 2 tid 100076 td 0xfffff80003ae2000 sched_switch() at sched_switch+0x942/frame 0xfffffe00004c88e0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004c8910 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004c8950 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004c8980 _sleep() at _sleep+0x367/frame 0xfffffe00004c8a30 crypto_proc() at crypto_proc+0x2ba/frame 0xfffffe00004c8a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004c8ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004c8ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command geom pid 13 tid 100060 td 0xfffff80003a5a000 sched_switch() at sched_switch+0x942/frame 0xfffffe00004788e0 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000478910 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000478950 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe0000478980 _sleep() at _sleep+0x367/frame 0xfffffe0000478a30 g_run_events() at g_run_events+0x4f/frame 0xfffffe0000478a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000478ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000478ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command geom pid 13 tid 100061 td 0xfffff80003a59560 sched_switch() at sched_switch+0x942/frame 0xfffffe000047d8e0 mi_switch() at mi_switch+0x18c/frame 0xfffffe000047d910 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000047d950 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe000047d980 _sleep() at _sleep+0x367/frame 0xfffffe000047da30 g_io_schedule_up() at g_io_schedule_up+0xa5/frame 0xfffffe000047da60 g_up_procbody() at g_up_procbody+0x6d/frame 0xfffffe000047da70 fork_exit() at fork_exit+0x84/frame 0xfffffe000047dab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000047dab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command geom pid 13 tid 100062 td 0xfffff80003a59000 sched_switch() at sched_switch+0x942/frame 0xfffffe00004828d0 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000482900 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000482940 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe0000482970 _sleep() at _sleep+0x367/frame 0xfffffe0000482a20 g_io_schedule_down() at g_io_schedule_down+0x14c/frame 0xfffffe0000482a60 g_down_procbody() at g_down_procbody+0x6d/frame 0xfffffe0000482a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000482ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000482ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100016 td 0xfffff80003946560 sched_switch() at sched_switch+0x942/frame 0xfffffe007899b9f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe007899ba20 ithread_loop() at ithread_loop+0x1f1/frame 0xfffffe007899ba70 fork_exit() at fork_exit+0x84/frame 0xfffffe007899bab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe007899bab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100017 td 0xfffff80003946000 sched_switch() at sched_switch+0x942/frame 0xfffffe00789a09f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789a0a20 ithread_loop() at ithread_loop+0x1f1/frame 0xfffffe00789a0a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789a0ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789a0ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100019 td 0xfffff80003976000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100046 td 0xfffff80003a51000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100047 td 0xfffff80003a50560 sched_switch() at sched_switch+0x942/frame 0xfffffe00004379f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000437a20 ithread_loop() at ithread_loop+0x1f1/frame 0xfffffe0000437a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000437ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000437ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100048 td 0xfffff80003a50000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100049 td 0xfffff80003a4f560 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100050 td 0xfffff80003a4f000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100051 td 0xfffff80003a4e560 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100052 td 0xfffff80003a4e000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100053 td 0xfffff80003a42560 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100054 td 0xfffff80003a42000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100055 td 0xfffff80003a5c560 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100056 td 0xfffff80003a5c000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100057 td 0xfffff80003a5b560 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100058 td 0xfffff80003a5b000 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100059 td 0xfffff80003a5a560 sched_switch() at sched_switch+0x942/frame 0xfffffe00004739f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000473a20 ithread_loop() at ithread_loop+0x1f1/frame 0xfffffe0000473a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000473ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000473ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100092 td 0xfffff80004038000 sched_switch() at sched_switch+0x942/frame 0xfffffe00005189f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000518a20 ithread_loop() at ithread_loop+0x1f1/frame 0xfffffe0000518a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000518ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000518ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100093 td 0xfffff80004037560 sched_switch() at sched_switch+0x942/frame 0xfffffe000051d9f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe000051da20 ithread_loop() at ithread_loop+0x1f1/frame 0xfffffe000051da70 fork_exit() at fork_exit+0x84/frame 0xfffffe000051dab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000051dab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100099 td 0xfffff8000402d560 sched_switch() at sched_switch+0x942/frame 0xfffffe000053b9f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe000053ba20 ithread_loop() at ithread_loop+0x1f1/frame 0xfffffe000053ba70 fork_exit() at fork_exit+0x84/frame 0xfffffe000053bab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000053bab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command intr pid 12 tid 100110 td 0xfffff80006096560 fork_trampoline() at fork_trampoline Tracing command intr pid 12 tid 100111 td 0xfffff80006096000 sched_switch() at sched_switch+0x942/frame 0xfffffe00005779f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000577a20 ithread_loop() at ithread_loop+0x1f1/frame 0xfffffe0000577a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000577ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000577ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100003 td 0xfffff80003938000 (CPU 0) kdb_alt_break_internal() at kdb_alt_break_internal+0x106/frame 0xfffffe007895a700 kdb_alt_break() at kdb_alt_break+0xb/frame 0xfffffe007895a710 uart_intr_rxready() at uart_intr_rxready+0x9a/frame 0xfffffe007895a740 uart_intr() at uart_intr+0x138/frame 0xfffffe007895a780 intr_event_handle() at intr_event_handle+0xbc/frame 0xfffffe007895a7d0 intr_execute_handlers() at intr_execute_handlers+0x58/frame 0xfffffe007895a800 lapic_handle_intr() at lapic_handle_intr+0x6d/frame 0xfffffe007895a840 apic_isr1_u() at apic_isr1_u+0xc1/frame 0xfffffe007895a910 acpi_cpu_idle() at acpi_cpu_idle+0x2ee/frame 0xfffffe007895a960 cpu_idle_acpi() at cpu_idle_acpi+0x3f/frame 0xfffffe007895a980 cpu_idle() at cpu_idle+0x8f/frame 0xfffffe007895a9a0 sched_idletd() at sched_idletd+0x517/frame 0xfffffe007895aa70 fork_exit() at fork_exit+0x84/frame 0xfffffe007895aab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe007895aab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100004 td 0xfffff80003937560 (CPU 1) cpustop_handler() at cpustop_handler+0x28/frame 0xfffffe00007ffdf0 ipi_nmi_handler() at ipi_nmi_handler+0x44/frame 0xfffffe00007ffe10 trap() at trap+0x49/frame 0xfffffe00007fff20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe00007fff20 --- trap 0x13, rip = 0xffffffff810c8726, rsp = 0xfffffe007895f910, rbp = 0xfffffe007895f910 --- acpi_cpu_c1() at acpi_cpu_c1+0x6/frame 0xfffffe007895f910 acpi_cpu_idle() at acpi_cpu_idle+0x2ee/frame 0xfffffe007895f960 cpu_idle_acpi() at cpu_idle_acpi+0x3f/frame 0xfffffe007895f980 cpu_idle() at cpu_idle+0x8f/frame 0xfffffe007895f9a0 sched_idletd() at sched_idletd+0x517/frame 0xfffffe007895fa70 fork_exit() at fork_exit+0x84/frame 0xfffffe007895fab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe007895fab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100005 td 0xfffff80003937000 (CPU 2) cpustop_handler() at cpustop_handler+0x28/frame 0xfffffe00071b2df0 ipi_nmi_handler() at ipi_nmi_handler+0x44/frame 0xfffffe00071b2e10 trap() at trap+0x49/frame 0xfffffe00071b2f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe00071b2f20 --- trap 0x13, rip = 0xffffffff810c8726, rsp = 0xfffffe0078964910, rbp = 0xfffffe0078964910 --- acpi_cpu_c1() at acpi_cpu_c1+0x6/frame 0xfffffe0078964910 acpi_cpu_idle() at acpi_cpu_idle+0x2ee/frame 0xfffffe0078964960 cpu_idle_acpi() at cpu_idle_acpi+0x3f/frame 0xfffffe0078964980 cpu_idle() at cpu_idle+0x8f/frame 0xfffffe00789649a0 sched_idletd() at sched_idletd+0x517/frame 0xfffffe0078964a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0078964ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0078964ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100006 td 0xfffff80003936560 (CPU 3) cpustop_handler() at cpustop_handler+0x28/frame 0xfffffe00071bbdf0 ipi_nmi_handler() at ipi_nmi_handler+0x44/frame 0xfffffe00071bbe10 trap() at trap+0x49/frame 0xfffffe00071bbf20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe00071bbf20 --- trap 0x13, rip = 0xffffffff810c8726, rsp = 0xfffffe0078969910, rbp = 0xfffffe0078969910 --- acpi_cpu_c1() at acpi_cpu_c1+0x6/frame 0xfffffe0078969910 acpi_cpu_idle() at acpi_cpu_idle+0x2ee/frame 0xfffffe0078969960 cpu_idle_acpi() at cpu_idle_acpi+0x3f/frame 0xfffffe0078969980 cpu_idle() at cpu_idle+0x8f/frame 0xfffffe00789699a0 sched_idletd() at sched_idletd+0x517/frame 0xfffffe0078969a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0078969ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0078969ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100007 td 0xfffff80003936000 (CPU 4) cpustop_handler() at cpustop_handler+0x28/frame 0xfffffe00071c4df0 ipi_nmi_handler() at ipi_nmi_handler+0x44/frame 0xfffffe00071c4e10 trap() at trap+0x49/frame 0xfffffe00071c4f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe00071c4f20 --- trap 0x13, rip = 0xffffffff810c8726, rsp = 0xfffffe007896e910, rbp = 0xfffffe007896e910 --- acpi_cpu_c1() at acpi_cpu_c1+0x6/frame 0xfffffe007896e910 acpi_cpu_idle() at acpi_cpu_idle+0x2ee/frame 0xfffffe007896e960 cpu_idle_acpi() at cpu_idle_acpi+0x3f/frame 0xfffffe007896e980 cpu_idle() at cpu_idle+0x8f/frame 0xfffffe007896e9a0 sched_idletd() at sched_idletd+0x517/frame 0xfffffe007896ea70 fork_exit() at fork_exit+0x84/frame 0xfffffe007896eab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe007896eab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100008 td 0xfffff80003935560 (CPU 5) cpustop_handler() at cpustop_handler+0x28/frame 0xfffffe00071cddf0 ipi_nmi_handler() at ipi_nmi_handler+0x44/frame 0xfffffe00071cde10 trap() at trap+0x49/frame 0xfffffe00071cdf20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe00071cdf20 --- trap 0x13, rip = 0xffffffff810c8726, rsp = 0xfffffe0078973910, rbp = 0xfffffe0078973910 --- acpi_cpu_c1() at acpi_cpu_c1+0x6/frame 0xfffffe0078973910 acpi_cpu_idle() at acpi_cpu_idle+0x2ee/frame 0xfffffe0078973960 cpu_idle_acpi() at cpu_idle_acpi+0x3f/frame 0xfffffe0078973980 cpu_idle() at cpu_idle+0x8f/frame 0xfffffe00789739a0 sched_idletd() at sched_idletd+0x517/frame 0xfffffe0078973a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0078973ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0078973ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100009 td 0xfffff80003935000 (CPU 6) cpustop_handler() at cpustop_handler+0x28/frame 0xfffffe00071d6df0 ipi_nmi_handler() at ipi_nmi_handler+0x44/frame 0xfffffe00071d6e10 trap() at trap+0x49/frame 0xfffffe00071d6f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe00071d6f20 --- trap 0x13, rip = 0xffffffff810c8726, rsp = 0xfffffe0078978910, rbp = 0xfffffe0078978910 --- acpi_cpu_c1() at acpi_cpu_c1+0x6/frame 0xfffffe0078978910 acpi_cpu_idle() at acpi_cpu_idle+0x2ee/frame 0xfffffe0078978960 cpu_idle_acpi() at cpu_idle_acpi+0x3f/frame 0xfffffe0078978980 cpu_idle() at cpu_idle+0x8f/frame 0xfffffe00789789a0 sched_idletd() at sched_idletd+0x517/frame 0xfffffe0078978a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0078978ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0078978ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100010 td 0xfffff80003949560 (CPU 7) cpustop_handler() at cpustop_handler+0x28/frame 0xfffffe00071dfdf0 ipi_nmi_handler() at ipi_nmi_handler+0x44/frame 0xfffffe00071dfe10 trap() at trap+0x49/frame 0xfffffe00071dff20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe00071dff20 --- trap 0x13, rip = 0xffffffff810c8726, rsp = 0xfffffe007897d910, rbp = 0xfffffe007897d910 --- acpi_cpu_c1() at acpi_cpu_c1+0x6/frame 0xfffffe007897d910 acpi_cpu_idle() at acpi_cpu_idle+0x2ee/frame 0xfffffe007897d960 cpu_idle_acpi() at cpu_idle_acpi+0x3f/frame 0xfffffe007897d980 cpu_idle() at cpu_idle+0x8f/frame 0xfffffe007897d9a0 sched_idletd() at sched_idletd+0x517/frame 0xfffffe007897da70 fork_exit() at fork_exit+0x84/frame 0xfffffe007897dab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe007897dab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100011 td 0xfffff80003949000 (CPU 8) cpustop_handler() at cpustop_handler+0x28/frame 0xfffffe00071e8df0 ipi_nmi_handler() at ipi_nmi_handler+0x44/frame 0xfffffe00071e8e10 trap() at trap+0x49/frame 0xfffffe00071e8f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe00071e8f20 --- trap 0x13, rip = 0xffffffff810c8726, rsp = 0xfffffe0078982910, rbp = 0xfffffe0078982910 --- acpi_cpu_c1() at acpi_cpu_c1+0x6/frame 0xfffffe0078982910 acpi_cpu_idle() at acpi_cpu_idle+0x2ee/frame 0xfffffe0078982960 cpu_idle_acpi() at cpu_idle_acpi+0x3f/frame 0xfffffe0078982980 cpu_idle() at cpu_idle+0x8f/frame 0xfffffe00789829a0 sched_idletd() at sched_idletd+0x517/frame 0xfffffe0078982a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0078982ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0078982ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100012 td 0xfffff80003948560 (CPU 9) cpustop_handler() at cpustop_handler+0x28/frame 0xfffffe00071f1df0 ipi_nmi_handler() at ipi_nmi_handler+0x44/frame 0xfffffe00071f1e10 trap() at trap+0x49/frame 0xfffffe00071f1f20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe00071f1f20 --- trap 0x13, rip = 0xffffffff810c8726, rsp = 0xfffffe0078987910, rbp = 0xfffffe0078987910 --- acpi_cpu_c1() at acpi_cpu_c1+0x6/frame 0xfffffe0078987910 acpi_cpu_idle() at acpi_cpu_idle+0x2ee/frame 0xfffffe0078987960 cpu_idle_acpi() at cpu_idle_acpi+0x3f/frame 0xfffffe0078987980 cpu_idle() at cpu_idle+0x8f/frame 0xfffffe00789879a0 sched_idletd() at sched_idletd+0x517/frame 0xfffffe0078987a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0078987ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0078987ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100013 td 0xfffff80003948000 (CPU 10) cpustop_handler() at cpustop_handler+0x28/frame 0xfffffe00071fadf0 ipi_nmi_handler() at ipi_nmi_handler+0x44/frame 0xfffffe00071fae10 trap() at trap+0x49/frame 0xfffffe00071faf20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe00071faf20 --- trap 0x13, rip = 0xffffffff810c8726, rsp = 0xfffffe007898c910, rbp = 0xfffffe007898c910 --- acpi_cpu_c1() at acpi_cpu_c1+0x6/frame 0xfffffe007898c910 acpi_cpu_idle() at acpi_cpu_idle+0x2ee/frame 0xfffffe007898c960 cpu_idle_acpi() at cpu_idle_acpi+0x3f/frame 0xfffffe007898c980 cpu_idle() at cpu_idle+0x8f/frame 0xfffffe007898c9a0 sched_idletd() at sched_idletd+0x517/frame 0xfffffe007898ca70 fork_exit() at fork_exit+0x84/frame 0xfffffe007898cab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe007898cab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command idle pid 11 tid 100014 td 0xfffff80003947560 (CPU 11) cpustop_handler() at cpustop_handler+0x28/frame 0xfffffe00071ffdf0 ipi_nmi_handler() at ipi_nmi_handler+0x44/frame 0xfffffe00071ffe10 trap() at trap+0x49/frame 0xfffffe00071fff20 nmi_calltrap() at nmi_calltrap+0x8/frame 0xfffffe00071fff20 --- trap 0x13, rip = 0xffffffff810c8726, rsp = 0xfffffe0078991910, rbp = 0xfffffe0078991910 --- acpi_cpu_c1() at acpi_cpu_c1+0x6/frame 0xfffffe0078991910 acpi_cpu_idle() at acpi_cpu_idle+0x2ee/frame 0xfffffe0078991960 cpu_idle_acpi() at cpu_idle_acpi+0x3f/frame 0xfffffe0078991980 cpu_idle() at cpu_idle+0x8f/frame 0xfffffe00789919a0 sched_idletd() at sched_idletd+0x517/frame 0xfffffe0078991a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0078991ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0078991ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command init pid 1 tid 100002 td 0xfffff80003938560 sched_switch() at sched_switch+0x942/frame 0xfffffe0078955550 mi_switch() at mi_switch+0x18c/frame 0xfffffe0078955580 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789555c0 sleepq_catch_signals() at sleepq_catch_signals+0x300/frame 0xfffffe0078955620 sleepq_wait_sig() at sleepq_wait_sig+0xf/frame 0xfffffe0078955650 _sleep() at _sleep+0x359/frame 0xfffffe0078955700 kern_wait6() at kern_wait6+0x4c4/frame 0xfffffe0078955790 sys_wait4() at sys_wait4+0x78/frame 0xfffffe0078955980 amd64_syscall() at amd64_syscall+0x7ab/frame 0xfffffe0078955ab0 fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe0078955ab0 --- syscall (7, FreeBSD ELF64, sys_wait4), rip = 0x2a71ca, rsp = 0x7fffffffe788, rbp = 0x7fffffffe7f0 --- Tracing command audit pid 10 tid 100001 td 0xfffff80003939000 sched_switch() at sched_switch+0x942/frame 0xfffffe0078950910 mi_switch() at mi_switch+0x18c/frame 0xfffffe0078950940 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0078950980 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789509b0 _cv_wait() at _cv_wait+0x203/frame 0xfffffe0078950a10 audit_worker() at audit_worker+0x93/frame 0xfffffe0078950a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0078950ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0078950ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100000 td 0xffffffff81f40060 sched_switch() at sched_switch+0x942/frame 0xffffffff824f39c0 mi_switch() at mi_switch+0x18c/frame 0xffffffff824f39f0 sleepq_switch() at sleepq_switch+0x10d/frame 0xffffffff824f3a30 sleepq_timedwait() at sleepq_timedwait+0x50/frame 0xffffffff824f3a70 _sleep() at _sleep+0x307/frame 0xffffffff824f3b20 swapper() at swapper+0x247/frame 0xffffffff824f3b70 btext() at btext+0x2c Tracing command kernel pid 0 tid 100015 td 0xfffff80003947000 sched_switch() at sched_switch+0x942/frame 0xfffffe00789968f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe0078996920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0078996960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe0078996990 _sleep() at _sleep+0x367/frame 0xfffffe0078996a40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe0078996a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0078996ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0078996ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100018 td 0xfffff80003939560 sched_switch() at sched_switch+0x942/frame 0xfffffe00789a58f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789a5920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789a5960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789a5990 _sleep() at _sleep+0x367/frame 0xfffffe00789a5a40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe00789a5a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789a5ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789a5ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100020 td 0xfffff80003975560 sched_switch() at sched_switch+0x942/frame 0xfffffe00789af8f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789af920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789af960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789af990 _sleep() at _sleep+0x367/frame 0xfffffe00789afa40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe00789afa70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789afab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789afab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100021 td 0xfffff80003975000 sched_switch() at sched_switch+0x942/frame 0xfffffe00789b4920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789b4950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789b4990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789b49c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789b4a40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789b4a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789b4ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789b4ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100022 td 0xfffff80003974560 sched_switch() at sched_switch+0x942/frame 0xfffffe00789b9920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789b9950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789b9990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789b99c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789b9a40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789b9a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789b9ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789b9ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100023 td 0xfffff80003974000 sched_switch() at sched_switch+0x942/frame 0xfffffe00789be920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789be950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789be990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789be9c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789bea40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789bea70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789beab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789beab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100024 td 0xfffff80003973560 sched_switch() at sched_switch+0x942/frame 0xfffffe00789c3920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789c3950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789c3990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789c39c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789c3a40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789c3a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789c3ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789c3ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100025 td 0xfffff80003973000 sched_switch() at sched_switch+0x942/frame 0xfffffe00789c8920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789c8950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789c8990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789c89c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789c8a40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789c8a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789c8ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789c8ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100026 td 0xfffff80003972560 sched_switch() at sched_switch+0x942/frame 0xfffffe00789cd920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789cd950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789cd990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789cd9c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789cda40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789cda70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789cdab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789cdab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100027 td 0xfffff80003972000 sched_switch() at sched_switch+0x942/frame 0xfffffe00789d2920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789d2950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789d2990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789d29c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789d2a40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789d2a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789d2ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789d2ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100028 td 0xfffff8000398a560 sched_switch() at sched_switch+0x942/frame 0xfffffe00789d7920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789d7950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789d7990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789d79c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789d7a40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789d7a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789d7ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789d7ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100029 td 0xfffff8000398a000 sched_switch() at sched_switch+0x942/frame 0xfffffe00789dc920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789dc950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789dc990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789dc9c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789dca40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789dca70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789dcab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789dcab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100030 td 0xfffff80003985560 sched_switch() at sched_switch+0x942/frame 0xfffffe00789e1920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789e1950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789e1990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789e19c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789e1a40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789e1a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789e1ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789e1ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100031 td 0xfffff80003985000 sched_switch() at sched_switch+0x942/frame 0xfffffe00789e6920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789e6950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789e6990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789e69c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789e6a40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789e6a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789e6ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789e6ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100032 td 0xfffff80003984560 sched_switch() at sched_switch+0x942/frame 0xfffffe00789eb920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789eb950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789eb990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789eb9c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789eba40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789eba70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789ebab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789ebab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100033 td 0xfffff80003984000 sched_switch() at sched_switch+0x942/frame 0xfffffe00789f0920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789f0950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789f0990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789f09c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789f0a40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789f0a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789f0ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789f0ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100034 td 0xfffff80003a1b000 sched_switch() at sched_switch+0x942/frame 0xfffffe00789f5920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789f5950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789f5990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789f59c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789f5a40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789f5a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789f5ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789f5ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100035 td 0xfffff80003a1a560 sched_switch() at sched_switch+0x942/frame 0xfffffe00789fa920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789fa950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789fa990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789fa9c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789faa40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789faa70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789faab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789faab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100036 td 0xfffff80003a1a000 sched_switch() at sched_switch+0x942/frame 0xfffffe00789ff920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00789ff950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00789ff990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00789ff9c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00789ffa40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe00789ffa70 fork_exit() at fork_exit+0x84/frame 0xfffffe00789ffab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00789ffab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100037 td 0xfffff80003a19560 sched_switch() at sched_switch+0x942/frame 0xfffffe0000405920 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000405950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000405990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004059c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe0000405a40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe0000405a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000405ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000405ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100038 td 0xfffff80003a19000 sched_switch() at sched_switch+0x942/frame 0xfffffe000040a920 mi_switch() at mi_switch+0x18c/frame 0xfffffe000040a950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000040a990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe000040a9c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe000040aa40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe000040aa70 fork_exit() at fork_exit+0x84/frame 0xfffffe000040aab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000040aab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100039 td 0xfffff80003a18560 sched_switch() at sched_switch+0x942/frame 0xfffffe000040f920 mi_switch() at mi_switch+0x18c/frame 0xfffffe000040f950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000040f990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe000040f9c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe000040fa40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe000040fa70 fork_exit() at fork_exit+0x84/frame 0xfffffe000040fab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000040fab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100040 td 0xfffff80003a18000 sched_switch() at sched_switch+0x942/frame 0xfffffe0000414920 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000414950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000414990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004149c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe0000414a40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe0000414a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000414ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000414ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100041 td 0xfffff80003a0c560 sched_switch() at sched_switch+0x942/frame 0xfffffe0000419920 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000419950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000419990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004199c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe0000419a40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe0000419a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000419ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000419ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100042 td 0xfffff80003a0c000 sched_switch() at sched_switch+0x942/frame 0xfffffe000041e920 mi_switch() at mi_switch+0x18c/frame 0xfffffe000041e950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000041e990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe000041e9c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe000041ea40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe000041ea70 fork_exit() at fork_exit+0x84/frame 0xfffffe000041eab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000041eab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100043 td 0xfffff80003a27560 sched_switch() at sched_switch+0x942/frame 0xfffffe0000423920 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000423950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000423990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004239c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe0000423a40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe0000423a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000423ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000423ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100044 td 0xfffff80003a27000 sched_switch() at sched_switch+0x942/frame 0xfffffe0000428920 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000428950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000428990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004289c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe0000428a40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe0000428a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000428ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000428ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100045 td 0xfffff80003a26560 sched_switch() at sched_switch+0x942/frame 0xfffffe000042d920 mi_switch() at mi_switch+0x18c/frame 0xfffffe000042d950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000042d990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe000042d9c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe000042da40 gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xb1/frame 0xfffffe000042da70 fork_exit() at fork_exit+0x84/frame 0xfffffe000042dab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000042dab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100063 td 0xfffff80003a51560 sched_switch() at sched_switch+0x942/frame 0xfffffe00004878f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000487920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000487960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe0000487990 _sleep() at _sleep+0x367/frame 0xfffffe0000487a40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe0000487a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000487ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000487ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100064 td 0xfffff80003adf000 sched_switch() at sched_switch+0x942/frame 0xfffffe000048c8f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe000048c920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000048c960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe000048c990 _sleep() at _sleep+0x367/frame 0xfffffe000048ca40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe000048ca70 fork_exit() at fork_exit+0x84/frame 0xfffffe000048cab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000048cab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100065 td 0xfffff80003ade560 sched_switch() at sched_switch+0x942/frame 0xfffffe00004918f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000491920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000491960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe0000491990 _sleep() at _sleep+0x367/frame 0xfffffe0000491a40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe0000491a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000491ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000491ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100066 td 0xfffff80003ade000 sched_switch() at sched_switch+0x942/frame 0xfffffe00004968f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe0000496920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe0000496960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe0000496990 _sleep() at _sleep+0x367/frame 0xfffffe0000496a40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe0000496a70 fork_exit() at fork_exit+0x84/frame 0xfffffe0000496ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe0000496ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100067 td 0xfffff80003add560 sched_switch() at sched_switch+0x942/frame 0xfffffe000049b8f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe000049b920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000049b960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe000049b990 _sleep() at _sleep+0x367/frame 0xfffffe000049ba40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe000049ba70 fork_exit() at fork_exit+0x84/frame 0xfffffe000049bab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000049bab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100068 td 0xfffff80003add000 sched_switch() at sched_switch+0x942/frame 0xfffffe00004a08f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004a0920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004a0960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004a0990 _sleep() at _sleep+0x367/frame 0xfffffe00004a0a40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe00004a0a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004a0ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004a0ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100069 td 0xfffff80003adc560 sched_switch() at sched_switch+0x942/frame 0xfffffe00004a58f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004a5920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004a5960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004a5990 _sleep() at _sleep+0x367/frame 0xfffffe00004a5a40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe00004a5a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004a5ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004a5ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100070 td 0xfffff80003adc000 sched_switch() at sched_switch+0x942/frame 0xfffffe00004aa8f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004aa920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004aa960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004aa990 _sleep() at _sleep+0x367/frame 0xfffffe00004aaa40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe00004aaa70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004aaab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004aaab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100071 td 0xfffff80003adb560 sched_switch() at sched_switch+0x942/frame 0xfffffe00004af8f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004af920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004af960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004af990 _sleep() at _sleep+0x367/frame 0xfffffe00004afa40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe00004afa70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004afab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004afab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100072 td 0xfffff80003adb000 sched_switch() at sched_switch+0x942/frame 0xfffffe00004b48f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004b4920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004b4960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004b4990 _sleep() at _sleep+0x367/frame 0xfffffe00004b4a40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe00004b4a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004b4ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004b4ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100073 td 0xfffff80003ae3560 sched_switch() at sched_switch+0x942/frame 0xfffffe00004b98f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004b9920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004b9960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004b9990 _sleep() at _sleep+0x367/frame 0xfffffe00004b9a40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe00004b9a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004b9ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004b9ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100074 td 0xfffff80003ae3000 sched_switch() at sched_switch+0x942/frame 0xfffffe00004be8f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004be920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004be960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004be990 _sleep() at _sleep+0x367/frame 0xfffffe00004bea40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe00004bea70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004beab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004beab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100075 td 0xfffff80003ae2560 sched_switch() at sched_switch+0x942/frame 0xfffffe00004c38f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00004c3920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00004c3960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00004c3990 _sleep() at _sleep+0x367/frame 0xfffffe00004c3a40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe00004c3a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00004c3ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00004c3ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100112 td 0xfffff8000608e560 sched_switch() at sched_switch+0x942/frame 0xfffffe000057c920 mi_switch() at mi_switch+0x18c/frame 0xfffffe000057c950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe000057c990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe000057c9c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe000057ca40 taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe000057ca70 fork_exit() at fork_exit+0x84/frame 0xfffffe000057cab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe000057cab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100119 td 0xfffff80013195000 sched_switch() at sched_switch+0x942/frame 0xfffffe00005a0920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005a0950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005a0990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005a09c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00005a0a40 taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00005a0a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005a0ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005a0ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100120 td 0xfffff80013194560 sched_switch() at sched_switch+0x942/frame 0xfffffe00005a5920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005a5950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005a5990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005a59c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00005a5a40 taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00005a5a70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005a5ab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005a5ab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100121 td 0xfffff80013194000 sched_switch() at sched_switch+0x942/frame 0xfffffe00005aa920 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005aa950 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005aa990 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005aa9c0 msleep_spin_sbt() at msleep_spin_sbt+0x1eb/frame 0xfffffe00005aaa40 taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00005aaa70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005aaab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005aaab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- Tracing command kernel pid 0 tid 100122 td 0xfffff80013193560 sched_switch() at sched_switch+0x942/frame 0xfffffe00005af8f0 mi_switch() at mi_switch+0x18c/frame 0xfffffe00005af920 sleepq_switch() at sleepq_switch+0x10d/frame 0xfffffe00005af960 sleepq_wait() at sleepq_wait+0x43/frame 0xfffffe00005af990 _sleep() at _sleep+0x367/frame 0xfffffe00005afa40 taskqueue_thread_loop() at taskqueue_thread_loop+0xe1/frame 0xfffffe00005afa70 fork_exit() at fork_exit+0x84/frame 0xfffffe00005afab0 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00005afab0 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- db:0:pho> dump Dumping 1950 out of 32625 MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91% Dump complete db> reset Index: /usr/src/sys/kern/vfs_bio.c =================================================================== --- /usr/src/sys/kern/vfs_bio.c (revision 330766) +++ /usr/src/sys/kern/vfs_bio.c (working copy) @@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -100,6 +101,7 @@ struct buf_ops buf_ops_bio = { .bop_bdflush = bufbdflush, }; +struct bufdomain; static struct buf *buf; /* buffer header pool */ extern struct buf *swbuf; /* Swap buffer header pool. */ caddr_t unmapped_buf; @@ -123,8 +125,8 @@ static int vfs_bio_clcheck(struct vnode *vp, int s daddr_t lblkno, daddr_t blkno); static void breada(struct vnode *, daddr_t *, int *, int, struct ucred *, int, void (*)(struct buf *)); -static int buf_flush(struct vnode *vp, int); -static int flushbufqueues(struct vnode *, int, int); +static int buf_flush(struct vnode *vp, struct bufdomain *, int); +static int flushbufqueues(struct vnode *, struct bufdomain *, int, int); static void buf_daemon(void); static __inline void bd_wakeup(void); static int sysctl_runningspace(SYSCTL_HANDLER_ARGS); @@ -133,6 +135,7 @@ static void bufkva_free(struct buf *); static int buf_import(void *, void **, int, int, int); static void buf_release(void *, void **, int); static void maxbcachebuf_adjust(void); +static inline struct bufdomain *bufdomain(struct buf *); static int sysctl_bufspace(SYSCTL_HANDLER_ARGS); int vmiodirenable = TRUE; @@ -147,22 +150,22 @@ static counter_u64_t bufkvaspace; SYSCTL_COUNTER_U64(_vfs, OID_AUTO, bufkvaspace, CTLFLAG_RD, &bufkvaspace, "Kernel virtual memory used for buffers"); static long maxbufspace; -SYSCTL_LONG(_vfs, OID_AUTO, maxbufspace, CTLFLAG_RW, &maxbufspace, 0, +SYSCTL_LONG(_vfs, OID_AUTO, maxbufspace, CTLFLAG_RD, &maxbufspace, 0, "Maximum allowed value of bufspace (including metadata)"); static long bufmallocspace; SYSCTL_LONG(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD, &bufmallocspace, 0, "Amount of malloced memory for buffers"); static long maxbufmallocspace; -SYSCTL_LONG(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW, &maxbufmallocspace, +SYSCTL_LONG(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RD, &maxbufmallocspace, 0, "Maximum amount of malloced memory for buffers"); static long lobufspace; -SYSCTL_LONG(_vfs, OID_AUTO, lobufspace, CTLFLAG_RW, &lobufspace, 0, +SYSCTL_LONG(_vfs, OID_AUTO, lobufspace, CTLFLAG_RD, &lobufspace, 0, "Minimum amount of buffers we want to have"); long hibufspace; -SYSCTL_LONG(_vfs, OID_AUTO, hibufspace, CTLFLAG_RW, &hibufspace, 0, +SYSCTL_LONG(_vfs, OID_AUTO, hibufspace, CTLFLAG_RD, &hibufspace, 0, "Maximum allowed value of bufspace (excluding metadata)"); long bufspacethresh; -SYSCTL_LONG(_vfs, OID_AUTO, bufspacethresh, CTLFLAG_RW, &bufspacethresh, +SYSCTL_LONG(_vfs, OID_AUTO, bufspacethresh, CTLFLAG_RD, &bufspacethresh, 0, "Bufspace consumed before waking the daemon to free some"); static counter_u64_t buffreekvacnt; SYSCTL_COUNTER_U64(_vfs, OID_AUTO, buffreekvacnt, CTLFLAG_RW, &buffreekvacnt, @@ -190,26 +193,27 @@ SYSCTL_INT(_vfs, OID_AUTO, altbufferflushes, CTLFL static int recursiveflushes; SYSCTL_INT(_vfs, OID_AUTO, recursiveflushes, CTLFLAG_RW, &recursiveflushes, 0, "Number of flushes skipped due to being recursive"); -static int numdirtybuffers; -SYSCTL_INT(_vfs, OID_AUTO, numdirtybuffers, CTLFLAG_RD, &numdirtybuffers, 0, +static int sysctl_numdirtybuffers(SYSCTL_HANDLER_ARGS); +SYSCTL_PROC(_vfs, OID_AUTO, numdirtybuffers, + CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RD, NULL, 0, sysctl_numdirtybuffers, "I", "Number of buffers that are dirty (has unwritten changes) at the moment"); static int lodirtybuffers; -SYSCTL_INT(_vfs, OID_AUTO, lodirtybuffers, CTLFLAG_RW, &lodirtybuffers, 0, +SYSCTL_INT(_vfs, OID_AUTO, lodirtybuffers, CTLFLAG_RD, &lodirtybuffers, 0, "How many buffers we want to have free before bufdaemon can sleep"); static int hidirtybuffers; -SYSCTL_INT(_vfs, OID_AUTO, hidirtybuffers, CTLFLAG_RW, &hidirtybuffers, 0, +SYSCTL_INT(_vfs, OID_AUTO, hidirtybuffers, CTLFLAG_RD, &hidirtybuffers, 0, "When the number of dirty buffers is considered severe"); int dirtybufthresh; -SYSCTL_INT(_vfs, OID_AUTO, dirtybufthresh, CTLFLAG_RW, &dirtybufthresh, +SYSCTL_INT(_vfs, OID_AUTO, dirtybufthresh, CTLFLAG_RD, &dirtybufthresh, 0, "Number of bdwrite to bawrite conversions to clear dirty buffers"); static int numfreebuffers; SYSCTL_INT(_vfs, OID_AUTO, numfreebuffers, CTLFLAG_RD, &numfreebuffers, 0, "Number of free buffers"); static int lofreebuffers; -SYSCTL_INT(_vfs, OID_AUTO, lofreebuffers, CTLFLAG_RW, &lofreebuffers, 0, +SYSCTL_INT(_vfs, OID_AUTO, lofreebuffers, CTLFLAG_RD, &lofreebuffers, 0, "Target number of free buffers"); static int hifreebuffers; -SYSCTL_INT(_vfs, OID_AUTO, hifreebuffers, CTLFLAG_RW, &hifreebuffers, 0, +SYSCTL_INT(_vfs, OID_AUTO, hifreebuffers, CTLFLAG_RD, &hifreebuffers, 0, "Threshold for clean buffer recycling"); static counter_u64_t getnewbufcalls; SYSCTL_COUNTER_U64(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RD, @@ -308,10 +312,10 @@ struct bufqueue { #define BQ_ASSERT_LOCKED(bq) mtx_assert(BQ_LOCKPTR((bq)), MA_OWNED) struct bufqueue __exclusive_cache_line bqempty; -struct bufqueue __exclusive_cache_line bqdirty; struct bufdomain { struct bufqueue bd_subq[MAXCPU + 1]; /* Per-cpu sub queues + global */ + struct bufqueue bd_dirtyq; struct bufqueue *bd_cleanq; struct mtx_padalign bd_run_lock; /* Constants */ @@ -321,10 +325,14 @@ struct bufdomain { long bd_bufspacethresh; int bd_hifreebuffers; int bd_lofreebuffers; + int bd_hidirtybuffers; + int bd_lodirtybuffers; + int bd_dirtybufthresh; int bd_lim; /* atomics */ int bd_wanted; - int __aligned(CACHE_LINE_SIZE) bd_running; + int __aligned(CACHE_LINE_SIZE) bd_numdirtybuffers; + int __aligned(CACHE_LINE_SIZE) bd_running; long __aligned(CACHE_LINE_SIZE) bd_bufspace; int __aligned(CACHE_LINE_SIZE) bd_freebuffers; } __aligned(CACHE_LINE_SIZE); @@ -336,15 +344,19 @@ struct bufdomain { #define BD_RUN_LOCKPTR(bd) (&(bd)->bd_run_lock) #define BD_RUN_LOCK(bd) mtx_lock(BD_RUN_LOCKPTR((bd))) #define BD_RUN_UNLOCK(bd) mtx_unlock(BD_RUN_LOCKPTR((bd))) -#define BD_DOMAIN(bd) (bd - bdclean) +#define BD_DOMAIN(bd) (bd - bdomain) -/* Maximum number of clean buffer domains. */ -#define CLEAN_DOMAINS 8 +/* Maximum number of buffer domains. */ +#define BUF_DOMAINS 8 +BITSET_DEFINE(bufdomainset, BUF_DOMAINS); +struct bufdomainset bdlodirty; /* Domains > lodirty */ +struct bufdomainset bdhidirty; /* Domains > hidirty */ + /* Configured number of clean queues. */ -static int __read_mostly clean_domains; +static int __read_mostly buf_domains; -struct bufdomain __exclusive_cache_line bdclean[CLEAN_DOMAINS]; +struct bufdomain __exclusive_cache_line bdomain[BUF_DOMAINS]; static void bq_remove(struct bufqueue *bq, struct buf *bp); static void bq_insert(struct bufqueue *bq, struct buf *bp, bool unlock); @@ -403,8 +415,8 @@ sysctl_bufspace(SYSCTL_HANDLER_ARGS) int i; lvalue = 0; - for (i = 0; i < clean_domains; i++) - lvalue += bdclean[i].bd_bufspace; + for (i = 0; i < buf_domains; i++) + lvalue += bdomain[i].bd_bufspace; if (sizeof(int) == sizeof(long) || req->oldlen >= sizeof(long)) return (sysctl_handle_long(oidp, &lvalue, 0, req)); if (lvalue > INT_MAX) @@ -421,12 +433,24 @@ sysctl_bufspace(SYSCTL_HANDLER_ARGS) int i; lvalue = 0; - for (i = 0; i < clean_domains; i++) - lvalue += bdclean[i].bd_bufspace; + for (i = 0; i < buf_domains; i++) + lvalue += bdomain[i].bd_bufspace; return (sysctl_handle_long(oidp, &lvalue, 0, req)); } #endif +static int +sysctl_numdirtybuffers(SYSCTL_HANDLER_ARGS) +{ + int value; + int i; + + value = 0; + for (i = 0; i < buf_domains; i++) + value += bdomain[i].bd_numdirtybuffers; + return (sysctl_handle_int(oidp, &value, 0, req)); +} + /* * bdirtywakeup: * @@ -444,6 +468,42 @@ bdirtywakeup(void) } /* + * bd_clear: + * + * Clear a domain from the appropriate bitsets when dirtybuffers + * is decremented. + */ +static void +bd_clear(struct bufdomain *bd) +{ + + mtx_lock(&bdirtylock); + if (bd->bd_numdirtybuffers <= bd->bd_lodirtybuffers) + BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty); + if (bd->bd_numdirtybuffers <= bd->bd_hidirtybuffers) + BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty); + mtx_unlock(&bdirtylock); +} + +/* + * bd_set: + * + * Set a domain in the appropriate bitsets when dirtybuffers + * is incremented. + */ +static void +bd_set(struct bufdomain *bd) +{ + + mtx_lock(&bdirtylock); + if (bd->bd_numdirtybuffers > bd->bd_lodirtybuffers) + BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty); + if (bd->bd_numdirtybuffers > bd->bd_hidirtybuffers) + BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty); + mtx_unlock(&bdirtylock); +} + +/* * bdirtysub: * * Decrement the numdirtybuffers count by one and wakeup any @@ -450,12 +510,17 @@ bdirtywakeup(void) * threads blocked in bwillwrite(). */ static void -bdirtysub(void) +bdirtysub(struct buf *bp) { + struct bufdomain *bd; + int num; - if (atomic_fetchadd_int(&numdirtybuffers, -1) == - (lodirtybuffers + hidirtybuffers) / 2) + bd = bufdomain(bp); + num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, -1); + if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2) bdirtywakeup(); + if (num == bd->bd_lodirtybuffers || num == bd->bd_hidirtybuffers) + bd_clear(bd); } /* @@ -465,16 +530,21 @@ static void * daemon if needed. */ static void -bdirtyadd(void) +bdirtyadd(struct buf *bp) { + struct bufdomain *bd; + int num; /* * Only do the wakeup once as we cross the boundary. The * buf daemon will keep running until the condition clears. */ - if (atomic_fetchadd_int(&numdirtybuffers, 1) == - (lodirtybuffers + hidirtybuffers) / 2) + bd = bufdomain(bp); + num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, 1); + if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2) bd_wakeup(); + if (num == bd->bd_lodirtybuffers || num == bd->bd_hidirtybuffers) + bd_set(bd); } /* @@ -539,7 +609,7 @@ bufspace_adjust(struct buf *bp, int bufsize) KASSERT((bp->b_flags & B_MALLOC) == 0, ("bufspace_adjust: malloc buf %p", bp)); - bd = &bdclean[bp->b_domain]; + bd = bufdomain(bp); diff = bufsize - bp->b_bufsize; if (diff < 0) { atomic_subtract_long(&bd->bd_bufspace, -diff); @@ -638,7 +708,7 @@ bufspace_wait(struct bufdomain *bd, struct vnode * * recursion. */ td->td_pflags |= TDP_BUFNEED | TDP_NORUNNINGBUF; - fl = buf_flush(vp, flushbufqtarget); + fl = buf_flush(vp, bd, flushbufqtarget); td->td_pflags &= norunbuf; BD_LOCK(bd); if (fl != 0) @@ -1025,7 +1095,6 @@ bufinit(void) ("maxbcachebuf (%d) must be >= MAXBSIZE (%d)\n", maxbcachebuf, MAXBSIZE)); bq_init(&bqempty, QUEUE_EMPTY, -1, "bufq empty lock"); - bq_init(&bqdirty, QUEUE_DIRTY, -1, "bufq dirty lock"); mtx_init(&rbreqlock, "runningbufspace lock", NULL, MTX_DEF); mtx_init(&bdlock, "buffer daemon lock", NULL, MTX_DEF); mtx_init(&bdirtylock, "dirty buf lock", NULL, MTX_DEF); @@ -1093,7 +1162,6 @@ bufinit(void) */ hidirtybuffers = nbuf / 4 + 20; dirtybufthresh = hidirtybuffers * 9 / 10; - numdirtybuffers = 0; /* * To support extreme low-memory systems, make sure hidirtybuffers * cannot eat up all available buffer space. This occurs when our @@ -1128,22 +1196,26 @@ bufinit(void) * One queue per-256mb up to the max. More queues gives better * concurrency but less accurate LRU. */ - clean_domains = MIN(howmany(maxbufspace, 256*1024*1024), CLEAN_DOMAINS); - for (i = 0 ; i < clean_domains; i++) { + buf_domains = MIN(howmany(maxbufspace, 256*1024*1024), BUF_DOMAINS); + for (i = 0 ; i < buf_domains; i++) { struct bufdomain *bd; - bd = &bdclean[i]; + bd = &bdomain[i]; bd_init(bd); - bd->bd_freebuffers = nbuf / clean_domains; - bd->bd_hifreebuffers = hifreebuffers / clean_domains; - bd->bd_lofreebuffers = lofreebuffers / clean_domains; + bd->bd_freebuffers = nbuf / buf_domains; + bd->bd_hifreebuffers = hifreebuffers / buf_domains; + bd->bd_lofreebuffers = lofreebuffers / buf_domains; bd->bd_bufspace = 0; - bd->bd_maxbufspace = maxbufspace / clean_domains; - bd->bd_hibufspace = hibufspace / clean_domains; - bd->bd_lobufspace = lobufspace / clean_domains; - bd->bd_bufspacethresh = bufspacethresh / clean_domains; + bd->bd_maxbufspace = maxbufspace / buf_domains; + bd->bd_hibufspace = hibufspace / buf_domains; + bd->bd_lobufspace = lobufspace / buf_domains; + bd->bd_bufspacethresh = bufspacethresh / buf_domains; + bd->bd_numdirtybuffers = 0; + bd->bd_hidirtybuffers = hidirtybuffers / buf_domains; + bd->bd_lodirtybuffers = lodirtybuffers / buf_domains; + bd->bd_dirtybufthresh = dirtybufthresh / buf_domains; /* Don't allow more than 2% of bufs in the per-cpu caches. */ - bd->bd_lim = nbuf / clean_domains / 50 / mp_ncpus; + bd->bd_lim = nbuf / buf_domains / 50 / mp_ncpus; } getnewbufcalls = counter_u64_alloc(M_WAITOK); getnewbufrestarts = counter_u64_alloc(M_WAITOK); @@ -1327,6 +1399,13 @@ bpmap_qenter(struct buf *bp) (vm_offset_t)(bp->b_offset & PAGE_MASK)); } +static inline struct bufdomain * +bufdomain(struct buf *bp) +{ + + return (&bdomain[bp->b_domain]); +} + static struct bufqueue * bufqueue(struct buf *bp) { @@ -1339,9 +1418,9 @@ bufqueue(struct buf *bp) case QUEUE_EMPTY: return (&bqempty); case QUEUE_DIRTY: - return (&bqdirty); + return (&bufdomain(bp)->bd_dirtyq); case QUEUE_CLEAN: - return (&bdclean[bp->b_domain].bd_subq[bp->b_subqueue]); + return (&bufdomain(bp)->bd_subq[bp->b_subqueue]); default: break; } @@ -1404,14 +1483,14 @@ binsfree(struct buf *bp, int qindex) bq_remove(bq, bp); BQ_UNLOCK(bq); } + bd = bufdomain(bp); if (qindex == QUEUE_CLEAN) { - bd = &bdclean[bp->b_domain]; if (bd->bd_lim != 0) bq = &bd->bd_subq[PCPU_GET(cpuid)]; else bq = bd->bd_cleanq; } else - bq = &bqdirty; + bq = &bd->bd_dirtyq; bq_insert(bq, bp, true); } @@ -1439,7 +1518,7 @@ buf_free(struct buf *bp) if (!LIST_EMPTY(&bp->b_dep)) buf_deallocate(bp); bufkva_free(bp); - atomic_add_int(&bdclean[bp->b_domain].bd_freebuffers, 1); + atomic_add_int(&bufdomain(bp)->bd_freebuffers, 1); BUF_UNLOCK(bp); uma_zfree(buf_zone, bp); } @@ -1715,9 +1794,10 @@ bd_init(struct bufdomain *bd) int domain; int i; - domain = bd - bdclean; + domain = bd - bdomain; bd->bd_cleanq = &bd->bd_subq[mp_maxid + 1]; bq_init(bd->bd_cleanq, QUEUE_CLEAN, mp_maxid + 1, "bufq clean lock"); + bq_init(&bd->bd_dirtyq, QUEUE_DIRTY, -1, "bufq dirty lock"); for (i = 0; i <= mp_maxid; i++) bq_init(&bd->bd_subq[i], QUEUE_CLEAN, i, "bufq clean subqueue lock"); @@ -1809,7 +1889,7 @@ bq_insert(struct bufqueue *bq, struct buf *bp, boo if (bp->b_qindex != QUEUE_NONE) panic("bq_insert: free buffer %p onto another queue?", bp); - bd = &bdclean[bp->b_domain]; + bd = bufdomain(bp); if (bp->b_flags & B_AGE) { /* Place this buf directly on the real queue. */ if (bq->bq_index == QUEUE_CLEAN) @@ -1926,8 +2006,8 @@ bufkva_reclaim(vmem_t *vmem, int flags) done = false; for (i = 0; i < 5; i++) { - for (q = 0; q < clean_domains; q++) - if (buf_recycle(&bdclean[q], true) != 0) + for (q = 0; q < buf_domains; q++) + if (buf_recycle(&bdomain[q], true) != 0) done = true; if (done) break; @@ -2319,7 +2399,7 @@ bdirty(struct buf *bp) if ((bp->b_flags & B_DELWRI) == 0) { bp->b_flags |= /* XXX B_DONE | */ B_DELWRI; reassignbuf(bp); - bdirtyadd(); + bdirtyadd(bp); } } @@ -2347,7 +2427,7 @@ bundirty(struct buf *bp) if (bp->b_flags & B_DELWRI) { bp->b_flags &= ~B_DELWRI; reassignbuf(bp); - bdirtysub(); + bdirtysub(bp); } /* * Since it is now being written, we can clear its deferred write flag. @@ -2419,9 +2499,9 @@ void bwillwrite(void) { - if (numdirtybuffers >= hidirtybuffers) { + if (buf_dirty_count_severe()) { mtx_lock(&bdirtylock); - while (numdirtybuffers >= hidirtybuffers) { + while (buf_dirty_count_severe()) { bdirtywait = 1; msleep(&bdirtywait, &bdirtylock, (PRIBIO + 4), "flswai", 0); @@ -2437,7 +2517,7 @@ int buf_dirty_count_severe(void) { - return(numdirtybuffers >= hidirtybuffers); + return (!BIT_EMPTY(BUF_DOMAINS, &bdhidirty)); } /* @@ -2522,7 +2602,7 @@ brelse(struct buf *bp) if (!LIST_EMPTY(&bp->b_dep)) buf_deallocate(bp); if (bp->b_flags & B_DELWRI) - bdirtysub(); + bdirtysub(bp); bp->b_flags &= ~(B_DELWRI | B_CACHE); if ((bp->b_flags & B_VMIO) == 0) { allocbuf(bp, 0); @@ -3135,9 +3215,9 @@ getnewbuf(struct vnode *vp, int slpflag, int slpti else metadata = false; if (vp == NULL) - bd = &bdclean[0]; + bd = &bdomain[0]; else - bd = &bdclean[vp->v_bufobj.bo_domain]; + bd = &bdomain[vp->v_bufobj.bo_domain]; counter_u64_add(getnewbufcalls, 1); reserved = false; @@ -3183,11 +3263,11 @@ static struct kproc_desc buf_kp = { SYSINIT(bufdaemon, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start, &buf_kp); static int -buf_flush(struct vnode *vp, int target) +buf_flush(struct vnode *vp, struct bufdomain *bd, int target) { int flushed; - flushed = flushbufqueues(vp, target, 0); + flushed = flushbufqueues(vp, bd, target, 0); if (flushed == 0) { /* * Could not find any buffers without rollback @@ -3196,7 +3276,7 @@ static int */ if (vp != NULL && target > 2) target /= 2; - flushbufqueues(vp, target, 1); + flushbufqueues(vp, bd, target, 1); } return (flushed); } @@ -3204,6 +3284,8 @@ static int static void buf_daemon() { + struct bufdomain *bd; + int speedupreq; int lodirty; int i; @@ -3216,11 +3298,11 @@ buf_daemon() /* * Start the buf clean daemons as children threads. */ - for (i = 0 ; i < clean_domains; i++) { + for (i = 0 ; i < buf_domains; i++) { int error; error = kthread_add((void (*)(void *))bufspace_daemon, - &bdclean[i], curproc, NULL, 0, 0, "bufspacedaemon-%d", i); + &bdomain[i], curproc, NULL, 0, 0, "bufspacedaemon-%d", i); if (error) panic("error %d spawning bufspace daemon", error); } @@ -3235,20 +3317,30 @@ buf_daemon() mtx_unlock(&bdlock); kproc_suspend_check(bufdaemonproc); - lodirty = lodirtybuffers; - if (bd_speedupreq) { - lodirty = numdirtybuffers / 2; - bd_speedupreq = 0; - } + /* - * Do the flush. Limit the amount of in-transit I/O we - * allow to build up, otherwise we would completely saturate - * the I/O system. + * Save speedupreq for this pass and reset to capture new + * requests. */ - while (numdirtybuffers > lodirty) { - if (buf_flush(NULL, numdirtybuffers - lodirty) == 0) - break; - kern_yield(PRI_USER); + speedupreq = bd_speedupreq; + bd_speedupreq = 0; + + /* + * Flush each domain sequentially according to its level and + * the speedup request. + */ + for (i = 0; i < buf_domains; i++) { + bd = &bdomain[i]; + if (speedupreq) + lodirty = bd->bd_numdirtybuffers / 2; + else + lodirty = bd->bd_lodirtybuffers; + while (bd->bd_numdirtybuffers > lodirty) { + if (buf_flush(NULL, bd, + bd->bd_numdirtybuffers - lodirty) == 0) + break; + kern_yield(PRI_USER); + } } /* @@ -3262,7 +3354,7 @@ buf_daemon() * to avoid endless loops on unlockable buffers. */ mtx_lock(&bdlock); - if (numdirtybuffers <= lodirtybuffers) { + if (!BIT_EMPTY(BUF_DOMAINS, &bdlodirty)) { /* * We reached our low water mark, reset the * request and sleep until we are needed again. @@ -3301,7 +3393,8 @@ SYSCTL_INT(_vfs, OID_AUTO, flushwithdeps, CTLFLAG_ 0, "Number of buffers flushed with dependecies that require rollbacks"); static int -flushbufqueues(struct vnode *lvp, int target, int flushdeps) +flushbufqueues(struct vnode *lvp, struct bufdomain *bd, int target, + int flushdeps) { struct bufqueue *bq; struct buf *sentinel; @@ -3314,7 +3407,7 @@ static int bool unlock; flushed = 0; - bq = &bqdirty; + bq = &bd->bd_dirtyq; bp = NULL; sentinel = malloc(sizeof(struct buf), M_TEMP, M_WAITOK | M_ZERO); sentinel->b_qindex = QUEUE_SENTINEL; @@ -3650,7 +3743,7 @@ bp_unmapped_get_kva(struct buf *bp, daddr_t blkno, panic("GB_NOWAIT_BD and GB_UNMAPPED %p", bp); } counter_u64_add(mappingrestarts, 1); - bufspace_wait(&bdclean[bp->b_domain], bp->b_vp, gbflags, 0, 0); + bufspace_wait(bufdomain(bp), bp->b_vp, gbflags, 0, 0); } has_addr: if (need_mapping) { @@ -3848,7 +3941,7 @@ loop: */ if (flags & GB_NOCREAT) return NULL; - if (bdclean[bo->bo_domain].bd_freebuffers == 0 && + if (bdomain[bo->bo_domain].bd_freebuffers == 0 && TD_IS_IDLETHREAD(curthread)) return NULL; @@ -3905,7 +3998,7 @@ loop: if (gbincore(bo, blkno)) { BO_UNLOCK(bo); bp->b_flags |= B_INVAL; - bufspace_release(&bdclean[bp->b_domain], maxsize); + bufspace_release(bufdomain(bp), maxsize); brelse(bp); goto loop; } @@ -3940,7 +4033,7 @@ loop: } allocbuf(bp, size); - bufspace_release(&bdclean[bp->b_domain], maxsize); + bufspace_release(bufdomain(bp), maxsize); bp->b_flags &= ~B_DONE; } CTR4(KTR_BUF, "getblk(%p, %ld, %d) = %p", vp, (long)blkno, size, bp); @@ -3969,7 +4062,7 @@ geteblk(int size, int flags) return (NULL); } allocbuf(bp, size); - bufspace_release(&bdclean[bp->b_domain], maxsize); + bufspace_release(bufdomain(bp), maxsize); bp->b_flags |= B_INVAL; /* b_dep cleared by getnewbuf() */ BUF_ASSERT_HELD(bp); return (bp); @@ -4838,7 +4931,7 @@ bufobj_init(struct bufobj *bo, void *private) static volatile int bufobj_cleanq; bo->bo_domain = - atomic_fetchadd_int(&bufobj_cleanq, 1) % clean_domains; + atomic_fetchadd_int(&bufobj_cleanq, 1) % buf_domains; rw_init(BO_LOCKPTR(bo), "bufobj interlock"); bo->bo_private = private; TAILQ_INIT(&bo->bo_clean.bv_hd); @@ -5183,10 +5276,9 @@ DB_SHOW_COMMAND(bufqueues, bufqueues) int i, j; db_printf("bqempty: %d\n", bqempty.bq_len); - db_printf("bqdirty: %d\n", bqdirty.bq_len); - for (i = 0; i < clean_domains; i++) { - bd = &bdclean[i]; + for (i = 0; i < buf_domains; i++) { + bd = &bdomain[i]; db_printf("Buf domain %d\n", i); db_printf("\tfreebufs\t%d\n", bd->bd_freebuffers); db_printf("\tlofreebufs\t%d\n", bd->bd_lofreebuffers); @@ -5198,7 +5290,13 @@ DB_SHOW_COMMAND(bufqueues, bufqueues) db_printf("\tlobufspace\t%ld\n", bd->bd_lobufspace); db_printf("\tbufspacethresh\t%ld\n", bd->bd_bufspacethresh); db_printf("\n"); + db_printf("\tnumdirtybuffers\t%d\n", bd->bd_numdirtybuffers); + db_printf("\tlodirtybuffers\t%d\n", bd->bd_lodirtybuffers); + db_printf("\thidirtybufferss\t%d\n", bd->bd_hidirtybuffers); + db_printf("\tdirtybufthresh\t%d\n", bd->bd_dirtybufthresh); + db_printf("\n"); db_printf("\tcleanq count\t%d\n", bd->bd_cleanq->bq_len); + db_printf("\tdirtyq count\t%d\n", bd->bd_dirtyq.bq_len); db_printf("\twakeup\t\t%d\n", bd->bd_wanted); db_printf("\tlim\t\t%d\n", bd->bd_lim); db_printf("\tCPU "); -- Test scenario: misc/fdatasync.sh