GENERIC HEAD from Sep 11 09:02 UTC, vmcore.80
Stress test with snapshots and a full disk.
GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2006 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD 7.0-CURRENT #2: Tue Sep 12 20:07:55 CEST 2006
pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.80-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf24 Stepping = 4
Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
Logical CPUs per core: 2
real memory = 1073676288 (1023 MB)
avail memory = 1032458240 (984 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
fxp0: link state changed to UP
panic: lock (sleep mutex) Softdep Lock not locked @ ufs/ffs/ffs_softdep.c:5362pid 657
cpuid = 1
KDB: enter: panic(syslo
[thread pid 40 tid 100048 ]
Stopped at kdb_enter+0x2b: nop
db> where
Tracing pid 40 tid 100048 td 0xc3f77510
kdb_enter(c0919d2a) at kdb_enter+0x2b
panic(c091e8ad,c0935ec2,c0932771,c0932787,14f2,...) at panic+0x14b
witness_unlock(c0a63158,8,c0932787,14f2) at witness_unlock+0xcf
_mtx_unlock_flags(c0a63158,0,c093277e,14f2,2,...) at _mtx_unlock_flags+0x8c
softdep_sync_metadata(c40df690) at softdep_sync_metadata+0x62f
ffs_syncvnode(c40df690,1) at ffs_syncvnode+0x33b
ffs_fsync(e43eec0c) at ffs_fsync+0x12
VOP_FSYNC_APV(c09d79e0,e43eec0c) at VOP_FSYNC_APV+0x9b
bufsync(c40df790,1,c3f77510) at bufsync+0x2e
bufobj_invalbuf(c40df790,1,c3f77510,0,0,...) at bufobj_invalbuf+0xb6
vinvalbuf(c40df690,1,c3f77510,0,0) at vinvalbuf+0x2a
vgonel(c40df690,7,0,c09ed1e0,c40df690,...) at vgonel+0xb9
vlrureclaim(c3fd4b50,c3f77510,c3f76d38,c06f279c,c06f279c,...) at vlrureclaim+0x2a2
vnlru_proc(0,e43eed38) at vnlru_proc+0x193
fork_exit(c06f279c,0,e43eed38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43eed6c, ebp = 0 ---
db> show alllocks
Process 657 (syslogd) thread 0xc4058a20 (100096)
exclusive sleep mutex vnode interlock r = 0 (0xc4165c98) locked @ kern/vfs_bio.c:1464
exclusive sleep mutex buf queue lock r = 0 (0xc0a56218) locked @ kern/vfs_bio.c:1445
db> where 657
Tracing pid 657 tid 100096 td 0xc4058a20
cpustop_handler(e66be618,c0894c12,da7a,3,a,...) at cpustop_handler+0x31
ipi_nmi_handler(da7a,3,a,0,0,...) at ipi_nmi_handler+0x28
trap(360008,c4050028,e66b0028,60,c3df3c00,...) at trap+0x3a
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc086c820, esp = 0xe66be660, ebp = 0xe66be670 ---
siointr1(c3df3c00,c0a7856c,0,c093c558,56e,...) at siointr1+0x230
siointr(c3df3c00) at siointr+0x21
intr_execute_handlers(c3c2c0c4,e66be6cc,4,e66be710,c08806e3,...) at intr_execute_handlers+0xe1
lapic_handle_intr(38) at lapic_handle_intr+0x30
Xapic_isr1() at Xapic_isr1+0x33
--- interrupt, eip = 0xc0889ea3, esp = 0xe66be70c, ebp = 0xe66be710 ---
spinlock_exit(3f8,e66be750,c086dafd,c0a7856c,0,...) at spinlock_exit+0x27
_mtx_unlock_spin_flags(c0a7856c,0,c093c558,a02) at _mtx_unlock_spin_flags+0xb1
sio_cnputc(c09e50e0,6f) at sio_cnputc+0xc1
cnputc(6f) at cnputc+0x4d
putchar(6f,e66be870) at putchar+0x52
kvprintf(c0931e3e,c06b5ad0,e66be870,a,e66be898,...) at kvprintf+0x6fa
log(3,c0931e3e,291,c4056ca4,0,...) at log+0x3c
ffs_fserr(c3fc9000,a10022,c09319f2,11bbcd92,0,...) at ffs_fserr+0x3a
ffs_realloccg(c45a76b4,1,0,2833ab7,0,2833a00,0,800,1000,c3c33a80,e66be9d4,c3fa5ca4,0,c0932025,28e) at ffs_realloccg+0x7ab
ffs_balloc_ufs2(c474d000,47ff,0,64,c3c33a80,...) at ffs_balloc_ufs2+0xd9f
ffs_write(e66bebb8) at ffs_write+0x2ac
VOP_WRITE_APV(c09d79e0,e66bebb8) at VOP_WRITE_APV+0x132
vn_write(c402e090,c4650c00,c3c33a80,0,c4058a20) at vn_write+0x1ea
dofilewrite(c4058a20,b,c402e090,c4650c00,ffffffff,...) at dofilewrite+0x7b
kern_writev(c4058a20,b,c4650c00,c4650c00,0,...) at kern_writev+0x36
writev(c4058a20,e66bed04) at writev+0x30
syscall(821003b,bfbf003b,bfbf003b,0,bfbfd070,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (121, FreeBSD ELF32, writev), eip = 0x2814e133, esp = 0xbfbfc98c, ebp = 0xbfbfd098 ---
db> show lockedvnods
Locked vnodes
0xc4043a80: tag devfs, type VCHR
usecount 1, writecount 0, refcount 57 mountedhere 0xc3faf100
flags ()
v_object 0xc406f258 ref 0 pages 2452
lock type devfs: EXCL (count 1) by thread 0xc3f77360 (pid 39)#0 0xc0689b04 at lockmgr+0x510
#1 0xc06eb8ad at vop_stdlock+0x21
#2 0xc08a80c7 at VOP_LOCK_APV+0x87
#3 0xc0700ab8 at vn_lock+0xac
#4 0xc06f4393 at sync_vnode+0xe3
#5 0xc06f4639 at sched_sync+0x1f5
#6 0xc0680494 at fork_exit+0xac
#7 0xc088038c at fork_trampoline+0x8
dev ad0s1d
0xc7c0b540: tag ufs, type VDIR
usecount 2, writecount 0, refcount 4 mountedhere 0
flags ()
v_object 0xc868ba50 ref 0 pages 1
lock type ufs: EXCL (count 1) by thread 0xc4107000 (pid 4045)#0 0xc0689b04 at lockmgr+0x510
#1 0xc07d9186 at ffs_lock+0x76
#2 0xc08a80c7 at VOP_LOCK_APV+0x87
#3 0xc0700ab8 at vn_lock+0xac
#4 0xc06fd4ee at getdirentries+0xee
#5 0xc0895992 at syscall+0x256
#6 0xc088037f at Xint0x80_syscall+0x1f
ino 577333, on dev ad0s1g
0xc4154690: tag ufs, type VREG
usecount 1, writecount 1, refcount 3 mountedhere 0
flags ()
v_object 0xc4118780 ref 0 pages 4
lock type ufs: EXCL (count 1) by thread 0xc3c3bd80 (pid 38)#0 0xc0689b04 at lockmgr+0x510
#1 0xc07d9186 at ffs_lock+0x76
#2 0xc08a80c7 at VOP_LOCK_APV+0x87
#3 0xc0700ab8 at vn_lock+0xac
#4 0xc06e5c4b at flushbufqueues+0x443
#5 0xc06e56c5 at buf_daemon+0x95
#6 0xc0680494 at fork_exit+0xac
#7 0xc088038c at fork_trampoline+0x8
ino 10551299, on dev ad0s1d
0xc474d000: tag ufs, type VREG
usecount 2, writecount 1, refcount 6 mountedhere 0
flags ()
v_object 0xc479ab40 ref 0 pages 5
lock type ufs: EXCL (count 1) by thread 0xc4058a20 (pid 657) with 1 pending#0 0xc0689b04 at lockmgr+0x510
#1 0xc07d9186 at ffs_lock+0x76
#2 0xc08a80c7 at VOP_LOCK_APV+0x87
#3 0xc0700ab8 at vn_lock+0xac
#4 0xc0700420 at vn_write+0x138
#5 0xc06c125b at dofilewrite+0x7b
#6 0xc06c10fe at kern_writev+0x36
#7 0xc06c10ac at writev+0x30
#8 0xc0895992 at syscall+0x256
#9 0xc088037f at Xint0x80_syscall+0x1f
ino 10551330, on dev ad0s1d
0xc4165bd0: tag ufs, type VREG
usecount 1, writecount 0, refcount 12 mountedhere 0
flags (VV_SYSTEM)
v_object 0xc4589618 ref 0 pages 248
lock type snaplk: EXCL (count 1) by thread 0xc4058a20 (pid 657) with 2 pending#0 0xc0689b04 at lockmgr+0x510
#1 0xc07cacca at ffs_copyonwrite+0x246
#2 0xc07d8c99 at ffs_geom_strategy+0xbd
#3 0xc06e3536 at bufwrite+0x15a
#4 0xc07d8bd2 at ffs_bufwrite+0x282
#5 0xc07c39e5 at ffs_update+0x349
#6 0xc07d4260 at softdep_request_cleanup+0x88
#7 0xc07bc560 at ffs_realloccg+0x700
#8 0xc07c2997 at ffs_balloc_ufs2+0xd9f
#9 0xc07d98e0 at ffs_write+0x2ac
#10 0xc08a6dca at VOP_WRITE_APV+0x132
#11 0xc07004d2 at vn_write+0x1ea
#12 0xc06c125b at dofilewrite+0x7b
#13 0xc06c10fe at kern_writev+0x36
#14 0xc06c10ac at writev+0x30
#15 0xc0895992 at syscall+0x256
#16 0xc088037f at Xint0x80_syscall+0x1f
ino 7, on dev ad0s1d
0xc40df3f0: tag ufs, type VDIR
usecount 1, writecount 0, refcount 2 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc3fea870 (pid 4075)#0 0xc0689b04 at lockmgr+0x510
#1 0xc07d9186 at ffs_lock+0x76
#2 0xc08a80c7 at VOP_LOCK_APV+0x87
#3 0xc0700ab8 at vn_lock+0xac
#4 0xc06f4c6a at vget+0xc2
#5 0xc06e8fda at cache_lookup+0x392
#6 0xc06e95d6 at vfs_cache_lookup+0x92
#7 0xc08a6145 at VOP_LOOKUP_APV+0xa1
#8 0xc06ed8ae at lookup+0x46e
#9 0xc06ed1de at namei+0x382
#10 0xc06fb081 at kern_stat+0x35
#11 0xc06fb02f at stat+0x1b
#12 0xc0895992 at syscall+0x256
#13 0xc088037f at Xint0x80_syscall+0x1f
ino 518144, on dev ad0s1d
0xc40df150: tag ufs, type VDIR
usecount 1, writecount 0, refcount 3 mountedhere 0
flags ()
lock type ufs: EXCL (count 1) by thread 0xc41a1360 (pid 982)#0 0xc0689b04 at lockmgr+0x510
#1 0xc07d9186 at ffs_lock+0x76
#2 0xc08a80c7 at VOP_LOCK_APV+0x87
#3 0xc0700ab8 at vn_lock+0xac
#4 0xc06f4c6a at vget+0xc2
#5 0xc06e8fda at cache_lookup+0x392
#6 0xc06e95d6 at vfs_cache_lookup+0x92
#7 0xc08a6145 at VOP_LOOKUP_APV+0xa1
#8 0xc06ed8ae at lookup+0x46e
#9 0xc06ed1de at namei+0x382
#10 0xc06fb081 at kern_stat+0x35
#11 0xc06fb02f at stat+0x1b
#12 0xc0895992 at syscall+0x256
#13 0xc088037f at Xint0x80_syscall+0x1f
ino 10551296, on dev ad0s1d
0xc40df690: tag ufs, type VDIR
usecount 0, writecount 0, refcount 5 mountedhere 0
flags (VI_DOOMED)
v_object 0xc184a1e0 ref 0 pages 1
lock type ufs: EXCL (count 1) by thread 0xc3f77510 (pid 40) with 2 pending#0 0xc0689b04 at lockmgr+0x510
#1 0xc07d9186 at ffs_lock+0x76
#2 0xc08a80c7 at VOP_LOCK_APV+0x87
#3 0xc06f2419 at vlrureclaim+0x20d
#4 0xc06f292f at vnlru_proc+0x193
#5 0xc0680494 at fork_exit+0xac
#6 0xc088038c at fork_trampoline+0x8
ino 518145, on dev ad0s1d
db> show lockedbufs
buf at 0xd7cdd374
b_flags = 0x200202a0<vmio,clusterok,done,delwri,cache>
b_error = 0, b_bufsize = 16384, b_bcount = 16384, b_resid = 0
b_bufobj = (0xc4043b80), b_data = 0xd88fc000, b_blkno = 168605920
b_npages = 4, pages(OBJ, IDX, PA): (0xc406f258, 0x141971c, 0x10b74000),(0xc406f258, 0x141971d, 0x1afb7000),(0xc406f258, 0x141971e, 0x40bf000),(0xc406f258, 0x141971f, 0x39148000)
lock type getblk: EXCL (count 1) by thread 0xc4058a20 (pid 657)#0 0xc0689b04 at lockmgr+0x510
#1 0xc06e6353 at getblk+0x12f
#2 0xc06e32df at breadn+0x2f
#3 0xc06e3138 at bread+0x20
#4 0xc07c38c4 at ffs_update+0x228
#5 0xc07d4260 at softdep_request_cleanup+0x88
#6 0xc07bc560 at ffs_realloccg+0x700
#7 0xc07c2997 at ffs_balloc_ufs2+0xd9f
#8 0xc07d98e0 at ffs_write+0x2ac
#9 0xc08a6dca at VOP_WRITE_APV+0x132
#10 0xc07004d2 at vn_write+0x1ea
#11 0xc06c125b at dofilewrite+0x7b
#12 0xc06c10fe at kern_writev+0x36
#13 0xc06c10ac at writev+0x30
#14 0xc0895992 at syscall+0x256
#15 0xc088037f at Xint0x80_syscall+0x1f
buf at 0xd7d97a1c
b_flags = 0x200200a0<vmio,clusterok,delwri,cache>
b_error = 28, b_bufsize = 2048, b_bcount = 2048, b_resid = 0
b_bufobj = (0xc474d100), b_data = 0xda564000, b_blkno = 168618716
b_npages = 1, pages(OBJ, IDX, PA): (0xc479ab40, 0x4, 0x272b0000)
lock type getblk: EXCL (count 1) by thread 0xc4058a20 (pid 657)#0 0xc0689b04 at lockmgr+0x510
#1 0xc06e6353 at getblk+0x12f
#2 0xc06e32df at breadn+0x2f
#3 0xc06e3138 at bread+0x20
#4 0xc07bbfed at ffs_realloccg+0x18d
#5 0xc07c2997 at ffs_balloc_ufs2+0xd9f
#6 0xc07d98e0 at ffs_write+0x2ac
#7 0xc08a6dca at VOP_WRITE_APV+0x132
#8 0xc07004d2 at vn_write+0x1ea
#9 0xc06c125b at dofilewrite+0x7b
#10 0xc06c10fe at kern_writev+0x36
#11 0xc06c10ac at writev+0x30
#12 0xc0895992 at syscall+0x256
#13 0xc088037f at Xint0x80_syscall+0x1f
buf at 0xd7db5ad0
b_flags = 0x20000220<vmio,done,cache>
b_error = 0, b_bufsize = 2048, b_bcount = 2048, b_resid = 0
b_bufobj = (0xc7c0b640), b_data = 0xda9f8000, b_blkno = 9089396
b_npages = 1, pages(OBJ, IDX, PA): (0xc868ba50, 0x0, 0x2ea29000)
lock type bufwait: EXCL (count 1) by thread 0xc4107000 (pid 4045)#0 0xc0689b04 at lockmgr+0x510
#1 0xc06e5008 at getnewbuf+0x144
#2 0xc06e65ff at getblk+0x3db
#3 0xc06e32df at breadn+0x2f
#4 0xc06e3138 at bread+0x20
#5 0xc07d941b at ffs_read+0x23f
#6 0xc08a6c27 at VOP_READ_APV+0x9b
#7 0xc07e4240 at ufs_readdir+0xd4
#8 0xc08a7cab at VOP_READDIR_APV+0x9b
#9 0xc06fd53f at getdirentries+0x13f
#10 0xc0895992 at syscall+0x256
#11 0xc088037f at Xint0x80_syscall+0x1f
buf at 0xd7dd9f98
b_flags = 0xa0020024<remfree,vmio,clusterok,cache,async>
b_error = 28, b_bufsize = 14336, b_bcount = 14336, b_resid = 0
b_bufobj = (0xc4154790), b_data = 0xdaf80000, b_blkno = 168619360
b_npages = 4, pages(OBJ, IDX, PA): (0xc4118780, 0x10, 0x1394b000),(0xc4118780, 0x11, 0x300c4000),(0xc4118780, 0x12, 0x210f5000),(0xc4118780, 0x13, 0x75f6000)
db> ps
pid ppid pgrp uid state wmesg wchan cmd
4075 4074 4074 2 S ufs 0xc40df6e8 sh
4074 4073 4074 2 Ss wait 0xc47ee69c sh
4073 840 840 0 S piperd 0xc4017c60 cron
4068 1692 1689 0 R+ rw
4045 4033 3867 0 R find
4040 4038 3867 0 S piperd 0xc410f948 cat
4038 4027 3867 0 S wait 0xc405569c sh
4037 4027 3867 0 S piperd 0xc404a948 sort
4036 4027 3867 0 S piperd 0xc4017000 sed
4034 4027 3867 0 S piperd 0xc4196c60 xargs
4033 4027 3867 0 S wait 0xc47ee8d0 sh
4027 4024 3867 0 S wait 0xc3fe8d38 sh
4026 4025 3867 0 S piperd 0xc3ff6630 mail
4025 4018 3867 0 S wait 0xc4004468 sh
4024 4018 3867 0 S wait 0xc47ebb04 sh
4018 4017 3867 0 S wait 0xc47eb468 sh
4017 3889 3867 0 S wait 0xc41a2b04 sh
3892 3890 3867 0 S piperd 0xc410e7bc mail
3890 3879 3867 0 S wait 0xc3fe8468 sh
3889 3879 3867 0 S wait 0xc4739234 sh
3879 3867 3867 0 S wait 0xc476fd38 sh
3867 3866 3867 0 Ss wait 0xc4106000 sh
3866 840 840 0 S piperd 0xc410e318 cron
1692 1691 1689 0 S+ wait 0xc4770d38 run
1691 1690 1689 0 S+ wait 0xc476ad38 run
1690 1689 1689 0 R+ run
1689 996 1689 0 S+ wait 0xc47eb69c sh
1058 1057 1058 1001 Ss+ ttyin 0xc3e02010 bash
1057 1050 1050 1001 S select 0xc0a55d1c sshd
1050 823 1050 0 SWs sbwait 0xc4112480 sshd
996 993 996 0 S+ wait 0xc3f78468 bash
993 992 993 0 SW+ pause 0xc4000d6c csh
992 986 992 1001 SW+ wait 0xc404cb04 su
986 985 986 1001 SWs+ wait 0xc4105b04 bash
985 983 983 1001 S select 0xc0a55d1c sshd
983 823 983 0 SWs sbwait 0xc4195d94 sshd
982 976 982 1001 S+ ufs 0xc474d058 tail
976 975 976 1001 SWs+ wait 0xc4105000 bash
975 973 973 1001 S select 0xc0a55d1c sshd
973 823 973 0 SWs sbwait 0xc411a718 sshd
972 1 972 0 SWs+ ttyin 0xc3dfe810 getty
971 1 971 0 SWs+ ttyin 0xc3e02810 getty
970 1 970 0 SWs+ ttyin 0xc3e01810 getty
969 1 969 0 SWs+ ttyin 0xc3e01010 getty
968 1 968 0 SWs+ ttyin 0xc3e00c10 getty
967 1 967 0 SWs+ ttyin 0xc3e02c10 getty
966 1 966 0 SWs+ ttyin 0xc3e03010 getty
965 1 965 0 SWs+ ttyin 0xc3e01c10 getty
948 1 948 0 SWs select 0xc0a55d1c inetd
919 1 919 0 SWs select 0xc0a55d1c moused
905 1 905 0 Rs watchdogd
879 873 879 1001 R+ top
873 872 873 1001 SWs+ wait 0xc404c234 bash
872 870 870 1001 S select 0xc0a55d1c sshd
870 823 870 0 SWs sbwait 0xc4113d94 sshd
840 1 840 0 Ss nanslp 0xc0a09244 cron
834 1 834 25 Ss pause 0xc4000034 sendmail
830 1 830 0 Ss select 0xc0a55d1c sendmail
823 1 823 0 SWs select 0xc0a55d1c sshd
803 1 803 0 Ss select 0xc0a55d1c ntpd
745 741 741 0 SW - 0xc4190400 nfsd
744 741 741 0 SW - 0xc4073200 nfsd
743 741 741 0 SW - 0xc4190600 nfsd
742 741 741 0 SW - 0xc4073000 nfsd
741 1 741 0 SWs select 0xc0a55d1c nfsd
739 1 739 0 SWs select 0xc0a55d1c mountd
680 1 680 0 Ss select 0xc0a55d1c rpcbind
657 1 657 0 Rs CPU 0 syslogd
547 1 547 0 SWs select 0xc0a55d1c devd
42 0 0 0 SL - 0xe43f4cfc [schedcpu]
41 0 0 0 SL ufs 0xc40df6e8 [softdepflush]
40 0 0 0 RL CPU 1 [vnlru]
39 0 0 0 SL snaplk 0xc4774314 [syncer]
38 0 0 0 SL snaplk 0xc4774314 [bufdaemon]
37 0 0 0 SL pgzero 0xc0a6bab0 [pagezero]
36 0 0 0 SL psleep 0xc0a639e0 [vmdaemon]
35 0 0 0 SL psleep 0xc0a639a0 [pagedaemon]
34 0 0 0 WL [irq7: ppc0]
33 0 0 0 SL - 0xc3d7e03c [fdc0]
32 0 0 0 WL [swi0: sio]
31 0 0 0 WL [irq12: psm0]
30 0 0 0 WL [irq1: atkbd0]
29 0 0 0 WL [irq15: ata1]
28 0 0 0 WL [irq14: ata0]
27 0 0 0 WL [irq17: fxp0]
26 0 0 0 SL usbtsk 0xc0a067c4 [usbtask]
25 0 0 0 SL usbevt 0xc3d6d210 [usb0]
24 0 0 0 WL [irq16: uhci0]
23 0 0 0 SL - 0xc3d5a280 [em0 taskq]
22 0 0 0 WL [irq9: acpi0]
21 0 0 0 WL [swi5: +]
9 0 0 0 SL - 0xc3c85700 [thread taskq]
20 0 0 0 WL [swi6: Giant taskq]
8 0 0 0 SL - 0xc3c85880 [acpi_task_2]
7 0 0 0 SL - 0xc3c85880 [acpi_task_1]
6 0 0 0 SL - 0xc3c85880 [acpi_task_0]
19 0 0 0 WL [swi6: task queue]
5 0 0 0 SL - 0xc3c85a00 [kqueue taskq]
18 0 0 0 WL [swi2: cambio]
17 0 0 0 SL - 0xc0a04400 [yarrow]
4 0 0 0 SL - 0xc0a06f7c [g_down]
3 0 0 0 SL - 0xc0a06f78 [g_up]
2 0 0 0 SL - 0xc0a06f70 [g_event]
16 0 0 0 WL [swi3: vm]
15 0 0 0 WL [swi4: clock sio]
14 0 0 0 WL [swi1: net]
13 0 0 0 RL [idle: cpu0]
12 0 0 0 RL [idle: cpu1]
11 0 0 0 RL CPU 2 [idle: cpu2]
10 0 0 0 RL CPU 3 [idle: cpu3]
1 0 1 0 SLs wait 0xc3c3a000 [init]
0 0 0 0 WLs [swapper]
4069 4068 1689 0 Z+ rw
db> show allpcpu
Current CPU: 1
cpuid = 0
curthread = 0xc4058a20: pid 657 "syslogd"
curpcb = 0xe66bed90
fpcurthread = none
idlethread = 0xc3c36a20: pid 13 "idle: cpu0"
APIC ID = 0
currentldt = 0x50
spin locks held:
cpuid = 1
curthread = 0xc3f77510: pid 40 "vnlru"
curpcb = 0xe43eed90
fpcurthread = none
idlethread = 0xc3c36870: pid 12 "idle: cpu1"
APIC ID = 1
currentldt = 0x50
spin locks held:
cpuid = 2
curthread = 0xc3c366c0: pid 11 "idle: cpu2"
curpcb = 0xe26cdd90
fpcurthread = none
idlethread = 0xc3c366c0: pid 11 "idle: cpu2"
APIC ID = 6
currentldt = 0x50
spin locks held:
cpuid = 3
curthread = 0xc3c36510: pid 10 "idle: cpu3"
curpcb = 0xe26cad90
fpcurthread = none
idlethread = 0xc3c36510: pid 10 "idle: cpu3"
APIC ID = 7
currentldt = 0x50
spin locks held:
db> allt
Tracing command sh pid 4075 tid 100076 td 0xc3fea870
sched_switch(c3fea870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c40df6e8,c0a08a30,0,c091d116,211,...) at sleepq_switch+0xcd
sleepq_wait(c40df6e8,0,c40df758,b7,c0917a4e,...) at sleepq_wait+0x46
msleep(c40df6e8,c0a08850,50,c0922001,0,...) at msleep+0x27d
acquire(e6616924,40,60000,c3fea870,0,...) at acquire+0x76
lockmgr(c40df6e8,2002,c40df758,c3fea870) at lockmgr+0x46e
ffs_lock(e661697c) at ffs_lock+0x76
VOP_LOCK_APV(c09d79e0,e661697c) at VOP_LOCK_APV+0x87
vn_lock(c40df690,2002,c3fea870,c40df690) at vn_lock+0xac
vget(c40df690,2002,c3fea870) at vget+0xc2
cache_lookup(c40df3f0,e6616bbc,e6616bd0) at cache_lookup+0x392
vfs_cache_lookup(e6616abc) at vfs_cache_lookup+0x92
VOP_LOOKUP_APV(c09d79e0,e6616abc) at VOP_LOOKUP_APV+0xa1
lookup(e6616ba8) at lookup+0x46e
namei(e6616ba8) at namei+0x382
kern_stat(c3fea870,82206dc,0,e6616c1c) at kern_stat+0x35
stat(c3fea870,e6616d04) at stat+0x1b
syscall(3b,3b,3b,805f99b,82206dc,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (188, FreeBSD ELF32, stat), eip = 0x281adc57, esp = 0xbfbfe93c, ebp = 0xbfbfea38 ---
Tracing command sh pid 4074 tid 100185 td 0xc47ec870
sched_switch(c47ec870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c47ee69c) at sleepq_switch+0xcd
sleepq_wait_sig(c47ee69c) at sleepq_wait_sig+0x1d
msleep(c47ee69c,c47ee704,15c,c091f1d9,0) at msleep+0x26e
kern_wait(c47ec870,ffffffff,e6850c2c,2,0) at kern_wait+0x7cf
wait4(c47ec870,e6850d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfebd8,fea,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfeb8c, ebp = 0xbfbfeba8 ---
Tracing command cron pid 4073 tid 100159 td 0xc4742510
sched_switch(c4742510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4017c60) at sleepq_switch+0xcd
sleepq_wait_sig(c4017c60) at sleepq_wait_sig+0x1d
msleep(c4017c60,c4017dd0,14c,c091eea9,0) at msleep+0x26e
pipe_read(c3fed480,e67cfc64,c418ec00,0,c4742510) at pipe_read+0x377
dofileread(c4742510,5,c3fed480,e67cfc64,ffffffff,...) at dofileread+0x89
kern_readv(c4742510,5,e67cfc64,8221000,1000,...) at kern_readv+0x36
read(c4742510,e67cfd04) at read+0x45
syscall(805003b,3b,bfbf003b,28188580,28188580,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28162117, esp = 0xbfbfe66c, ebp = 0xbfbfe688 ---
Tracing command rw pid 4068 tid 100147 td 0xc476ba20
sched_switch(c476ba20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a09244) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a09244) at sleepq_timedwait_sig+0x1e
msleep(c0a09244,0,15c,c091ae61,3e9,...) at msleep+0x244
kern_nanosleep(c476ba20,e67f3c74,e67f3c6c) at kern_nanosleep+0xab
nanosleep(c476ba20,e67f3d04) at nanosleep+0x4f
syscall(3b,3b,3b,2805288c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28138b2b, esp = 0xbfbfe8bc, ebp = 0xbfbfe8e8 ---
Tracing command find pid 4045 tid 100135 td 0xc4107000
sched_switch(c4107000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(d7db5ad0,c0a08a30,0,c091d116,211,...) at sleepq_switch+0xcd
sleepq_wait(d7db5ad0,0,c0922592,ead,8,...) at sleepq_wait+0x46
msleep(d7db5ad0,c0a561e8,4c,c0922ca6,0) at msleep+0x27d
bwait(d7db5ad0,4c,c0922ca6) at bwait+0x47
bufwait(d7db5ad0,0,e66d6b30,0,200,...) at bufwait+0x1a
breadn(c7c0b540,0,0,800,0,...) at breadn+0x11a
bread(c7c0b540,0,0,800,0,...) at bread+0x20
ffs_read(e66d6bac) at ffs_read+0x23f
VOP_READ_APV(c09d79e0,e66d6bac) at VOP_READ_APV+0x9b
ufs_readdir(e66d6c38) at ufs_readdir+0xd4
VOP_READDIR_APV(c09d79e0,e66d6c38) at VOP_READDIR_APV+0x9b
getdirentries(c4107000,e66d6d04) at getdirentries+0x13f
syscall(824003b,bfbf003b,bfbf003b,8250130,82403b0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (196, FreeBSD ELF32, getdirentries), eip = 0x2814089f, esp = 0xbfbfebfc, ebp = 0xbfbfec28 ---
Tracing command cat pid 4040 tid 100179 td 0xc47ed360
sched_switch(c47ed360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c410f948) at sleepq_switch+0xcd
sleepq_wait_sig(c410f948) at sleepq_wait_sig+0x1d
msleep(c410f948,c410fab8,14c,c091eea9,0) at msleep+0x26e
pipe_read(c43aaca8,e6862c64,c45c6000,0,c47ed360) at pipe_read+0x377
dofileread(c47ed360,0,c43aaca8,e6862c64,ffffffff,...) at dofileread+0x89
kern_readv(c47ed360,0,e6862c64,8201000,1000,...) at kern_readv+0x36
read(c47ed360,e6862d04) at read+0x45
syscall(3b,3b,3b,1,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28151117, esp = 0xbfbfecec, ebp = 0xbfbfee08 ---
Tracing command sh pid 4038 tid 100087 td 0xc4002bd0
sched_switch(c4002bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c405569c) at sleepq_switch+0xcd
sleepq_wait_sig(c405569c) at sleepq_wait_sig+0x1d
msleep(c405569c,c4055704,15c,c091f1d9,0) at msleep+0x26e
kern_wait(c4002bd0,ffffffff,e6661c2c,2,0) at kern_wait+0x7cf
wait4(c4002bd0,e6661d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe2b8,f1b,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe26c, ebp = 0xbfbfe288 ---
Tracing command sort pid 4037 tid 100184 td 0xc47eca20
sched_switch(c47eca20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c404a948) at sleepq_switch+0xcd
sleepq_wait_sig(c404a948) at sleepq_wait_sig+0x1d
msleep(c404a948,c404aab8,14c,c091eea9,0) at msleep+0x26e
pipe_read(c402e5e8,e6853c64,c45c6000,0,c47eca20) at pipe_read+0x377
dofileread(c47eca20,0,c402e5e8,e6853c64,ffffffff,...) at dofileread+0x89
kern_readv(c47eca20,0,e6853c64,8201000,1000,...) at kern_readv+0x36
read(c47eca20,e6853d04) at read+0x45
syscall(3b,3b,3b,100001,2816b7c0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x2815b117, esp = 0xbfbfe9ec, ebp = 0xbfbfea08 ---
Tracing command sed pid 4036 tid 100109 td 0xc4108510
sched_switch(c4108510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4017000) at sleepq_switch+0xcd
sleepq_wait_sig(c4017000) at sleepq_wait_sig+0x1d
msleep(c4017000,c4017170,14c,c091eea9,0) at msleep+0x26e
pipe_read(c3ffaea0,e66fac64,c45c6000,0,c4108510) at pipe_read+0x377
dofileread(c4108510,0,c3ffaea0,e66fac64,ffffffff,...) at dofileread+0x89
kern_readv(c4108510,0,e66fac64,8230000,1000,...) at kern_readv+0x36
read(c4108510,e66fad04) at read+0x45
syscall(3b,3b,3b,0,281657c0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28155117, esp = 0xbfbfeb4c, ebp = 0xbfbfeb68 ---
Tracing command xargs pid 4034 tid 100075 td 0xc3feaa20
sched_switch(c3feaa20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4196c60) at sleepq_switch+0xcd
sleepq_wait_sig(c4196c60) at sleepq_wait_sig+0x1d
msleep(c4196c60,c4196dd0,14c,c091eea9,0) at msleep+0x26e
pipe_read(c43dd9d8,e6619c64,c45c6000,0,c3feaa20) at pipe_read+0x377
dofileread(c3feaa20,0,c43dd9d8,e6619c64,ffffffff,...) at dofileread+0x89
kern_readv(c3feaa20,0,e6619c64,8201000,1000,...) at kern_readv+0x36
read(c3feaa20,e6619d04) at read+0x45
syscall(3b,3b,3b,8240040,281627c0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28152117, esp = 0xbfbfed0c, ebp = 0xbfbfed28 ---
Tracing command sh pid 4033 tid 100186 td 0xc47ec6c0
sched_switch(c47ec6c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c47ee8d0) at sleepq_switch+0xcd
sleepq_wait_sig(c47ee8d0) at sleepq_wait_sig+0x1d
msleep(c47ee8d0,c47ee938,15c,c091f1d9,0) at msleep+0x26e
kern_wait(c47ec6c0,ffffffff,e684dc2c,2,0) at kern_wait+0x7cf
wait4(c47ec6c0,e684dd04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe708,f1b,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe6bc, ebp = 0xbfbfe6d8 ---
Tracing command sh pid 4027 tid 100062 td 0xc3fe9870
sched_switch(c3fe9870,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3fe8d38) at sleepq_switch+0xcd
sleepq_wait_sig(c3fe8d38) at sleepq_wait_sig+0x1d
msleep(c3fe8d38,c3fe8da0,15c,c091f1d9,0) at msleep+0x26e
kern_wait(c3fe9870,ffffffff,e65fbc2c,2,0) at kern_wait+0x7cf
wait4(c3fe9870,e65fbd04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe9a8,f1b,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe95c, ebp = 0xbfbfe978 ---
Tracing command mail pid 4026 tid 100158 td 0xc47426c0
sched_switch(c47426c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3ff6630) at sleepq_switch+0xcd
sleepq_wait_sig(c3ff6630) at sleepq_wait_sig+0x1d
msleep(c3ff6630,c3ff67a0,14c,c091eea9,0) at msleep+0x26e
pipe_read(c43dda20,e67d2c64,c45c6000,0,c47426c0) at pipe_read+0x377
dofileread(c47426c0,0,c43dda20,e67d2c64,ffffffff,...) at dofileread+0x89
kern_readv(c47426c0,0,e67d2c64,8201000,1000,...) at kern_readv+0x36
read(c47426c0,e67d2d04) at read+0x45
syscall(3b,3b,3b,0,281707c0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28160117, esp = 0xbfbfdfbc, ebp = 0xbfbfdfd8 ---
Tracing command sh pid 4025 tid 100072 td 0xc4001000
sched_switch(c4001000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4004468) at sleepq_switch+0xcd
sleepq_wait_sig(c4004468) at sleepq_wait_sig+0x1d
msleep(c4004468,c40044d0,15c,c091f1d9,0) at msleep+0x26e
kern_wait(c4001000,ffffffff,e6631c2c,2,0) at kern_wait+0x7cf
wait4(c4001000,e6631d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe808,f1b,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe7bc, ebp = 0xbfbfe7d8 ---
Tracing command sh pid 4024 tid 100180 td 0xc47ed1b0
sched_switch(c47ed1b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c47ebb04) at sleepq_switch+0xcd
sleepq_wait_sig(c47ebb04) at sleepq_wait_sig+0x1d
msleep(c47ebb04,c47ebb6c,15c,c091f1d9,0) at msleep+0x26e
kern_wait(c47ed1b0,ffffffff,e685fc2c,2,0) at kern_wait+0x7cf
wait4(c47ed1b0,e685fd04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe4c8,f1b,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe47c, ebp = 0xbfbfe498 ---
Tracing command sh pid 4018 tid 100177 td 0xc47ed6c0
sched_switch(c47ed6c0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c47eb468) at sleepq_switch+0xcd
sleepq_wait_sig(c47eb468) at sleepq_wait_sig+0x1d
msleep(c47eb468,c47eb4d0,15c,c091f1d9,0) at msleep+0x26e
kern_wait(c47ed6c0,ffffffff,e6868c2c,2,0) at kern_wait+0x7cf
wait4(c47ed6c0,e6868d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfeb58,f1b,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfeb0c, ebp = 0xbfbfeb28 ---
Tracing command sh pid 4017 tid 100124 td 0xc41a0510
sched_switch(c41a0510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c41a2b04) at sleepq_switch+0xcd
sleepq_wait_sig(c41a2b04) at sleepq_wait_sig+0x1d
msleep(c41a2b04,c41a2b6c,15c,c091f1d9,0) at msleep+0x26e
kern_wait(c41a0510,ffffffff,e6751c2c,2,0) at kern_wait+0x7cf
wait4(c41a0510,e6751d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe9f8,f1b,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe9ac, ebp = 0xbfbfe9c8 ---
Tracing command mail pid 3892 tid 100091 td 0xc4002510
sched_switch(c4002510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c410e7bc) at sleepq_switch+0xcd
sleepq_wait_sig(c410e7bc) at sleepq_wait_sig+0x1d
msleep(c410e7bc,c410e92c,14c,c091eea9,0) at msleep+0x26e
pipe_read(c3ff2cf0,e6655c64,c45c6000,0,c4002510) at pipe_read+0x377
dofileread(c4002510,0,c3ff2cf0,e6655c64,ffffffff,...) at dofileread+0x89
kern_readv(c4002510,0,e6655c64,8201000,1000,...) at kern_readv+0x36
read(c4002510,e6655d04) at read+0x45
syscall(3b,3b,3b,0,281707c0,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28160117, esp = 0xbfbfdfdc, ebp = 0xbfbfdff8 ---
Tracing command sh pid 3890 tid 100058 td 0xc3fea000
sched_switch(c3fea000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3fe8468) at sleepq_switch+0xcd
sleepq_wait_sig(c3fe8468) at sleepq_wait_sig+0x1d
msleep(c3fe8468,c3fe84d0,15c,c091f1d9,0) at msleep+0x26e
kern_wait(c3fea000,ffffffff,e6607c2c,2,0) at kern_wait+0x7cf
wait4(c3fea000,e6607d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe848,f1b,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe7fc, ebp = 0xbfbfe818 ---
Tracing command sh pid 3889 tid 100134 td 0xc41071b0
sched_switch(c41071b0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4739234) at sleepq_switch+0xcd
sleepq_wait_sig(c4739234) at sleepq_wait_sig+0x1d
msleep(c4739234,c473929c,15c,c091f1d9,0) at msleep+0x26e
kern_wait(c41071b0,ffffffff,e66d9c2c,2,0) at kern_wait+0x7cf
wait4(c41071b0,e66d9d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe508,f1b,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe4bc, ebp = 0xbfbfe4d8 ---
Tracing command sh pid 3879 tid 100153 td 0xc476b000
sched_switch(c476b000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c476fd38) at sleepq_switch+0xcd
sleepq_wait_sig(c476fd38) at sleepq_wait_sig+0x1d
msleep(c476fd38,c476fda0,15c,c091f1d9,0) at msleep+0x26e
kern_wait(c476b000,ffffffff,e67e1c2c,2,0) at kern_wait+0x7cf
wait4(c476b000,e67e1d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfeb98,f1b,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfeb4c, ebp = 0xbfbfeb68 ---
Tracing command sh pid 3867 tid 100105 td 0xc3f77a20
sched_switch(c3f77a20,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4106000) at sleepq_switch+0xcd
sleepq_wait_sig(c4106000) at sleepq_wait_sig+0x1d
msleep(c4106000,c4106068,15c,c091f1d9,0) at msleep+0x26e
kern_wait(c3f77a20,ffffffff,e43f7c2c,2,0) at kern_wait+0x7cf
wait4(c3f77a20,e43f7d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfebe8,f1b,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfeb9c, ebp = 0xbfbfebb8 ---
Tracing command cron pid 3866 tid 100125 td 0xc41a0360
sched_switch(c41a0360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c410e318) at sleepq_switch+0xcd
sleepq_wait_sig(c410e318) at sleepq_wait_sig+0x1d
msleep(c410e318,c410e488,14c,c091eea9,0) at msleep+0x26e
pipe_read(c45cc708,e674ec64,c418ec00,0,c41a0360) at pipe_read+0x377
dofileread(c41a0360,5,c45cc708,e674ec64,ffffffff,...) at dofileread+0x89
kern_readv(c41a0360,5,e674ec64,8221000,1000,...) at kern_readv+0x36
read(c41a0360,e674ed04) at read+0x45
syscall(805003b,3b,bfbf003b,28188580,28188580,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x28162117, esp = 0xbfbfe66c, ebp = 0xbfbfe688 ---
Tracing command run pid 1692 tid 100160 td 0xc4772bd0
sched_switch(c4772bd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4770d38) at sleepq_switch+0xcd
sleepq_wait_sig(c4770d38) at sleepq_wait_sig+0x1d
msleep(c4770d38,c4770da0,15c,c091f1d9,0) at msleep+0x26e
kern_wait(c4772bd0,fe4,e682cc2c,0,0) at kern_wait+0x7cf
wait4(c4772bd0,e682cd04) at wait4+0x2a
syscall(bfbf003b,bfbf003b,bfbf003b,2805188c,bfbfe9d8,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x280df047, esp = 0xbfbfe3cc, ebp = 0xbfbfe3e8 ---
Tracing command run pid 1691 tid 100146 td 0xc476bbd0
sched_switch(c476bbd0,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c476ad38) at sleepq_switch+0xcd
sleepq_wait_sig(c476ad38) at sleepq_wait_sig+0x1d
msleep(c476ad38,c476ada0,15c,c091f1d9,0) at msleep+0x26e
kern_wait(c476bbd0,69c,e67f6c2c,0,0) at kern_wait+0x7cf
wait4(c476bbd0,e67f6d04) at wait4+0x2a
syscall(3b,3b,3b,2805188c,bfbfe9d8,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x280df047, esp = 0xbfbfe88c, ebp = 0xbfbfe8a8 ---
Tracing command run pid 1690 tid 100182 td 0xc47ecd80
sched_switch(c47ecd80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a09244) at sleepq_switch+0xcd
sleepq_timedwait_sig(c0a09244) at sleepq_timedwait_sig+0x1e
msleep(c0a09244,0,15c,c091ae61,3e9,...) at msleep+0x244
kern_nanosleep(c47ecd80,e6859c74,e6859c6c) at kern_nanosleep+0xab
nanosleep(c47ecd80,e6859d04) at nanosleep+0x4f
syscall(2805003b,3b,bfbf003b,2805188c,1,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (240, FreeBSD ELF32, nanosleep), eip = 0x28137b2b, esp = 0xbfbfe88c, ebp = 0xbfbfe8b8 ---
Tracing command sh pid 1689 tid 100178 td 0xc47ed510
sched_switch(c47ed510,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c47eb69c) at sleepq_switch+0xcd
sleepq_wait_sig(c47eb69c) at sleepq_wait_sig+0x1d
msleep(c47eb69c,c47eb704,15c,c091f1d9,0) at msleep+0x26e
kern_wait(c47ed510,ffffffff,e6865c2c,2,0) at kern_wait+0x7cf
wait4(c47ed510,e6865d04) at wait4+0x2a
syscall(3b,3b,3b,bfbfe898,699,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (7, FreeBSD ELF32, wait4), eip = 0x28148047, esp = 0xbfbfe84c, ebp = 0xbfbfe868 ---
Tracing command bash pid 1058 tid 100064 td 0xc4001d80
sched_switch(c4001d80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c3e02010) at sleepq_switch+0xcd
sleepq_wait_sig(c3e02010) at sleepq_wait_sig+0x1d
msleep(c3e02010,0,159,c09207e3,0) at msleep+0x26e
ttysleep(c3e02000,c3e02010,159,c09207e3,0,...) at ttysleep+0x21
ttread(c3e02000,e6649c64,0) at ttread+0x48f
ptsread(c457e700,e6649c64,0,c0a08a48,0,...) at ptsread+0x2d
giant_read(c457e700,e6649c64,0,1,c457e700,...) at giant_read+0x2d
devfs_read_f(c3ff2e10,e6649c64,c4048300,0,c4001d80) at devfs_read_f+0x62
dofileread(c4001d80,0,c3ff2e10,e6649c64,ffffffff,...) at dofileread+0x89
kern_readv(c4001d80,0,e6649c64,bfbfe82f,1,...) at kern_readv+0x36
read(c4001d80,e6649d04) at read+0x45
syscall(839003b,3b,bfbf003b,0,bfbfe82f,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x2830e117, esp = 0xbfbfe80c, ebp = 0xbfbfe838 ---
Tracing command sshd pid 1057 tid 100151 td 0xc476b360
sched_switch(c476b360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c0a55d1c) at sleepq_switch+0xcd
sleepq_wait_sig(c0a55d1c,246,c0a08a30,c091ec45,303,...) at sleepq_wait_sig+0x1d
cv_wait_sig(c0a55d1c,c0a55d04) at cv_wait_sig+0x18d
kern_select(c476b360,b,8201168,820116c,0,...) at kern_select+0x4bf
select(c476b360,e67e7d04) at select+0x44
syscall(bfbf003b,bfbf003b,bfbf003b,0,bfbfde7c,...) at syscall+0x256
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (93, FreeBSD ELF32, select), eip = 0x28389097, esp = 0xbfbfde0c, ebp = 0xbfbfde58 ---
Tracing command sshd pid 1050 tid 100131 td 0xc41076c0
db> show sleepchain 39
thread 100049 (pid 39, syncer) blocked on lk "snaplk" EXCL (count 1)
thread 100096 (pid 657, syslogd) running on CPU 0
db> where 39
Tracing pid 39 tid 100049 td 0xc3f77360
sched_switch(c3f77360,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4774314,c0a08a30,0,c091d116,211,...) at sleepq_switch+0xcd
sleepq_wait(c4774314,0,c4043b48,b7,c091cb27,...) at sleepq_wait+0x46
msleep(c4774314,c0a07e60,50,c093226d,0,...) at msleep+0x27d
acquire(e43ebb78,e0,60000,c3f77360,0,...) at acquire+0x76
lockmgr(c4774314,2022,c4043b48,c3f77360) at lockmgr+0x46e
ffs_copyonwrite(c4043a80,d7e30d34) at ffs_copyonwrite+0x246
ffs_geom_strategy(c4043b80,d7e30d34) at ffs_geom_strategy+0xbd
bufwrite(d7e30d34,4000,d7e30d34,e43ebc48,c06e4eb5,...) at bufwrite+0x15a
ffs_bufwrite(d7e30d34) at ffs_bufwrite+0x282
vfs_bio_awrite(d7e30d34) at vfs_bio_awrite+0x229
vop_stdfsync(e43ebcb4,c09ba440,e43ebc90,c08a7457,e43ebcb4,...) at vop_stdfsync+0x125
devfs_fsync(e43ebcb4) at devfs_fsync+0x23
VOP_FSYNC_APV(c09ba440,e43ebcb4) at VOP_FSYNC_APV+0x9b
sync_vnode(c4043b80,c3f77360) at sync_vnode+0x106
sched_sync(0,e43ebd38) at sched_sync+0x1f5
fork_exit(c06f4444,0,e43ebd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe43ebd6c, ebp = 0 ---
db> where 38
Tracing pid 38 tid 100027 td 0xc3c3bd80
sched_switch(c3c3bd80,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x280
sleepq_switch(c4774314,c0a08a30,0,c091d116,211,...) at sleepq_switch+0xcd
sleepq_wait(c4774314,0,c4043b48,c4043b48,b7,...) at sleepq_wait+0x46
msleep(c4774314,c0a07e60,50,c093226d,0,...) at msleep+0x27d
acquire(e270f9d8,e0,60000,c3c3bd80,0,...) at acquire+0x76
lockmgr(c4774314,2022,c4043b48,c3c3bd80) at lockmgr+0x46e
ffs_copyonwrite(c4043a80,d7dd9f98) at ffs_copyonwrite+0x246
ffs_geom_strategy(c4043b80,d7dd9f98,d7dd9ff8,e270fa7c,c09d7f20,...) at ffs_geom_strategy+0xbd
ufs_strategy(e270fa8c) at ufs_strategy+0xb5
VOP_STRATEGY_APV(c09d79e0,e270fa8c) at VOP_STRATEGY_APV+0xaf
bufstrategy(c4154790,d7dd9f98) at bufstrategy+0x55
bufwrite(d7dd9f98) at bufwrite+0x15a
vfs_bio_awrite(d7dd9f98,c0a56218,0,c0922589,882,...) at vfs_bio_awrite+0x229
flushbufqueues(2,0) at flushbufqueues+0x46a
buf_daemon(0,e270fd38) at buf_daemon+0x95
fork_exit(c06e5630,0,e270fd38) at fork_exit+0xac
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe270fd6c, ebp = 0 ---
db> call doadump
Physical memory: 1007 MB
Dumping 173 MB: 158 142 126 110 94 78 62 46 30 14
Dump complete
= 0xf
db> reset
cpu_reset: Restarting BSP
cpu_reset_proxy: Stopped CPU 1
(kgdb) f 12
#12 0xc07d373f in softdep_sync_metadata (vp=0xc40df690) at ../../../ufs/ffs/ffs_softdep.c:5362
5362 FREE_LOCK(&lk);
(kgdb) info loc
pagedep = (struct pagedep *) 0xc472f280
aip = (struct allocindir *) 0x12
bp = (struct buf *) 0xd7ed33f4
nbp = (struct buf *) 0xc40df758
wk = (struct worklist *) 0x0
i = 0x0
error = 0xc40df758
waitfor = 0x2
(kgdb) p lk
$1 = {mtx_object = {lo_name = 0xc0932771 "Softdep Lock", lo_type = 0xc0932771 "Softdep Lock", lo_flags = 0x1030000, lo_witness_data = {
lod_list = {stqe_next = 0xc0a188b8}, lod_witness = 0xc0a188b8}}, mtx_lock = 0x4, mtx_recurse = 0x0}
(kgdb) p &lk
$2 = (struct mtx *) 0xc0a63158
(kgdb) p *vp
$3 = {v_type = VDIR, v_tag = 0xc0922001 "ufs", v_op = 0xc09d79e0, v_data = 0xc40dd630, v_mount = 0xc3fd4b50, v_nmntvnodes = {
tqe_next = 0x0, tqe_prev = 0xc402da94}, v_un = {vu_mount = 0x0, vu_socket = 0x0, vu_cdev = 0x0, vu_fifoinfo = 0x0}, v_hashlist = {
le_next = 0x0, le_prev = 0xc83b72c0}, v_hash = 0x7e801, v_cache_src = {lh_first = 0x0}, v_cache_dst = {tqh_first = 0xc403eb6c,
tqh_last = 0xc403eb7c}, v_dd = 0xc40df3f0, v_cstart = 0x0, v_lasta = 0x0, v_lastw = 0x0, v_clen = 0x0, v_lock = {
lk_interlock = 0xc0a08850, lk_flags = 0x2040040, lk_sharecount = 0x0, lk_waitcount = 0x2, lk_exclusivecount = 0x1, lk_prio = 0x50,
lk_wmesg = 0xc0922001 "ufs", lk_timo = 0x33, lk_lockholder = 0xc3f77510, lk_newlock = 0x0, lk_stack = {depth = 0x7, pcs = {
0xc0689b04, 0xc07d9186, 0xc08a80c7, 0xc06f2419, 0xc06f292f, 0xc0680494, 0xc088038c, 0x0 <repeats 11 times>}}}, v_interlock = {
mtx_object = {lo_name = 0xc091e1ba "vnode interlock", lo_type = 0xc091e1ba "vnode interlock", lo_flags = 0x1030000,
lo_witness_data = {lod_list = {stqe_next = 0xc0a19010}, lod_witness = 0xc0a19010}}, mtx_lock = 0x4, mtx_recurse = 0x0},
v_vnlock = 0xc40df6e8, v_holdcnt = 0x5, v_usecount = 0x0, v_iflag = 0x80, v_vflag = 0x0, v_writecount = 0x0, v_freelist = {
tqe_next = 0x0, tqe_prev = 0xc3ffc398}, v_bufobj = {bo_mtx = 0xc40df758, bo_clean = {bv_hd = {tqh_first = 0x0,
tqh_last = 0xc40df794}, bv_root = 0x0, bv_cnt = 0x0}, bo_dirty = {bv_hd = {tqh_first = 0xd7ed33f4, tqh_last = 0xd7ed342c},
bv_root = 0xd7ed33f4, bv_cnt = 0x1}, bo_numoutput = 0x0, bo_flag = 0x1, bo_ops = 0xc09cabc4, bo_bsize = 0x4000,
bo_object = 0xc184a1e0, bo_synclist = {le_next = 0xc40df250, le_prev = 0xc41b0678}, bo_private = 0xc40df690,
__bo_vnode = 0xc40df690}, v_pollinfo = 0x0, v_label = 0x0}
(kgdb) l *ffs_realloccg+0x7ab
0xc07bc60b is in ffs_realloccg (../../../ufs/ffs/ffs_alloc.c:410).
405 UFS_UNLOCK(ump);
406 if (bp)
407 brelse(bp);
408 if (ppsratecheck(&lastfail, &curfail, 1)) {
409 ffs_fserr(fs, ip->i_number, "filesystem full");
410 uprintf("\n%s: write failed, filesystem is full\n",
411 fs->fs_fsmnt);
412 }
413 return (ENOSPC);
414 }