GENERIC HEAD from Jan 28 20:19 UTC + mpsafe_vfs = 1, vmcore.162
Double panic:
1) panic: Most recently used by inodedep
2) panic: thread 100190(swap):2 holds vm page queue mutex but isn't blocked on a lock
GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2005 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD 6.0-CURRENT #3: Fri Jan 28 21:21:46 CET 2005
pho@current.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
ACPI APIC Table: <A M I OEMAPIC >
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) Celeron(R) CPU 1.80GHz (1799.14-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf13 Stepping = 3
Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
real memory = 267583488 (255 MB)
avail memory = 252379136 (240 MB)
:
mount root from ufs:/dev/ad0s1a
WARNING: / was not properly dismounted
WARNING: /tmp was not properly dismounted
WARNING: /usr was not properly dismounted
WARNING: /var was not properly dismounted
/var: mount pending error: blocks 4 files 1
ad0: WARNING - WRITE_DMA interrupt was seen but timeout fired LBA=542695
Memory modified after free 0xc1d7c100(124) val=c23fa000 @ 0xc1d7c164
ppanic: thread 100190(swap):2 holds vm page queue mutex but isn't blocked on a lock
cpuid = 0
KDB: enter: panic
[thread pid 61 tid 100051 ]
Stopped at kdb_enter+0x2b: nop
db> where
Tracing pid 61 tid 100051 td 0xc162a170
kdb_enter(c08238a8) at kdb_enter+0x2b
panic(c0826b08,1875e,c238fd54,2,c083e5fb) at panic+0x14b
propagate_priority(c1a9e5c0,c08fb5ac,0,c0826a6b,192) at propagate_priority+0x10f
turnstile_adjust(c1a9e5c0,c0) at turnstile_adjust+0xef
sched_prio(c1a9e5c0,bf,c1a9e5c0,c1a9e5c0,cc6d9d0c) at sched_prio+0x54
resetpriority_thread(c1a9e5c0,c1a9b240) at resetpriority_thread+0x28
schedcpu(cc6d9d34,c1758dc8,cc6d9d34,c05ffc18,0) at schedcpu+0x18f
schedcpu_thread(0,cc6d9d48,0,c0622cb4,0) at schedcpu_thread+0x11
fork_exit(c0622cb4,0,cc6d9d48) at fork_exit+0xa4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xcc6d9d7c, ebp = 0 ---
db> where 100190
Tracing pid 29904 tid 100190 td 0xc23908a0
sched_switch(c23908a0,0,2) at sched_switch+0x17f
mi_switch(2,0,c08f42e0,0,c0824fe8) at mi_switch+0x264
critical_exit(c08c9260) at critical_exit+0x86
intr_execute_handlers(c08c9260,cf2adbb0,10,c112eb50,c112eb50) at intr_execute_handlers+0xba
atpic_handle_intr(0) at atpic_handle_intr+0x92
Xatpic_intr0() at Xatpic_intr0+0x20
--- interrupt, eip = 0xc076fabd, esp = 0xcf2adbf4, ebp = 0xcf2adc0c ---
vm_page_flag_clear(c112eb50,10) at vm_page_flag_clear+0x21
pmap_remove_all(c112eb50,c1412698,c1c18ad4,c27f1044,c1ad1960) at pmap_remove_all+0x157
vm_object_page_remove(c1c18ad4,10,0,869,0,0,c1c18ad4) at vm_object_page_remove+0x125
vm_map_entry_delete(c1ad1960,c27f1044,58,805f000,c27f1044) at vm_map_entry_delete+0xcc
vm_map_delete(c1ad1960,805f000,88b8000) at vm_map_delete+0x18f
obreak(c23908a0,cf2add14,1,b,283) at obreak+0x1a9
syscall(2f,2f,2f,88b8000,804b020) at syscall+0x213
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (17, FreeBSD ELF32, obreak), eip = 0x280ceef3, esp = 0xbfbfe8ac, ebp = 0xbfbfe8f8 ---
db> where 0x74e2
Tracing pid 29922 tid 100172 td 0xc1add2e0
sched_switch(c1add2e0,0,2) at sched_switch+0x17f
mi_switch(2,0,c08f42e0,0,c0824fe8) at mi_switch+0x264
critical_exit(c08c9260) at critical_exit+0x86
intr_execute_handlers(c08c9260,cf261924,c1add2e0,a23,c084188f) at intr_execute_handlers+0xba
atpic_handle_intr(0) at atpic_handle_intr+0x92
Xatpic_intr0() at Xatpic_intr0+0x20
--- interrupt, eip = 0xc06240a2, esp = 0xcf261968, ebp = 0xcf26196c ---
critical_exit(3f8,2580,1,cf2619b4,c07993a1) at critical_exit+0xb2
_mtx_unlock_spin_flags(c095c420,0,c084188f,a23) at _mtx_unlock_spin_flags+0xc9
siocnputc(c08c4860,70) at siocnputc+0xb9
cnputc(70) at cnputc+0x4d
putchar(70,cf261ac8) at putchar+0x52
kvprintf(c0823a9d,c062c7f4,cf261ac8,a,cf261ae8) at kvprintf+0x77
printf(c0823a9d,c08f4560,c08f4560,100,c083d005,cf261b1c) at printf+0x43
panic(c083d005,c083b16e,c083cfd6,c1d7c100,7c) at panic+0xef
mtrash_ctor(c1d7c100,80,0,402) at mtrash_ctor+0x4d
uma_zalloc_arg(c10526e0,0,402) at uma_zalloc_arg+0x14c
malloc(68,c08bcd60,402,c094f0c0,0) at malloc+0xae
inodedep_lookup(c167a000,193a1,1,cf261be0,c094f0c0) at inodedep_lookup+0xa7
softdep_change_linkcnt(c213b578,c6bc54f0,c65f3f88,c213b578,c1c39270) at softdep_change_linkcnt+0x31
ufs_dirremove(c17d83a8,c213b578,100800c,0,cf261c48) at ufs_dirremove+0x12d
ufs_remove(cf261c4c) at ufs_remove+0x4b
VOP_REMOVE_AP(cf261c4c) at VOP_REMOVE_AP+0x62
kern_unlink(c1add2e0,bfbfe940,0,cf261d40,c07bd6c3) at kern_unlink+0x167
unlink(c1add2e0,cf261d14,1,28,292) at unlink+0x12
syscall(2804002f,bfbf002f,bfbf002f,2804f6c0,bfbfeb14) at syscall+0x213
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (10, FreeBSD ELF32, unlink), eip = 0x280b7c33, esp = 0xbfbfe92c, ebp = 0xbfbfe9d8 ---
db> show alllocks
Process 29918 (creat) thread 0xc1a9e5c0 (100144)
exclusive sleep mutex vm object (standard object) r = 0 (0xc10619cc) locked @ kern/vfs_bio.c:3523
Process 29911 (swap) thread 0xc18a48a0 (100106)
exclusive sleep mutex vm object (standard object) r = 0 (0xc2532000) locked @ vm/vm_fault.c:901
exclusive sx user map r = 0 (0xc1a91620) locked @ vm/vm_map.c:2998
Process 29909 (swap) thread 0xc18a4000 (100100)
exclusive sleep mutex vm object (standard object) r = 0 (0xc1e6739c) locked @ vm/vm_fault.c:295
exclusive sx user map r = 0 (0xc23a274c) locked @ vm/vm_map.c:2998
Process 29904 (swap) thread 0xc23908a0 (100190)
exclusive sleep mutex vm page queue mutex r = 0 (0xc094fda0) locked @ vm/vm_object.c:1768
exclusive sleep mutex vm object (standard object) r = 0 (0xc1c18ad4) locked @ vm/vm_map.c:2193
exclusive sx user map r = 0 (0xc1ad19a4) locked @ vm/vm_unix.c:88
Process 29903 (swap) thread 0xc238e730 (100195)
exclusive sleep mutex vm object (standard object) r = 0 (0xc1bca210) locked @ vm/vm_fault.c:660
exclusive sx user map r = 0 (0xc1ad1878) locked @ vm/vm_map.c:2998
Process 29890 (swap) thread 0xc175a000 (100070)
exclusive sleep mutex vm object (standard object) r = 0 (0xc1f0239c) locked @ vm/vm_fault.c:295
exclusive sx user map r = 0 (0xc23a2bfc) locked @ vm/vm_map.c:2998
Process 29888 (swap) thread 0xc18a5170 (100111)
exclusive sleep mutex vm object (standard object) r = 0 (0xc1f6cbdc) locked @ vm/vm_fault.c:660
exclusive sx user map r = 0 (0xc27c73c8) locked @ vm/vm_map.c:2998
Process 29886 (swap) thread 0xc2391cf0 (100209)
exclusive sleep mutex vm object (standard object) r = 0 (0xc1b6b210) locked @ vm/vm_fault.c:295
exclusive sx user map r = 0 (0xc23a329c) locked @ vm/vm_map.c:2998
Process 29885 (swap) thread 0xc184c2e0 (100092)
exclusive sleep mutex vm object (standard object) r = 0 (0xc1d4e528) locked @ vm/vm_fault.c:295
exclusive sx user map r = 0 (0xc1fdfd28) locked @ vm/vm_map.c:2998
Process 29880 (swap) thread 0xc1add730 (100175)
exclusive sleep mutex vm object (standard object) r = 0 (0xc1d816b4) locked @ vm/vm_fault.c:901
exclusive sx user map r = 0 (0xc23a2d28) locked @ vm/vm_map.c:2998
Process 29879 (swap) thread 0xc2390450 (100187)
exclusive sleep mutex vm object (standard object) r = 0 (0xc1f5b18c) locked @ vm/vm_fault.c:660
exclusive sx user map r = 0 (0xc1ad1170) locked @ vm/vm_map.c:2998
db> show pcpu
cpuid = 0
curthread = 0xc162a170: pid 61 "schedcpu"
curpcb = 0xcc6d9da0
fpcurthread = none
idlethread = 0xc151f5c0: pid 11 "idle: cpu0"
APIC ID = 0
currentldt = 0x28
spin locks held:
db> call doadump
Dumping 255 MB
16 32 48 64 80 96 112 128 144 160 176 192 208 224 240
Dump complete
0xf
db> reset
(kgdb) l *mtrash_ctor+0x4d
0xc07633a5 is in mtrash_ctor (../../../vm/uma_dbg.c:136).
131
132 for (p = mem; cnt > 0; cnt--, p++)
133 if (*p != uma_junk) {
134 printf("Memory modified after free %p(%d) val=%x @ %p\n",
135 mem, size, *p, p);
136 panic("Most recently used by %s\n", (*ksp == NULL)?
137 "none" : (*ksp)->ks_shortdesc);
138 }
139 return (0);
140 }
(kgdb) x/40x 0xc1d7c100
0xc1d7c100: 0xdeadc0de 0xdeadc0de 0xdeadc0de 0xdeadc0de
0xc1d7c110: 0xdeadc0de 0xdeadc0de 0xdeadc0de 0xdeadc0de
0xc1d7c120: 0xdeadc0de 0xdeadc0de 0xdeadc0de 0xdeadc0de
0xc1d7c130: 0xdeadc0de 0xdeadc0de 0xdeadc0de 0xdeadc0de
0xc1d7c140: 0xdeadc0de 0xdeadc0de 0xdeadc0de 0xdeadc0de
0xc1d7c150: 0xdeadc0de 0xdeadc0de 0xdeadc0de 0xdeadc0de
0xc1d7c160: 0xdeadc0de 0xc23fa000 0xdeadc0de 0xdeadc0de
0xc1d7c170: 0xdeadc0de 0xdeadc0de 0xdeadc0de 0xc08bcd60
0xc1d7c180: 0xc1f15700 0xc664fc04 0x040d0001 0x00000000
0xc1d7c190: 0xc1703108 0xc167a000 0x0001e105 0xdead0001
index trace
------ -----
953 UNLOCK (spin mutex) sched lock r = 0 at ../../../kern/kern_switch.c:596
952 LOCK (spin mutex) sched lock r = 0 at ../../../kern/kern_intr.c:589
951 UNLOCK (spin mutex) callout r = 0 at ../../../kern/kern_timeout.c:305
950 LOCK (spin mutex) callout r = 0 at ../../../kern/kern_timeout.c:280
949 UNLOCK (sleep mutex) tcp r = 0 at ../../../netinet/tcp_subr.c:1367
948 UNLOCK (spin mutex) callout r = 0 at ../../../kern/kern_timeout.c:441
947 LOCK (spin mutex) callout r = 0 at ../../../kern/kern_timeout.c:400
946 LOCK (sleep mutex) tcp r = 0 at ../../../netinet/tcp_subr.c:1359
945 UNLOCK (spin mutex) callout r = 0 at ../../../kern/kern_timeout.c:245
944 LOCK (spin mutex) callout r = 0 at ../../../kern/kern_timeout.c:201
943 UNLOCK (spin mutex) sched lock r = 0 at ../../../kern/kern_intr.c:598
942 LOCK (spin mutex) sched lock r = 0 at ../../../kern/kern_switch.c:594
941 UNLOCK (spin mutex) icu r = 0 at ../../../i386/isa/atpic.c:267
940 LOCK (spin mutex) icu r = 0 at ../../../i386/isa/atpic.c:265
939 UNLOCK (spin mutex) sched lock r = 0 at ../../../kern/kern_intr.c:415
938 LOCK (spin mutex) sched lock r = 0 at ../../../kern/kern_intr.c:406
937 UNLOCK (sleep mutex) vnode interlock r = 0 at ../../../kern/vfs_subr.c:982
936 UNLOCK (sleep mutex) vnode interlock r = 0 at ../../../kern/vfs_subr.c:1953
935 LOCK (sleep mutex) vnode interlock r = 0 at ../../../kern/vfs_subr.c:1931
934 UNLOCK (sleep mutex) struct mount mtx r = 0 at ../../../kern/vfs_vnops.c:991
933 LOCK (sleep mutex) struct mount mtx r = 0 at ../../../kern/vfs_vnops.c:984
932 UNLOCK (sleep mutex) buffer daemon lock r = 0 at ../../../kern/vfs_bio.c:426
931 LOCK (sleep mutex) buffer daemon lock r = 0 at ../../../kern/vfs_bio.c:421
930 lockmgr(): lkp == 0xc6638be4 (lk_wmesg == "getblk"), flags == 0x6, td == 0xc2090b80 ../../../sys/buf.h:313
929 UNLOCK (sleep mutex) buf queue lock r = 0 at ../../../kern/vfs_bio.c:1579
928 bremfreel(0xc6638b84) vp 0xc1856270 flags 800000A0
927 LOCK (sleep mutex) buf queue lock r = 0 at ../../../kern/vfs_bio.c:1544
926 bqrelse(0xc6638b84) vp 0xc1856270 flags 800000A0
925 bdirty(0xc6638b84) vp 0xc1856270 flags 800000A0
924 UNLOCK (sleep mutex) vnode interlock r = 0 at ../../../kern/vfs_bio.c:1058
923 LOCK (sleep mutex) vnode interlock r = 0 at ../../../kern/vfs_bio.c:1023
922 bdwrite(0xc6638b84) vp 0xc1856270 flags 800000A0
921 UNLOCK (sleep mutex) Softdep Lock r = 0 at ../../../ufs/ffs/ffs_softdep.c:2794
920 LOCK (sleep mutex) Softdep Lock r = 0 at ../../../ufs/ffs/ffs_softdep.c:2847
919 UNLOCK (sleep mutex) dirrem r = 0 at ../../../kern/kern_malloc.c:213
918 LOCK (sleep mutex) dirrem r = 0 at ../../../kern/kern_malloc.c:203
917 UNLOCK (sleep mutex) UMA pcpu r = 0 at ../../../vm/uma_core.c:1846
916 UNLOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:1844
915 LOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:1842
914 LOCK (sleep mutex) UMA pcpu r = 0 at ../../../vm/uma_core.c:1825
913 UNLOCK (sleep mutex) Softdep Lock r = 0 at ../../../ufs/ffs/ffs_softdep.c:2838
912 LOCK (sleep mutex) Softdep Lock r = 0 at ../../../ufs/ffs/ffs_softdep.c:2834
911 UNLOCK (sleep mutex) Softdep Lock r = 0 at ../../../ufs/ffs/ffs_softdep.c:3039
910 LOCK (sleep mutex) Softdep Lock r = 0 at ../../../ufs/ffs/ffs_softdep.c:880
909 UNLOCK (sleep mutex) inodedep r = 0 at ../../../kern/kern_malloc.c:213
908 LOCK (sleep mutex) inodedep r = 0 at ../../../kern/kern_malloc.c:203
907 UNLOCK (sleep mutex) UMA pcpu r = 0 at ../../../vm/uma_core.c:1846
906 UNLOCK (sleep mutex) 128 r = 0 at ../../../vm/uma_core.c:1844
905 LOCK (sleep mutex) 128 r = 0 at ../../../vm/uma_core.c:1842
904 LOCK (sleep mutex) UMA pcpu r = 0 at ../../../vm/uma_core.c:1825
903 UNLOCK (sleep mutex) Softdep Lock r = 0 at ../../../ufs/ffs/ffs_softdep.c:877
902 LOCK (sleep mutex) Softdep Lock r = 0 at ../../../ufs/ffs/ffs_softdep.c:3034
901 UNLOCK (sleep mutex) dirhash r = 0 at ../../../ufs/ufs/ufs_dirhash.c:680
900 LOCK (sleep mutex) dirhash r = 0 at ../../../ufs/ufs/ufs_dirhash.c:663
899 getblk(0xc1856270, 0, 16384) = 0xc6638b84
898 bremfree(0xc6638b84) vp 0xc1856270 flags A0
897 acquire(): lkp == 0xc6638be4, extflags == 0x2010020, wanted == 0x100500
896 acquire(): lkp == 0xc6638be4, extflags == 0x2010020, wanted == 0x600
895 UNLOCK (sleep mutex) vnode interlock r = 0 at ../../../kern/kern_lock.c:238
894 lockmgr(): lkp == 0xc6638be4 (lk_wmesg == "getblk"), flags == 0x2090022, td == 0xc2090b80 ../../../sys/buf.h:298
893 LOCK (sleep mutex) vnode interlock r = 0 at ../../../kern/vfs_bio.c:2479
892 getblk(0xc1856270, 0, 16384)
891 breadn(0xc1856270, 70368744177664, 134528000)
890 UNLOCK (sleep mutex) struct mount mtx r = 0 at ../../../kern/vfs_vnops.c:925
889 LOCK (sleep mutex) struct mount mtx r = 0 at ../../../kern/vfs_vnops.c:907
888 UNLOCK (sleep mutex) UMA pcpu r = 0 at ../../../vm/uma_core.c:2276
887 UNLOCK (sleep mutex) NAMEI r = 0 at ../../../vm/uma_core.c:2274
886 LOCK (sleep mutex) NAMEI r = 0 at ../../../vm/uma_core.c:2269
885 LOCK (sleep mutex) UMA pcpu r = 0 at ../../../vm/uma_core.c:2251
884 UNLOCK (sleep mutex) vnode interlock r = 0 at ../../../kern/vfs_subr.c:1834
883 UNLOCK (sleep mutex) cdev r = 0 at ../../../kern/kern_conf.c:77
882 LOCK (sleep mutex) cdev r = 0 at ../../../kern/kern_conf.c:70
881 LOCK (sleep mutex) vnode interlock r = 0 at ../../../kern/vfs_subr.c:1832
880 lockmgr(): lkp == 0xc663b700 (lk_wmesg == "getblk"), flags == 0x6, td == 0xc2090b80 ../../../sys/buf.h:313
879 UNLOCK (sleep mutex) needsbuffer lock r = 0 at ../../../kern/vfs_bio.c:318
878 LOCK (sleep mutex) needsbuffer lock r = 0 at ../../../kern/vfs_bio.c:313
877 UNLOCK (sleep mutex) needsbuffer lock r = 0 at ../../../kern/vfs_bio.c:362
876 LOCK (sleep mutex) needsbuffer lock r = 0 at ../../../kern/vfs_bio.c:355
875 UNLOCK (sleep mutex) buf queue lock r = 0 at ../../../kern/vfs_bio.c:1579
874 UNLOCK (sleep mutex) vnode interlock r = 0 at ../../../kern/vfs_bio.c:1562
873 LOCK (sleep mutex) vnode interlock r = 0 at ../../../kern/vfs_bio.c:1560
872 bremfreel(0xc663b6a0) vp 0xc17ccaf8 flags A0020020
871 LOCK (sleep mutex) buf queue lock r = 0 at ../../../kern/vfs_bio.c:1544
870 bqrelse(0xc663b6a0) vp 0xc17ccaf8 flags A0020020
869 UNLOCK (sleep mutex) Softdep Lock r = 0 at ../../../ufs/ffs/ffs_softdep.c:4382
868 LOCK (sleep mutex) Softdep Lock r = 0 at ../../../ufs/ffs/ffs_softdep.c:4380
867 UNLOCK (sleep mutex) UMA pcpu r = 0 at ../../../vm/uma_core.c:1846
866 UNLOCK (sleep mutex) FFS2 dinode r = 0 at ../../../vm/uma_core.c:1844
865 LOCK (sleep mutex) FFS2 dinode r = 0 at ../../../vm/uma_core.c:1842
864 LOCK (sleep mutex) UMA pcpu r = 0 at ../../../vm/uma_core.c:1825
863 getblk(0xc17ccaf8, 1508128, 16384) = 0xc663b6a0
862 bremfree(0xc663b6a0) vp 0xc17ccaf8 flags 20020020
861 acquire(): lkp == 0xc663b700, extflags == 0x2010020, wanted == 0x100500
860 acquire(): lkp == 0xc663b700, extflags == 0x2010020, wanted == 0x600
859 UNLOCK (sleep mutex) vnode interlock r = 0 at ../../../kern/kern_lock.c:238
858 lockmgr(): lkp == 0xc663b700 (lk_wmesg == "getblk"), flags == 0x2090022, td == 0xc2090b80 ../../../sys/buf.h:298
857 LOCK (sleep mutex) vnode interlock r = 0 at ../../../kern/vfs_bio.c:2479
856 getblk(0xc17ccaf8, 1508128, 16384)