GENERIC 6.0-BETA5 from Oct 1 20:40 UTC, vmcore.41 mkdir + snapshot test wedge. KDB: debugger backends: ddb KDB: current backend: ddb Copyright (c) 1992-2005 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD 6.0-BETA5 #14: Sat Oct 1 22:52:27 CEST 2005 pho@crashbox.osted.lan:/usr/src/sys/i386/compile/Generic Timecounter "i8254" frequency 1193182 Hz quality 0 CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.80-MHz 686-class CPU) Origin = "GenuineIntel" Id = 0xf24 Stepping = 4 Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM> Hyperthreading: 2 logical CPUs real memory = 1073676288 (1023 MB) avail memory = 1041612800 (993 MB) : Trying to mount root from ufs:/dev/ad0s1a fxp0: link state changed to UP calcru: runtime went backwards from 2329396 usec to 2329314 usec for pid 15211 (thr1) calcru: runtime went backwards from 2329396 usec to 2329314 usec for pid 15211 (thr1) freebsd4_sigreturn: eflags = 0x0 freebsd4_sigreturn: eflags = 0x0 calcru: runtime went backwards from 3184614 usec to 3184584 usec for pid 20557 (thr1) calcru: runtime went backwards from 4431514 usec to 4431503 usec for pid 23197 (thr1) /tmp: mount pending error: blocks -140952 files -8205 sigreturn: eflags = 0x0 sigreturn: eflags = 0x0 sigreturn: eflags = 0x0 /tmp: mount pending error: blocks -141388 files -9361 /tmp: mount pending error: blocks -141388 files -9361 /tmp: mount pending error: blocks -147628 files -9361 /tmp: mount pending error: blocks -147628 files -9361 /tmp: mount pending error: blocks -153068 files -9362 /tmp: mount pending error: blocks -153068 files -9362 /tmp: mount pending error: blocks -153068 files -9362 /tmp: mount pending error: blocks -173196 files -9365 /tmp: mount pending error: blocks -173196 files -9365 ~KDB: enter: Line break on console [thread pid 14 tid 100002 ] Stopped at kdb_enter+0x2b: nop db> ps pid proc uid ppid pgrp flag stat wmesg wchan cmd 59882 c2449830 2 59880 59879 0004000 [SLPQ flswai 0xc093f4c4][SLP] rm 59880 c2b27000 2 59879 59879 0004000 [SLPQ wait 0xc2b27000][SLP] sh 59879 c6266c48 2 59878 59879 0004000 [SLPQ wait 0xc6266c48][SLP] sh 59878 c426ca3c 0 470 470 0000000 [SLPQ piperd 0xc4df1198][SLP] cron 59871 c6264418 2 59869 59868 0004000 [SLPQ flswai 0xc093f4c4][SLP] rm 59869 c426720c 2 59868 59868 0004000 [SLPQ wait 0xc426720c][SLP] sh 59868 c6339624 2 59867 59868 0004000 [SLPQ wait 0xc6339624][SLP] sh 59867 c4282624 0 470 470 0000000 [SLPQ piperd 0xc4e0b660][SLP] cron 59863 c5268418 1001 59858 59858 0000100 [SLPQ sbwait 0xc30d8d2c][SLP] sshd 59862 c4282000 2 59839 59835 0004000 [SLPQ flswai 0xc093f4c4][SLP] dd 59858 c5991c48 0 448 59858 0004100 [SLPQ flswai 0xc093f4c4][SLP] sshd 59839 c275e830 2 59835 59835 0004000 [SLPQ wait 0xc275e830][SLP] sh 59835 c2b2f20c 2 59832 59835 0004000 [SLPQ wait 0xc2b2f20c][SLP] sh 59832 c269e000 0 470 470 0000000 [SLPQ piperd 0xc248c660][SLP] cron 59798 c5268624 0 59797 59795 0000002 [SLPQ getblk 0xd63c7b18][SLP] mount 59797 c632e624 0 59795 59795 0004002 [SLPQ wait 0xc632e624][SLP] mount 59795 c2708418 0 34201 59795 0004002 [SLPQ wait 0xc2708418][SLP] sh 59782 c4267000 1001 59776 59770 0000002 [SLPQ getblk 0xd655e920][SLP] mkdir 59781 c6323c48 1001 59776 59770 0000002 [SLPQ snaplk 0xc5f48294][SLP] mkdir 59780 c6265624 1001 59776 59770 0000002 [SLPQ getblk 0xd649de70][SLP] mkdir 59779 c26a0624 1001 59776 59770 0000002 [SLPQ getblk 0xd649de70][SLP] mkdir 59778 c426cc48 1001 59776 59770 0000002 [SLPQ snaplk 0xc5f48294][SLP] mkdir 59777 c426c20c 1001 59776 59770 0000002 [SLPQ getblk 0xd649de70][SLP] mkdir 59776 c6262a3c 1001 59775 59770 0000002 [SLPQ wait 0xc6262a3c][SLP] mkdir 59775 c599120c 1001 59774 59770 0004002 [SLPQ wait 0xc599120c][SLP] mkdir 59774 c633920c 1001 59773 59770 0000002 [SLPQ wait 0xc633920c][SLP] run 59773 c5b20830 1001 59772 59770 0000002 [SLPQ wait 0xc5b20830][SLP] run 59772 c632320c 1001 59770 59770 0004002 [SLPQ nanslp 0xc093836c][SLP] run 59770 c6323418 1001 592 59770 0004002 [SLPQ wait 0xc6323418][SLP] sh 34392 c26a0a3c 0 0 0 0000204 [SLPQ mdwait 0xc38b6000][SLP] md0 34201 c2f7a20c 0 34198 34201 0004002 [SLPQ wait 0xc2f7a20c][SLP] bash 34198 c2f6920c 0 34197 34198 0004002 [SLPQ pause 0xc2f69240][SLP] csh 34197 c269ec48 1001 34041 34197 0004102 [SLPQ wait 0xc269ec48][SLP] su 34041 c2fa3c48 1001 34040 34041 0004002 [SLPQ wait 0xc2fa3c48][SLP] bash 34040 c2b28a3c 1001 34016 34016 0000100 [SLPQ select 0xc093ef04][SLP] sshd 34016 c2b2820c 0 448 34016 0004100 [SLPQ sbwait 0xc3385a64][SLP] sshd 592 c275ea3c 1001 591 592 0004002 [SLPQ wait 0xc275ea3c][SLP] bash 591 c2757a3c 1001 589 589 0000100 [SLPQ select 0xc093ef04][SLP] sshd 589 c26bd830 0 448 589 0004100 [SLPQ sbwait 0xc2abf4d4][SLP] sshd 588 c275dc48 1001 582 588 0004002 [SLPQ flswai 0xc093f4c4][SLP] top 582 c269ea3c 1001 581 582 0004002 [SLPQ wait 0xc269ea3c][SLP] bash 581 c275d418 1001 579 579 0000100 [SLPQ select 0xc093ef04][SLP] sshd 579 c2757000 0 448 579 0004100 [SLPQ sbwait 0xc283d370][SLP] sshd 578 c2708a3c 0 1 578 0004002 [SLPQ ttyin 0xc250f010][SLP] getty 577 c26bd000 0 1 577 0004002 [SLPQ ttyin 0xc2505010][SLP] getty 576 c270dc48 0 1 576 0004002 [SLPQ ttyin 0xc247b410][SLP] getty 575 c2449624 0 1 575 0004002 [SLPQ ttyin 0xc250e810][SLP] getty 574 c269e830 0 1 574 0004002 [SLPQ ttyin 0xc250e410][SLP] getty 573 c26bd418 0 1 573 0004002 [SLPQ ttyin 0xc250d410][SLP] getty 572 c26b420c 0 1 572 0004002 [SLPQ ttyin 0xc250d010][SLP] getty 571 c275d830 0 1 571 0004002 [SLPQ ttyin 0xc250e010][SLP] getty 557 c26a0418 0 1 557 0000000 [SLPQ select 0xc093ef04][SLP] inetd 536 c26b4a3c 0 1 536 0000000 [SLPQ select 0xc093ef04][SLP] moused 470 c26bc418 0 1 470 0000000 [SLPQ nanslp 0xc093836c][SLP] cron 458 c275d20c 25 1 458 0000100 [SLPQ pause 0xc275d240][SLP] sendmail 454 c26bc624 0 1 454 0000100 [SLPQ select 0xc093ef04][SLP] sendmail 448 c2757830 0 1 448 0000100 [SLPQ select 0xc093ef04][SLP] sshd 404 c275d624 0 1 404 0000000 [SLPQ select 0xc093ef04][SLP] usbd 313 c269e20c 0 1 313 0000000 [SLPQ select 0xc093ef04][SLP] devd 282 c2449c48 0 1 282 0000000 [SLPQ flswai 0xc093f4c4][SLP] syslogd 108 c239a20c 0 0 0 0000204 [SLPQ - 0xe5101d08][SLP] schedcpu 107 c239a418 0 0 0 0000204 [SLPQ - 0xc094748c][SLP] nfsiod 3 106 c239a624 0 0 0 0000204 [SLPQ - 0xc0947488][SLP] nfsiod 2 105 c239a830 0 0 0 0000204 [SLPQ - 0xc0947484][SLP] nfsiod 1 104 c239aa3c 0 0 0 0000204 [SLPQ - 0xc0947480][SLP] nfsiod 0 103 c239ac48 0 0 0 0000204 [SLPQ snaplk 0xc5f48294][SLP] syncer 102 c2447000 0 0 0 0000204 [SLPQ vlruwt 0xc2447000][SLP] vnlru 101 c244720c 0 0 0 0000204 [SLPQ snaplk 0xc5f48294][SLP] bufdaemon 100 c2447418 0 0 0 000020c [SLPQ pgzero 0xc094da44][SLP] pagezero 99 c2447624 0 0 0 0000204 [SLPQ psleep 0xc094d594][SLP] vmdaemon 98 c2447830 0 0 0 0000204 [SLPQ psleep 0xc094d550][SLP] pagedaemon 97 c2447a3c 0 0 0 0000204 [SLPQ - 0xc2446e3c][SLP] fdc0 96 c2447c48 0 0 0 0000204 [IWAIT] swi0: sio 95 c2449000 0 0 0 0000204 [SLPQ usbtsk 0xc09353a4][SLP] usbtask 94 c244920c 0 0 0 0000204 [SLPQ usbevt 0xc246e210][SLP] usb0 93 c2449418 0 0 0 0000204 [IWAIT] swi6: task queue 9 c2394000 0 0 0 0000204 [SLPQ - 0xc2370a80][SLP] acpi_task2 8 c239420c 0 0 0 0000204 [SLPQ - 0xc2370a80][SLP] acpi_task1 7 c2394418 0 0 0 0000204 [SLPQ - 0xc2370a80][SLP] acpi_task0 92 c2394624 0 0 0 0000204 [IWAIT] swi2: cambio 6 c2394830 0 0 0 0000204 [SLPQ - 0xc2370c00][SLP] kqueue taskq 91 c2394a3c 0 0 0 0000204 [IWAIT] swi5:+ 5 c2394c48 0 0 0 0000204 [SLPQ - 0xc2370d80][SLP] thread taskq 90 c2399000 0 0 0 0000204 [IWAIT] swi6:+ 89 c239920c 0 0 0 0000204 [SLPQ - 0xc09330c0][SLP] yarrow 4 c2399418 0 0 0 0000204 [SLPQ - 0xc0935888][SLP] g_down 3 c2399624 0 0 0 0000204 [SLPQ - 0xc0935884][SLP] g_up 2 c2399830 0 0 0 0000204 [SLPQ - 0xc093587c][SLP] g_event 88 c2399a3c 0 0 0 0000204 [IWAIT] swi1: net 87 c2399c48 0 0 0 0000204 [IWAIT] swi3: vm 86 c239a000 0 0 0 000020c [IWAIT] swi4: clock sio 85 c237f000 0 0 0 0000204 [IWAIT] irq71: 84 c237f20c 0 0 0 0000204 [IWAIT] irq70: 83 c237f418 0 0 0 0000204 [IWAIT] irq69: 82 c237f624 0 0 0 0000204 [IWAIT] irq68: 81 c237f830 0 0 0 0000204 [IWAIT] irq67: 80 c237fa3c 0 0 0 0000204 [IWAIT] irq66: 79 c237fc48 0 0 0 0000204 [IWAIT] irq65: 78 c2384000 0 0 0 0000204 [IWAIT] irq64: 77 c238420c 0 0 0 0000204 [IWAIT] irq63: 76 c2384418 0 0 0 0000204 [IWAIT] irq62: 75 c2384624 0 0 0 0000204 [IWAIT] irq61: 74 c2384830 0 0 0 0000204 [IWAIT] irq60: 73 c2384a3c 0 0 0 0000204 [IWAIT] irq59: 72 c2384c48 0 0 0 0000204 [IWAIT] irq58: 71 c235f20c 0 0 0 0000204 [IWAIT] irq57: 70 c235f418 0 0 0 0000204 [IWAIT] irq56: 69 c235f624 0 0 0 0000204 [IWAIT] irq55: 68 c235f830 0 0 0 0000204 [IWAIT] irq54: 67 c235fa3c 0 0 0 0000204 [IWAIT] irq53: 66 c235fc48 0 0 0 0000204 [IWAIT] irq52: 65 c2371000 0 0 0 0000204 [IWAIT] irq51: 64 c237120c 0 0 0 0000204 [IWAIT] irq50: 63 c2371418 0 0 0 0000204 [IWAIT] irq49: 62 c2371624 0 0 0 0000204 [IWAIT] irq48: em0 61 c2371830 0 0 0 0000204 [IWAIT] irq47: 60 c2371a3c 0 0 0 0000204 [IWAIT] irq46: 59 c2371c48 0 0 0 0000204 [IWAIT] irq45: 58 c2347624 0 0 0 0000204 [IWAIT] irq44: 57 c2347830 0 0 0 0000204 [IWAIT] irq43: 56 c2347a3c 0 0 0 0000204 [IWAIT] irq42: 55 c2347c48 0 0 0 0000204 [IWAIT] irq41: 54 c235e000 0 0 0 0000204 [IWAIT] irq40: 53 c235e20c 0 0 0 0000204 [IWAIT] irq39: 52 c235e418 0 0 0 0000204 [IWAIT] irq38: 51 c235e624 0 0 0 0000204 [IWAIT] irq37: 50 c235e830 0 0 0 0000204 [IWAIT] irq36: 49 c235ea3c 0 0 0 0000204 [IWAIT] irq35: 48 c235ec48 0 0 0 0000204 [IWAIT] irq34: 47 c235f000 0 0 0 0000204 [IWAIT] irq33: 46 c2338c48 0 0 0 0000204 [IWAIT] irq32: 45 c2346000 0 0 0 0000204 [IWAIT] irq31: 44 c234620c 0 0 0 0000204 [IWAIT] irq30: 43 c2346418 0 0 0 0000204 [IWAIT] irq29: 42 c2346624 0 0 0 0000204 [IWAIT] irq28: 41 c2346830 0 0 0 0000204 [IWAIT] irq27: 40 c2346a3c 0 0 0 0000204 [IWAIT] irq26: 39 c2346c48 0 0 0 0000204 [IWAIT] irq25: 38 c2347000 0 0 0 0000204 [IWAIT] irq24: 37 c234720c 0 0 0 0000204 [IWAIT] irq23: 36 c2347418 0 0 0 0000204 [IWAIT] irq22: 35 c232e624 0 0 0 0000204 [IWAIT] irq21: 34 c232e830 0 0 0 0000204 [IWAIT] irq20: 33 c232ea3c 0 0 0 0000204 [IWAIT] irq19: 32 c232ec48 0 0 0 0000204 [IWAIT] irq18: 31 c2338000 0 0 0 0000204 [IWAIT] irq17: fxp0 30 c233820c 0 0 0 0000204 [IWAIT] irq16: uhci0 29 c2338418 0 0 0 0000204 [IWAIT] irq15: ata1 28 c2338624 0 0 0 0000204 [IWAIT] irq14: ata0 27 c2338830 0 0 0 0000204 [IWAIT] irq13: 26 c2338a3c 0 0 0 0000204 [IWAIT] irq12: psm0 25 c22db20c 0 0 0 0000204 [IWAIT] irq11: 24 c22db418 0 0 0 0000204 [IWAIT] irq10: 23 c22db624 0 0 0 0000204 [IWAIT] irq9: acpi0 22 c22db830 0 0 0 0000204 [IWAIT] irq8: 21 c22dba3c 0 0 0 0000204 [IWAIT] irq7: ppc0 20 c22dbc48 0 0 0 0000204 [IWAIT] irq6: fdc0 19 c232e000 0 0 0 0000204 [IWAIT] irq5: 18 c232e20c 0 0 0 0000204 [IWAIT] irq4: sio0 17 c232e418 0 0 0 0000204 [IWAIT] irq3: sio1 16 c22d6000 0 0 0 0000204 [IWAIT] irq0: 15 c22d620c 0 0 0 0000204 [IWAIT] irq1: atkbd0 14 c22d6418 0 0 0 000020c [CPU 0] idle: cpu0 13 c22d6624 0 0 0 000020c [CPU 1] idle: cpu1 12 c22d6830 0 0 0 000020c [CPU 2] idle: cpu2 11 c22d6a3c 0 0 0 000020c [CPU 3] idle: cpu3 1 c22d6c48 0 0 1 0004200 [SLPQ wait 0xc22d6c48][SLP] init 10 c22db000 0 0 0 0000204 [SLPQ ktrace 0xc09362d8][SLP] ktrace 0 c0935980 0 0 0 0000200 [IWAIT] swapper db> where 59882 Tracing pid 59882 tid 100321 td 0xc6078180 sched_switch(c6078180,0,1) at sched_switch+0x14b mi_switch(1,0,c6078180,0,c6078180) at mi_switch+0x1ba sleepq_switch(c093f4c4) at sleepq_switch+0x133 sleepq_wait(c093f4c4,0,18100,0,4) at sleepq_wait+0xb msleep(c093f4c4,c093f4e0,50,c087d868,0) at msleep+0x362 bwillwrite(0,0,433fa064,0,0) at bwillwrite+0x107 kern_unlink(c6078180,bfbfef26,0,e7f3fd30,c081612b) at kern_unlink+0x11 unlink(c6078180,e7f3fd04,1,0,296) at unlink+0x12 syscall(3b,3b,3b,bfbfee8c,2) at syscall+0x2bf Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (10, FreeBSD ELF32, unlink), eip = 0x280b8717, esp = 0xbfbfed6c, ebp = 0xbfbfedf8 --- db> show pcpu 0 cpuid = 0 curthread = 0xc22d7c00: pid 14 "idle: cpu0" curpcb = 0xe35f8d90 fpcurthread = none idlethread = 0xc22d7c00: pid 14 "idle: cpu0" APIC ID = 0 currentldt = 0x50 db> show pcpu 1 cpuid = 1 curthread = 0xc22d7a80: pid 13 "idle: cpu1" curpcb = 0xe35f5d90 fpcurthread = none idlethread = 0xc22d7a80: pid 13 "idle: cpu1" APIC ID = 1 currentldt = 0x50 db> show pcpu 2 cpuid = 2 curthread = 0xc22d7900: pid 12 "idle: cpu2" curpcb = 0xe35f2d90 fpcurthread = none idlethread = 0xc22d7900: pid 12 "idle: cpu2" APIC ID = 6 currentldt = 0x50 db> show pcpu 3 cpuid = 3 curthread = 0xc22d7780: pid 11 "idle: cpu3" curpcb = 0xe35efd90 fpcurthread = none idlethread = 0xc22d7780: pid 11 "idle: cpu3" APIC ID = 7 currentldt = 0x50 db> call doadump Dumping 1023 MB (2 chunks) chunk 0: 1MB (159 pages) ... ok chunk 1: 1023MB (261872 pages) 1007 991 975 959 943 927 911 895 879 863 847 831 815 799 783 767 751 735 719 703 687 671 655 639 623 607 591 575 559 543 527 511 495 479 463 447 431 415 399 383 367 351 335 319 303 287 271 255 239 223 207 191 175 159 143 127 111 95 79 63 47 31 15 ... ok Dump complete = 0xf db> reset(kgdb) (kgdb) x/x 0xc093f4c4 0xc093f4c4 <needsbuffer>: 0x00000002 (kgdb) lock 0xc5f48294 $4 = {lk_interlock = 0xc0936d40, lk_flags = 0x20400c0, lk_sharecount = 0x0, lk_waitcount = 0x4, lk_exclusivecount = 0x1, lk_prio = 0x50, lk_wmesg = 0xc088aafd "snaplk", lk_timo = 0x33, lk_lockholder = 0xc3471180, lk_newlock = 0x0} Lock 0xc5f48294 is owned by pid 59798 (kgdb) lock 0xd649de70 $5 = {lk_interlock = 0xc0937910, lk_flags = 0x2040000, lk_sharecount = 0x0, lk_waitcount = 0x3, lk_exclusivecount = 0x1, lk_prio = 0x50, lk_wmesg = 0xc087da8f "getblk", lk_timo = 0x0, lk_lockholder = 0xc344e480, lk_newlock = 0x0} Lock 0xd649de70 is owned by pid 59778 (kgdb) x/20x 0xd63c7b18 0xd63c7b18: 0xc09368e4 0x02040000 0x00000000 0x00000001 0xd63c7b28: 0x00500001 0xc087da8f 0x00000000 0xfffffffe 0xd63c7b38: 0x00000000 0x00004000 0x00000000 0xd7f94000 0xd63c7b48: 0x00004000 0x00114080 0x00000000 0xc27a9990 0xd63c7b58: 0x00000000 0x00000000 0x00000000 0x00000000