GENERIC 7.0-CURRENT from Nov 4 09:15 UTC, vmcore.12

KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2005 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD 7.0-CURRENT #1: Fri Nov  4 10:21:52 CET 2005
    pho@crashbox.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
ACPI APIC Table: <A M I  OEMAPIC >
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) XEON(TM) CPU 1.80GHz (1799.80-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0xf24  Stepping = 4
  Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
  Logical CPUs per core: 2
real memory  = 1073676288 (1023 MB)
avail memory = 1041047552 (992 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
fxp0: link state changed to UP
panic: mb_dtor_pack: ref_cnt != 1
cpuid = 0
KDB: enter: panic
[thread pid 15 tid 100001 ]
Stopped at      kdb_enter+0x2b: nop
db> where
Tracing pid 15 tid 100001 td 0xc22d8d80
kdb_enter(c086ac6b) at kdb_enter+0x2b
panic(c0869507,c0821f99,c2464e00,e3685b48,c0790494) at panic+0x14b
mb_dtor_pack(c2464e00,100,0) at mb_dtor_pack+0xc1
uma_zfree_arg(c1040d20,c2464e00,0) at uma_zfree_arg+0x24
mb_free_ext(c2464e00) at mb_free_ext+0xad
sbdrop_locked(c27c3be8,3c00,c27c3c0c,0,c0871b1c) at sbdrop_locked+0xd0
sbdrop(c27c3be8,3c00,c2b98730,3d061532,0) at sbdrop+0x26
tcp_input(c3d7c500,14,12e,c3d7c500,0) at tcp_input+0x1089
ip_input(c3d7c500) at ip_input+0x5a1
netisr_processqueue(c0983e98) at netisr_processqueue+0x6e
swi_net(0) at swi_net+0xc2
ithread_execute_handlers(c22d7224,c232c000) at ithread_execute_handlers+0xe6
ithread_loop(c229d790,e3685d38,c229d790,c062d02c,0) at ithread_loop+0x67
fork_exit(c062d02c,c229d790,e3685d38) at fork_exit+0xa4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xe3685d6c, ebp = 0 ---
db> show alllocks
Process 666 (tcp) thread 0xc2bf3000 (100133)
exclusive sleep mutex so_rcv r = 0 (0xc280ecf8) locked @ kern/uipc_socket.c:1256
Process 664 (udp) thread 0xc268b000 (100143)
exclusive sleep mutex rtentry r = 0 (0xc27b5dc8) locked @ net/route.c:1264
exclusive sleep mutex inp (udpinp) r = 0 (0xc27b3798) locked @ netinet/udp_usrreq.c:765
Process 15 (swi1: net) thread 0xc22d8d80 (100001)
exclusive sleep mutex so_snd r = 0 (0xc27c3c0c) locked @ kern/uipc_socket2.c:1205
exclusive sleep mutex inp (tcpinp) r = 0 (0xc27f0630) locked @ netinet/tcp_input.c:742
db> where 666
Tracing pid 666 tid 100133 td 0xc2bf3000
sched_switch(c2bf3000,0,1) at sched_switch+0x183
mi_switch(1,0) at mi_switch+0x274
sleepq_switch(c280ed2c,0,e770db40,c0645c9e,c280ed2c) at sleepq_switch+0xe0
sleepq_wait_sig(c280ed2c,0,100,c280ecf8,584) at sleepq_wait_sig+0xc
msleep(c280ed2c,c280ecf8,158,c0871b39,0) at msleep+0x302
sbwait(c280ecd4,c280ecf8,c280ecd4,0,c280ecf8) at sbwait+0x4b
soreceive(c280ec84,0,e770dc60,0,0) at soreceive+0x2da
soo_read(c2676708,e770dc60,c2a75e00,0,c2bf3000) at soo_read+0x41
dofileread(c2bf3000,4,c2676708,e770dc60,ffffffff) at dofileread+0x85
kern_readv(c2bf3000,4,e770dc60,804e400,400) at kern_readv+0x36
read(c2bf3000,e770dd04,c2bf3000,0,2) at read+0x45
syscall(2805003b,bfbf003b,bfbf003b,28050288,bfbfea88) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (3, FreeBSD ELF32, read), eip = 0x2813222b, esp = 0xbfbfe8ec, ebp = 0xbfbfe948 ---
db> where 664
Tracing pid 664 tid 100143 td 0xc268b000
ipi_nmi_handler(c0982b80,114,0,0,c0934ae0) at ipi_nmi_handler+0x47
trap(c2680008,e7590028,28,c268b000,c0933f90) at trap+0x3d
calltrap() at calltrap+0x5
--- trap 0x13, eip = 0xc0638162, esp = 0xe7593bc4, ebp = 0xe7593bd0 ---
_mtx_lock_sleep(c0933f90,c268b000,0,c086b45c,f5) at _mtx_lock_sleep+0xf2
_mtx_lock_flags(c0933f90,0,c086b45c,f5,c0933060,4,c086785f,54c) at _mtx_lock_flags+0x85
_sx_xunlock(c0933060,c086785f,54c,12e,c2bfd500) at _sx_xunlock+0x2d
falloc(c268b000,e7593c6c,e7593c70,c2bf966c,e7593d30) at falloc+0x214
socket(c268b000,e7593d04,c268b000,0,2) at socket+0x22
syscall(2805003b,bfbf003b,bfbf003b,28050288,bfbfea88) at syscall+0x27e
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (97, FreeBSD ELF32, socket), eip = 0x28113dcf, esp = 0xbfbfe90c, ebp = 0xbfbfe958 ---
db> show allpcpu
Current CPU: 0

cpuid        = 0
curthread    = 0xc22d8d80: pid 15 "swi1: net"
curpcb       = 0xe3685d90
fpcurthread  = none
idlethread   = 0xc22d8c00: pid 14 "idle: cpu0"
APIC ID      = 0
currentldt   = 0x50
spin locks held:

cpuid        = 1
curthread    = 0xc23da900: pid 39 "bufdaemon"
curpcb       = 0xe50f8d90
fpcurthread  = none
idlethread   = 0xc22d8a80: pid 13 "idle: cpu1"
APIC ID      = 1
currentldt   = 0x50
spin locks held:

cpuid        = 2
curthread    = 0xc268b000: pid 664 "udp"
curpcb       = 0xe7593d90
fpcurthread  = none
idlethread   = 0xc22d8900: pid 12 "idle: cpu2"
APIC ID      = 6
currentldt   = 0x50
spin locks held:

cpuid        = 3
curthread    = 0xc2bf2c00: pid 660 "udp"
curpcb       = 0xe7707d90
fpcurthread  = none
idlethread   = 0xc22d8780: pid 11 "idle: cpu3"
APIC ID      = 7
currentldt   = 0x50
spin locks held:

db> ps
  pid   proc     uid  ppid  pgrp  flag   stat  wmesg    wchan  cmd
  675 c28d066c 1001   666   615 0000002 [Can run] tcp
  674 c2bf1224 1001   667   615 0008082 (threaded)  thr1
   thread 0xc2c60600 ksegrp 0xc26a36c0 [SLPQ kserel 0xc26a36f4][SLP]
   thread 0xc2c60780 ksegrp 0xc26a36c0 [SLPQ kserel 0xc26a36f4][SLP]
   thread 0xc2c60900 ksegrp 0xc26a36c0 [SLPQ pfault 0xc0990c98][SLP]
   thread 0xc2617600 ksegrp 0xc22db9c0 [SLPQ ksesigwait 0xc2bf12b0][SLP]
   thread 0xc2724900 ksegrp 0xc26a36c0 [SLPQ kserel 0xc26a36f4][SLP]
  673 c2bf1448 1001   667   615 0008082 (threaded)  thr1
   thread 0xc28cd780 ksegrp 0xc26a3660 [SLPQ kserel 0xc26a3694][SLP]
   thread 0xc28cd900 ksegrp 0xc26a3660 [SLPQ kserel 0xc26a3694][SLP]
   thread 0xc28cda80 ksegrp 0xc26a3660 [SLPQ pfault 0xc0990c98][SLP]
   thread 0xc28cdc00 ksegrp 0xc2bf75a0 [SLPQ ksesigwait 0xc2bf14d4][SLP]
   thread 0xc2bf3780 ksegrp 0xc26a3660 [SLPQ kserel 0xc26a3694][SLP]
  672 c2bf166c 1001   667   615 0008082 (threaded)  thr1
   thread 0xc2c60300 ksegrp 0xc26a3600 [SLPQ kserel 0xc26a3634][SLP]
   thread 0xc2617000 ksegrp 0xc26a3600 [SLPQ kserel 0xc26a3634][SLP]
   thread 0xc2617180 ksegrp 0xc26a3600 [SLPQ kserel 0xc26a3634][SLP]
   thread 0xc2617300 ksegrp 0xc26a3600 [SLPQ pfault 0xc0990c98][SLP]
   thread 0xc2617480 ksegrp 0xc22dba20 [SLPQ ksesigwait 0xc2bf16f8][SLP]
   thread 0xc2bf3600 ksegrp 0xc26a3600 [SLPQ kserel 0xc26a3634][SLP]
  671 c2bf1890 1001   667   615 0008082 (threaded)  thr1
   thread 0xc2edea80 ksegrp 0xc26a35a0 [SLPQ pfault 0xc0990c98][SLP]
   thread 0xc2edec00 ksegrp 0xc26a35a0 [SLPQ kserel 0xc26a35d4][SLP]
   thread 0xc2eded80 ksegrp 0xc26a35a0 [SLPQ kserel 0xc26a35d4][SLP]
   thread 0xc2bfc480 ksegrp 0xc22db8a0 [SLPQ ksesigwait 0xc2bf191c][SLP]
   thread 0xc2bf3480 ksegrp 0xc26a35a0 [SLPQ kserel 0xc26a35d4][SLP]
  670 c2bf1ab4 1001   667   615 0008082 (threaded)  thr1
   thread 0xc2edf180 ksegrp 0xc26a3540 [SLPQ kserel 0xc26a3574][SLP]
   thread 0xc2edf300 ksegrp 0xc26a3540 [SLPQ pfault 0xc0990c98][SLP]
   thread 0xc2edf480 ksegrp 0xc26a3540 [SLPQ kserel 0xc26a3574][SLP]
   thread 0xc2bfc300 ksegrp 0xc22db900 [SLPQ ksesigwait 0xc2bf1b40][SLP]
   thread 0xc2bf3300 ksegrp 0xc26a3540 [SLPQ kserel 0xc26a3574][SLP]
  669 c2bf9224 1001   667   615 0008082 (threaded)  thr1
   thread 0xc2bf2180 ksegrp 0xc247dde0 [SLPQ kserel 0xc247de14][SLP]
   thread 0xc2bf2300 ksegrp 0xc247dde0 [SLPQ kserel 0xc247de14][SLP]
   thread 0xc2bf2480 ksegrp 0xc247dde0 [SLPQ pfault 0xc0990c98][SLP]
   thread 0xc2617c00 ksegrp 0xc2bf7600 [SLPQ ksesigwait 0xc2bf92b0][SLP]
   thread 0xc268b300 ksegrp 0xc247dde0 [SLPQ kserel 0xc247de14][SLP]
  668 c2bf9448 1001   667   615 0008082 (threaded)  thr1
   thread 0xc2bfc600 ksegrp 0xc247dd80 [SLPQ kserel 0xc247ddb4][SLP]
   thread 0xc2bfcd80 ksegrp 0xc247dd80 [SLPQ kserel 0xc247ddb4][SLP]
   thread 0xc2c60000 ksegrp 0xc247dd80 [SLPQ pfault 0xc0990c98][SLP]
   thread 0xc2c60180 ksegrp 0xc22db960 [SLPQ ksesigwait 0xc2bf94d4][SLP]
   thread 0xc268b180 ksegrp 0xc247dd80 [SLPQ kserel 0xc247ddb4][SLP]
  667 c2bf1cd8 1001   627   615 0000002 [SLPQ wait 0xc2bf1cd8][SLP][SWAP] thr1
  666 c2bf6000 1001   665   615 0000002 [SLPQ sbwait 0xc280ed2c][SLP] tcp
  665 c2bf6224 1001   633   615 0000002 [SLPQ wait 0xc2bf6224][SLP][SWAP] tcp
  664 c2bf966c 1001   658   615 0000002 [CPU 2] udp
  663 c2bf9890 1001   658   615 0000002 [Can run] udp
  662 c2bf9ab4 1001   658   615 0000002 [Can run] udp
  661 c2bf9cd8 1001   658   615 0000002 [Can run] udp
  660 c2bf6448 1001   658   615 0000002 [CPU 3] udp
  659 c2bf666c 1001   658   615 0000002 [Can run] udp
  658 c28d0224 1001   628   615 0000002 [SLPQ wait 0xc28d0224][SLP][SWAP] udp
  657 c2728224 1001   646   615 0000002 [SLPQ biord 0xd636c2dc][SLP] creat
  656 c28d0890 1001   646   615 0000002 [SLPQ getblk 0xd636c33c][SLP] creat
  655 c281266c 1001   646   615 0000002 [SLPQ getblk 0xd636c33c][SLP] creat
  654 c28d0448 1001   646   615 0000002 [SLPQ getblk 0xd636c33c][SLP] creat
  653 c2bfa000 1001   643   615 0000002 [SLPQ pfault 0xc0990c98][SLP] mkdir
  652 c2bfa224 1001   643   615 0000002 [SLPQ biord 0xd6361d28][SLP] mkdir
  651 c2bfa448 1001   643   615 0000002 [SLPQ getblk 0xd6361d88][SLP] mkdir
  650 c2bfa66c 1001   643   615 0000002 [SLPQ pfault 0xc0990c98][SLP] mkdir
  649 c2bfa890 1001   643   615 0000002 [SLPQ getblk 0xd6361d88][SLP] mkdir
  648 c2bfaab4 1001   643   615 0000002 [SLPQ pfault 0xc0990c98][SLP] mkdir
  647 c2bfacd8 1001   643   615 0000002 [SLPQ getblk 0xd6361d88][SLP] mkdir
  646 c2bf6890 1001   625   615 0000002 [SLPQ wait 0xc2bf6890][SLP][SWAP] creat
  645 c2bff000 1001   643   615 0000002 [SLPQ pfault 0xc0990c98][SLP] mkdir
  644 c2bff224 1001   643   615 0000002 [SLPQ getblk 0xd6361d88][SLP] mkdir
  643 c2812890 1001   626   615 0000002 [SLPQ wait 0xc2812890][SLP][SWAP] mkdir
  642 c2bf6ab4 1001   636   615 0000002 [SLPQ pfault 0xc0990c98][SLP] swap
  641 c2bf6cd8 1001   636   615 0000002 [SLPQ pfault 0xc0990c98][SLP] swap
  640 c26a4cd8 1001   636   615 0000002 [SLPQ pfault 0xc0990c98][SLP] swap
  639 c2728ab4 1001   636   615 0000002 [SLPQ pfault 0xc0990c98][SLP] swap
  638 c2722cd8 1001   636   615 0000002 [SLPQ pfault 0xc0990c98][SLP] swap
  637 c2722000 1001   636   615 0000002 [SLPQ pfault 0xc0990c98][SLP] swap
  636 c247bab4 1001   624   615 0000002 [SLPQ wait 0xc247bab4][SLP][SWAP] swap
  633 c26a4000 1001   622   615 0004002 [SLPQ nanslp 0xc0935464][SLP] tcp
  628 c2729ab4 1001   622   615 0004002 [SLPQ nanslp 0xc0935464][SLP] udp
  627 c268acd8 1001   622   615 0004002 [SLPQ nanslp 0xc0935464][SLP] thr1
  626 c26a1000 1001   622   615 0004002 [SLPQ nanslp 0xc0935464][SLP] mkdir
  625 c268a448 1001   622   615 0004002 [SLPQ nanslp 0xc0935464][SLP] creat
  624 c28cbcd8 1001   622   615 0004002 [SLPQ nanslp 0xc0935464][SLP] swap
  622 c247b66c 1001   621   615 0000002 [SLPQ wait 0xc247b66c][SLP][SWAP] run
  621 c247bcd8 1001   620   615 0000002 [SLPQ wait 0xc247bcd8][SLP][SWAP] run
  620 c26a1890 1001   615   615 0004002 [SLPQ nanslp 0xc0935464][SLP] run
  615 c2811000 1001   609   615 0004002 [SLPQ wait 0xc2811000][SLP][SWAP] sh
  609 c26a1224 1001   608   609 0004002 [SLPQ wait 0xc26a1224][SLP][SWAP] bash
  608 c2729890 1001   606   606 0000100 [SLPQ select 0xc0982784][SLP][SWAP] sshd
  606 c2812cd8    0   474   606 0004100 [SLPQ sbwait 0xc27b14d4][SLP][SWAP] sshd
  605 c2811cd8 1001   599   605 0004002 [SLPQ select 0xc0982784][SLP] top
  599 c2616448 1001   598   599 0004002 [SLPQ wait 0xc2616448][SLP][SWAP] bash
  598 c28d0000 1001   596   596 0000100 [SLPQ select 0xc0982784][SLP] sshd
  596 c2729000    0   474   596 0004100 [SLPQ sbwait 0xc27b6bc8][SLP][SWAP] sshd
  595 c28cbab4    0     1   595 0004002 [SLPQ ttyin 0xc24a6010][SLP][SWAP] getty
  594 c26a4890    0     1   594 0004002 [SLPQ ttyin 0xc24a6410][SLP][SWAP] getty
  593 c28cb000    0     1   593 0004002 [SLPQ ttyin 0xc24a6810][SLP][SWAP] getty
  592 c28cb224    0     1   592 0004002 [SLPQ ttyin 0xc24a6c10][SLP][SWAP] getty
  591 c28cb448    0     1   591 0004002 [SLPQ ttyin 0xc24a7010][SLP][SWAP] getty
  590 c28cb66c    0     1   590 0004002 [SLPQ ttyin 0xc24a7410][SLP][SWAP] getty
  589 c26a4448    0     1   589 0004002 [SLPQ ttyin 0xc24a7810][SLP][SWAP] getty
  588 c272866c    0     1   588 0004002 [SLPQ ttyin 0xc24a7c10][SLP][SWAP] getty
  586 c2728890    0   584    47 0004002 [SLPQ pfault 0xc0990c98][SLP] sleep
  585 c272966c    0     1    47 0004002 [SLPQ piperd 0xc27334c8][SLP] logger
  584 c2811ab4    0   583    47 0000002 [SLPQ wait 0xc2811ab4][SLP][SWAP] sh
  583 c2616cd8    0     1    47 0000002 [SLPQ wait 0xc2616cd8][SLP][SWAP] sh
  575 c26a1448    0     1   575 0000000 [SLPQ select 0xc0982784][SLP][SWAP] inetd
  554 c2728000    0     1   554 0000000 [SLPQ select 0xc0982784][SLP][SWAP] moused
  500 c26a166c    0     1   500 0000000 [SWAP] cron
  484 c2728cd8   25     1   484 0000100 [SLPQ pause 0xc2728d0c][SLP][SWAP] sendmail
  480 c26a4ab4    0     1   480 0000100 [SLPQ pfault 0xc0990c98][SLP] sendmail
  474 c261666c    0     1   474 0000100 [SLPQ select 0xc0982784][SLP][SWAP] sshd
  424 c26a466c    0     1   424 0000000 [SLPQ select 0xc0982784][SLP] usbd
  408 c2812224    0   403   403 0000000 [SLPQ - 0xc263e200][SLP][SWAP] nfsd
  406 c2812448    0   403   403 0000000 [SLPQ - 0xc27fde00][SLP][SWAP] nfsd
  405 c268a66c    0   403   403 0000000 [SLPQ - 0xc263e400][SLP][SWAP] nfsd
  404 c247b890    0   403   403 0000000 [SLPQ - 0xc27fe000][SLP][SWAP] nfsd
  403 c2616ab4    0     1   403 0000000 [SLPQ select 0xc0982784][SLP][SWAP] nfsd
  401 c2616890    0     1   401 0000000 [SLPQ select 0xc0982784][SLP][SWAP] mountd
  334 c2722448    0     1   334 0000000 [SLPQ pfault 0xc0990c98][SLP] rpcbind
  303 c2722224    0     1   303 0000000 [SLPQ select 0xc0982784][SLP][SWAP] devd
  228 c2616224    0     1   228 0000000 [SLPQ pfault 0xc0990c98][SLP] syslogd
   46 c2615000    0     0     0 0000204 [SLPQ - 0xe5338d04][SLP] schedcpu
   45 c2615224    0     0     0 0000204 [SLPQ - 0xc098ae0c][SLP] nfsiod 3
   44 c2615448    0     0     0 0000204 [SLPQ - 0xc098ae08][SLP] nfsiod 2
   43 c261566c    0     0     0 0000204 [SLPQ - 0xc098ae04][SLP] nfsiod 1
   42 c2615890    0     0     0 0000204 [SLPQ - 0xc098ae00][SLP] nfsiod 0
   41 c2615ab4    0     0     0 0000204 [SLPQ syncer 0xc0935150][SLP] syncer
   40 c2615cd8    0     0     0 0000204 [SLPQ vlruwt 0xc2615cd8][SLP] vnlru
   39 c2616000    0     0     0 0000204 [CPU 1] bufdaemon
   38 c23d9cd8    0     0     0 000020c [SLPQ pgzero 0xc0991424][SLP] pagezero
   37 c2479000    0     0     0 0000204 [SLPQ psleep 0xc0990f74][SLP] vmdaemon
   36 c2479224    0     0     0 0000204 [SLPQ wswbuf0 0xc09906d4][SLP] pagedaemon
   35 c2479448    0     0     0 0000204 [IWAIT] irq7: ppc0
   34 c247966c    0     0     0 0000204 [SLPQ - 0xc23d0e3c][SLP] fdc0
   33 c2479890    0     0     0 0000204 [IWAIT] swi0: sio
   32 c2479ab4    0     0     0 0000204 [IWAIT] irq12: psm0
   31 c2479cd8    0     0     0 0000204 [IWAIT] irq1: atkbd0
   30 c247b000    0     0     0 0000204 [IWAIT] irq15: ata1
   29 c247b224    0     0     0 0000204 [IWAIT] irq14: ata0
   28 c247b448    0     0     0 0000204 [IWAIT] irq17: fxp0
   27 c232e66c    0     0     0 0000204 [SLPQ usbtsk 0xc0932084][SLP] usbtask
   26 c232e890    0     0     0 0000204 [SLPQ usbevt 0xc23e0210][SLP] usb0
   25 c232eab4    0     0     0 0000204 [IWAIT] irq16: uhci0
   24 c232ecd8    0     0     0 0000204 [IWAIT] irq48: em0
   23 c23d9000    0     0     0 0000204 [IWAIT] irq9: acpi0
   22 c23d9224    0     0     0 0000204 [IWAIT] swi5: Fast taskq
    9 c23d9448    0     0     0 0000204 [SLPQ - 0xc232d400][SLP] thread taskq
    8 c23d966c    0     0     0 0000204 [SLPQ - 0xc232d480][SLP] acpi_task2
    7 c23d9890    0     0     0 0000204 [SLPQ - 0xc232d480][SLP] acpi_task1
    6 c23d9ab4    0     0     0 0000204 [SLPQ - 0xc232d480][SLP] acpi_task0
   21 c22dc224    0     0     0 0000204 [IWAIT] swi6: Giant taskq
   20 c22dc448    0     0     0 0000204 [IWAIT] swi6: task queue
    5 c22dc66c    0     0     0 0000204 [SLPQ - 0xc232d700][SLP] kqueue taskq
   19 c22dc890    0     0     0 0000204 [IWAIT] swi2: cambio
   18 c22dcab4    0     0     0 0000204 [SLPQ - 0xc092fda0][SLP] yarrow
    4 c22dccd8    0     0     0 0000204 [RUNQ] g_down
    3 c232e000    0     0     0 0000204 [RUNQ] g_up
    2 c232e224    0     0     0 0000204 [SLPQ - 0xc09328bc][SLP] g_event
   17 c232e448    0     0     0 0000204 [IWAIT] swi3: vm
   16 c22d7000    0     0     0 000020c [RUNQ] swi4: clock sio
   15 c22d7224    0     0     0 0000204 [CPU 0] swi1: net
   14 c22d7448    0     0     0 000020c [Can run] idle: cpu0
   13 c22d766c    0     0     0 000020c [Can run] idle: cpu1
   12 c22d7890    0     0     0 000020c [Can run] idle: cpu2
   11 c22d7ab4    0     0     0 000020c [Can run] idle: cpu3
    1 c22d7cd8    0     0     1 0004200 [SLPQ wait 0xc22d7cd8][SLP] init
   10 c22dc000    0     0     0 0000204 [SLPQ ktrace 0xc0933338][SLP] ktrace
    0 c09329c0    0     0     0 0000200 [SLPQ vmwait 0xc0990c98][SLP] swapper
  634 c26a1cd8 1001   622   615 0006002 zomb[INACTIVE] lockf
  630 c268a890 1001   622   615 0006002 zomb[INACTIVE] pty
  631 c2722ab4 1001   622   615 0006002 zomb[INACTIVE] syscall
  635 c26a4224 1001   622   615 0006002 zomb[INACTIVE] symlink
  632 c268a224 1001   622   615 0006002 zomb[INACTIVE] sysctl
  629 c2729224 1001   622   615 0006002 zomb[INACTIVE] mkfifo
db> call doadump
Dumping 1023 MB (2 chunks)
  chunk 0: 1MB (159 pages) ... ok
  chunk 1: 1023MB (261872 pages) 1007 991 975 959 943 927 911 895 879 863 847 831 815 799 783 767 751 735 719 703 687 671 655 639 623 607 591 575 559 543 527 511 495 479 463 447 431 415 399 383 367 351 335 319 303 287 271 255 239 223 207 191 175 159 143 127 111 95 79 63 47 31 15 ... ok

Dump complete
= 0xf
db> reset

(kgdb) bt
#0  doadump () at pcpu.h:165
#1  0xc0469d73 in db_fncall (dummy1=0xc09ae980, dummy2=0x0, dummy3=0x0, dummy4=0xe3685918 "DYhãÈ\027~À0Yhã4Yhã\220\a")
    at ../../../ddb/db_command.c:489
#2  0xc0469b78 in db_command (last_cmdp=0xc091b924, cmd_table=0x0, aux_cmd_tablep=0xc0896750, aux_cmd_tablep_end=0xc089676c)
    at ../../../ddb/db_command.c:404
#3  0xc0469c40 in db_command_loop () at ../../../ddb/db_command.c:455
#4  0xc046b859 in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:228
#5  0xc0659a10 in kdb_trap (type=0x3, code=0x0, tf=0xe3685ab0) at ../../../kern/subr_kdb.c:485
#6  0xc07ff774 in trap (frame=
      {tf_fs = 0xe3680008, tf_es = 0xc0650028, tf_ds = 0xc0860028, tf_edi = 0xc0869507, tf_esi = 0x1, tf_ebp = 0xe3685af0, tf_isp = 0xe3685adc, tf_ebx = 0xe3685b1c, tf_edx = 0x0, tf_ecx = 0xc1033000, tf_eax = 0x12, tf_trapno = 0x3, tf_err = 0x0, tf_eip = 0xc0659717, tf_cs = 0x20, tf_eflags = 0x296, tf_esp = 0xe3685b10, tf_ss = 0xc063fc0f}) at ../../../i386/i386/trap.c:610
#7  0xc07ec6aa in calltrap () at ../../../i386/i386/exception.s:139
#8  0xc0659717 in kdb_enter (msg=0x12 <Address 0x12 out of bounds>) at cpufunc.h:60
#9  0xc063fc0f in panic (fmt=0xc0869507 "%s: ref_cnt != 1") at ../../../kern/kern_shutdown.c:549
#10 0xc0636e45 in mb_dtor_pack (mem=0xc2464e00, size=0x100, arg=0x0) at ../../../kern/kern_mbuf.c:336
#11 0xc0790494 in uma_zfree_arg (zone=0xc1040d20, item=0xc2464e00, udata=0x0) at ../../../vm/uma_core.c:2270
#12 0xc0677de1 in mb_free_ext (m=0xc2464e00) at uma.h:303
#13 0xc067f46c in sbdrop_locked (sb=0xc27c3be8, len=0x400) at mbuf.h:397
#14 0xc067f572 in sbdrop (sb=0xc27c3be8, len=0x3c00) at ../../../kern/uipc_socket2.c:1206
#15 0xc06daacd in tcp_input (m=0xc3d7c500, off0=0x14) at ../../../netinet/tcp_input.c:1199
#16 0xc06d3e65 in ip_input (m=0xc3d7c500) at ../../../netinet/ip_input.c:778
#17 0xc06b203e in netisr_processqueue (ni=0xc0983e98) at ../../../net/netisr.c:236
#18 0xc06b2226 in swi_net (dummy=0x0) at ../../../net/netisr.c:349
#19 0xc062cf62 in ithread_execute_handlers (p=0xc22d7224, ie=0xc232c000) at ../../../kern/kern_intr.c:662
#20 0xc062d093 in ithread_loop (arg=0xc229d790) at ../../../kern/kern_intr.c:745
#21 0xc062c1f4 in fork_exit (callout=0xc062d02c <ithread_loop>, arg=0xc229d790, frame=0xe3685d38) at ../../../kern/kern_fork.c:789
#22 0xc07ec70c in fork_trampoline () at ../../../i386/i386/exception.s:208
(kgdb) f 13
#13 0xc067f46c in sbdrop_locked (sb=0xc27c3be8, len=0x400) at mbuf.h:397
397                     mb_free_ext(m);
(kgdb) info loc
m = (struct mbuf *) 0x0
next = (struct mbuf *) 0x0
(kgdb) p *sb
$8 = {sb_sel = {si_thrlist = {tqe_next = 0x0, tqe_prev = 0x0}, si_thread = 0x0, si_note = {kl_list = {slh_first = 0x0},
      kl_lock = 0xc0627d04 <knlist_mtx_lock>, kl_unlock = 0xc0627d20 <knlist_mtx_unlock>, kl_locked = 0xc0627d3c <knlist_mtx_locked>,
      kl_lockarg = 0xc27c3c0c}, si_flags = 0x0}, sb_mtx = {mtx_object = {lo_class = 0xc08d0d04, lo_name = 0xc086e702 "so_snd",
      lo_type = 0xc086e702 "so_snd", lo_flags = 0x30000, lo_list = {tqe_next = 0xc27c3b94, tqe_prev = 0xc27f06f4},
      lo_witness = 0xc0945df0}, mtx_lock = 0xc22d8d80, mtx_recurse = 0x0}, sb_state = 0x0, sb_mb = 0xc242c200, sb_mbtail = 0xc245f100,
  sb_lastrecord = 0xc242c200, sb_cc = 0x400, sb_hiwat = 0xa800, sb_mbcnt = 0x900, sb_mbmax = 0x40000, sb_ctl = 0x0, sb_lowat = 0x800,
  sb_timeo = 0x0, sb_flags = 0x1}
(kgdb) up
#14 0xc067f572 in sbdrop (sb=0xc27c3be8, len=0x3c00) at ../../../kern/uipc_socket2.c:1206
1206            sbdrop_locked(sb, len);
(kgdb) up
#15 0xc06daacd in tcp_input (m=0xc3d7c500, off0=0x14) at ../../../netinet/tcp_input.c:1199
1199                                    sbdrop(&so->so_snd, acked);
(kgdb) p *so
$9 = {so_count = 0x1, so_type = 0x1, so_options = 0x4, so_linger = 0x0, so_state = 0x2, so_qstate = 0x0, so_pcb = 0xc27f05a0,
  so_proto = 0xc08dc1e8, so_head = 0x0, so_incomp = {tqh_first = 0x0, tqh_last = 0xc27c3b3c}, so_comp = {tqh_first = 0x0,
    tqh_last = 0xc27c3b44}, so_list = {tqe_next = 0x0, tqe_prev = 0x0}, so_qlen = 0x0, so_incqlen = 0x0, so_qlimit = 0x0, so_timeo = 0x0,
  so_error = 0x0, so_sigio = 0x0, so_oobmark = 0x0, so_aiojobq = {tqh_first = 0x0, tqh_last = 0xc27c3b68}, so_rcv = {sb_sel = {
      si_thrlist = {tqe_next = 0x0, tqe_prev = 0x0}, si_thread = 0x0, si_note = {kl_list = {slh_first = 0x0},
        kl_lock = 0xc0627d04 <knlist_mtx_lock>, kl_unlock = 0xc0627d20 <knlist_mtx_unlock>, kl_locked = 0xc0627d3c <knlist_mtx_locked>,
        kl_lockarg = 0xc27c3b94}, si_flags = 0x0}, sb_mtx = {mtx_object = {lo_class = 0xc08d0d04, lo_name = 0xc086e709 "so_rcv",
        lo_type = 0xc086e709 "so_rcv", lo_flags = 0x30000, lo_list = {tqe_next = 0xc27f0630, tqe_prev = 0xc27c3c1c},
        lo_witness = 0xc0945dc8}, mtx_lock = 0x4, mtx_recurse = 0x0}, sb_state = 0x0, sb_mb = 0x0, sb_mbtail = 0x0, sb_lastrecord = 0x0,
    sb_cc = 0x0, sb_hiwat = 0x11800, sb_mbcnt = 0x0, sb_mbmax = 0x40000, sb_ctl = 0x0, sb_lowat = 0x1, sb_timeo = 0x0, sb_flags = 0x0},
  so_snd = {sb_sel = {si_thrlist = {tqe_next = 0x0, tqe_prev = 0x0}, si_thread = 0x0, si_note = {kl_list = {slh_first = 0x0},
        kl_lock = 0xc0627d04 <knlist_mtx_lock>, kl_unlock = 0xc0627d20 <knlist_mtx_unlock>, kl_locked = 0xc0627d3c <knlist_mtx_locked>,
        kl_lockarg = 0xc27c3c0c}, si_flags = 0x0}, sb_mtx = {mtx_object = {lo_class = 0xc08d0d04, lo_name = 0xc086e702 "so_snd",
        lo_type = 0xc086e702 "so_snd", lo_flags = 0x30000, lo_list = {tqe_next = 0xc27c3b94, tqe_prev = 0xc27f06f4},
        lo_witness = 0xc0945df0}, mtx_lock = 0xc22d8d80, mtx_recurse = 0x0}, sb_state = 0x0, sb_mb = 0xc242c200, sb_mbtail = 0xc245f100,
    sb_lastrecord = 0xc242c200, sb_cc = 0x400, sb_hiwat = 0xa800, sb_mbcnt = 0x900, sb_mbmax = 0x40000, sb_ctl = 0x0, sb_lowat = 0x800,
    sb_timeo = 0x0, sb_flags = 0x1}, so_upcall = 0, so_upcallarg = 0x0, so_cred = 0xc2a75e00, so_label = 0x0, so_peerlabel = 0x0,
  so_gencnt = 0x85e, so_emuldata = 0x0, so_accf = 0x0}