GENERIC 6.0-BETA1 from Jul 30 05:48 UTC, vmcore.438

Memory modified after free 0xc1b7e400(508) val=c1b7c5cc @ 0xc1b7e410
Fixed in src/sys/ufs/ufs/ufs_dirhash.c,v 1.22 2005/08/17 08:48:42 iedowse

GDB: no debug ports present
KDB: debugger backends: ddb
KDB: current backend: ddb
Copyright (c) 1992-2005 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
        The Regents of the University of California. All rights reserved.
FreeBSD 6.0-BETA1 #0: Sat Jul 30 08:13:44 CEST 2005
    pho@current.osted.lan:/usr/src/sys/i386/compile/PHO
WARNING: WITNESS option enabled, expect reduced performance.
ACPI APIC Table: <A M I  OEMAPIC >
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) Celeron(R) CPU 1.80GHz (1799.15-MHz 686-class CPU)
  Origin = "GenuineIntel"  Id = 0xf13  Stepping = 3
  Features=0x3febfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM>
real memory  = 267583488 (255 MB)
avail memory = 252194816 (240 MB)
:
Trying to mount root from ufs:/dev/ad0s1a
WARNING: / was not properly dismounted
WARNING: /tmp was not properly dismounted
/tmp: mount pending error: blocks 14204 files 40873
WARNING: /usr was not properly dismounted
WARNING: /var was not properly dismounted
rl0: link state changed to DOWN
lock order reversal
 1st 0xc097a9a0 UMA lock (UMA lock) @ vm/uma_core.c:1494
 2nd 0xc1060144 system map (system map) @ vm/vm_map.c:2317
KDB: stack backtrace:
kdb_backtrace(0,ffffffff,c092fab8,c092fbf8,c08ba4a4) at kdb_backtrace+0x29
witness_checkorder(c1060144,9,c0870d0f,90d) at witness_checkorder+0x564
_mtx_lock_flags(c1060144,0,c0870d06,90d) at _mtx_lock_flags+0x5b
_vm_map_lock(c10600c0,c0870d06,90d) at _vm_map_lock+0x26
vm_map_remove(c10600c0,c1846000,c1847000,cc9d4c08,c077eb2d) at vm_map_remove+0x1f
kmem_free(c10600c0,c1846000,1000,cc9d4c38,c077e4da) at kmem_free+0x25
page_free(c1846000,1000,2) at page_free+0x29
zone_drain(c103d000) at zone_drain+0x26a
zone_foreach(c077e270,cc9d4cec,c079035b,c1655300,cc9d4c74) at zone_foreach+0x37
uma_reclaim(c1655300,cc9d4c74,0,c09260e0,cc9d4c80) at uma_reclaim+0x12
vm_pageout_scan(0,c097ae00,0,c08721df,5c3) at vm_pageout_scan+0x103
vm_pageout(0,cc9d4d38,0,c0791114,0) at vm_pageout+0x2c3
fork_exit(c0791114,0,cc9d4d38) at fork_exit+0xa0
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xcc9d4d6c, ebp = 0 ---
Memory modified after free 0xc1b7e400(508) val=c1b7c5cc @ 0xc1b7e410
panic: Most recently used by UFS dirhash

cpuid = 0
KDB: enter: panic
[thread pid 628 tid 100089 ]
Stopped at      kdb_enter+0x2b: nop
db> where
Tracing pid 628 tid 100089 td 0xc17c6c00
kdb_enter(c0855d8b) at kdb_enter+0x2b
panic(c087045b,c086ef55,c087042c,c1b7e400,1fc) at panic+0x14b
mtrash_ctor(c1b7e400,200,0,101) at mtrash_ctor+0x4d
uma_zalloc_arg(c104a5a0,0,101) at uma_zalloc_arg+0x10f
malloc(14c,c08f4480,101,5300,53) at malloc+0xae
ufsdirhash_build(c1ad3294) at ufsdirhash_build+0x25e
ufs_lookup(cf33da7c) at ufs_lookup+0xf9
VOP_CACHEDLOOKUP_APV(c08f42c0,cf33da7c) at VOP_CACHEDLOOKUP_APV+0x7e
vfs_cache_lookup(cf33db14,c1ad2000,cf33dbc8,cf33db30,c067f3fe) at vfs_cache_lookup+0xb2
VOP_LOOKUP_APV(c08f42c0,cf33db14) at VOP_LOOKUP_APV+0x87
lookup(cf33dba0,0,0,c17c6c00,c1b7c550) at lookup+0x3d6
namei(cf33dba0,805b1a8,0,0,c178abb0) at namei+0x35a
kern_lstat(c17c6c00,805b1a8,0,cf33dc74) at kern_lstat+0x47
lstat(c17c6c00,cf33dd04,2,4,296) at lstat+0x1b
syscall(3b,3b,3b,805b148,805b100) at syscall+0x22f
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (190, FreeBSD ELF32, lstat), eip = 0x28122ad7, esp = 0xbfbfe8cc, ebp = 0xbfbfe968 ---
db> show alllocks
db> show lockedvnods
Locked vnodes

0xc1ad2000: tag ufs, type VDIR
    usecount 3, writecount 0, refcount 12 mountedhere 0
    flags ()
    v_object 0xc1b966b4 ref 0 pages 28
     lock type ufs: EXCL (count 1) by thread 0xc17c6c00 (pid 628)
        ino 75456, on dev ad0s1e
db> ps
  pid   proc     uid  ppid  pgrp  flag   stat  wmesg    wchan  cmd
  628 c1878a3c 1001   627   627 0004002 [CPU 0] find
  627 c1878000 1001   619   627 0004002 [SLPQ wait 0xc1878000][SLP] sh
  619 c1879418 1001   618   619 0004002 [SLPQ wait 0xc1879418][SLP] bash
  618 c1a3e20c 1001   616   616 0000100 [SLPQ select 0xc096c884][SLP] sshd
  616 c1a42000    0   433   616 0004100 [SLPQ sbwait 0xc1a3820c][SLP] sshd
  615 c1a3e000 1001   594   615 0004002 [SLPQ select 0xc096c884][SLP] top
  594 c1879624 1001   593   594 0004002 [SLPQ wait 0xc1879624][SLP] bash
  593 c1879830 1001   591   591 0000100 [SLPQ select 0xc096c884][SLP] sshd
  591 c1879c48    0   433   591 0004100 [SLPQ sbwait 0xc185ce90][SLP] sshd
  572 c1878c48    0     1   572 0004002 [SLPQ ttyin 0xc16ea810][SLP] getty
  571 c1875a3c    0     1   571 0004002 [SLPQ ttyin 0xc16eac10][SLP] getty
  570 c17c3830    0     1   570 0004002 [SLPQ ttyin 0xc16eb010][SLP] getty
  569 c187820c    0     1   569 0004002 [SLPQ ttyin 0xc16eb410][SLP] getty
  568 c1875624    0     1   568 0004002 [SLPQ ttyin 0xc16e0410][SLP] getty
  567 c1878830    0     1   567 0004002 [SLPQ ttyin 0xc16d7c10][SLP] getty
  566 c1875830    0     1   566 0004002 [SLPQ ttyin 0xc16dfc10][SLP] getty
  565 c1875000    0     1   565 0004002 [SLPQ ttyin 0xc16e0810][SLP] getty
  533 c1875c48    0     1   533 0000000 [SLPQ select 0xc096c884][SLP] moused
  457 c187520c    0     1   457 0000000 [SLPQ nanslp 0xc091f56c][SLP] cron
  445 c1878624   25     1   445 0000100 [SLPQ pause 0xc1878658][SLP] sendmail
  439 c17c7418    0     1   439 0000100 [SLPQ select 0xc096c884][SLP] sendmail
  433 c17c7c48    0     1   433 0000100 [SLPQ select 0xc096c884][SLP] sshd
  415 c165420c    0     1   415 0000000 [SLPQ select 0xc096c884][SLP] ntpd
  386 c17c320c    0     1   386 0000000 [SLPQ select 0xc096c884][SLP] usbd
  366 c17c7830    0   361   361 0000000 [SLPQ - 0xc176e200][SLP] nfsd
  365 c17c7624    0   361   361 0000000 [SLPQ - 0xc176e400][SLP] nfsd
  364 c17c3c48    0   361   361 0000000 [SLPQ - 0xc176e600][SLP] nfsd
  363 c17c720c    0   361   361 0000000 [SLPQ - 0xc176e800][SLP] nfsd
  361 c17c7000    0     1   361 0000000 [SLPQ select 0xc096c884][SLP] nfsd
  359 c17c3624    0     1   359 0000000 [SLPQ select 0xc096c884][SLP] mountd
  290 c17c3418    0     1   290 0000000 [SLPQ select 0xc096c884][SLP] rpcbind
  272 c17c7a3c    0     1   272 0000000 [SLPQ select 0xc096c884][SLP] syslogd
  243 c17c3a3c    0     1   243 0000000 [SLPQ select 0xc096c884][SLP] devd
   61 c1654624    0     0     0 0000204 [SLPQ - 0xcc9f2d04][SLP] schedcpu
   60 c1654830    0     0     0 0000204 [SLPQ - 0xc0974d6c][SLP] nfsiod 3
   59 c1654a3c    0     0     0 0000204 [SLPQ - 0xc0974d68][SLP] nfsiod 2
   58 c1654c48    0     0     0 0000204 [SLPQ - 0xc0974d64][SLP] nfsiod 1
   57 c1772000    0     0     0 0000204 [SLPQ - 0xc0974d60][SLP] nfsiod 0
   56 c177220c    0     0     0 0000204 [SLPQ vlruwt 0xc177220c][SLP] vnlru
   55 c1772418    0     0     0 0000204 [SLPQ syncer 0xc091f2e0][SLP] syncer
   54 c1772624    0     0     0 0000204 [SLPQ psleep 0xc096cdcc][SLP] bufdaemon
   53 c1772830    0     0     0 000020c [SLPQ pgzero 0xc097b324][SLP] pagezero
   52 c1772a3c    0     0     0 0000204 [SLPQ psleep 0xc097ae74][SLP] vmdaemon
   51 c1772c48    0     0     0 0000204 [SLPQ psleep 0xc097ae30][SLP] pagedaemon
   50 c15a7624    0     0     0 0000204 [SLPQ - 0xc16a083c][SLP] fdc0
   49 c15a7830    0     0     0 0000204 [IWAIT] swi0: sio
   48 c15a7a3c    0     0     0 0000204 [SLPQ usbevt 0xc1649210][SLP] usb4
   47 c15a7c48    0     0     0 0000204 [SLPQ usbevt 0xc1696210][SLP] usb3
   46 c1651000    0     0     0 0000204 [SLPQ usbevt 0xc1684210][SLP] usb2
   45 c165120c    0     0     0 0000204 [SLPQ usbevt 0xc1685210][SLP] usb1
   44 c1651418    0     0     0 0000204 [SLPQ usbtsk 0xc091a244][SLP] usbtask
   43 c1651624    0     0     0 0000204 [SLPQ usbevt 0xc165a210][SLP] usb0
    9 c1651830    0     0     0 0000204 [SLPQ - 0xc164e000][SLP] kqueue taskq
    8 c1651a3c    0     0     0 0000204 [SLPQ - 0xc164e080][SLP] acpi_task2
    7 c1651c48    0     0     0 0000204 [SLPQ - 0xc164e080][SLP] acpi_task1
    6 c1654000    0     0     0 0000204 [SLPQ - 0xc164e080][SLP] acpi_task0
   42 c1598c48    0     0     0 0000204 [IWAIT] swi2: cambio
   41 c15a5000    0     0     0 0000204 [IWAIT] swi5:+
    5 c15a520c    0     0     0 0000204 [SLPQ - 0xc164e300][SLP] thread taskq
   40 c15a5418    0     0     0 0000204 [IWAIT] swi6:+
   39 c15a5624    0     0     0 0000204 [IWAIT] swi6: task queue
   38 c15a5830    0     0     0 0000204 [SLPQ - 0xc0917f60][SLP] yarrow
    4 c15a5a3c    0     0     0 0000204 [SLPQ - 0xc091ca68][SLP] g_down
    3 c15a5c48    0     0     0 0000204 [SLPQ - 0xc091ca64][SLP] g_up
    2 c15a7000    0     0     0 0000204 [SLPQ - 0xc091ca5c][SLP] g_event
   37 c15a720c    0     0     0 0000204 [IWAIT] swi3: vm
   36 c15a7418    0     0     0 000020c [RUNQ] swi4: clock sio
   35 c158d624    0     0     0 0000204 [IWAIT] swi1: net
   34 c158d830    0     0     0 0000204 [IWAIT] irq23: ehci0
   33 c158da3c    0     0     0 0000204 [IWAIT] irq22: rl0
   32 c158dc48    0     0     0 0000204 [IWAIT] irq21:
   31 c1598000    0     0     0 0000204 [IWAIT] irq20:
   30 c159820c    0     0     0 0000204 [IWAIT] irq19: uhci1
   29 c1598418    0     0     0 0000204 [IWAIT] irq18: uhci2
   28 c1598624    0     0     0 0000204 [IWAIT] irq17: pcm0
   27 c1598830    0     0     0 0000204 [IWAIT] irq16: uhci0 uhci3
   26 c1598a3c    0     0     0 0000204 [IWAIT] irq15: ata1
   25 c155c20c    0     0     0 0000204 [IWAIT] irq14: ata0
   24 c155c418    0     0     0 0000204 [IWAIT] irq13:
   23 c155c624    0     0     0 0000204 [IWAIT] irq12: psm0
   22 c155c830    0     0     0 0000204 [IWAIT] irq11:
   21 c155ca3c    0     0     0 0000204 [IWAIT] irq10:
   20 c155cc48    0     0     0 0000204 [IWAIT] irq9: acpi0
   19 c158d000    0     0     0 0000204 [IWAIT] irq8:
   18 c158d20c    0     0     0 0000204 [IWAIT] irq7: ppc0
   17 c158d418    0     0     0 0000204 [IWAIT] irq6: fdc0
   16 c1557000    0     0     0 0000204 [IWAIT] irq5:
   15 c155720c    0     0     0 0000204 [IWAIT] irq4: sio0
   14 c1557418    0     0     0 0000204 [IWAIT] irq3:
   13 c1557624    0     0     0 0000204 [IWAIT] irq0:
   12 c1557830    0     0     0 0000204 [IWAIT] irq1: atkbd0
   11 c1557a3c    0     0     0 000020c [Can run] idle: cpu0
    1 c1557c48    0     0     1 0004200 [SLPQ wait 0xc1557c48][SLP] init
   10 c155c000    0     0     0 0000204 [SLPQ ktrace 0xc091d4b8][SLP] ktrace
    0 c091cb60    0     0     0 0000200 [IWAIT] swapper
db> call doadump
Dumping 254 MB (2 chunks)
  chunk 0: 1MB (159 pages) ... ok
  chunk 1: 255MB (65072 pages) 239 223 207 191 175 159 143 127 111 95 79 63 47 31 15 ... ok

Dump complete
= 0xf
db> reset

(kgdb) bt
#0  doadump () at pcpu.h:165
#1  0xc0468ff3 in db_fncall (dummy1=0x0, dummy2=0x0, dummy3=0x0, dummy4=0xcf33d734 "`3(\b}L3P3\220\a")
    at ../../../ddb/db_command.c:489
#2  0xc0468df8 in db_command (last_cmdp=0xc0903b44, cmd_table=0x0, aux_cmd_tablep=0xc0881024, aux_cmd_tablep_end=0xc0881040)
    at ../../../ddb/db_command.c:349
#3  0xc0468ec0 in db_command_loop () at ../../../ddb/db_command.c:455
#4  0xc046aa61 in db_trap (type=0x3, code=0x0) at ../../../ddb/db_main.c:221
#5  0xc0649fa8 in kdb_trap (type=0x3, code=0x0, tf=0xcf33d878) at ../../../kern/subr_kdb.c:473
#6  0xc07ed8f0 in trap (frame=
      {tf_fs = 0xcf330008, tf_es = 0xc0640028, tf_ds = 0xc0850028, tf_edi = 0xc087045b, tf_esi = 0x1, tf_ebp = 0xcf33d8b8, tf_isp = 0xcf33d8a4, tf_ebx = 0xcf33d8e4, tf_edx = 0x0, tf_ecx = 0xc1033000, tf_eax = 0x12, tf_trapno = 0x3, tf_err = 0x0, tf_eip = 0xc0649d0f, tf_cs = 0x20, tf_eflags = 0x286, tf_esp = 0xcf33d8d8, tf_ss = 0xc0632107}) at ../../../i386/i386/trap.c:601
#7  0xc07db3aa in calltrap () at ../../../i386/i386/exception.s:139
#8  0xcf330008 in ?? ()
#9  0xc0640028 in link_elf_preload_parse_symbols (ef=0xcf33d8e4) at ../../../kern/link_elf.c:342
#10 0xc0632107 in panic (fmt=---Can't read userspace from dump, or kernel process---

) at ../../../kern/kern_shutdown.c:547
#11 0xc078150d in mtrash_ctor (mem=0xc1b7e400, size=0x0, arg=0x0, flags=0x101) at ../../../vm/uma_dbg.c:138
#12 0xc077f803 in uma_zalloc_arg (zone=0xc104a5a0, udata=0x0, flags=0x101) at ../../../vm/uma_core.c:1859
#13 0xc0628602 in malloc (size=0x150, mtp=0xc08f4480, flags=0x101) at uma.h:275
#14 0xc0772536 in ufsdirhash_build (ip=0xc1ad3294) at ../../../ufs/ufs/ufs_dirhash.c:191
#15 0xc077412d in ufs_lookup (ap=0xcf33da7c) at ../../../ufs/ufs/ufs_lookup.c:191
#16 0xc07fcd1a in VOP_CACHEDLOOKUP_APV (vop=0x0, a=0xcf33da7c) at vnode_if.c:150
#17 0xc067b52e in vfs_cache_lookup (ap=0x0) at vnode_if.h:82
#18 0xc07fcc63 in VOP_LOOKUP_APV (vop=0xc08f4800, a=0xcf33db14) at vnode_if.c:99
#19 0xc067f3fe in lookup (ndp=0xcf33dba0) at vnode_if.h:56
#20 0xc067edc6 in namei (ndp=0xcf33dba0) at ../../../kern/vfs_lookup.c:201
#21 0xc068ade7 in kern_lstat (td=0xc17c6c00, path=0x0, pathseg=UIO_USERSPACE, sbp=0xcf33dc74) at ../../../kern/vfs_syscalls.c:2102
#22 0xc068ad83 in lstat (td=0xc17c6c00, uap=0xcf33dd04) at ../../../kern/vfs_syscalls.c:2086
#23 0xc07ee0db in syscall (frame=
      {tf_fs = 0x3b, tf_es = 0x3b, tf_ds = 0x3b, tf_edi = 0x805b148, tf_esi = 0x805b100, tf_ebp = 0xbfbfe968, tf_isp = 0xcf33dd64, tf_ebx = 0x2813d4e8, tf_edx = 0x805b100, tf_ecx = 0x0, tf_eax = 0xbe, tf_trapno = 0x0, tf_err = 0x2, tf_eip = 0x28122ad7, tf_cs = 0x33, tf_eflags = 0x296, tf_esp = 0xbfbfe8cc, tf_ss = 0x3b}) at ../../../i386/i386/trap.c:986
#24 0xc07db3ff in Xint0x80_syscall () at ../../../i386/i386/exception.s:200
(kgdb) x/20 0xc1b7e400
0xc1b7e400:     0xdeadc0de      0xdeadc0de      0xdeadc0de      0xdeadc0de
0xc1b7e410:     0xc1b7c5cc      0xdeadc0de      0xdeadc0de      0xdeadc0de
0xc1b7e420:     0xdeadc0de      0xdeadc0de      0xdeadc0de      0xdeadc0de
0xc1b7e430:     0xdeadc0de      0xdeadc0de      0xdeadc0de      0xdeadc0de
0xc1b7e440:     0xdeadc0de      0xdeadc0de      0xdeadc0de      0xdeadc0de
(kgdb) x/20 0xc1b7c5cc
0xc1b7c5cc:     0xc08ba4a4      0xc08595ae      0xc08595ae      0x00030000
0xc1b7c5dc:     0x00000000      0xc1b7e410      0xc092fba8      0x00000004
0xc1b7c5ec:     0x00000000      0xc1b7c5a8      0x00000000      0x00000000
0xc1b7c5fc:     0x00000100      0x00000000      0x00000000      0x00000000
0xc1b7c60c:     0xc1a772d8      0xc1b7c5cc      0x00000000      0xc1b7c614