newtcp1-newtcp# pciconf -lv hostb0@pci0:0:0:0: class=0x060000 card=0x514d8086 chip=0x29a08086 rev=0x02 hdr=0x00 vendor = 'Intel Corporation' device = 'P965/G965 Memory Controller Hub' class = bridge subclass = HOST-PCI vgapci0@pci0:0:2:0: class=0x030000 card=0x514d8086 chip=0x29a28086 rev=0x02 hdr=0x00 vendor = 'Intel Corporation' device = 'G965 Integrated Graphics Controller' class = display subclass = VGA none0@pci0:0:3:0: class=0x078000 card=0x514d8086 chip=0x29a48086 rev=0x02 hdr=0x00 vendor = 'Intel Corporation' device = 'P965/G965 HECI Controller' class = simple comms em0@pci0:0:25:0: class=0x020000 card=0x00018086 chip=0x104b8086 rev=0x02 hdr=0x00 vendor = 'Intel Corporation' device = '82566DC Gigabit Network Connection' class = network subclass = ethernet uhci0@pci0:0:26:0: class=0x0c0300 card=0x514d8086 chip=0x28348086 rev=0x02 hdr=0x00 vendor = 'Intel Corporation' device = '82801H (ICH8 Family) USB UHCI' class = serial bus subclass = USB uhci1@pci0:0:26:1: class=0x0c0300 card=0x514d8086 chip=0x28358086 rev=0x02 hdr=0x00 vendor = 'Intel Corporation' device = '82801H (ICH8 Family) USB UHCI' class = serial bus subclass = USB ehci0@pci0:0:26:7: class=0x0c0320 card=0x514d8086 chip=0x283a8086 rev=0x02 hdr=0x00 vendor = 'Intel Corporation' device = '82801H (ICH8 Family) USB2 EHCI' class = serial bus subclass = USB none1@pci0:0:27:0: class=0x040300 card=0x25048086 chip=0x284b8086 rev=0x02 hdr=0x00 vendor = 'Intel Corporation' device = '82801H (ICH8 Family) HD Audio Controller' class = multimedia pcib1@pci0:0:28:0: class=0x060400 card=0x283f8086 chip=0x283f8086 rev=0x02 hdr=0x01 vendor = 'Intel Corporation' device = '82801H (ICH8 Family) PCI Express Port 1' class = bridge subclass = PCI-PCI pcib2@pci0:0:28:1: class=0x060400 card=0x28418086 chip=0x28418086 rev=0x02 hdr=0x01 vendor = 'Intel Corporation' device = '82801H (ICH8 Family) PCI Express Port 2' class = bridge subclass = PCI-PCI pcib3@pci0:0:28:2: class=0x060400 card=0x28438086 chip=0x28438086 rev=0x02 hdr=0x01 vendor = 'Intel Corporation' device = '82801H (ICH8 Family) PCI Express Port 3' class = bridge subclass = PCI-PCI pcib4@pci0:0:28:3: class=0x060400 card=0x28458086 chip=0x28458086 rev=0x02 hdr=0x01 vendor = 'Intel Corporation' device = '82801H (ICH8 Family) PCI Express Port 4' class = bridge subclass = PCI-PCI pcib5@pci0:0:28:4: class=0x060400 card=0x28478086 chip=0x28478086 rev=0x02 hdr=0x01 vendor = 'Intel Corporation' device = '82801H (ICH8 Family) PCI Express Port 5' class = bridge subclass = PCI-PCI uhci2@pci0:0:29:0: class=0x0c0300 card=0x514d8086 chip=0x28308086 rev=0x02 hdr=0x00 vendor = 'Intel Corporation' device = '82801H (ICH8 Family) USB UHCI' class = serial bus subclass = USB uhci3@pci0:0:29:1: class=0x0c0300 card=0x514d8086 chip=0x28318086 rev=0x02 hdr=0x00 vendor = 'Intel Corporation' device = '82801H (ICH8 Family) USB UHCI' class = serial bus subclass = USB uhci4@pci0:0:29:2: class=0x0c0300 card=0x514d8086 chip=0x28328086 rev=0x02 hdr=0x00 vendor = 'Intel Corporation' device = '82801H (ICH8 Family) USB UHCI' class = serial bus subclass = USB ehci1@pci0:0:29:7: class=0x0c0320 card=0x514d8086 chip=0x28368086 rev=0x02 hdr=0x00 vendor = 'Intel Corporation' device = '82801H (ICH8 Family) USB2 EHCI' class = serial bus subclass = USB pcib6@pci0:0:30:0: class=0x060401 card=0x514d8086 chip=0x244e8086 rev=0xf2 hdr=0x01 vendor = 'Intel Corporation' device = '82801BA/CA/DB/DBL/EB/ER/FB/GB/HB (ICH2/3/4/4/5/5/6/7/8), 63xxESB Hub Interface to PCI Bridge' class = bridge subclass = PCI-PCI isab0@pci0:0:31:0: class=0x060100 card=0x514d8086 chip=0x28128086 rev=0x02 hdr=0x00 vendor = 'Intel Corporation' device = '82801HH (ICH8DH) LPC Interface Controller' class = bridge subclass = PCI-ISA atapci1@pci0:0:31:2: class=0x010601 card=0x514d8086 chip=0x28218086 rev=0x02 hdr=0x00 vendor = 'Intel Corporation' device = 'ICH8R (ICH8 Family) SATA AHCI Controller' class = mass storage none2@pci0:0:31:3: class=0x0c0500 card=0x514d8086 chip=0x283e8086 rev=0x02 hdr=0x00 vendor = 'Intel Corporation' device = '82801H (ICH8 Family) SMBus Controller' class = serial bus subclass = SMBus atapci0@pci0:2:0:0: class=0x01018f card=0x610111ab chip=0x610111ab rev=0xb1 hdr=0x00 vendor = 'Marvell Semiconductor (Was: Galileo Technology Ltd)' class = mass storage subclass = ATA em1@pci0:6:0:0: class=0x020000 card=0x13768086 chip=0x107c8086 rev=0x05 hdr=0x00 vendor = 'Intel Corporation' device = 'PRO/1000 GT' class = network subclass = ethernet fwohci0@pci0:6:3:0: class=0x0c0010 card=0x514d8086 chip=0x8023104c rev=0x00 hdr=0x00 vendor = 'Texas Instruments (TI)' device = 'TSB43AB21/A IEEE1394a-2000 OHCI PHY/Link-Layer Ctrlr' class = serial bus subclass = FireWire newtcp1-newtcp# cat /var/crash/info.0 Dump header from device /dev/ad12s1b Architecture: i386 Architecture Version: 2 Dump Length: 71716864B (68 MB) Blocksize: 512 Dumptime: Thu Oct 25 11:57:29 2007 Hostname: newtcp1-newtcp.caia.swin.edu.au Magic: FreeBSD Kernel Dump Version String: FreeBSD 7.0-PRERELEASE #0: Thu Oct 25 11:38:50 EST 2007 root@newtcp1-newtcp.caia.swin.edu.au:/usr/src/sys/i386/compile/NEWTCP-YANDEX Panic String: _mtx_lock_sleep: recursed on non-recursive mutex em0 @ ../../../dev/em/if_em.c:2415 Dump Parity: 1894384915 Bounds: 0 Dump Status: good newtcp1-newtcp# kgdb /usr/src/sys/i386/compile/NEWTCP-YANDEX/kernel.debug /var/crash/vmcore.0 [GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"] GNU gdb 6.1.1 [FreeBSD] Copyright 2004 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "i386-marcel-freebsd". Unread portion of the kernel message buffer: [snip] Copyright (c) 1992-2007 The FreeBSD Project. Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 The Regents of the University of California. All rights reserved. FreeBSD is a registered trademark of The FreeBSD Foundation. FreeBSD 7.0-PRERELEASE #0: Thu Oct 25 11:38:50 EST 2007 root@newtcp1-newtcp.caia.swin.edu.au:/usr/src/sys/i386/compile/NEWTCP-YANDEX WARNING: WITNESS option enabled, expect reduced performance. Timecounter "i8254" frequency 1193182 Hz quality 0 CPU: Intel(R) Core(TM)2 CPU 6320 @ 1.86GHz (1876.01-MHz 686-class CPU) Origin = "GenuineIntel" Id = 0x6f6 Stepping = 6 Features=0xbfebfbff Features2=0xe3bd AMD Features=0x20100000 AMD Features2=0x1 Cores per package: 2 real memory = 1055166464 (1006 MB) avail memory = 1019117568 (971 MB) ACPI APIC Table: FreeBSD/SMP: Multiprocessor System Detected: 2 CPUs cpu0 (BSP): APIC ID: 0 cpu1 (AP): APIC ID: 1 ioapic0: Changing APIC ID to 2 ioapic0 irqs 0-23 on motherboard kbd1 at kbdmux0 ath_hal: 0.9.20.3 (AR5210, AR5211, AR5212, RF5111, RF5112, RF2413, RF5413) acpi0: on motherboard acpi0: [ITHREAD] acpi0: Power Button (fixed) Timecounter "ACPI-fast" frequency 3579545 Hz quality 1000 acpi_timer0: <24-bit timer at 3.579545MHz> port 0x408-0x40b on acpi0 cpu0: on acpi0 est0: on cpu0 p4tcc0: on cpu0 cpu1: on acpi0 est1: on cpu1 p4tcc1: on cpu1 acpi_button0: on acpi0 pcib0: port 0xcf8-0xcff on acpi0 pci0: on pcib0 vgapci0: port 0x3410-0x3417 mem 0x50200000-0x502fffff,0x40000000-0x4fffffff irq 16 at device 2.0 on pci0 agp0: on vgapci0 agp0: detected 7676k stolen memory agp0: aperture size is 256M pci0: at device 3.0 (no driver attached) em0: port 0x30e0-0x30ff mem 0x50300000-0x5031ffff,0x50324000-0x50324fff irq 20 at device 25.0 on pci0 em0: Ethernet address: 00:19:d1:e1:17:d3 em0: [FILTER] uhci0: port 0x30c0-0x30df irq 16 at device 26.0 on pci0 uhci0: [GIANT-LOCKED] uhci0: [ITHREAD] usb0: on uhci0 usb0: USB revision 1.0 uhub0: on usb0 uhub0: 2 ports with 2 removable, self powered uhci1: port 0x30a0-0x30bf irq 21 at device 26.1 on pci0 uhci1: [GIANT-LOCKED] uhci1: [ITHREAD] usb1: on uhci1 usb1: USB revision 1.0 uhub1: on usb1 uhub1: 2 ports with 2 removable, self powered ehci0: mem 0x50325c00-0x50325fff irq 18 at device 26.7 on pci0 ehci0: [GIANT-LOCKED] ehci0: [ITHREAD] usb2: EHCI version 1.0 usb2: companion controllers, 2 ports each: usb0 usb1 usb2: on ehci0 usb2: USB revision 2.0 uhub2: on usb2 uhub2: 4 ports with 4 removable, self powered pci0: at device 27.0 (no driver attached) pcib1: at device 28.0 on pci0 pci1: on pcib1 pcib2: at device 28.1 on pci0 pci2: on pcib2 atapci0: port 0x2018-0x201f,0x2024-0x2027,0x2010-0x2017,0x2020-0x2023,0x2000-0x200f mem 0x50100000-0x501001ff irq 17 at device 0.0 on pci2 atapci0: [ITHREAD] ata2: on atapci0 ata2: [ITHREAD] pcib3: at device 28.2 on pci0 pci3: on pcib3 pcib4: at device 28.3 on pci0 pci4: on pcib4 pcib5: at device 28.4 on pci0 pci5: on pcib5 uhci2: port 0x3080-0x309f irq 23 at device 29.0 on pci0 uhci2: [GIANT-LOCKED] uhci2: [ITHREAD] usb3: on uhci2 usb3: USB revision 1.0 uhub3: on usb3 uhub3: 2 ports with 2 removable, self powered uhci3: port 0x3060-0x307f irq 19 at device 29.1 on pci0 uhci3: [GIANT-LOCKED] uhci3: [ITHREAD] usb4: on uhci3 usb4: USB revision 1.0 uhub4: on usb4 uhub4: 2 ports with 2 removable, self powered uhci4: port 0x3040-0x305f irq 18 at device 29.2 on pci0 uhci4: [GIANT-LOCKED] uhci4: [ITHREAD] usb5: on uhci4 usb5: USB revision 1.0 uhub5: on usb5 uhub5: 2 ports with 2 removable, self powered ehci1: mem 0x50325800-0x50325bff irq 23 at device 29.7 on pci0 ehci1: [GIANT-LOCKED] ehci1: [ITHREAD] usb6: EHCI version 1.0 usb6: companion controllers, 2 ports each: usb3 usb4 usb5 usb6: on ehci1 usb6: USB revision 2.0 uhub6: on usb6 uhub6: 6 ports with 6 removable, self powered pcib6: at device 30.0 on pci0 pci6: on pcib6 em1: port 0x1000-0x103f mem 0x50020000-0x5003ffff,0x50000000-0x5001ffff irq 21 at device 0.0 on pci6 em1: Ethernet address: 00:1b:21:03:64:99 em1: [FILTER] fwohci0: mem 0x50044000-0x500447ff,0x50040000-0x50043fff irq 19 at device 3.0 on pci6 fwohci0: [FILTER] fwohci0: OHCI version 1.10 (ROM=0) fwohci0: No. of Isochronous channels is 4. fwohci0: EUI64 00:90:27:00:01:c7:e5:b0 fwohci0: Phy 1394a available S400, 2 ports. fwohci0: Link S400, max_rec 2048 bytes. firewire0: on fwohci0 fwe0: on firewire0 if_fwe0: Fake Ethernet address: 02:90:27:c7:e5:b0 fwe0: Ethernet address: 02:90:27:c7:e5:b0 fwip0: on firewire0 fwip0: Firewire address: 00:90:27:00:01:c7:e5:b0 @ 0xfffe00000000, S400, maxrec 2048 sbp0: on firewire0 dcons_crom0: on firewire0 dcons_crom0: bus_addr 0x125c000 fwohci0: Initiate bus reset fwohci0: BUS reset fwohci0: node_id=0xc800ffc0, gen=1, CYCLEMASTER mode isab0: at device 31.0 on pci0 isa0: on isab0 atapci1: port 0x3408-0x340f,0x341c-0x341f,0x3400-0x3407,0x3418-0x341b,0x3020-0x303f mem 0x50325000-0x503257ff irq 19 at device 31.2 on pci0 atapci1: [ITHREAD] atapci1: AHCI Version 01.10 controller with 6 ports detected ata3: on atapci1 ata3: [ITHREAD] ata4: on atapci1 ata4: [ITHREAD] ata5: on atapci1 ata5: [ITHREAD] ata6: on atapci1 ata6: [ITHREAD] ata7: on atapci1 ata7: [ITHREAD] ata8: on atapci1 ata8: [ITHREAD] pci0: at device 31.3 (no driver attached) atkbdc0: port 0x60,0x64 irq 1 on acpi0 atkbd0: irq 1 on atkbdc0 kbd0 at atkbd0 atkbd0: [GIANT-LOCKED] atkbd0: [ITHREAD] sio0: <16550A-compatible COM port> port 0x3f8-0x3ff irq 4 flags 0x10 on acpi0 sio0: type 16550A sio0: [FILTER] pmtimer0 on isa0 orm0: at iomem 0xcd800-0xce7ff pnpid ORM0000 on isa0 ata0 at port 0x1f0-0x1f7,0x3f6 irq 14 on isa0 ata0: [ITHREAD] ata1 at port 0x170-0x177,0x376 irq 15 on isa0 ata1: [ITHREAD] ppc0: at port 0x378-0x37f irq 7 on isa0 ppc0: SMC-like chipset (ECP/EPP/PS2/NIBBLE) in COMPATIBLE mode ppc0: FIFO with 16/16/8 bytes threshold ppbus0: on ppc0 ppi0: on ppbus0 plip0: on ppbus0 lpt0: on ppbus0 lpt0: Interrupt-driven port ppc0: [GIANT-LOCKED] ppc0: [ITHREAD] sc0: at flags 0x100 on isa0 sc0: VGA <16 virtual consoles, flags=0x300> sio1: configured irq 3 not in bitmap of probed irqs 0 sio1: port may not be enabled vga0: at port 0x3c0-0x3df iomem 0xa0000-0xbffff on isa0 Timecounters tick every 1.000 msec firewire0: 1 nodes, maxhop <= 0, cable IRM = 0 (me) firewire0: bus manager 0 (me) acquiring duplicate lock of same type: "network driver" 1st em0 @ dev/em/if_em.c:1515 2nd em0 @ dev/em/if_em.c:1516 KDB: stack backtrace: db_trace_self_wrapper(c0a9e41f,e241ec38,c078611e,c0aa07a9,c37e4390,...) at db_trace_self_wrapper+0x26 kdb_backtrace(c0aa07a9,c37e4390,c0a6f801,5ec,c0a9a752,...) at kdb_backtrace+0x29 witness_checkorder(c37a5ad4,9,c0a6f7f8,5ec,c0a9ca8e,...) at witness_checkorder+0x6de _mtx_lock_flags(c37a5ad4,0,c0a6f7f8,5ec,c37a5abc,...) at _mtx_lock_flags+0xbc em_handle_link(c37a5800,1,c0a9f611,50,c37a2b9c,...) at em_handle_link+0x63 taskqueue_run(c37a2b80,e241ecf8,c0733615,0,e241ecdc,...) at taskqueue_run+0x10b taskqueue_fast_run(0,e241ecdc,202,c0bb2ba0,c37a2b64,...) at taskqueue_fast_run+0x13 ithread_loop(c37a31c0,e241ed38,c0a9839b,314,c36c0aa0,...) at ithread_loop+0x1b5 fork_exit(c0733460,c37a31c0,e241ed38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe241ed70, ebp = 0 --- acd0: DMA limited to UDMA33, device found non-ATA66 cable acd0: DVDR at ata2-master UDMA33 ad12: 238475MB at ata6-master SATA150 SMP: AP CPU #1 Launched! WARNING: WITNESS option enabled, expect reduced performance. Trying to mount root from ufs:/dev/ad12s2a <118>Loading configuration files. <118>kernel dumps on /dev/ad12s1b <118>Entropy harvesting: <118> interrupts <118> ethernet <118> point_to_point <118> kickstart <118>. <118>swapon: adding /dev/ad12s1b as swap device <118>Starting file system checks: <118>/dev/ad12s2a: FILE SYSTEM CLEAN; SKIPPING CHECKS <118>/dev/ad12s2a: clean, 80417 free (1257 frags, 9895 blocks, 0.5% fragmentation) <118>/dev/ad12s2e: FILE SYSTEM CLEAN; SKIPPING CHECKS <118>/dev/ad12s2e: clean, 253807 free (31 frags, 31722 blocks, 0.0% fragmentation) <118>/dev/ad12s2f: FILE SYSTEM CLEAN; SKIPPING CHECKS <118>/dev/ad12s2f: clean, 24769438 free (38190 frags, 3091406 blocks, 0.1% fragmentation) <118>/dev/ad12s2d: FILE SYSTEM CLEAN; SKIPPING CHECKS <118>/dev/ad12s2d: clean, 991101 free (309 frags, 123849 blocks, 0.0% fragmentation) <118>Setting hostuuid: b024253a-0c28-11dc-92fa-00e018a3297b. <118>Setting hostid: 0x95afd096. <118>Mounting local file systems: <118>. <118>Setting hostname: newtcp1-newtcp.caia.swin.edu.au. <118>net.inet6.ip6.auto_linklocal: <118>1 <118> -> <118>0 <118> <118>kern.ipc.maxsockbuf: <118>262144 <118> -> <118>104857600 <118> <118>net.inet.tcp.inflight.enable: <118>1 <118> -> <118>0 <118> <118>net.inet.tcp.hostcache.expire: <118>3600 <118> -> <118>1 <118> <118>net.inet.tcp.recvbuf_auto: <118>1 <118> -> <118>0 <118> <118>net.inet.tcp.sendbuf_auto: <118>1 <118> -> <118>0 <118> <118>net.inet.tcp.tso: <118>1 <118> -> <118>0 <118> <118>net.inet.tcp.hostcache.prune: <118>300 <118> -> <118>5 <118> <118>net.inet.ip.fastforwarding: <118>0 <118> -> <118>1 <118> <118>lo0: flags=8049 metric 0 mtu 16384 <118> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x6 <118> inet6 ::1 prefixlen 128 <118> inet 127.0.0.1 netmask 0xff000000 <118>em0: flags=8843 metric 0 mtu 1500 <118> options=18b <118> ether 00:19:d1:e1:17:d3 <118> inet 136.186.229.190 netmask 0xffffff00 broadcast 136.186.229.255 <118> media: Ethernet autoselect <118> status: no carrier <118>em1: flags=8843 metric 0 mtu 1500 <118> options=8b <118> ether 00:1b:21:03:64:99 <118> inet 172.16.10.2 netmask 0xffffff00 broadcast 172.16.10.255 <118> media: Ethernet 1000baseTX (autoselect) <118> status: no carrier <118>add net default: gateway 136.186.229.1 <118>add net 172.16.11.0: gateway 172.16.10.1 <118>Additional routing options: <118>. <118>Starting devd. panic: _mtx_lock_sleep: recursed on non-recursive mutex em0 @ ../../../dev/em/if_em.c:2415 cpuid = 0 KDB: enter: panic [snip] (kgdb) backtrace #0 doadump () at pcpu.h:195 #1 0xc074e99e in boot (howto=260) at ../../../kern/kern_shutdown.c:409 #2 0xc074ec5b in panic (fmt=Variable "fmt" is not available. ) at ../../../kern/kern_shutdown.c:563 #3 0xc048cca7 in db_panic (addr=Could not find the frame base for "db_panic". ) at ../../../ddb/db_command.c:433 #4 0xc048d695 in db_command_loop () at ../../../ddb/db_command.c:401 #5 0xc048ee05 in db_trap (type=3, code=0) at ../../../ddb/db_main.c:222 #6 0xc07758f6 in kdb_trap (type=3, code=0, tf=0xe2403b94) at ../../../kern/subr_kdb.c:502 #7 0xc0a053ff in trap (frame=0xe2403b94) at ../../../i386/i386/trap.c:621 #8 0xc09eb81b in calltrap () at ../../../i386/i386/exception.s:139 #9 0xc0775a72 in kdb_enter (msg=0xc0a9b9c2 "panic") at cpufunc.h:60 #10 0xc074ec44 in panic (fmt=0xc0a9a8ce "_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n") at ../../../kern/kern_shutdown.c:547 #11 0xc0743347 in _mtx_lock_sleep (m=0xc37a5ad4, tid=3278625360, opts=0, file=0xc0a6f7f8 "../../../dev/em/if_em.c", line=2415) at ../../../kern/kern_mutex.c:310 #12 0xc074356f in _mtx_lock_flags (m=0xc37a5ad4, opts=0, file=0xc0a6f7f8 "../../../dev/em/if_em.c", line=2415) at ../../../kern/kern_mutex.c:186 #13 0xc0574020 in em_local_timer (arg=0xc37a5800) at ../../../dev/em/if_em.c:2415 #14 0xc0760706 in softclock (dummy=0x0) at ../../../kern/kern_timeout.c:274 #15 0xc0733615 in ithread_loop (arg=0xc36bb2b0) at ../../../kern/kern_intr.c:1036 #16 0xc0730ad8 in fork_exit (callout=0xc0733460 , arg=0xc36bb2b0, frame=0xe2403d38) at ../../../kern/kern_fork.c:796 #17 0xc09eb890 in fork_trampoline () at ../../../i386/i386/exception.s:205 (kgdb)