Happened on yesterday's 5.3-STABLE. It is a mount from SPARC/Solaris serving a ~600GB raw disk to FreeBSD. On the FreeBSD side were 10 iozone processes looping in automated mode (-a). Iozone runs fine up to 512K block until the server dies terrible death. After reboot the filesystem (UFS2 with softupdates) cannot be mounted or checked: ravell# fsck -t ufs /dev/ggate0 ** /dev/ggate0 Cannot find file system superblock ioctl (GCINFO): Inappropriate ioctl for device fsck_ufs: /dev/ggate0: can't read disk label ravell# dumpfs /dev/ggate0 dumpfs: /dev/ggate0: could not read superblock to fill out disk I attach some routine DDB checks. Kernel crash file available upon request (4GB download). As a side note, would it be hard to eliminate userland pages from the crashdump to both increase secrecy and shorten crashfile/dump duration? Can be done in both Solaris and Tru64, would be good to have in FreeBSD too. Regards, /S Fatal trap 12: page fault while in kernel mode cpuid = 2; apic id = 06 fault virtual address = 0x8a8a9304 fault code = supervisor read, page not present instruction pointer = 0x8:0xc05a1913 stack pointer = 0x10:0xf3d1793c frame pointer = 0x10:0xf3d17978 code segment = base 0x0, limit 0xfffff, type 0x1b = DPL 0, pres 1, def32 1, gran 1 processor eflags = interrupt enabled, resume, IOPL = 0 current process = 62173 (iozone) [thread pid 62173 tid 100279 ] Stopped at ffs_alloccg+0x1b: cmpl $0,0x4(%eax,%edx,1) db> db> doadump No such command db> call doadump Dumping 3839 MB 16 32 48 64 80 96 112 128 144 160 176 192 208 224 240 256 272 288 304 320 336 352 368 384 400 416 432 448 464 480 496 512 5284 Dump complete 0xf db> trace Tracing pid 62173 tid 100279 td 0xc6206e10 ffs_alloccg(c6c46dac,cc304030,a5a5a5ad,a5a5a5a5,4000) at ffs_alloccg+0x1b ffs_hashalloc(c6c46dac,cc304030,a5a5a5ad,a5a5a5a5,4000) at ffs_hashalloc+0x28 ffs_alloc(c6c46dac,20d,0,a5a5a5ad,a5a5a5a5) at ffs_alloc+0x175 ffs_balloc_ufs2(c6d73108,834000,0,4000,c64eb800) at ffs_balloc_ufs2+0x14b7 ffs_write(f3d17c10) at ffs_write+0x303 vn_write(c77dce58,f3d17c88,c64eb800,0,c6206e10) at vn_write+0x210 dofilewrite(c6206e10,c77dce58,3,8300000,10000) at dofilewrite+0xa8 write(c6206e10,f3d17d14,3,6eb5,292) at write+0x39 syscall(2f,2f,bfbf002f,0,10000) at syscall+0x283 Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (4, FreeBSD ELF32, write), eip = 0x280e85f3, esp = 0xbfbfe5dc, ebp = 0xbfbfe6f8 --- db> ps pid proc uarea uid ppid pgrp flag stat wmesg wchan cmd 62346 c62608d4 f3dbf000 0 62146 62346 0025002 stop[SUSP] vi 62174 c62601c4 f3d61000 0 62166 62166 0004002 [LOCK Giant c5382540] iozone 62173 c6204710 f3cdb000 0 62163 62163 0004002 [CPU 2] iozone 62172 c61b9a98 f3cbe000 0 62160 62160 0004002 [LOCK Giant c5382540] iozone 62171 c67d0388 f3e60000 0 62159 62159 0004002 [LOCK Giant c5382540] iozone 62170 c5655710 f3b5d000 0 62164 62164 0004002 [LOCK Giant c5382540] iozone 62169 c67d01c4 f3e5f000 0 62167 62167 0004002 [LOCK Giant c5382540] iozone 62168 c547154c f3ac6000 0 62165 62165 0004002 [LOCK Giant c5382540] iozone 62167 c54658d4 f3a9f000 0 62146 62167 0000002 [SLPQ pause 0xc546590c][SLP] zsh 62166 c5655e20 f3b61000 0 62146 62166 0000002 [SLPQ pause 0xc5655e58][SLP] zsh 62165 c627ba98 f3e17000 0 62146 62165 0000002 [SLPQ pause 0xc627bad0][SLP] zsh 62164 c67cde20 f3e54000 0 62146 62164 0000002 [SLPQ pause 0xc67cde58][SLP] zsh 62163 c627b000 f3d4c000 0 62146 62163 0000002 [SLPQ pause 0xc627b038][SLP] zsh 62162 c565ac5c f3bcc000 0 62157 62157 0004002 [LOCK Giant c5382540] iozone 62161 c6204c5c f3d1a000 0 62156 62156 0004002 [LOCK Giant c5382540] iozone 62160 c6204388 f3cd9000 0 62146 62160 0000002 [SLPQ pause 0xc62043c0][SLP] zsh 62159 c67d0000 f3e5e000 0 62146 62159 0000002 [SLPQ pause 0xc67d0038][SLP] zsh 62158 c67cd54c f3e4f000 0 62155 62155 0004002 [LOCK Giant c5382540] iozone 62157 c54691c4 f3aa4000 0 62146 62157 0000002 [SLPQ pause 0xc54691fc][SLP] zsh 62156 c62041c4 f3cd8000 0 62146 62156 0000002 [SLPQ pause 0xc62041fc][SLP] zsh 62155 c56551c4 f3b58000 0 62146 62155 0000002 [SLPQ pause 0xc56551fc][SLP] zsh 62146 c6207a98 f3d22000 0 62145 62146 0004002 [SLPQ ttyin 0xc5bcdc10][SLP] zsh 62145 c59fba98 f3c09000 0 1 62145 0000000 [SLPQ select 0xc06a7844][SLP] screen 62121 c625e8d4 f3d5c000 0 1 62121 0000000 [LOCK Giant c5382540] ggatec 55230 c61ba710 f3cc5000 0 1 55227 0000002 [SLPQ bpf 0xc65b4400][SLP] arpwatch 46756 c59fbe20 f3c0b000 25 1 46756 0000100 [SLPQ pause 0xc59fbe58][SLP] sendmail 45868 c67cd000 f3d37000 25 1 45868 0000100 [SLPQ pause 0xc67cd038][SLP] sendmail 45860 c54711c4 f3ac4000 0 1 45860 0000100 [SLPQ select 0xc06a7844][SLP] sendmail 39732 c627a388 f3d45000 0 39730 39732 0004002 [SLPQ ttyin 0xc560e610][SLP] zsh 39730 c59fa1c4 f3bd2000 0 401 39730 0000100 [SLPQ select 0xc06a7844][SLP] sshd 13920 c5465a98 f3aa0000 0 13856 13920 0004002 [SLPQ ttyin 0xc733f010][SLP] zsh 13856 c627b54c f3d4f000 0 401 13856 0000100 [SLPQ select 0xc06a7844][SLP] sshd 43780 c6204a98 f3d19000 0 1 43780 0000000 [SLPQ select 0xc06a7844][SLP] ntpd 9097 c546554c f3a9d000 0 9095 9097 0004002 [SLPQ ttyin 0xc53d5a10][SLP] zsh 9095 c61b9c5c f3cbf000 0 401 9095 0000100 [SLPQ select 0xc06a7844][SLP] sshd 527 c5469710 f3aa7000 70 526 519 0000002 [SLPQ select 0xc06a7844][SLP] postgres 526 c54bfc5c f3b38000 70 523 519 0000002 [SLPQ select 0xc06a7844][SLP] postgres 525 c54bfe20 f3b39000 70 523 519 0000002 [SLPQ select 0xc06a7844][SLP] postgres 523 c59fa710 f3bd5000 70 1 519 0004002 [SLPQ select 0xc06a7844][SLP] postgres 467 c546954c f3aa6000 0 1 467 0004002 [SLPQ ttyin 0xc53d5410][SLP] getty 466 c5656e20 f3bc4000 0 1 466 0004002 [SLPQ ttyin 0xc51a2a10][SLP] getty 465 c54c31c4 f3b3b000 0 1 465 0004002 [SLPQ ttyin 0xc5055210][SLP] getty 464 c5655a98 f3b5f000 0 1 464 0004002 [SLPQ ttyin 0xc5055410][SLP] getty 463 c50d7710 e908f000 0 1 463 0004002 [SLPQ ttyin 0xc5055610][SLP] getty 462 c5469e20 f3aab000 0 1 462 0004002 [SLPQ ttyin 0xc5055810][SLP] getty 461 c54c3388 f3b3c000 0 1 461 0004002 [SLPQ ttyin 0xc5055a10][SLP] getty 460 c54651c4 f3a9b000 0 1 460 0004002 [SLPQ ttyin 0xc5055c10][SLP] getty 459 c544ae20 f3a51000 0 1 459 0004002 [SLPQ ttyin 0xc5153010][SLP] getty 414 c565a000 f3bc5000 0 1 414 0000000 [SLPQ nanslp 0xc06a20ec][SLP] cron 401 c54708d4 f3abf000 0 1 401 0000100 [SLPQ select 0xc06a7844][SLP] sshd 292 c54c3c5c f3b41000 0 1 292 0000000 [SLPQ select 0xc06a7844][SLP] syslogd 272 c5469c5c f3aaa000 0 1 272 0000000 [SLPQ select 0xc06a7844][SLP] devd 86 c50d7e20 e9093000 0 0 0 0000204 [SLPQ - 0xe902dd18][SLP] schedcpu 85 c5385000 f17b2000 0 0 0 0000204 [SLPQ - 0xc06aa34c][SLP] nfsiod 3 84 c53851c4 f17d1000 0 0 0 0000204 [SLPQ - 0xc06aa348][SLP] nfsiod 2 83 c5385388 f17d2000 0 0 0 0000204 [SLPQ - 0xc06aa344][SLP] nfsiod 1 82 c538554c f17d3000 0 0 0 0000204 [SLPQ - 0xc06aa340][SLP] nfsiod 0 81 c5385710 f17d4000 0 0 0 0000204 [SLPQ syncer 0xc06a1e6c][SLP] syncer 80 c53858d4 f17d5000 0 0 0 0000204 [SLPQ vlruwt 0xc53858d4][SLP] vnlru 79 c5385a98 f17d6000 0 0 0 0000204 [SLPQ psleep 0xc06a7e0c][SLP] bufdaemon 78 c5385c5c f17d7000 0 0 0 000020c [SLPQ pgzero 0xc06b0c54][SLP] pagezero 77 c5385e20 f17d8000 0 0 0 0000204 [SLPQ psleep 0xc06b0ca8][SLP] vmdaemon 76 c5387000 f17d9000 0 0 0 0000204 [SLPQ psleep 0xc06b0c64][SLP] pagedaemon 75 c508ea98 e9043000 0 0 0 0000204 [IWAIT] swi0: sio 74 c508ec5c e9044000 0 0 0 0000204 [SLPQ - 0xc509b53c][SLP] fdc0 73 c508ee20 e9045000 0 0 0 0000204 [SLPQ aifthd 0xc508ee20][SLP] aac0aif 9 c50d3000 e9046000 0 0 0 0000204 [SLPQ actask 0xc078ea2c][SLP] acpi_task2 8 c50d31c4 e9047000 0 0 0 0000204 [SLPQ actask 0xc078ea2c][SLP] acpi_task1 7 c50d3388 e9048000 0 0 0 0000204 [SLPQ actask 0xc078ea2c][SLP] acpi_task0 6 c50d354c e9085000 0 0 0 0000204 [SLPQ - 0xc50fb2c0][SLP] thread taskq 72 c50d3710 e9086000 0 0 0 0000204 [IWAIT] swi6:+ 71 c50d38d4 e9087000 0 0 0 0000204 [IWAIT] swi6: acpitaskq 70 c50d3a98 e9088000 0 0 0 0000204 [IWAIT] swi6: task queue 5 c50d3c5c e9089000 0 0 0 0000204 [SLPQ - 0xc50fb480][SLP] kqueue taskq 69 c50d3e20 e908a000 0 0 0 0000204 [IWAIT] swi3: cambio 68 c50d7000 e908b000 0 0 0 0000204 [IWAIT] swi2: camnet 67 c50d71c4 e908c000 0 0 0 0000204 [IWAIT] swi6:+ 66 c50d7388 e908d000 0 0 0 0000204 [SLPQ - 0xc0699a60][SLP] yarrow 4 c507c1c4 e8ff9000 0 0 0 0000204 [SLPQ - 0xc069c6a8][SLP] g_down 3 c507c388 e8ffa000 0 0 0 0000204 [SLPQ - 0xc069c6a4][SLP] g_up 2 c507c54c e8ffb000 0 0 0 0000204 [SLPQ - 0xc069c69c][SLP] g_event 65 c507c710 e8ffc000 0 0 0 0000204 [IWAIT] swi1: net 64 c507c8d4 e8ffd000 0 0 0 0000204 [IWAIT] swi4: vm 63 c507ca98 e903a000 0 0 0 000020c [LOCK Giant c5382540] swi5: clock sio 62 c507cc5c e903b000 0 0 0 0000204 [IWAIT] irq0: clk 61 c507ce20 e903c000 0 0 0 0000204 [IWAIT] irq47: 60 c508e000 e903d000 0 0 0 0000204 [IWAIT] irq46: 59 c508e1c4 e903e000 0 0 0 0000204 [IWAIT] irq45: 58 c508e388 e903f000 0 0 0 0000204 [IWAIT] irq44: 57 c508e54c e9040000 0 0 0 0000204 [IWAIT] irq43: 56 c508e710 e9041000 0 0 0 0000204 [IWAIT] irq42: 55 c508e8d4 e9042000 0 0 0 0000204 [IWAIT] irq41: 54 c5067a98 e8fce000 0 0 0 0000204 [IWAIT] irq40: 53 c5067c5c e8fcf000 0 0 0 0000204 [IWAIT] irq39: 52 c5067e20 e8fd0000 0 0 0 0000204 [IWAIT] irq38: 51 c5079000 e8fd1000 0 0 0 0000204 [IWAIT] irq37: 50 c50791c4 e8ff0000 0 0 0 0000204 [IWAIT] irq36: 49 c5079388 e8ff1000 0 0 0 0000204 [IWAIT] irq35: 48 c507954c e8ff2000 0 0 0 0000204 [IWAIT] irq34: 47 c5079710 e8ff3000 0 0 0 0000204 [IWAIT] irq33: 46 c50798d4 e8ff4000 0 0 0 0000204 [IWAIT] irq32: 45 c5079a98 e8ff5000 0 0 0 0000204 [IWAIT] irq31: 44 c5079c5c e8ff6000 0 0 0 0000204 [IWAIT] irq30: aac0 43 c5079e20 e8ff7000 0 0 0 0000204 [IWAIT] irq29: bge1 42 c507c000 e8ff8000 0 0 0 0000204 [IWAIT] irq28: bge0 41 c505d54c e8fa4000 0 0 0 0000204 [IWAIT] irq27: 40 c505d710 e8fa5000 0 0 0 0000204 [IWAIT] irq26: 39 c505d8d4 e8fa6000 0 0 0 0000204 [IWAIT] irq25: 38 c505da98 e8fc5000 0 0 0 0000204 [IWAIT] irq24: 37 c505dc5c e8fc6000 0 0 0 0000204 [IWAIT] irq23: 36 c505de20 e8fc7000 0 0 0 0000204 [IWAIT] irq22: 35 c5067000 e8fc8000 0 0 0 0000204 [IWAIT] irq21: 34 c50671c4 e8fc9000 0 0 0 0000204 [IWAIT] irq20: 33 c5067388 e8fca000 0 0 0 0000204 [IWAIT] irq19: 32 c506754c e8fcb000 0 0 0 0000204 [IWAIT] irq18: 31 c5067710 e8fcc000 0 0 0 0000204 [IWAIT] irq17: 30 c50678d4 e8fcd000 0 0 0 0000204 [IWAIT] irq16: 29 c50091c4 e7559000 0 0 0 0000204 [IWAIT] irq15: ata1 28 c5009388 e755a000 0 0 0 0000204 [IWAIT] irq14: ata0 27 c500954c e7579000 0 0 0 0000204 [IWAIT] irq13: 26 c5009710 e757a000 0 0 0 0000204 [IWAIT] irq12: psm0 25 c50098d4 e757b000 0 0 0 0000204 [IWAIT] irq11: 24 c5009a98 e757c000 0 0 0 0000204 [IWAIT] irq10: 23 c5009c5c e757d000 0 0 0 0000204 [IWAIT] irq9: acpi0 22 c5009e20 e757e000 0 0 0 0000204 [IWAIT] irq8: rtc 21 c505d000 e8fa1000 0 0 0 0000204 [IWAIT] irq7: 20 c505d1c4 e8fa2000 0 0 0 0000204 [IWAIT] irq6: fdc0 19 c505d388 e8fa3000 0 0 0 0000204 [IWAIT] irq5: 18 c4fff000 e7513000 0 0 0 0000204 [IWAIT] irq4: sio0 17 c4fff1c4 e7550000 0 0 0 0000204 [IWAIT] irq3: sio1 16 c4fff388 e7551000 0 0 0 0000204 [IWAIT] irq2: 15 c4fff54c e7552000 0 0 0 0000204 [IWAIT] irq1: atkbd0 14 c4fff710 e7553000 0 0 0 000020c [CPU 0] idle: cpu0 13 c4fff8d4 e7554000 0 0 0 000020c [CPU 1] idle: cpu1 12 c4fffa98 e7555000 0 0 0 000020c [Can run] idle: cpu2 11 c4fffc5c e7556000 0 0 0 000020c [CPU 3] idle: cpu3 1 c4fffe20 e7557000 0 0 1 0004200 [SLPQ wait 0xc4fffe20][SLP] init 10 c5009000 e7558000 0 0 0 0000204 [SLPQ ktrace 0xc06a0098][SLP] ktrace 0 c069c7a0 c081f000 0 0 0 0000200 [SLPQ sched 0xc069c7a0][SLP] swapper db> help print p examine x search set write w delete d break dwatch watch dhwatch hwatch step s continue c until next match trace where call show ps gdb reset kill watchdog thread panic db> call sync forward_wakeup: Idle processor not found panic: pmap_invalidate_range: interrupts disabled cpuid = 2 KDB: stack backtrace: kdb_backtrace(e270ff5c,668bc042,ffc06014,c6206e10,f5) at kdb_backtrace+0x29 mi_switch(1,0,f3d17440,c04d89d8,c6206e10) at mi_switch+0xf6 sched_bind(c6206e10,0) at sched_bind+0x44 boot(104,104,c6206e10,dea1a000,dea1e000) at boot+0x64 panic(c06558ab,c0632bf0,dea1a000,4,d92fc71c) at panic+0x1b9 pmap_invalidate_range(c06c99a0,dea1a000,dea1e000,c6aadd68,f3d174b0) at pmap_invalidate_range+0x2b pmap_qremove(dea1a000,4) at pmap_qremove+0x3d vfs_vmio_release(d92fc71c) at vfs_vmio_release+0x19c getnewbuf(0,0,4000,4000,c6028000) at getnewbuf+0x2b6 getblk(c6cea528,6e30140,0,4000,0) at getblk+0x409 breadn(c6cea528,6e30140,0,4000,0) at breadn+0x31 bread(c6cea528,6e30140,0,4000,0) at bread+0x20 ffs_update(c6c2ea50,0,4,8000,0) at ffs_update+0x224 ffs_fsync(f3d1767c) at ffs_fsync+0x453 ffs_sync(c51a7c00,2,0,0,c51a7c00) at ffs_sync+0x209 sync(0,0,0,0,0,0,0,0,0,0) at sync+0xcb db_fncall(0,0,f3d17780,f3d17768,0) at db_fncall+0xf6 db_command(c06986c4,c06609c0,c065ace0,c065ace4,c0639140) at db_command+0x264 db_command_loop(0,0,f3d17820,f3d1780c,f3d17854) at db_command_loop+0x5c db_trap(c,0,10,f3d178fc,f3d1789c) at db_trap+0xdd kdb_trap(c,0,f3d178fc) at kdb_trap+0x8b trap_fatal(f3d178fc,8a8a9304,c6206e10,c5ac5960,c) at trap_fatal+0x279 trap_pfault(f3d178fc,0,8a8a9304) at trap_pfault+0x1d7 trap(18,2000010,10,1,c6b06000) at trap+0x2f1 calltrap() at calltrap+0x5 --- trap 0xc, eip = 0xc05a1913, esp = 0xf3d1793c, ebp = 0xf3d17978 --- ffs_alloccg(c6c46dac,cc304030,a5a5a5ad,a5a5a5a5,4000) at ffs_alloccg+0x1b ffs_hashalloc(c6c46dac,cc304030,a5a5a5ad,a5a5a5a5,4000) at ffs_hashalloc+0x28 ffs_alloc(c6c46dac,20d,0,a5a5a5ad,a5a5a5a5) at ffs_alloc+0x175 ffs_balloc_ufs2(c6d73108,834000,0,4000,c64eb800) at ffs_balloc_ufs2+0x14b7 ffs_write(f3d17c10) at ffs_write+0x303 vn_write(c77dce58,f3d17c88,c64eb800,0,c6206e10) at vn_write+0x210 dofilewrite(c6206e10,c77dce58,3,8300000,10000) at dofilewrite+0xa8 write(c6206e10,f3d17d14,3,6eb5,292) at write+0x39 syscall(2f,2f,bfbf002f,0,10000) at syscall+0x283 Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (4, FreeBSD ELF32, write), eip = 0x280e85f3, esp = 0xbfbfe5dc, ebp = 0xbfbfe6f8 --- db> re cpu_reset called on cpu#2 cpu_reset: Restarting BSP cpu_reset_proxy: Stopped CPU 2Received on Fri Jan 28 2005 - 13:48:01 UTC
This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:38:27 UTC