From badabe5ac033696ada62199bfa16cc140b5095aa Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Sun, 30 Dec 2018 12:12:54 +0100 Subject: [PATCH] pkg/report: improve parsing of "corrupted stack end detected inside scheduler" --- pkg/report/linux.go | 13 +++++++ pkg/report/testdata/linux/report/316 | 5 ++- pkg/report/testdata/linux/report/330 | 57 ++++++++++++++++++++++++++++ pkg/report/testdata/linux/report/331 | 40 +++++++++++++++++++ pkg/report/testdata/linux/report/332 | 44 +++++++++++++++++++++ pkg/report/testdata/linux/report/333 | 55 +++++++++++++++++++++++++++ pkg/report/testdata/linux/report/334 | 42 ++++++++++++++++++++ 7 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 pkg/report/testdata/linux/report/330 create mode 100644 pkg/report/testdata/linux/report/331 create mode 100644 pkg/report/testdata/linux/report/332 create mode 100644 pkg/report/testdata/linux/report/333 create mode 100644 pkg/report/testdata/linux/report/334 diff --git a/pkg/report/linux.go b/pkg/report/linux.go index 411f91867d31..757541a8688c 100644 --- a/pkg/report/linux.go +++ b/pkg/report/linux.go @@ -1212,6 +1212,19 @@ var linuxOopses = []*oops{ skip: []string{"stack_chk"}, }, }, + { + title: compile("Kernel panic - not syncing: corrupted stack end"), + report: compile("Kernel panic - not syncing: corrupted stack end detected inside scheduler"), + fmt: "kernel panic: corrupted stack end in %[1]v", + stack: &stackFmt{ + parts: []*regexp.Regexp{ + compile("Call Trace:"), + parseStackTrace, + }, + skip: []string{"schedule", "retint_kernel"}, + extractor: linuxStallFrameExtractor, + }, + }, { title: compile("Kernel panic - not syncing: Attempted to kill init!"), fmt: "kernel panic: Attempted to kill init!", diff --git a/pkg/report/testdata/linux/report/316 b/pkg/report/testdata/linux/report/316 index 4c7822c8d08a..7141db1e1787 100644 --- a/pkg/report/testdata/linux/report/316 +++ b/pkg/report/testdata/linux/report/316 @@ -1,4 +1,7 @@ -TITLE: kernel panic: corrupted stack end detected inside scheduler +# TODO: the arm64 crash has different capitalization for "Call trace:", +# so we don't recognize it as a stack trace. +TITLE: kernel panic: corrupted stack end in corrupted +CORRUPTED: Y [ 2231.649459] Kernel panic - not syncing: corrupted stack end detected inside scheduler [ 2231.657307] CPU: 185 PID: 11718 Comm: dbus-daemon Kdump: loaded Tainted: G W 4.20.0-rc2+ #4 diff --git a/pkg/report/testdata/linux/report/330 b/pkg/report/testdata/linux/report/330 new file mode 100644 index 000000000000..b1b340a87ded --- /dev/null +++ b/pkg/report/testdata/linux/report/330 @@ -0,0 +1,57 @@ +TITLE: kernel panic: corrupted stack end in wb_workfn + +[ 335.974403] Kernel panic - not syncing: corrupted stack end detected inside scheduler +[ 335.975933] CPU: 0 PID: 11678 Comm: kworker/u12:6 Not tainted 4.20.0+ #11 +[ 335.977172] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1 04/01/2014 +[ 335.978779] Workqueue: writeback wb_workfn (flush-8:0) +[ 335.979784] Call Trace: +[ 335.980287] dump_stack+0x1d3/0x2c6 +[ 335.982672] panic+0x2ad/0x55f +[ 335.986347] __schedule+0x1ec6/0x1ed0 +[ 335.993751] preempt_schedule_irq+0x87/0x110 +[ 335.994383] retint_kernel+0x1b/0x2d +[ 335.994939] RIP: 0010:__sanitizer_cov_trace_pc+0x26/0x50 +[ 335.995730] Code: 5d c3 66 90 55 48 89 e5 48 8b 75 08 65 48 8b 04 25 40 ee 01 00 65 8b 15 d8 05 80 7e 81 e2 00 01 1f 00 75 2b 8b 90 d8 12 00 00 <83> fa 02 75 20 48 8b 88 e0 12 00 00 8b 80 dc 12 00 00 48 8b 11 48 +[ 335.998549] RSP: 0018:ffff888044289348 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff02 +[ 335.999800] RAX: ffff88804400c3c0 RBX: 1ffff1100885126e RCX: 0000000000000000 +[ 336.000873] RDX: 0000000000000000 RSI: ffffffff81bb5329 RDI: 0000000000000007 +[ 336.001964] RBP: ffff888044289348 R08: ffff88804400c3c0 R09: fffff940002b91fe +[ 336.003010] R10: fffff940002b91fe R11: ffffea00015c8ff3 R12: ffffea00015c8fc0 +[ 336.004112] R13: ffff888044289410 R14: ffffea00015c8fc8 R15: ffffea00015c8fc0 +[ 336.006590] rmap_walk+0x119/0x350 +[ 336.009046] try_to_unmap+0x1f3/0xa40 +[ 336.020876] shrink_page_list+0x70ef/0xc2e0 +[ 336.138056] shrink_inactive_list+0x77b/0x1c60 +[ 336.143095] shrink_node_memcg+0x7a8/0x19a0 +[ 336.153262] syz-executor2 (20558) used greatest stack depth: 4808 bytes left +[ 336.153592] shrink_node+0x3e1/0x17f0 +[ 336.164797] do_try_to_free_pages+0x3df/0x12a0 +[ 336.168005] try_to_free_pages+0x4d0/0xb90 +[ 336.174332] __alloc_pages_slowpath+0xa5a/0x2db0 +[ 336.188121] __alloc_pages_nodemask+0xa89/0xde0 +[ 336.195476] alloc_pages_current+0x10c/0x210 +[ 336.196116] __page_cache_alloc+0x38c/0x5b0 +[ 336.199371] pagecache_get_page+0x396/0xf00 +[ 336.207507] ext4_mb_load_buddy_gfp+0xddf/0x1e70 +[ 336.227432] ext4_mb_find_by_goal+0x2fa/0x1040 +[ 336.234525] ext4_mb_regular_allocator+0x282/0x1590 +[ 336.241382] ext4_mb_new_blocks+0x1de3/0x4840 +[ 336.276757] ext4_ext_map_blocks+0x2eef/0x6180 +[ 336.291766] ext4_map_blocks+0x8f7/0x1b60 +[ 336.297211] ext4_writepages+0x2564/0x4170 +[ 336.308530] blkid (20588) used greatest stack depth: 3352 bytes left +[ 336.314367] do_writepages+0x9a/0x1a0 +[ 336.317308] __writeback_single_inode+0x20a/0x1660 +[ 336.338247] writeback_sb_inodes+0x71f/0x1210 +[ 336.344617] __writeback_inodes_wb+0x1b9/0x340 +[ 336.346588] wb_writeback+0xa73/0xfc0 +[ 336.356813] wb_workfn+0xd65/0x1790 +[ 336.370426] process_one_work+0xc90/0x1c40 +[ 336.387991] worker_thread+0x17f/0x1390 +[ 336.400947] kthread+0x35a/0x440 +[ 336.402013] ret_from_fork+0x3a/0x50 +[ 336.403121] Dumping ftrace buffer: +[ 336.403852] (ftrace buffer empty) +[ 336.404373] Kernel Offset: disabled +[ 336.404873] Rebooting in 86400 seconds.. + diff --git a/pkg/report/testdata/linux/report/331 b/pkg/report/testdata/linux/report/331 new file mode 100644 index 000000000000..1c6ad803a698 --- /dev/null +++ b/pkg/report/testdata/linux/report/331 @@ -0,0 +1,40 @@ +TITLE: kernel panic: corrupted stack end in wb_workfn + +[ 839.617079] Kernel panic - not syncing: corrupted stack end detected inside scheduler +[ 839.619019] CPU: 0 PID: 7895 Comm: kworker/u12:6 Not tainted 4.20.0+ #11 +[ 839.620030] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1 04/01/2014 +[ 839.621165] Workqueue: writeback wb_workfn (flush-8:0) +[ 839.621888] Call Trace: +[ 839.622253] dump_stack+0x1d3/0x2c6 +[ 839.623492] panic+0x2ad/0x55f +[ 839.627987] __schedule+0x1ec6/0x1ed0 +[ 839.633474] schedule+0xef/0x370 +[ 839.636799] schedule_timeout+0x140/0x260 +[ 839.638901] schedule_timeout_uninterruptible+0x75/0x90 +[ 839.639786] __alloc_pages_slowpath+0x14c9/0x2db0 +[ 839.653639] __alloc_pages_nodemask+0xa89/0xde0 +[ 839.658370] alloc_pages_current+0x10c/0x210 +[ 839.659154] __page_cache_alloc+0x38c/0x5b0 +[ 839.662126] pagecache_get_page+0x396/0xf00 +[ 839.670212] ext4_mb_load_buddy_gfp+0xa80/0x1e70 +[ 839.685979] ext4_mb_find_by_goal+0x2fa/0x1040 +[ 839.690920] ext4_mb_regular_allocator+0x282/0x1590 +[ 839.695735] ext4_mb_new_blocks+0x1de3/0x4840 +[ 839.723338] ext4_ext_map_blocks+0x2eef/0x6180 +[ 839.733924] ext4_map_blocks+0x8f7/0x1b60 +[ 839.737730] ext4_writepages+0x2564/0x4170 +[ 839.749193] do_writepages+0x9a/0x1a0 +[ 839.751229] __writeback_single_inode+0x20a/0x1660 +[ 839.765951] writeback_sb_inodes+0x71f/0x1210 +[ 839.771338] __writeback_inodes_wb+0x1b9/0x340 +[ 839.773483] wb_writeback+0xa73/0xfc0 +[ 839.784905] wb_workfn+0xd65/0x1790 +[ 839.802711] process_one_work+0xc90/0x1c40 +[ 839.825054] worker_thread+0x17f/0x1390 +[ 839.838890] kthread+0x35a/0x440 +[ 839.840051] ret_from_fork+0x3a/0x50 +[ 839.841422] Dumping ftrace buffer: +[ 839.842800] (ftrace buffer empty) +[ 839.843307] Kernel Offset: disabled +[ 839.843804] Rebooting in 86400 seconds.. + diff --git a/pkg/report/testdata/linux/report/332 b/pkg/report/testdata/linux/report/332 new file mode 100644 index 000000000000..6b52ef67ffdb --- /dev/null +++ b/pkg/report/testdata/linux/report/332 @@ -0,0 +1,44 @@ +TITLE: kernel panic: corrupted stack end in wb_workfn + +[ 281.244340] Kernel panic - not syncing: corrupted stack end detected inside scheduler +[ 281.245754] CPU: 2 PID: 6265 Comm: kworker/u12:4 Not tainted 4.20.0-rc7+ #6 +[ 281.246887] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1 04/01/2014 +[ 281.248240] Workqueue: writeback wb_workfn (flush-8:0) +[ 281.248992] Call Trace: +[ 281.249364] dump_stack+0x1d4/0x2b5 +[ 281.252261] panic+0x25e/0x49c +[ 281.255403] __schedule+0x1be8/0x21d0 +[ 281.263754] preempt_schedule_common+0x35/0xe0 +[ 281.264425] preempt_schedule+0x23/0x30 +[ 281.265010] ___preempt_schedule+0x16/0x18 +[ 281.265635] _raw_spin_unlock_irqrestore+0xbf/0xe0 +[ 281.266357] __remove_mapping+0x77b/0x17e0 +[ 281.291388] shrink_page_list+0x5232/0xa6b0 +[ 281.414732] shrink_inactive_list+0x997/0x1ab0 +[ 281.419009] shrink_node_memcg+0x9de/0x16a0 +[ 281.424799] shrink_node+0x3af/0x1530 +[ 281.433316] do_try_to_free_pages+0x3bc/0x1170 +[ 281.435723] try_to_free_pages+0x43c/0x9e0 +[ 281.442644] __alloc_pages_slowpath+0xa4c/0x2c10 +[ 281.459197] __alloc_pages_nodemask+0xa6c/0xe10 +[ 281.466504] alloc_pages_current+0xb6/0x1e0 +[ 281.467326] __page_cache_alloc+0x332/0x560 +[ 281.471049] pagecache_get_page+0x2af/0xdd0 +[ 281.487360] __getblk_gfp+0x36e/0xd50 +[ 281.497989] ext4_read_block_bitmap_nowait+0x2ed/0x1e10 +[ 281.509111] ext4_read_block_bitmap+0x23/0x80 +[ 281.509934] ext4_mb_mark_diskspace_used+0x180/0x10a0 +[ 281.512755] ext4_mb_new_blocks+0xeb7/0x4260 +[ 281.540189] ext4_ext_map_blocks+0x2776/0x5b00 +[ 281.556040] ext4_map_blocks+0xcaa/0x1860 +[ 281.559967] ext4_writepages+0x1e4c/0x3a20 +[ 281.575738] do_writepages+0xfc/0x170 +[ 281.578546] __writeback_single_inode+0x1cd/0x12e0 +[ 281.592498] writeback_sb_inodes+0x6c7/0x1040 +[ 281.598601] __writeback_inodes_wb+0x1a3/0x310 +[ 281.600816] wb_writeback+0x92c/0xe10 +[ 281.618064] wb_workfn+0xdf3/0x1600 +[ 281.635970] process_one_work+0xcf3/0x1be0 +[ 281.662614] worker_thread+0x17d/0x12f0 +[ 281.680989] kthread+0x354/0x430 +[ 281.682529] ret_from_fork+0x3a/0x50 diff --git a/pkg/report/testdata/linux/report/333 b/pkg/report/testdata/linux/report/333 new file mode 100644 index 000000000000..9b1febb5a1a7 --- /dev/null +++ b/pkg/report/testdata/linux/report/333 @@ -0,0 +1,55 @@ +TITLE: kernel panic: corrupted stack end in wb_workfn + +[ 186.195516] Kernel panic - not syncing: corrupted stack end detected inside scheduler +[ 186.203830] CPU: 0 PID: 21 Comm: kworker/u4:1 Not tainted 4.20.0+ #176 +[ 186.210506] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 186.219874] Workqueue: writeback wb_workfn (flush-8:0) +[ 186.225160] Call Trace: +[ 186.227755] dump_stack+0x1d3/0x2c6 +[ 186.236621] panic+0x2ad/0x55f +[ 186.263843] __schedule+0x1ec6/0x1ed0 +[ 186.311367] preempt_schedule_irq+0x87/0x110 +[ 186.315792] retint_kernel+0x1b/0x2d +[ 186.319510] RIP: 0010:lock_is_held_type+0x18b/0x210 +[ 186.324533] Code: ff df 41 c7 84 24 7c 08 00 00 00 00 00 00 48 89 fa 48 c1 ea 03 80 3c 02 00 75 63 48 83 3d 9c 09 0f 08 00 74 30 48 89 df 57 9d <0f> 1f 44 00 00 48 83 c4 08 44 89 e8 5b 41 5c 41 5d 5d c3 48 83 c4 +[ 186.343461] RSP: 0018:ffff8880a95dbe10 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 +[ 186.351185] RAX: dffffc0000000000 RBX: 0000000000000286 RCX: 0000000000000000 +[ 186.358478] RDX: 1ffffffff12e495e RSI: 0000000000000000 RDI: 0000000000000286 +[ 186.365783] RBP: ffff8880a95dbe30 R08: ffff8880a95d0580 R09: ffffed1015cc5b8f +[ 186.373065] R10: ffffed1015cc5b8f R11: ffff8880ae62dc7b R12: ffff8880a95d0580 +[ 186.380334] R13: 0000000000000001 R14: 0000000000000000 R15: ffffffffffffffff +[ 186.387658] rcu_read_lock_held+0xa9/0xc0 +[ 186.391823] list_lru_count_one+0x31a/0x450 +[ 186.414265] super_cache_count+0x1b9/0x2e0 +[ 186.418521] do_shrink_slab+0x1b1/0xd30 +[ 186.457922] shrink_slab+0x389/0x8c0 +[ 186.477028] shrink_node+0x647/0x17f0 +[ 186.529384] do_try_to_free_pages+0x3df/0x12a0 +[ 186.545944] try_to_free_pages+0x4d0/0xb90 +[ 186.591142] __alloc_pages_slowpath+0xa5a/0x2db0 +[ 186.663340] __alloc_pages_nodemask+0xa89/0xde0 +[ 186.702734] alloc_pages_current+0x10c/0x210 +[ 186.707168] __page_cache_alloc+0x38c/0x5b0 +[ 186.724259] pagecache_get_page+0x396/0xf00 +[ 186.805773] __getblk_gfp+0x3aa/0xd50 +[ 186.869886] __breadahead+0x8c/0x160 +[ 186.878296] __ext4_get_inode_loc+0xdc4/0x1330 +[ 186.891401] ext4_reserve_inode_write+0xfa/0x260 +[ 186.896182] ext4_mark_inode_dirty+0x19c/0xb20 +[ 186.993752] __ext4_ext_dirty+0x1f5/0x260 +[ 187.013327] ext4_ext_insert_extent+0x118d/0x45a0 +[ 187.047743] ext4_ext_map_blocks+0x32fc/0x6180 +[ 187.111537] ext4_map_blocks+0x8f7/0x1b60 +[ 187.134851] ext4_writepages+0x2564/0x4170 +[ 187.213830] do_writepages+0x9a/0x1a0 +[ 187.226388] __writeback_single_inode+0x20a/0x1660 +[ 187.291026] writeback_sb_inodes+0x71f/0x1210 +[ 187.326427] __writeback_inodes_wb+0x1b9/0x340 +[ 187.339862] wb_writeback+0xa73/0xfc0 +[ 187.442931] wb_workfn+0xee9/0x1790 +[ 187.539003] process_one_work+0xc90/0x1c40 +[ 187.654956] worker_thread+0x17f/0x1390 +[ 187.741419] kthread+0x35a/0x440 +[ 187.748617] ret_from_fork+0x3a/0x50 +[ 187.753622] Kernel Offset: disabled +[ 187.757359] Rebooting in 86400 seconds.. diff --git a/pkg/report/testdata/linux/report/334 b/pkg/report/testdata/linux/report/334 new file mode 100644 index 000000000000..318f8280160e --- /dev/null +++ b/pkg/report/testdata/linux/report/334 @@ -0,0 +1,42 @@ +TITLE: kernel panic: corrupted stack end in wb_workfn + +[ 200.999653] Kernel panic - not syncing: corrupted stack end detected inside scheduler +[ 201.007844] CPU: 1 PID: 7 Comm: kworker/u4:0 Not tainted 4.20.0+ #252 +[ 201.014420] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 201.023901] Workqueue: writeback wb_workfn (flush-8:0) +[ 201.029181] Call Trace: +[ 201.031778] dump_stack+0x1d3/0x2c6 +[ 201.040625] panic+0x2ad/0x55f +[ 201.076801] __schedule+0x1ec6/0x1ed0 +[ 201.120555] preempt_schedule_irq+0x87/0x110 +[ 201.124972] retint_kernel+0x1b/0x2d +[ 201.128689] RIP: 0010:__add_to_page_cache_locked+0x48c/0x1840 +[ 201.134577] Code: f8 83 e0 01 48 89 c6 48 89 85 80 fc ff ff e8 bb a7 e1 ff 48 8b 85 80 fc ff ff 4c 8b ad 88 fc ff ff 48 85 c0 0f 85 83 0e 00 00 6f a6 e1 ff 4c 89 ea 48 b8 00 00 00 00 00 fc ff df 48 c1 ea 03 +[ 201.153480] RSP: 0018:ffff8880a9e14668 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 201.161200] RAX: 0000000000000000 RBX: ffff8880a9e14a08 RCX: ffffffff819fff65 +[ 201.168506] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000007 +[ 201.175804] RBP: ffff8880a9e14a30 R08: ffff8880a9e061c0 R09: ffffed1015ce5b8f +[ 201.183079] R10: ffffea00052dfc80 R11: ffff8880ae72dc7b R12: ffff88821a1f6430 +[ 201.190351] R13: ffffea00052dfc80 R14: 0000000000000003 R15: dead000000000100 +[ 201.261080] add_to_page_cache_lru+0x355/0xdb0 +[ 201.264929] syz-executor642 invoked oom-killer: gfp_mask=0x6202c2(GFP_HIGHUSER|__GFP_NOWARN), nodemask=(null), order=0, oom_score_adj=1000 +[ 201.318813] syz-executor642 cpuset=syz4 mems_allowed=0-1 +[ 201.324288] pagecache_get_page+0x53a/0xf00 +[ 201.382778] ext4_mb_load_buddy_gfp+0xa80/0x1e70 +[ 201.495828] ext4_mb_regular_allocator+0x634/0x1590 +[ 201.520206] ext4_mb_new_blocks+0x1de3/0x4840 +[ 201.684191] ext4_ext_map_blocks+0x2eef/0x6180 +[ 201.747899] ext4_map_blocks+0x8f7/0x1b60 +[ 201.771231] ext4_writepages+0x2573/0x4180 +[ 201.850921] do_writepages+0x9a/0x1a0 +[ 201.863500] __writeback_single_inode+0x20a/0x1660 +[ 201.936685] writeback_sb_inodes+0x71f/0x1210 +[ 201.975790] __writeback_inodes_wb+0x1b9/0x340 +[ 201.989235] wb_writeback+0xa73/0xfc0 +[ 202.092390] wb_workfn+0xee9/0x1790 +[ 202.218262] process_one_work+0xc90/0x1c40 +[ 202.322376] worker_thread+0x17f/0x1390 +[ 202.396119] kthread+0x35a/0x440 +[ 202.403283] ret_from_fork+0x3a/0x50 +[ 202.408257] Kernel Offset: disabled +[ 202.412020] Rebooting in 86400 seconds..