KASAN: use-after-free Write in xfrm_hash_rebuild

[syzbot]

Hello,

syzbot found the following crash on:

HEAD commit: 601e6bcc Merge git://git.kernel.org/pub/scm/linux/kernel/g..
git tree: net-next
console output: https://syzkaller.appspot.com/x/log.txt?x=1534f3d0a00000
kernel config: https://syzkaller.appspot.com/x/.config?x=4005028a9d5ddac8
dashboard link: https://syzkaller.appspot.com/bug?extid=0165480d4ef07360eeda
compiler: gcc (GCC) 9.0.0 20181231 (experimental)

Unfortunately, I don't have any reproducer for this crash yet.

IMPORTANT: if you fix the bug, please add the following tag to the commit:
Reported-by: syzbot+016548...@syzkaller.appspotmail.com

==================================================================
BUG: KASAN: use-after-free in __write_once_size
include/linux/compiler.h:220 [inline]
BUG: KASAN: use-after-free in __hlist_del include/linux/list.h:713 [inline]
BUG: KASAN: use-after-free in hlist_del_rcu include/linux/rculist.h:455
[inline]
BUG: KASAN: use-after-free in xfrm_hash_rebuild+0xfff/0x10f0
net/xfrm/xfrm_policy.c:1317
Write of size 8 at addr ffff888098529100 by task kworker/0:0/5

CPU: 0 PID: 5 Comm: kworker/0:0 Not tainted 5.1.0+ #4
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS
Google 01/01/2011
Workqueue: events xfrm_hash_rebuild
Call Trace:
__dump_stack lib/dump_stack.c:77 [inline]
dump_stack+0x172/0x1f0 lib/dump_stack.c:113
print_address_description.cold+0x7c/0x20d mm/kasan/report.c:188
__kasan_report.cold+0x1b/0x40 mm/kasan/report.c:317
kasan_report+0x12/0x20 mm/kasan/common.c:614
__asan_report_store8_noabort+0x17/0x20 mm/kasan/generic_report.c:137
__write_once_size include/linux/compiler.h:220 [inline]
__hlist_del include/linux/list.h:713 [inline]
hlist_del_rcu include/linux/rculist.h:455 [inline]
xfrm_hash_rebuild+0xfff/0x10f0 net/xfrm/xfrm_policy.c:1317
process_one_work+0x98e/0x1790 kernel/workqueue.c:2268
worker_thread+0x98/0xe40 kernel/workqueue.c:2414
kthread+0x357/0x430 kernel/kthread.c:253
ret_from_fork+0x3a/0x50 arch/x86/entry/entry_64.S:352

Allocated by task 8152:
save_stack+0x23/0x90 mm/kasan/common.c:71
set_track mm/kasan/common.c:79 [inline]
__kasan_kmalloc mm/kasan/common.c:489 [inline]
__kasan_kmalloc.constprop.0+0xcf/0xe0 mm/kasan/common.c:462
kasan_kmalloc+0x9/0x10 mm/kasan/common.c:503
kmem_cache_alloc_node_trace+0x153/0x720 mm/slab.c:3630
kmalloc_node include/linux/slab.h:585 [inline]
kzalloc_node include/linux/slab.h:753 [inline]
__get_vm_area_node+0x12b/0x3a0 mm/vmalloc.c:1401
__vmalloc_node_range+0xd4/0x790 mm/vmalloc.c:1840
__vmalloc_node mm/vmalloc.c:1900 [inline]
__vmalloc_node_flags mm/vmalloc.c:1914 [inline]
vzalloc+0x6b/0x90 mm/vmalloc.c:1959
alloc_counters.isra.0+0x53/0x690 net/ipv6/netfilter/ip6_tables.c:819
copy_entries_to_user net/ipv4/netfilter/arp_tables.c:674 [inline]
get_entries net/ipv4/netfilter/arp_tables.c:861 [inline]
do_arpt_get_ctl+0x4a0/0x820 net/ipv4/netfilter/arp_tables.c:1482
nf_sockopt net/netfilter/nf_sockopt.c:104 [inline]
nf_getsockopt+0x80/0xe0 net/netfilter/nf_sockopt.c:122
ip_getsockopt net/ipv4/ip_sockglue.c:1574 [inline]
ip_getsockopt+0x176/0x1d0 net/ipv4/ip_sockglue.c:1554
tcp_getsockopt net/ipv4/tcp.c:3623 [inline]
tcp_getsockopt+0x95/0xf0 net/ipv4/tcp.c:3617
sock_common_getsockopt+0x9a/0xe0 net/core/sock.c:3089
__sys_getsockopt+0x168/0x250 net/socket.c:2115
__do_sys_getsockopt net/socket.c:2126 [inline]
__se_sys_getsockopt net/socket.c:2123 [inline]
__x64_sys_getsockopt+0xbe/0x150 net/socket.c:2123
do_syscall_64+0x103/0x670 arch/x86/entry/common.c:298
entry_SYSCALL_64_after_hwframe+0x49/0xbe

Freed by task 8152:
save_stack+0x23/0x90 mm/kasan/common.c:71
set_track mm/kasan/common.c:79 [inline]
__kasan_slab_free+0x102/0x150 mm/kasan/common.c:451
kasan_slab_free+0xe/0x10 mm/kasan/common.c:459
__cache_free mm/slab.c:3463 [inline]
kfree+0xcf/0x230 mm/slab.c:3786
__vunmap+0x704/0x9c0 mm/vmalloc.c:1617
__vfree+0x41/0xd0 mm/vmalloc.c:1658
vfree+0x5f/0x90 mm/vmalloc.c:1688
copy_entries_to_user net/ipv4/netfilter/arp_tables.c:706 [inline]
get_entries net/ipv4/netfilter/arp_tables.c:861 [inline]
do_arpt_get_ctl+0x67b/0x820 net/ipv4/netfilter/arp_tables.c:1482
nf_sockopt net/netfilter/nf_sockopt.c:104 [inline]
nf_getsockopt+0x80/0xe0 net/netfilter/nf_sockopt.c:122
ip_getsockopt net/ipv4/ip_sockglue.c:1574 [inline]
ip_getsockopt+0x176/0x1d0 net/ipv4/ip_sockglue.c:1554
tcp_getsockopt net/ipv4/tcp.c:3623 [inline]
tcp_getsockopt+0x95/0xf0 net/ipv4/tcp.c:3617
sock_common_getsockopt+0x9a/0xe0 net/core/sock.c:3089
__sys_getsockopt+0x168/0x250 net/socket.c:2115
__do_sys_getsockopt net/socket.c:2126 [inline]
__se_sys_getsockopt net/socket.c:2123 [inline]
__x64_sys_getsockopt+0xbe/0x150 net/socket.c:2123
do_syscall_64+0x103/0x670 arch/x86/entry/common.c:298
entry_SYSCALL_64_after_hwframe+0x49/0xbe

The buggy address belongs to the object at ffff888098529100
which belongs to the cache kmalloc-64 of size 64
The buggy address is located 0 bytes inside of
64-byte region [ffff888098529100, ffff888098529140)
The buggy address belongs to the page:
page:ffffea0002614a40 count:1 mapcount:0 mapping:ffff8880aa400340
index:0xffff888098529600
flags: 0x1fffc0000000200(slab)
raw: 01fffc0000000200 ffffea0002a22fc8 ffffea00026d2888 ffff8880aa400340
raw: ffff888098529600 ffff888098529000 000000010000001b 0000000000000000
page dumped because: kasan: bad access detected

Memory state around the buggy address:
ffff888098529000: 00 00 00 00 00 00 00 00 fc fc fc fc fc fc fc fc
ffff888098529080: 00 00 00 00 00 00 00 00 fc fc fc fc fc fc fc fc
> ffff888098529100: fb fb fb fb fb fb fb fb fc fc fc fc fc fc fc fc
^
ffff888098529180: fb fb fb fb fb fb fb fb fc fc fc fc fc fc fc fc
ffff888098529200: 00 00 00 00 00 00 00 00 fc fc fc fc fc fc fc fc
==================================================================


---
This bug is generated by a bot. It may contain errors.
See https://goo.gl/tpsmEJ for more information about syzbot.
syzbot engineers can be reached at syzk...@googlegroups.com.

syzbot will keep track of this bug report. See:
https://goo.gl/tpsmEJ#status for how to communicate with syzbot.

[syzbot]

syzbot has found a reproducer for the following crash on:

HEAD commit: 249155c2 Merge branch 'parisc-5.2-4' of git://git.kernel.o..
git tree: upstream
console output: https://syzkaller.appspot.com/x/log.txt?x=10f017c3a00000
kernel config: https://syzkaller.appspot.com/x/.config?x=9a31528e58cc12e2
dashboard link: https://syzkaller.appspot.com/bug?extid=0165480d4ef07360eeda
compiler: clang version 9.0.0 (/home/glider/llvm/clang
80fee25776c2fb61e74c1ecb1a523375c2500b69)
syz repro: https://syzkaller.appspot.com/x/repro.syz?x=16cf37c3a00000

IMPORTANT: if you fix the bug, please add the following tag to the commit:
Reported-by: syzbot+016548...@syzkaller.appspotmail.com

==================================================================
BUG: KASAN: use-after-free in __write_once_size

include/linux/compiler.h:221 [inline]
BUG: KASAN: use-after-free in __hlist_del include/linux/list.h:748 [inline]

BUG: KASAN: use-after-free in hlist_del_rcu include/linux/rculist.h:455
[inline]

BUG: KASAN: use-after-free in xfrm_hash_rebuild+0xa0d/0x1000
net/xfrm/xfrm_policy.c:1318
Write of size 8 at addr ffff888095e79c00 by task kworker/1:3/8066

CPU: 1 PID: 8066 Comm: kworker/1:3 Not tainted 5.2.0-rc6+ #7

Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS
Google 01/01/2011
Workqueue: events xfrm_hash_rebuild
Call Trace:
__dump_stack lib/dump_stack.c:77 [inline]

dump_stack+0x1d8/0x2f8 lib/dump_stack.c:113
print_address_description+0x6d/0x310 mm/kasan/report.c:188
__kasan_report+0x14b/0x1c0 mm/kasan/report.c:317
kasan_report+0x26/0x50 mm/kasan/common.c:614
__asan_report_store8_noabort+0x17/0x20 mm/kasan/generic_report.c:137
__write_once_size include/linux/compiler.h:221 [inline]
__hlist_del include/linux/list.h:748 [inline]
hlist_del_rcu include/linux/rculist.h:455 [inline]
xfrm_hash_rebuild+0xa0d/0x1000 net/xfrm/xfrm_policy.c:1318
process_one_work+0x814/0x1130 kernel/workqueue.c:2269
worker_thread+0xc01/0x1640 kernel/workqueue.c:2415
kthread+0x325/0x350 kernel/kthread.c:255
ret_from_fork+0x24/0x30 arch/x86/entry/entry_64.S:352

Allocated by task 8064:
save_stack mm/kasan/common.c:71 [inline]
set_track mm/kasan/common.c:79 [inline]
__kasan_kmalloc+0x11c/0x1b0 mm/kasan/common.c:489
kasan_kmalloc+0x9/0x10 mm/kasan/common.c:503
__do_kmalloc mm/slab.c:3660 [inline]
__kmalloc+0x23c/0x310 mm/slab.c:3669
kmalloc include/linux/slab.h:552 [inline]
kzalloc include/linux/slab.h:742 [inline]
xfrm_hash_alloc+0x38/0xe0 net/xfrm/xfrm_hash.c:21
xfrm_policy_init net/xfrm/xfrm_policy.c:4036 [inline]
xfrm_net_init+0x269/0xd60 net/xfrm/xfrm_policy.c:4120
ops_init+0x336/0x420 net/core/net_namespace.c:130
setup_net+0x212/0x690 net/core/net_namespace.c:316
copy_net_ns+0x224/0x380 net/core/net_namespace.c:439
create_new_namespaces+0x4ec/0x700 kernel/nsproxy.c:103
unshare_nsproxy_namespaces+0x12a/0x190 kernel/nsproxy.c:202
ksys_unshare+0x540/0xac0 kernel/fork.c:2692
__do_sys_unshare kernel/fork.c:2760 [inline]
__se_sys_unshare kernel/fork.c:2758 [inline]
__x64_sys_unshare+0x38/0x40 kernel/fork.c:2758
do_syscall_64+0xfe/0x140 arch/x86/entry/common.c:301
entry_SYSCALL_64_after_hwframe+0x49/0xbe

Freed by task 17:
save_stack mm/kasan/common.c:71 [inline]
set_track mm/kasan/common.c:79 [inline]
__kasan_slab_free+0x12a/0x1e0 mm/kasan/common.c:451
kasan_slab_free+0xe/0x10 mm/kasan/common.c:459
__cache_free mm/slab.c:3432 [inline]
kfree+0xae/0x120 mm/slab.c:3755
xfrm_hash_free+0x38/0xd0 net/xfrm/xfrm_hash.c:35
xfrm_bydst_resize net/xfrm/xfrm_policy.c:602 [inline]
xfrm_hash_resize+0x13f1/0x1840 net/xfrm/xfrm_policy.c:680
process_one_work+0x814/0x1130 kernel/workqueue.c:2269
worker_thread+0xc01/0x1640 kernel/workqueue.c:2415
kthread+0x325/0x350 kernel/kthread.c:255
ret_from_fork+0x24/0x30 arch/x86/entry/entry_64.S:352

The buggy address belongs to the object at ffff888095e79c00

which belongs to the cache kmalloc-64 of size 64
The buggy address is located 0 bytes inside of

64-byte region [ffff888095e79c00, ffff888095e79c40)

The buggy address belongs to the page:

page:ffffea0002579e40 refcount:1 mapcount:0 mapping:ffff8880aa400340
index:0x0
flags: 0x1fffc0000000200(slab)
raw: 01fffc0000000200 ffffea0002540888 ffffea0002907548 ffff8880aa400340
raw: 0000000000000000 ffff888095e79000 0000000100000020 0000000000000000

page dumped because: kasan: bad access detected

Memory state around the buggy address:

ffff888095e79b00: 00 00 00 00 00 00 00 00 fc fc fc fc fc fc fc fc
ffff888095e79b80: 00 00 00 00 00 00 00 00 fc fc fc fc fc fc fc fc
> ffff888095e79c00: fb fb fb fb fb fb fb fb fc fc fc fc fc fc fc fc
^
ffff888095e79c80: fb fb fb fb fb fb fb fb fc fc fc fc fc fc fc fc
ffff888095e79d00: 00 00 00 00 00 00 00 00 fc fc fc fc fc fc fc fc
==================================================================

[syzbot]

syzbot has bisected this bug to:

commit 1548bc4e0512700cf757192c106b3a20ab639223
Author: Florian Westphal <f...@strlen.de>
Date: Fri Jan 4 13:17:02 2019 +0000

xfrm: policy: delete inexact policies from inexact list on hash rebuild

bisection log: https://syzkaller.appspot.com/x/bisect.txt?x=1734cba9a00000
start commit: 249155c2 Merge branch 'parisc-5.2-4' of git://git.kernel.o..
git tree: upstream
final crash: https://syzkaller.appspot.com/x/report.txt?x=14b4cba9a00000
console output: https://syzkaller.appspot.com/x/log.txt?x=10b4cba9a00000

kernel config: https://syzkaller.appspot.com/x/.config?x=9a31528e58cc12e2
dashboard link: https://syzkaller.appspot.com/bug?extid=0165480d4ef07360eeda

syz repro: https://syzkaller.appspot.com/x/repro.syz?x=16cf37c3a00000

Reported-by: syzbot+016548...@syzkaller.appspotmail.com
Fixes: 1548bc4e0512 ("xfrm: policy: delete inexact policies from inexact
list on hash rebuild")

For information about bisection process see: https://goo.gl/tpsmEJ#bisection

[Florian Westphal]

syzbot <syzbot+016548...@syzkaller.appspotmail.com> wrote:
> syzbot has bisected this bug to:
>
> commit 1548bc4e0512700cf757192c106b3a20ab639223
> Author: Florian Westphal <f...@strlen.de>
> Date: Fri Jan 4 13:17:02 2019 +0000
>
> xfrm: policy: delete inexact policies from inexact list on hash rebuild

I'm looking at this now.

[Dmitry Vyukov]

On Tue, Jul 2, 2019 at 8:38 AM Hillf Danton <hda...@sina.com> wrote:
>
>
> On Wed, 26 Jun 2019 20:59:05 -0700 (PDT)

> --- a/net/xfrm/xfrm_policy.c
> +++ b/net/xfrm/xfrm_policy.c
> @@ -1203,6 +1203,11 @@ xfrm_policy_inexact_insert(struct xfrm_policy *policy, u8 dir, int excl)
> return delpol;
> }
>
> +static inline bool xfrm_policy_node_hashed(struct hlist_node *node)
> +{
> + return node->pprev && node->pprev != LIST_POISON2;

Is it right to open code LIST_POISON2 use here? As far as I see all
current uses of LIST_POISON2 are encapsulated in list functions.

> +}
> +
> static void xfrm_hash_rebuild(struct work_struct *work)
> {
> struct net *net = container_of(work, struct net,
> @@ -1315,7 +1320,9 @@ static void xfrm_hash_rebuild(struct work_struct *work)
> chain = policy_hash_bysel(net, &policy->selector,
> policy->family, dir);
>
> - hlist_del_rcu(&policy->bydst);
> + /* check bydst still hashed in case that policy survived bydst resize */
> + if (xfrm_policy_node_hashed(&policy->bydst))
> + hlist_del_rcu(&policy->bydst);
>
> if (!chain) {
> void *p = xfrm_policy_inexact_insert(policy, dir, 0);
> --

[Florian Westphal]

syzbot reported following spat:

BUG: KASAN: use-after-free in __write_once_size include/linux/compiler.h:221

BUG: KASAN: use-after-free in hlist_del_rcu include/linux/rculist.h:455

BUG: KASAN: use-after-free in xfrm_hash_rebuild+0xa0d/0x1000 net/xfrm/xfrm_policy.c:1318
Write of size 8 at addr ffff888095e79c00 by task kworker/1:3/8066

Workqueue: events xfrm_hash_rebuild
Call Trace:

__write_once_size include/linux/compiler.h:221 [inline]

hlist_del_rcu include/linux/rculist.h:455 [inline]
xfrm_hash_rebuild+0xa0d/0x1000 net/xfrm/xfrm_policy.c:1318
process_one_work+0x814/0x1130 kernel/workqueue.c:2269

Allocated by task 8064:
__kmalloc+0x23c/0x310 mm/slab.c:3669

kzalloc include/linux/slab.h:742 [inline]
xfrm_hash_alloc+0x38/0xe0 net/xfrm/xfrm_hash.c:21
xfrm_policy_init net/xfrm/xfrm_policy.c:4036 [inline]
xfrm_net_init+0x269/0xd60 net/xfrm/xfrm_policy.c:4120
ops_init+0x336/0x420 net/core/net_namespace.c:130
setup_net+0x212/0x690 net/core/net_namespace.c:316

The faulting address is the address of the old chain head,
free'd by xfrm_hash_resize().

In xfrm_hash_rehash(), chain heads get re-initialized without
any hlist_del_rcu:

for (i = hmask; i >= 0; i--)
INIT_HLIST_HEAD(odst + i);

Then, hlist_del_rcu() gets called on the about to-be-reinserted policy
when iterating the per-net list of policies.

hlist_del_rcu() will then make chain->first be nonzero again:

static inline void __hlist_del(struct hlist_node *n)
{
struct hlist_node *next = n->next; // address of next element in list
struct hlist_node **pprev = n->pprev;// location of previous elem, this
// can point at chain->first
WRITE_ONCE(*pprev, next); // chain->first points to next elem
if (next)
next->pprev = pprev;

Then, when we walk chainlist to find insertion point, we may find a
non-empty list even though we're supposedly reinserting the first
policy to an empty chain.

To fix this first unlink all exact and inexact policies instead of
zeroing the list heads.

Add the commands equivalent to the syzbot reproducer to xfrm_policy.sh,
without fix KASAN catches the corruption as it happens, SLUB poisoning
detects it a bit later.

Reported-by: syzbot+016548...@syzkaller.appspotmail.com
Fixes: 1548bc4e0512 ("xfrm: policy: delete inexact policies from inexact list on hash rebuild")
Signed-off-by: Florian Westphal <f...@strlen.de>
---
net/xfrm/xfrm_policy.c | 12 ++++++----
tools/testing/selftests/net/xfrm_policy.sh | 27 +++++++++++++++++++++-
2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index b1694d5d15d3..82be7780bbe8 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -1280,13 +1280,17 @@ static void xfrm_hash_rebuild(struct work_struct *work)

hlist_for_each_entry_safe(policy, n,
&net->xfrm.policy_inexact[dir],
- bydst_inexact_list)
+ bydst_inexact_list) {
+ hlist_del_rcu(&policy->bydst);
hlist_del_init(&policy->bydst_inexact_list);
+ }

hmask = net->xfrm.policy_bydst[dir].hmask;
odst = net->xfrm.policy_bydst[dir].table;
- for (i = hmask; i >= 0; i--)
- INIT_HLIST_HEAD(odst + i);
+ for (i = hmask; i >= 0; i--) {
+ hlist_for_each_entry_safe(policy, n, odst + i, bydst)
+ hlist_del_rcu(&policy->bydst);
+ }
if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
/* dir out => dst = remote, src = local */
net->xfrm.policy_bydst[dir].dbits4 = rbits4;
@@ -1315,8 +1319,6 @@ static void xfrm_hash_rebuild(struct work_struct *work)

chain = policy_hash_bysel(net, &policy->selector,
policy->family, dir);

- hlist_del_rcu(&policy->bydst);

-

if (!chain) {
void *p = xfrm_policy_inexact_insert(policy, dir, 0);

diff --git a/tools/testing/selftests/net/xfrm_policy.sh b/tools/testing/selftests/net/xfrm_policy.sh
index 71d7fdc513c1..5445943bf07f 100755
--- a/tools/testing/selftests/net/xfrm_policy.sh
+++ b/tools/testing/selftests/net/xfrm_policy.sh
@@ -257,6 +257,29 @@ check_exceptions()
return $lret
}

+check_hthresh_repeat()
+{
+ local log=$1
+ i=0
+
+ for i in $(seq 1 10);do
+ ip -net ns1 xfrm policy update src e000:0001::0000 dst ff01::0014:0000:0001 dir in tmpl src :: dst :: proto esp mode tunnel priority 100 action allow || break
+ ip -net ns1 xfrm policy set hthresh6 0 28 || break
+
+ ip -net ns1 xfrm policy update src e000:0001::0000 dst ff01::01 dir in tmpl src :: dst :: proto esp mode tunnel priority 100 action allow || break
+ ip -net ns1 xfrm policy set hthresh6 0 28 || break
+ done
+
+ if [ $i -ne 10 ] ;then
+ echo "FAIL: $log" 1>&2
+ ret=1
+ return 1
+ fi
+
+ echo "PASS: $log"
+ return 0
+}
+
#check for needed privileges
if [ "$(id -u)" -ne 0 ];then
echo "SKIP: Need root privileges"
@@ -404,7 +427,9 @@ for n in ns3 ns4;do
ip -net $n xfrm policy set hthresh4 32 32 hthresh6 128 128
sleep $((RANDOM%5))
done
-check_exceptions "exceptions and block policies after hresh change to normal"
+check_exceptions "exceptions and block policies after htresh change to normal"
+
+check_hthresh_repeat "policies with repeated htresh change"

for i in 1 2 3 4;do ip netns del ns$i;done

--
2.21.0

[Steffen Klassert]

Applied, thanks Florian!

[Hillf Danton]

No Sir.

> current uses of LIST_POISON2 are encapsulated in list functions.
>

You are right. The helper should have been tucked in linux/list.h.

--
Hillf