[Devel] [PATCH RH7] ms/vxlan: fix use-after-free on deletion

Pavel Tikhomirov ptikhomirov at virtuozzo.com
Wed Jun 14 12:19:38 MSK 2017


From: Mark Bloch <markb at mellanox.com>

Reproduced on debug kernel with KASAN:

[20903.934764] BUG: KASan: use after free in vxlan_dellink+0x5b2/0x5f0 [vxlan] at addr ffff880014180018
[20903.939483] Write of size 8 by task dockerd/392675
[20903.943568] page:ffffea0000506000 count:0 mapcount:-127 mapping:          (null) index:0x0
[20903.948279] page flags: 0x1fffff00000000()
[20903.952357] page dumped because: kasan: bad access detected
[20903.956635] CPU: 2 PID: 392675 Comm: dockerd ve: 54366ea9-3f08-446c-a794-02b8a9176a72 Not tainted 3.10.0-514.16.1.vz7.32.7.debug #1 32.7
[20903.965876] Hardware name: Virtuozzo KVM, BIOS 1.9.1-5.3.2.vz7.6 04/01/2014
[20903.970907]  ffff880014180018 00000000f359ad59 ffff8801a63d7438 ffffffff8255cce0
[20903.976184]  ffff8801a63d74b8 ffffffff8162352f ffff8801a63d74a0 ffffffff8133cc11
[20903.981393]  0000000000000297 0000000000000297 ffffffffa0f4a4f2 ffff8801bbf14ce8
[20903.986748] Call Trace:
[20903.991106]  [<ffffffff8255cce0>] dump_stack+0x1e/0x20
[20903.995961]  [<ffffffff8162352f>] kasan_report+0x4ff/0x540
[20904.000804]  [<ffffffff8133cc11>] ? lock_acquired+0x331/0xfd0
[20904.005656]  [<ffffffffa0f4a4f2>] ? vxlan_dellink+0x5b2/0x5f0 [vxlan]
[20904.010856]  [<ffffffff8162368c>] __asan_report_store8_noabort+0x1c/0x20
[20904.015990]  [<ffffffffa0f4a4f2>] vxlan_dellink+0x5b2/0x5f0 [vxlan]
[20904.021161]  [<ffffffffa0f49fd1>] ? vxlan_dellink+0x91/0x5f0 [vxlan]
[20904.026385]  [<ffffffffa0f49f40>] ? vxlan_fdb_dump+0x600/0x600 [vxlan]
[20904.031598]  [<ffffffff8217f9b8>] rtnl_delete_link+0xd8/0x150
[20904.036637]  [<ffffffff8217f8e0>] ? __rtnl_link_unregister+0x260/0x260
[20904.041786]  [<ffffffff81a4b8e2>] ? nla_parse+0x1f2/0x290
[20904.046826]  [<ffffffff82182d54>] rtnl_dellink+0x244/0x300
[20904.051865]  [<ffffffff82182b10>] ? rtnl_dump_all+0x3a0/0x3a0
[20904.057010]  [<ffffffff811b992d>] ? ns_capable+0xbd/0x100
[20904.062182]  [<ffffffff821ea0d2>] ? __netlink_ns_capable+0xe2/0x130
[20904.067412]  [<ffffffff82182b10>] ? rtnl_dump_all+0x3a0/0x3a0
[20904.072582]  [<ffffffff8218314a>] rtnetlink_rcv_msg+0x27a/0x770
[20904.077802]  [<ffffffff82182ed0>] ? rtnetlink_rcv+0x40/0x40
[20904.082916]  [<ffffffff82182eb0>] ? rtnetlink_rcv+0x20/0x40
[20904.087978]  [<ffffffff821f2190>] ? netlink_connect+0x500/0x500
[20904.093052]  [<ffffffff825660f0>] ? mutex_lock_interruptible_nested+0xfc0/0xfc0
[20904.098449]  [<ffffffff821fca15>] netlink_rcv_skb+0x2a5/0x3a0
[20904.103574]  [<ffffffff82182ed0>] ? rtnetlink_rcv+0x40/0x40
[20904.108663]  [<ffffffff82182ebf>] rtnetlink_rcv+0x2f/0x40
[20904.113659]  [<ffffffff821fa630>] netlink_unicast+0x420/0x4e0
[20904.118731]  [<ffffffff821fa210>] ? netlink_attachskb+0x680/0x680
[20904.123814]  [<ffffffff81622844>] ? kasan_check_write+0x14/0x20
[20904.128912]  [<ffffffff81a03a3b>] ? memcpy_fromiovec+0x11b/0x170
[20904.133906]  [<ffffffff821fb288>] netlink_sendmsg+0xb98/0x1bd0
[20904.138832]  [<ffffffff81345c79>] ? lock_acquire+0x169/0x460
[20904.143832]  [<ffffffff81282043>] ? rcu_read_lock+0x43/0xd0
[20904.148765]  [<ffffffff821fa6f0>] ? netlink_unicast+0x4e0/0x4e0
[20904.153705]  [<ffffffff81342d58>] ? __lock_acquire+0x6f8/0x2b40
[20904.158676]  [<ffffffff820df2dd>] sock_sendmsg+0x13d/0x1e0
[20904.163561]  [<ffffffff81342660>] ? debug_check_no_locks_freed+0x320/0x320
[20904.168663]  [<ffffffff820df1a0>] ? sockfd_lookup+0x160/0x160
[20904.173451]  [<ffffffff81221326>] ? __wake_up_bit+0xc6/0x110
[20904.178162]  [<ffffffff81221260>] ? wait_woken+0x1d0/0x1d0
[20904.182809]  [<ffffffff820e443e>] SYSC_sendto+0x22e/0x380
[20904.187373]  [<ffffffff820e4210>] ? SYSC_connect+0x2e0/0x2e0
[20904.191979]  [<ffffffff8168e79e>] ? fput+0x16e/0x1a0
[20904.196333]  [<ffffffff815a3a10>] ? copy_page_range+0x980/0x980
[20904.200689]  [<ffffffff81345c79>] ? lock_acquire+0x169/0x460
[20904.205067]  [<ffffffff8122faa4>] ? up_read+0x24/0x40
[20904.209250]  [<ffffffff8257e469>] ? __do_page_fault+0x1e9/0xb30
[20904.213532]  [<ffffffff81348efa>] ? lockdep_sys_exit+0x4a/0x100
[20904.217749]  [<ffffffff819f80c4>] ? lockdep_sys_exit_thunk+0x16/0x18
[20904.221985]  [<ffffffff81333d65>] ? trace_hardirqs_off_caller+0x1e5/0x2c0
[20904.226264]  [<ffffffff820e8625>] SyS_sendto+0x45/0x60
[20904.230307]  [<ffffffff8258f149>] system_call_fastpath+0x16/0x1b
[20904.234467] Memory state around the buggy address:
[20904.238463]  ffff88001417ff00: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc
[20904.242913]  ffff88001417ff80: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc
[20904.247317] >ffff880014180000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
[20904.251683]                             ^
[20904.255539]  ffff880014180080: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
[20904.260011]  ffff880014180100: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
[20904.264473] ==================================================================

In vxlan_dellink->hlist_del_rcu->__hlist_del: vxlan->hlist->pprev points to freed
memory (0xb88 is offset of pprev relative to struct net_device):

0xffffffffa0f4a17e <vxlan_dellink+574>: mov    0xb88(%r12),%r13
/usr/src/debug/kernel-3.10.0-514.16.1.el7/linux-3.10.0-514.16.1.vz7.32.7/drivers/net/vxlan.c: 3105
0xffffffffa0f4a186 <vxlan_dellink+582>: test   %r13,%r13
0xffffffffa0f4a189 <vxlan_dellink+585>: je     0xffffffffa0f4a23d <vxlan_dellink+765>
/usr/src/debug/kernel-3.10.0-514.16.1.el7/linux-3.10.0-514.16.1.vz7.32.7/drivers/net/vxlan.c: 3106
0xffffffffa0f4a18f <vxlan_dellink+591>: callq  0xffffffff813e20a0 <__sanitizer_cov_trace_pc>
/usr/src/debug/kernel-3.10.0-514.16.1.el7/linux-3.10.0-514.16.1.vz7.32.7/include/linux/list.h: 626
0xffffffffa0f4a194 <vxlan_dellink+596>: lea    0xb80(%r12),%rdi
0xffffffffa0f4a19c <vxlan_dellink+604>: movabs $0xdffffc0000000000,%rax
0xffffffffa0f4a1a6 <vxlan_dellink+614>: mov    %rdi,%rdx
0xffffffffa0f4a1a9 <vxlan_dellink+617>: shr    $0x3,%rdx
0xffffffffa0f4a1ad <vxlan_dellink+621>: cmpb   $0x0,(%rdx,%rax,1)
0xffffffffa0f4a1b1 <vxlan_dellink+625>: jne    0xffffffffa0f4a4f7 <vxlan_dellink+1463>
/usr/src/debug/kernel-3.10.0-514.16.1.el7/linux-3.10.0-514.16.1.vz7.32.7/include/linux/list.h: 628
0xffffffffa0f4a1b7 <vxlan_dellink+631>: mov    %r13,%rdx
0xffffffffa0f4a1ba <vxlan_dellink+634>: movabs $0xdffffc0000000000,%rax
/usr/src/debug/kernel-3.10.0-514.16.1.el7/linux-3.10.0-514.16.1.vz7.32.7/include/linux/list.h: 626
0xffffffffa0f4a1c4 <vxlan_dellink+644>: mov    0xb80(%r12),%r15
/usr/src/debug/kernel-3.10.0-514.16.1.el7/linux-3.10.0-514.16.1.vz7.32.7/include/linux/list.h: 628
0xffffffffa0f4a1cc <vxlan_dellink+652>: shr    $0x3,%rdx
0xffffffffa0f4a1d0 <vxlan_dellink+656>: cmpb   $0x0,(%rdx,%rax,1)
0xffffffffa0f4a1d4 <vxlan_dellink+660>: jne    0xffffffffa0f4a4ea <vxlan_dellink+1450>

0xffffffffa0f4a4ea <vxlan_dellink+1450>:        mov    %r13,%rdi
0xffffffffa0f4a4ed <vxlan_dellink+1453>:        callq  0xffffffff81623670 <__asan_report_store8_noabort>
0xffffffffa0f4a4f2 <vxlan_dellink+1458>:        jmpq   0xffffffffa0f4a1da <vxlan_dellink+666>

https://jira.sw.ru/browse/PSBM-67263

Adding a vxlan interface to a socket isn't symmetrical, while adding
is done in vxlan_open() the deletion is done in vxlan_dellink().
This can cause a use-after-free error when we close the vxlan
interface before deleting it.

We add vxlan_vs_del_dev() to match vxlan_vs_add_dev() and call
it from vxlan_stop() to match the call from vxlan_open().

Fixes: 56ef9c909b40 ("vxlan: Move socket initialization to within rtnl scope")
Acked-by: Jiri Benc <jbenc at redhat.com>
Tested-by: Roi Dayan <roid at mellanox.com>
Signed-off-by: Mark Bloch <markb at mellanox.com>
Acked-by: Roopa Prabhu <roopa at cumulusnetworks.com>
Signed-off-by: David S. Miller <davem at davemloft.net>
Signed-off-by: Pavel Tikhomirov <ptikhomirov at virtuozzo.com>
---
 drivers/net/vxlan.c | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c
index 5670d15..ec3a83b 100644
--- a/drivers/net/vxlan.c
+++ b/drivers/net/vxlan.c
@@ -77,6 +77,8 @@ static const u8 all_zeros_mac[ETH_ALEN + 2];
 
 static int vxlan_sock_add(struct vxlan_dev *vxlan);
 
+static void vxlan_vs_del_dev(struct vxlan_dev *vxlan);
+
 /* per-network namespace private data for this module */
 struct vxlan_net {
 	struct list_head  vxlan_list;
@@ -1065,6 +1067,8 @@ static void vxlan_sock_release(struct vxlan_dev *vxlan)
 
 	synchronize_net();
 
+	vxlan_vs_del_dev(vxlan);
+
 	if (ipv4) {
 		udp_tunnel_sock_release(vxlan->vn4_sock->sock);
 		kfree(vxlan->vn4_sock);
@@ -2307,6 +2311,15 @@ static void vxlan_cleanup(unsigned long arg)
 	mod_timer(&vxlan->age_timer, next_timer);
 }
 
+static void vxlan_vs_del_dev(struct vxlan_dev *vxlan)
+{
+	struct vxlan_net *vn = net_generic(vxlan->net, vxlan_net_id);
+
+	spin_lock(&vn->sock_lock);
+	hlist_del_init_rcu(&vxlan->hlist);
+	spin_unlock(&vn->sock_lock);
+}
+
 static void vxlan_vs_add_dev(struct vxlan_sock *vs, struct vxlan_dev *vxlan)
 {
 	struct vxlan_net *vn = net_generic(vxlan->net, vxlan_net_id);
@@ -3099,12 +3112,6 @@ static int vxlan_newlink(struct net *src_net, struct net_device *dev,
 static void vxlan_dellink(struct net_device *dev, struct list_head *head)
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
-	struct vxlan_net *vn = net_generic(vxlan->net, vxlan_net_id);
-
-	spin_lock(&vn->sock_lock);
-	if (!hlist_unhashed(&vxlan->hlist))
-		hlist_del_rcu(&vxlan->hlist);
-	spin_unlock(&vn->sock_lock);
 
 	gro_cells_destroy(&vxlan->gro_cells);
 	list_del(&vxlan->next);
-- 
2.9.4



More information about the Devel mailing list