[PATCH 5.15 014/203] net: deal with most data-races in sk_wait_event()

[Greg Kroah-Hartman]

From: Eric Dumazet <edum...@google.com>

[ Upstream commit d0ac89f6f9879fae316c155de77b5173b3e2c9c9 ]

__condition is evaluated twice in sk_wait_event() macro.

First invocation is lockless, and reads can race with writes,
as spotted by syzbot.

BUG: KCSAN: data-race in sk_stream_wait_connect / tcp_disconnect

write to 0xffff88812d83d6a0 of 4 bytes by task 9065 on cpu 1:
tcp_disconnect+0x2cd/0xdb0
inet_shutdown+0x19e/0x1f0 net/ipv4/af_inet.c:911
__sys_shutdown_sock net/socket.c:2343 [inline]
__sys_shutdown net/socket.c:2355 [inline]
__do_sys_shutdown net/socket.c:2363 [inline]
__se_sys_shutdown+0xf8/0x140 net/socket.c:2361
__x64_sys_shutdown+0x31/0x40 net/socket.c:2361
do_syscall_x64 arch/x86/entry/common.c:50 [inline]
do_syscall_64+0x41/0xc0 arch/x86/entry/common.c:80
entry_SYSCALL_64_after_hwframe+0x63/0xcd

read to 0xffff88812d83d6a0 of 4 bytes by task 9040 on cpu 0:
sk_stream_wait_connect+0x1de/0x3a0 net/core/stream.c:75
tcp_sendmsg_locked+0x2e4/0x2120 net/ipv4/tcp.c:1266
tcp_sendmsg+0x30/0x50 net/ipv4/tcp.c:1484
inet6_sendmsg+0x63/0x80 net/ipv6/af_inet6.c:651
sock_sendmsg_nosec net/socket.c:724 [inline]
sock_sendmsg net/socket.c:747 [inline]
__sys_sendto+0x246/0x300 net/socket.c:2142
__do_sys_sendto net/socket.c:2154 [inline]
__se_sys_sendto net/socket.c:2150 [inline]
__x64_sys_sendto+0x78/0x90 net/socket.c:2150
do_syscall_x64 arch/x86/entry/common.c:50 [inline]
do_syscall_64+0x41/0xc0 arch/x86/entry/common.c:80
entry_SYSCALL_64_after_hwframe+0x63/0xcd

value changed: 0x00000000 -> 0x00000068

Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2")
Reported-by: syzbot <syzk...@googlegroups.com>
Signed-off-by: Eric Dumazet <edum...@google.com>
Signed-off-by: David S. Miller <da...@davemloft.net>
Signed-off-by: Sasha Levin <sas...@kernel.org>
---
net/core/stream.c | 12 ++++++------
net/ipv4/tcp_bpf.c | 2 +-
net/llc/af_llc.c | 8 +++++---
net/smc/smc_close.c | 4 ++--
net/smc/smc_rx.c | 4 ++--
net/smc/smc_tx.c | 4 ++--
net/tipc/socket.c | 4 ++--
net/tls/tls_main.c | 3 ++-
8 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/net/core/stream.c b/net/core/stream.c
index cd60746877b1e..422ee97e4f2be 100644
--- a/net/core/stream.c
+++ b/net/core/stream.c
@@ -73,8 +73,8 @@ int sk_stream_wait_connect(struct sock *sk, long *timeo_p)
add_wait_queue(sk_sleep(sk), &wait);
sk->sk_write_pending++;
done = sk_wait_event(sk, timeo_p,
- !sk->sk_err &&
- !((1 << sk->sk_state) &
+ !READ_ONCE(sk->sk_err) &&
+ !((1 << READ_ONCE(sk->sk_state)) &
~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)), &wait);
remove_wait_queue(sk_sleep(sk), &wait);
sk->sk_write_pending--;
@@ -87,9 +87,9 @@ EXPORT_SYMBOL(sk_stream_wait_connect);
* sk_stream_closing - Return 1 if we still have things to send in our buffers.
* @sk: socket to verify
*/
-static inline int sk_stream_closing(struct sock *sk)
+static int sk_stream_closing(const struct sock *sk)
{
- return (1 << sk->sk_state) &
+ return (1 << READ_ONCE(sk->sk_state)) &
(TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK);
}

@@ -142,8 +142,8 @@ int sk_stream_wait_memory(struct sock *sk, long *timeo_p)

set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
sk->sk_write_pending++;
- sk_wait_event(sk, &current_timeo, sk->sk_err ||
- (sk->sk_shutdown & SEND_SHUTDOWN) ||
+ sk_wait_event(sk, &current_timeo, READ_ONCE(sk->sk_err) ||
+ (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) ||
(sk_stream_memory_free(sk) &&
!vm_wait), &wait);
sk->sk_write_pending--;
diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c
index 20ad554af3693..e3a9477293ce4 100644
--- a/net/ipv4/tcp_bpf.c
+++ b/net/ipv4/tcp_bpf.c
@@ -168,7 +168,7 @@ static int tcp_msg_wait_data(struct sock *sk, struct sk_psock *psock,
sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
ret = sk_wait_event(sk, &timeo,
!list_empty(&psock->ingress_msg) ||
- !skb_queue_empty(&sk->sk_receive_queue), &wait);
+ !skb_queue_empty_lockless(&sk->sk_receive_queue), &wait);
sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
remove_wait_queue(sk_sleep(sk), &wait);
return ret;
diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c
index 99305aadaa087..8b9a10d10036f 100644
--- a/net/llc/af_llc.c
+++ b/net/llc/af_llc.c
@@ -581,7 +581,8 @@ static int llc_ui_wait_for_disc(struct sock *sk, long timeout)

add_wait_queue(sk_sleep(sk), &wait);
while (1) {
- if (sk_wait_event(sk, &timeout, sk->sk_state == TCP_CLOSE, &wait))
+ if (sk_wait_event(sk, &timeout,
+ READ_ONCE(sk->sk_state) == TCP_CLOSE, &wait))
break;
rc = -ERESTARTSYS;
if (signal_pending(current))
@@ -601,7 +602,8 @@ static bool llc_ui_wait_for_conn(struct sock *sk, long timeout)

add_wait_queue(sk_sleep(sk), &wait);
while (1) {
- if (sk_wait_event(sk, &timeout, sk->sk_state != TCP_SYN_SENT, &wait))
+ if (sk_wait_event(sk, &timeout,
+ READ_ONCE(sk->sk_state) != TCP_SYN_SENT, &wait))
break;
if (signal_pending(current) || !timeout)
break;
@@ -620,7 +622,7 @@ static int llc_ui_wait_for_busy_core(struct sock *sk, long timeout)
while (1) {
rc = 0;
if (sk_wait_event(sk, &timeout,
- (sk->sk_shutdown & RCV_SHUTDOWN) ||
+ (READ_ONCE(sk->sk_shutdown) & RCV_SHUTDOWN) ||
(!llc_data_accept_state(llc->state) &&
!llc->remote_busy_flag &&
!llc->p_flag), &wait))
diff --git a/net/smc/smc_close.c b/net/smc/smc_close.c
index 84102db5bb314..149a59ecd299f 100644
--- a/net/smc/smc_close.c
+++ b/net/smc/smc_close.c
@@ -64,8 +64,8 @@ static void smc_close_stream_wait(struct smc_sock *smc, long timeout)

rc = sk_wait_event(sk, &timeout,
!smc_tx_prepared_sends(&smc->conn) ||
- sk->sk_err == ECONNABORTED ||
- sk->sk_err == ECONNRESET ||
+ READ_ONCE(sk->sk_err) == ECONNABORTED ||
+ READ_ONCE(sk->sk_err) == ECONNRESET ||
smc->conn.killed,
&wait);
if (rc)
diff --git a/net/smc/smc_rx.c b/net/smc/smc_rx.c
index 45b0575520da4..5b63c250ba604 100644
--- a/net/smc/smc_rx.c
+++ b/net/smc/smc_rx.c
@@ -204,9 +204,9 @@ int smc_rx_wait(struct smc_sock *smc, long *timeo,
sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
add_wait_queue(sk_sleep(sk), &wait);
rc = sk_wait_event(sk, timeo,
- sk->sk_err ||
+ READ_ONCE(sk->sk_err) ||
cflags->peer_conn_abort ||
- sk->sk_shutdown & RCV_SHUTDOWN ||
+ READ_ONCE(sk->sk_shutdown) & RCV_SHUTDOWN ||
conn->killed ||
fcrit(conn),
&wait);
diff --git a/net/smc/smc_tx.c b/net/smc/smc_tx.c
index 31ee76131a79e..a878ea084dbd6 100644
--- a/net/smc/smc_tx.c
+++ b/net/smc/smc_tx.c
@@ -113,8 +113,8 @@ static int smc_tx_wait(struct smc_sock *smc, int flags)
break; /* at least 1 byte of free & no urgent data */
set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
sk_wait_event(sk, &timeo,
- sk->sk_err ||
- (sk->sk_shutdown & SEND_SHUTDOWN) ||
+ READ_ONCE(sk->sk_err) ||
+ (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) ||
smc_cdc_rxed_any_close(conn) ||
(atomic_read(&conn->sndbuf_space) &&
!conn->urg_tx_pend),
diff --git a/net/tipc/socket.c b/net/tipc/socket.c
index f1c3b8eb4b3d3..b34857217fde4 100644
--- a/net/tipc/socket.c
+++ b/net/tipc/socket.c
@@ -313,9 +313,9 @@ static void tsk_rej_rx_queue(struct sock *sk, int error)
tipc_sk_respond(sk, skb, error);
}

-static bool tipc_sk_connected(struct sock *sk)
+static bool tipc_sk_connected(const struct sock *sk)
{
- return sk->sk_state == TIPC_ESTABLISHED;
+ return READ_ONCE(sk->sk_state) == TIPC_ESTABLISHED;
}

/* tipc_sk_type_connectionless - check if the socket is datagram socket
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index abd0c4557cb93..20b8ba4d1dfc4 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -92,7 +92,8 @@ int wait_on_pending_writer(struct sock *sk, long *timeo)
break;
}

- if (sk_wait_event(sk, timeo, !sk->sk_write_pending, &wait))
+ if (sk_wait_event(sk, timeo,
+ !READ_ONCE(sk->sk_write_pending), &wait))
break;
}
remove_wait_queue(sk_sleep(sk), &wait);
--
2.39.2

[Greg Kroah-Hartman]

index cbb268c15251c..5b05b889d31af 100644

index ebf9175119370..2e9547467edbe 100644

--- a/net/ipv4/tcp_bpf.c
+++ b/net/ipv4/tcp_bpf.c
@@ -168,7 +168,7 @@ static int tcp_msg_wait_data(struct sock *sk, struct sk_psock *psock,
sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
ret = sk_wait_event(sk, &timeo,
!list_empty(&psock->ingress_msg) ||
- !skb_queue_empty(&sk->sk_receive_queue), &wait);
+ !skb_queue_empty_lockless(&sk->sk_receive_queue), &wait);
sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
remove_wait_queue(sk_sleep(sk), &wait);
return ret;
diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c

index da7fe94bea2eb..9ffbc667be6cf 100644
--- a/net/llc/af_llc.c
+++ b/net/llc/af_llc.c
@@ -583,7 +583,8 @@ static int llc_ui_wait_for_disc(struct sock *sk, long timeout)

add_wait_queue(sk_sleep(sk), &wait);
while (1) {
- if (sk_wait_event(sk, &timeout, sk->sk_state == TCP_CLOSE, &wait))
+ if (sk_wait_event(sk, &timeout,
+ READ_ONCE(sk->sk_state) == TCP_CLOSE, &wait))
break;
rc = -ERESTARTSYS;
if (signal_pending(current))

@@ -603,7 +604,8 @@ static bool llc_ui_wait_for_conn(struct sock *sk, long timeout)

add_wait_queue(sk_sleep(sk), &wait);
while (1) {
- if (sk_wait_event(sk, &timeout, sk->sk_state != TCP_SYN_SENT, &wait))
+ if (sk_wait_event(sk, &timeout,
+ READ_ONCE(sk->sk_state) != TCP_SYN_SENT, &wait))
break;
if (signal_pending(current) || !timeout)
break;

@@ -622,7 +624,7 @@ static int llc_ui_wait_for_busy_core(struct sock *sk, long timeout)

while (1) {
rc = 0;
if (sk_wait_event(sk, &timeout,
- (sk->sk_shutdown & RCV_SHUTDOWN) ||
+ (READ_ONCE(sk->sk_shutdown) & RCV_SHUTDOWN) ||
(!llc_data_accept_state(llc->state) &&
!llc->remote_busy_flag &&
!llc->p_flag), &wait))
diff --git a/net/smc/smc_close.c b/net/smc/smc_close.c

index 31db7438857c9..dbdf03e8aa5b5 100644
--- a/net/smc/smc_close.c
+++ b/net/smc/smc_close.c
@@ -67,8 +67,8 @@ static void smc_close_stream_wait(struct smc_sock *smc, long timeout)

rc = sk_wait_event(sk, &timeout,
!smc_tx_prepared_sends(&smc->conn) ||
- sk->sk_err == ECONNABORTED ||
- sk->sk_err == ECONNRESET ||
+ READ_ONCE(sk->sk_err) == ECONNABORTED ||
+ READ_ONCE(sk->sk_err) == ECONNRESET ||
smc->conn.killed,
&wait);
if (rc)
diff --git a/net/smc/smc_rx.c b/net/smc/smc_rx.c

index 17c5aee7ee4f2..ffcc9996a3da3 100644
--- a/net/smc/smc_rx.c
+++ b/net/smc/smc_rx.c
@@ -263,9 +263,9 @@ int smc_rx_wait(struct smc_sock *smc, long *timeo,

sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
add_wait_queue(sk_sleep(sk), &wait);
rc = sk_wait_event(sk, timeo,
- sk->sk_err ||
+ READ_ONCE(sk->sk_err) ||
cflags->peer_conn_abort ||
- sk->sk_shutdown & RCV_SHUTDOWN ||
+ READ_ONCE(sk->sk_shutdown) & RCV_SHUTDOWN ||
conn->killed ||
fcrit(conn),
&wait);
diff --git a/net/smc/smc_tx.c b/net/smc/smc_tx.c

index f4b6a71ac488a..45128443f1f10 100644

--- a/net/smc/smc_tx.c
+++ b/net/smc/smc_tx.c
@@ -113,8 +113,8 @@ static int smc_tx_wait(struct smc_sock *smc, int flags)
break; /* at least 1 byte of free & no urgent data */
set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
sk_wait_event(sk, &timeo,
- sk->sk_err ||
- (sk->sk_shutdown & SEND_SHUTDOWN) ||
+ READ_ONCE(sk->sk_err) ||
+ (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) ||
smc_cdc_rxed_any_close(conn) ||
(atomic_read(&conn->sndbuf_space) &&
!conn->urg_tx_pend),
diff --git a/net/tipc/socket.c b/net/tipc/socket.c

index ff5bb9e4731c4..14027a7a7bef8 100644

--- a/net/tipc/socket.c
+++ b/net/tipc/socket.c
@@ -313,9 +313,9 @@ static void tsk_rej_rx_queue(struct sock *sk, int error)
tipc_sk_respond(sk, skb, error);
}

-static bool tipc_sk_connected(struct sock *sk)
+static bool tipc_sk_connected(const struct sock *sk)
{
- return sk->sk_state == TIPC_ESTABLISHED;
+ return READ_ONCE(sk->sk_state) == TIPC_ESTABLISHED;
}

/* tipc_sk_type_connectionless - check if the socket is datagram socket
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c

index b32c112984dd9..f2e7302a4d96b 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -111,7 +111,8 @@ int wait_on_pending_writer(struct sock *sk, long *timeo)

[Greg Kroah-Hartman]

index 434446ab14c57..f5c4e47df1650 100644

index 4380d32f5a5f9..9a2f3638d161d 100644
--- a/net/smc/smc_rx.c
+++ b/net/smc/smc_rx.c
@@ -267,9 +267,9 @@ int smc_rx_wait(struct smc_sock *smc, long *timeo,

index 37edfe10f8c6f..dd73d71c02a99 100644
--- a/net/tipc/socket.c
+++ b/net/tipc/socket.c
@@ -314,9 +314,9 @@ static void tsk_rej_rx_queue(struct sock *sk, int error)

[Greg Kroah-Hartman]

index cd60746877b1e..422ee97e4f2be 100644

index 926e29e84b40b..d0ca1fc325cd6 100644
--- a/net/ipv4/tcp_bpf.c
+++ b/net/ipv4/tcp_bpf.c
@@ -262,7 +262,7 @@ static int tcp_bpf_wait_data(struct sock *sk, struct sk_psock *psock,

sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
ret = sk_wait_event(sk, &timeo,
!list_empty(&psock->ingress_msg) ||
- !skb_queue_empty(&sk->sk_receive_queue), &wait);
+ !skb_queue_empty_lockless(&sk->sk_receive_queue), &wait);
sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
remove_wait_queue(sk_sleep(sk), &wait);
return ret;
diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c

index 99a37c411323e..01e26698285a0 100644
--- a/net/llc/af_llc.c
+++ b/net/llc/af_llc.c
@@ -582,7 +582,8 @@ static int llc_ui_wait_for_disc(struct sock *sk, long timeout)

add_wait_queue(sk_sleep(sk), &wait);
while (1) {
- if (sk_wait_event(sk, &timeout, sk->sk_state == TCP_CLOSE, &wait))
+ if (sk_wait_event(sk, &timeout,
+ READ_ONCE(sk->sk_state) == TCP_CLOSE, &wait))
break;
rc = -ERESTARTSYS;
if (signal_pending(current))

@@ -602,7 +603,8 @@ static bool llc_ui_wait_for_conn(struct sock *sk, long timeout)

add_wait_queue(sk_sleep(sk), &wait);
while (1) {
- if (sk_wait_event(sk, &timeout, sk->sk_state != TCP_SYN_SENT, &wait))
+ if (sk_wait_event(sk, &timeout,
+ READ_ONCE(sk->sk_state) != TCP_SYN_SENT, &wait))
break;
if (signal_pending(current) || !timeout)
break;

@@ -621,7 +623,7 @@ static int llc_ui_wait_for_busy_core(struct sock *sk, long timeout)

while (1) {
rc = 0;
if (sk_wait_event(sk, &timeout,
- (sk->sk_shutdown & RCV_SHUTDOWN) ||
+ (READ_ONCE(sk->sk_shutdown) & RCV_SHUTDOWN) ||
(!llc_data_accept_state(llc->state) &&
!llc->remote_busy_flag &&
!llc->p_flag), &wait))
diff --git a/net/smc/smc_close.c b/net/smc/smc_close.c

index 84102db5bb314..149a59ecd299f 100644
--- a/net/smc/smc_close.c
+++ b/net/smc/smc_close.c
@@ -64,8 +64,8 @@ static void smc_close_stream_wait(struct smc_sock *smc, long timeout)

rc = sk_wait_event(sk, &timeout,
!smc_tx_prepared_sends(&smc->conn) ||
- sk->sk_err == ECONNABORTED ||
- sk->sk_err == ECONNRESET ||
+ READ_ONCE(sk->sk_err) == ECONNABORTED ||
+ READ_ONCE(sk->sk_err) == ECONNRESET ||
smc->conn.killed,
&wait);
if (rc)
diff --git a/net/smc/smc_rx.c b/net/smc/smc_rx.c

index 7f7e983e42b1f..3757aff6c2f00 100644
--- a/net/smc/smc_rx.c
+++ b/net/smc/smc_rx.c
@@ -203,9 +203,9 @@ int smc_rx_wait(struct smc_sock *smc, long *timeo,

sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
add_wait_queue(sk_sleep(sk), &wait);
rc = sk_wait_event(sk, timeo,
- sk->sk_err ||
+ READ_ONCE(sk->sk_err) ||
cflags->peer_conn_abort ||
- sk->sk_shutdown & RCV_SHUTDOWN ||
+ READ_ONCE(sk->sk_shutdown) & RCV_SHUTDOWN ||
conn->killed ||
fcrit(conn),
&wait);
diff --git a/net/smc/smc_tx.c b/net/smc/smc_tx.c

index 52ef1fca0b604..2429f9fc7e0e7 100644
--- a/net/smc/smc_tx.c
+++ b/net/smc/smc_tx.c
@@ -110,8 +110,8 @@ static int smc_tx_wait(struct smc_sock *smc, int flags)

break; /* at least 1 byte of free & no urgent data */
set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
sk_wait_event(sk, &timeo,
- sk->sk_err ||
- (sk->sk_shutdown & SEND_SHUTDOWN) ||
+ READ_ONCE(sk->sk_err) ||
+ (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) ||
smc_cdc_rxed_any_close(conn) ||
(atomic_read(&conn->sndbuf_space) &&
!conn->urg_tx_pend),
diff --git a/net/tipc/socket.c b/net/tipc/socket.c

index 8f3c9fbb99165..7cf9b40b5c73b 100644
--- a/net/tipc/socket.c
+++ b/net/tipc/socket.c
@@ -300,9 +300,9 @@ static void tsk_rej_rx_queue(struct sock *sk, int error)

tipc_sk_respond(sk, skb, error);
}

-static bool tipc_sk_connected(struct sock *sk)
+static bool tipc_sk_connected(const struct sock *sk)
{
- return sk->sk_state == TIPC_ESTABLISHED;
+ return READ_ONCE(sk->sk_state) == TIPC_ESTABLISHED;
}

/* tipc_sk_type_connectionless - check if the socket is datagram socket
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c

index 54863e68f3040..7ee3c8b03a39e 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -92,7 +92,8 @@ int wait_on_pending_writer(struct sock *sk, long *timeo)