Rework ss_do_close function.

Previously we sent FIN to remote peer during socket
closing, but didn't wait for ack to this FIN. Moreover
if we could not transmit pending skbs we closed socket
and lost these skbs. Now we start to close socket and
wait until remote peer send ACK for our FIN.

fw/sock.c

@@ -545,11 +545,12 @@ EXPORT_SYMBOL(ss_send);
  *
  * Called with locked socket.
  */
-static void
-ss_do_close(struct sock *sk, int flags)
+static int
+ss_do_close(struct sock *sk, int flags, int wait_for_fin)
 {
 	struct sk_buff *skb;
 	int data_was_unread = 0;
+	u8 sk_shutdown = SHUTDOWN_MASK;
 
 	T_DBG2("[%d]: Close socket %p (%s): account=%d refcnt=%u\n",
 	       smp_processor_id(), sk, ss_statename[sk->sk_state],
@@ -572,9 +573,9 @@ ss_do_close(struct sock *sk, int flags)
 	 */
 	sk->sk_lock.owned = 1;
 
-	/* The below is mostly copy-paste from tcp_close(), 5.10.35. */
-	sk->sk_shutdown = SHUTDOWN_MASK;
+	swap(sk->sk_shutdown, sk_shutdown);
 
+	/* The below is mostly copy-paste from tcp_close(), 5.10.35. */
 	while ((skb = __skb_dequeue(&sk->sk_receive_queue))) {
 		u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq;
 
@@ -598,41 +599,15 @@ ss_do_close(struct sock *sk, int flags)
 		}
 		tcp_set_state(sk, TCP_CLOSE);
 		tcp_send_active_reset(sk, sk->sk_allocation);
+		goto adjudge_to_death;
+	} else if (tcp_close_state(sk)) {
+		tcp_send_fin(sk);
 	}
-	else if (tcp_close_state(sk)) {
-		/* The code below is taken from tcp_send_fin(), 5.10.35. */
-		struct sk_buff *skb, *tskb, *tail;
-		struct tcp_sock *tp = tcp_sk(sk);
 
-		tskb = tail = tcp_write_queue_tail(sk);
-		if (!tskb && tcp_under_memory_pressure(sk))
-			tskb = skb_rb_last(&sk->tcp_rtx_queue);
-
-		if (tskb) {
-			/* Send FIN with data if we have any. */
-			TCP_SKB_CB(tskb)->tcp_flags |= TCPHDR_FIN;
-			TCP_SKB_CB(tskb)->end_seq++;
-			tp->write_seq++;
-			if (!tail) {
-				WRITE_ONCE(tp->snd_nxt, tp->snd_nxt + 1);
-				goto adjudge_to_death;
-			}
-		}
-		else {
-			/* No data to send in the socket, allocate new skb. */
-			skb = alloc_skb_fclone(MAX_TCP_HEADER, sk->sk_allocation);
-			if (unlikely(!skb)) {
-				T_WARN("can't send FIN due to bad alloc");
-				goto adjudge_to_death;
-			}
-			INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
-			skb_reserve(skb, MAX_TCP_HEADER);
-			ss_forced_mem_schedule(sk, skb->truesize);
-			tcp_init_nondata_skb(skb, tp->write_seq,
-					     TCPHDR_ACK | TCPHDR_FIN);
-			tcp_queue_skb(sk, skb);
-		}
-		__tcp_push_pending_frames(sk, tcp_current_mss(sk), TCP_NAGLE_OFF);
+	if (wait_for_fin && sk_stream_closing(sk)) {
+		sk->sk_lock.owned = 0;
+		swap(sk->sk_shutdown, sk_shutdown);
+		return -EAGAIN;
 	}
 
 adjudge_to_death:
@@ -659,7 +634,7 @@ ss_do_close(struct sock *sk, int flags)
 			inet_csk_reset_keepalive_timer(sk, tmo - TCP_TIMEWAIT_LEN);
 		} else {
 			tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
-			return;
+			return 0;
 		}
 	}
 	if (sk->sk_state != TCP_CLOSE) {
@@ -687,6 +662,7 @@ ss_do_close(struct sock *sk, int flags)
 			tcp_write_queue_purge(sk);
 		inet_csk_destroy_sock(sk);
 	}
+	return 0;
 }
 
 /**
@@ -701,7 +677,8 @@ ss_do_close(struct sock *sk, int flags)
 static void
 ss_linkerror(struct sock *sk)
 {
-	ss_do_close(sk, 0);
+	int r = ss_do_close(sk, 0, false);
+	BUG_ON(r);
 	ss_conn_drop_guard_exit(sk);
 	sock_put(sk);	/* paired with ss_do_close() */
 }
@@ -1010,7 +987,8 @@ ss_tcp_state_change(struct sock *sk)
 		 * it on our own without calling upper layer hooks.
 		 */
 		if (ss_active_guard_enter(SS_V_ACT_NEWCONN)) {
-			ss_do_close(sk, 0);
+			r = ss_do_close(sk, 0, false);
+			BUG_ON(r);
 			sock_put(sk);
 			/*
 			 * The case of a connect to an upstream server that
@@ -1023,7 +1001,8 @@ ss_tcp_state_change(struct sock *sk)
 		}
 
 		if (!is_srv_sock && ss_active_guard_enter(SS_V_ACT_LIVECONN)) {
-			ss_do_close(sk, 0);
+			r = ss_do_close(sk, 0, false);
+			BUG_ON(r);
 			sock_put(sk);
 			ss_active_guard_exit(SS_V_ACT_NEWCONN);
 			return;
@@ -1074,6 +1053,14 @@ ss_tcp_state_change(struct sock *sk)
 		 */
 		ss_close(sk, SS_F_SYNC);
 	}
+	else if (sk->sk_state == TCP_FIN_WAIT2) {
+		/*
+		 * Received ack to FIN, continue to close connection.
+		 */
+		if (!skb_queue_empty(&sk->sk_receive_queue))
+			ss_tcp_process_data(sk);
+		ss_linkerror(sk);
+	}
 	else if (sk->sk_state == TCP_CLOSE) {
 		/*
 		 * We reach the state on regular tcp_close() (including the
@@ -1389,10 +1376,12 @@ EXPORT_SYMBOL(ss_getpeername);
 static void
 __sk_close_locked(struct sock *sk, int flags)
 {
-	ss_do_close(sk, flags);
+	int r = ss_do_close(sk, flags, true);
 	bh_unlock_sock(sk);
-	ss_conn_drop_guard_exit(sk);
-	sock_put(sk); /* paired with ss_do_close() */
+	if (!r) {
+		ss_conn_drop_guard_exit(sk);
+		sock_put(sk); /* paired with ss_do_close() */
+	}
 }
 
 static void