From a59c0a9296142cc7f30af23030c8bea85b5940f9 Mon Sep 17 00:00:00 2001
From: Amnon Hanuhov <ahanukov@redhat.com>
Date: Thu, 3 Jun 2021 16:57:41 +0300
Subject: [PATCH] crimson/net: Use out_q instead of pending_q

pending_q contains the same messages as in out_q and it is only used
for creating a bytestream out of these messages. We can just use out_q for that.

Signed-off-by: Amnon Hanuhov <ahanukov@redhat.com>
---
 src/crimson/net/Protocol.cc        | 31 +++++++++++++++++++++---------
 src/crimson/net/Protocol.h         |  7 +++++++
 src/crimson/net/SocketConnection.h |  1 -
 3 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/src/crimson/net/Protocol.cc b/src/crimson/net/Protocol.cc
index 50b5c45a335..75a60b32985 100644
--- a/src/crimson/net/Protocol.cc
+++ b/src/crimson/net/Protocol.cc
@@ -90,6 +90,26 @@ void Protocol::close(bool dispatch_reset,
   });
 }
 
+ceph::bufferlist Protocol::sweep_messages_and_move_to_sent(
+      size_t num_msgs,
+      bool require_keepalive,
+      std::optional<utime_t> keepalive_ack,
+      bool require_ack)
+{
+  ceph::bufferlist bl = do_sweep_messages(conn.out_q, 
+                                          num_msgs, 
+                                          require_keepalive, 
+                                          keepalive_ack, 
+                                          require_ack);
+  if (!conn.policy.lossy) {
+    conn.sent.insert(conn.sent.end(),
+                     conn.out_q.begin(),
+                     conn.out_q.end());
+  }
+  conn.out_q.clear();
+  return bl;
+}
+
 seastar::future<> Protocol::send(MessageRef msg)
 {
   if (write_state != write_state_t::drop) {
@@ -222,18 +242,11 @@ seastar::future<> Protocol::do_write_dispatch_sweep()
       if (unlikely(!still_queued)) {
         return try_exit_sweep();
       }
-      conn.pending_q.clear();
-      conn.pending_q.swap(conn.out_q);
-      if (!conn.policy.lossy) {
-        conn.sent.insert(conn.sent.end(),
-                         conn.pending_q.begin(),
-                         conn.pending_q.end());
-      }
       auto acked = ack_left;
       assert(acked == 0 || conn.in_seq > 0);
       // sweep all pending writes with the concrete Protocol
-      return socket->write(do_sweep_messages(
-          conn.pending_q, num_msgs, need_keepalive, keepalive_ack, acked > 0)
+      return socket->write(sweep_messages_and_move_to_sent(
+          num_msgs, need_keepalive, keepalive_ack, acked > 0)
       ).then([this, prv_keepalive_ack=keepalive_ack, acked] {
         need_keepalive = false;
         if (keepalive_ack == prv_keepalive_ack) {
diff --git a/src/crimson/net/Protocol.h b/src/crimson/net/Protocol.h
index dc4e4f2af8f..ce88629ba6c 100644
--- a/src/crimson/net/Protocol.h
+++ b/src/crimson/net/Protocol.h
@@ -65,6 +65,13 @@ class Protocol {
   virtual void notify_write() {};
 
   virtual void on_closed() {}
+ 
+ private:
+  ceph::bufferlist sweep_messages_and_move_to_sent(
+      size_t num_msgs,
+      bool require_keepalive,
+      std::optional<utime_t> keepalive_ack,
+      bool require_ack); 
 
  public:
   const proto_t proto_type;
diff --git a/src/crimson/net/SocketConnection.h b/src/crimson/net/SocketConnection.h
index 28a2491895e..068d8886ac4 100644
--- a/src/crimson/net/SocketConnection.h
+++ b/src/crimson/net/SocketConnection.h
@@ -45,7 +45,6 @@ class SocketConnection : public Connection {
 
   // messages to be resent after connection gets reset
   std::deque<MessageRef> out_q;
-  std::deque<MessageRef> pending_q;
   // messages sent, but not yet acked by peer
   std::deque<MessageRef> sent;