From 438b4e43cbbe8fa63764fef4563ed1b44ed36c4c Mon Sep 17 00:00:00 2001
From: Ruifeng Yang <yangruifeng.09209@h3c.com>
Date: Mon, 7 Sep 2015 09:05:38 +0800
Subject: [PATCH] msg: we should set the socket options before connect or
 listen in order to have it take effect.

Signed-off-by: Ruifeng Yang <149233652@qq.com>
---
 src/msg/async/AsyncConnection.cc |  1 -
 src/msg/async/AsyncMessenger.cc  |  3 +++
 src/msg/async/net_handler.cc     |  5 +++--
 src/msg/simple/Accepter.cc       | 10 ++++++++++
 src/msg/simple/Pipe.cc           |  5 +++--
 5 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/src/msg/async/AsyncConnection.cc b/src/msg/async/AsyncConnection.cc
index 36a19f80ccc..37258b337a8 100644
--- a/src/msg/async/AsyncConnection.cc
+++ b/src/msg/async/AsyncConnection.cc
@@ -975,7 +975,6 @@ int AsyncConnection::_process_connection()
         if (r < 0) {
           goto fail;
         }
-        net.set_socket_options(sd);
 
         center->create_file_event(sd, EVENT_READABLE, read_handler);
         state = STATE_CONNECTING_WAIT_BANNER;
diff --git a/src/msg/async/AsyncMessenger.cc b/src/msg/async/AsyncMessenger.cc
index c4be6a7746b..e5e393a4fb4 100644
--- a/src/msg/async/AsyncMessenger.cc
+++ b/src/msg/async/AsyncMessenger.cc
@@ -97,6 +97,9 @@ int Processor::bind(const entity_addr_t &bind_addr, const set<int>& avoid_ports)
     listen_sd = -1;
     return -errno;
   }
+
+  net.set_socket_options(listen_sd);
+
   // use whatever user specified (if anything)
   entity_addr_t listen_addr = bind_addr;
   listen_addr.set_family(family);
diff --git a/src/msg/async/net_handler.cc b/src/msg/async/net_handler.cc
index 8e6468cfde3..2639fdc3b2b 100644
--- a/src/msg/async/net_handler.cc
+++ b/src/msg/async/net_handler.cc
@@ -116,6 +116,9 @@ int NetHandler::generic_connect(const entity_addr_t& addr, bool nonblock)
       return ret;
     }
   }
+
+  set_socket_options(s);
+
   ret = ::connect(s, (sockaddr*)&addr.addr, addr.addr_size());
   if (ret < 0) {
     if (errno == EINPROGRESS && nonblock)
@@ -126,8 +129,6 @@ int NetHandler::generic_connect(const entity_addr_t& addr, bool nonblock)
     return -errno;
   }
 
-  set_socket_options(s);
-
   return s;
 }
 
diff --git a/src/msg/simple/Accepter.cc b/src/msg/simple/Accepter.cc
index 7d989a93691..3333693b706 100644
--- a/src/msg/simple/Accepter.cc
+++ b/src/msg/simple/Accepter.cc
@@ -140,6 +140,16 @@ int Accepter::bind(const entity_addr_t &bind_addr, const set<int>& avoid_ports)
     return rc;
   }
   
+  if (msgr->cct->_conf->ms_tcp_rcvbuf) {
+    int size = msgr->cct->_conf->ms_tcp_rcvbuf;
+    rc = ::setsockopt(listen_sd, SOL_SOCKET, SO_RCVBUF, (void*)&size, sizeof(size));
+    if (rc < 0)  {
+      rc = -errno;
+      lderr(msgr->cct) << "accepter.bind failed to set SO_RCVBUF to " << size << ": " << cpp_strerror(r) << dendl;
+      return rc;
+    }
+  }
+
   ldout(msgr->cct,10) << "accepter.bind bound to " << listen_addr << dendl;
 
   // listen!
diff --git a/src/msg/simple/Pipe.cc b/src/msg/simple/Pipe.cc
index a9b3b54e870..7d6419c3ee2 100644
--- a/src/msg/simple/Pipe.cc
+++ b/src/msg/simple/Pipe.cc
@@ -913,6 +913,9 @@ int Pipe::connect()
   }
 
   recv_reset();
+
+  set_socket_options();
+
   // connect!
   ldout(msgr->cct,10) << "connecting to " << peer_addr << dendl;
   rc = ::connect(sd, (sockaddr*)&peer_addr.addr, peer_addr.addr_size());
@@ -922,8 +925,6 @@ int Pipe::connect()
     goto fail;
   }
 
-  set_socket_options();
-
   // verify banner
   // FIXME: this should be non-blocking, or in some other way verify the banner as we get it.
   if (tcp_read((char*)&banner, strlen(CEPH_BANNER)) < 0) {