THRIFT-2789 TNonblockingServer leaks socket FD's under load
Client: C++
Patch: Sergey <drigh@deviantart.com> and Qiao Mu <qiaomuf@gmail.com>
diff --git a/lib/cpp/src/thrift/concurrency/ThreadManager.cpp b/lib/cpp/src/thrift/concurrency/ThreadManager.cpp
index 9ff2c9a..cc2c3c6 100644
--- a/lib/cpp/src/thrift/concurrency/ThreadManager.cpp
+++ b/lib/cpp/src/thrift/concurrency/ThreadManager.cpp
@@ -290,8 +290,10 @@
         if (task->state_ == ThreadManager::Task::EXECUTING) {
           try {
             task->run();
-          } catch (...) {
-            // XXX need to log this
+          } catch(const std::exception& e) {
+            GlobalOutput.printf("[ERROR] task->run() raised an exception: %s", e.what());
+          } catch(...) {
+            GlobalOutput.printf("[ERROR] task->run() raised an unknown exception");
           }
         }
       }
diff --git a/lib/cpp/src/thrift/server/TNonblockingServer.cpp b/lib/cpp/src/thrift/server/TNonblockingServer.cpp
index 1cfdef8..9833f10 100644
--- a/lib/cpp/src/thrift/server/TNonblockingServer.cpp
+++ b/lib/cpp/src/thrift/server/TNonblockingServer.cpp
@@ -283,6 +283,7 @@
   void forceClose() {
     appState_ = APP_CLOSE_CONNECTION;
     if (!notifyIOThread()) {
+      close();
       throw TException("TConnection::forceClose: failed write on notify pipe");
     }
   }
@@ -342,6 +343,8 @@
 
     // Signal completion back to the libevent thread via a pipe
     if (!connection_->notifyIOThread()) {
+      GlobalOutput.printf("TNonblockingServer: failed to notifyIOThread, closing.");
+      connection_->close();
       throw TException("TNonblockingServer::Task::run: failed write on notify pipe");
     }
   }
@@ -568,6 +571,9 @@
         // The ThreadManager is not ready to handle any more tasks (it's probably shutting down).
         GlobalOutput.printf("IllegalStateException: Server::process() %s", ise.what());
         close();
+      } catch (TimedOutException& to) {
+        GlobalOutput.printf("[ERROR] TimedOutException: Server::process() %s", to.what());
+        close();
       }
 
       // Set this connection idle so that libevent doesn't process more
@@ -969,7 +975,10 @@
     if (clientConnection->getIOThreadNumber() == 0) {
       clientConnection->transition();
     } else {
-      clientConnection->notifyIOThread();
+      if (!clientConnection->notifyIOThread()) {
+        GlobalOutput.perror("[ERROR] notifyIOThread failed on fresh connection, closing", errno);
+        returnConnection(clientConnection);
+      }
     }
 
     // addrLen is written by the accept() call, so needs to be set before the next call.