THRIFT-2789 TNonblockingServer leaks socket FD's under load
Client: C++
Patch: Sergey <drigh@deviantart.com> and Qiao Mu <qiaomuf@gmail.com>
diff --git a/lib/cpp/src/thrift/concurrency/ThreadManager.cpp b/lib/cpp/src/thrift/concurrency/ThreadManager.cpp
index 9ff2c9a..cc2c3c6 100644
--- a/lib/cpp/src/thrift/concurrency/ThreadManager.cpp
+++ b/lib/cpp/src/thrift/concurrency/ThreadManager.cpp
@@ -290,8 +290,10 @@
if (task->state_ == ThreadManager::Task::EXECUTING) {
try {
task->run();
- } catch (...) {
- // XXX need to log this
+ } catch(const std::exception& e) {
+ GlobalOutput.printf("[ERROR] task->run() raised an exception: %s", e.what());
+ } catch(...) {
+ GlobalOutput.printf("[ERROR] task->run() raised an unknown exception");
}
}
}
diff --git a/lib/cpp/src/thrift/server/TNonblockingServer.cpp b/lib/cpp/src/thrift/server/TNonblockingServer.cpp
index 1cfdef8..9833f10 100644
--- a/lib/cpp/src/thrift/server/TNonblockingServer.cpp
+++ b/lib/cpp/src/thrift/server/TNonblockingServer.cpp
@@ -283,6 +283,7 @@
void forceClose() {
appState_ = APP_CLOSE_CONNECTION;
if (!notifyIOThread()) {
+ close();
throw TException("TConnection::forceClose: failed write on notify pipe");
}
}
@@ -342,6 +343,8 @@
// Signal completion back to the libevent thread via a pipe
if (!connection_->notifyIOThread()) {
+ GlobalOutput.printf("TNonblockingServer: failed to notifyIOThread, closing.");
+ connection_->close();
throw TException("TNonblockingServer::Task::run: failed write on notify pipe");
}
}
@@ -568,6 +571,9 @@
// The ThreadManager is not ready to handle any more tasks (it's probably shutting down).
GlobalOutput.printf("IllegalStateException: Server::process() %s", ise.what());
close();
+ } catch (TimedOutException& to) {
+ GlobalOutput.printf("[ERROR] TimedOutException: Server::process() %s", to.what());
+ close();
}
// Set this connection idle so that libevent doesn't process more
@@ -969,7 +975,10 @@
if (clientConnection->getIOThreadNumber() == 0) {
clientConnection->transition();
} else {
- clientConnection->notifyIOThread();
+ if (!clientConnection->notifyIOThread()) {
+ GlobalOutput.perror("[ERROR] notifyIOThread failed on fresh connection, closing", errno);
+ returnConnection(clientConnection);
+ }
}
// addrLen is written by the accept() call, so needs to be set before the next call.