Fix Python 3.12 build issues in thrift Python (#3276)
- Add pyproject.toml with setuptools build requirement for PEP 517 compliance
- Replace distutils imports with setuptools equivalents
- Use setuptools error names directly (CompileError, ExecError, PlatformError)
- Fix macOS header collision with ntohll/htonll macros in endian.h
- Add a matrix of MacOS versions (macos-15-intel, macos-14, macos-15,
macos-26)
- Add a matrix of non-EOL Python versions for testing
- Remove MSVC2015 from the test matrix (very old).
- Support MSVC2022, the latest in AppVeyor.
- Upgrade tornado, twisted, and zope.interface versions to the first
that support Python 3.12.
- Try to make the test_socket, RunClientServer, and TestServer tests less flaky.
This fixes the ModuleNotFoundError: No module named 'distutils' error
when building thrift with Python 3.12+.
diff --git a/test/cpp/src/TestServer.cpp b/test/cpp/src/TestServer.cpp
index dc95af6..7bb40a0 100644
--- a/test/cpp/src/TestServer.cpp
+++ b/test/cpp/src/TestServer.cpp
@@ -596,7 +596,7 @@
memset(&sa, 0, sizeof(sa));
sa.sun_family = AF_UNIX;
strcpy(sa.sun_path, path.c_str());
- int rv = bind(socket_fd, (struct sockaddr*)&sa, sizeof(sa));
+ int rv = ::bind(socket_fd, (struct sockaddr*)&sa, sizeof(sa));
if (rv == -1) {
std::ostringstream os;
os << "Cannot bind domain socket: " << strerror(errno);
diff --git a/test/py.tornado/test_suite.py b/test/py.tornado/test_suite.py
index 0ee0a9b..fef09f0 100755
--- a/test/py.tornado/test_suite.py
+++ b/test/py.tornado/test_suite.py
@@ -37,7 +37,7 @@
sys.exit(0)
from tornado import gen
-from tornado.testing import AsyncTestCase, get_unused_port, gen_test
+from tornado.testing import AsyncTestCase, bind_unused_port, gen_test
from thrift import TTornado
from thrift.Thrift import TApplicationException
@@ -123,7 +123,8 @@
def setUp(self):
super(ThriftTestCase, self).setUp()
- self.port = get_unused_port()
+ sock, self.port = bind_unused_port()
+ sock.close()
# server
self.handler = TestHandler(self)
diff --git a/test/py/RunClientServer.py b/test/py/RunClientServer.py
index 278f06c..809c93b 100755
--- a/test/py/RunClientServer.py
+++ b/test/py/RunClientServer.py
@@ -102,6 +102,14 @@
raise Exception("Script subprocess failed, retcode=%d, args: %s" % (ret, ' '.join(script_args)))
+def pick_unused_port():
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.bind(('127.0.0.1', 0))
+ port = sock.getsockname()[1]
+ sock.close()
+ return port
+
+
def runServiceTest(libdir, genbase, genpydir, server_class, proto, port, use_zlib, use_ssl, verbose):
env = setup_pypath(libdir, os.path.join(genbase, genpydir))
# Build command line arguments
@@ -129,7 +137,14 @@
cli_args.append('--http=/')
if verbose > 0:
print('Testing server %s: %s' % (server_class, ' '.join(server_args)))
- serverproc = subprocess.Popen(server_args, env=env)
+ popen_kwargs = {'env': env}
+ # Windows uses process groups; POSIX starts a new session so we can killpg().
+ if platform.system() == 'Windows':
+ if hasattr(subprocess, 'CREATE_NEW_PROCESS_GROUP'):
+ popen_kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP
+ else:
+ popen_kwargs['start_new_session'] = True
+ serverproc = subprocess.Popen(server_args, **popen_kwargs)
def ensureServerAlive():
if serverproc.poll() is not None:
@@ -169,8 +184,12 @@
print('PY_GEN: %s' % genpydir, file=sys.stderr)
raise Exception("Client subprocess failed, retcode=%d, args: %s" % (ret, ' '.join(cli_args)))
finally:
- # check that server didn't die
- ensureServerAlive()
+ # check that server didn't die, but still attempt cleanup
+ cleanup_exc = None
+ try:
+ ensureServerAlive()
+ except Exception as exc:
+ cleanup_exc = exc
extra_sleep = EXTRA_DELAY.get(server_class, 0)
if extra_sleep > 0 and verbose > 0:
print('Giving %s (proto=%s,zlib=%s,ssl=%s) an extra %d seconds for child'
@@ -178,8 +197,17 @@
% (server_class, proto, use_zlib, use_ssl, extra_sleep))
time.sleep(extra_sleep)
sig = signal.SIGKILL if platform.system() != 'Windows' else signal.SIGABRT
- os.kill(serverproc.pid, sig)
+ try:
+ if platform.system() == 'Windows':
+ os.kill(serverproc.pid, sig)
+ else:
+ # POSIX: kill the whole process group to reap forked children.
+ os.killpg(serverproc.pid, sig)
+ except OSError:
+ pass
serverproc.wait()
+ if cleanup_exc:
+ raise cleanup_exc
class TestCases(object):
@@ -219,7 +247,8 @@
if self.verbose > 0:
print('\nTest run #%d: (includes %s) Server=%s, Proto=%s, zlib=%s, SSL=%s'
% (test_count, genpydir, try_server, try_proto, with_zlib, with_ssl))
- runServiceTest(self.libdir, self.genbase, genpydir, try_server, try_proto, self.port, with_zlib, with_ssl, self.verbose)
+ port = self.port if self.port else pick_unused_port()
+ runServiceTest(self.libdir, self.genbase, genpydir, try_server, try_proto, port, with_zlib, with_ssl, self.verbose)
if self.verbose > 0:
print('OK: Finished (includes %s) %s / %s proto / zlib=%s / SSL=%s. %d combinations tested.'
% (genpydir, try_server, try_proto, with_zlib, with_ssl, test_count))
@@ -255,7 +284,8 @@
if self.verbose > 0:
print('\nTest run #%d: (includes %s) Server=%s, Proto=%s, zlib=%s, SSL=%s'
% (test_count, genpydir, try_server, try_proto, with_zlib, with_ssl))
- runServiceTest(self.libdir, self.genbase, genpydir, try_server, try_proto, self.port, with_zlib, with_ssl)
+ port = self.port if self.port else pick_unused_port()
+ runServiceTest(self.libdir, self.genbase, genpydir, try_server, try_proto, port, with_zlib, with_ssl, self.verbose)
if self.verbose > 0:
print('OK: Finished (includes %s) %s / %s proto / zlib=%s / SSL=%s. %d combinations tested.'
% (genpydir, try_server, try_proto, with_zlib, with_ssl, test_count))
@@ -268,8 +298,8 @@
parser.add_option('--genpydirs', type='string', dest='genpydirs',
default='default,slots,oldstyle,no_utf8strings,dynamic,dynamicslots,enum,type_hints',
help='directory extensions for generated code, used as suffixes for \"gen-py-*\" added sys.path for individual tests')
- parser.add_option("--port", type="int", dest="port", default=9090,
- help="port number for server to listen on")
+ parser.add_option("--port", type="int", dest="port", default=0,
+ help="port number for server to listen on (0 = auto)")
parser.add_option('-v', '--verbose', action="store_const",
dest="verbose", const=2,
help="verbose output")
diff --git a/test/py/TestServer.py b/test/py/TestServer.py
index 3a2f639..c2723e5 100755
--- a/test/py/TestServer.py
+++ b/test/py/TestServer.py
@@ -28,6 +28,7 @@
from util import local_libpath
sys.path.insert(0, local_libpath())
from thrift.protocol import TProtocol, TProtocolDecorator
+from thrift.Thrift import TException
SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
@@ -328,7 +329,9 @@
tfactory = TTransport.TBufferedTransportFactory()
# if --zlib, then wrap server transport, and use a different transport factory
if options.zlib:
- transport = TZlibTransport.TZlibTransport(transport) # wrap with zlib
+ if server_type != "TProcessPoolServer":
+ transport = TZlibTransport.TZlibTransport(transport) # wrap with zlib
+ # Avoid wrapping the server transport for process pools; TZlibTransport isn't picklable on spawn.
tfactory = TZlibTransport.TZlibTransportFactory()
# do server-specific setup here: