THRIFT-3613 Port Python C extension to Python 3
Client: Python
Patch: Nobuaki Sukegawa

This closes #845
diff --git a/lib/py/setup.py b/lib/py/setup.py
index 67e9d52..ca10096 100644
--- a/lib/py/setup.py
+++ b/lib/py/setup.py
@@ -19,7 +19,6 @@
 # under the License.
 #
 
-import platform
 import sys
 try:
     from setuptools import setup, Extension
@@ -108,11 +107,7 @@
           )
 
 try:
-    with_binary = False
-    # Don't even try to build the C module unless we're on CPython 2.x.
-    # TODO: fix it for CPython 3.x
-    if platform.python_implementation() == 'CPython' and sys.version_info < (3,):
-        with_binary = True
+    with_binary = True
     run_setup(with_binary)
 except BuildFailed:
     print()
diff --git a/lib/py/src/compat.py b/lib/py/src/compat.py
index 42403ea..787149a 100644
--- a/lib/py/src/compat.py
+++ b/lib/py/src/compat.py
@@ -34,13 +34,7 @@
     from io import BytesIO as BufferIO
 
     def binary_to_str(bin_val):
-        try:
-            return bin_val.decode('utf8')
-        except:
-            return bin_val
+        return bin_val.decode('utf8')
 
     def str_to_binary(str_val):
-        try:
-            return bytes(str_val, 'utf8')
-        except:
-            return str_val
+        return bytes(str_val, 'utf8')
diff --git a/lib/py/src/ext/module.cpp b/lib/py/src/ext/module.cpp
index 82e3fe7..5ffc155 100644
--- a/lib/py/src/ext/module.cpp
+++ b/lib/py/src/ext/module.cpp
@@ -142,6 +142,24 @@
     {NULL, NULL, 0, NULL} /* Sentinel */
 };
 
+#if PY_MAJOR_VERSION >= 3
+
+static struct PyModuleDef ThriftFastBinaryDef = {PyModuleDef_HEAD_INIT,
+                                                 "thrift.protocol.fastbinary",
+                                                 NULL,
+                                                 0,
+                                                 ThriftFastBinaryMethods,
+                                                 NULL,
+                                                 NULL,
+                                                 NULL,
+                                                 NULL};
+
+#define INITERROR return NULL;
+
+PyObject* PyInit_fastbinary() {
+
+#else
+
 #define INITERROR return;
 
 void initfastbinary() {
@@ -150,6 +168,8 @@
   if (PycStringIO == NULL)
     INITERROR
 
+#endif
+
   const rlim_t kStackSize = 16 * 1024 * 1024; // min stack size = 16 MB
   struct rlimit rl;
   int result;
@@ -181,9 +201,16 @@
 #undef INIT_INTERN_STRING
 
   PyObject* module =
+#if PY_MAJOR_VERSION >= 3
+      PyModule_Create(&ThriftFastBinaryDef);
+#else
       Py_InitModule("thrift.protocol.fastbinary", ThriftFastBinaryMethods);
+#endif
   if (module == NULL)
     INITERROR;
 
+#if PY_MAJOR_VERSION >= 3
+  return module;
+#endif
 }
 }
diff --git a/lib/py/src/ext/protocol.tcc b/lib/py/src/ext/protocol.tcc
index 3df83a1..554ba6e 100644
--- a/lib/py/src/ext/protocol.tcc
+++ b/lib/py/src/ext/protocol.tcc
@@ -23,12 +23,18 @@
 #define CHECK_RANGE(v, min, max) (((v) <= (max)) && ((v) >= (min)))
 #define INIT_OUTBUF_SIZE 128
 
+#if PY_MAJOR_VERSION < 3
 #include <cStringIO.h>
+#else
+#include <algorithm>
+#endif
 
 namespace apache {
 namespace thrift {
 namespace py {
 
+#if PY_MAJOR_VERSION < 3
+
 namespace detail {
 
 inline bool input_check(PyObject* input) {
@@ -101,6 +107,82 @@
   return true;
 }
 
+#else
+
+namespace detail {
+
+inline bool input_check(PyObject* input) {
+  // TODO: Check for BytesIO type
+  return true;
+}
+
+inline EncodeBuffer* new_encode_buffer(size_t size) {
+  EncodeBuffer* buffer = new EncodeBuffer;
+  buffer->buf.reserve(size);
+  buffer->pos = 0;
+  return buffer;
+}
+
+struct bytesio {
+  PyObject_HEAD
+#if PY_MINOR_VERSION < 5
+      char* buf;
+#else
+      PyObject* buf;
+#endif
+  Py_ssize_t pos;
+  Py_ssize_t string_size;
+};
+
+inline int read_buffer(PyObject* buf, char** output, int len) {
+  bytesio* buf2 = reinterpret_cast<bytesio*>(buf);
+#if PY_MINOR_VERSION < 5
+  *output = buf2->buf + buf2->pos;
+#else
+  *output = PyBytes_AS_STRING(buf2->buf) + buf2->pos;
+#endif
+  Py_ssize_t pos0 = buf2->pos;
+  buf2->pos = std::min(buf2->pos + static_cast<Py_ssize_t>(len), buf2->string_size);
+  return static_cast<int>(buf2->pos - pos0);
+}
+}
+
+template <typename Impl>
+inline ProtocolBase<Impl>::~ProtocolBase() {
+  if (output_) {
+    delete output_;
+  }
+}
+
+template <typename Impl>
+inline bool ProtocolBase<Impl>::isUtf8(PyObject* typeargs) {
+  // while condition for py2 is "arg == 'UTF8'", it should be "arg != 'BINARY'" for py3.
+  // HACK: check the length and don't bother reading the value
+  return !PyUnicode_Check(typeargs) || PyUnicode_GET_LENGTH(typeargs) != 6;
+}
+
+template <typename Impl>
+PyObject* ProtocolBase<Impl>::getEncodedValue() {
+  return PyBytes_FromStringAndSize(output_->buf.data(), output_->buf.size());
+}
+
+template <typename Impl>
+inline bool ProtocolBase<Impl>::writeBuffer(char* data, size_t size) {
+  size_t need = size + output_->pos;
+  if (output_->buf.capacity() < need) {
+    try {
+      output_->buf.reserve(need);
+    } catch (std::bad_alloc& ex) {
+      PyErr_SetString(PyExc_MemoryError, "Failed to allocate write buffer");
+      return false;
+    }
+  }
+  std::copy(data, data + size, std::back_inserter(output_->buf));
+  return true;
+}
+
+#endif
+
 namespace detail {
 
 #define DECLARE_OP_SCOPE(name, op)                                                                 \
@@ -192,8 +274,8 @@
     return false;
   } else {
     // using building functions as this is a rare codepath
-    ScopedPyObject newiobuf(
-        PyObject_CallFunction(input_.refill_callable.get(), refill_signature, *output, rlen, len, NULL));
+    ScopedPyObject newiobuf(PyObject_CallFunction(input_.refill_callable.get(), refill_signature,
+                                                  *output, rlen, len, NULL));
     if (!newiobuf) {
       return false;
     }
diff --git a/lib/py/src/ext/types.cpp b/lib/py/src/ext/types.cpp
index f3a29a2..849ab2f 100644
--- a/lib/py/src/ext/types.cpp
+++ b/lib/py/src/ext/types.cpp
@@ -26,7 +26,11 @@
 
 PyObject* ThriftModule = NULL;
 
+#if PY_MAJOR_VERSION < 3
 char refill_signature[] = {'s', '#', 'i'};
+#else
+const char* refill_signature = "y#i";
+#endif
 
 bool parse_struct_item_spec(StructItemSpec* dest, PyObject* spec_tuple) {
   // i'd like to use ParseArgs here, but it seems to be a bottleneck.
diff --git a/lib/py/src/ext/types.h b/lib/py/src/ext/types.h
index 749bb68..0dd5d96 100644
--- a/lib/py/src/ext/types.h
+++ b/lib/py/src/ext/types.h
@@ -22,6 +22,18 @@
 
 #include <Python.h>
 
+#if PY_MAJOR_VERSION >= 3
+
+#include <vector>
+
+// TODO: better macros
+#define PyInt_AsLong(v) PyLong_AsLong(v)
+#define PyInt_FromLong(v) PyLong_FromLong(v)
+
+#define PyString_InternFromString(v) PyUnicode_InternFromString(v)
+
+#endif
+
 #define INTERN_STRING(value) _intern_##value
 
 #define INT_CONV_ERROR_OCCURRED(v) (((v) == -1) && PyErr_Occurred())
@@ -104,8 +116,16 @@
   ScopedPyObject refill_callable;
 };
 
+#if PY_MAJOR_VERSION < 3
 extern char refill_signature[3];
 typedef PyObject EncodeBuffer;
+#else
+extern const char* refill_signature;
+struct EncodeBuffer {
+  std::vector<char> buf;
+  size_t pos;
+};
+#endif
 
 /**
  * A cache of the spec_args for a set or list,
diff --git a/lib/py/test/_import_local_thrift.py b/lib/py/test/_import_local_thrift.py
index 1741669..d223122 100644
--- a/lib/py/test/_import_local_thrift.py
+++ b/lib/py/test/_import_local_thrift.py
@@ -1,13 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import glob
 import os
 import sys
 
-
 SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__))
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(SCRIPT_DIR)))
 
-if sys.version_info[0] == 2:
-    import glob
-    libdir = glob.glob(os.path.join(ROOT_DIR, 'lib', 'py', 'build', 'lib.*'))[0]
-    sys.path.insert(0, libdir)
-else:
-    sys.path.insert(0, os.path.join(ROOT_DIR, 'lib', 'py', 'build', 'lib'))
+for libpath in glob.glob(os.path.join(ROOT_DIR, 'lib', 'py', 'build', 'lib.*')):
+    if libpath.endswith('-%d.%d' % (sys.version_info[0], sys.version_info[1])):
+        sys.path.insert(0, libpath)
+        break
diff --git a/test/features/local_thrift/__init__.py b/test/features/local_thrift/__init__.py
index 0a0bb0b..c85cebe 100644
--- a/test/features/local_thrift/__init__.py
+++ b/test/features/local_thrift/__init__.py
@@ -1,14 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import glob
 import os
 import sys
 
-SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__))
-ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(SCRIPT_DIR)))
+_SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__))
+_ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(_SCRIPT_DIR)))
+_LIBDIR = os.path.join(_ROOT_DIR, 'lib', 'py', 'build', 'lib.*')
 
-if sys.version_info[0] == 2:
-    import glob
-    libdir = glob.glob(os.path.join(ROOT_DIR, 'lib', 'py', 'build', 'lib.*'))[0]
-    sys.path.insert(0, libdir)
-    thrift = __import__('thrift')
-else:
-    sys.path.insert(0, os.path.join(ROOT_DIR, 'lib', 'py', 'build', 'lib'))
-    thrift = __import__('thrift')
+for libpath in glob.glob(_LIBDIR):
+    if libpath.endswith('-%d.%d' % (sys.version_info[0], sys.version_info[1])):
+        sys.path.insert(0, libpath)
+        thrift = __import__('thrift')
+        break
diff --git a/test/known_failures_Linux.json b/test/known_failures_Linux.json
index 6cd595b..9ca31c4 100644
--- a/test/known_failures_Linux.json
+++ b/test/known_failures_Linux.json
@@ -60,8 +60,12 @@
   "csharp-nodejs_json_framed-ip-ssl",
   "csharp-perl_binary_buffered-ip-ssl",
   "csharp-perl_binary_framed-ip-ssl",
+  "csharp-py3_binary-accel_buffered-ip-ssl",
+  "csharp-py3_binary-accel_framed-ip-ssl",
   "csharp-py3_binary_buffered-ip-ssl",
   "csharp-py3_binary_framed-ip-ssl",
+  "csharp-py3_compact-accelc_buffered-ip-ssl",
+  "csharp-py3_compact-accelc_framed-ip-ssl",
   "csharp-py3_compact_buffered-ip-ssl",
   "csharp-py3_compact_framed-ip-ssl",
   "csharp-py3_json_buffered-ip-ssl",
@@ -131,6 +135,8 @@
   "py-perl_accel-binary_framed-ip-ssl",
   "py-perl_binary_buffered-ip-ssl",
   "py-perl_binary_framed-ip-ssl",
+  "py3-perl_accel-binary_buffered-ip-ssl",
+  "py3-perl_accel-binary_framed-ip-ssl",
   "py3-perl_binary_buffered-ip-ssl",
   "py3-perl_binary_framed-ip-ssl"
 ]
diff --git a/test/py/FastbinaryTest.py b/test/py/FastbinaryTest.py
index d688cd5..db3ef8b 100755
--- a/test/py/FastbinaryTest.py
+++ b/test/py/FastbinaryTest.py
@@ -122,7 +122,7 @@
     def _check_write(self, o):
         trans_fast = TTransport.TMemoryBuffer()
         trans_slow = TTransport.TMemoryBuffer()
-        prot_fast = self._fast(trans_fast)
+        prot_fast = self._fast(trans_fast, fallback=False)
         prot_slow = self._slow(trans_slow)
 
         o.write(prot_fast)
@@ -140,7 +140,7 @@
         slow_version_binary = prot.trans.getvalue()
 
         prot = self._fast(
-            TTransport.TMemoryBuffer(slow_version_binary))
+            TTransport.TMemoryBuffer(slow_version_binary), fallback=False)
         c = o.__class__()
         c.read(prot)
         if c != o:
@@ -152,7 +152,7 @@
 
         prot = self._fast(
             TTransport.TBufferedTransport(
-                TTransport.TMemoryBuffer(slow_version_binary)))
+                TTransport.TMemoryBuffer(slow_version_binary)), fallback=False)
         c = o.__class__()
         c.read(prot)
         if c != o:
@@ -187,7 +187,7 @@
         o = Backwards(**{"first_tag2": 4, "second_tag1": 2})
         trans_fast = TTransport.TMemoryBuffer()
         trans_slow = TTransport.TMemoryBuffer()
-        prot_fast = self._fast(trans_fast)
+        prot_fast = self._fast(trans_fast, fallback=False)
         prot_slow = self._slow(trans_slow)
 
         o.write(prot_fast)
@@ -196,7 +196,7 @@
         MINE = trans_fast.getvalue()
         assert id(ORIG) != id(MINE)
 
-        prot = self._fast(TTransport.TMemoryBuffer())
+        prot = self._fast(TTransport.TMemoryBuffer(), fallback=False)
         o.write(prot)
         prot = self._slow(
             TTransport.TMemoryBuffer(prot.trans.getvalue()))
@@ -218,12 +218,12 @@
 from __main__ import hm, rs, TDevNullTransport
 from thrift.protocol.{0} import {0}{1}
 trans = TDevNullTransport()
-prot = {0}{1}(trans)
+prot = {0}{1}(trans{2})
 """
 
-    setup_fast = setup.format(protocol, 'Accelerated')
+    setup_fast = setup.format(protocol, 'Accelerated', ', fallback=False')
     if not skip_slow:
-        setup_slow = setup.format(protocol, '')
+        setup_slow = setup.format(protocol, '', '')
 
     print("Starting Benchmarks")
 
diff --git a/test/py/RunClientServer.py b/test/py/RunClientServer.py
index 150f2be..d4a9cb2 100755
--- a/test/py/RunClientServer.py
+++ b/test/py/RunClientServer.py
@@ -22,7 +22,6 @@
 from __future__ import division
 from __future__ import print_function
 import copy
-import glob
 import os
 import signal
 import socket
@@ -31,10 +30,9 @@
 import time
 from optparse import OptionParser
 
+from util import local_libpath
+
 SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
-ROOT_DIR = os.path.dirname(os.path.dirname(SCRIPT_DIR))
-DEFAULT_LIBDIR_GLOB = os.path.join(ROOT_DIR, 'lib', 'py', 'build', 'lib.*')
-DEFAULT_LIBDIR_PY3 = os.path.join(ROOT_DIR, 'lib', 'py', 'build', 'lib')
 
 SCRIPTS = [
     'FastbinaryTest.py',
@@ -244,13 +242,6 @@
         return test_count
 
 
-def default_libdir():
-    if sys.version_info[0] == 2:
-        return glob.glob(DEFAULT_LIBDIR_GLOB)[0]
-    else:
-        return DEFAULT_LIBDIR_PY3
-
-
 def main():
     parser = OptionParser()
     parser.add_option('--all', action="store_true", dest='all')
@@ -265,7 +256,7 @@
     parser.add_option('-q', '--quiet', action="store_const",
                       dest="verbose", const=0,
                       help="minimal output")
-    parser.add_option('-L', '--libdir', dest="libdir", default=default_libdir(),
+    parser.add_option('-L', '--libdir', dest="libdir", default=local_libpath(),
                       help="directory path that contains Thrift Python library")
     parser.add_option('--gen-base', dest="gen_base", default=SCRIPT_DIR,
                       help="directory path that contains Thrift Python library")
diff --git a/test/py/SerializationTest.py b/test/py/SerializationTest.py
index d6308f0..f4f3a4f 100755
--- a/test/py/SerializationTest.py
+++ b/test/py/SerializationTest.py
@@ -292,7 +292,7 @@
 
 
 class AcceleratedBinaryTest(AbstractTest):
-    protocol_factory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
+    protocol_factory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory(fallback=False)
 
 
 class CompactProtocolTest(AbstractTest):
@@ -300,7 +300,7 @@
 
 
 class AcceleratedCompactTest(AbstractTest):
-    protocol_factory = TCompactProtocol.TCompactProtocolAcceleratedFactory()
+    protocol_factory = TCompactProtocol.TCompactProtocolAcceleratedFactory(fallback=False)
 
 
 class JSONProtocolTest(AbstractTest):
diff --git a/test/py/TestClient.py b/test/py/TestClient.py
index e83a880..5de9fa3 100755
--- a/test/py/TestClient.py
+++ b/test/py/TestClient.py
@@ -26,9 +26,9 @@
 import unittest
 from optparse import OptionParser
 
+from util import local_libpath
+
 SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
-ROOT_DIR = os.path.dirname(os.path.dirname(SCRIPT_DIR))
-DEFAULT_LIBDIR_GLOB = os.path.join(ROOT_DIR, 'lib', 'py', 'build', 'lib.*')
 
 
 class AbstractTest(unittest.TestCase):
@@ -268,12 +268,12 @@
 
 class AcceleratedBinaryTest(AbstractTest):
     def get_protocol(self, transport):
-        return TBinaryProtocol.TBinaryProtocolAcceleratedFactory().getProtocol(transport)
+        return TBinaryProtocol.TBinaryProtocolAcceleratedFactory(fallback=False).getProtocol(transport)
 
 
 class AcceleratedCompactTest(AbstractTest):
     def get_protocol(self, transport):
-        return TCompactProtocol.TCompactProtocolAcceleratedFactory().getProtocol(transport)
+        return TCompactProtocol.TCompactProtocolAcceleratedFactory(fallback=False).getProtocol(transport)
 
 
 def suite():
@@ -333,10 +333,7 @@
 
     if options.genpydir:
         sys.path.insert(0, os.path.join(SCRIPT_DIR, options.genpydir))
-    if options.libpydir:
-        sys.path.insert(0, glob.glob(options.libpydir)[0])
-    else:
-        sys.path.insert(0, glob.glob(DEFAULT_LIBDIR_GLOB)[0])
+    sys.path.insert(0, local_libpath())
 
     from ThriftTest import ThriftTest
     from ThriftTest.ttypes import Xtruct, Xtruct2, Numberz, Xception, Xception2
diff --git a/test/py/TestEof.py b/test/py/TestEof.py
index 8901613..cda1050 100755
--- a/test/py/TestEof.py
+++ b/test/py/TestEof.py
@@ -107,8 +107,8 @@
 
     def testBinaryProtocolAcceleratedBinaryEof(self):
         """Test that TBinaryProtocolAccelerated throws an EOFError when it reaches the end of the stream"""
-        self.eofTestHelper(TBinaryProtocol.TBinaryProtocolAcceleratedFactory())
-        self.eofTestHelperStress(TBinaryProtocol.TBinaryProtocolAcceleratedFactory())
+        self.eofTestHelper(TBinaryProtocol.TBinaryProtocolAcceleratedFactory(fallback=False))
+        self.eofTestHelperStress(TBinaryProtocol.TBinaryProtocolAcceleratedFactory(fallback=False))
 
     def testCompactProtocolEof(self):
         """Test that TCompactProtocol throws an EOFError when it reaches the end of the stream"""
@@ -117,8 +117,8 @@
 
     def testCompactProtocolAcceleratedCompactEof(self):
         """Test that TCompactProtocolAccelerated throws an EOFError when it reaches the end of the stream"""
-        self.eofTestHelper(TCompactProtocol.TCompactProtocolAcceleratedFactory())
-        self.eofTestHelperStress(TCompactProtocol.TCompactProtocolAcceleratedFactory())
+        self.eofTestHelper(TCompactProtocol.TCompactProtocolAcceleratedFactory(fallback=False))
+        self.eofTestHelperStress(TCompactProtocol.TCompactProtocolAcceleratedFactory(fallback=False))
 
 
 def suite():
diff --git a/test/py/TestFrozen.py b/test/py/TestFrozen.py
index f7c629b..e568e8c 100755
--- a/test/py/TestFrozen.py
+++ b/test/py/TestFrozen.py
@@ -102,12 +102,12 @@
 
 class TestFrozenAcceleratedBinary(TestFrozenBase):
     def protocol(self, trans):
-        return TBinaryProtocol.TBinaryProtocolAcceleratedFactory().getProtocol(trans)
+        return TBinaryProtocol.TBinaryProtocolAcceleratedFactory(fallback=False).getProtocol(trans)
 
 
 class TestFrozenAcceleratedCompact(TestFrozenBase):
     def protocol(self, trans):
-        return TCompactProtocol.TCompactProtocolAcceleratedFactory().getProtocol(trans)
+        return TCompactProtocol.TCompactProtocolAcceleratedFactory(fallback=False).getProtocol(trans)
 
 
 def suite():
diff --git a/test/py/TestServer.py b/test/py/TestServer.py
index 8819821..070560c 100755
--- a/test/py/TestServer.py
+++ b/test/py/TestServer.py
@@ -19,16 +19,15 @@
 # under the License.
 #
 from __future__ import division
-import glob
 import logging
 import os
 import sys
 import time
 from optparse import OptionParser
 
+from util import local_libpath
+
 SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
-ROOT_DIR = os.path.dirname(os.path.dirname(SCRIPT_DIR))
-DEFAULT_LIBDIR_GLOB = os.path.join(ROOT_DIR, 'lib', 'py', 'build', 'lib.*')
 
 
 class TestHandler(object):
@@ -300,10 +299,7 @@
     logging.basicConfig(level=options.verbose)
 
     sys.path.insert(0, os.path.join(SCRIPT_DIR, options.genpydir))
-    if options.libpydir:
-        sys.path.insert(0, glob.glob(options.libpydir)[0])
-    else:
-        sys.path.insert(0, glob.glob(DEFAULT_LIBDIR_GLOB)[0])
+    sys.path.insert(0, local_libpath())
 
     from ThriftTest import ThriftTest
     from ThriftTest.ttypes import Xtruct, Xception, Xception2, Insanity
diff --git a/test/py/util.py b/test/py/util.py
new file mode 100644
index 0000000..c2b3f5c
--- /dev/null
+++ b/test/py/util.py
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import glob
+import os
+import sys
+
+_SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
+_ROOT_DIR = os.path.dirname(os.path.dirname(_SCRIPT_DIR))
+
+
+def local_libpath():
+    globdir = os.path.join(_ROOT_DIR, 'lib', 'py', 'build', 'lib.*')
+    for libpath in glob.glob(globdir):
+        if libpath.endswith('-%d.%d' % (sys.version_info[0], sys.version_info[1])):
+            return libpath
diff --git a/test/tests.json b/test/tests.json
index 3104790..1eeb224 100644
--- a/test/tests.json
+++ b/test/tests.json
@@ -215,7 +215,6 @@
         "python3",
         "TestServer.py",
         "--verbose",
-        "--libpydir=../../lib/py/build/lib",
         "--genpydir=gen-py"
       ]
     },
@@ -225,7 +224,6 @@
         "python3",
         "TestClient.py",
         "--host=localhost",
-        "--libpydir=../../lib/py/build/lib",
         "--genpydir=gen-py"
       ]
     },
@@ -240,7 +238,9 @@
     "protocols": [
       "compact",
       "binary",
-      "json"
+      "json",
+      "binary:accel",
+      "compact:accelc"
     ],
     "workdir": "py"
   },