diff --git a/.gitignore b/.gitignore
index 228c6ac..a3062d8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -220,6 +220,7 @@
 /node_modules/
 /stamp-h1
 /test/status.html
+/test/results.json
 /test/c_glib/test_client
 /test/c_glib/test_server
 /test/cpp/StressTest
diff --git a/.travis.yml b/.travis.yml
index a5c728e..bb45961 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -55,11 +55,17 @@
 
   matrix:
     # Put it here because it's most time consuming
-    - TEST_NAME="make cross (gcc, automake)"
+    - TEST_NAME="make cross (automake)"
       CONFIG="--without-python"
       ALL_DEPS="yes"
       MAKE_TARGET="cross"
 
+    - TEST_NAME="make cross-py (automake)"
+      THRIFT_CROSSTEST_CONCURRENCY=6
+      CONFIG="--enable-tutorial=no --without-erlang --without-lua --without-haxe --without-d"
+      ALL_DEPS="yes"
+      MAKE_TARGET="cross2"
+
     # CMake builds
     - TEST_NAME="compiler (CMake + CPack)"
       CMAKE_CONFIG="-DBUILD_COMPILER=ON -DBUILD_LIBRARIES=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF"
@@ -86,7 +92,7 @@
     - TEST_NAME="Small Set (automake)"
       CONFIG="--without-erlang --without-haskell --without-python --without-go --without-lua --without-d --without-ruby --without-nodejs --without-java"
       ALL_DEPS="yes"
-    - TEST_NAME="dist (gcc, automake)"
+    - TEST_NAME="dist (automake)"
       CONFIG=""
       ALL_DEPS="yes"
       MAKE_TARGET="dist"
@@ -99,13 +105,13 @@
   exclude:
     # This one takes very long
     - compiler: gcc
-      env: TEST_NAME="make cross (gcc, automake)" CONFIG="--without-python" ALL_DEPS="yes" MAKE_TARGET="cross"
+      env: TEST_NAME="make cross (automake)" CONFIG="--without-python" ALL_DEPS="yes" MAKE_TARGET="cross"
 
     # Does not use native compiler, no need to do it twice
     - compiler: gcc
       env: TEST_NAME="compiler (mingw32-gcc, CMake + CPack)" CMAKE_CONFIG="-DCMAKE_TOOLCHAIN_FILE=../contrib/mingw32-toolchain.cmake -DBUILD_COMPILER=ON -DBUILD_LIBRARIES=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF"
     - compiler: gcc
-      env: TEST_NAME="dist (gcc, automake)" CONFIG="" ALL_DEPS="yes" MAKE_TARGET="dist"
+      env: TEST_NAME="dist (automake)" CONFIG="" ALL_DEPS="yes" MAKE_TARGET="dist"
 
   include:
     - env:
diff --git a/Makefile.am b/Makefile.am
index 0a9e431..bbe6f80 100755
--- a/Makefile.am
+++ b/Makefile.am
@@ -37,10 +37,18 @@
 print-version:
 	@echo $(VERSION)
 
+.PHONY: precross cross cross2
+precross-%: all
+	$(MAKE) -C $* precross
+precross: all precross-test precross-lib
 
-cross: check
+cross: precross
 	sh test/test.sh
 
+# TODO: generate --server and --client switches from "--with(out)-..." build flags
+cross2: precross
+	python test/test.py -s
+
 codespell_skip_files = \
 	*.jar \
 	*.class \
diff --git a/configure.ac b/configure.ac
index 6b26de5..1eaa359 100755
--- a/configure.ac
+++ b/configure.ac
@@ -692,6 +692,7 @@
   lib/c_glib/thrift_c_glib.pc
   lib/c_glib/test/Makefile
   lib/csharp/Makefile
+  lib/csharp/test/ThriftTest/Makefile
   lib/d/Makefile
   lib/d/test/Makefile
   lib/erl/Makefile
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 7b235d0..aa8b159 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -91,3 +91,7 @@
 	ocaml \
 	st \
 	ts
+
+precross-%:
+	$(MAKE) -C $* precross
+precross: precross-nodejs precross-csharp precross-java
diff --git a/lib/csharp/Makefile.am b/lib/csharp/Makefile.am
index 1c75aa1..5ce4276 100644
--- a/lib/csharp/Makefile.am
+++ b/lib/csharp/Makefile.am
@@ -17,6 +17,8 @@
 # under the License.
 #
 
+SUBDIRS = . test/ThriftTest
+
 THRIFTCODE= \
             src/Collections/THashSet.cs \
             src/Collections/TCollections.cs \
@@ -81,6 +83,9 @@
 clean-local:
 	$(RM) Thrift.dll
 
+precross:
+	$(MAKE) -C test/ThriftTest precross
+
 # run csharp tests?
 # check:
 # 	cd test/ThriftTest && ./maketest.sh
diff --git a/lib/csharp/test/ThriftTest/Makefile.am b/lib/csharp/test/ThriftTest/Makefile.am
new file mode 100644
index 0000000..fcde6fc
--- /dev/null
+++ b/lib/csharp/test/ThriftTest/Makefile.am
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+THRIFT = $(top_builddir)/compiler/cpp/thrift
+CSC=gmcs
+
+stubs: $(top_srcdir)/test/ThriftTest.thrift
+	$(THRIFT) --gen csharp -o . $(top_srcdir)/test/ThriftTest.thrift
+
+precross: TestClientServer.exe
+
+ThriftImpl.dll: stubs
+	$(CSC) /t:library /out:./ThriftImpl.dll /recurse:./gen-csharp/* /reference:../../Thrift.dll
+
+TestClientServer.exe: TestClient.cs TestServer.cs Program.cs ThriftImpl.dll
+	$(CSC)  /out:TestClientServer.exe /reference:../../Thrift.dll /reference:ThriftImpl.dll TestClient.cs TestServer.cs Program.cs
diff --git a/lib/csharp/test/ThriftTest/Program.cs b/lib/csharp/test/ThriftTest/Program.cs
index 3bf6796..5a4245b 100644
--- a/lib/csharp/test/ThriftTest/Program.cs
+++ b/lib/csharp/test/ThriftTest/Program.cs
@@ -31,12 +31,12 @@
 {
     class Program
     {
-        static void Main(string[] args)
+        static int Main(string[] args)
         {
             if (args.Length == 0)
             {
                 Console.WriteLine("must provide 'server' or 'client' arg");
-                return;
+                return -1;
             }
 
             string[] subArgs = new string[args.Length - 1];
@@ -46,16 +46,17 @@
             }
             if (args[0] == "client")
             {
-                TestClient.Execute(subArgs);
+                return TestClient.Execute(subArgs) ? 0 : 1;
             }
             else if (args[0] == "server")
             {
-                TestServer.Execute(subArgs);
+                return TestServer.Execute(subArgs) ? 0 : 1;
             }
             else
             {
                 Console.WriteLine("first argument must be 'server' or 'client'");
             }
+            return 0;
         }
     }
 }
diff --git a/lib/csharp/test/ThriftTest/TestClient.cs b/lib/csharp/test/ThriftTest/TestClient.cs
index a0ceb15..ec0696a 100644
--- a/lib/csharp/test/ThriftTest/TestClient.cs
+++ b/lib/csharp/test/ThriftTest/TestClient.cs
@@ -32,7 +32,7 @@
         private static int numIterations = 1;
         private static string protocol = "";
 
-        public static void Execute(string[] args)
+        public static bool Execute(string[] args)
         {
             try
             {
@@ -41,6 +41,7 @@
                 string url = null, pipe = null;
                 int numThreads = 1;
                 bool buffered = false, framed = false, encrypted = false;
+                string certPath = "../../../../../keys/server.pem";
 
                 try
                 {
@@ -96,6 +97,10 @@
                             encrypted = true;
                             Console.WriteLine("Using encrypted transport");
                         }
+                        else if (args[i].StartsWith("--cert="))
+                        {
+                            certPath = args[i].Substring("--cert=".Length);
+                        }
                     }
                 }
                 catch (Exception e)
@@ -119,7 +124,7 @@
                         else
                         {
                             if (encrypted)
-                                trans = new TTLSSocket(host, port, "../../../../../keys/client.pem");
+                                trans = new TTLSSocket(host, port, certPath);
                             else
                                 trans = new TSocket(host, port);
                         }
@@ -151,10 +156,12 @@
             catch (Exception outerEx)
             {
                 Console.WriteLine(outerEx.Message + " ST: " + outerEx.StackTrace);
+                return false;
             }
 
             Console.WriteLine();
             Console.WriteLine();
+            return true;
         }
 
         public static void ClientThread(object obj)
diff --git a/lib/csharp/test/ThriftTest/TestServer.cs b/lib/csharp/test/ThriftTest/TestServer.cs
index 2096cf8..0e9fe05 100644
--- a/lib/csharp/test/ThriftTest/TestServer.cs
+++ b/lib/csharp/test/ThriftTest/TestServer.cs
@@ -358,13 +358,14 @@
 
         } // class TestHandler
 
-        public static void Execute(string[] args)
+        public static bool Execute(string[] args)
         {
             try
             {
                 bool useBufferedSockets = false, useFramed = false, useEncryption = false, compact = false, json = false;
                 int port = 9090;
                 string pipe = null;
+                string certPath = "../../../../../keys/server.pem";
                 for (int i = 0; i < args.Length; i++)
                 {
                     if (args[i] == "-pipe")  // -pipe name
@@ -395,6 +396,10 @@
                     {
                         useEncryption = true;
                     }
+                    else if (args[i].StartsWith("--cert="))
+                    {
+                        certPath = args[i].Substring("--cert=".Length);
+                    }
                 }
 
                 // Processor
@@ -411,7 +416,7 @@
                 {
                     if (useEncryption)
                     {
-                        trans = new TTLSServerSocket(port, 0, useBufferedSockets, new X509Certificate2("../../../../../keys/server.pem"));
+                        trans = new TTLSServerSocket(port, 0, useBufferedSockets, new X509Certificate2(certPath));
                     }
                     else
                     {
@@ -461,8 +466,10 @@
             catch (Exception x)
             {
                 Console.Error.Write(x);
+                return false;
             }
             Console.WriteLine("done.");
+            return true;
         }
     }
 }
diff --git a/lib/go/Makefile.am b/lib/go/Makefile.am
index be2a2e5..bf173ba 100644
--- a/lib/go/Makefile.am
+++ b/lib/go/Makefile.am
@@ -33,7 +33,8 @@
 check-local:
 	$(GO) test ./thrift
 
-all-local: check-local
+all-local:
+	$(GO) build ./thrift
 
 EXTRA_DIST = \
 	thrift \
diff --git a/lib/go/test/Makefile.am b/lib/go/test/Makefile.am
index ef61249..38a0968 100644
--- a/lib/go/test/Makefile.am
+++ b/lib/go/test/Makefile.am
@@ -17,11 +17,11 @@
 # under the License.
 #
 
-THRIFT = $(top_srcdir)/compiler/cpp/thrift -out gopath/src/ --gen go:thrift_import=thrift
+THRIFT = $(top_builddir)/compiler/cpp/thrift
 THRIFTTEST = $(top_srcdir)/test/ThriftTest.thrift
 
 # Thrift for GO has problems with complex map keys: THRIFT-2063
-gopath: $(top_srcdir)/compiler/cpp/thrift $(THRIFTTEST) \
+gopath: $(THRIFT) $(THRIFTTEST) \
 				IncludesTest.thrift \
 				NamespacedTest.thrift \
 				MultiplexedProtocolTest.thrift \
diff --git a/lib/haxe/test/Makefile.am b/lib/haxe/test/Makefile.am
index 13b4266..91cfc90 100644
--- a/lib/haxe/test/Makefile.am
+++ b/lib/haxe/test/Makefile.am
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-THRIFT = $(top_srcdir)/compiler/cpp/thrift
+THRIFT = $(top_builddir)/compiler/cpp/thrift
 THRIFTCMD = $(THRIFT) --gen haxe -r
 THRIFTTEST = $(top_srcdir)/test/ThriftTest.thrift
 AGGR = $(top_srcdir)/contrib/async-test/aggr.thrift
diff --git a/lib/java/Makefile.am b/lib/java/Makefile.am
index cbec7af..63d40a6 100644
--- a/lib/java/Makefile.am
+++ b/lib/java/Makefile.am
@@ -19,6 +19,8 @@
 
 export CLASSPATH
 
+THRIFT = $(top_builddir)/compiler/cpp/thrift
+
 all-local:
 	$(ANT) $(ANT_FLAGS)
 
@@ -31,6 +33,9 @@
 	ANT=$(ANT) ; if test -z "$$ANT" ; then ANT=: ; fi ; \
 	$$ANT $(ANT_FLAGS) clean
 
+precross: $(THRIFT)
+	$(ANT) $(ANT_FLAGS) compile-test
+
 check-local: all
 	$(ANT) $(ANT_FLAGS) test
 
diff --git a/lib/nodejs/Makefile.am b/lib/nodejs/Makefile.am
index 9998348..b8e441b 100755
--- a/lib/nodejs/Makefile.am
+++ b/lib/nodejs/Makefile.am
@@ -16,14 +16,18 @@
 # under the License.
 
 
-THRIFT = $(top_srcdir)/compiler/cpp/thrift
+THRIFT = $(top_builddir)/compiler/cpp/thrift
 
-#stubs: $(top_srcdir)/test/ThriftTest.thrift
-#	$(THRIFT) --gen js:node -o test/ $(top_srcdir)/test/ThriftTest.thrift
+stubs: $(top_srcdir)/test/ThriftTest.thrift
+	$(THRIFT) --gen js:node -o test/ $(top_srcdir)/test/ThriftTest.thrift
 
 deps: $(top_srcdir)/package.json
 	$(NPM) install --no-bin-links $(top_srcdir)/
 
+all-local: deps
+
+precross: deps stubs
+
 check: deps
 	cd $(top_srcdir) && $(NPM) test && cd lib/nodejs
 
diff --git a/test/Makefile.am b/test/Makefile.am
index 23ec498..5bd2d0e 100755
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -104,3 +104,7 @@
 	ThriftTest.thrift \
 	FastbinaryTest.py \
 	README.md
+
+precross-%:
+	$(MAKE) -C $* precross
+precross: precross-py precross-rb precross-c_glib precross-cpp precross-perl precross-php precross-go
diff --git a/test/c_glib/Makefile.am b/test/c_glib/Makefile.am
index 252edb9..9412415 100755
--- a/test/c_glib/Makefile.am
+++ b/test/c_glib/Makefile.am
@@ -29,6 +29,8 @@
 
 libtestcglib_la_LIBADD = $(top_builddir)/lib/c_glib/libthrift_c_glib.la
 
+precross: test_client test_server
+
 check_PROGRAMS = \
 	test_client \
 	test_server
@@ -54,7 +56,7 @@
 #
 THRIFT = $(top_builddir)/compiler/cpp/thrift
 
-gen-c_glib/t_test_second_service.c  gen-c_glib/t_test_second_service.h  gen-c_glib/t_test_thrift_test.c  gen-c_glib/t_test_thrift_test.h  gen-c_glib/t_test_thrift_test_types.c  gen-c_glib/t_test_thrift_test_types.h: $(top_srcdir)/test/ThriftTest.thrift
+gen-c_glib/t_test_second_service.c  gen-c_glib/t_test_second_service.h  gen-c_glib/t_test_thrift_test.c  gen-c_glib/t_test_thrift_test.h  gen-c_glib/t_test_thrift_test_types.c  gen-c_glib/t_test_thrift_test_types.h: $(top_srcdir)/test/ThriftTest.thrift $(THRIFT)
 	$(THRIFT) --gen c_glib -r $<
 
 AM_CFLAGS = -g -Wall -Wextra $(GLIB_CFLAGS) $(GOBJECT_CFLAGS)
diff --git a/test/cpp/Makefile.am b/test/cpp/Makefile.am
index 7d57f5c..89fed8f 100755
--- a/test/cpp/Makefile.am
+++ b/test/cpp/Makefile.am
@@ -49,6 +49,8 @@
 
 libstresstestgencpp_la_LIBADD = $(top_builddir)/lib/cpp/libthrift.la
 
+precross: TestServer TestClient
+
 check_PROGRAMS = \
 	TestServer \
 	TestClient \
diff --git a/test/crossrunner/__init__.py b/test/crossrunner/__init__.py
new file mode 100644
index 0000000..06de2d0
--- /dev/null
+++ b/test/crossrunner/__init__.py
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from crossrunner.test import test_name
+from crossrunner.collect import collect_tests
+from crossrunner.run import TestDispatcher
+from crossrunner.report import generate_known_failures
+from crossrunner.report import load_known_failures
+from crossrunner.prepare import prepare
diff --git a/test/crossrunner/collect.py b/test/crossrunner/collect.py
new file mode 100644
index 0000000..80a82e7
--- /dev/null
+++ b/test/crossrunner/collect.py
@@ -0,0 +1,136 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import platform
+from itertools import product
+
+from crossrunner.util import merge_dict
+
+# Those keys are passed to execution as is.
+# Note that there are keys other than these, namely:
+# delay: After server is started, client start is delayed for the value
+# (seconds).
+# timeout: Test timeout after client is started (seconds).
+# platforms: Supported platforms. Should match platform.system() value.
+# protocols: list of supported protocols
+# transports: list of supported transports
+# sockets: list of supported sockets
+VALID_JSON_KEYS = [
+  'name',  # name of the library, typically a language name
+  'workdir',  # work directory where command is executed
+  'command',  # test command
+  'extra_args',  # args appended to command after other args are appended
+  'join_args',  # whether args should be passed as single concatenated string
+  'env',  # additional environmental variable
+]
+
+DEFAULT_DELAY = 1
+DEFAULT_TIMEOUT = 5
+
+
+def collect_testlibs(config, server_match, client_match):
+  """Collects server/client configurations from library configurations"""
+  def expand_libs(config):
+    for lib in config:
+      sv = lib.pop('server', None)
+      cl = lib.pop('client', None)
+      yield lib, sv, cl
+
+  def yield_testlibs(base_configs, configs, match):
+    for base, conf in zip(base_configs, configs):
+      if conf:
+        if not match or base['name'] in match:
+          platforms = conf.get('platforms') or base.get('platforms')
+          if not platforms or platform.system() in platforms:
+            yield merge_dict(base, conf)
+
+  libs, svs, cls = zip(*expand_libs(config))
+  servers = list(yield_testlibs(libs, svs, server_match))
+  clients = list(yield_testlibs(libs, cls, client_match))
+  return servers, clients
+
+
+def do_collect_tests(servers, clients):
+  def intersection(key, o1, o2):
+    """intersection of two collections.
+    collections are replaced with sets the first time"""
+    def cached_set(o, key):
+      v = o[key]
+      if not isinstance(v, set):
+        v = set(v)
+        o[key] = v
+      return v
+    return cached_set(o1, key) & cached_set(o2, key)
+
+  # each entry can be spec:impl (e.g. binary:accel)
+  def intersect_with_spec(key, o1, o2):
+    # store as set of (spec, impl) tuple
+    def cached_set(o):
+      def to_spec_impl_tuples(values):
+        for v in values:
+          spec, _, impl = v.partition(':')
+          yield spec, impl or spec
+      v = o[key]
+      if not isinstance(v, set):
+        v = set(to_spec_impl_tuples(set(v)))
+        o[key] = v
+      return v
+    for spec1, impl1 in cached_set(o1):
+      for spec2, impl2 in cached_set(o2):
+        if spec1 == spec2:
+          name = impl1 if impl1 == impl2 else '%s-%s' % (impl1, impl2)
+          yield name, impl1, impl2
+
+  def maybe_max(key, o1, o2, default):
+    """maximum of two if present, otherwise defult value"""
+    v1 = o1.get(key)
+    v2 = o2.get(key)
+    return max(v1, v2) if v1 and v2 else v1 or v2 or default
+
+  def filter_with_validkeys(o):
+    ret = {}
+    for key in VALID_JSON_KEYS:
+      if key in o:
+        ret[key] = o[key]
+    return ret
+
+  def merge_metadata(o, **ret):
+    for key in VALID_JSON_KEYS:
+      if key in o:
+        ret[key] = o[key]
+    return ret
+
+  for sv, cl in product(servers, clients):
+    for proto, proto1, proto2 in intersect_with_spec('protocols', sv, cl):
+      for trans, trans1, trans2 in intersect_with_spec('transports', sv, cl):
+        for sock in intersection('sockets', sv, cl):
+          yield {
+            'server': merge_metadata(sv, **{'protocol': proto1, 'transport': trans1}),
+            'client': merge_metadata(cl, **{'protocol': proto2, 'transport': trans2}),
+            'delay': maybe_max('delay', sv, cl, DEFAULT_DELAY),
+            'timeout': maybe_max('timeout', sv, cl, DEFAULT_TIMEOUT),
+            'protocol': proto,
+            'transport': trans,
+            'socket': sock
+          }
+
+
+def collect_tests(tests_dict, server_match, client_match):
+  sv, cl = collect_testlibs(tests_dict, server_match, client_match)
+  return list(do_collect_tests(sv, cl))
diff --git a/test/crossrunner/prepare.py b/test/crossrunner/prepare.py
new file mode 100644
index 0000000..6e4f6ee
--- /dev/null
+++ b/test/crossrunner/prepare.py
@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+import subprocess
+
+from crossrunner.collect import collect_testlibs
+
+
+def prepare(config_dict, testdir, server_match, client_match):
+  libs, libs2 = collect_testlibs(config_dict, server_match, client_match)
+  libs.extend(libs2)
+
+  def prepares():
+    for lib in libs:
+      pre = lib.get('prepare')
+      if pre:
+        yield pre, lib['workdir']
+
+  def files():
+    for lib in libs:
+      workdir = os.path.join(testdir, lib['workdir'])
+      for c in lib['command']:
+        if not c.startswith('-'):
+          p = os.path.join(workdir, c)
+          if not os.path.exists(p):
+            yield os.path.split(p)
+
+  def make(p):
+    d, f = p
+    with open(os.devnull, 'w') as devnull:
+      return subprocess.Popen(['make', f], cwd=d, stderr=devnull)
+
+  for pre, d in prepares():
+    subprocess.Popen(pre, cwd=d).wait()
+
+  for p in list(map(make, set(files()))):
+    p.wait()
+  return True
diff --git a/test/crossrunner/report.py b/test/crossrunner/report.py
new file mode 100644
index 0000000..da478fa
--- /dev/null
+++ b/test/crossrunner/report.py
@@ -0,0 +1,395 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import datetime
+import json
+import multiprocessing
+import os
+import platform
+import re
+import subprocess
+import sys
+import time
+import traceback
+
+from crossrunner.test import TestEntry
+
+LOG_DIR = 'log'
+RESULT_HTML = 'result.html'
+RESULT_JSON = 'results.json'
+FAIL_JSON = 'known_failures_%s.json'
+
+
+def generate_known_failures(testdir, overwrite, save, out):
+  def collect_failures(results):
+    success_index = 5
+    for r in results:
+      if not r[success_index]:
+        yield TestEntry.get_name(*r)
+  try:
+    with open(os.path.join(testdir, RESULT_JSON), 'r') as fp:
+      results = json.load(fp)
+  except IOError:
+    sys.stderr.write('Unable to load last result. Did you run tests ?\n')
+    return False
+  fails = collect_failures(results['results'])
+  if not overwrite:
+    known = load_known_failures(testdir)
+    known.extend(fails)
+    fails = known
+  fails_json = json.dumps(sorted(set(fails)), indent=2)
+  if save:
+    with open(os.path.join(testdir, FAIL_JSON % platform.system()), 'w+') as fp:
+      fp.write(fails_json)
+    sys.stdout.write('Successfully updated known failures.\n')
+  if out:
+    sys.stdout.write(fails_json)
+    sys.stdout.write('\n')
+  return True
+
+
+def load_known_failures(testdir):
+  try:
+    with open(os.path.join(testdir, FAIL_JSON % platform.system()), 'r') as fp:
+      return json.load(fp)
+  except IOError:
+    return []
+
+
+class TestReporter(object):
+  # Unfortunately, standard library doesn't handle timezone well
+  # DATETIME_FORMAT = '%a %b %d %H:%M:%S %Z %Y'
+  DATETIME_FORMAT = '%a %b %d %H:%M:%S %Y'
+
+  def __init__(self):
+    self._log = multiprocessing.get_logger()
+    self._lock = multiprocessing.Lock()
+
+  @classmethod
+  def test_logfile(cls, dir, test_name, prog_kind):
+    return os.path.realpath(os.path.join(
+      dir, 'log', '%s_%s.log' % (test_name, prog_kind)))
+
+  def _start(self):
+    self._start_time = time.time()
+
+  @property
+  def _elapsed(self):
+    return time.time() - self._start_time
+
+  @classmethod
+  def _format_date(cls):
+    return '%s' % datetime.datetime.now().strftime(cls.DATETIME_FORMAT)
+
+  def _print_date(self):
+    self.out.write('%s\n' % self._format_date())
+
+  def _print_bar(self, out=None):
+    (out or self.out).write(
+      '======================================================================\n')
+
+  def _print_exec_time(self):
+    self.out.write('Test execution took {:.1f} seconds.\n'.format(self._elapsed))
+
+
+class ExecReporter(TestReporter):
+  def __init__(self, testdir, test, prog):
+    super(ExecReporter, self).__init__()
+    self._test = test
+    self._prog = prog
+    self.logpath = self.test_logfile(testdir, test.name, prog.kind)
+    self.out = None
+
+  def begin(self):
+    self._start()
+    self._open()
+    if self.out and not self.out.closed:
+      self._print_header()
+    else:
+      self._log.debug('Output stream is not available.')
+
+  def end(self, returncode):
+    self._lock.acquire()
+    try:
+      if self.out and not self.out.closed:
+        self._print_footer(returncode)
+        self._close()
+        self.out = None
+      else:
+        self._log.debug('Output stream is not available.')
+    finally:
+      self._lock.release()
+
+  def killed(self):
+    self._lock.acquire()
+    try:
+      if self.out and not self.out.closed:
+        self._print_footer()
+        self._close()
+        self.out = None
+      else:
+        self._log.debug('Output stream is not available.')
+    finally:
+      self._lock.release()
+
+  _init_failure_exprs = {
+    'server': list(map(re.compile, [
+      '[Aa]ddress already in use',
+      'Could not bind',
+      'EADDRINUSE',
+    ])),
+    'client': list(map(re.compile, [
+      '[Cc]onnection refused',
+      'Could not connect to localhost',
+      'ECONNREFUSED',
+      'No such file or directory',  # domain socket
+    ])),
+  }
+
+  def maybe_false_positive(self):
+    """Searches through log file for socket bind error.
+    Returns True if suspicious expression is found, otherwise False"""
+    def match(line):
+      for expr in exprs:
+        if expr.search(line):
+          return True
+    try:
+      if self.out and not self.out.closed:
+        self.out.flush()
+      exprs = list(map(re.compile, self._init_failure_exprs[self._prog.kind]))
+
+      server_logfile = self.logpath
+      # need to handle unicode errors on Python 3
+      kwargs = {} if sys.version_info[0] < 3 else {'errors': 'replace'}
+      with open(server_logfile, 'r', **kwargs) as fp:
+        if any(map(match, fp)):
+          return True
+    except (KeyboardInterrupt, SystemExit):
+      raise
+    except Exception as ex:
+      self._log.warn('[%s]: Error while detecting false positive: %s' % (self._test.name, str(ex)))
+      self._log.info(traceback.print_exc())
+    return False
+
+  def _open(self):
+    self.out = open(self.logpath, 'w+')
+
+  def _close(self):
+    self.out.close()
+
+  def _print_header(self):
+    self._print_date()
+    self.out.write('Executing: %s\n' % ' '.join(self._prog.command))
+    self.out.write('Directory: %s\n' % self._prog.workdir)
+    self.out.write('config:delay: %s\n' % self._test.delay)
+    self.out.write('config:timeout: %s\n' % self._test.timeout)
+    self._print_bar()
+    self.out.flush()
+
+  def _print_footer(self, returncode=None):
+    self._print_bar()
+    if returncode is not None:
+      self.out.write('Return code: %d\n' % returncode)
+    else:
+      self.out.write('Process is killed.\n')
+    self._print_exec_time()
+    self._print_date()
+
+
+class SummaryReporter(TestReporter):
+  def __init__(self, testdir, concurrent=True):
+    super(SummaryReporter, self).__init__()
+    self.testdir = testdir
+    self.logdir = os.path.join(testdir, LOG_DIR)
+    self.out_path = os.path.join(testdir, RESULT_JSON)
+    self.concurrent = concurrent
+    self.out = sys.stdout
+    self._platform = platform.system()
+    self._revision = self._get_revision()
+    self._tests = []
+    if not os.path.exists(self.logdir):
+      os.mkdir(self.logdir)
+    self._known_failures = load_known_failures(testdir)
+    self._unexpected_success = []
+    self._unexpected_failure = []
+    self._expected_failure = []
+    self._print_header()
+
+  def _get_revision(self):
+    p = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'],
+                         cwd=self.testdir, stdout=subprocess.PIPE)
+    out, _ = p.communicate()
+    return out.strip()
+
+  def _format_test(self, test, with_result=True):
+    name = '%s-%s' % (test.server.name, test.client.name)
+    trans = '%s-%s' % (test.transport, test.socket)
+    if not with_result:
+      return '{:19s}{:13s}{:25s}'.format(name[:18], test.protocol[:12], trans[:24])
+    else:
+      result = 'success' if test.success else (
+          'timeout' if test.expired else 'failure')
+      result_string = '%s(%d)' % (result, test.returncode)
+      return '{:19s}{:13s}{:25s}{:s}\n'.format(name[:18], test.protocol[:12], trans[:24], result_string)
+
+  def _print_test_header(self):
+    self._print_bar()
+    self.out.write(
+      '{:19s}{:13s}{:25s}{:s}\n'.format('server-client:', 'protocol:', 'transport:', 'result:'))
+
+  def _print_header(self):
+    self._start()
+    self.out.writelines([
+      'Apache Thrift - Integration Test Suite\n',
+    ])
+    self._print_date()
+    self._print_test_header()
+
+  def _print_unexpected_failure(self):
+    if len(self._unexpected_failure) > 0:
+      self.out.writelines([
+        '*** Following %d failures were unexpected ***:\n' % len(self._unexpected_failure),
+        'If it is introduced by you, please fix it before submitting the code.\n',
+        # 'If not, please report at https://issues.apache.org/jira/browse/THRIFT\n',
+      ])
+      self._print_test_header()
+      for i in self._unexpected_failure:
+        self.out.write(self._format_test(self._tests[i]))
+      self._print_bar()
+    else:
+      self.out.write('No unexpected failures.\n')
+
+  def _print_unexpected_success(self):
+    if len(self._unexpected_success) > 0:
+      self.out.write(
+        'Following %d tests were known to fail but succeeded (it\'s normal):\n' % len(self._unexpected_success))
+      self._print_test_header()
+      for i in self._unexpected_success:
+        self.out.write(self._format_test(self._tests[i]))
+      self._print_bar()
+
+  def _print_footer(self):
+    fail_count = len(self._expected_failure) + len(self._unexpected_failure)
+    self._print_bar()
+    self._print_unexpected_success()
+    self._print_unexpected_failure()
+    self._write_html_data()
+    self._assemble_log('unexpected failures', self._unexpected_failure)
+    self._assemble_log('known failures', self._expected_failure)
+    self.out.writelines([
+      'You can browse results at:\n',
+      '\tfile://%s/%s\n' % (self.testdir, RESULT_HTML),
+      'Full log for each test is here:\n',
+      '\ttest/log/client_server_protocol_transport_client.log\n',
+      '\ttest/log/client_server_protocol_transport_server.log\n',
+      '%d failed of %d tests in total.\n' % (fail_count, len(self._tests)),
+    ])
+    self._print_exec_time()
+    self._print_date()
+
+  def _render_result(self, test):
+    return [
+      test.server.name,
+      test.client.name,
+      test.protocol,
+      test.transport,
+      test.socket,
+      test.success,
+      test.as_expected,
+      test.returncode,
+      {
+        'server': self.test_logfile(test.testdir, test.name, test.server.kind),
+        'client': self.test_logfile(test.testdir, test.name, test.client.kind),
+      },
+    ]
+
+  def _write_html_data(self):
+    """Writes JSON data to be read by result html"""
+    results = [self._render_result(r) for r in self._tests]
+    with open(self.out_path, 'w+') as fp:
+      fp.write(json.dumps({
+        'date': self._format_date(),
+        'revision': str(self._revision),
+        'platform': self._platform,
+        'duration': '{:.1f}'.format(self._elapsed),
+        'results': results,
+      }, indent=2))
+
+  def _assemble_log(self, title, indexes):
+    if len(indexes) > 0:
+      def add_prog_log(fp, test, prog_kind):
+        fp.write('*************************** %s message ***************************\n'
+                 % prog_kind)
+        path = self.test_logfile(self.testdir, test.name, prog_kind)
+        kwargs = {} if sys.version_info[0] < 3 else {'errors': 'replace'}
+        with open(path, 'r', **kwargs) as prog_fp:
+          fp.write(prog_fp.read())
+      filename = title.replace(' ', '_') + '.log'
+      with open(os.path.join(self.logdir, filename), 'w+') as fp:
+        for test in map(self._tests.__getitem__, indexes):
+          fp.write('TEST: [%s]\n' % test.name)
+          add_prog_log(fp, test, test.server.kind)
+          add_prog_log(fp, test, test.client.kind)
+          fp.write('**********************************************************************\n\n')
+      self.out.write('%s are logged to test/%s/%s\n' % (title.capitalize(), LOG_DIR, filename))
+
+  def end(self):
+    self._print_footer()
+    return len(self._unexpected_failure) == 0
+
+  def add_test(self, test_dict):
+    test = TestEntry(self.testdir, **test_dict)
+    self._lock.acquire()
+    try:
+      if not self.concurrent:
+        self.out.write(self._format_test(test, False))
+        self.out.flush()
+      self._tests.append(test)
+      return len(self._tests) - 1
+    finally:
+      self._lock.release()
+
+  def add_result(self, index, returncode, expired):
+    self._lock.acquire()
+    try:
+      failed = returncode is None or returncode != 0
+      test = self._tests[index]
+      known = test.name in self._known_failures
+      if failed:
+        if known:
+          self._log.debug('%s failed as expected' % test.name)
+          self._expected_failure.append(index)
+        else:
+          self._log.info('unexpected failure: %s' % test.name)
+          self._unexpected_failure.append(index)
+      elif known:
+        self._log.info('unexpected success: %s' % test.name)
+        self._unexpected_success.append(index)
+      test.success = not failed
+      test.returncode = returncode
+      test.expired = expired
+      test.as_expected = known == failed
+      if not self.concurrent:
+        result = 'success' if not failed else 'failure'
+        result_string = '%s(%d)' % (result, returncode)
+        self.out.write(result_string + '\n')
+      else:
+        self.out.write(self._format_test(test))
+    finally:
+      self._lock.release()
diff --git a/test/crossrunner/run.py b/test/crossrunner/run.py
new file mode 100644
index 0000000..e3300ba
--- /dev/null
+++ b/test/crossrunner/run.py
@@ -0,0 +1,317 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import contextlib
+import multiprocessing
+import multiprocessing.managers
+import os
+import platform
+import random
+import socket
+import signal
+import subprocess
+import threading
+import time
+import traceback
+
+from crossrunner.test import TestEntry, domain_socket_path
+from crossrunner.report import ExecReporter, SummaryReporter
+
+RESULT_TIMEOUT = 128
+RESULT_ERROR = 64
+
+
+class ExecutionContext(object):
+  def __init__(self, cmd, cwd, env, report):
+    self._log = multiprocessing.get_logger()
+    self.report = report
+    self.cmd = cmd
+    self.cwd = cwd
+    self.env = env
+    self.timer = None
+    self.expired = False
+
+  def _expire(self):
+    self._log.info('Timeout')
+    self.expired = True
+    self.kill()
+
+  def kill(self):
+    self._log.debug('Killing process : %d' % self.proc.pid)
+    if platform.system() != 'Windows':
+      try:
+        os.killpg(self.proc.pid, signal.SIGKILL)
+      except Exception as err:
+        self._log.info('Failed to kill process group : %s' % str(err))
+    try:
+      self.proc.kill()
+    except Exception as err:
+      self._log.info('Failed to kill process : %s' % str(err))
+    self.report.killed()
+
+  def _popen_args(self):
+    args = {
+      'cwd': self.cwd,
+      'env': self.env,
+      'stdout': self.report.out,
+      'stderr': subprocess.STDOUT,
+    }
+    # make sure child processes doesn't remain after killing
+    if platform.system() == 'Windows':
+      DETACHED_PROCESS = 0x00000008
+      args.update(creationflags=DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP)
+    else:
+      args.update(preexec_fn=os.setsid)
+    return args
+
+  def start(self, timeout=0):
+    self._log.debug('COMMAND: %s', ' '.join(self.cmd))
+    self._log.debug('WORKDIR: %s', self.cwd)
+    self._log.debug('LOGFILE: %s', self.report.logpath)
+    self.report.begin()
+    self.proc = subprocess.Popen(self.cmd, **self._popen_args())
+    if timeout > 0:
+      self.timer = threading.Timer(timeout, self._expire)
+      self.timer.start()
+    return self._scoped()
+
+  @contextlib.contextmanager
+  def _scoped(self):
+    yield self
+    self._log.debug('Killing scoped process')
+    self.kill()
+
+  def wait(self):
+    self.proc.communicate()
+    if self.timer:
+      self.timer.cancel()
+    self.report.end(self.returncode)
+
+  @property
+  def returncode(self):
+    return self.proc.returncode if self.proc else None
+
+
+def exec_context(port, testdir, test, prog):
+  report = ExecReporter(testdir, test, prog)
+  prog.build_command(port)
+  return ExecutionContext(prog.command, prog.workdir, prog.env, report)
+
+
+def run_test(testdir, test_dict, async=True, max_retry=3):
+  try:
+    logger = multiprocessing.get_logger()
+    retry_count = 0
+    test = TestEntry(testdir, **test_dict)
+    while True:
+      if stop.is_set():
+        logger.debug('Skipping because shutting down')
+        return None
+      logger.debug('Start')
+      with PortAllocator.alloc_port_scoped(ports, test.socket) as port:
+        logger.debug('Start with port %d' % port)
+        sv = exec_context(port, testdir, test, test.server)
+        cl = exec_context(port, testdir, test, test.client)
+
+        logger.debug('Starting server')
+        with sv.start():
+          if test.delay > 0:
+            logger.debug('Delaying client for %.2f seconds' % test.delay)
+            time.sleep(test.delay)
+          cl_retry_count = 0
+          cl_max_retry = 10
+          cl_retry_wait = 0.5
+          while True:
+            logger.debug('Starting client')
+            cl.start(test.timeout)
+            logger.debug('Waiting client')
+            cl.wait()
+            if not cl.report.maybe_false_positive() or cl_retry_count >= cl_max_retry:
+              if cl_retry_count > 0 and cl_retry_count < cl_max_retry:
+                logger.warn('[%s]: Connected after %d retry (%.2f sec each)' % (test.server.name, cl_retry_count, cl_retry_wait))
+              break
+            logger.debug('Server may not be ready, waiting %.2f second...' % cl_retry_wait)
+            time.sleep(cl_retry_wait)
+            cl_retry_count += 1
+
+      if not sv.report.maybe_false_positive() or retry_count >= max_retry:
+        logger.debug('Finish')
+        return RESULT_TIMEOUT if cl.expired else cl.proc.returncode
+      logger.warn('[%s]: Detected socket bind failure, retrying...' % test.server.name)
+      retry_count += 1
+  except (KeyboardInterrupt, SystemExit):
+    logger.info('Interrupted execution')
+    if not async:
+      raise
+    stop.set()
+    return None
+  except Exception as ex:
+    logger.warn('Error while executing test : %s' % str(ex))
+    if not async:
+      raise
+    logger.info(traceback.print_exc())
+    return RESULT_ERROR
+
+
+class PortAllocator(object):
+  def __init__(self):
+    self._log = multiprocessing.get_logger()
+    self._lock = multiprocessing.Lock()
+    self._ports = set()
+    self._dom_ports = set()
+    self._last_alloc = 0
+
+  def _get_tcp_port(self):
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.bind(('127.0.0.1', 0))
+    port = sock.getsockname()[1]
+    self._lock.acquire()
+    try:
+      ok = port not in self._ports
+      if ok:
+        self._ports.add(port)
+        self._last_alloc = time.time()
+    finally:
+      self._lock.release()
+      sock.close()
+    return port if ok else self._get_tcp_port()
+
+  def _get_domain_port(self):
+    port = random.randint(1024, 65536)
+    self._lock.acquire()
+    try:
+      ok = port not in self._dom_ports
+      if ok:
+        self._dom_ports.add(port)
+    finally:
+      self._lock.release()
+    return port if ok else self._get_domain_port()
+
+  def alloc_port(self, socket_type):
+    if socket_type == 'domain':
+      return self._get_domain_port()
+    else:
+      return self._get_tcp_port()
+
+  # static method for inter-process invokation
+  @staticmethod
+  @contextlib.contextmanager
+  def alloc_port_scoped(allocator, socket_type):
+    port = allocator.alloc_port(socket_type)
+    yield port
+    allocator.free_port(socket_type, port)
+
+  def free_port(self, socket_type, port):
+    self._log.debug('free_port')
+    self._lock.acquire()
+    try:
+      if socket_type == 'domain':
+        self._dom_ports.remove(port)
+        path = domain_socket_path(port)
+        if os.path.exists(path):
+          os.remove(path)
+      else:
+        self._ports.remove(port)
+    except IOError as err:
+      self._log.info('Error while freeing port : %s' % str(err))
+    finally:
+      self._lock.release()
+
+
+class NonAsyncResult(object):
+  def __init__(self, value):
+    self._value = value
+
+  def get(self, timeout=None):
+    return self._value
+
+  def wait(self, timeout=None):
+    pass
+
+  def ready(self):
+    return True
+
+  def successful(self):
+    return self._value == 0
+
+
+class TestDispatcher(object):
+  def __init__(self, testdir, concurrency):
+    self._log = multiprocessing.get_logger()
+    self.testdir = testdir
+    # seems needed for python 2.x to handle keyboard interrupt
+    self._stop = multiprocessing.Event()
+    self._async = concurrency > 1
+    if not self._async:
+      self._pool = None
+      global stop
+      global ports
+      stop = self._stop
+      ports = PortAllocator()
+    else:
+      self._m = multiprocessing.managers.BaseManager()
+      self._m.register('ports', PortAllocator)
+      self._m.start()
+      self._pool = multiprocessing.Pool(concurrency, self._pool_init, (self._m.address,))
+    self._report = SummaryReporter(testdir, concurrency > 1)
+    self._log.debug(
+        'TestDispatcher started with %d concurrent jobs' % concurrency)
+
+  def _pool_init(self, address):
+    global stop
+    global m
+    global ports
+    stop = self._stop
+    m = multiprocessing.managers.BaseManager(address)
+    m.connect()
+    ports = m.ports()
+
+  def _dispatch_sync(self, test, cont):
+    r = run_test(self.testdir, test, False)
+    cont(r)
+    return NonAsyncResult(r)
+
+  def _dispatch_async(self, test, cont):
+    return self._pool.apply_async(func=run_test, args=(self.testdir, test,), callback=cont)
+
+  def dispatch(self, test):
+    index = self._report.add_test(test)
+
+    def cont(r):
+      if not self._stop.is_set():
+        self._log.debug('freeing port')
+        self._log.debug('adding result')
+        self._report.add_result(index, r, r == RESULT_TIMEOUT)
+        self._log.debug('finish continuation')
+    fn = self._dispatch_async if self._async else self._dispatch_sync
+    return fn(test, cont)
+
+  def wait(self):
+    if self._async:
+      self._pool.close()
+      self._pool.join()
+      self._m.shutdown()
+    return self._report.end()
+
+  def terminate(self):
+    self._stop.set()
+    if self._async:
+      self._pool.terminate()
+      self._pool.join()
+      self._m.shutdown()
diff --git a/test/crossrunner/test.py b/test/crossrunner/test.py
new file mode 100644
index 0000000..512e664
--- /dev/null
+++ b/test/crossrunner/test.py
@@ -0,0 +1,136 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import copy
+import multiprocessing
+import os
+import sys
+
+from crossrunner.util import merge_dict
+
+
+def domain_socket_path(port):
+  return '/tmp/ThriftTest.thrift.%d' % port
+
+
+class TestProgram(object):
+  def __init__(self, kind, name, protocol, transport, socket, workdir, command, env=None,
+               extra_args=[], join_args=False, **kwargs):
+    self.kind = kind
+    self.name = name
+    self.protocol = protocol
+    self.transport = transport
+    self.socket = socket
+    self.workdir = workdir
+    self.command = None
+    self._base_command = self._fix_cmd_path(command)
+    if env:
+      self.env = copy.copy(os.environ)
+      self.env.update(env)
+    else:
+      self.env = os.environ
+    self._extra_args = extra_args
+    self._join_args = join_args
+
+  def _fix_cmd_path(self, cmd):
+    # if the arg is a file in the current directory, make it path
+    def abs_if_exists(arg):
+      p = os.path.join(self.workdir, arg)
+      return p if os.path.exists(p) else arg
+
+    if cmd[0] == 'python':
+      cmd[0] = sys.executable
+    else:
+      cmd[0] = abs_if_exists(cmd[0])
+    return cmd
+
+  def _socket_arg(self, socket, port):
+    return {
+      'ip-ssl': '--ssl',
+      'domain': '--domain-socket=%s' % domain_socket_path(port),
+    }.get(socket, None)
+
+  def build_command(self, port):
+    cmd = copy.copy(self._base_command)
+    args = []
+    args.append('--protocol=' + self.protocol)
+    args.append('--transport=' + self.transport)
+    socket_arg = self._socket_arg(self.socket, port)
+    if socket_arg:
+      args.append(socket_arg)
+    args.append('--port=%d' % port)
+    if self._join_args:
+      cmd.append('%s' % " ".join(args))
+    else:
+      cmd.extend(args)
+    if self._extra_args:
+      cmd.extend(self._extra_args)
+    self.command = cmd
+    return self.command
+
+
+class TestEntry(object):
+  def __init__(self, testdir, server, client, delay, timeout, **kwargs):
+    self.testdir = testdir
+    self._log = multiprocessing.get_logger()
+    self._config = kwargs
+    self.protocol = kwargs['protocol']
+    self.transport = kwargs['transport']
+    self.socket = kwargs['socket']
+    self.server = TestProgram('server', **self._fix_workdir(merge_dict(self._config, server)))
+    self.client = TestProgram('client', **self._fix_workdir(merge_dict(self._config, client)))
+    self.delay = delay
+    self.timeout = timeout
+    self._name = None
+    # results
+    self.success = None
+    self.as_expected = None
+    self.returncode = None
+    self.expired = False
+
+  def _fix_workdir(self, config):
+    key = 'workdir'
+    path = config.get(key, None)
+    if not path:
+      path = self.testdir
+    if os.path.isabs(path):
+      path = os.path.realpath(path)
+    else:
+      path = os.path.realpath(os.path.join(self.testdir, path))
+    config.update({key: path})
+    return config
+
+  @classmethod
+  def get_name(cls, server, client, proto, trans, sock, *args):
+    return '%s-%s_%s_%s-%s' % (server, client, proto, trans, sock)
+
+  @property
+  def name(self):
+    if not self._name:
+      self._name = self.get_name(
+          self.server.name, self.client.name, self.protocol, self.transport, self.socket)
+    return self._name
+
+  @property
+  def transport_name(self):
+    return '%s-%s' % (self.transport, self.socket)
+
+
+def test_name(server, client, protocol, transport, socket, **kwargs):
+  return TestEntry.get_name(server['name'], client['name'], protocol, transport, socket)
diff --git a/test/crossrunner/util.py b/test/crossrunner/util.py
new file mode 100644
index 0000000..750ed47
--- /dev/null
+++ b/test/crossrunner/util.py
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import copy
+
+
+def merge_dict(base, update):
+  """Update dict concatenating list values"""
+  res = copy.deepcopy(base)
+  for k, v in list(update.items()):
+    if k in list(res.keys()) and isinstance(v, list):
+      res[k].extend(v)
+    else:
+      res[k] = v
+  return res
diff --git a/test/erl/Makefile.am b/test/erl/Makefile.am
index a54e217..1940ce3 100644
--- a/test/erl/Makefile.am
+++ b/test/erl/Makefile.am
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-THRIFT = $(top_srcdir)/compiler/cpp/thrift
+THRIFT = $(top_builddir)/compiler/cpp/thrift
 REBAR = $(top_srcdir)/lib/erl/rebar
 
 THRIFT_FILES = $(wildcard ../*.thrift)
diff --git a/test/go/Makefile.am b/test/go/Makefile.am
index 66f81ad..7357f50 100644
--- a/test/go/Makefile.am
+++ b/test/go/Makefile.am
@@ -17,11 +17,11 @@
 # under the License.
 #
 
-THRIFT = $(top_srcdir)/compiler/cpp/thrift
+THRIFT = $(top_builddir)/compiler/cpp/thrift
 THRIFTCMD = $(THRIFT) -out src/gen --gen go:thrift_import=thrift
 THRIFTTEST = $(top_srcdir)/test/ThriftTest.thrift
 
-all: bin/testclient bin/testserver bin/stress
+precross: bin/testclient bin/testserver
 
 ThriftTest.thrift: $(THRIFTTEST)
 	grep -v list.*map.*list.*map $(THRIFTTEST) > ThriftTest.thrift
@@ -47,6 +47,8 @@
 clean-local:
 	$(RM) -r src/gen src/code.google.com src/thrift bin pkg gopath ThriftTest.thrift
 
+check_PROGRAMS: bin/testclient bin/testserver bin/stress
+
 check: gopath
 	GOPATH=`pwd` $(GO) test -v common/...
 
diff --git a/test/go/src/bin/testserver/main.go b/test/go/src/bin/testserver/main.go
index cd32f92..ebcd8e5 100644
--- a/test/go/src/bin/testserver/main.go
+++ b/test/go/src/bin/testserver/main.go
@@ -31,10 +31,11 @@
 var transport = flag.String("transport", "buffered", "Transport: buffered, framed, http")
 var protocol = flag.String("protocol", "binary", "Protocol: binary, compact, json")
 var ssl = flag.Bool("ssl", false, "Encrypted Transport using SSL")
+var certPath = flag.String("certPath", "keys", "Directory that contains SSL certificates")
 
 func main() {
 	flag.Parse()
-	server, err := common.StartServer(*host, *port, *domain_socket, *transport, *protocol, *ssl, common.PrintingHandler)
+	server, err := common.StartServer(*host, *port, *domain_socket, *transport, *protocol, *ssl, *certPath, common.PrintingHandler)
 	if err != nil {
 		log.Fatalf("Unable to start server: ", err)
 	}
diff --git a/test/go/src/common/clientserver_test.go b/test/go/src/common/clientserver_test.go
index 3b512ad..1b833c9 100644
--- a/test/go/src/common/clientserver_test.go
+++ b/test/go/src/common/clientserver_test.go
@@ -56,7 +56,7 @@
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 	handler := NewMockThriftTest(ctrl)
-	server, err := StartServer(unit.host, unit.port, unit.domain_socket, unit.transport, unit.protocol, unit.ssl, handler)
+	server, err := StartServer(unit.host, unit.port, unit.domain_socket, unit.transport, unit.protocol, unit.ssl, "../../../keys", handler)
 	if err != nil {
 		t.Errorf("Unable to start server", err)
 		t.FailNow()
diff --git a/test/go/src/common/server.go b/test/go/src/common/server.go
index e77cd37..d354b32 100644
--- a/test/go/src/common/server.go
+++ b/test/go/src/common/server.go
@@ -43,6 +43,7 @@
 	transport string,
 	protocol string,
 	ssl bool,
+	certPath string,
 	handler thrifttest.ThriftTest) (srv *thrift.TSimpleServer, err error) {
 
 	hostPort := fmt.Sprintf("%s:%d", host, port)
diff --git a/test/haxe/Makefile.am b/test/haxe/Makefile.am
index 9b7548b..1e537d3 100644
--- a/test/haxe/Makefile.am
+++ b/test/haxe/Makefile.am
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-THRIFT = $(top_srcdir)/compiler/cpp/thrift
+THRIFT = $(top_builddir)/compiler/cpp/thrift
 THRIFTCMD = $(THRIFT) --gen haxe -r
 THRIFTTEST = $(top_srcdir)/test/ThriftTest.thrift
 
diff --git a/test/hs/Makefile.am b/test/hs/Makefile.am
index b974ed8..e171248 100644
--- a/test/hs/Makefile.am
+++ b/test/hs/Makefile.am
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-THRIFT = $(top_srcdir)/compiler/cpp/thrift
+THRIFT = $(top_builddir)/compiler/cpp/thrift
 
 stubs: ../ConstantsDemo.thrift ../DebugProtoTest.thrift ../ThriftTest.thrift ../Include.thrift
 	$(THRIFT) --gen hs ../ConstantsDemo.thrift
@@ -36,6 +36,6 @@
 	$(RM) *.hi
 	$(RM) *.o
 
-all: check
+all-local: stubs
 	ghc -igen-hs TestServer.hs
 	ghc -igen-hs TestClient.hs
diff --git a/test/known_failures_Linux.json b/test/known_failures_Linux.json
new file mode 100644
index 0000000..a83c111
--- /dev/null
+++ b/test/known_failures_Linux.json
@@ -0,0 +1,664 @@
+[
+  "c_glib-c_glib_binary_buffered-ip",
+  "c_glib-c_glib_binary_framed-ip",
+  "c_glib-cpp_binary_buffered-ip",
+  "c_glib-cpp_binary_framed-ip",
+  "c_glib-csharp_binary_buffered-ip",
+  "c_glib-csharp_binary_framed-ip",
+  "c_glib-go_binary_buffered-ip",
+  "c_glib-go_binary_framed-ip",
+  "c_glib-hs_binary_buffered-ip",
+  "c_glib-hs_binary_framed-ip",
+  "c_glib-java_binary_buffered-ip",
+  "c_glib-java_binary_framed-fastframed-ip",
+  "c_glib-java_binary_framed-ip",
+  "c_glib-nodejs_binary_buffered-ip",
+  "c_glib-nodejs_binary_framed-ip",
+  "c_glib-perl_binary_buffered-ip",
+  "c_glib-php_binary_buffered-ip",
+  "c_glib-php_binary_framed-ip",
+  "c_glib-py_binary-accel_buffered-ip",
+  "c_glib-py_binary-accel_framed-ip",
+  "c_glib-py_binary_buffered-ip",
+  "c_glib-py_binary_framed-ip",
+  "c_glib-rb_binary-accel_buffered-ip",
+  "c_glib-rb_binary-accel_framed-ip",
+  "c_glib-rb_binary_buffered-ip",
+  "c_glib-rb_binary_framed-ip",
+  "cpp-cpp_binary_http-domain",
+  "cpp-cpp_binary_http-ip",
+  "cpp-cpp_compact_http-domain",
+  "cpp-cpp_compact_http-ip",
+  "cpp-cpp_json_http-domain",
+  "cpp-cpp_json_http-ip",
+  "cpp-csharp_binary_buffered-ip-ssl",
+  "cpp-csharp_binary_framed-ip-ssl",
+  "cpp-csharp_compact_buffered-ip-ssl",
+  "cpp-csharp_compact_framed-ip-ssl",
+  "cpp-csharp_json_buffered-ip-ssl",
+  "cpp-csharp_json_framed-ip-ssl",
+  "cpp-go_binary_buffered-ip",
+  "cpp-go_binary_buffered-ip-ssl",
+  "cpp-go_binary_framed-ip",
+  "cpp-go_binary_framed-ip-ssl",
+  "cpp-go_compact_buffered-ip",
+  "cpp-go_compact_buffered-ip-ssl",
+  "cpp-go_compact_framed-ip",
+  "cpp-go_compact_framed-ip-ssl",
+  "cpp-go_json_buffered-ip",
+  "cpp-go_json_buffered-ip-ssl",
+  "cpp-go_json_framed-ip",
+  "cpp-go_json_framed-ip-ssl",
+  "cpp-hs_binary_buffered-ip-ssl",
+  "cpp-hs_binary_framed-ip",
+  "cpp-hs_binary_framed-ip-ssl",
+  "cpp-hs_binary_http-evhttp-ip",
+  "cpp-hs_binary_http-evhttp-ip-ssl",
+  "cpp-hs_binary_http-ip",
+  "cpp-hs_binary_http-ip-ssl",
+  "cpp-hs_compact_buffered-ip-ssl",
+  "cpp-hs_compact_framed-ip",
+  "cpp-hs_compact_framed-ip-ssl",
+  "cpp-hs_compact_http-evhttp-ip",
+  "cpp-hs_compact_http-evhttp-ip-ssl",
+  "cpp-hs_compact_http-ip",
+  "cpp-hs_compact_http-ip-ssl",
+  "cpp-hs_json_buffered-ip-ssl",
+  "cpp-hs_json_framed-ip",
+  "cpp-hs_json_framed-ip-ssl",
+  "cpp-hs_json_http-evhttp-ip",
+  "cpp-hs_json_http-evhttp-ip-ssl",
+  "cpp-hs_json_http-ip",
+  "cpp-hs_json_http-ip-ssl",
+  "cpp-java_binary_http-ip",
+  "cpp-java_binary_http-ip-ssl",
+  "cpp-java_compact_http-ip",
+  "cpp-java_compact_http-ip-ssl",
+  "cpp-java_json_http-ip",
+  "cpp-java_json_http-ip-ssl",
+  "cpp-nodejs_json_buffered-ip",
+  "cpp-nodejs_json_buffered-ip-ssl",
+  "cpp-php_binary_framed-ip",
+  "cpp-rb_binary-accel_buffered-ip",
+  "cpp-rb_binary-accel_framed-ip",
+  "cpp-rb_binary_buffered-ip",
+  "cpp-rb_binary_framed-ip",
+  "cpp-rb_compact_buffered-ip",
+  "cpp-rb_compact_framed-ip",
+  "cpp-rb_json_buffered-ip",
+  "cpp-rb_json_framed-ip",
+  "csharp-c_glib_binary_buffered-ip",
+  "csharp-c_glib_binary_framed-ip",
+  "csharp-cpp_binary_buffered-ip",
+  "csharp-cpp_binary_buffered-ip-ssl",
+  "csharp-cpp_binary_framed-ip",
+  "csharp-cpp_binary_framed-ip-ssl",
+  "csharp-cpp_compact_buffered-ip",
+  "csharp-cpp_compact_buffered-ip-ssl",
+  "csharp-cpp_compact_framed-ip",
+  "csharp-cpp_compact_framed-ip-ssl",
+  "csharp-cpp_json_buffered-ip",
+  "csharp-cpp_json_buffered-ip-ssl",
+  "csharp-cpp_json_framed-ip",
+  "csharp-cpp_json_framed-ip-ssl",
+  "csharp-csharp_binary_buffered-ip-ssl",
+  "csharp-csharp_binary_framed-ip-ssl",
+  "csharp-csharp_compact_buffered-ip-ssl",
+  "csharp-csharp_compact_framed-ip-ssl",
+  "csharp-csharp_json_buffered-ip-ssl",
+  "csharp-csharp_json_framed-ip-ssl",
+  "csharp-go_binary_buffered-ip",
+  "csharp-go_binary_buffered-ip-ssl",
+  "csharp-go_binary_framed-ip",
+  "csharp-go_binary_framed-ip-ssl",
+  "csharp-go_compact_buffered-ip",
+  "csharp-go_compact_buffered-ip-ssl",
+  "csharp-go_compact_framed-ip",
+  "csharp-go_compact_framed-ip-ssl",
+  "csharp-go_json_buffered-ip",
+  "csharp-go_json_buffered-ip-ssl",
+  "csharp-go_json_framed-ip",
+  "csharp-go_json_framed-ip-ssl",
+  "csharp-hs_binary_buffered-ip",
+  "csharp-hs_binary_buffered-ip-ssl",
+  "csharp-hs_binary_framed-ip",
+  "csharp-hs_binary_framed-ip-ssl",
+  "csharp-hs_compact_buffered-ip",
+  "csharp-hs_compact_buffered-ip-ssl",
+  "csharp-hs_compact_framed-ip",
+  "csharp-hs_compact_framed-ip-ssl",
+  "csharp-hs_json_buffered-ip",
+  "csharp-hs_json_buffered-ip-ssl",
+  "csharp-hs_json_framed-ip",
+  "csharp-hs_json_framed-ip-ssl",
+  "csharp-java_binary_buffered-ip",
+  "csharp-java_binary_buffered-ip-ssl",
+  "csharp-java_binary_framed-fastframed-ip",
+  "csharp-java_binary_framed-fastframed-ip-ssl",
+  "csharp-java_binary_framed-ip",
+  "csharp-java_binary_framed-ip-ssl",
+  "csharp-java_compact_buffered-ip",
+  "csharp-java_compact_buffered-ip-ssl",
+  "csharp-java_compact_framed-fastframed-ip",
+  "csharp-java_compact_framed-fastframed-ip-ssl",
+  "csharp-java_compact_framed-ip",
+  "csharp-java_compact_framed-ip-ssl",
+  "csharp-java_json_buffered-ip",
+  "csharp-java_json_buffered-ip-ssl",
+  "csharp-java_json_framed-fastframed-ip",
+  "csharp-java_json_framed-fastframed-ip-ssl",
+  "csharp-java_json_framed-ip",
+  "csharp-java_json_framed-ip-ssl",
+  "csharp-nodejs_binary_buffered-ip",
+  "csharp-nodejs_binary_buffered-ip-ssl",
+  "csharp-nodejs_binary_framed-ip",
+  "csharp-nodejs_binary_framed-ip-ssl",
+  "csharp-nodejs_compact_buffered-ip",
+  "csharp-nodejs_compact_buffered-ip-ssl",
+  "csharp-nodejs_compact_framed-ip",
+  "csharp-nodejs_compact_framed-ip-ssl",
+  "csharp-nodejs_json_buffered-ip",
+  "csharp-nodejs_json_buffered-ip-ssl",
+  "csharp-nodejs_json_framed-ip",
+  "csharp-nodejs_json_framed-ip-ssl",
+  "csharp-php_binary_framed-ip",
+  "csharp-py_binary-accel_buffered-ip-ssl",
+  "csharp-py_binary-accel_framed-ip-ssl",
+  "csharp-py_binary_buffered-ip-ssl",
+  "csharp-py_binary_framed-ip-ssl",
+  "csharp-py_compact_buffered-ip-ssl",
+  "csharp-py_compact_framed-ip-ssl",
+  "csharp-py_json_buffered-ip-ssl",
+  "csharp-py_json_framed-ip-ssl",
+  "csharp-rb_binary-accel_buffered-ip",
+  "csharp-rb_binary-accel_framed-ip",
+  "csharp-rb_binary_buffered-ip",
+  "csharp-rb_binary_framed-ip",
+  "csharp-rb_compact_buffered-ip",
+  "csharp-rb_compact_framed-ip",
+  "csharp-rb_json_buffered-ip",
+  "csharp-rb_json_framed-ip",
+  "go-c_glib_binary_buffered-ip",
+  "go-c_glib_binary_framed-ip",
+  "go-cpp_binary_buffered-ip",
+  "go-cpp_binary_buffered-ip-ssl",
+  "go-cpp_binary_framed-ip",
+  "go-cpp_binary_framed-ip-ssl",
+  "go-cpp_compact_buffered-ip",
+  "go-cpp_compact_buffered-ip-ssl",
+  "go-cpp_compact_framed-ip",
+  "go-cpp_compact_framed-ip-ssl",
+  "go-cpp_json_buffered-ip",
+  "go-cpp_json_buffered-ip-ssl",
+  "go-cpp_json_framed-ip",
+  "go-cpp_json_framed-ip-ssl",
+  "go-csharp_binary_buffered-ip",
+  "go-csharp_binary_buffered-ip-ssl",
+  "go-csharp_binary_framed-ip",
+  "go-csharp_binary_framed-ip-ssl",
+  "go-csharp_compact_buffered-ip",
+  "go-csharp_compact_buffered-ip-ssl",
+  "go-csharp_compact_framed-ip",
+  "go-csharp_compact_framed-ip-ssl",
+  "go-csharp_json_buffered-ip",
+  "go-csharp_json_buffered-ip-ssl",
+  "go-csharp_json_framed-ip",
+  "go-csharp_json_framed-ip-ssl",
+  "go-hs_binary_buffered-ip-ssl",
+  "go-hs_binary_framed-ip",
+  "go-hs_binary_framed-ip-ssl",
+  "go-hs_compact_buffered-ip-ssl",
+  "go-hs_compact_framed-ip",
+  "go-hs_compact_framed-ip-ssl",
+  "go-hs_json_buffered-ip-ssl",
+  "go-hs_json_framed-ip",
+  "go-hs_json_framed-ip-ssl",
+  "go-java_binary_buffered-ip",
+  "go-java_binary_buffered-ip-ssl",
+  "go-java_binary_framed-fastframed-ip",
+  "go-java_binary_framed-fastframed-ip-ssl",
+  "go-java_binary_framed-ip",
+  "go-java_binary_framed-ip-ssl",
+  "go-java_compact_buffered-ip",
+  "go-java_compact_buffered-ip-ssl",
+  "go-java_compact_framed-fastframed-ip",
+  "go-java_compact_framed-fastframed-ip-ssl",
+  "go-java_compact_framed-ip",
+  "go-java_compact_framed-ip-ssl",
+  "go-java_json_buffered-ip",
+  "go-java_json_buffered-ip-ssl",
+  "go-java_json_framed-fastframed-ip",
+  "go-java_json_framed-fastframed-ip-ssl",
+  "go-java_json_framed-ip",
+  "go-java_json_framed-ip-ssl",
+  "go-nodejs_binary_buffered-ip",
+  "go-nodejs_binary_buffered-ip-ssl",
+  "go-nodejs_binary_framed-ip",
+  "go-nodejs_binary_framed-ip-ssl",
+  "go-nodejs_compact_buffered-ip",
+  "go-nodejs_compact_buffered-ip-ssl",
+  "go-nodejs_compact_framed-ip",
+  "go-nodejs_compact_framed-ip-ssl",
+  "go-nodejs_json_buffered-ip",
+  "go-nodejs_json_buffered-ip-ssl",
+  "go-nodejs_json_framed-ip",
+  "go-nodejs_json_framed-ip-ssl",
+  "go-perl_binary_buffered-ip",
+  "go-php_binary_buffered-ip",
+  "go-php_binary_framed-ip",
+  "go-py_json_buffered-ip",
+  "go-py_json_buffered-ip-ssl",
+  "go-py_json_framed-ip",
+  "go-py_json_framed-ip-ssl",
+  "go-rb_binary-accel_buffered-ip",
+  "go-rb_binary-accel_framed-ip",
+  "go-rb_binary_buffered-ip",
+  "go-rb_binary_framed-ip",
+  "go-rb_compact_buffered-ip",
+  "go-rb_compact_framed-ip",
+  "go-rb_json_buffered-ip",
+  "go-rb_json_framed-ip",
+  "hs-c_glib_binary_buffered-ip",
+  "hs-c_glib_binary_framed-ip",
+  "hs-cpp_binary_buffered-ip-ssl",
+  "hs-cpp_binary_evhttp-http-ip",
+  "hs-cpp_binary_evhttp-http-ip-ssl",
+  "hs-cpp_binary_framed-ip",
+  "hs-cpp_binary_framed-ip-ssl",
+  "hs-cpp_binary_http-ip",
+  "hs-cpp_binary_http-ip-ssl",
+  "hs-cpp_compact_buffered-ip-ssl",
+  "hs-cpp_compact_evhttp-http-ip",
+  "hs-cpp_compact_evhttp-http-ip-ssl",
+  "hs-cpp_compact_framed-ip",
+  "hs-cpp_compact_framed-ip-ssl",
+  "hs-cpp_compact_http-ip",
+  "hs-cpp_compact_http-ip-ssl",
+  "hs-cpp_json_buffered-ip-ssl",
+  "hs-cpp_json_evhttp-http-ip",
+  "hs-cpp_json_evhttp-http-ip-ssl",
+  "hs-cpp_json_framed-ip",
+  "hs-cpp_json_framed-ip-ssl",
+  "hs-cpp_json_http-ip",
+  "hs-cpp_json_http-ip-ssl",
+  "hs-csharp_binary_buffered-ip",
+  "hs-csharp_binary_buffered-ip-ssl",
+  "hs-csharp_binary_framed-ip",
+  "hs-csharp_binary_framed-ip-ssl",
+  "hs-csharp_compact_buffered-ip",
+  "hs-csharp_compact_buffered-ip-ssl",
+  "hs-csharp_compact_framed-ip",
+  "hs-csharp_compact_framed-ip-ssl",
+  "hs-csharp_json_buffered-ip",
+  "hs-csharp_json_buffered-ip-ssl",
+  "hs-csharp_json_framed-ip",
+  "hs-csharp_json_framed-ip-ssl",
+  "hs-go_binary_buffered-ip",
+  "hs-go_binary_buffered-ip-ssl",
+  "hs-go_binary_framed-ip",
+  "hs-go_binary_framed-ip-ssl",
+  "hs-go_compact_buffered-ip",
+  "hs-go_compact_buffered-ip-ssl",
+  "hs-go_compact_framed-ip",
+  "hs-go_compact_framed-ip-ssl",
+  "hs-go_json_buffered-ip",
+  "hs-go_json_buffered-ip-ssl",
+  "hs-go_json_framed-ip",
+  "hs-go_json_framed-ip-ssl",
+  "hs-java_binary_buffered-ip-ssl",
+  "hs-java_binary_evhttp-http-ip",
+  "hs-java_binary_evhttp-http-ip-ssl",
+  "hs-java_binary_framed-fastframed-ip",
+  "hs-java_binary_framed-fastframed-ip-ssl",
+  "hs-java_binary_framed-ip",
+  "hs-java_binary_framed-ip-ssl",
+  "hs-java_binary_http-ip",
+  "hs-java_binary_http-ip-ssl",
+  "hs-java_compact_buffered-ip-ssl",
+  "hs-java_compact_evhttp-http-ip",
+  "hs-java_compact_evhttp-http-ip-ssl",
+  "hs-java_compact_framed-fastframed-ip",
+  "hs-java_compact_framed-fastframed-ip-ssl",
+  "hs-java_compact_framed-ip",
+  "hs-java_compact_framed-ip-ssl",
+  "hs-java_compact_http-ip",
+  "hs-java_compact_http-ip-ssl",
+  "hs-java_json_buffered-ip-ssl",
+  "hs-java_json_evhttp-http-ip",
+  "hs-java_json_evhttp-http-ip-ssl",
+  "hs-java_json_framed-fastframed-ip",
+  "hs-java_json_framed-fastframed-ip-ssl",
+  "hs-java_json_framed-ip",
+  "hs-java_json_framed-ip-ssl",
+  "hs-java_json_http-ip",
+  "hs-java_json_http-ip-ssl",
+  "hs-nodejs_binary_buffered-ip",
+  "hs-nodejs_binary_buffered-ip-ssl",
+  "hs-nodejs_binary_framed-ip",
+  "hs-nodejs_binary_framed-ip-ssl",
+  "hs-nodejs_compact_buffered-ip",
+  "hs-nodejs_compact_buffered-ip-ssl",
+  "hs-nodejs_compact_framed-ip",
+  "hs-nodejs_compact_framed-ip-ssl",
+  "hs-nodejs_json_buffered-ip",
+  "hs-nodejs_json_buffered-ip-ssl",
+  "hs-nodejs_json_framed-ip",
+  "hs-nodejs_json_framed-ip-ssl",
+  "hs-py_binary-accel_buffered-ip-ssl",
+  "hs-py_binary-accel_framed-ip",
+  "hs-py_binary-accel_framed-ip-ssl",
+  "hs-py_binary_buffered-ip-ssl",
+  "hs-py_binary_framed-ip",
+  "hs-py_binary_framed-ip-ssl",
+  "hs-py_compact_buffered-ip-ssl",
+  "hs-py_compact_framed-ip",
+  "hs-py_compact_framed-ip-ssl",
+  "hs-py_json_buffered-ip-ssl",
+  "hs-py_json_framed-ip",
+  "hs-py_json_framed-ip-ssl",
+  "hs-rb_binary-accel_buffered-ip",
+  "hs-rb_binary-accel_framed-ip",
+  "hs-rb_binary_buffered-ip",
+  "hs-rb_binary_framed-ip",
+  "hs-rb_compact_buffered-ip",
+  "hs-rb_compact_framed-ip",
+  "hs-rb_json_buffered-ip",
+  "hs-rb_json_framed-ip",
+  "java-csharp_binary_buffered-ip-ssl",
+  "java-csharp_binary_fastframed-framed-ip-ssl",
+  "java-csharp_binary_framed-ip-ssl",
+  "java-csharp_compact_buffered-ip-ssl",
+  "java-csharp_compact_fastframed-framed-ip-ssl",
+  "java-csharp_compact_framed-ip-ssl",
+  "java-csharp_json_buffered-ip-ssl",
+  "java-csharp_json_fastframed-framed-ip-ssl",
+  "java-csharp_json_framed-ip-ssl",
+  "java-hs_binary_buffered-ip-ssl",
+  "java-hs_binary_fastframed-framed-ip",
+  "java-hs_binary_fastframed-framed-ip-ssl",
+  "java-hs_binary_framed-ip",
+  "java-hs_binary_framed-ip-ssl",
+  "java-hs_compact_buffered-ip-ssl",
+  "java-hs_compact_fastframed-framed-ip",
+  "java-hs_compact_fastframed-framed-ip-ssl",
+  "java-hs_compact_framed-ip",
+  "java-hs_compact_framed-ip-ssl",
+  "java-hs_json_buffered-ip-ssl",
+  "java-hs_json_fastframed-framed-ip",
+  "java-hs_json_fastframed-framed-ip-ssl",
+  "java-hs_json_framed-ip",
+  "java-hs_json_framed-ip-ssl",
+  "java-nodejs_json_buffered-ip",
+  "java-nodejs_json_buffered-ip-ssl",
+  "java-php_binary_fastframed-framed-ip",
+  "java-php_binary_framed-ip",
+  "java-rb_binary-accel_buffered-ip",
+  "java-rb_binary-accel_fastframed-framed-ip",
+  "java-rb_binary-accel_framed-ip",
+  "java-rb_binary_buffered-ip",
+  "java-rb_binary_fastframed-framed-ip",
+  "java-rb_binary_framed-ip",
+  "java-rb_compact_buffered-ip",
+  "java-rb_compact_fastframed-framed-ip",
+  "java-rb_compact_framed-ip",
+  "java-rb_json_buffered-ip",
+  "java-rb_json_fastframed-framed-ip",
+  "java-rb_json_framed-ip",
+  "nodejs-csharp_binary_buffered-ip",
+  "nodejs-csharp_binary_buffered-ip-ssl",
+  "nodejs-csharp_binary_framed-ip",
+  "nodejs-csharp_binary_framed-ip-ssl",
+  "nodejs-csharp_compact_buffered-ip",
+  "nodejs-csharp_compact_buffered-ip-ssl",
+  "nodejs-csharp_compact_framed-ip",
+  "nodejs-csharp_compact_framed-ip-ssl",
+  "nodejs-csharp_json_buffered-ip",
+  "nodejs-csharp_json_buffered-ip-ssl",
+  "nodejs-csharp_json_framed-ip",
+  "nodejs-csharp_json_framed-ip-ssl",
+  "nodejs-go_binary_buffered-ip",
+  "nodejs-go_binary_buffered-ip-ssl",
+  "nodejs-go_binary_framed-ip",
+  "nodejs-go_binary_framed-ip-ssl",
+  "nodejs-go_compact_buffered-ip",
+  "nodejs-go_compact_buffered-ip-ssl",
+  "nodejs-go_compact_framed-ip",
+  "nodejs-go_compact_framed-ip-ssl",
+  "nodejs-go_json_buffered-ip",
+  "nodejs-go_json_buffered-ip-ssl",
+  "nodejs-go_json_framed-ip",
+  "nodejs-go_json_framed-ip-ssl",
+  "nodejs-hs_binary_buffered-ip-ssl",
+  "nodejs-hs_binary_framed-ip",
+  "nodejs-hs_binary_framed-ip-ssl",
+  "nodejs-hs_compact_buffered-ip-ssl",
+  "nodejs-hs_compact_framed-ip",
+  "nodejs-hs_compact_framed-ip-ssl",
+  "nodejs-hs_json_buffered-ip",
+  "nodejs-hs_json_buffered-ip-ssl",
+  "nodejs-hs_json_framed-ip",
+  "nodejs-hs_json_framed-ip-ssl",
+  "nodejs-java_json_buffered-ip-ssl",
+  "nodejs-nodejs_json_buffered-ip-ssl",
+  "nodejs-php_binary_framed-ip",
+  "nodejs-py_compact_buffered-ip",
+  "nodejs-py_compact_buffered-ip-ssl",
+  "nodejs-py_compact_framed-ip",
+  "nodejs-py_compact_framed-ip-ssl",
+  "nodejs-py_json_buffered-ip",
+  "nodejs-py_json_buffered-ip-ssl",
+  "nodejs-py_json_framed-ip",
+  "nodejs-py_json_framed-ip-ssl",
+  "nodejs-rb_binary-accel_buffered-ip",
+  "nodejs-rb_binary-accel_framed-ip",
+  "nodejs-rb_binary_buffered-ip",
+  "nodejs-rb_binary_framed-ip",
+  "nodejs-rb_compact_buffered-ip",
+  "nodejs-rb_compact_framed-ip",
+  "nodejs-rb_json_buffered-ip",
+  "nodejs-rb_json_framed-ip",
+  "py-c_glib_accel-binary_buffered-ip",
+  "py-c_glib_accel-binary_framed-ip",
+  "py-c_glib_binary_buffered-ip",
+  "py-c_glib_binary_framed-ip",
+  "py-cpp_accel-binary_buffered-ip",
+  "py-cpp_accel-binary_buffered-ip-ssl",
+  "py-cpp_accel-binary_framed-ip",
+  "py-cpp_accel-binary_framed-ip-ssl",
+  "py-cpp_binary_buffered-ip",
+  "py-cpp_binary_buffered-ip-ssl",
+  "py-cpp_binary_framed-ip",
+  "py-cpp_binary_framed-ip-ssl",
+  "py-cpp_compact_buffered-ip",
+  "py-cpp_compact_buffered-ip-ssl",
+  "py-cpp_compact_framed-ip",
+  "py-cpp_compact_framed-ip-ssl",
+  "py-cpp_json_buffered-ip",
+  "py-cpp_json_buffered-ip-ssl",
+  "py-cpp_json_framed-ip",
+  "py-cpp_json_framed-ip-ssl",
+  "py-csharp_accel-binary_buffered-ip",
+  "py-csharp_accel-binary_buffered-ip-ssl",
+  "py-csharp_accel-binary_framed-ip",
+  "py-csharp_accel-binary_framed-ip-ssl",
+  "py-csharp_binary_buffered-ip",
+  "py-csharp_binary_buffered-ip-ssl",
+  "py-csharp_binary_framed-ip",
+  "py-csharp_binary_framed-ip-ssl",
+  "py-csharp_compact_buffered-ip",
+  "py-csharp_compact_buffered-ip-ssl",
+  "py-csharp_compact_framed-ip",
+  "py-csharp_compact_framed-ip-ssl",
+  "py-csharp_json_buffered-ip",
+  "py-csharp_json_buffered-ip-ssl",
+  "py-csharp_json_framed-ip",
+  "py-csharp_json_framed-ip-ssl",
+  "py-go_accel-binary_buffered-ip",
+  "py-go_accel-binary_buffered-ip-ssl",
+  "py-go_accel-binary_framed-ip",
+  "py-go_accel-binary_framed-ip-ssl",
+  "py-go_binary_buffered-ip",
+  "py-go_binary_buffered-ip-ssl",
+  "py-go_binary_framed-ip",
+  "py-go_binary_framed-ip-ssl",
+  "py-go_compact_buffered-ip",
+  "py-go_compact_buffered-ip-ssl",
+  "py-go_compact_framed-ip",
+  "py-go_compact_framed-ip-ssl",
+  "py-go_json_buffered-ip",
+  "py-go_json_buffered-ip-ssl",
+  "py-go_json_framed-ip",
+  "py-go_json_framed-ip-ssl",
+  "py-hs_accel-binary_buffered-ip",
+  "py-hs_accel-binary_buffered-ip-ssl",
+  "py-hs_accel-binary_framed-ip",
+  "py-hs_accel-binary_framed-ip-ssl",
+  "py-hs_binary_buffered-ip",
+  "py-hs_binary_buffered-ip-ssl",
+  "py-hs_binary_framed-ip",
+  "py-hs_binary_framed-ip-ssl",
+  "py-hs_compact_buffered-ip",
+  "py-hs_compact_buffered-ip-ssl",
+  "py-hs_compact_framed-ip",
+  "py-hs_compact_framed-ip-ssl",
+  "py-hs_json_buffered-ip",
+  "py-hs_json_buffered-ip-ssl",
+  "py-hs_json_framed-ip",
+  "py-hs_json_framed-ip-ssl",
+  "py-java_accel-binary_buffered-ip",
+  "py-java_accel-binary_buffered-ip-ssl",
+  "py-java_accel-binary_framed-fastframed-ip",
+  "py-java_accel-binary_framed-fastframed-ip-ssl",
+  "py-java_accel-binary_framed-ip",
+  "py-java_accel-binary_framed-ip-ssl",
+  "py-java_binary_buffered-ip",
+  "py-java_binary_buffered-ip-ssl",
+  "py-java_binary_framed-fastframed-ip",
+  "py-java_binary_framed-fastframed-ip-ssl",
+  "py-java_binary_framed-ip",
+  "py-java_binary_framed-ip-ssl",
+  "py-java_compact_buffered-ip",
+  "py-java_compact_buffered-ip-ssl",
+  "py-java_compact_framed-fastframed-ip",
+  "py-java_compact_framed-fastframed-ip-ssl",
+  "py-java_compact_framed-ip",
+  "py-java_compact_framed-ip-ssl",
+  "py-java_json_buffered-ip",
+  "py-java_json_buffered-ip-ssl",
+  "py-java_json_framed-fastframed-ip",
+  "py-java_json_framed-fastframed-ip-ssl",
+  "py-java_json_framed-ip",
+  "py-java_json_framed-ip-ssl",
+  "py-nodejs_accel-binary_buffered-ip",
+  "py-nodejs_accel-binary_buffered-ip-ssl",
+  "py-nodejs_accel-binary_framed-ip",
+  "py-nodejs_accel-binary_framed-ip-ssl",
+  "py-nodejs_binary_buffered-ip",
+  "py-nodejs_binary_buffered-ip-ssl",
+  "py-nodejs_binary_framed-ip",
+  "py-nodejs_binary_framed-ip-ssl",
+  "py-nodejs_compact_buffered-ip",
+  "py-nodejs_compact_buffered-ip-ssl",
+  "py-nodejs_compact_framed-ip",
+  "py-nodejs_compact_framed-ip-ssl",
+  "py-nodejs_json_buffered-ip",
+  "py-nodejs_json_buffered-ip-ssl",
+  "py-nodejs_json_framed-ip",
+  "py-nodejs_json_framed-ip-ssl",
+  "py-php_accel-binary_framed-ip",
+  "py-php_binary_framed-ip",
+  "py-rb_accel-binary_buffered-ip",
+  "py-rb_accel-binary_framed-ip",
+  "py-rb_accel_buffered-ip",
+  "py-rb_accel_framed-ip",
+  "py-rb_binary-accel_buffered-ip",
+  "py-rb_binary-accel_framed-ip",
+  "py-rb_binary_buffered-ip",
+  "py-rb_binary_framed-ip",
+  "py-rb_compact_buffered-ip",
+  "py-rb_compact_framed-ip",
+  "py-rb_json_buffered-ip",
+  "py-rb_json_framed-ip",
+  "rb-c_glib_accel-binary_buffered-ip",
+  "rb-c_glib_accel-binary_framed-ip",
+  "rb-c_glib_binary_buffered-ip",
+  "rb-c_glib_binary_framed-ip",
+  "rb-cpp_accel-binary_buffered-ip",
+  "rb-cpp_accel-binary_framed-ip",
+  "rb-cpp_binary_buffered-ip",
+  "rb-cpp_binary_framed-ip",
+  "rb-cpp_compact_buffered-ip",
+  "rb-cpp_compact_framed-ip",
+  "rb-cpp_json_buffered-ip",
+  "rb-cpp_json_framed-ip",
+  "rb-csharp_accel-binary_buffered-ip",
+  "rb-csharp_accel-binary_framed-ip",
+  "rb-csharp_binary_buffered-ip",
+  "rb-csharp_binary_framed-ip",
+  "rb-csharp_compact_buffered-ip",
+  "rb-csharp_compact_framed-ip",
+  "rb-csharp_json_buffered-ip",
+  "rb-csharp_json_framed-ip",
+  "rb-go_accel-binary_buffered-ip",
+  "rb-go_accel-binary_framed-ip",
+  "rb-go_binary_buffered-ip",
+  "rb-go_binary_framed-ip",
+  "rb-go_compact_buffered-ip",
+  "rb-go_compact_framed-ip",
+  "rb-go_json_buffered-ip",
+  "rb-go_json_framed-ip",
+  "rb-hs_accel-binary_buffered-ip",
+  "rb-hs_accel-binary_framed-ip",
+  "rb-hs_binary_buffered-ip",
+  "rb-hs_binary_framed-ip",
+  "rb-hs_compact_buffered-ip",
+  "rb-hs_compact_framed-ip",
+  "rb-hs_json_buffered-ip",
+  "rb-hs_json_framed-ip",
+  "rb-java_accel-binary_buffered-ip",
+  "rb-java_accel-binary_framed-fastframed-ip",
+  "rb-java_accel-binary_framed-ip",
+  "rb-java_binary_buffered-ip",
+  "rb-java_binary_framed-fastframed-ip",
+  "rb-java_binary_framed-ip",
+  "rb-java_compact_buffered-ip",
+  "rb-java_compact_framed-fastframed-ip",
+  "rb-java_compact_framed-ip",
+  "rb-java_json_buffered-ip",
+  "rb-java_json_framed-fastframed-ip",
+  "rb-java_json_framed-ip",
+  "rb-nodejs_accel-binary_buffered-ip",
+  "rb-nodejs_accel-binary_framed-ip",
+  "rb-nodejs_binary_buffered-ip",
+  "rb-nodejs_binary_framed-ip",
+  "rb-nodejs_compact_buffered-ip",
+  "rb-nodejs_compact_framed-ip",
+  "rb-nodejs_json_buffered-ip",
+  "rb-nodejs_json_framed-ip",
+  "rb-php_accel-binary_framed-ip",
+  "rb-php_binary_framed-ip",
+  "rb-py_accel-binary_buffered-ip",
+  "rb-py_accel-binary_framed-ip",
+  "rb-py_accel_buffered-ip",
+  "rb-py_accel_framed-ip",
+  "rb-py_binary-accel_buffered-ip",
+  "rb-py_binary-accel_framed-ip",
+  "rb-py_binary_buffered-ip",
+  "rb-py_binary_framed-ip",
+  "rb-py_compact_buffered-ip",
+  "rb-py_compact_framed-ip",
+  "rb-py_json_buffered-ip",
+  "rb-py_json_framed-ip",
+  "rb-rb_accel-binary_buffered-ip",
+  "rb-rb_accel-binary_framed-ip",
+  "rb-rb_accel_buffered-ip",
+  "rb-rb_accel_framed-ip",
+  "rb-rb_binary-accel_buffered-ip",
+  "rb-rb_binary-accel_framed-ip",
+  "rb-rb_binary_buffered-ip",
+  "rb-rb_binary_framed-ip",
+  "rb-rb_compact_buffered-ip",
+  "rb-rb_compact_framed-ip",
+  "rb-rb_json_buffered-ip",
+  "rb-rb_json_framed-ip"
+]
\ No newline at end of file
diff --git a/test/perl/Makefile.am b/test/perl/Makefile.am
index 291106b..d975f69 100644
--- a/test/perl/Makefile.am
+++ b/test/perl/Makefile.am
@@ -17,11 +17,13 @@
 # under the License.
 #
 
-THRIFT = $(top_srcdir)/compiler/cpp/thrift
+THRIFT = $(top_builddir)/compiler/cpp/thrift
 
 stubs: ../ThriftTest.thrift
 	$(THRIFT) --gen perl ../ThriftTest.thrift
 
+precross: stubs
+
 check: stubs
 
 clean-local:
diff --git a/test/perl/TestClient.pl b/test/perl/TestClient.pl
index ca1d47e..5a9a6f1 100644
--- a/test/perl/TestClient.pl
+++ b/test/perl/TestClient.pl
@@ -41,6 +41,11 @@
 my $host = 'localhost';
 my $port = 9090;
 
+foreach my $arg (@ARGV) {
+  if($arg =~ /^--port=([0-9]+)/) {
+    $port = $1;
+  }
+}
 
 my $socket = new Thrift::Socket($host, $port);
 
diff --git a/test/php/Makefile.am b/test/php/Makefile.am
index 1625903..11974da 100755
--- a/test/php/Makefile.am
+++ b/test/php/Makefile.am
@@ -17,12 +17,14 @@
 # under the License.
 #
 
-THRIFT = $(top_srcdir)/compiler/cpp/thrift
+THRIFT = $(top_builddir)/compiler/cpp/thrift
 
 stubs: ../ThriftTest.thrift
 	$(THRIFT) --gen php ../ThriftTest.thrift
 	$(THRIFT) --gen php:inlined ../ThriftTest.thrift
 
+precross: stubs
+
 check: stubs
 
 clean-local:
diff --git a/test/php/TestClient.php b/test/php/TestClient.php
index ea17435..4ec4eab 100755
--- a/test/php/TestClient.php
+++ b/test/php/TestClient.php
@@ -60,6 +60,14 @@
   $host = $argv[1];
 }
 
+foreach ($argv as $arg) {
+  if (substr($arg, 0, 7) == '--port=') {
+    $port = substr($arg, 7);
+  } else if (substr($arg, 0, 11) == '--transport=') {
+    $MODE = substr($arg, 11);
+  }
+}
+
 $hosts = array('localhost');
 
 $socket = new TSocket($host, $port);
diff --git a/test/py.twisted/Makefile.am b/test/py.twisted/Makefile.am
index 4723b7d..17baa59 100644
--- a/test/py.twisted/Makefile.am
+++ b/test/py.twisted/Makefile.am
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-THRIFT = $(top_srcdir)/compiler/cpp/thrift
+THRIFT = $(top_builddir)/compiler/cpp/thrift
 
 stubs: ../ThriftTest.thrift ../SmallTest.thrift
 	$(THRIFT) --gen py:twisted ../ThriftTest.thrift
diff --git a/test/py/Makefile.am b/test/py/Makefile.am
index 2fe9b5a..f8a3aa0 100755
--- a/test/py/Makefile.am
+++ b/test/py/Makefile.am
@@ -18,7 +18,7 @@
 #
 AUTOMAKE_OPTIONS = serial-tests
 
-THRIFT = $(top_srcdir)/compiler/cpp/thrift
+THRIFT = $(top_builddir)/compiler/cpp/thrift
 
 py_unit_tests = RunClientServer.py
 
@@ -38,6 +38,8 @@
         gen-py-dynamicslots/ThriftTest/__init__.py           \
         gen-py-dynamicslots/DebugProtoTest/__init__.py
 
+precross: $(THRIFT) $(thrift_gen)
+
 helper_scripts=                                 \
         TestClient.py                           \
         TestServer.py
@@ -52,27 +54,29 @@
 
 gen-py/%/__init__.py: ../%.thrift
 	$(THRIFT) --gen py  $<
-	test -d gen-py-default || mkdir gen-py-default
+
+gen-py-default/%/__init__.py: ../%.thrift
+	test -d gen-py-default || $(MKDIR_P) gen-py-default
 	$(THRIFT) --gen py -out gen-py-default $<
 
 gen-py-slots/%/__init__.py: ../%.thrift
-	test -d gen-py-slots || mkdir gen-py-slots
+	test -d gen-py-slots || $(MKDIR_P) gen-py-slots
 	$(THRIFT) --gen py:slots -out gen-py-slots $<
 
 gen-py-newstyle/%/__init__.py: ../%.thrift
-	test -d gen-py-newstyle || mkdir gen-py-newstyle
+	test -d gen-py-newstyle || $(MKDIR_P) gen-py-newstyle
 	$(THRIFT) --gen py:new_style -out gen-py-newstyle $<
 
 gen-py-newstyleslots/%/__init__.py: ../%.thrift
-	test -d gen-py-newstyleslots || mkdir gen-py-newstyleslots
+	test -d gen-py-newstyleslots || $(MKDIR_P) gen-py-newstyleslots
 	$(THRIFT) --gen py:new_style,slots -out gen-py-newstyleslots $<
 
 gen-py-dynamic/%/__init__.py: ../%.thrift
-	test -d gen-py-dynamic || mkdir gen-py-dynamic
+	test -d gen-py-dynamic || $(MKDIR_P) gen-py-dynamic
 	$(THRIFT) --gen py:dynamic -out gen-py-dynamic $<
 
 gen-py-dynamicslots/%/__init__.py: ../%.thrift
-	test -d gen-py-dynamicslots || mkdir gen-py-dynamicslots
+	test -d gen-py-dynamicslots || $(MKDIR_P) gen-py-dynamicslots
 	$(THRIFT) --gen py:dynamic,slots -out gen-py-dynamicslots $<
 
 clean-local:
diff --git a/test/rb/Makefile.am b/test/rb/Makefile.am
index 9cdd99b..7b74c6c 100644
--- a/test/rb/Makefile.am
+++ b/test/rb/Makefile.am
@@ -17,12 +17,14 @@
 # under the License.
 #
 
-THRIFT = $(top_srcdir)/compiler/cpp/thrift
+THRIFT = $(top_builddir)/compiler/cpp/thrift
 
-stubs: ../ThriftTest.thrift ../SmallTest.thrift
+stubs: $(THRIFT) ../ThriftTest.thrift ../SmallTest.thrift
 	$(THRIFT) --gen rb ../ThriftTest.thrift
 	$(THRIFT) --gen rb ../SmallTest.thrift
 
+precross: stubs
+
 check: stubs
 if HAVE_BUNDLER
 	$(BUNDLER) install
diff --git a/test/result.html b/test/result.html
index a2dac2c..0f918be 100644
--- a/test/result.html
+++ b/test/result.html
@@ -1,33 +1,74 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
 <!DOCTYPE HTML>
 <html>
 <head>
 <meta charset="utf-8">
 <title>Apache Thrift - integration test suite</title>
-<link rel="stylesheet" type="text/css" href="http://cdn.datatables.net/1.10.0/css/jquery.dataTables.css">
-<script type="text/javascript" charset="utf-8" src="http://code.jquery.com/jquery-1.10.2.min.js"></script>
-<script type="text/javascript" charset="utf-8" src="http://cdn.datatables.net/1.10.0/js/jquery.dataTables.js"></script>
+<link rel="stylesheet" type="text/css" href="http://cdn.datatables.net/1.10.4/css/jquery.dataTables.css">
+<script type="text/javascript" charset="utf-8" src="http://code.jquery.com/jquery-2.1.3.min.js"></script>
+<script type="text/javascript" charset="utf-8" src="http://cdn.datatables.net/1.10.4/js/jquery.dataTables.js"></script>
 <script>
- var test_data;
-
-    $(document).ready( function () {
-      $.getJSON('results.json', function(testData) {
-          testTable = $('#test_results').DataTable( {
-            data: testData,
-      "columnDefs": [
-            {
-                "render": function ( data, type, row ) {
-                    return data +' ('+ '<a href="'+row[5].Client+'">Client</a>,<a href="'+row[5].Server+'">Server</a>'+')';
+$.getJSON('results.json', function(results) {
+    $(document).ready(function() {
+        var transport = 3;
+        var socket = 4;
+        var success = 5;
+        var expected = 6;
+        var returnCode = 7;
+        var logFile = 8;
+        testTable = $('#test_results').DataTable({
+            data: results['results'],
+            columnDefs: [
+                {
+                    targets: 3,
+                    render: function(data, type, row) {
+                        return row[transport] + '-' + row[socket];
+                    },
                 },
-                "targets": 4
-            }
-        ]
-
-          });
-        $('#test_results_filter label input')
-          .focus()
-          .val('failure');
+                {
+                    targets: 4,
+                    render: function(data, type, row) {
+                        return (row[success] ? 'success' : 'failure')
+                                + '(' + (row[returnCode] == 128 ? 'timeout' : row[returnCode]) + ')'
+                                + '(<a href="' + row[logFile].server + '">Server</a>, '
+                                + '<a href="' + row[logFile].client + '">Client</a>)';
+                    },
+                },
+                {
+                    targets: 5,
+                    render: function(data, type, row) {
+                        // 'yes' rather than 'expected' to ease search
+                        return row[expected] ? 'yes' : 'unexpected';
+                    },
+                }
+            ],
         });
-      });
+        $('#test_results_filter label input').focus().val('unexpected failure');
+        $('#test_info').text(
+            "Test Date:     " + results['date'] + "\n" +
+            "Revision:      " + results['revision'] + "\n" +
+            "Platform:      " + results['platform'] + "\n" +
+            "Test duration: " + results['duration']) + " seconds";
+    });
+});
 </script>
 </head>
 <body>
@@ -40,8 +81,11 @@
             <th>Protocol</th>
             <th>Transport</th>
             <th>Result (log)</th>
+            <th>Expected</th>
         </tr>
     </thead>
 </table>
+<h2>Test Information</h2>
+<pre id="test_info"></pre>
 </body>
 </html>
diff --git a/test/test.py b/test/test.py
old mode 100644
new mode 100755
index c04ff8d..1176369
--- a/test/test.py
+++ b/test/test.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements. See the NOTICE file
@@ -19,285 +18,106 @@
 # under the License.
 #
 
-from __future__ import division
-import time
-import socket
-import subprocess
-import sys
-import os
-import signal
+# Apache Thrift - integration test suite
+#
+# tests different server-client, protocol and transport combinations
+#
+# This script supports python 2.7 and later.
+# python 3.x is recommended for better stability.
+#
+# TODO: eliminate a few 2.7 occurrences to support 2.6 ?
+#
+
 import json
-import platform
-import shutil
-import threading
-from optparse import OptionParser
+import logging
+import multiprocessing
+import optparse
+import os
+import sys
 
-parser = OptionParser()
-parser.add_option("--port", type="int", dest="port", default=9090,
-    help="port number for server to listen on")
-parser.add_option('-v', '--verbose', action="store_const",
-    dest="verbose", const=2,
-    help="verbose output")
-parser.add_option('-q', '--quiet', action="store_const",
-    dest="verbose", const=0,
-    help="minimal output")
-parser.add_option("--server", type="string", dest="servers", default="",
-    help="list of servers to test separated by commas, eg:- --server=cpp,java")
-parser.add_option("--client", type="string", dest="clients", default="",
-    help="list of clients to test separated by commas, eg:- --client=cpp,java")
-parser.set_defaults(verbose=1)
-options, args = parser.parse_args()
+import crossrunner
 
-if options.servers == "":
-  serversList = []
-else:
-  serversList = options.servers.split(",")
-if options.clients == "":
-  clientsList = []
-else:
-  clientsList = options.clients.split(",")
+TEST_DIR = os.path.realpath(os.path.dirname(__file__))
+CONFIG_PATH = os.path.join(TEST_DIR, 'tests.json')
 
-def relfile(fname):
-    return os.path.join(os.path.dirname(__file__), fname)
 
-def getSocketArgs(socket_type):
-  if socket_type == 'ip':
-    return ""
-  elif socket_type == 'ip-ssl':
-    return "--ssl"
-  elif socket_type == 'domain':
-    return "--domain-socket=/tmp/ThriftTest.thrift"
+def prepare(server_match, client_match):
+  with open(CONFIG_PATH, 'r') as fp:
+    j = json.load(fp)
+  return crossrunner.prepare(j, TEST_DIR, server_match, client_match)
 
-def runServiceTest(test_name, server_lib, server_executable, server_extra_args, client_lib,  client_executable, client_extra_args, server_protocol, client_protocol, transport, port, use_zlib, socket_type):
-  # Build command line arguments
-  server_args = []
-  cli_args = []
-  if server_lib == 'java':
-    server_args.append(server_executable[0])
-    server_args.append(server_executable[1])
-    server_args.append(relfile(server_executable[2]))
-    server_args.extend(['-Dtestargs','\"'])
+
+def run_tests(server_match, client_match, jobs, skip_known_failures):
+  logger = multiprocessing.get_logger()
+  logger.debug('Collecting tests')
+  with open(CONFIG_PATH, 'r') as fp:
+    j = json.load(fp)
+  tests = list(crossrunner.collect_tests(j, server_match, client_match))
+  if skip_known_failures:
+    known = crossrunner.load_known_failures(TEST_DIR)
+    tests = list(filter(lambda t: crossrunner.test_name(**t) not in known, tests))
+
+  dispatcher = crossrunner.TestDispatcher(TEST_DIR, jobs)
+  logger.debug('Executing %d tests' % len(tests))
+  try:
+    for r in [dispatcher.dispatch(test) for test in tests]:
+      r.wait()
+    logger.debug('Waiting for completion')
+    return dispatcher.wait()
+  except (KeyboardInterrupt, SystemExit):
+    logger.debug('Interrupted, shutting down')
+    dispatcher.terminate()
+    return False
+
+
+def default_concurrenty():
+  try:
+    return int(os.environ.get('THRIFT_CROSSTEST_CONCURRENCY'))
+  except (TypeError, ValueError):
+    # Since much time is spent sleeping, use many threads
+    return int(multiprocessing.cpu_count() * 1.25) + 1
+
+
+def main(argv):
+  parser = optparse.OptionParser()
+  parser.add_option('--server', type='string', dest='servers', default='',
+                    help='list of servers to test separated by commas, eg:- --server=cpp,java')
+  parser.add_option('--client', type='string', dest='clients', default='',
+                    help='list of clients to test separated by commas, eg:- --client=cpp,java')
+  parser.add_option('-s', '--skip-known-failures', action='store_true', dest='skip_known_failures',
+                    help='do not execute tests that are known to fail')
+  parser.add_option('-j', '--jobs', type='int', dest='jobs',
+                    default=default_concurrenty(),
+                    help='number of concurrent test executions')
+  g = optparse.OptionGroup(parser, 'Advanced')
+  g.add_option('-v', '--verbose', action='store_const',
+               dest='log_level', const=logging.DEBUG, default=logging.WARNING,
+               help='show debug output for test runner')
+  g.add_option('-P', '--print-expected-failures', choices=['merge', 'overwrite'],
+               dest='print_failures', default=None,
+               help="generate expected failures based on last result and print to stdout")
+  g.add_option('-U', '--update-expected-failures', choices=['merge', 'overwrite'],
+               dest='update_failures', default=None,
+               help="generate expected failures based on last result and save to default file location")
+  g.add_option('--prepare', action='store_true',
+               dest='prepare',
+               help="try to prepare files needed for cross test (experimental)")
+  parser.add_option_group(g)
+  logger = multiprocessing.log_to_stderr()
+  options, _ = parser.parse_args(argv)
+  server_match = options.servers.split(',') if options.servers else []
+  client_match = options.clients.split(',') if options.clients else []
+  logger.setLevel(options.log_level)
+
+  if options.prepare:
+    res = prepare(server_match, client_match)
+  elif options.update_failures or options.print_failures:
+    res = crossrunner.generate_known_failures(
+        TEST_DIR, options.update_failures == 'overwrite',
+        options.update_failures, options.print_failures)
   else:
-    server_args = [relfile(server_executable)]
-  if client_lib == 'java':
-    cli_args.append(client_executable[0])
-    cli_args.append(client_executable[1])
-    cli_args.append(relfile(client_executable[2]))
-    cli_args.extend(['-Dtestargs','\"'])
-  else:
-    cli_args = [relfile(client_executable)]
+    res = run_tests(server_match, client_match, options.jobs, options.skip_known_failures)
+  return 0 if res else 1
 
-  server_args.append('--protocol=%s' % server_protocol)
-  cli_args.append('--protocol=%s' % client_protocol)
-
-  for which in (server_args, cli_args):
-    which.append('--transport=%s' % transport)
-    which.append('--port=%d' % port) # default to 9090
-    if use_zlib:
-      which.append('--zlib')
-    if socket_type == 'ip-ssl':
-      which.append('--ssl')
-    elif socket_type == 'domain':
-      which.append('--domain-socket=/tmp/ThriftTest.thrift')
-#    if options.verbose == 0:
-#      which.append('-q')
-#    if options.verbose == 2:
-#      which.append('-v')
-  if server_lib == 'java':
-    server_args.append('\"')
-  if client_lib == 'java':
-    cli_args.append('\"')
-
-  server_args.extend(server_extra_args)
-  cli_args.extend(client_extra_args)
-
-  server_log=open(relfile("log/" + test_name + "_server.log"),"a")
-  client_log=open(relfile("log/" + test_name + "_client.log"),"a")
-
-  try:
-    if options.verbose > 0:
-      print 'Testing server: %s' % (' '.join(server_args))
-      serverproc = subprocess.Popen(server_args, stdout=server_log, stderr=server_log)
-    else:
-      serverproc = subprocess.Popen(server_args, stdout=server_log, stderr=server_log)
-  except OSError as e:
-    return "OS error({0}): {1}".format(e.errno, e.strerror)
-
-  def ensureServerAlive():
-    if serverproc.poll() is not None:
-      return 'Server subprocess died, args: %s' % (' '.join(server_args))
-
-  # Wait for the server to start accepting connections on the given port.
-  sock = socket.socket()
-  sleep_time = 0.1  # Seconds
-  max_attempts = 100
-  try:
-    attempt = 0
-
-    if socket_type != 'domain':
-      while sock.connect_ex(('127.0.0.1', port)) != 0:
-        attempt += 1
-        if attempt >= max_attempts:
-          return "TestServer not ready on port %d after %.2f seconds" % (port, sleep_time * attempt)
-        ensureServerAlive()
-        time.sleep(sleep_time)
-  finally:
-    sock.close()
-
-  try:
-    o = []
-    def target():
-      try:
-        if options.verbose > 0:
-          print 'Testing client: %s' % (' '.join(cli_args))
-          process = subprocess.Popen(cli_args, stdout=client_log, stderr=client_log)
-          o.append(process)
-          process.communicate()
-        else:
-          process = subprocess.Popen(cli_args, stdout=client_log, stderr=client_log)
-          o.append(process)
-          process.communicate()
-      except OSError as e:
-        return "OS error({0}): {1}".format(e.errno, e.strerror)
-      except:
-        return "Unexpected error:", sys.exc_info()[0]
-    thread = threading.Thread(target=target)
-    thread.start()
-
-    thread.join(10)
-    if thread.is_alive():
-      print 'Terminating process'
-      o[0].terminate()
-      thread.join()
-    if(len(o)==0):
-      return "Client subprocess failed, args: %s" % (' '.join(cli_args))
-    ret = o[0].returncode
-    if ret != 0:
-      return "Client subprocess failed, retcode=%d, args: %s" % (ret, ' '.join(cli_args))
-      #raise Exception("Client subprocess failed, retcode=%d, args: %s" % (ret, ' '.join(cli_args)))
-  finally:
-    # check that server didn't die
-    #ensureServerAlive()
-    extra_sleep = 0
-    if extra_sleep > 0 and options.verbose > 0:
-      print ('Giving (protocol=%s,zlib=%s,ssl=%s) an extra %d seconds for child'
-             'processes to terminate via alarm'
-             % (protocol, use_zlib, use_ssl, extra_sleep))
-      time.sleep(extra_sleep)
-    os.kill(serverproc.pid, signal.SIGTERM)
-    #serverproc.wait()
-  client_log.flush()
-  server_log.flush()
-  client_log.close()
-  server_log.close()
-
-test_count = 0
-failed = 0
-hard_fail_count = 0
-platform = platform.system()
-if os.path.exists(relfile('log')): shutil.rmtree(relfile('log'))
-os.makedirs(relfile('log'))
-if os.path.exists(relfile('results.json')): os.remove(relfile('results.json'))
-results_json = open(relfile("results.json"),"a")
-results_json.write("[\n")
-
-with open(relfile('tests.json')) as data_file:
-    data = json.load(data_file)
-
-#subprocess.call("export NODE_PATH=../lib/nodejs/test:../lib/nodejs/lib:${NODE_PATH}")
-count = 0
-for server in data["server"]:
-  if (server["lib"] in serversList or len(serversList) == 0) and platform in server["platform"]:
-    server_executable = server["executable"]
-    server_extra_args = ""
-    server_lib = server["lib"]
-    if "extra_args" in server:
-      server_extra_args = server["extra_args"]
-    for protocol in server["protocols"]:
-      for transport in server["transports"]:
-        for sock in server["sockets"]:
-          for client in data["client"]:
-            if (client["lib"] in clientsList or len(clientsList) == 0) and platform in client["platform"]:
-              client_executable = client["executable"]
-              client_extra_args = ""
-              client_lib = client["lib"]
-              if "extra_args" in client:
-                client_extra_args = client["extra_args"]
-              if protocol in client["protocols"]:
-                if transport in client["transports"]:
-                  if sock in client["sockets"]:
-                    if count != 0:
-                      results_json.write(",\n")
-                    count = 1
-                    results_json.write("\t[\n\t\t\"" + server_lib + "\",\n\t\t\"" + client_lib + "\",\n\t\t\"" + protocol + "\",\n\t\t\"" + transport + "-" + sock + "\",\n" )
-                    test_name = server_lib + "_" + client_lib + "_" + protocol + "_" + transport + "_" + sock
-                    ret = runServiceTest(test_name, server_lib, server_executable, server_extra_args, client_lib, client_executable, client_extra_args, protocol, protocol, transport, options.port, 0, sock)
-                    if ret != None:
-                      failed += 1
-                      if client["exit"] == "hard" and server["exit"] == "hard":
-                        hard_fail_count +=1
-                      print "Error: %s" % ret
-                      print "Using"
-                      print (' Server: %s --protocol=%s --transport=%s %s %s'
-                        % (server_executable, protocol, transport, getSocketArgs(sock), ' '.join(server_extra_args)))
-                      print (' Client: %s --protocol=%s --transport=%s %s %s'
-                        % (client_executable, protocol, transport, getSocketArgs(sock), ''.join(client_extra_args)))
-                      results_json.write("\t\t\"failure\",\n")
-                    else:
-                      results_json.write("\t\t\"success\",\n")
-                    results_json.write("\t\t{\n\t\t\t\"Client\":\"log/" + test_name + "_client.log\",\n\t\t\t\"Server\":\"log/" + test_name + "_server.log\"\n\t\t}\n\t]")
-                    test_count += 1
-              if protocol == 'binary' and 'accel' in client["protocols"]:
-                if transport in client["transports"]:
-                  if sock in client["sockets"]:
-                    if count != 0:
-                      results_json.write(",\n")
-                    count = 1
-                    results_json.write("\t[\n\t\t\"" + server_lib + "\",\n\t\t\"" + client_lib + "\",\n\t\t\"accel-binary\",\n\t\t\"" + transport + "-" + sock + "\",\n" )
-                    test_name = server_lib + "_" + client_lib + "_accel-binary_" + transport + "_" + sock
-                    ret = runServiceTest(test_name, server_lib,server_executable, server_extra_args, client_lib, client_executable, client_extra_args, protocol, 'accel', transport, options.port, 0, sock)
-
-                    if ret != None:
-                      failed += 1
-                      if client["exit"] == "hard" and server["exit"] == "hard":
-                        hard_fail_count +=1
-                      print "Error: %s" % ret
-                      print "Using"
-                      print (' Server: %s --protocol=%s --transport=%s %s %s'
-                        % (server_executable, protocol, transport, getSocketArgs(sock), ' '.join(server_extra_args)))
-                      print (' Client: %s --protocol=%s --transport=%s %s %s'
-                        % (client_executable, protocol, transport , getSocketArgs(sock), ''.join(client_extra_args)))
-                      results_json.write("\t\t\"failure\",\n")
-                    else:
-                      results_json.write("\t\t\"success\",\n")
-                    results_json.write("\t\t{\n\t\t\t\"Client\":\"log/" + test_name + "_client.log\",\n\t\t\t\"Server\":\"log/" + test_name + "_server.log\"\n\t\t}\n\t]")
-                    test_count += 1
-              if protocol == 'accel' and 'binary' in client["protocols"]:
-                if transport in client["transports"]:
-                  if sock in client["sockets"]:
-                    if count != 0:
-                      results_json.write(",\n")
-                    count = 1
-                    results_json.write("\t[\n\t\t\"" + server_lib + "\",\n\t\t\"" + client_lib + "\",\n\t\t\"binary-accel\",\n\t\t\"" + transport + "-" + sock + "\",\n" )
-                    test_name = server_lib + "_" + client_lib + "_binary-accel_" + transport + "_" + sock
-                    ret = runServiceTest(test_name, server_lib,server_executable, server_extra_args, client_lib, client_executable, client_extra_args, protocol, 'binary', transport, options.port, 0, sock)
-                    if ret != None:
-                      failed += 1
-                      if client["exit"] == "hard" and server["exit"] == "hard":
-                        hard_fail_count +=1
-                      print "Error: %s" % ret
-                      print "Using"
-                      print (' Server: %s --protocol=%s --transport=%s %s %s'
-                        % (server_executable, protocol, transport + sock, getSocketArgs(sock), ' '.join(server_extra_args)))
-                      print (' Client: %s --protocol=%s --transport=%s %s %s'
-                        % (client_executable, protocol, transport + sock, getSocketArgs(sock), ''.join(client_extra_args)))
-                      results_json.write("\t\t\"failure\",\n")
-                    else:
-                      results_json.write("\t\t\"success\",\n")
-                    results_json.write("\t\t{\n\t\t\t\"Client\":\"log/" + test_name + "_client.log\",\n\t\t\t\"Server\":\"log/" + test_name + "_server.log\"\n\t\t}\n\t]")
-                    test_count += 1
-results_json.write("\n]")
-results_json.flush()
-results_json.close()
-print '%s failed of %s tests in total' % (failed, test_count)
-sys.exit(hard_fail_count)
\ No newline at end of file
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))
diff --git a/test/tests.json b/test/tests.json
index 5abafdf..c428786 100644
--- a/test/tests.json
+++ b/test/tests.json
@@ -1,280 +1,344 @@
-{
-    "client": [
-        {
-            "description": "Python TestClient",
-            "lib": "py",
-            "executable": "py/TestClient.py",
-            "exit": "hard",
-            "extra_args":  ["--genpydir=gen-py"],
-            "protocols": [
-                "binary",
-                "compact",
-                "json",
-                "accel"
-            ],
-            "transports": [
-                "buffered",
-                "framed"
-            ],
-            "sockets": [
-                "ip",
-                "ip-ssl"
-            ],
-            "platform": [
-                "Linux"
-            ]
-        },
-        {
-            "description": "C++ TestClient",
-            "lib": "cpp",
-            "executable": "cpp/TestClient",
-            "exit": "hard",
-            "protocols": [
-                "binary",
-                "compact",
-                "json"
-            ],
-            "transports": [
-                "buffered",
-                "framed",
-                "http"
-            ],
-            "sockets": [
-                "ip",
-                "ip-ssl",
-                "domain"
-            ],
-            "platform": [
-                "Linux"
-            ]
-        },
-        {
-            "description": "Nodejs TestClient",
-            "lib": "nodejs",
-            "executable": "../lib/nodejs/test/client.js",
-            "exit": "soft",
-            "protocols": [
-                "binary",
-                "compact",
-                "json"
-            ],
-            "transports": [
-                "buffered",
-                "framed"
-            ],
-            "sockets": [
-                "ip",
-                "ip-ssl"
-            ],
-            "platform": [
-                "Linux"
-            ]
-        },
-        {
-            "description": "Ruby TestClient",
-            "lib": "ruby",
-            "executable": "rb/integration/TestClient.rb",
-            "exit": "soft",
-            "protocols": [
-                "binary",
-                "compact",
-                "json",
-                "accel"
-            ],
-            "transports": [
-                "buffered",
-                "framed"
-            ],
-            "sockets": [
-                "ip"
-            ],
-            "platform": [
-                "Linux"
-            ]
-        },
-        {
-            "description": "Java TestClient",
-            "lib": "java",
-            "executable": ["ant","-f","../lib/java/build.xml","-Dno-gen-thrift=\"\""],
-            "extra_args": ["run-testclient"],
-            "exit": "hard",
-            "protocols": [
-                "binary",
-                "compact",
-                "json"
-            ],
-            "transports": [
-                "buffered",
-                "framed",
-                "fastframed",
-                "http"
-            ],
-            "sockets": [
-                "ip",
-                "ip-ssl"
-            ],
-            "platform": [
-                "Linux"
-            ]
-        },
-        {
-            "description": "Haskell TestClient",
-            "lib": "hs",
-            "executable": "hs/TestClient",
-            "exit": "hard",
-            "protocols": [
-                "binary",
-                "compact",
-                "json"
-            ],
-            "transports": [
-                "buffered"
-            ],
-            "sockets": [
-                "ip"
-            ],
-            "platform": [
-                "Linux"
-            ]
-        }
+[
+  {
+    "name": "c_glib",
+    "platforms": [
+      "Linux"
     ],
-    "server": [
-        {
-            "description": "Python TSimpleServer",
-            "lib": "py",
-            "executable": "py/TestServer.py",
-            "extra_args": ["--genpydir=gen-py", "TSimpleServer"],
-            "extra_delay": 0,
-            "exit": "soft",
-            "protocols": [
-                "binary",
-                "compact",
-                "json",
-                "accel"
-            ],
-            "transports": [
-                "buffered",
-                "framed"
-            ],
-            "sockets": [
-                "ip",
-                "ip-ssl"
-            ],
-            "platform": [
-                "Linux"
-            ]
-        },
-        {
-            "description": "C++ TestServer",
-            "lib": "cpp",
-            "executable": "cpp/TestServer",
-            "exit": "hard",
-            "protocols": [
-                "binary",
-                "compact",
-                "json"
-            ],
-            "transports": [
-                "buffered",
-                "framed",
-                "http"
-            ],
-            "sockets": [
-                "ip",
-                "ip-ssl",
-                "domain"
-            ],
-            "platform": [
-                "Linux"
-            ]
-        },
-        {
-            "description": "Ruby TestServer",
-            "lib": "ruby",
-            "executable": "rb/integration/TestServer.rb",
-            "exit": "soft",
-            "protocols": [
-                "binary",
-                "compact",
-                "json",
-                "accel"
-            ],
-            "transports": [
-                "buffered",
-                "framed"
-            ],
-            "sockets": [
-                "ip"
-            ],
-            "platform": [
-                "Linux"
-            ]
-        },
-        {
-            "description": "Nodejs TestServer",
-            "lib": "nodejs",
-            "executable": "../lib/nodejs/test/server.js",
-            "exit": "soft",
-            "protocols": [
-                "binary",
-                "compact",
-                "json"
-            ],
-            "transports": [
-                "buffered",
-                "framed"
-            ],
-            "sockets": [
-                "ip",
-                "ip-ssl"
-            ],
-            "platform": [
-                "Linux"
-            ]
-        },
-        {
-            "description": "Java TestServer",
-            "lib": "java",
-            "executable": ["ant","-f","../lib/java/build.xml","-Dno-gen-thrift=\"\""],
-            "extra_args": ["run-testserver"],
-            "exit": "hard",
-            "protocols": [
-                "binary",
-                "compact",
-                "json"
-            ],
-            "transports": [
-                "buffered",
-                "framed",
-                "fastframed"
-            ],
-            "sockets": [
-                "ip",
-                "ip-ssl"
-            ],
-            "platform": [
-                "Linux"
-            ]
-        },
-        {
-            "description": "Haskell TestServer",
-            "lib": "hs",
-            "executable": "hs/TestServer",
-            "exit": "hard",
-            "protocols": [
-                "binary",
-                "compact",
-                "json"
-            ],
-            "transports": [
-                "buffered"
-            ],
-            "sockets": [
-                "ip"
-            ],
-            "platform": [
-                "Linux"
-            ]
-        }
-    ]
-}
+    "server": {
+      "delay": 1,
+      "command": [
+        "test_server"
+      ]
+    },
+    "client": {
+      "command": [
+        "test_client"
+      ]
+    },
+    "transports": [
+      "buffered",
+      "framed"
+    ],
+    "sockets": [
+      "ip"
+    ],
+    "protocols": [
+      "binary"
+    ],
+    "workdir": "c_glib"
+  },
+  {
+    "name": "go",
+    "server": {
+      "delay": 1,
+      "command": [
+        "testserver",
+        "--certPath=../../keys"
+      ]
+    },
+    "client": {
+      "timeout": 6,
+      "command": [
+        "testclient"
+      ]
+    },
+    "transports": [
+      "buffered",
+      "framed"
+    ],
+    "sockets": [
+      "ip",
+      "ip-ssl"
+    ],
+    "protocols": [
+      "binary",
+      "compact",
+      "json"
+    ],
+    "workdir": "go/bin"
+  },
+  {
+    "name": "java",
+    "join_args": true,
+    "command": [
+      "ant",
+      "-f",
+      "build.xml",
+      "-Dno-gen-thrift=\"\"",
+      "-Dtestargs"
+    ],
+    "prepare": [
+      "ant",
+      "-f",
+      "build.xml",
+      "compile-test"
+    ],
+    "server": {
+      "delay": 5,
+      "extra_args": ["run-testserver"]
+    },
+    "client": {
+      "timeout": 13,
+      "extra_args": ["run-testclient"],
+      "transports": [
+        "http"
+      ]
+    },
+    "transports": [
+      "buffered",
+      "framed",
+      "framed:fastframed"
+    ],
+    "sockets": [
+      "ip-ssl",
+      "ip"
+    ],
+    "protocols": [
+      "compact",
+      "binary",
+      "json"
+    ],
+    "workdir": "../lib/java"
+  },
+  {
+    "name": "nodejs",
+    "env": {
+      "NODE_PATH": "../lib"
+    },
+    "server": {
+      "delay": 1,
+      "command": [
+        "node",
+        "server.js",
+        "--type=tcp"
+      ]
+    },
+    "client": {
+      "timeout": 2.9,
+      "command": [
+        "node",
+        "client.js",
+        "--type=tcp"
+      ]
+    },
+    "transports": [
+      "buffered",
+      "framed"
+    ],
+    "sockets": [
+      "ip-ssl",
+      "ip"
+    ],
+    "protocols": [
+      "compact",
+      "binary",
+      "json"
+    ],
+    "workdir": "../lib/nodejs/test"
+  },
+  {
+    "name": "hs",
+    "server": {
+      "command": [
+        "TestServer"
+      ]
+    },
+    "client": {
+      "timeout": 6,
+      "command": [
+        "TestClient"
+      ]
+    },
+    "transports": [
+      "buffered",
+      "framed",
+      "http",
+      "http:evhttp"
+    ],
+    "sockets": [
+      "ip",
+      "ip-ssl"
+    ],
+    "protocols": [
+      "compact",
+      "binary",
+      "json"
+    ],
+    "workdir": "hs"
+  },
+  {
+    "name": "py",
+    "server": {
+      "delay": 1,
+      "extra_args": ["TSimpleServer"],
+      "command": [
+        "TestServer.py",
+        "--genpydir=gen-py"
+      ]
+    },
+    "client": {
+      "timeout": 10,
+      "command": [
+        "TestClient.py",
+        "--host=localhost",
+        "--genpydir=gen-py"
+      ]
+    },
+    "transports": [
+      "buffered",
+      "framed"
+    ],
+    "sockets": [
+      "ip-ssl",
+      "ip"
+    ],
+    "protocols": [
+      "compact",
+      "binary",
+      "json",
+      "binary:accel"
+    ],
+    "workdir": "py"
+  },
+  {
+    "name": "cpp",
+    "server": {
+      "delay": 2,
+      "command": [
+        "TestServer"
+      ]
+    },
+    "client": {
+      "timeout": 8,
+      "command": [
+        "TestClient"
+      ]
+    },
+    "transports": [
+      "buffered",
+      "http",
+      "framed"
+    ],
+    "sockets": [
+      "ip-ssl",
+      "ip",
+      "domain"
+    ],
+    "protocols": [
+      "compact",
+      "binary",
+      "json"
+    ],
+    "workdir": "cpp"
+  },
+  {
+    "name": "rb",
+    "server": {
+      "delay": 1,
+      "command": [
+        "ruby",
+        "../integration/TestServer.rb"
+      ]
+    },
+    "client": {
+      "timeout": 5,
+      "command": [
+        "ruby",
+        "../integration/TestClient.rb",
+        "--host=127.0.0.1"
+      ]
+    },
+    "transports": [
+      "buffered",
+      "framed"
+    ],
+    "sockets": [
+      "ip"
+    ],
+    "protocols": [
+      "compact",
+      "binary",
+      "json",
+      "binary:accel"
+    ],
+    "workdir": "rb/gen-rb"
+  },
+  {
+    "name": "csharp",
+    "env": {
+      "MONO_PATH": "../.."
+    },
+    "transports": [
+      "buffered",
+      "framed"
+    ],
+    "sockets": [
+      "ip",
+      "ip-ssl"
+    ],
+    "protocols": [
+      "binary",
+      "compact",
+      "json"
+    ],
+    "server": {
+      "delay": 3,
+      "command": [
+        "TestClientServer.exe",
+        "server",
+        "--cert=../../../../test/keys/server.pem"
+      ]
+    },
+    "client": {
+      "timeout": 9,
+      "command": [
+        "TestClientServer.exe",
+        "client",
+        "--cert=../../../../test/keys/client.pem"
+      ]
+    },
+    "workdir": "../lib/csharp/test/ThriftTest"
+  },
+  {
+    "name": "perl",
+    "client": {
+      "transports": [
+        "buffered"
+      ],
+      "sockets": [
+        "ip"
+      ],
+      "protocols": [
+        "binary"
+      ],
+      "command": [
+        "perl",
+        "-Igen-perl/",
+        "-I../../lib/perl/lib/",
+        "TestClient.pl"
+      ]
+    },
+    "workdir": "perl"
+  },
+  {
+    "name": "php",
+    "client": {
+      "timeout": 6,
+      "transports": [
+        "buffered",
+        "framed"
+      ],
+      "sockets": [
+        "ip"
+      ],
+      "protocols": [
+        "binary"
+      ],
+      "command": [
+        "php",
+        "TestClient.php"
+      ]
+    },
+    "workdir": "php"
+  }
+]