Implement episodic compilation for js code generation
diff --git a/compiler/cpp/Makefile.am b/compiler/cpp/Makefile.am
index 16d4d3a..9a3aeda 100644
--- a/compiler/cpp/Makefile.am
+++ b/compiler/cpp/Makefile.am
@@ -65,7 +65,7 @@
src/thrift/parse/t_type.h \
src/thrift/parse/t_typedef.cc \
src/thrift/parse/t_typedef.h \
- src/thrift/platform.h
+ src/thrift/platform.h
# Specific client generator source
thrift_SOURCES += src/thrift/generate/t_as3_generator.cc \
diff --git a/compiler/cpp/src/thrift/generate/t_js_generator.cc b/compiler/cpp/src/thrift/generate/t_js_generator.cc
index af402a4..88758b2 100644
--- a/compiler/cpp/src/thrift/generate/t_js_generator.cc
+++ b/compiler/cpp/src/thrift/generate/t_js_generator.cc
@@ -26,6 +26,7 @@
#include <vector>
#include <list>
#include <cassert>
+#include <unordered_map>
#include <stdlib.h>
#include <sys/stat.h>
@@ -38,9 +39,11 @@
using std::ostringstream;
using std::string;
using std::stringstream;
+using std::unordered_map;
using std::vector;
static const string endl = "\n"; // avoid ostream << std::endl flushes
+static const string episode_file_name = "thrift.js.episode";
// largest consecutive integer representable by a double (2 ^ 53 - 1)
static const int64_t max_safe_integer = 0x1fffffffffffff;
// smallest consecutive number representable by a double (-2 ^ 53 + 1)
@@ -65,6 +68,7 @@
gen_jquery_ = false;
gen_ts_ = false;
gen_es6_ = false;
+ gen_episode_file_ = false;
bool with_ns_ = false;
@@ -79,22 +83,26 @@
with_ns_ = true;
} else if( iter->first.compare("es6") == 0) {
gen_es6_ = true;
+ } else if( iter->first.compare("imports") == 0) {
+ parse_imports(program, iter->second);
+ } else if (iter->first.compare("thrift_package_output_directory") == 0) {
+ parse_thrift_package_output_directory(iter->second);
} else {
- throw "unknown option js:" + iter->first;
+ throw std::invalid_argument("unknown option js:" + iter->first);
}
}
if (gen_es6_ && gen_jquery_) {
- throw "Invalid switch: [-gen js:es6,jquery] options not compatible";
+ throw std::invalid_argument("invalid switch: [-gen js:es6,jquery] options not compatible");
}
if (gen_node_ && gen_jquery_) {
- throw "Invalid switch: [-gen js:node,jquery] options not compatible, try: [-gen js:node -gen "
- "js:jquery]";
+ throw std::invalid_argument("invalid switch: [-gen js:node,jquery] options not compatible, try: [-gen js:node -gen "
+ "js:jquery]");
}
if (!gen_node_ && with_ns_) {
- throw "Invalid switch: [-gen js:with_ns] is only valid when using node.js";
+ throw std::invalid_argument("invalid switch: [-gen js:with_ns] is only valid when using node.js");
}
// Depending on the processing flags, we will update these to be ES6 compatible
@@ -207,6 +215,7 @@
std::string ts_service_includes();
std::string render_includes();
std::string render_ts_includes();
+ std::string get_import_path(t_program* program);
std::string declare_field(t_field* tfield, bool init = false, bool obj = false);
std::string function_signature(t_function* tfunction,
std::string prefix = "",
@@ -215,6 +224,13 @@
std::string type_to_enum(t_type* ttype);
std::string make_valid_nodeJs_identifier(std::string const& name);
+ /**
+ * Helper parser functions
+ */
+
+ void parse_imports(t_program* program, const std::string& imports_string);
+ void parse_thrift_package_output_directory(const std::string& thrift_package_output_directory);
+
std::string autogen_comment() override {
return std::string("//\n") + "// Autogenerated by Thrift Compiler (" + THRIFT_VERSION + ")\n"
+ "//\n" + "// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n"
@@ -354,6 +370,11 @@
bool gen_es6_;
/**
+ * True if we will generate an episode file.
+ */
+ bool gen_episode_file_;
+
+ /**
* The name of the defined module(s), for TypeScript Definition Files.
*/
string ts_module_;
@@ -364,6 +385,21 @@
bool no_ns_;
/**
+ * The node modules to use when importing the previously generated files.
+ */
+ vector<string> imports;
+
+ /**
+ * Cache for imported modules.
+ */
+ unordered_map<string, string> module_name_2_import_path;
+
+ /**
+ * The prefix to use when generating the episode file.
+ */
+ string thrift_package_output_directory_;
+
+ /**
* The variable decorator for "const" variables. Will default to "var" if in an incompatible language.
*/
string js_const_type_;
@@ -381,6 +417,7 @@
/**
* File streams
*/
+ ofstream_with_content_based_conditional_update f_episode_;
ofstream_with_content_based_conditional_update f_types_;
ofstream_with_content_based_conditional_update f_service_;
ofstream_with_content_based_conditional_update f_types_ts_;
@@ -397,14 +434,23 @@
// Make output directory
MKDIR(get_out_dir().c_str());
- string outdir = get_out_dir();
+ const auto outdir = get_out_dir();
// Make output file(s)
- string f_types_name = outdir + program_->get_name() + "_types.js";
+ if (gen_episode_file_) {
+ const auto f_episode_file_path = outdir + episode_file_name;
+ f_episode_.open(f_episode_file_path);
+ }
+
+ const auto f_types_name = outdir + program_->get_name() + "_types.js";
f_types_.open(f_types_name.c_str());
+ if (gen_episode_file_) {
+ const auto types_module = program_->get_name() + "_types";
+ f_episode_ << types_module << ":" << thrift_package_output_directory_ << "/" << types_module << endl;
+ }
if (gen_ts_) {
- string f_types_ts_name = outdir + program_->get_name() + "_types.d.ts";
+ const auto f_types_ts_name = outdir + program_->get_name() + "_types.d.ts";
f_types_ts_.open(f_types_ts_name.c_str());
}
@@ -500,8 +546,7 @@
if (gen_node_) {
const vector<t_program*>& includes = program_->get_includes();
for (auto include : includes) {
- result += js_const_type_ + make_valid_nodeJs_identifier(include->get_name()) + "_ttypes = require('./" + include->get_name()
- + "_types');\n";
+ result += js_const_type_ + make_valid_nodeJs_identifier(include->get_name()) + "_ttypes = require('" + get_import_path(include) + "');\n";
}
if (includes.size() > 0) {
result += "\n";
@@ -522,8 +567,7 @@
}
const vector<t_program*>& includes = program_->get_includes();
for (auto include : includes) {
- result += "import " + make_valid_nodeJs_identifier(include->get_name()) + "_ttypes = require('./" + include->get_name()
- + "_types');\n";
+ result += "import " + make_valid_nodeJs_identifier(include->get_name()) + "_ttypes = require('" + get_import_path(include) + "');\n";
}
if (includes.size() > 0) {
result += "\n";
@@ -532,6 +576,21 @@
return result;
}
+string t_js_generator::get_import_path(t_program* program) {
+ const string import_file_name(program->get_name() + "_types");
+ if (program->get_recursive()) {
+ return "./" + import_file_name;
+ }
+
+ const string import_file_name_with_extension = import_file_name + ".js";
+
+ auto module_name_and_import_path_iterator = module_name_2_import_path.find(import_file_name);
+ if (module_name_and_import_path_iterator != module_name_2_import_path.end()) {
+ return module_name_and_import_path_iterator->second;
+ }
+ return "./" + import_file_name;
+}
+
/**
* Close up (or down) some filez.
*/
@@ -546,6 +605,9 @@
}
f_types_ts_.close();
}
+ if (gen_episode_file_){
+ f_episode_.close();
+ }
}
/**
@@ -657,7 +719,7 @@
}
break;
default:
- throw "compiler error: no const of base type " + t_base_type::t_base_name(tbase);
+ throw std::runtime_error("compiler error: no const of base type " + t_base_type::t_base_name(tbase));
}
} else if (type->is_enum()) {
out << value->get_integer();
@@ -676,7 +738,7 @@
}
}
if (field_type == NULL) {
- throw "type error: " + type->get_name() + " has no field " + v_iter->first->get_string();
+ throw std::runtime_error("type error: " + type->get_name() + " has no field " + v_iter->first->get_string());
}
if (v_iter != val.begin())
out << ",";
@@ -698,7 +760,7 @@
for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) {
if (v_iter != val.begin())
out << "," << endl;
-
+
if (ktype->is_base_type() && ((t_base_type*)get_true_type(ktype))->get_base() == t_base_type::TYPE_I64){
out << indent() << "\"" << v_iter->first->get_integer() << "\"";
} else {
@@ -846,7 +908,6 @@
f_types_ts_ << ts_indent() << (*m_iter)->get_name() << ": "
<< ts_get_type((*m_iter)->get_type()) << ";" << endl;
}
-
}
}
@@ -1124,6 +1185,9 @@
void t_js_generator::generate_service(t_service* tservice) {
string f_service_name = get_out_dir() + service_name_ + ".js";
f_service_.open(f_service_name.c_str());
+ if (gen_episode_file_) {
+ f_episode_ << service_name_ << ":" << thrift_package_output_directory_ << "/" << service_name_ << endl;
+ }
if (gen_ts_) {
string f_service_ts_name = get_out_dir() + service_name_ + ".d.ts";
@@ -2136,7 +2200,7 @@
t_type* type = get_true_type(tfield->get_type());
if (type->is_void()) {
- throw "CANNOT GENERATE DESERIALIZE CODE FOR void TYPE: " + prefix + tfield->get_name();
+ throw std::runtime_error("CANNOT GENERATE DESERIALIZE CODE FOR void TYPE: " + prefix + tfield->get_name());
}
string name = prefix + tfield->get_name();
@@ -2152,7 +2216,7 @@
t_base_type::t_base tbase = ((t_base_type*)type)->get_base();
switch (tbase) {
case t_base_type::TYPE_VOID:
- throw "compiler error: cannot serialize void field in a struct: " + name;
+ throw std::runtime_error("compiler error: cannot serialize void field in a struct: " + name);
break;
case t_base_type::TYPE_STRING:
out << (type->is_binary() ? "readBinary()" : "readString()");
@@ -2176,7 +2240,7 @@
out << "readDouble()";
break;
default:
- throw "compiler error: no JS name for base type " + t_base_type::t_base_name(tbase);
+ throw std::runtime_error("compiler error: no JS name for base type " + t_base_type::t_base_name(tbase));
}
} else if (type->is_enum()) {
out << "readI32()";
@@ -2317,7 +2381,7 @@
// Do nothing for void types
if (type->is_void()) {
- throw "CANNOT GENERATE SERIALIZE CODE FOR void TYPE: " + prefix + tfield->get_name();
+ throw std::runtime_error("CANNOT GENERATE SERIALIZE CODE FOR void TYPE: " + prefix + tfield->get_name());
}
if (type->is_struct() || type->is_xception()) {
@@ -2338,7 +2402,7 @@
t_base_type::t_base tbase = ((t_base_type*)type)->get_base();
switch (tbase) {
case t_base_type::TYPE_VOID:
- throw "compiler error: cannot serialize void field in a struct: " + name;
+ throw std::runtime_error("compiler error: cannot serialize void field in a struct: " + name);
break;
case t_base_type::TYPE_STRING:
out << (type->is_binary() ? "writeBinary(" : "writeString(") << name << ")";
@@ -2362,7 +2426,7 @@
out << "writeDouble(" << name << ")";
break;
default:
- throw "compiler error: no JS name for base type " + t_base_type::t_base_name(tbase);
+ throw std::runtime_error("compiler error: no JS name for base type " + t_base_type::t_base_name(tbase));
}
} else if (type->is_enum()) {
out << "writeI32(" << name << ")";
@@ -2510,7 +2574,7 @@
result += " = null";
break;
default:
- throw "compiler error: no JS initializer for base type " + t_base_type::t_base_name(tbase);
+ throw std::runtime_error("compiler error: no JS initializer for base type " + t_base_type::t_base_name(tbase));
}
} else if (type->is_enum()) {
result += " = null";
@@ -2589,7 +2653,7 @@
t_base_type::t_base tbase = ((t_base_type*)type)->get_base();
switch (tbase) {
case t_base_type::TYPE_VOID:
- throw "NO T_VOID CONSTRUCT";
+ throw std::runtime_error("NO T_VOID CONSTRUCT");
case t_base_type::TYPE_STRING:
return "Thrift.Type.STRING";
case t_base_type::TYPE_BOOL:
@@ -2617,7 +2681,7 @@
return "Thrift.Type.LIST";
}
- throw "INVALID TYPE IN type_to_enum: " + type->get_name();
+ throw std::runtime_error("INVALID TYPE IN type_to_enum: " + type->get_name());
}
/**
@@ -2785,10 +2849,80 @@
return str;
}
+void t_js_generator::parse_imports(t_program* program, const std::string& imports_string) {
+ if (program->get_recursive()) {
+ throw std::invalid_argument("[-gen js:imports=] option is not usable in recursive code generation mode");
+ }
+ std::stringstream sstream(imports_string);
+ std::string import;
+ while (std::getline(sstream, import, ':')) {
+ imports.emplace_back(import);
+ }
+ if (imports.empty()) {
+ throw std::invalid_argument("invalid usage: [-gen js:imports=] requires at least one path "
+ "(multiple paths are separated by ':')");
+ }
+ for (auto& import : imports) {
+ // Strip trailing '/'
+ if (!import.empty() && import[import.size() - 1] == '/') {
+ import = import.substr(0, import.size() - 1);
+ }
+ if (import.empty()) {
+ throw std::invalid_argument("empty paths are not allowed in imports");
+ }
+ std::ifstream episode_file;
+ string line;
+ const auto episode_file_path = import + "/" + episode_file_name;
+ episode_file.open(episode_file_path);
+ if (!episode_file) {
+ throw std::runtime_error("failed to open the file '" + episode_file_path + "'");
+ }
+ while (std::getline(episode_file, line)) {
+ const auto separator_position = line.find(':');
+ if (separator_position == string::npos) {
+ // could not find the separator in the line
+ throw std::runtime_error("the episode file '" + episode_file_path + "' is malformed, the line '" + line
+ + "' does not have a key:value separator ':'");
+ }
+ const auto module_name = line.substr(0, separator_position);
+ const auto import_path = line.substr(separator_position + 1);
+ if (module_name.empty()) {
+ throw std::runtime_error("the episode file '" + episode_file_path + "' is malformed, the module name is empty");
+ }
+ if (import_path.empty()) {
+ throw std::runtime_error("the episode file '" + episode_file_path + "' is malformed, the import path is empty");
+ }
+ const auto module_import_path = import.substr(import.find_last_of('/') + 1) + "/" + import_path;
+ const auto result = module_name_2_import_path.emplace(module_name, module_import_path);
+ if (!result.second) {
+ throw std::runtime_error("multiple providers of import path found for " + module_name
+ + "\n\t" + module_import_path + "\n\t" + result.first->second);
+ }
+ }
+ }
+}
+void t_js_generator::parse_thrift_package_output_directory(const std::string& thrift_package_output_directory) {
+ thrift_package_output_directory_ = thrift_package_output_directory;
+ // Strip trailing '/'
+ if (!thrift_package_output_directory_.empty() && thrift_package_output_directory_[thrift_package_output_directory_.size() - 1] == '/') {
+ thrift_package_output_directory_ = thrift_package_output_directory_.substr(0, thrift_package_output_directory_.size() - 1);
+ }
+ // Check that the thrift_package_output_directory is not empty after stripping
+ if (thrift_package_output_directory_.empty()) {
+ throw std::invalid_argument("the thrift_package_output_directory argument must not be empty");
+ } else {
+ gen_episode_file_ = true;
+ }
+}
+
THRIFT_REGISTER_GENERATOR(js,
"Javascript",
" jquery: Generate jQuery compatible code.\n"
" node: Generate node.js compatible code.\n"
" ts: Generate TypeScript definition files.\n"
" with_ns: Create global namespace objects when using node.js\n"
- " es6: Create ES6 code with Promises\n")
+ " es6: Create ES6 code with Promises\n"
+ " thrift_package_output_directory=<path>:\n"
+ " Generate episode file and use the <path> as prefix\n"
+ " imports=<paths_to_modules>:\n"
+ " ':' separated list of paths of modules that has episode files in their root\n")
diff --git a/compiler/cpp/src/thrift/main.cc b/compiler/cpp/src/thrift/main.cc
index 5b69dc5..03e0d6f 100644
--- a/compiler/cpp/src/thrift/main.cc
+++ b/compiler/cpp/src/thrift/main.cc
@@ -982,6 +982,7 @@
void generate(t_program* program, const vector<string>& generator_strings) {
// Oooohh, recursive code generation, hot!!
if (gen_recurse) {
+ program->set_recursive(true);
const vector<t_program*>& includes = program->get_includes();
for (auto include : includes) {
// Propagate output path from parent to child programs
@@ -1017,6 +1018,8 @@
failure("Error: %s\n", s.c_str());
} catch (const char* exc) {
failure("Error: %s\n", exc);
+ } catch (const std::invalid_argument& invalid_argument_exception) {
+ failure("Error: %s\n", invalid_argument_exception.what());
}
}
diff --git a/compiler/cpp/src/thrift/parse/t_program.h b/compiler/cpp/src/thrift/parse/t_program.h
index 13cb26e..140dc35 100644
--- a/compiler/cpp/src/thrift/parse/t_program.h
+++ b/compiler/cpp/src/thrift/parse/t_program.h
@@ -58,9 +58,9 @@
class t_program : public t_doc {
public:
t_program(std::string path, std::string name)
- : path_(path), name_(name), out_path_("./"), out_path_is_absolute_(false), scope_(new t_scope) {}
+ : path_(path), name_(name), out_path_("./"), out_path_is_absolute_(false), scope_(new t_scope), recursive_(false) {}
- t_program(std::string path) : path_(path), out_path_("./"), out_path_is_absolute_(false) {
+ t_program(std::string path) : path_(path), out_path_("./"), out_path_is_absolute_(false), recursive_(false) {
name_ = program_name(path);
scope_ = new t_scope();
}
@@ -355,6 +355,10 @@
const std::vector<std::string>& get_c_includes() const { return c_includes_; }
+ void set_recursive(const bool recursive) { recursive_ = recursive; }
+
+ bool get_recursive() const { return recursive_; }
+
private:
// File path
std::string path_;
@@ -400,6 +404,9 @@
// C extra includes
std::vector<std::string> c_includes_;
+
+ // Recursive code generation
+ bool recursive_;
};
#endif