GDScript: Reintroduce binary tokenization on export

This adds back a function available in 3.x: exporting the GDScript
files in a binary form by converting the tokens recognized by the
tokenizer into a data format.

It is enabled by default on export but can be manually disabled. The
format helps with loading times since, the tokens are easily
reconstructed, and with hiding the source code, since recovering it
would require a specialized tool. Code comments are not stored in this
format.

The `--test` command can also include a `--use-binary-tokens` flag
which will run the GDScript tests with the binary format instead of the
regular source code by converting them in-memory before the test runs.

editor/export/editor_export.cpp

@@ -85,6 +85,7 @@ void EditorExport::_save() {
 		config->set_value(section, "encryption_exclude_filters", preset->get_enc_ex_filter());
 		config->set_value(section, "encrypt_pck", preset->get_enc_pck());
 		config->set_value(section, "encrypt_directory", preset->get_enc_directory());
+		config->set_value(section, "script_export_mode", preset->get_script_export_mode());
 		credentials->set_value(section, "script_encryption_key", preset->get_script_encryption_key());
 
 		String option_section = "preset." + itos(i) + ".options";
@@ -269,6 +270,7 @@ void EditorExport::load_config() {
 		preset->set_include_filter(config->get_value(section, "include_filter"));
 		preset->set_exclude_filter(config->get_value(section, "exclude_filter"));
 		preset->set_export_path(config->get_value(section, "export_path", ""));
+		preset->set_script_export_mode(config->get_value(section, "script_export_mode", EditorExportPreset::MODE_SCRIPT_BINARY_TOKENS));
 
 		if (config->has_section_key(section, "encrypt_pck")) {
 			preset->set_enc_pck(config->get_value(section, "encrypt_pck"));

editor/export/editor_export_preset.cpp

@@ -323,6 +323,15 @@ String EditorExportPreset::get_script_encryption_key() const {
 	return script_key;
 }
 
+void EditorExportPreset::set_script_export_mode(int p_mode) {
+	script_mode = p_mode;
+	EditorExport::singleton->save_presets();
+}
+
+int EditorExportPreset::get_script_export_mode() const {
+	return script_mode;
+}
+
 Variant EditorExportPreset::get_or_env(const StringName &p_name, const String &p_env_var, bool *r_valid) const {
 	const String from_env = OS::get_singleton()->get_environment(p_env_var);
 	if (!from_env.is_empty()) {

editor/export/editor_export_preset.h

@@ -54,6 +54,11 @@ class EditorExportPreset : public RefCounted {
 		MODE_FILE_REMOVE,
 	};
 
+	enum ScriptExportMode {
+		MODE_SCRIPT_TEXT,
+		MODE_SCRIPT_BINARY_TOKENS,
+	};
+
 private:
 	Ref<EditorExportPlatform> platform;
 	ExportFilter export_filter = EXPORT_ALL_RESOURCES;
@@ -84,6 +89,7 @@ class EditorExportPreset : public RefCounted {
 	bool enc_directory = false;
 
 	String script_key;
+	int script_mode = MODE_SCRIPT_BINARY_TOKENS;
 
 protected:
 	bool _set(const StringName &p_name, const Variant &p_value);
@@ -152,6 +158,9 @@ class EditorExportPreset : public RefCounted {
 	void set_script_encryption_key(const String &p_key);
 	String get_script_encryption_key() const;
 
+	void set_script_export_mode(int p_mode);
+	int get_script_export_mode() const;
+
 	Variant get_or_env(const StringName &p_name, const String &p_env_var, bool *r_valid = nullptr) const;
 
 	// Return the preset's version number, or fall back to the

editor/export/project_export.cpp

@@ -383,6 +383,9 @@ void ProjectExportDialog::_edit_preset(int p_index) {
 		script_key_error->hide();
 	}
 
+	int script_export_mode = current->get_script_export_mode();
+	script_mode->select(script_export_mode);
+
 	updating = false;
 }
 
@@ -582,6 +585,19 @@ bool ProjectExportDialog::_validate_script_encryption_key(const String &p_key) {
 	return is_valid;
 }
 
+void ProjectExportDialog::_script_export_mode_changed(int p_mode) {
+	if (updating) {
+		return;
+	}
+
+	Ref<EditorExportPreset> current = get_current_preset();
+	ERR_FAIL_COND(current.is_null());
+
+	current->set_script_export_mode(p_mode);
+
+	_update_current_preset();
+}
+
 void ProjectExportDialog::_duplicate_preset() {
 	Ref<EditorExportPreset> current = get_current_preset();
 	if (current.is_null()) {
@@ -1328,7 +1344,7 @@ ProjectExportDialog::ProjectExportDialog() {
 	feature_vb->add_margin_child(TTR("Feature List:"), custom_feature_display, true);
 	sections->add_child(feature_vb);
 
-	// Script export parameters.
+	// Encryption export parameters.
 
 	VBoxContainer *sec_vb = memnew(VBoxContainer);
 	sec_vb->set_name(TTR("Encryption"));
@@ -1373,6 +1389,18 @@ ProjectExportDialog::ProjectExportDialog() {
 	sec_more_info->connect("pressed", callable_mp(this, &ProjectExportDialog::_open_key_help_link));
 	sec_vb->add_child(sec_more_info);
 
+	// Script export parameters.
+
+	VBoxContainer *script_vb = memnew(VBoxContainer);
+	script_vb->set_name(TTR("Scripts"));
+
+	script_mode = memnew(OptionButton);
+	script_vb->add_margin_child(TTR("GDScript Export Mode:"), script_mode);
+	script_mode->add_item(TTR("Text (easier debugging)"), (int)EditorExportPreset::MODE_SCRIPT_TEXT);
+	script_mode->add_item(TTR("Binary tokens (faster loading)"), (int)EditorExportPreset::MODE_SCRIPT_BINARY_TOKENS);
+	script_mode->connect("item_selected", callable_mp(this, &ProjectExportDialog::_script_export_mode_changed));
+	sections->add_child(script_vb);
+
 	sections->connect("tab_changed", callable_mp(this, &ProjectExportDialog::_tab_changed));
 
 	// Disable by default.

editor/export/project_export.h

@@ -160,6 +160,8 @@ class ProjectExportDialog : public ConfirmationDialog {
 	LineEdit *enc_in_filters = nullptr;
 	LineEdit *enc_ex_filters = nullptr;
 
+	OptionButton *script_mode = nullptr;
+
 	void _open_export_template_manager();
 
 	void _export_pck_zip();
@@ -183,6 +185,8 @@ class ProjectExportDialog : public ConfirmationDialog {
 	void _script_encryption_key_changed(const String &p_key);
 	bool _validate_script_encryption_key(const String &p_key);
 
+	void _script_export_mode_changed(int p_mode);
+
 	void _open_key_help_link();
 
 	void _tab_changed(int);

modules/gdscript/gdscript.cpp

@@ -35,6 +35,7 @@
 #include "gdscript_compiler.h"
 #include "gdscript_parser.h"
 #include "gdscript_rpc_callable.h"
+#include "gdscript_tokenizer_buffer.h"
 #include "gdscript_warning.h"
 
 #ifdef TOOLS_ENABLED
@@ -740,7 +741,12 @@ Error GDScript::reload(bool p_keep_state) {
 
 	valid = false;
 	GDScriptParser parser;
-	Error err = parser.parse(source, path, false);
+	Error err;
+	if (!binary_tokens.is_empty()) {
+		err = parser.parse_binary(binary_tokens, path);
+	} else {
+		err = parser.parse(source, path, false);
+	}
 	if (err) {
 		if (EngineDebugger::is_active()) {
 			GDScriptLanguage::get_singleton()->debug_break_parse(_get_debug_path(), parser.get_errors().front()->get().line, "Parser Error: " + parser.get_errors().front()->get().message);
@@ -1050,6 +1056,19 @@ Error GDScript::load_source_code(const String &p_path) {
 	return OK;
 }
 
+void GDScript::set_binary_tokens_source(const Vector<uint8_t> &p_binary_tokens) {
+	binary_tokens = p_binary_tokens;
+}
+
+const Vector<uint8_t> &GDScript::get_binary_tokens_source() const {
+	return binary_tokens;
+}
+
+Vector<uint8_t> GDScript::get_as_binary_tokens() const {
+	GDScriptTokenizerBuffer tokenizer;
+	return tokenizer.parse_code_string(source);
+}
+
 const HashMap<StringName, GDScriptFunction *> &GDScript::debug_get_member_functions() const {
 	return member_functions;
 }
@@ -2805,6 +2824,7 @@ Ref<Resource> ResourceFormatLoaderGDScript::load(const String &p_path, const Str
 
 void ResourceFormatLoaderGDScript::get_recognized_extensions(List<String> *p_extensions) const {
 	p_extensions->push_back("gd");
+	p_extensions->push_back("gdc");
 }
 
 bool ResourceFormatLoaderGDScript::handles_type(const String &p_type) const {
@@ -2813,7 +2833,7 @@ bool ResourceFormatLoaderGDScript::handles_type(const String &p_type) const {
 
 String ResourceFormatLoaderGDScript::get_resource_type(const String &p_path) const {
 	String el = p_path.get_extension().to_lower();
-	if (el == "gd") {
+	if (el == "gd" || el == "gdc") {
 		return "GDScript";
 	}
 	return "";

modules/gdscript/gdscript.h

@@ -176,6 +176,7 @@ class GDScript : public Script {
 	bool clearing = false;
 	//exported members
 	String source;
+	Vector<uint8_t> binary_tokens;
 	String path;
 	bool path_valid = false; // False if using default path.
 	StringName local_name; // Inner class identifier or `class_name`.
@@ -296,6 +297,10 @@ class GDScript : public Script {
 	String get_script_path() const;
 	Error load_source_code(const String &p_path);
 
+	void set_binary_tokens_source(const Vector<uint8_t> &p_binary_tokens);
+	const Vector<uint8_t> &get_binary_tokens_source() const;
+	Vector<uint8_t> get_as_binary_tokens() const;
+
 	bool get_property_default_value(const StringName &p_property, Variant &r_value) const override;
 
 	virtual void get_script_method_list(List<MethodInfo> *p_list) const override;

modules/gdscript/gdscript_cache.cpp

@@ -67,10 +67,15 @@ Error GDScriptParserRef::raise_status(Status p_new_status) {
 
 	while (p_new_status > status) {
 		switch (status) {
-			case EMPTY:
+			case EMPTY: {
 				status = PARSED;
-				result = parser->parse(GDScriptCache::get_source_code(path), path, false);
-				break;
+				String remapped_path = ResourceLoader::path_remap(path);
+				if (remapped_path.get_extension().to_lower() == "gdc") {
+					result = parser->parse_binary(GDScriptCache::get_binary_tokens(remapped_path), path);
+				} else {
+					result = parser->parse(GDScriptCache::get_source_code(remapped_path), path, false);
+				}
+			} break;
 			case PARSED: {
 				status = INHERITANCE_SOLVED;
 				Error inheritance_result = get_analyzer()->resolve_inheritance();
@@ -205,7 +210,8 @@ Ref<GDScriptParserRef> GDScriptCache::get_parser(const String &p_path, GDScriptP
 			return ref;
 		}
 	} else {
-		if (!FileAccess::exists(p_path)) {
+		String remapped_path = ResourceLoader::path_remap(p_path);
+		if (!FileAccess::exists(remapped_path)) {
 			r_error = ERR_FILE_NOT_FOUND;
 			return ref;
 		}
@@ -239,6 +245,20 @@ String GDScriptCache::get_source_code(const String &p_path) {
 	return source;
 }
 
+Vector<uint8_t> GDScriptCache::get_binary_tokens(const String &p_path) {
+	Vector<uint8_t> buffer;
+	Error err = OK;
+	Ref<FileAccess> f = FileAccess::open(p_path, FileAccess::READ, &err);
+	ERR_FAIL_COND_V_MSG(err != OK, buffer, "Failed to open binary GDScript file '" + p_path + "'.");
+
+	uint64_t len = f->get_length();
+	buffer.resize(len);
+	uint64_t read = f->get_buffer(buffer.ptrw(), buffer.size());
+	ERR_FAIL_COND_V_MSG(read != len, Vector<uint8_t>(), "Failed to read binary GDScript file '" + p_path + "'.");
+
+	return buffer;
+}
+
 Ref<GDScript> GDScriptCache::get_shallow_script(const String &p_path, Error &r_error, const String &p_owner) {
 	MutexLock lock(singleton->mutex);
 	if (!p_owner.is_empty()) {
@@ -251,10 +271,20 @@ Ref<GDScript> GDScriptCache::get_shallow_script(const String &p_path, Error &r_e
 		return singleton->shallow_gdscript_cache[p_path];
 	}
 
+	String remapped_path = ResourceLoader::path_remap(p_path);
+
 	Ref<GDScript> script;
 	script.instantiate();
 	script->set_path(p_path, true);
-	r_error = script->load_source_code(p_path);
+	if (remapped_path.get_extension().to_lower() == "gdc") {
+		Vector<uint8_t> buffer = get_binary_tokens(remapped_path);
+		if (buffer.is_empty()) {
+			r_error = ERR_FILE_CANT_READ;
+		}
+		script->set_binary_tokens_source(buffer);
+	} else {
+		r_error = script->load_source_code(remapped_path);
+	}
 
 	if (r_error) {
 		return Ref<GDScript>(); // Returns null and does not cache when the script fails to load.
@@ -294,9 +324,18 @@ Ref<GDScript> GDScriptCache::get_full_script(const String &p_path, Error &r_erro
 	}
 
 	if (p_update_from_disk) {
-		r_error = script->load_source_code(p_path);
-		if (r_error) {
-			return script;
+		if (p_path.get_extension().to_lower() == "gdc") {
+			Vector<uint8_t> buffer = get_binary_tokens(p_path);
+			if (buffer.is_empty()) {
+				r_error = ERR_FILE_CANT_READ;
+				return script;
+			}
+			script->set_binary_tokens_source(buffer);
+		} else {
+			r_error = script->load_source_code(p_path);
+			if (r_error) {
+				return script;
+			}
 		}
 	}
 

modules/gdscript/gdscript_cache.h

@@ -99,6 +99,7 @@ class GDScriptCache {
 	static void remove_script(const String &p_path);
 	static Ref<GDScriptParserRef> get_parser(const String &p_path, GDScriptParserRef::Status status, Error &r_error, const String &p_owner = String());
 	static String get_source_code(const String &p_path);
+	static Vector<uint8_t> get_binary_tokens(const String &p_path);
 	static Ref<GDScript> get_shallow_script(const String &p_path, Error &r_error, const String &p_owner = String());
 	static Ref<GDScript> get_full_script(const String &p_path, Error &r_error, const String &p_owner = String(), bool p_update_from_disk = false);
 	static Ref<GDScript> get_cached_script(const String &p_path);

modules/gdscript/gdscript_editor.cpp

@@ -210,7 +210,7 @@ bool GDScriptLanguage::supports_documentation() const {
 }
 
 int GDScriptLanguage::find_function(const String &p_function, const String &p_code) const {
-	GDScriptTokenizer tokenizer;
+	GDScriptTokenizerText tokenizer;
 	tokenizer.set_source_code(p_code);
 	int indent = 0;
 	GDScriptTokenizer::Token current = tokenizer.scan();