diff --git a/core/compressed_translation.cpp b/core/compressed_translation.cpp index a66997aa525..a92275565de 100644 --- a/core/compressed_translation.cpp +++ b/core/compressed_translation.cpp @@ -43,6 +43,8 @@ struct _PHashTranslationCmp { }; void PHashTranslation::generate(const Ref &p_from) { + // This method compresses a Translation instance. + // Right now it doesn't handle context or plurals, so Translation subclasses using plurals or context (i.e TranslationPO) shouldn't be compressed. #ifdef TOOLS_ENABLED List keys; p_from->get_message_list(&keys); @@ -212,7 +214,9 @@ bool PHashTranslation::_get(const StringName &p_name, Variant &r_ret) const { return true; } -StringName PHashTranslation::get_message(const StringName &p_src_text) const { +StringName PHashTranslation::get_message(const StringName &p_src_text, const StringName &p_context) const { + // p_context passed in is ignore. The use of context is not yet supported in PHashTranslation. + int htsize = hash_table.size(); if (htsize == 0) { @@ -267,6 +271,11 @@ StringName PHashTranslation::get_message(const StringName &p_src_text) const { } } +StringName PHashTranslation::get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context) const { + // The use of plurals translation is not yet supported in PHashTranslation. + return get_message(p_src_text, p_context); +} + void PHashTranslation::_get_property_list(List *p_list) const { p_list->push_back(PropertyInfo(Variant::PACKED_INT32_ARRAY, "hash_table")); p_list->push_back(PropertyInfo(Variant::PACKED_INT32_ARRAY, "bucket_table")); diff --git a/core/compressed_translation.h b/core/compressed_translation.h index 4f9c422e1ef..c8b3cd23308 100644 --- a/core/compressed_translation.h +++ b/core/compressed_translation.h @@ -79,7 +79,8 @@ protected: static void _bind_methods(); public: - virtual StringName get_message(const StringName &p_src_text) const override; //overridable for other implementations + virtual StringName get_message(const StringName &p_src_text, const StringName &p_context = "") const override; //overridable for other implementations + virtual StringName get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context = "") const override; void generate(const Ref &p_from); PHashTranslation() {} diff --git a/core/io/translation_loader_po.cpp b/core/io/translation_loader_po.cpp index 11aeddee09c..d8ddb213c31 100644 --- a/core/io/translation_loader_po.cpp +++ b/core/io/translation_loader_po.cpp @@ -32,26 +32,34 @@ #include "core/os/file_access.h" #include "core/translation.h" +#include "core/translation_po.h" RES TranslationLoaderPO::load_translation(FileAccess *f, Error *r_error) { enum Status { STATUS_NONE, STATUS_READING_ID, STATUS_READING_STRING, + STATUS_READING_CONTEXT, + STATUS_READING_PLURAL, }; Status status = STATUS_NONE; String msg_id; String msg_str; + String msg_context; + Vector msgs_plural; String config; if (r_error) { *r_error = ERR_FILE_CORRUPT; } - Ref translation = Ref(memnew(Translation)); + Ref translation = Ref(memnew(TranslationPO)); int line = 1; + int plural_forms = 0; + int plural_index = -1; + bool entered_context = false; bool skip_this = false; bool skip_next = false; bool is_eof = false; @@ -63,40 +71,107 @@ RES TranslationLoaderPO::load_translation(FileAccess *f, Error *r_error) { // If we reached last line and it's not a content line, break, otherwise let processing that last loop if (is_eof && l.empty()) { - if (status == STATUS_READING_ID) { + if (status == STATUS_READING_ID || status == STATUS_READING_CONTEXT || (status == STATUS_READING_PLURAL && plural_index != plural_forms - 1)) { memdelete(f); - ERR_FAIL_V_MSG(RES(), "Unexpected EOF while reading 'msgid' at: " + path + ":" + itos(line)); + ERR_FAIL_V_MSG(RES(), "Unexpected EOF while reading PO file at: " + path + ":" + itos(line)); } else { break; } } - if (l.begins_with("msgid")) { + if (l.begins_with("msgctxt")) { + if (status != STATUS_READING_STRING && status != STATUS_READING_PLURAL) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Unexpected 'msgctxt', was expecting 'msgid_plural' or 'msgstr' before 'msgctxt' while parsing: " + path + ":" + itos(line)); + } + + // In PO file, "msgctxt" appears before "msgid". If we encounter a "msgctxt", we add what we have read + // and set "entered_context" to true to prevent adding twice. + if (!skip_this && msg_id != "") { + if (status == STATUS_READING_STRING) { + translation->add_message(msg_id, msg_str, msg_context); + } else if (status == STATUS_READING_PLURAL) { + if (plural_index != plural_forms - 1) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Number of 'msgstr[]' doesn't match with number of plural forms: " + path + ":" + itos(line)); + } + translation->add_plural_message(msg_id, msgs_plural, msg_context); + } + } + msg_context = ""; + l = l.substr(7, l.length()).strip_edges(); + status = STATUS_READING_CONTEXT; + entered_context = true; + } + + if (l.begins_with("msgid_plural")) { + if (plural_forms == 0) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "PO file uses 'msgid_plural' but 'Plural-Forms' is invalid or missing in header: " + path + ":" + itos(line)); + } else if (status != STATUS_READING_ID) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Unexpected 'msgid_plural', was expecting 'msgid' before 'msgid_plural' while parsing: " + path + ":" + itos(line)); + } + // We don't record the message in "msgid_plural" itself as tr_n(), TTRN(), RTRN() interfaces provide the plural string already. + // We just have to reset variables related to plurals for "msgstr[]" later on. + l = l.substr(12, l.length()).strip_edges(); + plural_index = -1; + msgs_plural.clear(); + msgs_plural.resize(plural_forms); + status = STATUS_READING_PLURAL; + } else if (l.begins_with("msgid")) { if (status == STATUS_READING_ID) { memdelete(f); ERR_FAIL_V_MSG(RES(), "Unexpected 'msgid', was expecting 'msgstr' while parsing: " + path + ":" + itos(line)); } if (msg_id != "") { - if (!skip_this) { - translation->add_message(msg_id, msg_str); + if (!skip_this && !entered_context) { + if (status == STATUS_READING_STRING) { + translation->add_message(msg_id, msg_str, msg_context); + } else if (status == STATUS_READING_PLURAL) { + if (plural_index != plural_forms - 1) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Number of 'msgstr[]' doesn't match with number of plural forms: " + path + ":" + itos(line)); + } + translation->add_plural_message(msg_id, msgs_plural, msg_context); + } } } else if (config == "") { config = msg_str; + // Record plural rule. + int p_start = config.find("Plural-Forms"); + if (p_start != -1) { + int p_end = config.find("\n", p_start); + translation->set_plural_rule(config.substr(p_start, p_end - p_start)); + plural_forms = translation->get_plural_forms(); + } } l = l.substr(5, l.length()).strip_edges(); status = STATUS_READING_ID; + // If we did not encounter msgctxt, we reset context to empty to reset it. + if (!entered_context) { + msg_context = ""; + } msg_id = ""; msg_str = ""; skip_this = skip_next; skip_next = false; + entered_context = false; } - if (l.begins_with("msgstr")) { + if (l.begins_with("msgstr[")) { + if (status != STATUS_READING_PLURAL) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Unexpected 'msgstr[]', was expecting 'msgid_plural' before 'msgstr[]' while parsing: " + path + ":" + itos(line)); + } + plural_index++; // Increment to add to the next slot in vector msgs_plural. + l = l.substr(9, l.length()).strip_edges(); + } else if (l.begins_with("msgstr")) { if (status != STATUS_READING_ID) { memdelete(f); - ERR_FAIL_V_MSG(RES(), "Unexpected 'msgstr', was expecting 'msgid' while parsing: " + path + ":" + itos(line)); + ERR_FAIL_V_MSG(RES(), "Unexpected 'msgstr', was expecting 'msgid' before 'msgstr' while parsing: " + path + ":" + itos(line)); } l = l.substr(6, l.length()).strip_edges(); @@ -108,7 +183,7 @@ RES TranslationLoaderPO::load_translation(FileAccess *f, Error *r_error) { skip_next = true; } line++; - continue; //nothing to read or comment + continue; // Nothing to read or comment. } if (!l.begins_with("\"") || status == STATUS_NONE) { @@ -146,8 +221,12 @@ RES TranslationLoaderPO::load_translation(FileAccess *f, Error *r_error) { if (status == STATUS_READING_ID) { msg_id += l; - } else { + } else if (status == STATUS_READING_STRING) { msg_str += l; + } else if (status == STATUS_READING_CONTEXT) { + msg_context += l; + } else if (status == STATUS_READING_PLURAL && plural_index >= 0) { + msgs_plural.write[plural_index] = msgs_plural[plural_index] + l; } line++; @@ -155,14 +234,23 @@ RES TranslationLoaderPO::load_translation(FileAccess *f, Error *r_error) { memdelete(f); + // Add the last set of data from last iteration. if (status == STATUS_READING_STRING) { if (msg_id != "") { if (!skip_this) { - translation->add_message(msg_id, msg_str); + translation->add_message(msg_id, msg_str, msg_context); } } else if (config == "") { config = msg_str; } + } else if (status == STATUS_READING_PLURAL) { + if (!skip_this && msg_id != "") { + if (plural_index != plural_forms - 1) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Number of 'msgstr[]' doesn't match with number of plural forms: " + path + ":" + itos(line)); + } + translation->add_plural_message(msg_id, msgs_plural, msg_context); + } } ERR_FAIL_COND_V_MSG(config == "", RES(), "No config found in file: " + path + "."); diff --git a/core/object.cpp b/core/object.cpp index ff6d4a666f6..67c605c39ba 100644 --- a/core/object.cpp +++ b/core/object.cpp @@ -1432,12 +1432,22 @@ void Object::initialize_class() { initialized = true; } -StringName Object::tr(const StringName &p_message) const { +String Object::tr(const StringName &p_message, const StringName &p_context) const { if (!_can_translate || !TranslationServer::get_singleton()) { return p_message; } + return TranslationServer::get_singleton()->translate(p_message, p_context); +} - return TranslationServer::get_singleton()->translate(p_message); +String Object::tr_n(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context) const { + if (!_can_translate || !TranslationServer::get_singleton()) { + // Return message based on English plural rule if translation is not possible. + if (p_n == 1) { + return p_message; + } + return p_message_plural; + } + return TranslationServer::get_singleton()->translate_plural(p_message, p_message_plural, p_n, p_context); } void Object::_clear_internal_resource_paths(const Variant &p_var) { @@ -1578,7 +1588,8 @@ void Object::_bind_methods() { ClassDB::bind_method(D_METHOD("set_message_translation", "enable"), &Object::set_message_translation); ClassDB::bind_method(D_METHOD("can_translate_messages"), &Object::can_translate_messages); - ClassDB::bind_method(D_METHOD("tr", "message"), &Object::tr); + ClassDB::bind_method(D_METHOD("tr", "message", "context"), &Object::tr, DEFVAL("")); + ClassDB::bind_method(D_METHOD("tr_n", "message", "plural_message", "n", "context"), &Object::tr_n, DEFVAL("")); ClassDB::bind_method(D_METHOD("is_queued_for_deletion"), &Object::is_queued_for_deletion); diff --git a/core/object.h b/core/object.h index d9847d10aa9..f9a12da8f62 100644 --- a/core/object.h +++ b/core/object.h @@ -719,7 +719,8 @@ public: virtual void get_argument_options(const StringName &p_function, int p_idx, List *r_options) const; - StringName tr(const StringName &p_message) const; // translate message (internationalization) + String tr(const StringName &p_message, const StringName &p_context = "") const; // translate message (internationalization) + String tr_n(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context = "") const; bool _is_queued_for_deletion = false; // set to true by SceneTree::queue_delete() bool is_queued_for_deletion() const; diff --git a/core/translation.cpp b/core/translation.cpp index 4f835bd7b44..8c8ca06740d 100644 --- a/core/translation.cpp +++ b/core/translation.cpp @@ -794,17 +794,12 @@ static const char *locale_renames[][2] = { /////////////////////////////////////////////// -Vector Translation::_get_messages() const { - Vector msgs; - msgs.resize(translation_map.size() * 2); - int idx = 0; +Dictionary Translation::_get_messages() const { + Dictionary d; for (const Map::Element *E = translation_map.front(); E; E = E->next()) { - msgs.set(idx + 0, E->key()); - msgs.set(idx + 1, E->get()); - idx += 2; + d[E->key()] = E->value(); } - - return msgs; + return d; } Vector Translation::_get_message_list() const { @@ -819,14 +814,11 @@ Vector Translation::_get_message_list() const { return msgs; } -void Translation::_set_messages(const Vector &p_messages) { - int msg_count = p_messages.size(); - ERR_FAIL_COND(msg_count % 2); - - const String *r = p_messages.ptr(); - - for (int i = 0; i < msg_count; i += 2) { - add_message(r[i + 0], r[i + 1]); +void Translation::_set_messages(const Dictionary &p_messages) { + List keys; + p_messages.get_key_list(&keys); + for (auto E = keys.front(); E; E = E->next()) { + translation_map[E->get()] = p_messages[E->get()]; } } @@ -848,11 +840,21 @@ void Translation::set_locale(const String &p_locale) { } } -void Translation::add_message(const StringName &p_src_text, const StringName &p_xlated_text) { +void Translation::add_message(const StringName &p_src_text, const StringName &p_xlated_text, const StringName &p_context) { translation_map[p_src_text] = p_xlated_text; } -StringName Translation::get_message(const StringName &p_src_text) const { +void Translation::add_plural_message(const StringName &p_src_text, const Vector &p_plural_xlated_texts, const StringName &p_context) { + WARN_PRINT("Translation class doesn't handle plural messages. Calling add_plural_message() on a Translation instance is probably a mistake. \nUse a derived Translation class that handles plurals, such as TranslationPO class"); + ERR_FAIL_COND_MSG(p_plural_xlated_texts.empty(), "Parameter vector p_plural_xlated_texts passed in is empty."); + translation_map[p_src_text] = p_plural_xlated_texts[0]; +} + +StringName Translation::get_message(const StringName &p_src_text, const StringName &p_context) const { + if (p_context != StringName()) { + WARN_PRINT("Translation class doesn't handle context. Using context in get_message() on a Translation instance is probably a mistake. \nUse a derived Translation class that handles context, such as TranslationPO class"); + } + const Map::Element *E = translation_map.find(p_src_text); if (!E) { return StringName(); @@ -861,7 +863,16 @@ StringName Translation::get_message(const StringName &p_src_text) const { return E->get(); } -void Translation::erase_message(const StringName &p_src_text) { +StringName Translation::get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context) const { + WARN_PRINT("Translation class doesn't handle plural messages. Calling get_plural_message() on a Translation instance is probably a mistake. \nUse a derived Translation class that handles plurals, such as TranslationPO class"); + return get_message(p_src_text); +} + +void Translation::erase_message(const StringName &p_src_text, const StringName &p_context) { + if (p_context != StringName()) { + WARN_PRINT("Translation class doesn't handle context. Using context in erase_message() on a Translation instance is probably a mistake. \nUse a derived Translation class that handles context, such as TranslationPO class"); + } + translation_map.erase(p_src_text); } @@ -878,15 +889,17 @@ int Translation::get_message_count() const { void Translation::_bind_methods() { ClassDB::bind_method(D_METHOD("set_locale", "locale"), &Translation::set_locale); ClassDB::bind_method(D_METHOD("get_locale"), &Translation::get_locale); - ClassDB::bind_method(D_METHOD("add_message", "src_message", "xlated_message"), &Translation::add_message); - ClassDB::bind_method(D_METHOD("get_message", "src_message"), &Translation::get_message); - ClassDB::bind_method(D_METHOD("erase_message", "src_message"), &Translation::erase_message); + ClassDB::bind_method(D_METHOD("add_message", "src_message", "xlated_message", "context"), &Translation::add_message, DEFVAL("")); + ClassDB::bind_method(D_METHOD("add_plural_message", "src_message", "xlated_messages", "context"), &Translation::add_plural_message, DEFVAL("")); + ClassDB::bind_method(D_METHOD("get_message", "src_message", "context"), &Translation::get_message, DEFVAL("")); + ClassDB::bind_method(D_METHOD("get_plural_message", "src_message", "src_plural_message", "n", "context"), &Translation::get_plural_message, DEFVAL("")); + ClassDB::bind_method(D_METHOD("erase_message", "src_message", "context"), &Translation::erase_message, DEFVAL("")); ClassDB::bind_method(D_METHOD("get_message_list"), &Translation::_get_message_list); ClassDB::bind_method(D_METHOD("get_message_count"), &Translation::get_message_count); ClassDB::bind_method(D_METHOD("_set_messages"), &Translation::_set_messages); ClassDB::bind_method(D_METHOD("_get_messages"), &Translation::_get_messages); - ADD_PROPERTY(PropertyInfo(Variant::PACKED_STRING_ARRAY, "messages", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NOEDITOR | PROPERTY_USAGE_INTERNAL), "_set_messages", "_get_messages"); + ADD_PROPERTY(PropertyInfo(Variant::DICTIONARY, "messages", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NOEDITOR | PROPERTY_USAGE_INTERNAL), "_set_messages", "_get_messages"); ADD_PROPERTY(PropertyInfo(Variant::STRING, "locale"), "set_locale", "get_locale"); } @@ -1020,11 +1033,35 @@ void TranslationServer::remove_translation(const Ref &p_translation translations.erase(p_translation); } +Ref TranslationServer::get_translation_object(const String &p_locale) { + Ref res; + String lang = get_language_code(p_locale); + bool near_match_found = false; + + for (const Set>::Element *E = translations.front(); E; E = E->next()) { + const Ref &t = E->get(); + ERR_FAIL_COND_V(t.is_null(), nullptr); + String l = t->get_locale(); + + // Exact match. + if (l == p_locale) { + return t; + } + + // If near match found, keep that match, but keep looking to try to look for perfect match. + if (get_language_code(l) == lang && !near_match_found) { + res = t; + near_match_found = true; + } + } + return res; +} + void TranslationServer::clear() { translations.clear(); } -StringName TranslationServer::translate(const StringName &p_message) const { +StringName TranslationServer::translate(const StringName &p_message, const StringName &p_context) const { // Match given message against the translation catalog for the project locale. if (!enabled) { @@ -1033,6 +1070,46 @@ StringName TranslationServer::translate(const StringName &p_message) const { ERR_FAIL_COND_V_MSG(locale.length() < 2, p_message, "Could not translate message as configured locale '" + locale + "' is invalid."); + StringName res = _get_message_from_translations(p_message, p_context, locale, false); + + if (!res && fallback.length() >= 2) { + res = _get_message_from_translations(p_message, p_context, fallback, false); + } + + if (!res) { + return p_message; + } + + return res; +} + +StringName TranslationServer::translate_plural(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context) const { + if (!enabled) { + if (p_n == 1) { + return p_message; + } + return p_message_plural; + } + + ERR_FAIL_COND_V_MSG(locale.length() < 2, p_message, "Could not translate message as configured locale '" + locale + "' is invalid."); + + StringName res = _get_message_from_translations(p_message, p_context, locale, true, p_message_plural, p_n); + + if (!res && fallback.length() >= 2) { + res = _get_message_from_translations(p_message, p_context, fallback, true, p_message_plural, p_n); + } + + if (!res) { + if (p_n == 1) { + return p_message; + } + return p_message_plural; + } + + return res; +} + +StringName TranslationServer::_get_message_from_translations(const StringName &p_message, const StringName &p_context, const String &p_locale, bool plural, const String &p_message_plural, int p_n) const { // Locale can be of the form 'll_CC', i.e. language code and regional code, // e.g. 'en_US', 'en_GB', etc. It might also be simply 'll', e.g. 'en'. // To find the relevant translation, we look for those with locale starting @@ -1044,7 +1121,7 @@ StringName TranslationServer::translate(const StringName &p_message) const { // logic, so be sure to propagate changes there when changing things here. StringName res; - String lang = get_language_code(locale); + String lang = get_language_code(p_locale); bool near_match = false; for (const Set>::Element *E = translations.front(); E; E = E->next()) { @@ -1052,7 +1129,7 @@ StringName TranslationServer::translate(const StringName &p_message) const { ERR_FAIL_COND_V(t.is_null(), p_message); String l = t->get_locale(); - bool exact_match = (l == locale); + bool exact_match = (l == p_locale); if (!exact_match) { if (near_match) { continue; // Only near-match once, but keep looking for exact matches. @@ -1062,7 +1139,13 @@ StringName TranslationServer::translate(const StringName &p_message) const { } } - StringName r = t->get_message(p_message); + StringName r; + if (!plural) { + r = t->get_message(p_message, p_context); + } else { + r = t->get_plural_message(p_message, p_message_plural, p_n, p_context); + } + if (!r) { continue; } @@ -1075,44 +1158,6 @@ StringName TranslationServer::translate(const StringName &p_message) const { } } - if (!res && fallback.length() >= 2) { - // Try again with the fallback locale. - String fallback_lang = get_language_code(fallback); - near_match = false; - - for (const Set>::Element *E = translations.front(); E; E = E->next()) { - const Ref &t = E->get(); - ERR_FAIL_COND_V(t.is_null(), p_message); - String l = t->get_locale(); - - bool exact_match = (l == fallback); - if (!exact_match) { - if (near_match) { - continue; // Only near-match once, but keep looking for exact matches. - } - if (get_language_code(l) != fallback_lang) { - continue; // Language code does not match. - } - } - - StringName r = t->get_message(p_message); - if (!r) { - continue; - } - res = r; - - if (exact_match) { - break; - } else { - near_match = true; - } - } - } - - if (!res) { - return p_message; - } - return res; } @@ -1169,9 +1214,9 @@ void TranslationServer::set_tool_translation(const Ref &p_translati tool_translation = p_translation; } -StringName TranslationServer::tool_translate(const StringName &p_message) const { +StringName TranslationServer::tool_translate(const StringName &p_message, const StringName &p_context) const { if (tool_translation.is_valid()) { - StringName r = tool_translation->get_message(p_message); + StringName r = tool_translation->get_message(p_message, p_context); if (r) { return r; } @@ -1179,13 +1224,27 @@ StringName TranslationServer::tool_translate(const StringName &p_message) const return p_message; } +StringName TranslationServer::tool_translate_plural(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context) const { + if (tool_translation.is_valid()) { + StringName r = tool_translation->get_plural_message(p_message, p_message_plural, p_n, p_context); + if (r) { + return r; + } + } + + if (p_n == 1) { + return p_message; + } + return p_message_plural; +} + void TranslationServer::set_doc_translation(const Ref &p_translation) { doc_translation = p_translation; } -StringName TranslationServer::doc_translate(const StringName &p_message) const { +StringName TranslationServer::doc_translate(const StringName &p_message, const StringName &p_context) const { if (doc_translation.is_valid()) { - StringName r = doc_translation->get_message(p_message); + StringName r = doc_translation->get_message(p_message, p_context); if (r) { return r; } @@ -1193,16 +1252,32 @@ StringName TranslationServer::doc_translate(const StringName &p_message) const { return p_message; } +StringName TranslationServer::doc_translate_plural(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context) const { + if (doc_translation.is_valid()) { + StringName r = doc_translation->get_plural_message(p_message, p_message_plural, p_n, p_context); + if (r) { + return r; + } + } + + if (p_n == 1) { + return p_message; + } + return p_message_plural; +} + void TranslationServer::_bind_methods() { ClassDB::bind_method(D_METHOD("set_locale", "locale"), &TranslationServer::set_locale); ClassDB::bind_method(D_METHOD("get_locale"), &TranslationServer::get_locale); ClassDB::bind_method(D_METHOD("get_locale_name", "locale"), &TranslationServer::get_locale_name); - ClassDB::bind_method(D_METHOD("translate", "message"), &TranslationServer::translate); + ClassDB::bind_method(D_METHOD("translate", "message", "context"), &TranslationServer::translate, DEFVAL("")); + ClassDB::bind_method(D_METHOD("translate_plural", "message", "plural_message", "n", "context"), &TranslationServer::translate_plural, DEFVAL("")); ClassDB::bind_method(D_METHOD("add_translation", "translation"), &TranslationServer::add_translation); ClassDB::bind_method(D_METHOD("remove_translation", "translation"), &TranslationServer::remove_translation); + ClassDB::bind_method(D_METHOD("get_translation_object", "locale"), &TranslationServer::get_translation_object); ClassDB::bind_method(D_METHOD("clear"), &TranslationServer::clear); diff --git a/core/translation.h b/core/translation.h index 4f50a1a4bc0..cba25a434f0 100644 --- a/core/translation.h +++ b/core/translation.h @@ -41,10 +41,9 @@ class Translation : public Resource { String locale = "en"; Map translation_map; - Vector _get_message_list() const; - - Vector _get_messages() const; - void _set_messages(const Vector &p_messages); + virtual Vector _get_message_list() const; + virtual Dictionary _get_messages() const; + virtual void _set_messages(const Dictionary &p_messages); protected: static void _bind_methods(); @@ -53,12 +52,13 @@ public: void set_locale(const String &p_locale); _FORCE_INLINE_ String get_locale() const { return locale; } - void add_message(const StringName &p_src_text, const StringName &p_xlated_text); - virtual StringName get_message(const StringName &p_src_text) const; //overridable for other implementations - void erase_message(const StringName &p_src_text); - - void get_message_list(List *r_messages) const; - int get_message_count() const; + virtual void add_message(const StringName &p_src_text, const StringName &p_xlated_text, const StringName &p_context = ""); + virtual void add_plural_message(const StringName &p_src_text, const Vector &p_plural_xlated_texts, const StringName &p_context = ""); + virtual StringName get_message(const StringName &p_src_text, const StringName &p_context = "") const; //overridable for other implementations + virtual StringName get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context = "") const; + virtual void erase_message(const StringName &p_src_text, const StringName &p_context = ""); + virtual void get_message_list(List *r_messages) const; + virtual int get_message_count() const; Translation() {} }; @@ -80,6 +80,8 @@ class TranslationServer : public Object { static TranslationServer *singleton; bool _load_translations(const String &p_from); + StringName _get_message_from_translations(const StringName &p_message, const StringName &p_context, const String &p_locale, bool plural, const String &p_message_plural = "", int p_n = 0) const; + static void _bind_methods(); public: @@ -90,6 +92,7 @@ public: void set_locale(const String &p_locale); String get_locale() const; + Ref get_translation_object(const String &p_locale); String get_locale_name(const String &p_locale) const; @@ -98,7 +101,8 @@ public: void add_translation(const Ref &p_translation); void remove_translation(const Ref &p_translation); - StringName translate(const StringName &p_message) const; + StringName translate(const StringName &p_message, const StringName &p_context = "") const; + StringName translate_plural(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context = "") const; static Vector get_all_locales(); static Vector get_all_locale_names(); @@ -107,9 +111,11 @@ public: static String get_language_code(const String &p_locale); void set_tool_translation(const Ref &p_translation); - StringName tool_translate(const StringName &p_message) const; + StringName tool_translate(const StringName &p_message, const StringName &p_context = "") const; + StringName tool_translate_plural(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context = "") const; void set_doc_translation(const Ref &p_translation); - StringName doc_translate(const StringName &p_message) const; + StringName doc_translate(const StringName &p_message, const StringName &p_context = "") const; + StringName doc_translate_plural(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context = "") const; void setup(); diff --git a/core/translation_po.cpp b/core/translation_po.cpp new file mode 100644 index 00000000000..203f29026ba --- /dev/null +++ b/core/translation_po.cpp @@ -0,0 +1,312 @@ +/*************************************************************************/ +/* translation_po.cpp */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#include "translation_po.h" + +#include "core/os/file_access.h" + +#ifdef DEBUG_TRANSLATION_PO +void TranslationPO::print_translation_map() { + Error err; + FileAccess *file = FileAccess::open("translation_map_print_test.txt", FileAccess::WRITE, &err); + if (err != OK) { + ERR_PRINT("Failed to open translation_map_print_test.txt"); + return; + } + + file->store_line("NPlural : " + String::num_int64(this->get_plural_forms())); + file->store_line("Plural rule : " + this->get_plural_rule()); + file->store_line(""); + + List context_l; + translation_map.get_key_list(&context_l); + for (auto E = context_l.front(); E; E = E->next()) { + StringName ctx = E->get(); + file->store_line(" ===== Context: " + String::utf8(String(ctx).utf8()) + " ===== "); + const HashMap> &inner_map = translation_map[ctx]; + + List id_l; + inner_map.get_key_list(&id_l); + for (auto E2 = id_l.front(); E2; E2 = E2->next()) { + StringName id = E2->get(); + file->store_line("msgid: " + String::utf8(String(id).utf8())); + for (int i = 0; i < inner_map[id].size(); i++) { + file->store_line("msgstr[" + String::num_int64(i) + "]: " + String::utf8(String(inner_map[id][i]).utf8())); + } + file->store_line(""); + } + } + file->close(); +} +#endif + +Dictionary TranslationPO::_get_messages() const { + // Return translation_map as a Dictionary. + + Dictionary d; + + List context_l; + translation_map.get_key_list(&context_l); + for (auto E = context_l.front(); E; E = E->next()) { + StringName ctx = E->get(); + const HashMap> &id_str_map = translation_map[ctx]; + + Dictionary d2; + List id_l; + id_str_map.get_key_list(&id_l); + // Save list of id and strs associated with a context in a temporary dictionary. + for (auto E2 = id_l.front(); E2; E2 = E2->next()) { + StringName id = E2->get(); + d2[id] = id_str_map[id]; + } + + d[ctx] = d2; + } + + return d; +} + +void TranslationPO::_set_messages(const Dictionary &p_messages) { + // Construct translation_map from a Dictionary. + + List context_l; + p_messages.get_key_list(&context_l); + for (auto E = context_l.front(); E; E = E->next()) { + StringName ctx = E->get(); + const Dictionary &id_str_map = p_messages[ctx]; + + HashMap> temp_map; + List id_l; + id_str_map.get_key_list(&id_l); + for (auto E2 = id_l.front(); E2; E2 = E2->next()) { + StringName id = E2->get(); + temp_map[id] = id_str_map[id]; + } + + translation_map[ctx] = temp_map; + } +} + +Vector TranslationPO::_get_message_list() const { + // Return all keys in translation_map. + + List msgs; + get_message_list(&msgs); + + Vector v; + for (auto E = msgs.front(); E; E = E->next()) { + v.push_back(E->get()); + } + + return v; +} + +int TranslationPO::_get_plural_index(int p_n) const { + // Get a number between [0;number of plural forms). + + input_val.clear(); + input_val.push_back(p_n); + + Variant result; + for (int i = 0; i < equi_tests.size(); i++) { + Error err = expr->parse(equi_tests[i], input_name); + ERR_FAIL_COND_V_MSG(err != OK, 0, "Cannot parse expression. Error: " + expr->get_error_text()); + + result = expr->execute(input_val); + ERR_FAIL_COND_V_MSG(expr->has_execute_failed(), 0, "Cannot evaluate expression."); + + // Last expression. Variant result will either map to a bool or an integer, in both cases returning it will give the correct plural index. + if (i + 1 == equi_tests.size()) { + return result; + } + + if (bool(result)) { + return i; + } + } + + ERR_FAIL_V_MSG(0, "Unexpected. Function should have returned. Please report this bug."); +} + +void TranslationPO::_cache_plural_tests(const String &p_plural_rule) { + // Some examples of p_plural_rule passed in can have the form: + // "n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 && n%100<=99 ? 4 : 5" (Arabic) + // "n >= 2" (French) // When evaluating the last, esp careful with this one. + // "n != 1" (English) + int first_ques_mark = p_plural_rule.find("?"); + if (first_ques_mark == -1) { + equi_tests.push_back(p_plural_rule.strip_edges()); + return; + } + + String equi_test = p_plural_rule.substr(0, first_ques_mark).strip_edges(); + equi_tests.push_back(equi_test); + + String after_colon = p_plural_rule.substr(p_plural_rule.find(":") + 1, p_plural_rule.length()); + _cache_plural_tests(after_colon); +} + +void TranslationPO::set_plural_rule(const String &p_plural_rule) { + // Set plural_forms and plural_rule. + // p_plural_rule passed in has the form "Plural-Forms: nplurals=2; plural=(n >= 2);". + + int first_semi_col = p_plural_rule.find(";"); + plural_forms = p_plural_rule.substr(p_plural_rule.find("=") + 1, first_semi_col - (p_plural_rule.find("=") + 1)).to_int(); + + int expression_start = p_plural_rule.find("=", first_semi_col) + 1; + int second_semi_col = p_plural_rule.rfind(";"); + plural_rule = p_plural_rule.substr(expression_start, second_semi_col - expression_start); + + // Setup the cache to make evaluating plural rule faster later on. + plural_rule = plural_rule.replacen("(", ""); + plural_rule = plural_rule.replacen(")", ""); + _cache_plural_tests(plural_rule); + expr.instance(); + input_name.push_back("n"); +} + +void TranslationPO::add_message(const StringName &p_src_text, const StringName &p_xlated_text, const StringName &p_context) { + HashMap> &map_id_str = translation_map[p_context]; + + if (map_id_str.has(p_src_text)) { + WARN_PRINT("Double translations for \"" + String(p_src_text) + "\" under the same context \"" + String(p_context) + "\" for locale \"" + get_locale() + "\".\nThere should only be one unique translation for a given string under the same context."); + map_id_str[p_src_text].set(0, p_xlated_text); + } else { + map_id_str[p_src_text].push_back(p_xlated_text); + } +} + +void TranslationPO::add_plural_message(const StringName &p_src_text, const Vector &p_plural_xlated_texts, const StringName &p_context) { + ERR_FAIL_COND_MSG(p_plural_xlated_texts.size() != plural_forms, "Trying to add plural texts that don't match the required number of plural forms for locale \"" + get_locale() + "\""); + + HashMap> &map_id_str = translation_map[p_context]; + + if (map_id_str.has(p_src_text)) { + WARN_PRINT("Double translations for \"" + p_src_text + "\" under the same context \"" + p_context + "\" for locale " + get_locale() + ".\nThere should only be one unique translation for a given string under the same context."); + map_id_str[p_src_text].clear(); + } + + for (int i = 0; i < p_plural_xlated_texts.size(); i++) { + map_id_str[p_src_text].push_back(p_plural_xlated_texts[i]); + } +} + +int TranslationPO::get_plural_forms() const { + return plural_forms; +} + +String TranslationPO::get_plural_rule() const { + return plural_rule; +} + +StringName TranslationPO::get_message(const StringName &p_src_text, const StringName &p_context) const { + if (!translation_map.has(p_context) || !translation_map[p_context].has(p_src_text)) { + return StringName(); + } + ERR_FAIL_COND_V_MSG(translation_map[p_context][p_src_text].empty(), StringName(), "Source text \"" + String(p_src_text) + "\" is registered but doesn't have a translation. Please report this bug."); + + return translation_map[p_context][p_src_text][0]; +} + +StringName TranslationPO::get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context) const { + ERR_FAIL_COND_V_MSG(p_n < 0, StringName(), "N passed into translation to get a plural message should not be negative. For negative numbers, use singular translation please. Search \"gettext PO Plural Forms\" online for the documentation on translating negative numbers."); + + // If the query is the same as last time, return the cached result. + if (p_n == last_plural_n && p_context == last_plural_context && p_src_text == last_plural_key) { + return translation_map[p_context][p_src_text][last_plural_mapped_index]; + } + + if (!translation_map.has(p_context) || !translation_map[p_context].has(p_src_text)) { + return StringName(); + } + ERR_FAIL_COND_V_MSG(translation_map[p_context][p_src_text].empty(), StringName(), "Source text \"" + String(p_src_text) + "\" is registered but doesn't have a translation. Please report this bug."); + + if (translation_map[p_context][p_src_text].size() == 1) { + WARN_PRINT("Source string \"" + String(p_src_text) + "\" doesn't have plural translations. Use singular translation API for such as tr(), TTR() to translate \"" + String(p_src_text) + "\""); + return translation_map[p_context][p_src_text][0]; + } + + int plural_index = _get_plural_index(p_n); + ERR_FAIL_COND_V_MSG(plural_index < 0 || translation_map[p_context][p_src_text].size() < plural_index + 1, StringName(), "Plural index returned or number of plural translations is not valid. Please report this bug."); + + // Cache result so that if the next entry is the same, we can return directly. + // _get_plural_index(p_n) can get very costly, especially when evaluating long plural-rule (Arabic) + last_plural_key = p_src_text; + last_plural_context = p_context; + last_plural_n = p_n; + last_plural_mapped_index = plural_index; + + return translation_map[p_context][p_src_text][plural_index]; +} + +void TranslationPO::erase_message(const StringName &p_src_text, const StringName &p_context) { + if (!translation_map.has(p_context)) { + return; + } + + translation_map[p_context].erase(p_src_text); +} + +void TranslationPO::get_message_list(List *r_messages) const { + // PHashTranslation uses this function to get the list of msgid. + // Return all the keys of translation_map under "" context. + + List context_l; + translation_map.get_key_list(&context_l); + + for (auto E = context_l.front(); E; E = E->next()) { + if (String(E->get()) != "") { + continue; + } + + List msgid_l; + translation_map[E->get()].get_key_list(&msgid_l); + + for (auto E2 = msgid_l.front(); E2; E2 = E2->next()) { + r_messages->push_back(E2->get()); + } + } +} + +int TranslationPO::get_message_count() const { + List context_l; + translation_map.get_key_list(&context_l); + + int count = 0; + for (auto E = context_l.front(); E; E = E->next()) { + count += translation_map[E->get()].size(); + } + return count; +} + +void TranslationPO::_bind_methods() { + ClassDB::bind_method(D_METHOD("get_plural_forms"), &TranslationPO::get_plural_forms); + ClassDB::bind_method(D_METHOD("get_plural_rule"), &TranslationPO::get_plural_rule); +} diff --git a/core/translation_po.h b/core/translation_po.h new file mode 100644 index 00000000000..730635f63da --- /dev/null +++ b/core/translation_po.h @@ -0,0 +1,92 @@ +/*************************************************************************/ +/* translation_po.h */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#ifndef TRANSLATION_PO_H +#define TRANSLATION_PO_H + +//#define DEBUG_TRANSLATION_PO + +#include "core/math/expression.h" +#include "core/translation.h" + +class TranslationPO : public Translation { + GDCLASS(TranslationPO, Translation); + + // TLDR: Maps context to a list of source strings and translated strings. In PO terms, maps msgctxt to a list of msgid and msgstr. + // The first key corresponds to context, and the second key (of the contained HashMap) corresponds to source string. + // The value Vector in the second map stores the translated strings. Index 0, 1, 2 matches msgstr[0], msgstr[1], msgstr[2]... in the case of plurals. + // Otherwise index 0 mathes to msgstr in a singular translation. + // Strings without context have "" as first key. + HashMap>> translation_map; + + int plural_forms = 0; // 0 means no "Plural-Forms" is given in the PO header file. The min for all languages is 1. + String plural_rule; + + // Cache temporary variables related to _get_plural_index() to make it faster + Vector equi_tests; + Vector input_name; + mutable Ref expr; + mutable Array input_val; + mutable StringName last_plural_key; + mutable StringName last_plural_context; + mutable int last_plural_n = -1; // Set it to an impossible value at the beginning. + mutable int last_plural_mapped_index = 0; + + void _cache_plural_tests(const String &p_plural_rule); + int _get_plural_index(int p_n) const; + + Vector _get_message_list() const override; + Dictionary _get_messages() const override; + void _set_messages(const Dictionary &p_messages) override; + +protected: + static void _bind_methods(); + +public: + void get_message_list(List *r_messages) const override; + int get_message_count() const override; + void add_message(const StringName &p_src_text, const StringName &p_xlated_text, const StringName &p_context = "") override; + void add_plural_message(const StringName &p_src_text, const Vector &p_plural_xlated_texts, const StringName &p_context = "") override; + StringName get_message(const StringName &p_src_text, const StringName &p_context = "") const override; + StringName get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context = "") const override; + void erase_message(const StringName &p_src_text, const StringName &p_context = "") override; + + void set_plural_rule(const String &p_plural_rule); + int get_plural_forms() const; + String get_plural_rule() const; + +#ifdef DEBUG_TRANSLATION_PO + void print_translation_map(); +#endif + + TranslationPO() {} +}; + +#endif // TRANSLATION_PO_H diff --git a/core/ustring.cpp b/core/ustring.cpp index 957caf1015e..9d2d938eafe 100644 --- a/core/ustring.cpp +++ b/core/ustring.cpp @@ -4269,31 +4269,58 @@ String String::unquote() const { } #ifdef TOOLS_ENABLED -String TTR(const String &p_text) { +String TTR(const String &p_text, const String &p_context) { if (TranslationServer::get_singleton()) { - return TranslationServer::get_singleton()->tool_translate(p_text); + return TranslationServer::get_singleton()->tool_translate(p_text, p_context); } return p_text; } -String DTR(const String &p_text) { +String TTRN(const String &p_text, const String &p_text_plural, int p_n, const String &p_context) { + if (TranslationServer::get_singleton()) { + return TranslationServer::get_singleton()->tool_translate_plural(p_text, p_text_plural, p_n, p_context); + } + + // Return message based on English plural rule if translation is not possible. + if (p_n == 1) { + return p_text; + } + return p_text_plural; +} + +String DTR(const String &p_text, const String &p_context) { // Comes straight from the XML, so remove indentation and any trailing whitespace. const String text = p_text.dedent().strip_edges(); if (TranslationServer::get_singleton()) { - return TranslationServer::get_singleton()->doc_translate(text); + return TranslationServer::get_singleton()->doc_translate(text, p_context); } return text; } + +String DTRN(const String &p_text, const String &p_text_plural, int p_n, const String &p_context) { + const String text = p_text.dedent().strip_edges(); + const String text_plural = p_text_plural.dedent().strip_edges(); + + if (TranslationServer::get_singleton()) { + return TranslationServer::get_singleton()->doc_translate_plural(text, text_plural, p_n, p_context); + } + + // Return message based on English plural rule if translation is not possible. + if (p_n == 1) { + return text; + } + return text_plural; +} #endif -String RTR(const String &p_text) { +String RTR(const String &p_text, const String &p_context) { if (TranslationServer::get_singleton()) { - String rtr = TranslationServer::get_singleton()->tool_translate(p_text); + String rtr = TranslationServer::get_singleton()->tool_translate(p_text, p_context); if (rtr == String() || rtr == p_text) { - return TranslationServer::get_singleton()->translate(p_text); + return TranslationServer::get_singleton()->translate(p_text, p_context); } else { return rtr; } @@ -4301,3 +4328,20 @@ String RTR(const String &p_text) { return p_text; } + +String RTRN(const String &p_text, const String &p_text_plural, int p_n, const String &p_context) { + if (TranslationServer::get_singleton()) { + String rtr = TranslationServer::get_singleton()->tool_translate_plural(p_text, p_text_plural, p_n, p_context); + if (rtr == String() || rtr == p_text || rtr == p_text_plural) { + return TranslationServer::get_singleton()->translate_plural(p_text, p_text_plural, p_n, p_context); + } else { + return rtr; + } + } + + // Return message based on English plural rule if translation is not possible. + if (p_n == 1) { + return p_text; + } + return p_text_plural; +} diff --git a/core/ustring.h b/core/ustring.h index d37346fbd6e..7a1c1a5232f 100644 --- a/core/ustring.h +++ b/core/ustring.h @@ -410,8 +410,10 @@ _FORCE_INLINE_ bool is_str_less(const L *l_ptr, const R *r_ptr) { // and doc translate for the class reference (DTR). #ifdef TOOLS_ENABLED // Gets parsed. -String TTR(const String &); -String DTR(const String &); +String TTR(const String &p_text, const String &p_context = ""); +String TTRN(const String &p_text, const String &p_text_plural, int p_n, const String &p_context = ""); +String DTR(const String &p_text, const String &p_context = ""); +String DTRN(const String &p_text, const String &p_text_plural, int p_n, const String &p_context = ""); // Use for C strings. #define TTRC(m_value) (m_value) // Use to avoid parsing (for use later with C strings). @@ -419,13 +421,16 @@ String DTR(const String &); #else #define TTR(m_value) (String()) +#define TTRN(m_value) (String()) #define DTR(m_value) (String()) +#define DTRN(m_value) (String()) #define TTRC(m_value) (m_value) #define TTRGET(m_value) (m_value) #endif // Runtime translate for the public node API. -String RTR(const String &); +String RTR(const String &p_text, const String &p_context = ""); +String RTRN(const String &p_text, const String &p_text_plural, int p_n, const String &p_context = ""); bool is_symbol(CharType c); bool select_word(const String &p_s, int p_col, int &r_beg, int &r_end); diff --git a/doc/classes/EditorTranslationParserPlugin.xml b/doc/classes/EditorTranslationParserPlugin.xml index d40fc558de0..f5204e7babf 100644 --- a/doc/classes/EditorTranslationParserPlugin.xml +++ b/doc/classes/EditorTranslationParserPlugin.xml @@ -5,30 +5,41 @@ Plugins are registered via [method EditorPlugin.add_translation_parser_plugin] method. To define the parsing and string extraction logic, override the [method parse_file] method in script. + Add the extracted strings to argument [code]msgids[/code] or [code]msgids_context_plural[/code] if context or plural is used. + When adding to [code]msgids_context_plural[/code], you must add the data using the format [code]["A", "B", "C"][/code], where [code]A[/code] represents the extracted string, [code]B[/code] represents the context, and [code]C[/code] represents the plural version of the extracted string. If you want to add only context but not plural, put [code]""[/code] for the plural slot. The idea is the same if you only want to add plural but not context. See the code below for concrete examples. The extracted strings will be written into a POT file selected by user under "POT Generation" in "Localization" tab in "Project Settings" menu. - Below shows an example of a custom parser that extracts strings in a CSV file to write into a POT. + Below shows an example of a custom parser that extracts strings from a CSV file to write into a POT. [codeblock] tool extends EditorTranslationParserPlugin - func parse_file(path, extracted_strings): + func parse_file(path, msgids, msgids_context_plural): var file = File.new() file.open(path, File.READ) var text = file.get_as_text() var split_strs = text.split(",", false, 0) for s in split_strs: - extracted_strings.append(s) + msgids.append(s) #print("Extracted string: " + s) func get_recognized_extensions(): return ["csv"] [/codeblock] + To add a translatable string associated with context or plural, add it to [code]msgids_context_plural[/code]: + [codeblock] + # This will add a message with msgid "Test 1", msgctxt "context", and msgid_plural "test 1 plurals". + msgids_context_plural.append(["Test 1", "context", "test 1 plurals"]) + # This will add a message with msgid "A test without context" and msgid_plural "plurals". + msgids_context_plural.append(["A test without context", "", "plurals"]) + # This will add a message with msgid "Only with context" and msgctxt "a friendly context". + msgids_context_plural.append(["Only with context", "a friendly context", ""]) + [/codeblock] [b]Note:[/b] If you override parsing logic for standard script types (GDScript, C#, etc.), it would be better to load the [code]path[/code] argument using [method ResourceLoader.load]. This is because built-in scripts are loaded as [Resource] type, not [File] type. For example: [codeblock] - func parse_file(path, extracted_strings): + func parse_file(path, msgids, msgids_context_plural): var res = ResourceLoader.load(path, "Script") var text = res.get_source_code() # Parsing logic. @@ -53,7 +64,9 @@ - + + + Override this method to define a custom parsing logic to extract the translatable strings. diff --git a/doc/classes/Object.xml b/doc/classes/Object.xml index ca6b624359d..077067a0f4f 100644 --- a/doc/classes/Object.xml +++ b/doc/classes/Object.xml @@ -486,13 +486,35 @@ - + + + - Translates a message using translation catalogs configured in the Project Settings. + Translates a message using translation catalogs configured in the Project Settings. An additional context could be used to specify the translation context. Only works if message translation is enabled (which it is by default), otherwise it returns the [code]message[/code] unchanged. See [method set_message_translation]. + See https://docs.godotengine.org/en/latest/tutorials/i18n/internationalizing_games.html for examples of the usage of this method. + + + + + + + + + + + + + + + Translates a message involving plurals using translation catalogs configured in the Project Settings. An additional context could be used to specify the translation context. + Only works if message translation is enabled (which it is by default), otherwise it returns the [code]message[/code] or [code]plural_message[/code] unchanged. See [method set_message_translation]. + The number [code]n[/code] is the number or quantity of the plural object. It will be used to guide the translation system to fetch the correct plural form for the selected language. + [b]Note:[/b] Negative and floating-point values usually represent physical entities for which singular and plural don't clearly apply. In such cases, use [method tr]. + See https://docs.godotengine.org/en/latest/tutorials/i18n/localization_using_gettext.html for examples of the usage of this method. diff --git a/doc/classes/Translation.xml b/doc/classes/Translation.xml index 11245195bfc..1989a63362a 100644 --- a/doc/classes/Translation.xml +++ b/doc/classes/Translation.xml @@ -18,8 +18,25 @@ + + Adds a message if nonexistent, followed by its translation. + An additional context could be used to specify the translation context or differentiate polysemic words. + + + + + + + + + + + + + Adds a message involving plural translation if nonexistent, followed by its translation. + An additional context could be used to specify the translation context or differentiate polysemic words. @@ -27,6 +44,8 @@ + + Erases a message. @@ -36,6 +55,8 @@ + + Returns a message's translation. @@ -54,6 +75,22 @@ Returns all the messages (keys). + + + + + + + + + + + + + Returns a message's translation involving plurals. + The number [code]n[/code] is the number or quantity of the plural object. It will be used to guide the translation system to fetch the correct plural form for the selected language. + + diff --git a/doc/classes/TranslationServer.xml b/doc/classes/TranslationServer.xml index aaf7a4d160d..3369663af66 100644 --- a/doc/classes/TranslationServer.xml +++ b/doc/classes/TranslationServer.xml @@ -50,6 +50,16 @@ Returns a locale's language and its variant (e.g. [code]"en_US"[/code] would return [code]"English (United States)"[/code]). + + + + + + + Returns the [Translation] instance based on the [code]locale[/code] passed in. + It will return a [code]nullptr[/code] if there is no [Translation] instance that matches the [code]locale[/code]. + + @@ -73,8 +83,26 @@ + + - Returns the current locale's translation for the given message (key). + Returns the current locale's translation for the given message (key) and context. + + + + + + + + + + + + + + + Returns the current locale's translation for the given message (key), plural_message and context. + The number [code]n[/code] is the number or quantity of the plural object. It will be used to guide the translation system to fetch the correct plural form for the selected language. diff --git a/editor/editor_translation_parser.cpp b/editor/editor_translation_parser.cpp index da191fbc922..7a90d200003 100644 --- a/editor/editor_translation_parser.cpp +++ b/editor/editor_translation_parser.cpp @@ -37,15 +37,30 @@ EditorTranslationParser *EditorTranslationParser::singleton = nullptr; -Error EditorTranslationParserPlugin::parse_file(const String &p_path, Vector *r_extracted_strings) { +Error EditorTranslationParserPlugin::parse_file(const String &p_path, Vector *r_ids, Vector> *r_ids_ctx_plural) { if (!get_script_instance()) return ERR_UNAVAILABLE; if (get_script_instance()->has_method("parse_file")) { - Array extracted_strings; - get_script_instance()->call("parse_file", p_path, extracted_strings); - for (int i = 0; i < extracted_strings.size(); i++) { - r_extracted_strings->append(extracted_strings[i]); + Array ids; + Array ids_ctx_plural; + get_script_instance()->call("parse_file", p_path, ids, ids_ctx_plural); + + // Add user's extracted translatable messages. + for (int i = 0; i < ids.size(); i++) { + r_ids->append(ids[i]); + } + + // Add user's collected translatable messages with context or plurals. + for (int i = 0; i < ids_ctx_plural.size(); i++) { + Array arr = ids_ctx_plural[i]; + ERR_FAIL_COND_V_MSG(arr.size() != 3, ERR_INVALID_DATA, "Array entries written into `msgids_context_plural` in `parse_file()` method should have the form [\"message\", \"context\", \"plural message\"]"); + + Vector id_ctx_plural; + id_ctx_plural.push_back(arr[0]); + id_ctx_plural.push_back(arr[1]); + id_ctx_plural.push_back(arr[2]); + r_ids_ctx_plural->append(id_ctx_plural); } return OK; } else { @@ -69,7 +84,7 @@ void EditorTranslationParserPlugin::get_recognized_extensions(List *r_ex } void EditorTranslationParserPlugin::_bind_methods() { - ClassDB::add_virtual_method(get_class_static(), MethodInfo(Variant::NIL, "parse_file", PropertyInfo(Variant::STRING, "path"), PropertyInfo(Variant::ARRAY, "extracted_strings"))); + ClassDB::add_virtual_method(get_class_static(), MethodInfo(Variant::NIL, "parse_file", PropertyInfo(Variant::STRING, "path"), PropertyInfo(Variant::ARRAY, "msgids"), PropertyInfo(Variant::ARRAY, "msgids_context_plural"))); ClassDB::add_virtual_method(get_class_static(), MethodInfo(Variant::ARRAY, "get_recognized_extensions")); } diff --git a/editor/editor_translation_parser.h b/editor/editor_translation_parser.h index fb8aa6ec9bb..18f49b3803c 100644 --- a/editor/editor_translation_parser.h +++ b/editor/editor_translation_parser.h @@ -41,7 +41,7 @@ protected: static void _bind_methods(); public: - virtual Error parse_file(const String &p_path, Vector *r_extracted_strings); + virtual Error parse_file(const String &p_path, Vector *r_ids, Vector> *r_ids_ctx_plural); virtual void get_recognized_extensions(List *r_extensions) const; }; diff --git a/editor/plugins/packed_scene_translation_parser_plugin.cpp b/editor/plugins/packed_scene_translation_parser_plugin.cpp index 52af0008b72..608b5c3104e 100644 --- a/editor/plugins/packed_scene_translation_parser_plugin.cpp +++ b/editor/plugins/packed_scene_translation_parser_plugin.cpp @@ -37,7 +37,7 @@ void PackedSceneEditorTranslationParserPlugin::get_recognized_extensions(List *r_extracted_strings) { +Error PackedSceneEditorTranslationParserPlugin::parse_file(const String &p_path, Vector *r_ids, Vector> *r_ids_ctx_plural) { // Parse specific scene Node's properties (see in constructor) that are auto-translated by the engine when set. E.g Label's text property. // These properties are translated with the tr() function in the C++ code when being set or updated. @@ -71,8 +71,10 @@ Error PackedSceneEditorTranslationParserPlugin::parse_file(const String &p_path, String extension = s->get_language()->get_extension(); if (EditorTranslationParser::get_singleton()->can_parse(extension)) { Vector temp; - EditorTranslationParser::get_singleton()->get_parser(extension)->parse_file(s->get_path(), &temp); + Vector> ids_context_plural; + EditorTranslationParser::get_singleton()->get_parser(extension)->parse_file(s->get_path(), &temp, &ids_context_plural); parsed_strings.append_array(temp); + r_ids_ctx_plural->append_array(ids_context_plural); } } else if (property_name == "filters") { // Extract FileDialog's filters property with values in format "*.png ; PNG Images","*.gd ; GDScript Files". @@ -93,7 +95,7 @@ Error PackedSceneEditorTranslationParserPlugin::parse_file(const String &p_path, } } - r_extracted_strings->append_array(parsed_strings); + r_ids->append_array(parsed_strings); return OK; } diff --git a/editor/plugins/packed_scene_translation_parser_plugin.h b/editor/plugins/packed_scene_translation_parser_plugin.h index 2bd4dae9953..a0ffdf692c4 100644 --- a/editor/plugins/packed_scene_translation_parser_plugin.h +++ b/editor/plugins/packed_scene_translation_parser_plugin.h @@ -40,7 +40,7 @@ class PackedSceneEditorTranslationParserPlugin : public EditorTranslationParserP Set lookup_properties; public: - virtual Error parse_file(const String &p_path, Vector *r_extracted_strings) override; + virtual Error parse_file(const String &p_path, Vector *r_ids, Vector> *r_ids_ctx_plural) override; virtual void get_recognized_extensions(List *r_extensions) const override; PackedSceneEditorTranslationParserPlugin(); diff --git a/editor/pot_generator.cpp b/editor/pot_generator.cpp index f9b8722aada..f09750efdc0 100644 --- a/editor/pot_generator.cpp +++ b/editor/pot_generator.cpp @@ -31,23 +31,25 @@ #include "pot_generator.h" #include "core/error_macros.h" -#include "core/os/file_access.h" #include "core/project_settings.h" #include "editor_translation_parser.h" #include "plugins/packed_scene_translation_parser_plugin.h" POTGenerator *POTGenerator::singleton = nullptr; -//#define DEBUG_POT - #ifdef DEBUG_POT -void _print_all_translation_strings(const OrderedHashMap> &p_all_translation_strings) { - for (auto E_pair = p_all_translation_strings.front(); E_pair; E_pair = E_pair.next()) { - String msg = static_cast(E_pair.key()) + " : "; - for (Set::Element *E = E_pair.value().front(); E; E = E->next()) { - msg += E->get() + " "; +void POTGenerator::_print_all_translation_strings() { + for (auto E = all_translation_strings.front(); E; E = E.next()) { + Vector v_md = all_translation_strings[E.key()]; + for (int i = 0; i < v_md.size(); i++) { + print_line("++++++"); + print_line("msgid: " + E.key()); + print_line("context: " + v_md[i].ctx); + print_line("msgid_plural: " + v_md[i].plural); + for (Set::Element *E = v_md[i].locations.front(); E; E = E->next()) { + print_line("location: " + E->get()); + } } - print_line(msg); } } #endif @@ -65,27 +67,27 @@ void POTGenerator::generate_pot(const String &p_file) { // Collect all translatable strings according to files order in "POT Generation" setting. for (int i = 0; i < files.size(); i++) { - Vector translation_strings; + Vector msgids; + Vector> msgids_context_plural; String file_path = files[i]; String file_extension = file_path.get_extension(); if (EditorTranslationParser::get_singleton()->can_parse(file_extension)) { - EditorTranslationParser::get_singleton()->get_parser(file_extension)->parse_file(file_path, &translation_strings); + EditorTranslationParser::get_singleton()->get_parser(file_extension)->parse_file(file_path, &msgids, &msgids_context_plural); } else { ERR_PRINT("Unrecognized file extension " + file_extension + " in generate_pot()"); return; } - // Store translation strings parsed in this iteration along with their corresponding source file - to write into POT later on. - for (int j = 0; j < translation_strings.size(); j++) { - all_translation_strings[translation_strings[j]].insert(file_path); + for (int j = 0; j < msgids_context_plural.size(); j++) { + Vector entry = msgids_context_plural[j]; + _add_new_msgid(entry[0], entry[1], entry[2], file_path); + } + for (int j = 0; j < msgids.size(); j++) { + _add_new_msgid(msgids[j], "", "", file_path); } } -#ifdef DEBUG_POT - _print_all_translation_strings(all_translation_strings); -#endif - _write_to_pot(p_file); } @@ -119,37 +121,88 @@ void POTGenerator::_write_to_pot(const String &p_file) { file->store_string(header); - for (OrderedHashMap>::Element E_pair = all_translation_strings.front(); E_pair; E_pair = E_pair.next()) { - String msg = E_pair.key(); + for (OrderedHashMap>::Element E_pair = all_translation_strings.front(); E_pair; E_pair = E_pair.next()) { + String msgid = E_pair.key(); + Vector v_msgid_data = E_pair.value(); + for (int i = 0; i < v_msgid_data.size(); i++) { + String context = v_msgid_data[i].ctx; + String plural = v_msgid_data[i].plural; + const Set &locations = v_msgid_data[i].locations; - // Write file locations. - for (Set::Element *E = E_pair.value().front(); E; E = E->next()) { - file->store_line("#: " + E->get().trim_prefix("res://")); - } + // Write file locations. + for (Set::Element *E = locations.front(); E; E = E->next()) { + file->store_line("#: " + E->get().trim_prefix("res://")); + } - // Split \\n and \n. - Vector temp = msg.split("\\n"); - Vector msg_lines; - for (int i = 0; i < temp.size(); i++) { - msg_lines.append_array(temp[i].split("\n")); - if (i < temp.size() - 1) { - // Add \n. - msg_lines.set(msg_lines.size() - 1, msg_lines[msg_lines.size() - 1] + "\\n"); + // Write context. + if (!context.empty()) { + file->store_line("msgctxt \"" + context + "\""); + } + + // Write msgid. + _write_msgid(file, msgid, false); + + // Write msgid_plural + if (!plural.empty()) { + _write_msgid(file, plural, true); + file->store_line("msgstr[0] \"\""); + file->store_line("msgstr[1] \"\"\n"); + } else { + file->store_line("msgstr \"\"\n"); } } - - // Write msgid. - file->store_string("msgid "); - for (int i = 0; i < msg_lines.size(); i++) { - file->store_line("\"" + msg_lines[i] + "\""); - } - - file->store_line("msgstr \"\"\n"); } file->close(); } +void POTGenerator::_write_msgid(FileAccess *r_file, const String &p_id, bool p_plural) { + // Split \\n and \n. + Vector temp = p_id.split("\\n"); + Vector msg_lines; + for (int i = 0; i < temp.size(); i++) { + msg_lines.append_array(temp[i].split("\n")); + if (i < temp.size() - 1) { + // Add \n. + msg_lines.set(msg_lines.size() - 1, msg_lines[msg_lines.size() - 1] + "\\n"); + } + } + + if (p_plural) { + r_file->store_string("msgid_plural "); + } else { + r_file->store_string("msgid "); + } + + for (int i = 0; i < msg_lines.size(); i++) { + r_file->store_line("\"" + msg_lines[i] + "\""); + } +} + +void POTGenerator::_add_new_msgid(const String &p_msgid, const String &p_context, const String &p_plural, const String &p_location) { + // Insert new location if msgid under same context exists already. + if (all_translation_strings.has(p_msgid)) { + Vector &v_mdata = all_translation_strings[p_msgid]; + for (int i = 0; i < v_mdata.size(); i++) { + if (v_mdata[i].ctx == p_context) { + if (!v_mdata[i].plural.empty() && !p_plural.empty() && v_mdata[i].plural != p_plural) { + WARN_PRINT("Redefinition of plural message (msgid_plural), under the same message (msgid) and context (msgctxt)"); + } + v_mdata.write[i].locations.insert(p_location); + return; + } + } + } + + // Add a new entry of msgid, context, plural and location - context and plural might be empty if the inserted msgid doesn't associated + // context or plurals. + MsgidData mdata; + mdata.ctx = p_context; + mdata.plural = p_plural; + mdata.locations.insert(p_location); + all_translation_strings[p_msgid].push_back(mdata); +} + POTGenerator *POTGenerator::get_singleton() { if (!singleton) { singleton = memnew(POTGenerator); diff --git a/editor/pot_generator.h b/editor/pot_generator.h index abe1a21d417..8853b784ed6 100644 --- a/editor/pot_generator.h +++ b/editor/pot_generator.h @@ -32,14 +32,29 @@ #define POT_GENERATOR_H #include "core/ordered_hash_map.h" +#include "core/os/file_access.h" #include "core/set.h" +//#define DEBUG_POT + class POTGenerator { static POTGenerator *singleton; - // Stores all translatable strings and the source files containing them. - OrderedHashMap> all_translation_strings; + + struct MsgidData { + String ctx; + String plural; + Set locations; + }; + // Store msgid as key and the additional data around the msgid - if it's under a context, has plurals and its file locations. + OrderedHashMap> all_translation_strings; void _write_to_pot(const String &p_file); + void _write_msgid(FileAccess *r_file, const String &p_id, bool p_plural); + void _add_new_msgid(const String &p_msgid, const String &p_context, const String &p_plural, const String &p_location); + +#ifdef DEBUG_POT + void _print_all_translation_strings(); +#endif public: static POTGenerator *get_singleton(); diff --git a/editor/scene_tree_editor.cpp b/editor/scene_tree_editor.cpp index 7404c9779bb..a62448169da 100644 --- a/editor/scene_tree_editor.cpp +++ b/editor/scene_tree_editor.cpp @@ -258,27 +258,35 @@ bool SceneTreeEditor::_add_nodes(Node *p_node, TreeItem *p_parent) { int num_connections = p_node->get_persistent_signal_connection_count(); int num_groups = p_node->get_persistent_group_count(); + String msg_temp; + if (num_connections >= 1) { + Array arr; + arr.push_back(num_connections); + msg_temp += TTRN("Node has one connection.", "Node has {num} connections.", num_connections).format(arr, "{num}"); + msg_temp += " "; + } + if (num_groups >= 1) { + Array arr; + arr.push_back(num_groups); + msg_temp += TTRN("Node is in one group.", "Node is in {num} groups.", num_groups).format(arr, "{num}"); + } + if (num_connections >= 1 || num_groups >= 1) { + msg_temp += "\n" + TTR("Click to show signals dock."); + } + + Ref icon_temp; + auto signal_temp = BUTTON_SIGNALS; if (num_connections >= 1 && num_groups >= 1) { - item->add_button( - 0, - get_theme_icon("SignalsAndGroups", "EditorIcons"), - BUTTON_SIGNALS, - false, - vformat(TTR("Node has %s connection(s) and %s group(s).\nClick to show signals dock."), num_connections, num_groups)); + icon_temp = get_theme_icon("SignalsAndGroups", "EditorIcons"); } else if (num_connections >= 1) { - item->add_button( - 0, - get_theme_icon("Signals", "EditorIcons"), - BUTTON_SIGNALS, - false, - vformat(TTR("Node has %s connection(s).\nClick to show signals dock."), num_connections)); + icon_temp = get_theme_icon("Signals", "EditorIcons"); } else if (num_groups >= 1) { - item->add_button( - 0, - get_theme_icon("Groups", "EditorIcons"), - BUTTON_GROUPS, - false, - vformat(TTR("Node is in %s group(s).\nClick to show groups dock."), num_groups)); + icon_temp = get_theme_icon("Groups", "EditorIcons"); + signal_temp = BUTTON_GROUPS; + } + + if (num_connections >= 1 || num_groups >= 1) { + item->add_button(0, icon_temp, signal_temp, false, msg_temp); } } diff --git a/editor/translations/extract.py b/editor/translations/extract.py index 749bad5fff6..02ed65131fe 100755 --- a/editor/translations/extract.py +++ b/editor/translations/extract.py @@ -33,6 +33,7 @@ matches.sort() unique_str = [] unique_loc = {} +ctx_group = {} # Store msgctx, msg, and locations. main_po = """ # LANGUAGE translation of the Godot Engine editor. # Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. @@ -52,6 +53,34 @@ msgstr "" """ +def _write_message(msgctx, msg, msg_plural, location): + global main_po + main_po += "\n#: " + location + "\n" + if msgctx != "": + main_po += 'msgctxt "' + msgctx + '"\n' + main_po += 'msgid "' + msg + '"\n' + if msg_plural != "": + main_po += 'msgid_plural "' + msg_plural + '"\n' + main_po += 'msgstr[0] ""\n' + main_po += 'msgstr[1] ""\n' + else: + main_po += 'msgstr ""\n' + + +def _add_additional_location(msgctx, msg, location): + global main_po + # Add additional location to previous occurrence + msg_pos = -1 + if msgctx != "": + msg_pos = main_po.find('\nmsgctxt "' + msgctx + '"\nmsgid "' + msg + '"') + else: + msg_pos = main_po.find('\nmsgid "' + msg + '"') + + if msg_pos == -1: + print("Someone apparently thought writing Python was as easy as GDScript. Ping Akien.") + main_po = main_po[:msg_pos] + " " + location + main_po[msg_pos:] + + def process_file(f, fname): global main_po, unique_str, unique_loc @@ -60,10 +89,11 @@ def process_file(f, fname): lc = 1 while l: - patterns = ['RTR("', 'TTR("', 'TTRC("'] + patterns = ['RTR("', 'TTR("', 'TTRC("', 'TTRN("', 'RTRN("'] idx = 0 pos = 0 while pos >= 0: + # Loop until a pattern is found. If not, next line. pos = l.find(patterns[idx], pos) if pos == -1: if idx < len(patterns) - 1: @@ -72,29 +102,64 @@ def process_file(f, fname): continue pos += len(patterns[idx]) + # Read msg until " msg = "" while pos < len(l) and (l[pos] != '"' or l[pos - 1] == "\\"): msg += l[pos] pos += 1 + # Read plural. + msg_plural = "" + if patterns[idx] in ['TTRN("', 'RTRN("']: + pos = l.find('"', pos + 1) + pos += 1 + while pos < len(l) and (l[pos] != '"' or l[pos - 1] == "\\"): + msg_plural += l[pos] + pos += 1 + + # Read context. + msgctx = "" + pos += 1 + read_ctx = False + while pos < len(l): + if l[pos] == ")": + break + elif l[pos] == '"': + read_ctx = True + break + pos += 1 + + pos += 1 + if read_ctx: + while pos < len(l) and (l[pos] != '"' or l[pos - 1] == "\\"): + msgctx += l[pos] + pos += 1 + + # File location. location = os.path.relpath(fname).replace("\\", "/") if line_nb: location += ":" + str(lc) - if not msg in unique_str: - main_po += "\n#: " + location + "\n" - main_po += 'msgid "' + msg + '"\n' - main_po += 'msgstr ""\n' - unique_str.append(msg) - unique_loc[msg] = [location] - elif not location in unique_loc[msg]: - # Add additional location to previous occurrence too - msg_pos = main_po.find('\nmsgid "' + msg + '"') - if msg_pos == -1: - print("Someone apparently thought writing Python was as easy as GDScript. Ping Akien.") - main_po = main_po[:msg_pos] + " " + location + main_po[msg_pos:] - unique_loc[msg].append(location) - + if msgctx != "": + # If it's a new context or a new message within an existing context, then write new msgid. + # Else add location to existing msgid. + if not msgctx in ctx_group: + _write_message(msgctx, msg, msg_plural, location) + ctx_group[msgctx] = {msg: [location]} + elif not msg in ctx_group[msgctx]: + _write_message(msgctx, msg, msg_plural, location) + ctx_group[msgctx][msg] = [location] + elif not location in ctx_group[msgctx][msg]: + _add_additional_location(msgctx, msg, location) + ctx_group[msgctx][msg].append(location) + else: + if not msg in unique_str: + _write_message(msgctx, msg, msg_plural, location) + unique_str.append(msg) + unique_loc[msg] = [location] + elif not location in unique_loc[msg]: + _add_additional_location(msgctx, msg, location) + unique_loc[msg].append(location) l = f.readline() lc += 1 @@ -102,7 +167,7 @@ def process_file(f, fname): print("Updating the editor.pot template...") for fname in matches: - with open(fname, "r") as f: + with open(fname, "r", encoding="utf8") as f: process_file(f, fname) with open("editor.pot", "w") as f: diff --git a/modules/gdscript/editor/gdscript_translation_parser_plugin.cpp b/modules/gdscript/editor/gdscript_translation_parser_plugin.cpp index 6d454e43f27..944ed859f5a 100644 --- a/modules/gdscript/editor/gdscript_translation_parser_plugin.cpp +++ b/modules/gdscript/editor/gdscript_translation_parser_plugin.cpp @@ -37,9 +37,11 @@ void GDScriptEditorTranslationParserPlugin::get_recognized_extensions(Listget_recognized_extensions(r_extensions); } -Error GDScriptEditorTranslationParserPlugin::parse_file(const String &p_path, Vector *r_extracted_strings) { - // Parse and match all GDScript function API that involves translation string. - // E.g get_node("Label").text = "something", var test = tr("something"), "something" will be matched and collected. +Error GDScriptEditorTranslationParserPlugin::parse_file(const String &p_path, Vector *r_ids, Vector> *r_ids_ctx_plural) { + // Extract all translatable strings using the parsed tree from GDSriptParser. + // The strategy is to find all ExpressionNode and AssignmentNode from the tree and extract strings if relevant, i.e + // Search strings in ExpressionNode -> CallNode -> tr(), set_text(), set_placeholder() etc. + // Search strings in AssignmentNode -> text = "__", hint_tooltip = "__" etc. Error err; RES loaded_res = ResourceLoader::load(p_path, "", false, &err); @@ -48,108 +50,302 @@ Error GDScriptEditorTranslationParserPlugin::parse_file(const String &p_path, Ve return err; } + ids = r_ids; + ids_ctx_plural = r_ids_ctx_plural; Ref gdscript = loaded_res; String source_code = gdscript->get_source_code(); - Vector parsed_strings; - // Search translation strings with RegEx. - regex.clear(); - regex.compile(String("|").join(patterns)); - Array results = regex.search_all(source_code); - _get_captured_strings(results, &parsed_strings); - - // Special handling for FileDialog. - Vector temp; - _parse_file_dialog(source_code, &temp); - parsed_strings.append_array(temp); - - // Filter out / and + - String filter = "(?:\\\\\\n|\"[\\s\\\\]*\\+\\s*\")"; - regex.clear(); - regex.compile(filter); - for (int i = 0; i < parsed_strings.size(); i++) { - parsed_strings.set(i, regex.sub(parsed_strings[i], "", true)); + GDScriptParser parser; + err = parser.parse(source_code, p_path, false); + if (err != OK) { + ERR_PRINT("Failed to parse with GDScript with GDScriptParser."); + return err; } - r_extracted_strings->append_array(parsed_strings); + // Traverse through the parsed tree from GDScriptParser. + GDScriptParser::ClassNode *c = parser.get_tree(); + _traverse_class(c); return OK; } -void GDScriptEditorTranslationParserPlugin::_parse_file_dialog(const String &p_source_code, Vector *r_output) { - // FileDialog API has the form .filters = PackedStringArray(["*.png ; PNG Images","*.gd ; GDScript Files"]). - // First filter: Get "*.png ; PNG Images", "*.gd ; GDScript Files" from PackedStringArray. - regex.clear(); - regex.compile(String("|").join(file_dialog_patterns)); - Array results = regex.search_all(p_source_code); - - Vector temp; - _get_captured_strings(results, &temp); - String captured_strings = String(",").join(temp); - - // Second filter: Get the texts after semicolon from "*.png ; PNG Images","*.gd ; GDScript Files". - String second_filter = "\"[^;]+;" + text + "\""; - regex.clear(); - regex.compile(second_filter); - results = regex.search_all(captured_strings); - _get_captured_strings(results, r_output); - for (int i = 0; i < r_output->size(); i++) { - r_output->set(i, r_output->get(i).strip_edges()); +void GDScriptEditorTranslationParserPlugin::_traverse_class(const GDScriptParser::ClassNode *p_class) { + for (int i = 0; i < p_class->members.size(); i++) { + const GDScriptParser::ClassNode::Member &m = p_class->members[i]; + // There are 7 types of Member, but only class, function and variable can contain translatable strings. + switch (m.type) { + case GDScriptParser::ClassNode::Member::CLASS: + _traverse_class(m.m_class); + break; + case GDScriptParser::ClassNode::Member::FUNCTION: + _traverse_function(m.function); + break; + case GDScriptParser::ClassNode::Member::VARIABLE: + _read_variable(m.variable); + break; + default: + break; + } } } -void GDScriptEditorTranslationParserPlugin::_get_captured_strings(const Array &p_results, Vector *r_output) { - Ref result; - for (int i = 0; i < p_results.size(); i++) { - result = p_results[i]; - for (int j = 0; j < result->get_group_count(); j++) { - String s = result->get_string(j + 1); - // Prevent reading text with only spaces. - if (!s.strip_edges().empty()) { - r_output->push_back(s); +void GDScriptEditorTranslationParserPlugin::_traverse_function(const GDScriptParser::FunctionNode *p_func) { + _traverse_block(p_func->body); +} + +void GDScriptEditorTranslationParserPlugin::_read_variable(const GDScriptParser::VariableNode *p_var) { + _assess_expression(p_var->initializer); +} + +void GDScriptEditorTranslationParserPlugin::_traverse_block(const GDScriptParser::SuiteNode *p_suite) { + if (!p_suite) { + return; + } + + const Vector &statements = p_suite->statements; + for (int i = 0; i < statements.size(); i++) { + GDScriptParser::Node *statement = statements[i]; + + // Statements with Node type constant, break, continue, pass, breakpoint are skipped because they can't contain translatable strings. + switch (statement->type) { + case GDScriptParser::Node::VARIABLE: + _assess_expression(static_cast(statement)->initializer); + break; + case GDScriptParser::Node::IF: { + GDScriptParser::IfNode *if_node = static_cast(statement); + _assess_expression(if_node->condition); + //FIXME : if the elif logic is changed in GDScriptParser, then this probably will have to change as well. See GDScriptParser::TreePrinter::print_if(). + _traverse_block(if_node->true_block); + _traverse_block(if_node->false_block); + break; + } + case GDScriptParser::Node::FOR: { + GDScriptParser::ForNode *for_node = static_cast(statement); + _assess_expression(for_node->list); + _traverse_block(for_node->loop); + break; + } + case GDScriptParser::Node::WHILE: { + GDScriptParser::WhileNode *while_node = static_cast(statement); + _assess_expression(while_node->condition); + _traverse_block(while_node->loop); + break; + } + case GDScriptParser::Node::MATCH: { + GDScriptParser::MatchNode *match_node = static_cast(statement); + _assess_expression(match_node->test); + for (int j = 0; j < match_node->branches.size(); j++) { + _traverse_block(match_node->branches[j]->block); + } + break; + } + case GDScriptParser::Node::RETURN: + _assess_expression(static_cast(statement)->return_value); + break; + case GDScriptParser::Node::ASSERT: + _assess_expression((static_cast(statement))->condition); + break; + case GDScriptParser::Node::ASSIGNMENT: + _assess_assignment(static_cast(statement)); + break; + default: + if (statement->is_expression()) { + _assess_expression(static_cast(statement)); + } + break; + } + } +} + +void GDScriptEditorTranslationParserPlugin::_assess_expression(GDScriptParser::ExpressionNode *p_expression) { + // Explore all ExpressionNodes to find CallNodes which contain translation strings, such as tr(), set_text() etc. + // tr() can be embedded quite deep within multiple ExpressionNodes so need to dig down to search through all ExpressionNodes. + if (!p_expression) { + return; + } + + // ExpressionNode of type await, cast, get_node, identifier, literal, preload, self, subscript, unary are ignored as they can't be CallNode + // containing translation strings. + switch (p_expression->type) { + case GDScriptParser::Node::ARRAY: { + GDScriptParser::ArrayNode *array_node = static_cast(p_expression); + for (int i = 0; i < array_node->elements.size(); i++) { + _assess_expression(array_node->elements[i]); + } + break; + } + case GDScriptParser::Node::ASSIGNMENT: + _assess_assignment(static_cast(p_expression)); + break; + case GDScriptParser::Node::BINARY_OPERATOR: { + GDScriptParser::BinaryOpNode *binary_op_node = static_cast(p_expression); + _assess_expression(binary_op_node->left_operand); + _assess_expression(binary_op_node->right_operand); + break; + } + case GDScriptParser::Node::CALL: { + GDScriptParser::CallNode *call_node = static_cast(p_expression); + _extract_from_call(call_node); + for (int i = 0; i < call_node->arguments.size(); i++) { + _assess_expression(call_node->arguments[i]); + } + } break; + case GDScriptParser::Node::DICTIONARY: { + GDScriptParser::DictionaryNode *dict_node = static_cast(p_expression); + for (int i = 0; i < dict_node->elements.size(); i++) { + _assess_expression(dict_node->elements[i].key); + _assess_expression(dict_node->elements[i].value); + } + break; + } + case GDScriptParser::Node::TERNARY_OPERATOR: { + GDScriptParser::TernaryOpNode *ternary_op_node = static_cast(p_expression); + _assess_expression(ternary_op_node->condition); + _assess_expression(ternary_op_node->true_expr); + _assess_expression(ternary_op_node->false_expr); + break; + } + default: + break; + } +} + +void GDScriptEditorTranslationParserPlugin::_assess_assignment(GDScriptParser::AssignmentNode *p_assignment) { + // Extract the translatable strings coming from assignments. For example, get_node("Label").text = "____" + + StringName assignee_name; + if (p_assignment->assignee->type == GDScriptParser::Node::IDENTIFIER) { + assignee_name = static_cast(p_assignment->assignee)->name; + } else if (p_assignment->assignee->type == GDScriptParser::Node::SUBSCRIPT) { + assignee_name = static_cast(p_assignment->assignee)->attribute->name; + } + + if (assignment_patterns.has(assignee_name) && p_assignment->assigned_value->type == GDScriptParser::Node::LITERAL) { + // If the assignment is towards one of the extract patterns (text, hint_tooltip etc.), and the value is a string literal, we collect the string. + ids->push_back(static_cast(p_assignment->assigned_value)->value); + } else if (assignee_name == fd_filters && p_assignment->assigned_value->type == GDScriptParser::Node::CALL) { + // FileDialog.filters accepts assignment in the form of PackedStringArray. For example, + // get_node("FileDialog").filters = PackedStringArray(["*.png ; PNG Images","*.gd ; GDScript Files"]). + + GDScriptParser::CallNode *call_node = static_cast(p_assignment->assigned_value); + if (call_node->arguments[0]->type == GDScriptParser::Node::ARRAY) { + GDScriptParser::ArrayNode *array_node = static_cast(call_node->arguments[0]); + + // Extract the name in "extension ; name" of PackedStringArray. + for (int i = 0; i < array_node->elements.size(); i++) { + _extract_fd_literals(array_node->elements[i]); + } + } + } else { + // If the assignee is not in extract patterns or the assigned_value is not Literal type, try to see if the assigned_value contains tr(). + _assess_expression(p_assignment->assigned_value); + } +} + +void GDScriptEditorTranslationParserPlugin::_extract_from_call(GDScriptParser::CallNode *p_call) { + // Extract the translatable strings coming from function calls. For example: + // tr("___"), get_node("Label").set_text("____"), get_node("LineEdit").set_placeholder("____"). + + StringName function_name = p_call->function_name; + + // Variables for extracting tr() and tr_n(). + Vector id_ctx_plural; + id_ctx_plural.resize(3); + bool extract_id_ctx_plural = true; + + if (function_name == tr_func) { + // Extract from tr(id, ctx). + for (int i = 0; i < p_call->arguments.size(); i++) { + if (p_call->arguments[i]->type == GDScriptParser::Node::LITERAL) { + id_ctx_plural.write[i] = static_cast(p_call->arguments[i])->value; + } else { + // Avoid adding something like tr("Flying dragon", var_context_level_1). We want to extract both id and context together. + extract_id_ctx_plural = false; + } + } + if (extract_id_ctx_plural) { + ids_ctx_plural->push_back(id_ctx_plural); + } + } else if (function_name == trn_func) { + // Extract from tr_n(id, plural, n, ctx). + Vector indices; + indices.push_back(0); + indices.push_back(3); + indices.push_back(1); + for (int i = 0; i < indices.size(); i++) { + if (indices[i] >= p_call->arguments.size()) { + continue; + } + + if (p_call->arguments[indices[i]]->type == GDScriptParser::Node::LITERAL) { + id_ctx_plural.write[i] = static_cast(p_call->arguments[indices[i]])->value; + } else { + extract_id_ctx_plural = false; + } + } + if (extract_id_ctx_plural) { + ids_ctx_plural->push_back(id_ctx_plural); + } + } else if (first_arg_patterns.has(function_name)) { + // Extracting argument with only string literals. In other words, not extracting something like set_text("hello " + some_var). + if (p_call->arguments[0]->type == GDScriptParser::Node::LITERAL) { + ids->push_back(static_cast(p_call->arguments[0])->value); + } + } else if (second_arg_patterns.has(function_name)) { + if (p_call->arguments[1]->type == GDScriptParser::Node::LITERAL) { + ids->push_back(static_cast(p_call->arguments[1])->value); + } + } else if (function_name == fd_add_filter) { + // Extract the 'JPE Images' in this example - get_node("FileDialog").add_filter("*.jpg; JPE Images"). + _extract_fd_literals(p_call->arguments[0]); + + } else if (function_name == fd_set_filter && p_call->arguments[0]->type == GDScriptParser::Node::CALL) { + // FileDialog.set_filters() accepts assignment in the form of PackedStringArray. For example, + // get_node("FileDialog").set_filters( PackedStringArray(["*.png ; PNG Images","*.gd ; GDScript Files"])). + + GDScriptParser::CallNode *call_node = static_cast(p_call->arguments[0]); + if (call_node->arguments[0]->type == GDScriptParser::Node::ARRAY) { + GDScriptParser::ArrayNode *array_node = static_cast(call_node->arguments[0]); + for (int i = 0; i < array_node->elements.size(); i++) { + _extract_fd_literals(array_node->elements[i]); } } } } -GDScriptEditorTranslationParserPlugin::GDScriptEditorTranslationParserPlugin() { - // Regex search pattern templates. - // The extra complication in the regex pattern is to ensure that the matching works when users write over multiple lines, use tabs etc. - const String dot = "\\.[\\s\\\\]*"; - const String str_assign_template = "[\\s\\\\]*=[\\s\\\\]*\"" + text + "\""; - const String first_arg_template = "[\\s\\\\]*\\([\\s\\\\]*\"" + text + "\"[\\s\\S]*?\\)"; - const String second_arg_template = "[\\s\\\\]*\\([\\s\\S]+?,[\\s\\\\]*\"" + text + "\"[\\s\\S]*?\\)"; +void GDScriptEditorTranslationParserPlugin::_extract_fd_literals(GDScriptParser::ExpressionNode *p_expression) { + // Extract the name in "extension ; name". - // Common patterns. - patterns.push_back("tr" + first_arg_template); - patterns.push_back(dot + "text" + str_assign_template); - patterns.push_back(dot + "placeholder_text" + str_assign_template); - patterns.push_back(dot + "hint_tooltip" + str_assign_template); - patterns.push_back(dot + "set_text" + first_arg_template); - patterns.push_back(dot + "set_tooltip" + first_arg_template); - patterns.push_back(dot + "set_placeholder" + first_arg_template); - - // Tabs and TabContainer API. - patterns.push_back(dot + "set_tab_title" + second_arg_template); - patterns.push_back(dot + "add_tab" + first_arg_template); - - // PopupMenu API. - patterns.push_back(dot + "add_check_item" + first_arg_template); - patterns.push_back(dot + "add_icon_check_item" + second_arg_template); - patterns.push_back(dot + "add_icon_item" + second_arg_template); - patterns.push_back(dot + "add_icon_radio_check_item" + second_arg_template); - patterns.push_back(dot + "add_item" + first_arg_template); - patterns.push_back(dot + "add_multistate_item" + first_arg_template); - patterns.push_back(dot + "add_radio_check_item" + first_arg_template); - patterns.push_back(dot + "add_separator" + first_arg_template); - patterns.push_back(dot + "add_submenu_item" + first_arg_template); - patterns.push_back(dot + "set_item_text" + second_arg_template); - //patterns.push_back(dot + "set_item_tooltip" + second_arg_template); //no tr() behind this function. might be bug. - - // FileDialog API - special case. - const String fd_text = "((?:[\\s\\\\]*\"(?:[^\"\\\\]|\\\\[\\s\\S])*(?:\"[\\s\\\\]*\\+[\\s\\\\]*\"(?:[^\"\\\\]|\\\\[\\s\\S])*)*\"[\\s\\\\]*,?)*)"; - const String packed_string_array = "[\\s\\\\]*PackedStringArray[\\s\\\\]*\\([\\s\\\\]*\\[" + fd_text + "\\][\\s\\\\]*\\)"; - file_dialog_patterns.push_back(dot + "add_filter[\\s\\\\]*\\(" + fd_text + "[\\s\\\\]*\\)"); - file_dialog_patterns.push_back(dot + "filters[\\s\\\\]*=" + packed_string_array); - file_dialog_patterns.push_back(dot + "set_filters[\\s\\\\]*\\(" + packed_string_array + "[\\s\\\\]*\\)"); + if (p_expression->type == GDScriptParser::Node::LITERAL) { + String arg_val = String(static_cast(p_expression)->value); + PackedStringArray arr = arg_val.split(";", true); + if (arr.size() != 2) { + ERR_PRINT("Argument for setting FileDialog has bad format."); + return; + } + ids->push_back(arr[1].strip_edges()); + } +} + +GDScriptEditorTranslationParserPlugin::GDScriptEditorTranslationParserPlugin() { + assignment_patterns.insert("text"); + assignment_patterns.insert("placeholder_text"); + assignment_patterns.insert("hint_tooltip"); + + first_arg_patterns.insert("set_text"); + first_arg_patterns.insert("set_tooltip"); + first_arg_patterns.insert("set_placeholder"); + first_arg_patterns.insert("add_tab"); + first_arg_patterns.insert("add_check_item"); + first_arg_patterns.insert("add_item"); + first_arg_patterns.insert("add_multistate_item"); + first_arg_patterns.insert("add_radio_check_item"); + first_arg_patterns.insert("add_separator"); + first_arg_patterns.insert("add_submenu_item"); + + second_arg_patterns.insert("set_tab_title"); + second_arg_patterns.insert("add_icon_check_item"); + second_arg_patterns.insert("add_icon_item"); + second_arg_patterns.insert("add_icon_radio_check_item"); + second_arg_patterns.insert("set_item_text"); } diff --git a/modules/gdscript/editor/gdscript_translation_parser_plugin.h b/modules/gdscript/editor/gdscript_translation_parser_plugin.h index 9fa4b69f019..5ea416d4cc8 100644 --- a/modules/gdscript/editor/gdscript_translation_parser_plugin.h +++ b/modules/gdscript/editor/gdscript_translation_parser_plugin.h @@ -31,23 +31,40 @@ #ifndef GDSCRIPT_TRANSLATION_PARSER_PLUGIN_H #define GDSCRIPT_TRANSLATION_PARSER_PLUGIN_H +#include "core/set.h" #include "editor/editor_translation_parser.h" +#include "modules/gdscript/gdscript_parser.h" #include "modules/regex/regex.h" class GDScriptEditorTranslationParserPlugin : public EditorTranslationParserPlugin { GDCLASS(GDScriptEditorTranslationParserPlugin, EditorTranslationParserPlugin); - // Regex and search patterns that are used to match translation strings. - const String text = "((?:[^\"\\\\]|\\\\[\\s\\S])*(?:\"[\\s\\\\]*\\+[\\s\\\\]*\"(?:[^\"\\\\]|\\\\[\\s\\S])*)*)"; - RegEx regex; - Vector patterns; - Vector file_dialog_patterns; + Vector *ids; + Vector> *ids_ctx_plural; - void _parse_file_dialog(const String &p_source_code, Vector *r_output); - void _get_captured_strings(const Array &p_results, Vector *r_output); + // List of patterns used for extracting translation strings. + StringName tr_func = "tr"; + StringName trn_func = "tr_n"; + Set assignment_patterns; + Set first_arg_patterns; + Set second_arg_patterns; + // FileDialog patterns. + StringName fd_add_filter = "add_filter"; + StringName fd_set_filter = "set_filters"; + StringName fd_filters = "filters"; + + void _traverse_class(const GDScriptParser::ClassNode *p_class); + void _traverse_function(const GDScriptParser::FunctionNode *p_func); + void _traverse_block(const GDScriptParser::SuiteNode *p_suite); + + void _read_variable(const GDScriptParser::VariableNode *p_var); + void _assess_expression(GDScriptParser::ExpressionNode *p_expression); + void _assess_assignment(GDScriptParser::AssignmentNode *p_assignment); + void _extract_from_call(GDScriptParser::CallNode *p_call); + void _extract_fd_literals(GDScriptParser::ExpressionNode *p_expression); public: - virtual Error parse_file(const String &p_path, Vector *r_extracted_strings) override; + virtual Error parse_file(const String &p_path, Vector *r_ids, Vector> *r_ids_ctx_plural) override; virtual void get_recognized_extensions(List *r_extensions) const override; GDScriptEditorTranslationParserPlugin();