Improvements to GDScript identifier tokenization

This commit is contained in:
Ninni Pipping 2023-02-13 10:32:56 +01:00
parent e2e870c611
commit f68beeb7fa

View File

@ -559,6 +559,24 @@ GDScriptTokenizer::Token GDScriptTokenizer::potential_identifier() {
return make_identifier(name);
}
if (!only_ascii) {
// Kept here in case the order with push_error matters.
Token id = make_identifier(name);
#ifdef DEBUG_ENABLED
// Additional checks for identifiers but only in debug and if it's available in TextServer.
if (TS->has_feature(TextServer::FEATURE_UNICODE_SECURITY)) {
int64_t confusable = TS->is_confusable(name, keyword_list);
if (confusable >= 0) {
push_error(vformat(R"(Identifier "%s" is visually similar to the GDScript keyword "%s" and thus not allowed.)", name, keyword_list[confusable]));
}
}
#endif // DEBUG_ENABLED
// Cannot be a keyword, as keywords are ASCII only.
return id;
}
// Define some helper macros for the switch case.
#define KEYWORD_GROUP_CASE(char) \
break; \
@ -594,19 +612,7 @@ GDScriptTokenizer::Token GDScriptTokenizer::potential_identifier() {
}
// Not a keyword, so must be an identifier.
Token id = make_identifier(name);
#ifdef DEBUG_ENABLED
// Additional checks for identifiers but only in debug and if it's available in TextServer.
if (!only_ascii && TS->has_feature(TextServer::FEATURE_UNICODE_SECURITY)) {
int64_t confusable = TS->is_confusable(name, keyword_list);
if (confusable >= 0) {
push_error(vformat(R"(Identifier "%s" is visually similar to the GDScript keyword "%s" and thus not allowed.)", name, keyword_list[confusable]));
}
}
#endif // DEBUG_ENABLED
return id;
return make_identifier(name);
#undef KEYWORD_GROUP_CASE
#undef KEYWORD