Skip to content

Commit

Permalink
Port4ed from godot: Add binary MO translation file support. Add brotl…
Browse files Browse the repository at this point in the history 
…i decoder and WOFF2 support.

Use smaller .mo files instead of .po, if gettext is available.
Convert editor fonts to .woff2 format.
- bruvzg
godotengine/godot@fd2fba7
  • Loading branch information
@Relintai
Relintai committed Oct 2, 2023
1 parent 7df893a commit b4692f1
Show file tree
Hide file tree
Showing 53 changed files with 14,778 additions and 132 deletions.
320 changes: 201 additions & 119 deletions core/io/translation_loader_po.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,168 +34,249 @@
#include "core/string/translation.h"

RES TranslationLoaderPO::load_translation(FileAccess *f, bool p_use_context, Error *r_error) {
enum Status {
STATUS_NONE,
STATUS_READING_ID,
STATUS_READING_STRING,
STATUS_READING_CONTEXT,
};

Status status = STATUS_NONE;

String msg_id;
String msg_str;
String msg_context;
String config;

if (r_error) {
*r_error = ERR_FILE_CORRUPT;
}

const String path = f->get_path();

Ref<Translation> translation;
if (p_use_context) {
translation = Ref<Translation>(memnew(ContextTranslation));
} else {
translation.instance();
}

int line = 1;
bool entered_context = false;
bool skip_this = false;
bool skip_next = false;
bool is_eof = false;
const String path = f->get_path();
String config;

while (!is_eof) {
String l = f->get_line().strip_edges();
is_eof = f->eof_reached();
uint32_t magic = f->get_32();
if (magic == 0x950412de) {
// Load binary MO file.

// If we reached last line and it's not a content line, break, otherwise let processing that last loop
if (is_eof && l.empty()) {
if (status == STATUS_READING_ID || status == STATUS_READING_CONTEXT) {
memdelete(f);
ERR_FAIL_V_MSG(RES(), "Unexpected EOF while reading PO file at: " + path + ":" + itos(line));
} else {
break;
}
uint16_t version_maj = f->get_16();
uint16_t version_min = f->get_16();
if (version_maj > 1) {
ERR_FAIL_V_MSG(RES(), vformat("Unsupported MO file %s, version %d.%d.", path, version_maj, version_min));
}

if (l.begins_with("msgctxt")) {
if (status != STATUS_READING_STRING) {
memdelete(f);
ERR_FAIL_V_MSG(RES(), "Unexpected 'msgctxt', was expecting 'msgstr' before 'msgctxt' while parsing: " + path + ":" + itos(line));
uint32_t num_strings = f->get_32();
uint32_t id_table_offset = f->get_32();
uint32_t trans_table_offset = f->get_32();

// Read string tables.
for (uint32_t i = 0; i < num_strings; i++) {
String msg_id;
String msg_context;

// Read id strings and context.
{
Vector<uint8_t> data;
f->seek(id_table_offset + i * 8);
uint32_t str_start = 0;
uint32_t str_len = f->get_32();
uint32_t str_offset = f->get_32();

data.resize(str_len + 1);
f->seek(str_offset);
f->get_buffer(data.ptrw(), str_len);
data.write[str_len] = 0;

for (uint32_t j = 0; j < str_len + 1; j++) {
if (data[j] == 0x04) {
msg_context.parse_utf8((const char *)data.ptr(), j);
str_start = j + 1;
}
if (data[j] == 0x00) {
msg_id.parse_utf8((const char *)(data.ptr() + str_start), j - str_start);
break;
}
}
}

// In PO file, "msgctxt" appears before "msgid". If we encounter a "msgctxt", we add what we have read
// and set "entered_context" to true to prevent adding twice.
if (!skip_this && msg_id != "") {
translation->add_context_message(msg_id, msg_str, msg_context);
// Read translated strings.
{
Vector<uint8_t> data;
f->seek(trans_table_offset + i * 8);
uint32_t str_len = f->get_32();
uint32_t str_offset = f->get_32();

data.resize(str_len + 1);
f->seek(str_offset);
f->get_buffer(data.ptrw(), str_len);
data.write[str_len] = 0;

if (msg_id.empty()) {
config = String::utf8((const char *)data.ptr(), str_len);
} else {
for (uint32_t j = 0; j < str_len + 1; j++) {
if (data[j] == 0x00) {
translation->add_context_message(msg_id, String::utf8((const char *)data.ptr(), j), msg_context);
break;
}
}
}
}
msg_context = "";
l = l.substr(7, l.length()).strip_edges();
status = STATUS_READING_CONTEXT;
entered_context = true;
}

if (l.begins_with("msgid")) {
if (status == STATUS_READING_ID) {
memdelete(f);
ERR_FAIL_V_MSG(RES(), "Unexpected 'msgid', was expecting 'msgstr' while parsing: " + path + ":" + itos(line));
}
memdelete(f);
} else {
// Try to load as text PO file.
f->seek(0);

if (msg_id != "") {
if (!skip_this && !entered_context) {
translation->add_context_message(msg_id, msg_str, msg_context);
enum Status {
STATUS_NONE,
STATUS_READING_ID,
STATUS_READING_STRING,
STATUS_READING_CONTEXT,
};

Status status = STATUS_NONE;

String msg_id;
String msg_str;
String msg_context;

if (r_error) {
*r_error = ERR_FILE_CORRUPT;
}

int line = 1;
bool entered_context = false;
bool skip_this = false;
bool skip_next = false;
bool is_eof = false;

while (!is_eof) {
String l = f->get_line().strip_edges();
is_eof = f->eof_reached();

// If we reached last line and it's not a content line, break, otherwise let processing that last loop
if (is_eof && l.empty()) {
if (status == STATUS_READING_ID || status == STATUS_READING_CONTEXT) {
memdelete(f);
ERR_FAIL_V_MSG(RES(), "Unexpected EOF while reading PO file at: " + path + ":" + itos(line));
} else {
break;
}
} else if (config == "") {
config = msg_str;
}

l = l.substr(5, l.length()).strip_edges();
status = STATUS_READING_ID;
// If we did not encounter msgctxt, we reset context to empty to reset it.
if (!entered_context) {
if (l.begins_with("msgctxt")) {
if (status != STATUS_READING_STRING) {
memdelete(f);
ERR_FAIL_V_MSG(RES(), "Unexpected 'msgctxt', was expecting 'msgstr' before 'msgctxt' while parsing: " + path + ":" + itos(line));
}

// In PO file, "msgctxt" appears before "msgid". If we encounter a "msgctxt", we add what we have read
// and set "entered_context" to true to prevent adding twice.
if (!skip_this && msg_id != "") {
translation->add_context_message(msg_id, msg_str, msg_context);
}
msg_context = "";
l = l.substr(7, l.length()).strip_edges();
status = STATUS_READING_CONTEXT;
entered_context = true;
}
msg_id = "";
msg_str = "";
skip_this = skip_next;
skip_next = false;
entered_context = false;
}

if (l.begins_with("msgstr")) {
if (status != STATUS_READING_ID) {
memdelete(f);
ERR_FAIL_V_MSG(RES(), "Unexpected 'msgstr', was expecting 'msgid' before 'msgstr' while parsing: " + path + ":" + itos(line));
if (l.begins_with("msgid")) {
if (status == STATUS_READING_ID) {
memdelete(f);
ERR_FAIL_V_MSG(RES(), "Unexpected 'msgid', was expecting 'msgstr' while parsing: " + path + ":" + itos(line));
}

if (msg_id != "") {
if (!skip_this && !entered_context) {
translation->add_context_message(msg_id, msg_str, msg_context);
}
} else if (config == "") {
config = msg_str;
}

l = l.substr(5, l.length()).strip_edges();
status = STATUS_READING_ID;
// If we did not encounter msgctxt, we reset context to empty to reset it.
if (!entered_context) {
msg_context = "";
}
msg_id = "";
msg_str = "";
skip_this = skip_next;
skip_next = false;
entered_context = false;
}

l = l.substr(6, l.length()).strip_edges();
status = STATUS_READING_STRING;
}
if (l.begins_with("msgstr")) {
if (status != STATUS_READING_ID) {
memdelete(f);
ERR_FAIL_V_MSG(RES(), "Unexpected 'msgstr', was expecting 'msgid' before 'msgstr' while parsing: " + path + ":" + itos(line));
}

if (l == "" || l.begins_with("#")) {
if (l.find("fuzzy") != -1) {
skip_next = true;
l = l.substr(6, l.length()).strip_edges();
status = STATUS_READING_STRING;
}
line++;
continue; // Nothing to read or comment.
}

if (!l.begins_with("\"") || status == STATUS_NONE) {
memdelete(f);
ERR_FAIL_V_MSG(RES(), "Invalid line '" + l + "' while parsing: " + path + ":" + itos(line));
}
if (l == "" || l.begins_with("#")) {
if (l.find("fuzzy") != -1) {
skip_next = true;
}
line++;
continue; // Nothing to read or comment.
}

l = l.substr(1, l.length());
// Find final quote, ignoring escaped ones (\").
// The escape_next logic is necessary to properly parse things like \\"
// where the blackslash is the one being escaped, not the quote.
int end_pos = -1;
bool escape_next = false;
for (int i = 0; i < l.length(); i++) {
if (l[i] == '\\' && !escape_next) {
escape_next = true;
continue;
if (!l.begins_with("\"") || status == STATUS_NONE) {
memdelete(f);
ERR_FAIL_V_MSG(RES(), "Invalid line '" + l + "' while parsing: " + path + ":" + itos(line));
}

if (l[i] == '"' && !escape_next) {
end_pos = i;
break;
l = l.substr(1, l.length());
// Find final quote, ignoring escaped ones (\").
// The escape_next logic is necessary to properly parse things like \\"
// where the blackslash is the one being escaped, not the quote.
int end_pos = -1;
bool escape_next = false;
for (int i = 0; i < l.length(); i++) {
if (l[i] == '\\' && !escape_next) {
escape_next = true;
continue;
}

if (l[i] == '"' && !escape_next) {
end_pos = i;
break;
}

escape_next = false;
}

escape_next = false;
}
if (end_pos == -1) {
memdelete(f);
ERR_FAIL_V_MSG(RES(), "Expected '\"' at end of message while parsing: " + path + ":" + itos(line));
}

if (end_pos == -1) {
memdelete(f);
ERR_FAIL_V_MSG(RES(), "Expected '\"' at end of message while parsing: " + path + ":" + itos(line));
}
l = l.substr(0, end_pos);
l = l.c_unescape();

l = l.substr(0, end_pos);
l = l.c_unescape();
if (status == STATUS_READING_ID) {
msg_id += l;
} else if (status == STATUS_READING_STRING) {
msg_str += l;
} else if (status == STATUS_READING_CONTEXT) {
msg_context += l;
}

if (status == STATUS_READING_ID) {
msg_id += l;
} else if (status == STATUS_READING_STRING) {
msg_str += l;
} else if (status == STATUS_READING_CONTEXT) {
msg_context += l;
line++;
}

line++;
}

memdelete(f);
memdelete(f);

// Add the last set of data from last iteration.
if (status == STATUS_READING_STRING) {
if (msg_id != "") {
if (!skip_this) {
translation->add_context_message(msg_id, msg_str, msg_context);
// Add the last set of data from last iteration.
if (status == STATUS_READING_STRING) {
if (msg_id != "") {
if (!skip_this) {
translation->add_context_message(msg_id, msg_str, msg_context);
}
} else if (config == "") {
config = msg_str;
}
} else if (config == "") {
config = msg_str;
}
}

Expand Down Expand Up @@ -236,13 +317,14 @@ RES TranslationLoaderPO::load(const String &p_path, const String &p_original_pat

void TranslationLoaderPO::get_recognized_extensions(List<String> *p_extensions) const {
p_extensions->push_back("po");
p_extensions->push_back("mo");
}
bool TranslationLoaderPO::handles_type(const String &p_type) const {
return (p_type == "Translation");
}

String TranslationLoaderPO::get_resource_type(const String &p_path) const {
if (p_path.get_extension().to_lower() == "po") {
if (p_path.get_extension().to_lower() == "po" || p_path.get_extension().to_lower() == "mo") {
return "Translation";
}
return "";
Expand Down
2 changes: 1 addition & 1 deletion doc/classes/DynamicFont.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
</brief_description>
<description>
DynamicFont renders vector font files dynamically at runtime instead of using a prerendered texture atlas like [BitmapFont]. This trades the faster loading time of [BitmapFont]s for the ability to change font parameters like size and spacing during runtime. [DynamicFontData] is used for referencing the font file paths. DynamicFont also supports defining one or more fallback fonts, which will be used when displaying a character not supported by the main font.
DynamicFont uses the [url=https://www.freetype.org/]FreeType[/url] library for rasterization. Supported formats are TrueType ([code].ttf[/code]), OpenType ([code].otf[/code]) and Web Open Font Format 1 ([code].woff[/code]). Web Open Font Format 2 ([code].woff2[/code]) is [i]not[/i] supported.
DynamicFont uses the [url=https://www.freetype.org/]FreeType[/url] library for rasterization. Supported formats are TrueType ([code].ttf[/code]), OpenType ([code].otf[/code]), Web Open Font Format 1 ([code].woff[/code]), and Web Open Font Format 2 ([code].woff2[/code]).
[codeblock]
var dynamic_font = DynamicFont.new()
dynamic_font.font_data = load("res://BarlowCondensed-Bold.ttf")
Expand Down
Loading

0 comments on commit b4692f1

Please sign in to comment.