From c6264ee41aef655acf4d1f540fbf7b8867e63fdf Mon Sep 17 00:00:00 2001 From: Evgencheg <73418250+Evgencheg@users.noreply.github.com> Date: Tue, 15 Oct 2024 18:41:27 +0300 Subject: [PATCH] Fix locale scripts (#2648) Co-authored-by: mhamster <81412348+mhamsterr@users.noreply.github.com> Co-authored-by: Morb0 <14136326+Morb0@users.noreply.github.com> --- Tools/ss14_ru/__init__.py | 2 +- Tools/ss14_ru/clean_duplicates.py | 120 ++++++++++++++++++ Tools/ss14_ru/clean_empty.py | 61 +++++++++ Tools/ss14_ru/file.py | 43 ++++++- Tools/ss14_ru/fluentast.py | 2 +- Tools/ss14_ru/fluentastcomparer.py | 2 +- Tools/ss14_ru/fluentastmanager.py | 2 +- Tools/ss14_ru/fluentformatter.py | 2 +- Tools/ss14_ru/keyfinder.py | 2 +- .../lokalise_fluent_ast_comparer_manager.py | 2 +- Tools/ss14_ru/lokalise_project.py | 2 +- Tools/ss14_ru/lokalisemodels.py | 2 +- Tools/ss14_ru/project.py | 2 +- Tools/ss14_ru/requirements.txt | Bin 142 -> 79 bytes Tools/ss14_ru/translation.bat | 9 ++ Tools/ss14_ru/translation.sh | 12 ++ Tools/ss14_ru/translationsassembler.py | 2 +- Tools/ss14_ru/yamlextractor.py | 22 +++- Tools/ss14_ru/yamlmodels.py | 2 +- 19 files changed, 271 insertions(+), 20 deletions(-) create mode 100644 Tools/ss14_ru/clean_duplicates.py create mode 100644 Tools/ss14_ru/clean_empty.py create mode 100644 Tools/ss14_ru/translation.bat create mode 100644 Tools/ss14_ru/translation.sh diff --git a/Tools/ss14_ru/__init__.py b/Tools/ss14_ru/__init__.py index 8fc809dcf53..c7fe81101f3 100644 --- a/Tools/ss14_ru/__init__.py +++ b/Tools/ss14_ru/__init__.py @@ -1 +1 @@ -from fluentformatter import FluentFile, FluentFormatter +from fluentformatter import FluentFile, FluentFormatter diff --git a/Tools/ss14_ru/clean_duplicates.py b/Tools/ss14_ru/clean_duplicates.py new file mode 100644 index 00000000000..b43909e8a3b --- /dev/null +++ b/Tools/ss14_ru/clean_duplicates.py @@ -0,0 +1,120 @@ +import os +import re +import chardet +from datetime import datetime + +def find_top_level_dir(start_dir): + marker_file = 'SpaceStation14.sln' + current_dir = start_dir + while True: + if marker_file in os.listdir(current_dir): + return current_dir + parent_dir = os.path.dirname(current_dir) + if parent_dir == current_dir: + print(f"Не удалось найти {marker_file} начиная с {start_dir}") + exit(-1) + current_dir = parent_dir + +def find_ftl_files(root_dir): + ftl_files = [] + for root, dirs, files in os.walk(root_dir): + for file in files: + if file.endswith('.ftl'): + ftl_files.append(os.path.join(root, file)) + return ftl_files + +def detect_encoding(file_path): + with open(file_path, 'rb') as file: + raw_data = file.read() + return chardet.detect(raw_data)['encoding'] + +def parse_ent_blocks(file_path): + try: + encoding = detect_encoding(file_path) + with open(file_path, 'r', encoding=encoding) as file: + content = file.read() + except UnicodeDecodeError: + print(f"Ошибка при чтении файла {file_path}. Попытка чтения в UTF-8.") + try: + with open(file_path, 'r', encoding='utf-8') as file: + content = file.read() + except UnicodeDecodeError: + print(f"Не удалось прочитать файл {file_path}. Пропускаем.") + return {} + + ent_blocks = {} + current_ent = None + current_block = [] + + for line in content.split('\n'): + if line.startswith('ent-'): + if current_ent: + ent_blocks[current_ent] = '\n'.join(current_block) + current_ent = line.split('=')[0].strip() + current_block = [line] + elif current_ent and (line.strip().startswith('.desc') or line.strip().startswith('.suffix')): + current_block.append(line) + elif line.strip() == '': + if current_ent: + ent_blocks[current_ent] = '\n'.join(current_block) + current_ent = None + current_block = [] + else: + if current_ent: + ent_blocks[current_ent] = '\n'.join(current_block) + current_ent = None + current_block = [] + + if current_ent: + ent_blocks[current_ent] = '\n'.join(current_block) + + return ent_blocks + +def remove_duplicates(root_dir): + ftl_files = find_ftl_files(root_dir) + all_ents = {} + removed_duplicates = [] + + for file_path in ftl_files: + ent_blocks = parse_ent_blocks(file_path) + for ent, block in ent_blocks.items(): + if ent not in all_ents: + all_ents[ent] = (file_path, block) + + for file_path in ftl_files: + try: + encoding = detect_encoding(file_path) + with open(file_path, 'r', encoding=encoding) as file: + content = file.read() + + ent_blocks = parse_ent_blocks(file_path) + for ent, block in ent_blocks.items(): + if all_ents[ent][0] != file_path: + content = content.replace(block, '') + removed_duplicates.append((ent, file_path, block)) + + content = re.sub(r'\n{3,}', '\n\n', content) + + with open(file_path, 'w', encoding=encoding) as file: + file.write(content) + except Exception as e: + print(f"Ошибка при обработке файла {file_path}: {str(e)}") + + # Сохранение лога удаленных дубликатов + log_filename = f"removed_duplicates_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" + with open(log_filename, 'w', encoding='utf-8') as log_file: + for ent, file_path, block in removed_duplicates: + log_file.write(f"Удален дубликат: {ent}\n") + log_file.write(f"Файл: {file_path}\n") + log_file.write("Содержимое:\n") + log_file.write(block) + log_file.write("\n\n") + + print(f"Обработка завершена. Проверено файлов: {len(ftl_files)}") + print(f"Лог удаленных дубликатов сохранен в файл: {log_filename}") + +if __name__ == "__main__": + script_dir = os.path.dirname(os.path.abspath(__file__)) + main_folder = find_top_level_dir(script_dir) + root_dir = os.path.join(main_folder, "Resources\\Locale\\ru-RU") + remove_duplicates(root_dir) diff --git a/Tools/ss14_ru/clean_empty.py b/Tools/ss14_ru/clean_empty.py new file mode 100644 index 00000000000..d6f2efd7c30 --- /dev/null +++ b/Tools/ss14_ru/clean_empty.py @@ -0,0 +1,61 @@ +import os +import logging +from datetime import datetime + +def find_top_level_dir(start_dir): + marker_file = 'SpaceStation14.sln' + current_dir = start_dir + while True: + if marker_file in os.listdir(current_dir): + return current_dir + parent_dir = os.path.dirname(current_dir) + if parent_dir == current_dir: + print(f"Не удалось найти {marker_file} начиная с {start_dir}") + exit(-1) + current_dir = parent_dir +def setup_logging(): + log_filename = f"cleanup_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" + logging.basicConfig(filename=log_filename, level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s') + console = logging.StreamHandler() + console.setLevel(logging.INFO) + logging.getLogger('').addHandler(console) + return log_filename + +def remove_empty_files_and_folders(path): + removed_files = 0 + removed_folders = 0 + + for root, dirs, files in os.walk(path, topdown=False): + # Удаление пустых файлов + for file in files: + file_path = os.path.join(root, file) + if os.path.getsize(file_path) == 0: + try: + os.remove(file_path) + logging.info(f"Удален пустой файл: {file_path}") + removed_files += 1 + except Exception as e: + logging.error(f"Ошибка при удалении файла {file_path}: {str(e)}") + + # Удаление пустых папок + if not os.listdir(root): + try: + os.rmdir(root) + logging.info(f"Удалена пустая папка: {root}") + removed_folders += 1 + except Exception as e: + logging.error(f"Ошибка при удалении папки {root}: {str(e)}") + + return removed_files, removed_folders + +if __name__ == "__main__": + script_dir = os.path.dirname(os.path.abspath(__file__)) + main_folder = find_top_level_dir(script_dir) + root_dir = os.path.join(main_folder, "Resources\\Locale") + log_file = setup_logging() + + logging.info(f"Начало очистки в директории: {root_dir}") + files_removed, folders_removed = remove_empty_files_and_folders(root_dir) + logging.info(f"Очистка завершена. Удалено файлов: {files_removed}, удалено папок: {folders_removed}") + print(f"Лог операций сохранен в файл: {log_file}") diff --git a/Tools/ss14_ru/file.py b/Tools/ss14_ru/file.py index bbb53c79b1c..abaf8e1d730 100644 --- a/Tools/ss14_ru/file.py +++ b/Tools/ss14_ru/file.py @@ -1,8 +1,9 @@ -import typing +import typing from fluent.syntax import ast from yamlmodels import YAMLElements import os +import re class File: @@ -42,17 +43,53 @@ def get_name(self): class FluentFile(File): def __init__(self, full_path): super().__init__(full_path) + + self.newline_exceptions_regex = re.compile(r"^\s*[\[\]{}#%^*]") + self.newline_remover_tag = "%ERASE_NEWLINE%" + self.newline_remover_regex = re.compile(r"\n?\s*" + self.newline_remover_tag) + + "%ERASE_NEWLINE%" + self.full_path = full_path + def kludge(self, element): + return str.replace( + element.value, + self.prefixed_newline, + self.prefixed_newline_substitute + ) + + def parse_data(self, file_data: typing.AnyStr): from fluent.syntax import FluentParser - return FluentParser().parse(file_data) + parsed_data = FluentParser().parse(file_data) + + for body_element in parsed_data.body: + if not isinstance(body_element, ast.Term) and not isinstance(body_element, ast.Message): + continue + + if not len(body_element.value.elements): + continue + + first_element = body_element.value.elements[0] + if not isinstance(first_element, ast.TextElement): + continue + + if not self.newline_exceptions_regex.match(first_element.value): + continue + + first_element.value = f"{self.newline_remover_tag}{first_element.value}" + + return parsed_data def serialize_data(self, parsed_file_data: ast.Resource): from fluent.syntax import FluentSerializer - return FluentSerializer(with_junk=True).serialize(parsed_file_data) + serialized_data = FluentSerializer(with_junk=True).serialize(parsed_file_data) + serialized_data = self.newline_remover_regex.sub(' ', serialized_data) + + return serialized_data def read_serialized_data(self): return self.serialize_data(self.parse_data(self.read_data())) diff --git a/Tools/ss14_ru/fluentast.py b/Tools/ss14_ru/fluentast.py index 20576b47051..ac2597f7667 100644 --- a/Tools/ss14_ru/fluentast.py +++ b/Tools/ss14_ru/fluentast.py @@ -1,4 +1,4 @@ -import typing +import typing from fluent.syntax import ast, FluentParser, FluentSerializer from lokalisemodels import LokaliseKey diff --git a/Tools/ss14_ru/fluentastcomparer.py b/Tools/ss14_ru/fluentastcomparer.py index 21fa9701757..0b4c49b3417 100644 --- a/Tools/ss14_ru/fluentastcomparer.py +++ b/Tools/ss14_ru/fluentastcomparer.py @@ -1,4 +1,4 @@ -from fluent.syntax import ast +from fluent.syntax import ast from fluentast import FluentAstAbstract from pydash import py_ diff --git a/Tools/ss14_ru/fluentastmanager.py b/Tools/ss14_ru/fluentastmanager.py index 76cf4f7bcdf..8ae13154d1e 100644 --- a/Tools/ss14_ru/fluentastmanager.py +++ b/Tools/ss14_ru/fluentastmanager.py @@ -1,4 +1,4 @@ -from fluent.syntax import ast +from fluent.syntax import ast from fluentast import FluentAstAbstract diff --git a/Tools/ss14_ru/fluentformatter.py b/Tools/ss14_ru/fluentformatter.py index f7a96dbb6c9..dd043b0e5df 100644 --- a/Tools/ss14_ru/fluentformatter.py +++ b/Tools/ss14_ru/fluentformatter.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3 # Форматтер, приводящий fluent-файлы (.ftl) в соответствие стайлгайду # path - путь к папке, содержащий форматируемые файлы. Для форматирования всего проекта, необходимо заменить значение на root_dir_path diff --git a/Tools/ss14_ru/keyfinder.py b/Tools/ss14_ru/keyfinder.py index b06d4282868..317cec76dc3 100644 --- a/Tools/ss14_ru/keyfinder.py +++ b/Tools/ss14_ru/keyfinder.py @@ -1,4 +1,4 @@ -import typing +import typing import logging from pydash import py_ diff --git a/Tools/ss14_ru/lokalise_fluent_ast_comparer_manager.py b/Tools/ss14_ru/lokalise_fluent_ast_comparer_manager.py index cdbc84d5bd8..86e39aba45c 100644 --- a/Tools/ss14_ru/lokalise_fluent_ast_comparer_manager.py +++ b/Tools/ss14_ru/lokalise_fluent_ast_comparer_manager.py @@ -1,4 +1,4 @@ -from fluent.syntax import ast +from fluent.syntax import ast from fluentast import FluentAstMessage from fluentastcomparer import FluentAstComparer diff --git a/Tools/ss14_ru/lokalise_project.py b/Tools/ss14_ru/lokalise_project.py index 05a4f4d285e..bed995793a1 100644 --- a/Tools/ss14_ru/lokalise_project.py +++ b/Tools/ss14_ru/lokalise_project.py @@ -1,4 +1,4 @@ -import lokalise +import lokalise import typing from lokalisemodels import LokaliseKey from pydash import py_ diff --git a/Tools/ss14_ru/lokalisemodels.py b/Tools/ss14_ru/lokalisemodels.py index 98e238cc18c..44846cd94ac 100644 --- a/Tools/ss14_ru/lokalisemodels.py +++ b/Tools/ss14_ru/lokalisemodels.py @@ -1,4 +1,4 @@ -import typing +import typing import os from pydash import py_ from project import Project diff --git a/Tools/ss14_ru/project.py b/Tools/ss14_ru/project.py index 7fcc9c7c329..0b4f67f4abf 100644 --- a/Tools/ss14_ru/project.py +++ b/Tools/ss14_ru/project.py @@ -1,4 +1,4 @@ -import pathlib +import pathlib import os import glob from file import FluentFile diff --git a/Tools/ss14_ru/requirements.txt b/Tools/ss14_ru/requirements.txt index c68858f01804994cdda64a0739e37adf7324799a..dae3410246f9f370b3899a1cb6d8604a5171f6d4 100644 GIT binary patch literal 79 zcmaFAdw*I^X=+}Ht*wEcp@E(OS3zY;VsVD8t%V*?fGeOf($Uw)*47NnE2%8V%uA0? dttd&&E6&W%E4H;Y(X#|