Mixed encodings support (v0.1.5)

* implemented support for MTF files that contain `UTF-8` and `CP-1252` characters (fixes #6)
* the `--ignore-errors` option now ignores **all** types of exceptions and prints conversion statistics
* current Biped conversion rate increased to 99%

README.md

@@ -17,18 +17,24 @@ record sheet (e.g. the structure pips are missing).
 `mtf2json` does not simply create a 1:1 JSON version of the MTF data (that wouldn't be possible anyway) but restructures the data and
 adds information that is required for creating record sheets (see examples below).
 
-## Limitations and Supported Versions
+## Current State
+### Conversion Rates
 
-### Quad Mechs
-Currently, `mtf2json` can only convert MTF files of **biped** mechs.
+| Chassis Type | Conversion Rate |
+|--------------|---------------|
+| Biped | 99% (3945 / 3954) |
+| Quad | Not supported |
+| Tripod | Not supported |
+| LAM | Not supported |
 
-### Supported MegaMek Version
+### Latest Supported MegaMekLab Version
 `0.49.19.1`
 
 ### Testing
 Testing `mtf2json` is challenging, mostly because the MTF format is so loosely specified. I'm still not sure if I've actually seen all
-possible keys and understood all supported value syntaxes. Since there are over 4000 MTF files in MegaMek, I can't test and manually
-verify them all. If you have trouble converting an MTF file, create a Github issue and append the file, so I can verify and fix the issue.
+possible keys and understood all supported value syntaxes. Since there are over 4000 MTF files in MegaMek, I can't manually verify all
+conversion results. If you have trouble converting an MTF file, create a Github issue and append the file, so I can verify and fix the
+issue.
 
 ## JSON Structure and Examples
 
@@ -177,7 +183,7 @@ JSON:
 },
 ```
 Armor type, tech base (if available) and pips are stored in the `armor` section. In case of patchwork armor, each location will have a `type` key:
-```
+```json
 "armor": {
     "type": "Patchwork",
     "left_arm": {
@@ -392,9 +398,9 @@ To convert an MTF file to JSON, use the following command:
 mtf2json --mtf-file <path_to_mtf_file> [--convert] [--json-file <path_to_json_file]
 ```
 
-To query JSON data in the terminal (e.g. armor data), pipe the output into `jq`:
+To query JSON data in the terminal (e.g. armor pips of left arm), pipe the output into `jq`:
 ```sh
-mtf2json --mtf-file <path_to_mtf_file> | jq .armor
+mtf2json --mtf-file <path_to_mtf_file> | jq .armor.left_arm.pips
 ```
 
 ### Library

mtf2json/cli.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 import os
 from .mtf2json import read_mtf, write_json, ConversionError, version, mm_version
-from typing import Optional
+from typing import Optional, List, Tuple
 
 
 def create_parser() -> argparse.ArgumentParser:
@@ -41,7 +41,7 @@ def create_parser() -> argparse.ArgumentParser:
                         help="Recursively convert MTF files in subdirectories.")
     parser.add_argument('--ignore-errors', '-i',
                         action='store_true',
-                        help="Continue converting files even if an error occurs.")
+                        help="Ignore errors during conversion (continue with next file). Print statistics afterwards.")
     return parser
 
 
@@ -53,7 +53,7 @@ def convert_dir(mtf_dir: Path,
     Convert all MTF files in the `mtf_dir` folder to JSON (and subfolders if `recursive` is True).
     The JSON files have the same name but suffix '.json' instead of '.mtf'.
     If `json_dir` is given, write the JSON file to that directory.
-    If 'ignore_errors' is True, continue with the next file in case of a ConversionError.
+    If 'ignore_errors' is True, continue with the next file in case of an exception.
     """
     if not mtf_dir.is_dir():
         raise ValueError(f"'{mtf_dir}' is not a directory.")
@@ -64,10 +64,14 @@ def convert_dir(mtf_dir: Path,
         elif not json_dir.is_dir():
             raise ValueError(f"'{json_dir}' is not a directory.")
 
+    num_files = num_success = 0
+    error_files: List[Tuple[str, str]] = []
     error_occured = False
     for root, _, files in os.walk(mtf_dir):
+        files.sort()
         for file in files:
             if file.endswith('.mtf'):
+                num_files += 1
                 mtf_path = Path(root) / file
                 if json_dir:
                     relative_path = mtf_path.relative_to(mtf_dir)
@@ -79,14 +83,23 @@ def convert_dir(mtf_dir: Path,
                     print(f"'{mtf_path}' -> '{json_path}' ...  ", end='')
                     data = read_mtf(mtf_path)
                     write_json(data, json_path)
+                    num_success += 1
                     print("SUCCESS")
-                except ConversionError as e:
+                except Exception as ex:
                     error_occured = True
-                    print(f"ERROR: {e}")
+                    error_files.append((str(mtf_path), str(ex)))
+                    print(f"ERROR: {ex}")
                     if not ignore_errors:
                         return 1
         if not recursive:
             break
+    if ignore_errors:
+        # print statistics
+        print(f"> Converted {num_success} of {num_files} files.")
+        if len(error_files) > 0:
+            print("> Failed to convert:")
+            for f, e in error_files:
+                print(f"  {f} ({e})")
     return 1 if error_occured else 0
 
 

mtf2json/mtf2json.py

@@ -4,12 +4,13 @@
 """
 import json
 import re
+import codecs
 from math import ceil
 from pathlib import Path
 from typing import Dict, Any, Tuple, Union, Optional, List, cast, TextIO
 
 
-version = "0.1.4"
+version = "0.1.5"
 mm_version = "0.49.19.1"
 
 
@@ -71,6 +72,14 @@ class ConversionError(Exception):
 string_keys = ['model']
 
 
+def mixed_decoder(error: UnicodeError) -> Tuple[str, int]:
+    bs: bytes = error.object[error.start: error.end]
+    return bs.decode("cp1252"), error.start + 1
+
+
+codecs.register_error("mixed", mixed_decoder)
+
+
 def __rename_keys(obj: Any) -> Any:
     """
     Rename the keys in the given object according to
@@ -794,7 +803,7 @@ def read_mtf(path: Path) -> Dict[str, Any]:
     mech_data: Dict[str, Any] = {}
 
     current_section = None
-    with open(path, 'r') as file:
+    with open(path, 'r', encoding='utf8', errors='mixed') as file:
         __check_compat(file)
         for line in file:
             line = line.strip()

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "mtf2json"
-version = "0.1.4"
+version = "0.1.5"
 description = "Convert the MegaMek MTF format to JSON."
 authors = ["juk0de <[email protected]>"]
 license = "MIT"