diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 33cdf84..b84fb94 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.1.2 +current_version = 1.2.0 [bumpversion:file:setup.py] diff --git a/.gitignore b/.gitignore index 82a7beb..a01c33f 100644 --- a/.gitignore +++ b/.gitignore @@ -33,8 +33,11 @@ waf-* tmp_* # Test tools -.tox -.cache +.tox/ +.cache/ .pytest_cache .coverage* -htmlcov/ \ No newline at end of file +htmlcov/ + +# Documentation +dev_doc/ diff --git a/CHANGELOG b/CHANGELOG index 40ec7ff..2a3e1d6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,24 @@ forthcoming ----------------------------------- - prophyc: Python codecs source code is pep8-compliant + +1.2.0 +----------------------------------- +- prophyc.generators.prophy: introduced prophy language generator +- prophyc.generators.python: + * generated source code is pep8-compliant (at least much more than it was before) + * generated imports do not use asterisk, implicitly lists imported names + * generated file contains annotation about fact of being generated by prophy +- prophyc.parsers.isar collects docstrings +- prophyc.model: + * refactor performed, shaped nodes' inheritance tree + * each node gets reproducible repr() (added renew package as dependency) + * each node gets str implementation that returns its prophy language representation + * each node may have a doc-string + * node name and docstring can be defined in unicode (special care in python2) + * Constant can be defined with numbers in hexadecimal literal format + + 1.1.2 ----------------------------------- - prophyc: python3 bug in C++ frontend fixed (libclang invocation contained unicode strings) diff --git a/MANIFEST.in b/MANIFEST.in index e971c0d..77c1567 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,7 +2,7 @@ include *.py include *.sh include .coveragerc include .bumpversion.cfg - +include requirements.txt include AUTHORS include HOWTO include CHANGELOG diff --git a/docs/conf.py b/docs/conf.py index ae70777..c791629 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,9 +51,9 @@ # built documents. # # The short X.Y version. -version = '1.1.2' +version = '1.2.0' # The full version, including alpha/beta/rc tags. -release = '1.1.2' +release = '1.2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/prophy/__init__.py b/prophy/__init__.py index 96ca72d..da60e58 100644 --- a/prophy/__init__.py +++ b/prophy/__init__.py @@ -35,4 +35,4 @@ 'with_metaclass', ] -__version__ = '1.1.2' +__version__ = '1.2.0' diff --git a/prophy/descriptor.py b/prophy/descriptor.py index f5b6fda..374f3d0 100644 --- a/prophy/descriptor.py +++ b/prophy/descriptor.py @@ -33,11 +33,11 @@ def evaluate_codecs(self): codec_kind.BYTES: (encode_bytes, decode_bytes), codec_kind.SCALAR: (encode_scalar, decode_scalar) } - kind = codec_kind.classify(self.type) - assert kind in all_codecs - self.encode_fcn, self.decode_fcn = all_codecs.get(kind) + kind_ = codec_kind.classify(self.type) + assert kind_ in all_codecs + self.encode_fcn, self.decode_fcn = all_codecs.get(kind_) - if kind == codec_kind.OPTIONAL: + if kind_ == codec_kind.OPTIONAL: base_kind = codec_kind.classify(self.type.__bases__[0]) opt_encode, opt_decode = all_codecs.get(base_kind) diff --git a/prophy/generators.py b/prophy/generators.py index 55dfda7..e561b44 100644 --- a/prophy/generators.py +++ b/prophy/generators.py @@ -1,3 +1,5 @@ +import collections + from .base_array import base_array from .composite import codec_kind, distance_to_next_multiply, struct_packed from .descriptor import DescriptorField @@ -69,14 +71,16 @@ def _build_up_implementation(cls): def validate(cls): for name, number in cls._enumerators: if not isinstance(name, str): - raise ProphyError("enum member's first argument has to be string") + msg = "enum ({}) member's first argument has to be string, got '{}'" + raise ProphyError(msg.format(cls.__name__, type(name).__name__)) if not isinstance(number, (int, long)): - raise ProphyError("enum member's second argument has to be an integer") + msg = "enum member's ({}.{}) second argument has to be an integer, got '{}'" + raise ProphyError(msg.format(cls.__name__, name, type(number).__name__)) - names = set(name for name, _ in cls._enumerators) - if len(names) < len(cls._enumerators): - raise ProphyError("names overlap in '{}' enum".format(cls.__name__)) + duplicates = ", ".join(_list_duplicates(name for name, _ in cls._enumerators)) + if duplicates: + raise ProphyError("names overlap in '{}' enum, duplicates: {}".format(cls.__name__, duplicates)) def add_attributes(self): def check(cls, value): @@ -107,9 +111,11 @@ class struct_generator(_composite_generator_base): def validate(cls): for field in cls._descriptor: if not isinstance(field.name, str): - raise ProphyError("member name must be a string type") + msg = "struct ({}) member's name must be a string type, got: '{}'" + raise ProphyError(msg.format(cls.__name__, type(field.name).__name__)) if not hasattr(field.type, "_is_prophy_object"): - raise ProphyError("member type must be a prophy object, is: {!r}".format(field.type)) + msg = "struct member's ({}.{}) type must be a prophy object, is: {!r}" + raise ProphyError(msg.format(cls.__name__, field.name, field.type)) types = list(cls._types()) for type_ in types[:-1]: @@ -397,3 +403,8 @@ def setter(self, new_value): self._fields[field.name] = new_value setattr(cls, field.name, property(getter, setter)) + + +def _list_duplicates(iterable): + iterable = list(iterable) + return sorted((collections.Counter(iterable) - collections.Counter(set(iterable))).keys()) diff --git a/prophy/scalar.py b/prophy/scalar.py index 51804dd..afd157a 100644 --- a/prophy/scalar.py +++ b/prophy/scalar.py @@ -45,7 +45,7 @@ def check(value): if not isinstance(value, (int, long)): raise ProphyError("not an int") if not min_ <= value <= max_: - raise ProphyError("out of bounds") + raise ProphyError("value: {} out of {}B integer's bounds: [{}, {}]".format(value, size, min_, max_)) return value cls._check = check diff --git a/prophy/tests/test_array_bound.py b/prophy/tests/test_array_bound.py index d920607..528fdba 100644 --- a/prophy/tests/test_array_bound.py +++ b/prophy/tests/test_array_bound.py @@ -55,7 +55,7 @@ def test_bound_scalar_array_assignment(BoundScalarArray): x.value = 10 with pytest.raises(Exception, match="not an int"): x.value[0] = "will fail type check" - with pytest.raises(Exception, match="out of bounds"): + with pytest.raises(Exception, match=r"value: -1 out of 4B integer's bounds: \[0, 4294967295\]"): x.value[0] = -1 with pytest.raises(Exception, match="not an int"): x.value[:] = [1, 2, "abc"] @@ -109,7 +109,7 @@ class LengthFieldAfter(prophy.with_metaclass(prophy.struct_generator, prophy.str def test_bound_scalar_array_bad_sizer_type(): - msg = "member type must be a prophy object, is: 'not_an_int'" + msg = r"struct member's \(_.not_an_int\) type must be a prophy object, is: 'not_an_int'" with pytest.raises(Exception, match=msg): class _(prophy.with_metaclass(prophy.struct_generator, prophy.struct_packed)): _descriptor = [("not_an_int", "not_an_int"), diff --git a/prophy/tests/test_array_fixed.py b/prophy/tests/test_array_fixed.py index 2beee37..4c9e6d7 100644 --- a/prophy/tests/test_array_fixed.py +++ b/prophy/tests/test_array_fixed.py @@ -70,7 +70,7 @@ def test_fixed_scalar_array_assignment(FixedScalarArray): x.value[:] = (10,) with pytest.raises(Exception, match="not an int"): x.value[0] = "will fail type check" - with pytest.raises(Exception, match="out of bounds"): + with pytest.raises(Exception, match=r"value: -1 out of 4B integer's bounds: \[0, 4294967295\]"): x.value[0] = -1 y = FixedScalarArray() diff --git a/prophy/tests/test_enum.py b/prophy/tests/test_enum.py index 8233568..1f78284 100644 --- a/prophy/tests/test_enum.py +++ b/prophy/tests/test_enum.py @@ -110,31 +110,35 @@ class NoEnumerators(prophy.with_metaclass(prophy.enum_generator, prophy.enum)): def test_enum_invalid_name(): - msg = "enum member's first argument has to be string" + msg = r"enum \(BadNamed\) member's first argument has to be string, got 'float'" with pytest.raises(prophy.ProphyError, match=msg): - class _(prophy.with_metaclass(prophy.enum_generator, prophy.enum)): + class BadNamed(prophy.with_metaclass(prophy.enum_generator, prophy.enum)): _enumerators = [(3.14159, 1), ("correct_name", 2)] def test_enum_invalid_value(): - msg = "enum member's second argument has to be an integer" + msg = r"enum member's \(TheEnum.invalid_value\) second argument has to be an integer, got 'float'" with pytest.raises(prophy.ProphyError, match=msg): - class _(prophy.with_metaclass(prophy.enum_generator, prophy.enum)): + class TheEnum(prophy.with_metaclass(prophy.enum_generator, prophy.enum)): _enumerators = [("correct_value", 1), ("invalid_value", 3.14159)] def test_enum_names_overlap(): - msg = "names overlap in 'NamesOverlapping' enum" + msg = "names overlap in 'NamesOverlapping' enum, duplicates: OtherDuplicate, SameName" with pytest.raises(prophy.ProphyError, match=msg): class NamesOverlapping(prophy.with_metaclass(prophy.enum_generator, prophy.enum)): - _enumerators = [("NamesOverlapping_Overlap", 1), - ("NamesOverlapping_Overlap", 2)] + _enumerators = [("SameName", 1), + ("SameName", 2), + ("SameName", 3), + ("OtherDuplicate", 4), + ("OtherDuplicate", 5), + ("ValidName", 6)] def test_enum_value_out_of_bounds(): - msg = "out of bounds" + msg = r"value: 4294967296 out of 4B integer's bounds: \[0, 4294967295\]" with pytest.raises(prophy.ProphyError, match=msg): class _(prophy.with_metaclass(prophy.enum_generator, prophy.enum)): _enumerators = [("OutOfBounds", 0xFFFFFFFF + 1)] diff --git a/prophy/tests/test_integers.py b/prophy/tests/test_integers.py index aa35921..132dc15 100644 --- a/prophy/tests/test_integers.py +++ b/prophy/tests/test_integers.py @@ -2,19 +2,19 @@ import pytest -@pytest.mark.parametrize('IntType, min_, max_', [ - (prophy.i8, -(0x80), 0x7F), - (prophy.i16, -(0x8000), 0x7FFF), - (prophy.i32, -(0x80000000), 0x7FFFFFFF), - (prophy.i64, -(0x8000000000000000), 0x7FFFFFFFFFFFFFFF), +@pytest.mark.parametrize('integer_type, min_, max_', [ + (prophy.i8, -0x80, 0x7F), + (prophy.i16, -0x8000, 0x7FFF), + (prophy.i32, -0x80000000, 0x7FFFFFFF), + (prophy.i64, -0x8000000000000000, 0x7FFFFFFFFFFFFFFF), (prophy.u8, 0, 0xFF), (prophy.u16, 0, 0xFFFF), (prophy.u32, 0, 0xFFFFFFFF), (prophy.u64, 0, 0xFFFFFFFFFFFFFFFF) ]) -def test_integer(IntType, min_, max_): +def test_integer(integer_type, min_, max_): class X(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): - _descriptor = [("value", IntType)] + _descriptor = [("value", integer_type)] x = X() assert x.value == 0 @@ -23,17 +23,14 @@ class X(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): x.value = min_ assert x.value == min_ - with pytest.raises(prophy.ProphyError) as e: + with pytest.raises(prophy.ProphyError, match="not an int"): x.value = "123" - assert "not an int" in str(e.value) - with pytest.raises(prophy.ProphyError) as e: + with pytest.raises(prophy.ProphyError, match=r"value: \d+ out of \dB integer's bounds: \[-?\d+, \d+\]"): x.value = max_ + 1 - assert "out of bounds" in str(e.value) - with pytest.raises(prophy.ProphyError) as e: + with pytest.raises(prophy.ProphyError, match=r"value: -?\d+ out of \dB integer's bounds: \[-?\d+, \d+\]"): x.value = min_ - 1 - assert "out of bounds" in str(e.value) y = X() y.value = 42 @@ -41,7 +38,7 @@ class X(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): assert y.value == min_ -@pytest.mark.parametrize('IntType, a, encoded_a, b, encoded_b, too_short, too_long', [ +@pytest.mark.parametrize('integer_type, a, encoded_a, b, encoded_b, too_short, too_long', [ (prophy.i8, 1, b"\x01", (-1), b"\xff", @@ -83,9 +80,9 @@ class X(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): b"\xff\xff\xff\xff\xff\xff\xff", b"\xff\xff\xff\xff\xff\xff\xff\xff\xff") ]) -def test_integer_codec(IntType, a, encoded_a, b, encoded_b, too_short, too_long): +def test_integer_codec(integer_type, a, encoded_a, b, encoded_b, too_short, too_long): class X(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): - _descriptor = [("value", IntType)] + _descriptor = [("value", integer_type)] x = X() x.value = 8 @@ -101,10 +98,8 @@ class X(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): x.decode(encoded_b, ">") assert x.value == b - with pytest.raises(prophy.ProphyError) as e: + with pytest.raises(prophy.ProphyError, match="X: too few bytes to decode integer"): x.decode(too_short, ">") - assert "too few bytes to decode integer" in str(e.value) - with pytest.raises(prophy.ProphyError) as e: + with pytest.raises(prophy.ProphyError, match="not all bytes of X read"): x.decode(too_long, ">") - assert "not all bytes of {} read".format(X.__name__) in str(e.value) diff --git a/prophy/tests/test_struct.py b/prophy/tests/test_struct.py index 4dd1db2..710c169 100644 --- a/prophy/tests/test_struct.py +++ b/prophy/tests/test_struct.py @@ -424,7 +424,7 @@ class X(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): def test_bad_struct_member_name(): - with pytest.raises(prophy.ProphyError, match="member name must be a string type"): + with pytest.raises(prophy.ProphyError, match=r"struct \(X\) member's name must be a string type, got: 'float'"): class X(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [(3.14159, prophy.u32)] diff --git a/prophy/tests/test_version.py b/prophy/tests/test_version.py index 3ae7031..e2dbd58 100644 --- a/prophy/tests/test_version.py +++ b/prophy/tests/test_version.py @@ -1,3 +1,3 @@ def test_version_number(): import prophy - assert prophy.__version__ == '1.1.2' + assert prophy.__version__ == '1.2.0' diff --git a/prophy_cpp/wscript b/prophy_cpp/wscript index d03a856..2b2c01d 100644 --- a/prophy_cpp/wscript +++ b/prophy_cpp/wscript @@ -4,7 +4,7 @@ from waflib.Task import Task from waflib.Tools import waf_unit_test APPNAME = 'prophy-cpp' -VERSION = '1.1.2' +VERSION = '1.2.0' def options(ctx): ctx.load('compiler_cxx python waf_unit_test') diff --git a/prophyc/__init__.py b/prophyc/__init__.py index 0a0e7ed..cd74368 100644 --- a/prophyc/__init__.py +++ b/prophyc/__init__.py @@ -1,16 +1,15 @@ #! /usr/bin/env python -from contextlib import contextmanager import os import sys +from contextlib import contextmanager from . import model from . import options from .file_processor import FileProcessor from .generators.base import GenerateError - -__version__ = '1.1.2' +__version__ = '1.2.0' class ProphycError(Exception): @@ -37,64 +36,75 @@ def main(args): if opts.version: print("prophyc {}".format(__version__)) - return + return {} if not opts.input_files: emit.error("missing input file") + if opts.isar_includes and not opts.sack: + emit.error('Isar defines inclusion is supported only in "sack" compilation mode.') + serializers = get_serializers(opts) - patcher = get_patcher(opts) - if not serializers: + if not serializers and not opts.void_out: emit.error("missing output directives") - supplementary_nodes = [] - if opts.isar_includes: - if not opts.sack: - emit.error('Isar defines inclusion is supported only in "sack" compilation mode.') - - isar_parser = get_isar_parser(emit) - model_parser = ModelParser(isar_parser, patcher, emit) - file_parser = FileProcessor(model_parser, opts.include_dirs) - - for input_file in opts.isar_includes: - with error_on_exception(emit): - include_nodes = file_parser(input_file) - basename = get_basename(input_file) - supplementary_nodes.append(model.Include(basename, include_nodes)) + patcher = get_patcher(opts.patch) + supplementary_nodes = create_supplements(emit, opts.isar_includes, opts.include_dirs, patcher) - for include_name, include_nodes in flatten_included_defs(supplementary_nodes): - generate_target_files(emit, serializers, include_name, include_nodes) + model_nodes = dict(flatten_included_defs(supplementary_nodes)) - parser = get_target_parser(emit, opts, supplementary_nodes) - model_parser = ModelParser(parser, patcher, emit) - file_parser = FileProcessor(model_parser, opts.include_dirs) + source_parser = get_target_parser(emit, opts, supplementary_nodes) + model_parser = model.ModelParser(source_parser, patcher, emit) + file_processor_ = FileProcessor(model_parser, opts.include_dirs) for input_file in opts.input_files: with error_on_exception(emit): - nodes = file_parser(input_file) - generate_target_files(emit, serializers, get_basename(input_file), nodes) + nodes = file_processor_(input_file) + basename = get_basename(input_file) + model_nodes[basename] = nodes + generate_target_files(emit, serializers, model_nodes) -def get_isar_parser(emit): - from prophyc.parsers.isar import IsarParser - return IsarParser(warn=emit.warn) + return model_nodes + + +def create_supplements(emit, isar_includes, include_dirs, patcher): + isar_parser = get_isar_parser(emit) + model_parser = model.ModelParser(isar_parser, patcher, emit) + file_parser = FileProcessor(model_parser, include_dirs) + supplementary_nodes = [] + for input_file in isar_includes: + with error_on_exception(emit): + include_nodes = file_parser(input_file) + basename = get_basename(input_file) + supplementary_nodes.append(model.Include(basename, include_nodes)) + return supplementary_nodes def get_target_parser(emit, opts, supplementary_nodes): if opts.isar: return get_isar_parser(emit) elif opts.sack: - from prophyc.parsers.sack import SackParser - status = SackParser.check() - if not status: - emit.error(status.error) - return SackParser(opts.include_dirs, warn=emit.warn, include_tree=supplementary_nodes) + return get_sack_parser(emit, supplementary_nodes, opts.include_dirs) else: from prophyc.parsers.prophy import ProphyParser return ProphyParser() +def get_isar_parser(emit): + from prophyc.parsers.isar import IsarParser + return IsarParser(warn=emit.warn) + + +def get_sack_parser(emit, supplementary_nodes, sack_include_dirs): + from prophyc.parsers.sack import SackParser + status = SackParser.check() + if not status: + emit.error(status.error) + return SackParser(sack_include_dirs, warn=emit.warn, include_tree=supplementary_nodes) + + def get_serializers(opts): serializers = [] if opts.python_out: @@ -106,33 +116,19 @@ def get_serializers(opts): if opts.cpp_full_out: from prophyc.generators.cpp_full import CppFullGenerator serializers.append(CppFullGenerator(opts.cpp_full_out)) + if opts.prophy_out: + from prophyc.generators.prophy import SchemaGenerator + serializers.append(SchemaGenerator(opts.prophy_out)) return serializers -def get_patcher(opts): - if opts.patch: +def get_patcher(patch_file_path): + if patch_file_path: from prophyc import patch - patches = patch.parse(opts.patch) + patches = patch.parse(patch_file_path) return lambda nodes: patch.patch(nodes, patches) -class ModelParser(): - def __init__(self, parser, patcher, emit): - self.parser = parser - self.patcher = patcher - self.emit = emit - - def __call__(self, *parse_args): - nodes = self.parser.parse(*parse_args) - if self.patcher: - self.patcher(nodes) - model.topological_sort(nodes) - model.cross_reference(nodes, warn=self.emit.warn) - model.evaluate_kinds(nodes) - model.evaluate_sizes(nodes, warn=self.emit.warn) - return nodes - - def get_basename(path): return os.path.splitext(os.path.basename(path))[0] @@ -141,19 +137,21 @@ def flatten_included_defs(supple_nodes): def get_nodes_and_names(nodes_list): for elem in nodes_list: if isinstance(elem, model.Include): - yield elem - for sub_elem in get_nodes_and_names(elem.nodes): + yield elem.name, elem.members + for sub_elem in get_nodes_and_names(elem.members): yield sub_elem + """ pass trough a dictionary to avoid duplicates """ return tuple(dict(get_nodes_and_names(supple_nodes)).items()) -def generate_target_files(emit, serializers, basename, nodes): - for serializer in serializers: - try: - serializer.serialize(nodes, basename) - except GenerateError as e: - emit.error(str(e)) +def generate_target_files(emit, serializers, model_nodes): + for basename, nodes in model_nodes.items(): + for serializer in serializers: + try: + serializer.serialize(nodes, basename) + except GenerateError as e: + emit.error(str(e)) @contextmanager diff --git a/prophyc/calc.py b/prophyc/calc.py index fa02ec0..7aaf384 100644 --- a/prophyc/calc.py +++ b/prophyc/calc.py @@ -6,62 +6,66 @@ class ParseError(Exception): pass -class Parser(object): - """ - Base class for a lexer/parser that has the rules defined as methods - """ - tokens = () - precedence = () +class Calc(object): + tokens = ('NAME', 'CONST16', 'CONST10', 'LSHIFT', 'RSHIFT') + precedence = ( + ('left', '+', '-'), + ('left', '*', '/'), + ('left', 'LSHIFT', 'RSHIFT'), + ('right', 'UMINUS'), + ) + + literals = ['+', '-', '*', '/', '(', ')'] + + t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' + t_LSHIFT = r'<<' + t_RSHIFT = r'>>' def __init__(self): self.lexer = lex.lex(module=self, debug=0) self.parser = yacc.yacc(module=self, tabmodule='parsetab_calc', write_tables=0, debug=0) + self.vars = {} - def eval(self, expr): + def eval(self, expr, vars_): + self.vars = vars_ return self.parser.parse(expr, lexer=self.lexer) + @staticmethod + def t_CONST16(t): + r"""0x[0-9a-fA-F]+""" + t.value = int(t.value, 16) + return t -class Calc(Parser): - tokens = ('NAME', 'NUMBER', 'LSHIFT', 'RSHIFT') - - literals = ['+', '-', '*', '/', '(', ')'] - - t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' - t_LSHIFT = r'<<' - t_RSHIFT = r'>>' - - def t_NUMBER(self, t): - r'\d+' + @staticmethod + def t_CONST10(t): + r"""\d+""" t.value = int(t.value) return t t_ignore = " \t" - def t_newline(self, t): - r'\n+' + @staticmethod + def t_newline(t): + r"""\n+""" t.lexer.lineno += t.value.count("\n") - def t_error(self, t): + @staticmethod + def t_error(t): raise ParseError('illegal character %s' % t.value[0]) - precedence = ( - ('left', '+', '-'), - ('left', '*', '/'), - ('left', 'LSHIFT', 'RSHIFT'), - ('right', 'UMINUS'), - ) - - def p_statement_expr(self, p): - 'statement : expression' + @staticmethod + def p_statement_expr(p): + """statement : expression""" p[0] = p[1] - def p_expression_binop(self, p): - '''expression : expression '+' expression + @staticmethod + def p_expression_binop(p): + """expression : expression '+' expression | expression '-' expression | expression '*' expression | expression '/' expression | expression LSHIFT expression - | expression RSHIFT expression''' + | expression RSHIFT expression""" if p[2] == '+': p[0] = p[1] + p[3] elif p[2] == '-': @@ -75,20 +79,24 @@ def p_expression_binop(self, p): elif p[2] == '>>': p[0] = p[1] >> p[3] - def p_expression_uminus(self, p): - "expression : '-' expression %prec UMINUS" + @staticmethod + def p_expression_uminus(p): + """expression : '-' expression %prec UMINUS""" p[0] = -p[2] - def p_expression_group(self, p): - "expression : '(' expression ')'" + @staticmethod + def p_expression_group(p): + """expression : '(' expression ')'""" p[0] = p[2] - def p_expression_number(self, p): - "expression : NUMBER" + @staticmethod + def p_expression_number(p): + """expression : CONST10 + | CONST16""" p[0] = p[1] def p_expression_name(self, p): - "expression : NAME" + """expression : NAME""" try: p[0] = p[1] while not isinstance(p[0], int): @@ -99,10 +107,6 @@ def p_expression_name(self, p): def p_error(self, p): raise ParseError("syntax error at '%s'" % p.value) - def eval(self, expr, vars_): - self.vars = vars_ - return super(Calc, self).eval(expr) - calc = Calc() diff --git a/prophyc/file_processor.py b/prophyc/file_processor.py index 70fd024..1ab3f99 100644 --- a/prophyc/file_processor.py +++ b/prophyc/file_processor.py @@ -1,3 +1,4 @@ +import codecs import os from contextlib import contextmanager @@ -43,7 +44,7 @@ class FileProcessor(object): def __init__(self, process_content, include_dirs): self.process_content = process_content '''Function object accepting arguments (content, path, process_file)''' - self.include_dirs = [_ for _ in include_dirs] + self.include_dirs = [d for d in include_dirs] self.files = {} '''Absolute paths are keys, values are results of process_content calls''' @@ -79,6 +80,8 @@ def _process_file(self, path): return self.files[abspath] self.files[abspath] = None - result = self.process_content(open(path).read(), path, lambda leaf: self.process_leaf(leaf)) + with codecs.open(path, 'r', encoding='utf-8') as f: + content = f.read() + result = self.process_content(content, path, lambda leaf: self.process_leaf(leaf)) self.files[abspath] = result return result diff --git a/prophyc/generators/base.py b/prophyc/generators/base.py index e979ad6..3576768 100644 --- a/prophyc/generators/base.py +++ b/prophyc/generators/base.py @@ -1,4 +1,6 @@ +import codecs import os + from prophyc import model @@ -7,7 +9,7 @@ class GenerateError(Exception): def _write_file(file_path, string): - with open(file_path, "w") as f: + with codecs.open(file_path, "w", encoding="utf-8") as f: f.write(string) @@ -33,8 +35,8 @@ def check_nodes(self, nodes): class GeneratorBase(GeneratorAbc): - def __init__(self, outpu_directory="."): - self.output_dir = outpu_directory + def __init__(self, output_directory="."): + self.output_dir = output_directory def serialize(self, nodes, base_name): self.check_nodes(nodes) @@ -74,19 +76,23 @@ class TranslatorBase(TranslatorAbc): } def __call__(self, nodes, base_name): + nodes = self._move_includes_to_front(nodes) content = self._process_nodes(nodes, base_name) return self._block_post_process(content, base_name, nodes) def _process_nodes(self, nodes, base_name): - def collect(): - previous_node = None - for node, translated_node in self._nodes_dispatcher(nodes, base_name): - if self._prepend_newline(previous_node, node): - yield '\n' - yield translated_node + "\n" - previous_node = node + render = "" + previous_node_type = None + for node_type_name, translated_node in self._nodes_dispatcher(nodes, base_name): + lines_splitter = self._make_lines_splitter(previous_node_type, node_type_name) + if translated_node: + render += lines_splitter + translated_node + + previous_node_type = node_type_name - return ''.join(collect()) + if nodes and render: + render += "\n" + return render @classmethod def _block_post_process(cls, content, base_name, nodes): @@ -96,28 +102,41 @@ def _block_post_process(cls, content, base_name, nodes): return content @classmethod - def _prepend_newline(cls, previous_node, current_node): - if previous_node: - is_different_type = type(previous_node) is not type(current_node) - is_enum_struct_or_union = isinstance(previous_node, (model.Enum, model.Struct, model.Union)) - return is_different_type or is_enum_struct_or_union + def _make_lines_splitter(cls, previous_node_type, current_node_type): + if not previous_node_type: + return "" + + if previous_node_type != current_node_type: + return "\n\n" + + return "\n" def _nodes_dispatcher(self, nodes, base_name): for block_translator_class in self.prerequisite_translators: prerequisite_block_translator = block_translator_class() translated_block = prerequisite_block_translator(nodes, base_name) - if translated_block: - yield "prerequisite block", translated_block + if nodes and translated_block: + last_node_name = type(nodes[-1]).__name__ + yield last_node_name, translated_block for node in nodes: handler = self._get_translation_handler(node) if handler: translated_node = handler(node) - yield node, translated_node + yield type(node).__name__, translated_node def _get_translation_handler(self, node): - if type(node) not in self._translation_methods_map: - raise GenerateError("Unknown node type: {}".format(type(node).__name__)) - - translation_method_name = self._translation_methods_map[type(node)] + translation_method_name = self._translation_methods_map.get(type(node), None) + assert translation_method_name, "Unknown node type: {}".format(type(node).__name__) return getattr(self, translation_method_name, None) + + @staticmethod + def _move_includes_to_front(nodes): + includes = [] + others = [] + for node in nodes: + if isinstance(node, model.Include): + includes.append(node) + else: + others.append(node) + return includes + others diff --git a/prophyc/generators/cpp.py b/prophyc/generators/cpp.py index a93a0fc..7593f21 100644 --- a/prophyc/generators/cpp.py +++ b/prophyc/generators/cpp.py @@ -1,7 +1,6 @@ from prophyc import model from prophyc.generators.base import GenerateError, GeneratorBase, TranslatorBase - primitive_types = { 'u8': 'uint8_t', 'u16': 'uint16_t', @@ -88,7 +87,7 @@ def translate_constant(self, constant): return 'enum {{ {} = {} }};'.format(constant.name, _to_literal(constant.value)) def translate_typedef(self, typedef): - tp = primitive_types.get(typedef.type_, typedef.type_) + tp = primitive_types.get(typedef.type_name, typedef.type_name) return 'typedef {} {};'.format(tp, typedef.name) def translate_enum(self, enum): @@ -111,8 +110,8 @@ def build_annotation(member): annotation = 'greedy array' return annotation - typename = primitive_types.get(member.type_, member.type_) - if member.array: + typename = primitive_types.get(member.type_name, member.type_name) + if member.is_array: annotation = build_annotation(member) size = member.size or 1 if annotation: @@ -123,7 +122,7 @@ def build_annotation(member): field = '{0} {1};\n'.format(typename, member.name) if member.optional: field = 'prophy::bool_t has_{0};\n'.format(member.name) + field - if member.padding > 0: + if member.padding is not None and member.padding > 0: field += padder.generate_padding(member.padding) return field @@ -154,7 +153,7 @@ def gen_disc(member): return 'discriminator_{0} = {1}'.format(member.name, member.discriminator) def gen_member(member): - typename = primitive_types.get(member.type_, member.type_) + typename = primitive_types.get(member.type_name, member.type_name) return '{0} {1};\n'.format(typename, member.name) enum_fields = _indent(',\n'.join(gen_disc(mem) for mem in union.members), 4) @@ -165,6 +164,16 @@ def gen_member(member): return UNION_DEF_TEMPLATE.format(align=union.alignment, name=union.name, parts=_indent(parts, 4)) + @classmethod + def _make_lines_splitter(cls, previous_node_type, current_node_type): + if not previous_node_type: + return "" + + if previous_node_type != current_node_type: + return "\n\n" + + return "\n" + ENUM_SWAP_TEMPLATE = """\ template <> inline {0}* swap<{0}>({0}* in) {{ swap(reinterpret_cast(in)); return in + 1; }}""" @@ -183,8 +192,8 @@ def gen_member(member): class _HppSwapDeclarations(TranslatorBase): block_template = HPP_SWAPS_BLOCK_TEMPLATE - def _prepend_newline(self, _, __): - return False + def _make_lines_splitter(self, previous_node_type, _): + return "\n" if previous_node_type else "" def translate_enum(self, enum): return ENUM_SWAP_TEMPLATE.format(enum.name) @@ -198,7 +207,7 @@ def translate_union(self, union): def _member_access_statement(member): out = '&payload->%s' % member.name - if isinstance(member, model.StructMember) and member.array: + if isinstance(member, model.StructMember) and member.is_array: out = out[1:] return out @@ -251,7 +260,7 @@ class _CppSwapTranslator(TranslatorBase): def translate_struct(self, struct): def gen_member(member, delimiters=[]): - if member.array: + if member.is_array: is_dynamic = member.kind == model.Kind.DYNAMIC swap_mode = 'dynamic' if is_dynamic else 'fixed' if member.bound: @@ -276,7 +285,7 @@ def gen_last_member(name, last_mem, delimiters=[]): name, _member_access_statement(last_mem) ) - elif last_mem.kind == model.Kind.DYNAMIC or last_mem.dynamic: + elif last_mem.kind == model.Kind.DYNAMIC or last_mem.is_dynamic: return 'return cast<{0}*>({1});\n'.format( name, gen_member(last_mem, delimiters) diff --git a/prophyc/generators/cpp_full.py b/prophyc/generators/cpp_full.py index cefdc05..8787487 100644 --- a/prophyc/generators/cpp_full.py +++ b/prophyc/generators/cpp_full.py @@ -2,7 +2,6 @@ from prophyc.model import DISC_SIZE, BUILTIN_SIZES from prophyc.generators.base import GenerateError, GeneratorBase, TranslatorBase - BUILTIN2C = { 'i8': 'int8_t', 'i16': 'int16_t', @@ -29,7 +28,7 @@ def _get_initializer(m): m = m.definition if isinstance(m.definition, model.Enum): return m.definition.members[0].name - if m.type_ in BUILTIN2C: + if m.type_name in BUILTIN2C: return '' return None @@ -53,15 +52,15 @@ def _get_byte_size(node): node = _get_leaf(node) if isinstance(node, model.Enum): return DISC_SIZE - elif hasattr(node, 'type_'): - return BUILTIN_SIZES.get(node.type_) + elif hasattr(node, 'type_name'): + return BUILTIN_SIZES.get(node.type_name) else: return node.byte_size def _get_cpp_builtin_type(node): """Gets C++ float or int type from stdint.h, or throws miserably.""" - return BUILTIN2C[_get_leaf(node).type_] + return BUILTIN2C[_get_leaf(node).type_name] def _to_literal(value): @@ -78,7 +77,8 @@ def translate_include(self, include): return '#include "{}.ppf.hpp"'.format(include.name) -HPP_DEFS_TEMPLATE = """\ +class _HppDefinitionsTranslator(TranslatorBase): + block_template = """\ namespace prophy {{ namespace generated @@ -89,10 +89,6 @@ def translate_include(self, include): }} // namespace prophy """ - -class _HppDefinitionsTranslator(TranslatorBase): - block_template = HPP_DEFS_TEMPLATE - def translate_constant(self, constant): return 'enum {{ {} = {} }};'.format(constant.name, _to_literal(constant.value)) @@ -101,44 +97,57 @@ def translate_enum(self, enum): return 'enum {0}\n'.format(enum.name) + '{\n' + body + '};' def translate_typedef(self, node): - return 'typedef {0} {1};'.format(BUILTIN2C.get(node.type_, node.type_), node.name) + return 'typedef {0} {1};'.format(BUILTIN2C.get(node.type_name, node.type_name), node.name) def translate_struct(self, node): return ( - 'struct {0} : public prophy::detail::message<{0}>\n'.format(node.name) + - '{\n' + - _indent( - '\n'.join(( - 'enum {{ encoded_byte_size = {0} }};\n'.format(generate_struct_encoded_byte_size(node)), - generate_struct_fields(node), - generate_struct_constructor(node), - 'size_t get_byte_size() const\n' + - '{\n' + - _indent(generate_struct_get_byte_size(node)) + - '}\n' - )) - ) + - '};' + 'struct {0} : public prophy::detail::message<{0}>\n'.format(node.name) + + '{\n' + + _indent( + '\n'.join(( + 'enum {{ encoded_byte_size = {0} }};\n'.format(generate_struct_encoded_byte_size(node)), + generate_struct_fields(node), + generate_struct_constructor(node), + 'size_t get_byte_size() const\n' + + '{\n' + + _indent(generate_struct_get_byte_size(node)) + + '}\n' + )) + ) + + '};' ) def translate_union(self, node): return ( - 'struct {0} : public prophy::detail::message<{0}>\n'.format(node.name) + - '{\n' + - _indent( - '\n'.join(( - 'enum {{ encoded_byte_size = {0} }};\n'.format(generate_union_encoded_byte_size(node)), - generate_union_fields(node), - generate_union_constructor(node), - 'size_t get_byte_size() const\n' + - '{\n' + - _indent(generate_union_get_byte_size(node)) + - '}\n' - )) - ) + - '};' + 'struct {0} : public prophy::detail::message<{0}>\n'.format(node.name) + + '{\n' + + _indent( + '\n'.join(( + 'enum {{ encoded_byte_size = {0} }};\n'.format(generate_union_encoded_byte_size(node)), + generate_union_fields(node), + generate_union_constructor(node), + 'size_t get_byte_size() const\n' + + '{\n' + + _indent(generate_union_get_byte_size(node)) + + '}\n' + )) + ) + + '};' ) + @classmethod + def _make_lines_splitter(cls, previous_node_type, current_node_type): + if not previous_node_type: + return "" + + if previous_node_type in ("Enum", "Struct", "Union") or current_node_type in ("Enum", "Struct", "Union"): + return "\n\n\n" + + if previous_node_type != current_node_type: + return "\n\n" + + return "\n" + HPP_HEADER_TEMPLATE = """\ #ifndef _PROPHY_GENERATED_FULL_{base_name}_HPP @@ -167,6 +176,14 @@ class _HppTranslator(TranslatorBase): _HppDefinitionsTranslator ] + @classmethod + def _make_lines_splitter(cls, previous_node_type, current_node_type): + if not previous_node_type: + return "" + if previous_node_type in ("Enum", "Struct", "Union") or current_node_type in ("Enum", "Struct", "Union"): + return "\n\n" + return TranslatorBase._make_lines_splitter(previous_node_type, current_node_type) + CPP_SOURCE_TEMPLATE = """\ #include "{base_name}.ppf.hpp" @@ -194,123 +211,122 @@ class _CppTranslator(TranslatorBase): def translate_enum(self, node): return ( - 'template <>\n' + - 'const char* print_traits<{0}>::to_literal({0} x)\n'.format(node.name) + - '{\n' + - _indent( - 'switch (x)\n' + + 'template <>\n' + + 'const char* print_traits<{0}>::to_literal({0} x)\n'.format(node.name) + '{\n' + _indent( - ''.join('case {0}: return "{0}";\n'.format(m.name) for m in node.members) + - 'default: return 0;\n' + 'switch (x)\n' + + '{\n' + + _indent( + ''.join('case {0}: return "{0}";\n'.format(m.name) for m in node.members) + + 'default: return 0;\n' + ) + + '}\n' ) + - '}\n' - ) + - '}' + '}' ) def translate_struct(self, node): def encode_impl(node): + cast_template = 'template uint8_t* message_impl<{0}>::encode<{1}>(const {0}& x, uint8_t* pos);\n' return ( - 'template <>\n' + - 'template \n' + - 'uint8_t* message_impl<{0}>::encode(const {0}& x, uint8_t* pos)\n'.format(node.name) + - '{\n' + - _indent( - generate_struct_encode(node) + - 'return pos;\n' - ) + - '}\n' + - ''.join( - 'template uint8_t* message_impl<{0}>::encode<{1}>(const {0}& x, uint8_t* pos);\n'.format( - node.name, e) - for e in ('native', 'little', 'big') - ) + 'template <>\n' + + 'template \n' + + 'uint8_t* message_impl<{0}>::encode(const {0}& x, uint8_t* pos)\n'.format(node.name) + + '{\n' + + _indent( + generate_struct_encode(node) + + 'return pos;\n' + ) + + '}\n' + + ''.join(cast_template.format(node.name, e) for e in ('native', 'little', 'big')) ) def decode_impl(node): + cast_template = 'template bool message_impl<{0}>::decode<{1}>' \ + '({0}& x, const uint8_t*& pos, const uint8_t* end);\n' return ( - 'template <>\n' + - 'template \n' + - 'bool message_impl<{0}>::decode({0}& x, const uint8_t*& pos, const uint8_t* end)\n'.format(node.name) + - '{\n' + - _indent( - 'return (\n' + - _indent(generate_struct_decode(node)) + - ');\n' - ) + - '}\n' + - ''.join( - 'template bool message_impl<{0}>::decode<{1}>({0}& x, const uint8_t*& pos, const uint8_t* end);\n'.format( - node.name, e) - for e in ('native', 'little', 'big') - ) + 'template <>\n' + + 'template \n' + + 'bool message_impl<{0}>::decode({0}& x, const uint8_t*& pos, const uint8_t* end)\n'.format( + node.name) + + '{\n' + + _indent( + 'return (\n' + + _indent(generate_struct_decode(node)) + + ');\n' + ) + + '}\n' + ''.join(cast_template.format(node.name, e) for e in ('native', 'little', 'big')) ) def print_impl(node): + cast_template = 'template void message_impl<{0}>::print(const {0}& x, std::ostream& out, size_t indent);' return ( - 'template <>\n' + - 'void message_impl<{0}>::print(const {0}& x, std::ostream& out, size_t indent)\n'.format(node.name) + - '{\n' + - _indent(generate_struct_print(node)) + - '}\n' + - 'template void message_impl<{0}>::print(const {0}& x, std::ostream& out, size_t indent);'.format( - node.name) + 'template <>\n' + + 'void message_impl<{0}>::print(const {0}& x, std::ostream& out, size_t indent)\n'.format( + node.name) + + '{\n' + + _indent(generate_struct_print(node)) + + '}\n' + cast_template.format(node.name) ) + return ( - encode_impl(node) + '\n' + - decode_impl(node) + '\n' + - print_impl(node) + encode_impl(node) + '\n' + + decode_impl(node) + '\n' + + print_impl(node) ) def translate_union(self, node): def encode_impl(node): return ( - 'template <>\n' + - 'template \n' + - 'uint8_t* message_impl<{0}>::encode(const {0}& x, uint8_t* pos)\n'.format(node.name) + - '{\n' + - _indent( - generate_union_encode(node) + - 'return pos;\n' - ) + - '}\n' + - ''.join( - 'template uint8_t* message_impl<{0}>::encode<{1}>(const {0}& x, uint8_t* pos);\n'.format( - node.name, e) - for e in ('native', 'little', 'big') - ) + 'template <>\n' + + 'template \n' + + 'uint8_t* message_impl<{0}>::encode(const {0}& x, uint8_t* pos)\n'.format(node.name) + + '{\n' + + _indent( + generate_union_encode(node) + + 'return pos;\n' + ) + + '}\n' + + ''.join( + 'template uint8_t* message_impl<{0}>::encode<{1}>(const {0}& x, uint8_t* pos);\n'.format( + node.name, e) + for e in ('native', 'little', 'big') + ) ) def decode_impl(node): return ( - 'template <>\n' + - 'template \n' + - 'bool message_impl<{0}>::decode({0}& x, const uint8_t*& pos, const uint8_t* end)\n'.format(node.name) + - '{\n' + - _indent(generate_union_decode(node)) + - '}\n' + - ''.join( - 'template bool message_impl<{0}>::decode<{1}>({0}& x, const uint8_t*& pos, const uint8_t* end);\n'.format( - node.name, e) - for e in ('native', 'little', 'big') - ) + 'template <>\n' + + 'template \n' + + 'bool message_impl<{0}>::decode({0}& x, const uint8_t*& pos, const uint8_t* end)\n'.format( + node.name) + + '{\n' + + _indent(generate_union_decode(node)) + + '}\n' + + ''.join( + 'template bool message_impl<{0}>::decode<{1}>({0}& x, const uint8_t*& pos, const uint8_t* end);\n'.format( + node.name, e) + for e in ('native', 'little', 'big') + ) ) def print_impl(node): return ( - 'template <>\n' + - 'void message_impl<{0}>::print(const {0}& x, std::ostream& out, size_t indent)\n'.format(node.name) + - '{\n' + - _indent(generate_union_print(node)) + - '}\n' + - 'template void message_impl<{0}>::print(const {0}& x, std::ostream& out, size_t indent);\n'.format( - node.name) + 'template <>\n' + + 'void message_impl<{0}>::print(const {0}& x, std::ostream& out, size_t indent)\n'.format( + node.name) + + '{\n' + + _indent(generate_union_print(node)) + + '}\n' + + 'template void message_impl<{0}>::print(const {0}& x, std::ostream& out, size_t indent);\n'.format( + node.name) ) + return ( - encode_impl(node) + '\n' + - decode_impl(node) + '\n' + - print_impl(node) + encode_impl(node) + '\n' + + decode_impl(node) + '\n' + + print_impl(node) ) @@ -320,18 +336,17 @@ def generate_struct_encode(node): delimiters = {bound[m.name].name: (m, _get_cpp_builtin_type(m)) for m in node.members if m.name in bound} for m in node.members: - if m.fixed: + if m.is_fixed: text += 'pos = do_encode(pos, x.{0}.data(), {1});\n'.format(m.name, m.size) - elif m.dynamic: + elif m.is_dynamic: d, dcpptype = delimiters[m.name] text += 'pos = do_encode(pos, x.{0}.data(), {1}(x.{0}.size()));\n'.format(m.name, dcpptype) - elif m.limited: + elif m.is_limited: d, dcpptype = delimiters[m.name] dcpptype = _get_cpp_builtin_type(d) - text += ( - 'do_encode(pos, x.{0}.data(), {2}(std::min(x.{0}.size(), size_t({1}))));\n'.format(m.name, m.size, dcpptype) + - 'pos = pos + {0};\n'.format(m.byte_size) - ) + text += ('do_encode(pos, x.{0}.data(), {2}(std::min(x.{0}.size(), size_t({1}))));\n' + 'pos = pos + {3};\n').format(m.name, m.size, dcpptype, m.byte_size) + elif m.greedy: text += 'pos = do_encode(pos, x.{0}.data(), x.{0}.size());\n'.format(m.name) elif m.optional: @@ -339,12 +354,12 @@ def generate_struct_encode(node): elif m.name in bound: b = bound[m.name] mcpptype = _get_cpp_builtin_type(m) - if b.dynamic: + if b.is_dynamic: text += 'pos = do_encode(pos, {1}(x.{0}.size()));\n'.format(b.name, mcpptype) else: text += ( 'pos = do_encode(pos, {2}(std::min(x.{0}.size(), size_t({1}))));\n' - .format(b.name, b.size, mcpptype) + .format(b.name, b.size, mcpptype) ) else: text += 'pos = do_encode(pos, x.{0});\n'.format(m.name) @@ -360,11 +375,11 @@ def generate_struct_decode(node): text = [] bound = {m.bound: m for m in node.members if m.bound} for m in node.members: - if m.fixed: + if m.is_fixed: text.append('do_decode(x.{0}.data(), {1}, pos, end)'.format(m.name, m.size)) - elif m.dynamic: + elif m.is_dynamic: text.append('do_decode(x.{0}.data(), x.{0}.size(), pos, end)'.format(m.name)) - elif m.limited: + elif m.is_limited: text.append('do_decode_in_place(x.{0}.data(), x.{0}.size(), pos, end)'.format(m.name)) text.append('do_decode_advance({0}, pos, end)'.format(m.byte_size)) elif m.greedy: @@ -374,7 +389,7 @@ def generate_struct_decode(node): elif m.name in bound: b = bound[m.name] mcpptype = _get_cpp_builtin_type(m) - if b.dynamic: + if b.is_dynamic: text.append('do_decode_resize(x.{0}, pos, end)'.format(b.name, mcpptype)) else: text.append('do_decode_resize(x.{0}, pos, end, {1})'.format(b.name, b.size, mcpptype)) @@ -392,16 +407,16 @@ def generate_struct_print(node): text = '' bound = {m.bound: m for m in node.members if m.bound} for m in node.members: - if m.array: - if m.fixed: + if m.is_array: + if m.is_fixed: inner = 'x.{0}.data(), size_t({1})'.format(m.name, m.size) - elif m.dynamic: + elif m.is_dynamic: inner = 'x.{0}.data(), x.{0}.size()'.format(m.name) - elif m.limited: + elif m.is_limited: inner = 'x.{0}.data(), std::min(x.{0}.size(), size_t({1}))'.format(m.name, m.size) elif m.greedy: inner = 'x.{0}.data(), x.{0}.size()'.format(m.name) - if m.type_ == 'byte': + if m.type_name == 'byte': inner = inner.join(('std::make_pair(', ')')) text += 'do_print(out, indent, "{0}", {1});\n'.format(m.name, inner) elif m.optional: @@ -422,12 +437,12 @@ def generate_struct_get_byte_size(node): elems = [] for m in node.members: if m.kind == model.Kind.FIXED: - if m.dynamic or m.greedy: + if m.is_dynamic or m.greedy: elems += ['{0}.size() * {1}'.format(m.name, _get_byte_size(m))] else: bytes_ += m.byte_size + m.padding else: - if m.dynamic or m.greedy: + if m.is_dynamic or m.greedy: elems += [ 'std::accumulate({0}.begin(), {0}.end(), size_t(), prophy::detail::byte_size())'.format(m.name) ] @@ -451,20 +466,21 @@ def generate_struct_fields(node): bound = {m.bound: m for m in node.members if m.bound} text = '' for m in node.members: - if m.fixed: - text += 'array<{0}, {1}> {2};\n'.format(BUILTIN2C.get(m.type_, m.type_), m.size, m.name) - elif m.dynamic: - text += 'std::vector<{0}> {1};\n'.format(BUILTIN2C.get(m.type_, m.type_), m.name) - elif m.limited: - text += 'std::vector<{0}> {1}; /// limit {2}\n'.format(BUILTIN2C.get(m.type_, m.type_), m.name, m.size) + if m.is_fixed: + text += 'array<{0}, {1}> {2};\n'.format(BUILTIN2C.get(m.type_name, m.type_name), m.size, m.name) + elif m.is_dynamic: + text += 'std::vector<{0}> {1};\n'.format(BUILTIN2C.get(m.type_name, m.type_name), m.name) + elif m.is_limited: + text += 'std::vector<{0}> {1}; /// limit {2}\n'.format(BUILTIN2C.get(m.type_name, m.type_name), m.name, + m.size) elif m.greedy: - text += 'std::vector<{0}> {1}; /// greedy\n'.format(BUILTIN2C.get(m.type_, m.type_), m.name) + text += 'std::vector<{0}> {1}; /// greedy\n'.format(BUILTIN2C.get(m.type_name, m.type_name), m.name) elif m.optional: - text += 'optional<{0}> {1};\n'.format(BUILTIN2C.get(m.type_, m.type_), m.name) + text += 'optional<{0}> {1};\n'.format(BUILTIN2C.get(m.type_name, m.type_name), m.name) elif m.name in bound: pass else: - text += '{0} {1};\n'.format(BUILTIN2C.get(m.type_, m.type_), m.name) + text += '{0} {1};\n'.format(BUILTIN2C.get(m.type_name, m.type_name), m.name) return text @@ -474,19 +490,19 @@ def add_to_default(lst, m, init): lst.append('{0}({1})'.format(m.name, init)) def add_to_full(lst, const_ref, m, fmt='{0}'): - lst.append((const_ref, fmt.format(BUILTIN2C.get(m.type_, m.type_)), m.name)) + lst.append((const_ref, fmt.format(BUILTIN2C.get(m.type_name, m.type_name)), m.name)) bound = {m.bound: m for m in node.members if m.bound} default_ctor = [] full_ctor = [] for m in node.members: init = _get_initializer(m) - if m.fixed: + if m.is_fixed: add_to_default(default_ctor, m, '') add_to_full(full_ctor, True, m, 'array<{0}, %s>' % m.size) - elif m.dynamic: + elif m.is_dynamic: add_to_full(full_ctor, True, m, 'std::vector<{0}>') - elif m.limited: + elif m.is_limited: add_to_full(full_ctor, True, m, 'std::vector<{0}>') elif m.greedy: add_to_full(full_ctor, True, m, 'std::vector<{0}>') @@ -498,15 +514,15 @@ def add_to_full(lst, const_ref, m, fmt='{0}'): add_to_default(default_ctor, m, init) add_to_full(full_ctor, init is None, m) return ( - (default_ctor and - '{0}(): {1} {{ }}\n'.format(node.name, ', '.join(default_ctor)) or - '{0}() {{ }}\n'.format(node.name)) + - '{0}({1}): {2} {{ }}\n'.format( - node.name, - ', '.join(_const_refize(const_ref, tp) + ' _%s' % - idx for idx, (const_ref, tp, _) in enumerate(full_ctor, 1)), - ', '.join('{0}(_{1})'.format(name, idx) for idx, (_, _, name) in enumerate(full_ctor, 1)) - ) + (default_ctor and + '{0}(): {1} {{ }}\n'.format(node.name, ', '.join(default_ctor)) or + '{0}() {{ }}\n'.format(node.name)) + + '{0}({1}): {2} {{ }}\n'.format( + node.name, + ', '.join(_const_refize(const_ref, tp) + ' _%s' % + idx for idx, (const_ref, tp, _) in enumerate(full_ctor, 1)), + ', '.join('{0}(_{1})'.format(name, idx) for idx, (_, _, name) in enumerate(full_ctor, 1)) + ) ) @@ -518,14 +534,14 @@ def gen_case(member): .format(node.name, member.name)) return ( - 'if (!do_decode(x.discriminator, pos, end)) return false;\n' + - (discpad and 'if (!do_decode_advance({0}, pos, end)) return false;\n'.format(discpad) or '') + - 'switch (x.discriminator)\n' + - '{\n' + - ''.join(' ' + gen_case(m) for m in node.members) + - ' ' + 'default: return false;\n' + - '}\n' + - 'return do_decode_advance({0}, pos, end);\n'.format(node.byte_size - DISC_SIZE - discpad) + 'if (!do_decode(x.discriminator, pos, end)) return false;\n' + + (discpad and 'if (!do_decode_advance({0}, pos, end)) return false;\n'.format(discpad) or '') + + 'switch (x.discriminator)\n' + + '{\n' + + ''.join(' ' + gen_case(m) for m in node.members) + + ' ' + 'default: return false;\n' + + '}\n' + + 'return do_decode_advance({0}, pos, end);\n'.format(node.byte_size - DISC_SIZE - discpad) ) @@ -537,13 +553,13 @@ def gen_case(member): .format(node.name, member.name)) return ( - 'pos = do_encode(pos, x.discriminator);\n' + - (discpad and 'pos = pos + {0};\n'.format(discpad) or '') + - 'switch (x.discriminator)\n' + - '{\n' + - ''.join(' ' + gen_case(m) for m in node.members) + - '}\n' + - 'pos = pos + {0};\n'.format(node.byte_size - DISC_SIZE - discpad) + 'pos = do_encode(pos, x.discriminator);\n' + + (discpad and 'pos = pos + {0};\n'.format(discpad) or '') + + 'switch (x.discriminator)\n' + + '{\n' + + ''.join(' ' + gen_case(m) for m in node.members) + + '}\n' + + 'pos = pos + {0};\n'.format(node.byte_size - DISC_SIZE - discpad) ) @@ -553,10 +569,10 @@ def gen_case(member): .format(node.name, member.name)) return ( - 'switch (x.discriminator)\n' + - '{\n' + - ''.join(' ' + gen_case(m) for m in node.members) + - '}\n' + 'switch (x.discriminator)\n' + + '{\n' + + ''.join(' ' + gen_case(m) for m in node.members) + + '}\n' ) @@ -574,26 +590,22 @@ def generate_union_fields(node): 'static const prophy::detail::int2type discriminator_{0}_t;\n'.format(m.name) for m in node.members ) - fields = ''.join('{0} {1};\n'.format(BUILTIN2C.get(m.type_, m.type_), m.name) for m in node.members) + fields = ''.join('{0} {1};\n'.format(BUILTIN2C.get(m.type_name, m.type_name), m.name) for m in node.members) return 'enum _discriminator\n{\n' + _indent(body) + '} discriminator;\n' + '\n' + disc_defs + '\n' + fields def generate_union_constructor(node): inits = [_get_initializer(m) for m in node.members] return ( - '{0}(): {1} {{ }}\n'.format(node.name, ', '.join( - ['discriminator(discriminator_{0})'.format(node.members[0].name)] + - ['{0}({1})'.format(m.name, init) for m, init in zip(node.members, inits) if init is not None] - )) + - ''.join( - '{0}(prophy::detail::int2type, {2} _1): discriminator(discriminator_{1}), {1}(_1) {{ }}\n' - .format( - node.name, - m.name, - _const_refize(init is None, BUILTIN2C.get(m.type_, m.type_)) + '{0}(): {1} {{ }}\n'.format(node.name, ', '.join( + ['discriminator(discriminator_{0})'.format(node.members[0].name)] + + ['{0}({1})'.format(m.name, init) for m, init in zip(node.members, inits) if init is not None] + )) + + ''.join( + '{0}(prophy::detail::int2type, {2} _1): discriminator(discriminator_{1}), {1}(_1) {{ }}\n' + .format(node.name, m.name, _const_refize(init is None, BUILTIN2C.get(m.type_name, m.type_name))) + for m, init in zip(node.members, inits) ) - for m, init in zip(node.members, inits) - ) ) diff --git a/prophyc/generators/prophy.py b/prophyc/generators/prophy.py new file mode 100644 index 0000000..a179932 --- /dev/null +++ b/prophyc/generators/prophy.py @@ -0,0 +1,151 @@ +from collections import namedtuple + +from prophyc.generators import base, word_wrap + +INDENT_STR = " " +MAX_LINE_WIDTH = 100 + +DocStr = namedtuple("DocStr", "block, inline") + + +def _form_doc(model_node, max_inl_docstring_len, indent_level): + block_doc, inline_doc = "", "" + if model_node.docstring: + if len(model_node.docstring) <= max_inl_docstring_len and "\n" not in model_node.docstring: + inline_doc = u" // {}".format(model_node.docstring) + + elif model_node.docstring: + block_doc = u"\n" + "".join( + _gen_multi_line_doc(model_node.docstring, indent_level=indent_level, block_header=model_node.name)) + + return DocStr(block_doc, inline_doc) + + +schema_line_breaker = word_wrap.BreakLinesByWidth(MAX_LINE_WIDTH, " ", "/* ", " * ", " ", " */") + + +@schema_line_breaker +def _gen_multi_line_doc(block_comment_text, indent_level=0, block_header=""): + assert "\n" not in block_header, "Will not work with line breaks in header bar." + + if block_header: + if len(block_comment_text) >= 250: + schema_line_breaker.make_a_bar("-" if indent_level else "=", block_header) + yield block_header + + for paragraph in block_comment_text.split("\n"): + yield paragraph + + +def _columnizer(model_node, column_splitter, max_line_width=100): + members_table = [column_splitter(m) for m in model_node.members] + widths = [max(len(str(r)) for r in g) for g in zip(*members_table)] + max_inline_comment_width = max_line_width - sum(widths) + + for member, columns in zip(model_node.members, members_table): + doc = _form_doc(member, max_inline_comment_width, indent_level=1) + + if doc.block: + yield doc.block + yield "\n" + INDENT_STR + + for is_not_last, (cell_width, cell_str) in enumerate(zip(widths, columns), 1 - len(columns)): + + yield cell_str + + padding = " " * (max(0, cell_width - len(cell_str))) + if is_not_last: + yield padding + elif doc.inline: + yield padding + doc.inline + + if model_node.members: + yield "\n" + + +def generate_schema_container(model_node, designator, column_splitter): + if model_node.docstring: + block_docstring = "".join(_gen_multi_line_doc(model_node.docstring, indent_level=0, + block_header=model_node.name)) + if block_docstring: + block_docstring += "\n" + else: + block_docstring = "" + members = "".join(_columnizer(model_node, column_splitter, max_line_width=100)) + return "{}{} {} {{{}}};".format(block_docstring, designator, model_node.name, members) + + +class SchemaTranslator(base.TranslatorBase): + block_template = u'''{content}''' + + @staticmethod + def translate_include(include): + doc = _form_doc(include, 50, indent_level=0) + return u"{d.block}#include \"{0.name}\"{d.inline}".format(include, d=doc) + + @staticmethod + def translate_constant(constant): + doc = _form_doc(constant, max_inl_docstring_len=50, indent_level=0) + return u"{d.block}\n{0.name} = {0.value};{d.inline}".format(constant, d=doc) + + @staticmethod + def translate_enum(enumerator): + def column_selector(member): + value = " = {};".format(member.value) + return member.name, value + + return generate_schema_container(enumerator, "enum", column_selector) + + @staticmethod + def translate_struct(struct): + def column_selector(member): + type_ = member.value + if member.optional: + type_ += "*" + + if member.is_fixed: + name = '{m.name}[{m.size}];' + elif member.is_limited: + name = '{m.name}<{m.size}>;' + elif member.is_dynamic: + name = '{m.name}<@{m.bound}>;' + elif member.greedy: + name = '{m.name}<...>;' + else: + name = '{m.name};' + + return type_, " ", name.format(m=member) + + return generate_schema_container(struct, "struct", column_selector) + + @staticmethod + def translate_union(union): + def column_selector(member): + discriminator = "{}: ".format(member.discriminator) + field_type = member.value + field_name = " {};".format(member.name) + return discriminator, field_type, field_name + + return generate_schema_container(union, "union", column_selector) + + @classmethod + def _make_lines_splitter(cls, previous_node_type, current_node_type): + if not previous_node_type: + return "" + + if previous_node_type == "Include" and current_node_type != "Include": + return "\n\n" + + if previous_node_type in ("Struct", "Union") or current_node_type in ("Enum", "Struct", "Union"): + return "\n\n\n" + + if previous_node_type != current_node_type: + return "\n\n" + + return "\n" + + +class SchemaGenerator(base.GeneratorBase): + top_level_translators = { + '.prophy': SchemaTranslator, + } diff --git a/prophyc/generators/python.py b/prophyc/generators/python.py index 676f56e..8114da9 100644 --- a/prophyc/generators/python.py +++ b/prophyc/generators/python.py @@ -1,7 +1,6 @@ - +from prophyc.generators import word_wrap from prophyc.generators.base import GeneratorBase, TranslatorBase - libname = "prophy" primitive_types = {x + y: "%s.%s" % (libname, x + y) for x in "uir" for y in ["8", "16", "32", "64"]} primitive_types['byte'] = '%s.u8' % libname @@ -10,15 +9,7 @@ def _make_list(repr_function, members): if not members: return "[]" - - single_indent = " " * 4 - member_indent = 2 * single_indent - - def make_lines(): - for e in members: - yield "\n%s%s" % (member_indent, repr_function(e)) - - return "[%s\n%s]" % (",".join(make_lines()), single_indent) + return "[%s\n ]" % "".join("\n %s," % repr_function(e) for e in members) def _form_enum_member(member): @@ -30,16 +21,16 @@ def _generate_enum_constants(members): def _form_struct_member(member): - prefixed_type = primitive_types.get(member.type_, member.type_) + prefixed_type = primitive_types.get(member.type_name, member.type_name) if member.optional: prefixed_type = "%s.optional(%s)" % (libname, prefixed_type) - if member.array: + if member.is_array: elem_strs = [] if member.bound: - elem_strs.append("bound = '%s'" % member.bound) + elem_strs.append("bound='%s'" % member.bound) if member.size: - elem_strs.append("size = %s" % member.size) - if member.type_ == 'byte': + elem_strs.append("size=%s" % member.size) + if member.type_name == 'byte': prefixed_type = '%s.bytes(%s)' % (libname, ', '.join(elem_strs)) else: prefixed_type = '%s.array(%s)' % (libname, ', '.join([prefixed_type] + elem_strs)) @@ -47,60 +38,110 @@ def _form_struct_member(member): def _form_union_member(member): - prefixed_type = "%s.%s" % (libname, member.type_) if member.type_ in primitive_types else member.type_ + prefixed_type = "%s.%s" % (libname, member.type_name) if member.type_name in primitive_types else member.type_name return "('%s', %s, %s)" % (member.name, prefixed_type, member.discriminator) -ENUM_TEMPLATE = """\ -class {enum_name}({libname}.with_metaclass({libname}.enum_generator, {libname}.enum)): - _enumerators = {members_list} - -{constants}""" +import_breaker = word_wrap.BreakLinesByWidth(80, " ", "", " ", " ") -STRUCT_TEMPLATE = """\ -class {struct_name}({libname}.with_metaclass({libname}.struct_generator, {libname}.struct)): - _descriptor = {members_list}""" -UNION_TEMPLATE = """\ -class {union_name}({libname}.with_metaclass({libname}.union_generator, {libname}.union)): - _descriptor = {members_list}""" +class _PythonTranslator(TranslatorBase): + block_template = """\ +# This file has been generated by prophyc. -PYTHON_FILE_TEMPLATE = """\ import {0} {{content}}""".format(libname) + def __init__(self): + self.included_symbols = set() -class _PythonTranslator(TranslatorBase): - block_template = PYTHON_FILE_TEMPLATE + def __call__(self, nodes, base_name): + self.included_symbols = set() + return super(_PythonTranslator, self).__call__(nodes, base_name) def translate_include(self, include): - return "from %s import *" % include.name.split("/")[-1] - - def translate_constant(self, constant): - return "%s = %s" % constant - - def translate_typedef(self, typedef): - if typedef.type_ in primitive_types: - value = "{0}.{1}".format(libname, typedef.type_) + included = list(sorted(n.name for n in include.defined_symbols() if n.name not in self.included_symbols)) + self.included_symbols.update(included) + if not included: + return "" + statement_begin = "from %s import " % include.name.split("/")[-1] + symbols = ", ".join(included) + if len(statement_begin) + len(symbols) <= 80: + return statement_begin + symbols + + @import_breaker + def importer(): + yield symbols + + return "%s(\n%s)" % (statement_begin, "".join(importer())) + + @staticmethod + def translate_constant(constant): + line = "%s = %s" % (constant.name, constant.value) + doc = constant.docstring + if doc: + # todo: it's a false assumption that it fits in single line + line += " \'\'\'{}\'\'\'".format(doc) + return line + + @staticmethod + def translate_typedef(typedef): + if typedef.type_name in primitive_types: + value = "{0}.{1}".format(libname, typedef.type_name) else: - value = typedef.type_ + value = typedef.type_name return "%s = %s" % (typedef.name, value) - def translate_enum(self, enum): + @staticmethod + def translate_enum(enum): members_list = _make_list(_form_enum_member, enum.members) constants_list = _generate_enum_constants(enum.members) - return ENUM_TEMPLATE.format(libname=libname, enum_name=enum.name, members_list=members_list, - constants=constants_list) + template = """\ +class {enum_name}({libname}.with_metaclass({libname}.enum_generator, {libname}.enum)): + _enumerators = {members_list} + - def translate_struct(self, struct): +{constants}""" + return template.format(libname=libname, enum_name=enum.name, members_list=members_list, + constants=constants_list) + + @staticmethod + def translate_struct(struct): members_list = _make_list(_form_struct_member, struct.members) - return STRUCT_TEMPLATE.format(libname=libname, struct_name=struct.name, members_list=members_list) + template = """\ +class {struct_name}({libname}.with_metaclass({libname}.struct_generator, {libname}.struct)): + _descriptor = {members_list}""" + return template.format(libname=libname, struct_name=struct.name, members_list=members_list) - def translate_union(self, union): + @staticmethod + def translate_union(union): members_list = _make_list(_form_union_member, union.members) - return UNION_TEMPLATE.format(libname=libname, union_name=union.name, members_list=members_list) + union_template = """\ +class {union_name}({libname}.with_metaclass({libname}.union_generator, {libname}.union)): + _descriptor = {members_list}""" + return union_template.format(libname=libname, union_name=union.name, members_list=members_list) + + @classmethod + def _make_lines_splitter(cls, previous_node_type, current_node_type): + if not previous_node_type: + return "" + + if previous_node_type == "Include" and current_node_type != "Include": + return "\n\n" + + if previous_node_type in ("Struct", "Union") or current_node_type in ("Enum", "Struct", "Union"): + return "\n\n\n" + + if previous_node_type == "Enum" and current_node_type == "Constant": + # Enum ends with several constants, so they join as the same type + return "\n" + + if previous_node_type != current_node_type: + return "\n\n" + + return "\n" class PythonGenerator(GeneratorBase): diff --git a/prophyc/generators/word_wrap.py b/prophyc/generators/word_wrap.py new file mode 100644 index 0000000..780fb52 --- /dev/null +++ b/prophyc/generators/word_wrap.py @@ -0,0 +1,169 @@ +from collections import deque +from contextlib import contextmanager +from functools import wraps + +from prophyc import six + +BREAKABLE_SPACE = " " + + +class BreakLinesByWidth(object): + def __init__( + self, + max_line_width=100, + indent_str=" ", + block_start_token="", + hard_indent_tail="", + soft_indent_tail="", + block_end_token="", + ): + self.max_line_width = max_line_width + self.indent_str = indent_str + self.hard_indent_tail = hard_indent_tail + self.soft_indent_tail = soft_indent_tail + self.block_start_token = block_start_token + self.block_end_token = block_end_token + + self.indent_level = None + self.line_abs_pos = None + self.line_rel_pos = None + self._markup_queue = None + + def _init(self, indent_level): + self.indent_level = indent_level + self.line_abs_pos = 0 + self.line_rel_pos = 0 + self._markup_queue = deque() + + def __call__(self, decorated_generator): + @wraps(decorated_generator) + def sub_generator(*p, **k): + """ Is supposed to decorate a generator that: + - yields paragraphs + - and controls markup elements via the decorator class instance. + """ + self._init(k.pop('indent_level', 0)) + + if self.block_start_token: + self.open_block() + + for paragraph in decorated_generator(*p, **k): + while self._markup_queue: + yield self._markup_queue.popleft() + + if paragraph: + with self.being_a_paragraph(): + soft_lines = paragraph.split("\n") + for is_not_last, soft_line in enumerate(soft_lines, 1 - len(soft_lines)): + for word in soft_line.split(BREAKABLE_SPACE): + if self.line_rel_pos > 0: + if self.line_would_overflow(word): + self.break_line() + self.make_soft_line_indent() + + if self.line_rel_pos > 0 or word == "": + self._advance(BREAKABLE_SPACE) + + self._advance(word) + if is_not_last: + self.break_line() + self.make_soft_line_indent() + + while self._markup_queue: + yield self._markup_queue.popleft() + + if self.block_end_token: + self.close_block() + + while self._markup_queue: + yield self._markup_queue.popleft() + + return sub_generator + + def line_would_overflow(self, word): + return self.line_abs_pos + len(word) >= self.max_line_width + + def make_a_bar(self, char_="=", title=""): + assert isinstance(char_, six.string_types), "Bar character has to be a string." + assert len(char_) == 1, "Bar character has to be a single character." + line = "{} {} ".format(char_ * 4, title) if title else "" + padding_width = self.max_line_width - len(line) - self._hard_indent_len + bar = line + char_ * padding_width + + with self.being_a_paragraph(): + self._advance(bar) + + @contextmanager + def being_a_paragraph(self): + if not self.block_just_started: + self.make_paragraph_indent() + try: + yield + finally: + self.break_line() + + @property + def block_just_started(self): + return self.line_abs_pos == self._hard_indent_len + + def break_line(self): + self.line_abs_pos = 0 + self.line_rel_pos = 0 + self._markup_queue.append("\n") + + def _make_indent_w_tail(self, tail): + self._advance(self._base_indent() + tail, False) + + def make_paragraph_indent(self): + self._make_indent_w_tail(self.hard_indent_tail) + + def make_soft_line_indent(self): + self._make_indent_w_tail(self.soft_indent_tail) + + def open_block(self): + self._make_indent_w_tail(self.block_start_token) + + def close_block(self): + if not self.block_just_started: + self._make_indent_w_tail(self.block_end_token) + else: + self._advance(self.block_end_token, False) + + def _base_indent(self): + return self.indent_str * self.indent_level + + @property + def _hard_indent_len(self): + return len(self._base_indent()) + len(self.hard_indent_tail) + + def _advance(self, text, increment_rel_pos=True): + self.line_abs_pos += len(text) + if increment_rel_pos: + # intended to avoid counting indentation length + self.line_rel_pos += len(text) + self._markup_queue.append(text) + + +def split_long_string(long_string, max_line_width=80): + lines = long_string.split("\n") + if not any(len(line) > max_line_width for line in lines): + for not_last, line in enumerate(lines, 1 - len(lines)): + yield line + ("\n" if not_last else "") + else: + words = long_string.split(" ") + if not any(len(word) > max_line_width for word in words): + line, pos = "", 0 + for not_last, word in enumerate(words, 1 - len(words)): + word += " " if not_last else "" + line += word + pos += len(word) + if pos >= 80: + yield line + line, pos = "", 0 + if line: + yield line + else: + pos = 0 + while pos < len(long_string): + yield long_string[pos:pos + max_line_width] + pos += max_line_width diff --git a/prophyc/model.py b/prophyc/model.py index 3b99edb..9dff99f 100644 --- a/prophyc/model.py +++ b/prophyc/model.py @@ -1,8 +1,8 @@ -from collections import namedtuple from itertools import islice -from .six import ifilter, reduce -from . import calc +import renew + +from . import calc, six """ Exception types """ @@ -14,8 +14,13 @@ def __init__(self, errors): """Collection of 2-tuples of location and message.""" +class ModelError(Exception): + """ General error for model tree construction errors. """ + pass + + class Kind: - """ Determines struct member wire format type. """ + """ Determines struct member wire format type (bytes stiffness). """ FIXED = 0 DYNAMIC = 1 UNLIMITED = 2 @@ -42,195 +47,332 @@ class Kind: """ Enum byte size. """ ENUM_SIZE = BUILTIN_SIZES['u32'] -""" Model consists of 6 kinds of symbols: -Includes, Constants, Enums, Typedefs, Structs, Unions. """ +model_repr = renew.make_renew_reprs(namespace="prophyc.model") -Include = namedtuple("Include", ["name", "nodes"]) -Constant = namedtuple("Constant", ["name", "value"]) +@model_repr +class ModelNode(object): + """ The lowermost base for each type of prophyc model. """ + __slots__ = "name", "_value", "docstring" + _eq_attributes = "name", "_value" + _str_pattern = None + def __init__(self, name, value, docstring=None): + self.name = _check_string(name, "model node name") + self._value = value + self.docstring = _check_string(docstring, "doc string") -class Enum(object): + @property + def value(self): + return self._value + + def __str__(self): + return self.schema_repr() - def __init__(self, name, members): - self.name = name - self.members = members + def schema_repr(self): + return self._str_pattern.format(s=self) def __eq__(self, other): - return ((self.name == other.name) and - (self.members == other.members)) + if isinstance(other, self.__class__): + return all(getattr(self, a) == getattr(other, a) for a in self._eq_attributes) - def __repr__(self): - return self.name + ''.join(('\n {}'.format(x) for x in self.members)) + '\n' + if isinstance(other, tuple) and len(other) in (2, 3): + return all(getattr(self, a) == other[i] for i, a in enumerate(self._eq_attributes)) + def __ne__(self, other): + return not self.__eq__(other) -class EnumMember(object): + def dependencies(self): + raise NotImplementedError("To be overridden in %s class." % self.__class__.__name__) - def __init__(self, name, value): - self.name = name - self.value = value - def __eq__(self, other): - return ((self.name == other.name) and - (self.value == other.value)) +class Constant(ModelNode): + _str_pattern = "const {s.name} = {s.value!r};" + __slots__ = () - def __repr__(self): - return '{0} {1}'.format(self.name, self.value) + def eval_int(self, all_constants): + try: + return int(self.value) + except ValueError: + try: + return calc.eval(self.value, all_constants) + except calc.ParseError: + return None + def dependencies(self): + def sub_(x, y): + return x.replace(y, " ") -class Typedef(object): + for symbol in six.reduce(sub_, "()+-", self.value).split(): + if not symbol.isdigit(): + yield symbol - def __init__(self, name, type_, **kwargs): - self.name = name - self.type_ = type_ - if 'definition' in kwargs: - self.definition = kwargs['definition'] - def __eq__(self, other): - return ((self.name == other.name) and - (self.type_ == other.type_)) +class EnumMember(Constant): + _str_pattern = "{s.name} = {s.value!r};" + __slots__ = () - def __repr__(self): - return '{0} {1}'.format(self.type_, self.name) +class _Serializable(ModelNode): + __slots__ = "kind", "byte_size", "alignment" -class Struct(object): + def __init__(self, name, value, docstring=None): + super(_Serializable, self).__init__(name, value, docstring) + self.alignment = None + """byte size of field influenced by array: multiplied by fixed/limited size, 0 if dynamic/greedy""" + self.byte_size = None + """type kind, not influenced by array or optional""" + self.kind = None + self.calc_wire_stiffness() - def __init__(self, name, members): - self.name = name - self.members = members + @classmethod + def calc_wire_stiffness(cls): + raise NotImplementedError("Abstract method to be overriden in %s" % cls.__name__) - self.kind = evaluate_struct_kind(self) - self.byte_size = None - """byte size of complete struct, dynamic/greedy arrays assumed empty""" +@model_repr +class Typedef(_Serializable): + _str_pattern = "typedef {s.type_name} {s.name};" + _eq_attributes = "name", "type_name", "definition" + __slots__ = "definition", - self.alignment = None + def __init__(self, name, type_name, definition=None, docstring=None): + if definition is not None: + if not isinstance(definition, six.string_types + (Typedef, Enum, Struct, Union)): + msg = "{}.definition should be string, Typedef, Enum, Struct or Union, got: {}." + raise ModelError(msg.format(self.__class__.__name__, type(definition).__name__)) + self.definition = definition + super(Typedef, self).__init__(name, type_name, docstring) - def __eq__(self, other): - return ((self.name == other.name) and - (self.members == other.members)) + @property + def type_name(self): + return self._value + + @type_name.setter + def type_name(self, new_value): + _check_string(new_value, "Type designator") + self._value = new_value + + @property + def lowermost_typedef(self): + lowermost = self.definition + while isinstance(lowermost, Typedef): + lowermost = lowermost.definition + return lowermost + + def calc_wire_stiffness(self): + """ Typedef propagates stiffness kinds of its lowermost type """ + self.kind = Kind.FIXED + if self.definition and isinstance(self.lowermost_typedef, Struct): + self.kind = self.lowermost_typedef.kind - def __repr__(self): - return self.name + ''.join(('\n {}'.format(x) for x in self.members)) + '\n' + def dependencies(self): + yield self.type_name -class StructMember(object): +@model_repr +class StructMember(Typedef): + __slots__ = "bound", "size", "greedy", "optional", "numeric_size", "padding" + _eq_attributes = "name", "_value", "bound", "size", "greedy", "optional", "definition" - def __init__(self, name, type_, - bound=None, size=None, - unlimited=False, optional=False, - definition=None): - assert(sum((bool(bound or size), unlimited, optional)) <= 1) + def __init__(self, name, type_name, definition=None, bound=None, size=None, greedy=False, optional=False, + docstring=None): + assert sum((bool(bound or size), greedy, optional)) <= 1, "Over-constraint" + assert isinstance(optional, bool), "'optional' argument value has to be boolean" + assert isinstance(greedy, bool), "'greedy' argument value has to be boolean" - self.name = name - self.type_ = type_ - self.array = bool(bound or size or unlimited) + super(StructMember, self).__init__(name, type_name, definition, docstring) self.bound = bound self.size = size + self.greedy = greedy self.optional = optional - self.definition = definition - - self.numeric_size = None """integral number indicating array size (it may be a string with enum member or constant name)""" - - self.kind = evaluate_member_kind(self) - """type kind, not influenced by array or optional""" - - self.byte_size = None - """byte size of field influenced by array: multiplied by fixed/limited size, 0 if dynamic/greedy""" - - self.alignment = None - - self.padding = None + self.numeric_size = None """amount of bytes to add before next field. If field dynamic: negative alignment of next field""" + self.padding = None - def __eq__(self, other): - return ((self.name == other.name) and - (self.type_ == other.type_) and - (self.array == other.array) and - (self.bound == other.bound) and - (self.size == other.size) and - (self.optional == other.optional)) - - def __repr__(self): - fmts = { - (False, False, False, False): '{0} {1}', - (True, False, True, False): '{0} {1}[{3}]', - (True, True, False, False): '{0} {1}<>({2})', - (True, True, True, False): '{0} {1}<{3}>({2})', - (True, False, False, False): '{0} {1}<...>', - (False, False, False, True): '{0}* {1}' - } - fmt = fmts[(self.array, bool(self.bound), bool(self.size), self.optional)] - return fmt.format(self.type_, self.name, self.bound, self.size) + @property + def is_array(self): + return self.bound or self.size or self.greedy @property - def fixed(self): + def is_fixed(self): return not self.bound and self.size @property - def dynamic(self): + def is_limited(self): + return self.bound and self.size + + @property + def is_dynamic(self): return self.bound and not self.size + def __str__(self): + return self.schema_repr() + + def schema_repr(self): + if self.optional: + return '{s.type_name}* {s.name};'.format(s=self) + if self.is_fixed: + return '{s.type_name} {s.name}[{s.size}];'.format(s=self) + if self.is_limited: + return '{s.type_name} {s.name}<{s.size}>;'.format(s=self) + if self.is_dynamic: + return '{s.type_name} {s.name}<@{s.bound}>;'.format(s=self) + if self.greedy: + return '{s.type_name} {s.name}<...>;'.format(s=self) + + return '{s.type_name} {s.name};'.format(s=self) + + +@model_repr +class UnionMember(Typedef): + _str_pattern = '{s.discriminator}: {s.type_name} {s.name};' + _eq_attributes = "name", "type_name", "discriminator" + __slots__ = "discriminator", + + def __init__(self, name, type_name, discriminator, definition=None, docstring=None): + super(UnionMember, self).__init__(name, type_name, definition, docstring) + self.discriminator = discriminator + + +""" Composite kinds """ + + +@model_repr +class _Container(ModelNode): + """ Anything that represents a collection of members. """ + _member_type_restriction = None + _collection_kind = None + _str_pattern = "{s._collection_kind} {s.name} {{\n{s._str_members}}};\n" + __slots__ = () + + def __init__(self, name, members, docstring=None): + super(_Container, self).__init__(name, members, docstring) + self._check_members_type(members) + self._check_members_duplication(members) + + def _check_members_type(self, members): + if not isinstance(members, list): + msg = "{s._collection_kind} '{s.name}' members must be a list, got {t} instead." + raise ModelError(msg.format(s=self, t=type(members).__name__)) + + for index, member in enumerate(members): + if not isinstance(member, self._member_type_restriction): + msg = "Each member of {s._collection_kind} '{s.name}' has to be a {et} instance. Got {gt} at index {i}." + raise ModelError( + msg.format(s=self, et=self._member_type_restriction.__name__, gt=type(member).__name__, i=index)) + + def _check_members_duplication(self, members): + meet_identifiers = set() + for member in members: + if member.name in meet_identifiers: + msg = "Duplicated '{m.name}' identifier in {s._collection_kind} {s.name}." + raise ModelError(msg.format(m=member, s=self)) + meet_identifiers.add(member.name) + @property - def limited(self): - return self.bound and self.size + def value(self): + forbidden = "Use of value property is forbidden for {0}. Use '{0}.members' instead." + raise ModelError(forbidden.format(self.__class__.__name__)) @property - def greedy(self): - return self.array and not self.bound and not self.size + def members(self): + return self._value + @property + def _str_members(self): + return "".join(" {}\n".format(m) for m in self.members) -class Union(object): + def dependencies(self): + for member in self.members: + for dependency in member.dependencies(): + yield dependency - def __init__(self, name, members): - self.name = name - self.members = members - self.kind = Kind.FIXED - self.byte_size = None - self.alignment = None +class Include(_Container): + _member_type_restriction = ModelNode + _collection_kind = "include" + _str_pattern = "#include {s.name};\n" + __slots__ = () - def __eq__(self, other): - return ((self.name == other.name) and - (self.members == other.members)) + def _check_members_duplication(self, members): + """ Include is allowed to get duplicated definitions. """ + + def dependencies(self): + """ No need to provide any dependencies to include. """ + return [] - def __repr__(self): - return self.name + ''.join(('\n {}'.format(x) for x in self.members)) + '\n' + def defined_symbols(self): + """ Generate collection of symbols delivered by this include. """ + for member in self.members: + if isinstance(member, Include): + for symbol in member.defined_symbols(): + yield symbol + else: + yield member -class UnionMember(object): +class Enum(_Container): + _member_type_restriction = EnumMember + _collection_kind = "enum" + __slots__ = () - def __init__(self, name, type_, discriminator, - definition=None): - self.name = name - self.type_ = type_ - self.discriminator = discriminator + def dependencies(self): + for member in self.members: + yield member.name - self.definition = definition - self.kind = evaluate_member_kind(self) - self.byte_size = None - self.alignment = None - def __eq__(self, other): - return ((self.name == other.name) and - (self.type_ == other.type_) and - (self.discriminator == other.discriminator)) +class _SerializableContainer(_Container, _Serializable): + __slots__ = () - def __repr__(self): - return '{0}: {1} {2}'.format(self.discriminator, self.type_, self.name) + def calc_wire_stiffness(self): + self._check_members_type(self.members) + """ Evaluate stiffness of the node. """ + self.kind = Kind.FIXED + if isinstance(self, Struct): + if self.members: + for member in self.members: + member.calc_wire_stiffness() + + if self.members[-1].greedy: + self.kind = Kind.UNLIMITED + elif any(x.is_dynamic for x in self.members): + self.kind = Kind.DYNAMIC + else: + self.kind = max(x.kind for x in self.members) + + +class Struct(_SerializableContainer): + _member_type_restriction = StructMember + _collection_kind = "struct" + __slots__ = () + + +class Union(_SerializableContainer): + _member_type_restriction = UnionMember + _collection_kind = "union" + __slots__ = () """ Utils """ -def split_after(nodes, pred): +def _check_string(docstring, what_): + if docstring is None: + return + if not isinstance(docstring, six.string_types): + msg = "Got {} of '{}' type, expected string." + raise ModelError(msg.format(what_, type(docstring).__name__)) + return six.decode_string(docstring) + + +def split_after(nodes, predicate): part = [] for x in nodes: part.append(x) - if pred(x): + if predicate(x): yield part part = [] if part: @@ -247,57 +389,57 @@ def null_warn(_): def topological_sort(nodes): """Sorts nodes.""" - def get_include_deps(_): - return [] - - def get_constant_deps(constant): - return filter(lambda x: not x.isdigit(), - reduce(lambda x, y: x.replace(y, " "), "()+-", constant.value).split()) - - def get_typedef_deps(typedef): - return [typedef.type_] - - def get_enum_deps(enum): - return [member.name for member in enum.members] - - def get_struct_deps(struct): - return [member.type_ for member in struct.members] - - def get_union_deps(union): - return [member.type_ for member in union.members] + def find_first_dep(dependency, start_index): + for i, n in enumerate(islice(nodes, start_index, None), start_index): + if n.name == dependency: + return i - def get_deps(node): - if isinstance(node, Include): - return get_include_deps(node) - elif isinstance(node, Constant): - return get_constant_deps(node) - elif isinstance(node, Typedef): - return get_typedef_deps(node) - elif isinstance(node, Enum): - return get_enum_deps(node) - elif isinstance(node, Struct): - return get_struct_deps(node) - elif isinstance(node, Union): - return get_union_deps(node) - - def model_sort_rotate(nodes, known, available, index): + def model_sort_rotate(): node = nodes[index] - for dep in get_deps(node): + for dep in node.dependencies(): if dep not in known and dep in available: - found_index, _ = next(ifilter(lambda x: x[1].name == dep, - enumerate(islice(nodes, index + 1, None), start=index + 1))) - nodes.insert(index, nodes.pop(found_index)) + found_index = find_first_dep(dep, index + 1) + if found_index: + nodes.insert(index, nodes.pop(found_index)) return True known.add(node.name) - return False known = set(x + y for x in "uir" for y in ["8", "16", "32", "64"]) available = set(node.name for node in nodes) - index = 0 - max_index = len(nodes) - while index < max_index: - if not model_sort_rotate(nodes, known, available, index): - index = index + 1 + for index in range(len(nodes)): + while model_sort_rotate(): + pass + + +def _make_types_index(nodes_): + """ Creates flat dictionary that maps model objects to its name. """ + included = set() + for node_ in nodes_: + if isinstance(node_, Include): + if node_.name not in included: + included.add(node_.name) + for included_name, included_type in _make_types_index(node_.members): + yield included_name, included_type + else: + yield node_.name, node_ + + +def _collect_constants(nodes_, constants=None): + constants = constants or {} + included = set() + for node_ in nodes_: + if isinstance(node_, Include) and node_.name not in included: + included.add(node_.name) + constants.update(_collect_constants(node_.members, constants)) + + elif isinstance(node_, Constant): + constants[node_.name] = node_.eval_int(constants) + + elif isinstance(node_, Enum): + for member in node_.members: + constants[member.name] = member.eval_int(constants) + + return constants def cross_reference(nodes, warn=null_warn): @@ -305,76 +447,25 @@ def cross_reference(nodes, warn=null_warn): Adds definition reference to Typedef and StructMember. Adds numeric_size to StructMember if it's a sized array. """ + types_index = dict(_make_types_index(nodes)) + constants = _collect_constants(nodes) - def get_type_defitinions(nodes): - included = set() - types = {} - - def add_nodes_level(nodes): - for node in nodes: - if isinstance(node, Include): - if node.name not in included: - included.add(node.name) - add_nodes_level(node.nodes) - else: - types[node.name] = node - - add_nodes_level(nodes) - return types - - def get_constant_definitions(nodes): - def eval_int(x, constants): - try: - return int(x) - except ValueError: - try: - return calc.eval(x, constants) - except calc.ParseError: - return None - - included = set() - constants = {} - - def add_constants_level(nodes): - for node in nodes: - if isinstance(node, Include): - if node.name not in included: - included.add(node.name) - add_constants_level(node.nodes) - elif isinstance(node, Constant): - constants[node.name] = eval_int(node.value, constants) - elif isinstance(node, Enum): - for member in node.members: - constants[member.name] = eval_int(member.value, constants) - - add_constants_level(nodes) - return constants - - types = get_type_defitinions(nodes) - constants = get_constant_definitions(nodes) - - def cross_reference_types(node): - if node.type_ in BUILTIN_SIZES: - node.definition = None + def cross_reference_types(node_): + if node_.type_name in BUILTIN_SIZES: + node_.definition = None return - found = types.get(node.type_) + found = types_index.get(node_.type_name) if not found: - warn("type '%s' not found" % node.type_) - node.definition = found + warn("type '%s' not found" % node_.type_name) + node_.definition = found - def evaluate_array_sizes(node): - def to_int(x): + def evaluate_array_sizes(node_): + if node_.size: try: - return int(x) - except ValueError: - val = constants.get(x) - return val is not None and val or calc.eval(x, constants) - if node.size: - try: - node.numeric_size = to_int(node.size) + node_.numeric_size = to_int(node_.size, constants) except calc.ParseError as e: warn(str(e)) - node.numeric_size = None + node_.numeric_size = None for node in nodes: if isinstance(node, Typedef): @@ -385,42 +476,22 @@ def to_int(x): elif isinstance(node, Union): list(map(cross_reference_types, node.members)) + return constants -def evaluate_struct_kind(node): - """Adds kind to Struct. Requires cross referenced nodes.""" - if node.members: - if node.members[-1].greedy: - return Kind.UNLIMITED - elif any(x.dynamic for x in node.members): - return Kind.DYNAMIC - else: - return max(x.kind for x in node.members) - else: - return Kind.FIXED +def to_int(x, constants): + try: + return int(x) + except ValueError: + val = constants.get(x) + return val if val is not None else calc.eval(x, constants) -def evaluate_member_kind(member): - """Adds kind to StructMember. Requires cross referenced nodes.""" - def evaluate_node_kind(node): - while isinstance(node, Typedef): - node = node.definition - if isinstance(node, Struct): - return node.kind - else: - return Kind.FIXED - if member.definition: - return evaluate_node_kind(member.definition) - else: - return Kind.FIXED - -def evaluate_kinds(nodes): +def evaluate_stiffness_kinds(nodes): """Adds kind to all Structs and StructMembers. Requires cross referenced nodes.""" for node in nodes: if isinstance(node, Struct): - for member in node.members: - member.kind = evaluate_member_kind(member) - node.kind = evaluate_struct_kind(node) + node.calc_wire_stiffness() def evaluate_sizes(nodes, warn=null_warn): @@ -428,86 +499,89 @@ def evaluate_sizes(nodes, warn=null_warn): Adds byte_size and alignment to Struct, StructMember, Union, UnionMember. Requires cross referenced nodes and evaluated kinds. """ - def evaluate_node_size(node, parent, member): - while isinstance(node, Typedef) and node.definition: - node = node.definition - if isinstance(node, (Struct, Union)): - return (node.byte_size, node.alignment) - elif isinstance(node, Enum): - return (ENUM_SIZE, ENUM_SIZE) - elif node.type_ in BUILTIN_SIZES: - byte_size = BUILTIN_SIZES[node.type_] - return (byte_size, byte_size) + + def evaluate_node_size(node_, parent, member): + while isinstance(node_, Typedef) and node_.definition: + node_ = node_.definition + if isinstance(node_, (Struct, Union)): + return node_.byte_size, node_.alignment + elif isinstance(node_, Enum): + return ENUM_SIZE, ENUM_SIZE + elif node_.type_name in BUILTIN_SIZES: + byte_size = BUILTIN_SIZES[node_.type_name] + return byte_size, byte_size else: # unknown type, e.g. empty typedef - warn('%s::%s has unknown type "%s"' % (parent.name, member.name, node.type_)) - return (None, None) + warn('%s::%s has unknown type "%s"' % (parent.name, member.name, node_.type_name)) + return None, None def evaluate_array_and_optional_size(member): - if member.array and member.byte_size is not None: + if member.is_array and member.byte_size is not None: member.byte_size = member.numeric_size and (member.byte_size * member.numeric_size) or 0 elif member.optional: member.alignment = max(DISC_SIZE, member.alignment) member.byte_size = member.byte_size + member.alignment - def evaluate_member_size(node, member): + def evaluate_member_size(node_, member): if isinstance(member, StructMember) and (member.size and member.numeric_size is None): # unknown array size - warn('%s::%s array has unknown size "%s"' % (node.name, member.name, member.size)) + warn('%s::%s array has unknown size "%s"' % (node_.name, member.name, member.size)) size_alignment = (None, None) elif member.definition: - size_alignment = evaluate_node_size(node=member.definition, parent=node, member=member) - elif member.type_ in BUILTIN_SIZES: - byte_size = BUILTIN_SIZES[member.type_] + size_alignment = evaluate_node_size(node_=member.definition, parent=node_, member=member) + + elif member.type_name in BUILTIN_SIZES: + byte_size = BUILTIN_SIZES[member.type_name] size_alignment = (byte_size, byte_size) else: # unknown type - warn('%s::%s has unknown type "%s"' % (node.name, member.name, member.type_)) + warn('%s::%s has unknown type "%s"' % (node_.name, member.name, member.type_name)) size_alignment = (None, None) member.byte_size, member.alignment = size_alignment return size_alignment != (None, None) - def evaluate_empty_size(node): - node.byte_size, node.alignment = (None, None) + def evaluate_empty_size(node_): + node_.byte_size, node_.alignment = (None, None) - def evaluate_members_sizes(node): - if not all([evaluate_member_size(node, mem) for mem in node.members]): - evaluate_empty_size(node) + def evaluate_members_sizes(node_): + if not all([evaluate_member_size(node_, mem_) for mem_ in node_.members]): + evaluate_empty_size(node_) return False return True - def evaluate_partial_padding_size(node): - parts = split_after(node.members, lambda x: (x.kind == Kind.DYNAMIC) or (x.array and not x.size)) + def evaluate_partial_padding_size(node_): + parts = split_after(node_.members, lambda m: (m.kind == Kind.DYNAMIC) or (m.is_array and not m.size)) for part in [x for x in parts][1:]: part[0].alignment = max(part[0].alignment, max(x.alignment for x in part)) - def evaluate_struct_size(node): + def evaluate_struct_size(node_): def is_member_dynamic(m): - return m.dynamic or m.greedy or m.kind != Kind.FIXED - alignment = node.members and max(x.alignment for x in node.members) or 1 + return m.is_dynamic or m.greedy or m.kind != Kind.FIXED + + alignment = node_.members and max(x.alignment for x in node_.members) or 1 byte_size = 0 - prev_member = node.members and node.members[0] or None - for member in node.members: + prev_member = node_.members and node_.members[0] or None + for member in node_.members: padding = (member.alignment - byte_size % member.alignment) % member.alignment byte_size += member.byte_size + padding if is_member_dynamic(prev_member) and (prev_member.alignment < member.alignment): - prev_member.padding = -(member.alignment) + prev_member.padding = -member.alignment else: prev_member.padding = padding prev_member = member - if node.members: + if node_.members: padding = (alignment - byte_size % alignment) % alignment byte_size += padding - if any(is_member_dynamic(m) for m in node.members): - prev_member.padding = (node.members[-1].alignment < alignment) and (-alignment) or 0 + if any(is_member_dynamic(m) for m in node_.members): + prev_member.padding = (node_.members[-1].alignment < alignment) and (-alignment) or 0 else: prev_member.padding = padding - node.byte_size, node.alignment = byte_size, alignment + node_.byte_size, node_.alignment = byte_size, alignment - def evaluate_union_size(node): - node.alignment = max(DISC_SIZE, node.members and max(x.alignment for x in node.members) or 1) - node.byte_size = (node.members and max(x.byte_size for x in node.members) or 0) + node.alignment - node.byte_size = int((node.byte_size + node.alignment - 1) / node.alignment) * node.alignment + def evaluate_union_size(node_): + node_.alignment = max(DISC_SIZE, node_.members and max(x.alignment for x in node_.members) or 1) + node_.byte_size = (node_.members and max(x.byte_size for x in node_.members) or 0) + node_.alignment + node_.byte_size = int((node_.byte_size + node_.alignment - 1) / node_.alignment) * node_.alignment for node in nodes: if isinstance(node, Struct): @@ -518,6 +592,8 @@ def evaluate_union_size(node): elif isinstance(node, Union): if evaluate_members_sizes(node): evaluate_union_size(node) + elif isinstance(node, Include): + evaluate_sizes(node.members, warn) def partition(members): @@ -530,9 +606,31 @@ def partition(members): current = main for member in members[:-1]: current.append(member) - if member.kind == Kind.DYNAMIC or member.dynamic: + if member.kind == Kind.DYNAMIC or member.is_dynamic: current = [] parts.append(current) if members: current.append(members[-1]) return main, parts + + +def evaluate_model(nodes, warn_emitter=lambda x: None): + topological_sort(nodes) + constants = cross_reference(nodes, warn_emitter) + evaluate_stiffness_kinds(nodes) + evaluate_sizes(nodes, warn_emitter) + return nodes, constants + + +class ModelParser(object): + def __init__(self, parser, patcher, emit): + self.parser = parser + self.patcher = patcher + self.emit = emit + + def __call__(self, *parse_args): + nodes = self.parser.parse(*parse_args) + if self.patcher: + self.patcher(nodes) + nodes, _ = evaluate_model(nodes, self.emit.warn) + return nodes diff --git a/prophyc/options.py b/prophyc/options.py index 586b825..b887636 100644 --- a/prophyc/options.py +++ b/prophyc/options.py @@ -34,6 +34,7 @@ def error(self, message): group.add_argument('--isar', action='store_true', help='Parse input files as isar xml.') + group.add_argument('--sack', action='store_true', help='Parse input files as sack C++.') @@ -45,7 +46,7 @@ def error(self, message): action='append', default=[], help=('Add the directory to the list of directories to be ' - 'searched for included files.')) + 'searched for included files.')) parser.add_argument('-S', '--include_isar', metavar='XMLFILE', @@ -60,14 +61,19 @@ def error(self, message): type=readable_file, help=("File with instructions changing definitions of prophy " "messages after parsing. It's needed in sack and isar " - "modes, since C++ and isar xml are unable to express " - "all prophy features.")) + "modes, since C++ and isar xml are unable to express " + "all prophy features.")) parser.add_argument('--python_out', metavar='OUT_DIR', type=readable_dir, help='Generate Python source files.') + parser.add_argument('--prophy_out', + metavar='OUT_DIR', + type=readable_dir, + help='Generate prophy schema source files.') + parser.add_argument('--cpp_out', metavar='OUT_DIR', type=readable_dir, @@ -78,6 +84,10 @@ def error(self, message): type=readable_dir, help='Generate C++ full object-based codec header and source files.') + parser.add_argument('--void_out', + action='store_true', + help='Allow compilation without generating any files.') + parser.add_argument('--version', action='store_true', help='Show version information and exit.') diff --git a/prophyc/parsers/isar.py b/prophyc/parsers/isar.py index 8a3f472..3c80a13 100644 --- a/prophyc/parsers/isar.py +++ b/prophyc/parsers/isar.py @@ -1,8 +1,7 @@ import os import xml.etree.ElementTree as ElementTree -from ..six import reduce -from prophyc import model +from prophyc import model, six from prophyc.file_processor import CyclicIncludeError, FileNotFoundError @@ -58,121 +57,164 @@ def expand_operators(string_): "64 bit float": "r64"} -def make_include(elem, process_file, warn): - path = elem.get("href") - try: - nodes = process_file(path) - except (CyclicIncludeError, FileNotFoundError) as e: - if warn: - warn(str(e)) - nodes = [] - return model.Include(os.path.splitext(path)[0], nodes) - - -def make_constant(elem): - return model.Constant(elem.get("name"), expand_operators(elem.get("value"))) - - -def make_typedef(elem): - if "type" in elem.attrib: - return model.Typedef(elem.get("name"), elem.get("type")) - elif "primitiveType" in elem.attrib and elem.get("name") not in primitive_types.values(): - return model.Typedef(elem.get("name"), primitive_types[elem.get("primitiveType")]) - - -def make_enum_member(elem): - value = elem.get('value') - try: - int_value = int(value, 0) - if int_value < 0: - value = "0x{:X}".format(0x100000000 + int_value) - except ValueError: - pass - return model.EnumMember(elem.get("name"), expand_operators(value)) - - -def _check_for_duplicates(enum): - values = set() - for m in enum.members: - if m.value in values: - raise ValueError("Duplicate Enum value in '{}', value '{}'.".format(enum.name, m.value)) - values.add(m.value) - - -def make_enum(elem): - if len(elem): - enum = model.Enum(elem.get("name"), [make_enum_member(member) for member in elem]) - _check_for_duplicates(enum) +def make_include(xml_elem, process_file, warn): + if "include" in xml_elem.tag: + path = xml_elem.get("href") + try: + nodes = process_file(path) + except (CyclicIncludeError, FileNotFoundError) as e: + if warn: + warn(str(e)) + nodes = [] + return model.Include(os.path.splitext(path)[0], nodes, docstring=get_docstr(xml_elem)) + + +def get_docstr(xml_elem): + return six.decode_string(xml_elem.get("comment", "")) or None + + +def make_constant(xml_elem): + return model.Constant( + xml_elem.get("name"), + expand_operators(xml_elem.get("value")), + docstring=get_docstr(xml_elem) + ) + + +def make_typedef(xml_elem): + if "type" in xml_elem.attrib: + return model.Typedef( + xml_elem.get("name"), + xml_elem.get("type"), + docstring=get_docstr(xml_elem) + ) + + elif "primitiveType" in xml_elem.attrib and xml_elem.get("name") not in primitive_types.values(): + return model.Typedef( + xml_elem.get("name"), + primitive_types[xml_elem.get("primitiveType")], + docstring=get_docstr(xml_elem) + ) + + +def make_enum(xml_elem): + def check_for_duplicates(enum_obj): + values = set() + for m in enum_obj.members: + if m.value in values: + raise ValueError("Duplicate Enum value in '{}', value '{}'.".format(enum_obj.name, m.value)) + values.add(m.value) + + if len(xml_elem): + members = [] + for member in xml_elem: + value = member.get('value') + try: + int_value = int(value, 0) + if int_value < 0: + value = "0x{:X}".format(0x100000000 + int_value) + except ValueError: + pass + members.append(model.EnumMember( + member.get("name"), + expand_operators(value), + docstring=get_docstr(member)) + ) + + enum = model.Enum(xml_elem.get("name"), members, docstring=get_docstr(xml_elem)) + check_for_duplicates(enum) return enum -def make_struct_members(elem, dynamic_array=False): - members = [] - ename = elem.get("name") - etype = elem.get("type") - optional = elem.get("optional") +def make_struct_members(xml_elem, dynamic_array=False): + xml_elem_name = xml_elem.get("name") + xml_elem_type = xml_elem.get("type") + optional = xml_elem.get("optional") optional = bool(optional) and optional.lower() == "true" - dimension = elem.find("dimension") - if dimension is not None: - size = dimension.get("size", None) - if "size2" in dimension.attrib: - size = "{}*{}".format(size, dimension.get("size2", None)) - if optional: - members.append(model.StructMember("has_" + ename, "u32")) - - sizer_name = dimension.get("variableSizeFieldName", None) - if sizer_name and "@" in sizer_name[0]: - members.append(model.StructMember(ename, etype, bound=sizer_name[1:])) - - elif size and "THIS_IS_VARIABLE_SIZE_ARRAY" in size: - sizer_name = "numOf" + ename[0].upper() + ename[1:] - members.append(model.StructMember(ename, etype, bound=sizer_name)) - - elif "isVariableSize" in dimension.attrib: - type_ = dimension.get("variableSizeFieldType", "u32") - sizer_name = dimension.get("variableSizeFieldName", ename + "_len") - members.append(model.StructMember(sizer_name, type_)) - members.append(model.StructMember(ename, etype, bound=sizer_name, size=None if dynamic_array else size)) + dimension = xml_elem.find("dimension") + comment = get_docstr(xml_elem) + + def collect(): + if dimension is None: + yield model.StructMember(xml_elem_name, xml_elem_type, optional=optional, docstring=comment) else: - members.append(model.StructMember(ename, etype, size=size)) - else: - members.append(model.StructMember(ename, etype, optional=optional)) - return members + size = dimension.get("size", None) + size2 = dimension.get("size2", None) + if size2: + size = "{}*{}".format(size, size2) + if optional: + yield model.StructMember("has_" + xml_elem_name, "u32", docstring="implicit enabler for optional field") + + sizer_name = dimension.get("variableSizeFieldName", None) + if sizer_name and "@" in sizer_name[0]: + yield model.StructMember(xml_elem_name, xml_elem_type, bound=sizer_name[1:], docstring=comment) + + elif size and "THIS_IS_VARIABLE_SIZE_ARRAY" in size: + sizer_name = "numOf" + xml_elem_name[0].upper() + xml_elem_name[1:] + yield model.StructMember(xml_elem_name, xml_elem_type, bound=sizer_name, docstring=comment) + + elif "isVariableSize" in dimension.attrib: + type_ = dimension.get("variableSizeFieldType", "u32") + sizer_name = dimension.get("variableSizeFieldName", xml_elem_name + "_len") + yield model.StructMember(sizer_name, type_, docstring=comment) + size_ = None if dynamic_array else size + yield model.StructMember(xml_elem_name, xml_elem_type, bound=sizer_name, size=size_, docstring=comment) + + else: + yield model.StructMember(xml_elem_name, xml_elem_type, size=size, docstring=comment) + + return list(collect()) + + +def make_struct(xml_elem, last_member_array_is_dynamic=False): + if len(xml_elem): + members = [] + for member in xml_elem: + for sub_ in make_struct_members(member, last_member_array_is_dynamic): + members.append(sub_) + return model.Struct(xml_elem.get("name"), members, docstring=get_docstr(xml_elem)) + + +def make_union(xml_elem): + if len(xml_elem): + members = [] + for member in xml_elem: + members.append(model.UnionMember( + member.get("name"), + member.get("type"), + member.get("discriminatorValue"), + docstring=get_docstr(member), + )) + return model.Union(xml_elem.get('name'), members, docstring=get_docstr(xml_elem)) -def make_struct(elem, last_member_array_is_dynamic=False): - if len(elem): - members = reduce(lambda x, y: x + y, (make_struct_members(member) for member in elem[:-1]), []) - members += make_struct_members(elem[-1], last_member_array_is_dynamic) - return model.Struct(elem.get("name"), members) +class IsarParser(object): + def __init__(self, warn=None): + self.warn = warn -def make_union_member(elem): - return model.UnionMember(elem.get("name"), elem.get("type"), elem.get("discriminatorValue")) + def parse(self, content, _, process_file): + def collect(): + root = ElementTree.fromstring(content) + for xml_elem in root.iterfind('.//*[@href]'): + yield make_include(xml_elem, process_file, self.warn) + for xml_elem in root.iterfind('.//constant'): + yield make_constant(xml_elem) -def make_union(elem): - if len(elem): - return model.Union(elem.get('name'), [make_union_member(member) for member in elem]) + for xml_elem in root.iterfind('.//typedef'): + yield make_typedef(xml_elem) + for xml_elem in root.iterfind('.//enum'): + yield make_enum(xml_elem) -class IsarParser(object): + for xml_elem in root.iterfind('.//struct'): + yield make_struct(xml_elem) - def __init__(self, warn=None): - self.warn = warn + for xml_elem in root.iterfind('.//union'): + yield make_union(xml_elem) + + for xml_elem in root.iterfind('.//message'): + yield make_struct(xml_elem, last_member_array_is_dynamic=True) - def __get_model(self, root, process_file): - nodes = [] - nodes += [make_include(elem, process_file, self.warn) - for elem in filter(lambda elem: "include" in elem.tag, root.iterfind('.//*[@href]'))] - nodes += [make_constant(elem) for elem in root.iterfind('.//constant')] - nodes += filter(None, (make_typedef(elem) for elem in root.iterfind('.//typedef'))) - nodes += filter(None, (make_enum(elem) for elem in root.iterfind('.//enum'))) - nodes += filter(None, (make_struct(elem) for elem in root.iterfind('.//struct'))) - nodes += filter(None, (make_union(elem) for elem in root.iterfind('.//union'))) - nodes += filter(None, (make_struct(elem, last_member_array_is_dynamic=True) - for elem in root.iterfind('.//message'))) - return nodes - - def parse(self, content, path, process_file): - return self.__get_model(ElementTree.fromstring(content), process_file) + return [element for element in collect() if element] diff --git a/prophyc/parsers/prophy.py b/prophyc/parsers/prophy.py index 463d142..0ab25ad 100755 --- a/prophyc/parsers/prophy.py +++ b/prophyc/parsers/prophy.py @@ -4,10 +4,7 @@ import ply.lex as lex import ply.yacc as yacc -from prophyc.six import ifilter -from prophyc.model import Include, Constant, Typedef, Enum, EnumMember, Struct, StructMember, Union, UnionMember, \ - Kind, ParseError -from prophyc.file_processor import CyclicIncludeError, FileNotFoundError +from prophyc import file_processor, model, six def get_column(input_, pos): @@ -15,7 +12,6 @@ def get_column(input_, pos): class Parser(object): - literals = ['+', '-', '*', '/', '(', ')', '#'] keywords = ( @@ -140,10 +136,9 @@ def _validate_struct_members(self, members): ) fieldnames.add(name) if member.bound: - bound, _, _ = next(ifilter(lambda m: m[0].name == member.bound, members[:i]), - (None, None, None)) + bound, _, __ = next(six.ifilter(lambda m: m[0].name == member.bound, members[:i]), (None, None, None)) if bound: - self._parser_check(self._is_type_sizer_compatible(bound.type_), + self._parser_check(self._is_type_sizer_compatible(bound.type_name), "Sizer of '{}' has to be of (unsigned) integer type".format(name), line, pos) else: @@ -152,7 +147,7 @@ def _validate_struct_members(self, members): for member, line, pos in members[:-1]: self._parser_check( - not member.greedy and member.kind != Kind.UNLIMITED, + not member.greedy and member.kind != model.Kind.UNLIMITED, "greedy array field '{}' not last".format(member.name), line, pos ) @@ -160,8 +155,8 @@ def _validate_struct_members(self, members): def _is_type_sizer_compatible(self, typename): if typename in {type_ + width for type_ in 'ui' for width in ['8', '16', '32', '64']}: return True - elif typename in self.typedecls and isinstance(self.typedecls[typename], Typedef): - return self._is_type_sizer_compatible(self.typedecls[typename].type_) + elif typename in self.typedecls and isinstance(self.typedecls[typename], model.Typedef): + return self._is_type_sizer_compatible(self.typedecls[typename].type_name) else: return False @@ -192,31 +187,31 @@ def p_include_def(self, t): try: nodes = self.parse_file(path) - except (CyclicIncludeError, FileNotFoundError) as e: + except (file_processor.CyclicIncludeError, file_processor.FileNotFoundError) as e: self._parser_error(str(e), t.lineno(3), t.lexpos(3)) nodes = [] for node in nodes: - if isinstance(node, Constant): + if isinstance(node, model.Constant): self.constdecls[node.name] = node - if isinstance(node, Enum): + if isinstance(node, model.Enum): for mem in node.members: self.constdecls[mem.name] = mem - if isinstance(node, (Typedef, Enum, Struct, Union)): + if isinstance(node, (model.Typedef, model.Enum, model.Struct, model.Union)): self.typedecls[node.name] = node - node = Include(stem, nodes) + node = model.Include(stem, nodes) self.nodes.append(node) def p_constant_def(self, t): '''constant_def : CONST unique_id EQUALS expression SEMI''' - node = Constant(t[2], str(t[4])) + node = model.Constant(t[2], str(t[4])) self.constdecls[t[2]] = node self.nodes.append(node) def p_enum_def(self, t): '''enum_def : ENUM unique_id enum_body SEMI''' - node = Enum(t[2], t[3]) + node = model.Enum(t[2], t[3]) self.typedecls[t[2]] = node self.nodes.append(node) @@ -234,13 +229,13 @@ def p_enum_member_list_2(self, t): def p_enum_member(self, t): '''enum_member : unique_id EQUALS expression''' - member = EnumMember(t[1], str(t[3])) + member = model.EnumMember(t[1], str(t[3])) self.constdecls[t[1]] = member t[0] = member def p_typedef_def(self, t): '''typedef_def : TYPEDEF type_spec unique_id SEMI''' - node = Typedef(t[3], t[2][0], definition=t[2][1]) + node = model.Typedef(t[3], t[2][0], definition=t[2][1]) self.typedecls[t[3]] = node self.nodes.append(node) @@ -248,10 +243,12 @@ def p_struct_def(self, t): '''struct_def : STRUCT unique_id struct_body SEMI''' self._validate_struct_members(t[3]) - - node = Struct(t[2], [x for x, _, _ in t[3]]) - self.typedecls[t[2]] = node - self.nodes.append(node) + try: + node = model.Struct(t[2], [x for x, _, _ in t[3]]) + self.typedecls[t[2]] = node + self.nodes.append(node) + except model.ModelError as e: # actual raise is postponed till end of parsing + self._parser_error(str(e), t.lexer.lineno, 0) def p_struct_body(self, t): '''struct_body : LBRACE struct_member_list RBRACE''' @@ -267,45 +264,45 @@ def p_struct_member_list_2(self, t): def p_struct_member_1(self, t): '''struct_member : type_spec ID''' - t[0] = [(StructMember(t[2], t[1][0], definition=t[1][1]), t.lineno(2), t.lexpos(2))] + t[0] = [(model.StructMember(t[2], t[1][0], definition=t[1][1]), t.lineno(2), t.lexpos(2))] def p_struct_member_2(self, t): '''struct_member : bytes ID LBRACKET positive_expression RBRACKET | type_spec ID LBRACKET positive_expression RBRACKET''' - t[0] = [(StructMember(t[2], t[1][0], size=str(t[4]), definition=t[1][1]), t.lineno(2), t.lexpos(2))] + t[0] = [(model.StructMember(t[2], t[1][0], size=str(t[4]), definition=t[1][1]), t.lineno(2), t.lexpos(2))] def p_struct_member_3(self, t): '''struct_member : bytes ID LT AT ID GT | type_spec ID LT AT ID GT''' t[0] = [ - (StructMember(t[2], t[1][0], bound=t[5], definition=t[1][1]), t.lineno(2), t.lexpos(2)) + (model.StructMember(t[2], t[1][0], bound=t[5], definition=t[1][1]), t.lineno(2), t.lexpos(2)) ] def p_struct_member_4(self, t): '''struct_member : bytes ID LT GT | type_spec ID LT GT''' t[0] = [ - (StructMember('num_of_' + t[2], 'u32', definition=None), t.lineno(2), t.lexpos(2)), - (StructMember(t[2], t[1][0], bound='num_of_' + t[2], definition=t[1][1]), t.lineno(2), t.lexpos(2)) + (model.StructMember('num_of_' + t[2], 'u32', definition=None), t.lineno(2), t.lexpos(2)), + (model.StructMember(t[2], t[1][0], bound='num_of_' + t[2], definition=t[1][1]), t.lineno(2), t.lexpos(2)) ] def p_struct_member_5(self, t): '''struct_member : bytes ID LT positive_expression GT | type_spec ID LT positive_expression GT''' t[0] = [ - (StructMember('num_of_' + t[2], 'u32', definition=None), t.lineno(2), t.lexpos(2)), - (StructMember(t[2], t[1][0], bound='num_of_' + t[2], - size=str(t[4]), definition=t[1][1]), t.lineno(2), t.lexpos(2)) + (model.StructMember('num_of_' + t[2], 'u32', definition=None), t.lineno(2), t.lexpos(2)), + (model.StructMember(t[2], t[1][0], bound='num_of_' + t[2], size=str(t[4]), definition=t[1][1]), + t.lineno(2), t.lexpos(2)) ] def p_struct_member_6(self, t): '''struct_member : bytes ID LT DOTS GT | type_spec ID LT DOTS GT''' - t[0] = [(StructMember(t[2], t[1][0], unlimited=True, definition=t[1][1]), t.lineno(2), t.lexpos(2))] + t[0] = [(model.StructMember(t[2], t[1][0], greedy=True, definition=t[1][1]), t.lineno(2), t.lexpos(2))] def p_struct_member_7(self, t): '''struct_member : type_spec '*' ID''' - t[0] = [(StructMember(t[3], t[1][0], optional=True, definition=t[1][1]), t.lineno(3), t.lexpos(3))] + t[0] = [(model.StructMember(t[3], t[1][0], optional=True, definition=t[1][1]), t.lineno(3), t.lexpos(3))] def p_bytes(self, t): '''bytes : BYTES''' @@ -332,12 +329,12 @@ def p_union_def(self, t): discriminatorvalues.add(member.discriminator) for member, line, pos in t[3]: self._parser_check( - member.kind == Kind.FIXED, + member.kind == model.Kind.FIXED, "dynamic union arm '{}'".format(member.name), line, pos ) - node = Union(t[2], [x for x, _, _ in t[3]]) + node = model.Union(t[2], [x for x, _, _ in t[3]]) self.typedecls[t[2]] = node self.nodes.append(node) @@ -355,7 +352,7 @@ def p_union_member_list_2(self, t): def p_union_member(self, t): '''union_member : expression COLON type_spec ID''' - t[0] = (UnionMember(t[4], t[3][0], str(t[1]), definition=t[3][1]), t.lineno(4), t.lexpos(4)) + t[0] = (model.UnionMember(t[4], t[3][0], str(t[1]), definition=t[3][1]), t.lineno(4), t.lexpos(4)) def p_type_spec_1(self, t): '''type_spec : U8 @@ -486,29 +483,27 @@ def p_error(self, t): @contextmanager -def allocate_parser(parsers=[]): +def allocate_parser(parsers_stack): """ Creating parsers is very expensive, so there is a need to reuse them. On the other hand, recursive parser usage requires a unique one for each level of recursion. Static stack of parsers seems to solve the issue. """ - parser = parsers and parsers.pop() or Parser() + parser = parsers_stack and parsers_stack.pop() or Parser() try: yield parser finally: - parsers.append(parser) - - -def build_model(input_, parse_error_prefix, parse_file): - with allocate_parser() as parser: - parser.parse(input_, parse_error_prefix, parse_file) - if parser.errors: - raise ParseError(parser.errors) - return parser.nodes + parsers_stack.append(parser) class ProphyParser(object): - def parse(self, content, path, parse_file): - return build_model(content, path, parse_file) + @staticmethod + def parse(content, parse_error_prefix, parse_file): + parsers_stack = [] + with allocate_parser(parsers_stack) as parser: + parser.parse(content, parse_error_prefix, parse_file) + if parser.errors: + raise model.ParseError(parser.errors) + return parser.nodes diff --git a/prophyc/parsers/sack.py b/prophyc/parsers/sack.py index 4e8fb39..3d4776a 100644 --- a/prophyc/parsers/sack.py +++ b/prophyc/parsers/sack.py @@ -1,12 +1,12 @@ +import ctypes.util import os import re -import ctypes.util -from .clang.cindex import Config, Index, CursorKind, TypeKind, TranslationUnitLoadError, LibclangError +from contextlib import contextmanager from prophyc import model from prophyc.generators.cpp import _HppDefinitionsTranslator from prophyc.six import to_bytes -from contextlib import contextmanager +from .clang import cindex class SackParserError(Exception): @@ -29,13 +29,13 @@ def remove_nodes(self, node_names_to_remove): class Builder(object): unambiguous_builtins = { - TypeKind.UCHAR: 'u8', - TypeKind.SCHAR: 'i8', - TypeKind.CHAR_S: 'i8', - TypeKind.POINTER: 'u32', - TypeKind.FLOAT: 'r32', - TypeKind.DOUBLE: 'r64', - TypeKind.BOOL: 'i32' + cindex.TypeKind.UCHAR: 'u8', + cindex.TypeKind.SCHAR: 'i8', + cindex.TypeKind.CHAR_S: 'i8', + cindex.TypeKind.POINTER: 'u32', + cindex.TypeKind.FLOAT: 'r32', + cindex.TypeKind.DOUBLE: 'r64', + cindex.TypeKind.BOOL: 'i32' } def __init__(self, tree_model): @@ -61,36 +61,36 @@ def dive_deeper(method): method(decl) return name - if tp.kind is TypeKind.TYPEDEF: + if tp.kind is cindex.TypeKind.TYPEDEF: return self.get_type_name(decl.underlying_typedef_type) - elif tp.kind in (TypeKind.UNEXPOSED, TypeKind.ELABORATED, TypeKind.RECORD): + elif tp.kind in (cindex.TypeKind.UNEXPOSED, cindex.TypeKind.ELABORATED, cindex.TypeKind.RECORD): - if decl.kind in (CursorKind.STRUCT_DECL, CursorKind.CLASS_DECL): + if decl.kind in (cindex.CursorKind.STRUCT_DECL, cindex.CursorKind.CLASS_DECL): return dive_deeper(self.add_struct) - elif decl.kind is CursorKind.UNION_DECL: + elif decl.kind is cindex.CursorKind.UNION_DECL: return dive_deeper(self.add_union) - elif decl.kind is CursorKind.ENUM_DECL: + elif decl.kind is cindex.CursorKind.ENUM_DECL: return self.get_type_name(decl.type) - elif decl.kind is CursorKind.TYPEDEF_DECL: + elif decl.kind is cindex.CursorKind.TYPEDEF_DECL: return self.get_type_name(decl.underlying_typedef_type) else: raise SackParserError("Unknown declaration, {} {}".format(tp.spelling, decl.kind)) - elif tp.kind in (TypeKind.CONSTANTARRAY, TypeKind.INCOMPLETEARRAY): + elif tp.kind in (cindex.TypeKind.CONSTANTARRAY, cindex.TypeKind.INCOMPLETEARRAY): return self.get_type_name(tp.element_type) - elif tp.kind is TypeKind.ENUM: + elif tp.kind is cindex.TypeKind.ENUM: return dive_deeper(self.add_enum) - if tp.kind in (TypeKind.USHORT, TypeKind.UINT, TypeKind.ULONG, TypeKind.ULONGLONG): + if tp.kind in (cindex.TypeKind.USHORT, cindex.TypeKind.UINT, cindex.TypeKind.ULONG, cindex.TypeKind.ULONGLONG): return 'u%d' % (tp.get_size() * 8) - elif tp.kind in (TypeKind.SHORT, TypeKind.INT, TypeKind.LONG, TypeKind.LONGLONG): + elif tp.kind in (cindex.TypeKind.SHORT, cindex.TypeKind.INT, cindex.TypeKind.LONG, cindex.TypeKind.LONGLONG): return 'i%d' % (tp.get_size() * 8) return self.unambiguous_builtins[tp.kind] @@ -111,9 +111,8 @@ def enum_member(cursor): def add_struct(self, cursor): def array_length(tp): - if tp.kind is TypeKind.CONSTANTARRAY: + if tp.kind is cindex.TypeKind.CONSTANTARRAY: return tp.element_count - return None def struct_member(cursor_): name = cursor_.spelling.decode() @@ -122,7 +121,7 @@ def struct_member(cursor_): return model.StructMember(name, type_name, size=array_len) members = [struct_member(x) for x in cursor.get_children() - if x.kind is CursorKind.FIELD_DECL and not x.is_bitfield()] + if x.kind is cindex.CursorKind.FIELD_DECL and not x.is_bitfield()] node = model.Struct(Builder.alphanumeric_name(cursor), members) self.tree.add_node(node) @@ -133,20 +132,21 @@ def union_member(cursor, disc): return model.UnionMember(name, type_name, str(disc)) members = [union_member(x, i) for i, x in enumerate(cursor.get_children()) - if x.kind is CursorKind.FIELD_DECL] + if x.kind is cindex.CursorKind.FIELD_DECL] node = model.Union(Builder.alphanumeric_name(cursor), members) self.tree.add_node(node) def build_model(self, translation_unit): for cursor in translation_unit.cursor.get_children(): - if cursor.kind is CursorKind.UNEXPOSED_DECL: + if cursor.kind is cindex.CursorKind.UNEXPOSED_DECL: for in_cursor in cursor.get_children(): - if in_cursor.kind is CursorKind.STRUCT_DECL and in_cursor.spelling and in_cursor.is_definition(): - self.add_struct(in_cursor) + if in_cursor.kind is cindex.CursorKind.STRUCT_DECL: + if in_cursor.spelling and in_cursor.is_definition(): + self.add_struct(in_cursor) if cursor.spelling and cursor.is_definition(): - if cursor.kind is CursorKind.STRUCT_DECL: + if cursor.kind is cindex.CursorKind.STRUCT_DECL: self.add_struct(cursor) - if cursor.kind is CursorKind.ENUM_DECL: + if cursor.kind is cindex.CursorKind.ENUM_DECL: self.add_enum(cursor) @@ -163,7 +163,7 @@ def __init__(self, include_tree): def flatten_nodes(nodes_list): for node in nodes_list: if isinstance(node, model.Include): - for node in SupplementaryDefs.flatten_nodes(node.nodes): + for node in SupplementaryDefs.flatten_nodes(node.members): yield node else: yield node @@ -224,11 +224,11 @@ def __bool__(self): __nonzero__ = __bool__ def _check_libclang(): - testconf = Config() + testconf = cindex.Config() try: testconf.get_cindex_library() return True - except LibclangError: + except cindex.LibclangError: return False import platform @@ -238,14 +238,14 @@ def _check_libclang(): return SackParserStatus("sack input requires libclang and it's not installed") return SackParserStatus() - def __init__(self, include_dirs=[], warn=None, include_tree=[]): - self.include_dirs = include_dirs + def __init__(self, include_dirs=None, warn=None, include_tree=None): + self.include_dirs = include_dirs or [] self.warn = warn - self.supples = SupplementaryDefs(include_tree) + self.supples = SupplementaryDefs(include_tree or []) def parse(self, content, path, _): args_ = [to_bytes("-I" + x) for x in self.include_dirs] - index = Index.create() + index = cindex.Index.create() with self.supples.implicit_supplementation(content) as (content_, tree): builder = Builder(tree) path = path.encode() @@ -253,7 +253,7 @@ def parse(self, content, path, _): try: translation_unit = index.parse(path, args_, unsaved_files=((path, content_),)) - except TranslationUnitLoadError: + except cindex.TranslationUnitLoadError: raise model.ParseError([(path.decode(), 'error parsing translation unit')]) self.print_diagnostics(path, translation_unit) @@ -285,7 +285,7 @@ def _get_location(self, location, target_path): def _setup_libclang(): if os.environ.get('PROPHY_NOCLANG'): - Config.set_library_file('prophy_noclang') + cindex.Config.set_library_file('prophy_noclang') return versions = [None, '3.5', '3.4', '3.3', '3.2', '3.6', '3.7', '3.8', '3.9'] @@ -293,7 +293,7 @@ def _setup_libclang(): name = v and 'clang-' + v or 'clang' libname = ctypes.util.find_library(name) if libname: - Config.set_library_file(libname) + cindex.Config.set_library_file(libname) break diff --git a/prophyc/patch.py b/prophyc/patch.py index 98b76a1..e6ae894 100644 --- a/prophyc/patch.py +++ b/prophyc/patch.py @@ -1,217 +1,218 @@ -from collections import namedtuple +import codecs +from collections import namedtuple, defaultdict + from . import model Action = namedtuple("Action", ["action", "params"]) def parse(filename): - def make_item(line): - words = line.split() - name, action = words[:2] - params = words[2:] - return name, Action(action, params) - patches = {} - for name, action in (make_item(line) for line in open(filename) if line.strip()): - patches.setdefault(name, []).append(action) - return patches - - -def patch(nodes, patchdict): + patches = defaultdict(list) + with codecs.open(filename, "r", encoding="utf-8") as f: + for line in f.readlines(): + if line.strip(): + words = line.split() + name, action = words[:2] + params = words[2:] + patches[name].append(Action(action, params)) + return dict(patches) + + +def patch(nodes, patch_dict): for idx, node in enumerate(nodes): - patches = patchdict.get(node.name) + patches = patch_dict.get(node.name) if patches: nodes[idx] = _apply(node, patches) def _apply(node, patches): - for patch in patches: - action = _actions.get(patch.action) + for patch_ in patches: + action = _actions.get(patch_.action) if not action: - raise Exception("Unknown action: %s %s" % (node.name, patch)) - node = action(node, patch) + raise Exception("Unknown action: %s %s" % (node.name, patch_)) + node = action(node, patch_) return node -def _type(node, patch): +def _type(node, patch_): if not isinstance(node, model.Struct): - raise Exception("Can change field only in struct: %s %s" % (node.name, patch)) + raise Exception("Can change field only in struct: %s %s" % (node.name, patch_)) - if len(patch.params) != 2: - raise Exception("Change field must have 2 params: %s %s" % (node.name, patch)) - name, tp = patch.params + if len(patch_.params) != 2: + raise Exception("Change field must have 2 params: %s %s" % (node.name, patch_)) + name, tp = patch_.params i, member = next((x for x in enumerate(node.members) if x[1].name == name), (None, None)) if not member: - raise Exception("Member not found: %s %s" % (node.name, patch)) + raise Exception("Member not found: %s %s" % (node.name, patch_)) mem = node.members[i] - mem.type_ = tp + mem.type_name = tp return node -def _insert(node, patch): +def _insert(node, patch_): if not isinstance(node, model.Struct): - raise Exception("Can insert field only in struct: %s %s" % (node.name, patch)) + raise Exception("Can insert field only in struct: %s %s" % (node.name, patch_)) - if len(patch.params) != 3: - raise Exception("Change field must have 3 params: %s %s" % (node.name, patch)) - index, name, tp = patch.params + if len(patch_.params) != 3: + raise Exception("Change field must have 3 params: %s %s" % (node.name, patch_)) + index, name, tp = patch_.params if not _is_int(index): - raise Exception("Index is not a number: %s %s" % (node.name, patch)) + raise Exception("Index is not a number: %s %s" % (node.name, patch_)) index = int(index) node.members.insert(index, model.StructMember(name, tp)) return node -def _remove(node, patch): +def _remove(node, patch_): if not isinstance(node, model.Struct): - raise Exception("Can remove field only in struct: %s %s" % (node.name, patch)) + raise Exception("Can remove field only in struct: %s %s" % (node.name, patch_)) - if len(patch.params) != 1: - raise Exception("Remove field must have 1 param: %s %s" % (node.name, patch)) - name, = patch.params + if len(patch_.params) != 1: + raise Exception("Remove field must have 1 param: %s %s" % (node.name, patch_)) + name, = patch_.params i, member = next((x for x in enumerate(node.members) if x[1].name == name), (None, None)) if not member: - raise Exception("Member not found: %s %s" % (node.name, patch)) + raise Exception("Member not found: %s %s" % (node.name, patch_)) del node.members[i] return node -def _dynamic(node, patch): +def _dynamic(node, patch_): if not isinstance(node, model.Struct): - raise Exception("Can change field only in struct: %s %s" % (node.name, patch)) + raise Exception("Can change field only in struct: %s %s" % (node.name, patch_)) - if len(patch.params) != 2: - raise Exception("Change field must have 2 params: %s %s" % (node.name, patch)) - name, len_name = patch.params + if len(patch_.params) != 2: + raise Exception("Change field must have 2 params: %s %s" % (node.name, patch_)) + name, len_name = patch_.params i, member = next((x for x in enumerate(node.members) if x[1].name == name), (None, None)) if not member: - raise Exception("Member not found: %s %s" % (node.name, patch)) + raise Exception("Member not found: %s %s" % (node.name, patch_)) mem = node.members[i] - mem.array = True mem.bound = len_name mem.size = None mem.optional = False return node -def _greedy(node, patch): +def _greedy(node, patch_): if not isinstance(node, model.Struct): - raise Exception("Can change field only in struct: %s %s" % (node.name, patch)) + raise Exception("Can change field only in struct: %s %s" % (node.name, patch_)) - if len(patch.params) != 1: - raise Exception("Change field must have 1 params: %s %s" % (node.name, patch)) - name, = patch.params + if len(patch_.params) != 1: + raise Exception("Change field must have 1 params: %s %s" % (node.name, patch_)) + name, = patch_.params i, member = next((x for x in enumerate(node.members) if x[1].name == name), (None, None)) if not member: - raise Exception("Member not found: %s %s" % (node.name, patch)) + raise Exception("Member not found: %s %s" % (node.name, patch_)) mem = node.members[i] - mem.array = True + mem.greedy = True mem.bound = None mem.size = None mem.optional = False return node -def _static(node, patch): +def _static(node, patch_): if not isinstance(node, model.Struct): - raise Exception("Can change field only in struct: %s %s" % (node.name, patch)) + raise Exception("Can change field only in struct: %s %s" % (node.name, patch_)) - if len(patch.params) != 2: - raise Exception("Change field must have 2 params: %s %s" % (node.name, patch)) - name, size = patch.params + if len(patch_.params) != 2: + raise Exception("Change field must have 2 params: %s %s" % (node.name, patch_)) + name, size = patch_.params i, member = next((x for x in enumerate(node.members) if x[1].name == name), (None, None)) if not member: - raise Exception("Member not found: %s %s" % (node.name, patch)) + raise Exception("Member not found: %s %s" % (node.name, patch_)) node.members[i].bound = None node.members[i].size = None mem = node.members[i] - mem.array = True mem.bound = None mem.size = size mem.optional = False return node -def _limited(node, patch): +def _limited(node, patch_): if not isinstance(node, model.Struct): - raise Exception("Can change field only in struct: %s %s" % (node.name, patch)) - - if len(patch.params) != 2: - raise Exception("Change field must have 2 params: %s %s" % (node.name, patch)) - name, len_array = patch.params + raise Exception("Can change field only in struct: %s %s" % (node.name, patch_)) - sizer_found = len(tuple(x for x in node.members if x.name == len_array)) - if not sizer_found: - raise Exception("Array len member not found: %s %s" % (node.name, patch)) + if len(patch_.params) != 2: + raise Exception("Change field must have 2 params: %s %s" % (node.name, patch_)) + name, len_array = patch_.params i, member = next((x for x in enumerate(node.members) if x[1].name == name), (None, None)) if not member: - raise Exception("Member not found: %s %s" % (node.name, patch)) + raise Exception("Member not found: %s %s" % (node.name, patch_)) + + sizer_found = len(tuple(x for x in node.members[:i] if x.name == len_array)) + if not sizer_found: + raise Exception("Array len member not found: %s %s" % (node.name, patch_)) mem = node.members[i] - mem.array = True mem.bound = len_array mem.optional = False return node -def _struct(node, patch): +def _struct(node, patch_): if not isinstance(node, model.Union): - raise Exception("Can only change union to struct: %s" % (node.name)) + raise Exception("Can only change union to struct: %s" % node.name) - if len(patch.params): - raise Exception("Change union to struct takes no params: %s" % (node.name)) + if len(patch_.params): + raise Exception("Change union to struct takes no params: %s" % node.name) def to_struct_member(member): - return model.StructMember(name=member.name, type_=member.type_, definition=member.definition) + return model.StructMember(name=member.name, type_name=member.type_name, definition=member.definition) return model.Struct(node.name, [to_struct_member(mem) for mem in node.members]) -def _rename(node, patch): - def rename_node(node, new_name): - node.name = patch.params[0] - return node +def _rename(node, patch_): + def rename_node(node_, new_name): + node_.name = new_name + return node_ - def rename_field(node, orig_name, new_name): - if not isinstance(node, (model.Struct, model.Union)): - raise Exception("Can rename fields only in composites: %s %s" % (node.name, patch)) - member = next((x for x in node.members if x.name == orig_name), None) + def rename_field(node_, orig_name, new_name): + if not isinstance(node_, (model.Struct, model.Union)): + raise Exception("Can rename fields only in composites: %s %s" % (node_.name, patch_)) + member = next((x for x in node_.members if x.name == orig_name), None) if not member: - raise Exception("Member not found: %s %s" % (node.name, patch)) + raise Exception("Member not found: %s %s" % (node_.name, patch_)) member.name = new_name - return node + return node_ - if len(patch.params) == 1: - return rename_node(node, patch.params[0]) + if len(patch_.params) == 1: + return rename_node(node, patch_.params[0]) - if len(patch.params) == 2: - return rename_field(node, patch.params[0], patch.params[1]) + if len(patch_.params) == 2: + return rename_field(node, patch_.params[0], patch_.params[1]) - raise Exception("Rename must have 1 or 2 params: %s %s" % (node.name, patch)) + raise Exception("Rename must have 1 or 2 params: %s %s" % (node.name, patch_)) -_actions = {'type': _type, - 'insert': _insert, - 'remove': _remove, - 'greedy': _greedy, - 'static': _static, - 'limited': _limited, - 'dynamic': _dynamic, - 'struct': _struct, - 'rename': _rename} +_actions = { + 'type': _type, + 'insert': _insert, + 'remove': _remove, + 'greedy': _greedy, + 'static': _static, + 'limited': _limited, + 'dynamic': _dynamic, + 'struct': _struct, + 'rename': _rename +} def _is_int(s): diff --git a/prophyc/six.py b/prophyc/six.py index 16760eb..d0507e8 100644 --- a/prophyc/six.py +++ b/prophyc/six.py @@ -2,14 +2,40 @@ if sys.version < '3': # pragma: no cover from itertools import ifilter + reduce = reduce + string_types = (str, unicode) + def to_bytes(str_): return str_ + + def decode_string(str_or_unicode_or_bytes): + if isinstance(str_or_unicode_or_bytes, str): + if any(ord(c) > 127 for c in str_or_unicode_or_bytes): + return str_or_unicode_or_bytes.decode("utf-8") + return str_or_unicode_or_bytes + if isinstance(str_or_unicode_or_bytes, bytes): + return str_or_unicode_or_bytes.decode("utf-8") + if isinstance(str_or_unicode_or_bytes, unicode): + return str_or_unicode_or_bytes + raise TypeError("Got text as %s, expected string." % type(str_or_unicode_or_bytes).__name__) + else: # pragma: no cover ifilter = filter from functools import reduce + string_types = (str,) + + + def decode_string(str_or_bytes): + if isinstance(str_or_bytes, bytes): + return str_or_bytes.decode("utf-8") + if isinstance(str_or_bytes, string_types): + return str_or_bytes + raise TypeError("Got text as %s, expected string." % type(str_or_bytes).__name__) + + def to_bytes(str_): return bytes(str_, 'utf-8') diff --git a/prophyc/tests/conftest.py b/prophyc/tests/conftest.py index 4f3b4db..62abc18 100644 --- a/prophyc/tests/conftest.py +++ b/prophyc/tests/conftest.py @@ -1,25 +1,48 @@ import os -import pytest import subprocess import sys -import prophyc +import pytest +import prophyc +from prophyc import model, six main_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) sys.path.insert(0, main_dir) +def pytest_assertrepr_compare(op, left, right): + """ + It's quite common that we compare string representation of documents. + Native pytest's comparison failure representation is at least unreadable for this purpose. + This prints compared strings only in case of test's failure. + """ + if op == "==" and isinstance(left, six.string_types) and isinstance(right, six.string_types): + if len(left) > 60 or len(right) > 60: + print("--- (left):\n{}---".format(left)) + print("does not match currently defined reference:") + print("--- (right):\n{}---".format(right)) + + def check_libclang(): from prophyc.parsers.sack import SackParser return SackParser.check() -def pytest_namespace(): - return { - 'clang_installed': pytest.mark.skipif(not check_libclang(), reason="clang not installed"), - 'clang_not_installed': pytest.mark.skipif(check_libclang(), reason="clang installed") - } +@pytest.fixture +def if_clang_installed(): + result = check_libclang() + if not result: + pytest.skip("clang not installed") + return result + + +@pytest.fixture +def if_clang_not_installed(): + result = check_libclang() + if result: + pytest.skip("clang installed") + return result @pytest.yield_fixture @@ -32,13 +55,6 @@ def tmpdir_cwd(tmpdir): os.chdir(orig_dir) -@pytest.yield_fixture -def tmpfiles_cwd(tmpdir_cwd): - def create_tmp_files(*files_to_create): - return map(tmpdir_cwd.join, files_to_create) - yield create_tmp_files - - @pytest.fixture def dummy_file(tmpdir_cwd): the_file = tmpdir_cwd.join("input") @@ -48,7 +64,6 @@ def dummy_file(tmpdir_cwd): @pytest.fixture def sys_capture(capsys): - class Capture(object): def __enter__(self): capsys.readouterr() @@ -65,10 +80,11 @@ def __exit__(self, _, exc_value, __): def get(self): return self.code, self.out, self.err + return Capture -@pytest.fixture(params=["py_code"]) +@pytest.fixture(params=["py_code", "subprocess"]) def call_prophyc(request, sys_capture): if request.param == "subprocess": @@ -89,3 +105,74 @@ def call_captured(call_args): return cp.get() return call_captured + + +@pytest.fixture +def lorem_with_breaks(): + return "\n".join([ + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt " + "ut labore et dolore magna aliqua. Libero nunc consequat interdum varius sit. Maecenas accumsan " + "lacus vel facilisis:", + " - Dui ut ornare,", + " - Lectus,", + " - Malesuada pellentesque,", + "", + "", + "Elit eget gravida cum sociis natoque penatibus et. Netus et malesuada fames ac turpis egestas sed.", + "Egestas integer eget aliquet.", + ]) + + +@pytest.fixture +def larger_model(lorem_with_breaks): + return [ + model.Typedef('a', 'i16'), + model.Typedef('c', 'a'), + model.Include('some_defs', [ + model.Struct('IncludedStruct', [ + model.StructMember('member1', 'r32', docstring='doc for member1'), + model.StructMember('member2', 'u64', docstring='docstring for member1') + ]), + model.Typedef('c', 'a'), + ]), + model.Include('cplx', [ + model.Struct('cint16_t', [ + model.StructMember('re', 'i16', docstring='real'), + model.StructMember('im', 'i16', docstring='imaginary') + ]), + model.Struct('cint32_t', [ + model.StructMember('re', 'i32', docstring='real'), + model.StructMember('im', 'i32', docstring='imaginary') + ]), + ]), + model.Union('the_union', [ + model.UnionMember('a', 'IncludedStruct', 0), + model.UnionMember('field_with_a_long_name', 'cint16_t', 1, docstring="Shorter"), + model.UnionMember('field_with_a_longer_name', 'cint32_t', 2, docstring="Longer description"), + model.UnionMember('other', 'i32', 4090, docstring='This one has larger discriminator'), + ], "spec for that union"), + model.Enum('E1', [ + model.EnumMember('E1_A', '0', 'enum1 constant value A'), + model.EnumMember('E1_B_has_a_long_name', '1', 'enum1 constant va3lue B'), + model.EnumMember('E1_C_desc', '2', lorem_with_breaks[:150]), + ], "Enumerator is a model type that is not supposed to be serialized. Its definition represents yet another " + "syntax variation for typing a constant. Of course elements of it's type are serializable " + "(as int32)"), + model.Enum('E2', [ + model.EnumMember('E2_A', '0', "Short\nmultiline\ndoc"), + ]), + model.Constant('CONST_A', '6'), + model.Constant('CONST_B', '0'), + model.Struct('StructMemberKinds', [ + model.StructMember('member_without_docstring', 'i16'), + model.StructMember('ext_size', 'i16', docstring='arbitrary sizer for dynamic arrays'), + model.StructMember('optional_element', 'cint16_t', optional=True, docstring='optional array'), + model.StructMember('fixed_array', 'cint16_t', size=3, docstring='Array with static size.'), + model.StructMember('samples', 'cint16_t', bound='ext_size', docstring='dynamic (ext.sized) array'), + model.StructMember('limited_array', 'r64', size=4, bound='ext_size', docstring='Has statically ' + 'evaluable maximum size.'), + model.StructMember('greedy', 'cint16_t', greedy=True, docstring='Represents array of arbitrary ' + 'number of elements. Buffer size ' + 'must be multiply of element size.'), + ], lorem_with_breaks[:400]), + ] diff --git a/prophyc/tests/generators/test_base.py b/prophyc/tests/generators/test_base.py deleted file mode 100644 index d130ab7..0000000 --- a/prophyc/tests/generators/test_base.py +++ /dev/null @@ -1,136 +0,0 @@ -import pytest - -from prophyc.generators import base -from prophyc import model -from prophyc.generators.base import GenerateError - - -def make_dummy_translation_method(name): - template = '<{}> [{{content}}]'.format(name) - - def dummy_translation(node): - return template.format(content=node) - return dummy_translation - - -@pytest.fixture -def straigth_generator(): - - class EmptyTranslator(base.TranslatorBase): - pass - - class StraightTranslator(base.TranslatorBase): - block_template = '''scope {base_name} {{\n{content}\n}} // endscope {base_name}\n''' - - def __init__(self): - for model_type, translator_name in base.TranslatorBase._translation_methods_map.items(): - method = make_dummy_translation_method(model_type.__name__) - setattr(self, translator_name, method) - - class StraightGenerator(base.GeneratorBase): - top_level_translators = { - '.st': StraightTranslator, - '.em': EmptyTranslator - } - - return StraightGenerator - - -@pytest.fixture -def serialize(mocker, straigth_generator): - scoper = type('scoper', (), {'writes': {}}) - - def write_mock(file_path, file_content): - scoper.writes.update({file_path: file_content}) - - mocker.patch.object(base, '_write_file', side_effect=write_mock) - mocker.patch.object(base.os.path, 'isdir', return_value=True) - - def process(nodes, base_name='mainer'): - model.cross_reference(nodes) - model.evaluate_kinds(nodes) - model.evaluate_sizes(nodes) - gen = straigth_generator('fake_out') - gen.check_nodes(nodes) - gen.serialize(nodes, base_name) - - return scoper.writes - return process - - -TRANSLATION_MODEL_A = [] -TRANSLATION_WRITES_A = { - 'fake_out/mainer.em': '', - 'fake_out/mainer.st': 'scope mainer {\n\n} // endscope mainer\n' -} - -TRANSLATION_MODEL_B = [ - model.Include('a', [model.Typedef('a', 'b')]), - model.Constant('CONST_A', '0'), -] -TRANSLATION_WRITES_B = { - 'fake_out/mainer.em': '', - 'fake_out/mainer.st': """scope mainer { - [Include(name='a', nodes=[b a])] - - [Constant(name='CONST_A', value='0')] - -} // endscope mainer -""", -} - -TRANSLATION_MODEL_C = [ - model.Typedef('a', 'b'), - model.Typedef('c', 'd'), - model.Enum('E1', [ - model.EnumMember('E1_A', '0'), - model.EnumMember('E1_B', '1')]), - model.Enum('E2', [ - model.EnumMember('E2_A', '0')]), - model.Constant('CONST_A', '0'), - model.Constant('CONST_B', '0') -] - -TRANSLATION_WRITES_C = { - 'fake_out/mainer.em': '', - 'fake_out/mainer.st': '''scope mainer { - [b a] - [d c] - - [E1 - E1_A 0 - E1_B 1 -] - - [E2 - E2_A 0 -] - - [Constant(name='CONST_A', value='0')] - [Constant(name='CONST_B', value='0')] - -} // endscope mainer -'''} - - -@pytest.mark.parametrize('nodes, writes', [ - (TRANSLATION_MODEL_A, TRANSLATION_WRITES_A), - (TRANSLATION_MODEL_B, TRANSLATION_WRITES_B), - (TRANSLATION_MODEL_C, TRANSLATION_WRITES_C) -]) -def test_straigth_translation(serialize, nodes, writes): - assert serialize(nodes) == writes - - -UNKNOWN_NONE_RAISE_TEST = [ - ([234], 'Unknown node type: int'), - ([[12]], 'Unknown node type: list'), - ([None], 'Unknown node type: NoneType') -] - - -@pytest.mark.parametrize('nodes, match', UNKNOWN_NONE_RAISE_TEST) -def test_raise_on_unknown_node(straigth_generator, nodes, match): - gen = straigth_generator('.') - with pytest.raises(GenerateError, match=match): - gen.serialize(nodes, '') diff --git a/prophyc/tests/generators/test_cpp.py b/prophyc/tests/generators/test_cpp.py index acc1d34..c26c58c 100644 --- a/prophyc/tests/generators/test_cpp.py +++ b/prophyc/tests/generators/test_cpp.py @@ -1,6 +1,7 @@ import pytest + from prophyc import model -from prophyc.generators.cpp import CppGenerator, GenerateError, _Padder, _HppTranslator,\ +from prophyc.generators.cpp import CppGenerator, GenerateError, _Padder, _HppTranslator, \ _HppDefinitionsTranslator, _HppSwapDeclarations, _HppIncludesTranslator, _CppSwapTranslator hpp_translator = _HppTranslator() @@ -8,9 +9,9 @@ def process(nodes): - model.cross_reference(nodes) - model.evaluate_kinds(nodes) - model.evaluate_sizes(nodes) + def warn(things): + print(things) + nodes, _ = model.evaluate_model(nodes, warn) CppGenerator().check_nodes(nodes) return nodes @@ -238,7 +239,7 @@ def test_definitions_struct_with_byte(): def test_definitions_struct_with_byte_array(): nodes = process([ - model.Struct("Struct", [model.StructMember("a", "byte", unlimited=True)]) + model.Struct("Struct", [model.StructMember("a", "byte", greedy=True)]) ]) assert generate_definitions(nodes) == """\ @@ -478,7 +479,10 @@ def test_definitions_newlines(): nodes = process([ model.Typedef("a", "b"), model.Typedef("c", "d"), - model.Enum("E1", [model.EnumMember("E1_A", "0")]), + model.Enum("E1", [ + model.EnumMember("E1_A", "0"), + model.EnumMember("E1_B", "1"), + ]), model.Enum("E2", [model.EnumMember("E2_A", "0")]), model.Constant("CONST_A", "0"), model.Typedef("e", "f"), @@ -494,9 +498,9 @@ def test_definitions_newlines(): enum E1 { - E1_A = 0 + E1_A = 0, + E1_B = 1u }; - enum E2 { E2_A = 0 @@ -513,7 +517,6 @@ def test_definitions_newlines(): { uint32_t a; }; - PROPHY_STRUCT(4) B { uint32_t b; @@ -687,7 +690,7 @@ def test_swap_struct_with_greedy_array(): model.Typedef('Y', 'u32'), model.Struct("X", [ (model.StructMember("x", "u8")), - (model.StructMember("y", "Y", unlimited=True)) + (model.StructMember("y", "Y", greedy=True)) ]), model.Struct("Z", [ (model.StructMember("z", "X")) @@ -701,7 +704,6 @@ def test_swap_struct_with_greedy_array(): swap(&payload->x); return cast(payload->y); } - template <> Z* swap(Z* payload) { @@ -763,7 +765,7 @@ def test_swap_enum_in_greedy_array(): model.EnumMember("E1_A", "0") ]), model.Struct("EnumGreedyArray", [ - (model.StructMember("x", "E1", unlimited=True)) + (model.StructMember("x", "E1", greedy=True)) ]) ]) @@ -819,7 +821,6 @@ def test_swap_struct_with_dynamic_element(): swap(&payload->num_of_x); return cast(swap_n_fixed(payload->x, payload->num_of_x)); } - template <> X* swap(X* payload) { @@ -847,7 +848,6 @@ def test_swap_struct_with_dynamic_array_of_dynamic_elements(): swap(&payload->num_of_x); return cast(swap_n_fixed(payload->x, payload->num_of_x)); } - template <> X* swap(X* payload) { @@ -1031,7 +1031,6 @@ def test_swap_struct_with_many_dynamic_fields(): swap(&payload->num_of_x); return cast(swap_n_fixed(payload->x, payload->num_of_x)); } - inline X::part2* swap(X::part2* payload) { return cast(swap(&payload->y)); @@ -1084,12 +1083,6 @@ def test_generate_empty_file(): #include -namespace prophy -{ - - -} // namespace prophy - #endif /* _PROPHY_GENERATED_TestEmpty_HPP */ """ @@ -1157,6 +1150,130 @@ def test_generate_file(): """ +def test_rendering_larger_model(larger_model): + assert generate_hpp(process(larger_model), "TestFile") == """\ +#ifndef _PROPHY_GENERATED_TestFile_HPP +#define _PROPHY_GENERATED_TestFile_HPP + +#include + +#include "some_defs.pp.hpp" +#include "cplx.pp.hpp" + +typedef int16_t a; +typedef a c; + +PROPHY_STRUCT(8) the_union +{ + enum _discriminator + { + discriminator_a = 0, + discriminator_field_with_a_long_name = 1, + discriminator_field_with_a_longer_name = 2, + discriminator_other = 4090 + } discriminator; + + uint32_t _padding0; /// manual padding to ensure natural alignment layout + + union + { + IncludedStruct a; + cint16_t field_with_a_long_name; + cint32_t field_with_a_longer_name; + int32_t other; + }; +}; + +enum E1 +{ + E1_A = 0, + E1_B_has_a_long_name = 1u, + E1_C_desc = 2u +}; +enum E2 +{ + E2_A = 0 +}; + +enum { CONST_A = 6u }; +enum { CONST_B = 0 }; + +PROPHY_STRUCT(8) StructMemberKinds +{ + int16_t member_without_docstring; + int16_t ext_size; + prophy::bool_t has_optional_element; + cint16_t optional_element; + cint16_t fixed_array[3]; + cint16_t samples[1]; /// dynamic array, size in ext_size + + PROPHY_STRUCT(8) part2 + { + double limited_array[4]; /// limited array, size in ext_size + cint16_t greedy[1]; /// greedy array + } _2; +}; + +namespace prophy +{ + +template <> the_union* swap(the_union*); +template <> inline E1* swap(E1* in) { swap(reinterpret_cast(in)); return in + 1; } +template <> inline E2* swap(E2* in) { swap(reinterpret_cast(in)); return in + 1; } +template <> StructMemberKinds* swap(StructMemberKinds*); + +} // namespace prophy + +#endif /* _PROPHY_GENERATED_TestFile_HPP */ +""" + assert generate_cpp(process(larger_model), "TestFile") == """\ +#include + +#include "TestFile.pp.hpp" + +using namespace prophy::detail; + +namespace prophy +{ + +template <> +the_union* swap(the_union* payload) +{ + swap(reinterpret_cast(&payload->discriminator)); + switch (payload->discriminator) + { + case the_union::discriminator_a: swap(&payload->a); break; + case the_union::discriminator_field_with_a_long_name: swap(&payload->field_with_a_long_name); break; + case the_union::discriminator_field_with_a_longer_name: swap(&payload->field_with_a_longer_name); break; + case the_union::discriminator_other: swap(&payload->other); break; + default: break; + } + return payload + 1; +} + +inline StructMemberKinds::part2* swap(StructMemberKinds::part2* payload, size_t ext_size) +{ + swap_n_fixed(payload->limited_array, ext_size); + return cast(payload->greedy); +} + +template <> +StructMemberKinds* swap(StructMemberKinds* payload) +{ + swap(&payload->member_without_docstring); + swap(&payload->ext_size); + swap(&payload->has_optional_element); + if (payload->has_optional_element) swap(&payload->optional_element); + swap_n_fixed(payload->fixed_array, 3); + StructMemberKinds::part2* part2 = cast(swap_n_fixed(payload->samples, \ +payload->ext_size)); + return cast(swap(part2, payload->ext_size)); +} + +} // namespace prophy +""" + + def test_struct_size_error(): nodes = [ model.Struct('X', [ diff --git a/prophyc/tests/generators/test_cpp_full.py b/prophyc/tests/generators/test_cpp_full.py index d46251b..ff52e2b 100644 --- a/prophyc/tests/generators/test_cpp_full.py +++ b/prophyc/tests/generators/test_cpp_full.py @@ -47,7 +47,7 @@ def generate_hpp(nodes, base_name): def process(nodes): model.cross_reference(nodes) - model.evaluate_kinds(nodes) + model.evaluate_stiffness_kinds(nodes) model.evaluate_sizes(nodes) return nodes @@ -351,7 +351,7 @@ def Builtin(): model.StructMember('x', 'u32', size=2, bound='num_of_x') ]), model.Struct('BuiltinGreedy', [ - model.StructMember('x', 'u32', unlimited=True) + model.StructMember('x', 'u32', greedy=True) ]) ]) @@ -379,7 +379,7 @@ def Fixcomp(): model.StructMember('x', 'Builtin', size=2, bound='num_of_x') ]), model.Struct('FixcompGreedy', [ - model.StructMember('x', 'Builtin', unlimited=True) + model.StructMember('x', 'Builtin', greedy=True) ]) ]) @@ -399,7 +399,7 @@ def Dyncomp(): model.StructMember('x', 'BuiltinDynamic', bound='num_of_x') ]), model.Struct('DyncompGreedy', [ - model.StructMember('x', 'BuiltinDynamic', unlimited=True) + model.StructMember('x', 'BuiltinDynamic', greedy=True) ]) ]) @@ -471,7 +471,7 @@ def Bytes(): model.StructMember('x', 'byte', size=4, bound='num_of_x') ]), model.Struct('BytesGreedy', [ - model.StructMember('x', 'byte', unlimited=True) + model.StructMember('x', 'byte', greedy=True) ]) ]) @@ -497,7 +497,7 @@ def Endpad(): ]), model.Struct('EndpadGreedy', [ model.StructMember('x', 'u32'), - model.StructMember('y', 'u8', unlimited=True) + model.StructMember('y', 'u8', greedy=True) ]) ]) @@ -2311,16 +2311,19 @@ def test_generate_hpp_newlines(): typedef b a; typedef d c; + enum E1 { E1_A = 0 }; + enum E2 { E2_A = 0 }; + enum { CONST_A = 0 }; typedef f e; @@ -2328,6 +2331,7 @@ def test_generate_hpp_newlines(): enum { CONST_B = 0 }; enum { CONST_C = 0 }; + struct A : public prophy::detail::message { enum { encoded_byte_size = 4 }; @@ -2343,6 +2347,7 @@ def test_generate_hpp_newlines(): } }; + struct B : public prophy::detail::message { enum { encoded_byte_size = 4 }; @@ -2461,7 +2466,6 @@ def test_generate_cpp(Struct): do_print(out, indent, "y", x.y); } template void message_impl::print(const X& x, std::ostream& out, size_t indent); - template <> template uint8_t* message_impl::encode(const Y& x, uint8_t* pos) @@ -2535,6 +2539,7 @@ def test_generate_hpp_with_included_struct(): #include "Input.ppf.hpp" #include "Input2.ppf.hpp" + namespace prophy { namespace generated diff --git a/prophyc/tests/generators/test_prophy_gen.py b/prophyc/tests/generators/test_prophy_gen.py new file mode 100644 index 0000000..01dc338 --- /dev/null +++ b/prophyc/tests/generators/test_prophy_gen.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8 -*- + +# flake8: noqa W291 + +import pytest +from prophyc import model + +from prophyc.generators import base, prophy + + +@pytest.fixture +def serialize(mocker): + writes = {} + + def write_mock(file_path, file_content): + writes.update({file_path: file_content}) + + mocker.patch.object(base, '_write_file', side_effect=write_mock) + mocker.patch.object(base.os.path, 'isdir', return_value=True) + + def process(nodes, gen, base_name='mainer'): + model.evaluate_model(nodes) + gen.check_nodes(nodes) + gen.serialize(nodes, base_name) + return writes + + return process + + +@pytest.fixture +def schema_gen(serialize): + def perform_test(nodes): + gen = prophy.SchemaGenerator() + writes_ = serialize(nodes, gen, "schema_test") + assert len(writes_) == 1 + file_names = list(writes_.keys()) + return writes_.get("./schema_test.prophy", "File not written, but: {}".format(file_names[0])) + + return perform_test + + +def test_empty_tree(schema_gen): + assert schema_gen([]) == '' + + +def test_structs(schema_gen): + model_nodes = [ + model.Struct('X', [ + model.StructMember('x', 'u32'), + model.StructMember('y', 'u64', size=2), + ]), + model.Struct('Y', [ + model.StructMember('q', 'u16'), + ]), + ] + assert schema_gen(model_nodes) == """\ +struct X { + u32 x; + u64 y[2]; +}; + + +struct Y { + u16 q; +}; +""" + + +LOREM_W_BREAKS = "\n".join([ + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt " + "ut labore et dolore magna aliqua. Libero nunc consequat interdum varius sit. Maecenas accumsan " + "lacus vel facilisis:", + " - Dui ut ornare,", + " - Lectus,", + " - Malesuada pellentesque,", + "", "", + "Elit eget gravida cum sociis natoque penatibus et. Netus et malesuada fames ac turpis egestas sed.", + "Neque ornare aenean euismod elementum nisi quis eleifend. Arcu dictum varius duis at " + "consectetur lorem. Nam at lectus urna duis convallis convallis. At lectus urna duis convallis " + "convallis tellus. At urna condimentum mattis pellentesque id nibh tortor id. Fames ac turpis.", + "Egestas integer eget aliquet.", +]) + + +def test_larger_model(schema_gen, larger_model): + assert schema_gen(larger_model) == """\ +#include "some_defs" +#include "cplx" + +/* the_union + * spec for that union + */ +union the_union { + 0: IncludedStruct a; + 1: cint16_t field_with_a_long_name; // Shorter + 2: cint32_t field_with_a_longer_name; // Longer description + 4090: i32 other; // This one has larger discriminator +}; + + +/* E1 + * Enumerator is a model type that is not supposed to be serialized. Its definition represents yet + another syntax variation for typing a constant. Of course elements of it's type are serializable + (as int32) + */ +enum E1 { + E1_A = 0; // enum1 constant value A + E1_B_has_a_long_name = 1; // enum1 constant va3lue B + /* E1_C_desc + * Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut + labore et dolore magna aliqua. Libero nunc consequat inte + */ + E1_C_desc = 2; +}; + + +enum E2 { + /* E2_A + * Short + * multiline + * doc + */ + E2_A = 0; +}; + + +CONST_A = 6; + +CONST_B = 0; + + +/* ==== StructMemberKinds ========================================================================== + * StructMemberKinds + * Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut + labore et dolore magna aliqua. Libero nunc consequat interdum varius sit. Maecenas accumsan lacus + vel facilisis: + * - Dui ut ornare, + * - Lectus, + * - Malesuada pellentesque, + * Elit eget gravida cum sociis natoque penatibus et. Netus et malesuada fames ac turpis egestas + sed. + * Egestas integer eget aliquet. + */ +struct StructMemberKinds { + i16 member_without_docstring; + i16 ext_size; // arbitrary sizer for dynamic arrays + cint16_t* optional_element; // optional array + cint16_t fixed_array[3]; // Array with static size. + cint16_t samples<@ext_size>; // dynamic (ext.sized) array + r64 limited_array<4>; // Has statically evaluable maximum size. + /* greedy + * Represents array of arbitrary number of elements. Buffer size must be multiply of element + size. + */ + cint16_t greedy<...>; +}; +""" + + +def test_constants_layout(schema_gen): + input_model = [ + model.Constant('CONSTANT_0', "NULL"), + model.Constant('CONST_D', "1", "This one gets a description."), + model.Constant('CONST_E', "512"), + model.Constant('CONSTANT_B', "23", "This one gets longer description. " * 4), + model.Constant('CONST_C', "NO DESC"), + model.Constant('D', "CONSTANT_B"), + ] + + assert schema_gen(input_model) == """\ + +CONSTANT_0 = NULL; + +CONST_D = 1; // This one gets a description. + +CONST_E = 512; + +/* CONSTANT_B + * This one gets longer description. This one gets longer description. This one gets longer + description. This one gets longer description. + */ +CONSTANT_B = 23; + +CONST_C = NO DESC; + +D = CONSTANT_B; +""" diff --git a/prophyc/tests/generators/test_python.py b/prophyc/tests/generators/test_python.py index e54de8b..18b3ec9 100644 --- a/prophyc/tests/generators/test_python.py +++ b/prophyc/tests/generators/test_python.py @@ -8,29 +8,47 @@ def serialize(nodes): def test_includes_rendering(): + common_include = model.Include("foo", [ + model.Constant("symbol_1", 1), + model.Constant("number_12", 12), + ]) nodes = [ - model.Include("szydlo", []), - model.Include("root/nowe_mydlo", []), - model.Include("../root/nowiejsze_powidlo", []), + common_include, + model.Include("root/ni_knights", [ + model.Include("../root/rabbit", [ + common_include, + model.Constant("pi", "3.14159"), + model.Typedef("definition", "things", "r32", "docstring"), + ]), + model.Constant("symbol_2", 2), + ]), + model.Include("../root/baz_bar", []), + model.Include("many/numbers", [model.Constant("number_%s" % n, n) for n in reversed(range(20))]), ] ref = """\ -from szydlo import * -from nowe_mydlo import * -from nowiejsze_powidlo import * +from foo import number_12, symbol_1 +from ni_knights import definition, pi, symbol_2 +from numbers import ( + number_0, number_1, number_10, number_11, number_13, number_14, number_15, + number_16, number_17, number_18, number_19, number_2, number_3, number_4, + number_5, number_6, number_7, number_8, number_9 +) """ - assert ref == serialize(nodes) + # call twice to check if 'duplication avoidance' machinery in _PythonTranslator.translate_include works ok + assert serialize(nodes) == ref + assert serialize(nodes) == ref def test_constants_rendering(): - nodes = [model.Constant("CONST_A", "0"), - model.Constant("CONST_B", "31")] + nodes = [model.Constant("CONST_A", "0", "first constant"), + model.Constant("CONST_B", "31", "second constant")] ref = """\ -CONST_A = 0 -CONST_B = 31 +CONST_A = 0 '''first constant''' +CONST_B = 31 '''second constant''' """ - assert ref == serialize(nodes) + assert serialize(nodes) == ref def test_typedefs_rendering(): @@ -39,7 +57,7 @@ def test_typedefs_rendering(): ref = """\ a = b """ - assert ref == serialize(nodes) + assert serialize(nodes) == ref def test_enums_rendering(): @@ -52,14 +70,15 @@ class EEnum(prophy.with_metaclass(prophy.enum_generator, prophy.enum)): _enumerators = [ ('EEnum_A', 0), ('EEnum_B', 1), - ('EEnum_C', 2) + ('EEnum_C', 2), ] + EEnum_A = 0 EEnum_B = 1 EEnum_C = 2 """ - assert ref == serialize(nodes) + assert serialize(nodes) == ref def test_struct_rendering(): @@ -74,10 +93,10 @@ class Struct(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): ('a', prophy.u8), ('b', prophy.i64), ('c', prophy.r32), - ('d', TTypeX) + ('d', TTypeX), ] """ - assert ref == serialize(nodes) + assert serialize(nodes) == ref def test_struct_rendering_with_dynamic_array(): @@ -88,10 +107,10 @@ def test_struct_rendering_with_dynamic_array(): class Struct(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ ('tmpName', TNumberOfItems), - ('a', prophy.array(prophy.u8, bound = 'tmpName')) + ('a', prophy.array(prophy.u8, bound='tmpName')), ] """ - assert ref == serialize(nodes) + assert serialize(nodes) == ref def test_struct_rendering_with_dynamic_arrays_bounded_by_the_same_member(): @@ -105,11 +124,11 @@ class Struct(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ ('numOfElements', TNumberOfItems), ('tmpName', prophy.u32), - ('a', prophy.array(prophy.u8, bound = 'numOfElements')), - ('b', prophy.array(prophy.r32, bound = 'numOfElements')) + ('a', prophy.array(prophy.u8, bound='numOfElements')), + ('b', prophy.array(prophy.r32, bound='numOfElements')), ] """ - assert ref == serialize(nodes) + assert serialize(nodes) == ref def test_struct_rendering_with_static_array(): @@ -118,10 +137,10 @@ def test_struct_rendering_with_static_array(): ref = """\ class Struct(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ - ('a', prophy.array(prophy.u8, size = NUM_OF_ARRAY_ELEMS)) + ('a', prophy.array(prophy.u8, size=NUM_OF_ARRAY_ELEMS)), ] """ - assert ref == serialize(nodes) + assert serialize(nodes) == ref def test_struct_rendering_with_limited_array(): @@ -132,10 +151,10 @@ def test_struct_rendering_with_limited_array(): class Struct(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ ('a_len', prophy.u8), - ('a', prophy.array(prophy.u8, bound = 'a_len', size = NUM_OF_ARRAY_ELEMS)) + ('a', prophy.array(prophy.u8, bound='a_len', size=NUM_OF_ARRAY_ELEMS)), ] """ - assert ref == serialize(nodes) + assert serialize(nodes) == ref def test_struct_rendering_with_optional(): @@ -144,10 +163,10 @@ def test_struct_rendering_with_optional(): ref = """\ class Struct(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ - ('a', prophy.optional(prophy.u32)) + ('a', prophy.optional(prophy.u32)), ] """ - assert ref == serialize(nodes) + assert serialize(nodes) == ref def test_struct_rendering_with_byte(): @@ -156,22 +175,22 @@ def test_struct_rendering_with_byte(): ref = """\ class Struct(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ - ('a', prophy.u8) + ('a', prophy.u8), ] """ - assert ref == serialize(nodes) + assert serialize(nodes) == ref def test_struct_rendering_with_byte_array(): - nodes = [model.Struct("Struct", [model.StructMember("a", "byte", unlimited=True)])] + nodes = [model.Struct("Struct", [model.StructMember("a", "byte", greedy=True)])] ref = """\ class Struct(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ - ('a', prophy.bytes()) + ('a', prophy.bytes()), ] """ - assert ref == serialize(nodes) + assert serialize(nodes) == ref def test_union_rendering(): @@ -184,10 +203,10 @@ class U(prophy.with_metaclass(prophy.union_generator, prophy.union)): _descriptor = [ ('a', A, 0), ('b', B, 1), - ('c', C, 2) + ('c', C, 2), ] """ - assert ref == serialize(nodes) + assert serialize(nodes) == ref def test_union_rendering_2(): @@ -200,17 +219,17 @@ class U(prophy.with_metaclass(prophy.union_generator, prophy.union)): _descriptor = [ ('a', prophy.i8, 0), ('b', prophy.u32, 1), - ('c', prophy.r64, 2) + ('c', prophy.r64, 2), ] """ - assert ref == serialize(nodes) + assert serialize(nodes) == ref -def test_of_PythonGenerator(): +def test_python_translator_1(): ih = [] th = [] for x in range(20, 200, 60): - ih.append(model.Include("test_include_" + str(x), [])) + ih.append(model.Include("test_include_" + str(x), [model.Constant("n_%s" % x, x, "doc")])) th.append(model.Typedef("td_elem_name_" + str(x), "td_elem_val_" + str(x))) th.append(model.Typedef("td_elem_name_" + str(x), "i_td_elem_val_" + str(x))) th.append(model.Typedef("td_elem_name_" + str(x), "u_td_elem_val_" + str(x))) @@ -234,11 +253,13 @@ def test_of_PythonGenerator(): output = python_translator(nodes, "") ref = """\ +# This file has been generated by prophyc. + import prophy -from test_include_20 import * -from test_include_80 import * -from test_include_140 import * +from test_include_20 import n_20 +from test_include_80 import n_80 +from test_include_140 import n_140 C_A = 5 C_B = 5 @@ -254,22 +275,87 @@ def test_of_PythonGenerator(): td_elem_name_140 = i_td_elem_val_140 td_elem_name_140 = u_td_elem_val_140 + class test(prophy.with_metaclass(prophy.enum_generator, prophy.enum)): _enumerators = [ ('elem_1', val_1), ('elem_31', val_31), ('elem_61', val_61), - ('elem_91', val_91) + ('elem_91', val_91), ] + elem_1 = val_1 elem_31 = val_31 elem_61 = val_61 elem_91 = val_91 + class MAC_L2CallConfigResp(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ - ('messageResult', SMessageResult) + ('messageResult', SMessageResult), ] """ assert output == ref + + +def test_python_translator_2(larger_model): + python_translator = _PythonTranslator() + output = python_translator(larger_model, "") + + assert output == """\ +# This file has been generated by prophyc. + +import prophy + +from some_defs import IncludedStruct, c +from cplx import cint16_t, cint32_t + +a = prophy.i16 +c = a + + +class the_union(prophy.with_metaclass(prophy.union_generator, prophy.union)): + _descriptor = [ + ('a', IncludedStruct, 0), + ('field_with_a_long_name', cint16_t, 1), + ('field_with_a_longer_name', cint32_t, 2), + ('other', prophy.i32, 4090), + ] + + +class E1(prophy.with_metaclass(prophy.enum_generator, prophy.enum)): + _enumerators = [ + ('E1_A', 0), + ('E1_B_has_a_long_name', 1), + ('E1_C_desc', 2), + ] + + +E1_A = 0 +E1_B_has_a_long_name = 1 +E1_C_desc = 2 + + +class E2(prophy.with_metaclass(prophy.enum_generator, prophy.enum)): + _enumerators = [ + ('E2_A', 0), + ] + + +E2_A = 0 +CONST_A = 6 +CONST_B = 0 + + +class StructMemberKinds(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): + _descriptor = [ + ('member_without_docstring', prophy.i16), + ('ext_size', prophy.i16), + ('optional_element', prophy.optional(cint16_t)), + ('fixed_array', prophy.array(cint16_t, size=3)), + ('samples', prophy.array(cint16_t, bound='ext_size')), + ('limited_array', prophy.array(prophy.r64, bound='ext_size', size=4)), + ('greedy', prophy.array(cint16_t)), + ] +""" diff --git a/prophyc/tests/generators/test_word_wrap.py b/prophyc/tests/generators/test_word_wrap.py new file mode 100644 index 0000000..131692e --- /dev/null +++ b/prophyc/tests/generators/test_word_wrap.py @@ -0,0 +1,384 @@ +import pytest + +from prophyc.generators import word_wrap + + +# flake8: noqa W291 + + +@pytest.fixture +def foo_generator(): + foo_breaker = word_wrap.BreakLinesByWidth( + max_line_width=40, + soft_indent_tail=" " * 7, + ) + + @foo_breaker + def foo_writer(raw_paragraphs): + if raw_paragraphs: + foo_breaker.make_a_bar(".", "Foo writes:") + for line_index, line in enumerate(raw_paragraphs.split("\n")): + yield "l0x{:02X}: {}".format(line_index, line) + + return foo_writer + + +FOO_GEN_TEST = ["""\ +.... Foo writes: ....................... +l0x00: Lorem ipsum dolor sit amet, + consectetur adipiscing elit, sed + do eiusmod tempor incididunt ut + labore et dolore magna aliqua. + Libero nunc consequat interdum + varius sit. Maecenas accumsan + lacus vel facilisis: +l0x01: - Dui ut ornare, +l0x02: - Lectus, +l0x03: - Malesuada pellentesque, +l0x04: +l0x05: +l0x06: Elit eget gravida cum sociis + natoque penatibus et. Netus et + malesuada fames ac turpis egestas + sed. +l0x07: Egestas integer eget aliquet. +""", """\ + .... Foo writes: ................... + l0x00: Lorem ipsum dolor sit amet, + consectetur adipiscing elit, + sed do eiusmod tempor + incididunt ut labore et + dolore magna aliqua. Libero + nunc consequat interdum + varius sit. Maecenas accumsan + lacus vel facilisis: + l0x01: - Dui ut ornare, + l0x02: - Lectus, + l0x03: - Malesuada pellentesque, + l0x04: + l0x05: + l0x06: Elit eget gravida cum sociis + natoque penatibus et. Netus + et malesuada fames ac turpis + egestas sed. + l0x07: Egestas integer eget aliquet. +""", """\ + .... Foo writes: ............... + l0x00: Lorem ipsum dolor sit + amet, consectetur + adipiscing elit, sed do + eiusmod tempor incididunt + ut labore et dolore magna + aliqua. Libero nunc + consequat interdum varius + sit. Maecenas accumsan + lacus vel facilisis: + l0x01: - Dui ut ornare, + l0x02: - Lectus, + l0x03: - Malesuada + pellentesque, + l0x04: + l0x05: + l0x06: Elit eget gravida cum + sociis natoque penatibus + et. Netus et malesuada + fames ac turpis egestas + sed. + l0x07: Egestas integer eget + aliquet. +""" + + ] + + +@pytest.mark.parametrize("level, reference_str", enumerate(FOO_GEN_TEST), + ids=[str(i) for i in range(len(FOO_GEN_TEST))]) +def test_foo_generator(foo_generator, lorem_with_breaks, level, reference_str): + assert render_str(foo_generator, lorem_with_breaks, level) == reference_str + + +@pytest.fixture +def c_breaker(): + c_breaker = word_wrap.BreakLinesByWidth( + 40, " ", "/* ", " * ", " ", " */" + ) + return c_breaker + + +def test_c_breaker1(c_breaker): + @c_breaker + def my_gen(): + c_breaker.make_a_bar(title="that bar") + yield "thing" + c_breaker.make_a_bar(".", title="sub bar 1") + c_breaker.make_a_bar(".", title="sub bar 2") + yield "multi\nline\nparagraph" + + assert "".join(my_gen(indent_level=2)) == """\ + /* ==== that bar =============== + * thing + * .... sub bar 1 .............. + * .... sub bar 2 .............. + * multi + line + paragraph + */""" + + +def test_c_breaker2(c_breaker): + @c_breaker + def my_gen(): + c_breaker.make_a_bar(title="that bar") + return iter(()) + + assert "".join(my_gen(indent_level=2)) == """\ + /* ==== that bar =============== + */""" + + +def test_c_breaker3(c_breaker): + @c_breaker + def my_gen(): + return iter(()) + + assert "".join(my_gen(indent_level=2)) == """\ + /* */""" + + +@pytest.fixture +def c_alike_generator(c_breaker): + @c_breaker + def c_writer(raw_paragraphs): + for line in raw_paragraphs.split("\n"): + yield line + + return c_writer + + +@pytest.mark.parametrize("indent, expected_result", enumerate([ + "/* */", + " /* */", + " /* */", + " /* */", +])) +def test_c_generator_empty(c_alike_generator, indent, expected_result): + assert render_str(c_alike_generator, "", indent) == expected_result + + +def test_wraps_with_overflow(c_alike_generator): + input_str = ( + "Lorem ipsum dolor sit amet, consecteturadipiscingelitseddoeiusmodtempor ut " + "labore et dolore magna aliqua. Liberonuncconsequatinterdumvariussit. Maecenas accumsan lacus") + + assert render_str(c_alike_generator, input_str, 3) == """\ + /* Lorem ipsum dolor sit + amet, + consecteturadipiscingelitseddoeiusmodtempor + ut labore et dolore magna + aliqua. + Liberonuncconsequatinterdumvariussit. + Maecenas accumsan lacus + */""" + + +def test_c_generator_0(c_alike_generator, lorem_with_breaks): + reference_str = """\ +/* Lorem ipsum dolor sit amet, + consectetur adipiscing elit, sed do + eiusmod tempor incididunt ut labore + et dolore magna aliqua. Libero nunc + consequat interdum varius sit. + Maecenas accumsan lacus vel + facilisis: + * - Dui ut ornare, + * - Lectus, + * - Malesuada pellentesque, + * Elit eget gravida cum sociis natoque + penatibus et. Netus et malesuada + fames ac turpis egestas sed. + * Egestas integer eget aliquet. + */""" + assert render_str(c_alike_generator, lorem_with_breaks, 0) == reference_str + + +def test_c_generator_2(c_alike_generator, lorem_with_breaks): + reference_str = """\ + /* Lorem ipsum dolor sit amet, + consectetur adipiscing elit, + sed do eiusmod tempor + incididunt ut labore et + dolore magna aliqua. Libero + nunc consequat interdum + varius sit. Maecenas accumsan + lacus vel facilisis: + * - Dui ut ornare, + * - Lectus, + * - Malesuada pellentesque, + * Elit eget gravida cum sociis + natoque penatibus et. Netus + et malesuada fames ac turpis + egestas sed. + * Egestas integer eget aliquet. + */""" + assert render_str(c_alike_generator, lorem_with_breaks, 2) == reference_str + + +def render_str(gen_, input_str, indent_level): + return "".join(gen_(input_str, indent_level=indent_level)) + + +@pytest.fixture +def fooing_gen(): + breaker = word_wrap.BreakLinesByWidth( + max_line_width=40, + hard_indent_tail=" % ", + soft_indent_tail=" | ", + ) + + @breaker + def fooing(raw_paragraphs): + breaker.make_a_bar("=", "that things") + yield ">fooing start:" + for line in raw_paragraphs.split("\n"): + yield line + + yield ":fooing end<" + breaker.make_a_bar("^") + + return fooing + + +def test_empty_text_word_wrap(fooing_gen): + assert "".join(fooing_gen("", indent_level=0)) == """\ + % ==== that things ==================== + % >fooing start: + % :fooing end< + % ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +""" + + +LOREM_SINGLE_LINE = ( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt " + "ut labore et dolore magna aliqua. Libero nunc consequat interdum varius sit. Maecenas accumsan " + "lacus vel facilisis.\n" +) + +FOOING_TEST = [ + """\ + % ==== that things ==================== + % >fooing start: + % Lorem ipsum dolor sit amet, + | consectetur adipiscing elit, sed do + | eiusmod tempor incididunt ut labore + | et dolore magna aliqua. Libero nunc + | consequat interdum varius sit. + | Maecenas accumsan lacus vel + | facilisis. + % :fooing end< + % ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +""", """\ + % ==== that things ================ + % >fooing start: + % Lorem ipsum dolor sit amet, + | consectetur adipiscing elit, sed + | do eiusmod tempor incididunt ut + | labore et dolore magna aliqua. + | Libero nunc consequat interdum + | varius sit. Maecenas accumsan + | lacus vel facilisis. + % :fooing end< + % ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +""", """\ + % ==== that things ============ + % >fooing start: + % Lorem ipsum dolor sit amet, + | consectetur adipiscing elit, + | sed do eiusmod tempor + | incididunt ut labore et + | dolore magna aliqua. Libero + | nunc consequat interdum + | varius sit. Maecenas accumsan + | lacus vel facilisis. + % :fooing end< + % ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +""", """\ + % ==== that things ======== + % >fooing start: + % Lorem ipsum dolor sit + | amet, consectetur + | adipiscing elit, sed do + | eiusmod tempor incididunt + | ut labore et dolore magna + | aliqua. Libero nunc + | consequat interdum varius + | sit. Maecenas accumsan + | lacus vel facilisis. + % :fooing end< + % ^^^^^^^^^^^^^^^^^^^^^^^^^ +""" +] + + +@pytest.mark.parametrize("indent, reference_str", enumerate(FOOING_TEST), ids=[str(i) for i in range(len(FOOING_TEST))]) +def test_single_line_slpit_0(fooing_gen, indent, reference_str): + assert "".join(fooing_gen(LOREM_SINGLE_LINE, indent_level=indent)) == reference_str + + +@pytest.mark.parametrize("string", [ + "", " ", "\n", "abcd", "abcd efgh", "abcd\nefgh", "\nabcd\nefgh\n", "\n \t \n" +]) +def test_split_short_string_not_modified(string): + assert "".join(word_wrap.split_long_string(string)) == string + + +def test_split_string_with_short_lines(): + INPUT = """\ + | Lorem ipsum dolor sit + | amet, consectetur + | adipiscing elit, sed do + | eiusmod tempor incididunt + | ut labore et dolore magna + | aliqua. +""" + assert list(word_wrap.split_long_string(INPUT)) == [ + ' | Lorem ipsum dolor sit\n', + ' | amet, consectetur\n', + ' | adipiscing elit, sed do\n', + ' | eiusmod tempor incididunt\n', + ' | ut labore et dolore magna\n', + ' | aliqua.\n', + '' + ] + + +def test_breaking_long_string_no_breaks(): + parts = list(word_wrap.split_long_string(LOREM_SINGLE_LINE)) + assert "".join(parts) == LOREM_SINGLE_LINE + assert parts == [ + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ', + 'ut labore et dolore magna aliqua. Libero nunc consequat interdum varius sit. Maecenas ', + 'accumsan lacus vel facilisis.\n' + ] + + +def test_breaking_long_string(lorem_with_breaks): + parts = list(word_wrap.split_long_string(lorem_with_breaks)) + assert "".join(parts) == lorem_with_breaks + assert parts == [ + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ', + 'ut labore et dolore magna aliqua. Libero nunc consequat interdum varius sit. Maecenas ', + 'accumsan lacus vel facilisis:\n - Dui ut ornare,\n - Lectus,\n - Malesuada pellentesque,\n\n\nElit ', + 'eget gravida cum sociis natoque penatibus et. Netus et malesuada fames ac turpis ', + 'egestas sed.\nEgestas integer eget aliquet.' + ] + + +def test_hard_breaking_string(): + text = "".join(chr(c) for s, e in ('AZ', '09', 'az') * 4 for c in range(ord(s), ord(e))) + parts = list(word_wrap.split_long_string(text)) + assert "".join(parts) == text + assert parts == [ + 'ABCDEFGHIJKLMNOPQRSTUVWXY012345678abcdefghijklmnopqrstuvwxyABCDEFGHIJKLMNOPQRSTU', + 'VWXY012345678abcdefghijklmnopqrstuvwxyABCDEFGHIJKLMNOPQRSTUVWXY012345678abcdefgh', + 'ijklmnopqrstuvwxyABCDEFGHIJKLMNOPQRSTUVWXY012345678abcdefghijklmnopqrstuvwxy' + ] diff --git a/prophyc/tests/parsers/test_isar.py b/prophyc/tests/parsers/test_isar.py index 57a7217..963d596 100644 --- a/prophyc/tests/parsers/test_isar.py +++ b/prophyc/tests/parsers/test_isar.py @@ -24,7 +24,7 @@ def test_includes_parsing(): """ nodes = parse(xml, process_file=lambda path: [model.Typedef("a", "u8")]) - assert [ + assert nodes == [ ("mydlo", [model.Typedef("a", "u8")]), ("szydlo", [model.Typedef("a", "u8")]), ("powidlo", [model.Typedef("a", "u8")]), @@ -32,7 +32,7 @@ def test_includes_parsing(): ("../entliczek/petliczek", [model.Typedef("a", "u8")]), ("../my.basename", [model.Typedef("a", "u8")]), ("noext", [model.Typedef("a", "u8")]), - ] == nodes + ] def test_includes_call_file_process_with_proper_path(): @@ -61,6 +61,7 @@ def test_includes_cyclic_include_error(): def process_file_with_error(path): raise CyclicIncludeError(path) + warnings = [] nodes = parse( @@ -81,6 +82,7 @@ def test_includes_file_not_found_error(): def process_file_with_error(path): raise FileNotFoundError(path) + warnings = [] nodes = parse( @@ -172,9 +174,8 @@ def test_enums_parsing_repeated_value(): """ - with pytest.raises(ValueError) as e: + with pytest.raises(ValueError, match="Duplicate Enum value"): parse(xml) - assert "Duplicate Enum value" in str(e) def test_struct_parsing(): @@ -519,9 +520,7 @@ def test_empty_elemens_parsing(): """ - nodes = parse(xml) - - assert len(nodes) == 0 + assert parse(xml) == [] def test_primitive_types(): diff --git a/prophyc/tests/parsers/test_prophy.py b/prophyc/tests/parsers/test_prophy.py index 57c1eeb..63add4a 100644 --- a/prophyc/tests/parsers/test_prophy.py +++ b/prophyc/tests/parsers/test_prophy.py @@ -73,7 +73,7 @@ def test_typedefs_parsing(): assert parse(content) == [ model.Typedef("x", "u32"), - model.Typedef("y", "x") + model.Typedef("y", "x", model.Typedef('x', 'u32')) ] @@ -154,7 +154,7 @@ def test_structs_with_dynamic_array_parsing(): model.Typedef('x_t', 'u32'), model.Struct('test', [ model.StructMember('num_of_x', 'u32'), - model.StructMember('x', 'x_t', bound='num_of_x'), + model.StructMember('x', 'x_t', model.Typedef('x_t', 'u32'), bound='num_of_x'), model.StructMember('num_of_y', 'u32'), model.StructMember('y', 'byte', bound='num_of_y') ]) @@ -178,7 +178,7 @@ def test_structs_with_dynamic_arrays_bounded_by_the_same_member_parsing(): model.Struct('test', [ model.StructMember('num_of_elements', 'u32'), model.StructMember('dummy', 'u16'), - model.StructMember('x', 'x_t', bound='num_of_elements'), + model.StructMember('x', 'x_t', model.Typedef('x_t', 'u32'), bound='num_of_elements'), model.StructMember('y', 'byte', bound='num_of_elements') ]) ] @@ -199,9 +199,15 @@ def test_structs_with_dynamic_arrays_bounded_by_the_same_member_parsing_typedef_ assert parse(content) == [ model.Typedef('num_of_elements_t', 'i32'), - model.Typedef('sz_t', 'num_of_elements_t'), + model.Typedef('sz_t', 'num_of_elements_t', model.Typedef('num_of_elements_t', 'i32', None)), model.Struct('ExtSized', [ - model.StructMember('sz', 'sz_t'), + model.StructMember( + 'sz', 'sz_t', model.Typedef( + 'sz_t', 'num_of_elements_t', model.Typedef( + 'num_of_elements_t', 'i32', None + ) + ) + ), model.StructMember('one', 'u32', bound='sz'), model.StructMember('two', 'u16', bound='sz'), model.StructMember('three', 'i32', bound='sz') @@ -245,7 +251,7 @@ def test_structs_with_greedy_array_parsing(): assert parse(content) == [ model.Struct('test', [ - model.StructMember('x', 'u32', unlimited=True) + model.StructMember('x', 'u32', greedy=True) ]) ] @@ -549,8 +555,9 @@ def test_error_union_empty(): def test_error_union_repeated_arm_name(): - with pytest.raises(ParseError) as e: - parse('union test { 1: u32 x; 2: u32 x; };') + with pytest.raises(model.ModelError, match="Duplicated 'x' identifier in union test."): + with pytest.raises(ParseError) as e: + parse('union test { 1: u32 x; 2: u32 x; };') assert ":1:31", "field 'x' redefined" == e.value.errors[0] @@ -581,6 +588,7 @@ def test_multiple_errors(): """) assert e.value.errors == [ ("test.prophy:1:23", "field 'x' redefined"), + ('test.prophy:2:1', "Duplicated 'x' identifier in struct X."), ("test.prophy:2:11", "constant 'unknown' was not declared"), ("test.prophy:3:9", "syntax error at '2'") ] @@ -731,10 +739,12 @@ def test_include_errors(): with pytest.raises(ParseError) as e: def parse_file(path): raise FileNotFoundError(path) + parse('#include "imnotthere"', parse_file) assert e.value.errors == [("test.prophy:1:10", "file imnotthere not found")] with pytest.raises(ParseError) as e: def parse_file(path): raise CyclicIncludeError(path) + parse('#include "includemeagain"', parse_file) assert e.value.errors == [("test.prophy:1:10", "file includemeagain included again during parsing")] diff --git a/prophyc/tests/parsers/test_sack.py b/prophyc/tests/parsers/test_sack.py index 468e557..345384f 100644 --- a/prophyc/tests/parsers/test_sack.py +++ b/prophyc/tests/parsers/test_sack.py @@ -9,17 +9,8 @@ def parse(content, extension='.hpp', warn=None, includes=[]): return SackParser(warn=warn, include_dirs=includes).parse(content, file_name + extension, None) -class contains_cmp(object): - def __init__(self, x): - self.x = x - - def __eq__(self, other): - return any((self.x in other, other in self.x)) - - @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_simple_struct(extension): +def test_simple_struct(if_clang_installed, extension): hpp = """\ #include struct X @@ -39,8 +30,7 @@ def test_simple_struct(extension): @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_ints(extension): +def test_ints(if_clang_installed, extension): hpp = """\ #include struct X @@ -84,8 +74,7 @@ def test_ints(extension): @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_nested_typedefs(extension): +def test_nested_typedefs(if_clang_installed, extension): hpp = """\ typedef int my_int; typedef my_int i_like_typedefs; @@ -99,8 +88,7 @@ def test_nested_typedefs(extension): @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_typedefed_struct(extension): +def test_typedefed_struct(if_clang_installed, extension): hpp = """\ #include typedef struct @@ -118,8 +106,7 @@ def test_typedefed_struct(extension): ] -@pytest.clang_installed -def test_namespaced_struct(): +def test_namespaced_struct(if_clang_installed, ): hpp = """\ #include namespace m @@ -144,8 +131,7 @@ def test_namespaced_struct(): @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_array(extension): +def test_array(if_clang_installed, extension): hpp = """\ #include struct X @@ -161,8 +147,7 @@ def test_array(extension): @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_enum(extension): +def test_enum(if_clang_installed, extension): hpp = """\ enum Enum { @@ -188,8 +173,7 @@ def test_enum(extension): @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_typedefed_enum(extension): +def test_typedefed_enum(if_clang_installed, extension): hpp = """\ typedef enum Enum { @@ -214,8 +198,7 @@ def test_typedefed_enum(extension): ] -@pytest.clang_installed -def test_namespaced_enum(): +def test_namespaced_enum(if_clang_installed, ): hpp = """\ namespace m { @@ -246,8 +229,7 @@ def test_namespaced_enum(): ] -@pytest.clang_installed -def test_namespaced_typedef(): +def test_namespaced_typedef(if_clang_installed, ): hpp = """\ #include namespace N { typedef uint32_t u32; } @@ -264,8 +246,7 @@ def test_namespaced_typedef(): @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_enum_with_negative_one_values(extension): +def test_enum_with_negative_one_values(if_clang_installed, extension): hpp = """\ enum Enum { @@ -291,8 +272,7 @@ def test_enum_with_negative_one_values(extension): @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_multiple_enums(extension): +def test_multiple_enums(if_clang_installed, extension): hpp = """\ typedef enum Enum { @@ -322,8 +302,7 @@ def test_multiple_enums(extension): @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_c_enum(extension): +def test_c_enum(if_clang_installed, extension): hpp = """\ typedef enum { @@ -349,8 +328,7 @@ def test_c_enum(extension): @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_union(extension): +def test_union(if_clang_installed, extension): hpp = """\ #include union Union @@ -377,8 +355,7 @@ def test_union(extension): @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_typedefed_union(extension): +def test_typedefed_union(if_clang_installed, extension): hpp = """\ #include typedef union @@ -401,8 +378,7 @@ def test_typedefed_union(extension): @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_multiple_structs(extension): +def test_multiple_structs(if_clang_installed, extension): hpp = """\ #include struct X @@ -433,8 +409,7 @@ def test_multiple_structs(extension): ] -@pytest.clang_installed -def test_class_template(): +def test_class_template(if_clang_installed, ): hpp = """\ #include #include @@ -464,8 +439,7 @@ class A @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_c_struct(extension): +def test_c_struct(if_clang_installed, extension): hpp = """\ #ifdef __cplusplus extern "C" { @@ -486,9 +460,7 @@ def test_c_struct(extension): ] -@pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_struct_with_anonymous_struct(extension): +def test_struct_with_anonymous_struct(if_clang_installed, ): hpp = """\ struct X { @@ -498,20 +470,26 @@ def test_struct_with_anonymous_struct(extension): } a[3]; }; """ - Anonymous = contains_cmp("X__anonymous__") - assert parse(hpp, extension) == [ - model.Struct(Anonymous, [ + assert parse(hpp, '.h') == [ + model.Struct("X__anonymous__at__test__h__3__5__", [ + model.StructMember("b", "i8") + ]), + model.Struct("X", [ + model.StructMember("a", "X__anonymous__at__test__h__3__5__", size=3) + ]) + ] + assert parse(hpp, '.hpp') == [ + model.Struct("X__anonymous__struct__at__test__hpp__3__5__", [ model.StructMember("b", "i8") ]), model.Struct("X", [ - model.StructMember("a", Anonymous, size=3) + model.StructMember("a", "X__anonymous__struct__at__test__hpp__3__5__", size=3) ]) ] @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_struct_with_incomplete_array(extension): +def test_struct_with_incomplete_array(if_clang_installed, extension): hpp = """\ struct X { @@ -526,8 +504,7 @@ def test_struct_with_incomplete_array(extension): @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_struct_with_incomplete_array_in_file_with_hyphen(extension): +def test_struct_with_incomplete_array_in_file_with_hyphen(if_clang_installed, extension): hpp = """\ struct X { @@ -543,8 +520,7 @@ def test_struct_with_incomplete_array_in_file_with_hyphen(extension): @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_forward_declared_struct(extension): +def test_forward_declared_struct(if_clang_installed, extension): hpp = """\ struct X; """ @@ -552,8 +528,7 @@ def test_forward_declared_struct(extension): @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_omit_bitfields(extension): +def test_omit_bitfields(if_clang_installed, extension): hpp = """\ struct X { @@ -578,8 +553,7 @@ def __call__(self, msg, location='prophyc'): ('.c', 'type specifier missing, defaults to \'int\''), ('.cpp', 'C++ requires a type specifier for all declarations'), ]) -@pytest.clang_installed -def test_diagnostics_error(extension, expected): +def test_diagnostics_error(if_clang_installed, extension, expected): content = """\ unknown; """ @@ -592,8 +566,7 @@ def test_diagnostics_error(extension, expected): @pytest.mark.parametrize('extension', [('.c'), ('.cpp')]) -@pytest.clang_installed -def test_diagnostics_warning(extension): +def test_diagnostics_warning(if_clang_installed, extension): content = """\ int foo() { @@ -608,16 +581,14 @@ def test_diagnostics_warning(extension): assert warn.warnings == [('test' + extension + ':4:1', 'control reaches end of non-void function')] -@pytest.clang_installed -def test_libclang_parsing_error(): +def test_libclang_parsing_error(if_clang_installed, ): with pytest.raises(model.ParseError) as e: parse('content', '') assert e.value.errors == [('test', 'error parsing translation unit')] @pytest.mark.parametrize('extension', [('.h'), ('.hpp')]) -@pytest.clang_installed -def test_include_dirs(extension, tmpdir): +def test_include_dirs(if_clang_installed, extension, tmpdir): dependency_hpp = '''\ typedef double Double; ''' diff --git a/prophyc/tests/test_calc.py b/prophyc/tests/test_calc.py index 2e2effd..f812211 100644 --- a/prophyc/tests/test_calc.py +++ b/prophyc/tests/test_calc.py @@ -11,6 +11,11 @@ def test_calc_numeric(): assert calc.eval('10 >> 2', {}) == 2 assert calc.eval('-10', {}) == -10 assert calc.eval('\n-10\n', {}) == -10 + assert calc.eval('0x0', {}) == 0 + assert calc.eval('0x3', {}) == 3 + assert calc.eval('\n-0x1a\n', {}) == -26 + assert calc.eval('0x2 + 0x00ff', {}) == 257 + assert calc.eval('0x2 - 0xff', {}) == -253 def test_calc_parens(): @@ -27,14 +32,14 @@ def test_calc_nested_variables(): def test_calc_errors(): - with pytest.raises(calc.ParseError) as e: + with pytest.raises(calc.ParseError, match="illegal character &"): calc.eval('&', {}) - assert 'illegal' in str(e.value) - with pytest.raises(calc.ParseError) as e: + with pytest.raises(calc.ParseError, match=r"syntax error at '\+'"): calc.eval('++', {}) - assert 'syntax' in str(e.value) - with pytest.raises(calc.ParseError) as e: + with pytest.raises(calc.ParseError, match="numeric constant 'unknown' not found"): calc.eval('unknown', {}) - assert 'not found' in str(e.value) + + with pytest.raises(calc.ParseError, match="numeric constant 'x2' not found"): + calc.eval('2 + x2', {}) diff --git a/prophyc/tests/test_model.py b/prophyc/tests/test_model.py index 136dfe0..206b49a 100644 --- a/prophyc/tests/test_model.py +++ b/prophyc/tests/test_model.py @@ -1,28 +1,87 @@ +# -*- coding: utf-8 -*- + +import pytest +import renew + +import prophyc # noqa from prophyc import model -def test_typedef_repr(): +def test_node_eq(): + a = model.Enum("name", [], "docstr 1") + b = model.Enum("name", [], "docstr 2") + c = model.Struct("name", [], "docstr 3") + + assert a == b + assert a != c + + +def assert_repr_reproduces(object_): + """ The second assertion assumes that __eq__ checks actual equality. In fact - it skips docstring. """ + assert repr(eval(repr(object_))) == repr(object_) + assert eval(repr(object_)) == object_ + return True + + +def test_typedef_str(): typedef = model.Typedef("my_typedef", "u8") - assert str(typedef) == "u8 my_typedef" + assert str(typedef) == "typedef u8 my_typedef;" + + +def test_typedef_repr(): + assert_repr_reproduces(model.Typedef('my_typedef', 'u8', docstring='comment')) def test_struct_repr(): - struct = model.Struct("MyStruct", [ + struct_a = model.Struct("MyStruct", [ model.StructMember("a", "u8"), - model.StructMember("b", "u16", bound='xlen'), - model.StructMember("c", "u32", size=5), - model.StructMember("d", "u64", bound='xlen', size=5), - model.StructMember("e", "UU", unlimited=True), - model.StructMember("f", "UUUU", optional=True) - ]) - assert str(struct) == """\ -MyStruct - u8 a - u16 b<>(xlen) - u32 c[5] - u64 d<5>(xlen) - UU e<...> - UUUU* f + model.StructMember("b", "cint16_t"), + model.StructMember("c", "u32", size=3, docstring="no_docstring"), + ], docstring="no_docstring_") + + assert repr(struct_a) == """\ +prophyc.model.Struct( + 'MyStruct', + [ + prophyc.model.StructMember('a', 'u8'), + prophyc.model.StructMember('b', 'cint16_t'), + prophyc.model.StructMember('c', 'u32', size=3, docstring='no_docstring'), + ], + 'no_docstring_', +)""" + + assert_repr_reproduces(struct_a) + assert struct_a == eval(repr(struct_a)) + + assert_repr_reproduces(struct_a) + + +def test_struct_str(): + struct_with_arrays = model.Struct("MyStruct", [ + model.StructMember("sizer_field", "u8"), + model.StructMember("b", "UserDefinedType"), + model.StructMember("c", "UserDefinedType", optional=True), + model.StructMember("fixed_array", "u32", size=3, docstring="no_docstring"), + model.StructMember("dynamic_array", "u32", bound='num_of_dynamic_array'), + model.StructMember("limited_array", "r64", bound='num_of_limited_array', size=3), + model.StructMember("ext_sized_1", "i32", bound="sizer_field"), + model.StructMember("ext_sized_2", "i16", bound="sizer_field"), + model.StructMember("greedy_array", "u8", greedy=True), + ], docstring="no_docstring_") + + # todo: its against documentation + assert str(struct_with_arrays) == """\ +struct MyStruct { + u8 sizer_field; + UserDefinedType b; + UserDefinedType* c; + u32 fixed_array[3]; + u32 dynamic_array<@num_of_dynamic_array>; + r64 limited_array<3>; + i32 ext_sized_1<@sizer_field>; + i16 ext_sized_2<@sizer_field>; + u8 greedy_array<...>; +}; """ @@ -30,28 +89,198 @@ def test_union_repr(): union = model.Union("MyUnion", [ model.UnionMember("a", "u8", 1), model.UnionMember("b", "u16", 2), - model.UnionMember("c", "u32", 3) + model.UnionMember("c", "u32", 3, docstring="deff") ]) - assert str(union.members[0]) == "1: u8 a" - assert str(union.members[1]) == "2: u16 b" - assert str(union.members[2]) == "3: u32 c" + assert_repr_reproduces(union) assert str(union) == """\ -MyUnion - 1: u8 a - 2: u16 b - 3: u32 c +union MyUnion { + 1: u8 a; + 2: u16 b; + 3: u32 c; +}; +""" + + +def test_larger_model_str(larger_model): + processed, _ = model.evaluate_model(larger_model) + assert "\n".join(str(node) for node in processed) == """\ +typedef i16 a; +typedef a c; +#include some_defs; + +#include cplx; + +union the_union { + 0: IncludedStruct a; + 1: cint16_t field_with_a_long_name; + 2: cint32_t field_with_a_longer_name; + 4090: i32 other; +}; + +enum E1 { + E1_A = '0'; + E1_B_has_a_long_name = '1'; + E1_C_desc = '2'; +}; + +enum E2 { + E2_A = '0'; +}; + +const CONST_A = '6'; +const CONST_B = '0'; +struct StructMemberKinds { + i16 member_without_docstring; + i16 ext_size; + cint16_t* optional_element; + cint16_t fixed_array[3]; + cint16_t samples<@ext_size>; + r64 limited_array<4>; + cint16_t greedy<...>; +}; """ +def test_larger_model_repr(larger_model): + assert renew.reproduction(larger_model) == r"""[ + prophyc.model.Typedef('a', 'i16'), + prophyc.model.Typedef('c', 'a'), + prophyc.model.Include( + 'some_defs', + [ + prophyc.model.Struct( + 'IncludedStruct', + [ + prophyc.model.StructMember('member1', 'r32', docstring='doc for member1'), + prophyc.model.StructMember('member2', 'u64', docstring='docstring for member1'), + ], + ), + prophyc.model.Typedef('c', 'a'), + ], + ), + prophyc.model.Include( + 'cplx', + [ + prophyc.model.Struct( + 'cint16_t', + [ + prophyc.model.StructMember('re', 'i16', docstring='real'), + prophyc.model.StructMember('im', 'i16', docstring='imaginary'), + ], + ), + prophyc.model.Struct( + 'cint32_t', + [ + prophyc.model.StructMember('re', 'i32', docstring='real'), + prophyc.model.StructMember('im', 'i32', docstring='imaginary'), + ], + ), + ], + ), + prophyc.model.Union( + 'the_union', + [ + prophyc.model.UnionMember('a', 'IncludedStruct', 0), + prophyc.model.UnionMember('field_with_a_long_name', 'cint16_t', 1, docstring='Shorter'), + prophyc.model.UnionMember('field_with_a_longer_name', 'cint32_t', 2, docstring='Longer description'), + prophyc.model.UnionMember('other', 'i32', 4090, docstring='This one has larger discriminator'), + ], + 'spec for that union', + ), + prophyc.model.Enum( + 'E1', + [ + prophyc.model.EnumMember('E1_A', '0', 'enum1 constant value A'), + prophyc.model.EnumMember('E1_B_has_a_long_name', '1', 'enum1 constant va3lue B'), + prophyc.model.EnumMember( + 'E1_C_desc', + '2', + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ' + 'ut labore et dolore magna aliqua. Libero nunc consequat inte' + ), + ], + 'Enumerator is a model type that is not supposed to be serialized. Its definition ' + 'represents yet another syntax variation for typing a constant. Of course elements ' + "of it's type are serializable (as int32)" + ), + prophyc.model.Enum('E2', [prophyc.model.EnumMember('E2_A', '0', 'Short\nmultiline\ndoc')]), + prophyc.model.Constant('CONST_A', '6'), + prophyc.model.Constant('CONST_B', '0'), + prophyc.model.Struct( + 'StructMemberKinds', + [ + prophyc.model.StructMember('member_without_docstring', 'i16'), + prophyc.model.StructMember('ext_size', 'i16', docstring='arbitrary sizer for dynamic arrays'), + prophyc.model.StructMember( + 'optional_element', + 'cint16_t', + optional=True, + docstring='optional array', + ), + prophyc.model.StructMember('fixed_array', 'cint16_t', size=3, docstring='Array with static size.'), + prophyc.model.StructMember( + 'samples', + 'cint16_t', + bound='ext_size', + docstring='dynamic (ext.sized) array', + ), + prophyc.model.StructMember( + 'limited_array', + 'r64', + bound='ext_size', + size=4, + docstring='Has statically evaluable maximum size.', + ), + prophyc.model.StructMember( + 'greedy', + 'cint16_t', + greedy=True, + docstring='Represents array of arbitrary number of elements. Buffer size must be multiply of ' + 'element size.' + ), + ], + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ' + 'ut labore et dolore magna aliqua. Libero nunc consequat interdum varius sit. Maecenas ' + 'accumsan lacus vel facilisis:\n - Dui ut ornare,\n - Lectus,\n - Malesuada pellentesque,\n\n\nElit ' + 'eget gravida cum sociis natoque penatibus et. Netus et malesuada fames ac turpis ' + 'egestas sed.\nEgestas integer eget aliquet.' + ), +]""" + assert_repr_reproduces(larger_model) + + +NODES_CONSTRUCTORS = [ + (model.Constant, ("ño", "value")), + (model.EnumMember, ("ño", "value")), + (model.Typedef, ("ño", "value")), + (model.StructMember, ("ño", "value")), + (model.UnionMember, ("ño", "tp_", 2)), + (model.Include, ("ño", [])), + (model.Struct, ("ño", [],)), + (model.Union, ("ño", [])), + (model.Enum, ("ño", [])), +] + + +@pytest.mark.parametrize("k, args", NODES_CONSTRUCTORS) +def test_unicode_handling(k, args): + a = k(*args, docstring="used weird letter from 'jalapeño' word") + b = k(*args, docstring="used weird letter from 'jalapeño' word") + assert a == b + assert a.docstring == b.docstring + + def test_split_after(): generator = model.split_after([1, 42, 2, 3, 42, 42, 5], lambda x: x == 42) assert [x for x in generator] == [[1, 42], [2, 3, 42], [42], [5]] def test_model_sort_enums(): - nodes = [model.Typedef("B", "A"), - model.Typedef("C", "A"), - model.Enum("A", [])] + nodes = [ + model.Typedef("B", "A"), + model.Typedef("C", "A"), + model.Enum("A", []), + ] model.topological_sort(nodes) @@ -59,121 +288,159 @@ def test_model_sort_enums(): def test_model_sort_typedefs(): - nodes = [model.Typedef("A", "X"), - model.Typedef("C", "B"), - model.Typedef("B", "A"), - model.Typedef("E", "D"), - model.Typedef("D", "C")] + nodes = [ + model.Typedef("A", "X"), + model.Typedef("C", "B"), + model.Typedef("B", "A"), + model.Typedef("E", "D"), + model.Typedef("D", "C"), + ] model.topological_sort(nodes) - assert ["A", "B", "C", "D", "E"] == [node.name for node in nodes] + assert [node.name for node in nodes] == ["A", "B", "C", "D", "E"] + assert [dep for node in nodes for dep in node.dependencies()] == ["X", "A", "B", "C", "D"] def test_model_sort_structs(): - nodes = [model.Struct("C", [model.StructMember("a", "B"), - model.StructMember("b", "A"), - model.StructMember("c", "D")]), - model.Struct("B", [model.StructMember("a", "X"), - model.StructMember("b", "A"), - model.StructMember("c", "Y")]), - model.Struct("A", [model.StructMember("a", "X"), - model.StructMember("b", "Y"), - model.StructMember("c", "Z")])] + nodes = [ + model.Struct("C", [ + model.StructMember("a", "B"), + model.StructMember("b", "A"), + model.StructMember("c", "D"), + ]), + model.Struct("B", [ + model.StructMember("a", "X"), + model.StructMember("b", "A"), + model.StructMember("c", "Y"), + ]), + model.Struct("A", [ + model.StructMember("a", "X"), + model.StructMember("b", "Y"), + model.StructMember("c", "Z"), + ]), + ] model.topological_sort(nodes) - assert ["A", "B", "C"] == [node.name for node in nodes] + assert [node.name for node in nodes] == ["A", "B", "C"] + assert [tuple(node.dependencies()) for node in nodes] == [('X', 'Y', 'Z'), ('X', 'A', 'Y'), ('B', 'A', 'D')] def test_model_sort_struct_with_two_deps(): - nodes = [model.Struct("C", [model.StructMember("a", "B")]), - model.Struct("B", [model.StructMember("a", "A")]), - model.Struct("A", [model.StructMember("a", "X")])] + nodes = [ + model.Struct("C", [model.StructMember("a", "B")]), + model.Struct("B", [model.StructMember("a", "A")]), + model.Struct("A", [model.StructMember("a", "X")]), + ] model.topological_sort(nodes) - assert ["A", "B", "C"] == [node.name for node in nodes] + assert [node.name for node in nodes] == ["A", "B", "C"] def test_model_sort_struct_with_multiple_dependencies(): - nodes = [model.Struct("D", [model.StructMember("a", "A"), - model.StructMember("b", "B"), - model.StructMember("c", "C")]), - model.Struct("C", [model.StructMember("a", "A"), - model.StructMember("b", "B")]), - model.Struct("B", [model.StructMember("a", "A")]), - model.Typedef("A", "TTypeX")] + nodes = [ + model.Struct("D", [ + model.StructMember("a", "A"), + model.StructMember("b", "B"), + model.StructMember("c", "C"), + ]), + model.Struct("C", [ + model.StructMember("a", "A"), + model.StructMember("b", "B"), + ]), + model.Struct("B", [ + model.StructMember("a", "A"), + ]), + model.Typedef("A", "TTypeX"), + ] model.topological_sort(nodes) - assert ["A", "B", "C", "D"] == [node.name for node in nodes] + assert [node.name for node in nodes] == ["A", "B", "C", "D"] def test_model_sort_union(): - nodes = [model.Typedef("C", "B"), - model.Union("B", [model.UnionMember("a", "A", "0"), - model.UnionMember("b", "A", "1")]), - model.Struct("A", [model.StructMember("a", "X")])] + nodes = [ + model.Typedef("C", "B"), + model.Union("B", [ + model.UnionMember("a", "A", "0"), + model.UnionMember("b", "A", "1"), + ]), + model.Struct("A", [ + model.StructMember("a", "X"), + ]), + ] model.topological_sort(nodes) - assert ["A", "B", "C"] == [node.name for node in nodes] + assert [node.name for node in nodes] == ["A", "B", "C"] def test_model_sort_constants(): - nodes = [model.Constant("C_C", "C_A + C_B"), - model.Constant("C_A", "1"), - model.Constant("C_B", "2")] + nodes = [ + model.Constant("C_C", "C_A + C_B"), + model.Constant("C_A", "1"), + model.Constant("C_B", "2"), + ] model.topological_sort(nodes) - assert [("C_A", "1"), ("C_B", "2"), ("C_C", "C_A + C_B")] == nodes + assert nodes == [("C_A", "1"), ("C_B", "2"), ("C_C", "C_A + C_B")] def test_cross_reference_structs(): nodes = [ model.Struct("A", [ - model.StructMember("a", "u8") + model.StructMember("a", "u8"), ]), model.Struct("B", [ model.StructMember("a", "A"), - model.StructMember("b", "u8") + model.StructMember("b", "u8"), ]), model.Struct("C", [ model.StructMember("a", "A"), model.StructMember("b", "B"), - model.StructMember("c", "NON_EXISTENT") + model.StructMember("c", "NON_EXISTENT"), ]), model.Struct("D", [ model.StructMember("a", "A"), model.StructMember("b", "B"), - model.StructMember("c", "C") + model.StructMember("c", "C"), ]) ] - model.cross_reference(nodes) + constants = model.cross_reference(nodes) + assert [n.name for n in nodes] == ['A', 'B', 'C', 'D'] definition_names = [[x.definition.name if x.definition else None for x in y.members] for y in nodes] assert definition_names == [ [None], ['A', None], ['A', 'B', None], - ['A', 'B', 'C'] + ['A', 'B', 'C'], + ] + assert [tuple(n.dependencies()) for n in nodes] == [ + ('u8',), + ('A', 'u8'), + ('A', 'B', 'NON_EXISTENT'), + ('A', 'B', 'C'), ] + assert constants == {} def test_cross_reference_typedef(): nodes = [ model.Struct("A", [ - model.StructMember("a", "u8") + model.StructMember("a", "u8"), ]), model.Typedef("B", "A"), model.Struct("C", [ model.StructMember("a", "A"), - model.StructMember("b", "B") + model.StructMember("b", "B"), ]), - model.Typedef("D", "B") + model.Typedef("D", "B"), ] model.cross_reference(nodes) @@ -192,7 +459,7 @@ def test_cross_symbols_from_includes(): ]), model.Struct('ola', [ model.StructMember('a', 'ala'), - ]) + ]), ]), model.Struct('ula', [ model.StructMember('a', 'ola'), @@ -202,38 +469,47 @@ def test_cross_symbols_from_includes(): model.cross_reference(nodes) - assert nodes[1].members[0].definition.name == 'ola' - assert nodes[1].members[1].definition.name == 'ala' + assert nodes[1].members[0].definition == model.Struct('ola', [model.StructMember('a', 'ala')], '') + assert nodes[1].members[1].definition == model.Typedef('ala', 'u32') # cross-reference only needs to link definitions of first level of nodes - assert nodes[0].nodes[1].members[0].definition is None + assert nodes[0].members[1].members[0] == model.StructMember('a', 'ala', None) + assert nodes[0].members[1].members[0].definition is None def test_cross_reference_array_size_from_includes(): nodes = [ model.Include('x', [ model.Include('y', [ - model.Constant('NUM', '3'), + model.Constant('NUM_HEX', '0xf'), + model.Constant('NUM_DEC', '3'), ]), model.Enum('E', [ - model.EnumMember('E1', '1'), - model.EnumMember('E3', 'NUM') + model.EnumMember('E1', 'NUM_HEX'), + model.EnumMember('E3', 'NUM_DEC'), ]), ]), model.Struct('X', [ - model.StructMember('x', 'u32', size='NUM'), + model.StructMember('x', 'u32', size='NUM_DEC'), model.StructMember('y', 'u32', size='E1'), model.StructMember('z', 'u32', size='UNKNOWN'), - model.StructMember('a', 'u32', size='E3') + model.StructMember('a', 'u32', size='E3'), ]) ] - model.cross_reference(nodes) + constants = model.cross_reference(nodes) assert nodes[1].members[0].numeric_size == 3 - assert nodes[1].members[1].numeric_size == 1 + assert nodes[1].members[1].numeric_size == 15 assert nodes[1].members[2].numeric_size is None assert nodes[1].members[3].numeric_size == 3 + assert constants == { + 'E1': 15, + 'E3': 3, + 'NUM_DEC': 3, + 'NUM_HEX': 15, + } + def test_cross_reference_numeric_size_of_expression(): nodes = [ @@ -242,19 +518,20 @@ def test_cross_reference_numeric_size_of_expression(): model.Constant('C', 'A*B'), model.Struct('X', [ model.StructMember('x', 'u32', size='C'), - ]) + ]), ] - model.cross_reference(nodes) + constants = model.cross_reference(nodes) assert nodes[3].members[0].numeric_size == 180 + assert constants == {'A': 12, 'B': 15, 'C': 180} def test_cross_reference_expression_as_array_size(): nodes = [ model.Struct('X', [ model.StructMember('x', 'u32', size='2 * 3'), - ]) + ]), ] model.cross_reference(nodes) @@ -335,12 +612,12 @@ def test_evaluate_kinds_arrays(): model.StructMember("d", "u8", bound="d_len", size="5"), model.StructMember("e_len", "u8"), model.StructMember("e", "u8", bound="e_len"), - model.StructMember("f", "u8", unlimited=True) - ]) + model.StructMember("f", "u8", greedy=True), + ]), ] model.cross_reference(nodes) - model.evaluate_kinds(nodes) + model.evaluate_stiffness_kinds(nodes) assert [x.kind for x in nodes[0].members] == [ model.Kind.FIXED, @@ -357,22 +634,22 @@ def test_evaluate_kinds_arrays(): def test_evaluate_kinds_struct_records(): nodes = [ model.Struct("Fix", [ - model.StructMember("a", "u8") + model.StructMember("a", "u8"), ]), model.Struct("Dyn", [ model.StructMember("a_len", "u8"), - model.StructMember("a", "u8", bound="a_len") + model.StructMember("a", "u8", bound="a_len"), ]), model.Struct("X", [ model.StructMember("a", "Dyn"), model.StructMember("b_len", "u8"), model.StructMember("b", "Fix", bound="b_len"), - model.StructMember("c", "Fix", unlimited=True) - ]) + model.StructMember("c", "Fix", greedy=True), + ]), ] model.cross_reference(nodes) - model.evaluate_kinds(nodes) + model.evaluate_stiffness_kinds(nodes) assert [x.kind for x in nodes] == [ model.Kind.FIXED, @@ -393,48 +670,48 @@ def test_evaluate_kinds_with_typedefs(): model.Struct("Empty", []), model.Struct("Dynamic", [ model.StructMember("a_len", "u8"), - model.StructMember("a", "u8", bound="a_len") + model.StructMember("a", "u8", bound="a_len"), ]), model.Struct("Fixed", [ - model.StructMember("a", "u8", size="10") + model.StructMember("a", "u8", size="10"), ]), model.Struct("Limited", [ model.StructMember("a_len", "u8"), - model.StructMember("a", "u8", bound="a_len", size="10") + model.StructMember("a", "u8", bound="a_len", size="10"), ]), model.Struct("Greedy", [ - model.StructMember("a", "byte", unlimited=True) + model.StructMember("a", "byte", greedy=True), ]), model.Struct("DynamicWrapper", [ - model.StructMember("a", "Dynamic") + model.StructMember("a", "Dynamic"), ]), model.Struct("GreedyWrapper", [ - model.StructMember("a", "Greedy") + model.StructMember("a", "Greedy"), ]), model.Struct("GreedyDynamic", [ - model.StructMember("a", "Dynamic", unlimited=True) + model.StructMember("a", "Dynamic", greedy=True), ]), model.Typedef("TU8", "u8"), model.Typedef("TDynamic", "Dynamic"), model.Typedef("TGreedy", "Greedy"), model.Struct("TypedefedU8", [ - model.StructMember("a", "TU8") + model.StructMember("a", "TU8"), ]), model.Struct("TypedefedDynamic", [ - model.StructMember("a", "TDynamic") + model.StructMember("a", "TDynamic"), ]), model.Struct("TypedefedGreedy", [ - model.StructMember("a", "TGreedy") + model.StructMember("a", "TGreedy"), ]), model.Typedef("TTDynamic", "TDynamic"), model.Typedef("TTTDynamic", "TTDynamic"), model.Struct("DeeplyTypedefed", [ - model.StructMember("a", "TTTDynamic") + model.StructMember("a", "TTTDynamic"), ]), ] model.cross_reference(nodes) - model.evaluate_kinds(nodes) + model.evaluate_stiffness_kinds(nodes) assert [x.kind for x in nodes if isinstance(x, model.Struct)] == [ model.Kind.FIXED, @@ -457,12 +734,12 @@ def test_partition_fixed(): model.Struct("Fixed", [ model.StructMember("a", "u8"), model.StructMember("b", "u8"), - model.StructMember("c", "u8") + model.StructMember("c", "u8"), ]) ] model.cross_reference(nodes) - model.evaluate_kinds(nodes) + model.evaluate_stiffness_kinds(nodes) main, parts = model.partition(nodes[0].members) assert [x.name for x in main] == ["a", "b", "c"] @@ -477,12 +754,12 @@ def test_partition_many_arrays(): model.StructMember("num_of_b", "u8"), model.StructMember("b", "u8", bound="num_of_b"), model.StructMember("num_of_c", "u8"), - model.StructMember("c", "u8", bound="num_of_c") + model.StructMember("c", "u8", bound="num_of_c"), ]), ] model.cross_reference(nodes) - model.evaluate_kinds(nodes) + model.evaluate_stiffness_kinds(nodes) main, parts = model.partition(nodes[0].members) assert [x.name for x in main] == ["num_of_a", "a"] @@ -495,12 +772,12 @@ def test_partition_many_arrays_mixed(): model.StructMember("num_of_a", "u8"), model.StructMember("num_of_b", "u8"), model.StructMember("a", "u8", bound="num_of_a"), - model.StructMember("b", "u8", bound="num_of_b") + model.StructMember("b", "u8", bound="num_of_b"), ]), ] model.cross_reference(nodes) - model.evaluate_kinds(nodes) + model.evaluate_stiffness_kinds(nodes) main, parts = model.partition(nodes[0].members) assert [x.name for x in main] == ["num_of_a", "num_of_b", "a"] @@ -511,17 +788,17 @@ def test_partition_dynamic_struct(): nodes = [ model.Struct("Dynamic", [ model.StructMember("num_of_a", "u8"), - model.StructMember("a", "u8", bound="num_of_a") + model.StructMember("a", "u8", bound="num_of_a"), ]), model.Struct("X", [ model.StructMember("a", "u8"), model.StructMember("b", "Dynamic"), - model.StructMember("c", "u8") + model.StructMember("c", "u8"), ]) ] model.cross_reference(nodes) - model.evaluate_kinds(nodes) + model.evaluate_stiffness_kinds(nodes) main, parts = model.partition(nodes[1].members) assert [x.name for x in main] == ["a", "b"] @@ -532,17 +809,17 @@ def test_partition_many_dynamic_structs(): nodes = [ model.Struct("Dynamic", [ model.StructMember("num_of_a", "u8"), - model.StructMember("a", "u8", bound="num_of_a") + model.StructMember("a", "u8", bound="num_of_a"), ]), model.Struct("X", [ model.StructMember("a", "Dynamic"), model.StructMember("b", "Dynamic"), - model.StructMember("c", "Dynamic") - ]) + model.StructMember("c", "Dynamic"), + ]), ] model.cross_reference(nodes) - model.evaluate_kinds(nodes) + model.evaluate_stiffness_kinds(nodes) main, parts = model.partition(nodes[1].members) assert [x.name for x in main] == ["a"] @@ -551,7 +828,7 @@ def test_partition_many_dynamic_structs(): def process(nodes, warn=None): model.cross_reference(nodes) - model.evaluate_kinds(nodes) + model.evaluate_stiffness_kinds(nodes) model.evaluate_sizes(nodes, **(warn and {'warn': warn} or {})) return nodes @@ -564,9 +841,9 @@ def process_with_warnings(nodes): def get_size_alignment_padding(node): return ( - isinstance(node, model.StructMember) and - (node.byte_size, node.alignment, node.padding) or - (node.byte_size, node.alignment) + isinstance(node, model.StructMember) and + (node.byte_size, node.alignment, node.padding) or + (node.byte_size, node.alignment) ) @@ -578,13 +855,13 @@ def test_evaluate_sizes_struct(): nodes = process([ model.Struct('X', [ model.StructMember('x', 'u16'), - model.StructMember('y', 'u8') - ]) + model.StructMember('y', 'u8'), + ]), ]) assert list(map(get_size_alignment_padding, get_members_and_node(nodes[0]))) == [ (2, 2, 0), (1, 1, 1), - (4, 2) + (4, 2), ] @@ -596,12 +873,12 @@ def test_evaluate_sizes_nested_struct(): model.Struct('X', [ model.StructMember('x', 'u8'), model.StructMember('y', 'U16'), - ]) + ]), ]) assert list(map(get_size_alignment_padding, get_members_and_node(nodes[1]))) == [ (1, 1, 1), (2, 2, 0), - (4, 2) + (4, 2), ] @@ -609,13 +886,13 @@ def test_evaluate_sizes_fixed_array(): nodes = process([ model.Struct('X', [ model.StructMember('x', 'u32'), - model.StructMember('y', 'u8', size='3') - ]) + model.StructMember('y', 'u8', size='3'), + ]), ]) assert list(map(get_size_alignment_padding, get_members_and_node(nodes[0]))) == [ (4, 4, 0), (3, 1, 1), - (8, 4) + (8, 4), ] @@ -628,28 +905,28 @@ def test_evaluate_sizes_dynamic_array(): model.Struct('Y', [ model.StructMember('x', 'u8'), model.StructMember('y', 'X'), - model.StructMember('z', 'u8') + model.StructMember('z', 'u8'), ]), model.Struct('Z', [ model.StructMember('x', 'X'), - model.StructMember('y', 'u64') - ]) + model.StructMember('y', 'u64'), + ]), ]) assert list(map(get_size_alignment_padding, get_members_and_node(nodes[0]))) == [ (4, 4, 0), (0, 1, -4), - (4, 4) + (4, 4), ] assert list(map(get_size_alignment_padding, get_members_and_node(nodes[1]))) == [ (1, 1, 3), (4, 4, 0), (1, 1, -4), - (12, 4) + (12, 4), ] assert list(map(get_size_alignment_padding, get_members_and_node(nodes[2]))) == [ (4, 4, -8), (8, 8, 0), - (16, 8) + (16, 8), ] @@ -658,12 +935,12 @@ def test_evaluate_sizes_limited_array(): model.Struct('X', [ model.StructMember('num_of_x', 'u32'), model.StructMember('x', 'u8', bound='num_of_x', size='2'), - ]) + ]), ]) assert list(map(get_size_alignment_padding, get_members_and_node(nodes[0]))) == [ (4, 4, 0), (2, 1, 2), - (8, 4) + (8, 4), ] @@ -671,13 +948,13 @@ def test_evaluate_sizes_greedy_array(): nodes = process([ model.Struct('X', [ model.StructMember('num_of_x', 'u32'), - model.StructMember('x', 'u8', unlimited=True), - ]) + model.StructMember('x', 'u8', greedy=True), + ]), ]) assert list(map(get_size_alignment_padding, get_members_and_node(nodes[0]))) == [ (4, 4, 0), (0, 1, -4), - (4, 4) + (4, 4), ] @@ -685,7 +962,7 @@ def test_evaluate_sizes_partial_padding(): nodes = process([ model.Struct('D', [ model.StructMember('num_of_x', 'u32'), - model.StructMember('x', 'u32', bound='num_of_x') + model.StructMember('x', 'u32', bound='num_of_x'), ]), model.Struct('X', [ model.StructMember('num_of_x', 'u32'), @@ -722,14 +999,14 @@ def test_evaluate_sizes_partial_padding(): (0, 1, -8), (1, 8, 7), (8, 8, 0), - (24, 8) + (24, 8), ] assert list(map(get_size_alignment_padding, get_members_and_node(nodes[2]))) == [ (4, 4, 0), (0, 1, -8), (4, 8, 4), (0, 8, 0), - (16, 8) + (16, 8), ] assert list(map(get_size_alignment_padding, get_members_and_node(nodes[3]))) == [ (4, 4, 0), @@ -741,13 +1018,13 @@ def test_evaluate_sizes_partial_padding(): (1, 2, 0), (1, 1, 0), (2, 2, -8), - (40, 8) + (40, 8), ] assert list(map(get_size_alignment_padding, get_members_and_node(nodes[4]))) == [ (4, 4, 0), (0, 2, 0), (2, 2, -4), - (8, 4) + (8, 4), ] @@ -764,28 +1041,28 @@ def test_evaluate_sizes_typedef(): ]) assert list(map(get_size_alignment_padding, get_members_and_node(nodes[1]))) == [ (4, 4, 0), - (4, 4) + (4, 4), ] assert list(map(get_size_alignment_padding, get_members_and_node(nodes[3]))) == [ (4, 4, 0), - (4, 4) + (4, 4), ] def test_evaluate_sizes_enum(): nodes = process([ model.Enum('E', [ - model.EnumMember('E1', '1') + model.EnumMember('E1', '1'), ]), model.Struct('X', [ model.StructMember('x', 'E'), model.StructMember('y', 'i8'), - ]) + ]), ]) assert list(map(get_size_alignment_padding, get_members_and_node(nodes[1]))) == [ (4, 4, 0), (1, 1, 3), - (8, 4) + (8, 4), ] @@ -794,12 +1071,12 @@ def test_evaluate_sizes_floats(): model.Struct('X', [ model.StructMember('x', 'r32'), model.StructMember('y', 'r64'), - ]) + ]), ]) assert list(map(get_size_alignment_padding, get_members_and_node(nodes[0]))) == [ (4, 4, 4), (8, 8, 0), - (16, 8) + (16, 8), ] @@ -809,43 +1086,43 @@ def test_evaluate_sizes_bytes(): model.StructMember('x', 'byte'), model.StructMember('y', 'byte', size=3), model.StructMember('num_of_z', 'u32'), - model.StructMember('z', 'byte', bound='num_of_z') - ]) + model.StructMember('z', 'byte', bound='num_of_z'), + ]), ]) assert list(map(get_size_alignment_padding, get_members_and_node(nodes[0]))) == [ (1, 1, 0), (3, 1, 0), (4, 4, 0), (0, 1, -4), - (8, 4) + (8, 4), ] def test_evaluate_sizes_optional(): nodes = process([ model.Struct('X', [ - model.StructMember('x', 'u32') + model.StructMember('x', 'u32'), ]), model.Struct('O1', [ model.StructMember('x', 'u8', optional=True), model.StructMember('y', 'u16', optional=True), model.StructMember('z', 'u32', optional=True), - model.StructMember('a', 'u64', optional=True) + model.StructMember('a', 'u64', optional=True), ]), model.Struct('O2', [ - model.StructMember('x', 'X', optional=True) - ]) + model.StructMember('x', 'X', optional=True), + ]), ]) assert list(map(get_size_alignment_padding, get_members_and_node(nodes[1]))) == [ (5, 4, 3), (6, 4, 2), (8, 4, 0), (16, 8, 0), - (40, 8) + (40, 8), ] assert list(map(get_size_alignment_padding, get_members_and_node(nodes[2]))) == [ (8, 4, 0), - (8, 4) + (8, 4), ] @@ -854,51 +1131,51 @@ def test_evaluate_sizes_union(): model.Union('X', [ model.UnionMember('x', 'u32', '1'), model.UnionMember('y', 'u32', '2'), - model.UnionMember('z', 'u32', '3') + model.UnionMember('z', 'u32', '3'), ]), model.Union('Y', [ - model.UnionMember('x', 'u64', '1') + model.UnionMember('x', 'u64', '1'), ]), model.Union('Z', [ model.UnionMember('x', 'X', '1'), - model.UnionMember('y', 'Y', '2') - ]) + model.UnionMember('y', 'Y', '2'), + ]), ]) assert list(map(get_size_alignment_padding, get_members_and_node(nodes[0]))) == [ (4, 4), (4, 4), (4, 4), - (8, 4) + (8, 4), ] assert list(map(get_size_alignment_padding, get_members_and_node(nodes[1]))) == [ (8, 8), - (16, 8) + (16, 8), ] assert list(map(get_size_alignment_padding, get_members_and_node(nodes[2]))) == [ (8, 4), (16, 8), - (24, 8) + (24, 8), ] def test_evaluate_sizes_union_with_padding(): nodes = process([ model.Union('X', [ - model.UnionMember('x', 'u8', '1') + model.UnionMember('x', 'u8', '1'), ]), model.Union('Y', [ model.UnionMember('x', 'u8', '1'), - model.UnionMember('y', 'u64', '2') - ]) + model.UnionMember('y', 'u64', '2'), + ]), ]) assert list(map(get_size_alignment_padding, get_members_and_node(nodes[0]))) == [ (1, 1), - (8, 4) + (8, 4), ] assert list(map(get_size_alignment_padding, get_members_and_node(nodes[1]))) == [ (1, 1), (8, 8), - (16, 8) + (16, 8), ] @@ -907,12 +1184,8 @@ def test_evaluate_sizes_empty(): model.Struct('X', []), model.Union('X', []), ]) - assert list(map(get_size_alignment_padding, get_members_and_node(nodes[0]))) == [ - (0, 1) - ] - assert list(map(get_size_alignment_padding, get_members_and_node(nodes[1]))) == [ - (4, 4) - ] + assert list(map(get_size_alignment_padding, get_members_and_node(nodes[0]))) == [(0, 1)] + assert list(map(get_size_alignment_padding, get_members_and_node(nodes[1]))) == [(4, 4)] def test_evaluate_sizes_unknown(): @@ -925,7 +1198,7 @@ def test_evaluate_sizes_unknown(): model.Union('Y', [ model.UnionMember('x', 'u32', '1'), model.UnionMember('y', 'U', '2'), - model.UnionMember('z', 'u32', '3') + model.UnionMember('z', 'u32', '3'), ]), model.Typedef('U16', 'U'), model.Struct('Z', [ @@ -938,25 +1211,25 @@ def test_evaluate_sizes_unknown(): 'X::y has unknown type "U"', 'Y::y has unknown type "U"', 'Z::x has unknown type "U"', - 'Z::y has unknown type "Unknown"' + 'Z::y has unknown type "Unknown"', ] assert list(map(get_size_alignment_padding, get_members_and_node(nodes[0]))) == [ (1, 1, None), (None, None, None), (4, 4, None), - (None, None) + (None, None), ] assert list(map(get_size_alignment_padding, get_members_and_node(nodes[1]))) == [ (4, 4), (None, None), (4, 4), - (None, None) + (None, None), ] assert list(map(get_size_alignment_padding, get_members_and_node(nodes[3]))) == [ (None, None, None), (None, None, None), - (None, None) + (None, None), ] @@ -965,30 +1238,29 @@ def test_evaluate_sizes_array_with_named_size(): model.Constant('NUM', '3'), model.Enum('E', [ model.EnumMember('E1', '1'), - model.EnumMember('E3', 'NUM') + model.EnumMember('E3', 'NUM'), ]), model.Struct('X', [ model.StructMember('x', 'u32', size='NUM'), model.StructMember('y', 'u32', size='E1'), - model.StructMember('z', 'u32', size='E3') + model.StructMember('z', 'u32', size='E3'), ]), model.Struct('Y', [ model.StructMember('x', 'u32', size='UNKNOWN'), - model.StructMember('y', 'u32') - ]) - + model.StructMember('y', 'u32'), + ]), ]) assert list(map(get_size_alignment_padding, get_members_and_node(nodes[2]))) == [ (12, 4, 0), (4, 4, 0), (12, 4, 0), - (28, 4) + (28, 4), ] assert list(map(get_size_alignment_padding, get_members_and_node(nodes[3]))) == [ (None, None, None), (4, 4, None), - (None, None) + (None, None), ] @@ -996,30 +1268,113 @@ def test_evaluate_sizes_with_include(): nodes = process([ model.Include('input', [ model.Enum('E', [ - model.EnumMember('E1', '1') - ]) + model.EnumMember('E1', '1'), + ]), ]), model.Struct('X', [ model.StructMember('x', 'E'), model.StructMember('y', 'i8'), - ]) + ]), ]) assert list(map(get_size_alignment_padding, get_members_and_node(nodes[1]))) == [ (4, 4, 0), (1, 1, 3), - (8, 4) + (8, 4), ] -def test_enum_repr(): - E = model.Enum('TheEnum', [ - model.EnumMember('E1', '1'), +def test_enum_str_and_repr(): + enum = model.Enum('TheEnum', [ + model.EnumMember('E1', 1), model.EnumMember('E2', '2'), - model.EnumMember('E3', '3') + model.EnumMember('E3', '3'), ]) - assert repr(E) == """\ -TheEnum - E1 1 - E2 2 - E3 3 + assert str(enum) == """\ +enum TheEnum { + E1 = 1; + E2 = '2'; + E3 = '3'; +}; """ + assert_repr_reproduces(enum) + + +def test_wrong_struct_members_type_definition(): + expected_msg = "struct 'A' members must be a list, got str instead." + with pytest.raises(model.ModelError, match=expected_msg): + model.Struct("A", "string") + + +def test_wrong_struct_member_type(): + expected_msg = "Each member of struct 'A' has to be a StructMember instance. Got str at index 1." + with pytest.raises(model.ModelError, match=expected_msg): + model.Struct("A", [ + model.StructMember('field_name', 'u32'), + "string", + ]) + expected_msg = "Each member of struct 'A' has to be a StructMember instance. Got UnionMember at index 0." + with pytest.raises(model.ModelError, match=expected_msg): + model.Struct("A", [ + model.UnionMember('field_name', 'u32', 2), + ]) + + +def test_wrong_union_member_type(): + expected_msg = "Each member of union 'U' has to be a UnionMember instance. Got str at index 1." + with pytest.raises(model.ModelError, match=expected_msg): + model.Union("U", [ + model.UnionMember('field_name', 'u32', 0), + "string", + ]) + + expected_msg = "Each member of union 'U' has to be a UnionMember instance. Got StructMember at index 0." + with pytest.raises(model.ModelError, match=expected_msg): + model.Union("U", [ + model.StructMember('field_name', 'u32'), + ]) + + +def test_duplicated_identifiers_struct(): + with pytest.raises(model.ModelError, match="Duplicated 'field_name' identifier in struct A"): + model.Struct("A", [ + model.StructMember('field_name', 'u32'), + model.StructMember('field_name', 'u16'), + ]) + + +def test_duplicated_identifiers_union(): + with pytest.raises(model.ModelError, match="Duplicated 'field_name' identifier in union U"): + model.Union("U", [ + model.UnionMember('field_name', 'u32', 0), + model.UnionMember('field_name', 'u16', 1), + ]) + + +def test_bad_creation(): + with pytest.raises(model.ModelError, match="Got model node name of 'float' type, expected string."): + model.Struct(3.14159, []) + + +def test_bad_creation_typedef(): + msg = "Typedef.definition should be string, Typedef, Enum, Struct or Union, got: float." + with pytest.raises(model.ModelError, match=msg): + model.Typedef("a", "b", 3.14159) + + +def test_not_implemented(): + a = model.ModelNode("a", "b", "c") + + with pytest.raises(NotImplementedError, match="To be overridden in ModelNode class."): + a.dependencies() + + +def test_not_implemented2(): + with pytest.raises(NotImplementedError, match="Abstract method to be overriden in _Serializable"): + model._Serializable.calc_wire_stiffness() + + +def test_bad_attribute(): + a = model.Include("this", []) + expected_message = "Use of value property is forbidden for Include. Use 'Include.members' instead." + with pytest.raises(model.ModelError, match=expected_message): + a.value diff --git a/prophyc/tests/test_patch.py b/prophyc/tests/test_patch.py index bfcc954..ec6bb39 100644 --- a/prophyc/tests/test_patch.py +++ b/prophyc/tests/test_patch.py @@ -237,7 +237,7 @@ def test_make_field_greedy_array(): assert [model.Struct('MyStruct', [model.StructMember('field1', 'u32'), model.StructMember('field2', 'u32'), - model.StructMember('field3', 'u32', unlimited=True)])] == nodes + model.StructMember('field3', 'u32', greedy=True)])] == nodes def test_make_field_greedy_array_not_a_struct(): @@ -333,6 +333,17 @@ def test_make_field_limited_array(): model.StructMember('field3', 'u32', bound='field2', size='20')])] == nodes +def test_fail_to_make_field_limited_array(): + nodes = [model.Struct("MyStruct", [model.StructMember("field1", "u32"), + model.StructMember("field2", "u32"), + model.StructMember("after", "u32")])] + + patches = {'MyStruct': [patch.Action('limited', ['field2', 'after'])]} + + with pytest.raises(Exception, match="Array len member not found: "): + patch.patch(nodes, patches) + + def test_make_field_limited_array_not_a_struct(): nodes = [model.Typedef("MyStruct", "MyRealStruct")] patches = {'MyStruct': [patch.Action('limited', ['field3', 'field2'])]} @@ -404,8 +415,8 @@ def test_change_union_to_struct_and_remove_field(): def test_change_union_to_struct_not_a_union(): - nodes = [model.Struct("MyStruct", [model.StructMember("field1", "u32", 1), - model.StructMember("field2", "u32", 2)])] + nodes = [model.Struct("MyStruct", [model.StructMember("field1", "u32"), + model.StructMember("field2", "u32")])] patches = {'MyStruct': [patch.Action('struct', [])]} with pytest.raises(Exception) as e: diff --git a/prophyc/tests/test_prophyc.py b/prophyc/tests/test_prophyc.py index 2b74468..44427fe 100644 --- a/prophyc/tests/test_prophyc.py +++ b/prophyc/tests/test_prophyc.py @@ -1,9 +1,10 @@ import os -import pytest main_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) empty_python_output = """\ +# This file has been generated by prophyc. + import prophy """ @@ -11,7 +12,7 @@ def test_showing_version(call_prophyc): ret, out, err = call_prophyc(["--version"]) - expected_version = '1.1.2' + expected_version = '1.2.0' assert ret == 0 assert out == 'prophyc %s\n' % expected_version assert err == "" @@ -127,13 +128,14 @@ def test_isar_patch(call_prophyc, tmpdir_cwd): assert tmpdir_cwd.join("input.py").read() == empty_python_output + """\ class A(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ - ('a', prophy.u8) + ('a', prophy.u8), ] + class B(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ ('a', prophy.u8), - ('b', prophy.array(A, bound = 'a')) + ('b', prophy.array(A, bound='a')), ] """ @@ -251,8 +253,7 @@ def test_isar_with_includes(call_prophyc, tmpdir_cwd): """ in tmpdir_cwd.join("input.ppf.hpp").read() -@pytest.clang_installed -def test_sack_compiles_single_empty_hpp(call_prophyc, tmpdir_cwd): +def test_sack_compiles_single_empty_hpp(if_clang_installed, call_prophyc, tmpdir_cwd): tmpdir_cwd.join("input.hpp").write("") ret, out, err = call_prophyc(["--sack", "--python_out", str(tmpdir_cwd), @@ -264,8 +265,7 @@ def test_sack_compiles_single_empty_hpp(call_prophyc, tmpdir_cwd): assert tmpdir_cwd.join("input.py").read() == empty_python_output -@pytest.clang_installed -def test_sack_patch(call_prophyc, tmpdir_cwd): +def test_sack_patch(if_clang_installed, call_prophyc, tmpdir_cwd): tmpdir_cwd.join("input.hpp").write("""\ struct X { @@ -285,13 +285,12 @@ def test_sack_patch(call_prophyc, tmpdir_cwd): assert tmpdir_cwd.join("input.py").read() == empty_python_output + """\ class X(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ - ('x', prophy.r64) + ('x', prophy.r64), ] """ -@pytest.clang_installed -def test_multiple_outputs(call_prophyc, tmpdir_cwd): +def test_multiple_outputs(if_clang_installed, call_prophyc, tmpdir_cwd): tmpdir_cwd.join("input.xml").write(""" @@ -308,11 +307,13 @@ def test_multiple_outputs(call_prophyc, tmpdir_cwd): assert out == "" assert err == "" assert tmpdir_cwd.join("input.py").read() == """\ +# This file has been generated by prophyc. + import prophy class Test(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ - ('x', prophy.u32) + ('x', prophy.u32), ] """ assert tmpdir_cwd.join("input.pp.hpp").read() == """\ @@ -356,8 +357,7 @@ class Test(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): """ -@pytest.clang_not_installed -def test_clang_not_installed(call_prophyc, tmpdir_cwd): +def test_clang_not_installed(if_clang_not_installed, call_prophyc, tmpdir_cwd): tmpdir_cwd.join("input.hpp").write("") ret, out, err = call_prophyc(["--sack", "--python_out", str(tmpdir_cwd), @@ -365,7 +365,7 @@ def test_clang_not_installed(call_prophyc, tmpdir_cwd): assert ret == 1 assert out == "" - assert err == "prophyc: error: %s\n" % pytest.clang_not_installed.args[0].error + assert err == "prophyc: error: %s\n" % if_clang_not_installed.error def test_prophy_language(call_prophyc, tmpdir_cwd): @@ -389,19 +389,22 @@ def test_prophy_language(call_prophyc, tmpdir_cwd): assert out == "" assert err == "" assert tmpdir_cwd.join("input.py").read() == """\ +# This file has been generated by prophyc. + import prophy class X(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ - ('x', prophy.array(prophy.u32, size = 5)), + ('x', prophy.array(prophy.u32, size=5)), ('num_of_y', prophy.u32), - ('y', prophy.array(prophy.u64, bound = 'num_of_y', size = 2)) + ('y', prophy.array(prophy.u64, bound='num_of_y', size=2)), ] + class U(prophy.with_metaclass(prophy.union_generator, prophy.union)): _descriptor = [ ('x', X, 1), - ('y', prophy.u32, 2) + ('y', prophy.u32, 2), ] """ assert tmpdir_cwd.join("input.pp.hpp").read() == """\ @@ -498,8 +501,7 @@ def test_prophy_parse_errors(call_prophyc, tmpdir_cwd): assert not os.path.exists("input.py") -@pytest.clang_installed -def test_sack_parse_warnings(call_prophyc, tmpdir_cwd): +def test_sack_parse_warnings(if_clang_installed, call_prophyc, tmpdir_cwd): tmpdir_cwd.join("input.cpp").write("""\ int foo() { int x; } rubbish; @@ -516,8 +518,7 @@ def test_sack_parse_warnings(call_prophyc, tmpdir_cwd): assert os.path.exists("input.py") -@pytest.clang_installed -def test_sack_parse_errors(call_prophyc, tmpdir_cwd): +def test_sack_parse_errors(if_clang_installed, call_prophyc, tmpdir_cwd): tmpdir_cwd.join("input.unknown").write("") ret, out, err = call_prophyc(['--python_out', str(tmpdir_cwd), '--sack', @@ -568,6 +569,7 @@ def test_cpp_full_out(call_prophyc, tmpdir_cwd): enum { MAX = 4u }; + struct X : public prophy::detail::message { enum { encoded_byte_size = 16 }; diff --git a/prophyc/tests/test_prophyc_with_prophy.py b/prophyc/tests/test_prophyc_with_prophy.py index 29cdc93..a78e61b 100644 --- a/prophyc/tests/test_prophyc_with_prophy.py +++ b/prophyc/tests/test_prophyc_with_prophy.py @@ -1,5 +1,4 @@ import os -import pytest opd = os.path.dirname opr = os.path.realpath @@ -51,8 +50,7 @@ def test_isar_input(tmpdir_cwd, call_prophyc): assert b"\x00\x00\x00\x01\x12\x31\x00\x00" == s.encode(">") -@pytest.clang_installed -def test_sack_input(tmpdir_cwd, call_prophyc): +def test_sack_input(if_clang_installed, tmpdir_cwd, call_prophyc): content = """\ #include struct X diff --git a/prophyc/tests/test_supplementation.py b/prophyc/tests/test_supplementation.py index 235e3fc..26de33d 100644 --- a/prophyc/tests/test_supplementation.py +++ b/prophyc/tests/test_supplementation.py @@ -3,7 +3,6 @@ import os import pytest - IsarTestItem = namedtuple("IsarTestItem", "file_name_base, input_xml, expected_py") ISAR_TEST_SET_1 = [ @@ -31,6 +30,8 @@ """, expected_py="""\ +# This file has been generated by prophyc. + import prophy IsarCONST_A = 4 @@ -39,28 +40,30 @@ IsarDefA = prophy.r32 IsarDefB = prophy.r64 + class EIsarDefEnum(prophy.with_metaclass(prophy.enum_generator, prophy.enum)): _enumerators = [ ('EIsarDefEnum_A', 0), ('EIsarDefEnum_B', 1), ('EIsarDefEnum_C', 2), - ('EIsarDefEnum_D', 4) + ('EIsarDefEnum_D', 4), ] + EIsarDefEnum_A = 0 EIsarDefEnum_B = 1 EIsarDefEnum_C = 2 EIsarDefEnum_D = 4 + class IsarDefC(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ ('ifC_a', prophy.u16), ('ifC_b_len', prophy.u32), - ('ifC_b', prophy.array(prophy.u64, bound = 'ifC_b_len')) + ('ifC_b', prophy.array(prophy.u64, bound='ifC_b_len')), ] """), - IsarTestItem( file_name_base="included_by_sack_a", input_xml="""\ @@ -80,20 +83,25 @@ class IsarDefC(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): """, expected_py="""\ +# This file has been generated by prophyc. + import prophy -from isar_root_defs import * +from isar_root_defs import ( + EIsarDefEnum, IsarCONST_A, IsarCONST_B, IsarDefA, IsarDefB, IsarDefC +) class IsarK(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ ('ifK_a', prophy.u8), - ('ifK_B', prophy.array(IsarDefB, size = IsarCONST_A)) + ('ifK_B', prophy.array(IsarDefB, size=IsarCONST_A)), ] + class IsarL(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ ('numOfItems', prophy.u32), - ('theBItems', prophy.array(IsarDefB, bound = 'numOfItems')) + ('theBItems', prophy.array(IsarDefB, bound='numOfItems')), ] """), @@ -112,16 +120,20 @@ class IsarL(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): """, expected_py="""\ +# This file has been generated by prophyc. + import prophy -from isar_root_defs import * +from isar_root_defs import ( + EIsarDefEnum, IsarCONST_A, IsarCONST_B, IsarDefA, IsarDefB, IsarDefC +) class IsarV(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ ('ifV_a', IsarDefA), ('ifV_b_len', prophy.u32), - ('ifV_b', prophy.array(IsarDefB, bound = 'ifV_b_len', size = IsarCONST_B)), - ('ifV_c', EIsarDefEnum) + ('ifV_b', prophy.array(IsarDefB, bound='ifV_b_len', size=IsarCONST_B)), + ('ifV_c', EIsarDefEnum), ] """) ] @@ -144,8 +156,7 @@ def check_isars_generated(isar_test_set): return check_isars_generated -@pytest.clang_installed -def test_sack_supples(isar_test_helper, tmpdir_cwd, call_prophyc): +def test_sack_supplements(if_clang_installed, isar_test_helper, tmpdir_cwd, call_prophyc): cpp = tmpdir_cwd.join('the_sack.hpp') cppy = tmpdir_cwd.join('the_sack.py') cpp.write("""\ @@ -167,23 +178,27 @@ def test_sack_supples(isar_test_helper, tmpdir_cwd, call_prophyc): assert err == "" assert ret == 0 assert cppy.read() == """\ +# This file has been generated by prophyc. + import prophy -from included_by_sack_a import * -from included_by_sack_b import * +from included_by_sack_a import ( + EIsarDefEnum, IsarCONST_A, IsarCONST_B, IsarDefA, IsarDefB, IsarDefC, IsarK, + IsarL +) +from included_by_sack_b import IsarV class cppX(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ ('defined_in_xml', IsarK), ('defined_deeper_in_xmls', IsarDefC), ('regular_type', prophy.u16), - ('typedefed_deeper_in_xmls', IsarDefA) + ('typedefed_deeper_in_xmls', IsarDefA), ] """ -@pytest.clang_installed -def test_sack_supples_array_size_from_isar(isar_test_helper, tmpdir_cwd, call_prophyc): +def test_sack_supples_array_size_from_isar(if_clang_installed, isar_test_helper, tmpdir_cwd, call_prophyc): cpp = tmpdir_cwd.join('the_sack.hpp') cppy = tmpdir_cwd.join('the_sack.py') cpp.write("""\ @@ -203,21 +218,25 @@ def test_sack_supples_array_size_from_isar(isar_test_helper, tmpdir_cwd, call_pr assert err == "" assert ret == 0 assert cppy.read() == """\ +# This file has been generated by prophyc. + import prophy -from included_by_sack_a import * -from included_by_sack_b import * +from included_by_sack_a import ( + EIsarDefEnum, IsarCONST_A, IsarCONST_B, IsarDefA, IsarDefB, IsarDefC, IsarK, + IsarL +) +from included_by_sack_b import IsarV class cppX(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ - ('type_from_isar', prophy.array(IsarV, size = 4)), - ('regular_type', prophy.array(prophy.u16, size = 16)) + ('type_from_isar', prophy.array(IsarV, size=4)), + ('regular_type', prophy.array(prophy.u16, size=16)), ] """ -@pytest.clang_installed -def test_sack_supples_static_consts_values_from_isar(isar_test_helper, tmpdir_cwd, call_prophyc): +def test_sack_supples_static_consts_values_from_isar(if_clang_installed, isar_test_helper, tmpdir_cwd, call_prophyc): cpp = tmpdir_cwd.join('the_sack.hpp') cppy = tmpdir_cwd.join('the_sack.py') cpp.write("""\ @@ -247,26 +266,31 @@ def test_sack_supples_static_consts_values_from_isar(isar_test_helper, tmpdir_cw assert ret == 0 assert cppy.read() == """\ +# This file has been generated by prophyc. + import prophy -from included_by_sack_a import * -from included_by_sack_b import * +from included_by_sack_a import ( + EIsarDefEnum, IsarCONST_A, IsarCONST_B, IsarDefA, IsarDefB, IsarDefC, IsarK, + IsarL +) +from included_by_sack_b import IsarV class constants(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [] + class cppX(prophy.with_metaclass(prophy.struct_generator, prophy.struct)): _descriptor = [ - ('straigth', prophy.array(prophy.u8, size = 2)), - ('tricky1', prophy.array(prophy.u16, size = 4)), - ('tricky2', prophy.array(prophy.u16, size = 20)), - ('tricky3', prophy.array(prophy.u16, size = 64)) + ('straigth', prophy.array(prophy.u8, size=2)), + ('tricky1', prophy.array(prophy.u16, size=4)), + ('tricky2', prophy.array(prophy.u16, size=20)), + ('tricky3', prophy.array(prophy.u16, size=64)), ] """ -@pytest.clang_installed -def test_warns_with_supplementation(isar_test_helper, tmpdir_cwd, call_prophyc): +def test_warns_with_supplementation(if_clang_installed, isar_test_helper, tmpdir_cwd, call_prophyc): cpp = tmpdir_cwd.join('the_sack.hpp') cpp.write("""\ #include diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..87a3da1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +ply +renew>=0.3.0 diff --git a/setup.py b/setup.py index aba046c..0f992ab 100644 --- a/setup.py +++ b/setup.py @@ -1,25 +1,23 @@ from setuptools import setup, find_packages -# flake8: noqa - long_description = open('README.rst').read() setup( - name = 'prophy', - version = '1.1.2', - author = 'Krzysztof Laskowski', - author_email = 'aurzenligl@gmail.com', - maintainer = 'Krzysztof Laskowski', - maintainer_email = 'aurzenligl@gmail.com', - license = 'MIT license', - url = 'https://github.com/aurzenligl/prophy', - description = 'prophy: fast serialization protocol', - long_description = long_description, - packages = find_packages(), - requires = ['ply'], - install_requires = ['ply'], - keywords = 'idl codec binary data protocol compiler', - classifiers = [ + name='prophy', + version='1.2.0', + author='Krzysztof Laskowski', + author_email='aurzenligl@gmail.com', + maintainer='Krzysztof Laskowski', + maintainer_email='aurzenligl@gmail.com', + license='MIT license', + url='https://github.com/aurzenligl/prophy', + description='prophy: fast serialization protocol', + long_description=long_description, + long_description_content_type='text/x-rst', + packages=find_packages(), + install_requires=['ply', 'renew>=0.3.0'], + keywords='idl codec binary data protocol compiler', + classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Intended Audience :: Telecommunications Industry', @@ -43,7 +41,7 @@ 'Operating System :: OS Independent', 'License :: OSI Approved :: MIT License', ], - entry_points = { + entry_points={ 'console_scripts': [ 'prophyc = prophyc.__main__:entry_main' ], diff --git a/tox.ini b/tox.ini index a977d00..80564a7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,20 @@ +[flake8] +max-line-length = 120 + [tox] envlist = check, py27,py34,py35,py36,pypy,noclang, coverage +skip_missing_interpreters = true [testenv] usedevelop = True deps = - pytest<4.0 - pytest-mock + -r{toxinidir}/requirements.txt pdbpp + pytest + pytest-mock rpdb setenv = noclang: PROPHY_NOCLANG=1 @@ -18,13 +23,17 @@ commands = [testenv:check] deps = + -r{toxinidir}/requirements.txt + check-manifest docutils flake8 - readme-renderer - check-manifest + readme-renderer[md] + twine commands = check-manifest - python setup.py check --strict --metadata --restructuredtext + python setup.py check --strict --metadata + python setup.py sdist + twine check dist/* flake8 prophy prophyc setup.py [testenv:coverage] @@ -34,17 +43,23 @@ changedir = . whitelist_externals = sh deps = - ply - pytest<4.0 - pytest-mock + -r{toxinidir}/requirements.txt coverage coveralls + pytest + pytest-mock setenv = COVERAGE_PROCESS_START={toxinidir}/.coveragerc commands = sh -c 'echo "import coverage; coverage.process_startup()" > {envsitepackagesdir}/../sitecustomize.py' - coverage run -m pytest prophy prophyc + coverage run -m pytest {posargs} prophy prophyc coverage combine . prophyc coverage report -m coverage html sh -c 'if [ $COVERALLS_REPO_TOKEN ]; then coveralls; fi' + +[testenv:dev_doc] +deps = + epydoc +commands = + epydoc -v --graph=all --out dev_doc prophy prophyc