Skip to content

Commit

Permalink
Added schemas to all yamcan dictionaries
Browse files Browse the repository at this point in the history
  • Loading branch information
@JoshLafleur
JoshLafleur committed Feb 5, 2025
1 parent 25272e6 commit a309921
Show file tree
Hide file tree
Showing 21 changed files with 264 additions and 57 deletions.
230 changes: 203 additions & 27 deletions network/NetworkGen/NetworkGen.py
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
from argparse import ArgumentParser, Namespace
from os import makedirs
from os import makedirs, path
from pathlib import Path
from typing import Dict, Iterator, Tuple
import pickle
import copy
from schema import Schema, Or, Optional, And

from mako import template
from mako.lookup import TemplateLookup
Expand All @@ -29,35 +30,84 @@
discrete_values = {}
templates = {}

BUS_SCHEMA = Schema({
"name": str,
"description": str,
"defaultEndianness": str,
"baudrate": Or(1000000, 500000),
})

NODE_SCHEMA = Schema({
"description": str,
"onBuses": Or(str, list[str]),
Optional("duplicateNode"): int,
})

SIGNAL_SCHEMA = Schema({
"description": str,
Optional("unit"): str,
Optional("nativeRepresentation"): dict,
Optional("discreteValues"): str,
Optional("continuous"): bool,
Optional("validationRole"): str,
})

MESSAGE_SCHEMA = Schema({
Optional("description"): str,
Optional("cycleTimeMs"): int,
"id": int,
Optional("lengthBytes"): int,
Optional("sourceBuses"): Or(str, list[str]),
Optional("signals"): dict,
Optional("unscheduled"): bool,
Optional("template"): str,
})

RX_FILE_SCHEMA = Schema({
"messages": Or(dict, None),
"signals": Or(dict, None),
})

RX_ITEM_SCHEMA = Schema({
Optional("sourceBuses"): Or(str, list[str]),
Optional("unrecorded"): bool,
Optional("node"): Or(int, list[int])
})

# if this gets set during the build, we will fail at the end
ERROR = False


def generate_discrete_values(data_dir: Path) -> None:
def generate_discrete_values(definition_dir: Path) -> None:
"""Load discrete values from discrete-values.yaml"""
global ERROR
global discrete_values
discrete_values = DiscreteValues()

with open(data_dir.joinpath("discrete_values.yaml"), "r") as fd:
with open(definition_dir.joinpath("discrete_values.yaml"), "r") as fd:
dvs: Dict[str, Dict[str, int]] = safe_load(fd)

for name, val in dvs.items():
try:
discrete_value = DiscreteValue(name, val)
for vals, _ in val.items():
if any(c for c in vals if c.islower()):
raise Exception(f"Discrete Value '{name}' cannot contain lower case in enum '{vals}'")
if any(c for c in vals if not c.isalnum() and c != '_'):
raise Exception(f"Discrete Value '{name}' may only contain capital letters, numbers, or underscores in enum '{vals}'")
setattr(discrete_values, name, discrete_value)
except ValueError as e:
print(e)
ERROR = True

def generate_templates(data_dir: Path) -> None:
def generate_templates(definition_dir: Path) -> None:
"""Load discrete values from signals.yaml"""
global ERROR
global templates

with open(data_dir.joinpath("signals.yaml"), "r") as fd:
with open(definition_dir.joinpath("signals.yaml"), "r") as fd:
templates["signals"] = safe_load(fd)["signals"]
with open(data_dir.joinpath("messages.yaml"), "r") as fd:
with open(definition_dir.joinpath("messages.yaml"), "r") as fd:
messages = safe_load(fd)["messages"]
templates["messages"] = {}
for message, definition in messages.items():
Expand All @@ -76,27 +126,46 @@ def generate_templates(data_dir: Path) -> None:
else:
templates["messages"][message] = definition

def generate_can_buses(data_dir: Path) -> None:
def generate_can_buses(definition_dir: Path) -> None:
"""Generate CAN buses based on yaml files"""
can_buses = data_dir.joinpath("buses").glob("*.yaml")
global ERROR
error = False
can_buses = definition_dir.joinpath("buses").glob("*.yaml")
for bus_file_path in can_buses:
with open(bus_file_path, "r") as bus_file:
can_bus_def = safe_load(bus_file)
try:
BUS_SCHEMA.validate(can_bus_def)
try:
Endianess[can_bus_def["defaultEndianness"]]
except:
raise Exception(f"Endianness '{can_bus_def["defaultEndianness"]}' is not valid. Endianness can be { [ e.name for e in Endianess ] }.")
except Exception as e:
print(f"CANbus configuration file '{bus_file_path}' is invalid.")
print(f"CAN Bus Schema Error: {e}")
error = True
continue

try:
can_bus_defs[can_bus_def["name"]] = CanBus(can_bus_def)
except Exception as e:
# FIXME: except the specific exception we're expecting here
raise e

if error:
ERROR = True
raise Exception("Error generating CAN bus', review previous errors...")


def generate_can_nodes(data_dir: Path) -> None:
def generate_can_nodes(definition_dir: Path) -> None:
"""
Generate CAN nodes based on yaml files
"""
global ERROR
error = False

nodes = [
dir for dir in data_dir.joinpath("data/components").iterdir() if dir.is_dir()
dir for dir in definition_dir.joinpath("data/components").iterdir() if dir.is_dir()
]

for node in nodes:
Expand Down Expand Up @@ -141,6 +210,21 @@ def generate_can_nodes(data_dir: Path) -> None:
node_def = safe_load(fd)
node_dict.update(node_def)

try:
NODE_SCHEMA.validate(node_def)
if type(node_def["onBuses"]) is list:
for bus in node_def["onBuses"]:
if bus not in can_bus_defs:
raise Exception(f"Node '{node.name}' is on bus '{bus}' but that bus is not defined.")
else:
if node_def["onBuses"] not in can_bus_defs:
raise Exception(f"Node '{node.name}' is on bus '{node_def["onBuses"]}' but that bus is not defined.")
except Exception as e:
print(f"CAN node configuration file for node '{node.name}' is invalid.")
print(f"CAN Node Error: {e}")
error = True
continue

if "duplicateNode" in node_dict:
for i in range(0, node_def["duplicateNode"]):
# create one object and add it to all buses so that each bus will have the same object
Expand All @@ -155,12 +239,16 @@ def generate_can_nodes(data_dir: Path) -> None:
can_node.on_buses = node_def["onBuses"]
can_nodes[node.name] = can_node

if error:
ERROR = True
raise Exception("Error generating CAN nodes, review previous errors...")


def process_node(node: CanNode):
"""Process the signals and messages associated with a given CAN node"""
global ERROR
global templates
error = False

sig_file = SIG_FILE.format(name=node.name)
msg_file = MESSAGE_FILE.format(name=node.name)
Expand All @@ -171,6 +259,31 @@ def process_node(node: CanNode):
with open(node.def_files[sig_file], "r") as signals_file:
signals_dict = safe_load(signals_file)["signals"]

if signals_dict is not None:
for sig, definition in signals_dict.items():
try:
SIGNAL_SCHEMA.validate(definition)
if "nativeRepresentation" not in definition and "discreteValues" not in definition:
raise Exception("Signal '{sig}' in '{node.name}' has neither a discreteValues or nativeRepresentation.")
if "unit" in definition:
try:
Units(definition["unit"])
except:
raise Exception(f"Unit '{definition["unit"]}' is not an accepted unit.")
if "validationRole" in definition:
try:
ValidationRole(definition["validationRole"])
except:
raise Exception(f"Validation role '{definition["validationRole"]}' is not valid.")
except Exception as e:
print(f"CAN signal definition for '{sig}' in node '{node.name}' is invalid.")
print(f"CAN Signal Schema Error: {e}")
error = True
continue
if error:
ERROR = True
raise Exception("Error processing node signals, see previous errors...")

signals = {}

if not signals_dict:
Expand All @@ -188,6 +301,20 @@ def process_node(node: CanNode):
with open(node.def_files[msg_file], "r") as messages_file:
messages_dict = safe_load(messages_file).get("messages", None)

for msg, definition in messages_dict.items():
try:
MESSAGE_SCHEMA.validate(definition)
if "lengthBytes" in definition and (definition["lengthBytes"] < 1 or definition["lengthBytes"] > 8):
raise Exception("Message length must be greater than 0 and less than or equal to 8")
except Exception as e:
print(f"CAN message definition for '{msg}' in node '{node.name}' is invalid.")
print(f"Message Schema Error: {e}")
error = True
continue
if error:
ERROR = True
raise Exception("Error processing node messages, see previous errors...")

if not messages_dict:
print(f"No messages found in message file for node '{node.name}'")
ERROR = True
Expand All @@ -208,16 +335,28 @@ def process_node(node: CanNode):
definition.update(templates["messages"][definition["template"]])

definition["id"] = definition["id"] + node.offset;
if "sourceBuses" in definition:
ls = []
if type(definition["sourceBuses"]) is str:
ls.append(definition["sourceBuses"])
else:
ls = definition["sourceBuses"]
for bus in ls:
if bus not in can_bus_defs:
print(f"Message '{msg_name}' has an undefined bus '{bus}'.")
ERROR = True
break
if bus not in node.on_buses:
print(f"Message '{msg_name}' is on bus '{bus}' but node '{node.name}' is not on that bus.")
ERROR = True
break

for existing_node in can_nodes:
for msg in can_nodes[existing_node].messages:
if definition["id"] == can_nodes[existing_node].messages[msg].id:
#if "sourceBuses" in definition:
#for bus in definition["sourceBuses"]:
#if bus in can_nodes[existing_node].messages[msg].source_buses:
print(f"Message {msg_name} has the same ID as {msg}")
ERROR = True
break
print(f"Message {msg_name} has the same ID as {msg}")
ERROR = True
break
msg_obj = CanMessage(node, msg_name, definition)

if msg_obj.signals is None:
Expand Down Expand Up @@ -297,21 +436,61 @@ def process_node(node: CanNode):
def process_receivers(bus: CanBus, node: CanNode):
"""Process signals received by a node"""
global ERROR
error = False

rx_file = RX_FILE.format(name=node.name)
if rx_file not in node.def_files:
print(f"Note: rx file not found for node '{node.name}'")
return
with open(node.def_files[rx_file], "r") as fd:
rx_sig_file = safe_load(fd)
try:
RX_FILE_SCHEMA.validate(rx_sig_file)
except Exception as e:
print(f"CAN rx definition for node '{node.name}' is invalid.")
print(f"File Schema Error: {e}")
raise e

with open(node.def_files[rx_file], "r") as fd:
# making this a dictionary because eventually we'll
# probably want to be able to gateway messages from
# one bus to another, and this would be the place to
# define the gateway node
rx_sig_dict = safe_load(fd)["signals"] or {}
rx_sig_dict = rx_sig_file["signals"] if rx_sig_file["signals"] is not None else {}

for sig, definition in rx_sig_dict.items():
if definition is not None:
try:
RX_ITEM_SCHEMA.validate(definition)
if "sourceBuses" in definition and definition["sourceBuses"] not in can_bus_defs:
raise Exception(f"Source bus {definition["sourceBuses"]} is not defined in the network.")
except Exception as e:
print(f"CAN signal reception definition for '{sig}' in node '{node.name}' is invalid.")
print(f"CAN RX Signal Error: {e}")
error = True
continue
if error:
ERROR = True
raise Exception("Error processing node receivers, see previous errors...")


with open(node.def_files[rx_file], "r") as fd:
rx_msg_dict = safe_load(fd)["messages"] or {}
rx_msg_dict = rx_sig_file["messages"] if rx_sig_file["messages"] is not None else {}

for msg, definition in rx_msg_dict.items():
if definition is not None:
try:
RX_ITEM_SCHEMA.validate(definition)
if "sourceBuses" in definition and definition["sourceBuses"] not in can_bus_defs:
raise Exception(f"Source bus {definition["sourceBuses"]} is not defined in the network.")
except Exception as e:
print(f"CAN message reception definition for '{msg}' in node '{node.name}' is invalid.")
print(f"CAN RX Message Error: {e}")
error = True
continue
if error:
ERROR = True
raise Exception("Error processing node receivers, see previous errors...")

for sig_name in rx_sig_dict.keys():
if sig_name not in bus.signals:
Expand Down Expand Up @@ -422,7 +601,7 @@ def parse_args() -> Namespace:

parser.add_argument(
"--data-dir",
dest="data_dir",
dest="definition_dir",
type=Path,
help="Path to the network definition directory",
)
Expand Down Expand Up @@ -483,10 +662,10 @@ def parse_args() -> Namespace:


def parseNetwork(args, lookup):
generate_discrete_values(args.data_dir)
generate_templates(args.data_dir.joinpath("data/templates"))
generate_can_buses(args.data_dir)
generate_can_nodes(args.data_dir)
generate_discrete_values(args.definition_dir)
generate_templates(args.definition_dir.joinpath("data/templates"))
generate_can_buses(args.definition_dir)
generate_can_nodes(args.definition_dir)

for node in can_nodes.values():
process_node(node)
Expand All @@ -496,8 +675,6 @@ def parseNetwork(args, lookup):
if not node.processed:
process_receivers(bus, node)

generate_dbcs(lookup, bus, args.output_dir)


def main():
"""Main function"""
Expand All @@ -509,12 +686,11 @@ def main():

# parse arguments
args = parse_args()
lookup = TemplateLookup(directories=[args.data_dir.joinpath("templates")])
lookup = TemplateLookup(directories=[ path.dirname(__file__) + "/templates" ])

if args.build:
print(f"Parsing network...")
parseNetwork(args, lookup)

for bus in can_bus_defs.values():
generate_dbcs(lookup, bus, args.output_dir)
if args.cache_dir:
Expand Down
Loading

0 comments on commit a309921

Please sign in to comment.