From a4e7e35a308535f72176e289d841716629e92bf9 Mon Sep 17 00:00:00 2001 From: Yaman Umuroglu Date: Tue, 21 May 2024 14:46:50 +0200 Subject: [PATCH] [Test] fix changes return style for inference cost --- src/qonnx/util/inference_cost.py | 2 +- tests/analysis/test_inference_cost.py | 152 +++++++++++--------- tests/analysis/test_matmul_mac_cost.py | 2 +- tests/transformation/test_pruning.py | 4 +- tests/transformation/test_quantize_graph.py | 4 +- 5 files changed, 88 insertions(+), 76 deletions(-) diff --git a/src/qonnx/util/inference_cost.py b/src/qonnx/util/inference_cost.py index 12f1b56d..8041ecdc 100644 --- a/src/qonnx/util/inference_cost.py +++ b/src/qonnx/util/inference_cost.py @@ -133,7 +133,7 @@ def inference_cost( if "unsupported" in res: res["unsupported"] = str(res["unsupported"]) combined_results[i] = res - else: + elif i in ["optype_cost", "node_cost"]: per_optype_or_node_breakdown = {} for optype, op_res in res.items(): bops, macs = compute_bops_and_macs(op_res) diff --git a/tests/analysis/test_inference_cost.py b/tests/analysis/test_inference_cost.py index a94f57f4..572d2e14 100644 --- a/tests/analysis/test_inference_cost.py +++ b/tests/analysis/test_inference_cost.py @@ -34,90 +34,102 @@ model_details_infcost = { "FINN-CNV_W2A2": { "expected_sparse": { - "op_mac_SCALEDINT<8>_INT2": 1345500.0, - "mem_w_INT2": 908033.0, - "mem_o_SCALEDINT<32>": 57600.0, - "op_mac_INT2_INT2": 35615771.0, - "mem_o_INT32": 85002.0, - "unsupported": "set()", - "discount_sparsity": True, - "total_bops": 163991084.0, - "total_macs": 36961271.0, - "total_mem_w_bits": 1816066.0, - "total_mem_w_elems": 908033.0, - "total_mem_o_bits": 4563264.0, - "total_mem_o_elems": 142602.0, + "total_cost": { + "op_mac_SCALEDINT<8>_INT2": 1345500.0, + "mem_w_INT2": 908033.0, + "mem_o_SCALEDINT<32>": 57600.0, + "op_mac_INT2_INT2": 35615771.0, + "mem_o_INT32": 85002.0, + "unsupported": "set()", + "discount_sparsity": True, + "total_bops": 163991084.0, + "total_macs": 36961271.0, + "total_mem_w_bits": 1816066.0, + "total_mem_w_elems": 908033.0, + "total_mem_o_bits": 4563264.0, + "total_mem_o_elems": 142602.0, + } }, "expected_dense": { - "op_mac_SCALEDINT<8>_INT2": 1555200.0, - "mem_w_INT2": 1542848.0, - "mem_o_SCALEDINT<32>": 57600.0, - "op_mac_INT2_INT2": 57906176.0, - "mem_o_INT32": 85002.0, - "unsupported": "set()", - "discount_sparsity": False, - "total_bops": 256507904.0, - "total_macs": 59461376.0, - "total_mem_w_bits": 3085696.0, - "total_mem_w_elems": 1542848.0, - "total_mem_o_bits": 4563264.0, - "total_mem_o_elems": 142602.0, + "total_cost": { + "op_mac_SCALEDINT<8>_INT2": 1555200.0, + "mem_w_INT2": 1542848.0, + "mem_o_SCALEDINT<32>": 57600.0, + "op_mac_INT2_INT2": 57906176.0, + "mem_o_INT32": 85002.0, + "unsupported": "set()", + "discount_sparsity": False, + "total_bops": 256507904.0, + "total_macs": 59461376.0, + "total_mem_w_bits": 3085696.0, + "total_mem_w_elems": 1542848.0, + "total_mem_o_bits": 4563264.0, + "total_mem_o_elems": 142602.0, + } }, }, "FINN-TFC_W2A2": { "expected_sparse": { - "op_mac_INT2_INT2": 22355.0, - "mem_w_INT2": 22355.0, - "mem_o_INT32": 202.0, - "unsupported": "set()", - "discount_sparsity": True, - "total_bops": 89420.0, - "total_macs": 22355.0, - "total_mem_w_bits": 44710.0, - "total_mem_w_elems": 22355.0, - "total_mem_o_bits": 6464.0, - "total_mem_o_elems": 202.0, + "total_cost": { + "op_mac_INT2_INT2": 22355.0, + "mem_w_INT2": 22355.0, + "mem_o_INT32": 202.0, + "unsupported": "set()", + "discount_sparsity": True, + "total_bops": 89420.0, + "total_macs": 22355.0, + "total_mem_w_bits": 44710.0, + "total_mem_w_elems": 22355.0, + "total_mem_o_bits": 6464.0, + "total_mem_o_elems": 202.0, + } }, "expected_dense": { - "op_mac_INT2_INT2": 59008.0, - "mem_w_INT2": 59008.0, - "mem_o_INT32": 202.0, - "unsupported": "set()", - "discount_sparsity": False, - "total_bops": 236032.0, - "total_macs": 59008.0, - "total_mem_w_bits": 118016.0, - "total_mem_w_elems": 59008.0, - "total_mem_o_bits": 6464.0, - "total_mem_o_elems": 202.0, + "total_cost": { + "op_mac_INT2_INT2": 59008.0, + "mem_w_INT2": 59008.0, + "mem_o_INT32": 202.0, + "unsupported": "set()", + "discount_sparsity": False, + "total_bops": 236032.0, + "total_macs": 59008.0, + "total_mem_w_bits": 118016.0, + "total_mem_w_elems": 59008.0, + "total_mem_o_bits": 6464.0, + "total_mem_o_elems": 202.0, + } }, }, "RadioML_VGG10": { "expected_sparse": { - "op_mac_SCALEDINT<8>_SCALEDINT<8>": 12620311.0, - "mem_w_SCALEDINT<8>": 155617.0, - "mem_o_SCALEDINT<32>": 130328.0, - "unsupported": "set()", - "discount_sparsity": True, - "total_bops": 807699904.0, - "total_macs": 12620311.0, - "total_mem_w_bits": 1244936.0, - "total_mem_w_elems": 155617.0, - "total_mem_o_bits": 4170496.0, - "total_mem_o_elems": 130328.0, + "total_cost": { + "unsupported": "set()", + "discount_sparsity": True, + "op_mac_SCALEDINT<8>_SCALEDINT<8>": 12620311.0, + "mem_w_SCALEDINT<8>": 155617.0, + "mem_o_SCALEDINT<32>": 130328.0, + "total_bops": 807699904.0, + "total_macs": 12620311.0, + "total_mem_w_bits": 1244936.0, + "total_mem_w_elems": 155617.0, + "total_mem_o_bits": 4170496.0, + "total_mem_o_elems": 130328.0, + } }, "expected_dense": { - "op_mac_SCALEDINT<8>_SCALEDINT<8>": 12864512.0, - "mem_w_SCALEDINT<8>": 159104.0, - "mem_o_SCALEDINT<32>": 130328.0, - "unsupported": "set()", - "discount_sparsity": False, - "total_bops": 823328768.0, - "total_macs": 12864512.0, - "total_mem_w_bits": 1272832.0, - "total_mem_w_elems": 159104.0, - "total_mem_o_bits": 4170496.0, - "total_mem_o_elems": 130328.0, + "total_cost": { + "unsupported": "set()", + "discount_sparsity": False, + "op_mac_SCALEDINT<8>_SCALEDINT<8>": 12864512.0, + "mem_w_SCALEDINT<8>": 159104.0, + "mem_o_SCALEDINT<32>": 130328.0, + "total_bops": 823328768.0, + "total_macs": 12864512.0, + "total_mem_w_bits": 1272832.0, + "total_mem_w_elems": 159104.0, + "total_mem_o_bits": 4170496.0, + "total_mem_o_elems": 130328.0, + } }, }, } diff --git a/tests/analysis/test_matmul_mac_cost.py b/tests/analysis/test_matmul_mac_cost.py index fbdc0d2a..ff7dbc2f 100644 --- a/tests/analysis/test_matmul_mac_cost.py +++ b/tests/analysis/test_matmul_mac_cost.py @@ -40,6 +40,6 @@ def test_matmul_mac_cost(): cleaned_model = cleanup_model(model) # Two Matmul layers with shape (i_shape, w_shape, o_shape), # L1: ([4, 64, 32], [4, 32, 64], [4, 64, 64]) and L2: ([4, 64, 64], [4, 64, 32], [4, 64, 32]) - inf_cost_dict = infc.inference_cost(cleaned_model, discount_sparsity=False) + inf_cost_dict = infc.inference_cost(cleaned_model, discount_sparsity=False)["total_cost"] mac_cost = inf_cost_dict["op_mac_FLOAT32_FLOAT32"] # Expected mac cost 4*32*64*64 + 4*64*64*32 = 1048576 assert mac_cost == 1048576.0, "Error: discrepancy in mac cost." diff --git a/tests/transformation/test_pruning.py b/tests/transformation/test_pruning.py index 85f9afc9..b2fdbcd8 100644 --- a/tests/transformation/test_pruning.py +++ b/tests/transformation/test_pruning.py @@ -90,7 +90,7 @@ def test_pruning_mnv1(): # do cleanup including folding quantized weights model = cleanup_model(model, False) inp, golden = get_golden_in_and_output("MobileNetv1-w4a4") - cost0 = inference_cost(model, discount_sparsity=False) + cost0 = inference_cost(model, discount_sparsity=False)["total_cost"] assert cost0["op_mac_SCALEDINT<8>_SCALEDINT<8>"] == 10645344.0 assert cost0["mem_w_SCALEDINT<8>"] == 864.0 assert cost0["op_mac_SCALEDINT<4>_SCALEDINT<4>"] == 556357408.0 @@ -105,7 +105,7 @@ def test_pruning_mnv1(): } model = model.transform(PruneChannels(prune_spec)) - cost1 = inference_cost(model, discount_sparsity=False) + cost1 = inference_cost(model, discount_sparsity=False)["total_cost"] assert cost1["op_mac_SCALEDINT<8>_SCALEDINT<8>"] == 7318674.0 assert cost1["mem_w_SCALEDINT<8>"] == 594.0 assert cost1["op_mac_SCALEDINT<4>_SCALEDINT<4>"] == 546053216.0 diff --git a/tests/transformation/test_quantize_graph.py b/tests/transformation/test_quantize_graph.py index 867f9b34..5278194d 100644 --- a/tests/transformation/test_quantize_graph.py +++ b/tests/transformation/test_quantize_graph.py @@ -120,14 +120,14 @@ def to_verify(model, test_details): def test_quantize_graph(test_model): test_details = model_details[test_model] model = download_model(test_model, do_cleanup=True, return_modelwrapper=True) - original_model_inf_cost = inference_cost(model, discount_sparsity=False) + original_model_inf_cost = inference_cost(model, discount_sparsity=False)["total_cost"] nodes_pos = test_details["test_input"] model = model.transform(QuantizeGraph(nodes_pos)) quantnodes_added = len(model.get_nodes_by_op_type("Quant")) assert quantnodes_added == 10 # 10 positions are specified. verification = to_verify(model, nodes_pos) assert verification == "Success" - inf_cost = inference_cost(model, discount_sparsity=False) + inf_cost = inference_cost(model, discount_sparsity=False)["total_cost"] assert ( inf_cost["total_macs"] == original_model_inf_cost["total_macs"] ) # "1814073344.0" must be same as the original model.