diff --git a/.gitignore b/.gitignore index fcb1641c..932ab96b 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,10 @@ synth.txt synth.v application_fpga_par.json application_fpga_par.txt +tb_application_fpga_sim.fst +tb_application_fpga_sim.fst.hier +tb_verilated/ +verilated/ *.o *.asc *.bin diff --git a/hw/application_fpga/Makefile b/hw/application_fpga/Makefile index 8f40c1ea..31a8f906 100644 --- a/hw/application_fpga/Makefile +++ b/hw/application_fpga/Makefile @@ -73,14 +73,23 @@ ICE40_SIM_CELLS = $(shell yosys-config --datdir/ice40/cells_sim.v) # FPGA specific source files. -FPGA_SRC = \ +FPGA_VERILOG_SRCS = \ $(P)/rtl/application_fpga.v \ - $(P)/core/clk_reset_gen/rtl/clk_reset_gen.v + $(P)/core/clk_reset_gen/rtl/clk_reset_gen.v \ + $(P)/core/trng/rtl/trng.v + +# Testbench simulation specific source files. +SIM_VERILOG_SRCS = \ + $(P)/tb/tb_application_fpga_sim.v \ + $(P)/tb/application_fpga_sim.v \ + $(P)/tb/reset_gen_sim.v \ + $(P)/tb/trng_sim.v # Verilator simulation specific source files. -VERILATOR_FPGA_SRC = \ +VERILATOR_VERILOG_SRCS = \ $(P)/tb/application_fpga_sim.v \ - $(P)/tb/reset_gen_sim.v + $(P)/tb/reset_gen_sim.v \ + $(P)/tb/trng_sim.v # Common verilog source files. VERILOG_SRCS = \ @@ -97,8 +106,7 @@ VERILOG_SRCS = \ $(P)/core/tk1/rtl/udi_rom.v \ $(P)/core/uart/rtl/uart_core.v \ $(P)/core/uart/rtl/uart_fifo.v \ - $(P)/core/uart/rtl/uart.v \ - $(P)/core/trng/rtl/trng.v + $(P)/core/uart/rtl/uart.v # PicoRV32 verilog source file PICORV32_SRCS = \ @@ -176,6 +184,10 @@ $(TESTFW_OBJS): $(FIRMWARE_DEPS) firmware.elf: $(FIRMWARE_OBJS) $(P)/fw/tk1/firmware.lds $(CC) $(CFLAGS) $(FIRMWARE_OBJS) $(LDFLAGS) -o $@ +simfirmware.elf: CFLAGS += -DSIMULATION +simfirmware.elf: $(FIRMWARE_OBJS) $(P)/fw/tk1/firmware.lds + $(CC) $(CFLAGS) $(FIRMWARE_OBJS) $(LDFLAGS) -o $@ + qemu_firmware.elf: CFLAGS += -DQEMU_CONSOLE qemu_firmware.elf: firmware.elf mv firmware.elf qemu_firmware.elf @@ -216,6 +228,8 @@ bram_fw.hex: firmware.hex: firmware.bin firmware_size_mismatch python3 $(P)/tools/makehex/makehex.py $< $(BRAM_FW_SIZE) > $@ +simfirmware.hex: simfirmware.bin simfirmware_size_mismatch + python3 $(P)/tools/makehex/makehex.py $< $(BRAM_FW_SIZE) > $@ testfw.hex: testfw.bin testfw_size_mismatch python3 $(P)/tools/makehex/makehex.py $< $(BRAM_FW_SIZE) > $@ @@ -249,7 +263,11 @@ LINT_FLAGS = \ --timescale 1ns/1ns \ -DNO_ICE40_DEFAULT_ASSIGNMENTS -lint: $(FPGA_SRC) $(VERILOG_SRCS) $(PICORV32_SRCS) $(ICE40_SIM_CELLS) +lint: $(FPGA_VERILOG_SRCS) \ + $(SIM_VERILOG_SRCS) \ + $(VERILOG_SRCS) \ + $(PICORV32_SRCS) \ + $(ICE40_SIM_CELLS) $(LINT) $(LINT_FLAGS) \ -DBRAM_FW_SIZE=$(BRAM_FW_SIZE) \ -DFIRMWARE_HEX=\"$(P)/firmware.hex\" \ @@ -278,13 +296,13 @@ CHECK_FORMAT_FLAGS = \ --inplace \ --verify -fmt: $(FPGA_SRC) $(VERILATOR_FPGA_SRC) $(VERILOG_SRCS) +fmt: $(FPGA_VERILOG_SRCS) $(SIM_VERILOG_SRCS) $(VERILATOR_VERILOG_SRCS) $(VERILOG_SRCS) $(FORMAT) $(FORMAT_FLAGS) $^ .PHONY: fmt # Temporary fix using grep, since the verible with --verify flag only returns # error if the last file is malformatted. -checkfmt: $(FPGA_SRC) $(VERILATOR_FPGA_SRC) $(VERILOG_SRCS) +checkfmt: $(FPGA_VERILOG_SRCS) $(SIM_VERILOG_SRCS) $(VERILATOR_VERILOG_SRCS) $(VERILOG_SRCS) $(FORMAT) $(CHECK_FORMAT_FLAGS) $^ 2>&1 | \ grep "Needs formatting" && exit 1 || true .PHONY: checkfmt @@ -292,7 +310,7 @@ checkfmt: $(FPGA_SRC) $(VERILATOR_FPGA_SRC) $(VERILOG_SRCS) #------------------------------------------------------------------- # Build Verilator compiled simulation for the design. #------------------------------------------------------------------- -verilator: $(VERILATOR_FPGA_SRC) $(VERILOG_SRCS) $(PICORV32_SRCS) \ +verilator: $(VERILATOR_VERILOG_SRCS) $(VERILOG_SRCS) $(PICORV32_SRCS) \ firmware.hex $(ICE40_SIM_CELLS) \ $(P)/tb/application_fpga_verilator.cc verilator \ @@ -301,6 +319,7 @@ verilator: $(VERILATOR_FPGA_SRC) $(VERILOG_SRCS) $(PICORV32_SRCS) \ -Wall \ -Wno-COMBDLY \ -Wno-lint \ + -Wno-UNOPTFLAT \ -DBRAM_FW_SIZE=$(BRAM_FW_SIZE) \ -DFIRMWARE_HEX=\"$(P)/firmware.hex\" \ -DUDS_HEX=\"$(P)/data/uds.hex\" \ @@ -308,10 +327,10 @@ verilator: $(VERILATOR_FPGA_SRC) $(VERILOG_SRCS) $(PICORV32_SRCS) \ --cc \ --exe \ --Mdir verilated \ - --top-module application_fpga \ + --top-module application_fpga_sim \ $(filter %.v, $^) \ $(filter %.cc, $^) - make -C verilated -f Vapplication_fpga.mk + make -C verilated -f Vapplication_fpga_sim.mk .PHONY: verilator #------------------------------------------------------------------- @@ -334,7 +353,7 @@ tb: YOSYS_FLAG ?= -synth.json: $(FPGA_SRC) $(VERILOG_SRCS) $(PICORV32_SRCS) bram_fw.hex \ +synth.json: $(FPGA_VERILOG_SRCS) $(VERILOG_SRCS) $(PICORV32_SRCS) bram_fw.hex \ $(P)/data/uds.hex $(P)/data/udi.hex $(YOSYS_PATH)yosys \ -v3 \ @@ -381,41 +400,43 @@ application_fpga_testfw.bin: application_fpga.asc bram_fw.hex testfw.hex @-$(RM) $<.tmp #------------------------------------------------------------------- -# post-synthesis functional simulation. +# Build testbench simulation for the design #------------------------------------------------------------------- -synth_tb.vvp: $(P)/tb/tb_application_fpga.v synth.json - iverilog \ - -o $@ \ - -s tb_application_fpga synth.v $(P)/tb/tb_application_fpga.v \ +tb_application_fpga: $(SIM_VERILOG_SRCS) \ + $(VERILOG_SRCS) \ + $(PICORV32_SRCS) \ + $(ICE40_SIM_CELLS) \ + simfirmware.hex + python3 ./tools/app_bin_to_spram_hex.py \ + ./tb/app.bin \ + ./tb/output_spram0.hex \ + ./tb/output_spram1.hex \ + ./tb/output_spram2.hex \ + ./tb/output_spram3.hex \ + || { echo -e "\n -- Put your app.bin to simulate in the \"tb\" directory\n"; false; } + verilator \ + -j $(shell nproc --ignore=1) \ + --binary \ + --cc \ + --exe \ + --Mdir tb_verilated \ + --trace-fst \ + --trace-structs \ + --top-module tb_application_fpga_sim \ + --timescale 1ns/1ns \ + --timing \ + -Wno-WIDTHEXPAND \ + -Wno-UNOPTFLAT \ -DNO_ICE40_DEFAULT_ASSIGNMENTS \ - $(ICE40_SIM_CELLS) - chmod -x $@ - -synth_sim: synth_tb.vvp - vvp -N $< -.PHONY: synth_sim - -synth_sim_vcd: synth_tb.vvp - vvp -N $< +vcd -.PHONY: synth_sim_vcd - -#------------------------------------------------------------------- -# post-place and route functional simulation. -#------------------------------------------------------------------- -route.v: application_fpga.asc $(P)/data/$(PIN_FILE) - icebox_vlog -L -n application_fpga -sp $(P)/data/$(PIN_FILE) $< > $@ - -route_tb.vvp: route.v tb/tb_application_fpga.v - iverilog -o $@ -s tb_application_fpga $^ $(ICE40_SIM_CELLS) - chmod -x $@ - -route_sim: route_tb.vvp - vvp -N $< -.PHONY: route_sim - -route_sim_vcd: route_tb.vvp - vvp -N $< +vcd -.PHONY: route_sim_vcd + -DAPP_SIZE=$(shell ls -l tb/app.bin| awk '{print $$5}') \ + -DBRAM_FW_SIZE=$(BRAM_FW_SIZE) \ + -DFIRMWARE_HEX=\"$(P)/simfirmware.hex\" \ + -DUDS_HEX=\"$(P)/data/uds.hex\" \ + -DUDI_HEX=\"$(P)/data/udi.hex\" \ + $(filter %.v, $^) + make -C tb_verilated -f Vtb_application_fpga_sim.mk + ./tb_verilated/Vtb_application_fpga_sim \ + && { echo -e "\n -- Wave simulation saved to tb_application_fpga_sim.fst\n"; true; } #------------------------------------------------------------------- # FPGA device programming. @@ -449,14 +470,11 @@ view: tb_application_fpga_vcd #------------------------------------------------------------------- # Cleanup. #------------------------------------------------------------------- -clean: clean_fw clean_tb +clean: clean_sim clean_fw clean_tb rm -f bram_fw.hex - rm -f synth.{v,json,txt} route.v application_fpga.{asc,bin,vcd} application_fpga_testfw.bin - rm -f tb_application_fpga.vvp synth_tb.vvp route_tb.vvp + rm -f synth.{v,json,txt} application_fpga.{asc,bin} application_fpga_testfw.bin rm -f application_fpga_par.{json,txt} - rm -f *.vcd rm -f lint_issues.txt - rm -rf verilated rm -f tools/tpt/*.hex rm -rf tools/tpt/__pycache__ .PHONY: clean @@ -469,6 +487,15 @@ clean_fw: rm -f qemu_firmware.elf .PHONY: clean_fw +clean_sim: + rm -f simfirmware.{elf,elf.map,bin,hex} + rm -f tb_application_fpga_sim.fst + rm -f tb_application_fpga_sim.fst.hier + rm -f tb/output_spram*.hex + rm -rf tb_verilated + rm -rf verilated +.PHONY: clean_sim + clean_tb: make -C core/timer/toolruns clean make -C core/tk1/toolruns clean @@ -487,20 +514,21 @@ help: @echo "" @echo "Supported targets:" @echo "------------------" - @echo "all Build all targets." - @echo "check Run static analysis on firmware." - @echo "splint Run splint static analysis on firmware." - @echo "firmware.elf Build firmware ELF file." - @echo "firmware.hex Build firmware converted to hex, to be included in bitstream." - @echo "bram_fw.hex Build a fake BRAM file that will be filled in later after place-n-route." - @echo "verilator Build Verilator simulation program" - @echo "lint Run lint on Verilog source files." - @echo "tb Run all testbenches" - @echo "prog_flash Program device flash with FGPA bitstream including firmware (using the RPi Pico-based programmer)." - @echo "prog_flash_testfw Program device flash as above, but with testfw." - @echo "clean Delete all generated files." - @echo "clean_fw Delete only generated files for firmware. Useful for fw devs." - @echo "clean_tb Delete only generated files for testbenches." + @echo "all Build all targets." + @echo "check Run static analysis on firmware." + @echo "splint Run splint static analysis on firmware." + @echo "firmware.elf Build firmware ELF file." + @echo "firmware.hex Build firmware converted to hex, to be included in bitstream." + @echo "bram_fw.hex Build a fake BRAM file that will be filled in later after place-n-route." + @echo "verilator Build Verilator simulation program" + @echo "tb_application_fpga Build testbench simulation for the design" + @echo "lint Run lint on Verilog source files." + @echo "tb Run all testbenches" + @echo "prog_flash Program device flash with FGPA bitstream including firmware (using the RPi Pico-based programmer)." + @echo "prog_flash_testfw Program device flash as above, but with testfw." + @echo "clean Delete all generated files." + @echo "clean_fw Delete only generated files for firmware. Useful for fw devs." + @echo "clean_tb Delete only generated files for testbenches." #======================================================================= # EOF Makefile diff --git a/hw/application_fpga/application_fpga.bin.sha256 b/hw/application_fpga/application_fpga.bin.sha256 index 6b3aa343..08caf501 100644 --- a/hw/application_fpga/application_fpga.bin.sha256 +++ b/hw/application_fpga/application_fpga.bin.sha256 @@ -1 +1 @@ -2a3928e8587c8d5d7eca6048c6448dc1d16d52a9a25d3ca5026bec3e7fb14ef3 application_fpga.bin +44086edb70377991b57d3f1c231f743fcf0c2c9d2303843ec133f76cc42449a8 application_fpga.bin diff --git a/hw/application_fpga/core/tk1/rtl/tk1.v b/hw/application_fpga/core/tk1/rtl/tk1.v index 4f481b5d..7e39ce04 100644 --- a/hw/application_fpga/core/tk1/rtl/tk1.v +++ b/hw/application_fpga/core/tk1/rtl/tk1.v @@ -13,7 +13,9 @@ `default_nettype none -module tk1 ( +module tk1 #( + parameter [31:0] APP_SIZE = 32'h0 +) ( input wire clk, input wire reset_n, @@ -255,7 +257,7 @@ module tk1 ( gpio3_reg <= 1'h0; gpio4_reg <= 1'h0; app_start_reg <= 32'h0; - app_size_reg <= 32'h0; + app_size_reg <= APP_SIZE; blake2s_addr_reg <= 32'h0; cdi_mem[0] <= 32'h0; cdi_mem[1] <= 32'h0; diff --git a/hw/application_fpga/fw/tk1/main.c b/hw/application_fpga/fw/tk1/main.c index 524b0dc6..fc50c4b7 100644 --- a/hw/application_fpga/fw/tk1/main.c +++ b/hw/application_fpga/fw/tk1/main.c @@ -54,7 +54,9 @@ static enum state loading_commands(const struct frame_header *hdr, const uint8_t *cmd, enum state state, struct context *ctx); static void run(const struct context *ctx); +#if !defined(SIMULATION) static uint32_t xorwow(uint32_t state, uint32_t acc); +#endif static void scramble_ram(void); static void print_hw_version(void) @@ -364,6 +366,7 @@ static void run(const struct context *ctx) __builtin_unreachable(); } +#if !defined(SIMULATION) static uint32_t xorwow(uint32_t state, uint32_t acc) { state ^= state << 13; @@ -372,9 +375,13 @@ static uint32_t xorwow(uint32_t state, uint32_t acc) state += acc; return state; } +#endif static void scramble_ram(void) { + // Can't fill RAM if we are simulating, data has already been loaded + // into RAM. +#if !defined(SIMULATION) uint32_t *ram = (uint32_t *)(TK1_RAM_BASE); // Fill RAM with random data @@ -386,6 +393,7 @@ static void scramble_ram(void) data_state = xorwow(data_state, data_acc); ram[w] = data_state; } +#endif // Set RAM address and data scrambling parameters *ram_addr_rand = rnd_word(); @@ -414,6 +422,10 @@ int main(void) scramble_ram(); +#if defined(SIMULATION) + run(&ctx); +#endif + for (;;) { switch (state) { case FW_STATE_INITIAL: diff --git a/hw/application_fpga/tb/application_fpga_sim.v b/hw/application_fpga/tb/application_fpga_sim.v index 3f948716..00da22d0 100644 --- a/hw/application_fpga/tb/application_fpga_sim.v +++ b/hw/application_fpga/tb/application_fpga_sim.v @@ -23,20 +23,21 @@ `define verbose(debug_command) `endif +`ifndef APP_SIZE +`define APP_SIZE 0 +`endif -module application_fpga ( +module application_fpga_sim ( input wire clk, - output wire valid, - output wire [03 : 0] wstrb, - output wire [31 : 0] addr, - output wire [31 : 0] wdata, - output wire [31 : 0] rdata, - output wire ready, - output wire interface_rx, input wire interface_tx, + output wire spi_ss, + output wire spi_sck, + output wire spi_mosi, + input wire spi_miso, + input wire touch_event, input wire app_gpio1, @@ -59,14 +60,18 @@ module application_fpga ( localparam RESERVED_PREFIX = 2'h2; localparam MMIO_PREFIX = 2'h3; - // MMIO core mem sub-prefixes. + // MMIO core sub-prefixes. localparam TRNG_PREFIX = 6'h00; localparam TIMER_PREFIX = 6'h01; localparam UDS_PREFIX = 6'h02; localparam UART_PREFIX = 6'h03; localparam TOUCH_SENSE_PREFIX = 6'h04; + localparam FW_RAM_PREFIX = 6'h10; localparam TK1_PREFIX = 6'h3f; + // Instruction used to cause a trap. + localparam ILLEGAL_INSTRUCTION = 32'h0; + //---------------------------------------------------------------- // Registers, memories with associated wires. @@ -83,93 +88,84 @@ module application_fpga ( //---------------------------------------------------------------- wire reset_n; + /* verilator lint_off UNOPTFLAT */ + wire cpu_trap; wire cpu_valid; - wire [03 : 0] cpu_wstrb; + wire cpu_instr; + wire [ 3 : 0] cpu_wstrb; + /* verilator lint_off UNUSED */ wire [31 : 0] cpu_addr; wire [31 : 0] cpu_wdata; - /* verilator lint_off UNOPTFLAT */ reg rom_cs; - /* verilator lint_on UNOPTFLAT */ reg [11 : 0] rom_address; wire [31 : 0] rom_read_data; wire rom_ready; reg ram_cs; reg [ 3 : 0] ram_we; - reg [14 : 0] ram_address; + reg [15 : 0] ram_address; reg [31 : 0] ram_write_data; wire [31 : 0] ram_read_data; wire ram_ready; - /* verilator lint_off UNOPTFLAT */ reg trng_cs; - /* verilator lint_on UNOPTFLAT */ reg trng_we; reg [ 7 : 0] trng_address; reg [31 : 0] trng_write_data; wire [31 : 0] trng_read_data; wire trng_ready; - /* verilator lint_off UNOPTFLAT */ reg timer_cs; - /* verilator lint_on UNOPTFLAT */ reg timer_we; reg [ 7 : 0] timer_address; reg [31 : 0] timer_write_data; wire [31 : 0] timer_read_data; wire timer_ready; - /* verilator lint_off UNOPTFLAT */ reg uds_cs; - /* verilator lint_on UNOPTFLAT */ - reg [ 7 : 0] uds_address; + reg [ 2 : 0] uds_address; wire [31 : 0] uds_read_data; wire uds_ready; - /* verilator lint_off UNOPTFLAT */ reg uart_cs; - /* verilator lint_on UNOPTFLAT */ reg uart_we; reg [ 7 : 0] uart_address; reg [31 : 0] uart_write_data; wire [31 : 0] uart_read_data; wire uart_ready; - /* verilator lint_off UNOPTFLAT */ + reg fw_ram_cs; + reg [ 3 : 0] fw_ram_we; + reg [ 8 : 0] fw_ram_address; + reg [31 : 0] fw_ram_write_data; + wire [31 : 0] fw_ram_read_data; + wire fw_ram_ready; + reg touch_sense_cs; - /* verilator lint_on UNOPTFLAT */ reg touch_sense_we; reg [ 7 : 0] touch_sense_address; wire [31 : 0] touch_sense_read_data; wire touch_sense_ready; - /* verilator lint_off UNOPTFLAT */ reg tk1_cs; - /* verilator lint_on UNOPTFLAT */ reg tk1_we; reg [ 7 : 0] tk1_address; reg [31 : 0] tk1_write_data; wire [31 : 0] tk1_read_data; wire tk1_ready; wire system_mode; - - - //---------------------------------------------------------------- - // Concurrent assignments. - //---------------------------------------------------------------- - assign valid = cpu_valid; - assign wstrb = cpu_wstrb; - assign addr = cpu_addr; - assign wdata = cpu_wdata; - assign rdata = muxed_rdata_reg; - assign ready = muxed_ready_reg; + wire force_trap; + wire [14 : 0] ram_addr_rand; + wire [31 : 0] ram_data_rand; + wire tk1_system_reset; + /* verilator lint_on UNOPTFLAT */ //---------------------------------------------------------------- // Module instantiations. //---------------------------------------------------------------- - reset_gen #( + reset_gen_sim #( .RESET_CYCLES(200) ) reset_gen_inst ( .clk (clk), @@ -179,35 +175,31 @@ module application_fpga ( picorv32 #( .ENABLE_COUNTERS(0), - .LATCHED_MEM_RDATA(0), .TWO_STAGE_SHIFT(0), - .TWO_CYCLE_ALU(0), - .CATCH_MISALIGN(0), - .CATCH_ILLINSN(0), - .COMPRESSED_ISA(1), - .ENABLE_MUL(1), - .ENABLE_DIV(0), - .BARREL_SHIFTER(0) + .CATCH_MISALIGN (0), + .COMPRESSED_ISA (1), + .ENABLE_FAST_MUL(1), + .BARREL_SHIFTER (1) ) cpu ( .clk(clk), .resetn(reset_n), + .trap(cpu_trap), .mem_valid(cpu_valid), + .mem_ready(muxed_ready_reg), .mem_addr (cpu_addr), .mem_wdata(cpu_wdata), .mem_wstrb(cpu_wstrb), .mem_rdata(muxed_rdata_reg), - .mem_ready(muxed_ready_reg), + .mem_instr(cpu_instr), - // Defined unsed ports. Makes lint happy, - // but still needs to help lint with empty ports. + // Defined unused ports. Makes lint happy. But + // we still needs to help lint with empty ports. /* verilator lint_off PINCONNECTEMPTY */ .irq(32'h0), .eoi(), - .trap(), .trace_valid(), .trace_data(), - .mem_instr(), .mem_la_read(), .mem_la_write(), .mem_la_addr(), @@ -226,6 +218,9 @@ module application_fpga ( rom rom_inst ( + .clk(clk), + .reset_n(reset_n), + .cs(rom_cs), .address(rom_address), .read_data(rom_read_data), @@ -237,6 +232,9 @@ module application_fpga ( .clk(clk), .reset_n(reset_n), + .ram_addr_rand(ram_addr_rand), + .ram_data_rand(ram_data_rand), + .cs(ram_cs), .we(ram_we), .address(ram_address), @@ -246,6 +244,33 @@ module application_fpga ( ); + fw_ram fw_ram_inst ( + .clk(clk), + .reset_n(reset_n), + + .system_mode(system_mode), + + .cs(fw_ram_cs), + .we(fw_ram_we), + .address(fw_ram_address), + .write_data(fw_ram_write_data), + .read_data(fw_ram_read_data), + .ready(fw_ram_ready) + ); + + + trng_sim trng_inst ( + .clk(clk), + .reset_n(reset_n), + .cs(trng_cs), + .we(trng_we), + .address(trng_address), + .write_data(trng_write_data), + .read_data(trng_read_data), + .ready(trng_ready) + ); + + timer timer_inst ( .clk(clk), .reset_n(reset_n), @@ -263,6 +288,8 @@ module application_fpga ( .clk(clk), .reset_n(reset_n), + .system_mode(system_mode), + .cs(uds_cs), .address(uds_address), .read_data(uds_read_data), @@ -300,12 +327,30 @@ module application_fpga ( ); - tk1 tk1_inst ( + tk1 #( + .APP_SIZE(`APP_SIZE) + ) tk1_inst ( .clk(clk), .reset_n(reset_n), .system_mode(system_mode), + .cpu_addr (cpu_addr), + .cpu_instr (cpu_instr), + .cpu_valid (cpu_valid), + .cpu_trap (cpu_trap), + .force_trap(force_trap), + + .system_reset(tk1_system_reset), + + .ram_addr_rand(ram_addr_rand), + .ram_data_rand(ram_data_rand), + + .spi_ss (spi_ss), + .spi_sck (spi_sck), + .spi_mosi(spi_mosi), + .spi_miso(spi_miso), + .led_r(led_r), .led_g(led_g), .led_b(led_b), @@ -330,13 +375,12 @@ module application_fpga ( //---------------------------------------------------------------- always @(posedge clk) begin : reg_update if (!reset_n) begin - muxed_ready_reg <= 1'h0; muxed_rdata_reg <= 32'h0; + muxed_ready_reg <= 1'h0; end - else begin - muxed_ready_reg <= muxed_ready_new; muxed_rdata_reg <= muxed_rdata_new; + muxed_ready_reg <= muxed_ready_new; end end @@ -348,7 +392,9 @@ module application_fpga ( always @* begin : cpu_mem_ctrl reg [1 : 0] area_prefix; reg [5 : 0] core_prefix; + reg [255:0] ascii_state; + ascii_state = ""; area_prefix = cpu_addr[31 : 30]; core_prefix = cpu_addr[29 : 24]; @@ -359,10 +405,15 @@ module application_fpga ( rom_address = cpu_addr[13 : 2]; ram_cs = 1'h0; - ram_we = cpu_wstrb; - ram_address = cpu_addr[16 : 2]; + ram_we = 4'h0; + ram_address = cpu_addr[17 : 2]; ram_write_data = cpu_wdata; + fw_ram_cs = 1'h0; + fw_ram_we = cpu_wstrb; + fw_ram_address = cpu_addr[10 : 2]; + fw_ram_write_data = cpu_wdata; + trng_cs = 1'h0; trng_we = |cpu_wstrb; trng_address = cpu_addr[9 : 2]; @@ -374,7 +425,7 @@ module application_fpga ( timer_write_data = cpu_wdata; uds_cs = 1'h0; - uds_address = cpu_addr[9 : 2]; + uds_address = cpu_addr[4 : 2]; uart_cs = 1'h0; uart_we = |cpu_wstrb; @@ -390,92 +441,123 @@ module application_fpga ( tk1_address = cpu_addr[9 : 2]; tk1_write_data = cpu_wdata; + // Two stage mux implementing read and + // write access performed based on the address + // from the CPU. if (cpu_valid && !muxed_ready_reg) begin - case (area_prefix) - ROM_PREFIX: begin - `verbose($display("Access to ROM area");) - rom_cs = 1'h1; - muxed_rdata_new = rom_read_data; - muxed_ready_new = rom_ready; - end - - RAM_PREFIX: begin - `verbose($display("Access to RAM area");) - ram_cs = 1'h1; - muxed_rdata_new = ram_read_data; - muxed_ready_new = ram_ready; - end - - RESERVED_PREFIX: begin - `verbose($display("Access to RESERVED area");) - muxed_rdata_new = 32'h00000000; - muxed_ready_new = 1'h1; - end - - MMIO_PREFIX: begin - `verbose($display("Access to MMIO area");) - case (core_prefix) - TRNG_PREFIX: begin - `verbose($display("Access to TRNG core");) - trng_cs = 1'h1; - muxed_rdata_new = trng_read_data; - muxed_ready_new = trng_ready; - end - - TIMER_PREFIX: begin - `verbose($display("Access to TIMER core");) - timer_cs = 1'h1; - muxed_rdata_new = timer_read_data; - muxed_ready_new = timer_ready; - end - - UDS_PREFIX: begin - `verbose($display("Access to UDS core");) - uds_cs = 1'h1; - muxed_rdata_new = uds_read_data; - muxed_ready_new = uds_ready; - end - - UART_PREFIX: begin - `verbose($display("Access to UART core");) - uart_cs = 1'h1; - muxed_rdata_new = uart_read_data; - muxed_ready_new = uart_ready; - end - - TOUCH_SENSE_PREFIX: begin - `verbose($display("Access to TOUCH_SENSE core");) - touch_sense_cs = 1'h1; - muxed_rdata_new = touch_sense_read_data; - muxed_ready_new = touch_sense_ready; - end - - TK1_PREFIX: begin - `verbose($display("Access to TK1 core");) - tk1_cs = 1'h1; - muxed_rdata_new = tk1_read_data; - muxed_ready_new = tk1_ready; - end - - default: begin - `verbose($display("UNDEFINED MMIO");) - muxed_rdata_new = 32'h00000000; - muxed_ready_new = 1'h1; - end - endcase // case (core_prefix) - end // case: MMIO_PREFIX - - default: begin - `verbose($display("UNDEFINED AREA");) - muxed_rdata_new = 32'h0; - muxed_ready_new = 1'h1; - end - endcase // case (area_prefix) - end + if (force_trap) begin + `verbose($display("Force trap");) + ascii_state = "Force trap"; + muxed_rdata_new = ILLEGAL_INSTRUCTION; + muxed_ready_new = 1'h1; + end + else begin + case (area_prefix) + ROM_PREFIX: begin + `verbose($display("Access to ROM area");) + ascii_state = "ROM area"; + rom_cs = 1'h1; + muxed_rdata_new = rom_read_data; + muxed_ready_new = rom_ready; + end + + RAM_PREFIX: begin + `verbose($display("Access to RAM area");) + ascii_state = "RAM area"; + ram_cs = 1'h1; + ram_we = cpu_wstrb; + muxed_rdata_new = ram_read_data; + muxed_ready_new = ram_ready; + end + + RESERVED_PREFIX: begin + `verbose($display("Access to RESERVED area");) + ascii_state = "RESERVED area"; + muxed_rdata_new = 32'h0; + muxed_ready_new = 1'h1; + end + + MMIO_PREFIX: begin + `verbose($display("Access to MMIO area");) + case (core_prefix) + TRNG_PREFIX: begin + `verbose($display("Access to TRNG core");) + ascii_state = "TRNG core"; + trng_cs = 1'h1; + muxed_rdata_new = trng_read_data; + muxed_ready_new = trng_ready; + end + + TIMER_PREFIX: begin + `verbose($display("Access to TIMER core");) + ascii_state = "TIMER core"; + timer_cs = 1'h1; + muxed_rdata_new = timer_read_data; + muxed_ready_new = timer_ready; + end + + UDS_PREFIX: begin + `verbose($display("Access to UDS core");) + ascii_state = "UDS core"; + uds_cs = 1'h1; + muxed_rdata_new = uds_read_data; + muxed_ready_new = uds_ready; + end + + UART_PREFIX: begin + `verbose($display("Access to UART core");) + ascii_state = "UART core"; + uart_cs = 1'h1; + muxed_rdata_new = uart_read_data; + muxed_ready_new = uart_ready; + end + + TOUCH_SENSE_PREFIX: begin + `verbose($display("Access to TOUCH_SENSE core");) + ascii_state = "TOUCH_SENSE core"; + touch_sense_cs = 1'h1; + muxed_rdata_new = touch_sense_read_data; + muxed_ready_new = touch_sense_ready; + end + + FW_RAM_PREFIX: begin + `verbose($display("Access to FW_RAM core");) + ascii_state = "FW_RAM core"; + fw_ram_cs = 1'h1; + muxed_rdata_new = fw_ram_read_data; + muxed_ready_new = fw_ram_ready; + end + + TK1_PREFIX: begin + `verbose($display("Access to TK1 core");) + ascii_state = "TK1 core"; + tk1_cs = 1'h1; + muxed_rdata_new = tk1_read_data; + muxed_ready_new = tk1_ready; + end + + default: begin + `verbose($display("UNDEFINED MMIO");) + ascii_state = "UNDEFINED MMIO"; + muxed_rdata_new = 32'h0; + muxed_ready_new = 1'h1; + end + endcase // case (core_prefix) + end // case: MMIO_PREFIX + + default: begin + `verbose($display("UNDEFINED AREA");) + ascii_state = "UNDEFINED AREA"; + muxed_rdata_new = 32'h0; + muxed_ready_new = 1'h1; + end + endcase // case (area_prefix) + end // if (force_trap) begin end else begin + end // if (cpu_valid && !muxed_ready_reg) begin end endmodule // application_fpga //====================================================================== -// EOF application_fpga.v +// EOF application_fpga_sim.v //====================================================================== diff --git a/hw/application_fpga/tb/application_fpga_verilator.cc b/hw/application_fpga/tb/application_fpga_verilator.cc index 29cd1bec..e3d0d686 100644 --- a/hw/application_fpga/tb/application_fpga_verilator.cc +++ b/hw/application_fpga/tb/application_fpga_verilator.cc @@ -21,12 +21,15 @@ #include #include -#include "Vapplication_fpga.h" +#include "Vapplication_fpga_sim.h" #include "verilated.h" -// Clock: 18 MHz, 62500 bps -// Divisor = 18E6 / 62500 = 288 -#define BIT_DIV 288 +// Clock: 21 MHz, 62500 bps +// Divisor = 21E6 / 62500 = 336 +#define CPU_CLOCK 21000000 +#define BAUD_RATE 62500 +#define BIT_DIV (CPU_CLOCK/BAUD_RATE) + struct uart { int bit_div; @@ -294,13 +297,15 @@ int main(int argc, char **argv, char **env) { Verilated::commandArgs(argc, argv); int r = 0, g = 0, b = 0; - Vapplication_fpga top; + Vapplication_fpga_sim top; struct uart u; struct pty p; int err; if (signal(SIGUSR1, sighandler) == SIG_ERR) return -1; + printf("cpu clock: %d\n", CPU_CLOCK); + printf("baud rate: %d\n", BAUD_RATE); printf("generate touch event: \"$ kill -USR1 %d\"\n", (int)getpid()); err = pty_init(&p); diff --git a/hw/application_fpga/tb/reset_gen_sim.v b/hw/application_fpga/tb/reset_gen_sim.v index 6187d4ce..de573d64 100644 --- a/hw/application_fpga/tb/reset_gen_sim.v +++ b/hw/application_fpga/tb/reset_gen_sim.v @@ -1,8 +1,8 @@ //====================================================================== // // reset_gen_sim.v -// ---------------- -// Reset generator Verilator simulation of the application_fpga. +// --------------- +// Reset generator simulation of the application_fpga. // // // Author: Joachim Strombergson @@ -13,7 +13,7 @@ `default_nettype none -module reset_gen #( +module reset_gen_sim #( parameter RESET_CYCLES = 200 ) ( input wire clk, @@ -63,8 +63,8 @@ module reset_gen #( end end -endmodule // reset_gen +endmodule // reset_gen_sim //====================================================================== -// EOF reset_gen.v +// EOF reset_gen_sim.v //====================================================================== diff --git a/hw/application_fpga/tb/tb_application_fpga_sim.v b/hw/application_fpga/tb/tb_application_fpga_sim.v new file mode 100644 index 00000000..32605fb4 --- /dev/null +++ b/hw/application_fpga/tb/tb_application_fpga_sim.v @@ -0,0 +1,112 @@ +//====================================================================== +// +// tb_application_fpga_sim.v +// ------------------------- +// Top level module of the application_fpga. +// The design exposes a UART interface to allow a host to +// send commands and receive responses as needed load, execute and +// communicate with applications. +// +// +// Copyright (C) 2022 - Tillitis AB +// SPDX-License-Identifier: GPL-2.0-only +// +//====================================================================== + +`timescale 1ns / 1ns + +module tb_application_fpga_sim (); + + //---------------------------------------------------------------- + // Internal constant and parameter definitions. + //---------------------------------------------------------------- + parameter CLK_HALF_PERIOD = 1; + parameter CLK_PERIOD = 2 * CLK_HALF_PERIOD; + + //---------------------------------------------------------------- + // Register and Wire declarations. + //---------------------------------------------------------------- + reg tb_clk = 0; + wire tb_interface_rx; + reg tb_interface_tx = 1'h1; // Set to 1 to simulate inactive UART + wire tb_spi_ss; + wire tb_spi_sck; + wire tb_spi_mosi; + reg tb_spi_miso; + reg tb_touch_event; + reg tb_app_gpio1; + reg tb_app_gpio2; + wire tb_app_gpio3; + wire tb_app_gpio4; + wire tb_led_r; + wire tb_led_g; + wire tb_led_b; + + //---------------------------------------------------------------- + // Device Under Test. + //---------------------------------------------------------------- + application_fpga_sim dut ( + .clk(tb_clk), + .interface_rx(tb_interface_rx), + .interface_tx(tb_interface_tx), + .spi_ss(tb_spi_ss), + .spi_sck(tb_spi_sck), + .spi_mosi(tb_spi_mosi), + .spi_miso(tb_spi_miso), + .touch_event(tb_touch_event), + .app_gpio1(tb_app_gpio1), + .app_gpio2(tb_app_gpio2), + .app_gpio3(tb_app_gpio3), + .app_gpio4(tb_app_gpio4), + .led_r(tb_led_r), + .led_g(tb_led_g), + .led_b(tb_led_b) + ); + + //---------------------------------------------------------------- + // clk_gen + // + // Always running clock generator process. + //---------------------------------------------------------------- + always begin : clk_gen + #CLK_HALF_PERIOD; + tb_clk = !tb_clk; + end // clk_gen + + //---------------------------------------------------------------- + // finish + // + // End simulation + //---------------------------------------------------------------- + initial begin + // End simulation after XXX time units (set by timescale) + #20000000; + $display("TIMEOUT"); + $finish; + end + + //---------------------------------------------------------------- + // Fill memories with data + //---------------------------------------------------------------- + initial begin + $readmemh("tb/output_spram0.hex", dut.ram_inst.spram0.mem); + $readmemh("tb/output_spram1.hex", dut.ram_inst.spram1.mem); + $readmemh("tb/output_spram2.hex", dut.ram_inst.spram2.mem); + $readmemh("tb/output_spram3.hex", dut.ram_inst.spram3.mem); + end + + //---------------------------------------------------------------- + // dumpfile + // + // Save waveform file + //---------------------------------------------------------------- + initial begin + $dumpfile("tb_application_fpga_sim.fst"); + $dumpvars(0, tb_application_fpga_sim); + end + +endmodule // tb_application_fpga_sim + +//====================================================================== +// EOF tb_application_fpga_sim.v +//====================================================================== diff --git a/hw/application_fpga/tb/trng_sim.v b/hw/application_fpga/tb/trng_sim.v new file mode 100644 index 00000000..1e6dd897 --- /dev/null +++ b/hw/application_fpga/tb/trng_sim.v @@ -0,0 +1,160 @@ +//====================================================================== +// +// trng_sim.v +// ---------- +// TRNG simulation of the application_fpga. +// +// +// Copyright (C) 2022 - Tillitis AB +// SPDX-License-Identifier: GPL-2.0-only +// +//====================================================================== + +`default_nettype none + +module trng_sim ( + input wire clk, + input wire reset_n, + + input wire cs, + input wire we, + input wire [ 7 : 0] address, + /* verilator lint_off UNUSED */ + input wire [31 : 0] write_data, + /* verilator lint_on UNUSED */ + output wire [31 : 0] read_data, + output wire ready +); + + + //---------------------------------------------------------------- + // Internal constant and parameter definitions. + //---------------------------------------------------------------- + // API + localparam ADDR_STATUS = 8'h09; + localparam ADDR_ENTROPY = 8'h20; + + // Total number of ROSCs will be 2 x NUM_ROSC. + localparam SAMPLE_CYCLES = 16'h1000; + localparam NUM_ROSC = 16; + localparam SKIP_BITS = 32; + + localparam CTRL_SAMPLE1 = 0; + localparam CTRL_SAMPLE2 = 1; + localparam CTRL_DATA_READY = 2; + + + //---------------------------------------------------------------- + // Registers with associated wires. + //---------------------------------------------------------------- + reg [15 : 0] cycle_ctr_reg; + reg [15 : 0] cycle_ctr_new; + reg cycle_ctr_done; + reg cycle_ctr_rst; + + reg [7 : 0] bit_ctr_reg; + reg [7 : 0] bit_ctr_new; + reg bit_ctr_inc; + reg bit_ctr_rst; + reg bit_ctr_we; + + reg [31 : 0] entropy_reg; + reg [31 : 0] entropy_new; + reg entropy_we; + + reg [1 : 0] sample1_reg; + reg [1 : 0] sample1_new; + reg sample1_we; + + reg [1 : 0] sample2_reg; + reg [1 : 0] sample2_new; + reg sample2_we; + + reg data_ready_reg; + reg data_ready_new; + reg data_ready_we; + + reg [1 : 0] trng_ctrl_reg; + reg [1 : 0] trng_ctrl_new; + reg trng_ctrl_we; + + //---------------------------------------------------------------- + // Wires. + //---------------------------------------------------------------- + reg [31 : 0] tmp_read_data; + reg tmp_ready; + + /* verilator lint_off UNOPTFLAT */ + wire [(NUM_ROSC - 1) : 0] f; + /* verilator lint_on UNOPTFLAT */ + + /* verilator lint_off UNOPTFLAT */ + wire [(NUM_ROSC - 1) : 0] g; + /* verilator lint_on UNOPTFLAT */ + + // Simulation of TRNG with 32-bit LFSR with polynomial: x^32 + x^22 + x^2 + x + 1 + wire feedback = entropy_reg[31] ^ entropy_reg[21] ^ entropy_reg[1] ^ entropy_reg[0]; + //---------------------------------------------------------------- + // Concurrent connectivity for ports etc. + //---------------------------------------------------------------- + assign read_data = tmp_read_data; + assign ready = tmp_ready; + + + //--------------------------------------------------------------- + // reg_update + //--------------------------------------------------------------- + always @(posedge clk) begin : reg_update + if (!reset_n) begin + cycle_ctr_reg <= 16'h0; + bit_ctr_reg <= 8'h0; + sample1_reg <= 2'h0; + sample2_reg <= 2'h0; + entropy_reg <= 32'hDEADBEEF; // Reset LFSR to a non-zero seed + data_ready_reg <= 1'h1; + trng_ctrl_reg <= CTRL_SAMPLE1; + end + else begin + if (cs) begin + if (!we) begin + if (address == ADDR_ENTROPY) begin + entropy_reg <= {entropy_reg[30:0], feedback}; // Shift left with feedback + end + end + end + end + end + + + //---------------------------------------------------------------- + // api + // + // The interface command decoding logic. + //---------------------------------------------------------------- + always @* begin : api + bit_ctr_rst = 1'h0; + tmp_read_data = 32'h0; + tmp_ready = 1'h0; + + if (cs) begin + tmp_ready = 1'h1; + + if (!we) begin + if (address == ADDR_STATUS) begin + tmp_read_data = {31'h0, data_ready_reg}; + end + + if (address == ADDR_ENTROPY) begin + tmp_read_data = entropy_reg; + bit_ctr_rst = 1'h1; + end + end + end + end // api + + +endmodule // trng + +//====================================================================== +// EOF trng_sim.v +//====================================================================== diff --git a/hw/application_fpga/tools/app_bin_to_spram_hex.py b/hw/application_fpga/tools/app_bin_to_spram_hex.py new file mode 100644 index 00000000..a71c8f03 --- /dev/null +++ b/hw/application_fpga/tools/app_bin_to_spram_hex.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Tillitis AB +# SPDX-License-Identifier: BSD-2-Clause + +import argparse +import sys + +arg_parser = argparse.ArgumentParser( + description=( + "Converts one TKey app binary into four files suitable to load into SPRAM" + " during simulation." + ) +) + +arg_parser.add_argument("app_bin") +arg_parser.add_argument("output_spram0_hex") +arg_parser.add_argument("output_spram1_hex") +arg_parser.add_argument("output_spram2_hex") +arg_parser.add_argument("output_spram3_hex") + +args = arg_parser.parse_args() + + +def abort(msg: str, exitcode: int): + sys.stderr.write(msg + "\n") + sys.exit(exitcode) + + +with ( + open(args.app_bin, "rb") as app, + open(args.output_spram0_hex, "w") as spram0, + open(args.output_spram1_hex, "w") as spram1, + open(args.output_spram2_hex, "w") as spram2, + open(args.output_spram3_hex, "w") as spram3, +): + + # For debug + # app = open("app.bin", "rb") + # spram0 = open("output_spram0_hex", "w") + # spram1 = open("output_spram1_hex", "w") + # spram2 = open("output_spram2_hex", "w") + # spram3 = open("output_spram3_hex", "w") + + SPRAM_SIZE_BYTES = int(256 * 1024 / 8) + RAM_SIZE_BYTES = SPRAM_SIZE_BYTES * 4 + + rows, cols = int(SPRAM_SIZE_BYTES / 2), 2 + ram0 = [[0] * cols for _ in range(rows)] + ram1 = [[0] * cols for _ in range(rows)] + ram2 = [[0] * cols for _ in range(rows)] + ram3 = [[0] * cols for _ in range(rows)] + + # Input data is little endian + input_data_le = app.read() + + if len(input_data_le) % 2 != 0: + abort("Error: File size not a multiple of 2 bytes", -1) + + if len(input_data_le) > RAM_SIZE_BYTES: + abort("Error: File does not fit in RAM", -1) + + # Zero-pad input to RAM size + input_data_le += b"\x00" * (RAM_SIZE_BYTES - len(input_data_le)) + + # Set ram_addr_rand and ram_data_rand to the values the simulated TRNG + # will deliver. + ram_addr_rand = 0xDEADBEEF + ram_data_rand = 0xBD5B7DDE + + # Convert RAM + cpu_addr_data_le = input_data_le[0:RAM_SIZE_BYTES] + + for cpu_addr in range(0, int(RAM_SIZE_BYTES / 4), 4): + + # Data is little endian + word_data_le = cpu_addr_data_le[cpu_addr : cpu_addr + 4] + + # Get ram address + ram_addr = cpu_addr >> 2 + + # Convert ram_addr to little endian for calculation + ram_addr_le = ram_addr.to_bytes(4, byteorder="little", signed=False) + + # Convert ram_data_rand to little endian for calculation + ram_data_rand_le = ram_data_rand.to_bytes(4, byteorder="little", signed=False) + + # Concatenate two ram_addr and convert to little endian for calculation + ram_addr_double = ((ram_addr << 16) & 0xFFFF0000) | (ram_addr & 0x0000FFFF) + ram_addr_double_le = ram_addr_double.to_bytes( + 4, byteorder="little", signed=False + ) + + # XOR word_data_le, ram_data_rand_le and ram_addr_le + # Explanation: + # 1. Zip the byte strings: zip(word_data_le, ram_data_rand_le, ram_addr_double_le) creates + # an iterator of tuples with corresponding elements from the byte strings. + # 2. Apply XOR: Use a generator expression (a ^ b ^ c for a, b, c in ...) to XOR the values in each tuple. + # 3. Convert back to bytes: The bytes constructor collects the XORed values into a new byte string. + scrambled_word_data_le = bytes( + a ^ b ^ c + for a, b, c in zip(word_data_le, ram_data_rand_le, ram_addr_double_le) + ) + + # Convert ram_addr_rand to little endian for calculation + ram_addr_rand_le = ram_addr_rand.to_bytes(4, byteorder="little", signed=False) + + # XOR ram_addr_le and ram_addr_rand_le + scrambled_ram_addr_le = bytes( + a ^ b for a, b in zip(ram_addr_le, ram_addr_rand_le) + ) + + # Get scrambled_ram_addr as int + scrambled_ram_addr = int.from_bytes( + scrambled_ram_addr_le, byteorder="little", signed=False + ) + + # In hardware the highest bits are discarded since the memory is only 128K bytes + scrambled_ram_addr = scrambled_ram_addr & 0x7FFF + + # Check bit 14 for which pair of ram blocks the data should be stored in + if not (scrambled_ram_addr & 0x4000): + ram0[scrambled_ram_addr] = ( + scrambled_word_data_le[0], + scrambled_word_data_le[1], + ) + ram1[scrambled_ram_addr] = ( + scrambled_word_data_le[2], + scrambled_word_data_le[3], + ) + else: + ram2[scrambled_ram_addr] = ( + scrambled_word_data_le[0], + scrambled_word_data_le[1], + ) + ram3[scrambled_ram_addr] = ( + scrambled_word_data_le[2], + scrambled_word_data_le[3], + ) + + for line in range(0, rows, 1): + spram0.write(f"{ram0[line][1]:02x}") + spram0.write(f"{ram0[line][0]:02x}\n") + spram1.write(f"{ram1[line][1]:02x}") + spram1.write(f"{ram1[line][0]:02x}\n") + spram2.write(f"{ram2[line][1]:02x}") + spram2.write(f"{ram2[line][0]:02x}\n") + spram3.write(f"{ram3[line][1]:02x}") + spram3.write(f"{ram3[line][0]:02x}\n")