From 3d2c912b4fd46b64d6d2c2e422bc0f7d9fda22d2 Mon Sep 17 00:00:00 2001 From: Luke Wren Date: Thu, 9 Dec 2021 22:25:18 +0000 Subject: [PATCH] Add test script to make it easier to add software testcases --- hdl/hazard3_core.v | 1 + test/sim/amo_timer_irq/Makefile | 5 -- test/sim/common/src_only_app.mk | 22 +++--- test/sim/common/tb_cxxrtl_io.h | 2 + test/sim/ecall_simple/Makefile | 5 -- test/sim/lr_sc_smoke/Makefile | 5 -- test/sim/sw_testcases/.gitignore | 1 + test/sim/{amo_smoke => sw_testcases}/Makefile | 4 +- test/sim/sw_testcases/Readme.md | 20 +++++ .../main.c => sw_testcases/amo_smoke.c} | 2 - .../sw_testcases/amo_smoke.expected_output | 12 +++ .../amo_smoke.gtkw | 3 +- .../main.c => sw_testcases/amo_timer_irq.c} | 8 +- .../amo_timer_irq.gtkw | 0 test/sim/sw_testcases/cleantests | 2 + .../main.c => sw_testcases/ecall_simple.c} | 2 +- .../sw_testcases/ecall_simple.expected_output | 11 +++ .../ecall_simple.gtkw} | 3 +- test/sim/sw_testcases/hellow.c | 6 ++ test/sim/sw_testcases/hellow.expected_output | 1 + .../main.c => sw_testcases/lr_sc_smoke.c} | 0 .../lr_sc_smoke.gtkw | 3 +- test/sim/sw_testcases/runtests | 76 +++++++++++++++++++ .../main.c => sw_testcases/wfi_loop.c} | 0 .../sim/sw_testcases/wfi_loop.expected_output | 12 +++ test/sim/tb_cxxrtl/Makefile | 4 +- test/sim/wfi_loop/Makefile | 5 -- 27 files changed, 167 insertions(+), 48 deletions(-) delete mode 100644 test/sim/amo_timer_irq/Makefile delete mode 100644 test/sim/ecall_simple/Makefile delete mode 100644 test/sim/lr_sc_smoke/Makefile create mode 100644 test/sim/sw_testcases/.gitignore rename test/sim/{amo_smoke => sw_testcases}/Makefile (56%) create mode 100644 test/sim/sw_testcases/Readme.md rename test/sim/{amo_smoke/main.c => sw_testcases/amo_smoke.c} (83%) create mode 100644 test/sim/sw_testcases/amo_smoke.expected_output rename test/sim/{amo_smoke => sw_testcases}/amo_smoke.gtkw (82%) rename test/sim/{amo_timer_irq/main.c => sw_testcases/amo_timer_irq.c} (74%) rename test/sim/{amo_timer_irq => sw_testcases}/amo_timer_irq.gtkw (100%) create mode 100755 test/sim/sw_testcases/cleantests rename test/sim/{ecall_simple/main.c => sw_testcases/ecall_simple.c} (93%) create mode 100644 test/sim/sw_testcases/ecall_simple.expected_output rename test/sim/{ecall_simple/ecall_simple_run.gtkw => sw_testcases/ecall_simple.gtkw} (90%) create mode 100644 test/sim/sw_testcases/hellow.c create mode 100644 test/sim/sw_testcases/hellow.expected_output rename test/sim/{lr_sc_smoke/main.c => sw_testcases/lr_sc_smoke.c} (100%) rename test/sim/{lr_sc_smoke => sw_testcases}/lr_sc_smoke.gtkw (81%) create mode 100755 test/sim/sw_testcases/runtests rename test/sim/{wfi_loop/main.c => sw_testcases/wfi_loop.c} (100%) create mode 100644 test/sim/sw_testcases/wfi_loop.expected_output delete mode 100644 test/sim/wfi_loop/Makefile diff --git a/hdl/hazard3_core.v b/hdl/hazard3_core.v index 74209de..f7b8790 100644 --- a/hdl/hazard3_core.v +++ b/hdl/hazard3_core.v @@ -863,6 +863,7 @@ always @ (*) begin {MEMOP_LBU , 2'b10}: m_rdata_pick_sext = {{24{1'b0 }}, bus_rdata_d[23:16]}; {MEMOP_LBU , 2'b11}: m_rdata_pick_sext = {{24{1'b0 }}, bus_rdata_d[31:24]}; {MEMOP_LW , 2'bzz}: m_rdata_pick_sext = bus_rdata_d; + {MEMOP_LR_W, 2'bzz}: m_rdata_pick_sext = bus_rdata_d; default: m_rdata_pick_sext = 32'hxxxx_xxxx; endcase diff --git a/test/sim/amo_timer_irq/Makefile b/test/sim/amo_timer_irq/Makefile deleted file mode 100644 index e950845..0000000 --- a/test/sim/amo_timer_irq/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -SRCS := ../common/init.S main.c -APP := amo_timer_irq -CCFLAGS = -march=rv32imac -Os - -include ../common/src_only_app.mk diff --git a/test/sim/common/src_only_app.mk b/test/sim/common/src_only_app.mk index 815de6e..040969f 100644 --- a/test/sim/common/src_only_app.mk +++ b/test/sim/common/src_only_app.mk @@ -12,6 +12,7 @@ CROSS_PREFIX ?= riscv32-unknown-elf- TBDIR ?= ../tb_cxxrtl INCDIR ?= ../common MAX_CYCLES ?= 100000 +TMP_PREFIX ?= tmp/ ############################################################################### @@ -20,29 +21,30 @@ MAX_CYCLES ?= 100000 all: run -run: $(APP).bin - $(TBDIR)/tb $(APP).bin $(APP)_run.vcd --cycles $(MAX_CYCLES) +run: $(TMP_PREFIX)$(APP).bin + $(TBDIR)/tb $(TMP_PREFIX)$(APP).bin $(TMP_PREFIX)$(APP)_run.vcd --cycles $(MAX_CYCLES) view: run - gtkwave $(APP)_run.vcd + gtkwave $(TMP_PREFIX)$(APP)_run.vcd -bin: $(APP).bin +bin: $(TMP_PREFIX)$(APP).bin tb: $(MAKE) -C $(TBDIR) tb clean: - rm -f $(APP).elf $(APP).bin $(APP).dis $(APP)_run.vcd + rm -rf $(TMP_PREFIX) clean_tb: clean $(MAKE) -C $(TBDIR) clean ############################################################################### -$(APP).bin: $(APP).elf +$(TMP_PREFIX)$(APP).bin: $(TMP_PREFIX)$(APP).elf $(CROSS_PREFIX)objcopy -O binary $^ $@ - $(CROSS_PREFIX)objdump -h $(APP).elf > $(APP).dis - $(CROSS_PREFIX)objdump -d $(APP).elf >> $(APP).dis + $(CROSS_PREFIX)objdump -h $^ > $(TMP_PREFIX)$(APP).dis + $(CROSS_PREFIX)objdump -d $^ >> $(TMP_PREFIX)$(APP).dis -$(APP).elf: $(SRCS) $(wildcard %.h) - $(CROSS_PREFIX)gcc $(CCFLAGS) $(SRCS) -T $(LDSCRIPT) $(addprefix -I,$(INCDIR)) -o $(APP).elf +$(TMP_PREFIX)$(APP).elf: $(SRCS) $(wildcard %.h) + mkdir -p $(TMP_PREFIX) + $(CROSS_PREFIX)gcc $(CCFLAGS) $(SRCS) -T $(LDSCRIPT) $(addprefix -I,$(INCDIR)) -o $@ diff --git a/test/sim/common/tb_cxxrtl_io.h b/test/sim/common/tb_cxxrtl_io.h index 330b83a..c550ea9 100644 --- a/test/sim/common/tb_cxxrtl_io.h +++ b/test/sim/common/tb_cxxrtl_io.h @@ -55,4 +55,6 @@ static inline void tb_printf(const char *fmt, ...) { va_end(args); } +#define tb_assert(cond, ...) if (!(cond)) {tb_printf(__VA_ARGS__); tb_exit(-1);} + #endif diff --git a/test/sim/ecall_simple/Makefile b/test/sim/ecall_simple/Makefile deleted file mode 100644 index 9baf65a..0000000 --- a/test/sim/ecall_simple/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -SRCS := ../common/init.S main.c -APP := ecall_simple -CCFLAGS = -march=rv32ic - -include ../common/src_only_app.mk diff --git a/test/sim/lr_sc_smoke/Makefile b/test/sim/lr_sc_smoke/Makefile deleted file mode 100644 index c24a110..0000000 --- a/test/sim/lr_sc_smoke/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -SRCS := ../common/init.S main.c -APP := lr_sc_smoke -CCFLAGS = -march=rv32imac -Os - -include ../common/src_only_app.mk diff --git a/test/sim/sw_testcases/.gitignore b/test/sim/sw_testcases/.gitignore new file mode 100644 index 0000000..a9a5aec --- /dev/null +++ b/test/sim/sw_testcases/.gitignore @@ -0,0 +1 @@ +tmp diff --git a/test/sim/amo_smoke/Makefile b/test/sim/sw_testcases/Makefile similarity index 56% rename from test/sim/amo_smoke/Makefile rename to test/sim/sw_testcases/Makefile index 0c83f94..4c02243 100644 --- a/test/sim/amo_smoke/Makefile +++ b/test/sim/sw_testcases/Makefile @@ -1,5 +1,5 @@ -SRCS := ../common/init.S main.c -APP := amo_smoke +APP := hellow +SRCS := ../common/init.S $(APP).c CCFLAGS = -march=rv32imac -Os include ../common/src_only_app.mk diff --git a/test/sim/sw_testcases/Readme.md b/test/sim/sw_testcases/Readme.md new file mode 100644 index 0000000..f91b8b7 --- /dev/null +++ b/test/sim/sw_testcases/Readme.md @@ -0,0 +1,20 @@ +Software Testcases +================== + +A smorgasbord of software testcases for various features and cases that aren't well-covered by upstream tests such as `riscv-arch-test`, the `riscv-test` end-to-end debug tests or `riscv-formal`. Each test consists of one C file. + +Some tests have an expected text output associated with them -- the test passes if this text output matches, and `main()` exits with a zero return code. Other tests are completely self-checking, reporting pass/fail only with the return code from `main()`. This means there is _no point_ running these tests if the processor is in a fundamentally broken state (e.g. doesn't pass ISA compliance) and can't be trusted to check itself. + +To run the tests: + +```bash +./runtests +``` + +This will first rebuild the simulator (`../tb_cxxrtl/`) if needed, then build and run all the software testcases, then print out a summary of test pass/fail status. The `./run_tests` executable itself returns a successful exit code if and only if all tests passed. A VCD trace and printf log will be created for each test, with the same name as the test, for debugging failures. + +To clean up the junk: + +```bash +./cleantests +``` diff --git a/test/sim/amo_smoke/main.c b/test/sim/sw_testcases/amo_smoke.c similarity index 83% rename from test/sim/amo_smoke/main.c rename to test/sim/sw_testcases/amo_smoke.c index a871064..8e721f7 100644 --- a/test/sim/amo_smoke/main.c +++ b/test/sim/sw_testcases/amo_smoke.c @@ -3,8 +3,6 @@ volatile uint32_t scratch[2]; -#define test_assert(cond, ...) if (!(cond)) {tb_printf(__VA_ARGS__); return -1;} - int main() { scratch[0] = 0; diff --git a/test/sim/sw_testcases/amo_smoke.expected_output b/test/sim/sw_testcases/amo_smoke.expected_output new file mode 100644 index 0000000..382de62 --- /dev/null +++ b/test/sim/sw_testcases/amo_smoke.expected_output @@ -0,0 +1,12 @@ +Initial value: 0 +amoadd.w rd, 1, (&addr) -> fetched 0 +amoadd.w rd, 2, (&addr) -> fetched 1 +amoadd.w rd, 3, (&addr) -> fetched 3 +amoadd.w rd, 4, (&addr) -> fetched 6 +amoadd.w rd, 5, (&addr) -> fetched 10 +amoadd.w rd, 6, (&addr) -> fetched 15 +amoadd.w rd, 7, (&addr) -> fetched 21 +amoadd.w rd, 8, (&addr) -> fetched 28 +amoadd.w rd, 9, (&addr) -> fetched 36 +amoadd.w rd, 10, (&addr) -> fetched 45 +Final value: 55 diff --git a/test/sim/amo_smoke/amo_smoke.gtkw b/test/sim/sw_testcases/amo_smoke.gtkw similarity index 82% rename from test/sim/amo_smoke/amo_smoke.gtkw rename to test/sim/sw_testcases/amo_smoke.gtkw index 8c1c611..685ab9f 100644 --- a/test/sim/amo_smoke/amo_smoke.gtkw +++ b/test/sim/sw_testcases/amo_smoke.gtkw @@ -2,10 +2,9 @@ [*] GTKWave Analyzer v3.3.103 (w)1999-2019 BSI [*] Sat Dec 4 23:25:19 2021 [*] -[dumpfile] "/home/luke/proj/hazard3/test/sim/amo_smoke/amo_smoke_run.vcd" +[dumpfile] "tmp/amo_smoke_run.vcd" [dumpfile_mtime] "Sat Dec 4 23:21:36 2021" [dumpfile_size] 6246687 -[savefile] "/home/luke/proj/hazard3/test/sim/amo_smoke/amo_smoke.gtkw" [timestart] 420 [size] 1975 1095 [pos] -1 -1 diff --git a/test/sim/amo_timer_irq/main.c b/test/sim/sw_testcases/amo_timer_irq.c similarity index 74% rename from test/sim/amo_timer_irq/main.c rename to test/sim/sw_testcases/amo_timer_irq.c index 4534e3b..a81fb09 100644 --- a/test/sim/amo_timer_irq/main.c +++ b/test/sim/sw_testcases/amo_timer_irq.c @@ -7,8 +7,6 @@ #define IRQ_INTERVAL 201 #define N_AMOS 1000 -#define test_assert(cond, ...) if (!(cond)) {tb_printf(__VA_ARGS__); return -1;} - volatile uint32_t amo_count, irq_count; void __attribute__((interrupt)) isr_machine_timer() { @@ -32,14 +30,14 @@ int main() { : "=r" (fetch) : "r" (1), "r" (&amo_count) ); - test_assert(fetch == i, "Bad fetch, expected %u, got %u\n", i, fetch); + tb_assert(fetch == i, "Bad fetch, expected %u, got %u\n", i, fetch); } asm volatile ("csrci mstatus, 0x8"); uint32_t current_time = mm_timer->mtime; tb_printf("At time %u, received %u IRQs\n", current_time, irq_count); - test_assert(current_time / IRQ_INTERVAL + 1 == irq_count, "Bad IRQ count\n"); - test_assert(amo_count == N_AMOS, "Bad final AMO count %u\n", N_AMOS); + tb_assert(current_time / IRQ_INTERVAL + 1 == irq_count, "Bad IRQ count\n"); + tb_assert(amo_count == N_AMOS, "Bad final AMO count %u\n", N_AMOS); return 0; } diff --git a/test/sim/amo_timer_irq/amo_timer_irq.gtkw b/test/sim/sw_testcases/amo_timer_irq.gtkw similarity index 100% rename from test/sim/amo_timer_irq/amo_timer_irq.gtkw rename to test/sim/sw_testcases/amo_timer_irq.gtkw diff --git a/test/sim/sw_testcases/cleantests b/test/sim/sw_testcases/cleantests new file mode 100755 index 0000000..ce78ec2 --- /dev/null +++ b/test/sim/sw_testcases/cleantests @@ -0,0 +1,2 @@ +#!/bin/bash +rm -rf tmp diff --git a/test/sim/ecall_simple/main.c b/test/sim/sw_testcases/ecall_simple.c similarity index 93% rename from test/sim/ecall_simple/main.c rename to test/sim/sw_testcases/ecall_simple.c index d1ce642..2867f00 100644 --- a/test/sim/ecall_simple/main.c +++ b/test/sim/sw_testcases/ecall_simple.c @@ -5,7 +5,7 @@ #define read_csr(csrname) ({ \ uint32_t __csr_tmp_u32; \ - __asm__ ("csrr %0, " #csrname : "=r" (__csr_tmp_u32)); \ + asm volatile ("csrr %0, " #csrname : "=r" (__csr_tmp_u32)); \ __csr_tmp_u32; \ }) diff --git a/test/sim/sw_testcases/ecall_simple.expected_output b/test/sim/sw_testcases/ecall_simple.expected_output new file mode 100644 index 0000000..0e20eac --- /dev/null +++ b/test/sim/sw_testcases/ecall_simple.expected_output @@ -0,0 +1,11 @@ +mcause initial value: +00000000 +Handling ecall. Call number: +00000123 +Handling ecall. Call number: +00000456 +Handling ecall. Call number: +deadbeef +Finished making calls. +mcause final value: +0000000b diff --git a/test/sim/ecall_simple/ecall_simple_run.gtkw b/test/sim/sw_testcases/ecall_simple.gtkw similarity index 90% rename from test/sim/ecall_simple/ecall_simple_run.gtkw rename to test/sim/sw_testcases/ecall_simple.gtkw index ffee99e..e4b7a42 100644 --- a/test/sim/ecall_simple/ecall_simple_run.gtkw +++ b/test/sim/sw_testcases/ecall_simple.gtkw @@ -2,10 +2,9 @@ [*] GTKWave Analyzer v3.3.103 (w)1999-2019 BSI [*] Sat May 22 07:37:18 2021 [*] -[dumpfile] "/home/luke/proj/hazard3/test/ecall_simple/ecall_simple_run.vcd" +[dumpfile] "tmp/ecall_simple_run.vcd" [dumpfile_mtime] "Sat May 22 07:33:26 2021" [dumpfile_size] 1269546 -[savefile] "/home/luke/proj/hazard3/test/ecall_simple/ecall_simple_run.gtkw" [timestart] 314 [size] 2560 1403 [pos] -1 -1 diff --git a/test/sim/sw_testcases/hellow.c b/test/sim/sw_testcases/hellow.c new file mode 100644 index 0000000..18467e7 --- /dev/null +++ b/test/sim/sw_testcases/hellow.c @@ -0,0 +1,6 @@ +#include "tb_cxxrtl_io.h" + +int main() { + tb_puts("Hello world from Hazard3 + CXXRTL!\n"); + return 0; +} diff --git a/test/sim/sw_testcases/hellow.expected_output b/test/sim/sw_testcases/hellow.expected_output new file mode 100644 index 0000000..daefc0a --- /dev/null +++ b/test/sim/sw_testcases/hellow.expected_output @@ -0,0 +1 @@ +Hello world from Hazard3 + CXXRTL! diff --git a/test/sim/lr_sc_smoke/main.c b/test/sim/sw_testcases/lr_sc_smoke.c similarity index 100% rename from test/sim/lr_sc_smoke/main.c rename to test/sim/sw_testcases/lr_sc_smoke.c diff --git a/test/sim/lr_sc_smoke/lr_sc_smoke.gtkw b/test/sim/sw_testcases/lr_sc_smoke.gtkw similarity index 81% rename from test/sim/lr_sc_smoke/lr_sc_smoke.gtkw rename to test/sim/sw_testcases/lr_sc_smoke.gtkw index c668479..9c696bf 100644 --- a/test/sim/lr_sc_smoke/lr_sc_smoke.gtkw +++ b/test/sim/sw_testcases/lr_sc_smoke.gtkw @@ -2,10 +2,9 @@ [*] GTKWave Analyzer v3.3.103 (w)1999-2019 BSI [*] Sat Dec 4 14:31:51 2021 [*] -[dumpfile] "/home/luke/proj/hazard3/test/sim/lr_sc_smoke/lr_sc_smoke_run.vcd" +[dumpfile] "tmp/lr_sc_smoke_run.vcd" [dumpfile_mtime] "Sat Dec 4 14:20:04 2021" [dumpfile_size] 2577335 -[savefile] "/home/luke/proj/hazard3/test/sim/lr_sc_smoke/lr_sc_smoke.gtkw" [timestart] 842 [size] 1975 1095 [pos] -1 -1 diff --git a/test/sim/sw_testcases/runtests b/test/sim/sw_testcases/runtests new file mode 100755 index 0000000..2e6b72c --- /dev/null +++ b/test/sim/sw_testcases/runtests @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import sys + +testlist = [] + +for path in os.listdir(): + if os.path.isfile(path) and path.endswith(".c"): + testlist.append(path[:-2]) + +testlist = sorted(testlist) + +tb_build_ret = subprocess.run( + ["make", "-C", "../tb_cxxrtl", "tb"], + timeout=120 +) +if tb_build_ret.returncode != 0: + sys.exit("Failed.") + +all_passed = True + +for test in testlist: + sys.stdout.write(f"{test:<30}") + + test_build_ret = subprocess.run( + ["make", f"APP={test}", f"tmp/{test}.bin"], + stdout=subprocess.DEVNULL + ) + if test_build_ret.returncode != 0: + print("\033[33m[MK ERR]\033[39m") + all_passed = False + continue + + test_run_ret = subprocess.run( + ["../tb_cxxrtl/tb", f"tmp/{test}.bin", f"tmp/{test}_run.vcd", "--cycles", "1000000"], + stdout = subprocess.PIPE, + timeout=10 + ) + with open(f"tmp/{test}.log", "wb") as f: + f.write(test_run_ret.stdout) + + # Testbench itself should always exit successfully. + if test_run_ret.returncode != 0: + sys.exit("Negative return code from testbench!") + + # Pass if the program under test has zero exit code AND its output matches + # the expected output (if there is an expected_output file) + + output_lines = test_run_ret.stdout.decode("utf-8").strip().splitlines() + returncode = -1 + if len(output_lines) >= 2: + exit_line = output_lines[-2] + if exit_line.startswith("CPU requested halt"): + try: + returncode = int(exit_line.split(" ")[-1]) + except: + pass + if returncode != 0: + print("\033[31m[BADRET]\033[39m") + all_passed = False + continue + + if os.path.exists(f"{test}.expected_output"): + expected_lines = open(f"{test}.expected_output").read().strip().splitlines() + if expected_lines != output_lines[:-2]: + print("\033[31m[BADOUT]\033[39m") + all_passed = False + continue + + print("\033[32m[PASSED]\033[39m") + +sys.exit(not all_passed) + + diff --git a/test/sim/wfi_loop/main.c b/test/sim/sw_testcases/wfi_loop.c similarity index 100% rename from test/sim/wfi_loop/main.c rename to test/sim/sw_testcases/wfi_loop.c diff --git a/test/sim/sw_testcases/wfi_loop.expected_output b/test/sim/sw_testcases/wfi_loop.expected_output new file mode 100644 index 0000000..3194e19 --- /dev/null +++ b/test/sim/sw_testcases/wfi_loop.expected_output @@ -0,0 +1,12 @@ +Enabling IRQS... +IRQ 1 +IRQ 2 +IRQ 3 +IRQ 4 +IRQ 5 +IRQ 6 +IRQ 7 +IRQ 8 +IRQ 9 +IRQ 10 +Took 10 IRQs, span 9 times diff --git a/test/sim/tb_cxxrtl/Makefile b/test/sim/tb_cxxrtl/Makefile index 55bd6f6..ff6d9ac 100644 --- a/test/sim/tb_cxxrtl/Makefile +++ b/test/sim/tb_cxxrtl/Makefile @@ -16,7 +16,7 @@ MUL_FAST := 1 MULH_FAST := 1 REDUCED_BYPASS := 0 -.PHONY: clean tb all +.PHONY: clean all all: tb @@ -36,7 +36,7 @@ SYNTH_CMD += chparam -set MUL_FAST $(MUL_FAST) $(TOP); SYNTH_CMD += chparam -set MULH_FAST $(MULH_FAST) $(TOP); SYNTH_CMD += write_cxxrtl dut.cpp -dut.cpp: +dut.cpp: $(shell listfiles ../../../hdl/hazard3.f); yosys -p "$(SYNTH_CMD)" 2>&1 > cxxrtl.log clean:: diff --git a/test/sim/wfi_loop/Makefile b/test/sim/wfi_loop/Makefile deleted file mode 100644 index 054b352..0000000 --- a/test/sim/wfi_loop/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -SRCS := ../common/init.S main.c -APP := wfi_loop -CCFLAGS = -march=rv32imc -Os - -include ../common/src_only_app.mk