Add test script to make it easier to add software testcases

hdl/hazard3_core.v

@@ -863,6 +863,7 @@ always @ (*) begin
 		{MEMOP_LBU , 2'b10}: m_rdata_pick_sext = {{24{1'b0           }}, bus_rdata_d[23:16]};
 		{MEMOP_LBU , 2'b11}: m_rdata_pick_sext = {{24{1'b0           }}, bus_rdata_d[31:24]};
 		{MEMOP_LW  , 2'bzz}: m_rdata_pick_sext = bus_rdata_d;
+		{MEMOP_LR_W, 2'bzz}: m_rdata_pick_sext = bus_rdata_d;
 		default:             m_rdata_pick_sext = 32'hxxxx_xxxx;
 	endcase
 

test/sim/amo_timer_irq/Makefile

@@ -1,5 +0,0 @@
-SRCS := ../common/init.S main.c
-APP  := amo_timer_irq
-CCFLAGS = -march=rv32imac -Os
-
-include ../common/src_only_app.mk

test/sim/common/src_only_app.mk

@@ -12,6 +12,7 @@ CROSS_PREFIX ?= riscv32-unknown-elf-
 TBDIR        ?= ../tb_cxxrtl
 INCDIR       ?= ../common
 MAX_CYCLES   ?= 100000
+TMP_PREFIX   ?= tmp/
 
 ###############################################################################
 
@@ -20,29 +21,30 @@ MAX_CYCLES   ?= 100000
 
 all: run
 
-run: $(APP).bin
+run: $(TMP_PREFIX)$(APP).bin
-	$(TBDIR)/tb $(APP).bin $(APP)_run.vcd --cycles $(MAX_CYCLES)
+	$(TBDIR)/tb $(TMP_PREFIX)$(APP).bin $(TMP_PREFIX)$(APP)_run.vcd --cycles $(MAX_CYCLES)
 
 view: run
-	gtkwave $(APP)_run.vcd
+	gtkwave $(TMP_PREFIX)$(APP)_run.vcd
 
-bin: $(APP).bin
+bin: $(TMP_PREFIX)$(APP).bin
 
 tb:
 	$(MAKE) -C $(TBDIR) tb
 
 clean:
-	rm -f $(APP).elf $(APP).bin $(APP).dis $(APP)_run.vcd
+	rm -rf $(TMP_PREFIX)
 
 clean_tb: clean
 	$(MAKE) -C $(TBDIR) clean
 
 ###############################################################################
 
-$(APP).bin: $(APP).elf
+$(TMP_PREFIX)$(APP).bin: $(TMP_PREFIX)$(APP).elf
 	$(CROSS_PREFIX)objcopy -O binary $^ $@
-	$(CROSS_PREFIX)objdump -h $(APP).elf > $(APP).dis
+	$(CROSS_PREFIX)objdump -h $^ > $(TMP_PREFIX)$(APP).dis
-	$(CROSS_PREFIX)objdump -d $(APP).elf >> $(APP).dis
+	$(CROSS_PREFIX)objdump -d $^ >> $(TMP_PREFIX)$(APP).dis
 
-$(APP).elf: $(SRCS) $(wildcard %.h)
+$(TMP_PREFIX)$(APP).elf: $(SRCS) $(wildcard %.h)
-	$(CROSS_PREFIX)gcc $(CCFLAGS) $(SRCS) -T $(LDSCRIPT) $(addprefix -I,$(INCDIR)) -o $(APP).elf
+	mkdir -p $(TMP_PREFIX)
+	$(CROSS_PREFIX)gcc $(CCFLAGS) $(SRCS) -T $(LDSCRIPT) $(addprefix -I,$(INCDIR)) -o $@

test/sim/common/tb_cxxrtl_io.h

@@ -55,4 +55,6 @@ static inline void tb_printf(const char *fmt, ...) {
 	va_end(args);
 }
 
+#define tb_assert(cond, ...) if (!(cond)) {tb_printf(__VA_ARGS__); tb_exit(-1);}
+
 #endif

test/sim/ecall_simple/Makefile

@@ -1,5 +0,0 @@
-SRCS := ../common/init.S main.c
-APP  := ecall_simple
-CCFLAGS = -march=rv32ic
-
-include ../common/src_only_app.mk

test/sim/lr_sc_smoke/Makefile

@@ -1,5 +0,0 @@
-SRCS := ../common/init.S main.c
-APP  := lr_sc_smoke
-CCFLAGS = -march=rv32imac -Os
-
-include ../common/src_only_app.mk

test/sim/sw_testcases/.gitignore

@@ -0,0 +1 @@
+tmp

test/sim/sw_testcases/Makefile

@@ -1,5 +1,5 @@
-SRCS := ../common/init.S main.c
+APP  := hellow
-APP  := amo_smoke
+SRCS := ../common/init.S $(APP).c
 CCFLAGS = -march=rv32imac -Os
 
 include ../common/src_only_app.mk

test/sim/sw_testcases/Readme.md

@@ -0,0 +1,20 @@
+Software Testcases
+==================
+
+A smorgasbord of software testcases for various features and cases that aren't well-covered by upstream tests such as `riscv-arch-test`, the `riscv-test` end-to-end debug tests or `riscv-formal`. Each test consists of one C file.
+
+Some tests have an expected text output associated with them -- the test passes if this text output matches, and `main()` exits with a zero return code. Other tests are completely self-checking, reporting pass/fail only with the return code from `main()`. This means there is _no point_ running these tests if the processor is in a fundamentally broken state (e.g. doesn't pass ISA compliance) and can't be trusted to check itself. 
+
+To run the tests:
+
+```bash
+./runtests
+```
+
+This will first rebuild the simulator (`../tb_cxxrtl/`) if needed, then build and run all the software testcases, then print out a summary of test pass/fail status. The `./run_tests` executable itself returns a successful exit code if and only if all tests passed. A VCD trace and printf log will be created for each test, with the same name as the test, for debugging failures.
+
+To clean up the junk:
+
+```bash
+./cleantests
+```

test/sim/sw_testcases/amo_smoke.c

@@ -3,8 +3,6 @@
 
 volatile uint32_t scratch[2];
 
-#define test_assert(cond, ...) if (!(cond)) {tb_printf(__VA_ARGS__); return -1;}
-
 int main() {
 
 	scratch[0] = 0;

test/sim/sw_testcases/amo_smoke.expected_output

@@ -0,0 +1,12 @@
+Initial value: 0
+amoadd.w rd, 1, (&addr) -> fetched 0
+amoadd.w rd, 2, (&addr) -> fetched 1
+amoadd.w rd, 3, (&addr) -> fetched 3
+amoadd.w rd, 4, (&addr) -> fetched 6
+amoadd.w rd, 5, (&addr) -> fetched 10
+amoadd.w rd, 6, (&addr) -> fetched 15
+amoadd.w rd, 7, (&addr) -> fetched 21
+amoadd.w rd, 8, (&addr) -> fetched 28
+amoadd.w rd, 9, (&addr) -> fetched 36
+amoadd.w rd, 10, (&addr) -> fetched 45
+Final value: 55

test/sim/sw_testcases/amo_smoke.gtkw

@@ -2,10 +2,9 @@
 [*] GTKWave Analyzer v3.3.103 (w)1999-2019 BSI
 [*] Sat Dec  4 23:25:19 2021
 [*]
-[dumpfile] "/home/luke/proj/hazard3/test/sim/amo_smoke/amo_smoke_run.vcd"
+[dumpfile] "tmp/amo_smoke_run.vcd"
 [dumpfile_mtime] "Sat Dec  4 23:21:36 2021"
 [dumpfile_size] 6246687
-[savefile] "/home/luke/proj/hazard3/test/sim/amo_smoke/amo_smoke.gtkw"
 [timestart] 420
 [size] 1975 1095
 [pos] -1 -1

test/sim/sw_testcases/amo_timer_irq.c

@@ -7,8 +7,6 @@
 #define IRQ_INTERVAL 201
 #define N_AMOS 1000
 
-#define test_assert(cond, ...) if (!(cond)) {tb_printf(__VA_ARGS__); return -1;}
-
 volatile uint32_t amo_count, irq_count;
 
 void __attribute__((interrupt)) isr_machine_timer() {
@@ -32,14 +30,14 @@ int main() {
 			: "=r" (fetch)
 			: "r" (1), "r" (&amo_count)
 		);
-		test_assert(fetch == i, "Bad fetch, expected %u, got %u\n", i, fetch);
+		tb_assert(fetch == i, "Bad fetch, expected %u, got %u\n", i, fetch);
 	}
 
 	asm volatile ("csrci mstatus, 0x8");
 	uint32_t current_time = mm_timer->mtime;
 	tb_printf("At time %u, received %u IRQs\n", current_time, irq_count);
-	test_assert(current_time / IRQ_INTERVAL + 1 == irq_count, "Bad IRQ count\n");
+	tb_assert(current_time / IRQ_INTERVAL + 1 == irq_count, "Bad IRQ count\n");
-	test_assert(amo_count == N_AMOS, "Bad final AMO count %u\n", N_AMOS);
+	tb_assert(amo_count == N_AMOS, "Bad final AMO count %u\n", N_AMOS);
 
 	return 0;
 }

test/sim/sw_testcases/cleantests

@@ -0,0 +1,2 @@
+#!/bin/bash
+rm -rf tmp

test/sim/sw_testcases/ecall_simple.c

@@ -5,7 +5,7 @@
 
 #define read_csr(csrname) ({ \
   uint32_t __csr_tmp_u32; \
-  __asm__ ("csrr %0, " #csrname : "=r" (__csr_tmp_u32)); \
+  asm volatile ("csrr %0, " #csrname : "=r" (__csr_tmp_u32)); \
   __csr_tmp_u32; \
 })
 

test/sim/sw_testcases/ecall_simple.expected_output

@@ -0,0 +1,11 @@
+mcause initial value:
+00000000
+Handling ecall. Call number:
+00000123
+Handling ecall. Call number:
+00000456
+Handling ecall. Call number:
+deadbeef
+Finished making calls.
+mcause final value:
+0000000b

test/sim/sw_testcases/ecall_simple.gtkw

@@ -2,10 +2,9 @@
 [*] GTKWave Analyzer v3.3.103 (w)1999-2019 BSI
 [*] Sat May 22 07:37:18 2021
 [*]
-[dumpfile] "/home/luke/proj/hazard3/test/ecall_simple/ecall_simple_run.vcd"
+[dumpfile] "tmp/ecall_simple_run.vcd"
 [dumpfile_mtime] "Sat May 22 07:33:26 2021"
 [dumpfile_size] 1269546
-[savefile] "/home/luke/proj/hazard3/test/ecall_simple/ecall_simple_run.gtkw"
 [timestart] 314
 [size] 2560 1403
 [pos] -1 -1

test/sim/sw_testcases/hellow.c

@@ -0,0 +1,6 @@
+#include "tb_cxxrtl_io.h"
+
+int main() {
+	tb_puts("Hello world from Hazard3 + CXXRTL!\n");
+	return 0;
+}

test/sim/sw_testcases/hellow.expected_output

@@ -0,0 +1 @@
+Hello world from Hazard3 + CXXRTL!

test/sim/sw_testcases/lr_sc_smoke.gtkw

@@ -2,10 +2,9 @@
 [*] GTKWave Analyzer v3.3.103 (w)1999-2019 BSI
 [*] Sat Dec  4 14:31:51 2021
 [*]
-[dumpfile] "/home/luke/proj/hazard3/test/sim/lr_sc_smoke/lr_sc_smoke_run.vcd"
+[dumpfile] "tmp/lr_sc_smoke_run.vcd"
 [dumpfile_mtime] "Sat Dec  4 14:20:04 2021"
 [dumpfile_size] 2577335
-[savefile] "/home/luke/proj/hazard3/test/sim/lr_sc_smoke/lr_sc_smoke.gtkw"
 [timestart] 842
 [size] 1975 1095
 [pos] -1 -1

test/sim/sw_testcases/runtests

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import sys
+
+testlist = []
+
+for path in os.listdir():
+	if os.path.isfile(path) and path.endswith(".c"):
+		testlist.append(path[:-2])
+
+testlist = sorted(testlist)
+
+tb_build_ret = subprocess.run(
+	["make", "-C", "../tb_cxxrtl", "tb"],
+	timeout=120
+)
+if tb_build_ret.returncode != 0:
+	sys.exit("Failed.")
+
+all_passed = True
+
+for test in testlist:
+	sys.stdout.write(f"{test:<30}")
+
+	test_build_ret = subprocess.run(
+		["make", f"APP={test}", f"tmp/{test}.bin"],
+		stdout=subprocess.DEVNULL
+	)
+	if test_build_ret.returncode != 0:
+		print("\033[33m[MK ERR]\033[39m")
+		all_passed = False
+		continue
+
+	test_run_ret = subprocess.run(
+		["../tb_cxxrtl/tb", f"tmp/{test}.bin", f"tmp/{test}_run.vcd", "--cycles", "1000000"],
+		stdout = subprocess.PIPE,
+		timeout=10
+	)
+	with open(f"tmp/{test}.log", "wb") as f:
+		f.write(test_run_ret.stdout)
+
+	# Testbench itself should always exit successfully.
+	if test_run_ret.returncode != 0:
+		sys.exit("Negative return code from testbench!")
+
+	# Pass if the program under test has zero exit code AND its output matches
+	# the expected output (if there is an expected_output file)
+
+	output_lines = test_run_ret.stdout.decode("utf-8").strip().splitlines()
+	returncode = -1
+	if len(output_lines) >= 2:
+		exit_line = output_lines[-2]
+		if exit_line.startswith("CPU requested halt"):
+			try:
+				returncode = int(exit_line.split(" ")[-1])
+			except:
+				pass
+	if returncode != 0:
+		print("\033[31m[BADRET]\033[39m")
+		all_passed = False
+		continue
+
+	if os.path.exists(f"{test}.expected_output"):
+		expected_lines = open(f"{test}.expected_output").read().strip().splitlines()
+		if expected_lines != output_lines[:-2]:
+			print("\033[31m[BADOUT]\033[39m")
+			all_passed = False
+			continue
+
+	print("\033[32m[PASSED]\033[39m")
+
+sys.exit(not all_passed)
+
+

test/sim/sw_testcases/wfi_loop.expected_output

@@ -0,0 +1,12 @@
+Enabling IRQS...
+IRQ 1
+IRQ 2
+IRQ 3
+IRQ 4
+IRQ 5
+IRQ 6
+IRQ 7
+IRQ 8
+IRQ 9
+IRQ 10
+Took 10 IRQs, span 9 times

test/sim/tb_cxxrtl/Makefile

@@ -16,7 +16,7 @@ MUL_FAST         := 1
 MULH_FAST        := 1
 REDUCED_BYPASS   := 0
 
-.PHONY: clean tb all
+.PHONY: clean all
 
 all: tb
 
@@ -36,7 +36,7 @@ SYNTH_CMD += chparam -set MUL_FAST $(MUL_FAST) $(TOP);
 SYNTH_CMD += chparam -set MULH_FAST $(MULH_FAST) $(TOP);
 SYNTH_CMD += write_cxxrtl dut.cpp
 
-dut.cpp:
+dut.cpp: $(shell listfiles ../../../hdl/hazard3.f);
 	yosys -p "$(SYNTH_CMD)" 2>&1 > cxxrtl.log
 
 clean::

test/sim/wfi_loop/Makefile

@@ -1,5 +0,0 @@
-SRCS := ../common/init.S main.c
-APP  := wfi_loop
-CCFLAGS = -march=rv32imc -Os
-
-include ../common/src_only_app.mk