Move expected_output into tests inline

test/sim/sw_testcases/Readme.md

@@ -3,7 +3,26 @@ Software Testcases
 
 A smorgasbord of software testcases for various features and cases that aren't well-covered by upstream tests such as `riscv-arch-test`, the `riscv-test` end-to-end debug tests or `riscv-formal`. Each test consists of one C file.
 
-Some tests have an expected text output associated with them -- the test passes if this text output matches, and `main()` exits with a zero return code. Other tests are completely self-checking, reporting pass/fail only with the return code from `main()`. This means there is _no point_ running these tests if the processor is in a fundamentally broken state (e.g. doesn't pass ISA compliance) and can't be trusted to check itself. 
+Some tests have an expected text output associated with them -- the test passes if this text output matches, and `main()` exits with a zero return code. Other tests are completely self-checking, reporting pass/fail only with the return code from `main()`. This means there is _no point_ running these tests if the processor is in a fundamentally broken state (e.g. doesn't pass ISA compliance) and can't be trusted to check itself.
+
+For example, `hellow.c`:
+
+```c
+#include "tb_cxxrtl_io.h"
+
+/*EXPECTED-OUTPUT***************************************************************
+
+Hello world from Hazard3 + CXXRTL!
+
+*******************************************************************************/
+
+int main() {
+	tb_puts("Hello world from Hazard3 + CXXRTL!\n");
+	return 0;
+}
+```
+
+The contents of the `EXPECTED-OUTPUT` comment is simply compared with the logged text from `tb_puts`, `tb_printf` etc. Tests might log a range of output here, such as `mcause` values in exceptions.
 
 To run the tests:
 

test/sim/sw_testcases/amo_smoke.c

@@ -1,6 +1,23 @@
 #include "tb_cxxrtl_io.h"
 #include <stdint.h>
 
+/*EXPECTED-OUTPUT***************************************************************
+
+Initial value: 0
+amoadd.w rd, 1, (&addr) -> fetched 0
+amoadd.w rd, 2, (&addr) -> fetched 1
+amoadd.w rd, 3, (&addr) -> fetched 3
+amoadd.w rd, 4, (&addr) -> fetched 6
+amoadd.w rd, 5, (&addr) -> fetched 10
+amoadd.w rd, 6, (&addr) -> fetched 15
+amoadd.w rd, 7, (&addr) -> fetched 21
+amoadd.w rd, 8, (&addr) -> fetched 28
+amoadd.w rd, 9, (&addr) -> fetched 36
+amoadd.w rd, 10, (&addr) -> fetched 45
+Final value: 55
+
+*******************************************************************************/
+
 volatile uint32_t scratch[2];
 
 int main() {

test/sim/sw_testcases/amo_smoke.expected_output

@@ -1,12 +0,0 @@
-Initial value: 0
-amoadd.w rd, 1, (&addr) -> fetched 0
-amoadd.w rd, 2, (&addr) -> fetched 1
-amoadd.w rd, 3, (&addr) -> fetched 3
-amoadd.w rd, 4, (&addr) -> fetched 6
-amoadd.w rd, 5, (&addr) -> fetched 10
-amoadd.w rd, 6, (&addr) -> fetched 15
-amoadd.w rd, 7, (&addr) -> fetched 21
-amoadd.w rd, 8, (&addr) -> fetched 28
-amoadd.w rd, 9, (&addr) -> fetched 36
-amoadd.w rd, 10, (&addr) -> fetched 45
-Final value: 55

test/sim/sw_testcases/csr_id.c

@@ -4,6 +4,17 @@
 // This is a new CSR for priv-1.12. Most compilers (maybe binutils?) don't know about it.
 #define mconfigptr 0xf15
 
+/*EXPECTED-OUTPUT***************************************************************
+
+mvendorid:  deadbeef
+marchid:    0000001b
+mimpid:     12345678
+mhartid:    00000000
+mconfigptr: 9abcdef0
+misa:       40801105
+
+*******************************************************************************/
+
 int main() {
 	// Expected value: 32'hdeadbeef, set in tb Makefile
 	tb_printf("mvendorid:  %08x\n", read_csr(mvendorid ));

test/sim/sw_testcases/csr_id.expected_output

@@ -1,6 +0,0 @@
-mvendorid:  deadbeef
-marchid:    0000001b
-mimpid:     12345678
-mhartid:    00000000
-mconfigptr: 9abcdef0
-misa:       40801105

test/sim/sw_testcases/ebreak.c

@@ -1,6 +1,21 @@
 #include "tb_cxxrtl_io.h"
 #include "hazard3_csr.h"
 
+/*EXPECTED-OUTPUT***************************************************************
+
+Entering test section
+mcause = 3
+Offset into test: 0, 32-bit ebreak
+mcause = 3
+Offset into test: 4, 16-bit ebreak
+mcause = 3
+Offset into test: 6, 32-bit ebreak
+mcause = 3
+Offset into test: 10, 16-bit ebreak
+Done
+
+*******************************************************************************/
+
 // This is naked so we can take its address and get accurate offsets for the
 // breakpoints, which we can then check. Otherwise, we would have issues with
 // the size of the prologue potentially varying between builds etc.

test/sim/sw_testcases/ebreak.expected_output

@@ -1,10 +0,0 @@
-Entering test section
-mcause = 3
-Offset into test: 0, 32-bit ebreak
-mcause = 3
-Offset into test: 4, 16-bit ebreak
-mcause = 3
-Offset into test: 6, 32-bit ebreak
-mcause = 3
-Offset into test: 10, 16-bit ebreak
-Done

test/sim/sw_testcases/ecall_simple.c

@@ -3,6 +3,22 @@
 
 #include <stdint.h>
 
+/*EXPECTED-OUTPUT***************************************************************
+
+mcause initial value:
+00000000
+Handling ecall. Call number:
+00000123
+Handling ecall. Call number:
+00000456
+Handling ecall. Call number:
+deadbeef
+Finished making calls.
+mcause final value:
+0000000b
+
+*******************************************************************************/
+
 void __attribute__((interrupt)) handle_exception() {
 	uint32_t call_num;
 	asm volatile ("mv %0, a7" : "=r" (call_num));

test/sim/sw_testcases/ecall_simple.expected_output

@@ -1,11 +0,0 @@
-mcause initial value:
-00000000
-Handling ecall. Call number:
-00000123
-Handling ecall. Call number:
-00000456
-Handling ecall. Call number:
-deadbeef
-Finished making calls.
-mcause final value:
-0000000b

test/sim/sw_testcases/hellow.c

@@ -1,5 +1,11 @@
 #include "tb_cxxrtl_io.h"
 
+/*EXPECTED-OUTPUT***************************************************************
+
+Hello world from Hazard3 + CXXRTL!
+
+*******************************************************************************/
+
 int main() {
 	tb_puts("Hello world from Hazard3 + CXXRTL!\n");
 	return 0;

test/sim/sw_testcases/hellow.expected_output

@@ -1 +0,0 @@
-Hello world from Hazard3 + CXXRTL!

test/sim/sw_testcases/illegal_instr.c

@@ -1,6 +1,26 @@
 #include "tb_cxxrtl_io.h"
 #include "hazard3_csr.h"
 
+/*EXPECTED-OUTPUT***************************************************************
+
+1: defined illegal all-zeroes
+Exception, mcause = 2
+16-bit illegal instruction: 0000
+2: defined illegal all-ones
+Exception, mcause = 2
+32-bit illegal instruction: ffffffff
+3: unimplemented CSR 0xfff
+Exception, mcause = 2
+32-bit illegal instruction: fff027f3
+4: write to read-only CSR
+Exception, mcause = 2
+32-bit illegal instruction: f1101073
+5: unimplemented instruction (F extension)
+Exception, mcause = 2
+32-bit illegal instruction: 00052087
+
+*******************************************************************************/
+
 int main() {
 	tb_puts("1: defined illegal all-zeroes\n");
 	asm volatile (".hword 0x0000");

test/sim/sw_testcases/illegal_instr.expected_output

@@ -1,15 +0,0 @@
-1: defined illegal all-zeroes
-Exception, mcause = 2
-16-bit illegal instruction: 0000
-2: defined illegal all-ones
-Exception, mcause = 2
-32-bit illegal instruction: ffffffff
-3: unimplemented CSR 0xfff
-Exception, mcause = 2
-32-bit illegal instruction: fff027f3
-4: write to read-only CSR
-Exception, mcause = 2
-32-bit illegal instruction: f1101073
-5: unimplemented instruction (F extension)
-Exception, mcause = 2
-32-bit illegal instruction: 00052087

test/sim/sw_testcases/instr_access_fault.c

@@ -3,6 +3,13 @@
 
 #include <stdint.h>
 
+/*EXPECTED-OUTPUT***************************************************************
+
+mcause = 1
+mepc = 56789abc
+
+*******************************************************************************/
+
 int main() {
 	uintptr_t illegal_addr = 0x56789abc;
 	asm volatile ("jr %0" : : "r" (illegal_addr));

test/sim/sw_testcases/instr_access_fault.expected_output

@@ -1,2 +0,0 @@
-mcause = 1
-mepc = 56789abc

test/sim/sw_testcases/load_misalign_halfword.c

@@ -1,6 +1,29 @@
 #include "tb_cxxrtl_io.h"
 #include "hazard3_csr.h"
 
+/*EXPECTED-OUTPUT***************************************************************
+
+Load halfword signed, 1 byte offset
+-> exception, mcause = 4
+Result: 00000000
+Load halfword signed, 3 byte offset
+-> exception, mcause = 4
+Result: 00000000
+Load halfword signed aligned (sanity check)
+Result: ffffcdef
+Result: 00001234
+Load halfword unsigned, 1 byte offset
+-> exception, mcause = 4
+Result: 00000000
+Load halfword unsigned, 3 byte offset
+-> exception, mcause = 4
+Result: 00000000
+Load halfword unsigned aligned (sanity check)
+Result: 0000cdef
+Result: 00001234
+
+*******************************************************************************/
+
 int main() {
 	volatile uint32_t target_word = 0x1234cdefu;
 	volatile uint32_t result_word = 0;

test/sim/sw_testcases/load_misalign_halfword.expected_output

@@ -1,18 +0,0 @@
-Load halfword signed, 1 byte offset
--> exception, mcause = 4
-Result: 00000000
-Load halfword signed, 3 byte offset
--> exception, mcause = 4
-Result: 00000000
-Load halfword signed aligned (sanity check)
-Result: ffffcdef
-Result: 00001234
-Load halfword unsigned, 1 byte offset
--> exception, mcause = 4
-Result: 00000000
-Load halfword unsigned, 3 byte offset
--> exception, mcause = 4
-Result: 00000000
-Load halfword unsigned aligned (sanity check)
-Result: 0000cdef
-Result: 00001234

test/sim/sw_testcases/load_misalign_word.c

@@ -1,6 +1,22 @@
 #include "tb_cxxrtl_io.h"
 #include "hazard3_csr.h"
 
+/*EXPECTED-OUTPUT***************************************************************
+
+Load word, 1 byte offset
+-> exception, mcause = 4
+Result: 00000000
+Load word, 2 byte offset
+-> exception, mcause = 4
+Result: 00000000
+Load word, 3 byte offset
+-> exception, mcause = 4
+Result: 00000000
+Load word aligned (sanity check)
+Result: ffffffff
+
+*******************************************************************************/
+
 int main() {
 	volatile uint32_t target_word = -1u;
 	volatile uint32_t result_word = 0;

test/sim/sw_testcases/load_misalign_word.expected_output

@@ -1,11 +0,0 @@
-Load word, 1 byte offset
--> exception, mcause = 4
-Result: 00000000
-Load word, 2 byte offset
--> exception, mcause = 4
-Result: 00000000
-Load word, 3 byte offset
--> exception, mcause = 4
-Result: 00000000
-Load word aligned (sanity check)
-Result: ffffffff

test/sim/sw_testcases/load_store_fault.c

@@ -4,6 +4,28 @@
 // Check load/stores which generate a bus fault generate an exception, and
 // report the correct mcause and mepc.
 
+/*EXPECTED-OUTPUT***************************************************************
+
+-> exception, mcause = 7
+exception instr: 0007a023
+-> exception, mcause = 7
+exception instr: 00079023
+-> exception, mcause = 7
+exception instr: 00078023
+-> exception, mcause = 5
+exception instr: 0007a003
+-> exception, mcause = 5
+exception instr: 00079003
+-> exception, mcause = 5
+exception instr: 0007d003
+-> exception, mcause = 5
+exception instr: 00078003
+-> exception, mcause = 5
+exception instr: 0007c003
+Done.
+
+*******************************************************************************/
+
 int main() {
 	// Word-aligned address which generates an access fault. Constrained to a
 	// particular register, because the instructions appear in the test log to

test/sim/sw_testcases/load_store_fault.expected_output

@@ -1,17 +0,0 @@
--> exception, mcause = 7
-exception instr: 0007a023
--> exception, mcause = 7
-exception instr: 00079023
--> exception, mcause = 7
-exception instr: 00078023
--> exception, mcause = 5
-exception instr: 0007a003
--> exception, mcause = 5
-exception instr: 00079003
--> exception, mcause = 5
-exception instr: 0007d003
--> exception, mcause = 5
-exception instr: 00078003
--> exception, mcause = 5
-exception instr: 0007c003
-Done.

test/sim/sw_testcases/lr_sc_fault.c

@@ -1,10 +1,27 @@
 #include "tb_cxxrtl_io.h"
 #include "hazard3_csr.h"
 
-// Check lr/sc which generate a bus fault generate an exception, and
+// Check lr/sc which encounter bus faults generate exceptions, and report the
-// report the correct mcause and mepc.
+// correct mcause and mepc.
 
-// Calling convention abuse to get stable register allocation without cursed register keyword
+/*EXPECTED-OUTPUT***************************************************************
+
+Failed load, suppressed store
+-> exception, mcause = 5
+exception instr: 100627af
+sc.w result: 0
+Good load, failed store
+-> exception, mcause = 7
+exception instr: 18a5a52f
+sc.w result: 123
+Repeated failed store
+sc.w result: 0
+
+*******************************************************************************/
+
+// Calling convention abuse to get stable register allocation without cursed
+// register keyword. We need stable registers because the excepting
+// instructions are in the test log.
 uint32_t __attribute__((naked)) do_lr_sc(uint32_t initial_sc, uint32_t *dst, const uint32_t *src) {
 	asm volatile (
 		// a5 used as a dumpster

test/sim/sw_testcases/lr_sc_fault.expected_output

@@ -1,10 +0,0 @@
-Failed load, suppressed store
--> exception, mcause = 5
-exception instr: 100627af
-sc.w result: 0
-Good load, failed store
--> exception, mcause = 7
-exception instr: 18a5a52f
-sc.w result: 123
-Repeated failed store
-sc.w result: 0

test/sim/sw_testcases/lr_sc_smoke.c

@@ -1,9 +1,20 @@
 #include "tb_cxxrtl_io.h"
 #include <stdint.h>
 
-volatile uint32_t scratch[2];
+/*EXPECTED-OUTPUT***************************************************************
 
-#define test_assert(cond, ...) if (!(cond)) {tb_printf(__VA_ARGS__); return -1;}
+Test 1: lr.w -> nop -> sc.w
+OK
+Test 2: lr.w -> sc.w
+OK
+Test 3: sc.w with no preceding lr.w
+OK
+Test 4: lr.w -> sw -> sc.w
+OK
+
+*******************************************************************************/
+
+volatile uint32_t scratch[2];
 
 int main() {
 	uint32_t load_result, success;
@@ -13,12 +24,12 @@ int main() {
 		"lr.w %0, (%2)\n"
 		"nop\n"
 		"sc.w %1, %3, (%2)\n"
-		: "=r" (load_result), "=r" (success)
+		: "=&r" (load_result), "=r" (success)
 		: "r" (&scratch[0]), "r" (0x5678)
 	);
-	test_assert(load_result == 0x1234, "Bad load result %08x\n", load_result);
+	tb_assert(load_result == 0x1234, "Bad load result %08x\n", load_result);
-	test_assert(scratch[0] == 0x5678, "Store didn't write memory\n");
+	tb_assert(scratch[0] == 0x5678, "Store didn't write memory\n");
-	test_assert(success == 1, "Should report success\n");
+	tb_assert(success == 1, "Should report success\n");
 	tb_puts("OK\n");
 
 	tb_puts("Test 2: lr.w -> sc.w\n");
@@ -26,12 +37,12 @@ int main() {
 	asm volatile (
 		"lr.w %0, (%2)\n"
 		"sc.w %1, %3, (%2)\n"
-		: "=r" (load_result), "=r" (success)
+		: "=&r" (load_result), "=r" (success)
 		: "r" (&scratch[0]), "r" (0xa5a5)
 	);
-	test_assert(load_result == 0xabcd, "Bad load result %08x\n", load_result);
+	tb_assert(load_result == 0xabcd, "Bad load result %08x\n", load_result);
-	test_assert(scratch[0] == 0xa5a5, "Store didn't write memory\n");
+	tb_assert(scratch[0] == 0xa5a5, "Store didn't write memory\n");
-	test_assert(success == 1, "Should report success\n");
+	tb_assert(success == 1, "Should report success\n");
 	tb_puts("OK\n");
 
 	tb_puts("Test 3: sc.w with no preceding lr.w\n");
@@ -41,8 +52,8 @@ int main() {
 		: "=r" (success)
 		: "r" (&scratch[0]), "r" (0x5678)
 	);
-	test_assert(scratch[0] == 0x1234, "Store shouldn't write memory\n");
+	tb_assert(scratch[0] == 0x1234, "Store shouldn't write memory\n");
-	test_assert(success == 0, "Should report failure\n");
+	tb_assert(success == 0, "Should report failure\n");
 	tb_puts("OK\n");
 
 	// Reservation is only cleared by other harts' stores.
@@ -53,12 +64,12 @@ int main() {
 		"lr.w %0, (%2)\n"
 		"sw %3, 4(%2)\n"
 		"sc.w %1, %4, (%2)\n"
-		: "=r" (load_result), "=r" (success)
+		: "=&r" (load_result), "=r" (success)
 		: "r" (&scratch[0]), "r" (0xabcd), "r" (0x5678)
 	);
-	test_assert(scratch[1] == 0xabcd, "Regular store should succeed\n");
+	tb_assert(scratch[1] == 0xabcd, "Regular store should succeed\n");
-	test_assert(scratch[0] == 0x5678, "sc didn't write memory\n");
+	tb_assert(scratch[0] == 0x5678, "sc didn't write memory\n");
-	test_assert(success == 1, "Should report success\n");
+	tb_assert(success == 1, "Should report success\n");
 	tb_puts("OK\n");
 
 	return 0;

test/sim/sw_testcases/runtests

@@ -62,9 +62,22 @@ for test in testlist:
 		all_passed = False
 		continue
 
-	if os.path.exists(f"{test}.expected_output"):
+	test_src = open(f"{test}.c").read()
-		expected_lines = open(f"{test}.expected_output").read().strip().splitlines()
+	if "/*EXPECTED-OUTPUT" in test_src:
-		if expected_lines != output_lines[:-2]:
+		good_output = True
+		try:
+			expected_start = test_src.find("/*EXPECTED-OUTPUT")
+			expected_end = test_src.find("*/", expected_start)
+			expected_lines = test_src[expected_start:expected_end + 1].splitlines()[1:-1]
+			while expected_lines[0].strip() == "":
+				del expected_lines[0]
+			while expected_lines[-1].strip() == "":
+				del expected_lines[-1]
+			if expected_lines != output_lines[:-2]:
+				good_output = False
+		except:
+			good_output = False
+		if not good_output:
 			print("\033[31m[BADOUT]\033[39m")
 			all_passed = False
 			continue

test/sim/sw_testcases/store_misalign_halfword.c

@@ -1,6 +1,22 @@
 #include "tb_cxxrtl_io.h"
 #include "hazard3_csr.h"
 
+// Check misaligned halfword stores generate exception with correct mcause
+
+/*EXPECTED-OUTPUT***************************************************************
+
+Store halfword, 1 byte offset
+-> exception, mcause = 6
+Target value: ffffffff
+Store halfword, 3 byte offset
+-> exception, mcause = 6
+Target value: ffffffff
+Aligned store halfword, sanity check
+Target value: ffff0000
+Target value: 00000000
+
+*******************************************************************************/
+
 int main() {
 	volatile uint32_t target_word = -1u;
 	tb_puts("Store halfword, 1 byte offset\n");

test/sim/sw_testcases/store_misalign_halfword.expected_output

@@ -1,9 +0,0 @@
-Store halfword, 1 byte offset
--> exception, mcause = 6
-Target value: ffffffff
-Store halfword, 3 byte offset
--> exception, mcause = 6
-Target value: ffffffff
-Aligned store halfword, sanity check
-Target value: ffff0000
-Target value: 00000000

test/sim/sw_testcases/store_misalign_word.c

@@ -1,6 +1,24 @@
 #include "tb_cxxrtl_io.h"
 #include "hazard3_csr.h"
 
+// Check misaligned word stores generate exception with correct mcause
+
+/*EXPECTED-OUTPUT***************************************************************
+
+Store word, 1 byte offset
+-> exception, mcause = 6
+Target value: ffffffff
+Store word, 2 byte offset
+-> exception, mcause = 6
+Target value: ffffffff
+Store word, 3 byte offset
+-> exception, mcause = 6
+Target value: ffffffff
+Aligned store word, sanity check
+Target value: 00000000
+
+*******************************************************************************/
+
 int main() {
 	volatile uint32_t target_word = -1u;
 	tb_puts("Store word, 1 byte offset\n");

test/sim/sw_testcases/store_misalign_word.expected_output

@@ -1,11 +0,0 @@
-Store word, 1 byte offset
--> exception, mcause = 6
-Target value: ffffffff
-Store word, 2 byte offset
--> exception, mcause = 6
-Target value: ffffffff
-Store word, 3 byte offset
--> exception, mcause = 6
-Target value: ffffffff
-Aligned store word, sanity check
-Target value: 00000000

test/sim/sw_testcases/wfi_loop.c

@@ -1,5 +1,22 @@
 #include "tb_cxxrtl_io.h"
 
+/*EXPECTED-OUTPUT***************************************************************
+
+Enabling IRQS...
+IRQ 1
+IRQ 2
+IRQ 3
+IRQ 4
+IRQ 5
+IRQ 6
+IRQ 7
+IRQ 8
+IRQ 9
+IRQ 10
+Took 10 IRQs, span 9 times
+
+*******************************************************************************/
+
 #define TIMER_INTERVAL 1000
 #define MAX_IRQ_COUNT 10