Add trace disassembly annotation script for rvcpp, and add runtests support for passing flags to tb, and running post-processing commands on test results.

test/sim/rvcpp/Makefile

@@ -4,7 +4,9 @@ EXECUTABLE:=rvcpp
 .SUFFIXES:
 .PHONY: all clean tb
 
-all:
+all: $(EXECUTABLE)
+
+$(EXECUTABLE): $(SRCS) $(wildcard include/*.h)
 	g++ -std=c++17 -O3 -Wall -Wextra -I include $(SRCS) -o $(EXECUTABLE)
 
 # To match tb_cxxrtl/Makefile:

test/sim/rvcpp/scripts/annotate_trace.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+import argparse
+import re
+import sys
+
+# Script for annotating rvcpp trace output with the contents of an objdump
+# disassembly file. Multiple disassembly files can be passed, in which case
+# they will be merged. (Usually these files would be for non-overlapping
+# address ranges: if the files overlap, the later file in command line order
+# takes precedence for the overlapping address.)
+
+parser = argparse.ArgumentParser()
+parser.add_argument("logfile", help="Raw log file to be annotated, output from rvcpp --trace")
+parser.add_argument("out", help="Output path for annotated log file (pass - for stdout)")
+parser.add_argument("-d", "--dis", action="append", help="Specify a disassembly file (output of objdump -d) with which to annotate the log")
+args = parser.parse_args()
+
+if args.dis is None:
+	sys.exit("At least one disassembly file must be specified")
+
+label_dict = {}
+instr_dict = {}
+for dispath in args.dis:
+	for l in open(dispath).readlines():
+		if re.match(r"^\s*[0-9a-f]+:", l):
+			instruction_addr = int(l.split(":")[0], 16)
+			instruction_text = " ".join(l.strip().split()[2:])
+			instr_dict[instruction_addr] = instruction_text
+		elif re.match("^[0-9a-f]+ <", l):
+			label_addr = int(l.split()[0], 16)
+			label_text = l.split("<")[-1].strip("\n>:")
+			label_dict[label_addr] = label_text
+
+ifile = open(args.logfile)
+if args.out == "-":
+	ofile = sys.stdout
+else:
+	ofile = open(args.out, "w")
+
+for l in ifile.readlines():
+	# Not an addressed line, so just pass it through unmodified.
+	if not re.match("^[0-9a-f]{8}:", l):
+		ofile.write(l)
+		continue
+	addr = int(l[:8], 16)
+	# If there is a label, there ought also be an instruction to be labelled.
+	# assert(not (addr in label_dict and addr not in instr_dict))
+	# Not an address we know about, so pass it through unmodified.
+	if addr not in label_dict and addr not in instr_dict:
+		ofile.write(l)
+		continue
+	# Removed label lines for now as the label is usually present on the jump instruction
+	# if addr in label_dict:
+	# 	ofile.write(" " * 42 + label_dict[addr] + ":\n")
+	if addr in instr_dict:
+		ofile.write(l.strip() + " " * 2 + instr_dict[addr] + "\n")
+
+

test/sim/sw_testcases/runtests

@@ -1,16 +1,31 @@
 #!/usr/bin/env python3
 
+import argparse
 import os
+import shlex
 import subprocess
 import sys
-import argparse
 
 args = sys.argv[1:]
 
-parser = argparse.ArgumentParser()
+parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter)
 parser.add_argument("tests", nargs="*", help="List of tests to run. Empty to run all tests. Each test corresponds to one C file.")
 parser.add_argument("--vcd", action="store_true", help="Pass --vcd flag to simulator, to generate waveform dumps.")
 parser.add_argument("--tb", default="../tb_cxxrtl/tb", help="Pass tb executable to run tests.")
+parser.add_argument("--tbarg", action="append", default=[], help="Extra argument to pass to tb executable. Can pass --tbarg=xxx multiple times to pass multiple arguments.")
+parser.add_argument("--postcmd", action="append", default=[], help="Add a command to run post-simulation, e.g. log file processing. The string TEST is expanded to the test result file name, minus any file extensions.")
+parser.epilog = """
+Example command lines:
+
+Run all tests:
+./runtests
+
+Just run hello world, and generate waves:
+./runtests hello_world --vcd
+
+Run under rvcpp, enable instruction tracing, and post-process log using disassembly:
+./runtests --tb ../rvcpp/rvcpp --tbarg=--trace --postcmd="../rvcpp/scripts/annotate_trace.py TEST.log TEST_annotated.log -d TEST.dis"
+"""
 args = parser.parse_args()
 
 testlist = args.tests
@@ -38,21 +53,23 @@ if tb_build_ret.returncode != 0:
 
 all_passed = True
 
+passed_test_count = 0
 for test in testlist:
 	sys.stdout.write(f"{test:<30}")
-
+	failed = False
 	test_build_ret = subprocess.run(
 		["make", f"APP={test}", f"tmp/{test}.bin"],
 		stdout=subprocess.DEVNULL
 	)
 	if test_build_ret.returncode != 0:
 		print("\033[33m[MK ERR]\033[39m")
-		all_passed = False
+		failed = True
-		continue
 
+	if not failed:
 		cmdline = [args.tb, "--bin", f"tmp/{test}.bin", "--cycles", "1000000"]
 		if args.vcd:
 			cmdline += ["--vcd", f"tmp/{test}.vcd"]
+		cmdline += args.tbarg
 
 		try:
 			test_run_ret = subprocess.run(
@@ -65,18 +82,18 @@ for test in testlist:
 				f.write(test_run_ret.stdout)
 		except subprocess.TimeoutExpired:
 			print("\033[31m[TIMOUT]\033[39m")
-		all_passed = False
+			failed = True
-		continue
 
 	# Testbench itself should always exit successfully.
+	if not failed:
 		if test_run_ret.returncode != 0:
 			print("Negative return code from testbench!")
-		all_passed = False
+			failed = True
-		continue
 
 	# Pass if the program under test has zero exit code AND its output matches
 	# the expected output (if there is an expected_output file)
 
+	if not failed:
 		output_lines = test_run_ret.stdout.decode("utf-8").strip().splitlines()
 		returncode = -1
 		if len(output_lines) >= 2:
@@ -88,9 +105,9 @@ for test in testlist:
 					pass
 		if returncode != 0:
 			print("\033[31m[BADRET]\033[39m")
-		all_passed = False
+			failed = True
-		continue
 
+	if not failed:
 		test_src = open(f"{test}.c").read()
 		if "/*EXPECTED-OUTPUT" in test_src:
 			good_output = True
@@ -123,10 +140,19 @@ for test in testlist:
 				good_output = False
 			if not good_output:
 				print("\033[31m[BADOUT]\033[39m")
-			all_passed = False
+				failed = True
-			continue
 
+	if not failed:
 		print("\033[32m[PASSED]\033[39m")
+		passed_test_count += 1
+
+	# Post-processing commands are run regardless of success. Their return
+	# codes are ignored.
+	for postcmd in args.postcmd:
+		postcmd = postcmd.replace("TEST", f"tmp/{test}")
+		subprocess.run(shlex.split(postcmd))
+
+print(f"\nPassed: {passed_test_count} out of {len(testlist)}")
 
 sys.exit(not all_passed)