#!/usr/bin/env python3
import argparse
import sys
import subprocess
import re
from pathlib import Path
from dataclasses import dataclass
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKCYAN = '\033[96m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
class Test:
pass
@dataclass
class TestGroup:
tests: list[Test]
directory: Path
name: str
c_test_file: Path # The C file to compile and use for this test
dat_test_file: Path # The C file to compile and use for this test
def __str__(self):
return self.name
@dataclass
class Test:
group: TestGroup
name: str
input_file: Path
output_file: Path
expected_file: Path
def __str__(self):
return f"{self.group.name}.{self.name}"
@dataclass
class Validation:
test: Test
expected: list[str]
actual: list[str]
matches: bool
def find_tests(groups_dir: Path, programs_dir: Path, out_dir: Path, group_name: str|None, test_name: str|None) -> list[TestGroup]:
group_names: list[Path] = []
if group_name is None:
group_names = [f for f in groups_dir.iterdir() if f.is_dir()]
else:
group_names = [groups_dir / group_name]
groups: list[TestGroup] = []
for group_dir in group_names:
tests: list[Test] = []
group = TestGroup(
tests = tests,
directory = group_dir,
name = group_dir.name,
c_test_file = programs_dir / f"{group_dir.name}.c",
dat_test_file = programs_dir / "bin" / f"{group_dir.name}.dat",
)
test_names = []
if test_name is None:
test_names = [f.name[:-len("-input.dat")] for f in group_dir.iterdir() if f.is_file() and f.name.endswith("-input.dat")]
else:
test_names = [test_name]
for test_name in test_names:
test = Test(
group,
test_name,
group_dir / f"{test_name}-input.dat",
out_dir / f"{test_name}-output.dat",
group_dir / f"{test_name}-expected.dat",
)
if not test.input_file.exists() or not test.expected_file.exists():
continue
tests.append(test)
groups.append(group)
return groups
def validate_test(test: Test) -> Validation:
expected = test.expected_file.read_text()
actual = test.output_file.read_text()
expected_arr = list(filter(lambda word: word != "", re.split(r"[\n ]+", expected)))
actual_arr = re.split(r"[\n ]+", actual)
# trim leading
actual_arr = actual_arr[:len(expected_arr)]
return Validation(
test = test,
expected = expected_arr,
actual = actual_arr,
matches = (actual_arr == expected_arr)
)
def compile_test(project_dir: Path, comp_list: Path, out_dir: Path, test: Test) -> bool:
generics = {
'CPU_PROGRAM_PATH': f"\\\"{test.group.dat_test_file}\\\"",
'CPU_PROGRAM_NAME': f"\\\"{test.group.dat_test_file.stem}\\\"",
'MEMORY_LOAD_FILE': 1,
'MEMORY_LOAD_FILE_PATH': f"\\\"{test.input_file}\\\"",
'MEMORY_WRITE_FILE': 1,
'MEMORY_WRITE_FILE_PATH': f"\\\"{test.output_file}\\\"",
}
params = []
for gname, gvalue in generics.items():
params.append(f"-G{gname}={gvalue}")
params.append("--binary")
params.append("--Mdir")
params.append(f"{out_dir}")
params.append("-o")
params.append(f"test_{test.group.name}_{test.name}")
params.append("--top")
params.append("tb_cpu_program")
for line in comp_list.read_text().split('\n'):
if line != "":
params.append(f"{project_dir / line}")
return subprocess.run(
str.join(" ", [ "verilator" ] + params),
stdout = subprocess.DEVNULL,
shell = True,
check = True,
).returncode == 0
def run_test(out_dir: Path, test: Test) -> bool:
return subprocess.run(
[out_dir / f"test_{test.group.name}_{test.name}"],
stdout = subprocess.DEVNULL,
shell = True,
check = True,
).returncode == 0
def compile_program(make_dir: Path, group: TestGroup) -> bool:
return subprocess.run(
["make", "-C", make_dir, group.dat_test_file.relative_to(make_dir)],
stdout = subprocess.DEVNULL,
stderr = subprocess.DEVNULL,
) == 0
# Program
parser = argparse.ArgumentParser("Test simple RISC-V processor written in Verilog.")
parser.add_argument(
"command",
choices = [ "run", "list"],
help = "What to do. Either run the test, or run testcases based on filter."
)
parser.add_argument(
"-f",
"--filter",
type = str,
nargs = "*",
help = "Filter, should be in group.test format."
)
parser.add_argument(
"-t",
"--type",
choices = ["custom", "official"],
default = "custom",
help = "Type of the testcases, either custom testcases or official riscv selftests.",
)
args = parser.parse_args()
here = Path(__file__).parent
project_dir = here.parent
programs_dir = project_dir / "programs"
out_dir = here / "out"
groups_dir = here / "custom"
# TODO support multiple tests
group_name, test_name = args.filter[0].split('.') if args.filter is not None else (None, None)
test_groups: list[TestGroup] = find_tests(groups_dir, programs_dir, out_dir, group_name, test_name)
# Official
# TODO
# Custom
if args.command == "list":
print("Found these tests:")
for group in test_groups:
for test in group.tests:
print(f" {test}")
sys.exit(0)
for group in test_groups:
compile_program(project_dir, group)
for test in group.tests:
compile_test(project_dir, here / "comp_list.lst", out_dir, test)
run_test(out_dir, test)
validation = validate_test(test)
if validation.matches:
print(f"{test.group.name}.{test.name} {bcolors.OKGREEN}passed{bcolors.ENDC}")
else:
print(f"{test.group.name}.{test.name} {bcolors.FAIL}failed{bcolors.ENDC}")
print(f" Got {validation.actual}. Expected {validation.expected}")