scripts: Add regression test runner/checker

This commit is contained in:
Connor McLaughlin 2021-08-04 20:17:05 +10:00
parent 6cba6986d3
commit 43c310af4b
2 changed files with 152 additions and 0 deletions

View file

@ -0,0 +1,81 @@
import argparse
import glob
import sys
import os
import re
import hashlib
from pathlib import Path
def compare_frames(path1, path2):
try:
with open(path1, "rb") as f:
hash1 = hashlib.md5(f.read()).digest()
with open(path2, "rb") as f:
hash2 = hashlib.md5(f.read()).digest()
#print(hash1, hash2)
return hash1 == hash2
except:
return False
def check_regression_test(baselinedir, testdir, name):
#print("Checking '%s'..." % name)
dir1 = os.path.join(baselinedir, name)
dir2 = os.path.join(testdir, name)
if not os.path.isdir(dir2):
#print("*** %s is missing in test set" % name)
return False
images = glob.glob(os.path.join(dir1, "frame_*.png"))
for imagepath in images:
imagename = Path(imagepath).name
matches = re.match("frame_([0-9]+).png", imagename)
if matches is None:
continue
framenum = int(matches[1])
path1 = os.path.join(dir1, imagename)
path2 = os.path.join(dir2, imagename)
if not os.path.isfile(path2):
print("--- Frame %u for %s is missing in test set" % (framenum, name))
return False
if not compare_frames(path1, path2):
print("*** Difference in frame %u for %s" % (framenum, name))
return False
return True
def check_regression_tests(baselinedir, testdir):
gamedirs = glob.glob(baselinedir + "/*", recursive=False)
success = 0
failure = 0
for gamedir in gamedirs:
name = Path(gamedir).name
if check_regression_test(baselinedir, testdir, name):
success += 1
else:
failure += 1
return (failure == 0)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Check frame dump images for regression tests")
parser.add_argument("-baselinedir", action="store", required=True, help="Directory containing baseline frames to check against")
parser.add_argument("-testdir", action="store", required=True, help="Directory containing frames to check")
args = parser.parse_args()
if not check_regression_tests(os.path.realpath(args.baselinedir), os.path.realpath(args.testdir)):
sys.exit(1)
else:
sys.exit(0)

View file

@ -0,0 +1,71 @@
import argparse
import glob
import sys
import os
import subprocess
import multiprocessing
from functools import partial
def is_game_path(path):
idx = path.rfind('.')
if idx < 0:
return False
extension = path[idx + 1:].strip().lower()
return extension in ["cue", "chd"]
def run_regression_test(runner, destdir, dump_interval, frames, gamepath):
args = [runner,
"-renderer", "software",
"-log", "verbose",
"-dumpdir", destdir,
"-dumpinterval", str(dump_interval),
"-frames", str(frames),
"--", gamepath
]
print("Running '%s'" % (" ".join(args)))
subprocess.run(args)
def run_regression_tests(runner, gamedir, destdir, dump_interval, frames, parallel=1):
paths = glob.glob(gamedir + "/*.*", recursive=True)
gamepaths = list(filter(is_game_path, paths))
if not os.path.isdir(destdir) and not os.mkdir(destdir):
print("Failed to create directory")
return False
print("Found %u games" % len(gamepaths))
if parallel <= 1:
for game in gamepaths:
run_regression_test(runner, destdir, dump_interval, frames, game)
else:
print("Processing %u games on %u processors" % (len(gamepaths), parallel))
func = partial(run_regression_test, runner, destdir, dump_interval, frames)
pool = multiprocessing.Pool(parallel)
pool.map(func, gamepaths)
pool.close()
return True
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Generate frame dump images for regression tests")
parser.add_argument("-runner", action="store", required=True, help="Path to DuckStation regression test runner")
parser.add_argument("-gamedir", action="store", required=True, help="Directory containing game images")
parser.add_argument("-destdir", action="store", required=True, help="Base directory to dump frames to")
parser.add_argument("-dumpinterval", action="store", type=int, required=True, help="Interval to dump frames at")
parser.add_argument("-frames", action="store", type=int, default=3600, help="Number of frames to run")
parser.add_argument("-parallel", action="store", type=int, default=1, help="Number of proceeses to run")
args = parser.parse_args()
if not run_regression_tests(args.runner, os.path.realpath(args.gamedir), os.path.realpath(args.destdir), args.dumpinterval, args.frames, args.parallel):
sys.exit(1)
else:
sys.exit(0)