diff --git a/scripts/check_regression_tests.py b/scripts/check_regression_tests.py new file mode 100644 index 000000000..c33f07f4f --- /dev/null +++ b/scripts/check_regression_tests.py @@ -0,0 +1,81 @@ +import argparse +import glob +import sys +import os +import re +import hashlib + +from pathlib import Path + +def compare_frames(path1, path2): + try: + with open(path1, "rb") as f: + hash1 = hashlib.md5(f.read()).digest() + with open(path2, "rb") as f: + hash2 = hashlib.md5(f.read()).digest() + + #print(hash1, hash2) + return hash1 == hash2 + except: + return False + + +def check_regression_test(baselinedir, testdir, name): + #print("Checking '%s'..." % name) + + dir1 = os.path.join(baselinedir, name) + dir2 = os.path.join(testdir, name) + if not os.path.isdir(dir2): + #print("*** %s is missing in test set" % name) + return False + + images = glob.glob(os.path.join(dir1, "frame_*.png")) + for imagepath in images: + imagename = Path(imagepath).name + matches = re.match("frame_([0-9]+).png", imagename) + if matches is None: + continue + + framenum = int(matches[1]) + + path1 = os.path.join(dir1, imagename) + path2 = os.path.join(dir2, imagename) + if not os.path.isfile(path2): + print("--- Frame %u for %s is missing in test set" % (framenum, name)) + return False + + if not compare_frames(path1, path2): + print("*** Difference in frame %u for %s" % (framenum, name)) + return False + + return True + + +def check_regression_tests(baselinedir, testdir): + gamedirs = glob.glob(baselinedir + "/*", recursive=False) + + success = 0 + failure = 0 + + for gamedir in gamedirs: + name = Path(gamedir).name + if check_regression_test(baselinedir, testdir, name): + success += 1 + else: + failure += 1 + + return (failure == 0) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Check frame dump images for regression tests") + parser.add_argument("-baselinedir", action="store", required=True, help="Directory containing baseline frames to check against") + parser.add_argument("-testdir", action="store", required=True, help="Directory containing frames to check") + + args = parser.parse_args() + + if not check_regression_tests(os.path.realpath(args.baselinedir), os.path.realpath(args.testdir)): + sys.exit(1) + else: + sys.exit(0) + diff --git a/scripts/run_regression_tests.py b/scripts/run_regression_tests.py new file mode 100644 index 000000000..f9b8dbec3 --- /dev/null +++ b/scripts/run_regression_tests.py @@ -0,0 +1,71 @@ +import argparse +import glob +import sys +import os +import subprocess +import multiprocessing +from functools import partial + +def is_game_path(path): + idx = path.rfind('.') + if idx < 0: + return False + + extension = path[idx + 1:].strip().lower() + return extension in ["cue", "chd"] + + +def run_regression_test(runner, destdir, dump_interval, frames, gamepath): + args = [runner, + "-renderer", "software", + "-log", "verbose", + "-dumpdir", destdir, + "-dumpinterval", str(dump_interval), + "-frames", str(frames), + "--", gamepath + ] + + print("Running '%s'" % (" ".join(args))) + subprocess.run(args) + + +def run_regression_tests(runner, gamedir, destdir, dump_interval, frames, parallel=1): + paths = glob.glob(gamedir + "/*.*", recursive=True) + gamepaths = list(filter(is_game_path, paths)) + + if not os.path.isdir(destdir) and not os.mkdir(destdir): + print("Failed to create directory") + return False + + print("Found %u games" % len(gamepaths)) + + if parallel <= 1: + for game in gamepaths: + run_regression_test(runner, destdir, dump_interval, frames, game) + else: + print("Processing %u games on %u processors" % (len(gamepaths), parallel)) + func = partial(run_regression_test, runner, destdir, dump_interval, frames) + pool = multiprocessing.Pool(parallel) + pool.map(func, gamepaths) + pool.close() + + + return True + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate frame dump images for regression tests") + parser.add_argument("-runner", action="store", required=True, help="Path to DuckStation regression test runner") + parser.add_argument("-gamedir", action="store", required=True, help="Directory containing game images") + parser.add_argument("-destdir", action="store", required=True, help="Base directory to dump frames to") + parser.add_argument("-dumpinterval", action="store", type=int, required=True, help="Interval to dump frames at") + parser.add_argument("-frames", action="store", type=int, default=3600, help="Number of frames to run") + parser.add_argument("-parallel", action="store", type=int, default=1, help="Number of proceeses to run") + + args = parser.parse_args() + + if not run_regression_tests(args.runner, os.path.realpath(args.gamedir), os.path.realpath(args.destdir), args.dumpinterval, args.frames, args.parallel): + sys.exit(1) + else: + sys.exit(0) +