scripts: Add regression test runner/checker

scripts/check_regression_tests.py

@@ -0,0 +1,81 @@
+import argparse
+import glob
+import sys
+import os
+import re
+import hashlib
+
+from pathlib import Path
+
+def compare_frames(path1, path2):
+    try:
+        with open(path1, "rb") as f:
+            hash1 = hashlib.md5(f.read()).digest()
+        with open(path2, "rb") as f:
+            hash2 = hashlib.md5(f.read()).digest()
+
+        #print(hash1, hash2)
+        return hash1 == hash2
+    except:
+        return False
+
+
+def check_regression_test(baselinedir, testdir, name):
+    #print("Checking '%s'..." % name)
+
+    dir1 = os.path.join(baselinedir, name)
+    dir2 = os.path.join(testdir, name)
+    if not os.path.isdir(dir2):
+        #print("*** %s is missing in test set" % name)
+        return False
+
+    images = glob.glob(os.path.join(dir1, "frame_*.png"))
+    for imagepath in images:
+        imagename = Path(imagepath).name
+        matches = re.match("frame_([0-9]+).png", imagename)
+        if matches is None:
+            continue
+
+        framenum = int(matches[1])
+        
+        path1 = os.path.join(dir1, imagename)
+        path2 = os.path.join(dir2, imagename)
+        if not os.path.isfile(path2):
+            print("--- Frame %u for %s is missing in test set" % (framenum, name))
+            return False
+
+        if not compare_frames(path1, path2):
+            print("*** Difference in frame %u for %s" % (framenum, name))
+            return False
+
+    return True
+
+
+def check_regression_tests(baselinedir, testdir):
+    gamedirs = glob.glob(baselinedir + "/*", recursive=False)
+    
+    success = 0
+    failure = 0
+
+    for gamedir in gamedirs:
+        name = Path(gamedir).name
+        if check_regression_test(baselinedir, testdir, name):
+            success += 1
+        else:
+            failure += 1
+
+    return (failure == 0)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Check frame dump images for regression tests")
+    parser.add_argument("-baselinedir", action="store", required=True, help="Directory containing baseline frames to check against")
+    parser.add_argument("-testdir", action="store", required=True, help="Directory containing frames to check")
+
+    args = parser.parse_args()
+
+    if not check_regression_tests(os.path.realpath(args.baselinedir), os.path.realpath(args.testdir)):
+        sys.exit(1)
+    else:
+        sys.exit(0)
+

scripts/run_regression_tests.py

@@ -0,0 +1,71 @@
+import argparse
+import glob
+import sys
+import os
+import subprocess
+import multiprocessing
+from functools import partial
+
+def is_game_path(path):
+    idx = path.rfind('.')
+    if idx < 0:
+        return False
+
+    extension = path[idx + 1:].strip().lower()
+    return extension in ["cue", "chd"]
+
+
+def run_regression_test(runner, destdir, dump_interval, frames, gamepath):
+    args = [runner,
+            "-renderer", "software",
+            "-log", "verbose",
+            "-dumpdir", destdir,
+            "-dumpinterval", str(dump_interval),
+            "-frames", str(frames),
+            "--", gamepath
+    ]
+
+    print("Running '%s'" % (" ".join(args)))
+    subprocess.run(args)
+
+
+def run_regression_tests(runner, gamedir, destdir, dump_interval, frames, parallel=1):
+    paths = glob.glob(gamedir + "/*.*", recursive=True)
+    gamepaths = list(filter(is_game_path, paths))
+
+    if not os.path.isdir(destdir) and not os.mkdir(destdir):
+        print("Failed to create directory")
+        return False
+
+    print("Found %u games" % len(gamepaths))
+
+    if parallel <= 1:
+        for game in gamepaths:
+            run_regression_test(runner, destdir, dump_interval, frames, game)
+    else:
+        print("Processing %u games on %u processors" % (len(gamepaths), parallel))
+        func = partial(run_regression_test, runner, destdir, dump_interval, frames)
+        pool = multiprocessing.Pool(parallel)
+        pool.map(func, gamepaths)
+        pool.close()
+
+
+    return True
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Generate frame dump images for regression tests")
+    parser.add_argument("-runner", action="store", required=True, help="Path to DuckStation regression test runner")
+    parser.add_argument("-gamedir", action="store", required=True, help="Directory containing game images")
+    parser.add_argument("-destdir", action="store", required=True, help="Base directory to dump frames to")
+    parser.add_argument("-dumpinterval", action="store", type=int, required=True, help="Interval to dump frames at")
+    parser.add_argument("-frames", action="store", type=int, default=3600, help="Number of frames to run")
+    parser.add_argument("-parallel", action="store", type=int, default=1, help="Number of proceeses to run")
+
+    args = parser.parse_args()
+
+    if not run_regression_tests(args.runner, os.path.realpath(args.gamedir), os.path.realpath(args.destdir), args.dumpinterval, args.frames, args.parallel):
+        sys.exit(1)
+    else:
+        sys.exit(0)
+