supermodel_build_bot.py: New build bot script for GitHub

Scripts/supermodel_build_bot.py

@@ -1,8 +1,7 @@
 #
 # Supermodel
 # A Sega Model 3 Arcade Emulator.
-# Copyright 2011-2021 Bart Trzynadlowski, Nik Henson, Ian Curtis,
+# Copyright 2003-2022 The Supermodel Team
-#                     Harry Tuttle, and Spindizzi
 #
 # This file is part of Supermodel.
 #
@@ -21,9 +20,9 @@
 #
 
 #
-# update_model3emu_snapshot.py
+# supermodel_build_bot.py
 #
-# SVN snapshot build script. Checks latest SVN revision against Supermodel3.com
+# Git snapshot build script. Checks latest Git commit against Supermodel3.com
 # download page and builds and uploads a snapshot if needed. Intended to be run
 # at least once daily as part of an automated job.
 #
@@ -35,9 +34,11 @@
 #   - mingw64/mingw-w64-x86_64-SDL2_net package
 #   - msys/subversion package
 #   - msys/zip package
+#   - git
 #
 # To perform a test run:
-#   - Download https://supermodel3.com/Download.html
+#   - Download https://supermodel3.com/Download.html to the directory from
+#     which the script will be run.
 #   - Run: python update_model3emu_snapshot.py --working-dir=c:\tmp\build --test-run
 #
 
@@ -66,7 +67,7 @@ class Bash:
   def execute(self, working_dir, command, log=False, print_output=False):
     import subprocess
     working_dir = working_dir.replace("\\", "/")  # convert to UNIX-style path
-    invocation = self.bash_path + " -l -c \"" + "cd " + working_dir + " && " + command + "\""
+    invocation = self.bash_path + " -l -c '" + "cd " + working_dir + " && " + command + "'"
     if log:
       print("Executing: %s" % invocation)
     result = subprocess.run(invocation, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
@@ -74,7 +75,7 @@ class Bash:
       print(result.stdout.decode())
     return result
 
-def get_web_page(url, test_run):
+def get_web_page(test_run):
   if test_run:
     with open("Download.html") as fp:
       html = fp.read()
@@ -91,43 +92,43 @@ def get_comment(line):
     return line[comment_begin+4:comment_end].strip()
   return None
 
-def add_new_file(html, filename, version, svn_revision):
+def add_new_file(html, filename, version, git_sha, date):
-  # Scan for: <!-- BEGIN_SVN_SNAPSHOTS $TEMPLATE -->
+  # Scan for: <!-- BEGIN_GIT_SNAPSHOTS $TEMPLATE -->
   lines = html.splitlines()
   for i in range(len(lines)):
     line = lines[i]
     text = get_comment(line)
-    if text and text.startswith("BEGIN_SVN_SNAPSHOTS "):
+    if text and text.startswith("BEGIN_GIT_SNAPSHOTS "):
-      template = text[len("BEGIN_SVN_SNAPSHOTS "):]
+      template = text[len("BEGIN_GIT_SNAPSHOTS "):]
-      new_row = template.replace("$FILENAME", filename).replace("$VERSION", version).replace("$REVISION", svn_revision)
+      new_row = template.replace("$FILENAME", filename).replace("$VERSION", version).replace("$GIT_SHA", git_sha).replace("$DATE", date)
-      svn_tag = "<!-- SVN_REVISION=%s -->" % svn_revision
+      git_tag = "<!-- GIT_SHA=%s -->" % git_sha
-      lines.insert(i + 1, new_row + " " + svn_tag)
+      lines.insert(i + 1, new_row + " " + git_tag)
       html = "\n".join(lines)
       return html
-  raise ParseError("BEGIN_SVN_SNAPSHOTS not found")
+  raise ParseError("BEGIN_GIT_SNAPSHOTS not found")
 
-def get_uploaded_svn_revisions(html):
+def get_uploaded_git_shas(html):
-  # Scan for all lines with: <!-- SVN_REVISION=$REVISION -->
+  # Scan for all lines with: <!-- GIT_SHA=$SHA -->
-  revisions = []
+  shas = []
   for line in html.splitlines():
     text = get_comment(line)
-    if text and text.startswith("SVN_REVISION"):
+    if text and text.startswith("GIT_SHA"):
       tokens = text.split("=")
       if len(tokens) == 2:
-        revision = tokens[1]
+        sha = tokens[1]
-        revisions.append(revision)
+        shas.append(sha)
       else:
-        raise ParseError("Error parsing SVN_REVISION")
+        raise ParseError("Error parsing GIT_SHA")
-  revisions.reverse()
+  shas.reverse()
-  return revisions
+  return shas
 
-def create_change_log(bash, repo_dir, file_path, uploaded_revisions, current_revision):
+def create_change_log(bash, repo_dir, file_path, uploaded_shas, current_sha):
-  from_revision = uploaded_revisions[0] if len(uploaded_revisions) > 0 else current_revision
+  # Log from first commit after 0.2a release until now
-  result = bash.execute(working_dir=repo_dir, command="svn log -r " + current_revision + ":" + from_revision)
+  result = bash.execute(working_dir=repo_dir, command="git -c color.ui=never log 06594a5..." + current_sha + " --no-decorate --pretty=\"Commit: %H%nAuthor: %an%nDate:   %ad%n%n%w(80,4,4)%B\"")
   if result.returncode != 0:
-    return PackageError("Unable to obtain SVN log")
+    return PackageError("Unable to obtain Git log")
   change_log = result.stdout.decode().strip()
-  with open(file_path, "w") as fp:
+  with open(file_path, "w", encoding="utf-8") as fp:
     header = "\n\n" \
       "  ####                                                      ###           ###\n" \
       " ##  ##                                                      ##            ##\n" \
@@ -140,10 +141,9 @@ def create_change_log(bash, repo_dir, file_path, uploaded_revisions, current_rev
       "\n" \
       "                       A Sega Model 3 Arcade Emulator.\n" \
       "\n" \
-      "        Copyright 2011-2021 Bart Trzynadlowski, Nik Henson, Ian Curtis,\n" \
+      "                   Copyright 2003-2022 The Supermodel Team\n" \
-      "                          Harry Tuttle, and Spindizzi\n" \
       "\n" \
-      "                                SVN CHANGE LOG\n" \
+      "                                 CHANGE LOG\n" \
       "\n" \
       "\n"
     fp.write(header)
@@ -157,7 +157,7 @@ def upload(html_file, zip_file_path, username, password):
   from ftplib import FTP
   ftp = FTP("supermodel3.com")
   ftp.login(username, password)
-  ftp.cwd("public_html/Files/SVN_Snapshots")
+  ftp.cwd("public_html/Files/Git_Snapshots")
   with open(zip_file_path, "rb") as fp:
     ftp.storbinary("STOR " + os.path.basename(zip_file_path), fp)
   ftp.cwd("../../")
@@ -177,15 +177,17 @@ def confirm_package_contents(package_dir, package_files):
 
 def rmdir(dir):
   def delete_readonly(action, name, exc):
+    # https://stackoverflow.com/questions/2656322/shutil-rmtree-fails-on-windows-with-access-is-denied
     import stat
-    os.chmod(name, stat.S_IWRITE)
+    if not os.access(name, os.W_OK):
-    os.remove(name)
+      os.chmod(name, stat.S_IWUSR)
-  for root, dirs, files in os.walk(dir):  # we expect .svn to be write-protected
+      action(name)
-    if '.svn' in dirs:
+  for root, dirs, files in os.walk(dir):  # we expect .git to be write-protected
-      shutil.rmtree(root+'\.svn',onerror=delete_readonly)
+    if '.git' in dirs:
+      shutil.rmtree(root+'\.git',onerror=delete_readonly)
   shutil.rmtree(dir)
 
-def update_svn_snapshot(working_dir, username, password, test_run, make):
+def update_git_snapshot(working_dir, username, password, test_run, make):
   failed = False
   print("Starting %s in working directory: %s" % ("test run" if test_run else "release process", working_dir))
 
@@ -193,27 +195,34 @@ def update_svn_snapshot(working_dir, username, password, test_run, make):
     bash = Bash(bash_path="c:\\msys64\\usr\\bin\\bash.exe")
     repo_dir = os.path.join(working_dir, "model3emu")
 
-    # Fetch Supermodel download page and compare most recent uploaded SVN snapshot against current SVN revision
+    # Clone Git repo. Unfortunately need to always do this in order to get short SHA
-    print("Fetching Supermodel download page...")
+    result = bash.execute(working_dir=working_dir, command="git clone https://github.com/trzy/Supermodel.git model3emu")
-    html = get_web_page(url="https://supermodel3.com/Forum/Download.html", test_run=test_run)
-    uploaded_revisions = get_uploaded_svn_revisions(html=html)
-    last_uploaded_revision = uploaded_revisions[-1] if len(uploaded_revisions) > 0 else None
-    print("Checking current SVN revision...")
-    result = bash.execute(working_dir=working_dir, command="svn info --show-item revision https://svn.code.sf.net/p/model3emu/code/trunk")
     if result.returncode != 0:
-      raise CheckoutError("SVN revision check failed")
+      raise CheckoutError("Git clone failed")
-    current_revision = result.stdout.decode().strip()
 
-    # If SVN has a newer version, or if performing a test run, build it and update the web page
+    # Fetch Supermodel download page and compare most recent uploaded Git snapshot against current Git sha
-    if current_revision == last_uploaded_revision and (not test_run):
+    print("Fetching Supermodel download page...")
-      print("Nothing to do. Revision already uploaded: %s" % current_revision)
+    html = get_web_page(test_run=test_run)
+    uploaded_shas = get_uploaded_git_shas(html=html)
+    last_uploaded_sha = uploaded_shas[-1] if len(uploaded_shas) > 0 else None
+    print("Checking current Git SHA...")
+    result = bash.execute(working_dir=repo_dir, command="git rev-parse --short HEAD")
+    if result.returncode != 0:
+      raise CheckoutError("Git HEAD SHA check failed")
+    current_sha = result.stdout.decode().strip()
+
+    # Get date of commit
+    result = bash.execute(working_dir=repo_dir, command="git show -s --format=%as HEAD")
+    if result.returncode != 0:
+      raise CheckoutError("Unable to obtain HEAD commit date")
+    current_date = result.stdout.decode().strip()
+
+    # If Git has a newer version, or if performing a test run, build it and update the web page
+    if current_sha == last_uploaded_sha and (not test_run):
+      print("Nothing to do. Current Git SHA already uploaded: %s" % current_sha)
     else:
       # Check out
-      print("SVN revision is %s but uploaded revision is %s" % (current_revision, last_uploaded_revision))
+      print("Git SHA is %s but last uploaded SHA is %s" % (current_sha, last_uploaded_sha))
-      print("Checking out Supermodel...")
-      result = bash.execute(working_dir=working_dir, command="svn co https://svn.code.sf.net/p/model3emu/code/trunk model3emu")
-      if result.returncode != 0:
-        raise CheckoutError("SVN checkout failed")
 
       # Build
       print("Building Supermodel...")
@@ -221,7 +230,7 @@ def update_svn_snapshot(working_dir, username, password, test_run, make):
       if result.returncode != 0:
         raise BuildError("Failed to obtain version")
       version = result.stdout.decode().strip()
-      result = bash.execute(working_dir=repo_dir, command=make + " -f Makefiles/Makefile.Win32 release NET_BOARD=1")
+      result = bash.execute(working_dir=repo_dir, command=make + " -f Makefiles/Makefile.Win32 release NET_BOARD=1", print_output=False)
       if result.returncode != 0:
         raise BuildError("Build failed")
 
@@ -230,7 +239,7 @@ def update_svn_snapshot(working_dir, username, password, test_run, make):
       pkg_dir = os.path.join(working_dir, "pkg")
       bash.execute(working_dir=working_dir, command="mkdir pkg && mkdir pkg/Config && mkdir pkg/NVRAM && mkdir pkg/Saves && mkdir pkg/ROMs")
       change_log_file_path = os.path.join(pkg_dir, "CHANGES.txt")
-      create_change_log(bash, repo_dir=repo_dir, file_path=change_log_file_path, uploaded_revisions=uploaded_revisions, current_revision=current_revision)
+      create_change_log(bash, repo_dir=repo_dir, file_path=change_log_file_path, uploaded_shas=uploaded_shas, current_sha=current_sha)
       bash.execute(working_dir=working_dir, command="cp model3emu/Config/Supermodel.ini pkg/Config && cp model3emu/Config/Games.xml pkg/Config")
       bash.execute(working_dir=working_dir, command="echo NVRAM files go here. >pkg/NVRAM/DIR.txt")
       bash.execute(working_dir=working_dir, command="echo Save states go here. >pkg/Saves/DIR.txt")
@@ -263,7 +272,7 @@ def update_svn_snapshot(working_dir, username, password, test_run, make):
 
       # Update the web page
       print("Updating Download page HTML...")
-      html = add_new_file(html=html, filename=zip_file, version=version, svn_revision=current_revision)
+      html = add_new_file(html=html, filename=zip_file, version=version, git_sha=current_sha, date=current_date)
       html_file_path = os.path.join(working_dir, "Download.updated.html")
       write_html_file(html=html, file_path=html_file_path)
       if test_run:
@@ -300,10 +309,10 @@ if __name__ == "__main__":
       if os.path.exists(options.working_dir):
         raise Exception("Specified working directory (%s) already exists. This script requires a non-existent path that can safely be overwritten and then deleted." % options.working_dir)
       os.makedirs(options.working_dir)
-      failed = update_svn_snapshot(working_dir=options.working_dir, username=options.username, password=options.password, test_run=options.test_run, make=options.make)
+      failed = update_git_snapshot(working_dir=options.working_dir, username=options.username, password=options.password, test_run=options.test_run, make=options.make)
     else:
       with tempfile.TemporaryDirectory() as working_dir:
-        failed = update_svn_snapshot(working_dir=working_dir, username=options.username, password=options.password, test_run=options.test_run, make=options.make)
+        failed = update_git_snapshot(working_dir=working_dir, username=options.username, password=options.password, test_run=options.test_run, make=options.make)
     # Retry until success
     if not failed:
       break