Xargon/Supermodel
1
0
Fork 0
mirror of https://github.com/RetroDECK/Supermodel.git synced 2024-11-22 05:45:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

supermodel_build_bot.py: New build bot script for GitHub

Browse source
This commit is contained in:
Bart Trzynadlowski 2022-06-23 14:06:07 -07:00
parent 35a7e8a60b
commit 46eff8c5eb
1 changed files with 321 additions and 312 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

131
Scripts/update_model3emu_snapshot.py → Scripts/supermodel_build_bot.py
View file

@ -1,8 +1,7 @@
#
# Supermodel
# A Sega Model 3 Arcade Emulator.
# Copyright 2011-2021 Bart Trzynadlowski, Nik Henson, Ian Curtis,
# Harry Tuttle, and Spindizzi
# Copyright 2003-2022 The Supermodel Team
#
# This file is part of Supermodel.
#
@ -21,9 +20,9 @@
#
#
# update_model3emu_snapshot.py
# supermodel_build_bot.py
#
# SVN snapshot build script. Checks latest SVN revision against Supermodel3.com
# Git snapshot build script. Checks latest Git commit against Supermodel3.com
# download page and builds and uploads a snapshot if needed. Intended to be run
# at least once daily as part of an automated job.
#
@ -35,9 +34,11 @@
# - mingw64/mingw-w64-x86_64-SDL2_net package
# - msys/subversion package
# - msys/zip package
# - git
#
# To perform a test run:
# - Download https://supermodel3.com/Download.html
# - Download https://supermodel3.com/Download.html to the directory from
# which the script will be run.
# - Run: python update_model3emu_snapshot.py --working-dir=c:\tmp\build --test-run
#
@ -66,7 +67,7 @@ class Bash:
def execute(self, working_dir, command, log=False, print_output=False):
import subprocess
working_dir = working_dir.replace("\\", "/") # convert to UNIX-style path
invocation = self.bash_path + " -l -c \"" + "cd " + working_dir + " && " + command + "\""
invocation = self.bash_path + " -l -c '" + "cd " + working_dir + " && " + command + "'"
if log:
print("Executing: %s" % invocation)
result = subprocess.run(invocation, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
@ -74,7 +75,7 @@ class Bash:
print(result.stdout.decode())
return result
def get_web_page(url, test_run):
def get_web_page(test_run):
if test_run:
with open("Download.html") as fp:
html = fp.read()
@ -91,43 +92,43 @@ def get_comment(line):
return line[comment_begin+4:comment_end].strip()
return None
def add_new_file(html, filename, version, svn_revision):
# Scan for: <!-- BEGIN_SVN_SNAPSHOTS $TEMPLATE -->
def add_new_file(html, filename, version, git_sha, date):
# Scan for: <!-- BEGIN_GIT_SNAPSHOTS $TEMPLATE -->
lines = html.splitlines()
for i in range(len(lines)):
line = lines[i]
text = get_comment(line)
if text and text.startswith("BEGIN_SVN_SNAPSHOTS "):
template = text[len("BEGIN_SVN_SNAPSHOTS "):]
new_row = template.replace("$FILENAME", filename).replace("$VERSION", version).replace("$REVISION", svn_revision)
svn_tag = "<!-- SVN_REVISION=%s -->" % svn_revision
lines.insert(i + 1, new_row + " " + svn_tag)
if text and text.startswith("BEGIN_GIT_SNAPSHOTS "):
template = text[len("BEGIN_GIT_SNAPSHOTS "):]
new_row = template.replace("$FILENAME", filename).replace("$VERSION", version).replace("$GIT_SHA", git_sha).replace("$DATE", date)
git_tag = "<!-- GIT_SHA=%s -->" % git_sha
lines.insert(i + 1, new_row + " " + git_tag)
html = "\n".join(lines)
return html
raise ParseError("BEGIN_SVN_SNAPSHOTS not found")
raise ParseError("BEGIN_GIT_SNAPSHOTS not found")
def get_uploaded_svn_revisions(html):
# Scan for all lines with: <!-- SVN_REVISION=$REVISION -->
revisions = []
def get_uploaded_git_shas(html):
# Scan for all lines with: <!-- GIT_SHA=$SHA -->
shas = []
for line in html.splitlines():
text = get_comment(line)
if text and text.startswith("SVN_REVISION"):
if text and text.startswith("GIT_SHA"):
tokens = text.split("=")
if len(tokens) == 2:
revision = tokens[1]
revisions.append(revision)
sha = tokens[1]
shas.append(sha)
else:
raise ParseError("Error parsing SVN_REVISION")
revisions.reverse()
return revisions
raise ParseError("Error parsing GIT_SHA")
shas.reverse()
return shas
def create_change_log(bash, repo_dir, file_path, uploaded_revisions, current_revision):
from_revision = uploaded_revisions[0] if len(uploaded_revisions) > 0 else current_revision
result = bash.execute(working_dir=repo_dir, command="svn log -r " + current_revision + ":" + from_revision)
def create_change_log(bash, repo_dir, file_path, uploaded_shas, current_sha):
# Log from first commit after 0.2a release until now
result = bash.execute(working_dir=repo_dir, command="git -c color.ui=never log 06594a5..." + current_sha + " --no-decorate --pretty=\"Commit: %H%nAuthor: %an%nDate: %ad%n%n%w(80,4,4)%B\"")
if result.returncode != 0:
return PackageError("Unable to obtain SVN log")
return PackageError("Unable to obtain Git log")
change_log = result.stdout.decode().strip()
with open(file_path, "w") as fp:
with open(file_path, "w", encoding="utf-8") as fp:
header = "\n\n" \
" #### ### ###\n" \
" ## ## ## ##\n" \
@ -140,10 +141,9 @@ def create_change_log(bash, repo_dir, file_path, uploaded_revisions, current_rev
"\n" \
" A Sega Model 3 Arcade Emulator.\n" \
"\n" \
" Copyright 2011-2021 Bart Trzynadlowski, Nik Henson, Ian Curtis,\n" \
" Harry Tuttle, and Spindizzi\n" \
" Copyright 2003-2022 The Supermodel Team\n" \
"\n" \
" SVN CHANGE LOG\n" \
" CHANGE LOG\n" \
"\n" \
"\n"
fp.write(header)
@ -157,7 +157,7 @@ def upload(html_file, zip_file_path, username, password):
from ftplib import FTP
ftp = FTP("supermodel3.com")
ftp.login(username, password)
ftp.cwd("public_html/Files/SVN_Snapshots")
ftp.cwd("public_html/Files/Git_Snapshots")
with open(zip_file_path, "rb") as fp:
ftp.storbinary("STOR " + os.path.basename(zip_file_path), fp)
ftp.cwd("../../")
@ -177,15 +177,17 @@ def confirm_package_contents(package_dir, package_files):
def rmdir(dir):
def delete_readonly(action, name, exc):
# https://stackoverflow.com/questions/2656322/shutil-rmtree-fails-on-windows-with-access-is-denied
import stat
os.chmod(name, stat.S_IWRITE)
os.remove(name)
for root, dirs, files in os.walk(dir): # we expect .svn to be write-protected
if '.svn' in dirs:
shutil.rmtree(root+'\.svn',onerror=delete_readonly)
if not os.access(name, os.W_OK):
os.chmod(name, stat.S_IWUSR)
action(name)
for root, dirs, files in os.walk(dir): # we expect .git to be write-protected
if '.git' in dirs:
shutil.rmtree(root+'\.git',onerror=delete_readonly)
shutil.rmtree(dir)
def update_svn_snapshot(working_dir, username, password, test_run, make):
def update_git_snapshot(working_dir, username, password, test_run, make):
failed = False
print("Starting %s in working directory: %s" % ("test run" if test_run else "release process", working_dir))
@ -193,27 +195,34 @@ def update_svn_snapshot(working_dir, username, password, test_run, make):
bash = Bash(bash_path="c:\\msys64\\usr\\bin\\bash.exe")
repo_dir = os.path.join(working_dir, "model3emu")
# Fetch Supermodel download page and compare most recent uploaded SVN snapshot against current SVN revision
print("Fetching Supermodel download page...")
html = get_web_page(url="https://supermodel3.com/Forum/Download.html", test_run=test_run)
uploaded_revisions = get_uploaded_svn_revisions(html=html)
last_uploaded_revision = uploaded_revisions[-1] if len(uploaded_revisions) > 0 else None
print("Checking current SVN revision...")
result = bash.execute(working_dir=working_dir, command="svn info --show-item revision https://svn.code.sf.net/p/model3emu/code/trunk")
# Clone Git repo. Unfortunately need to always do this in order to get short SHA
result = bash.execute(working_dir=working_dir, command="git clone https://github.com/trzy/Supermodel.git model3emu")
if result.returncode != 0:
raise CheckoutError("SVN revision check failed")
current_revision = result.stdout.decode().strip()
raise CheckoutError("Git clone failed")
# If SVN has a newer version, or if performing a test run, build it and update the web page
if current_revision == last_uploaded_revision and (not test_run):
print("Nothing to do. Revision already uploaded: %s" % current_revision)
# Fetch Supermodel download page and compare most recent uploaded Git snapshot against current Git sha
print("Fetching Supermodel download page...")
html = get_web_page(test_run=test_run)
uploaded_shas = get_uploaded_git_shas(html=html)
last_uploaded_sha = uploaded_shas[-1] if len(uploaded_shas) > 0 else None
print("Checking current Git SHA...")
result = bash.execute(working_dir=repo_dir, command="git rev-parse --short HEAD")
if result.returncode != 0:
raise CheckoutError("Git HEAD SHA check failed")
current_sha = result.stdout.decode().strip()
# Get date of commit
result = bash.execute(working_dir=repo_dir, command="git show -s --format=%as HEAD")
if result.returncode != 0:
raise CheckoutError("Unable to obtain HEAD commit date")
current_date = result.stdout.decode().strip()
# If Git has a newer version, or if performing a test run, build it and update the web page
if current_sha == last_uploaded_sha and (not test_run):
print("Nothing to do. Current Git SHA already uploaded: %s" % current_sha)
else:
# Check out
print("SVN revision is %s but uploaded revision is %s" % (current_revision, last_uploaded_revision))
print("Checking out Supermodel...")
result = bash.execute(working_dir=working_dir, command="svn co https://svn.code.sf.net/p/model3emu/code/trunk model3emu")
if result.returncode != 0:
raise CheckoutError("SVN checkout failed")
print("Git SHA is %s but last uploaded SHA is %s" % (current_sha, last_uploaded_sha))
# Build
print("Building Supermodel...")
@ -221,7 +230,7 @@ def update_svn_snapshot(working_dir, username, password, test_run, make):
if result.returncode != 0:
raise BuildError("Failed to obtain version")
version = result.stdout.decode().strip()
result = bash.execute(working_dir=repo_dir, command=make + " -f Makefiles/Makefile.Win32 release NET_BOARD=1")
result = bash.execute(working_dir=repo_dir, command=make + " -f Makefiles/Makefile.Win32 release NET_BOARD=1", print_output=False)
if result.returncode != 0:
raise BuildError("Build failed")
@ -230,7 +239,7 @@ def update_svn_snapshot(working_dir, username, password, test_run, make):
pkg_dir = os.path.join(working_dir, "pkg")
bash.execute(working_dir=working_dir, command="mkdir pkg && mkdir pkg/Config && mkdir pkg/NVRAM && mkdir pkg/Saves && mkdir pkg/ROMs")
change_log_file_path = os.path.join(pkg_dir, "CHANGES.txt")
create_change_log(bash, repo_dir=repo_dir, file_path=change_log_file_path, uploaded_revisions=uploaded_revisions, current_revision=current_revision)
create_change_log(bash, repo_dir=repo_dir, file_path=change_log_file_path, uploaded_shas=uploaded_shas, current_sha=current_sha)
bash.execute(working_dir=working_dir, command="cp model3emu/Config/Supermodel.ini pkg/Config && cp model3emu/Config/Games.xml pkg/Config")
bash.execute(working_dir=working_dir, command="echo NVRAM files go here. >pkg/NVRAM/DIR.txt")
bash.execute(working_dir=working_dir, command="echo Save states go here. >pkg/Saves/DIR.txt")
@ -263,7 +272,7 @@ def update_svn_snapshot(working_dir, username, password, test_run, make):
# Update the web page
print("Updating Download page HTML...")
html = add_new_file(html=html, filename=zip_file, version=version, svn_revision=current_revision)
html = add_new_file(html=html, filename=zip_file, version=version, git_sha=current_sha, date=current_date)
html_file_path = os.path.join(working_dir, "Download.updated.html")
write_html_file(html=html, file_path=html_file_path)
if test_run:
@ -300,10 +309,10 @@ if __name__ == "__main__":
if os.path.exists(options.working_dir):
raise Exception("Specified working directory (%s) already exists. This script requires a non-existent path that can safely be overwritten and then deleted." % options.working_dir)
os.makedirs(options.working_dir)
failed = update_svn_snapshot(working_dir=options.working_dir, username=options.username, password=options.password, test_run=options.test_run, make=options.make)
failed = update_git_snapshot(working_dir=options.working_dir, username=options.username, password=options.password, test_run=options.test_run, make=options.make)
else:
with tempfile.TemporaryDirectory() as working_dir:
failed = update_svn_snapshot(working_dir=working_dir, username=options.username, password=options.password, test_run=options.test_run, make=options.make)
failed = update_git_snapshot(working_dir=working_dir, username=options.username, password=options.password, test_run=options.test_run, make=options.make)
# Retry until success
if not failed:
break