Xargon/RetroDECK
1
0
Fork 0
mirror of https://github.com/RetroDECK/RetroDECK.git synced 2024-11-24 23:15:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

AUTOMATIONS: updated automation tools

Browse source
This commit is contained in:
XargonWan 2024-02-05 14:52:19 +01:00
parent b9bf9f17c5
commit 214a583ee8
11 changed files with 154 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
automation_tools/automation_task_list.cfg
View file

@ -6,7 +6,10 @@ hash^SAMEDUCKSHAPLACEHOLDER^https://buildbot.libretro.com/nightly/linux/x86_64/l
hash^PPSSPPBIOSHASHPLACEHOLDER^https://github.com/hrydgard/ppsspp/archive/refs/heads/master.zip
hash^MSXBIOSHASHPLACEHOLDER^http://bluemsx.msxblue.com/rel_download/blueMSXv282full.zip
hash^XEMUHDDHASHPLACEHOLDER^https://github.com/mborgerson/xemu-hdd-image/releases/latest/download/xbox_hdd.qcow2.zip
hash^VITA3KSHAPLACEHOLDER^https://github.com/Vita3K/Vita3K/releases/download/continuous/ubuntu-latest.zip
hash^RANIGHTLYCORESPLACEHOLDER^https://buildbot.libretro.com/nightly/linux/x86_64/RetroArch_cores.7z
hash^RETRODECKMAMEPLACEHOLDER^"https://github.com/XargonWan/RetroDECK-MAME/releases/download/$(curl -s https://api.github.com/repos/XargonWan/RetroDECK-MAME/releases/latest | grep -oP '"tag_name": "\K(.*?)(?=")')/RetroDECK-MAME-Artifact.tar.gz"
url^RETRODECKMAMEURLPLACEHOLDER^"https://github.com/XargonWan/RetroDECK-MAME/releases/download/$(curl -s https://api.github.com/repos/XargonWan/RetroDECK-MAME/releases/latest | grep -oP '"tag_name": "\K(.*?)(?=")')/RetroDECK-MAME-Artifact.tar.gz"
latestcommit^UNIVERSALDYNAMICINPUTCOMMITPLACEHOLDER^https://github.com/Venomalia/UniversalDynamicInput^main
outside_info^VERSIONPLACEHOLDER^${GITHUB_WORKSPACE}/buildid
outside_file^VERSIONPLACEHOLDER^${GITHUB_WORKSPACE}/buildid
branch^THISBRANCH

10
automation_tools/cooker_build_id.sh Normal file
View file

@ -0,0 +1,10 @@
#!/bin/bash
word1=$(shuf -n 1 ${GITHUB_WORKSPACE}/automation_tools/codename_wordlist.txt)
capitalized_word1="$(tr '[:lower:]' '[:upper:]' <<< ${word1:0:1})${word1:1}"
word2=$(shuf -n 1 ${GITHUB_WORKSPACE}/automation_tools/codename_wordlist.txt)
capitalized_word2="$(tr '[:lower:]' '[:upper:]' <<< ${word2:0:1})${word2:1}"
result=$capitalized_word1$capitalized_word2
echo $result > ${GITHUB_WORKSPACE}/buildid
echo "buildid=$result" >> $GITHUB_ENV
echo "VersionID is $result"

4
automation_tools/flatpak_build_bundle.sh
View file

@ -3,7 +3,7 @@
# This is building the bundle RetroDECK.flatpak after the download and build steps are done
if [ "${GITHUB_REF##*/}" = "main" ]; then
flatpak build-bundle ${GITHUB_WORKSPACE}/local RetroDECK.flatpak net.retrodeck.retrodeck
flatpak build-bundle "${GITHUB_WORKSPACE}/.local" "$GITHUB_WORKSPACE/RetroDECK.flatpak" net.retrodeck.retrodeck
else
flatpak build-bundle ${GITHUB_WORKSPACE}/local RetroDECK-cooker.flatpak net.retrodeck.retrodeck
flatpak build-bundle "${GITHUB_WORKSPACE}/.local" "$GITHUB_WORKSPACE/RetroDECK-cooker.flatpak" net.retrodeck.retrodeck
fi

12
automation_tools/flatpak_build_bundle_persistent.sh Executable file
View file

@ -0,0 +1,12 @@
#!/bin/bash
# This is building the bundle RetroDECK.flatpak after the download and build steps are done
BUILD_DIR="$HOME/cooker-persistent"
mkdir -p "$BUILD_DIR"
if [ "${GITHUB_REF##*/}" = "main" ]; then
flatpak build-bundle "${BUILD_DIR}/.local" "$GITHUB_WORKSPACE/RetroDECK.flatpak" net.retrodeck.retrodeck
else
flatpak build-bundle "${BUILD_DIR}/.local" "$GITHUB_WORKSPACE/RetroDECK-cooker.flatpak" net.retrodeck.retrodeck
fi

7
automation_tools/flatpak_build_download_only.sh
View file

@ -6,18 +6,19 @@ git config --global protocol.file.allow always
if [[ "${GITHUB_REF##*/}" == "main" ]]; then
BUNDLE_NAME="RetroDECK.flatpak"
FOLDER=retrodeck-flatpak-cooker
FOLDER=retrodeck-flatpak
else
BUNDLE_NAME="RetroDECK-cooker.flatpak"
FOLDER=retrodeck-flatpak-cooker
fi
mkdir -vp "${GITHUB_WORKSPACE}"/{local,retrodeck-flatpak-cooker}
mkdir -vp "${GITHUB_WORKSPACE}"/{.local,retrodeck-flatpak-cooker}
flatpak-builder --user --force-clean \
--install-deps-from=flathub \
--install-deps-from=flathub-beta \
--repo="${GITHUB_WORKSPACE}/local" \
--repo="${GITHUB_WORKSPACE}/.local" \
--download-only \
"${GITHUB_WORKSPACE}/${FOLDER}" \
net.retrodeck.retrodeck.yml

26
automation_tools/flatpak_build_download_only_persistent.sh Executable file
View file

@ -0,0 +1,26 @@
#!/bin/bash
# This script is downloading the needed files to prepare the manifest build
git config --global protocol.file.allow always
if [[ "${GITHUB_REF##*/}" == "main" ]]; then
BUNDLE_NAME="RetroDECK.flatpak"
FOLDER=retrodeck-flatpak
else
BUNDLE_NAME="RetroDECK-cooker.flatpak"
FOLDER=retrodeck-flatpak-cooker
fi
BUILD_DIR="$HOME/cooker-persistent"
mkdir -p "$BUILD_DIR"
mkdir -vp "${BUILD_DIR}"/{.local,retrodeck-flatpak-cooker}
flatpak-builder --user --force-clean \
--install-deps-from=flathub \
--install-deps-from=flathub-beta \
--repo="${BUILD_DIR}/.local" \
--download-only \
"${BUILD_DIR}/${FOLDER}" \
"${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml"

8
automation_tools/flatpak_build_only.sh
View file

@ -2,7 +2,7 @@
# This script is downloading the needed files to prepare the manifest build
git config --global protocol.file.allow always
git config protocol.file.allow always
if [ "${GITHUB_REF##*/}" = "main" ]; then
BUNDLE_NAME="RetroDECK.flatpak"
@ -12,13 +12,13 @@ else
FOLDER=retrodeck-flatpak-cooker
fi
mkdir -vp ${GITHUB_WORKSPACE}/local
mkdir -vp ${GITHUB_WORKSPACE}/.local
mkdir -vp ${GITHUB_WORKSPACE}/"$FOLDER"
flatpak-builder --user --force-clean \
--install-deps-from=flathub \
--install-deps-from=flathub-beta \
--repo=${GITHUB_WORKSPACE}/local \
--repo=${GITHUB_WORKSPACE}/.local \
--disable-download \
${GITHUB_WORKSPACE}/"$FOLDER" \
"${GITHUB_WORKSPACE}/$FOLDER" \
net.retrodeck.retrodeck.yml

25
automation_tools/flatpak_build_only_persistent.sh Executable file
View file

@ -0,0 +1,25 @@
#!/bin/bash
# This script is downloading the needed files to prepare the manifest build
git config protocol.file.allow always
if [ "${GITHUB_REF##*/}" = "main" ]; then
BUNDLE_NAME="RetroDECK.flatpak"
FOLDER=retrodeck-flatpak-main
else
BUNDLE_NAME="RetroDECK-cooker.flatpak"
FOLDER=retrodeck-flatpak-cooker
fi
BUILD_DIR="$HOME/cooker-persistent"
mkdir -vp ${$BUILD_DIR}/.local
mkdir -vp ${$BUILD_DIR}/"$FOLDER"
flatpak-builder --user --force-clean \
--install-deps-from=flathub \
--install-deps-from=flathub-beta \
--repo="${BUILD_DIR}/.local" \
--disable-download \
"${BUILD_DIR}/${FOLDER}" \
"${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml"

33
automation_tools/main_version_checker.sh
View file

@ -1,13 +1,10 @@
#!/bin/bash
# This script is intended to gather version information from various sources:
# RetroDECK repository
# Appdata.xml file
# Manifest YAML file
# It consists of three functions, each responsible for retrieving a specific version-related data.
# This script is used to check that the versions are correct and stopping the pipeline if something is wrong.
# This is designed to be run on the main pipeline to check that everything is in order before building RetroDECK.
source automation_tools/version_extractor.sh
# Set the file paths
appdata="net.retrodeck.retrodeck.appdata.xml"
manifest="net.retrodeck.retrodeck.yml"
manifest_content=$(cat "$manifest")
@ -35,30 +32,6 @@ compare_versions() {
return 0 # Versions are equal
}
fetch_repo_version(){
# Getting latest RetroDECK release info
LATEST_RELEASE=$(curl -s "https://api.github.com/repos/XargonWan/RetroDECK/releases/latest")
# Extracting tag name from the latest release
repo_version=$(echo "$LATEST_RELEASE" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
# Printing results
echo "$repo_version"
}
fetch_appdata_version(){
# Extract the version from the net.retrodeck.retrodeck.appdata.xml file
appdata_version=$(grep -oPm1 "(?<=<release version=\")[^\"]+" "$appdata")
echo "$appdata_version"
}
fetch_manifest_version(){
# Use awk to extract the value of the first iteration of VERSION variable
manifest_version=$(echo "$manifest_content" | awk '/VERSION=/ && !/#/ { sub(/.*VERSION=/, ""); sub(/#.*/, ""); print; exit }')
# Trim leading and trailing whitespace
manifest_version=$(echo "$manifest_version" | awk '{gsub(/[^0-9.a-zA-Z]/,""); print}')
echo "$manifest_version"
}
repo_version=$(fetch_repo_version)
echo -e "Online repository:\t$repo_version"

5
automation_tools/manifest_validation.sh
View file

@ -1,5 +0,0 @@
#!/bin/bash
flatpak install -y flathub org.flatpak.Builder
flatpak run --command=appstream-util org.flatpak.Builder validate -v net.retrodeck.retrodeck.appdata.xml

72
automation_tools/pre_build_automation.sh
View file

@ -3,20 +3,33 @@
# For the file paths to work correctly, call this script with this command from the cloned repo folder root:
# sh automation_tools/pre_build_automation.sh
# Different actions need different information in the task list file
# branch: This changes the placeholder text to the currently-detected GIT branch if an automated build was started from a PR environment.
# hash: Finds the SHA256 hash of a file online and updates the placeholder in the manifest.
# Needs the URL of the file, in this line format: hash^PLACEHOLDERTEXT^url
# latestcommit: Finds the most recent commit of a git repo and updated the placeholder in the manifest.
# Needs the URL of the repo and the branch to find the latest commit from, in this line format: latestcommit^PLACEHOLDERTEXT^url^branch
# latestappimage: Finds the download URL and SHA256 hash of the latest AppImage release from a git repo
# latestappimage: Finds the download URL and SHA256 hash of the latest AppImage release from a git repo.
# Needs the API URL of the repo, in this line format: latestappimage^PLACEHOLDERTEXT^https://api.github.com/repos/<owner-name>/<repo-name>/releases/latest
# As this command updates two different placeholders (one for the URL, one for the file hash) in the manifest,
# the URL that would be used in the above example is "PLACEHOLDERTEXT" and the hash placeholder text would be "HASHPLACEHOLDERTEXT"
# The "HASH" prefix of the placeholder text is hardcoded in the script
# outside_file: Prints the contents of a file from the build environment (such as the buildid file) and replaces the placeholder text with those contents.
# outside_env_var: Gets the value of an environmental variable from the build environment (the output of "echo $var" from the terminal) and replaces the placeholder text with that value.
# custom_command: Runs a single command explicitly as written in the $URL field of the task list, including variable and command expansion. This should work the same as if you were runnig the command directly from the terminal.
# This command does not need a PLACEHOLDERTEXT field in the task list, so needs to be in this syntax: custom_command^^$COMMAND
# url: This is used to calculate a dynamic URL and the value to the $caluculated_url environmental variable, for use in other subsequent commands.
rd_manifest=${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml
automation_task_list=${GITHUB_WORKSPACE}/automation_tools/automation_task_list.cfg
current_branch=$(git rev-parse --abbrev-ref HEAD)
# During the PR automated tests instead of the branch name is returned "HEAD", fixing it
if [ $current_branch == "HEAD" ]; then
echo "Looks like we are on a PR environment, retrieving the branch name from which the PR is raised."
current_branch=$(echo $GITHUB_REF | sed 's@refs/heads/@@')
echo "The branch name from which the PR is raised is \"$current_branch\"."
fi
echo "Manifest location: $rd_manifest"
echo "Automation task list location: $automation_task_list"
echo
@ -28,21 +41,28 @@ echo
while IFS="^" read -r action placeholder url branch
do
if [[ ! $action == "#"* ]] && [[ ! -z "$action" ]]; then
if [[ "$action" == "branch" ]]; then
case "$action" in
"branch" )
echo
echo "Placeholder text: $placeholder"
echo "Current branch:" "$current_branch"
echo
/bin/sed -i 's^'"$placeholder"'^'"$current_branch"'^g' $rd_manifest
elif [[ "$action" == "hash" ]]; then
;;
"hash" )
echo
echo "Placeholder text: $placeholder"
echo "URL to hash: $url"
calculated_url=$(eval echo "$url") # in case the url has to be calculated from an expression
echo "URL to hash: $calculated_url"
echo
hash=$(curl -sL "$url" | sha256sum | cut -d ' ' -f1)
hash=$(curl -sL "$calculated_url" | sha256sum | cut -d ' ' -f1)
echo "Hash found: $hash"
/bin/sed -i 's^'"$placeholder"'^'"$hash"'^' $rd_manifest
elif [[ "$action" == "latestcommit" ]]; then
;;
"latestcommit" )
echo
echo "Placeholder text: $placeholder"
echo "Repo to get latest commit from: $url branch: $branch"
@ -50,7 +70,9 @@ do
commit=$(git ls-remote "$url" "$branch" | cut -f1)
echo "Commit found: $commit"
/bin/sed -i 's^'"$placeholder"'^'"$commit"'^' $rd_manifest
elif [[ "$action" == "latestappimage" ]]; then
;;
"latestappimage" )
echo
echo "Placeholder text: $placeholder"
echo "Repo to look for AppImage releases: $url"
@ -61,7 +83,9 @@ do
appimagehash=$(curl -sL "$appimageurl" | sha256sum | cut -d ' ' -f1)
echo "AppImage hash found: $appimagehash"
/bin/sed -i 's^'"HASHFOR$placeholder"'^'"$appimagehash"'^' $rd_manifest
elif [[ "$action" == "outside_info" ]]; then
;;
"outside_file" )
if [[ "$url" = \$* ]]; then # If value is a reference to a variable name
eval url="$url"
fi
@ -70,6 +94,36 @@ do
echo "Information being injected: $(cat $url)"
echo
/bin/sed -i 's^'"$placeholder"'^'"$(cat $url)"'^' $rd_manifest
fi
;;
"outside_env_var" )
if [[ "$url" = \$* ]]; then # If value is a reference to a variable name
eval url="$url"
fi
echo
echo "Placeholder text: $placeholder"
echo "Information being injected: $(echo $url)"
echo
/bin/sed -i 's^'"$placeholder"'^'"$(echo $url)"'^' $rd_manifest
;;
"custom_command" )
echo
echo "Command to run: $url"
echo
eval "$url"
;;
"url" )
# this is used to calculate a dynamic url
echo
echo "Placeholder text: $placeholder"
calculated_url=$(eval echo "$url")
echo "Information being injected: $calculated_url"
echo
/bin/sed -i 's^'"$placeholder"'^'"$calculated_url"'^' $rd_manifest
;;
esac
fi
done < "$automation_task_list"