AUTOMATIONS: updated automation tools

automation_tools/automation_task_list.cfg

@@ -6,7 +6,10 @@ hash^SAMEDUCKSHAPLACEHOLDER^https://buildbot.libretro.com/nightly/linux/x86_64/l
 hash^PPSSPPBIOSHASHPLACEHOLDER^https://github.com/hrydgard/ppsspp/archive/refs/heads/master.zip
 hash^MSXBIOSHASHPLACEHOLDER^http://bluemsx.msxblue.com/rel_download/blueMSXv282full.zip
 hash^XEMUHDDHASHPLACEHOLDER^https://github.com/mborgerson/xemu-hdd-image/releases/latest/download/xbox_hdd.qcow2.zip
+hash^VITA3KSHAPLACEHOLDER^https://github.com/Vita3K/Vita3K/releases/download/continuous/ubuntu-latest.zip
 hash^RANIGHTLYCORESPLACEHOLDER^https://buildbot.libretro.com/nightly/linux/x86_64/RetroArch_cores.7z
+hash^RETRODECKMAMEPLACEHOLDER^"https://github.com/XargonWan/RetroDECK-MAME/releases/download/$(curl -s https://api.github.com/repos/XargonWan/RetroDECK-MAME/releases/latest | grep -oP '"tag_name": "\K(.*?)(?=")')/RetroDECK-MAME-Artifact.tar.gz"
+url^RETRODECKMAMEURLPLACEHOLDER^"https://github.com/XargonWan/RetroDECK-MAME/releases/download/$(curl -s https://api.github.com/repos/XargonWan/RetroDECK-MAME/releases/latest | grep -oP '"tag_name": "\K(.*?)(?=")')/RetroDECK-MAME-Artifact.tar.gz"
 latestcommit^UNIVERSALDYNAMICINPUTCOMMITPLACEHOLDER^https://github.com/Venomalia/UniversalDynamicInput^main
-outside_info^VERSIONPLACEHOLDER^${GITHUB_WORKSPACE}/buildid
+outside_file^VERSIONPLACEHOLDER^${GITHUB_WORKSPACE}/buildid
 branch^THISBRANCH

automation_tools/cooker_build_id.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+word1=$(shuf -n 1 ${GITHUB_WORKSPACE}/automation_tools/codename_wordlist.txt)
+capitalized_word1="$(tr '[:lower:]' '[:upper:]' <<< ${word1:0:1})${word1:1}"
+word2=$(shuf -n 1 ${GITHUB_WORKSPACE}/automation_tools/codename_wordlist.txt)
+capitalized_word2="$(tr '[:lower:]' '[:upper:]' <<< ${word2:0:1})${word2:1}"
+result=$capitalized_word1$capitalized_word2
+echo $result > ${GITHUB_WORKSPACE}/buildid
+echo "buildid=$result" >> $GITHUB_ENV
+echo "VersionID is $result"

automation_tools/flatpak_build_bundle.sh

@@ -3,7 +3,7 @@
 # This is building the bundle RetroDECK.flatpak after the download and build steps are done
 
 if [ "${GITHUB_REF##*/}" = "main" ]; then
-    flatpak build-bundle ${GITHUB_WORKSPACE}/local RetroDECK.flatpak net.retrodeck.retrodeck
+    flatpak build-bundle "${GITHUB_WORKSPACE}/.local" "$GITHUB_WORKSPACE/RetroDECK.flatpak" net.retrodeck.retrodeck
 else
-    flatpak build-bundle ${GITHUB_WORKSPACE}/local RetroDECK-cooker.flatpak net.retrodeck.retrodeck
+    flatpak build-bundle "${GITHUB_WORKSPACE}/.local" "$GITHUB_WORKSPACE/RetroDECK-cooker.flatpak" net.retrodeck.retrodeck
 fi

automation_tools/flatpak_build_bundle_persistent.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# This is building the bundle RetroDECK.flatpak after the download and build steps are done
+
+BUILD_DIR="$HOME/cooker-persistent"
+mkdir -p "$BUILD_DIR"
+
+if [ "${GITHUB_REF##*/}" = "main" ]; then
+    flatpak build-bundle "${BUILD_DIR}/.local" "$GITHUB_WORKSPACE/RetroDECK.flatpak" net.retrodeck.retrodeck
+else
+    flatpak build-bundle "${BUILD_DIR}/.local" "$GITHUB_WORKSPACE/RetroDECK-cooker.flatpak" net.retrodeck.retrodeck
+fi

automation_tools/flatpak_build_download_only.sh

@@ -6,18 +6,19 @@ git config --global protocol.file.allow always
 
 if [[ "${GITHUB_REF##*/}" == "main" ]]; then
     BUNDLE_NAME="RetroDECK.flatpak"
-    FOLDER=retrodeck-flatpak-cooker
+    FOLDER=retrodeck-flatpak
 else
     BUNDLE_NAME="RetroDECK-cooker.flatpak"
     FOLDER=retrodeck-flatpak-cooker
 fi
 
-mkdir -vp "${GITHUB_WORKSPACE}"/{local,retrodeck-flatpak-cooker}
+mkdir -vp "${GITHUB_WORKSPACE}"/{.local,retrodeck-flatpak-cooker}
 
 flatpak-builder --user --force-clean \
     --install-deps-from=flathub \
     --install-deps-from=flathub-beta \
-    --repo="${GITHUB_WORKSPACE}/local" \
+    --repo="${GITHUB_WORKSPACE}/.local" \
     --download-only \
     "${GITHUB_WORKSPACE}/${FOLDER}" \
     net.retrodeck.retrodeck.yml
+    

automation_tools/flatpak_build_download_only_persistent.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# This script is downloading the needed files to prepare the manifest build
+
+git config --global protocol.file.allow always
+
+if [[ "${GITHUB_REF##*/}" == "main" ]]; then
+    BUNDLE_NAME="RetroDECK.flatpak"
+    FOLDER=retrodeck-flatpak
+else
+    BUNDLE_NAME="RetroDECK-cooker.flatpak"
+    FOLDER=retrodeck-flatpak-cooker
+fi
+
+BUILD_DIR="$HOME/cooker-persistent"
+mkdir -p "$BUILD_DIR"
+mkdir -vp "${BUILD_DIR}"/{.local,retrodeck-flatpak-cooker}
+
+flatpak-builder --user --force-clean \
+--install-deps-from=flathub \
+--install-deps-from=flathub-beta \
+--repo="${BUILD_DIR}/.local" \
+--download-only \
+"${BUILD_DIR}/${FOLDER}" \
+"${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml"
+

automation_tools/flatpak_build_only.sh

@@ -2,7 +2,7 @@
 
 # This script is downloading the needed files to prepare the manifest build
 
-git config --global protocol.file.allow always
+git config protocol.file.allow always
 
 if [ "${GITHUB_REF##*/}" = "main" ]; then
     BUNDLE_NAME="RetroDECK.flatpak"
@@ -12,13 +12,13 @@ else
     FOLDER=retrodeck-flatpak-cooker
 fi
 
-mkdir -vp ${GITHUB_WORKSPACE}/local
+mkdir -vp ${GITHUB_WORKSPACE}/.local
 mkdir -vp ${GITHUB_WORKSPACE}/"$FOLDER"
 
 flatpak-builder --user --force-clean \
     --install-deps-from=flathub \
     --install-deps-from=flathub-beta \
-    --repo=${GITHUB_WORKSPACE}/local \
+    --repo=${GITHUB_WORKSPACE}/.local \
     --disable-download \
-    ${GITHUB_WORKSPACE}/"$FOLDER" \
+    "${GITHUB_WORKSPACE}/$FOLDER" \
     net.retrodeck.retrodeck.yml

automation_tools/flatpak_build_only_persistent.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# This script is downloading the needed files to prepare the manifest build
+
+git config protocol.file.allow always
+
+if [ "${GITHUB_REF##*/}" = "main" ]; then
+    BUNDLE_NAME="RetroDECK.flatpak"
+    FOLDER=retrodeck-flatpak-main
+else
+    BUNDLE_NAME="RetroDECK-cooker.flatpak"
+    FOLDER=retrodeck-flatpak-cooker
+fi
+
+BUILD_DIR="$HOME/cooker-persistent"
+mkdir -vp ${$BUILD_DIR}/.local
+mkdir -vp ${$BUILD_DIR}/"$FOLDER"
+
+flatpak-builder --user --force-clean \
+--install-deps-from=flathub \
+--install-deps-from=flathub-beta \
+--repo="${BUILD_DIR}/.local" \
+--disable-download \
+"${BUILD_DIR}/${FOLDER}" \
+"${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml"

automation_tools/main_version_checker.sh

@@ -1,13 +1,10 @@
 #!/bin/bash
-
+# This script is used to check that the versions are correct and stopping the pipeline if something is wrong.
-# This script is intended to gather version information from various sources:
+# This is designed to be run on the main pipeline to check that everything is in order before building RetroDECK.
-# RetroDECK repository
-# Appdata.xml file
-# Manifest YAML file
-# It consists of three functions, each responsible for retrieving a specific version-related data.
 
 source automation_tools/version_extractor.sh
 
+# Set the file paths
 appdata="net.retrodeck.retrodeck.appdata.xml"
 manifest="net.retrodeck.retrodeck.yml"
 manifest_content=$(cat "$manifest")
@@ -35,30 +32,6 @@ compare_versions() {
     return 0  # Versions are equal
 }
 
-
-fetch_repo_version(){
-    # Getting latest RetroDECK release info
-    LATEST_RELEASE=$(curl -s "https://api.github.com/repos/XargonWan/RetroDECK/releases/latest")
-    # Extracting tag name from the latest release
-    repo_version=$(echo "$LATEST_RELEASE" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
-    # Printing results
-    echo "$repo_version"
-}
-
-fetch_appdata_version(){
-    # Extract the version from the net.retrodeck.retrodeck.appdata.xml file
-    appdata_version=$(grep -oPm1 "(?<=<release version=\")[^\"]+" "$appdata")
-    echo "$appdata_version"
-}
-
-fetch_manifest_version(){
-    # Use awk to extract the value of the first iteration of VERSION variable
-    manifest_version=$(echo "$manifest_content" | awk '/VERSION=/ && !/#/ { sub(/.*VERSION=/, ""); sub(/#.*/, ""); print; exit }')
-    # Trim leading and trailing whitespace
-    manifest_version=$(echo "$manifest_version" | awk '{gsub(/[^0-9.a-zA-Z]/,""); print}')
-    echo "$manifest_version"
-}
-
 repo_version=$(fetch_repo_version)
 echo -e "Online repository:\t$repo_version"
 

automation_tools/manifest_validation.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-flatpak install -y flathub org.flatpak.Builder
-
-flatpak run --command=appstream-util org.flatpak.Builder validate -v net.retrodeck.retrodeck.appdata.xml

automation_tools/pre_build_automation.sh

@@ -3,20 +3,33 @@
 # For the file paths to work correctly, call this script with this command from the cloned repo folder root:
 # sh automation_tools/pre_build_automation.sh
 # Different actions need different information in the task list file
+# branch: This changes the placeholder text to the currently-detected GIT branch if an automated build was started from a PR environment.
 # hash: Finds the SHA256 hash of a file online and updates the placeholder in the manifest. 
 #     Needs the URL of the file, in this line format: hash^PLACEHOLDERTEXT^url
 # latestcommit: Finds the most recent commit of a git repo and updated the placeholder in the manifest.
 #     Needs the URL of the repo and the branch to find the latest commit from, in this line format: latestcommit^PLACEHOLDERTEXT^url^branch
-# latestappimage: Finds the download URL and SHA256 hash of the latest AppImage release from a git repo
+# latestappimage: Finds the download URL and SHA256 hash of the latest AppImage release from a git repo.
 #     Needs the API URL of the repo, in this line format: latestappimage^PLACEHOLDERTEXT^https://api.github.com/repos/<owner-name>/<repo-name>/releases/latest
 #     As this command updates two different placeholders (one for the URL, one for the file hash) in the manifest, 
 #     the URL that would be used in the above example is "PLACEHOLDERTEXT" and the hash placeholder text would be "HASHPLACEHOLDERTEXT"
 #     The "HASH" prefix of the placeholder text is hardcoded in the script
+# outside_file: Prints the contents of a file from the build environment (such as the buildid file) and replaces the placeholder text with those contents.
+# outside_env_var: Gets the value of an environmental variable from the build environment (the output of "echo $var" from the terminal) and replaces the placeholder text with that value.
+# custom_command: Runs a single command explicitly as written in the $URL field of the task list, including variable and command expansion. This should work the same as if you were runnig the command directly from the terminal.
+#     This command does not need a PLACEHOLDERTEXT field in the task list, so needs to be in this syntax: custom_command^^$COMMAND
+# url: This is used to calculate a dynamic URL and the value to the $caluculated_url environmental variable, for use in other subsequent commands.
 
 rd_manifest=${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml
 automation_task_list=${GITHUB_WORKSPACE}/automation_tools/automation_task_list.cfg
 current_branch=$(git rev-parse --abbrev-ref HEAD)
 
+# During the PR automated tests instead of the branch name is returned "HEAD", fixing it
+if [ $current_branch == "HEAD" ]; then
+  echo "Looks like we are on a PR environment, retrieving the branch name from which the PR is raised."
+  current_branch=$(echo $GITHUB_REF | sed 's@refs/heads/@@')
+  echo "The branch name from which the PR is raised is \"$current_branch\"."
+fi
+
 echo "Manifest location: $rd_manifest"
 echo "Automation task list location: $automation_task_list"
 echo
@@ -28,21 +41,28 @@ echo
 while IFS="^" read -r action placeholder url branch
 do
   if [[ ! $action == "#"* ]] && [[ ! -z "$action" ]]; then
-    if [[ "$action" == "branch" ]]; then
+    case "$action" in
+
+    "branch" )
       echo
       echo "Placeholder text: $placeholder"
       echo "Current branch:" "$current_branch"
       echo
       /bin/sed -i 's^'"$placeholder"'^'"$current_branch"'^g' $rd_manifest
-    elif [[ "$action" == "hash" ]]; then
+    ;;
+
+    "hash" )
       echo
       echo "Placeholder text: $placeholder"
-      echo "URL to hash: $url"
+      calculated_url=$(eval echo "$url")  # in case the url has to be calculated from an expression
+      echo "URL to hash: $calculated_url"
       echo
-      hash=$(curl -sL "$url" | sha256sum | cut -d ' ' -f1)
+      hash=$(curl -sL "$calculated_url" | sha256sum | cut -d ' ' -f1)
       echo "Hash found: $hash"
       /bin/sed -i 's^'"$placeholder"'^'"$hash"'^' $rd_manifest
-    elif [[ "$action" == "latestcommit" ]]; then
+    ;;
+
+    "latestcommit" )
       echo
       echo "Placeholder text: $placeholder"
       echo "Repo to get latest commit from: $url branch: $branch"
@@ -50,7 +70,9 @@ do
       commit=$(git ls-remote "$url" "$branch" | cut -f1)
       echo "Commit found: $commit"
       /bin/sed -i 's^'"$placeholder"'^'"$commit"'^' $rd_manifest
-    elif [[ "$action" == "latestappimage" ]]; then
+    ;;
+
+    "latestappimage" )
       echo
       echo "Placeholder text: $placeholder"
       echo "Repo to look for AppImage releases: $url"
@@ -61,7 +83,9 @@ do
       appimagehash=$(curl -sL "$appimageurl" | sha256sum | cut -d ' ' -f1)
       echo "AppImage hash found: $appimagehash"
       /bin/sed -i 's^'"HASHFOR$placeholder"'^'"$appimagehash"'^' $rd_manifest
-    elif [[ "$action" == "outside_info" ]]; then
+    ;;
+
+    "outside_file" )
       if [[ "$url" = \$* ]]; then # If value is a reference to a variable name
         eval url="$url"
       fi
@@ -70,6 +94,36 @@ do
       echo "Information being injected: $(cat $url)"
       echo
       /bin/sed -i 's^'"$placeholder"'^'"$(cat $url)"'^' $rd_manifest
-    fi
+    ;;
+
+    "outside_env_var" )
+      if [[ "$url" = \$* ]]; then # If value is a reference to a variable name
+        eval url="$url"
+      fi
+      echo
+      echo "Placeholder text: $placeholder"
+      echo "Information being injected: $(echo $url)"
+      echo
+      /bin/sed -i 's^'"$placeholder"'^'"$(echo $url)"'^' $rd_manifest
+    ;;
+
+    "custom_command" )
+      echo
+      echo "Command to run: $url"
+      echo
+      eval "$url"
+    ;;
+
+    "url" )
+      # this is used to calculate a dynamic url
+      echo
+      echo "Placeholder text: $placeholder"
+      calculated_url=$(eval echo "$url")
+      echo "Information being injected: $calculated_url"
+      echo
+      /bin/sed -i 's^'"$placeholder"'^'"$calculated_url"'^' $rd_manifest
+    ;;
+
+    esac
   fi
 done < "$automation_task_list"