Merge 49404701fb into 82195a562c

.github/workflows/cooker-selfhosted.yml

@@ -30,6 +30,7 @@ jobs:
 
   Building_RetroDECK:
     runs-on: self-hosted
+    #runs-on: ubuntu-latest
     steps:
 
       # Circumventing this bug: https://github.com/flatpak/flatpak-builder/issues/317
@@ -115,14 +116,6 @@ jobs:
       - name: Build flatpak
         run: "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/flatpak_build_only.sh"
 
-      - name: Create Artifact for flathub
-        run: | 
-          tar -czf ${GITHUB_WORKSPACE}/RetroDECK-Artifact-cooker.tar.gz -C ${GITHUB_WORKSPACE}/retrodeck-flatpak-cooker .
-          hash=($(sha256sum ${GITHUB_WORKSPACE}/RetroDECK-Artifact-cooker.tar.gz))
-          echo $hash > ${GITHUB_WORKSPACE}/RetroDECK-Artifact-cooker.sha
-          mv -f RetroDECK-Artifact-cooker.* ${{ secrets.ARTIFACT_REPO }}
-        continue-on-error: true
-
       - name: Create Bundle
         run: "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/flatpak_build_bundle.sh"
 
@@ -183,6 +176,14 @@ jobs:
           path: RetroDECK-cooker.flatpak
         continue-on-error: true
 
+      - name: Create Artifact for flathub
+        run: | 
+          tar -czf ${GITHUB_WORKSPACE}/RetroDECK-Artifact-cooker.tar.gz -C ${GITHUB_WORKSPACE}/retrodeck-flatpak-cooker .
+          hash=($(sha256sum ${GITHUB_WORKSPACE}/RetroDECK-Artifact-cooker.tar.gz))
+          echo $hash > ${GITHUB_WORKSPACE}/RetroDECK-Artifact-cooker.sha
+          mv -f RetroDECK-Artifact-cooker.* ${{ secrets.ARTIFACT_REPO }}
+        continue-on-error: true
+
       # - name: Upload RetroDECK-cooker.flatpak to Gitea Release
       #   run: |
       #     # Set variables for Gitea host, organization, repository, access token, and release details

.gitmodules

@@ -1,9 +0,0 @@
-[submodule "rd-submodules/shared-modules"]
-	path = rd-submodules/shared-modules
-	url = https://github.com/flathub/shared-modules.git
-[submodule "rd-submodules/retroarch"]
-	path = rd-submodules/retroarch
-	url = https://github.com/flathub/org.libretro.RetroArch.git
-[submodule "rd-submodules/ryujinx"]
-	path = rd-submodules/ryujinx
-	url = https://github.com/flathub/org.ryujinx.Ryujinx

README.md

@@ -102,7 +102,7 @@ Read up here on how to help out, there is something for everyone:
 | [RetroDECK-website](https://github.com/XargonWan/RetroDECK-website)  | Where our website source code is |
 | [RetroDECK on Flathub](https://github.com/flathub/net.retrodeck.retrodeck) | Our little space in Flathub's organization                 |
 | [RetroDECK-MAME](https://github.com/XargonWan/RetroDECK-MAME)   | MAME is big to build, so it's pre-built here and then imported in the build |
-| [RetroDECK-ES-DE](https://github.com/XargonWan/RetroDECK-ES-DE)  | This is where we store our patched fork of ES-DE          |
+| [RetroDECK-ES-DE](https://github.com/RetroDECK/ES-DE)  | This is where we store our patched fork of ES-DE          |
 
 
 # The RetroDECK Wiki

automation_tools/automation_task_list.cfg

@@ -1,15 +1,34 @@
 # The proper format for this file is
 # ACTION^PLACEHOLDERTEXT^URL^REPO(Optional)
 # hash^DOOMSHAPLACEHOLDER^https://buildbot.libretro.com/assets/cores/DOOM/Doom%20%28Shareware%29.zip
-#hash^DUCKSTATIONSHAPLACEHOLDER^https://github.com/stenzek/duckstation/releases/download/preview/DuckStation-x64.AppImage
+# Please leave some newlines in the end of this file or it will break
+
+latestcommit^UNIVERSALDYNAMICINPUTCOMMITPLACEHOLDER^https://github.com/Venomalia/UniversalDynamicInput^main
+outside_file^VERSIONPLACEHOLDER^${GITHUB_WORKSPACE}/buildid
+branch^THISBRANCH
+
+hash^RASHAPLACEHOLDER^https://buildbot.libretro.com/nightly/linux/x86_64/RetroArch.7z
 hash^SAMEDUCKSHAPLACEHOLDER^https://buildbot.libretro.com/nightly/linux/x86_64/latest/sameduck_libretro.so.zip
 hash^PPSSPPBIOSHASHPLACEHOLDER^https://github.com/hrydgard/ppsspp/archive/refs/heads/master.zip
 hash^MSXBIOSHASHPLACEHOLDER^http://bluemsx.msxblue.com/rel_download/blueMSXv282full.zip
 hash^XEMUHDDHASHPLACEHOLDER^https://github.com/mborgerson/xemu-hdd-image/releases/latest/download/xbox_hdd.qcow2.zip
 hash^VITA3KSHAPLACEHOLDER^https://github.com/Vita3K/Vita3K/releases/download/continuous/ubuntu-latest.zip
 hash^RANIGHTLYCORESPLACEHOLDER^https://buildbot.libretro.com/nightly/linux/x86_64/RetroArch_cores.7z
-hash^RETRODECKMAMEPLACEHOLDER^"https://github.com/XargonWan/RetroDECK-MAME/releases/download/$(curl -s https://api.github.com/repos/XargonWan/RetroDECK-MAME/releases/latest | grep -oP '"tag_name": "\K(.*?)(?=")')/RetroDECK-MAME-Artifact.tar.gz"
+
-url^RETRODECKMAMEURLPLACEHOLDER^"https://github.com/XargonWan/RetroDECK-MAME/releases/download/$(curl -s https://api.github.com/repos/XargonWan/RetroDECK-MAME/releases/latest | grep -oP '"tag_name": "\K(.*?)(?=")')/RetroDECK-MAME-Artifact.tar.gz"
+latestghreleasesha^RETRODECKESDEPLACEHOLDER^https://api.github.com/repos/RetroDECK/ES-DE/releases/latest^RetroDECK-ES-DE-Artifact.tar.gz
-latestcommit^UNIVERSALDYNAMICINPUTCOMMITPLACEHOLDER^https://github.com/Venomalia/UniversalDynamicInput^main
+latestghreleasesha^RETRODECKPPSSPPHASH^https://api.github.com/repos/RetroDECK/org.ppsspp.PPSSPP/releases/latest^RetroDECK-PPSSPP-Artifact.tar.gz
-outside_file^VERSIONPLACEHOLDER^${GITHUB_WORKSPACE}/buildid
+latestghreleasesha^RETRODECKDOLPHINPLACEHOLDER^https://api.github.com/repos/RetroDECK/org.DolphinEmu.dolphin-emu/releases/latest^RetroDECK-dolphin-Artifact.tar.gz
-branch^THISBRANCH
+latestghreleasesha^RETRODECKSOLARUSPLACEHOLDER^https://api.github.com/repos/RetroDECK/org.solarus_games.solarus.Launcher/releases/latest^RetroDECK-solarus-Artifact.tar.gz
+latestghreleasesha^RETRODECKPRIMEHACKPLACEHOLDER^https://api.github.com/repos/RetroDECK/io.github.shiiion.primehack/releases/latest^RetroDECK-primehack-Artifact.tar.gz
+latestghreleasesha^RETRODECKGZDOOMSHASPLACEHOLDER^https://api.github.com/repos/RetroDECK/org.zdoom.GZDoom/releases/latest^RetroDECK-gzdoom-Artifact.tar.gz
+latestghreleasesha^RETRODECKMELONDSSHAPLACEHOLDER^https://api.github.com/repos/RetroDECK/net.kuribo64.melonDS/releases/latest^RetroDECK-melonds-Artifact.tar.gz
+latestghreleasesha^RETRODECKMAMEPLACEHOLDER^https://api.github.com/repos/XargonWan/RetroDECK-MAME/releases/latest^RetroDECK-MAME-Artifact.tar.gz
+
+latestghrelease^RETRODECKMAMEURLPLACEHOLDER^https://api.github.com/repos/XargonWan/RetroDECK-MAME/releases/latest^RetroDECK-MAME-Artifact.tar.gz
+latestghrelease^RETRODECKESDEURLPLACEHOLDER^https://api.github.com/repos/RetroDECK/ES-DE/releases/latest^RetroDECK-ES-DE-Artifact.tar.gz
+latestghrelease^RETRODECKPPSSPPURL^https://api.github.com/repos/RetroDECK/org.ppsspp.PPSSPP/releases/latest^RetroDECK-PPSSPP-Artifact.tar.gz
+latestghrelease^RETRODECKDOLPHINURLPLACEHOLDER^https://api.github.com/repos/RetroDECK/org.DolphinEmu.dolphin-emu/releases/latest^RetroDECK-dolphin-Artifact.tar.gz
+latestghrelease^RETRODECKSOLARUSURLPLACEHOLDER^https://api.github.com/repos/RetroDECK/org.solarus_games.solarus.Launcher/releases/latest^RetroDECK-solarus-Artifact.tar.gz
+latestghrelease^RETRODECKPRIMEHACKURLPLACEHOLDER^https://api.github.com/repos/RetroDECK/io.github.shiiion.primehack/releases/latest^RetroDECK-primehack-Artifact.tar.gz
+latestghrelease^RETRODECKGZDOOMSURLPLACEHOLDER^https://api.github.com/repos/RetroDECK/org.zdoom.GZDoom/releases/latest^RetroDECK-gzdoom-Artifact.tar.gz
+latestghrelease^RETRODECKMELONDSURLPLACEHOLDER^https://api.github.com/repos/RetroDECK/net.kuribo64.melonDS/releases/latest^RetroDECK-melonds-Artifact.tar.gz

automation_tools/pre_build_automation.sh

@@ -1,46 +1,57 @@
 #!/bin/bash
 
+# Set script to exit immediately on any error
+set -e
+
 # For the file paths to work correctly, call this script with this command from the cloned repo folder root:
 # sh automation_tools/pre_build_automation.sh
 # Different actions need different information in the task list file
 # branch: This changes the placeholder text to the currently-detected GIT branch if an automated build was started from a PR environment.
-# hash: Finds the SHA256 hash of a file online and updates the placeholder in the manifest. 
+# hash: Finds the SHA256 hash of a file online and updates the placeholder in the manifest.
 #     Needs the URL of the file, in this line format: hash^PLACEHOLDERTEXT^url
-# latestcommit: Finds the most recent commit of a git repo and updated the placeholder in the manifest.
+# latestcommit: Finds the most recent commit of a git repo and updates the placeholder in the manifest.
 #     Needs the URL of the repo and the branch to find the latest commit from, in this line format: latestcommit^PLACEHOLDERTEXT^url^branch
 # latestghtag: Finds the most recent tag on a GitHub repo, for repos that don't have normal releases, but also shouldn't use the latest commit
 #     Needs the URL of the repo, in this line format: latestghtag^PLACEHOLDERTEXT^url
 # latestghrelease: Finds the download URL and SHA256 hash of the latest release from a git repo.
-#     Needs the API URL of the repo, in this line format: latestappimage^PLACEHOLDERTEXT^https://api.github.com/repos/<owner-name>/<repo-name>/releases/latest^<file suffix>
+#     Needs the API URL of the repo, in this line format: latestghrelease^PLACEHOLDERTEXT^https://api.github.com/repos/<owner-name>/<repo-name>/releases/latest^<file suffix>
-#     As this command updates two different placeholders (one for the URL, one for the file hash) in the manifest, 
+#     As this command updates two different placeholders (one for the URL, one for the file hash) in the manifest,
 #     the URL that would be used in the above example is "PLACEHOLDERTEXT" and the hash placeholder text would be "HASHPLACEHOLDERTEXT"
-#     The "HASH" prefix of the placeholder text is hardcoded in the script
+#     The "HASH" prefix of the hash placeholder text is hardcoded in the script.
 #     The <file_suffix> will be the file extension or other identifying suffix at the end of the file name that can be used to select from multiple releases.
 #     Example: If there are these file options for a given release:
 #     yuzu-mainline-20240205-149629642.AppImage
 #     yuzu-linux-20240205-149629642-source.tar.xz
-#     yuzu-linux-20240205-149629642-debug.tar.xz 
+#     yuzu-linux-20240205-149629642-debug.tar.xz
 #     Entering "AppImage" (without quotes) for the <file_suffix> will identify yuzu-mainline-20240205-149629642.AppImage
 #     Entering "source-.tar.xz" (without quotes) for the <file_suffix> will identify yuzu-linux-20240205-149629642-source.tar.xz
 #     Entering "debug-tar.xz" (without quotes) for the <file_suffix> will identify yuzu-linux-20240205-149629642-debug.tar.xz
-#     As a file extension like ".tar.zx" can apply to multiple file options, the entire part that is appended to each release name should be included.
+#     As a file extension like ".tar.xz" can apply to multiple file options, the entire part that is appended to each release name should be included.
 #     The <file_suffix> will also only consider entries where the given suffix is at the end of the file name. So "AppImage" will identify "file.AppImage" but not "file.AppImage.zsync"
+# latestghreleasesha: Finds the SHA256 hash of a specific asset in the latest release from a git repo.
+#     Needs the API URL of the repo, in this line format: latestghreleasesha^PLACEHOLDERTEXT^https://api.github.com/repos/<owner-name>/<repo-name>/releases/latest^<file suffix>
+#     This command updates the placeholder in the manifest with the SHA256 hash of the specified asset.
 # outside_file: Prints the contents of a file from the build environment (such as the buildid file) and replaces the placeholder text with those contents.
 # outside_env_var: Gets the value of an environmental variable from the build environment (the output of "echo $var" from the terminal) and replaces the placeholder text with that value.
-# custom_command: Runs a single command explicitly as written in the $URL field of the task list, including variable and command expansion. This should work the same as if you were runnig the command directly from the terminal.
+# custom_command: Runs a single command explicitly as written in the $URL field of the task list, including variable and command expansion. This should work the same as if you were running the command directly from the terminal.
 #     This command does not need a PLACEHOLDERTEXT field in the task list, so needs to be in this syntax: custom_command^^$COMMAND
-# url: This is used to calculate a dynamic URL and the value to the $caluculated_url environmental variable, for use in other subsequent commands.
+# url: This is used to calculate a dynamic URL and the value to the $calculated_url environmental variable, for use in other subsequent commands.
 
-rd_manifest=${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml
+# Define paths
-automation_task_list=${GITHUB_WORKSPACE}/automation_tools/automation_task_list.cfg
+rd_manifest="${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml"
-current_branch=$(git rev-parse --abbrev-ref HEAD)
+automation_task_list="${GITHUB_WORKSPACE}/automation_tools/automation_task_list.cfg"
 
-# During the PR automated tests instead of the branch name is returned "HEAD", fixing it
+# Retrieve current git branch
-if [ $current_branch == "HEAD" ]; then
+get_current_branch() {
-  echo "Looks like we are on a PR environment, retrieving the branch name from which the PR is raised."
+  local branch=$(git rev-parse --abbrev-ref HEAD)
-  current_branch=$(echo $GITHUB_REF | sed 's@refs/heads/@@')
+  if [ "$branch" == "HEAD" ]; then
-  echo "The branch name from which the PR is raised is \"$current_branch\"."
+    echo "$GITHUB_REF" | sed 's@refs/heads/@@'
-fi
+  else
+    echo "$branch"
+  fi
+}
+
+current_branch=$(get_current_branch)
 
 echo "Manifest location: $rd_manifest"
 echo "Automation task list location: $automation_task_list"
@@ -49,103 +60,130 @@ echo "Task list contents:"
 cat "$automation_task_list"
 echo
 
-# Update all collected information
+# Functions to handle different actions
-while IFS="^" read -r action placeholder url branch || [[ -n "$action" ]];
+handle_branch() {
-do
+  local placeholder="$1"
-  if [[ ! $action == "#"* ]] && [[ ! -z "$action" ]]; then
+  echo "Replacing placeholder $placeholder with branch $current_branch"
+  /bin/sed -i 's^'"$placeholder"'^'"$current_branch"'^g' "$rd_manifest"
+}
+
+handle_hash() {
+  local placeholder="$1"
+  local url="$2"
+  local calculated_url=$(eval echo "$url")
+  local hash=$(curl -sL "$calculated_url" | sha256sum | cut -d ' ' -f1)
+  echo "Replacing placeholder $placeholder with hash $hash"
+  /bin/sed -i 's^'"$placeholder"'^'"$hash"'^g' "$rd_manifest"
+}
+
+handle_latestcommit() {
+  local placeholder="$1"
+  local url="$2"
+  local branch="$3"
+  local commit=$(git ls-remote "$url" "$branch" | cut -f1)
+  echo "Replacing placeholder $placeholder with latest commit $commit"
+  /bin/sed -i 's^'"$placeholder"'^'"$commit"'^g' "$rd_manifest"
+}
+
+handle_latestghtag() {
+  local placeholder="$1"
+  local url="$2"
+  local tag=$(git ls-remote --tags "$url" | tail -n 1 | cut -f2 | sed 's|refs/tags/||')
+  echo "Replacing placeholder $placeholder with latest tag $tag"
+  /bin/sed -i 's^'"$placeholder"'^'"$tag"'^g' "$rd_manifest"
+}
+
+handle_latestghrelease() {
+  local placeholder="$1"
+  local url="$2"
+  local suffix="$3"
+  echo "Fetching release data from: $url"
+  local release_data=$(curl -s "$url")
+  echo "Release data fetched."
+  local ghreleaseurl=$(echo "$release_data" | jq -r ".assets[] | select(.name | endswith(\"$suffix\")).browser_download_url")
+  
+  if [[ -z "$ghreleaseurl" ]]; then
+    echo "Error: No asset found with suffix $suffix"
+    exit 1
+  fi
+  
+  local ghreleasehash=$(curl -sL "$ghreleaseurl" | sha256sum | cut -d ' ' -f1)
+  
+  echo "Replacing placeholder $placeholder with URL $ghreleaseurl and hash $ghreleasehash"
+  /bin/sed -i 's^'"$placeholder"'^'"$ghreleaseurl"'^g' "$rd_manifest"
+  /bin/sed -i 's^'"HASHFOR$placeholder"'^'"$ghreleasehash"'^g' "$rd_manifest"
+}
+
+handle_latestghreleasesha() {
+  local placeholder="$1"
+  local url="$2"
+  local suffix="$3"
+  echo "Fetching release data from: $url"
+  local release_data=$(curl -s "$url")
+  echo "Release data fetched."
+  local ghreleaseurl=$(echo "$release_data" | jq -r ".assets[] | select(.name | endswith(\"$suffix\")).browser_download_url")
+  
+  if [[ -z "$ghreleaseurl" ]]; then
+    echo "Error: No asset found with suffix $suffix"
+    exit 1
+  fi
+  
+  local ghreleasehash=$(curl -sL "$ghreleaseurl" | sha256sum | cut -d ' ' -f1)
+  
+  echo "Replacing placeholder $placeholder with hash $ghreleasehash"
+  /bin/sed -i 's^'"$placeholder"'^'"$ghreleasehash"'^g' "$rd_manifest"
+}
+
+handle_outside_file() {
+  local placeholder="$1"
+  local file_path="$2"
+  if [[ "$file_path" == \$* ]]; then
+    eval file_path="$file_path"
+  fi
+  local content=$(cat "$file_path")
+  echo "Replacing placeholder $placeholder with content of file $file_path"
+  /bin/sed -i 's^'"$placeholder"'^'"$content"'^g' "$rd_manifest"
+}
+
+handle_outside_env_var() {
+  local placeholder="$1"
+  local var_name="$2"
+  if [[ "$var_name" == \$* ]]; then
+    eval var_name="$var_name"
+  fi
+  local value=$(echo "$var_name")
+  echo "Replacing placeholder $placeholder with environment variable $value"
+  /bin/sed -i 's^'"$placeholder"'^'"$value"'^g' "$rd_manifest"
+}
+
+handle_custom_command() {
+  local command="$1"
+  echo "Executing custom command: $command"
+  eval "$command"
+}
+
+handle_url() {
+  local placeholder="$1"
+  local url="$2"
+  local calculated_url=$(eval echo "$url")
+  echo "Replacing placeholder $placeholder with calculated URL $calculated_url"
+  /bin/sed -i 's^'"$placeholder"'^'"$calculated_url"'^g' "$rd_manifest"
+}
+
+# Process the task list
+while IFS="^" read -r action placeholder url branch || [[ -n "$action" ]]; do
+  if [[ ! "$action" == "#"* ]] && [[ -n "$action" ]]; then
     case "$action" in
-
+      "branch" ) handle_branch "$placeholder" ;;
-    "branch" )
+      "hash" ) handle_hash "$placeholder" "$url" ;;
-      echo
+      "latestcommit" ) handle_latestcommit "$placeholder" "$url" "$branch" ;;
-      echo "Placeholder text: $placeholder"
+      "latestghtag" ) handle_latestghtag "$placeholder" "$url" ;;
-      echo "Current branch:" "$current_branch"
+      "latestghrelease" ) handle_latestghrelease "$placeholder" "$url" "$branch" ;;
-      echo
+      "latestghreleasesha" ) handle_latestghreleasesha "$placeholder" "$url" "$branch" ;;
-      /bin/sed -i 's^'"$placeholder"'^'"$current_branch"'^g' $rd_manifest
+      "outside_file" ) handle_outside_file "$placeholder" "$url" ;;
-    ;;
+      "outside_env_var" ) handle_outside_env_var "$placeholder" "$url" ;;
-
+      "custom_command" ) handle_custom_command "$url" ;;
-    "hash" )
+      "url" ) handle_url "$placeholder" "$url" ;;
-      echo
-      echo "Placeholder text: $placeholder"
-      calculated_url=$(eval echo "$url")  # in case the url has to be calculated from an expression
-      echo "URL to hash: $calculated_url"
-      echo
-      hash=$(curl -sL "$calculated_url" | sha256sum | cut -d ' ' -f1)
-      echo "Hash found: $hash"
-      /bin/sed -i 's^'"$placeholder"'^'"$hash"'^' $rd_manifest
-    ;;
-
-    "latestcommit" )
-      echo
-      echo "Placeholder text: $placeholder"
-      echo "Repo to get latest commit from: $url branch: $branch"
-      echo
-      commit=$(git ls-remote "$url" "$branch" | cut -f1)
-      echo "Commit found: $commit"
-      /bin/sed -i 's^'"$placeholder"'^'"$commit"'^' $rd_manifest
-    ;;
-
-    "latestghtag" )
-      echo
-      echo "Placeholder text: $placeholder"
-      echo "Repo to get the latest tag from: $url"
-      echo
-      tag=$(git ls-remote "$url" | tail -n 1 | cut -f2 | sed 's|refs/tags/||')
-      echo "Tag found: $tag"
-      /bin/sed -i 's^'"$placeholder"'^'"$tag"'^' $rd_manifest
-    ;;
-
-    "latestghrelease" )
-      echo
-      echo "Placeholder text: $placeholder"
-      echo "Repo to look for latest releases: $url"
-      echo
-      ghreleaseurl=$(curl -s "$url" | grep browser_download_url | grep "$branch\""$ | cut -d : -f 2,3 | tr -d \" | sed -n 1p | tr -d ' ')
-      echo "GitHub release URL found: $ghreleaseurl"
-      /bin/sed -i 's^'"$placeholder"'^'"$ghreleaseurl"'^' $rd_manifest
-      ghreleasehash=$(curl -sL "$ghreleaseurl" | sha256sum | cut -d ' ' -f1)
-      echo "GitHub release hash found: $ghreleasehash"
-      /bin/sed -i 's^'"HASHFOR$placeholder"'^'"$ghreleasehash"'^' $rd_manifest
-    ;;
-
-    "outside_file" )
-      if [[ "$url" = \$* ]]; then # If value is a reference to a variable name
-        eval url="$url"
-      fi
-      echo
-      echo "Placeholder text: $placeholder"
-      echo "Information being injected: $(cat $url)"
-      echo
-      /bin/sed -i 's^'"$placeholder"'^'"$(cat $url)"'^' $rd_manifest
-    ;;
-
-    "outside_env_var" )
-      if [[ "$url" = \$* ]]; then # If value is a reference to a variable name
-        eval url="$url"
-      fi
-      echo
-      echo "Placeholder text: $placeholder"
-      echo "Information being injected: $(echo $url)"
-      echo
-      /bin/sed -i 's^'"$placeholder"'^'"$(echo $url)"'^' $rd_manifest
-    ;;
-
-    "custom_command" )
-      echo
-      echo "Command to run: $url"
-      echo
-      eval "$url"
-    ;;
-
-    "url" )
-      # this is used to calculate a dynamic url
-      echo
-      echo "Placeholder text: $placeholder"
-      calculated_url=$(eval echo "$url")
-      echo "Information being injected: $calculated_url"
-      echo
-      /bin/sed -i 's^'"$placeholder"'^'"$calculated_url"'^' $rd_manifest
-    ;;
-
     esac
   fi
 done < "$automation_task_list"

developer_toolbox/build_retrodeck_locally.sh

@@ -4,7 +4,7 @@
 
 # Check if script is running with elevated privileges
 if [ "$EUID" -ne 0 ]; then
-    echo "The build might fail without some superuser permissions, please run me with sudo. Continue without sudo? [y/N,]"
+    echo "The build might fail without some superuser permissions, please run me with sudo. Continue without sudo? [y/N]"
     read -r continue_without_sudo
     if [[ "$continue_without_sudo" != "y" ]]; then
         exit 1

functions/global.sh

@@ -2,6 +2,10 @@
 
 # This file is containing some global function needed for the script such as the config file tools
 
+# pathing the retrodeck components provided libraries
+# now disabled as we are importing everything in /app/lib. In case we are breaking something we need to restore this approach
+# export LD_LIBRARY_PATH="/app/retrodeck/lib:/app/retrodeck/lib/debug:/app/retrodeck/lib/pkgconfig:$LD_LIBRARY_PATH"
+
 source /app/libexec/050_save_migration.sh
 source /app/libexec/checks.sh
 source /app/libexec/compression.sh
@@ -18,7 +22,7 @@ source /app/libexec/configurator_functions.sh
 # Static variables
 rd_conf="/var/config/retrodeck/retrodeck.cfg"                                                                         # RetroDECK config file path
 rd_conf_backup="/var/config/retrodeck/retrodeck.bak"                                                                  # Backup of RetroDECK config file from update
-rd_logs_folder="/var/config/retrodeck/logs"                                                                                  # Static location to write all RetroDECK-related logs
+rd_logs_folder="/var/config/retrodeck/logs"                                                                           # Static location to write all RetroDECK-related logs
 emuconfigs="/app/retrodeck/emu-configs"                                                                               # folder with all the default emulator configs
 rd_defaults="$emuconfigs/defaults/retrodeck/retrodeck.cfg"                                                            # A default RetroDECK config file
 rd_update_patch="/var/config/retrodeck/rd_update.patch"                                                               # A static location for the temporary patch file used during retrodeck.cfg updates

net.retrodeck.retrodeck.appdata.xml

@@ -73,6 +73,14 @@
             <p>Changes:</p>
             <ul>
                <li>WELCOME TO cooker-0.9.0b!</li>
+               <li>RetroDECK is on a diet: we saved around 600mb</li>
+            </ul>
+            <p>Developers, developers, developers:</p>
+            <ul>
+               <li>Manifest was completely redone, more on that on the MONTH blog post</li>
+               <li>RetroDECK now is migrated to an organization on GitHub, as before, please check the MONTH blog post</li>
+               <li>pre_build_automation script weas reworked and additional functions are added</li>
+               <li>Contributing is now easier and the manifest build time is down to about 40m (was 3h30)</li>
             </ul>
             <p>Known issues:</p>
             <ul>

old/removed-modules.yml

@@ -172,4 +172,182 @@
     - type: file
       path: rd-submodules/ryujinx/ryujinx-wrapper
 
-  # Ryujinx - END
+  # Ryujinx - END
+
+  # Needed from ES-DE 2.1.0+
+  - name: libpoppler-glib
+    buildsystem: cmake-ninja
+    config-opts:
+      - -DENABLE_BOOST=OFF
+    sources:
+      - type: archive
+        # original link not working, found a mirror
+        url: https://poppler.freedesktop.org/poppler-22.11.0.tar.xz
+        #url: https://gitlab.freedesktop.org/poppler/poppler/-/archive/poppler-22.11.0/poppler-poppler-22.11.0.tar.bz2
+        sha256: 093ba9844ed774285517361c15e21a31ba4df278a499263d4403cca74f2da828
+    cleanup:
+      - /lib/pkgconfig
+      - /include
+      - '*.a'
+      - '*.la'
+
+
+  # dependency of: XEMU, MelonDS (4.7.0)
+  - name: libslirp
+    buildsystem: meson
+    cleanup:
+      - /include
+    sources:
+      - type: archive
+        url: https://gitlab.freedesktop.org/slirp/libslirp/-/archive/v4.8.0/libslirp-v4.8.0.tar.gz
+        sha256: 2a98852e65666db313481943e7a1997abff0183bd9bea80caec1b5da89fda28c
+        x-checker-data:
+          type: anitya
+          project-id: 96796
+          stable-only: true
+          url-template: https://gitlab.freedesktop.org/slirp/libslirp/-/archive/v$version/libslirp-v$version.tar.gz
+
+  # dependency of: CEMU (1.0.26), DOLPHIN (1.0.27)
+  - name: libusb
+    config-opts:
+      - --disable-static
+    cleanup:
+      - /include
+      - /lib/*.la
+      - /lib/pkgconfig
+    sources:
+      - type: archive
+        url: https://github.com/libusb/libusb/releases/download/v1.0.27/libusb-1.0.27.tar.bz2
+        sha256: ffaa41d741a8a3bee244ac8e54a72ea05bf2879663c098c82fc5757853441575
+        x-checker-data:
+          type: anitya
+          project-id: 1749
+          stable-only: true
+          url-template: https://github.com/libusb/libusb/releases/download/v$version/libusb-$version.tar.bz2
+
+    # Dolphin - START
+  # why from source: AppImage not provided, only Flatpak
+  # https://github.com/flathub/org.DolphinEmu.dolphin-emu
+  # WHEN UPADTING: remember to update rd-submodules/dolphin contents
+
+  - name: xdg-screensaver-shim
+    buildsystem: meson
+    sources:
+      - type: archive
+        url: https://github.com/Unrud/xdg-screensaver-shim/archive/0.0.2.tar.gz
+        sha256: 0ed2a69fe6ee6cbffd2fe16f85116db737f17fb1e79bfb812d893cf15c728399
+
+  - name: dolphin-emu
+    buildsystem: cmake-ninja
+    config-opts:
+      - -DCMAKE_BUILD_TYPE=Release
+      - -DENABLE_ALSA=OFF
+      - -DENABLE_SDL=ON
+      - -DENABLE_EVDEV=ON
+      - -DDISTRIBUTOR=Flathub
+    cleanup:
+      - /share/man
+    post-install:
+      - install -D -t ${FLATPAK_DEST}/bin/ dolphin-emu-wrapper
+      - sed -i -e 's/viewBox="0 0 1024.02 571.29"/viewBox="0 -285.645 1024 1024" width="2048"
+        height="2048"/g' /app/share/icons/hicolor/scalable/apps/dolphin-emu.svg
+    sources:
+      - type: git
+        # Sometimes Dolphin or its submodules clone are failing in https so it must done in ssh
+        # fatal: remote transport reported error
+        # url: ssh://git@github.com/dolphin-emu/dolphin.git
+        url: https://github.com/dolphin-emu/dolphin.git
+        commit: cc3ff347b437fd80f3a2880d1bbc7ba2d5f191f4
+        # x-checker-data:
+        #   type: json
+        #   url: https://dolphin-emu.org/update/latest/beta
+        #   commit-query: .hash
+        #   version-query: .shortrev
+        #   timestamp-query: .date
+        #   is-main-source: true
+      # detects whether dolphin is running in a flatpak sandbox
+      # and makes it use xdg directories if it is.
+      # prevents dolphin from attempting to write conf files
+      # in non-writable paths, typically happens when a user
+      # has leftover files from a previous non-flatpak install
+      - type: patch
+        path: rd-submodules/dolphin/detectflatpak.patch
+      # version strings must match exactly for online multiplayer
+      - type: patch
+        path: rd-submodules/dolphin/nodirtyversion.patch
+      - type: script
+        commands:
+          - |
+            for i in {0..9}; do
+              test -S $XDG_RUNTIME_DIR/discord-ipc-$i ||
+                ln -sf {app/com.discordapp.Discord,$XDG_RUNTIME_DIR}/discord-ipc-$i;
+            done
+            dolphin-emu "$@"
+        dest-filename: dolphin-emu-wrapper
+
+  - name: universal_dynamic_input
+    buildsystem: simple
+    build-commands:
+      - mkdir -p ${FLATPAK_DEST}/retrodeck/extras/DynamicInputTextures
+      - cp -rf * ${FLATPAK_DEST}/retrodeck/extras/DynamicInputTextures/
+    sources:
+      - type: git
+        url: https://github.com/Venomalia/UniversalDynamicInput.git
+        commit: UNIVERSALDYNAMICINPUTCOMMITPLACEHOLDER
+
+  # Dolphin - END
+
+  # SOLARUS - START
+  # why from source: they only provide a snap
+  # https://github.com/flathub/org.solarus_games.solarus.Launcher
+
+  - name: luajit
+    no-autogen: true
+    cleanup:
+      - /bin
+      - /lib/*.a
+      - /include
+      - /lib/pkgconfig
+      - /share/man
+    sources:
+      - type: archive
+        url: https://luajit.org/download/LuaJIT-2.0.5.tar.gz
+        sha256: 874b1f8297c697821f561f9b73b57ffd419ed8f4278c82e05b48806d30c1e979
+      - type: shell
+        commands:
+          - sed -i 's|/usr/local|/app|' ./Makefile
+
+  - name: libmodplug
+    cleanup:
+      - /include
+    sources:
+      - type: archive
+        url: https://download.sourceforge.net/project/modplug-xmms/libmodplug/0.8.9.0/libmodplug-0.8.9.0.tar.gz
+        sha512: 880e10154fd367ee24ace53ca7e32a5c1fee7dfff1f934e6499944c12779427da63e2f55f8d6ce25db0a42a5b4424053bf64305b31dbfc4ef6a8909924d655fd
+
+  - name: physfs
+    buildsystem: cmake-ninja
+    config-opts:
+      - -DPHYSFS_BUILD_TEST=OFF
+    sources:
+      - type: archive
+        url: https://icculus.org/physfs/downloads/physfs-3.0.2.tar.bz2
+        sha256: 304df76206d633df5360e738b138c94e82ccf086e50ba84f456d3f8432f9f863
+        size: 194888
+        x-checker-data:
+          type: anitya
+          project-id: 3631
+          url-template: https://icculus.org/physfs/downloads/physfs-$version.tar.bz2
+
+  - name: solarus
+    buildsystem: cmake-ninja
+    builddir: true
+    config-opts:
+      - -DSOLARUS_GUI=OFF
+    sources:
+      - type: git
+        url: https://gitlab.com/solarus-games/solarus.git
+        tag: v1.6.5
+        commit: 3aec70b0556a8d7aed7903d1a3e4d9a18c5d1649
+
+  # SOLARUS - END

rd-submodules/dolphin/detectflatpak.patch

@@ -1,33 +0,0 @@
-From f2d3b349d8487cf0ee50b36ba024ac513d36f826 Mon Sep 17 00:00:00 2001
-From: Carles Pastor <cpbadosa@gmail.com>
-Date: Sat, 20 Aug 2022 13:35:32 +0200
-Subject: [PATCH] Detect we are running inside a flatpak sandbox
-
----
- Source/Core/UICommon/UICommon.cpp | 4 ++--
- 1 file changed, 2 insertions(+), 2 deletions(-)
-
-diff --git a/Source/Core/UICommon/UICommon.cpp b/Source/Core/UICommon/UICommon.cpp
-index 8d9dc2abb8..8a9d39a3d5 100644
---- a/Source/Core/UICommon/UICommon.cpp
-+++ b/Source/Core/UICommon/UICommon.cpp
-@@ -348,7 +348,7 @@ void SetUserDirectory(std::string custom_path)
-     //    -> Use GetExeDirectory()/User
-     // 2. $DOLPHIN_EMU_USERPATH is set
-     //    -> Use $DOLPHIN_EMU_USERPATH
--    // 3. ~/.dolphin-emu directory exists
-+    // 3. ~/.dolphin-emu directory exists, and we're not in flatpak
-     //    -> Use ~/.dolphin-emu
-     // 4. Default
-     //    -> Use XDG basedir, see
-@@ -381,7 +381,7 @@ void SetUserDirectory(std::string custom_path)
-     {
-       user_path = home_path + "." DOLPHIN_DATA_DIR DIR_SEP;
- 
--      if (!File::Exists(user_path))
-+      if (File::Exists("/.flatpak-info") || !File::Exists(user_path))
-       {
-         const char* data_home = getenv("XDG_DATA_HOME");
-         std::string data_path =
--- 
-2.37.1

rd-submodules/dolphin/nodirtyversion.patch

@@ -1,13 +0,0 @@
-diff --git a/CMake/ScmRevGen.cmake b/CMake/ScmRevGen.cmake
-index aacf7491ce..503791cda8 100644
---- a/CMake/ScmRevGen.cmake
-+++ b/CMake/ScmRevGen.cmake
-@@ -7,7 +7,7 @@ if(GIT_FOUND)
-       OUTPUT_VARIABLE DOLPHIN_WC_REVISION
-       OUTPUT_STRIP_TRAILING_WHITESPACE)
-   # defines DOLPHIN_WC_DESCRIBE
--  execute_process(WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMAND ${GIT_EXECUTABLE} describe --always --long --dirty
-+  execute_process(WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMAND ${GIT_EXECUTABLE} describe --always --long
-       OUTPUT_VARIABLE DOLPHIN_WC_DESCRIBE
-       OUTPUT_STRIP_TRAILING_WHITESPACE)
- 

rd-submodules/melonds/hotkeys.patch

@@ -1,18 +0,0 @@
-diff -au1r melonDS/src/frontend/qt_sdl/main.cpp melonDS/src/frontend/qt_sdl/main.cpp
---- melonDS/src/frontend/qt_sdl/main.cpp	2023-05-06 17:27:24.822375255 +0200
-+++ melonDS/src/frontend/qt_sdl/main.cpp	2023-05-06 17:47:19.587498189 +0200
-@@ -1467,3 +1467,3 @@
-             actSaveState[0] = submenu->addAction("File...");
--            actSaveState[0]->setShortcut(QKeySequence(Qt::ShiftModifier | Qt::Key_F9));
-+            actSaveState[0]->setShortcut(QKeySequence(Qt::CTRL | Qt::Key_S));
-             actSaveState[0]->setData(QVariant(0));
-@@ -1483,3 +1483,3 @@
-             actLoadState[0] = submenu->addAction("File...");
--            actLoadState[0]->setShortcut(QKeySequence(Qt::Key_F9));
-+            actLoadState[0]->setShortcut(QKeySequence(Qt::CTRL | Qt::Key_A));
-             actLoadState[0]->setData(QVariant(0));
-@@ -1489,3 +1489,3 @@
-         actUndoStateLoad = menu->addAction("Undo state load");
--        actUndoStateLoad->setShortcut(QKeySequence(Qt::Key_F12));
-+        actUndoStateLoad->setShortcut(QKeySequence(Qt::CTRL | Qt::Key_8));
-         connect(actUndoStateLoad, &QAction::triggered, this, &MainWindow::onUndoStateLoad);

rd-submodules/primehack/detectflatpak.patch

@@ -1,33 +0,0 @@
-From f2d3b349d8487cf0ee50b36ba024ac513d36f826 Mon Sep 17 00:00:00 2001
-From: Carles Pastor <cpbadosa@gmail.com>
-Date: Sat, 20 Aug 2022 13:35:32 +0200
-Subject: [PATCH] Detect we are running inside a flatpak sandbox
-
----
- Source/Core/UICommon/UICommon.cpp | 4 ++--
- 1 file changed, 2 insertions(+), 2 deletions(-)
-
-diff --git a/Source/Core/UICommon/UICommon.cpp b/Source/Core/UICommon/UICommon.cpp
-index 8d9dc2abb8..8a9d39a3d5 100644
---- a/Source/Core/UICommon/UICommon.cpp
-+++ b/Source/Core/UICommon/UICommon.cpp
-@@ -348,7 +348,7 @@ void SetUserDirectory(std::string custom_path)
-     //    -> Use GetExeDirectory()/User
-     // 2. $DOLPHIN_EMU_USERPATH is set
-     //    -> Use $DOLPHIN_EMU_USERPATH
--    // 3. ~/.dolphin-emu directory exists
-+    // 3. ~/.dolphin-emu directory exists, and we're not in flatpak
-     //    -> Use ~/.dolphin-emu
-     // 4. Default
-     //    -> Use XDG basedir, see
-@@ -381,7 +381,7 @@ void SetUserDirectory(std::string custom_path)
-     {
-       user_path = home_path + "." DOLPHIN_DATA_DIR DIR_SEP;
- 
--      if (!File::Exists(user_path))
-+      if (File::Exists("/.flatpak-info") || !File::Exists(user_path))
-       {
-         const char* data_home = getenv("XDG_DATA_HOME");
-         std::string data_path =
--- 
-2.37.1

rd-submodules/primehack/nodirtyversion.patch

@@ -1,12 +0,0 @@
-diff --git a/CMake/ScmRevGen.cmake b/CMake/ScmRevGen.cmake
-index aacf7491ce..503791cda8 100644
---- a/CMake/ScmRevGen.cmake
-+++ b/CMake/ScmRevGen.cmake
-@@ -7,7 +7,7 @@ if(GIT_FOUND)
-       OUTPUT_VARIABLE DOLPHIN_WC_REVISION
-       OUTPUT_STRIP_TRAILING_WHITESPACE)
-   # defines DOLPHIN_WC_DESCRIBE
--  execute_process(WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMAND ${GIT_EXECUTABLE} describe --always --long --dirty
-+  execute_process(WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMAND ${GIT_EXECUTABLE} describe --always --long
-       OUTPUT_VARIABLE DOLPHIN_WC_DESCRIBE
-       OUTPUT_STRIP_TRAILING_WHITESPACE)

rd-submodules/retroarch

@@ -1 +0,0 @@
-Subproject commit eec4bcca8c9e514467fb62cca4a7b2d12c6a2328

rd-submodules/ryujinx

@@ -1 +0,0 @@
-Subproject commit 41761a7dc36ec8520bdaca796d9d0b1219b66a07

rd-submodules/shared-modules

@@ -1 +0,0 @@
-Subproject commit 782d3cc04ccdd8071017f622d4bacd35faecbd86