BUILD: pre_build_automation is now called manifest_placeholder_replacer + added hash caching for local builds

2025-04-10 19:15:12 +00:00 · 2025-01-09 09:45:59 +09:00 · 2025-01-09 09:45:59 +09:00 · c1260d0857
parent 912d8c28ca
commit c1260d0857
6 changed files with 54 additions and 11 deletions
--- a/.github/workflows/build_retrodeck.yml
+++ b/.github/workflows/build_retrodeck.yml
@ -147,7 +147,7 @@ jobs:
        run: "cp ${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml ${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml.bak"

      - name: Run pre-build automation tasks
-        run: "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/pre_build_automation.sh"
+        run: "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/manifest_placeholder_replacer.sh"

      - name: "Adding flatpak portal for automated updates (Cooker only)"
        if: github.ref != 'refs/heads/main'
@ -269,7 +269,7 @@ jobs:
          echo "Recalculating hashes and retrying download..."
          rm -f "{GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml"
          cp "${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml.bak" "${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml"
-          "${GITHUB_WORKSPACE}/automation_tools/pre_build_automation.sh"
+          "${GITHUB_WORKSPACE}/automation_tools/manifest_placeholder_replacer.sh"
          "${GITHUB_WORKSPACE}/automation_tools/flatpak_build_download_only.sh"

      - name: Build flatpak
--- a/.gitignore
+++ b/.gitignore
@ -26,6 +26,7 @@ incconfigs/
 *.flatpak
 *.flatpak.sha
 net.retrodeck.retrodeck.cached.yml
+placeholders.cache

 # Python #
 ##########
--- a/automation_tools/manifest_placeholder_replacer.sh
+++ b/automation_tools/manifest_placeholder_replacer.sh
@ -4,7 +4,7 @@
 set -e

 # For the file paths to work correctly, call this script with this command from the cloned repo folder root:
-# sh automation_tools/pre_build_automation.sh
+# sh automation_tools/manifest_placeholder_replacer.sh
 # Different actions need different information in the task list file
 # branch: This changes the placeholder text to the currently-detected GIT branch if an automated build was started from a PR environment.
 # hash: Finds the SHA256 hash of a file online and updates the placeholder in the manifest.
@ -40,6 +40,12 @@ set -e
 # Define paths
 rd_manifest="${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml"
 automation_task_list="${GITHUB_WORKSPACE}/automation_tools/automation_task_list.cfg"
+cache_file="${GITHUB_WORKSPACE}/placeholders.cache"
+
+# Check if cache file exists
+if [ -f "$cache_file" ]; then
+  echo "Warning: Cache file $cache_file is being used. If you encounter issues with hashes, consider deleting this file."
+fi

 # Retrieve current git branch
 get_current_branch() {
@ -71,6 +77,34 @@ echo "Task list contents:"
 cat "$automation_task_list"
 echo

+# Function to get hash from cache or calculate it
+get_hash() {
+  local url="$1"
+  local hash
+
+  # Check if cache should be used and if cache file exists
+  # the use_cache variable is initialized by build_retrodeck_locally only so in the pipeline it will never use cache
+  if [ "$use_cache" == "true" ] && [ -f "$cache_file" ]; then
+    # Try to retrieve hash from cache
+    hash=$(grep "^$url " "$cache_file" | cut -d ' ' -f2)
+    if [ -n "$hash" ]; then
+      echo "Cache file exists, using cached hash for $url"
+    fi
+  fi
+
+  # If hash is not found in cache, calculate it
+  if [ -z "$hash" ]; then
+    hash=$(curl -sL "$url" | sha256sum | cut -d ' ' -f1)
+    # Save the calculated hash to cache if caching is enabled
+    if [ "$use_cache" == "true" ]; then
+      echo "$url $hash" >> "$cache_file"
+    fi
+  fi
+
+  # Return the hash
+  echo "$hash"
+}
+
 # Functions to handle different actions
 handle_branch() {
  local placeholder="$1"
@ -82,7 +116,7 @@ handle_hash() {
  local placeholder="$1"
  local url="$2"
  local calculated_url=$(eval echo "$url")
-  local hash=$(curl -sL "$calculated_url" | sha256sum | cut -d ' ' -f1)
+  local hash=$(get_hash "$calculated_url")
  echo "Replacing placeholder $placeholder with hash $hash"
  /bin/sed -i 's^'"$placeholder"'^'"$hash"'^g' "$rd_manifest"
 }
@ -118,7 +152,7 @@ handle_latestghrelease() {
    exit 1
  fi
  
-  local ghreleasehash=$(curl -sL "$ghreleaseurl" | sha256sum | cut -d ' ' -f1)
+  local ghreleasehash=$(get_hash "$ghreleaseurl")
  
  echo "Replacing placeholder $placeholder with URL $ghreleaseurl and hash $ghreleasehash"
  /bin/sed -i 's^'"$placeholder"'^'"$ghreleaseurl"'^g' "$rd_manifest"
@ -139,7 +173,7 @@ handle_latestghreleasesha() {
    exit 1
  fi
  
-  local ghreleasehash=$(curl -sL "$ghreleaseurl" | sha256sum | cut -d ' ' -f1)
+  local ghreleasehash=$(get_hash "$ghreleaseurl")
  
  echo "Replacing placeholder $placeholder with hash $ghreleasehash"
  /bin/sed -i 's^'"$placeholder"'^'"$ghreleasehash"'^g' "$rd_manifest"
--- a/developer_toolbox/build_retrodeck_locally.sh
+++ b/developer_toolbox/build_retrodeck_locally.sh
@ -10,6 +10,14 @@
 #     fi
 # fi

+read -rp "Do you want to use the hashes cache? If you're unsure just say no [Y/n]" use_cache_input
+use_cache_input=${use_cache_input:-Y}
+if [[ "$use_cache_input" =~ ^[Yy]$ ]]; then
+    export use_cache="true"
+else
+    export use_cache="false"
+fi
+
 git submodule update --init --recursive

 export GITHUB_WORKSPACE="."
@ -23,7 +31,7 @@ cp net.retrodeck.retrodeck.yml net.retrodeck.retrodeck.yml.bak

 automation_tools/install_dependencies.sh
 automation_tools/cooker_build_id.sh
-automation_tools/pre_build_automation.sh
+automation_tools/manifest_placeholder_replacer.sh
 automation_tools/cooker_flatpak_portal_add.sh
 # THIS SCRIPT IS BROKEN HENCE DISABLED FTM
 # automation_tools/appdata_management.sh
--- a/old/disabled-workflows/cooker-selfhosted-persistent.yml
+++ b/old/disabled-workflows/cooker-selfhosted-persistent.yml
@ -78,7 +78,7 @@ jobs:
        run: "cp ${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml ${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml.bak"

      - name: Run pre-build automation tasks
-        run : "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/pre_build_automation.sh"
+        run : "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/manifest_placeholder_replacer.sh"

      - name: "Adding flatpak portal for automated updates (cooker only)"
        run: "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/cooker_flatpak_portal_add.sh"
@ -102,7 +102,7 @@ jobs:
          echo "Recalculating hashes and retrying download..."
          rm -f "{GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml"
          cp "${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml.bak" "${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml"
-          "${GITHUB_WORKSPACE}/automation_tools/pre_build_automation.sh"
+          "${GITHUB_WORKSPACE}/automation_tools/manifest_placeholder_replacer.sh"
          "${GITHUB_WORKSPACE}/automation_tools/flatpak_build_download_only_persistent.sh"

      - name: Build flatpak
--- a/old/disabled-workflows/cooker-selfhosted.yml
+++ b/old/disabled-workflows/cooker-selfhosted.yml
@ -80,7 +80,7 @@ jobs:
        run: "cp ${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml ${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml.bak"

      - name: Run pre-build automation tasks
-        run : "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/pre_build_automation.sh"
+        run : "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/manifest_placeholder_replacer.sh"

      - name: "Adding flatpak portal for automated updates (cooker only)"
        run: "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/cooker_flatpak_portal_add.sh"
@ -104,7 +104,7 @@ jobs:
          echo "Recalculating hashes and retrying download..."
          rm -f "{GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml"
          cp "${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml.bak" "${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml"
-          "${GITHUB_WORKSPACE}/automation_tools/pre_build_automation.sh"
+          "${GITHUB_WORKSPACE}/automation_tools/manifest_placeholder_replacer.sh"
          "${GITHUB_WORKSPACE}/automation_tools/flatpak_build_download_only.sh"

      - name: Build flatpak