From c1260d0857d480796e3ee99bee41caa249adc860 Mon Sep 17 00:00:00 2001 From: XargonWan Date: Thu, 9 Jan 2025 09:45:59 +0900 Subject: [PATCH] BUILD: pre_build_automation is now called manifest_placeholder_replacer + added hash caching for local builds --- .github/workflows/build_retrodeck.yml | 4 +- .gitignore | 1 + ...on.sh => manifest_placeholder_replacer.sh} | 42 +++++++++++++++++-- developer_toolbox/build_retrodeck_locally.sh | 10 ++++- .../cooker-selfhosted-persistent.yml | 4 +- old/disabled-workflows/cooker-selfhosted.yml | 4 +- 6 files changed, 54 insertions(+), 11 deletions(-) rename automation_tools/{pre_build_automation.sh => manifest_placeholder_replacer.sh} (87%) diff --git a/.github/workflows/build_retrodeck.yml b/.github/workflows/build_retrodeck.yml index c0457f5e..ce03facc 100644 --- a/.github/workflows/build_retrodeck.yml +++ b/.github/workflows/build_retrodeck.yml @@ -147,7 +147,7 @@ jobs: run: "cp ${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml ${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml.bak" - name: Run pre-build automation tasks - run: "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/pre_build_automation.sh" + run: "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/manifest_placeholder_replacer.sh" - name: "Adding flatpak portal for automated updates (Cooker only)" if: github.ref != 'refs/heads/main' @@ -269,7 +269,7 @@ jobs: echo "Recalculating hashes and retrying download..." rm -f "{GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml" cp "${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml.bak" "${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml" - "${GITHUB_WORKSPACE}/automation_tools/pre_build_automation.sh" + "${GITHUB_WORKSPACE}/automation_tools/manifest_placeholder_replacer.sh" "${GITHUB_WORKSPACE}/automation_tools/flatpak_build_download_only.sh" - name: Build flatpak diff --git a/.gitignore b/.gitignore index 17a6e2a2..510b780f 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ incconfigs/ *.flatpak *.flatpak.sha net.retrodeck.retrodeck.cached.yml +placeholders.cache # Python # ########## diff --git a/automation_tools/pre_build_automation.sh b/automation_tools/manifest_placeholder_replacer.sh similarity index 87% rename from automation_tools/pre_build_automation.sh rename to automation_tools/manifest_placeholder_replacer.sh index 26c50c78..a6c135dd 100755 --- a/automation_tools/pre_build_automation.sh +++ b/automation_tools/manifest_placeholder_replacer.sh @@ -4,7 +4,7 @@ set -e # For the file paths to work correctly, call this script with this command from the cloned repo folder root: -# sh automation_tools/pre_build_automation.sh +# sh automation_tools/manifest_placeholder_replacer.sh # Different actions need different information in the task list file # branch: This changes the placeholder text to the currently-detected GIT branch if an automated build was started from a PR environment. # hash: Finds the SHA256 hash of a file online and updates the placeholder in the manifest. @@ -40,6 +40,12 @@ set -e # Define paths rd_manifest="${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml" automation_task_list="${GITHUB_WORKSPACE}/automation_tools/automation_task_list.cfg" +cache_file="${GITHUB_WORKSPACE}/placeholders.cache" + +# Check if cache file exists +if [ -f "$cache_file" ]; then + echo "Warning: Cache file $cache_file is being used. If you encounter issues with hashes, consider deleting this file." +fi # Retrieve current git branch get_current_branch() { @@ -71,6 +77,34 @@ echo "Task list contents:" cat "$automation_task_list" echo +# Function to get hash from cache or calculate it +get_hash() { + local url="$1" + local hash + + # Check if cache should be used and if cache file exists + # the use_cache variable is initialized by build_retrodeck_locally only so in the pipeline it will never use cache + if [ "$use_cache" == "true" ] && [ -f "$cache_file" ]; then + # Try to retrieve hash from cache + hash=$(grep "^$url " "$cache_file" | cut -d ' ' -f2) + if [ -n "$hash" ]; then + echo "Cache file exists, using cached hash for $url" + fi + fi + + # If hash is not found in cache, calculate it + if [ -z "$hash" ]; then + hash=$(curl -sL "$url" | sha256sum | cut -d ' ' -f1) + # Save the calculated hash to cache if caching is enabled + if [ "$use_cache" == "true" ]; then + echo "$url $hash" >> "$cache_file" + fi + fi + + # Return the hash + echo "$hash" +} + # Functions to handle different actions handle_branch() { local placeholder="$1" @@ -82,7 +116,7 @@ handle_hash() { local placeholder="$1" local url="$2" local calculated_url=$(eval echo "$url") - local hash=$(curl -sL "$calculated_url" | sha256sum | cut -d ' ' -f1) + local hash=$(get_hash "$calculated_url") echo "Replacing placeholder $placeholder with hash $hash" /bin/sed -i 's^'"$placeholder"'^'"$hash"'^g' "$rd_manifest" } @@ -118,7 +152,7 @@ handle_latestghrelease() { exit 1 fi - local ghreleasehash=$(curl -sL "$ghreleaseurl" | sha256sum | cut -d ' ' -f1) + local ghreleasehash=$(get_hash "$ghreleaseurl") echo "Replacing placeholder $placeholder with URL $ghreleaseurl and hash $ghreleasehash" /bin/sed -i 's^'"$placeholder"'^'"$ghreleaseurl"'^g' "$rd_manifest" @@ -139,7 +173,7 @@ handle_latestghreleasesha() { exit 1 fi - local ghreleasehash=$(curl -sL "$ghreleaseurl" | sha256sum | cut -d ' ' -f1) + local ghreleasehash=$(get_hash "$ghreleaseurl") echo "Replacing placeholder $placeholder with hash $ghreleasehash" /bin/sed -i 's^'"$placeholder"'^'"$ghreleasehash"'^g' "$rd_manifest" diff --git a/developer_toolbox/build_retrodeck_locally.sh b/developer_toolbox/build_retrodeck_locally.sh index 18899f75..c8a5da10 100755 --- a/developer_toolbox/build_retrodeck_locally.sh +++ b/developer_toolbox/build_retrodeck_locally.sh @@ -10,6 +10,14 @@ # fi # fi +read -rp "Do you want to use the hashes cache? If you're unsure just say no [Y/n]" use_cache_input +use_cache_input=${use_cache_input:-Y} +if [[ "$use_cache_input" =~ ^[Yy]$ ]]; then + export use_cache="true" +else + export use_cache="false" +fi + git submodule update --init --recursive export GITHUB_WORKSPACE="." @@ -23,7 +31,7 @@ cp net.retrodeck.retrodeck.yml net.retrodeck.retrodeck.yml.bak automation_tools/install_dependencies.sh automation_tools/cooker_build_id.sh -automation_tools/pre_build_automation.sh +automation_tools/manifest_placeholder_replacer.sh automation_tools/cooker_flatpak_portal_add.sh # THIS SCRIPT IS BROKEN HENCE DISABLED FTM # automation_tools/appdata_management.sh diff --git a/old/disabled-workflows/cooker-selfhosted-persistent.yml b/old/disabled-workflows/cooker-selfhosted-persistent.yml index 1c89a17a..a3752c82 100644 --- a/old/disabled-workflows/cooker-selfhosted-persistent.yml +++ b/old/disabled-workflows/cooker-selfhosted-persistent.yml @@ -78,7 +78,7 @@ jobs: run: "cp ${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml ${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml.bak" - name: Run pre-build automation tasks - run : "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/pre_build_automation.sh" + run : "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/manifest_placeholder_replacer.sh" - name: "Adding flatpak portal for automated updates (cooker only)" run: "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/cooker_flatpak_portal_add.sh" @@ -102,7 +102,7 @@ jobs: echo "Recalculating hashes and retrying download..." rm -f "{GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml" cp "${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml.bak" "${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml" - "${GITHUB_WORKSPACE}/automation_tools/pre_build_automation.sh" + "${GITHUB_WORKSPACE}/automation_tools/manifest_placeholder_replacer.sh" "${GITHUB_WORKSPACE}/automation_tools/flatpak_build_download_only_persistent.sh" - name: Build flatpak diff --git a/old/disabled-workflows/cooker-selfhosted.yml b/old/disabled-workflows/cooker-selfhosted.yml index c264940f..a512a9dd 100644 --- a/old/disabled-workflows/cooker-selfhosted.yml +++ b/old/disabled-workflows/cooker-selfhosted.yml @@ -80,7 +80,7 @@ jobs: run: "cp ${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml ${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml.bak" - name: Run pre-build automation tasks - run : "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/pre_build_automation.sh" + run : "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/manifest_placeholder_replacer.sh" - name: "Adding flatpak portal for automated updates (cooker only)" run: "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/cooker_flatpak_portal_add.sh" @@ -104,7 +104,7 @@ jobs: echo "Recalculating hashes and retrying download..." rm -f "{GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml" cp "${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml.bak" "${GITHUB_WORKSPACE}/net.retrodeck.retrodeck.yml" - "${GITHUB_WORKSPACE}/automation_tools/pre_build_automation.sh" + "${GITHUB_WORKSPACE}/automation_tools/manifest_placeholder_replacer.sh" "${GITHUB_WORKSPACE}/automation_tools/flatpak_build_download_only.sh" - name: Build flatpak