From 558382c83ba79a8e3895d12b2cdac6dc75636ab0 Mon Sep 17 00:00:00 2001 From: XargonWan Date: Wed, 25 Sep 2024 10:10:46 +0900 Subject: [PATCH] Added automation tool for local build [skip ci] --- .github/workflows/build_es-de.yml | 3 + automation_tools/automation_task_list.cfg | 7 + automation_tools/pre_build_automation.sh | 257 ++++++++++++++++++++++ developer_toolbox/build_locally.sh | 46 ++++ es-de-module.yml | 4 +- 5 files changed, 315 insertions(+), 2 deletions(-) create mode 100644 automation_tools/automation_task_list.cfg create mode 100644 automation_tools/pre_build_automation.sh create mode 100755 developer_toolbox/build_locally.sh diff --git a/.github/workflows/build_es-de.yml b/.github/workflows/build_es-de.yml index 07b797a60..afd3fdfbb 100644 --- a/.github/workflows/build_es-de.yml +++ b/.github/workflows/build_es-de.yml @@ -23,6 +23,9 @@ jobs: - name: "Install dependencies" run: "automation_tools/install_dependencies.sh" + - name: Run pre-build automation tasks + run : "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/pre_build_automation.sh" + - name: "Creating ES-DE manifest" run: "/bin/bash ${GITHUB_WORKSPACE}/automation_tools/update_es-de_manifest.sh" diff --git a/automation_tools/automation_task_list.cfg b/automation_tools/automation_task_list.cfg new file mode 100644 index 000000000..7ae79f32f --- /dev/null +++ b/automation_tools/automation_task_list.cfg @@ -0,0 +1,7 @@ +# The proper format for this file is +# ACTION^PLACEHOLDERTEXT^URL^REPO(Optional) +# hash^DOOMSHAPLACEHOLDER^https://buildbot.libretro.com/assets/cores/DOOM/Doom%20%28Shareware%29.zip +# Please leave some newlines in the end of this file or it will break + +branch^THISBRANCH +THISREPO^THISREPO diff --git a/automation_tools/pre_build_automation.sh b/automation_tools/pre_build_automation.sh new file mode 100644 index 000000000..d0ba4d7e3 --- /dev/null +++ b/automation_tools/pre_build_automation.sh @@ -0,0 +1,257 @@ +#!/bin/bash + +# Set script to exit immediately on any error +set -e + +# For the file paths to work correctly, call this script with this command from the cloned repo folder root: +# sh automation_tools/pre_build_automation.sh +# Different actions need different information in the task list file +# branch: This changes the placeholder text to the currently-detected GIT branch if an automated build was started from a PR environment. +# hash: Finds the SHA256 hash of a file online and updates the placeholder in the manifest. +# Needs the URL of the file, in this line format: hash^PLACEHOLDERTEXT^url +# latestcommit: Finds the most recent commit of a git repo and updates the placeholder in the manifest. +# Needs the URL of the repo and the branch to find the latest commit from, in this line format: latestcommit^PLACEHOLDERTEXT^url^branch +# latestghtag: Finds the most recent tag on a GitHub repo, for repos that don't have normal releases, but also shouldn't use the latest commit +# Needs the URL of the repo, in this line format: latestghtag^PLACEHOLDERTEXT^url +# latestghrelease: Finds the download URL and SHA256 hash of the latest release from a git repo. +# Needs the API URL of the repo, in this line format: latestghrelease^PLACEHOLDERTEXT^https://api.github.com/repos///releases/latest^ +# As this command updates two different placeholders (one for the URL, one for the file hash) in the manifest, +# the URL that would be used in the above example is "PLACEHOLDERTEXT" and the hash placeholder text would be "HASHPLACEHOLDERTEXT" +# The "HASH" prefix of the hash placeholder text is hardcoded in the script. +# The will be the file extension or other identifying suffix at the end of the file name that can be used to select from multiple releases. +# Example: If there are these file options for a given release: +# yuzu-mainline-20240205-149629642.AppImage +# yuzu-linux-20240205-149629642-source.tar.xz +# yuzu-linux-20240205-149629642-debug.tar.xz +# Entering "AppImage" (without quotes) for the will identify yuzu-mainline-20240205-149629642.AppImage +# Entering "source-.tar.xz" (without quotes) for the will identify yuzu-linux-20240205-149629642-source.tar.xz +# Entering "debug-tar.xz" (without quotes) for the will identify yuzu-linux-20240205-149629642-debug.tar.xz +# As a file extension like ".tar.xz" can apply to multiple file options, the entire part that is appended to each release name should be included. +# The will also only consider entries where the given suffix is at the end of the file name. So "AppImage" will identify "file.AppImage" but not "file.AppImage.zsync" +# latestghreleasesha: Finds the SHA256 hash of a specific asset in the latest release from a git repo. +# Needs the API URL of the repo, in this line format: latestghreleasesha^PLACEHOLDERTEXT^https://api.github.com/repos///releases/latest^ +# This command updates the placeholder in the manifest with the SHA256 hash of the specified asset. +# outside_file: Prints the contents of a file from the build environment (such as the buildid file) and replaces the placeholder text with those contents. +# outside_env_var: Gets the value of an environmental variable from the build environment (the output of "echo $var" from the terminal) and replaces the placeholder text with that value. +# custom_command: Runs a single command explicitly as written in the $URL field of the task list, including variable and command expansion. This should work the same as if you were running the command directly from the terminal. +# This command does not need a PLACEHOLDERTEXT field in the task list, so needs to be in this syntax: custom_command^^$COMMAND +# url: This is used to calculate a dynamic URL and the value to the $calculated_url environmental variable, for use in other subsequent commands. + +# Define paths +rd_manifest="${GITHUB_WORKSPACE}/es-de-module.yml" +automation_task_list="${GITHUB_WORKSPACE}/automation_tools/automation_task_list.cfg" + +# Retrieve current git branch +get_current_branch() { + local branch=$(git rev-parse --abbrev-ref HEAD) + if [ "$branch" == "HEAD" ]; then + echo "$GITHUB_REF" | sed 's@refs/heads/@@' + else + echo "$branch" + fi +} + +# Retrieve the repository URL +get_repo_url() { + local repo_url=$(git config --get remote.origin.url) + # Convert SSH URL to HTTPS if needed + if [[ "$repo_url" == git@* ]]; then + repo_url=$(echo "$repo_url" | sed -e 's|git@|https://|' -e 's|:|/|') + fi + echo "$repo_url" +} + +current_branch=$(get_current_branch) +current_repo_url=$(get_repo_url) + +echo "Manifest location: $rd_manifest" +echo "Automation task list location: $automation_task_list" +echo +echo "Task list contents:" +cat "$automation_task_list" +echo + +# Functions to handle different actions +handle_branch() { + local placeholder="$1" + echo "Replacing placeholder $placeholder with branch $current_branch" + /bin/sed -i 's^'"$placeholder"'^'"$current_branch"'^g' "$rd_manifest" +} + +handle_hash() { + local placeholder="$1" + local url="$2" + local calculated_url=$(eval echo "$url") + local hash=$(curl -sL "$calculated_url" | sha256sum | cut -d ' ' -f1) + echo "Replacing placeholder $placeholder with hash $hash" + /bin/sed -i 's^'"$placeholder"'^'"$hash"'^g' "$rd_manifest" +} + +handle_latestcommit() { + local placeholder="$1" + local url="$2" + local branch="$3" + local commit=$(git ls-remote "$url" "$branch" | cut -f1) + echo "Replacing placeholder $placeholder with latest commit $commit" + /bin/sed -i 's^'"$placeholder"'^'"$commit"'^g' "$rd_manifest" +} + +handle_latestghtag() { + local placeholder="$1" + local url="$2" + local tag=$(git ls-remote --tags "$url" | tail -n 1 | cut -f2 | sed 's|refs/tags/||') + echo "Replacing placeholder $placeholder with latest tag $tag" + /bin/sed -i 's^'"$placeholder"'^'"$tag"'^g' "$rd_manifest" +} + +handle_latestghrelease() { + local placeholder="$1" + local api_url="$2" + local pattern="$3" + echo "Fetching release data from: $api_url" + + # Fetch the release data from GitHub API + local release_data=$(curl -s "$api_url") + echo "Release data fetched." + + # Find the matching asset using the pattern + local ghreleaseurl=$(echo "$release_data" | jq -r ".assets[] | select(.name | test(\"$pattern\")).browser_download_url") + + if [[ -z "$ghreleaseurl" ]]; then + echo "Error: No asset found matching pattern $pattern" + exit 1 + fi + + # Download the file and compute its hash + local ghreleasehash=$(curl -sL "$ghreleaseurl" | sha256sum | cut -d ' ' -f1) + + echo "Replacing placeholder $placeholder with URL $ghreleaseurl and hash $ghreleasehash" + /bin/sed -i 's^'"$placeholder"'^'"$ghreleaseurl"'^g' "$rd_manifest" + /bin/sed -i 's^'"HASHFOR$placeholder"'^'"$ghreleasehash"'^g' "$rd_manifest" +} + +handle_latestghreleasesha() { + local placeholder="$1" + local api_url="$2" + local pattern="$3" + echo "Fetching release data from: $api_url" + + # Fetch the release data from GitHub API + local release_data=$(curl -s "$api_url") + echo "Release data fetched." + + # Find the matching asset using the pattern + local ghreleaseurl=$(echo "$release_data" | jq -r ".assets[] | select(.name | test(\"$pattern\")).browser_download_url") + + if [[ -z "$ghreleaseurl" ]]; then + echo "Error: No asset found matching pattern $pattern" + exit 1 + fi + + # Download the file and compute its hash + echo "Downloading asset to compute hash: $ghreleaseurl" + local ghreleasehash=$(curl -sL "$ghreleaseurl" | sha256sum | cut -d ' ' -f1) + + echo "Replacing placeholder $placeholder with hash $ghreleasehash" + /bin/sed -i 's^'"$placeholder"'^'"$ghreleasehash"'^g' "$rd_manifest" +} + +handle_outside_file() { + local placeholder="$1" + local file_path="$2" + if [[ "$file_path" == \$* ]]; then + eval file_path="$file_path" + fi + local content=$(cat "$file_path") + echo "Replacing placeholder $placeholder with content of file $file_path" + /bin/sed -i 's^'"$placeholder"'^'"$content"'^g' "$rd_manifest" +} + +handle_outside_env_var() { + local placeholder="$1" + local var_name="$2" + if [[ "$var_name" == \$* ]]; then + eval var_name="$var_name" + fi + local value=$(echo "$var_name") + echo "Replacing placeholder $placeholder with environment variable $value" + /bin/sed -i 's^'"$placeholder"'^'"$value"'^g' "$rd_manifest" +} + +handle_custom_command() { + local command="$1" + echo "Executing custom command: $command" + eval "$command" +} + +handle_url() { + local placeholder="$1" + local url="$2" + local calculated_url=$(eval echo "$url") + echo "Replacing placeholder $placeholder with calculated URL $calculated_url" + /bin/sed -i 's^'"$placeholder"'^'"$calculated_url"'^g' "$rd_manifest" +} + +# Handle the THISREPO placeholder +handle_thisrepo() { + local placeholder="$1" + echo "Replacing placeholder $placeholder with repository URL $current_repo_url" + /bin/sed -i 's^'"$placeholder"'^'"$current_repo_url"'^g' "$rd_manifest" +} + +# New function to handle the latest artifact from GitHub Actions +handle_latestghaartifact() { + local placeholder_url="$1" + local placeholder_hash="$2" + local workflow_url="$3" + local artifact_name="$4" + + echo "Fetching workflow runs from: $workflow_url" + workflow_runs_url=$(echo "$workflow_url" | sed 's/github.com/api.github.com\/repos/' | sed 's/actions\/workflows\/[^\/]*$/actions\/runs/') + + local runs_data=$(curl -s "$workflow_runs_url") + local latest_run_url=$(echo "$runs_data" | jq -r ".workflow_runs[0].artifacts_url") + + if [[ -z "$latest_run_url" ]]; then + echo "Error: No workflow runs found" + exit 1 + fi + + echo "Fetching artifacts from the latest run: $latest_run_url" + local artifacts_data=$(curl -s "$latest_run_url") + local artifact_url=$(echo "$artifacts_data" | jq -r ".artifacts[] | select(.name == \"$artifact_name\").archive_download_url") + + if [[ -z "$artifact_url" ]]; then + echo "Error: No artifact found with name $artifact_name" + exit 1 + fi + + echo "Downloading the artifact to calculate the hash..." + local artifact_hash=$(curl -sL "$artifact_url" | sha256sum | cut -d ' ' -f1) + + echo "Replacing placeholder $placeholder_url with artifact URL $artifact_url" + echo "Replacing placeholder $placeholder_hash with artifact hash $artifact_hash" + /bin/sed -i 's^'"$placeholder_url"'^'"$artifact_url"'^g' "$rd_manifest" + /bin/sed -i 's^'"$placeholder_hash"'^'"$artifact_hash"'^g' "$rd_manifest" +} + +# Process the task list +while IFS="^" read -r action placeholder url branch artifact_name || [[ -n "$action" ]]; do + if [[ ! "$action" == "#"* ]] && [[ -n "$action" ]]; then + case "$action" in + "branch" ) handle_branch "$placeholder" ;; + "hash" ) handle_hash "$placeholder" "$url" ;; + "latestcommit" ) handle_latestcommit "$placeholder" "$url" "$branch" ;; + "latestghtag" ) handle_latestghtag "$placeholder" "$url" ;; + "latestghrelease" ) handle_latestghrelease "$placeholder" "$url" "$branch" ;; + "latestghreleasesha" ) handle_latestghreleasesha "$placeholder" "$url" "$branch" ;; + "outside_file" ) handle_outside_file "$placeholder" "$url" ;; + "outside_env_var" ) handle_outside_env_var "$placeholder" "$url" ;; + "custom_command" ) handle_custom_command "$url" ;; + "url" ) handle_url "$placeholder" "$url" ;; + "THISREPO" ) handle_thisrepo "$placeholder" ;; + "latestghaartifact" ) handle_latestghaartifact "$placeholder" "$branch" "$url" "$artifact_name" ;; + esac + fi +done < "$automation_task_list" + diff --git a/developer_toolbox/build_locally.sh b/developer_toolbox/build_locally.sh new file mode 100755 index 000000000..bb7166bef --- /dev/null +++ b/developer_toolbox/build_locally.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# WARNING: run this script from the project root folder, not from here!! + +# Check if script is running with elevated privileges +if [ "$EUID" -ne 0 ]; then + echo "The build might fail without some superuser permissions, please run me with sudo. Continue without sudo? [y/N]" + read -r continue_without_sudo + if [[ "$continue_without_sudo" != "y" ]]; then + exit 1 + fi +fi + +git submodule update --init --recursive + +export GITHUB_WORKSPACE="." + +# Initialize the Flatpak repo +ostree init --mode=archive-z2 --repo=${GITHUB_WORKSPACE}/retrodeck-repo + +cp net.retrodeck.retrodeck.appdata.xml net.retrodeck.retrodeck.appdata.xml.bak +cp net.retrodeck.retrodeck.yml net.retrodeck.retrodeck.yml.bak + +automation_tools/install_dependencies.sh +#automation_tools/cooker_build_id.sh +automation_tools/pre_build_automation.sh +#automation_tools/cooker_flatpak_portal_add.sh +# THIS SCRIPT IS BROKEN HENCE DISABLED FTM +# automation_tools/appdata_management.sh +#automation_tools/flatpak_build_download_only.sh +#automation_tools/flatpak_build_only.sh +#automation_tools/flatpak_build_bundle.sh + +flatpak-builder --user --force-clean \ + --install-deps-from=flathub \ + --install-deps-from=flathub-beta \ + --repo="${GITHUB_WORKSPACE}/retrodeck-repo" \ + "${GITHUB_WORKSPACE}/${FOLDER}" \ + net.retrodeck.es-de.yml + +flatpak build-bundle "${GITHUB_WORKSPACE}/retrodeck-repo" "$GITHUB_WORKSPACE/RetroDECK-ES-DE.flatpak" net.retrodeck.es-de + +rm -f net.retrodeck.retrodeck.appdata.xml +rm -f net.retrodeck.retrodeck.yml +cp net.retrodeck.retrodeck.appdata.xml.bak net.retrodeck.retrodeck.appdata.xml +cp net.retrodeck.retrodeck.yml.bak net.retrodeck.retrodeck.yml diff --git a/es-de-module.yml b/es-de-module.yml index d802040c5..a4a98c3d7 100644 --- a/es-de-module.yml +++ b/es-de-module.yml @@ -107,8 +107,8 @@ modules: - es-core sources: - type: git - url: https://github.com/RetroDECK/ES-DE - branch: retrodeck-main + url: THISREPO + branch: THISBRANCH - type: shell # ES-DE - END \ No newline at end of file