mirror of
https://github.com/RetroDECK/RetroVERSE.git
synced 2026-10-16 17:16:39 +00:00
1502 lines
59 KiB
YAML
1502 lines
59 KiB
YAML
name: "RetroVERSE: Build Inventory"
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
branch_override:
|
|
description: 'Override branch (leave empty to use selected branch from dropdown)'
|
|
required: false
|
|
type: string
|
|
|
|
dry_run:
|
|
description: 'Dry run: build and validate but do not create release'
|
|
required: false
|
|
type: boolean
|
|
default: true
|
|
|
|
push:
|
|
branches:
|
|
- main
|
|
- cooker
|
|
|
|
permissions:
|
|
contents: write
|
|
|
|
env:
|
|
SOURCE_ID: official
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The Alchemist is not vendored here. It is checked out from the components
|
|
# repo at a ref pinned in alchemist/alchemist_ref, so the RetroVERSE repo carries
|
|
# only its own version_policy.sh (and optional version_pins.sh), which are
|
|
# overlaid onto the checkout.
|
|
#
|
|
# The ref is pinned rather than floating on main: an unpinned tool could
|
|
# change build output with no commit to this repo, and reuse would not notice
|
|
# because the fingerprint covers item folders only. A release could then mix
|
|
# artifacts built by two different tool versions.
|
|
# ---------------------------------------------------------------------------
|
|
ALCHEMIST_REPO: RetroDECK/components
|
|
ALCHEMIST_PATH: .alchemist
|
|
ALCHEMIST_DIR: .alchemist/automation-tools/alchemist
|
|
|
|
jobs:
|
|
# ===========================================================================
|
|
# Check whether this event should trigger a build
|
|
# ===========================================================================
|
|
check-trigger:
|
|
name: Check Build Trigger
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
should_build: ${{ steps.check.outputs.should_build }}
|
|
is_dry_run: ${{ steps.check.outputs.is_dry_run }}
|
|
steps:
|
|
- name: Checkout code
|
|
if: github.event_name == 'push'
|
|
uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Evaluate trigger
|
|
id: check
|
|
run: |
|
|
BUILD_TRIGGER="build"
|
|
RELEASE_TRIGGER="release"
|
|
DRY_RUN_TRIGGER="dry-run"
|
|
|
|
# --- Manual dispatch ---
|
|
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
|
|
echo "Manual trigger"
|
|
echo "should_build=true" >> $GITHUB_OUTPUT
|
|
echo "is_dry_run=${{ inputs.dry_run }}" >> $GITHUB_OUTPUT
|
|
exit 0
|
|
fi
|
|
|
|
# --- Push event ---
|
|
COMMITS=$(git log --format=%B ${{ github.event.before }}..${{ github.event.after }})
|
|
|
|
# PR merge commits always trigger a build
|
|
if echo "$COMMITS" | grep -qiE "^Merge pull request #"; then
|
|
echo "PR merge detected: auto-building"
|
|
echo "should_build=true" >> $GITHUB_OUTPUT
|
|
echo "is_dry_run=false" >> $GITHUB_OUTPUT
|
|
exit 0
|
|
fi
|
|
|
|
if echo "$COMMITS" | grep -qiE "\[${DRY_RUN_TRIGGER}\]"; then
|
|
echo "Found [dry-run] trigger in commit messages"
|
|
echo "should_build=true" >> $GITHUB_OUTPUT
|
|
echo "is_dry_run=true" >> $GITHUB_OUTPUT
|
|
exit 0
|
|
fi
|
|
|
|
if echo "$COMMITS" | grep -qiE "\[(${RELEASE_TRIGGER}|${BUILD_TRIGGER})\]"; then
|
|
echo "Found build/release trigger in commit messages"
|
|
echo "should_build=true" >> $GITHUB_OUTPUT
|
|
echo "is_dry_run=false" >> $GITHUB_OUTPUT
|
|
exit 0
|
|
fi
|
|
|
|
echo "No build trigger found in commit messages: skipping"
|
|
echo "should_build=false" >> $GITHUB_OUTPUT
|
|
echo "is_dry_run=false" >> $GITHUB_OUTPUT
|
|
|
|
# ===========================================================================
|
|
# Determine build parameters and validate sources
|
|
# ===========================================================================
|
|
configure:
|
|
name: Configure Build and Validate Files
|
|
needs: check-trigger
|
|
if: needs.check-trigger.outputs.should_build == 'true'
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
branch: ${{ steps.config.outputs.branch }}
|
|
is_dry_run: ${{ needs.check-trigger.outputs.is_dry_run }}
|
|
alchemist_ref: ${{ steps.alchemist-ref.outputs.ref }}
|
|
steps:
|
|
- name: Determine build parameters
|
|
id: config
|
|
run: |
|
|
if [[ -n "${{ inputs.branch_override }}" ]]; then
|
|
BRANCH="${{ inputs.branch_override }}"
|
|
else
|
|
BRANCH="${{ github.ref_name }}"
|
|
fi
|
|
|
|
echo "branch=$BRANCH" >> $GITHUB_OUTPUT
|
|
|
|
echo ""
|
|
echo "=========================================="
|
|
echo "Build Configuration"
|
|
echo "=========================================="
|
|
echo " Trigger: ${{ github.event_name }}"
|
|
echo " Branch: $BRANCH"
|
|
echo " Dry run: ${{ needs.check-trigger.outputs.is_dry_run }}"
|
|
echo "=========================================="
|
|
|
|
- name: Checkout code
|
|
uses: actions/checkout@v6
|
|
with:
|
|
ref: ${{ steps.config.outputs.branch }}
|
|
|
|
# -----------------------------------------------------------------------
|
|
# The pinned Alchemist ref. A tag or full commit SHA.
|
|
# -----------------------------------------------------------------------
|
|
- name: Read pinned Alchemist ref
|
|
id: alchemist-ref
|
|
run: |
|
|
REF_FILE="alchemist/alchemist_ref"
|
|
|
|
if [[ ! -f "$REF_FILE" ]]; then
|
|
echo "ERROR: $REF_FILE not found: the Alchemist ref must be pinned in this repo"
|
|
exit 1
|
|
fi
|
|
|
|
REF=$(grep -vE '^\s*(#|$)' "$REF_FILE" | head -n1 | tr -d '[:space:]')
|
|
|
|
if [[ -z "$REF" ]]; then
|
|
echo "ERROR: $REF_FILE is empty"
|
|
exit 1
|
|
fi
|
|
|
|
case "$REF" in
|
|
main|cooker|epicure|HEAD)
|
|
echo "ERROR: '$REF' is a branch, not a pin. Use a tag or a full commit SHA."
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
echo "ref=$REF" >> $GITHUB_OUTPUT
|
|
echo "Alchemist pinned to: $REF"
|
|
|
|
- name: Validate version policy is present
|
|
run: |
|
|
# load_versions.sh checks for version_policy.sh at source time, before any recipe is read, and exits if it is missing. This repo therefore carries its own even if it holds only data items.
|
|
# A comment-only file is enough.
|
|
if [[ ! -f "alchemist/version_policy.sh" ]]; then
|
|
echo "ERROR: alchemist/version_policy.sh not found"
|
|
echo "The Alchemist exits at source time without it, even when no item needs a policy."
|
|
exit 1
|
|
fi
|
|
|
|
echo "Version policy present"
|
|
|
|
- name: Validate Bash scripts
|
|
run: |
|
|
echo "Validating Bash scripts..."
|
|
|
|
ERRORS=0
|
|
FILES=0
|
|
while IFS= read -r script; do
|
|
if ! bash -n "$script" 2>/dev/null; then
|
|
echo "FAILED: $script"
|
|
ERRORS=$((ERRORS + 1))
|
|
else
|
|
FILES=$((FILES + 1))
|
|
fi
|
|
done < <(find . -type f -name "*.sh")
|
|
|
|
if [[ "$ERRORS" -gt 0 ]]; then
|
|
echo "ERROR: $ERRORS script(s) failed syntax validation"
|
|
exit 1
|
|
fi
|
|
|
|
echo "$FILES Bash scripts passed syntax validation"
|
|
|
|
- name: Validate JSON files
|
|
run: |
|
|
echo "Validating JSON files..."
|
|
|
|
ERRORS=0
|
|
FILES=0
|
|
while IFS= read -r json; do
|
|
if ! jq empty "$json" 2>/dev/null; then
|
|
echo "FAILED: $json"
|
|
ERRORS=$((ERRORS + 1))
|
|
else
|
|
FILES=$((FILES + 1))
|
|
fi
|
|
done < <(find . -type f -name "*.json")
|
|
|
|
if [[ "$ERRORS" -gt 0 ]]; then
|
|
echo "ERROR: $ERRORS JSON file(s) failed validation"
|
|
exit 1
|
|
fi
|
|
|
|
echo "$FILES JSON files passed validation"
|
|
|
|
# ---------------------------------------------------------------------
|
|
# Static manifest checks. Anything requiring a built artifact runs in the build job instead.
|
|
# See "Validate item against artifact".
|
|
# ---------------------------------------------------------------------
|
|
- name: Validate RetroVERSE manifests
|
|
run: |
|
|
echo "Validating RetroVERSE manifests..."
|
|
|
|
KNOWN_POLICIES="sync replace preserve_if_modified install_once"
|
|
ERRORS=0
|
|
ITEMS=0
|
|
|
|
for manifest in */retroverse_manifest.json; do
|
|
[[ -f "$manifest" ]] || continue
|
|
|
|
item_dir=$(dirname "$manifest")
|
|
item=$(basename "$item_dir")
|
|
ITEMS=$((ITEMS + 1))
|
|
|
|
fail() {
|
|
echo " FAILED [$item]: $1"
|
|
ERRORS=$((ERRORS + 1))
|
|
}
|
|
|
|
id=$(jq -r '.id // empty' "$manifest")
|
|
class=$(jq -r '.class // empty' "$manifest")
|
|
version=$(jq -r '.version // empty' "$manifest")
|
|
name=$(jq -r '.name // empty' "$manifest")
|
|
|
|
[[ -n "$id" ]] || fail "missing 'id'"
|
|
[[ -n "$class" ]] || fail "missing 'class'"
|
|
[[ -n "$version" ]] || fail "missing 'version'"
|
|
[[ -n "$name" ]] || fail "missing 'name'"
|
|
|
|
if [[ -n "$id" && "$id" != "$item" ]]; then
|
|
fail "id '$id' does not match folder name '$item'"
|
|
fi
|
|
|
|
if [[ ! -f "$item_dir/component_recipe.json" ]]; then
|
|
fail "no component_recipe.json"
|
|
fi
|
|
|
|
file_count=$(jq '.files // [] | length' "$manifest")
|
|
|
|
case "$class" in
|
|
data)
|
|
if [[ "$file_count" -eq 0 ]]; then
|
|
fail "class 'data' declares no files[]"
|
|
fi
|
|
;;
|
|
component)
|
|
if [[ "$file_count" -ne 0 ]]; then
|
|
fail "class 'component' must not declare files[]: deployment belongs to the external component installer"
|
|
fi
|
|
|
|
# RetroVERSE version is read from retroverse_manifest.json for both classes so the aggregator needs no class-specific logic.
|
|
# For components it must agree with the packaged manifest.
|
|
cm="$item_dir/component_manifest.json"
|
|
if [[ ! -f "$cm" ]]; then
|
|
fail "class 'component' has no component_manifest.json"
|
|
else
|
|
cm_version=$(jq -r '.version // empty' "$cm")
|
|
if [[ "$cm_version" != "$version" ]]; then
|
|
fail "version mismatch: retroverse_manifest.json '$version' vs component_manifest.json '$cm_version'"
|
|
fi
|
|
fi
|
|
;;
|
|
*)
|
|
fail "unknown class '$class'"
|
|
;;
|
|
esac
|
|
|
|
# Policy vocabulary
|
|
while IFS= read -r policy; do
|
|
[[ -n "$policy" ]] || continue
|
|
if ! grep -qw "$policy" <<< "$KNOWN_POLICIES"; then
|
|
fail "unknown policy '$policy'"
|
|
fi
|
|
done < <(jq -r '.files // [] | .[] | .policy // "sync"' "$manifest")
|
|
|
|
# Destination must be present and carry no obvious empty expansion
|
|
while IFS= read -r dest; do
|
|
if [[ -z "$dest" ]]; then
|
|
fail "files[] entry with empty destination"
|
|
fi
|
|
done < <(jq -r '.files // [] | .[] | .destination // ""' "$manifest")
|
|
|
|
# Requirement syntax
|
|
while IFS= read -r req; do
|
|
[[ -n "$req" ]] || continue
|
|
case "$req" in
|
|
component:*|item:*|client_schema:*) ;;
|
|
*) fail "unparseable requirement '$req'" ;;
|
|
esac
|
|
done < <(jq -r '.requires // [] | .[]' "$manifest")
|
|
done
|
|
|
|
if [[ "$ITEMS" -eq 0 ]]; then
|
|
echo "ERROR: No RetroVERSE items found"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "$ERRORS" -gt 0 ]]; then
|
|
echo ""
|
|
echo "ERROR: $ERRORS manifest validation error(s)"
|
|
exit 1
|
|
fi
|
|
|
|
echo "$ITEMS item manifests passed validation"
|
|
|
|
- name: Validate item requirements resolve
|
|
run: |
|
|
echo "Validating item requirements..."
|
|
|
|
# Build the set of item ids present in this inventory
|
|
INVENTORY=$(for m in */retroverse_manifest.json; do
|
|
[[ -f "$m" ]] && jq -r '.id' "$m"
|
|
done)
|
|
|
|
ERRORS=0
|
|
|
|
for manifest in */retroverse_manifest.json; do
|
|
[[ -f "$manifest" ]] || continue
|
|
item=$(jq -r '.id' "$manifest")
|
|
|
|
while IFS= read -r req; do
|
|
[[ "$req" == item:* ]] || continue
|
|
target="${req#item:}"
|
|
|
|
if ! grep -qx "$target" <<< "$INVENTORY"; then
|
|
echo " FAILED [$item]: requires item '$target' which is not in the inventory"
|
|
ERRORS=$((ERRORS + 1))
|
|
fi
|
|
done < <(jq -r '.requires // [] | .[]' "$manifest")
|
|
done
|
|
|
|
if [[ "$ERRORS" -gt 0 ]]; then
|
|
echo "ERROR: $ERRORS unresolved item requirement(s)"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Item requirements resolve"
|
|
|
|
# ===========================================================================
|
|
# Collect item metadata, fingerprints and upstream versions
|
|
# ===========================================================================
|
|
# Replaces the components repo's resolve-versions job. There is no version
|
|
# resolution for the RetroVERSE version, it is read from the manifest. Component
|
|
# items still resolve an upstream version through the pinned policy chain,
|
|
# for the record and for the bump check.
|
|
# ===========================================================================
|
|
collect-metadata:
|
|
name: Collect Item Metadata
|
|
needs: configure
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
outputs:
|
|
items-json: ${{ steps.collect.outputs.items }}
|
|
prev-json: ${{ steps.get-prev.outputs.prev }}
|
|
prev-alchemist-ref: ${{ steps.get-prev.outputs.prev_alchemist_ref }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v6
|
|
with:
|
|
ref: ${{ needs.configure.outputs.branch }}
|
|
fetch-depth: 0
|
|
|
|
# -----------------------------------------------------------------------
|
|
# Alchemist checkout. sparse-checkout keeps the rest of the components
|
|
# repo out of the workspace, and the nested .git is harmless: alchemist.sh
|
|
# resolves REPO_ROOT from its working directory, which stays the RetroVERSE
|
|
# repo root, so $REPO_ROOT/$COMPONENT_NAME still points at the item folder.
|
|
# -----------------------------------------------------------------------
|
|
- name: Checkout Alchemist
|
|
uses: actions/checkout@v6
|
|
with:
|
|
repository: ${{ env.ALCHEMIST_REPO }}
|
|
ref: ${{ needs.configure.outputs.alchemist_ref }}
|
|
path: ${{ env.ALCHEMIST_PATH }}
|
|
sparse-checkout: automation-tools/alchemist
|
|
sparse-checkout-cone-mode: true
|
|
|
|
- name: Overlay RetroVERSE version policy
|
|
run: |
|
|
# version_policy.sh and version_pins.sh are read from the Alchemist's
|
|
# own SCRIPT_DIR, so the components repo's copies arrive with the
|
|
# checkout and must be replaced by this repo's.
|
|
if [[ ! -x "$ALCHEMIST_DIR/alchemist.sh" ]]; then
|
|
echo "ERROR: alchemist.sh not found or not executable at $ALCHEMIST_DIR"
|
|
exit 1
|
|
fi
|
|
|
|
rm -f "$ALCHEMIST_DIR/version_policy.sh" "$ALCHEMIST_DIR/version_pins.sh"
|
|
cp alchemist/version_policy.sh "$ALCHEMIST_DIR/version_policy.sh"
|
|
|
|
if [[ -f alchemist/version_pins.sh ]]; then
|
|
cp alchemist/version_pins.sh "$ALCHEMIST_DIR/version_pins.sh"
|
|
echo "Version pins overlaid"
|
|
fi
|
|
|
|
echo "Alchemist ready at ${{ needs.configure.outputs.alchemist_ref }}"
|
|
|
|
- name: Get previous release metadata
|
|
id: get-prev
|
|
run: |
|
|
BRANCH_NAME="${{ needs.configure.outputs.branch }}"
|
|
|
|
echo "Looking for previous release on branch: ${BRANCH_NAME}"
|
|
|
|
PREV_RELEASE=$(gh release list --limit 100 --json tagName \
|
|
-q "[.[] | select(.tagName | startswith(\"${BRANCH_NAME}-\"))] | .[0].tagName")
|
|
|
|
if [[ -z "$PREV_RELEASE" || "$PREV_RELEASE" == "null" ]]; then
|
|
echo "No previous release found for branch ${BRANCH_NAME}"
|
|
echo "prev={}" >> $GITHUB_OUTPUT
|
|
echo "prev_alchemist_ref=" >> $GITHUB_OUTPUT
|
|
exit 0
|
|
fi
|
|
|
|
echo "Previous release found: ${PREV_RELEASE}"
|
|
|
|
if gh release download "$PREV_RELEASE" -p "retroverse.json" 2>/dev/null; then
|
|
# Reduce the published array back to an id-keyed object for lookups
|
|
PREV=$(jq -c '.items | map({(.id): .}) | add // {}' retroverse.json)
|
|
echo "prev=$PREV" >> $GITHUB_OUTPUT
|
|
echo "Previous inventory: $(jq 'keys | length' <<< "$PREV") items"
|
|
|
|
# Tool version the previous release was built with. A change to it invalidates every reuse (see setup-matrix)
|
|
PREV_ALCHEMIST=$(jq -r '.alchemist_ref // ""' retroverse.json)
|
|
echo "prev_alchemist_ref=$PREV_ALCHEMIST" >> $GITHUB_OUTPUT
|
|
echo "Previous Alchemist ref: ${PREV_ALCHEMIST:-<unrecorded>}"
|
|
else
|
|
echo "No retroverse.json found in previous release"
|
|
echo "prev={}" >> $GITHUB_OUTPUT
|
|
echo "prev_alchemist_ref=" >> $GITHUB_OUTPUT
|
|
fi
|
|
|
|
- name: Check for component items
|
|
id: has-components
|
|
run: |
|
|
if grep -l '"class"[[:space:]]*:[[:space:]]*"component"' */retroverse_manifest.json >/dev/null 2>&1; then
|
|
echo "found=true" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "found=false" >> $GITHUB_OUTPUT
|
|
echo "No component items: skipping build dependency install"
|
|
fi
|
|
|
|
- name: Install build dependencies
|
|
if: steps.has-components.outputs.found == 'true'
|
|
run: |
|
|
echo "Installing build dependencies..."
|
|
|
|
unset pkg_mgr
|
|
for candidate in apt pacman dnf; do
|
|
command -v "$candidate" &> /dev/null && pkg_mgr="$candidate" && break
|
|
done
|
|
|
|
case "${pkg_mgr:-}" in
|
|
apt)
|
|
sudo add-apt-repository -y ppa:flatpak/stable
|
|
sudo apt update
|
|
sudo apt install -y flatpak flatpak-builder p7zip-full xmlstarlet bzip2 curl jq unzip
|
|
;;
|
|
pacman)
|
|
sudo pacman -Sy --noconfirm flatpak flatpak-builder p7zip xmlstarlet bzip2 curl jq unzip
|
|
;;
|
|
dnf)
|
|
sudo dnf install -y flatpak flatpak-builder p7zip p7zip-plugins xmlstarlet bzip2 curl jq unzip
|
|
;;
|
|
*)
|
|
echo "ERROR: No supported package manager found (apt, pacman, dnf)"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
flatpak remote-add --user --if-not-exists flathub https://flathub.org/repo/flathub.flatpakrepo
|
|
flatpak remote-add --user --if-not-exists flathub-beta https://flathub.org/beta-repo/flathub-beta.flatpakrepo
|
|
|
|
echo "Build dependencies installed"
|
|
|
|
- name: Collect metadata and fingerprints
|
|
id: collect
|
|
env:
|
|
PREV: ${{ steps.get-prev.outputs.prev }}
|
|
run: |
|
|
echo "Collecting item metadata..."
|
|
|
|
metadata='{}'
|
|
|
|
for manifest in */retroverse_manifest.json; do
|
|
[[ -f "$manifest" ]] || continue
|
|
|
|
item_dir=$(dirname "$manifest")
|
|
item=$(basename "$item_dir")
|
|
class=$(jq -r '.class' "$manifest")
|
|
version=$(jq -r '.version' "$manifest")
|
|
|
|
echo ""
|
|
echo "Processing $item ($class)..."
|
|
|
|
# -----------------------------------------------------------------
|
|
# Fingerprint covers only what the Alchemist consumes. The RetroVERSE
|
|
# manifest and artwork are excluded: they are republished from
|
|
# source every release, so a description or thumbnail edit must not
|
|
# invalidate an otherwise identical artifact.
|
|
# -----------------------------------------------------------------
|
|
fingerprint=$(git ls-tree -r HEAD -- "$item_dir" \
|
|
| grep -v 'retroverse_manifest\.json$' \
|
|
| grep -v "^.*\s${item_dir}/artwork/" \
|
|
| git hash-object --stdin)
|
|
|
|
# -----------------------------------------------------------------
|
|
# Component items resolve an upstream version through the pinned
|
|
# policy chain. Data items have no upstream at all.
|
|
# -----------------------------------------------------------------
|
|
upstream=""
|
|
if [[ "$class" == "component" ]]; then
|
|
upstream=$("$ALCHEMIST_DIR/alchemist.sh" \
|
|
-f "$item_dir/component_recipe.json" --resolve-versions)
|
|
|
|
if [[ -z "$upstream" ]]; then
|
|
echo "ERROR: Failed to resolve upstream version for $item"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
prev_built_on=$(jq -r --arg i "$item" '.[$i].built_on // empty' <<< "$PREV")
|
|
|
|
metadata=$(jq \
|
|
--arg item "$item" \
|
|
--arg class "$class" \
|
|
--arg version "$version" \
|
|
--arg fingerprint "$fingerprint" \
|
|
--arg upstream "$upstream" \
|
|
--arg built_on "$prev_built_on" \
|
|
--arg dir "$item_dir" \
|
|
'.[$item] = {
|
|
id: $item,
|
|
dir: $dir,
|
|
class: $class,
|
|
version: $version,
|
|
fingerprint: $fingerprint,
|
|
upstream_version: $upstream,
|
|
built_on: $built_on
|
|
}' <<< "$metadata")
|
|
|
|
echo " Version: $version"
|
|
echo " Fingerprint: ${fingerprint:0:12}..."
|
|
[[ -n "$upstream" ]] && echo " Upstream: $upstream"
|
|
done
|
|
|
|
echo "items=$(jq -c . <<< "$metadata")" >> $GITHUB_OUTPUT
|
|
|
|
echo ""
|
|
echo "Item versions:"
|
|
jq -r 'to_entries[] | " \(.key): \(.value.version)"' <<< "$metadata"
|
|
|
|
# ---------------------------------------------------------------------
|
|
# The bump check is the exact inverse of the reuse condition. If the
|
|
# fingerprint or the resolved upstream version moved while the manifest
|
|
# version did not, the package changed without being versioned.
|
|
#
|
|
# The upstream check cannot fold into the fingerprint: version_policy.sh
|
|
# and version_pins.sh live beside the Alchemist, outside the item folder,
|
|
# so a pin bump leaves the fingerprint untouched.
|
|
# ---------------------------------------------------------------------
|
|
- name: Enforce manifest version bumps
|
|
env:
|
|
CURRENT: ${{ steps.collect.outputs.items }}
|
|
PREV: ${{ steps.get-prev.outputs.prev }}
|
|
run: |
|
|
if [[ "$PREV" == "{}" ]]; then
|
|
echo "No previous release: skipping bump check"
|
|
exit 0
|
|
fi
|
|
|
|
echo "Checking manifest version bumps..."
|
|
|
|
ERRORS=0
|
|
|
|
while IFS= read -r item; do
|
|
cur_version=$(jq -r --arg i "$item" '.[$i].version' <<< "$CURRENT")
|
|
cur_fp=$(jq -r --arg i "$item" '.[$i].fingerprint' <<< "$CURRENT")
|
|
cur_up=$(jq -r --arg i "$item" '.[$i].upstream_version // ""' <<< "$CURRENT")
|
|
|
|
prev_version=$(jq -r --arg i "$item" '.[$i].version // empty' <<< "$PREV")
|
|
prev_fp=$(jq -r --arg i "$item" '.[$i].fingerprint // empty' <<< "$PREV")
|
|
prev_up=$(jq -r --arg i "$item" '.[$i].upstream_version // ""' <<< "$PREV")
|
|
|
|
# New item: nothing to compare against
|
|
if [[ -z "$prev_version" ]]; then
|
|
continue
|
|
fi
|
|
|
|
[[ "$cur_version" != "$prev_version" ]] && continue
|
|
|
|
if [[ "$cur_fp" != "$prev_fp" ]]; then
|
|
echo " FAILED [$item]: content changed but version is still $cur_version"
|
|
ERRORS=$((ERRORS + 1))
|
|
elif [[ "$cur_up" != "$prev_up" ]]; then
|
|
echo " FAILED [$item]: upstream version $prev_up -> $cur_up but version is still $cur_version"
|
|
ERRORS=$((ERRORS + 1))
|
|
fi
|
|
done < <(jq -r 'keys[]' <<< "$CURRENT")
|
|
|
|
if [[ "$ERRORS" -gt 0 ]]; then
|
|
echo ""
|
|
echo "ERROR: $ERRORS item(s) changed without a manifest version bump"
|
|
exit 1
|
|
fi
|
|
|
|
echo "All changed items carry a version bump"
|
|
|
|
# ===========================================================================
|
|
# Build the item matrix with reuse detection
|
|
# ===========================================================================
|
|
setup-matrix:
|
|
name: Setup Build Matrix
|
|
needs: [configure, collect-metadata]
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
build-matrix: ${{ steps.set-matrix.outputs.build }}
|
|
steps:
|
|
- name: Build item matrix
|
|
id: set-matrix
|
|
env:
|
|
CURRENT: ${{ needs.collect-metadata.outputs.items-json }}
|
|
PREV: ${{ needs.collect-metadata.outputs.prev-json }}
|
|
ALCHEMIST_REF: ${{ needs.configure.outputs.alchemist_ref }}
|
|
PREV_ALCHEMIST_REF: ${{ needs.collect-metadata.outputs.prev-alchemist-ref }}
|
|
run: |
|
|
echo "Current: $(jq 'keys | length' <<< "$CURRENT") items"
|
|
echo "Previous: $(jq 'keys | length' <<< "$PREV") items"
|
|
echo ""
|
|
|
|
# ---------------------------------------------------------------
|
|
# A change to the pinned Alchemist invalidates every reuse.
|
|
#
|
|
# The tool is not covered by any item's fingerprint, so without
|
|
# this a release could carry some artifacts built by the new
|
|
# Alchemist and some by the old one. Rebuilding everything keeps a
|
|
# release internally consistent, at the cost of one full rebuild
|
|
# per Alchemist bump.
|
|
#
|
|
# This does NOT force version bumps: the bump check compares
|
|
# fingerprint and upstream version, neither of which the tool ref
|
|
# touches. A tool change rebuilds without demanding a bump on
|
|
# every item in the RetroVERSE.
|
|
# ---------------------------------------------------------------
|
|
ALCHEMIST_CHANGED=false
|
|
if [[ -n "$PREV_ALCHEMIST_REF" && "$PREV_ALCHEMIST_REF" != "$ALCHEMIST_REF" ]]; then
|
|
ALCHEMIST_CHANGED=true
|
|
echo "Alchemist ref changed: $PREV_ALCHEMIST_REF -> $ALCHEMIST_REF"
|
|
echo "Reuse disabled for this release: rebuilding every item"
|
|
echo ""
|
|
elif [[ -z "$PREV_ALCHEMIST_REF" && "$PREV" != "{}" ]]; then
|
|
ALCHEMIST_CHANGED=true
|
|
echo "Previous release recorded no Alchemist ref: rebuilding every item"
|
|
echo ""
|
|
fi
|
|
|
|
entries=$(jq -c \
|
|
--argjson prev "$PREV" \
|
|
--argjson alchemist_changed "$ALCHEMIST_CHANGED" '
|
|
to_entries | map(
|
|
.value as $c |
|
|
($prev[.key] // {}) as $p |
|
|
{
|
|
item: $c.id,
|
|
dir: $c.dir,
|
|
class: $c.class,
|
|
version: $c.version,
|
|
can_reuse: (
|
|
($alchemist_changed | not) and
|
|
($p.version // "") == $c.version and
|
|
($p.fingerprint // "") == $c.fingerprint and
|
|
($p.upstream_version // "") == ($c.upstream_version // "")
|
|
)
|
|
}
|
|
)
|
|
' <<< "$CURRENT")
|
|
|
|
count=$(jq 'length' <<< "$entries")
|
|
if [[ "$count" -eq 0 ]]; then
|
|
echo "ERROR: No items found"
|
|
exit 1
|
|
fi
|
|
|
|
reuse_count=$(jq '[.[] | select(.can_reuse)] | length' <<< "$entries")
|
|
build_count=$(jq '[.[] | select(.can_reuse | not)] | length' <<< "$entries")
|
|
|
|
echo "Build matrix:"
|
|
jq -r '.[] | " \(.item) \(.version) [\(.class)] - \(if .can_reuse then "REUSE" else "BUILD" end)"' <<< "$entries"
|
|
|
|
echo ""
|
|
echo "=========================================="
|
|
echo "Matrix Summary"
|
|
echo " Reuse: $reuse_count"
|
|
echo " Build: $build_count"
|
|
echo " Total: $count"
|
|
echo "=========================================="
|
|
|
|
echo "build=$entries" >> $GITHUB_OUTPUT
|
|
|
|
# ===========================================================================
|
|
# Build each item (or reuse from previous release)
|
|
# ===========================================================================
|
|
# fail-fast is false: the RetroVERSE may hold hundreds of items and one malformed
|
|
# manifest should surface every other failure in the same run.
|
|
# ===========================================================================
|
|
build-items:
|
|
name: Build ${{ matrix.item }}
|
|
needs: [configure, collect-metadata, setup-matrix]
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
strategy:
|
|
matrix:
|
|
include: ${{ fromJSON(needs.setup-matrix.outputs.build-matrix) }}
|
|
fail-fast: false
|
|
max-parallel: 20
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v6
|
|
with:
|
|
ref: ${{ needs.configure.outputs.branch }}
|
|
fetch-depth: 0
|
|
|
|
- name: Checkout Alchemist
|
|
if: matrix.can_reuse == false
|
|
uses: actions/checkout@v6
|
|
with:
|
|
repository: ${{ env.ALCHEMIST_REPO }}
|
|
ref: ${{ needs.configure.outputs.alchemist_ref }}
|
|
path: ${{ env.ALCHEMIST_PATH }}
|
|
sparse-checkout: automation-tools/alchemist
|
|
sparse-checkout-cone-mode: true
|
|
|
|
- name: Overlay RetroVERSE version policy
|
|
if: matrix.can_reuse == false
|
|
run: |
|
|
if [[ ! -x "$ALCHEMIST_DIR/alchemist.sh" ]]; then
|
|
echo "ERROR: alchemist.sh not found or not executable at $ALCHEMIST_DIR"
|
|
exit 1
|
|
fi
|
|
|
|
rm -f "$ALCHEMIST_DIR/version_policy.sh" "$ALCHEMIST_DIR/version_pins.sh"
|
|
cp alchemist/version_policy.sh "$ALCHEMIST_DIR/version_policy.sh"
|
|
|
|
if [[ -f alchemist/version_pins.sh ]]; then
|
|
cp alchemist/version_pins.sh "$ALCHEMIST_DIR/version_pins.sh"
|
|
fi
|
|
|
|
echo "Alchemist ready at ${{ needs.configure.outputs.alchemist_ref }}"
|
|
|
|
- name: Install build dependencies
|
|
if: matrix.can_reuse == false
|
|
run: |
|
|
echo "Installing build dependencies..."
|
|
|
|
unset pkg_mgr
|
|
for candidate in apt pacman dnf; do
|
|
command -v "$candidate" &> /dev/null && pkg_mgr="$candidate" && break
|
|
done
|
|
|
|
case "${pkg_mgr:-}" in
|
|
apt)
|
|
sudo add-apt-repository -y ppa:flatpak/stable
|
|
sudo apt update
|
|
sudo apt install -y flatpak flatpak-builder p7zip-full xmlstarlet bzip2 curl jq unzip
|
|
;;
|
|
pacman)
|
|
sudo pacman -Sy --noconfirm flatpak flatpak-builder p7zip xmlstarlet bzip2 curl jq unzip
|
|
;;
|
|
dnf)
|
|
sudo dnf install -y flatpak flatpak-builder p7zip p7zip-plugins xmlstarlet bzip2 curl jq unzip
|
|
;;
|
|
*)
|
|
echo "ERROR: No supported package manager found (apt, pacman, dnf)"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
flatpak remote-add --user --if-not-exists flathub https://flathub.org/repo/flathub.flatpakrepo
|
|
flatpak remote-add --user --if-not-exists flathub-beta https://flathub.org/beta-repo/flathub-beta.flatpakrepo
|
|
|
|
echo "Build dependencies installed"
|
|
|
|
- name: Download previous artifact
|
|
if: matrix.can_reuse == true
|
|
run: |
|
|
ITEM="${{ matrix.item }}"
|
|
BRANCH_NAME="${{ needs.configure.outputs.branch }}"
|
|
|
|
echo "Reusing $ITEM from previous release"
|
|
|
|
PREV_RELEASE=$(gh release list --limit 100 --json tagName \
|
|
-q "[.[] | select(.tagName | startswith(\"${BRANCH_NAME}-\"))] | .[0].tagName")
|
|
|
|
if [[ -z "$PREV_RELEASE" || "$PREV_RELEASE" == "null" ]]; then
|
|
echo "ERROR: No previous release found but can_reuse is true"
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "${ITEM}/artifacts"
|
|
|
|
gh release download "$PREV_RELEASE" \
|
|
-p "${ITEM}.tar.gz" \
|
|
-p "${ITEM}.tar.gz.sha" \
|
|
-D "${ITEM}/artifacts/"
|
|
|
|
echo "Reused assets downloaded"
|
|
|
|
# ---------------------------------------------------------------------
|
|
# Component items run through the Alchemist unchanged, so their archive
|
|
# is byte-for-byte the shape the external component installer expects.
|
|
# ---------------------------------------------------------------------
|
|
- name: Build component item
|
|
if: matrix.can_reuse == false && matrix.class == 'component'
|
|
run: |
|
|
ITEM="${{ matrix.item }}"
|
|
|
|
echo "Building component item $ITEM..."
|
|
|
|
"$ALCHEMIST_DIR/alchemist.sh" -f "${{ matrix.dir }}/component_recipe.json"
|
|
|
|
echo "Build complete"
|
|
|
|
# ---------------------------------------------------------------------
|
|
# Data items stop before compression so each deployed file can be hashed
|
|
# while still uncompressed. The workflow then compresses exactly as the
|
|
# Alchemist would, so collection and verification stay class-agnostic.
|
|
#
|
|
# The output directory MUST exist before invoking with -o: the Alchemist
|
|
# runs realpath on it, and export masks the failure from set -e, leaving
|
|
# WORKDIR empty and the artifact root at the filesystem root.
|
|
# ---------------------------------------------------------------------
|
|
- name: Build data item
|
|
if: matrix.can_reuse == false && matrix.class == 'data'
|
|
run: |
|
|
ITEM="${{ matrix.item }}"
|
|
|
|
echo "Building data item $ITEM..."
|
|
|
|
ITEM_WORKDIR="$GITHUB_WORKSPACE/.build/$ITEM"
|
|
mkdir -p "$ITEM_WORKDIR"
|
|
|
|
"$ALCHEMIST_DIR/alchemist.sh" \
|
|
-f "${{ matrix.dir }}/component_recipe.json" \
|
|
--dry-run \
|
|
-o "$ITEM_WORKDIR"
|
|
|
|
ARTIFACT_ROOT="$ITEM_WORKDIR/$ITEM-artifact"
|
|
|
|
if [[ ! -d "$ARTIFACT_ROOT" ]]; then
|
|
echo "ERROR: Artifact root not found: $ARTIFACT_ROOT"
|
|
exit 1
|
|
fi
|
|
|
|
# Hash keys are relative to the artifact root so they line up with
|
|
# files[].source in the manifest.
|
|
(
|
|
cd "$ARTIFACT_ROOT"
|
|
find . -type f -printf '%P\n' | sort | while IFS= read -r f; do
|
|
printf '%s %s\n' "$(sha256sum "$f" | awk '{print $1}')" "$f"
|
|
done
|
|
) > "$ITEM_WORKDIR/file_hashes.txt"
|
|
|
|
echo "Hashed $(wc -l < "$ITEM_WORKDIR/file_hashes.txt") file(s)"
|
|
|
|
OUT_DIR="$ITEM/artifacts"
|
|
mkdir -p "$OUT_DIR"
|
|
|
|
tar -czf "$OUT_DIR/$ITEM.tar.gz" -C "$ARTIFACT_ROOT" .
|
|
|
|
# Matches the Alchemist's own output format (bare filename)
|
|
( cd "$OUT_DIR" && sha256sum "$ITEM.tar.gz" > "$ITEM.tar.gz.sha" )
|
|
|
|
cp "$ITEM_WORKDIR/file_hashes.txt" "$OUT_DIR/$ITEM.file_hashes.txt"
|
|
|
|
echo "Build complete"
|
|
|
|
# ---------------------------------------------------------------------
|
|
# Validation runs on every item, reused or not. The RetroVERSE manifest sits
|
|
# outside the fingerprint, so an edit to files[].source alone would
|
|
# otherwise reuse the old artifact and never revalidate the paths.
|
|
# ---------------------------------------------------------------------
|
|
- name: Validate item against artifact
|
|
run: |
|
|
ITEM="${{ matrix.item }}"
|
|
MANIFEST="${{ matrix.dir }}/retroverse_manifest.json"
|
|
ARCHIVE="$ITEM/artifacts/$ITEM.tar.gz"
|
|
|
|
if [[ ! -f "$ARCHIVE" ]]; then
|
|
echo "ERROR: Archive not found: $ARCHIVE"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "${{ matrix.class }}" != "data" ]]; then
|
|
echo "Component item: no files[] to cross-check"
|
|
exit 0
|
|
fi
|
|
|
|
EXTRACT_DIR=$(mktemp -d)
|
|
tar -xzf "$ARCHIVE" -C "$EXTRACT_DIR"
|
|
|
|
ERRORS=0
|
|
|
|
# Every declared source must exist in the artifact
|
|
while IFS= read -r src; do
|
|
[[ -n "$src" ]] || continue
|
|
|
|
if [[ "$src" == */ ]]; then
|
|
if [[ ! -d "$EXTRACT_DIR/${src%/}" ]]; then
|
|
echo " FAILED: files[].source '$src' is not a directory in the artifact"
|
|
ERRORS=$((ERRORS + 1))
|
|
fi
|
|
elif [[ ! -e "$EXTRACT_DIR/$src" ]]; then
|
|
echo " FAILED: files[].source '$src' not found in the artifact"
|
|
ERRORS=$((ERRORS + 1))
|
|
fi
|
|
done < <(jq -r '.files[] | .source' "$MANIFEST")
|
|
|
|
# Nothing in the artifact may be left undeclared
|
|
DECLARED=$(jq -r '.files[] | .source' "$MANIFEST")
|
|
|
|
while IFS= read -r actual; do
|
|
covered=false
|
|
while IFS= read -r src; do
|
|
if [[ "$src" == */ ]]; then
|
|
[[ "$actual" == "${src}"* ]] && covered=true && break
|
|
else
|
|
[[ "$actual" == "$src" ]] && covered=true && break
|
|
fi
|
|
done <<< "$DECLARED"
|
|
|
|
if [[ "$covered" == false ]]; then
|
|
echo " FAILED: '$actual' is in the artifact but not declared in files[]"
|
|
ERRORS=$((ERRORS + 1))
|
|
fi
|
|
done < <(cd "$EXTRACT_DIR" && find . -type f -printf '%P\n')
|
|
|
|
rm -rf "$EXTRACT_DIR"
|
|
|
|
if [[ "$ERRORS" -gt 0 ]]; then
|
|
echo "ERROR: $ERRORS manifest/artifact mismatch(es) for $ITEM"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Manifest and artifact agree"
|
|
|
|
- name: Verify artifacts
|
|
run: |
|
|
ITEM="${{ matrix.item }}"
|
|
|
|
if [[ ! -f "$ITEM/artifacts/$ITEM.tar.gz" ]]; then
|
|
echo "ERROR: Archive not found: $ITEM/artifacts/$ITEM.tar.gz"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ! -f "$ITEM/artifacts/$ITEM.tar.gz.sha" ]]; then
|
|
echo "ERROR: SHA file not found: $ITEM/artifacts/$ITEM.tar.gz.sha"
|
|
exit 1
|
|
fi
|
|
|
|
if ! grep -q "$ITEM.tar.gz" "$ITEM/artifacts/$ITEM.tar.gz.sha"; then
|
|
echo "ERROR: SHA file does not reference expected archive filename"
|
|
exit 1
|
|
fi
|
|
|
|
ARCHIVE_SIZE=$(du -h "$ITEM/artifacts/$ITEM.tar.gz" | cut -f1)
|
|
echo "Artifacts verified ($ARCHIVE_SIZE)"
|
|
|
|
# ---------------------------------------------------------------------
|
|
# Artwork is staged as a single item-named file rather than an artwork/
|
|
# directory, so collection in create-release cannot collide no matter how
|
|
# download-artifact lays the tree out. It runs for reused items too:
|
|
# artwork sits outside the fingerprint and is republished every release.
|
|
# ---------------------------------------------------------------------
|
|
- name: Stage artwork
|
|
run: |
|
|
ITEM="${{ matrix.item }}"
|
|
MANIFEST="${{ matrix.dir }}/retroverse_manifest.json"
|
|
|
|
ART_REL=$(jq -r '.artwork // empty' "$MANIFEST")
|
|
|
|
if [[ -z "$ART_REL" ]]; then
|
|
echo "No artwork declared for $ITEM"
|
|
exit 0
|
|
fi
|
|
|
|
ART_SRC="${{ matrix.dir }}/$ART_REL"
|
|
|
|
if [[ ! -f "$ART_SRC" ]]; then
|
|
echo "ERROR: artwork declared as '$ART_REL' but not found at $ART_SRC"
|
|
exit 1
|
|
fi
|
|
|
|
EXT="${ART_SRC##*.}"
|
|
if [[ "$EXT" == "$ART_SRC" ]]; then
|
|
echo "ERROR: artwork '$ART_REL' has no file extension"
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "$ITEM/artifacts"
|
|
cp "$ART_SRC" "$ITEM/artifacts/$ITEM.artwork.$EXT"
|
|
|
|
echo "Artwork staged as $ITEM.artwork.$EXT"
|
|
|
|
- name: Upload artifacts
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: ${{ matrix.item }}-artifacts
|
|
path: ${{ matrix.item }}/artifacts/
|
|
retention-days: 1
|
|
if-no-files-found: error
|
|
|
|
# ===========================================================================
|
|
# Create release
|
|
# ===========================================================================
|
|
create-release:
|
|
name: Create Release
|
|
needs: [configure, collect-metadata, build-items]
|
|
if: needs.configure.outputs.is_dry_run == 'false'
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
outputs:
|
|
release_tag: ${{ steps.tag.outputs.tag }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v6
|
|
with:
|
|
ref: ${{ needs.configure.outputs.branch }}
|
|
|
|
- name: Generate release tag
|
|
id: tag
|
|
run: |
|
|
BRANCH_NAME="${{ needs.configure.outputs.branch }}"
|
|
BRANCH_SAFE=$(echo "$BRANCH_NAME" | sed 's/\//-/g')
|
|
TIMESTAMP=$(date -u +"%Y%m%d-%H%M%S")
|
|
|
|
echo "tag=${BRANCH_SAFE}-${TIMESTAMP}" >> $GITHUB_OUTPUT
|
|
echo "timestamp=$(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> $GITHUB_OUTPUT
|
|
echo "branch=$BRANCH_NAME" >> $GITHUB_OUTPUT
|
|
|
|
echo "Release tag: ${BRANCH_SAFE}-${TIMESTAMP}"
|
|
|
|
# ---------------------------------------------------------------------
|
|
# main publishes as latest, which the client's stable channel follows.
|
|
# cooker publishes as prerelease, which the beta channel follows.
|
|
# ---------------------------------------------------------------------
|
|
- name: Determine release type
|
|
id: release-type
|
|
run: |
|
|
case "${{ steps.tag.outputs.branch }}" in
|
|
main)
|
|
echo "is_latest=true" >> $GITHUB_OUTPUT
|
|
echo "is_prerelease=false" >> $GITHUB_OUTPUT
|
|
echo "Release type: Latest (stable channel)"
|
|
;;
|
|
*)
|
|
echo "is_latest=false" >> $GITHUB_OUTPUT
|
|
echo "is_prerelease=true" >> $GITHUB_OUTPUT
|
|
echo "Release type: Pre-release (beta channel)"
|
|
;;
|
|
esac
|
|
|
|
- name: Download all item artifacts
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
path: downloaded-artifacts/
|
|
pattern: '*-artifacts'
|
|
|
|
- name: Collect and verify artifacts
|
|
env:
|
|
CURRENT: ${{ needs.collect-metadata.outputs.items-json }}
|
|
run: |
|
|
# nullglob: without it, an unmatched glob passes through literally and
|
|
# the checks below report a missing SHA for a file called "*.tar.gz"
|
|
# instead of saying nothing was collected.
|
|
shopt -s nullglob
|
|
|
|
mkdir -p release-assets artwork-staging
|
|
|
|
echo "Downloaded artifact tree:"
|
|
find downloaded-artifacts/ -maxdepth 3 | sort | sed 's/^/ /'
|
|
echo ""
|
|
|
|
# -------------------------------------------------------------------
|
|
# Collection is layout-agnostic on purpose.
|
|
#
|
|
# download-artifact may or may not place each artifact in its own
|
|
# subdirectory depending on version and pattern handling, so nothing
|
|
# here depends on the tree shape. Every file an item contributes is
|
|
# already named after that item, so a flattened download and a nested
|
|
# one both collect correctly.
|
|
# -------------------------------------------------------------------
|
|
while IFS= read -r file; do
|
|
cp "$file" release-assets/
|
|
done < <(find downloaded-artifacts/ -type f \
|
|
\( -name '*.tar.gz' -o -name '*.tar.gz.sha' -o -name '*.file_hashes.txt' \))
|
|
|
|
while IFS= read -r art; do
|
|
cp "$art" artwork-staging/
|
|
done < <(find downloaded-artifacts/ -type f -name '*.artwork.*')
|
|
|
|
# Item archives only. The artwork bundle is also a .tar.gz and is
|
|
# built later in this job; excluding it by name keeps the inventory
|
|
# check correct if these steps are ever reordered.
|
|
ARCHIVES=()
|
|
for candidate in release-assets/*.tar.gz; do
|
|
[[ "$(basename "$candidate")" == "artwork.tar.gz" ]] && continue
|
|
ARCHIVES+=("$candidate")
|
|
done
|
|
|
|
if [[ "${#ARCHIVES[@]}" -eq 0 ]]; then
|
|
echo "ERROR: no artifacts were collected from downloaded-artifacts/"
|
|
echo "The download step produced nothing matching *.tar.gz - see the tree above."
|
|
exit 1
|
|
fi
|
|
|
|
MISSING=0
|
|
for archive in "${ARCHIVES[@]}"; do
|
|
if [[ ! -f "${archive}.sha" ]]; then
|
|
echo "ERROR: Missing SHA file for $(basename "$archive")"
|
|
MISSING=$((MISSING + 1))
|
|
fi
|
|
done
|
|
|
|
if [[ "$MISSING" -gt 0 ]]; then
|
|
echo "ERROR: $MISSING artifact(s) missing SHA files"
|
|
exit 1
|
|
fi
|
|
|
|
# -------------------------------------------------------------------
|
|
# Inventory completeness.
|
|
#
|
|
# A failed matrix leg already fails build-items and skips this job, so
|
|
# this is not guarding against item build failures. It guards against
|
|
# an item silently never reaching release-assets. A download pattern
|
|
# that misses one, a collection bug, a matrix entry that produced no
|
|
# leg. None of those fail anything on their own, and the result would
|
|
# be a smaller inventory that every client reads as those items having
|
|
# been removed from the RetroVERSE.
|
|
# -------------------------------------------------------------------
|
|
jq -r 'keys[]' <<< "$CURRENT" | sort > /tmp/expected_items
|
|
printf '%s\n' "${ARCHIVES[@]}" \
|
|
| xargs -n1 basename \
|
|
| sed 's/\.tar\.gz$//' \
|
|
| sort > /tmp/collected_items
|
|
|
|
INVENTORY_ERRORS=0
|
|
|
|
if [[ -s /tmp/expected_items ]]; then
|
|
while IFS= read -r item; do
|
|
[[ -n "$item" ]] || continue
|
|
echo "ERROR: '$item' is in the inventory but has no artifact"
|
|
INVENTORY_ERRORS=$((INVENTORY_ERRORS + 1))
|
|
done < <(comm -23 /tmp/expected_items /tmp/collected_items)
|
|
fi
|
|
|
|
while IFS= read -r item; do
|
|
[[ -n "$item" ]] || continue
|
|
echo "ERROR: artifact '$item' has no matching inventory entry"
|
|
INVENTORY_ERRORS=$((INVENTORY_ERRORS + 1))
|
|
done < <(comm -13 /tmp/expected_items /tmp/collected_items)
|
|
|
|
if [[ "$INVENTORY_ERRORS" -gt 0 ]]; then
|
|
echo ""
|
|
echo "ERROR: inventory is incomplete: refusing to publish a partial set"
|
|
echo " Expected: $(wc -l < /tmp/expected_items) items"
|
|
echo " Collected: $(wc -l < /tmp/collected_items) items"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Inventory complete: $(wc -l < /tmp/collected_items) items"
|
|
|
|
# ---------------------------------------------------------------------
|
|
# The bundle is a flat set of <item_id>.artwork.<ext> files, so the client
|
|
# finds an item's thumbnail by name without walking a directory tree.
|
|
# ---------------------------------------------------------------------
|
|
- name: Build artwork bundle
|
|
run: |
|
|
if [[ -d artwork-staging && -n "$(ls -A artwork-staging)" ]]; then
|
|
tar -czf release-assets/artwork.tar.gz -C artwork-staging .
|
|
( cd release-assets && sha256sum artwork.tar.gz > artwork.tar.gz.sha )
|
|
echo "Artwork bundle built: $(ls -1 artwork-staging | wc -l) image(s)"
|
|
else
|
|
echo "No artwork to bundle"
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------
|
|
# Aggregation. Authored fields come from each retroverse_manifest.json;
|
|
# sha256, payload_size, url and file_hashes are derived here.
|
|
#
|
|
# file_hashes for a REUSED item are carried forward from the previous
|
|
# release. A reused item is never unpacked and rehashed. Omitting this
|
|
# would not fail the build; it would silently ship empty hash maps for
|
|
# the most stable items in the RetroVERSE.
|
|
# ---------------------------------------------------------------------
|
|
- name: Generate retroverse.json
|
|
env:
|
|
CURRENT: ${{ needs.collect-metadata.outputs.items-json }}
|
|
PREV: ${{ needs.collect-metadata.outputs.prev-json }}
|
|
run: |
|
|
RELEASE_TAG="${{ steps.tag.outputs.tag }}"
|
|
BUILD_TIMESTAMP="${{ steps.tag.outputs.timestamp }}"
|
|
BASE_URL="https://github.com/${{ github.repository }}/releases/download/${RELEASE_TAG}"
|
|
|
|
echo "Generating retroverse.json..."
|
|
|
|
entries='[]'
|
|
|
|
while IFS= read -r item; do
|
|
manifest_path=$(jq -r --arg i "$item" '.[$i].dir' <<< "$CURRENT")/retroverse_manifest.json
|
|
manifest=$(jq -c . "$manifest_path")
|
|
|
|
sha=$(awk '{print $1}' "release-assets/${item}.tar.gz.sha")
|
|
size=$(stat -c%s "release-assets/${item}.tar.gz")
|
|
url="${BASE_URL}/${item}.tar.gz"
|
|
|
|
cur_fp=$(jq -r --arg i "$item" '.[$i].fingerprint' <<< "$CURRENT")
|
|
cur_up=$(jq -r --arg i "$item" '.[$i].upstream_version // ""' <<< "$CURRENT")
|
|
cur_ver=$(jq -r --arg i "$item" '.[$i].version' <<< "$CURRENT")
|
|
|
|
prev_fp=$(jq -r --arg i "$item" '.[$i].fingerprint // empty' <<< "$PREV")
|
|
prev_ver=$(jq -r --arg i "$item" '.[$i].version // empty' <<< "$PREV")
|
|
|
|
if [[ -n "$prev_fp" && "$cur_fp" == "$prev_fp" && "$cur_ver" == "$prev_ver" ]]; then
|
|
status="reused"
|
|
built_on=$(jq -r --arg i "$item" '.[$i].built_on // empty' <<< "$PREV")
|
|
file_hashes=$(jq -c --arg i "$item" '.[$i].file_hashes // {}' <<< "$PREV")
|
|
else
|
|
status="new"
|
|
built_on="$BUILD_TIMESTAMP"
|
|
|
|
if [[ -f "release-assets/${item}.file_hashes.txt" ]]; then
|
|
file_hashes=$(jq -R -s '
|
|
split("\n")
|
|
| map(select(length > 0)
|
|
| capture("^(?<h>[0-9a-f]{64}) (?<f>.*)$")
|
|
| {(.f): .h})
|
|
| add // {}
|
|
' < "release-assets/${item}.file_hashes.txt")
|
|
else
|
|
file_hashes='{}'
|
|
fi
|
|
fi
|
|
|
|
# Data items must never end up with an empty hash map
|
|
class=$(jq -r '.class' <<< "$manifest")
|
|
if [[ "$class" == "data" && "$file_hashes" == "{}" ]]; then
|
|
echo "ERROR: $item is a data item with no file_hashes"
|
|
exit 1
|
|
fi
|
|
|
|
# The authored 'artwork' value is a repo path the client cannot see.
|
|
# Replace it with the file's name inside artwork.tar.gz, or null
|
|
# when the item ships none.
|
|
artwork_file=""
|
|
for candidate in artwork-staging/"$item".artwork.*; do
|
|
[[ -f "$candidate" ]] || continue
|
|
artwork_file=$(basename "$candidate")
|
|
break
|
|
done
|
|
|
|
entry=$(jq -c \
|
|
--arg sha "$sha" \
|
|
--argjson size "$size" \
|
|
--arg url "$url" \
|
|
--arg upstream "$cur_up" \
|
|
--arg fingerprint "$cur_fp" \
|
|
--arg built_on "$built_on" \
|
|
--arg status "$status" \
|
|
--arg source_id "$SOURCE_ID" \
|
|
--arg artwork "$artwork_file" \
|
|
--argjson file_hashes "$file_hashes" \
|
|
'. + {
|
|
source_id: $source_id,
|
|
sha256: $sha,
|
|
payload_size: $size,
|
|
url: $url,
|
|
upstream_version: $upstream,
|
|
fingerprint: $fingerprint,
|
|
built_on: $built_on,
|
|
status: $status,
|
|
artwork: (if $artwork == "" then null else $artwork end),
|
|
file_hashes: $file_hashes
|
|
}' <<< "$manifest")
|
|
|
|
entries=$(jq -c --argjson e "$entry" '. += [$e]' <<< "$entries")
|
|
done < <(jq -r 'keys[]' <<< "$CURRENT")
|
|
|
|
# alchemist_ref is recorded so the next release can tell whether the
|
|
# tool changed. Without it, reuse cannot be invalidated on a tool bump.
|
|
jq -n \
|
|
--arg schema "1" \
|
|
--arg source_id "$SOURCE_ID" \
|
|
--arg tag "$RELEASE_TAG" \
|
|
--arg built_on "$BUILD_TIMESTAMP" \
|
|
--arg alchemist_ref "${{ needs.configure.outputs.alchemist_ref }}" \
|
|
--argjson items "$entries" \
|
|
'{
|
|
retroverse_schema_version: ($schema | tonumber),
|
|
source_id: $source_id,
|
|
release_tag: $tag,
|
|
built_on: $built_on,
|
|
alchemist_ref: $alchemist_ref,
|
|
items: $items
|
|
}' > release-assets/retroverse.json
|
|
|
|
# Per-item hash files are build intermediates, not release assets
|
|
rm -f release-assets/*.file_hashes.txt
|
|
|
|
echo "retroverse.json generated: $(jq '.items | length' release-assets/retroverse.json) items"
|
|
|
|
- name: Generate release notes
|
|
run: |
|
|
RELEASE_TAG="${{ steps.tag.outputs.tag }}"
|
|
METADATA=$(jq '.items' release-assets/retroverse.json)
|
|
|
|
NEW_COUNT=$(jq '[.[] | select(.status == "new")] | length' <<< "$METADATA")
|
|
REUSED_COUNT=$(jq '[.[] | select(.status == "reused")] | length' <<< "$METADATA")
|
|
TOTAL_COUNT=$(jq 'length' <<< "$METADATA")
|
|
|
|
cat > release_notes.md <<EOF
|
|
## RetroVERSE ${RELEASE_TAG}
|
|
|
|
**Branch:** \`${{ needs.configure.outputs.branch }}\`
|
|
**Commit:** \`${{ github.sha }}\`
|
|
**Built On:** ${{ steps.tag.outputs.timestamp }}
|
|
**Items:** ${TOTAL_COUNT} total (${NEW_COUNT} new, ${REUSED_COUNT} reused)
|
|
|
|
---
|
|
|
|
## Inventory
|
|
|
|
| Item | Class | Version | Upstream | Status |
|
|
|------|-------|---------|----------|--------|
|
|
EOF
|
|
|
|
jq -r '
|
|
sort_by(.id) | .[] |
|
|
"| \(.name) | \(.class) | \(.version) | \(if .upstream_version == "" then "---" else .upstream_version end) | \(if .status == "new" then "New Build" else "Reused" end) |"
|
|
' <<< "$METADATA" >> release_notes.md
|
|
|
|
echo "Release notes generated"
|
|
|
|
- name: Create release
|
|
run: |
|
|
RELEASE_TAG="${{ steps.tag.outputs.tag }}"
|
|
|
|
FLAGS=""
|
|
[[ "${{ steps.release-type.outputs.is_prerelease }}" == "true" ]] && FLAGS="$FLAGS --prerelease"
|
|
[[ "${{ steps.release-type.outputs.is_latest }}" == "true" ]] && FLAGS="$FLAGS --latest"
|
|
|
|
gh release create "$RELEASE_TAG" \
|
|
--title "RetroVERSE ${RELEASE_TAG}" \
|
|
--notes-file release_notes.md \
|
|
$FLAGS \
|
|
release-assets/*
|
|
|
|
echo ""
|
|
echo "=========================================="
|
|
echo "Release created"
|
|
echo " URL: https://github.com/${{ github.repository }}/releases/tag/${RELEASE_TAG}"
|
|
echo "=========================================="
|
|
|
|
# ===========================================================================
|
|
# Dry-run summary
|
|
# ===========================================================================
|
|
dry-run-summary:
|
|
name: Dry-Run Summary
|
|
needs: [configure, collect-metadata, build-items]
|
|
if: needs.configure.outputs.is_dry_run == 'true'
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Show dry-run results
|
|
env:
|
|
CURRENT: ${{ needs.collect-metadata.outputs.items-json }}
|
|
run: |
|
|
echo "=========================================="
|
|
echo "DRY RUN - Build Complete (no release created)"
|
|
echo "=========================================="
|
|
echo ""
|
|
echo "Branch: ${{ needs.configure.outputs.branch }}"
|
|
echo ""
|
|
echo "Items built/validated:"
|
|
jq -r 'to_entries[] | " \(.key) [\(.value.class)]: \(.value.version)"' <<< "$CURRENT"
|
|
echo ""
|
|
echo "Dry-run complete: no release was created"
|
|
echo "=========================================="
|
|
|
|
# ===========================================================================
|
|
# Summary
|
|
# ===========================================================================
|
|
summary:
|
|
name: Build Summary
|
|
needs: [check-trigger, configure, setup-matrix, build-items, create-release]
|
|
if: always() && needs.check-trigger.outputs.should_build == 'true'
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Generate summary
|
|
env:
|
|
BUILD_MATRIX: ${{ needs.setup-matrix.outputs.build-matrix }}
|
|
run: |
|
|
IS_DRY_RUN="${{ needs.configure.outputs.is_dry_run }}"
|
|
RELEASE_RESULT="${{ needs.create-release.result }}"
|
|
|
|
# A tag only exists once create-release has actually published. It is
|
|
# empty on a dry run (the job is skipped) and on a failed release.
|
|
RELEASE_TAG="${{ needs.create-release.outputs.release_tag }}"
|
|
RELEASE_URL=""
|
|
|
|
if [[ -n "$RELEASE_TAG" && "$RELEASE_RESULT" == "success" ]]; then
|
|
RELEASE_URL="${{ github.server_url }}/${{ github.repository }}/releases/tag/${RELEASE_TAG}"
|
|
fi
|
|
|
|
echo "## RetroVERSE Build Summary" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
|
|
if [[ "$IS_DRY_RUN" == "true" ]]; then
|
|
echo "> **Dry run** - no release was created" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
elif [[ -n "$RELEASE_URL" ]]; then
|
|
echo "**[View release ${RELEASE_TAG}](${RELEASE_URL})**" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
fi
|
|
|
|
echo "### Parameters" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Parameter | Value |" >> $GITHUB_STEP_SUMMARY
|
|
echo "|-----------|-------|" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Branch | \`${{ needs.configure.outputs.branch }}\` |" >> $GITHUB_STEP_SUMMARY
|
|
|
|
if [[ -n "$RELEASE_URL" ]]; then
|
|
echo "| Release | [\`${RELEASE_TAG}\`](${RELEASE_URL}) |" >> $GITHUB_STEP_SUMMARY
|
|
elif [[ -n "$RELEASE_TAG" ]]; then
|
|
echo "| Release | \`${RELEASE_TAG}\` (publish failed) |" >> $GITHUB_STEP_SUMMARY
|
|
else
|
|
echo "| Release | (not created) |" >> $GITHUB_STEP_SUMMARY
|
|
fi
|
|
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
|
|
if [[ -n "$BUILD_MATRIX" && "$BUILD_MATRIX" != "null" ]]; then
|
|
TOTAL=$(jq 'length' <<< "$BUILD_MATRIX")
|
|
REUSE_COUNT=$(jq '[.[] | select(.can_reuse)] | length' <<< "$BUILD_MATRIX")
|
|
BUILD_COUNT=$(jq '[.[] | select(.can_reuse | not)] | length' <<< "$BUILD_MATRIX")
|
|
|
|
echo "### Items ($TOTAL total: $BUILD_COUNT built, $REUSE_COUNT reused)" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Item | Class | Version | Status |" >> $GITHUB_STEP_SUMMARY
|
|
echo "|------|-------|---------|--------|" >> $GITHUB_STEP_SUMMARY
|
|
|
|
jq -r '
|
|
sort_by(.item) | .[] |
|
|
"| \(.item) | \(.class) | \(.version) | \(if .can_reuse then "Reused" else "Built" end) |"
|
|
' <<< "$BUILD_MATRIX" >> $GITHUB_STEP_SUMMARY
|
|
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
fi
|
|
|
|
echo "### Results" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Stage | Result |" >> $GITHUB_STEP_SUMMARY
|
|
echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Build | ${{ needs.build-items.result }} |" >> $GITHUB_STEP_SUMMARY
|
|
|
|
if [[ "$IS_DRY_RUN" != "true" ]]; then
|
|
echo "| Release | ${{ needs.create-release.result }} |" >> $GITHUB_STEP_SUMMARY
|
|
fi
|