Xargon/RetroVERSE
1
0
Fork 0
mirror of https://github.com/RetroDECK/RetroVERSE.git synced 2026-10-16 17:16:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update workflow to pull components Alchemist rather than having another in RetroVERSE repo

Browse source
This commit is contained in:
icenine451 2026-09-22 14:58:12 -04:00
parent e8c9aedd2d
commit 512ed1821c
1 changed files with 204 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

222
.github/workflows/retroverse-build.yml vendored
View file

@ -9,7 +9,7 @@ on:
type: string
dry_run:
description: 'Dry run - build and validate but do not create release'
description: 'Dry run: build and validate but do not create release'
required: false
type: boolean
default: true
@ -25,6 +25,21 @@ permissions:
env:
SOURCE_ID: official
# ---------------------------------------------------------------------------
# The Alchemist is not vendored here. It is checked out from the components
# repo at a ref pinned in alchemist/alchemist_ref, so the store repo carries
# only its own version_policy.sh (and optional version_pins.sh), which are
# overlaid onto the checkout.
#
# The ref is pinned rather than floating on main: an unpinned tool could
# change build output with no commit to this repo, and reuse would not notice
# because the fingerprint covers item folders only. A release could then mix
# artifacts built by two different tool versions.
# ---------------------------------------------------------------------------
ALCHEMIST_REPO: RetroDECK/components
ALCHEMIST_PATH: .alchemist
ALCHEMIST_DIR: .alchemist/automation-tools/alchemist
jobs:
# ===========================================================================
# Check whether this event should trigger a build
@ -62,7 +77,7 @@ jobs:
# PR merge commits always trigger a build
if echo "$COMMITS" | grep -qiE "^Merge pull request #"; then
echo "PR merge detected - auto-building"
echo "PR merge detected: auto-building"
echo "should_build=true" >> $GITHUB_OUTPUT
echo "is_dry_run=false" >> $GITHUB_OUTPUT
exit 0
@ -82,7 +97,7 @@ jobs:
exit 0
fi
echo "No build trigger found in commit messages - skipping"
echo "No build trigger found in commit messages: skipping"
echo "should_build=false" >> $GITHUB_OUTPUT
echo "is_dry_run=false" >> $GITHUB_OUTPUT
@ -97,6 +112,7 @@ jobs:
outputs:
branch: ${{ steps.config.outputs.branch }}
is_dry_run: ${{ needs.check-trigger.outputs.is_dry_run }}
alchemist_ref: ${{ steps.alchemist-ref.outputs.ref }}
steps:
- name: Determine build parameters
id: config
@ -123,6 +139,48 @@ jobs:
with:
ref: ${{ steps.config.outputs.branch }}
# -----------------------------------------------------------------------
# The pinned Alchemist ref. A tag or full commit SHA.
# -----------------------------------------------------------------------
- name: Read pinned Alchemist ref
id: alchemist-ref
run: |
REF_FILE="alchemist/alchemist_ref"
if [[ ! -f "$REF_FILE" ]]; then
echo "ERROR: $REF_FILE not found: the Alchemist ref must be pinned in this repo"
exit 1
fi
REF=$(grep -vE '^\s*(#|$)' "$REF_FILE" | head -n1 | tr -d '[:space:]')
if [[ -z "$REF" ]]; then
echo "ERROR: $REF_FILE is empty"
exit 1
fi
case "$REF" in
main|cooker|epicure|HEAD)
echo "ERROR: '$REF' is a branch, not a pin. Use a tag or a full commit SHA."
exit 1
;;
esac
echo "ref=$REF" >> $GITHUB_OUTPUT
echo "Alchemist pinned to: $REF"
- name: Validate version policy is present
run: |
# load_versions.sh checks for version_policy.sh at source time, before any recipe is read, and exits if it is missing. This repo therefore carries its own even if it holds only data items.
# A comment-only file is enough.
if [[ ! -f "alchemist/version_policy.sh" ]]; then
echo "ERROR: alchemist/version_policy.sh not found"
echo "The Alchemist exits at source time without it, even when no item needs a policy."
exit 1
fi
echo "Version policy present"
- name: Validate Bash scripts
run: |
echo "Validating Bash scripts..."
@ -219,7 +277,7 @@ jobs:
;;
component)
if [[ "$file_count" -ne 0 ]]; then
fail "class 'component' must not declare files[] - deployment belongs to the external component installer"
fail "class 'component' must not declare files[]: deployment belongs to the external component installer"
fi
# Store version is read from retroverse_manifest.json for both classes so the aggregator needs no class-specific logic.
@ -327,6 +385,7 @@ jobs:
outputs:
items-json: ${{ steps.collect.outputs.items }}
prev-json: ${{ steps.get-prev.outputs.prev }}
prev-alchemist-ref: ${{ steps.get-prev.outputs.prev_alchemist_ref }}
steps:
- name: Checkout code
uses: actions/checkout@v6
@ -334,6 +393,41 @@ jobs:
ref: ${{ needs.configure.outputs.branch }}
fetch-depth: 0
# -----------------------------------------------------------------------
# Alchemist checkout. sparse-checkout keeps the rest of the components
# repo out of the workspace, and the nested .git is harmless: alchemist.sh
# resolves REPO_ROOT from its working directory, which stays the store
# repo root, so $REPO_ROOT/$COMPONENT_NAME still points at the item folder.
# -----------------------------------------------------------------------
- name: Checkout Alchemist
uses: actions/checkout@v6
with:
repository: ${{ env.ALCHEMIST_REPO }}
ref: ${{ needs.configure.outputs.alchemist_ref }}
path: ${{ env.ALCHEMIST_PATH }}
sparse-checkout: automation-tools/alchemist
sparse-checkout-cone-mode: true
- name: Overlay store version policy
run: |
# version_policy.sh and version_pins.sh are read from the Alchemist's
# own SCRIPT_DIR, so the components repo's copies arrive with the
# checkout and must be replaced by this repo's.
if [[ ! -x "$ALCHEMIST_DIR/alchemist.sh" ]]; then
echo "ERROR: alchemist.sh not found or not executable at $ALCHEMIST_DIR"
exit 1
fi
rm -f "$ALCHEMIST_DIR/version_policy.sh" "$ALCHEMIST_DIR/version_pins.sh"
cp alchemist/version_policy.sh "$ALCHEMIST_DIR/version_policy.sh"
if [[ -f alchemist/version_pins.sh ]]; then
cp alchemist/version_pins.sh "$ALCHEMIST_DIR/version_pins.sh"
echo "Version pins overlaid"
fi
echo "Alchemist ready at ${{ needs.configure.outputs.alchemist_ref }}"
- name: Get previous release metadata
id: get-prev
run: |
@ -347,6 +441,7 @@ jobs:
if [[ -z "$PREV_RELEASE" || "$PREV_RELEASE" == "null" ]]; then
echo "No previous release found for branch ${BRANCH_NAME}"
echo "prev={}" >> $GITHUB_OUTPUT
echo "prev_alchemist_ref=" >> $GITHUB_OUTPUT
exit 0
fi
@ -357,9 +452,15 @@ jobs:
PREV=$(jq -c '.items | map({(.id): .}) | add // {}' retroverse.json)
echo "prev=$PREV" >> $GITHUB_OUTPUT
echo "Previous inventory: $(jq 'keys | length' <<< "$PREV") items"
# Tool version the previous release was built with. A change to it invalidates every reuse (see setup-matrix)
PREV_ALCHEMIST=$(jq -r '.alchemist_ref // ""' retroverse.json)
echo "prev_alchemist_ref=$PREV_ALCHEMIST" >> $GITHUB_OUTPUT
echo "Previous Alchemist ref: ${PREV_ALCHEMIST:-<unrecorded>}"
else
echo "No retroverse.json found in previous release"
echo "prev={}" >> $GITHUB_OUTPUT
echo "prev_alchemist_ref=" >> $GITHUB_OUTPUT
fi
- name: Check for component items
@ -369,7 +470,7 @@ jobs:
echo "found=true" >> $GITHUB_OUTPUT
else
echo "found=false" >> $GITHUB_OUTPUT
echo "No component items - skipping build dependency install"
echo "No component items: skipping build dependency install"
fi
- name: Install build dependencies
@ -427,7 +528,9 @@ jobs:
# -----------------------------------------------------------------
# Fingerprint covers only what the Alchemist consumes. The store
# manifest and artwork are excluded.
# manifest and artwork are excluded: they are republished from
# source every release, so a description or thumbnail edit must not
# invalidate an otherwise identical artifact.
# -----------------------------------------------------------------
fingerprint=$(git ls-tree -r HEAD -- "$item_dir" \
| grep -v 'retroverse_manifest\.json$' \
@ -440,7 +543,7 @@ jobs:
# -----------------------------------------------------------------
upstream=""
if [[ "$class" == "component" ]]; then
upstream=$(automation-tools/alchemist/alchemist.sh \
upstream=$("$ALCHEMIST_DIR/alchemist.sh" \
-f "$item_dir/component_recipe.json" --resolve-versions)
if [[ -z "$upstream" ]]; then
@ -495,7 +598,7 @@ jobs:
PREV: ${{ steps.get-prev.outputs.prev }}
run: |
if [[ "$PREV" == "{}" ]]; then
echo "No previous release - skipping bump check"
echo "No previous release: skipping bump check"
exit 0
fi
@ -512,7 +615,7 @@ jobs:
prev_fp=$(jq -r --arg i "$item" '.[$i].fingerprint // empty' <<< "$PREV")
prev_up=$(jq -r --arg i "$item" '.[$i].upstream_version // ""' <<< "$PREV")
# New item - nothing to compare against
# New item: nothing to compare against
if [[ -z "$prev_version" ]]; then
continue
fi
@ -551,12 +654,42 @@ jobs:
env:
CURRENT: ${{ needs.collect-metadata.outputs.items-json }}
PREV: ${{ needs.collect-metadata.outputs.prev-json }}
ALCHEMIST_REF: ${{ needs.configure.outputs.alchemist_ref }}
PREV_ALCHEMIST_REF: ${{ needs.collect-metadata.outputs.prev-alchemist-ref }}
run: |
echo "Current: $(jq 'keys | length' <<< "$CURRENT") items"
echo "Previous: $(jq 'keys | length' <<< "$PREV") items"
echo ""
entries=$(jq -c --argjson prev "$PREV" '
# ---------------------------------------------------------------
# A change to the pinned Alchemist invalidates every reuse.
#
# The tool is not covered by any item's fingerprint, so without
# this a release could carry some artifacts built by the new
# Alchemist and some by the old one. Rebuilding everything keeps a
# release internally consistent, at the cost of one full rebuild
# per Alchemist bump.
#
# This does NOT force version bumps: the bump check compares
# fingerprint and upstream version, neither of which the tool ref
# touches. A tool change rebuilds without demanding a bump on
# every item in the store.
# ---------------------------------------------------------------
ALCHEMIST_CHANGED=false
if [[ -n "$PREV_ALCHEMIST_REF" && "$PREV_ALCHEMIST_REF" != "$ALCHEMIST_REF" ]]; then
ALCHEMIST_CHANGED=true
echo "Alchemist ref changed: $PREV_ALCHEMIST_REF -> $ALCHEMIST_REF"
echo "Reuse disabled for this release: rebuilding every item"
echo ""
elif [[ -z "$PREV_ALCHEMIST_REF" && "$PREV" != "{}" ]]; then
ALCHEMIST_CHANGED=true
echo "Previous release recorded no Alchemist ref: rebuilding every item"
echo ""
fi
entries=$(jq -c \
--argjson prev "$PREV" \
--argjson alchemist_changed "$ALCHEMIST_CHANGED" '
to_entries | map(
.value as $c |
($prev[.key] // {}) as $p |
@ -566,6 +699,7 @@ jobs:
class: $c.class,
version: $c.version,
can_reuse: (
($alchemist_changed | not) and
($p.version // "") == $c.version and
($p.fingerprint // "") == $c.fingerprint and
($p.upstream_version // "") == ($c.upstream_version // "")
@ -599,6 +733,9 @@ jobs:
# ===========================================================================
# Build each item (or reuse from previous release)
# ===========================================================================
# fail-fast is false: a store may hold hundreds of items and one malformed
# manifest should surface every other failure in the same run.
# ===========================================================================
build-items:
name: Build ${{ matrix.item }}
needs: [configure, collect-metadata, setup-matrix]
@ -617,6 +754,33 @@ jobs:
ref: ${{ needs.configure.outputs.branch }}
fetch-depth: 0
- name: Checkout Alchemist
if: matrix.can_reuse == false
uses: actions/checkout@v6
with:
repository: ${{ env.ALCHEMIST_REPO }}
ref: ${{ needs.configure.outputs.alchemist_ref }}
path: ${{ env.ALCHEMIST_PATH }}
sparse-checkout: automation-tools/alchemist
sparse-checkout-cone-mode: true
- name: Overlay store version policy
if: matrix.can_reuse == false
run: |
if [[ ! -x "$ALCHEMIST_DIR/alchemist.sh" ]]; then
echo "ERROR: alchemist.sh not found or not executable at $ALCHEMIST_DIR"
exit 1
fi
rm -f "$ALCHEMIST_DIR/version_policy.sh" "$ALCHEMIST_DIR/version_pins.sh"
cp alchemist/version_policy.sh "$ALCHEMIST_DIR/version_policy.sh"
if [[ -f alchemist/version_pins.sh ]]; then
cp alchemist/version_pins.sh "$ALCHEMIST_DIR/version_pins.sh"
fi
echo "Alchemist ready at ${{ needs.configure.outputs.alchemist_ref }}"
- name: Install build dependencies
if: matrix.can_reuse == false
run: |
@ -676,7 +840,8 @@ jobs:
echo "Reused assets downloaded"
# ---------------------------------------------------------------------
# Component items run through the Alchemist unchanged.
# Component items run through the Alchemist unchanged, so their archive
# is byte-for-byte the shape the external component installer expects.
# ---------------------------------------------------------------------
- name: Build component item
if: matrix.can_reuse == false && matrix.class == 'component'
@ -685,7 +850,7 @@ jobs:
echo "Building component item $ITEM..."
automation-tools/alchemist/alchemist.sh -f "${{ matrix.dir }}/component_recipe.json"
"$ALCHEMIST_DIR/alchemist.sh" -f "${{ matrix.dir }}/component_recipe.json"
echo "Build complete"
@ -693,6 +858,10 @@ jobs:
# Data items stop before compression so each deployed file can be hashed
# while still uncompressed. The workflow then compresses exactly as the
# Alchemist would, so collection and verification stay class-agnostic.
#
# The output directory MUST exist before invoking with -o: the Alchemist
# runs realpath on it, and export masks the failure from set -e, leaving
# WORKDIR empty and the artifact root at the filesystem root.
# ---------------------------------------------------------------------
- name: Build data item
if: matrix.can_reuse == false && matrix.class == 'data'
@ -704,7 +873,7 @@ jobs:
ITEM_WORKDIR="$GITHUB_WORKSPACE/.build/$ITEM"
mkdir -p "$ITEM_WORKDIR"
automation-tools/alchemist/alchemist.sh \
"$ALCHEMIST_DIR/alchemist.sh" \
-f "${{ matrix.dir }}/component_recipe.json" \
--dry-run \
-o "$ITEM_WORKDIR"
@ -756,7 +925,7 @@ jobs:
fi
if [[ "${{ matrix.class }}" != "data" ]]; then
echo "Component item - no files[] to cross-check"
echo "Component item: no files[] to cross-check"
exit 0
fi
@ -948,7 +1117,15 @@ jobs:
fi
# -------------------------------------------------------------------
# Check for inventory completeness.
# Inventory completeness.
#
# A failed matrix leg already fails build-items and skips this job, so
# this is not guarding against item build failures. It guards against
# an item silently never reaching release-assets. A download pattern
# that misses one, a collection bug, a matrix entry that produced no
# leg. None of those fail anything on their own, and the result would
# be a smaller inventory that every client reads as those items having
# been removed from the store.
# -------------------------------------------------------------------
jq -r 'keys[]' <<< "$CURRENT" | sort > /tmp/expected_items
ls release-assets/*.tar.gz \
@ -974,7 +1151,7 @@ jobs:
if [[ "$INVENTORY_ERRORS" -gt 0 ]]; then
echo ""
echo "ERROR: inventory is incomplete - refusing to publish a partial store"
echo "ERROR: inventory is incomplete: refusing to publish a partial store"
echo " Expected: $(wc -l < /tmp/expected_items) items"
echo " Collected: $(wc -l < /tmp/collected_items) items"
exit 1
@ -992,8 +1169,13 @@ jobs:
fi
# ---------------------------------------------------------------------
# Collect store metadata. Authored fields come from each retroverse_manifest.json;
# Aggregation. Authored fields come from each retroverse_manifest.json;
# sha256, payload_size, url and file_hashes are derived here.
#
# file_hashes for a REUSED item are carried forward from the previous
# release. A reused item is never unpacked and rehashed. Omitting this
# would not fail the build; it would silently ship empty hash maps for
# the most stable items in the store.
# ---------------------------------------------------------------------
- name: Generate retroverse.json
env:
@ -1076,17 +1258,21 @@ jobs:
entries=$(jq -c --argjson e "$entry" '. += [$e]' <<< "$entries")
done < <(jq -r 'keys[]' <<< "$CURRENT")
# alchemist_ref is recorded so the next release can tell whether the
# tool changed. Without it, reuse cannot be invalidated on a tool bump.
jq -n \
--arg schema "1" \
--arg source_id "$SOURCE_ID" \
--arg tag "$RELEASE_TAG" \
--arg built_on "$BUILD_TIMESTAMP" \
--arg alchemist_ref "${{ needs.configure.outputs.alchemist_ref }}" \
--argjson items "$entries" \
'{
store_schema_version: ($schema | tonumber),
source_id: $source_id,
release_tag: $tag,
built_on: $built_on,
alchemist_ref: $alchemist_ref,
items: $items
}' > release-assets/retroverse.json
@ -1169,7 +1355,7 @@ jobs:
echo "Items built/validated:"
jq -r 'to_entries[] | " \(.key) [\(.value.class)]: \(.value.version)"' <<< "$CURRENT"
echo ""
echo "Dry-run complete - no release was created"
echo "Dry-run complete: no release was created"
echo "=========================================="
# ===========================================================================