diff --git a/.github/workflows/retroverse-build.yml b/.github/workflows/retroverse-build.yml index 14e629f..6527369 100644 --- a/.github/workflows/retroverse-build.yml +++ b/.github/workflows/retroverse-build.yml @@ -9,7 +9,7 @@ on: type: string dry_run: - description: 'Dry run - build and validate but do not create release' + description: 'Dry run: build and validate but do not create release' required: false type: boolean default: true @@ -25,6 +25,21 @@ permissions: env: SOURCE_ID: official + # --------------------------------------------------------------------------- + # The Alchemist is not vendored here. It is checked out from the components + # repo at a ref pinned in alchemist/alchemist_ref, so the store repo carries + # only its own version_policy.sh (and optional version_pins.sh), which are + # overlaid onto the checkout. + # + # The ref is pinned rather than floating on main: an unpinned tool could + # change build output with no commit to this repo, and reuse would not notice + # because the fingerprint covers item folders only. A release could then mix + # artifacts built by two different tool versions. + # --------------------------------------------------------------------------- + ALCHEMIST_REPO: RetroDECK/components + ALCHEMIST_PATH: .alchemist + ALCHEMIST_DIR: .alchemist/automation-tools/alchemist + jobs: # =========================================================================== # Check whether this event should trigger a build @@ -62,7 +77,7 @@ jobs: # PR merge commits always trigger a build if echo "$COMMITS" | grep -qiE "^Merge pull request #"; then - echo "PR merge detected - auto-building" + echo "PR merge detected: auto-building" echo "should_build=true" >> $GITHUB_OUTPUT echo "is_dry_run=false" >> $GITHUB_OUTPUT exit 0 @@ -82,7 +97,7 @@ jobs: exit 0 fi - echo "No build trigger found in commit messages - skipping" + echo "No build trigger found in commit messages: skipping" echo "should_build=false" >> $GITHUB_OUTPUT echo "is_dry_run=false" >> $GITHUB_OUTPUT @@ -97,6 +112,7 @@ jobs: outputs: branch: ${{ steps.config.outputs.branch }} is_dry_run: ${{ needs.check-trigger.outputs.is_dry_run }} + alchemist_ref: ${{ steps.alchemist-ref.outputs.ref }} steps: - name: Determine build parameters id: config @@ -123,6 +139,48 @@ jobs: with: ref: ${{ steps.config.outputs.branch }} + # ----------------------------------------------------------------------- + # The pinned Alchemist ref. A tag or full commit SHA. + # ----------------------------------------------------------------------- + - name: Read pinned Alchemist ref + id: alchemist-ref + run: | + REF_FILE="alchemist/alchemist_ref" + + if [[ ! -f "$REF_FILE" ]]; then + echo "ERROR: $REF_FILE not found: the Alchemist ref must be pinned in this repo" + exit 1 + fi + + REF=$(grep -vE '^\s*(#|$)' "$REF_FILE" | head -n1 | tr -d '[:space:]') + + if [[ -z "$REF" ]]; then + echo "ERROR: $REF_FILE is empty" + exit 1 + fi + + case "$REF" in + main|cooker|epicure|HEAD) + echo "ERROR: '$REF' is a branch, not a pin. Use a tag or a full commit SHA." + exit 1 + ;; + esac + + echo "ref=$REF" >> $GITHUB_OUTPUT + echo "Alchemist pinned to: $REF" + + - name: Validate version policy is present + run: | + # load_versions.sh checks for version_policy.sh at source time, before any recipe is read, and exits if it is missing. This repo therefore carries its own even if it holds only data items. + # A comment-only file is enough. + if [[ ! -f "alchemist/version_policy.sh" ]]; then + echo "ERROR: alchemist/version_policy.sh not found" + echo "The Alchemist exits at source time without it, even when no item needs a policy." + exit 1 + fi + + echo "Version policy present" + - name: Validate Bash scripts run: | echo "Validating Bash scripts..." @@ -219,7 +277,7 @@ jobs: ;; component) if [[ "$file_count" -ne 0 ]]; then - fail "class 'component' must not declare files[] - deployment belongs to the external component installer" + fail "class 'component' must not declare files[]: deployment belongs to the external component installer" fi # Store version is read from retroverse_manifest.json for both classes so the aggregator needs no class-specific logic. @@ -327,6 +385,7 @@ jobs: outputs: items-json: ${{ steps.collect.outputs.items }} prev-json: ${{ steps.get-prev.outputs.prev }} + prev-alchemist-ref: ${{ steps.get-prev.outputs.prev_alchemist_ref }} steps: - name: Checkout code uses: actions/checkout@v6 @@ -334,6 +393,41 @@ jobs: ref: ${{ needs.configure.outputs.branch }} fetch-depth: 0 + # ----------------------------------------------------------------------- + # Alchemist checkout. sparse-checkout keeps the rest of the components + # repo out of the workspace, and the nested .git is harmless: alchemist.sh + # resolves REPO_ROOT from its working directory, which stays the store + # repo root, so $REPO_ROOT/$COMPONENT_NAME still points at the item folder. + # ----------------------------------------------------------------------- + - name: Checkout Alchemist + uses: actions/checkout@v6 + with: + repository: ${{ env.ALCHEMIST_REPO }} + ref: ${{ needs.configure.outputs.alchemist_ref }} + path: ${{ env.ALCHEMIST_PATH }} + sparse-checkout: automation-tools/alchemist + sparse-checkout-cone-mode: true + + - name: Overlay store version policy + run: | + # version_policy.sh and version_pins.sh are read from the Alchemist's + # own SCRIPT_DIR, so the components repo's copies arrive with the + # checkout and must be replaced by this repo's. + if [[ ! -x "$ALCHEMIST_DIR/alchemist.sh" ]]; then + echo "ERROR: alchemist.sh not found or not executable at $ALCHEMIST_DIR" + exit 1 + fi + + rm -f "$ALCHEMIST_DIR/version_policy.sh" "$ALCHEMIST_DIR/version_pins.sh" + cp alchemist/version_policy.sh "$ALCHEMIST_DIR/version_policy.sh" + + if [[ -f alchemist/version_pins.sh ]]; then + cp alchemist/version_pins.sh "$ALCHEMIST_DIR/version_pins.sh" + echo "Version pins overlaid" + fi + + echo "Alchemist ready at ${{ needs.configure.outputs.alchemist_ref }}" + - name: Get previous release metadata id: get-prev run: | @@ -347,6 +441,7 @@ jobs: if [[ -z "$PREV_RELEASE" || "$PREV_RELEASE" == "null" ]]; then echo "No previous release found for branch ${BRANCH_NAME}" echo "prev={}" >> $GITHUB_OUTPUT + echo "prev_alchemist_ref=" >> $GITHUB_OUTPUT exit 0 fi @@ -357,9 +452,15 @@ jobs: PREV=$(jq -c '.items | map({(.id): .}) | add // {}' retroverse.json) echo "prev=$PREV" >> $GITHUB_OUTPUT echo "Previous inventory: $(jq 'keys | length' <<< "$PREV") items" + + # Tool version the previous release was built with. A change to it invalidates every reuse (see setup-matrix) + PREV_ALCHEMIST=$(jq -r '.alchemist_ref // ""' retroverse.json) + echo "prev_alchemist_ref=$PREV_ALCHEMIST" >> $GITHUB_OUTPUT + echo "Previous Alchemist ref: ${PREV_ALCHEMIST:-}" else echo "No retroverse.json found in previous release" echo "prev={}" >> $GITHUB_OUTPUT + echo "prev_alchemist_ref=" >> $GITHUB_OUTPUT fi - name: Check for component items @@ -369,7 +470,7 @@ jobs: echo "found=true" >> $GITHUB_OUTPUT else echo "found=false" >> $GITHUB_OUTPUT - echo "No component items - skipping build dependency install" + echo "No component items: skipping build dependency install" fi - name: Install build dependencies @@ -427,7 +528,9 @@ jobs: # ----------------------------------------------------------------- # Fingerprint covers only what the Alchemist consumes. The store - # manifest and artwork are excluded. + # manifest and artwork are excluded: they are republished from + # source every release, so a description or thumbnail edit must not + # invalidate an otherwise identical artifact. # ----------------------------------------------------------------- fingerprint=$(git ls-tree -r HEAD -- "$item_dir" \ | grep -v 'retroverse_manifest\.json$' \ @@ -440,7 +543,7 @@ jobs: # ----------------------------------------------------------------- upstream="" if [[ "$class" == "component" ]]; then - upstream=$(automation-tools/alchemist/alchemist.sh \ + upstream=$("$ALCHEMIST_DIR/alchemist.sh" \ -f "$item_dir/component_recipe.json" --resolve-versions) if [[ -z "$upstream" ]]; then @@ -495,7 +598,7 @@ jobs: PREV: ${{ steps.get-prev.outputs.prev }} run: | if [[ "$PREV" == "{}" ]]; then - echo "No previous release - skipping bump check" + echo "No previous release: skipping bump check" exit 0 fi @@ -512,7 +615,7 @@ jobs: prev_fp=$(jq -r --arg i "$item" '.[$i].fingerprint // empty' <<< "$PREV") prev_up=$(jq -r --arg i "$item" '.[$i].upstream_version // ""' <<< "$PREV") - # New item - nothing to compare against + # New item: nothing to compare against if [[ -z "$prev_version" ]]; then continue fi @@ -551,12 +654,42 @@ jobs: env: CURRENT: ${{ needs.collect-metadata.outputs.items-json }} PREV: ${{ needs.collect-metadata.outputs.prev-json }} + ALCHEMIST_REF: ${{ needs.configure.outputs.alchemist_ref }} + PREV_ALCHEMIST_REF: ${{ needs.collect-metadata.outputs.prev-alchemist-ref }} run: | echo "Current: $(jq 'keys | length' <<< "$CURRENT") items" echo "Previous: $(jq 'keys | length' <<< "$PREV") items" echo "" - entries=$(jq -c --argjson prev "$PREV" ' + # --------------------------------------------------------------- + # A change to the pinned Alchemist invalidates every reuse. + # + # The tool is not covered by any item's fingerprint, so without + # this a release could carry some artifacts built by the new + # Alchemist and some by the old one. Rebuilding everything keeps a + # release internally consistent, at the cost of one full rebuild + # per Alchemist bump. + # + # This does NOT force version bumps: the bump check compares + # fingerprint and upstream version, neither of which the tool ref + # touches. A tool change rebuilds without demanding a bump on + # every item in the store. + # --------------------------------------------------------------- + ALCHEMIST_CHANGED=false + if [[ -n "$PREV_ALCHEMIST_REF" && "$PREV_ALCHEMIST_REF" != "$ALCHEMIST_REF" ]]; then + ALCHEMIST_CHANGED=true + echo "Alchemist ref changed: $PREV_ALCHEMIST_REF -> $ALCHEMIST_REF" + echo "Reuse disabled for this release: rebuilding every item" + echo "" + elif [[ -z "$PREV_ALCHEMIST_REF" && "$PREV" != "{}" ]]; then + ALCHEMIST_CHANGED=true + echo "Previous release recorded no Alchemist ref: rebuilding every item" + echo "" + fi + + entries=$(jq -c \ + --argjson prev "$PREV" \ + --argjson alchemist_changed "$ALCHEMIST_CHANGED" ' to_entries | map( .value as $c | ($prev[.key] // {}) as $p | @@ -566,6 +699,7 @@ jobs: class: $c.class, version: $c.version, can_reuse: ( + ($alchemist_changed | not) and ($p.version // "") == $c.version and ($p.fingerprint // "") == $c.fingerprint and ($p.upstream_version // "") == ($c.upstream_version // "") @@ -599,6 +733,9 @@ jobs: # =========================================================================== # Build each item (or reuse from previous release) # =========================================================================== + # fail-fast is false: a store may hold hundreds of items and one malformed + # manifest should surface every other failure in the same run. + # =========================================================================== build-items: name: Build ${{ matrix.item }} needs: [configure, collect-metadata, setup-matrix] @@ -617,6 +754,33 @@ jobs: ref: ${{ needs.configure.outputs.branch }} fetch-depth: 0 + - name: Checkout Alchemist + if: matrix.can_reuse == false + uses: actions/checkout@v6 + with: + repository: ${{ env.ALCHEMIST_REPO }} + ref: ${{ needs.configure.outputs.alchemist_ref }} + path: ${{ env.ALCHEMIST_PATH }} + sparse-checkout: automation-tools/alchemist + sparse-checkout-cone-mode: true + + - name: Overlay store version policy + if: matrix.can_reuse == false + run: | + if [[ ! -x "$ALCHEMIST_DIR/alchemist.sh" ]]; then + echo "ERROR: alchemist.sh not found or not executable at $ALCHEMIST_DIR" + exit 1 + fi + + rm -f "$ALCHEMIST_DIR/version_policy.sh" "$ALCHEMIST_DIR/version_pins.sh" + cp alchemist/version_policy.sh "$ALCHEMIST_DIR/version_policy.sh" + + if [[ -f alchemist/version_pins.sh ]]; then + cp alchemist/version_pins.sh "$ALCHEMIST_DIR/version_pins.sh" + fi + + echo "Alchemist ready at ${{ needs.configure.outputs.alchemist_ref }}" + - name: Install build dependencies if: matrix.can_reuse == false run: | @@ -676,7 +840,8 @@ jobs: echo "Reused assets downloaded" # --------------------------------------------------------------------- - # Component items run through the Alchemist unchanged. + # Component items run through the Alchemist unchanged, so their archive + # is byte-for-byte the shape the external component installer expects. # --------------------------------------------------------------------- - name: Build component item if: matrix.can_reuse == false && matrix.class == 'component' @@ -685,7 +850,7 @@ jobs: echo "Building component item $ITEM..." - automation-tools/alchemist/alchemist.sh -f "${{ matrix.dir }}/component_recipe.json" + "$ALCHEMIST_DIR/alchemist.sh" -f "${{ matrix.dir }}/component_recipe.json" echo "Build complete" @@ -693,6 +858,10 @@ jobs: # Data items stop before compression so each deployed file can be hashed # while still uncompressed. The workflow then compresses exactly as the # Alchemist would, so collection and verification stay class-agnostic. + # + # The output directory MUST exist before invoking with -o: the Alchemist + # runs realpath on it, and export masks the failure from set -e, leaving + # WORKDIR empty and the artifact root at the filesystem root. # --------------------------------------------------------------------- - name: Build data item if: matrix.can_reuse == false && matrix.class == 'data' @@ -704,7 +873,7 @@ jobs: ITEM_WORKDIR="$GITHUB_WORKSPACE/.build/$ITEM" mkdir -p "$ITEM_WORKDIR" - automation-tools/alchemist/alchemist.sh \ + "$ALCHEMIST_DIR/alchemist.sh" \ -f "${{ matrix.dir }}/component_recipe.json" \ --dry-run \ -o "$ITEM_WORKDIR" @@ -756,7 +925,7 @@ jobs: fi if [[ "${{ matrix.class }}" != "data" ]]; then - echo "Component item - no files[] to cross-check" + echo "Component item: no files[] to cross-check" exit 0 fi @@ -948,7 +1117,15 @@ jobs: fi # ------------------------------------------------------------------- - # Check for inventory completeness. + # Inventory completeness. + # + # A failed matrix leg already fails build-items and skips this job, so + # this is not guarding against item build failures. It guards against + # an item silently never reaching release-assets. A download pattern + # that misses one, a collection bug, a matrix entry that produced no + # leg. None of those fail anything on their own, and the result would + # be a smaller inventory that every client reads as those items having + # been removed from the store. # ------------------------------------------------------------------- jq -r 'keys[]' <<< "$CURRENT" | sort > /tmp/expected_items ls release-assets/*.tar.gz \ @@ -974,7 +1151,7 @@ jobs: if [[ "$INVENTORY_ERRORS" -gt 0 ]]; then echo "" - echo "ERROR: inventory is incomplete - refusing to publish a partial store" + echo "ERROR: inventory is incomplete: refusing to publish a partial store" echo " Expected: $(wc -l < /tmp/expected_items) items" echo " Collected: $(wc -l < /tmp/collected_items) items" exit 1 @@ -992,8 +1169,13 @@ jobs: fi # --------------------------------------------------------------------- - # Collect store metadata. Authored fields come from each retroverse_manifest.json; + # Aggregation. Authored fields come from each retroverse_manifest.json; # sha256, payload_size, url and file_hashes are derived here. + # + # file_hashes for a REUSED item are carried forward from the previous + # release. A reused item is never unpacked and rehashed. Omitting this + # would not fail the build; it would silently ship empty hash maps for + # the most stable items in the store. # --------------------------------------------------------------------- - name: Generate retroverse.json env: @@ -1076,17 +1258,21 @@ jobs: entries=$(jq -c --argjson e "$entry" '. += [$e]' <<< "$entries") done < <(jq -r 'keys[]' <<< "$CURRENT") + # alchemist_ref is recorded so the next release can tell whether the + # tool changed. Without it, reuse cannot be invalidated on a tool bump. jq -n \ --arg schema "1" \ --arg source_id "$SOURCE_ID" \ --arg tag "$RELEASE_TAG" \ --arg built_on "$BUILD_TIMESTAMP" \ + --arg alchemist_ref "${{ needs.configure.outputs.alchemist_ref }}" \ --argjson items "$entries" \ '{ store_schema_version: ($schema | tonumber), source_id: $source_id, release_tag: $tag, built_on: $built_on, + alchemist_ref: $alchemist_ref, items: $items }' > release-assets/retroverse.json @@ -1169,7 +1355,7 @@ jobs: echo "Items built/validated:" jq -r 'to_entries[] | " \(.key) [\(.value.class)]: \(.value.version)"' <<< "$CURRENT" echo "" - echo "Dry-run complete - no release was created" + echo "Dry-run complete: no release was created" echo "==========================================" # ===========================================================================