diff --git a/automation_tools/libman.sh b/automation_tools/libman.sh new file mode 100644 index 00000000..ed395fee --- /dev/null +++ b/automation_tools/libman.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +# Be aware that this script deletes the source directory after copying the files. It is intended to be used only by the flatpak builder. + + +# List of user-defined libraries to exclude +excluded_libraries=("libselinux.so.1") + +# Define target directory +target_dir="${FLATPAK_DEST}/lib" + + +echo "Worry not, LibMan is here!" + +# Set default destination if FLATPAK_DEST is not set +if [ -z "$FLATPAK_DEST" ]; then + export FLATPAK_DEST="/app" +fi + +# Check if source directory is provided +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 0 +fi + +# Ensure the target directory exists +if ! mkdir -p "$target_dir"; then + echo "Error: Failed to create target directory $target_dir" + exit 0 +fi + +# Function to check if a file is in the excluded libraries list +is_excluded() { + local file="$1" + for excluded in "${excluded_libraries[@]}"; do + if [[ "$excluded" == "$file" ]]; then + return 0 + fi + done + return 1 +} + +# Find and copy files +copied_files=() +failed_files=() + +# First, copy all regular files +for file in $(find "$1" -type f -name "*.so*" ! -type l); do + # Define destination file path + dest_file="$target_dir/$(basename "$file")" + + # Skip if the file is in the list of excluded libraries + if is_excluded "$(basename "$file")"; then + reason="library is in the exclusion list" + echo "Skipped $file as it is $reason" + failed_files+=("$file, $reason") + continue + fi + + # Skip if the destination file already exists + if [ -e "$dest_file" ]; then + echo "Skipped $file as $dest_file already exists" + continue + fi + + # Attempt to copy the file + if install -D "$file" "$dest_file" 2>error_log; then + echo "Copied $file to $dest_file" + copied_files+=("$file") + else + error_message=$(error_log; then + echo "Copied symlink target $symlink_target to $dest_symlink_target" + copied_files+=("$symlink_target") + else + error_message=$(error_log; then + echo "Created symlink $dest_file -> $dest_symlink_target" + copied_files+=("$file") + else + error_message=$(RetroDECK now is migrated to an organization on GitHub, as before, please check the October blog post
  • pre_build_automation script was reworked and additional functions are added
  • Contributing is now easier and the manifest build time is down to about 1h (was 3h30)
  • +
  • Introduced a new library management system called LibMan to make sure that each compoent is bringing its own libraries while built from the manifest
  • Known issues:

      diff --git a/net.retrodeck.retrodeck.yml b/net.retrodeck.retrodeck.yml index baf805a9..2a88d39f 100644 --- a/net.retrodeck.retrodeck.yml +++ b/net.retrodeck.retrodeck.yml @@ -53,7 +53,7 @@ cleanup: - /src - '*.a' - '*.la' - - /app/retrodeck/tmplib + - /app/bin/libman.sh modules: @@ -70,11 +70,12 @@ modules: # [ ] Update the VERSION variable on line containing "VERSION=THISBRANCH" # [ ] Update the appdata.xml with the version number and notes - - name: version-initialization + - name: retrodeck-initialization buildsystem: simple build-commands: - | + # VERSION INITIALIZATION # on main please update this with the version variable, eg: VERSION=0.8.0b # on cooker will be VERSION=cooker-0.9.0b for example VERSION=cooker-0.9.0b @@ -88,6 +89,9 @@ modules: echo $VERSION >> ${FLATPAK_DEST}/retrodeck/version cat ${FLATPAK_DEST}/retrodeck/version echo "Version is $VERSION" + + # LIBMAN INSTALLATION + install -Dm755 "automation_tools/libman.sh" "/app/bin/libman.sh" sources: - type: git url: THISREPO @@ -229,21 +233,22 @@ modules: - name: retroarch buildsystem: simple build-commands: + # Step 1: Create required directories - mkdir -p "${FLATPAK_DEST}/share/libretro/" + # Step 2: Copy RetroArch configuration files - cp -r ./RetroArch-Linux-x86_64.AppImage.home/.config/retroarch/* "${FLATPAK_DEST}/share/libretro/" + # Step 3: Extract the AppImage - chmod +x ./*.AppImage - ./*.AppImage --appimage-extract - - mkdir -p "${FLATPAK_DEST}/retrodeck/tmplib" "${FLATPAK_DEST}/retrodeck/tmplib/debug" - - mv "squashfs-root/usr/lib/"* "${FLATPAK_DEST}/retrodeck/tmplib" - - cp -r squashfs-root/usr/* "${FLATPAK_DEST}" + # Step 4: Use libman.sh to manage libraries + - /app/bin/libman.sh "squashfs-root/usr/lib" + # Step 5: Copy remaining extracted files to the Flatpak destination + - find squashfs-root/usr/ -exec sh -c 'install -Dm755 "{}" "${FLATPAK_DEST}/$(echo "{}" | sed "s|^squashfs-root/usr||")"' \; sources: - type: archive url: https://buildbot.libretro.com/stable/1.20.0/linux/x86_64/RetroArch.7z - #sha256: RASHAPLACEHOLDER sha256: 809b3e9f02a9849719453d0f189a0edc544ad3235c8ce75a79488e710ba9668a - # Not part of the offical RetroArch AppImage - # TODO: outsource me - name: libbz2 no-autogen: true @@ -341,11 +346,15 @@ modules: - name: retrodeck-ppsspp buildsystem: simple build-commands: + # Step 1: Remove unused manifest file - rm -f "files/manifest.json" - - mkdir -p "${FLATPAK_DEST}/retrodeck/tmplib" "${FLATPAK_DEST}/retrodeck/tmplib/debug" - - "cp -rnL files/lib/* ${FLATPAK_DEST}/retrodeck/tmplib || echo 'Warning: Some files could not be copied, but the build will continue.'" + # Step 2: Use libman.sh to copy libraries to the RetroDeck library folder + - /app/bin/libman.sh "files/lib" + # Step 3: Clean up source library folder - rm -rf "files/lib" + # Step 4: Make binaries executable - chmod +x "files/bin/"* + # Step 5: Copy remaining files to Flatpak destination - cp -r files/* "${FLATPAK_DEST}" sources: - type: archive @@ -359,11 +368,14 @@ modules: - name: pcsx2 buildsystem: simple build-commands: + # Step 1: Ensure the AppImage is executable - chmod +x *.AppImage + # Step 2: Extract the AppImage - ./*.AppImage --appimage-extract - - mkdir -p "${FLATPAK_DEST}/retrodeck/tmplib" "${FLATPAK_DEST}/retrodeck/tmplib/debug" - - mv "squashfs-root/usr/lib/"* "${FLATPAK_DEST}/retrodeck/tmplib" - - cp -r squashfs-root/usr/* "${FLATPAK_DEST}" + # Step 3: Use libman.sh to handle library files + - /app/bin/libman.sh "squashfs-root/usr/lib" + # Step 4: Copy the remaining extracted files to the Flatpak destination + - find squashfs-root/usr/ -exec sh -c 'install -Dm755 "{}" "${FLATPAK_DEST}/$(echo "{}" | sed "s|^squashfs-root/usr||")"' \; sources: - type: file url: https://github.com/PCSX2/pcsx2/releases/download/v2.3.53/pcsx2-v2.3.53-linux-appimage-x64-Qt.AppImage @@ -376,13 +388,15 @@ modules: - name: retrodeck-dolphin buildsystem: simple build-commands: + # Step 1: Remove any existing manifest.json file - rm -f "files/manifest.json" - - mkdir -p "${FLATPAK_DEST}/retrodeck/tmplib" "${FLATPAK_DEST}/retrodeck/tmplib/debug" - - "cp -rnL files/lib/debug ${FLATPAK_DEST}/retrodeck/tmplib || echo 'Warning: Some files could not be copied, but the build will continue.'" - - rm -rf "files/lib/debug" - - "cp -rnL files/lib/* ${FLATPAK_DEST}/retrodeck/tmplib || echo 'Warning: Some files could not be copied, but the build will continue.'" + # Step 2: Use libman.sh to manage libraries + - /app/bin/libman.sh "files/lib" + # Step 3: removing libraries folder that have been already moved - rm -rf "files/lib" + # Step 4: Ensure binaries are executable - chmod +x "files/bin/"* + # Step 5: Copy all remaining files to the Flatpak destination - cp -r files/* "${FLATPAK_DEST}" sources: - type: archive @@ -395,14 +409,17 @@ modules: - name: retrodeck-primehack buildsystem: simple build-commands: + # Step 1: Remove any existing manifest.json file - rm -f "files/manifest.json" - - mkdir -p "${FLATPAK_DEST}/retrodeck/tmplib" "${FLATPAK_DEST}/retrodeck/tmplib/debug" + # Step 2: Remove unnecessary files - rm -rf "files/lib/pkgconfig" - - "cp -rnL files/lib/debug ${FLATPAK_DEST}/retrodeck/tmplib || echo 'Warning: Some files could not be copied, but the build will continue.'" - - rm -rf "files/lib/debug" - - "cp -rnL files/lib/* ${FLATPAK_DEST}/retrodeck/tmplib || echo 'Warning: Some files could not be copied, but the build will continue.'" + # Step 3: Use libman.sh to manage libraries + - /app/bin/libman.sh "files/lib" + # Step 4: removing libraries folder that have been already moved - rm -rf "files/lib" + # Step 5: Ensure binaries are executable - chmod +x "files/bin/"* + # Step 6: Copy all remaining files to the Flatpak destination - cp -r files/* "${FLATPAK_DEST}" sources: - type: archive @@ -415,11 +432,15 @@ modules: - name: retrodeck-rpcs3 buildsystem: simple build-commands: + # Step 1: Remove any existing manifest.json file - rm -f "files/manifest.json" - - mkdir -p "${FLATPAK_DEST}/retrodeck/tmplib" "${FLATPAK_DEST}/retrodeck/tmplib/debug" - - "cp -rnL files/lib/* ${FLATPAK_DEST}/retrodeck/tmplib || echo 'Warning: Some files could not be copied, but the build will continue.'" + # Step 2: Use libman.sh to manage libraries + - /app/bin/libman.sh "files/lib" + # Step 3: removing libraries folder that have been already moved - rm -rf "files/lib" + # Step 4: Ensure binaries are executable - chmod +x "files/bin/"* + # Step 5: Copy all remaining files to the Flatpak destination - cp -r files/* "${FLATPAK_DEST}" sources: - type: archive @@ -432,11 +453,15 @@ modules: - name: melonds buildsystem: simple build-commands: + # Step 1: Remove any existing manifest.json file - rm -f "files/manifest.json" - - mkdir -p "${FLATPAK_DEST}/retrodeck/tmplib/" - - "cp -rnL files/lib/* ${FLATPAK_DEST}/retrodeck/tmplib || echo 'Warning: Some files could not be copied, but the build will continue.'" + # Step 2: Use libman.sh to manage libraries + - /app/bin/libman.sh "files/lib" + # Step 3: removing libraries folder that have been already moved - rm -rf "files/lib" + # Step 4: Ensure binaries are executable - chmod +x "files/bin/"* + # Step 5: Copy all remaining files to the Flatpak destination - cp -r files/* "${FLATPAK_DEST}/" sources: - type: archive @@ -450,11 +475,13 @@ modules: - name: duckstation buildsystem: simple build-commands: + # Step 1: Ensure the AppImage is executable and extract it - chmod +x *.AppImage - ./*.AppImage --appimage-extract - - mkdir -p "${FLATPAK_DEST}/retrodeck/tmplib" "${FLATPAK_DEST}/retrodeck/tmplib/debug" - - mv "squashfs-root/usr/lib/"* "${FLATPAK_DEST}/retrodeck/tmplib" - - cp -r squashfs-root/usr/* "${FLATPAK_DEST}" + # Step 2: Use libman.sh to manage libraries + - /app/bin/libman.sh "squashfs-root/usr/lib" + # Step 3: Copy remaining extracted files to the Flatpak destination + - find squashfs-root/usr/ -exec sh -c 'install -Dm755 "{}" "${FLATPAK_DEST}/$(echo "{}" | sed "s|^squashfs-root/usr||")"' \; sources: - type: file url: https://github.com/RetroDECK/Duckstation/releases/download/preview/DuckStation-x64.AppImage @@ -467,14 +494,17 @@ modules: - name: cemu buildsystem: simple build-commands: + # Step 1: Ensure the AppImage is executable and extract it - chmod +x *.AppImage - ./*.AppImage --appimage-extract - - mkdir -p "${FLATPAK_DEST}/retrodeck/tmplib" "${FLATPAK_DEST}/retrodeck/tmplib/debug" - - rm -rf "squashfs-root/usr/lib/girepository-1.0" # causes issues, I think we already have this - - mv "squashfs-root/usr/lib/"* "${FLATPAK_DEST}/retrodeck/tmplib" - - cp -r squashfs-root/usr/* "${FLATPAK_DEST}" + # Step 2: Remove unnecessary files causing potential issues + - rm -rf "squashfs-root/usr/lib/girepository-1.0" + # Step 3: Use libman.sh to manage libraries + - /app/bin/libman.sh "squashfs-root/usr/lib" + # Step 4: Copy remaining extracted files to the Flatpak destination + - find squashfs-root/usr/ -exec sh -c 'install -Dm755 "{}" "${FLATPAK_DEST}/$(echo "{}" | sed "s|^squashfs-root/usr||")"' \; + # Step 5: Install the wrapper script - install -Dm755 Cemu-wrapper "${FLATPAK_DEST}/bin/" - #TODO: do we need this wrapper? squashfs-root/apprun-hooks/linuxdeploy-plugin-gtk.sh sources: - type: file url: https://github.com/cemu-project/Cemu/releases/download/v2.4/Cemu-2.4-x86_64.AppImage @@ -517,12 +547,18 @@ modules: - name: retrodeck-solarus buildsystem: simple build-commands: + # Step 1: Remove any existing manifest.json file - rm -f "files/manifest.json" - - mkdir -p "${FLATPAK_DEST}/retrodeck/tmplib" "${FLATPAK_DEST}/retrodeck/tmplib/debug" - #- cp -r "files/lib/debug/lib/*" "${FLATPAK_DEST}/retrodeck/tmplib/debug/" + # Step 2: Remove unnecessary files - rm -rf "files/lib/debug" "files/lib/pkgconfig" + # Step 3: Use libman.sh to manage libraries + - /app/bin/libman.sh "files/lib" + # Step 4: removing libraries folder that have been already moved + - rm -rf "files/lib" + # Step 5: Ensure binaries are executable - chmod +x "files/bin/"* - - "cp -rnL files/lib/* ${FLATPAK_DEST}/retrodeck/tmplib || echo 'Warning: Some files could not be copied, but the build will continue.'" + # Step 6: Copy all remaining files to the Flatpak destination + - cp -r files/* "${FLATPAK_DEST}" sources: - type: archive url: https://github.com/RetroDECK/org.solarus_games.solarus.Launcher/releases/latest/download/RetroDECK-solarus-Artifact.tar.gz @@ -535,12 +571,17 @@ modules: - name: retrodeck-gzdoom buildsystem: simple build-commands: + # Step 1: Remove any existing manifest.json file - rm -f "files/manifest.json" - - mkdir -p "${FLATPAK_DEST}/retrodeck/tmplib" "${FLATPAK_DEST}/retrodeck/tmplib/debug" + # Step 2: Remove unnecessary files - rm -rf "files/lib/cmake" "files/lib/pkgconfig" "files/lib/debug" - - "cp -rnL files/lib/* ${FLATPAK_DEST}/retrodeck/tmplib || echo 'Warning: Some files could not be copied, but the build will continue.'" + # Step 3: Use libman.sh to manage libraries + - /app/bin/libman.sh "files/lib" + # Step 4: removing libraries folder that have been already moved - rm -rf "files/lib" + # Step 5: Ensure binaries are executable - chmod +x "files/bin/"* + # Step 6: Copy all remaining files to the Flatpak destination - cp -r files/* "${FLATPAK_DEST}" sources: - type: archive @@ -549,15 +590,20 @@ modules: # Pancakes - - name: Pancakes + - name: pancakes buildsystem: simple build-commands: + # Step 1: Create the Flatpak destination directory - mkdir -p "${FLATPAK_DEST}" - - mv *.so* "${FLATPAK_DEST}/retrodeck/tmplib/" + # Step 2: Extract and manage library files + - /app/bin/libman.sh "." + # Step 3: Move and set up the binary + - rm -rf *.dll.config - mv R*x* "${FLATPAK_DEST}/bin/" - chmod +x "${FLATPAK_DEST}/bin/"R*x* + # Step 4: Set up license directory and move license files - mkdir -p "${FLATPAK_DEST}/retrodeck/licenses" - - mv LICENSE.txt "${FLATPAK_DEST}/retrodeck/licenses" + - mv LICENSE.txt THIRDPARTY.md "${FLATPAK_DEST}/retrodeck/licenses" sources: - type: archive url: https://github.com/RetroDECK/Pancakes-bin/releases/download/1.1.0.1403/pancakes-Release-linux_x64.tar.gz @@ -569,11 +615,13 @@ modules: - name: xemu buildsystem: simple build-commands: + # Step 1: Ensure the AppImage is executable and extract it - chmod +x *.AppImage - ./*.AppImage --appimage-extract - - mkdir -p "${FLATPAK_DEST}/retrodeck/tmplib" "${FLATPAK_DEST}/retrodeck/tmplib/debug" - - mv "squashfs-root/usr/lib/"* "${FLATPAK_DEST}/retrodeck/tmplib" - - cp -r squashfs-root/usr/* "${FLATPAK_DEST}/" + # Step 2: Use libman.sh to manage libraries + - /app/bin/libman.sh "squashfs-root/usr/lib" + # Step 3: Copy remaining extracted files to the Flatpak destination + - find squashfs-root/usr/ -exec sh -c 'install -Dm755 "{}" "${FLATPAK_DEST}/$(echo "{}" | sed "s|^squashfs-root/usr||")"' \; sources: - type: file url: https://github.com/xemu-project/xemu/releases/download/v0.8.5/xemu-v0.8.5-x86_64.AppImage @@ -594,7 +642,11 @@ modules: - name: retrodeck-mame buildsystem: simple build-commands: + # Step 1: Remove any existing manifest.json file - rm -f files/manifest.json + # Step 2: Use libman.sh to manage libraries - REMOVED AS MAME DON'T BRING ANY LIBS WITH IT + # - /app/bin/libman.sh "files/lib" + # Step 3: Copy the managed libraries and all other files to the Flatpak destination - cp -rn files/* ${FLATPAK_DEST} sources: - type: archive @@ -631,12 +683,15 @@ modules: - name: ES-DE buildsystem: simple build-commands: + # Step 1: Remove any existing manifest.json file - rm -f "files/manifest.json" - - mkdir -p "${FLATPAK_DEST}/retrodeck/tmplib" "${FLATPAK_DEST}/retrodeck/tmplib/debug" - - cp -rfL files/lib/* ${FLATPAK_DEST}/retrodeck/tmplib - - cp -rfL files/lib/* ${FLATPAK_DEST}/retrodeck/tmplib + # Step 2: Use libman.sh to manage libraries + - /app/bin/libman.sh "files/lib" + # Step 3: Clean up source library folder - rm -rf "files/lib" + # Step 4: Ensure binaries are executable - chmod +x "files/bin/"* + # Step 5: Copy all remaining files to the Flatpak destination - cp -r files/* "${FLATPAK_DEST}" sources: - type: archive @@ -668,53 +723,6 @@ modules: url: https://github.com/ruffle-rs/ruffle/releases/download/nightly-2024-10-29/ruffle-nightly-2024_10_29-linux-x86_64.tar.gz sha256: 5ab0fc46b07d022d4b0ff355e88175cd77b8bb1663612f31d160f8001dc472bb - - name: fetch-missing-libs - buildsystem: simple - build-commands: - - | - # Search for missing libraries in our libtmp folder - missing_libs=$(find /app/bin -type f -exec ldd {} + 2>/dev/null | grep 'not found' | awk '$1 ~ /\.so/ {print $1}' | sort | uniq) - echo "$missing_libs" - - # Manually specified libraries that are not automatically detected - manually_imported="libpostproc.so.56 libswscale.so.6 libshaderc_shared.so.1 libbz2.so.1.0 libaio.so.1 ld-linux.so.2 libvpx.so.9 libzmusic.so.1 libicuuc.so.73 libLLVM.so.18.1" - - # Combine detected missing libraries and manually specified ones - missing_libs="$missing_libs $manually_imported" - echo -e "Missing libs and manually imported:\n$missing_libs" - - # Source paths - src_dir="${FLATPAK_DEST}/retrodeck/tmplib" - src_debug_dir="${FLATPAK_DEST}/retrodeck/tmplib/debug" - - # Destination paths - dest_dir="${FLATPAK_DEST}/lib" - dest_debug_dir="${FLATPAK_DEST}/retrodeck/lib/debug" - - mkdir -p "$dest_dir" "$dest_debug_dir" - - copy_missing_libs() { - local src=$1 - local src_debug=$2 - local dest=$3 - for lib in $missing_libs; do - if [[ -f "$src/$lib" ]]; then - cp "$src/$lib" "$dest" && echo "Copied \"$lib\" to \"$dest\"" || echo "Error copying \"$lib\"" - elif [[ -f "$src_debug/$lib.debug" ]]; then - cp "$src_debug/$lib.debug" "$dest" && echo "Copied debug version of \"$lib\" to \"$dest\"" || echo "Error copying debug version of \"$lib\"" - else - echo "Library \"$lib\" not found in \"$src\" or \"$src_debug\"" - fi - done - } - - # Copy libraries from source to destination - copy_missing_libs "$src_dir" "$src_debug_dir" "$dest_dir" - copy_missing_libs "$src_debug_dir" "$dest_debug_dir" - - # Remove temporary source directory to free up space - rm -rf "$src_dir" - - name: retrodeck buildsystem: simple build-commands: