From 8c3bc5585671a1d2a4d4be7ac4604444a7a52e54 Mon Sep 17 00:00:00 2001 From: Marco Matthijs <47783400+MatthijsMarco@users.noreply.github.com> Date: Mon, 8 Jul 2024 15:09:06 +0200 Subject: [PATCH] CLOUD_SYNC: added --- .github/workflows/emulator-updates.yaml | 56 +++++++++++++++ automation_tools/automation_task_list.cfg | 2 +- functions/cloud_sync.sh | 76 ++++++++++++++++++++ functions/global.sh | 1 + net.retrodeck.retrodeck.appdata.xml | 1 + net.retrodeck.retrodeck.yml | 37 ++++------ tools/configurator.sh | 87 +++++++++++++++++++++-- 7 files changed, 230 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/emulator-updates.yaml diff --git a/.github/workflows/emulator-updates.yaml b/.github/workflows/emulator-updates.yaml new file mode 100644 index 00000000..d55cd4a7 --- /dev/null +++ b/.github/workflows/emulator-updates.yaml @@ -0,0 +1,56 @@ +name: Check Emulator Updates + +on: + # schedule: + # - cron: '0 18 * * *' + push: + branches: + - main + workflow_dispatch: + +jobs: + check-updates: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flatpak-builder-tools + + - name: Check for emulator updates + run: | + flatpak-builder --check-updates ./net.retrodeck.retrodeck.yml --modules "modulo1,modulo2" # Sostituisci modulo1,modulo2 con i tuoi moduli specifici + + - name: Configure Git + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Create or update branch + run: | + git fetch origin + if git rev-parse --verify origin/feat/auto-emulator-updates; then + git branch -D feat/auto-emulator-updates + fi + git checkout -b feat/auto-emulator-updates + git add . + git commit -m "Update emulators" + git push --set-upstream origin feat/auto-emulator-updates --force + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + source: feat/auto-emulator-updates + destination: cooker-0.8.2b + title: 'Automated Emulator Updates' + body: 'This PR contains automated updates for specified emulators.' diff --git a/automation_tools/automation_task_list.cfg b/automation_tools/automation_task_list.cfg index 1b955846..2284832c 100644 --- a/automation_tools/automation_task_list.cfg +++ b/automation_tools/automation_task_list.cfg @@ -1,7 +1,7 @@ # The proper format for this file is # ACTION^PLACEHOLDERTEXT^URL^REPO(Optional) # hash^DOOMSHAPLACEHOLDER^https://buildbot.libretro.com/assets/cores/DOOM/Doom%20%28Shareware%29.zip -hash^DUCKSTATIONSHAPLACEHOLDER^https://github.com/stenzek/duckstation/releases/download/preview/DuckStation-x64.AppImage +#hash^DUCKSTATIONSHAPLACEHOLDER^https://github.com/stenzek/duckstation/releases/download/preview/DuckStation-x64.AppImage hash^SAMEDUCKSHAPLACEHOLDER^https://buildbot.libretro.com/nightly/linux/x86_64/latest/sameduck_libretro.so.zip hash^PPSSPPBIOSHASHPLACEHOLDER^https://github.com/hrydgard/ppsspp/archive/refs/heads/master.zip hash^MSXBIOSHASHPLACEHOLDER^http://bluemsx.msxblue.com/rel_download/blueMSXv282full.zip diff --git a/functions/cloud_sync.sh b/functions/cloud_sync.sh index 05a7907c..5c7231a5 100644 --- a/functions/cloud_sync.sh +++ b/functions/cloud_sync.sh @@ -1,2 +1,78 @@ #!/bin/bash +#TODO: +# - remove hard code +# - add multi-user support +# - save remote name somewhere (and maybe make customisable remote name) +# - optional: back up to mutliple remotes (by replacing $rdhome with a different remote) +# - add exclusion options (to have multiple playthroughs for different devices) + + +#--backup-dir requires v1.66 +#Options for resync-mode (also requires v1.66): +# - path1 (local files always win), +# - path2 (remote files always win), +# - newer (newer files always win), +# - older (older files always win), +# - larger (larger files always win), +# - smaller (smaller files always win) + +set_cloud() { # 1=cloud-provider 2=resync-mode 3=username 4=password 5=host/URL 6=port + #TODO: only trigger this log when browser authentication needed + log i "Configurator: Opening browser and authenticating" + case $1 in + box) + rclone --fast-list --ignore-checksum config create RetroDECK box + ;; + dropbox) + rclone --fast-list --ignore-checksum config create RetroDECK dropbox + ;; + drive) + rclone --fast-list --ignore-checksum config create RetroDECK drive scope=drive + ;; + onedrive) + rclone --fast-list --ignore-checksum config create RetroDECK onedrive drive_type=personal access_scopes=Files.ReadWrite,offline_access + ;; + #TODO for ftp, smb, webdav: check how passwords are stored, is it secure? + ftp) + rclone --fast-list --ignore-checksum config create RetroDECK ftp host=$5 port=$6 username=$3 password=$4 + ;; + smb) + rclone --fast-list --ignore-checksum config create RetroDECK smb host=$5 port=$6 username=$3 password=$4 + ;; + fastmail | nextcloud | owncloud | sharepoint | sharepoint-ntlm | rclone | other) + # TODO: add filtering to allow both base URL and webdav URL to be added (nextcloud-instance.com and nextcloud-instance.com/remote.php/dav/files/USERNAME/ would both be valid inputs) + rclone --fast-list --ignore-checksum config create RetroDECK webdav url=$5 username=$3 password=$4 provider=$1 + ;; + *) + exit + ;; + esac + rclone mkdir RetroDECK:/RetroDECK + rclone mkdir RetroDECK:/RetroDECK_backup + touch $rdhome/RCLONE_TEST + mkdir $rdhome/../retrodeck_backup + #TODO: discuss which other directories are eligable for syncing, and add them to the include flag (or user choice). + rclone --copy-links --check-first bisync --resync --resync-mode $2 $rdhome RetroDECK:/RetroDECK --include "{saves,screenshots}/**" --backup-dir1 ~/retrodeck_backup --backup-dir2 RetroDECK:/RetroDECK_backup +} + +unset_cloud() { + rclone config delete RetroDECK +} + + +#Theoretically, you only need to push to the cloud after quitting a game, and pull just before starting. The bisync option has some nice extra options, however, so the preferable workflow needs to be discussed. + +sync_cloud() { #1=resolver type (none, newer, older, larger, smaller, path1, path2) + # --max-delete PERCENT: Safety check on maximum percentage of deleted files allowed. If exceeded, the bisync run will abort. (default: 50%)? + rclone --copy-links --check-first bisync --recover --no-slow-hash --check-access --conflict-resolve $1 $rdhome RetroDECK:/RetroDECK --include "{saves,screenshots}/**" --backup-dir1 ~/retrodeck_backup --backup-dir2 RetroDECK:/RetroDECK_backup +} + +# --update: Skip files that are newer on the destination? +push_cloud() { + rclone --copy-links --check-first sync --check-first $rdhome RetroDECK:/RetroDECK --include "{saves,screenshots}/**" --backup-dir RetroDECK:/RetroDECK_backup +} + +pull_cloud() { + rclone --copy-links --check-first sync --check-first RetroDECK:/RetroDECK $rdhome --include "{saves,screenshots}/**" --backup-dir ~/retrodeck_backup +} diff --git a/functions/global.sh b/functions/global.sh index 52db4426..01fcc38a 100644 --- a/functions/global.sh +++ b/functions/global.sh @@ -14,6 +14,7 @@ source /app/libexec/post_update.sh source /app/libexec/prepare_component.sh source /app/libexec/presets.sh source /app/libexec/configurator_functions.sh +source /app/libexec/cloud_sync.sh # Static variables rd_conf="/var/config/retrodeck/retrodeck.cfg" # RetroDECK config file path diff --git a/net.retrodeck.retrodeck.appdata.xml b/net.retrodeck.retrodeck.appdata.xml index 1856f694..9818c1e9 100644 --- a/net.retrodeck.retrodeck.appdata.xml +++ b/net.retrodeck.retrodeck.appdata.xml @@ -83,6 +83,7 @@
  • FRAMEWORK: fixed an issue where second level subfolders inside retrodeck folder were not correctly placed, this caused errors on ES-DE as themes and downloaded_media folder could not be found after a reset or first install
  • CEMU: now audio should work correctly when docking and undocking the Steam Deck
  • CEMU: added microphone blow input to R3
    • +
  • DUCKSTATION: now out of the rolling release
  • INFRASTRUCTURE: we now have a self-hosted Gitea repo in case fo disaster recovery

    • Known issues:

    diff --git a/net.retrodeck.retrodeck.yml b/net.retrodeck.retrodeck.yml index 9a5aa49c..5397aca3 100644 --- a/net.retrodeck.retrodeck.yml +++ b/net.retrodeck.retrodeck.yml @@ -167,8 +167,8 @@ modules: - cp rclone ${FLATPAK_DEST}/bin/ sources: - type: archive - url: https://github.com/rclone/rclone/releases/download/v1.61.1/rclone-v1.61.1-linux-amd64.zip - sha256: 6d6455e1cb69eb0615a52cc046a296395e44d50c0f32627ba8590c677ddf50a9 + url: https://github.com/rclone/rclone/releases/download/v1.67.0/rclone-v1.67.0-linux-amd64.zip + sha256: 07c23d21a94d70113d949253478e13261c54d14d72023bb14d96a8da5f3e7722 # Source: https://github.com/flathub/com.valvesoftware.Steam.Utility.steamtinkerlaunch/blob/129c9192f874405d21612d95f9749dc2bcaf8cea/modules/rsync.yml#L5 - name: rsync @@ -310,7 +310,7 @@ modules: # dependency of: CEMU, RPCS3 - rd-submodules/shared-modules/glew/glew.json - # dependency of: XEMU, MelonDS + # dependency of: XEMU, MelonDS (4.7.0) - name: libslirp buildsystem: meson cleanup: @@ -343,16 +343,6 @@ modules: stable-only: true url-template: https://github.com/libusb/libusb/releases/download/v$version/libusb-$version.tar.bz2 - # needed for chdman - # - name: libFLAC - # buildsystem: cmake-ninja - # config-opts: - # - -Dbuildtype=release - # sources: - # - type: archive - # url: https://downloads.xiph.org/releases/flac/flac-1.4.3.tar.xz - # sha256: 6c58e69cd22348f441b861092b825e591d0b822e106de6eb0ee4d05d27205b70 - # ES-DE - START # https://gitlab.com/es-de/emulationstation-de @@ -690,8 +680,8 @@ modules: - ln -s "${FLATPAK_DEST}/pcsx2-qt/usr/bin/pcsx2-qt" "${FLATPAK_DEST}/bin/pcsx2-qt" sources: - type: file - url: https://github.com/PCSX2/pcsx2/releases/download/v1.7.5900/pcsx2-v1.7.5900-linux-appimage-x64-Qt.AppImage - sha256: a4012fbdd175536cac99ef7f0b5fb65aa8303b4c5b8cc31918a05341d61bbb26 + url: https://github.com/PCSX2/pcsx2/releases/download/v1.7.5967/pcsx2-v1.7.5967-linux-appimage-x64-Qt.AppImage + sha256: 7af0257204fac749efd1d6a309e287378f56bae54446902a5f34aafe17ce9915 # PCSX2 - END @@ -718,14 +708,15 @@ modules: - /share/man post-install: - install -D -t ${FLATPAK_DEST}/bin/ dolphin-emu-wrapper - - sed -i -e 's/"2048"/"512"/g' /app/share/icons/hicolor/scalable/apps/dolphin-emu.svg + - sed -i -e 's/viewBox="0 0 1024.02 571.29"/viewBox="0 -285.645 1024 1024" width="2048" + height="2048"/g' /app/share/icons/hicolor/scalable/apps/dolphin-emu.svg sources: - type: git # Sometimes Dolphin or its submodules clone are failing in https so it must done in ssh # fatal: remote transport reported error # url: ssh://git@github.com/dolphin-emu/dolphin.git url: https://github.com/dolphin-emu/dolphin.git - commit: a9544510468740b77cf06ef28daaa65fe247fd32 + commit: b92e354389bb7c0bd114a8631b8af110d3cb3a14 x-checker-data: type: json url: https://dolphin-emu.org/update/latest/beta @@ -959,8 +950,8 @@ modules: - ln -s "${FLATPAK_DEST}/duckstation/usr/bin/duckstation-qt" "${FLATPAK_DEST}/bin/duckstation-qt" sources: - type: file - url: https://github.com/stenzek/duckstation/releases/download/preview/DuckStation-x64.AppImage - sha256: DUCKSTATIONSHAPLACEHOLDER + url: https://github.com/stenzek/duckstation/releases/download/v0.1-6937/DuckStation-x64.AppImage + sha256: 01afc0c7c597d510455018d9ce902e4a3c06a51d35162ed8c025c3b65ae2218b # Duckstation-AppImage - END @@ -1070,8 +1061,8 @@ modules: x-checker-data: type: git tag-pattern: ^v([\d.]+-\d+)$ - tag: v2.0-86 - commit: 1672f969bbc4a683e4a852aa2e145c1e6f9f68e6 + tag: v2.0-88 + commit: aefbb918beb8718af8f190a73018ff63bf801d95 disable-submodules: true - type: git url: https://github.com/mozilla/cubeb @@ -1237,8 +1228,8 @@ modules: ln -sv ${FLATPAK_DEST}/ryujinx/Ryujinx ${FLATPAK_DEST}/bin/Ryujinx.sh sources: - type: file - url: https://github.com/Ryujinx/release-channel-master/releases/download/1.1.1330/ryujinx-1.1.1330-linux_x64.tar.gz - sha256: 1e9158a3f55ff128e089ac3778dfb35127f2cab9d59bc52b5fe63c7b7ac2f6cb + url: https://github.com/Ryujinx/release-channel-master/releases/download/1.1.1343/ryujinx-1.1.1343-linux_x64.tar.gz + sha256: 3cd2172d2f42a0fb93a83bffe6bd8fb2d1d81fb45afc80e0068a245c8f755ac1 # Ryujinx Appimage - END diff --git a/tools/configurator.sh b/tools/configurator.sh index 573dfef1..1890d6dc 100644 --- a/tools/configurator.sh +++ b/tools/configurator.sh @@ -572,6 +572,9 @@ configurator_retrodeck_tools_dialog() { if [[ $(get_setting_value "$rd_conf" "akai_ponzu" "retrodeck" "options") == "true" ]]; then choices+=("Ponzu - Remove Citra" "Run Ponzu to remove Citra from RetroDECK. Configurations and saves will be mantained.") fi + if [[ $(rclone listremotes) =~ "RetroDECK:" ]]; then + choices+=("Cloud: Manual Sync" "Run a manual sync with the configured cloud instance. Functionality is in ALPHA.") + fi choice=$(rd_zenity --list --title="RetroDECK Configurator Utility - RetroDECK: Tools" --cancel-label="Back" \ --window-icon="/app/share/icons/hicolor/scalable/apps/net.retrodeck.retrodeck.svg" --width=1200 --height=720 \ @@ -691,13 +694,17 @@ configurator_retrodeck_tools_dialog() { configurator_online_update_setting_dialog ;; -"Ponzu - Remove Yuzu" ) - ponzu_remove "yuzu" -;; + "Ponzu - Remove Yuzu" ) + ponzu_remove "yuzu" + ;; -"Ponzu - Remove Citra" ) - ponzu_remove "citra" -;; + "Ponzu - Remove Citra" ) + ponzu_remove "citra" + ;; + + "Cloud: Manual Sync" ) + sync_cloud newer + ;; "" ) # No selection made or Back button clicked log i "Configurator: going back" @@ -1414,6 +1421,7 @@ configurator_developer_dialog() { --column="Choice" --column="Description" \ "Change Multi-user mode" "Enable or disable multi-user support" \ "Change Update Channel" "Change between normal and cooker builds" \ + "Configure Cloud Sync" "Enable, disable, or edit cloud configuration" \ "Browse the Wiki" "Browse the RetroDECK wiki online" \ "Install RetroDECK Starter Pack" "Install the optional RetroDECK starter pack" ) @@ -1429,6 +1437,11 @@ configurator_developer_dialog() { configurator_online_update_channel_dialog ;; + "Configure Cloud Sync" ) + log i "Configurator: opening \"$choice\" menu" + configurator_cloud_sync_dialog + ;; + "Browse the Wiki" ) log i "Configurator: opening \"$choice\" menu" xdg-open "https://github.com/XargonWan/RetroDECK/wiki" @@ -1506,6 +1519,68 @@ configurator_online_update_channel_dialog() { fi } +configurator_cloud_sync_dialog() { + if [[ $(rclone listremotes) =~ "RetroDECK:" ]]; then + rd_zenity --question \ + --no-wrap --window-icon="/app/share/icons/hicolor/scalable/apps/net.retrodeck.retrodeck.svg" \ + --title "RetroDECK Configurator - RetroDECK Cloud Sync" \ + --text="You currently have cloud sync set up. Would you like to disable cloud sync?\n\nDisabling cloud syncing and then setting it up again has NOT been tested. Please backup your data.\n\n(We recognise the irony of this statement.)" + + if [ $? == 0 ] # User clicked "Yes" + then + unset_cloud + else # User clicked "Cancel" + configurator_developer_dialog + fi + else + rd_zenity --question \ + --no-wrap --window-icon="/app/share/icons/hicolor/scalable/apps/net.retrodeck.retrodeck.svg" \ + --title "RetroDECK Configurator - RetroDECK Change Update Branch" \ + --text="No cloud sync config was detected. Would you like to set it up?\n\nThis functionality is in ALPHA, and RetroDECK is not responsible for any lost data. You have been warned." + + if [ $? == 0 ] # User clicked "Yes" + then + configurator_cloud_provider_dialog + else # User clicked "Cancel" + configurator_developer_dialog + fi + fi +} + +configurator_cloud_provider_dialog() { + choice=$(rd_zenity --list --title="RetroDECK Cloud Cloud Sync - Cloud Provider" --cancel-label="Back" \ + --window-icon="/app/share/icons/hicolor/scalable/apps/net.retrodeck.retrodeck.svg" --width=1200 --height=720 \ + --column="Choice" \ + "Box" \ + "Dropbox" \ + "Google Drive" \ + "OneDrive" ) + + case $choice in + + "Box" ) + set_cloud box newer + ;; + + "Dropbox" ) + set_cloud dropbox newer + ;; + + "Google Drive" ) + set_cloud drive newer + ;; + + "OneDrive" ) + set_cloud onedrive newer + ;; + + "" ) # No selection made or Back button clicked + log i "Configurator: going back" + configurator_developer_dialog + ;; + esac +} + # START THE CONFIGURATOR configurator_welcome_dialog