From 3fd347d1f6e97761fb97b02dfbe53746c7edd39e Mon Sep 17 00:00:00 2001 From: nathan Date: Thu, 14 May 2026 12:45:43 -0700 Subject: [PATCH] Implement Linastorage project backups --- AGRARIAN_DEVELOPMENT_ROADMAP.md | 14 +- Docs/Ops/AgrarianProjectBackupRunbook.md | 99 ++++++++ .../systemd/agrarian-project-backup.service | 11 + .../systemd/agrarian-project-backup.timer | 11 + Scripts/agrarian_project_backup.sh | 220 ++++++++++++++++++ 5 files changed, 347 insertions(+), 8 deletions(-) create mode 100644 Docs/Ops/AgrarianProjectBackupRunbook.md create mode 100644 Operations/systemd/agrarian-project-backup.service create mode 100644 Operations/systemd/agrarian-project-backup.timer create mode 100755 Scripts/agrarian_project_backup.sh diff --git a/AGRARIAN_DEVELOPMENT_ROADMAP.md b/AGRARIAN_DEVELOPMENT_ROADMAP.md index a5bd718..891ba94 100644 --- a/AGRARIAN_DEVELOPMENT_ROADMAP.md +++ b/AGRARIAN_DEVELOPMENT_ROADMAP.md @@ -168,13 +168,12 @@ Major version 0.01 work already completed: Remaining version 0.01 cleanup before moving deeper into new gameplay: - [ ] Decide whether to keep current Unreal template variants or remove unused starter variants. -- [!] Create protected `main` branch. Blocked while the repo remains private on the current GitHub plan; GitHub API reports this requires GitHub Pro or making the repository public. - [x] Decide whether to create/use a long-lived `dev` branch. Decision: do not use one yet; use `main` plus short-lived task branches until team size or release channels require a staging branch. - [x] Finish branch naming conventions. - [x] Finish commit message conventions. - [x] Define GitHub/LFS free-tier storage guardrails. - [x] Define backup expectations for NAS and repository. -- [ ] Implement Linastorage incremental project backup job. +- [x] Implement Linastorage incremental project backup job. - [ ] Implement quiesced VM backup job for Windows-Builder and Ubuntu-Codex. - [ ] Create repeatable dedicated server build instructions. - [~] Finish required plugin documentation. @@ -196,12 +195,11 @@ Goal: Prepare the project so all future development is controlled, recoverable, - [x] Create `.gitignore` for Unreal Engine. - [x] Ensure `Binaries/`, `Intermediate/`, `Saved/`, and `DerivedDataCache/` are excluded unless intentionally needed. - [x] Commit clean baseline project. -- [!] Create protected `main` branch. Blocked while the repo remains private on the current GitHub plan; GitHub API reports this requires GitHub Pro or making the repository public. - [x] Create `dev` branch if we want staging before main. Decision: do not create a long-lived `dev` branch yet. - [x] Define branch naming conventions. - [x] Define commit message conventions. - [x] Define GitHub/LFS free-tier storage guardrails. -- [ ] Define backup expectations for NAS and repo. +- [x] Define backup expectations for NAS and repo. - [x] Confirm this roadmap file is committed or otherwise backed up. ## 0.2 Engine And Tooling Decisions @@ -314,7 +312,7 @@ Current tooling decisions: - [ ] Stabilize Windows-Builder network/RDP behavior under GPU passthrough. - [x] Decide and document VM snapshot cadence before major engine/tool changes. - [x] Define Unraid share backup policy. -- [ ] Implement Linastorage incremental project backup job with deleted-file retention. +- [x] Implement Linastorage incremental project backup job with deleted-file retention. - [ ] Implement quiesced VM backup job for Windows-Builder and Ubuntu-Codex. - [ ] Add recurring restore-test log for project and VM backups. - [ ] Define GitHub branch protection and review rules. @@ -1266,6 +1264,7 @@ These tracks run across all phases and must not be left as afterthoughts. - [ ] Add one-command Linux dedicated server build wrapper. - [x] Define investor-demo build trigger at version milestone completion. - [x] Add smoke-test command for build artifacts. +- [ ] Enable protected `main` branch once revenue or a paid GitHub plan justifies the cost. ## L. Storefront Development Distribution @@ -1410,13 +1409,12 @@ Current version: `0.01 Foundation Baseline` Earliest incomplete foundation items: -- [!] Create protected `main` branch. Blocked while the repo remains private on the current GitHub plan; GitHub API reports this requires GitHub Pro or making the repository public. - [x] Decide whether to create/use a long-lived `dev` branch. Decision: no long-lived `dev` branch yet. - [x] Finish branch naming conventions. - [x] Finish commit message conventions. - [x] Define GitHub/LFS free-tier storage guardrails. - [x] Define backup expectations for NAS and repository. -- [ ] Implement Linastorage incremental project backup job. +- [x] Implement Linastorage incremental project backup job. - [ ] Implement quiesced VM backup job for Windows-Builder and Ubuntu-Codex. - [ ] Create repeatable dedicated server build instructions. - [~] Finish required plugin documentation. @@ -1435,4 +1433,4 @@ Earliest incomplete foundation items: Immediate next item: -- [ ] Implement Linastorage incremental project backup job. +- [ ] Implement quiesced VM backup job for Windows-Builder and Ubuntu-Codex. diff --git a/Docs/Ops/AgrarianProjectBackupRunbook.md b/Docs/Ops/AgrarianProjectBackupRunbook.md new file mode 100644 index 0000000..e58b3b8 --- /dev/null +++ b/Docs/Ops/AgrarianProjectBackupRunbook.md @@ -0,0 +1,99 @@ +# Agrarian Project Backup Runbook + +Agrarian project backups run from the Codex/dev host and write encrypted, +deduplicated restic snapshots to Linastorage. + +## Paths + +- Source: `/mnt/projects/AgrarianGameBulid` +- NAS mount: `/mnt/backups/linastorage` +- Backup root: `/mnt/backups/linastorage/backups/agrarian-game/project` +- Restic repository: `/mnt/backups/linastorage/backups/agrarian-game/project/restic-repository` +- State files: `/mnt/backups/linastorage/backups/agrarian-game/project/state` +- Password file: `/root/.backup-secrets/agrarian-project-restic.password` +- Script: `/usr/local/sbin/agrarian-project-backup` +- Service: `agrarian-project-backup.service` +- Timer: `agrarian-project-backup.timer` + +## Schedule + +The timer runs every two hours with a small randomized delay. The script exits +without creating a snapshot when there are no changes compared to the latest +snapshot. + +The first implementation attempt used `rsync --link-dest`, but Linastorage over +SMB did not support the hard-link operations required for that model. The active +job uses restic instead. Restic stores encrypted, deduplicated chunks as normal +files, which is a better fit for this SMB target. + +Manual snapshots can be created before risky work: + +```bash +sudo /usr/local/sbin/agrarian-project-backup --manual +``` + +Force a scheduled-style snapshot even if no changes are detected: + +```bash +sudo /usr/local/sbin/agrarian-project-backup --force +``` + +Check whether a snapshot would be created: + +```bash +sudo /usr/local/sbin/agrarian-project-backup --dry-run +``` + +## Included + +- Unreal project files. +- Git metadata, including unpushed local history when practical. +- Local source data under `Data/`, including raw terrain source inputs excluded + from GitHub. +- Scripts, docs, configs, plugins, content, and project metadata. + +## Excluded + +- `DerivedDataCache/` +- `Intermediate/` +- `Saved/` +- `Binaries/` +- `Builds/` +- transient Git LFS temp files +- local Visual Studio temp files + +## Retention + +The script uses restic retention: + +- All snapshots within 7 days. +- Daily snapshots for 30 days. +- Weekly snapshots for 12 weeks. +- Monthly snapshots for 12 months. + +The script records a source signature and skips scheduled runs when no project +changes are detected. + +## Restore Test + +Restore one file to a temporary path: + +```bash +mkdir -p /tmp/agrarian-project-restore-test +sudo sh -c 'RESTIC_PASSWORD_FILE=/root/.backup-secrets/agrarian-project-restic.password \ + restic -r /mnt/backups/linastorage/backups/agrarian-game/project/restic-repository \ + dump latest /mnt/projects/AgrarianGameBulid/AGRARIAN_DEVELOPMENT_ROADMAP.md \ + > /tmp/agrarian-project-restore-test/AGRARIAN_DEVELOPMENT_ROADMAP.md' +diff -q /mnt/projects/AgrarianGameBulid/AGRARIAN_DEVELOPMENT_ROADMAP.md \ + /tmp/agrarian-project-restore-test/AGRARIAN_DEVELOPMENT_ROADMAP.md +``` + +List snapshots: + +```bash +sudo RESTIC_PASSWORD_FILE=/root/.backup-secrets/agrarian-project-restic.password \ + restic -r /mnt/backups/linastorage/backups/agrarian-game/project/restic-repository \ + snapshots --tag agrarian-game --tag project +``` + +Record restore tests in the handoff log with the snapshot name and result. diff --git a/Operations/systemd/agrarian-project-backup.service b/Operations/systemd/agrarian-project-backup.service new file mode 100644 index 0000000..0d9a517 --- /dev/null +++ b/Operations/systemd/agrarian-project-backup.service @@ -0,0 +1,11 @@ +[Unit] +Description=Agrarian project incremental backup to Linastorage +Wants=network-online.target +After=network-online.target mnt-backups-linastorage.automount + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/agrarian-project-backup +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 diff --git a/Operations/systemd/agrarian-project-backup.timer b/Operations/systemd/agrarian-project-backup.timer new file mode 100644 index 0000000..d8647a8 --- /dev/null +++ b/Operations/systemd/agrarian-project-backup.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run Agrarian project backups every two hours when changes exist + +[Timer] +OnCalendar=*-*-* 00/2:13:00 +Persistent=true +RandomizedDelaySec=10m +Unit=agrarian-project-backup.service + +[Install] +WantedBy=timers.target diff --git a/Scripts/agrarian_project_backup.sh b/Scripts/agrarian_project_backup.sh new file mode 100755 index 0000000..cc3f054 --- /dev/null +++ b/Scripts/agrarian_project_backup.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +umask 077 + +SOURCE_DIR="${AGRARIAN_PROJECT_SOURCE:-/mnt/projects/AgrarianGameBulid}" +MOUNT_POINT="${AGRARIAN_BACKUP_MOUNT:-/mnt/backups/linastorage}" +BACKUP_ROOT="${AGRARIAN_BACKUP_ROOT:-$MOUNT_POINT/backups/agrarian-game/project}" +RESTIC_REPOSITORY="${AGRARIAN_RESTIC_REPOSITORY:-$BACKUP_ROOT/restic-repository}" +RESTIC_PASSWORD_FILE="${AGRARIAN_RESTIC_PASSWORD_FILE:-/root/.backup-secrets/agrarian-project-restic.password}" +STATE_DIR="${AGRARIAN_BACKUP_STATE_DIR:-$BACKUP_ROOT/state}" +LOCK_FILE="${AGRARIAN_BACKUP_LOCK:-/run/lock/agrarian-project-backup.lock}" +LOG_TAG="${AGRARIAN_BACKUP_LOG_TAG:-agrarian-project-backup}" + +DRY_RUN=0 +FORCE=0 +MANUAL=0 + +EXCLUDES=( + --exclude "$SOURCE_DIR/DerivedDataCache" + --exclude "$SOURCE_DIR/Intermediate" + --exclude "$SOURCE_DIR/Saved" + --exclude "$SOURCE_DIR/Binaries" + --exclude "$SOURCE_DIR/Builds" + --exclude "$SOURCE_DIR/.git/index.lock" + --exclude "$SOURCE_DIR/.git/lfs/tmp" + --exclude "$SOURCE_DIR/.git/logs" + --exclude "$SOURCE_DIR/.vs" + --exclude "$SOURCE_DIR/AGRARIAN_BACKUP_MANIFEST.txt" + --exclude "$SOURCE_DIR/AGRARIAN_CRITICAL_SHA256SUMS" + --exclude "$SOURCE_DIR/AGRARIAN_RSYNC_LOG.txt" +) + +log() { + printf '[%s] %s\n' "$(date -Is)" "$*" +} + +die() { + log "ERROR: $*" + exit 1 +} + +usage() { + cat <<'USAGE' +Usage: agrarian_project_backup.sh [--force] [--manual] [--dry-run] + +Creates an incremental deduplicated restic backup of the Agrarian Unreal project +on Linastorage. Generated Unreal cache/build folders are excluded; Git metadata +and local source data are included. +USAGE +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --force) + FORCE=1 + ;; + --manual) + MANUAL=1 + FORCE=1 + ;; + --dry-run) + DRY_RUN=1 + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "Unknown argument: $1" + ;; + esac + shift + done +} + +restic_cmd() { + RESTIC_PASSWORD_FILE="$RESTIC_PASSWORD_FILE" \ + restic -r "$RESTIC_REPOSITORY" "$@" +} + +require_environment() { + [[ -d "$SOURCE_DIR" ]] || die "Project source directory not found: $SOURCE_DIR" + [[ -f "$SOURCE_DIR/AgrarianGame.uproject" ]] || die "Missing AgrarianGame.uproject under $SOURCE_DIR" + findmnt -rn "$MOUNT_POINT" >/dev/null || die "$MOUNT_POINT is not mounted" + mkdir -p "$BACKUP_ROOT" "$STATE_DIR" "$(dirname "$RESTIC_PASSWORD_FILE")" + [[ -w "$BACKUP_ROOT" ]] || die "$BACKUP_ROOT is not writable" + command -v restic >/dev/null || die "restic is required" +} + +ensure_password_file() { + if [[ -f "$RESTIC_PASSWORD_FILE" ]]; then + chmod 0600 "$RESTIC_PASSWORD_FILE" + return 0 + fi + + log "Creating root-owned restic password file: $RESTIC_PASSWORD_FILE" + openssl rand -base64 48 > "$RESTIC_PASSWORD_FILE" + chmod 0600 "$RESTIC_PASSWORD_FILE" +} + +ensure_repository() { + if [[ -f "$RESTIC_REPOSITORY/config" ]]; then + return 0 + fi + + log "Initializing restic repository: $RESTIC_REPOSITORY" + mkdir -p "$RESTIC_REPOSITORY" + restic_cmd init +} + +source_signature() { + local tmp + tmp="$(mktemp)" + + { + git -C "$SOURCE_DIR" rev-parse HEAD 2>/dev/null || true + git -C "$SOURCE_DIR" status --porcelain=v1 2>/dev/null || true + find "$SOURCE_DIR" \ + \( -path "$SOURCE_DIR/DerivedDataCache" \ + -o -path "$SOURCE_DIR/Intermediate" \ + -o -path "$SOURCE_DIR/Saved" \ + -o -path "$SOURCE_DIR/Binaries" \ + -o -path "$SOURCE_DIR/Builds" \ + -o -path "$SOURCE_DIR/.git/lfs/tmp" \ + -o -path "$SOURCE_DIR/.git/logs" \ + -o -path "$SOURCE_DIR/.vs" \) -prune \ + -o -type f -printf '%P\t%s\t%T@\n' 2>/dev/null \ + | sort + } > "$tmp" + + sha256sum "$tmp" | awk '{print $1}' + rm -f -- "$tmp" +} + +write_manifest() { + local signature="$1" + local manifest="$STATE_DIR/LATEST_MANIFEST.txt" + + { + echo "backup_timestamp=$(date -Is)" + echo "host=$(hostname -f 2>/dev/null || hostname)" + echo "source=$SOURCE_DIR" + echo "mount=$MOUNT_POINT" + echo "repository=$RESTIC_REPOSITORY" + echo "signature=$signature" + echo "backup_mode=$([[ "$MANUAL" == "1" ]] && echo manual || echo scheduled)" + echo "git_head=$(git -C "$SOURCE_DIR" rev-parse HEAD 2>/dev/null || echo unavailable)" + echo + echo "[excluded]" + printf '%s\n' "${EXCLUDES[@]}" + } > "$manifest" +} + +backup_project() { + local signature="$1" + local mode_tag="scheduled" + [[ "$MANUAL" == "1" ]] && mode_tag="manual" + + log "Starting restic backup for Agrarian project" + restic_cmd backup \ + --one-file-system \ + --tag agrarian-game \ + --tag project \ + --tag "$mode_tag" \ + "${EXCLUDES[@]}" \ + "$SOURCE_DIR" + + write_manifest "$signature" + printf '%s\n' "$signature" > "$STATE_DIR/LAST_SOURCE_SIGNATURE" + + log "Applying restic retention policy" + restic_cmd forget \ + --tag agrarian-game \ + --tag project \ + --keep-within 7d \ + --keep-daily 30 \ + --keep-weekly 12 \ + --keep-monthly 12 \ + --prune + + restic_cmd snapshots --tag agrarian-game --tag project > "$STATE_DIR/LATEST_SNAPSHOTS.txt" + log "Project backup completed" + logger -t "$LOG_TAG" "restic project backup completed successfully" +} + +main() { + parse_args "$@" + + exec 9>"$LOCK_FILE" + if ! flock -n 9; then + log "Another Agrarian project backup is already running; skipping this trigger" + logger -t "$LOG_TAG" "another project backup is already running; skipped" + exit 0 + fi + + require_environment + ensure_password_file + ensure_repository + + local signature previous_signature + signature="$(source_signature)" + previous_signature="$(cat "$STATE_DIR/LAST_SOURCE_SIGNATURE" 2>/dev/null || true)" + + if [[ "$FORCE" != "1" && "$signature" == "$previous_signature" ]]; then + log "No project changes detected since latest restic backup; skipping" + logger -t "$LOG_TAG" "no project changes detected; skipped" + exit 0 + fi + + if [[ "$DRY_RUN" == "1" ]]; then + log "Dry run: project changes detected; restic snapshot would be created" + exit 0 + fi + + backup_project "$signature" +} + +main "$@"