docs/operations.md

@@ -87,9 +87,9 @@ auto-mount.
 running from cron, a systemd timer, or the supplied
 `scripts/librenotes-backup.{service,timer}` units.
 
-Required tools on the backup host: `sqlite3`, `tar`, `gzip`, and
-optionally `rclone` for off-site copy. On Debian/Ubuntu:
-`apt install sqlite3 rclone`.
+Required tools on the backup host: `sqlite3`, `tar`, `gzip`,
+`rsync`, and optionally `rclone` for object-store off-site copy.
+On Debian/Ubuntu: `apt install sqlite3 rsync rclone`.
 
 ### What is backed up
 
@@ -103,10 +103,25 @@ optionally `rclone` for off-site copy. On Debian/Ubuntu:
 
 ### Off-site copy
 
-Set `BACKUP_REMOTE` to an `rclone` destination (e.g.
-`s3:librenotes-backups`). When set, the script invokes
-`rclone copy` to upload the archive after creation. Without it,
-backups stay on the local disk only.
+The script supports two off-site transports; either or both can be
+configured. Without either, backups stay on local disk only.
+
+- `BACKUP_REMOTE` — `rclone` destination (e.g.
+  `s3:librenotes-backups`, `b2:librenotes`). Uses `rclone copy` to
+  upload each new archive. Best for object-store targets.
+- `BACKUP_REMOTE_RSYNC` — `rsync` destination (e.g.
+  `backup@rsync.net:librenotes/`). Each run rsyncs into
+  `<root>/YYYY-MM-DD/` and passes
+  `--link-dest=../<previous-day>/`, so unchanged archives become
+  hard links — daily snapshots cost (almost) nothing extra. Set
+  `BACKUP_REMOTE_SSH_KEY` to the path of a passphrase-less private
+  key if the host's default identity is wrong (recommended for
+  rsync.net-style restricted accounts).
+
+For netcup, the operator's choice is captured in
+[issue #41](https://git.librete.ch/public/librenotes/issues/41).
+Until a target is chosen, leave both unset and run the timer
+locally to verify archives generate cleanly.
 
 ### Retention
 
@@ -139,3 +154,63 @@ Confirm with:
 ```sh
 systemctl list-timers librenotes-backup.timer
 ```
+
+### Restore procedure
+
+A restore is a four-step process. Run it on a *spare* directory
+(e.g. `/srv/librenotes-restore-YYYYMMDD/`) — never overwrite the
+live `/srv/librenotes/data` or `/srv/librenotes/state` until the
+parallel stack passes verification.
+
+```sh
+# 1. Pick an archive (local or pulled back from the off-host target).
+ARCHIVE=/var/backups/librenotes/librenotes-20260415-031712.tar.gz
+
+# 2. Lay out a parallel stack dir.
+RESTORE_DIR=/srv/librenotes-restore
+mkdir -p "$RESTORE_DIR"/{data,state}
+chown -R 65532:65532 "$RESTORE_DIR"/{data,state}    # distroless nonroot uid
+
+# 3. Unpack the archive into a scratch dir, then move the pieces
+#    into place.
+SCRATCH=$(mktemp -d)
+tar -C "$SCRATCH" -xzf "$ARCHIVE"
+install -m 0644 -o 65532 -g 65532 "$SCRATCH/librenotes.db" \
+        "$RESTORE_DIR/state/librenotes.db"
+tar -C "$RESTORE_DIR/data" -xzf "$SCRATCH/notes.tar.gz"
+chown -R 65532:65532 "$RESTORE_DIR/data"
+rm -rf "$SCRATCH"
+
+# 4. Bring up a parallel compose stack (dedicated docker compose
+#    project name + non-conflicting LIBRENOTES_BASE_URL) and verify.
+cd "$RESTORE_DIR"
+cp /srv/librenotes/.env .env
+perl -i -pe 's|^LIBRENOTES_BASE_URL=.*|LIBRENOTES_BASE_URL=http://localhost:18080|' .env
+cp /srv/librenotes/{compose.yaml,compose.netcup.yaml} .
+# Override port + project name so the parallel stack does not
+# clash with the live one.
+docker compose -p librenotes-restore \
+  -f compose.yaml \
+  --profile=restore \
+  up -d \
+  -e LIBRENOTES_PORT=18080
+curl -fsS http://localhost:18080/healthz
+# Sign in with a known test user, click around, confirm note state.
+docker compose -p librenotes-restore down
+```
+
+Once the parallel stack proves the archive is intact, swap into
+production with a maintenance window:
+
+```sh
+docker compose -f compose.yaml -f compose.netcup.yaml down
+mv /srv/librenotes/data /srv/librenotes/data.bak-$(date +%s)
+mv /srv/librenotes/state /srv/librenotes/state.bak-$(date +%s)
+mv /srv/librenotes-restore/data /srv/librenotes/data
+mv /srv/librenotes-restore/state /srv/librenotes/state
+docker compose -f compose.yaml -f compose.netcup.yaml up -d
+curl -fsS https://ln.cloud.librete.ch/healthz
+```
+
+If anything goes wrong, the `.bak-*` directories are still there
+to restore from.

scripts/backup.sh

@@ -13,10 +13,18 @@
 # Optional env:
 #   BACKUP_DIR           where to write archives (default /var/backups/librenotes)
 #   BACKUP_REMOTE        rclone target for off-site copy (e.g. s3:bucket/path)
+#   BACKUP_REMOTE_RSYNC  rsync destination for hard-link-deduplicated
+#                        off-host copy, e.g. backup@rsync.net:librenotes/
+#                        Layout written at the target:
+#                          <root>/YYYY-MM-DD/librenotes-<ts>.tar.gz
+#                        Each run passes --link-dest=../<previous>/ so
+#                        unchanged archives cost zero extra bytes.
+#   BACKUP_REMOTE_SSH_KEY path to a private SSH key used for the rsync
+#                        leg (passed via -e "ssh -i <key>").
 #   BACKUP_VERSION       version string written into info.txt
 #
 # The script is intentionally a single self-contained file so it
-# can run on a minimal host with only sqlite3, tar, gzip, and
+# can run on a minimal host with only sqlite3, tar, gzip, rsync, and
 # (optionally) rclone.
 
 set -euo pipefail
@@ -58,3 +66,22 @@ if [ -n "${BACKUP_REMOTE:-}" ]; then
   rclone copy "$archive" "$BACKUP_REMOTE" --quiet
   echo "uploaded to $BACKUP_REMOTE"
 fi
+
+if [ -n "${BACKUP_REMOTE_RSYNC:-}" ]; then
+  # Compute today / previous-day directory names. Layout at the
+  # remote target is <root>/YYYY-MM-DD/. We pass --link-dest pointing
+  # at yesterday's dir so unchanged archives become hard links —
+  # ~free for daily snapshots that are mostly identical.
+  today="$(date -u +%Y-%m-%d)"
+  yesterday="$(date -u -d 'yesterday' +%Y-%m-%d 2>/dev/null \
+              || date -u -v-1d +%Y-%m-%d)"
+  ssh_opts=()
+  if [ -n "${BACKUP_REMOTE_SSH_KEY:-}" ]; then
+    ssh_opts=(-e "ssh -i $BACKUP_REMOTE_SSH_KEY -o StrictHostKeyChecking=accept-new")
+  fi
+  # Strip any trailing slash so we control the join.
+  remote="${BACKUP_REMOTE_RSYNC%/}"
+  rsync -a --link-dest="../$yesterday/" "${ssh_opts[@]}" \
+        "$archive" "$remote/$today/"
+  echo "rsync'd $archive -> $remote/$today/"
+fi

scripts/librenotes-backup.timer

@@ -2,9 +2,10 @@
 Description=Run librenotes backup nightly
 
 [Timer]
-# 03:17 UTC nightly with up to 5min jitter so multiple machines on
-# the same schedule don't all hit the off-site target at once.
-OnCalendar=*-*-* 03:17:00 UTC
+# 03:00 Europe/Berlin nightly with up to 5min jitter so multiple
+# machines on the same schedule don't all hit the off-site target
+# at once.
+OnCalendar=*-*-* 03:00:00 Europe/Berlin
 RandomizedDelaySec=5min
 Persistent=true
 Unit=librenotes-backup.service