diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 0000000..c2655de --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,99 @@ +name: Deploy + +on: + push: + branches: [main] + tags: ["v*"] + +# Required repository secrets: +# REGISTRY registry hostname, e.g. registry.librete.ch +# REGISTRY_USER robot account +# REGISTRY_PASS robot token +# DEPLOY_HOST deployment SSH target, e.g. root@librenot.es +# DEPLOY_KEY private SSH key (PEM, no passphrase) +# DEPLOY_PATH remote directory containing the compose stack +# HEALTH_URL public URL to verify post-deploy, e.g. +# https://librenot.es/healthz +# +# Tag pushes deploy the tag (vX.Y.Z); main-branch pushes deploy +# the rolling :main image. Set image to immutable tag so rollback +# is just `docker compose -f ... up -d` with the previous tag. + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 15 + if: ${{ vars.DEPLOY_ENABLED == 'true' }} + steps: + - uses: actions/checkout@v4 + + - uses: docker/setup-buildx-action@v3 + + - name: Log in to registry + uses: docker/login-action@v3 + with: + registry: ${{ secrets.REGISTRY }} + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASS }} + + - name: Compute tags + id: tags + run: | + BASE="${{ secrets.REGISTRY }}/librenotes" + if [[ "${GITHUB_REF}" == refs/tags/* ]]; then + TAG="${GITHUB_REF##refs/tags/}" + echo "tags=${BASE}:${TAG},${BASE}:latest" >> "$GITHUB_OUTPUT" + echo "version=${TAG}" >> "$GITHUB_OUTPUT" + else + echo "tags=${BASE}:main,${BASE}:${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + echo "version=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + fi + + - uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: ${{ steps.tags.outputs.tags }} + build-args: | + VERSION=${{ steps.tags.outputs.version }} + BUILDTIME=${{ github.event.head_commit.timestamp }} + + deploy: + runs-on: ubuntu-latest + needs: build + timeout-minutes: 10 + if: ${{ vars.DEPLOY_ENABLED == 'true' }} + steps: + - name: Configure SSH + run: | + mkdir -p ~/.ssh + echo "${{ secrets.DEPLOY_KEY }}" > ~/.ssh/id_deploy + chmod 600 ~/.ssh/id_deploy + ssh-keyscan -H "${{ secrets.DEPLOY_HOST#*@ }}" >> ~/.ssh/known_hosts || true + + - name: Pull and restart on deploy host + env: + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }} + run: | + ssh -i ~/.ssh/id_deploy "$DEPLOY_HOST" \ + "cd $DEPLOY_PATH && \ + docker compose -f docker-compose.yml -f docker-compose.prod.yml pull && \ + docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d --remove-orphans" + + - name: Verify health + env: + HEALTH_URL: ${{ secrets.HEALTH_URL }} + run: | + # Give the new container ~30s to come up, then poll for + # a 200 from /healthz. Failure aborts the workflow which + # is the alert. + for i in $(seq 1 12); do + if curl -fsS "$HEALTH_URL" >/dev/null; then + echo "deploy verified" + exit 0 + fi + sleep 5 + done + echo "deploy verification failed" + exit 1 diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..996a4a3 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,117 @@ +# Operations + +This document covers deployment, backups, and rollback for the +librenot.es production environment. For developer / contributor +docs see the top-level [README](../README.md). + +## Deployment + +The repository ships a CI workflow at `.gitea/workflows/deploy.yml` +that builds and pushes a Docker image on every push to `main` and +on every `vX.Y.Z` tag, then SSHes to the deployment host and runs +`docker compose pull && up -d`. + +The workflow is gated on the repository variable +`DEPLOY_ENABLED=true`. Deployment is opt-in; flipping the variable +disables the workflow without removing the file. + +### Required secrets + +The workflow expects these secrets and variables on the repo: + +| Name | Type | Purpose | +| ---- | ---- | ------- | +| `REGISTRY` | secret | hostname of the OCI registry | +| `REGISTRY_USER` | secret | robot account | +| `REGISTRY_PASS` | secret | robot token | +| `DEPLOY_HOST` | secret | `user@host` SSH target | +| `DEPLOY_KEY` | secret | passphrase-less private key | +| `DEPLOY_PATH` | secret | absolute path on host with `docker-compose.*.yml` | +| `HEALTH_URL` | secret | e.g. `https://librenot.es/healthz` | +| `DEPLOY_ENABLED` | variable | `true` to enable the workflow | + +### Production compose stack + +On the deployment host, place `docker-compose.yml` and +`docker-compose.prod.yml` from this repo at `$DEPLOY_PATH`, +together with an `.env` file containing the runtime configuration +(JWT secret, SMTP credentials, public base URL, image tag). + +Bring it up with: + +```sh +docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d +``` + +### Rollback + +Image tags are immutable per commit / version. To roll back, set +`LIBRENOTES_IMAGE` to the previous tag in `.env` and run the same +`up -d` command. The deployment workflow does not auto-rollback +on health-check failure — failed health alerts the operator via +the workflow itself, who can then redeploy the prior tag manually. + +```sh +# Example: roll back to v0.1.2 +sed -i 's/^LIBRENOTES_IMAGE=.*/LIBRENOTES_IMAGE=registry.librete.ch\/librenotes:v0.1.2/' .env +docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d +``` + +## Backups + +`scripts/backup.sh` is a self-contained backup driver suitable for +running from cron, a systemd timer, or the supplied +`scripts/librenotes-backup.{service,timer}` units. + +Required tools on the backup host: `sqlite3`, `tar`, `gzip`, and +optionally `rclone` for off-site copy. On Debian/Ubuntu: +`apt install sqlite3 rclone`. + +### What is backed up + +- The SQLite database (`$LIBRENOTES_DB`) via the SQLite + `.backup` command, which produces a consistent online snapshot + without needing to stop the application. +- The per-tenant note directory (`$LIBRENOTES_DATA_DIR`) as a + gzipped tar. +- An `info.txt` in each archive recording the backup timestamp, + hostname, and version. + +### Off-site copy + +Set `BACKUP_REMOTE` to an `rclone` destination (e.g. +`s3:librenotes-backups`). When set, the script invokes +`rclone copy` to upload the archive after creation. Without it, +backups stay on the local disk only. + +### Retention + +`scripts/backup-prune.sh` enforces the policy "keep 30 daily ++ 12 monthly". Run it from the same timer as the backup. Files +keep the form `librenotes-YYYYMMDD-HHMMSS.tar.gz` so the prune +script can sort and select by name alone. + +### Restore test + +`scripts/backup-restore-test.sh` picks the most recent archive, +extracts it into a scratch directory, runs `sqlite3 .schema` on +the database to confirm it is readable, and verifies that the +note tar lists at least one entry. It is wired into a separate +weekly timer so a silent backup-corruption regression cannot hide +indefinitely. + +### Systemd + +Drop `scripts/librenotes-backup.{service,timer}` into +`/etc/systemd/system/`, then: + +```sh +systemctl daemon-reload +systemctl enable --now librenotes-backup.timer +``` + +Confirm with: + +```sh +systemctl list-timers librenotes-backup.timer +``` diff --git a/scripts/backup-prune.sh b/scripts/backup-prune.sh new file mode 100755 index 0000000..0c9a222 --- /dev/null +++ b/scripts/backup-prune.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# backup-prune.sh — enforce retention "keep 30 daily + 12 monthly". +# +# Walks $BACKUP_DIR for librenotes-YYYYMMDD-HHMMSS.tar.gz and +# deletes archives outside the retention policy. +# - keep the most recent N daily archives (default 30) +# - additionally keep the most recent archive of each of the +# last M distinct months (default 12) +# Anything else is removed. + +set -euo pipefail + +BACKUP_DIR="${BACKUP_DIR:-/var/backups/librenotes}" +DAILY_KEEP="${DAILY_KEEP:-30}" +MONTHLY_KEEP="${MONTHLY_KEEP:-12}" + +if [ ! -d "$BACKUP_DIR" ]; then + echo "backup dir $BACKUP_DIR does not exist; nothing to prune" + exit 0 +fi + +cd "$BACKUP_DIR" + +# Sort archives by name (date is in the filename, lexicographic +# order == chronological order). +mapfile -t archives < <(ls -1 librenotes-*.tar.gz 2>/dev/null | sort) + +declare -A keep +# Keep the newest DAILY_KEEP outright. +for a in "${archives[@]: -$DAILY_KEEP}"; do + keep["$a"]=1 +done + +# Walk archives newest-first, recording one per month until we +# have MONTHLY_KEEP distinct months. +declare -A month_seen +months_kept=0 +for ((i=${#archives[@]}-1; i>=0; i--)); do + a="${archives[$i]}" + # filename: librenotes-YYYYMMDD-HHMMSS.tar.gz -> YYYYMM + ym="${a:11:6}" + if [ -z "${month_seen[$ym]:-}" ] && [ "$months_kept" -lt "$MONTHLY_KEEP" ]; then + keep["$a"]=1 + month_seen[$ym]=1 + months_kept=$((months_kept + 1)) + fi +done + +removed=0 +for a in "${archives[@]}"; do + if [ -z "${keep[$a]:-}" ]; then + rm -f "$a" + removed=$((removed + 1)) + fi +done + +echo "kept ${#keep[@]} archives, removed $removed" diff --git a/scripts/backup-restore-test.sh b/scripts/backup-restore-test.sh new file mode 100755 index 0000000..b460ec8 --- /dev/null +++ b/scripts/backup-restore-test.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# backup-restore-test.sh — verify the most recent backup is usable. +# +# Failure of this script is an alert. It does NOT touch the live +# database or data dir. +# +# Steps: +# 1. Pick the newest archive in $BACKUP_DIR. +# 2. Extract into a scratch tmpdir. +# 3. Run sqlite3 ".schema" against the snapshotted DB; non-empty +# output means the file is readable as SQLite. +# 4. Run tar -tzf on notes.tar.gz; we expect at least one entry. +# 5. Print the info.txt header so logs include backup metadata. + +set -euo pipefail + +BACKUP_DIR="${BACKUP_DIR:-/var/backups/librenotes}" + +archive="$(ls -1t "$BACKUP_DIR"/librenotes-*.tar.gz 2>/dev/null | head -n 1 || true)" +if [ -z "$archive" ]; then + echo "no archives in $BACKUP_DIR" >&2 + exit 1 +fi + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +tar -C "$work" -xzf "$archive" + +if [ ! -f "$work/librenotes.db" ]; then + echo "archive missing librenotes.db: $archive" >&2 + exit 1 +fi +schema="$(sqlite3 "$work/librenotes.db" ".schema" || true)" +if [ -z "$schema" ]; then + echo "sqlite .schema returned empty for $archive" >&2 + exit 1 +fi + +if [ ! -f "$work/notes.tar.gz" ]; then + echo "archive missing notes.tar.gz: $archive" >&2 + exit 1 +fi +n="$(tar -tzf "$work/notes.tar.gz" | wc -l)" +if [ "$n" -lt 1 ]; then + echo "notes.tar.gz contains no entries: $archive" >&2 + exit 1 +fi + +echo "verified $archive" +[ -f "$work/info.txt" ] && cat "$work/info.txt" diff --git a/scripts/backup.sh b/scripts/backup.sh new file mode 100755 index 0000000..09bd70e --- /dev/null +++ b/scripts/backup.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# backup.sh — daily backup of librenotes state. +# +# Produces $BACKUP_DIR/librenotes-YYYYMMDD-HHMMSS.tar.gz containing: +# - librenotes.db (consistent SQLite .backup snapshot) +# - notes.tar.gz (per-tenant note files) +# - info.txt (timestamp, hostname, version) +# +# Required env: +# LIBRENOTES_DB path to the SQLite database +# LIBRENOTES_DATA_DIR path to the per-tenant note directory +# +# Optional env: +# BACKUP_DIR where to write archives (default /var/backups/librenotes) +# BACKUP_REMOTE rclone target for off-site copy (e.g. s3:bucket/path) +# BACKUP_VERSION version string written into info.txt +# +# The script is intentionally a single self-contained file so it +# can run on a minimal host with only sqlite3, tar, gzip, and +# (optionally) rclone. + +set -euo pipefail + +: "${LIBRENOTES_DB:?LIBRENOTES_DB is required}" +: "${LIBRENOTES_DATA_DIR:?LIBRENOTES_DATA_DIR is required}" +BACKUP_DIR="${BACKUP_DIR:-/var/backups/librenotes}" +BACKUP_VERSION="${BACKUP_VERSION:-unknown}" + +mkdir -p "$BACKUP_DIR" + +ts="$(date -u +%Y%m%d-%H%M%S)" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +# Online SQLite snapshot. .backup is atomic from the application's +# perspective even while writes are happening. +sqlite3 "$LIBRENOTES_DB" ".backup '$work/librenotes.db'" + +# Notes archive. Use --warning=no-file-changed because per-user +# files may be touched concurrently; we still get a consistent +# point-in-time view per file thanks to tar's read semantics. +tar --warning=no-file-changed -C "$LIBRENOTES_DATA_DIR" -czf "$work/notes.tar.gz" . + +cat > "$work/info.txt" </dev/null || stat -f%z "$work/librenotes.db") +notes_size:$(stat -c%s "$work/notes.tar.gz" 2>/dev/null || stat -f%z "$work/notes.tar.gz") +EOF + +archive="$BACKUP_DIR/librenotes-$ts.tar.gz" +tar -C "$work" -czf "$archive" librenotes.db notes.tar.gz info.txt + +echo "wrote $archive ($(stat -c%s "$archive" 2>/dev/null || stat -f%z "$archive") bytes)" + +if [ -n "${BACKUP_REMOTE:-}" ]; then + rclone copy "$archive" "$BACKUP_REMOTE" --quiet + echo "uploaded to $BACKUP_REMOTE" +fi diff --git a/scripts/librenotes-backup-verify.service b/scripts/librenotes-backup-verify.service new file mode 100644 index 0000000..cfbc0e8 --- /dev/null +++ b/scripts/librenotes-backup-verify.service @@ -0,0 +1,12 @@ +[Unit] +Description=librenotes weekly backup-restore test + +[Service] +Type=oneshot +EnvironmentFile=-/etc/librenotes/backup.env +ExecStart=/usr/local/bin/librenotes-backup-restore-test.sh +TimeoutStartSec=10min +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +NoNewPrivileges=true diff --git a/scripts/librenotes-backup-verify.timer b/scripts/librenotes-backup-verify.timer new file mode 100644 index 0000000..ab837f0 --- /dev/null +++ b/scripts/librenotes-backup-verify.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Weekly verification of latest librenotes backup + +[Timer] +OnCalendar=Mon *-*-* 04:00:00 UTC +Persistent=true +Unit=librenotes-backup-verify.service + +[Install] +WantedBy=timers.target diff --git a/scripts/librenotes-backup.service b/scripts/librenotes-backup.service new file mode 100644 index 0000000..e212075 --- /dev/null +++ b/scripts/librenotes-backup.service @@ -0,0 +1,19 @@ +[Unit] +Description=librenotes daily backup +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +EnvironmentFile=-/etc/librenotes/backup.env +ExecStart=/usr/local/bin/librenotes-backup.sh +ExecStartPost=/usr/local/bin/librenotes-backup-prune.sh +SuccessExitStatus=0 +TimeoutStartSec=30min +# Backup driver only reads from the live data dir; never writes +# back. Sandbox accordingly. +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/var/backups/librenotes +NoNewPrivileges=true diff --git a/scripts/librenotes-backup.timer b/scripts/librenotes-backup.timer new file mode 100644 index 0000000..2150a38 --- /dev/null +++ b/scripts/librenotes-backup.timer @@ -0,0 +1,13 @@ +[Unit] +Description=Run librenotes backup nightly + +[Timer] +# 03:17 UTC nightly with up to 5min jitter so multiple machines on +# the same schedule don't all hit the off-site target at once. +OnCalendar=*-*-* 03:17:00 UTC +RandomizedDelaySec=5min +Persistent=true +Unit=librenotes-backup.service + +[Install] +WantedBy=timers.target