From 3c5ab83de27570f0a8dea4cb2238373c5e9930f9 Mon Sep 17 00:00:00 2001 From: XTAO Deployment Bot Date: Fri, 17 Jul 2026 00:19:46 -0600 Subject: [PATCH] Add Phase 03 Nextcloud desired state --- SHA256SUMS | 29 ++-- edge-reconciler/reconcile_cloudflare_dns.py | 1 + kuma/configure_kuma_public_status.py | 1 + nextcloud/compose.yaml | 58 ++++++++ nextcloud/nextcloud-traefik.yml | 33 +++++ phase03/accept-nextcloud-functional.sh | 102 +++++++++++++++ phase03/backup-restore-nextcloud.sh | 138 ++++++++++++++++++++ phase03/deploy-nextcloud.sh | 123 +++++++++++++++++ phase03/reconcile-nextcloud-secrets.sh | 135 +++++++++++++++++++ secret-agent/secret_agent_runner.py | 9 +- 10 files changed, 617 insertions(+), 12 deletions(-) create mode 100644 nextcloud/compose.yaml create mode 100644 nextcloud/nextcloud-traefik.yml create mode 100644 phase03/accept-nextcloud-functional.sh create mode 100644 phase03/backup-restore-nextcloud.sh create mode 100644 phase03/deploy-nextcloud.sh create mode 100644 phase03/reconcile-nextcloud-secrets.sh diff --git a/SHA256SUMS b/SHA256SUMS index 076e402..c4fea3b 100644 --- a/SHA256SUMS +++ b/SHA256SUMS @@ -1,11 +1,18 @@ -0aec27176d0bb0053a592fe9e9bcc37118b27f55541a6825eed5bb5cef6a60c7 state/implementation/desired-state/README.md -9d4d0f119d29eb5622978d531e27998f97b986106c8426d6ef01dc7dfe5ba937 state/implementation/desired-state/SHA256SUMS -4020b538e0499d12195375b5146634799cd96929826ede63bee3d1fd973c518b state/implementation/desired-state/backup-controller/xtao-backup-controller -9a434c629f955a7298995fe48415cb8bba95b12594684a8bb120ee37514c3f3a state/implementation/desired-state/edge-reconciler/reconcile_cloudflare_dns.py -0b19202e23cc88d1247c0359a20bacabfbd9074c985bcf5777d994371dabb041 state/implementation/desired-state/edge/compose.yaml -99c2d0f3829437d6f6617c55672b535d0e109a512fd55218b69e43d92319a953 state/implementation/desired-state/edge/traefik-dynamic.yml -73a1906f4a950e2e7f3cc17782f64bbab0403461d8016c2223fcb751255400c9 state/implementation/desired-state/edge/traefik.yml -256dc4be0585b645d62c7765ae0579e109ee07039a65dbcf171c25debdc3b731 state/implementation/desired-state/kuma/configure_kuma_public_status.py -10172af7ee895c1d593f654d4ab8cb5195d40d15cc65d1a09fb2c210b426edd2 state/implementation/desired-state/secret-agent/install-root-unlock.sh -8f59112e44e725de60f476b6a43215a598b6f0328387437c80bfdd299964eebb state/implementation/desired-state/secret-agent/secret_agent_runner.py -5128c2db987e30d08e3d6fe1af89febb71d701955f969ca88081aafd37390337 state/implementation/desired-state/secret-agent/xtao-secret-agent +0aec27176d0bb0053a592fe9e9bcc37118b27f55541a6825eed5bb5cef6a60c7 ./README.md +7b688eaa85148703bacf6693fa4e6ac78a25303a8a367811498e0b5466efaa42 ./SHA256SUMS +4020b538e0499d12195375b5146634799cd96929826ede63bee3d1fd973c518b ./backup-controller/xtao-backup-controller +a97bcca22987b503b1194125807def8280ec1b5b06181f3a1fc846112e54b9d3 ./edge-reconciler/reconcile_cloudflare_dns.py +0b19202e23cc88d1247c0359a20bacabfbd9074c985bcf5777d994371dabb041 ./edge/compose.yaml +99c2d0f3829437d6f6617c55672b535d0e109a512fd55218b69e43d92319a953 ./edge/traefik-dynamic.yml +73a1906f4a950e2e7f3cc17782f64bbab0403461d8016c2223fcb751255400c9 ./edge/traefik.yml +1c35be0d4a14be831c64243be5a7425a94709d22c938a38216f73e0d3ece0db9 ./kuma/configure_kuma_public_status.py +74290bf244f856663563e367b72b1df9068f2b045f9c9417a55760cf53e484aa ./nextcloud/compose.yaml +8a8d0e728a2d44e135f91014f15efc301ee41fa44d507a98ec048cec412e838d ./nextcloud/nextcloud-traefik.yml +b8b94750f5c26c4ae4f7690e50a18342fda294c31a0d88ef48301f6542b8349d ./phase03/accept-nextcloud-functional.sh +3180cf06892e08de66c34d05f11de4e6cfaa892d157094a9063a9309f1f08802 ./phase03/backup-restore-nextcloud.sh +3ba8ec05d052ce136bbe98c00a7f09557e50a0a8f617262db538be6a8849eae5 ./phase03/deploy-nextcloud.sh +34d8d2a211b3d9313ad4beae5f775ce5e3eed7f6bb1e970924fc9dd49a3fc63d ./phase03/reconcile-nextcloud-secrets.sh +9eef426ca8c2d929f7fded4ad2886d855bafe8e7b291dc9c199277e963a9a268 ./publish_to_forgejo.sh +10172af7ee895c1d593f654d4ab8cb5195d40d15cc65d1a09fb2c210b426edd2 ./secret-agent/install-root-unlock.sh +49321466dcd62e14254012f5d594d7602d39555da92df6eac0e562d97831657e ./secret-agent/secret_agent_runner.py +5128c2db987e30d08e3d6fe1af89febb71d701955f969ca88081aafd37390337 ./secret-agent/xtao-secret-agent diff --git a/edge-reconciler/reconcile_cloudflare_dns.py b/edge-reconciler/reconcile_cloudflare_dns.py index 92202ae..919047b 100755 --- a/edge-reconciler/reconcile_cloudflare_dns.py +++ b/edge-reconciler/reconcile_cloudflare_dns.py @@ -39,6 +39,7 @@ def main(): raise SystemExit(f"FAIL zone_lookup count={len(zones)}") zone_id = zones[0]["id"] desired = { + f"cloud.{args.zone}": {"type": "A", "content": args.public_ip, "proxied": False, "ttl": 300}, f"git.{args.zone}": {"type": "A", "content": args.public_ip, "proxied": False, "ttl": 300}, f"status.{args.zone}": {"type": "A", "content": args.public_ip, "proxied": False, "ttl": 300}, } diff --git a/kuma/configure_kuma_public_status.py b/kuma/configure_kuma_public_status.py index dd48e2f..83b1f2a 100755 --- a/kuma/configure_kuma_public_status.py +++ b/kuma/configure_kuma_public_status.py @@ -13,6 +13,7 @@ monitors = [ ('Traefik HTTPS','https://git.xtao.net/', 'http', 1), ('Forgejo','https://git.xtao.net/', 'http', 1), ('Status Page','https://status.xtao.net/status/xtao', 'http', 1), + ('Nextcloud','https://cloud.xtao.net/status.php', 'http', 0), ] for name, url, typ, ignore_tls in monitors: row = cur.execute("select id from monitor where name=?", (name,)).fetchone() diff --git a/nextcloud/compose.yaml b/nextcloud/compose.yaml new file mode 100644 index 0000000..0fce146 --- /dev/null +++ b/nextcloud/compose.yaml @@ -0,0 +1,58 @@ +name: nextcloud + +services: + nextcloud-db: + image: postgres@sha256:7a396fd264a2067788b6551122b50f162bf6136312c7fc9d74381cb92c648382 + restart: unless-stopped + env_file: + - /volume1/xtao-data/nextcloud/secrets/postgres.env + networks: + - nextcloud_internal + volumes: + - /volume1/xtao-data/nextcloud/postgres:/var/lib/postgresql/data + + nextcloud-redis: + image: redis@sha256:b1addbe72465a718643cff9e60a58e6df1841e29d6d7d60c9a85d8d72f08d1a7 + restart: unless-stopped + command: ["redis-server", "/usr/local/etc/redis/redis.conf"] + networks: + - nextcloud_internal + volumes: + - /volume1/xtao-data/nextcloud/redis:/data + - /volume1/xtao-data/nextcloud/secrets/redis.conf:/usr/local/etc/redis/redis.conf:ro + + nextcloud-web: + image: nextcloud@sha256:cbe9aaba3bca1093510ff51b48c84abb412b70fb037c1b77b77e703b1ee63bb7 + restart: unless-stopped + depends_on: + - nextcloud-db + - nextcloud-redis + env_file: + - /volume1/xtao-data/nextcloud/secrets/nextcloud.env + networks: + - nextcloud_internal + - xtao_edge_internal + volumes: + - /volume1/xtao-data/nextcloud/html:/var/www/html + - /volume1/xtao-data/nextcloud/data:/var/www/html/data + + nextcloud-cron: + image: nextcloud@sha256:cbe9aaba3bca1093510ff51b48c84abb412b70fb037c1b77b77e703b1ee63bb7 + restart: unless-stopped + depends_on: + - nextcloud-db + - nextcloud-redis + entrypoint: /cron.sh + env_file: + - /volume1/xtao-data/nextcloud/secrets/nextcloud.env + networks: + - nextcloud_internal + volumes: + - /volume1/xtao-data/nextcloud/html:/var/www/html + - /volume1/xtao-data/nextcloud/data:/var/www/html/data + +networks: + nextcloud_internal: + driver: bridge + xtao_edge_internal: + external: true diff --git a/nextcloud/nextcloud-traefik.yml b/nextcloud/nextcloud-traefik.yml new file mode 100644 index 0000000..b4f9a33 --- /dev/null +++ b/nextcloud/nextcloud-traefik.yml @@ -0,0 +1,33 @@ +http: + routers: + nextcloud: + rule: Host(`cloud.xtao.net`) + entryPoints: [websecure] + service: nextcloud + middlewares: [nextcloud-headers] + tls: {} + nextcloud-wellknown-dav: + rule: Host(`cloud.xtao.net`) && (Path(`/.well-known/caldav`) || Path(`/.well-known/carddav`)) + entryPoints: [websecure] + service: nextcloud + middlewares: [nextcloud-dav-redirect] + tls: {} + services: + nextcloud: + loadBalancer: + passHostHeader: true + servers: + - url: http://nextcloud-web:80 + middlewares: + nextcloud-dav-redirect: + redirectRegex: + regex: "^https://cloud\\.xtao\\.net/\\.well-known/(card|cal)dav/?$" + replacement: "https://cloud.xtao.net/remote.php/dav/" + permanent: true + nextcloud-headers: + headers: + stsSeconds: 15552000 + stsIncludeSubdomains: true + stsPreload: true + customRequestHeaders: + X-Forwarded-Proto: "https" diff --git a/phase03/accept-nextcloud-functional.sh b/phase03/accept-nextcloud-functional.sh new file mode 100644 index 0000000..925e99e --- /dev/null +++ b/phase03/accept-nextcloud-functional.sh @@ -0,0 +1,102 @@ +#!/bin/sh +set -eu + +agent=/usr/local/bin/xtao-secret-agent +docker_bin=/usr/local/bin/docker +base=https://cloud.xtao.net +user=xtao-admin +work=$(mktemp -d /tmp/xtao-nextcloud-accept.XXXXXX) +cleanup() { + rm -rf "$work" +} +trap cleanup EXIT INT TERM + +[ "$(id -u)" = "0" ] || { echo "FAIL must_run_as_root"; exit 1; } +[ -x "$agent" ] || { echo "FAIL missing_secret_agent"; exit 2; } +command -v curl >/dev/null || { echo "FAIL missing_curl"; exit 2; } +command -v python3 >/dev/null || { echo "FAIL missing_python3"; exit 2; } + +netrc="$work/netrc" +{ + printf 'machine cloud.xtao.net login ' + "$agent" raw-field --entry infra/nextcloud/admin --field username + printf ' password ' + "$agent" raw-field --entry infra/nextcloud/admin --field password + printf '\n' +} > "$netrc" +chmod 600 "$netrc" + +status_json="$work/status.json" +curl -fsS "$base/status.php" -o "$status_json" +python3 - "$status_json" <<'PY' +import json, sys +data=json.load(open(sys.argv[1], encoding="utf-8")) +assert data["installed"] is True and data["maintenance"] is False and data["needsDbUpgrade"] is False +print("PASS status installed=true maintenance=false needsDbUpgrade=false") +PY + +ts=$(date -u +%Y%m%dT%H%M%SZ) +small="phase03-small-$ts.bin" +large="phase03-large-$ts.bin" +dd if=/dev/urandom of="$work/$small" bs=1M count=2 status=none +sha256sum "$work/$small" > "$work/$small.sha256" + +curl -fsS --netrc-file "$netrc" -T "$work/$small" "$base/remote.php/dav/files/$user/$small" +curl -fsS --netrc-file "$netrc" "$base/remote.php/dav/files/$user/$small" -o "$work/$small.download" +sha256sum -c "$work/$small.sha256" --ignore-missing >/dev/null +printf 'PASS webdav_upload_download file=%s size=2MiB\n' "$small" + +share_json="$work/share.json" +curl -fsS --netrc-file "$netrc" \ + -H 'OCS-APIRequest: true' \ + -H 'Accept: application/json' \ + -d "path=/$small" \ + -d 'shareType=3' \ + "$base/ocs/v2.php/apps/files_sharing/api/v1/shares" -o "$share_json" +share_url=$(python3 - "$share_json" <<'PY' +import json, sys +data=json.load(open(sys.argv[1], encoding="utf-8")) +meta=data["ocs"]["meta"] +if meta.get("statuscode") not in (100, 200): + print("FAIL share_meta " + json.dumps(meta, ensure_ascii=False), file=sys.stderr) + raise SystemExit(3) +print(data["ocs"]["data"]["url"]) +PY +) +curl -fsSL "$share_url/download" -o "$work/$small.share-download" +cmp "$work/$small" "$work/$small.share-download" +printf 'PASS share_link_create_download file=%s\n' "$small" + +for path in \ + "/remote.php/dav/files/$user/" \ + "/remote.php/dav/calendars/$user/" \ + "/remote.php/dav/addressbooks/users/$user/"; do + code=$(curl -sS --netrc-file "$netrc" -o "$work/propf.out" -w '%{http_code}' -X PROPFIND -H 'Depth: 0' "$base$path") + [ "$code" = "207" ] || { echo "FAIL propfind path=$path code=$code"; exit 3; } +done +printf 'PASS webdav_caldav_carddav_propfind code=207\n' + +for path in /.well-known/caldav /.well-known/carddav; do + code=$(curl -sS -o /dev/null -w '%{http_code}' -I "$base$path") + case "$code" in 301|302|307|308) ;; *) echo "FAIL well_known path=$path code=$code"; exit 3;; esac +done +printf 'PASS well_known_redirect caldav_carddav\n' + +dd if=/dev/urandom of="$work/$large" bs=1M count=64 status=none +sha256sum "$work/$large" > "$work/$large.sha256" +set +e +timeout 1 curl -sS --netrc-file "$netrc" -T "$work/$large" "$base/remote.php/dav/files/$user/interrupted-$large" >/dev/null 2>&1 +interrupted_rc=$? +set -e +case "$interrupted_rc" in 0|124|28) ;; *) echo "FAIL interrupted_transfer rc=$interrupted_rc"; exit 3;; esac +curl -fsS --netrc-file "$netrc" -T "$work/$large" "$base/remote.php/dav/files/$user/$large" +curl -fsS --netrc-file "$netrc" "$base/remote.php/dav/files/$user/$large" -o "$work/$large.download" +sha256sum -c "$work/$large.sha256" --ignore-missing >/dev/null +printf 'PASS large_file_and_interrupted_transfer size=64MiB interrupted_rc=%s\n' "$interrupted_rc" + +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php -f /var/www/html/cron.php +lastcron=$("$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:app:get core lastcron) +[ -n "$lastcron" ] && [ "$lastcron" != "0" ] || { echo "FAIL cron_lastcron_empty"; exit 3; } +printf 'PASS cron_exec lastcron=%s\n' "$lastcron" + +printf 'PASS nextcloud_functional_acceptance values=redacted\n' diff --git a/phase03/backup-restore-nextcloud.sh b/phase03/backup-restore-nextcloud.sh new file mode 100644 index 0000000..ca627f5 --- /dev/null +++ b/phase03/backup-restore-nextcloud.sh @@ -0,0 +1,138 @@ +#!/bin/sh +set -eu + +[ "$(id -u)" = "0" ] || { echo "FAIL must_run_as_root"; exit 1; } +umask 077 + +rclone_image='rclone/rclone@sha256:623378ad0ff3ebd5cebf77720843c0e02edfe46e2d5b5ac6bed54c6371780dfb' +restic_image='restic/restic@sha256:424a4e1fcc6fe2557b5614239dc71a2c793acb33a83ea217171bd7edc1862dcb' +docker_bin=/usr/local/bin/docker +agent=/usr/local/bin/xtao-secret-agent +base=/volume1/xtao-data/backup-controller +bin="$base/bin" +backup_root=/volume2/xtao-backup/nextcloud +tmp=$(mktemp -d /tmp/xtao-nextcloud-backup.XXXXXX) +trap 'rm -rf "$tmp"' EXIT HUP INT TERM + +mkdir -p "$bin" "$backup_root/dumps" "$backup_root/restores" "$backup_root/logs" +chmod 700 "$backup_root" "$backup_root/dumps" "$backup_root/restores" "$backup_root/logs" + +"$agent" raw-field --entry external/google-drive/offsite-backup --field rclone_token_json > "$tmp/token.json" +"$agent" raw-field --entry external/google-drive/offsite-backup --field root_folder_id > "$tmp/root_folder_id" +"$agent" raw-field --entry external/google-drive/offsite-backup --field oauth_client_id > "$tmp/oauth_client_id" +"$agent" raw-field --entry external/google-drive/offsite-backup --field oauth_client_secret > "$tmp/oauth_client_secret" +"$agent" raw-field --entry backup/google-drive/restic-repository --field password > "$tmp/restic.pass" +chmod 600 "$tmp/token.json" "$tmp/root_folder_id" "$tmp/oauth_client_id" "$tmp/oauth_client_secret" "$tmp/restic.pass" + +python3 - "$tmp/token.json" "$tmp/root_folder_id" "$tmp/oauth_client_id" "$tmp/oauth_client_secret" "$tmp/rclone.conf" <<'PY' +import json, pathlib, sys +token_path, root_path, client_id_path, client_secret_path, config_path = map(pathlib.Path, sys.argv[1:]) +token = token_path.read_text(encoding="utf-8").strip() +json.loads(token) +root = root_path.read_text(encoding="utf-8").strip() +if not root or root == "CODEX_GENERATE": + raise SystemExit("FAIL invalid_root_folder_id") +client_id = client_id_path.read_text(encoding="utf-8").strip() +client_secret = client_secret_path.read_text(encoding="utf-8").strip() +client_lines = "" +if client_id and client_id != "NONE": + client_lines += f"client_id = {client_id}\n" +if client_secret: + client_lines += f"client_secret = {client_secret}\n" +config_path.write_text( + "[xtao-gdrive]\n" + "type = drive\n" + "scope = drive.file\n" + f"{client_lines}" + f"root_folder_id = {root}\n" + f"token = {token}\n", + encoding="utf-8", +) +PY +chmod 600 "$tmp/rclone.conf" + +if [ ! -x "$bin/rclone" ]; then + cid=$("$docker_bin" create --entrypoint /bin/sh "$rclone_image" -c true) + "$docker_bin" cp "$cid:/usr/local/bin/rclone" "$tmp/rclone" + "$docker_bin" rm "$cid" >/dev/null + install -m 700 "$tmp/rclone" "$bin/rclone" +fi + +dbuser=$("$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php -r 'include "/var/www/html/config/config.php"; echo $CONFIG["dbuser"];') +dbname=$("$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php -r 'include "/var/www/html/config/config.php"; echo $CONFIG["dbname"];') +case "$dbuser" in *[!A-Za-z0-9_-]*|"") echo "FAIL unexpected_dbuser"; exit 3;; esac +case "$dbname" in *[!A-Za-z0-9_-]*|"") echo "FAIL unexpected_dbname"; exit 3;; esac + +ts=$(date -u +%Y%m%dT%H%M%SZ) +dump="$backup_root/dumps/nextcloud-$ts.sql" +"$docker_bin" exec -i nextcloud-nextcloud-db-1 pg_dump -U nextcloud "$dbname" > "$dump" +chmod 600 "$dump" + +sample=$(find /volume1/xtao-data/nextcloud/data/xtao-admin/files -maxdepth 1 -type f -name 'phase03-small-*.bin' | sort | tail -n 1) +[ -n "$sample" ] || { echo "FAIL missing_uploaded_canary_file"; exit 3; } +sample_name=$(basename "$sample") +sha256sum "$sample" | awk '{print $1}' > "$tmp/sample.sha256" + +restic_env="-e RCLONE_CONFIG=/config/rclone/rclone.conf -e RESTIC_PASSWORD_FILE=/secrets/restic.pass -e RESTIC_REPOSITORY=rclone:xtao-gdrive:restic/nas" +set +e +"$docker_bin" run --rm --network bridge \ + -v "$tmp:/config/rclone" \ + -v "$tmp/restic.pass:/secrets/restic.pass:ro" \ + -v "$bin/rclone:/usr/local/bin/rclone:ro" \ + $restic_env \ + "$restic_image" init >/tmp/xtao-nextcloud-restic-init.out 2>&1 +rc=$? +set -e +if [ "$rc" -ne 0 ] && ! grep -Eq 'already initialized|config file already exists' /tmp/xtao-nextcloud-restic-init.out; then + cat /tmp/xtao-nextcloud-restic-init.out + echo "FAIL restic_init" + exit 2 +fi + +"$docker_bin" run --rm --network bridge \ + -v "$tmp:/config/rclone" \ + -v "$tmp/restic.pass:/secrets/restic.pass:ro" \ + -v "$bin/rclone:/usr/local/bin/rclone:ro" \ + -v "/volume1/xtao-data/nextcloud/data/xtao-admin/files:/backup/files:ro" \ + -v "/volume1/xtao-data/nextcloud/html/config:/backup/config:ro" \ + -v "$backup_root/dumps:/backup/dbdumps:ro" \ + $restic_env \ + "$restic_image" backup --tag phase03-nextcloud-$ts /backup/files /backup/config /backup/dbdumps >/tmp/xtao-nextcloud-restic-backup.out + +restore="$backup_root/restores/restore-$ts" +mkdir -p "$restore" +chmod 700 "$restore" +"$docker_bin" run --rm --network bridge \ + -v "$tmp:/config/rclone" \ + -v "$tmp/restic.pass:/secrets/restic.pass:ro" \ + -v "$bin/rclone:/usr/local/bin/rclone:ro" \ + -v "$restore:/restore" \ + $restic_env \ + "$restic_image" restore latest --tag "phase03-nextcloud-$ts" --target /restore >/tmp/xtao-nextcloud-restic-restore.out + +restored_sample="$restore/backup/files/$sample_name" +[ -f "$restored_sample" ] || { echo "FAIL restored_sample_missing"; exit 3; } +sha256sum "$restored_sample" | awk '{print $1}' > "$tmp/restored.sha256" +cmp "$tmp/sample.sha256" "$tmp/restored.sha256" +[ -f "$restore/backup/config/config.php" ] || { echo "FAIL restored_config_missing"; exit 3; } +[ -f "$restore/backup/dbdumps/$(basename "$dump")" ] || { echo "FAIL restored_dump_missing"; exit 3; } + +restore_db="nc_restore_$(date -u +%H%M%S)" +"$docker_bin" exec -i nextcloud-nextcloud-db-1 createdb -U nextcloud "$restore_db" +set +e +"$docker_bin" exec -i nextcloud-nextcloud-db-1 psql -U nextcloud "$restore_db" < "$restore/backup/dbdumps/$(basename "$dump")" >/tmp/xtao-nextcloud-db-restore.out 2>&1 +restore_rc=$? +set -e +if [ "$restore_rc" -ne 0 ]; then + "$docker_bin" exec -i nextcloud-nextcloud-db-1 dropdb -U nextcloud "$restore_db" || true + cat /tmp/xtao-nextcloud-db-restore.out + echo "FAIL isolated_db_restore" + exit 3 +fi +user_count=$("$docker_bin" exec -i nextcloud-nextcloud-db-1 psql -U nextcloud "$restore_db" -Atc "select count(*) from oc_users where uid='xtao-admin';") +file_count=$("$docker_bin" exec -i nextcloud-nextcloud-db-1 psql -U nextcloud "$restore_db" -Atc "select count(*) from oc_filecache where path='files/$sample_name';") +"$docker_bin" exec -i nextcloud-nextcloud-db-1 dropdb -U nextcloud "$restore_db" +[ "$user_count" = "1" ] || { echo "FAIL restored_user_missing"; exit 3; } +[ "$file_count" != "0" ] || { echo "FAIL restored_filecache_missing"; exit 3; } + +printf 'PASS nextcloud_backup_restore restic=PASS isolated_db=PASS restored_file=%s values=redacted\n' "$sample_name" diff --git a/phase03/deploy-nextcloud.sh b/phase03/deploy-nextcloud.sh new file mode 100644 index 0000000..1e6b107 --- /dev/null +++ b/phase03/deploy-nextcloud.sh @@ -0,0 +1,123 @@ +#!/bin/sh +set -eu + +src_dir=${1:-/tmp/xtao-phase03} +compose_src="$src_dir/compose.yaml" +traefik_src="$src_dir/nextcloud-traefik.yml" +runner_src="$src_dir/secret_agent_runner_phase03.py" + +docker_bin=/usr/local/bin/docker +compose_bin=/usr/local/bin/docker-compose +agent=/usr/local/bin/xtao-secret-agent +ts=$(date -u +%Y%m%dT%H%M%SZ) + +[ "$(id -u)" = "0" ] || { echo "FAIL must_run_as_root"; exit 1; } +[ -r "$compose_src" ] || { echo "FAIL missing_compose_src"; exit 2; } +[ -r "$traefik_src" ] || { echo "FAIL missing_traefik_src"; exit 2; } +[ -r "$runner_src" ] || { echo "FAIL missing_runner_src"; exit 2; } +[ -x "$agent" ] || { echo "FAIL missing_secret_agent"; exit 2; } + +install -d -m 700 /volume1/xtao-data/nextcloud +install -d -m 700 /volume1/xtao-data/nextcloud/secrets +install -d -m 700 /volume1/xtao-data/nextcloud/html +install -d -m 700 /volume1/xtao-data/nextcloud/data +install -d -m 700 /volume1/xtao-data/nextcloud/postgres +install -d -m 700 /volume1/xtao-data/nextcloud/redis +install -d -m 700 /volume1/xtao-data/nextcloud/backups +install -d -m 700 /volume1/xtao-data/traefik/dynamic + +runner_dst=/volume1/xtao-data/secret-agent/bin/secret_agent_runner.py +[ -f "$runner_dst" ] && cp -p "$runner_dst" "$runner_dst.bak.phase03.$ts" +install -m 600 "$runner_src" "$runner_dst" + +"$agent" probe +"$agent" upsert --entry infra/nextcloud/admin +"$agent" upsert --entry infra/nextcloud/postgres +"$agent" upsert --entry infra/nextcloud/redis +"$agent" check --entry infra/nextcloud/admin +"$agent" check --entry infra/nextcloud/postgres +"$agent" check --entry infra/nextcloud/redis + +tmpdir=$(mktemp -d /volume1/xtao-data/nextcloud/secrets/.tmp.XXXXXX) +chmod 700 "$tmpdir" + +postgres_env="$tmpdir/postgres.env" +nextcloud_env="$tmpdir/nextcloud.env" +redis_conf="$tmpdir/redis.conf" + +{ + printf 'POSTGRES_DB=' + "$agent" raw-field --entry infra/nextcloud/postgres --field database + printf '\nPOSTGRES_USER=' + "$agent" raw-field --entry infra/nextcloud/postgres --field username + printf '\nPOSTGRES_PASSWORD=' + "$agent" raw-field --entry infra/nextcloud/postgres --field password + printf '\n' +} > "$postgres_env" + +{ + printf 'POSTGRES_HOST=nextcloud-db\n' + printf 'POSTGRES_DB=' + "$agent" raw-field --entry infra/nextcloud/postgres --field database + printf '\nPOSTGRES_USER=' + "$agent" raw-field --entry infra/nextcloud/postgres --field username + printf '\nPOSTGRES_PASSWORD=' + "$agent" raw-field --entry infra/nextcloud/postgres --field password + printf '\nREDIS_HOST=nextcloud-redis\nREDIS_HOST_PORT=6379\nREDIS_HOST_PASSWORD=' + "$agent" raw-field --entry infra/nextcloud/redis --field password + printf '\nNEXTCLOUD_ADMIN_USER=' + "$agent" raw-field --entry infra/nextcloud/admin --field username + printf '\nNEXTCLOUD_ADMIN_PASSWORD=' + "$agent" raw-field --entry infra/nextcloud/admin --field password + printf '\nNEXTCLOUD_TRUSTED_DOMAINS=cloud.xtao.net\n' + printf 'TRUSTED_PROXIES=172.16.0.0/12 192.168.1.0/24\n' + printf 'OVERWRITEHOST=cloud.xtao.net\nOVERWRITEPROTOCOL=https\nOVERWRITECLIURL=https://cloud.xtao.net\n' + printf 'APACHE_DISABLE_REWRITE_IP=1\nPHP_MEMORY_LIMIT=1024M\nPHP_UPLOAD_LIMIT=10G\n' +} > "$nextcloud_env" + +{ + printf 'appendonly yes\n' + printf 'requirepass ' + "$agent" raw-field --entry infra/nextcloud/redis --field password + printf '\n' +} > "$redis_conf" + +chmod 600 "$postgres_env" "$nextcloud_env" "$redis_conf" +mv "$postgres_env" /volume1/xtao-data/nextcloud/secrets/postgres.env +mv "$nextcloud_env" /volume1/xtao-data/nextcloud/secrets/nextcloud.env +mv "$redis_conf" /volume1/xtao-data/nextcloud/secrets/redis.conf +chown 999:0 /volume1/xtao-data/nextcloud/secrets/redis.conf +chmod 400 /volume1/xtao-data/nextcloud/secrets/redis.conf +rmdir "$tmpdir" + +[ -f /volume1/xtao-data/nextcloud/compose.yaml ] && cp -p /volume1/xtao-data/nextcloud/compose.yaml "/volume1/xtao-data/nextcloud/compose.yaml.bak.phase03.$ts" +install -m 600 "$compose_src" /volume1/xtao-data/nextcloud/compose.yaml + +[ -f /volume1/xtao-data/traefik/dynamic/nextcloud.yml ] && cp -p /volume1/xtao-data/traefik/dynamic/nextcloud.yml "/volume1/xtao-data/traefik/dynamic/nextcloud.yml.bak.phase03.$ts" +install -m 600 "$traefik_src" /volume1/xtao-data/traefik/dynamic/nextcloud.yml + +cd /volume1/xtao-data/nextcloud +"$compose_bin" -f compose.yaml config >/dev/null +"$compose_bin" -f compose.yaml pull +"$compose_bin" -f compose.yaml up -d + +for i in $(seq 1 90); do + if "$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ status >/tmp/xtao-nextcloud-occ-status 2>&1; then + break + fi + sleep 4 +done + +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ background:cron +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set trusted_domains 1 --value=cloud.xtao.net +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set trusted_proxies 0 --value=172.16.0.0/12 +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set trusted_proxies 1 --value=192.168.1.0/24 +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set overwritehost --value=cloud.xtao.net +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set overwriteprotocol --value=https +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set overwrite.cli.url --value=https://cloud.xtao.net +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set htaccess.RewriteBase --value=/ +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set maintenance_window_start --type=integer --value=8 +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ maintenance:update:htaccess +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ status + +printf 'PASS nextcloud_deploy route=cloud.xtao.net secrets=redacted compose=/volume1/xtao-data/nextcloud/compose.yaml\n' diff --git a/phase03/reconcile-nextcloud-secrets.sh b/phase03/reconcile-nextcloud-secrets.sh new file mode 100644 index 0000000..2d87e68 --- /dev/null +++ b/phase03/reconcile-nextcloud-secrets.sh @@ -0,0 +1,135 @@ +#!/bin/sh +set -eu + +src_dir=${1:-/tmp/xtao-phase03} +runner_src="$src_dir/secret_agent_runner_phase03.py" +agent=/usr/local/bin/xtao-secret-agent +docker_bin=/usr/local/bin/docker +compose_bin=/usr/local/bin/docker-compose +ts=$(date -u +%Y%m%dT%H%M%SZ) + +[ "$(id -u)" = "0" ] || { echo "FAIL must_run_as_root"; exit 1; } +[ -r "$runner_src" ] || { echo "FAIL missing_runner_src"; exit 2; } + +runner_dst=/volume1/xtao-data/secret-agent/bin/secret_agent_runner.py +cp -p "$runner_dst" "$runner_dst.bak.phase03.reconcile.$ts" +install -m 600 "$runner_src" "$runner_dst" + +"$agent" upsert --entry infra/nextcloud/admin +"$agent" upsert --entry infra/nextcloud/postgres +"$agent" upsert --entry infra/nextcloud/redis +"$agent" check --entry infra/nextcloud/admin +"$agent" check --entry infra/nextcloud/postgres +"$agent" check --entry infra/nextcloud/redis + +tmpdir=$(mktemp -d /volume1/xtao-data/nextcloud/secrets/.reconcile.XXXXXX) +chmod 700 "$tmpdir" +cleanup() { + rm -rf "$tmpdir" +} +trap cleanup EXIT INT TERM + +"$agent" raw-field --entry infra/nextcloud/postgres --field password > "$tmpdir/dbpass" +"$agent" raw-field --entry infra/nextcloud/redis --field password > "$tmpdir/redispass" +"$agent" raw-field --entry infra/nextcloud/admin --field password > "$tmpdir/adminpass" +chmod 600 "$tmpdir/dbpass" "$tmpdir/redispass" "$tmpdir/adminpass" + +dbuser=$("$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php -r 'include "/var/www/html/config/config.php"; echo $CONFIG["dbuser"];') +dbname=$("$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php -r 'include "/var/www/html/config/config.php"; echo $CONFIG["dbname"];') +case "$dbuser" in + *[!A-Za-z0-9_-]*|"") echo "FAIL unexpected_dbuser"; exit 3;; +esac +case "$dbname" in + *[!A-Za-z0-9_-]*|"") echo "FAIL unexpected_dbname"; exit 3;; +esac + +if grep -q "'" "$tmpdir/dbpass"; then + echo "FAIL unexpected_dbpass_character" + exit 3 +fi +awk -v u="$dbuser" '{printf "ALTER USER \"%s\" WITH PASSWORD '"'"'%s'"'"';\n", u, $0}' "$tmpdir/dbpass" > "$tmpdir/alter.sql" +chmod 600 "$tmpdir/alter.sql" +"$docker_bin" exec -i nextcloud-nextcloud-db-1 psql -U nextcloud -d "$dbname" < "$tmpdir/alter.sql" >/dev/null + +{ + printf 'POSTGRES_HOST=nextcloud-db\nPOSTGRES_DB=%s\nPOSTGRES_USER=%s\nPOSTGRES_PASSWORD=' "$dbname" "$dbuser" + cat "$tmpdir/dbpass" + printf '\nREDIS_HOST=nextcloud-redis\nREDIS_HOST_PORT=6379\nREDIS_HOST_PASSWORD=' + cat "$tmpdir/redispass" + printf '\nNEXTCLOUD_ADMIN_USER=xtao-admin\nNEXTCLOUD_ADMIN_PASSWORD=' + cat "$tmpdir/adminpass" + printf '\nNEXTCLOUD_TRUSTED_DOMAINS=cloud.xtao.net\n' + printf 'TRUSTED_PROXIES=172.16.0.0/12 192.168.1.0/24\n' + printf 'OVERWRITEHOST=cloud.xtao.net\nOVERWRITEPROTOCOL=https\nOVERWRITECLIURL=https://cloud.xtao.net\n' + printf 'APACHE_DISABLE_REWRITE_IP=1\nPHP_MEMORY_LIMIT=1024M\nPHP_UPLOAD_LIMIT=10G\n' +} > "$tmpdir/nextcloud.env" +chmod 600 "$tmpdir/nextcloud.env" +mv "$tmpdir/nextcloud.env" /volume1/xtao-data/nextcloud/secrets/nextcloud.env + +{ + printf 'POSTGRES_DB=nextcloud\nPOSTGRES_USER=nextcloud\nPOSTGRES_PASSWORD=' + cat "$tmpdir/dbpass" + printf '\n' +} > "$tmpdir/postgres.env" +chmod 600 "$tmpdir/postgres.env" +mv "$tmpdir/postgres.env" /volume1/xtao-data/nextcloud/secrets/postgres.env + +{ + printf 'appendonly yes\nrequirepass ' + cat "$tmpdir/redispass" + printf '\n' +} > "$tmpdir/redis.conf" +chmod 400 "$tmpdir/redis.conf" +chown 999:0 "$tmpdir/redis.conf" +mv "$tmpdir/redis.conf" /volume1/xtao-data/nextcloud/secrets/redis.conf + +cd /volume1/xtao-data/nextcloud +"$compose_bin" -f compose.yaml up -d nextcloud-redis +"$docker_bin" restart nextcloud-nextcloud-redis-1 >/dev/null +sleep 3 + +cat "$tmpdir/dbpass" | "$docker_bin" exec -i -u www-data nextcloud-nextcloud-web-1 php -r ' +$v = trim(stream_get_contents(STDIN)); +include "/var/www/html/config/config.php"; +$CONFIG["dbpassword"] = $v; +$out = "/dev/null + +"$compose_bin" -f compose.yaml up -d +sleep 5 + +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ background:cron +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set trusted_domains 1 --value=cloud.xtao.net +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set trusted_proxies 0 --value=172.16.0.0/12 +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set trusted_proxies 1 --value=192.168.1.0/24 +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set overwritehost --value=cloud.xtao.net +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set overwriteprotocol --value=https +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set overwrite.cli.url --value=https://cloud.xtao.net +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set htaccess.RewriteBase --value=/ +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set maintenance_window_start --type=integer --value=8 +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ maintenance:update:htaccess +"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ status + +printf 'PASS nextcloud_secret_reconcile values=redacted\n' diff --git a/secret-agent/secret_agent_runner.py b/secret-agent/secret_agent_runner.py index f4008f1..000e826 100644 --- a/secret-agent/secret_agent_runner.py +++ b/secret-agent/secret_agent_runner.py @@ -18,6 +18,9 @@ ALLOWED = { "infra/traefik/monitor": {"username": "traefik-monitor", "token": "token"}, "infra/forgejo/admin": {"username": "forgejo-admin", "password": "secret"}, "infra/forgejo/postgres": {"username": "forgejo", "database": "forgejo", "password": "secret"}, + "infra/nextcloud/admin": {"username": "xtao-admin", "password": "secret"}, + "infra/nextcloud/postgres": {"username": "nextcloud", "database": "nextcloud", "password": "secret"}, + "infra/nextcloud/redis": {"password": "secret"}, "backup/google-drive/restic-repository": {"repository": "gdrive:xtao-backup/restic/nas", "password": "secret"}, } READ_ALLOWED = { @@ -76,7 +79,11 @@ def upsert(kp, path): entry.notes = "generated_by=nas-secret-agent" for key, value in profile.items(): if value in ("secret", "token"): - entry.set_custom_property(key, gen(value), protect=True) + current = entry.get_custom_property(key) + if current in (None, ""): + entry.set_custom_property(key, gen(value), protect=True) + elif not entry.is_custom_property_protected(key): + raise SystemExit(f"FAIL unprotected_existing_field path={path} field={key}") else: entry.set_custom_property(key, value, protect=False) return sorted(profile)