Add Phase 03 Nextcloud desired state
This commit is contained in:
XTAO Deployment Bot
parent
1b159fc3d6
commit
3c5ab83de2
10 changed files with 617 additions and 12 deletions
29
SHA256SUMS
29
SHA256SUMS
|
|
@ -1,11 +1,18 @@
|
|||
0aec27176d0bb0053a592fe9e9bcc37118b27f55541a6825eed5bb5cef6a60c7 state/implementation/desired-state/README.md
|
||||
9d4d0f119d29eb5622978d531e27998f97b986106c8426d6ef01dc7dfe5ba937 state/implementation/desired-state/SHA256SUMS
|
||||
4020b538e0499d12195375b5146634799cd96929826ede63bee3d1fd973c518b state/implementation/desired-state/backup-controller/xtao-backup-controller
|
||||
9a434c629f955a7298995fe48415cb8bba95b12594684a8bb120ee37514c3f3a state/implementation/desired-state/edge-reconciler/reconcile_cloudflare_dns.py
|
||||
0b19202e23cc88d1247c0359a20bacabfbd9074c985bcf5777d994371dabb041 state/implementation/desired-state/edge/compose.yaml
|
||||
99c2d0f3829437d6f6617c55672b535d0e109a512fd55218b69e43d92319a953 state/implementation/desired-state/edge/traefik-dynamic.yml
|
||||
73a1906f4a950e2e7f3cc17782f64bbab0403461d8016c2223fcb751255400c9 state/implementation/desired-state/edge/traefik.yml
|
||||
256dc4be0585b645d62c7765ae0579e109ee07039a65dbcf171c25debdc3b731 state/implementation/desired-state/kuma/configure_kuma_public_status.py
|
||||
10172af7ee895c1d593f654d4ab8cb5195d40d15cc65d1a09fb2c210b426edd2 state/implementation/desired-state/secret-agent/install-root-unlock.sh
|
||||
8f59112e44e725de60f476b6a43215a598b6f0328387437c80bfdd299964eebb state/implementation/desired-state/secret-agent/secret_agent_runner.py
|
||||
5128c2db987e30d08e3d6fe1af89febb71d701955f969ca88081aafd37390337 state/implementation/desired-state/secret-agent/xtao-secret-agent
|
||||
0aec27176d0bb0053a592fe9e9bcc37118b27f55541a6825eed5bb5cef6a60c7 ./README.md
|
||||
7b688eaa85148703bacf6693fa4e6ac78a25303a8a367811498e0b5466efaa42 ./SHA256SUMS
|
||||
4020b538e0499d12195375b5146634799cd96929826ede63bee3d1fd973c518b ./backup-controller/xtao-backup-controller
|
||||
a97bcca22987b503b1194125807def8280ec1b5b06181f3a1fc846112e54b9d3 ./edge-reconciler/reconcile_cloudflare_dns.py
|
||||
0b19202e23cc88d1247c0359a20bacabfbd9074c985bcf5777d994371dabb041 ./edge/compose.yaml
|
||||
99c2d0f3829437d6f6617c55672b535d0e109a512fd55218b69e43d92319a953 ./edge/traefik-dynamic.yml
|
||||
73a1906f4a950e2e7f3cc17782f64bbab0403461d8016c2223fcb751255400c9 ./edge/traefik.yml
|
||||
1c35be0d4a14be831c64243be5a7425a94709d22c938a38216f73e0d3ece0db9 ./kuma/configure_kuma_public_status.py
|
||||
74290bf244f856663563e367b72b1df9068f2b045f9c9417a55760cf53e484aa ./nextcloud/compose.yaml
|
||||
8a8d0e728a2d44e135f91014f15efc301ee41fa44d507a98ec048cec412e838d ./nextcloud/nextcloud-traefik.yml
|
||||
b8b94750f5c26c4ae4f7690e50a18342fda294c31a0d88ef48301f6542b8349d ./phase03/accept-nextcloud-functional.sh
|
||||
3180cf06892e08de66c34d05f11de4e6cfaa892d157094a9063a9309f1f08802 ./phase03/backup-restore-nextcloud.sh
|
||||
3ba8ec05d052ce136bbe98c00a7f09557e50a0a8f617262db538be6a8849eae5 ./phase03/deploy-nextcloud.sh
|
||||
34d8d2a211b3d9313ad4beae5f775ce5e3eed7f6bb1e970924fc9dd49a3fc63d ./phase03/reconcile-nextcloud-secrets.sh
|
||||
9eef426ca8c2d929f7fded4ad2886d855bafe8e7b291dc9c199277e963a9a268 ./publish_to_forgejo.sh
|
||||
10172af7ee895c1d593f654d4ab8cb5195d40d15cc65d1a09fb2c210b426edd2 ./secret-agent/install-root-unlock.sh
|
||||
49321466dcd62e14254012f5d594d7602d39555da92df6eac0e562d97831657e ./secret-agent/secret_agent_runner.py
|
||||
5128c2db987e30d08e3d6fe1af89febb71d701955f969ca88081aafd37390337 ./secret-agent/xtao-secret-agent
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ def main():
|
|||
raise SystemExit(f"FAIL zone_lookup count={len(zones)}")
|
||||
zone_id = zones[0]["id"]
|
||||
desired = {
|
||||
f"cloud.{args.zone}": {"type": "A", "content": args.public_ip, "proxied": False, "ttl": 300},
|
||||
f"git.{args.zone}": {"type": "A", "content": args.public_ip, "proxied": False, "ttl": 300},
|
||||
f"status.{args.zone}": {"type": "A", "content": args.public_ip, "proxied": False, "ttl": 300},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ monitors = [
|
|||
('Traefik HTTPS','https://git.xtao.net/', 'http', 1),
|
||||
('Forgejo','https://git.xtao.net/', 'http', 1),
|
||||
('Status Page','https://status.xtao.net/status/xtao', 'http', 1),
|
||||
('Nextcloud','https://cloud.xtao.net/status.php', 'http', 0),
|
||||
]
|
||||
for name, url, typ, ignore_tls in monitors:
|
||||
row = cur.execute("select id from monitor where name=?", (name,)).fetchone()
|
||||
|
|
|
|||
58
nextcloud/compose.yaml
Normal file
58
nextcloud/compose.yaml
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
name: nextcloud
|
||||
|
||||
services:
|
||||
nextcloud-db:
|
||||
image: postgres@sha256:7a396fd264a2067788b6551122b50f162bf6136312c7fc9d74381cb92c648382
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- /volume1/xtao-data/nextcloud/secrets/postgres.env
|
||||
networks:
|
||||
- nextcloud_internal
|
||||
volumes:
|
||||
- /volume1/xtao-data/nextcloud/postgres:/var/lib/postgresql/data
|
||||
|
||||
nextcloud-redis:
|
||||
image: redis@sha256:b1addbe72465a718643cff9e60a58e6df1841e29d6d7d60c9a85d8d72f08d1a7
|
||||
restart: unless-stopped
|
||||
command: ["redis-server", "/usr/local/etc/redis/redis.conf"]
|
||||
networks:
|
||||
- nextcloud_internal
|
||||
volumes:
|
||||
- /volume1/xtao-data/nextcloud/redis:/data
|
||||
- /volume1/xtao-data/nextcloud/secrets/redis.conf:/usr/local/etc/redis/redis.conf:ro
|
||||
|
||||
nextcloud-web:
|
||||
image: nextcloud@sha256:cbe9aaba3bca1093510ff51b48c84abb412b70fb037c1b77b77e703b1ee63bb7
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- nextcloud-db
|
||||
- nextcloud-redis
|
||||
env_file:
|
||||
- /volume1/xtao-data/nextcloud/secrets/nextcloud.env
|
||||
networks:
|
||||
- nextcloud_internal
|
||||
- xtao_edge_internal
|
||||
volumes:
|
||||
- /volume1/xtao-data/nextcloud/html:/var/www/html
|
||||
- /volume1/xtao-data/nextcloud/data:/var/www/html/data
|
||||
|
||||
nextcloud-cron:
|
||||
image: nextcloud@sha256:cbe9aaba3bca1093510ff51b48c84abb412b70fb037c1b77b77e703b1ee63bb7
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- nextcloud-db
|
||||
- nextcloud-redis
|
||||
entrypoint: /cron.sh
|
||||
env_file:
|
||||
- /volume1/xtao-data/nextcloud/secrets/nextcloud.env
|
||||
networks:
|
||||
- nextcloud_internal
|
||||
volumes:
|
||||
- /volume1/xtao-data/nextcloud/html:/var/www/html
|
||||
- /volume1/xtao-data/nextcloud/data:/var/www/html/data
|
||||
|
||||
networks:
|
||||
nextcloud_internal:
|
||||
driver: bridge
|
||||
xtao_edge_internal:
|
||||
external: true
|
||||
33
nextcloud/nextcloud-traefik.yml
Normal file
33
nextcloud/nextcloud-traefik.yml
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
http:
|
||||
routers:
|
||||
nextcloud:
|
||||
rule: Host(`cloud.xtao.net`)
|
||||
entryPoints: [websecure]
|
||||
service: nextcloud
|
||||
middlewares: [nextcloud-headers]
|
||||
tls: {}
|
||||
nextcloud-wellknown-dav:
|
||||
rule: Host(`cloud.xtao.net`) && (Path(`/.well-known/caldav`) || Path(`/.well-known/carddav`))
|
||||
entryPoints: [websecure]
|
||||
service: nextcloud
|
||||
middlewares: [nextcloud-dav-redirect]
|
||||
tls: {}
|
||||
services:
|
||||
nextcloud:
|
||||
loadBalancer:
|
||||
passHostHeader: true
|
||||
servers:
|
||||
- url: http://nextcloud-web:80
|
||||
middlewares:
|
||||
nextcloud-dav-redirect:
|
||||
redirectRegex:
|
||||
regex: "^https://cloud\\.xtao\\.net/\\.well-known/(card|cal)dav/?$"
|
||||
replacement: "https://cloud.xtao.net/remote.php/dav/"
|
||||
permanent: true
|
||||
nextcloud-headers:
|
||||
headers:
|
||||
stsSeconds: 15552000
|
||||
stsIncludeSubdomains: true
|
||||
stsPreload: true
|
||||
customRequestHeaders:
|
||||
X-Forwarded-Proto: "https"
|
||||
102
phase03/accept-nextcloud-functional.sh
Normal file
102
phase03/accept-nextcloud-functional.sh
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
agent=/usr/local/bin/xtao-secret-agent
|
||||
docker_bin=/usr/local/bin/docker
|
||||
base=https://cloud.xtao.net
|
||||
user=xtao-admin
|
||||
work=$(mktemp -d /tmp/xtao-nextcloud-accept.XXXXXX)
|
||||
cleanup() {
|
||||
rm -rf "$work"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
[ "$(id -u)" = "0" ] || { echo "FAIL must_run_as_root"; exit 1; }
|
||||
[ -x "$agent" ] || { echo "FAIL missing_secret_agent"; exit 2; }
|
||||
command -v curl >/dev/null || { echo "FAIL missing_curl"; exit 2; }
|
||||
command -v python3 >/dev/null || { echo "FAIL missing_python3"; exit 2; }
|
||||
|
||||
netrc="$work/netrc"
|
||||
{
|
||||
printf 'machine cloud.xtao.net login '
|
||||
"$agent" raw-field --entry infra/nextcloud/admin --field username
|
||||
printf ' password '
|
||||
"$agent" raw-field --entry infra/nextcloud/admin --field password
|
||||
printf '\n'
|
||||
} > "$netrc"
|
||||
chmod 600 "$netrc"
|
||||
|
||||
status_json="$work/status.json"
|
||||
curl -fsS "$base/status.php" -o "$status_json"
|
||||
python3 - "$status_json" <<'PY'
|
||||
import json, sys
|
||||
data=json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
assert data["installed"] is True and data["maintenance"] is False and data["needsDbUpgrade"] is False
|
||||
print("PASS status installed=true maintenance=false needsDbUpgrade=false")
|
||||
PY
|
||||
|
||||
ts=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
small="phase03-small-$ts.bin"
|
||||
large="phase03-large-$ts.bin"
|
||||
dd if=/dev/urandom of="$work/$small" bs=1M count=2 status=none
|
||||
sha256sum "$work/$small" > "$work/$small.sha256"
|
||||
|
||||
curl -fsS --netrc-file "$netrc" -T "$work/$small" "$base/remote.php/dav/files/$user/$small"
|
||||
curl -fsS --netrc-file "$netrc" "$base/remote.php/dav/files/$user/$small" -o "$work/$small.download"
|
||||
sha256sum -c "$work/$small.sha256" --ignore-missing >/dev/null
|
||||
printf 'PASS webdav_upload_download file=%s size=2MiB\n' "$small"
|
||||
|
||||
share_json="$work/share.json"
|
||||
curl -fsS --netrc-file "$netrc" \
|
||||
-H 'OCS-APIRequest: true' \
|
||||
-H 'Accept: application/json' \
|
||||
-d "path=/$small" \
|
||||
-d 'shareType=3' \
|
||||
"$base/ocs/v2.php/apps/files_sharing/api/v1/shares" -o "$share_json"
|
||||
share_url=$(python3 - "$share_json" <<'PY'
|
||||
import json, sys
|
||||
data=json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
meta=data["ocs"]["meta"]
|
||||
if meta.get("statuscode") not in (100, 200):
|
||||
print("FAIL share_meta " + json.dumps(meta, ensure_ascii=False), file=sys.stderr)
|
||||
raise SystemExit(3)
|
||||
print(data["ocs"]["data"]["url"])
|
||||
PY
|
||||
)
|
||||
curl -fsSL "$share_url/download" -o "$work/$small.share-download"
|
||||
cmp "$work/$small" "$work/$small.share-download"
|
||||
printf 'PASS share_link_create_download file=%s\n' "$small"
|
||||
|
||||
for path in \
|
||||
"/remote.php/dav/files/$user/" \
|
||||
"/remote.php/dav/calendars/$user/" \
|
||||
"/remote.php/dav/addressbooks/users/$user/"; do
|
||||
code=$(curl -sS --netrc-file "$netrc" -o "$work/propf.out" -w '%{http_code}' -X PROPFIND -H 'Depth: 0' "$base$path")
|
||||
[ "$code" = "207" ] || { echo "FAIL propfind path=$path code=$code"; exit 3; }
|
||||
done
|
||||
printf 'PASS webdav_caldav_carddav_propfind code=207\n'
|
||||
|
||||
for path in /.well-known/caldav /.well-known/carddav; do
|
||||
code=$(curl -sS -o /dev/null -w '%{http_code}' -I "$base$path")
|
||||
case "$code" in 301|302|307|308) ;; *) echo "FAIL well_known path=$path code=$code"; exit 3;; esac
|
||||
done
|
||||
printf 'PASS well_known_redirect caldav_carddav\n'
|
||||
|
||||
dd if=/dev/urandom of="$work/$large" bs=1M count=64 status=none
|
||||
sha256sum "$work/$large" > "$work/$large.sha256"
|
||||
set +e
|
||||
timeout 1 curl -sS --netrc-file "$netrc" -T "$work/$large" "$base/remote.php/dav/files/$user/interrupted-$large" >/dev/null 2>&1
|
||||
interrupted_rc=$?
|
||||
set -e
|
||||
case "$interrupted_rc" in 0|124|28) ;; *) echo "FAIL interrupted_transfer rc=$interrupted_rc"; exit 3;; esac
|
||||
curl -fsS --netrc-file "$netrc" -T "$work/$large" "$base/remote.php/dav/files/$user/$large"
|
||||
curl -fsS --netrc-file "$netrc" "$base/remote.php/dav/files/$user/$large" -o "$work/$large.download"
|
||||
sha256sum -c "$work/$large.sha256" --ignore-missing >/dev/null
|
||||
printf 'PASS large_file_and_interrupted_transfer size=64MiB interrupted_rc=%s\n' "$interrupted_rc"
|
||||
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php -f /var/www/html/cron.php
|
||||
lastcron=$("$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:app:get core lastcron)
|
||||
[ -n "$lastcron" ] && [ "$lastcron" != "0" ] || { echo "FAIL cron_lastcron_empty"; exit 3; }
|
||||
printf 'PASS cron_exec lastcron=%s\n' "$lastcron"
|
||||
|
||||
printf 'PASS nextcloud_functional_acceptance values=redacted\n'
|
||||
138
phase03/backup-restore-nextcloud.sh
Normal file
138
phase03/backup-restore-nextcloud.sh
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
[ "$(id -u)" = "0" ] || { echo "FAIL must_run_as_root"; exit 1; }
|
||||
umask 077
|
||||
|
||||
rclone_image='rclone/rclone@sha256:623378ad0ff3ebd5cebf77720843c0e02edfe46e2d5b5ac6bed54c6371780dfb'
|
||||
restic_image='restic/restic@sha256:424a4e1fcc6fe2557b5614239dc71a2c793acb33a83ea217171bd7edc1862dcb'
|
||||
docker_bin=/usr/local/bin/docker
|
||||
agent=/usr/local/bin/xtao-secret-agent
|
||||
base=/volume1/xtao-data/backup-controller
|
||||
bin="$base/bin"
|
||||
backup_root=/volume2/xtao-backup/nextcloud
|
||||
tmp=$(mktemp -d /tmp/xtao-nextcloud-backup.XXXXXX)
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT TERM
|
||||
|
||||
mkdir -p "$bin" "$backup_root/dumps" "$backup_root/restores" "$backup_root/logs"
|
||||
chmod 700 "$backup_root" "$backup_root/dumps" "$backup_root/restores" "$backup_root/logs"
|
||||
|
||||
"$agent" raw-field --entry external/google-drive/offsite-backup --field rclone_token_json > "$tmp/token.json"
|
||||
"$agent" raw-field --entry external/google-drive/offsite-backup --field root_folder_id > "$tmp/root_folder_id"
|
||||
"$agent" raw-field --entry external/google-drive/offsite-backup --field oauth_client_id > "$tmp/oauth_client_id"
|
||||
"$agent" raw-field --entry external/google-drive/offsite-backup --field oauth_client_secret > "$tmp/oauth_client_secret"
|
||||
"$agent" raw-field --entry backup/google-drive/restic-repository --field password > "$tmp/restic.pass"
|
||||
chmod 600 "$tmp/token.json" "$tmp/root_folder_id" "$tmp/oauth_client_id" "$tmp/oauth_client_secret" "$tmp/restic.pass"
|
||||
|
||||
python3 - "$tmp/token.json" "$tmp/root_folder_id" "$tmp/oauth_client_id" "$tmp/oauth_client_secret" "$tmp/rclone.conf" <<'PY'
|
||||
import json, pathlib, sys
|
||||
token_path, root_path, client_id_path, client_secret_path, config_path = map(pathlib.Path, sys.argv[1:])
|
||||
token = token_path.read_text(encoding="utf-8").strip()
|
||||
json.loads(token)
|
||||
root = root_path.read_text(encoding="utf-8").strip()
|
||||
if not root or root == "CODEX_GENERATE":
|
||||
raise SystemExit("FAIL invalid_root_folder_id")
|
||||
client_id = client_id_path.read_text(encoding="utf-8").strip()
|
||||
client_secret = client_secret_path.read_text(encoding="utf-8").strip()
|
||||
client_lines = ""
|
||||
if client_id and client_id != "NONE":
|
||||
client_lines += f"client_id = {client_id}\n"
|
||||
if client_secret:
|
||||
client_lines += f"client_secret = {client_secret}\n"
|
||||
config_path.write_text(
|
||||
"[xtao-gdrive]\n"
|
||||
"type = drive\n"
|
||||
"scope = drive.file\n"
|
||||
f"{client_lines}"
|
||||
f"root_folder_id = {root}\n"
|
||||
f"token = {token}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
PY
|
||||
chmod 600 "$tmp/rclone.conf"
|
||||
|
||||
if [ ! -x "$bin/rclone" ]; then
|
||||
cid=$("$docker_bin" create --entrypoint /bin/sh "$rclone_image" -c true)
|
||||
"$docker_bin" cp "$cid:/usr/local/bin/rclone" "$tmp/rclone"
|
||||
"$docker_bin" rm "$cid" >/dev/null
|
||||
install -m 700 "$tmp/rclone" "$bin/rclone"
|
||||
fi
|
||||
|
||||
dbuser=$("$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php -r 'include "/var/www/html/config/config.php"; echo $CONFIG["dbuser"];')
|
||||
dbname=$("$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php -r 'include "/var/www/html/config/config.php"; echo $CONFIG["dbname"];')
|
||||
case "$dbuser" in *[!A-Za-z0-9_-]*|"") echo "FAIL unexpected_dbuser"; exit 3;; esac
|
||||
case "$dbname" in *[!A-Za-z0-9_-]*|"") echo "FAIL unexpected_dbname"; exit 3;; esac
|
||||
|
||||
ts=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
dump="$backup_root/dumps/nextcloud-$ts.sql"
|
||||
"$docker_bin" exec -i nextcloud-nextcloud-db-1 pg_dump -U nextcloud "$dbname" > "$dump"
|
||||
chmod 600 "$dump"
|
||||
|
||||
sample=$(find /volume1/xtao-data/nextcloud/data/xtao-admin/files -maxdepth 1 -type f -name 'phase03-small-*.bin' | sort | tail -n 1)
|
||||
[ -n "$sample" ] || { echo "FAIL missing_uploaded_canary_file"; exit 3; }
|
||||
sample_name=$(basename "$sample")
|
||||
sha256sum "$sample" | awk '{print $1}' > "$tmp/sample.sha256"
|
||||
|
||||
restic_env="-e RCLONE_CONFIG=/config/rclone/rclone.conf -e RESTIC_PASSWORD_FILE=/secrets/restic.pass -e RESTIC_REPOSITORY=rclone:xtao-gdrive:restic/nas"
|
||||
set +e
|
||||
"$docker_bin" run --rm --network bridge \
|
||||
-v "$tmp:/config/rclone" \
|
||||
-v "$tmp/restic.pass:/secrets/restic.pass:ro" \
|
||||
-v "$bin/rclone:/usr/local/bin/rclone:ro" \
|
||||
$restic_env \
|
||||
"$restic_image" init >/tmp/xtao-nextcloud-restic-init.out 2>&1
|
||||
rc=$?
|
||||
set -e
|
||||
if [ "$rc" -ne 0 ] && ! grep -Eq 'already initialized|config file already exists' /tmp/xtao-nextcloud-restic-init.out; then
|
||||
cat /tmp/xtao-nextcloud-restic-init.out
|
||||
echo "FAIL restic_init"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
"$docker_bin" run --rm --network bridge \
|
||||
-v "$tmp:/config/rclone" \
|
||||
-v "$tmp/restic.pass:/secrets/restic.pass:ro" \
|
||||
-v "$bin/rclone:/usr/local/bin/rclone:ro" \
|
||||
-v "/volume1/xtao-data/nextcloud/data/xtao-admin/files:/backup/files:ro" \
|
||||
-v "/volume1/xtao-data/nextcloud/html/config:/backup/config:ro" \
|
||||
-v "$backup_root/dumps:/backup/dbdumps:ro" \
|
||||
$restic_env \
|
||||
"$restic_image" backup --tag phase03-nextcloud-$ts /backup/files /backup/config /backup/dbdumps >/tmp/xtao-nextcloud-restic-backup.out
|
||||
|
||||
restore="$backup_root/restores/restore-$ts"
|
||||
mkdir -p "$restore"
|
||||
chmod 700 "$restore"
|
||||
"$docker_bin" run --rm --network bridge \
|
||||
-v "$tmp:/config/rclone" \
|
||||
-v "$tmp/restic.pass:/secrets/restic.pass:ro" \
|
||||
-v "$bin/rclone:/usr/local/bin/rclone:ro" \
|
||||
-v "$restore:/restore" \
|
||||
$restic_env \
|
||||
"$restic_image" restore latest --tag "phase03-nextcloud-$ts" --target /restore >/tmp/xtao-nextcloud-restic-restore.out
|
||||
|
||||
restored_sample="$restore/backup/files/$sample_name"
|
||||
[ -f "$restored_sample" ] || { echo "FAIL restored_sample_missing"; exit 3; }
|
||||
sha256sum "$restored_sample" | awk '{print $1}' > "$tmp/restored.sha256"
|
||||
cmp "$tmp/sample.sha256" "$tmp/restored.sha256"
|
||||
[ -f "$restore/backup/config/config.php" ] || { echo "FAIL restored_config_missing"; exit 3; }
|
||||
[ -f "$restore/backup/dbdumps/$(basename "$dump")" ] || { echo "FAIL restored_dump_missing"; exit 3; }
|
||||
|
||||
restore_db="nc_restore_$(date -u +%H%M%S)"
|
||||
"$docker_bin" exec -i nextcloud-nextcloud-db-1 createdb -U nextcloud "$restore_db"
|
||||
set +e
|
||||
"$docker_bin" exec -i nextcloud-nextcloud-db-1 psql -U nextcloud "$restore_db" < "$restore/backup/dbdumps/$(basename "$dump")" >/tmp/xtao-nextcloud-db-restore.out 2>&1
|
||||
restore_rc=$?
|
||||
set -e
|
||||
if [ "$restore_rc" -ne 0 ]; then
|
||||
"$docker_bin" exec -i nextcloud-nextcloud-db-1 dropdb -U nextcloud "$restore_db" || true
|
||||
cat /tmp/xtao-nextcloud-db-restore.out
|
||||
echo "FAIL isolated_db_restore"
|
||||
exit 3
|
||||
fi
|
||||
user_count=$("$docker_bin" exec -i nextcloud-nextcloud-db-1 psql -U nextcloud "$restore_db" -Atc "select count(*) from oc_users where uid='xtao-admin';")
|
||||
file_count=$("$docker_bin" exec -i nextcloud-nextcloud-db-1 psql -U nextcloud "$restore_db" -Atc "select count(*) from oc_filecache where path='files/$sample_name';")
|
||||
"$docker_bin" exec -i nextcloud-nextcloud-db-1 dropdb -U nextcloud "$restore_db"
|
||||
[ "$user_count" = "1" ] || { echo "FAIL restored_user_missing"; exit 3; }
|
||||
[ "$file_count" != "0" ] || { echo "FAIL restored_filecache_missing"; exit 3; }
|
||||
|
||||
printf 'PASS nextcloud_backup_restore restic=PASS isolated_db=PASS restored_file=%s values=redacted\n' "$sample_name"
|
||||
123
phase03/deploy-nextcloud.sh
Normal file
123
phase03/deploy-nextcloud.sh
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
src_dir=${1:-/tmp/xtao-phase03}
|
||||
compose_src="$src_dir/compose.yaml"
|
||||
traefik_src="$src_dir/nextcloud-traefik.yml"
|
||||
runner_src="$src_dir/secret_agent_runner_phase03.py"
|
||||
|
||||
docker_bin=/usr/local/bin/docker
|
||||
compose_bin=/usr/local/bin/docker-compose
|
||||
agent=/usr/local/bin/xtao-secret-agent
|
||||
ts=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
|
||||
[ "$(id -u)" = "0" ] || { echo "FAIL must_run_as_root"; exit 1; }
|
||||
[ -r "$compose_src" ] || { echo "FAIL missing_compose_src"; exit 2; }
|
||||
[ -r "$traefik_src" ] || { echo "FAIL missing_traefik_src"; exit 2; }
|
||||
[ -r "$runner_src" ] || { echo "FAIL missing_runner_src"; exit 2; }
|
||||
[ -x "$agent" ] || { echo "FAIL missing_secret_agent"; exit 2; }
|
||||
|
||||
install -d -m 700 /volume1/xtao-data/nextcloud
|
||||
install -d -m 700 /volume1/xtao-data/nextcloud/secrets
|
||||
install -d -m 700 /volume1/xtao-data/nextcloud/html
|
||||
install -d -m 700 /volume1/xtao-data/nextcloud/data
|
||||
install -d -m 700 /volume1/xtao-data/nextcloud/postgres
|
||||
install -d -m 700 /volume1/xtao-data/nextcloud/redis
|
||||
install -d -m 700 /volume1/xtao-data/nextcloud/backups
|
||||
install -d -m 700 /volume1/xtao-data/traefik/dynamic
|
||||
|
||||
runner_dst=/volume1/xtao-data/secret-agent/bin/secret_agent_runner.py
|
||||
[ -f "$runner_dst" ] && cp -p "$runner_dst" "$runner_dst.bak.phase03.$ts"
|
||||
install -m 600 "$runner_src" "$runner_dst"
|
||||
|
||||
"$agent" probe
|
||||
"$agent" upsert --entry infra/nextcloud/admin
|
||||
"$agent" upsert --entry infra/nextcloud/postgres
|
||||
"$agent" upsert --entry infra/nextcloud/redis
|
||||
"$agent" check --entry infra/nextcloud/admin
|
||||
"$agent" check --entry infra/nextcloud/postgres
|
||||
"$agent" check --entry infra/nextcloud/redis
|
||||
|
||||
tmpdir=$(mktemp -d /volume1/xtao-data/nextcloud/secrets/.tmp.XXXXXX)
|
||||
chmod 700 "$tmpdir"
|
||||
|
||||
postgres_env="$tmpdir/postgres.env"
|
||||
nextcloud_env="$tmpdir/nextcloud.env"
|
||||
redis_conf="$tmpdir/redis.conf"
|
||||
|
||||
{
|
||||
printf 'POSTGRES_DB='
|
||||
"$agent" raw-field --entry infra/nextcloud/postgres --field database
|
||||
printf '\nPOSTGRES_USER='
|
||||
"$agent" raw-field --entry infra/nextcloud/postgres --field username
|
||||
printf '\nPOSTGRES_PASSWORD='
|
||||
"$agent" raw-field --entry infra/nextcloud/postgres --field password
|
||||
printf '\n'
|
||||
} > "$postgres_env"
|
||||
|
||||
{
|
||||
printf 'POSTGRES_HOST=nextcloud-db\n'
|
||||
printf 'POSTGRES_DB='
|
||||
"$agent" raw-field --entry infra/nextcloud/postgres --field database
|
||||
printf '\nPOSTGRES_USER='
|
||||
"$agent" raw-field --entry infra/nextcloud/postgres --field username
|
||||
printf '\nPOSTGRES_PASSWORD='
|
||||
"$agent" raw-field --entry infra/nextcloud/postgres --field password
|
||||
printf '\nREDIS_HOST=nextcloud-redis\nREDIS_HOST_PORT=6379\nREDIS_HOST_PASSWORD='
|
||||
"$agent" raw-field --entry infra/nextcloud/redis --field password
|
||||
printf '\nNEXTCLOUD_ADMIN_USER='
|
||||
"$agent" raw-field --entry infra/nextcloud/admin --field username
|
||||
printf '\nNEXTCLOUD_ADMIN_PASSWORD='
|
||||
"$agent" raw-field --entry infra/nextcloud/admin --field password
|
||||
printf '\nNEXTCLOUD_TRUSTED_DOMAINS=cloud.xtao.net\n'
|
||||
printf 'TRUSTED_PROXIES=172.16.0.0/12 192.168.1.0/24\n'
|
||||
printf 'OVERWRITEHOST=cloud.xtao.net\nOVERWRITEPROTOCOL=https\nOVERWRITECLIURL=https://cloud.xtao.net\n'
|
||||
printf 'APACHE_DISABLE_REWRITE_IP=1\nPHP_MEMORY_LIMIT=1024M\nPHP_UPLOAD_LIMIT=10G\n'
|
||||
} > "$nextcloud_env"
|
||||
|
||||
{
|
||||
printf 'appendonly yes\n'
|
||||
printf 'requirepass '
|
||||
"$agent" raw-field --entry infra/nextcloud/redis --field password
|
||||
printf '\n'
|
||||
} > "$redis_conf"
|
||||
|
||||
chmod 600 "$postgres_env" "$nextcloud_env" "$redis_conf"
|
||||
mv "$postgres_env" /volume1/xtao-data/nextcloud/secrets/postgres.env
|
||||
mv "$nextcloud_env" /volume1/xtao-data/nextcloud/secrets/nextcloud.env
|
||||
mv "$redis_conf" /volume1/xtao-data/nextcloud/secrets/redis.conf
|
||||
chown 999:0 /volume1/xtao-data/nextcloud/secrets/redis.conf
|
||||
chmod 400 /volume1/xtao-data/nextcloud/secrets/redis.conf
|
||||
rmdir "$tmpdir"
|
||||
|
||||
[ -f /volume1/xtao-data/nextcloud/compose.yaml ] && cp -p /volume1/xtao-data/nextcloud/compose.yaml "/volume1/xtao-data/nextcloud/compose.yaml.bak.phase03.$ts"
|
||||
install -m 600 "$compose_src" /volume1/xtao-data/nextcloud/compose.yaml
|
||||
|
||||
[ -f /volume1/xtao-data/traefik/dynamic/nextcloud.yml ] && cp -p /volume1/xtao-data/traefik/dynamic/nextcloud.yml "/volume1/xtao-data/traefik/dynamic/nextcloud.yml.bak.phase03.$ts"
|
||||
install -m 600 "$traefik_src" /volume1/xtao-data/traefik/dynamic/nextcloud.yml
|
||||
|
||||
cd /volume1/xtao-data/nextcloud
|
||||
"$compose_bin" -f compose.yaml config >/dev/null
|
||||
"$compose_bin" -f compose.yaml pull
|
||||
"$compose_bin" -f compose.yaml up -d
|
||||
|
||||
for i in $(seq 1 90); do
|
||||
if "$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ status >/tmp/xtao-nextcloud-occ-status 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 4
|
||||
done
|
||||
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ background:cron
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set trusted_domains 1 --value=cloud.xtao.net
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set trusted_proxies 0 --value=172.16.0.0/12
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set trusted_proxies 1 --value=192.168.1.0/24
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set overwritehost --value=cloud.xtao.net
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set overwriteprotocol --value=https
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set overwrite.cli.url --value=https://cloud.xtao.net
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set htaccess.RewriteBase --value=/
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set maintenance_window_start --type=integer --value=8
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ maintenance:update:htaccess
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ status
|
||||
|
||||
printf 'PASS nextcloud_deploy route=cloud.xtao.net secrets=redacted compose=/volume1/xtao-data/nextcloud/compose.yaml\n'
|
||||
135
phase03/reconcile-nextcloud-secrets.sh
Normal file
135
phase03/reconcile-nextcloud-secrets.sh
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
src_dir=${1:-/tmp/xtao-phase03}
|
||||
runner_src="$src_dir/secret_agent_runner_phase03.py"
|
||||
agent=/usr/local/bin/xtao-secret-agent
|
||||
docker_bin=/usr/local/bin/docker
|
||||
compose_bin=/usr/local/bin/docker-compose
|
||||
ts=$(date -u +%Y%m%dT%H%M%SZ)
|
||||
|
||||
[ "$(id -u)" = "0" ] || { echo "FAIL must_run_as_root"; exit 1; }
|
||||
[ -r "$runner_src" ] || { echo "FAIL missing_runner_src"; exit 2; }
|
||||
|
||||
runner_dst=/volume1/xtao-data/secret-agent/bin/secret_agent_runner.py
|
||||
cp -p "$runner_dst" "$runner_dst.bak.phase03.reconcile.$ts"
|
||||
install -m 600 "$runner_src" "$runner_dst"
|
||||
|
||||
"$agent" upsert --entry infra/nextcloud/admin
|
||||
"$agent" upsert --entry infra/nextcloud/postgres
|
||||
"$agent" upsert --entry infra/nextcloud/redis
|
||||
"$agent" check --entry infra/nextcloud/admin
|
||||
"$agent" check --entry infra/nextcloud/postgres
|
||||
"$agent" check --entry infra/nextcloud/redis
|
||||
|
||||
tmpdir=$(mktemp -d /volume1/xtao-data/nextcloud/secrets/.reconcile.XXXXXX)
|
||||
chmod 700 "$tmpdir"
|
||||
cleanup() {
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
"$agent" raw-field --entry infra/nextcloud/postgres --field password > "$tmpdir/dbpass"
|
||||
"$agent" raw-field --entry infra/nextcloud/redis --field password > "$tmpdir/redispass"
|
||||
"$agent" raw-field --entry infra/nextcloud/admin --field password > "$tmpdir/adminpass"
|
||||
chmod 600 "$tmpdir/dbpass" "$tmpdir/redispass" "$tmpdir/adminpass"
|
||||
|
||||
dbuser=$("$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php -r 'include "/var/www/html/config/config.php"; echo $CONFIG["dbuser"];')
|
||||
dbname=$("$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php -r 'include "/var/www/html/config/config.php"; echo $CONFIG["dbname"];')
|
||||
case "$dbuser" in
|
||||
*[!A-Za-z0-9_-]*|"") echo "FAIL unexpected_dbuser"; exit 3;;
|
||||
esac
|
||||
case "$dbname" in
|
||||
*[!A-Za-z0-9_-]*|"") echo "FAIL unexpected_dbname"; exit 3;;
|
||||
esac
|
||||
|
||||
if grep -q "'" "$tmpdir/dbpass"; then
|
||||
echo "FAIL unexpected_dbpass_character"
|
||||
exit 3
|
||||
fi
|
||||
awk -v u="$dbuser" '{printf "ALTER USER \"%s\" WITH PASSWORD '"'"'%s'"'"';\n", u, $0}' "$tmpdir/dbpass" > "$tmpdir/alter.sql"
|
||||
chmod 600 "$tmpdir/alter.sql"
|
||||
"$docker_bin" exec -i nextcloud-nextcloud-db-1 psql -U nextcloud -d "$dbname" < "$tmpdir/alter.sql" >/dev/null
|
||||
|
||||
{
|
||||
printf 'POSTGRES_HOST=nextcloud-db\nPOSTGRES_DB=%s\nPOSTGRES_USER=%s\nPOSTGRES_PASSWORD=' "$dbname" "$dbuser"
|
||||
cat "$tmpdir/dbpass"
|
||||
printf '\nREDIS_HOST=nextcloud-redis\nREDIS_HOST_PORT=6379\nREDIS_HOST_PASSWORD='
|
||||
cat "$tmpdir/redispass"
|
||||
printf '\nNEXTCLOUD_ADMIN_USER=xtao-admin\nNEXTCLOUD_ADMIN_PASSWORD='
|
||||
cat "$tmpdir/adminpass"
|
||||
printf '\nNEXTCLOUD_TRUSTED_DOMAINS=cloud.xtao.net\n'
|
||||
printf 'TRUSTED_PROXIES=172.16.0.0/12 192.168.1.0/24\n'
|
||||
printf 'OVERWRITEHOST=cloud.xtao.net\nOVERWRITEPROTOCOL=https\nOVERWRITECLIURL=https://cloud.xtao.net\n'
|
||||
printf 'APACHE_DISABLE_REWRITE_IP=1\nPHP_MEMORY_LIMIT=1024M\nPHP_UPLOAD_LIMIT=10G\n'
|
||||
} > "$tmpdir/nextcloud.env"
|
||||
chmod 600 "$tmpdir/nextcloud.env"
|
||||
mv "$tmpdir/nextcloud.env" /volume1/xtao-data/nextcloud/secrets/nextcloud.env
|
||||
|
||||
{
|
||||
printf 'POSTGRES_DB=nextcloud\nPOSTGRES_USER=nextcloud\nPOSTGRES_PASSWORD='
|
||||
cat "$tmpdir/dbpass"
|
||||
printf '\n'
|
||||
} > "$tmpdir/postgres.env"
|
||||
chmod 600 "$tmpdir/postgres.env"
|
||||
mv "$tmpdir/postgres.env" /volume1/xtao-data/nextcloud/secrets/postgres.env
|
||||
|
||||
{
|
||||
printf 'appendonly yes\nrequirepass '
|
||||
cat "$tmpdir/redispass"
|
||||
printf '\n'
|
||||
} > "$tmpdir/redis.conf"
|
||||
chmod 400 "$tmpdir/redis.conf"
|
||||
chown 999:0 "$tmpdir/redis.conf"
|
||||
mv "$tmpdir/redis.conf" /volume1/xtao-data/nextcloud/secrets/redis.conf
|
||||
|
||||
cd /volume1/xtao-data/nextcloud
|
||||
"$compose_bin" -f compose.yaml up -d nextcloud-redis
|
||||
"$docker_bin" restart nextcloud-nextcloud-redis-1 >/dev/null
|
||||
sleep 3
|
||||
|
||||
cat "$tmpdir/dbpass" | "$docker_bin" exec -i -u www-data nextcloud-nextcloud-web-1 php -r '
|
||||
$v = trim(stream_get_contents(STDIN));
|
||||
include "/var/www/html/config/config.php";
|
||||
$CONFIG["dbpassword"] = $v;
|
||||
$out = "<?php\n$"."CONFIG = " . var_export($CONFIG, true) . ";\n";
|
||||
file_put_contents("/var/www/html/config/config.php", $out);
|
||||
'
|
||||
|
||||
cat "$tmpdir/redispass" | "$docker_bin" exec -i -u www-data nextcloud-nextcloud-web-1 php -r '
|
||||
$v = trim(stream_get_contents(STDIN));
|
||||
include "/var/www/html/config/config.php";
|
||||
if (!isset($CONFIG["redis"]) || !is_array($CONFIG["redis"])) {
|
||||
$CONFIG["redis"] = array();
|
||||
}
|
||||
$CONFIG["redis"]["host"] = "nextcloud-redis";
|
||||
$CONFIG["redis"]["port"] = 6379;
|
||||
$CONFIG["redis"]["password"] = $v;
|
||||
$CONFIG["memcache.locking"] = "\\OC\\Memcache\\Redis";
|
||||
$out = "<?php\n$"."CONFIG = " . var_export($CONFIG, true) . ";\n";
|
||||
file_put_contents("/var/www/html/config/config.php", $out);
|
||||
'
|
||||
|
||||
{
|
||||
cat "$tmpdir/adminpass"
|
||||
printf '\n'
|
||||
cat "$tmpdir/adminpass"
|
||||
printf '\n'
|
||||
} | "$docker_bin" exec -i -u www-data nextcloud-nextcloud-web-1 php occ user:resetpassword xtao-admin >/dev/null
|
||||
|
||||
"$compose_bin" -f compose.yaml up -d
|
||||
sleep 5
|
||||
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ background:cron
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set trusted_domains 1 --value=cloud.xtao.net
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set trusted_proxies 0 --value=172.16.0.0/12
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set trusted_proxies 1 --value=192.168.1.0/24
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set overwritehost --value=cloud.xtao.net
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set overwriteprotocol --value=https
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set overwrite.cli.url --value=https://cloud.xtao.net
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set htaccess.RewriteBase --value=/
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ config:system:set maintenance_window_start --type=integer --value=8
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ maintenance:update:htaccess
|
||||
"$docker_bin" exec -u www-data nextcloud-nextcloud-web-1 php occ status
|
||||
|
||||
printf 'PASS nextcloud_secret_reconcile values=redacted\n'
|
||||
|
|
@ -18,6 +18,9 @@ ALLOWED = {
|
|||
"infra/traefik/monitor": {"username": "traefik-monitor", "token": "token"},
|
||||
"infra/forgejo/admin": {"username": "forgejo-admin", "password": "secret"},
|
||||
"infra/forgejo/postgres": {"username": "forgejo", "database": "forgejo", "password": "secret"},
|
||||
"infra/nextcloud/admin": {"username": "xtao-admin", "password": "secret"},
|
||||
"infra/nextcloud/postgres": {"username": "nextcloud", "database": "nextcloud", "password": "secret"},
|
||||
"infra/nextcloud/redis": {"password": "secret"},
|
||||
"backup/google-drive/restic-repository": {"repository": "gdrive:xtao-backup/restic/nas", "password": "secret"},
|
||||
}
|
||||
READ_ALLOWED = {
|
||||
|
|
@ -76,7 +79,11 @@ def upsert(kp, path):
|
|||
entry.notes = "generated_by=nas-secret-agent"
|
||||
for key, value in profile.items():
|
||||
if value in ("secret", "token"):
|
||||
entry.set_custom_property(key, gen(value), protect=True)
|
||||
current = entry.get_custom_property(key)
|
||||
if current in (None, ""):
|
||||
entry.set_custom_property(key, gen(value), protect=True)
|
||||
elif not entry.is_custom_property_protected(key):
|
||||
raise SystemExit(f"FAIL unprotected_existing_field path={path} field={key}")
|
||||
else:
|
||||
entry.set_custom_property(key, value, protect=False)
|
||||
return sorted(profile)
|
||||
|
|
|
|||
Loading…
Reference in a new issue