From 1b159fc3d66f35230aa845c2fce3cdebdcef1746 Mon Sep 17 00:00:00 2001 From: XTAO Deployment Bot Date: Thu, 16 Jul 2026 23:33:08 -0600 Subject: [PATCH] phase02 nas desired state --- README.md | 3 + SHA256SUMS | 11 ++ backup-controller/xtao-backup-controller | 115 +++++++++++ edge-reconciler/reconcile_cloudflare_dns.py | 81 ++++++++ edge/compose.yaml | 82 ++++++++ edge/traefik-dynamic.yml | 52 +++++ edge/traefik.yml | 27 +++ kuma/configure_kuma_public_status.py | 27 +++ secret-agent/install-root-unlock.sh | 35 ++++ secret-agent/secret_agent_runner.py | 206 ++++++++++++++++++++ secret-agent/xtao-secret-agent | 55 ++++++ 11 files changed, 694 insertions(+) create mode 100644 README.md create mode 100644 SHA256SUMS create mode 100755 backup-controller/xtao-backup-controller create mode 100755 edge-reconciler/reconcile_cloudflare_dns.py create mode 100644 edge/compose.yaml create mode 100644 edge/traefik-dynamic.yml create mode 100644 edge/traefik.yml create mode 100755 kuma/configure_kuma_public_status.py create mode 100755 secret-agent/install-root-unlock.sh create mode 100644 secret-agent/secret_agent_runner.py create mode 100755 secret-agent/xtao-secret-agent diff --git a/README.md b/README.md new file mode 100644 index 0000000..07e5ffa --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# XTAO NAS Phase 02 Desired State + +Non-secret desired state for Traefik, Uptime Kuma public status, Forgejo edge stack, Secret Agent, Edge Reconciler and Backup Controller. Secret values are referenced by KDBX paths only and are not stored here. diff --git a/SHA256SUMS b/SHA256SUMS new file mode 100644 index 0000000..076e402 --- /dev/null +++ b/SHA256SUMS @@ -0,0 +1,11 @@ +0aec27176d0bb0053a592fe9e9bcc37118b27f55541a6825eed5bb5cef6a60c7 state/implementation/desired-state/README.md +9d4d0f119d29eb5622978d531e27998f97b986106c8426d6ef01dc7dfe5ba937 state/implementation/desired-state/SHA256SUMS +4020b538e0499d12195375b5146634799cd96929826ede63bee3d1fd973c518b state/implementation/desired-state/backup-controller/xtao-backup-controller +9a434c629f955a7298995fe48415cb8bba95b12594684a8bb120ee37514c3f3a state/implementation/desired-state/edge-reconciler/reconcile_cloudflare_dns.py +0b19202e23cc88d1247c0359a20bacabfbd9074c985bcf5777d994371dabb041 state/implementation/desired-state/edge/compose.yaml +99c2d0f3829437d6f6617c55672b535d0e109a512fd55218b69e43d92319a953 state/implementation/desired-state/edge/traefik-dynamic.yml +73a1906f4a950e2e7f3cc17782f64bbab0403461d8016c2223fcb751255400c9 state/implementation/desired-state/edge/traefik.yml +256dc4be0585b645d62c7765ae0579e109ee07039a65dbcf171c25debdc3b731 state/implementation/desired-state/kuma/configure_kuma_public_status.py +10172af7ee895c1d593f654d4ab8cb5195d40d15cc65d1a09fb2c210b426edd2 state/implementation/desired-state/secret-agent/install-root-unlock.sh +8f59112e44e725de60f476b6a43215a598b6f0328387437c80bfdd299964eebb state/implementation/desired-state/secret-agent/secret_agent_runner.py +5128c2db987e30d08e3d6fe1af89febb71d701955f969ca88081aafd37390337 state/implementation/desired-state/secret-agent/xtao-secret-agent diff --git a/backup-controller/xtao-backup-controller b/backup-controller/xtao-backup-controller new file mode 100755 index 0000000..83fbf11 --- /dev/null +++ b/backup-controller/xtao-backup-controller @@ -0,0 +1,115 @@ +#!/bin/sh +set -eu + +[ "$(id -u)" = "0" ] || { echo "FAIL must_run_as_root"; exit 1; } +umask 077 + +rclone_image='rclone/rclone@sha256:623378ad0ff3ebd5cebf77720843c0e02edfe46e2d5b5ac6bed54c6371780dfb' +restic_image='restic/restic@sha256:424a4e1fcc6fe2557b5614239dc71a2c793acb33a83ea217171bd7edc1862dcb' +base=/volume1/xtao-data/backup-controller +ssd=/volume2/xtao-backup/backup-controller +bin="$base/bin" +tmp=$(mktemp -d /tmp/xtao-backup-controller.XXXXXX) +trap 'rm -rf "$tmp"' EXIT HUP INT TERM + +mkdir -p "$bin" "$base/canary-source" "$ssd/restores" "$ssd/logs" +chmod 700 "$base" "$bin" "$base/canary-source" "$ssd" "$ssd/restores" "$ssd/logs" + +/usr/local/bin/xtao-secret-agent raw-field --entry external/google-drive/offsite-backup --field rclone_token_json > "$tmp/token.json" +/usr/local/bin/xtao-secret-agent raw-field --entry external/google-drive/offsite-backup --field root_folder_id > "$tmp/root_folder_id" +/usr/local/bin/xtao-secret-agent raw-field --entry external/google-drive/offsite-backup --field oauth_client_id > "$tmp/oauth_client_id" +/usr/local/bin/xtao-secret-agent raw-field --entry external/google-drive/offsite-backup --field oauth_client_secret > "$tmp/oauth_client_secret" +/usr/local/bin/xtao-secret-agent raw-field --entry backup/google-drive/restic-repository --field password > "$tmp/restic.pass" +chmod 600 "$tmp/token.json" "$tmp/root_folder_id" "$tmp/oauth_client_id" "$tmp/oauth_client_secret" "$tmp/restic.pass" + +python3 - "$tmp/token.json" "$tmp/root_folder_id" "$tmp/oauth_client_id" "$tmp/oauth_client_secret" "$tmp/rclone.conf" <<'PY' +import json, pathlib, sys +token_path, root_path, client_id_path, client_secret_path, config_path = map(pathlib.Path, sys.argv[1:]) +token = token_path.read_text(encoding="utf-8").strip() +json.loads(token) +root = root_path.read_text(encoding="utf-8").strip() +if not root or root == "CODEX_GENERATE": + raise SystemExit("FAIL invalid_root_folder_id") +client_id = client_id_path.read_text(encoding="utf-8").strip() +client_secret = client_secret_path.read_text(encoding="utf-8").strip() +client_lines = "" +if client_id and client_id != "NONE": + client_lines += f"client_id = {client_id}\n" +if client_secret: + client_lines += f"client_secret = {client_secret}\n" +config_path.write_text( + "[xtao-gdrive]\n" + "type = drive\n" + "scope = drive.file\n" + f"{client_lines}" + f"root_folder_id = {root}\n" + f"token = {token}\n", + encoding="utf-8", +) +PY +chmod 600 "$tmp/rclone.conf" + +if [ ! -x "$bin/rclone" ]; then + cid=$(/usr/local/bin/docker create --entrypoint /bin/sh "$rclone_image" -c true) + /usr/local/bin/docker cp "$cid:/usr/local/bin/rclone" "$tmp/rclone" + /usr/local/bin/docker rm "$cid" >/dev/null + install -m 700 "$tmp/rclone" "$bin/rclone" +fi + +canary_name="phase02-canary-$(date -u +%Y%m%dT%H%M%SZ).txt" +printf 'xtao backup controller canary %s\n' "$canary_name" > "$tmp/canary.txt" +sha256sum "$tmp/canary.txt" | awk '{print $1}' > "$tmp/canary.sha256" + +/usr/local/bin/docker run --rm --network bridge \ + -v "$tmp:/config/rclone" \ + -v "$tmp:/work" \ + "$rclone_image" copyto /work/canary.txt "xtao-gdrive:phase02-canary/$canary_name" >/dev/null +/usr/local/bin/docker run --rm --network bridge \ + -v "$tmp:/config/rclone" \ + -v "$tmp:/work" \ + "$rclone_image" copyto "xtao-gdrive:phase02-canary/$canary_name" /work/canary.download >/dev/null +sha256sum "$tmp/canary.download" | awk '{print $1}' > "$tmp/canary.download.sha256" +cmp "$tmp/canary.sha256" "$tmp/canary.download.sha256" +/usr/local/bin/docker run --rm --network bridge \ + -v "$tmp:/config/rclone" \ + "$rclone_image" deletefile "xtao-gdrive:phase02-canary/$canary_name" >/dev/null + +sample="$base/canary-source/sample.txt" +printf 'xtao restic restore sample %s\n' "$canary_name" > "$sample" +sha256sum "$sample" | awk '{print $1}' > "$tmp/sample.sha256" + +restic_env="-e RCLONE_CONFIG=/config/rclone/rclone.conf -e RESTIC_PASSWORD_FILE=/secrets/restic.pass -e RESTIC_REPOSITORY=rclone:xtao-gdrive:restic/nas" +set +e +/usr/local/bin/docker run --rm --network bridge \ + -v "$tmp:/config/rclone" \ + -v "$tmp/restic.pass:/secrets/restic.pass:ro" \ + -v "$bin/rclone:/usr/local/bin/rclone:ro" \ + $restic_env \ + "$restic_image" init >/tmp/xtao-restic-init.out 2>&1 +rc=$? +set -e +if [ "$rc" -ne 0 ] && ! grep -Eq 'already initialized|config file already exists' /tmp/xtao-restic-init.out; then + cat /tmp/xtao-restic-init.out + echo "FAIL restic_init" + exit 2 +fi +/usr/local/bin/docker run --rm --network bridge \ + -v "$tmp:/config/rclone" \ + -v "$tmp/restic.pass:/secrets/restic.pass:ro" \ + -v "$bin/rclone:/usr/local/bin/rclone:ro" \ + -v "$base/canary-source:/data/canary-source:ro" \ + $restic_env \ + "$restic_image" backup /data/canary-source >/tmp/xtao-restic-backup.out +restore="$tmp/restore" +mkdir -p "$restore" +/usr/local/bin/docker run --rm --network bridge \ + -v "$tmp:/config/rclone" \ + -v "$tmp/restic.pass:/secrets/restic.pass:ro" \ + -v "$bin/rclone:/usr/local/bin/rclone:ro" \ + -v "$restore:/restore" \ + $restic_env \ + "$restic_image" restore latest --target /restore >/tmp/xtao-restic-restore.out +sha256sum "$restore/data/canary-source/sample.txt" | awk '{print $1}' > "$tmp/restore.sha256" +cmp "$tmp/sample.sha256" "$tmp/restore.sha256" +cp "$tmp/restore.sha256" "$ssd/logs/last-restore.sha256" +echo "PASS backup_controller gdrive_canary=PASS restic_backup_restore=PASS values=redacted" diff --git a/edge-reconciler/reconcile_cloudflare_dns.py b/edge-reconciler/reconcile_cloudflare_dns.py new file mode 100755 index 0000000..92202ae --- /dev/null +++ b/edge-reconciler/reconcile_cloudflare_dns.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +import argparse +import ipaddress +import json +import sys +import urllib.parse +import urllib.request + + +def request(method, base, path, token, payload=None): + data = None + headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + if payload is not None: + data = json.dumps(payload, separators=(",", ":")).encode("utf-8") + req = urllib.request.Request(base.rstrip("/") + path, data=data, headers=headers, method=method) + with urllib.request.urlopen(req, timeout=30) as resp: + body = json.loads(resp.read().decode("utf-8")) + if not body.get("success"): + raise SystemExit("FAIL cloudflare_api_error") + return body["result"] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--api-base", required=True) + parser.add_argument("--zone", required=True) + parser.add_argument("--public-ip", required=True) + parser.add_argument("--apply", action="store_true") + parser.add_argument("--output", required=True) + args = parser.parse_args() + + token = sys.stdin.read().strip() + if not token: + raise SystemExit("FAIL missing_token_stdin") + ipaddress.ip_address(args.public_ip) + + zones = request("GET", args.api_base, "/zones?name=" + urllib.parse.quote(args.zone), token) + if len(zones) != 1: + raise SystemExit(f"FAIL zone_lookup count={len(zones)}") + zone_id = zones[0]["id"] + desired = { + f"git.{args.zone}": {"type": "A", "content": args.public_ip, "proxied": False, "ttl": 300}, + f"status.{args.zone}": {"type": "A", "content": args.public_ip, "proxied": False, "ttl": 300}, + } + + existing = request("GET", args.api_base, f"/zones/{zone_id}/dns_records?per_page=500", token) + by_name_type = {(r["name"], r["type"]): r for r in existing} + plan = [] + for name, spec in sorted(desired.items()): + current = by_name_type.get((name, spec["type"])) + if current is None: + plan.append({"action": "create", "name": name, **spec}) + else: + drift = any(current.get(k) != v for k, v in spec.items()) + if drift: + plan.append({"action": "update", "id": current["id"], "name": name, **spec}) + + if args.apply: + for item in plan: + payload = {k: item[k] for k in ("type", "name", "content", "ttl", "proxied")} + if item["action"] == "create": + request("POST", args.api_base, f"/zones/{zone_id}/dns_records", token, payload) + else: + request("PUT", args.api_base, f"/zones/{zone_id}/dns_records/{item['id']}", token, payload) + + result = { + "zone": args.zone, + "zone_id": zone_id, + "public_ip": args.public_ip, + "apply": bool(args.apply), + "changes": [{k: v for k, v in item.items() if k != "id"} for item in plan], + "preserved_record_types": ["MX", "TXT", "SPF", "DKIM", "DMARC", "verification"], + } + with open(args.output, "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + f.write("\n") + print(f"PASS edge_reconciler apply={bool(args.apply)} changes={len(plan)} zone={args.zone} values=redacted") + + +if __name__ == "__main__": + main() diff --git a/edge/compose.yaml b/edge/compose.yaml new file mode 100644 index 0000000..a0759a6 --- /dev/null +++ b/edge/compose.yaml @@ -0,0 +1,82 @@ +services: + traefik: + image: traefik@sha256:8516638b18e67e999d293e4ff0e5baf7807674cd4bdd3d36d448497bcbf0a174 + restart: unless-stopped + command: + - --configFile=/etc/traefik/traefik.yml + volumes: + - /volume1/xtao-data/traefik/traefik.yml:/etc/traefik/traefik.yml:ro + - /volume1/xtao-data/traefik/dynamic:/etc/traefik/dynamic:ro + - /volume1/xtao-data/traefik/certs:/certs:ro + networks: + xtao_edge_macvlan: + ipv4_address: 192.168.1.202 + xtao_edge_internal: {} + + uptime-kuma: + image: louislam/uptime-kuma@sha256:b4c3a4d186b4612b3a7bbb39c56a634626276615c21871c837a86e4a43d8e047 + restart: unless-stopped + volumes: + - /volume1/xtao-data/uptime-kuma:/app/data + networks: + - xtao_edge_internal + + forgejo-db: + image: postgres@sha256:f5aa7cac1dc299a77761ada5a5cf9c6ef4064bc85c89fed49c587dd2381d6f05 + restart: unless-stopped + env_file: + - /volume1/xtao-data/secrets/forgejo-postgres.env + environment: + POSTGRES_USER: forgejo + POSTGRES_DB: forgejo + volumes: + - /volume1/xtao-data/postgres/forgejo:/var/lib/postgresql/data + networks: + - xtao_edge_internal + + forgejo: + image: codeberg.org/forgejo/forgejo@sha256:0f07dbf37c585fd0f90b5bb27d66540fd0cf197f5ee79b493be9ee18787797f9 + restart: unless-stopped + depends_on: + - forgejo-db + env_file: + - /volume1/xtao-data/secrets/forgejo-postgres.env + environment: + USER_UID: "1028" + USER_GID: "100" + FORGEJO__server__DOMAIN: git.xtao.net + FORGEJO__server__ROOT_URL: https://git.xtao.net/ + FORGEJO__server__HTTP_PORT: "3000" + FORGEJO__service__DISABLE_REGISTRATION: "true" + FORGEJO__security__INSTALL_LOCK: "true" + FORGEJO__database__DB_TYPE: postgres + FORGEJO__database__HOST: forgejo-db:5432 + FORGEJO__database__NAME: forgejo + FORGEJO__database__USER: forgejo + volumes: + - /volume1/xtao-data/forgejo:/data + networks: + - xtao_edge_internal + + registry: + image: registry@sha256:46faa9a1ae6813194b53921a370f2f4f8c5e1aae228a89bceafef5847a6a3278 + restart: unless-stopped + volumes: + - /volume1/xtao-data/registry:/var/lib/registry + networks: + - xtao_edge_internal + +networks: + xtao_edge_internal: + name: xtao_edge_internal + driver: bridge + xtao_edge_macvlan: + name: xtao_edge_macvlan + driver: macvlan + driver_opts: + parent: ovs_eth0 + ipam: + config: + - subnet: 192.168.1.0/24 + gateway: 192.168.1.1 + ip_range: 192.168.1.202/32 diff --git a/edge/traefik-dynamic.yml b/edge/traefik-dynamic.yml new file mode 100644 index 0000000..3e98c98 --- /dev/null +++ b/edge/traefik-dynamic.yml @@ -0,0 +1,52 @@ +tls: + certificates: + - certFile: /certs/shadow.crt + keyFile: /certs/shadow.key + +http: + routers: + forgejo: + rule: Host(`git.xtao.net`) + entryPoints: [websecure] + service: forgejo + tls: {} + kuma-status: + rule: Host(`status.xtao.net`) && PathPrefix(`/status`) + entryPoints: [websecure] + service: uptime-kuma + tls: {} + kuma-status-root: + rule: Host(`status.xtao.net`) && Path(`/`) + entryPoints: [websecure] + middlewares: [status-root-redirect] + service: noop + tls: {} + reject: + rule: PathPrefix(`/`) + entryPoints: [websecure] + priority: 1 + service: reject-sink + tls: {} + services: + forgejo: + loadBalancer: + servers: + - url: http://forgejo:3000 + uptime-kuma: + loadBalancer: + servers: + - url: http://uptime-kuma:3001 + noop: + loadBalancer: + servers: + - url: http://127.0.0.1:9 + reject-sink: + loadBalancer: + servers: + - url: http://127.0.0.1:9 + middlewares: + status-root-redirect: + redirectRegex: + regex: "^https://status\\.xtao\\.net/$" + replacement: "https://status.xtao.net/status/xtao" + permanent: true diff --git a/edge/traefik.yml b/edge/traefik.yml new file mode 100644 index 0000000..3a2363d --- /dev/null +++ b/edge/traefik.yml @@ -0,0 +1,27 @@ +global: + checkNewVersion: false + sendAnonymousUsage: false + +entryPoints: + web: + address: ":80" + http: + redirections: + entryPoint: + to: websecure + scheme: https + websecure: + address: ":443" + +api: + dashboard: true + +providers: + file: + directory: /etc/traefik/dynamic + watch: true + +log: + level: INFO + +accessLog: {} diff --git a/kuma/configure_kuma_public_status.py b/kuma/configure_kuma_public_status.py new file mode 100755 index 0000000..dd48e2f --- /dev/null +++ b/kuma/configure_kuma_public_status.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +import sqlite3 +p = '/volume1/xtao-data/uptime-kuma/kuma.db' +con = sqlite3.connect(p) +con.row_factory = sqlite3.Row +cur = con.cursor() +cur.execute("insert or ignore into status_page (slug,title,description,icon,theme,published,search_engine_index,show_tags,footer_text,show_powered_by,show_certificate_expiry) values (?,?,?,?,?,?,?,?,?,?,?)", ('xtao','XTAO Status','XTAO edge and control plane status','/icon.svg','light',1,0,0,'',0,1)) +page_id = cur.execute("select id from status_page where slug=?", ('xtao',)).fetchone()['id'] +cur.execute("insert or ignore into status_page_cname (status_page_id,domain) values (?,?)", (page_id,'status.xtao.net')) +cur.execute("insert or ignore into `group` (name,public,active,weight,status_page_id) values (?,?,?,?,?)", ('NAS Edge',1,1,1000,page_id)) +group_id = cur.execute("select id from `group` where name=? and status_page_id=?", ('NAS Edge', page_id)).fetchone()['id'] +monitors = [ + ('Traefik HTTPS','https://git.xtao.net/', 'http', 1), + ('Forgejo','https://git.xtao.net/', 'http', 1), + ('Status Page','https://status.xtao.net/status/xtao', 'http', 1), +] +for name, url, typ, ignore_tls in monitors: + row = cur.execute("select id from monitor where name=?", (name,)).fetchone() + if row is None: + cur.execute("insert into monitor (name,active,interval,url,type,ignore_tls,maxretries,retry_interval,accepted_statuscodes_json,method) values (?,?,?,?,?,?,?,?,?,?)", (name,1,60,url,typ,ignore_tls,1,60,'[\"200-299\"]','GET')) + mon_id = cur.lastrowid + else: + mon_id = row['id'] + cur.execute("update monitor set active=1, interval=60, url=?, type=?, ignore_tls=?, accepted_statuscodes_json='[\"200-299\"]', method='GET' where id=?", (url,typ,ignore_tls,mon_id)) + cur.execute("insert or ignore into monitor_group (monitor_id,group_id,weight,send_url) values (?,?,?,?)", (mon_id,group_id,1000,0)) +con.commit() +print(f"PASS kuma_public_status_configured slug=xtao monitors={len(monitors)} values=redacted") diff --git a/secret-agent/install-root-unlock.sh b/secret-agent/install-root-unlock.sh new file mode 100755 index 0000000..74a02ad --- /dev/null +++ b/secret-agent/install-root-unlock.sh @@ -0,0 +1,35 @@ +#!/bin/sh +set -eu + +umask 077 +root_dir=/root/.xtao-secret-agent +shared_dir=/volume1/xtao-data/secret-agent +backup_dir=/volume2/xtao-backup/secret-agent/kdbx +cred_tmp="$root_dir/master.key.$$" + +[ "$(id -u)" = "0" ] || { echo "FAIL must_run_as_root"; exit 1; } + +mkdir -p "$root_dir" "$root_dir/locks" "$root_dir/db" "$shared_dir/bin" "$shared_dir/runtime" "$shared_dir/logs" "$backup_dir" +chmod 700 "$root_dir" "$root_dir/locks" "$root_dir/db" "$shared_dir" "$shared_dir/bin" "$shared_dir/runtime" "$shared_dir/logs" "$backup_dir" + +cat > "$cred_tmp" +chmod 600 "$cred_tmp" +if [ ! -s "$cred_tmp" ]; then + rm -f "$cred_tmp" + echo "FAIL empty_unlock_input" + exit 2 +fi +mv "$cred_tmp" "$root_dir/master.key" +chown root:root "$root_dir/master.key" +chmod 600 "$root_dir/master.key" + +cat > "$root_dir/config.env" <<'EOF' +XTAO_SECRET_AGENT_DB=/root/.xtao-secret-agent/db/xtao-automation.kdbx +XTAO_SECRET_AGENT_PY_IMAGE=python@sha256:cab2dbf575e971934a81e4622f5aba17aa7929719bd7e31033a3a83b97fd0464 +XTAO_SECRET_AGENT_SHARED=/volume1/xtao-data/secret-agent +XTAO_SECRET_AGENT_BACKUP=/volume2/xtao-backup/secret-agent/kdbx +EOF +chown root:root "$root_dir/config.env" +chmod 600 "$root_dir/config.env" + +echo "PASS root_unlock_installed path=/root/.xtao-secret-agent/master.key mode=0600" diff --git a/secret-agent/secret_agent_runner.py b/secret-agent/secret_agent_runner.py new file mode 100644 index 0000000..f4008f1 --- /dev/null +++ b/secret-agent/secret_agent_runner.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +import argparse +import os +import secrets +import shutil +import string +import sys +import tempfile +import time +from pathlib import Path + +from pykeepass import PyKeePass + +ALLOWED = { + "infra/nas/automation": {"username": "xtao-deploy", "password": "secret"}, + "infra/xtao-app/automation": {"username": "xtao-deploy", "password": "secret"}, + "infra/xtao-ai/automation": {"username": "xtao-deploy", "password": "secret"}, + "infra/traefik/monitor": {"username": "traefik-monitor", "token": "token"}, + "infra/forgejo/admin": {"username": "forgejo-admin", "password": "secret"}, + "infra/forgejo/postgres": {"username": "forgejo", "database": "forgejo", "password": "secret"}, + "backup/google-drive/restic-repository": {"repository": "gdrive:xtao-backup/restic/nas", "password": "secret"}, +} +READ_ALLOWED = { + **{path: set(profile) for path, profile in ALLOWED.items()}, + "external/google-drive/offsite-backup": { + "oauth_client_id", + "oauth_client_secret", + "rclone_token_json", + "root_folder_id", + }, +} +ALPHABET = string.ascii_letters + string.digits + "-_" + + +def master_from_stdin(): + value = sys.stdin.read().rstrip("\r\n") + if not value: + raise SystemExit("FAIL missing_master_stdin") + return value + + +def ensure_group(kp, parts): + group = kp.root_group + for name in parts: + found = kp.find_groups(name=name, group=group, recursive=False, first=True) + group = found or kp.add_group(group, name) + return group + + +def find_entry(kp, path): + group = kp.root_group + parts = path.split("/") + for name in parts[:-1]: + group = kp.find_groups(name=name, group=group, recursive=False, first=True) + if group is None: + return None + return kp.find_entries(title=parts[-1], group=group, recursive=False, first=True) + + +def gen(kind): + if kind == "token": + return secrets.token_urlsafe(32) + return "".join(secrets.choice(ALPHABET) for _ in range(40)) + + +def upsert(kp, path): + if path not in ALLOWED: + raise SystemExit(f"FAIL disallowed_entry path={path}") + profile = ALLOWED[path] + group = ensure_group(kp, path.split("/")[:-1]) + entry = kp.find_entries(title=path.split("/")[-1], group=group, recursive=False, first=True) + if entry is None: + entry = kp.add_entry(group, path.split("/")[-1], "", "", force_creation=True) + entry.username = profile.get("username", "") + entry.url = f"credential://{path}" + entry.notes = "generated_by=nas-secret-agent" + for key, value in profile.items(): + if value in ("secret", "token"): + entry.set_custom_property(key, gen(value), protect=True) + else: + entry.set_custom_property(key, value, protect=False) + return sorted(profile) + + +def protected_fields(kp, path): + entry = find_entry(kp, path) + if entry is None: + raise SystemExit(f"FAIL entry_missing path={path}") + fields = [] + for key in sorted((ALLOWED.get(path) or {}).keys()): + value = entry.get_custom_property(key) + if value in (None, ""): + raise SystemExit(f"FAIL empty_field path={path} field={key}") + if key in ("password", "token"): + if not entry.is_custom_property_protected(key): + raise SystemExit(f"FAIL unprotected_field path={path} field={key}") + fields.append(key) + return fields + + +def backup_file(db, backup_dir): + backup_dir.mkdir(parents=True, exist_ok=True) + os.chmod(backup_dir, 0o700) + ts = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) + target = backup_dir / f"{Path(db).name}.{ts}.bak.kdbx" + shutil.copy2(db, target) + os.chmod(target, 0o600) + return target + + +def atomic_upsert(db, master, backup_dir, entry_path): + db = Path(db) + backup = backup_file(db, Path(backup_dir)) + fd, tmp_name = tempfile.mkstemp(prefix=db.name + ".", suffix=".candidate", dir=str(db.parent)) + os.close(fd) + candidate = Path(tmp_name) + try: + shutil.copy2(db, candidate) + os.chmod(candidate, 0o600) + kp = PyKeePass(str(candidate), password=master) + fields = upsert(kp, entry_path) + kp.save() + reopened = PyKeePass(str(candidate), password=master) + protected_fields(reopened, entry_path) + os.replace(candidate, db) + PyKeePass(str(db), password=master) + return backup, fields + except Exception: + if candidate.exists(): + candidate.unlink() + raise + + +def cmd_probe(args, master): + kp = PyKeePass(args.db, password=master) + count = len(kp.entries) + print(f"PASS secret_agent_probe entries={count} values=redacted") + + +def cmd_check(args, master): + kp = PyKeePass(args.db, password=master) + fields = protected_fields(kp, args.entry) + print(f"PASS field_check entry={args.entry} protected={','.join(fields) or 'none'} values=redacted") + + +def cmd_upsert(args, master): + backup, fields = atomic_upsert(args.db, master, args.backup_dir, args.entry) + print(f"PASS upsert entry={args.entry} fields={','.join(fields)} backup_ref={backup.name} values=redacted") + + +def cmd_rollback_test(args, master): + before = Path(args.db).read_bytes() + backup, fields = atomic_upsert(args.db, master, args.backup_dir, "infra/traefik/monitor") + shutil.copy2(backup, args.db) + os.chmod(args.db, 0o600) + PyKeePass(args.db, password=master) + after = Path(args.db).read_bytes() + if before != after: + raise SystemExit("FAIL rollback_mismatch") + print(f"PASS atomic_write_readback_rollback entry=infra/traefik/monitor fields={','.join(fields)} backup_ref={backup.name} values=redacted") + + +def cmd_raw_field(args, master): + allowed_fields = READ_ALLOWED.get(args.entry) + if not allowed_fields or args.field not in allowed_fields: + raise SystemExit(f"FAIL read_disallowed entry={args.entry} field={args.field}") + kp = PyKeePass(args.db, password=master) + entry = find_entry(kp, args.entry) + if entry is None: + raise SystemExit(f"FAIL entry_missing path={args.entry}") + value = entry.get_custom_property(args.field) + if value in (None, ""): + raise SystemExit(f"FAIL empty_field entry={args.entry} field={args.field}") + sys.stdout.write(value) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--db", required=True) + parser.add_argument("--backup-dir", required=True) + sub = parser.add_subparsers(dest="cmd", required=True) + sub.add_parser("probe") + check = sub.add_parser("check") + check.add_argument("--entry", required=True) + upsert_p = sub.add_parser("upsert") + upsert_p.add_argument("--entry", required=True) + sub.add_parser("rollback-test") + raw = sub.add_parser("raw-field") + raw.add_argument("--entry", required=True) + raw.add_argument("--field", required=True) + args = parser.parse_args() + master = master_from_stdin() + if args.cmd == "probe": + cmd_probe(args, master) + elif args.cmd == "check": + cmd_check(args, master) + elif args.cmd == "upsert": + cmd_upsert(args, master) + elif args.cmd == "rollback-test": + cmd_rollback_test(args, master) + elif args.cmd == "raw-field": + cmd_raw_field(args, master) + + +if __name__ == "__main__": + main() diff --git a/secret-agent/xtao-secret-agent b/secret-agent/xtao-secret-agent new file mode 100755 index 0000000..3b00dc9 --- /dev/null +++ b/secret-agent/xtao-secret-agent @@ -0,0 +1,55 @@ +#!/bin/sh +set -eu + +[ "$(id -u)" = "0" ] || { echo "FAIL must_run_as_root"; exit 1; } + +config=/root/.xtao-secret-agent/config.env +[ -r "$config" ] || { echo "FAIL missing_config"; exit 2; } +. "$config" + +master=/root/.xtao-secret-agent/master.key +shared=${XTAO_SECRET_AGENT_SHARED:-/volume1/xtao-data/secret-agent} +db=${XTAO_SECRET_AGENT_DB:-/root/.xtao-secret-agent/db/xtao-automation.kdbx} +image=${XTAO_SECRET_AGENT_PY_IMAGE:-python@sha256:cab2dbf575e971934a81e4622f5aba17aa7929719bd7e31033a3a83b97fd0464} +backup=${XTAO_SECRET_AGENT_BACKUP:-/volume2/xtao-backup/secret-agent/kdbx} +db_dir=$(dirname "$db") +db_name=$(basename "$db") +site="$shared/runtime/site" +wheels="$shared/runtime/wheels" +runner="$shared/bin/secret_agent_runner.py" +lock=/root/.xtao-secret-agent/locks/agent.lock + +[ -s "$master" ] || { echo "FAIL missing_unlock_input"; exit 2; } +[ -s "$db" ] || { echo "FAIL missing_kdbx"; exit 2; } +[ -r "$runner" ] || { echo "FAIL missing_runner"; exit 2; } +mkdir -p "$site" "$backup" "$(dirname "$lock")" +chmod 700 "$shared" "$shared/runtime" "$shared/bin" "$backup" "$(dirname "$lock")" + +ensure_runtime() { + if [ -f "$site/pykeepass/__init__.py" ]; then + return 0 + fi + [ -d "$wheels" ] || { echo "FAIL missing_wheels"; exit 2; } + /usr/local/bin/docker run --rm --network none \ + -v "$wheels:/wheels:ro" \ + -v "$site:/site" \ + "$image" \ + python -m pip install --root-user-action ignore --no-index --find-links /wheels --target /site pykeepass >/dev/null + [ -f "$site/pykeepass/__init__.py" ] || { echo "FAIL runtime_install"; exit 2; } +} + +ensure_runtime + +cmd=${1:-probe} +shift || true + +exec 9>"$lock" +flock -x 9 +cat "$master" | /usr/local/bin/docker run --rm -i --network none \ + -v "$db_dir:/kdbx" \ + -v "$backup:/backup" \ + -v "$site:/site:ro" \ + -v "$runner:/runner/secret_agent_runner.py:ro" \ + -e PYTHONPATH=/site \ + "$image" \ + python /runner/secret_agent_runner.py --db "/kdbx/$db_name" --backup-dir /backup "$cmd" "$@"