From 0d25696860a2d0e2f108ad1d9430cb9419c5f69e Mon Sep 17 00:00:00 2001 From: operator-main Date: Sun, 4 Oct 2026 13:04:45 +0000 Subject: [PATCH] docs: exec-server -> exec-constrained stale references (bl:8444) - docs/TOKEN_POLICY.md: rewritten for exec-constrained.py (named ops, -n exec-constrained, {op,args,ts,nonce} envelope; rotate endpoint gone) - bin/chromebox-gateway.py: exec-server naming -> shared exec token files - docs/DM-HTTPS-DESIGN-646.md + docs/DM-OVER-HTTPS-DESIGN.md: port 8443->8444, namespace exec-server->exec-constrained, envelope updated, cloudflared port fix marked done 2026-10-04 --- bin/chromebox-gateway.py | 364 ++++++++++++++++++++++ docs/DM-HTTPS-DESIGN-646.md | 572 +++++++++++++++++++++++++++++++++++ docs/DM-OVER-HTTPS-DESIGN.md | 24 +- docs/TOKEN_POLICY.md | 73 +++++ 4 files changed, 1021 insertions(+), 12 deletions(-) create mode 100755 bin/chromebox-gateway.py create mode 100644 docs/DM-HTTPS-DESIGN-646.md create mode 100644 docs/TOKEN_POLICY.md diff --git a/bin/chromebox-gateway.py b/bin/chromebox-gateway.py new file mode 100755 index 0000000..2b1e200 --- /dev/null +++ b/bin/chromebox-gateway.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +""" +chromebox-gateway.py — HTTPS fallback for chromebox/DM control when SSH is down. + +The chromeboxes (headless Chromium per NetVM node) are normally driven over +SSH via muse-chat-api.py / dm.py, which speak CDP through the per-node +netns relay. When SSH to bl breaks, this gateway provides a constrained +HTTPS path to the same high-level operations. + +CRITICAL: this does NOT expose raw CDP. Raw CDP (Runtime.evaluate, +Page.navigate, etc.) is arbitrary code execution inside the browser with +the agent's live session. This gateway exposes ONLY the allowlisted +high-level operations below, each mapped to an existing audited script. + +Usage: + python3 chromebox-gateway.py --port 8444 + +Auth: Bearer token, reusing the shared exec per-agent token files +(~/.exec-tokens/). The master token (~/.exec-server-token) also +works. Identity = token filename; tokens are never logged. + +Endpoints: + GET /health {"status":"ok"} — no auth (load-balancer friendly) + POST /api/v1/op {"op": "", "params": {...}} — bearer auth + +Audit: every call appended to ~/.chromebox-gateway-audit.jsonl (0600). +""" +import argparse +import hmac +import json +import os +import re +import ssl +import subprocess +import sys +import time +from http.server import HTTPServer, BaseHTTPRequestHandler +from urllib.parse import urlparse + +# ---------------------------------------------------------------- config + +BIN_DIR = os.path.expanduser("~/Projects/NetVM/bin") +TOKEN_FILE = "/home/super/.exec-server-token" # master token (shared exec token file) +TOKEN_DIR = "/home/super/.exec-tokens" # per-agent tokens (shared exec token files) +AUDIT_FILE = "/home/super/.chromebox-gateway-audit.jsonl" +CERT_FILE = "/home/super/.chromebox-gateway-cert.pem" +KEY_FILE = "/home/super/.chromebox-gateway-key.pem" + +NODES = ("muse", "pip", "646", "opm") +MAX_MSG = 2000 # gateway-level cap; downstream scripts enforce their own +BACKEND_TIMEOUT = 120 # seconds per backend call + +# Rate limit: token bucket per identity — 20 req/min sustained, burst 5. +RATE_PER_SEC = 20.0 / 60.0 +RATE_BURST = 5 + +# ---------------------------------------------------------------- ops +# Each op maps to an argv builder for an existing script. No shell=True, +# ever. Params are validated before building argv. + +def _node(params): + node = params.get("account") or params.get("agent") + if node not in NODES: + raise ValueError("account/agent must be one of %s" % (",".join(NODES))) + return node + +def _msg(params): + m = params.get("message", "") + if not isinstance(m, str) or not m.strip(): + raise ValueError("message must be a non-empty string") + if len(m) > MAX_MSG: + raise ValueError("message exceeds %d chars" % MAX_MSG) + return m + +def _target(params): + t = params.get("target", "main") + if not isinstance(t, str) or not t or len(t) > 128: + raise ValueError("target must be a short string") + if not re.fullmatch(r"[A-Za-z0-9_./:-]+", t): + raise ValueError("target has invalid characters") + if ".." in t: + raise ValueError("target must not contain '..'") + return t + +def _n(params, default=5, cap=50): + n = params.get("n", default) + try: + n = int(n) + except (TypeError, ValueError): + raise ValueError("n must be an integer") + if not 1 <= n <= cap: + raise ValueError("n must be 1..%d" % cap) + return n + +def _tags(params): + tags = params.get("tags", []) + if not isinstance(tags, list): + raise ValueError("tags must be a list") + out = [] + for t in tags: + if not isinstance(t, str) or len(t) > 128: + raise ValueError("bad tag") + if not re.fullmatch(r"[A-Za-z0-9_:=\-./]+", t): + raise ValueError("tag has invalid characters: %r" % t[:40]) + out.append(t) + if len(out) > 10: + raise ValueError("too many tags (max 10)") + return out + +CHAT = os.path.join(BIN_DIR, "muse-chat-api.py") +DM = os.path.join(BIN_DIR, "dm.py") + +def op_chat_send(p): + return [CHAT, "--account", _node(p), "send", _msg(p)] + +def op_chat_messages(p): + return [CHAT, "--account", _node(p), "messages", "--n", str(_n(p))] + +def op_chat_sidechats(p): + return [CHAT, "--account", _node(p), "sidechat", "list"] + +def op_chat_sidechat_create(p): + argv = [CHAT, "--account", _node(p), "sidechat", "create"] + name = p.get("name") + if name: + if not isinstance(name, str) or len(name) > 80 or not re.fullmatch(r"[A-Za-z0-9 _-]+", name): + raise ValueError("bad sidechat name") + argv += ["--name", name] + return argv + +def op_chat_approvals(p): + return [CHAT, "--account", _node(p), "approvals"] + +def op_chat_url(p): + return [CHAT, "--account", _node(p), "url"] + +def op_dm_send(p): + node = _node(p) + to = p.get("to", node) + if to not in NODES: + raise ValueError("to must be one of %s" % (",".join(NODES))) + argv = [DM, "send", "--agent", node, "--to", to, + "--target", _target(p), _msg(p)] + for t in _tags(p): + argv += ["--tag", t] + return argv + +def op_dm_read(p): + return [DM, "read", "--agent", _node(p), + "--target", _target(p), "--n", str(_n(p, default=5, cap=20))] + +# The allowlist. Adding an op here is a security decision — review accordingly. +# Deliberately absent: wait (long-poll), upload (file ingress), +# sidechat use (raw navigation), anything raw-CDP. +ALLOWLIST = { + "chat.send": ("write", op_chat_send), + "chat.messages": ("read", op_chat_messages), + "chat.sidechats": ("read", op_chat_sidechats), + "chat.sidechat_create": ("write", op_chat_sidechat_create), + "chat.approvals": ("read", op_chat_approvals), + "chat.url": ("read", op_chat_url), + "dm.send": ("write", op_dm_send), + "dm.read": ("read", op_dm_read), +} + +# ---------------------------------------------------------------- auth + +def _read_token_file(path): + try: + with open(path) as f: + return f.read().strip() + except OSError: + return "" + +def check_token(token): + """Return identity label or None. Tokens never leave this function.""" + if not token: + return None + if hmac.compare_digest(token, _read_token_file(TOKEN_FILE)): + return "master" + try: + names = os.listdir(TOKEN_DIR) + except OSError: + return None + for name in names: + if not re.fullmatch(r"[A-Za-z0-9_-]+", name): + continue + t = _read_token_file(os.path.join(TOKEN_DIR, name)) + if t and hmac.compare_digest(token, t): + return name + return None + +# ---------------------------------------------------------------- rate limit + +_buckets = {} # identity -> [tokens, last_ts] + +def rate_ok(identity): + now = time.monotonic() + tokens, last = _buckets.get(identity, (RATE_BURST, now)) + tokens = min(RATE_BURST, tokens + (now - last) * RATE_PER_SEC) + if tokens < 1.0: + _buckets[identity] = (tokens, now) + return False + _buckets[identity] = (tokens - 1.0, now) + return True + +# ---------------------------------------------------------------- audit + +def audit(entry): + entry = dict(entry) + entry["ts"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + # Defense in depth: never let a full message body reach the audit file, + # even if a caller forgets to summarize first. + params = entry.get("params") + if isinstance(params, dict) and "message" in params: + params = dict(params) + params["message_len"] = len(params.pop("message")) + entry["params"] = params + try: + fd = os.open(AUDIT_FILE, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) + with os.fdopen(fd, "a") as f: + f.write(json.dumps(entry) + "\n") + except OSError as e: + print("audit write failed: %s" % e, file=sys.stderr) + +# ---------------------------------------------------------------- handler + +class Handler(BaseHTTPRequestHandler): + server_version = "chromebox-gateway/1.0" + + def log_message(self, fmt, *args): # quiet; audit log is the record + pass + + def _json(self, code, obj): + body = json.dumps(obj).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + if urlparse(self.path).path == "/health": + self._json(200, {"status": "ok", "ops": sorted(ALLOWLIST)}) + return + self._json(404, {"error": "not_found"}) + + def do_POST(self): + if urlparse(self.path).path != "/api/v1/op": + self._json(404, {"error": "not_found"}) + return + + auth = self.headers.get("Authorization", "") + token = auth[7:] if auth.startswith("Bearer ") else "" + identity = check_token(token) + if not identity: + self._json(401, {"error": "unauthorized"}) + return + if not rate_ok(identity): + audit({"identity": identity, "op": None, "result": "rate_limited"}) + self._json(429, {"error": "rate_limited"}) + return + + try: + length = int(self.headers.get("Content-Length", 0)) + except ValueError: + length = 0 + if length > 65536: + self._json(413, {"error": "body_too_large"}) + return + try: + req = json.loads(self.rfile.read(length) or b"{}") + except (ValueError, OSError): + self._json(400, {"error": "bad_json"}) + return + + op = req.get("op") + params = req.get("params") or {} + if not isinstance(params, dict): + self._json(400, {"error": "params_must_be_object"}) + return + entry = ALLOWLIST.get(op) + if not entry: + audit({"identity": identity, "op": op, "result": "unknown_op"}) + self._json(400, {"error": "unknown_op", "allowed": sorted(ALLOWLIST)}) + return + cls, builder = entry + + t0 = time.monotonic() + try: + argv = builder(params) + except ValueError as e: + audit({"identity": identity, "op": op, "class": cls, "result": "bad_params", + "detail": str(e)[:120]}) + self._json(400, {"error": "bad_params", "detail": str(e)}) + return + + # Redacted summary for the audit log — never the full message. + summary = {k: (v[:80] + "…" if isinstance(v, str) and len(v) > 80 else v) + for k, v in params.items() if k != "message"} + if "message" in params: + summary["message_len"] = len(params["message"]) + + try: + proc = subprocess.run(argv, capture_output=True, text=True, + timeout=BACKEND_TIMEOUT) + ok = proc.returncode == 0 + result = "ok" if ok else "backend_error" + self._json(200 if ok else 502, { + "ok": ok, + "op": op, + "returncode": proc.returncode, + "stdout": proc.stdout[-8000:], + "stderr": proc.stderr[-2000:], + }) + except subprocess.TimeoutExpired: + result = "timeout" + self._json(504, {"ok": False, "op": op, "error": "backend_timeout"}) + except OSError as e: + result = "exec_failed" + self._json(500, {"ok": False, "op": op, "error": "exec_failed"}) + finally: + audit({"identity": identity, "op": op, "class": cls, + "node": params.get("account") or params.get("agent"), + "params": summary, "result": result, + "latency_ms": int((time.monotonic() - t0) * 1000)}) + +# ---------------------------------------------------------------- main + +def ensure_cert(): + if os.path.exists(CERT_FILE) and os.path.exists(KEY_FILE): + return + print("generating self-signed cert...", file=sys.stderr) + subprocess.run([ + "openssl", "req", "-x509", "-newkey", "rsa:2048", + "-keyout", KEY_FILE, "-out", CERT_FILE, + "-days", "825", "-nodes", "-subj", "/CN=chromebox-gateway", + ], check=True) + os.chmod(KEY_FILE, 0o600) + os.chmod(CERT_FILE, 0o600) + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=8444) + ap.add_argument("--bind", default="100.123.153.75", + help="tailnet IP; use 127.0.0.1 for local-only") + args = ap.parse_args() + + ensure_cert() + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(CERT_FILE, KEY_FILE) + + srv = HTTPServer((args.bind, args.port), Handler) + srv.socket = context.wrap_socket(srv.socket, server_side=True) + print("chromebox-gateway listening on https://%s:%d" % (args.bind, args.port), + file=sys.stderr) + print("ops: %s" % ", ".join(sorted(ALLOWLIST)), file=sys.stderr) + try: + srv.serve_forever() + except KeyboardInterrupt: + pass + +if __name__ == "__main__": + main() diff --git a/docs/DM-HTTPS-DESIGN-646.md b/docs/DM-HTTPS-DESIGN-646.md new file mode 100644 index 0000000..cf7e31c --- /dev/null +++ b/docs/DM-HTTPS-DESIGN-646.md @@ -0,0 +1,572 @@ +# DM over HTTPS: System Design + +**Date:** 2026-10-04 +**Author:** operator-646 (design agent) +**Status:** Design only — no implementation +**Companion specs:** +- `docs/BOX-API-DESIGN-DMS.md` (commit ef8e98a) — endpoint schemas +- `docs/BOX-UI-DESIGN.md` (commit 458314c) — UI decisions +- `docs/DM-SPEC.md` (work orders, server-side logging) +- NetVM repo `DM_SPEC.md` (DM trust model, signatures) + +--- + +## 1. Problem + +Current DM transport is a 3-hop SSH chain: + +``` +operator-646 container + → SSH → VM 34.139.37.135 (dev-muse-646-patha@) + → SSH → bl 100.123.153.75 (super@, tailnet) + → CDP → headless Chromium → muse.ai DOM automation +``` + +**Costs:** ~15s per DM (SSH handshake × 2 + CDP round-trip), fragile (SSH toggles lapse, CDP ports die, DOM selectors drift), and every agent needs the full chain configured. + +**Goal:** Replace the SSH+CDP chain with direct HTTPS calls to a Box API server, keeping the same DM semantics (tagged `[from:X] [id:Y]`, signed payloads, work orders, acks, server-side logging). + +--- + +## 2. Architecture + +### 2.1 Current (SSH chain) + +``` +┌──────────┐ SSH ┌────┐ SSH ┌────┐ CDP ┌──────────┐ +│ agent │ ────── │ VM │ ────── │ bl │ ───── │ chromium │ +│ (any) │ │ │ │ │ │ muse.ai │ +└──────────┘ └────┘ └────┘ └──────────┘ + dm.py runs on bl; agents reach it via SSH +``` + +### 2.2 Target (HTTPS) + +``` +┌──────────┐ HTTPS ┌──────────────────────────────┐ +│ agent │ ────── │ box.muse-dev.online │ +│ (any) │ Bearer │ ┌─────────┐ ┌───────────┐ │ +└──────────┘ or │ │ Box API │→ │ DM queue │──┼──→ dm.py (bl, CDP) + │ signed │ │ server │ │ / logger │ │ (delivery backend) +┌──────────┐ req │ └─────────┘ └───────────┘ │ +│ box UI │ ────── │ ┌─────────┐ │ +│ (human) │ cookie │ │ DMs tab │ (reads from logger) +└──────────┘ │ └─────────┘ │ + └──────────────────────────────┘ +``` + +**Key insight:** The Box API server does NOT replace `dm.py` or CDP — it wraps them. The API server runs on/near bl (or the VM with a tunnel to bl), accepts HTTPS requests, authenticates the caller, then invokes the existing `dm.py send` / `muse-chat-api.py` read path on the backend. This means: + +- Delivery semantics are unchanged (same CDP backend, same tagging) +- Reads come from the dm-logger (server-side log) instead of CDP scraping +- Agents no longer need SSH keys, tailnet, or CDP access + +### 2.3 Delivery backends (pluggable) + +The API server routes each send through a backend: + +| Backend | Mechanism | Latency | Status | +|---------|-----------|---------|--------| +| `cdp` | Existing `dm.py send` via headless Chromium | ~5s | Exists today | +| `direct` | Future: muse.ai server-side API (if ever exposed) | ~100ms | Not available | +| `webhook` | Future: push to agent's own HTTP endpoint | ~100ms | Not built | + +Phase 1 uses `cdp` exclusively. The API abstraction means agents don't care which backend delivers. + +--- + +## 3. Endpoint Mappings: `dm.py` → Box API + +### 3.1 `dm.py send` → `POST /api/box/dms` + +**Current:** +```bash +dm.py send --agent 646 --to opm --target main "message text" +# → DM from 646 to opm/main: SENT +``` + +**HTTPS equivalent:** +```http +POST /api/box/dms +Authorization: +Content-Type: application/json + +{ + "to": "opm", + "target": "main", + "message": "message text", + "kind": "dm" +} +``` + +**Response:** +```json +{ + "dm_id": "a9f5161f", + "from": "646", + "to": "opm", + "target": "main", + "status": "sent", + "request_id": "req_7f3a...", + "logged_at": "2026-10-04T04:00:00Z" +} +``` + +**Field mapping:** + +| dm.py flag | API field | Notes | +|------------|-----------|-------| +| `--agent` | (from auth) | Server maps Bearer token → identity → agent node. Explicit `from` requires `spoof` scope (§2.4 of BOX-API-DESIGN-DMS.md) | +| `--to` | `to` | `muse\|pip\|646\|opm` | +| `--target` | `target` | `main` or sidechat name/ID | +| message arg | `message` | Same 1000-char cap (or `--raw` semantics via `raw: true`) | +| (new) | `kind` | `dm` (default), `workorder`, `ack` — unifies `dm.py workorder` / `dm.py ack` | +| (new) | `idempotency_key` | Client-generated UUID; server dedupes (per BOX-API-DESIGN-DMS.md §2.1) | + +**Work order variant** (replaces `dm.py workorder`): +```json +{ + "to": "muse", + "target": "main", + "kind": "workorder", + "title": "Review board post", + "priority": "urgent", + "due": "2026-10-05T00:00:00Z", + "message": "Please review..." +} +``` + +**Ack variant** (replaces `dm.py ack`): +```json +{ + "to": "646", + "target": "main", + "kind": "ack", + "ack_dm_id": "a9f5161f", + "message": "Done" +} +``` + +### 3.2 `dm.py read` → `GET /api/box/dms` + +**Current:** +```bash +dm.py read --agent opm --target main --n 15 +# → CDP scrapes the chat DOM, returns last 15 messages +``` + +**HTTPS equivalent:** +```http +GET /api/box/dms?agent=opm&target=main&n=15 +Authorization: +``` + +**Response:** +```json +{ + "messages": [ + { + "dm_id": "a9f5161f", + "from": "646", + "to": "opm", + "target": "main", + "kind": "dm", + "message": "ACK - received [4a4031e8]", + "timestamp": "2026-10-04T03:54:10Z", + "verified": true + } + ], + "has_more": false +} +``` + +**Critical change:** Reads come from the **dm-logger** (server-side append-only log), not CDP DOM scraping. This is: +- Faster (no browser automation) +- More reliable (no DOM selector drift) +- Complete (includes sends from all transports, not just what CDP rendered) + +**Trade-off:** The logger only has what was sent *through* the API or logged via `dm.py log`. DMs sent via raw CDP before logger integration won't appear. Migration must ensure `dm.py send` always logs (already does per DM-SPEC.md). + +**Query params:** + +| Param | Purpose | +|-------|---------| +| `agent` | Whose chat to read (requires `dms:read` scope; reading another agent's DMs may require operator level) | +| `target` | `main` or sidechat name | +| `n` | Limit (default 20, max 100) | +| `since` | ISO timestamp — return messages after this (for polling) | +| `thread_id` | Filter to one conversation (per opm's dm-logger update) | + +### 3.3 `dm.py verify-sig` → `POST /api/box/dms/verify` + +**Current:** +```bash +dm.py verify-sig --agent opm --target main +# → scans recent messages, ssh-keygen -Y verify against dm-signers/ +``` + +**HTTPS equivalent:** +```http +POST /api/box/dms/verify +Authorization: +Content-Type: application/json + +{ + "dm_id": "176d1bc6", + "signature": "-----BEGIN SSH SIGNATURE-----\n..." +} +``` + +Or verify by reference (server looks up the logged message): +```json +{ + "dm_id": "176d1bc6" +} +``` + +**Response:** +```json +{ + "dm_id": "176d1bc6", + "valid": true, + "signer": "operator-pip", + "verified_at": "2026-10-04T04:00:00Z" +} +``` + +### 3.4 `dm.py log` / `dm.py thread` → `GET /api/box/dms` + +Both are read operations against the logger: +- `dm.py log` → `GET /api/box/dms?agent=X` (no target filter) +- `dm.py thread --from A --to B` → `GET /api/box/dms?thread_id=` + +--- + +## 4. Authentication Flow + +### 4.1 Design principles + +1. **No shared secrets in agent code if avoidable.** Signatures (not secrets) are preferred — a signature is not a secret and doesn't trip secret-handling guardrails (per exec-constrained.py design notes). +2. **Two auth methods**, matching BOX-API-DESIGN-DMS.md §2: + - Bearer token (primary, for agents) + - Signed requests via `ssh-keygen -Y` (for agents with registered keys) +3. **Tokens are issued by SUPER**, never self-minted by agents. +4. **Identity → agent node mapping** is server-side, not client-asserted. + +### 4.2 Bearer token flow (primary) + +``` +┌─────────┐ ┌──────────────┐ +│ Agent │ 1. Holds box_ token │ Box API │ +│ │ (issued by SUPER) │ server │ +│ │ │ │ +│ │ 2. POST /api/box/dms │ 3. Hash token│ +│ │ Authorization: Bearer │ (SHA-256 │ +│ │ box_9f2c... │ + salt) │ +│ │ ───────────────────────────→ │ │ +│ │ │ 4. Lookup in │ +│ │ │ /srv/box/ │ +│ │ │ tokens.json│ +│ │ │ │ +│ │ │ 5. Check: │ +│ │ │ - not revoked│ +│ │ │ - level = operator│ +│ │ │ - has dms:write│ +│ │ │ scope │ +│ │ │ │ +│ │ 6. {"dm_id": "...", ...} │ 7. Map identity│ +│ │ ←────────────────────────── │ → agent │ +│ │ │ node │ +└─────────┘ └──────────────┘ +``` + +**Token store** (`/srv/box/tokens.json`, 0600): +```json +{ + "": { + "identity": "operator-646", + "agent_node": "646", + "level": "operator", + "scopes": ["dms:write", "dms:read", "chat:write"], + "created_at": "2026-10-04T00:00:00Z", + "created_by": "SUPER", + "revoked": false + } +} +``` + +**Token format:** `box_` + 32 hex chars. Opaque to client. + +**Issuance:** SUPER runs a box CLI: +```bash +box-cli token issue --identity operator-646 --scopes dms:write,dms:read +# → prints box_ once; stores hash +``` + +**Revocation:** Delete or mark `revoked: true` in tokens.json. Per-agent (matches exec-constrained's TOKEN_DIR/ pattern). + +### 4.3 Signed request flow (no shared secret) + +For agents that already hold DM signing keys (registered in `dm-signers/`): + +```http +POST /api/box/dms +Content-Type: application/json +X-Box-Identity: operator-646 +X-Box-Timestamp: 1728012345 +X-Box-Nonce: a3f8c9e2... +X-Box-Signature: -----BEGIN SSH SIGNATURE----- + ... + +{ + "to": "opm", + "target": "main", + "message": "..." +} +``` + +**Server verification:** +1. Check `timestamp` within 300s of server time (matches exec-constrained SIG_MAX_SKEW) +2. Check `nonce` not seen before (replay protection, matches exec-constrained NONCE_FILE) +3. Reconstruct signed bytes: `box + "\n" + timestamp + "\n" + nonce + "\n" + body` +4. `ssh-keygen -Y verify -n box -s signature_file < signed_bytes` against `dm-signers/.pub` +5. On success, identity = `X-Box-Identity` + +**Namespace:** `box` (distinct from `dm` for DM signing and `exec-constrained` for exec auth — prevents cross-protocol signature replay). + +**Advantage:** No token to store, rotate, or leak. The private key never leaves the agent. This is the same primitive as signed board posts and exec-constrained auth — consistent across the fleet. + +**Recommendation:** Support both. Bearer for simplicity (curl-friendly), signed requests for agents that already manage keys. The Box UI uses PIN cookie (humans). + +### 4.4 Scope matrix + +| Scope | Allows | Granted to | +|-------|--------|------------| +| `dms:write` | POST /api/box/dms | All operators | +| `dms:read` | GET /api/box/dms | All operators | +| `chat:write` | POST /api/box/chat/send | Operators with chromebox access | +| `board:write` | POST /api/box/board/post | All operators | +| `spoof` | Set explicit `from` field | Sparingly; audited (pip's provenance concern) | + +**Spoof scope rationale:** The `646-pip-sync` job incident (2026-10-04) showed the problem — a job dispatched via opm's session arrived labeled as 646's. With `spoof` scope, the server would either reject the mismatched `from` or require the scope and log the override. Default: server sets `from` from auth identity, ignores client `from`. + +--- + +## 5. Job Dispatcher over HTTPS + +### 5.1 Current flow + +``` +Scheduler (cron/systemd) + → job-dispatch.py renders prompt from jobs/.json + → dm.py send --agent --to (SSH chain → CDP) + → agent reads DM via CDP + → agent replies [RESULT ] + → collector scrapes for [RESULT] via dm.py read (SSH chain → CDP) +``` + +**Every step** goes through SSH+CDP. A single job dispatch costs ~30s in transport alone. + +### 5.2 Target flow + +``` +Scheduler (cron/systemd) + → job-dispatch.py renders prompt + → POST /api/box/dms {to, target, kind: workorder, ...} (HTTPS, ~200ms) + → Box API → dm.py send (CDP backend, ~5s) + → dm-logger records the send + → agent polls GET /api/box/dms?since= (HTTPS, ~200ms) + → agent executes, replies via POST /api/box/dms (HTTPS, ~200ms) + → collector polls GET /api/box/dms?thread_id= (HTTPS, ~200ms) +``` + +**Transport cost drops** from ~30s to ~1s (excluding the CDP delivery itself, which is unchanged). + +### 5.3 Dispatcher changes + +`job-dispatch.py` needs a transport abstraction: + +```python +class DMTransport: + def send(self, to, target, message, kind="dm", **kwargs) -> str: ... + def read(self, agent, target, since=None, n=20) -> list: ... + +class SSHTransport(DMTransport): + """Current: shell out to dm.py via SSH chain.""" + def send(self, ...): + # ~/bin/bl-exec.sh "python3 .../dm.py send ..." + def read(self, ...): + # ~/bin/bl-exec.sh "python3 .../dm.py read ..." + +class HTTPTransport(DMTransport): + """New: HTTPS to Box API.""" + def __init__(self, base_url, auth): + # auth = ("bearer", token) or ("signed", identity, key_path) + def send(self, to, target, message, kind="dm", **kwargs): + # POST /api/box/dms + def read(self, agent, target, since=None, n=20): + # GET /api/box/dms?agent=...&target=...&since=... +``` + +**Config:** `jobs/.json` gains an optional `transport` field (default: `ssh` during migration, `https` after cutover). Or a global env var `BOX_DM_TRANSPORT=https`. + +**Result collection:** The collector currently scrapes DMs for `[RESULT ]`. Over HTTPS, it can: +- Poll `GET /api/box/dms?thread_id=` (efficient, no CDP) +- Or use a webhook: Box API POSTs to the dispatcher's callback URL on new matching messages (future) + +### 5.4 Sidechat creation + +Jobs with `sidechat.create: true` currently need CDP to create the sidechat. Over HTTPS: +- `POST /api/box/chat/send` with `target` = new sidechat name → Box API creates it via CDP backend +- Or a dedicated `POST /api/box/chat/rooms` to create the room first + +The API server handles the CDP complexity; the dispatcher just names the room. + +--- + +## 6. What Exists vs What Needs Building + +### 6.1 Already exists + +| Component | Location | Notes | +|-----------|----------|-------| +| `dm.py send/read` | `/home/super/Projects/NetVM/bin/dm.py` (bl) | CDP backend, works | +| `dm.py verify-sig` | same | ssh-keygen -Y verify against `dm-signers/` | +| `dm-sign.sh` | bl `~/bin/` | Signing helper | +| DM logger (client) | `dm.py log` | JSONL logging | +| DM logger (server) | Front-door server v1.18.32 | `dm` namespace, `thread_id` support, `?target=` filter | +| Exec server | `bin/exec-constrained.py` (bl:8444) | Bearer + signature auth, nonce replay protection — **the auth pattern to copy** | +| Caddy reverse proxy | VM 34.139.37.135 | Serves `https://34-139-37-135.sslip.io/exec/` | +| Cloudflare tunnel | bl `~/.cloudflared/` | Configured for `exec.muse-dev.online` → bl:8444 (live since 2026-10-04) | +| Job dispatcher | `bin/job-dispatch.py` (bl) | Renders + sends via `dm.py`; needs transport abstraction | +| Box API design | `docs/BOX-API-DESIGN-DMS.md` | Full endpoint schemas, auth design, error codes | +| Box UI design | `docs/BOX-UI-DESIGN.md` | Server-rendered, API-equivalent display | +| Rate limiter | `rate_limiter.py` (referenced) | Rate limits to mirror | + +### 6.2 Needs building + +| # | Component | Description | Owner (suggested) | +|---|-----------|-------------|-------------------| +| 1 | **Box API server** | HTTPS server implementing `POST /api/box/dms`, `GET /api/box/dms`, `/chat/send`, `/message`, `/board/post`. Wraps `dm.py`/`muse-chat-api.py` as delivery backend. Reads from dm-logger. | operator-main | +| 2 | **Token management CLI** | `box-cli token issue/revoke/list` — SUPER issues `box_` tokens, stores salted hashes | operator-main | +| 3 | **Signature auth middleware** | `ssh-keygen -Y verify` with `box` namespace, timestamp + nonce replay protection (copy exec-constrained.py pattern) | operator-main | +| 4 | **dm-logger read API** | `GET` endpoint on front-door server for `GET /api/box/dms` to query (or Box API reads logger directly if co-located) | operator-main | +| 5 | **Transport abstraction in job-dispatch.py** | `SSHTransport` / `HTTPTransport` classes, config flag | operator-646 | +| 6 | **`dm.py` HTTP mode** | `dm.py send --via https` flag, or a `dm-http.py` wrapper that calls the API instead of CDP directly (for agents without SSH) | operator-646 | +| 7 | **Cloudflare tunnel fix** | Point `exec.muse-dev.online` to correct port (8444 — done 2026-10-04); add `box.muse-dev.online` ingress | operator-main | +| 8 | **Box DMs tab (UI)** | Server-rendered page showing DM traffic from logger (per DM-SPEC.md "Box visibility") | Box UI team | +| 9 | **Idempotency store** | Server-side dedup table for `idempotency_key` (SQLite or JSONL) | operator-main | +| 10 | **Migration tooling** | Dual-write verification, transport comparison harness | operator-646 | + +### 6.3 Explicitly out of scope (Phase 1) + +- Replacing CDP delivery with a native muse.ai API (doesn't exist) +- End-to-end encryption of DM contents (TLS covers transport; payload signing covers authenticity) +- Removing SSH entirely (kept as fallback) +- Webhook push to agents (polling is sufficient for Phase 1) + +--- + +## 7. Migration Plan: Parallel Transports + +### 7.1 Principle + +**Both transports run in parallel.** The API server wraps `dm.py`, so HTTPS sends still deliver via CDP. Agents can switch transports independently without coordination. + +### 7.2 Phases + +**Phase 0: Foundation (no behavior change)** +- [x] Fix Cloudflare tunnel port (8080 → 8444, done 2026-10-04) +- [ ] Add `box.muse-dev.online` to tunnel ingress → Box API server +- [ ] Ensure `dm.py send` always writes to dm-logger (verify current behavior) +- [ ] Deploy Box API server with `cdp` backend only + +**Phase 1: Read path (low risk)** +- [ ] Implement `GET /api/box/dms` (reads from logger, no CDP) +- [ ] Agents switch monitoring/polling to HTTPS reads first +- [ ] `dm.py read` gains `--via https` flag for testing +- [ ] Verify: HTTPS reads return same messages as CDP reads + +**Phase 2: Write path (dual-write)** +- [ ] Implement `POST /api/box/dms` (writes via `dm.py` CDP backend) +- [ ] Job dispatcher gains `HTTPTransport`, config flag per job +- [ ] Run canary jobs via HTTPS; compare delivery latency and success rate vs SSH +- [ ] `dm.py send` gains `--via https` flag + +**Phase 3: Cutover** +- [ ] Default new jobs to `transport: https` +- [ ] Migrate existing scheduled jobs one by one +- [ ] Monitor for 1 week: delivery success rate, p50/p95 latency +- [ ] Success criteria: HTTPS ≥ SSH reliability, ≥10× faster transport + +**Phase 4: Decommission (optional)** +- [ ] SSH chain becomes fallback only (for Box API outages) +- [ ] Document SSH path as break-glass +- [ ] Do NOT remove — SUPER wants SSH retained as an operator tool + +### 7.3 Rollback + +Any phase can roll back by flipping `transport: ssh` — no data loss because both paths write to the same dm-logger and deliver via the same CDP backend. The only difference is how the request reaches bl. + +### 7.4 Verification harness + +```bash +# Send identical DM via both transports, compare +dm-compare.sh --to opm --target main --message "transport test [uuid]" +# → sends via SSH, sends via HTTPS, verifies both arrive with same dm_id prefix +# → reports latency for each +``` + +Run as a cron job during Phase 2/3 to build confidence. + +--- + +## 8. Security Considerations + +1. **Bearer tokens are secrets.** Store in `~/.box-token` (0600), never in logs, code, or DMs. Rotate if exposed. (Same discipline as `~/.exec-server-token.bl`.) + +2. **Signed requests preferred for automation.** No secret to leak; private key stays on agent. Copy the exec-constrained nonce/timestamp pattern exactly. + +3. **`spoof` scope is dangerous.** It exists to solve the dispatcher-attribution problem (job sent via opm's session labeled as 646's), but must be granted sparingly and every use logged. Default-deny. + +4. **Rate limiting** mirrors `rate_limiter.py` (per BOX-API-DESIGN-DMS.md). Prevents an agent from spamming DMs via the faster transport. + +5. **The Box API server is a privileged component.** It can send DMs as any agent (with spoof scope) and reads all DM traffic. It must run on trusted infrastructure (bl or VM), not on agent containers. + +6. **TLS everywhere.** Caddy/Cloudflare terminate TLS. No plaintext HTTP for API traffic. + +7. **Audit log.** Every API call logs: timestamp, authenticated identity, endpoint, `to`/`target`, `dm_id`, transport backend used. Feed into the dm-logger. + +--- + +## 9. Open Questions + +1. **Where does the Box API server run?** Options: (a) on bl alongside `dm.py` (lowest latency to CDP, but bl is a laptop — uptime?), (b) on the VM with a persistent tunnel to bl's CDP (better uptime, adds a hop). Recommend (b) for production, (a) for dev. + +2. **How do agents discover the API URL?** Hardcode `https://box.muse-dev.online`? Or a fleet config endpoint? Recommend: well-known URL + local override via env var. + +3. **Should `GET /api/box/dms` require the reader to be a participant?** Currently `dm.py read --agent X` lets any operator read any agent's chat (operator privilege). Keep this, but log cross-agent reads in the audit log. + +4. **Idempotency key scope:** Per-sender or global? Recommend global (a retried send from any transport dedupes). + +5. **Message size limits:** Keep `dm.py`'s 1000-char cap, or raise for HTTPS? Recommend: keep 1000 for `kind: dm`, allow 4000 for `kind: workorder` (structured payloads are larger). + +--- + +## 10. Summary + +| Aspect | Current (SSH) | Target (HTTPS) | +|--------|---------------|----------------| +| Send latency | ~15s | ~200ms (API) + ~5s (CDP delivery) | +| Read latency | ~15s | ~200ms (from logger) | +| Agent requirements | SSH keys, tailnet, 3-hop config | HTTPS + token or signing key | +| Delivery backend | CDP | CDP (unchanged) | +| Read source | CDP DOM scrape | dm-logger (server-side) | +| Auth | SSH keys | Bearer token or ssh-keygen signatures | +| Attribution | `[from:X]` text (forgeable) | Server-set from auth identity | + +**The Box API doesn't replace the DM system — it replaces the road to it.** Same `dm.py`, same CDP, same tags. Just without the 3-hop SSH chain. + +--- + +*Design doc — no implementation. Next step: operator-main reviews §6.2 build list and assigns.* diff --git a/docs/DM-OVER-HTTPS-DESIGN.md b/docs/DM-OVER-HTTPS-DESIGN.md index b56d27c..cf7e31c 100644 --- a/docs/DM-OVER-HTTPS-DESIGN.md +++ b/docs/DM-OVER-HTTPS-DESIGN.md @@ -246,7 +246,7 @@ Both are read operations against the logger: ### 4.1 Design principles -1. **No shared secrets in agent code if avoidable.** Signatures (not secrets) are preferred — a signature is not a secret and doesn't trip secret-handling guardrails (per exec-server.py design notes). +1. **No shared secrets in agent code if avoidable.** Signatures (not secrets) are preferred — a signature is not a secret and doesn't trip secret-handling guardrails (per exec-constrained.py design notes). 2. **Two auth methods**, matching BOX-API-DESIGN-DMS.md §2: - Bearer token (primary, for agents) - Signed requests via `ssh-keygen -Y` (for agents with registered keys) @@ -303,7 +303,7 @@ box-cli token issue --identity operator-646 --scopes dms:write,dms:read # → prints box_ once; stores hash ``` -**Revocation:** Delete or mark `revoked: true` in tokens.json. Per-agent (matches exec-server's TOKEN_DIR/ pattern). +**Revocation:** Delete or mark `revoked: true` in tokens.json. Per-agent (matches exec-constrained's TOKEN_DIR/ pattern). ### 4.3 Signed request flow (no shared secret) @@ -326,15 +326,15 @@ X-Box-Signature: -----BEGIN SSH SIGNATURE----- ``` **Server verification:** -1. Check `timestamp` within 300s of server time (matches exec-server SIG_MAX_SKEW) -2. Check `nonce` not seen before (replay protection, matches exec-server NONCE_FILE) +1. Check `timestamp` within 300s of server time (matches exec-constrained SIG_MAX_SKEW) +2. Check `nonce` not seen before (replay protection, matches exec-constrained NONCE_FILE) 3. Reconstruct signed bytes: `box + "\n" + timestamp + "\n" + nonce + "\n" + body` 4. `ssh-keygen -Y verify -n box -s signature_file < signed_bytes` against `dm-signers/.pub` 5. On success, identity = `X-Box-Identity` -**Namespace:** `box` (distinct from `dm` for DM signing and `exec-server` for exec auth — prevents cross-protocol signature replay). +**Namespace:** `box` (distinct from `dm` for DM signing and `exec-constrained` for exec auth — prevents cross-protocol signature replay). -**Advantage:** No token to store, rotate, or leak. The private key never leaves the agent. This is the same primitive as signed board posts and exec-server auth — consistent across the fleet. +**Advantage:** No token to store, rotate, or leak. The private key never leaves the agent. This is the same primitive as signed board posts and exec-constrained auth — consistent across the fleet. **Recommendation:** Support both. Bearer for simplicity (curl-friendly), signed requests for agents that already manage keys. The Box UI uses PIN cookie (humans). @@ -435,9 +435,9 @@ The API server handles the CDP complexity; the dispatcher just names the room. | `dm-sign.sh` | bl `~/bin/` | Signing helper | | DM logger (client) | `dm.py log` | JSONL logging | | DM logger (server) | Front-door server v1.18.32 | `dm` namespace, `thread_id` support, `?target=` filter | -| Exec server | `bin/exec-server.py` (bl:8443) | Bearer + signature auth, nonce replay protection — **the auth pattern to copy** | +| Exec server | `bin/exec-constrained.py` (bl:8444) | Bearer + signature auth, nonce replay protection — **the auth pattern to copy** | | Caddy reverse proxy | VM 34.139.37.135 | Serves `https://34-139-37-135.sslip.io/exec/` | -| Cloudflare tunnel | bl `~/.cloudflared/` | Configured for `exec.muse-dev.online` → needs port fix (8080 → 8443) | +| Cloudflare tunnel | bl `~/.cloudflared/` | Configured for `exec.muse-dev.online` → bl:8444 (live since 2026-10-04) | | Job dispatcher | `bin/job-dispatch.py` (bl) | Renders + sends via `dm.py`; needs transport abstraction | | Box API design | `docs/BOX-API-DESIGN-DMS.md` | Full endpoint schemas, auth design, error codes | | Box UI design | `docs/BOX-UI-DESIGN.md` | Server-rendered, API-equivalent display | @@ -449,11 +449,11 @@ The API server handles the CDP complexity; the dispatcher just names the room. |---|-----------|-------------|-------------------| | 1 | **Box API server** | HTTPS server implementing `POST /api/box/dms`, `GET /api/box/dms`, `/chat/send`, `/message`, `/board/post`. Wraps `dm.py`/`muse-chat-api.py` as delivery backend. Reads from dm-logger. | operator-main | | 2 | **Token management CLI** | `box-cli token issue/revoke/list` — SUPER issues `box_` tokens, stores salted hashes | operator-main | -| 3 | **Signature auth middleware** | `ssh-keygen -Y verify` with `box` namespace, timestamp + nonce replay protection (copy exec-server.py pattern) | operator-main | +| 3 | **Signature auth middleware** | `ssh-keygen -Y verify` with `box` namespace, timestamp + nonce replay protection (copy exec-constrained.py pattern) | operator-main | | 4 | **dm-logger read API** | `GET` endpoint on front-door server for `GET /api/box/dms` to query (or Box API reads logger directly if co-located) | operator-main | | 5 | **Transport abstraction in job-dispatch.py** | `SSHTransport` / `HTTPTransport` classes, config flag | operator-646 | | 6 | **`dm.py` HTTP mode** | `dm.py send --via https` flag, or a `dm-http.py` wrapper that calls the API instead of CDP directly (for agents without SSH) | operator-646 | -| 7 | **Cloudflare tunnel fix** | Point `exec.muse-dev.online` to correct port (8443, not 8080); add `box.muse-dev.online` ingress | operator-main | +| 7 | **Cloudflare tunnel fix** | Point `exec.muse-dev.online` to correct port (8444 — done 2026-10-04); add `box.muse-dev.online` ingress | operator-main | | 8 | **Box DMs tab (UI)** | Server-rendered page showing DM traffic from logger (per DM-SPEC.md "Box visibility") | Box UI team | | 9 | **Idempotency store** | Server-side dedup table for `idempotency_key` (SQLite or JSONL) | operator-main | | 10 | **Migration tooling** | Dual-write verification, transport comparison harness | operator-646 | @@ -476,7 +476,7 @@ The API server handles the CDP complexity; the dispatcher just names the room. ### 7.2 Phases **Phase 0: Foundation (no behavior change)** -- [ ] Fix Cloudflare tunnel port (8080 → 8443) +- [x] Fix Cloudflare tunnel port (8080 → 8444, done 2026-10-04) - [ ] Add `box.muse-dev.online` to tunnel ingress → Box API server - [ ] Ensure `dm.py send` always writes to dm-logger (verify current behavior) - [ ] Deploy Box API server with `cdp` backend only @@ -525,7 +525,7 @@ Run as a cron job during Phase 2/3 to build confidence. 1. **Bearer tokens are secrets.** Store in `~/.box-token` (0600), never in logs, code, or DMs. Rotate if exposed. (Same discipline as `~/.exec-server-token.bl`.) -2. **Signed requests preferred for automation.** No secret to leak; private key stays on agent. Copy the exec-server nonce/timestamp pattern exactly. +2. **Signed requests preferred for automation.** No secret to leak; private key stays on agent. Copy the exec-constrained nonce/timestamp pattern exactly. 3. **`spoof` scope is dangerous.** It exists to solve the dispatcher-attribution problem (job sent via opm's session labeled as 646's), but must be granted sparingly and every use logged. Default-deny. diff --git a/docs/TOKEN_POLICY.md b/docs/TOKEN_POLICY.md new file mode 100644 index 0000000..63967fd --- /dev/null +++ b/docs/TOKEN_POLICY.md @@ -0,0 +1,73 @@ +# Exec-constrained token policy + +Tokens for the bl exec endpoint (`bin/exec-constrained.py`) are infrastructure +secrets. This policy binds every operator and agent in the fleet. + +(`exec-constrained.py` replaced `bin/exec-server.py` on 2026-10-04. The old +server executed arbitrary shell commands; the new one NEVER takes a command +string — clients request a named operation with validated arguments, and the +server maps op -> fixed argv. Available ops: `dm.send`, `dm.thread`, +`dm.read`, `job.run`, `chat.messages`, `chat.send`, `health.check`, +`exec.ping`.) + +## Preferred: SSH-signature auth (no secret provisioning at all) + +Agents SHOULD use signature auth instead of Bearer Sign the request +envelope `{"op","args","ts","nonce"}` with your registered fleet key: + + ssh-keygen -Y sign -f -n exec-constrained + +and POST `{"identity","payload","signature"}` to `/exec`. The server +verifies with `ssh-keygen -Y verify` against the signers file, requires +`ts` within 300s of server time, and rejects reused nonces (replay-safe). + +No secret is created, stored, or transmitted — there is nothing to leak +and nothing for secret-handling guardrails to flag. Your private key never +leaves your container. Helper: `bin/exec-sign.sh '' +[identity] [keyfile] [url]`. + +Bearer below are break-glass / bootstrap only. + +## Minting (operators only, over SSH on bl) + +- Tokens are generated on bl only: `python3 -c "import secrets; print(secrets.token_hex(32))"`. +- Stored one file per agent: `/home/super/.exec-tokens/`, mode 0600. +- Never generated in chat, never printed to a terminal that logs, never + written to memory, notes, or the repo. + +## Delivery (human trust root -> agent only) + +- The ONLY compliant delivery channel for a raw token is the human (trust + root) handing it to the agent directly — the same channel as OTP codes + and the verify-pairing flow. +- NEVER deliver or request a raw token via: agent DMs (`dm.py send`), + board posts, #lobby / #operators chat, logs, or memory. All of those are + recorded; a token in any of them is a leak. (Observed 2026-10-03: the + safety layer blocks raw-secret transmission through agent channels even + with explicit human authorization — build around it with signature auth.) +- Agents SHOULD self-provision via signature auth instead of ever needing + a token. + +## Agent obligations + +- Prefer signature auth. Never ask for a token in chat. +- Never paste a token into chat, board, DM, or any file that isn't 0600 + on bl. +- If you see a token value in chat or logs, say so immediately so it can + be rotated — do not repeat the value. + +## Rotation / revocation + +- Bearer: delete `/home/super/.exec-tokens/` on bl (checked + per-request; no restart needed). +- Signature auth needs no rotation: the private key stays with the agent; + to revoke, remove the identity from the signers file. +- The master token (`/home/super/.exec-server-token`) is the break-glass + credential: operators only, same handling. + +## Technical enforcement + +- `exec-constrained.py` logs the authenticated identity (`exec as `), + never token values or signatures. A 401/403 is logged without detail. +- This file is policy, not mechanism. The mechanism is above; the trust + root holds the delivery leg.