docs: exec-server -> exec-constrained stale references (bl:8444)
- docs/TOKEN_POLICY.md: rewritten for exec-constrained.py (named ops,
-n exec-constrained, {op,args,ts,nonce} envelope; rotate endpoint gone)
- bin/chromebox-gateway.py: exec-server naming -> shared exec token files
- docs/DM-HTTPS-DESIGN-646.md + docs/DM-OVER-HTTPS-DESIGN.md: port
8443->8444, namespace exec-server->exec-constrained, envelope updated,
cloudflared port fix marked done 2026-10-04
This commit is contained in:
Executable
+364
@@ -0,0 +1,364 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
chromebox-gateway.py — HTTPS fallback for chromebox/DM control when SSH is down.
|
||||||
|
|
||||||
|
The chromeboxes (headless Chromium per NetVM node) are normally driven over
|
||||||
|
SSH via muse-chat-api.py / dm.py, which speak CDP through the per-node
|
||||||
|
netns relay. When SSH to bl breaks, this gateway provides a constrained
|
||||||
|
HTTPS path to the same high-level operations.
|
||||||
|
|
||||||
|
CRITICAL: this does NOT expose raw CDP. Raw CDP (Runtime.evaluate,
|
||||||
|
Page.navigate, etc.) is arbitrary code execution inside the browser with
|
||||||
|
the agent's live session. This gateway exposes ONLY the allowlisted
|
||||||
|
high-level operations below, each mapped to an existing audited script.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python3 chromebox-gateway.py --port 8444
|
||||||
|
|
||||||
|
Auth: Bearer token, reusing the shared exec per-agent token files
|
||||||
|
(~/.exec-tokens/<agent>). The master token (~/.exec-server-token) also
|
||||||
|
works. Identity = token filename; tokens are never logged.
|
||||||
|
|
||||||
|
Endpoints:
|
||||||
|
GET /health {"status":"ok"} — no auth (load-balancer friendly)
|
||||||
|
POST /api/v1/op {"op": "<name>", "params": {...}} — bearer auth
|
||||||
|
|
||||||
|
Audit: every call appended to ~/.chromebox-gateway-audit.jsonl (0600).
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import hmac
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import ssl
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- config
|
||||||
|
|
||||||
|
BIN_DIR = os.path.expanduser("~/Projects/NetVM/bin")
|
||||||
|
TOKEN_FILE = "/home/super/.exec-server-token" # master token (shared exec token file)
|
||||||
|
TOKEN_DIR = "/home/super/.exec-tokens" # per-agent tokens (shared exec token files)
|
||||||
|
AUDIT_FILE = "/home/super/.chromebox-gateway-audit.jsonl"
|
||||||
|
CERT_FILE = "/home/super/.chromebox-gateway-cert.pem"
|
||||||
|
KEY_FILE = "/home/super/.chromebox-gateway-key.pem"
|
||||||
|
|
||||||
|
NODES = ("muse", "pip", "646", "opm")
|
||||||
|
MAX_MSG = 2000 # gateway-level cap; downstream scripts enforce their own
|
||||||
|
BACKEND_TIMEOUT = 120 # seconds per backend call
|
||||||
|
|
||||||
|
# Rate limit: token bucket per identity — 20 req/min sustained, burst 5.
|
||||||
|
RATE_PER_SEC = 20.0 / 60.0
|
||||||
|
RATE_BURST = 5
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- ops
|
||||||
|
# Each op maps to an argv builder for an existing script. No shell=True,
|
||||||
|
# ever. Params are validated before building argv.
|
||||||
|
|
||||||
|
def _node(params):
|
||||||
|
node = params.get("account") or params.get("agent")
|
||||||
|
if node not in NODES:
|
||||||
|
raise ValueError("account/agent must be one of %s" % (",".join(NODES)))
|
||||||
|
return node
|
||||||
|
|
||||||
|
def _msg(params):
|
||||||
|
m = params.get("message", "")
|
||||||
|
if not isinstance(m, str) or not m.strip():
|
||||||
|
raise ValueError("message must be a non-empty string")
|
||||||
|
if len(m) > MAX_MSG:
|
||||||
|
raise ValueError("message exceeds %d chars" % MAX_MSG)
|
||||||
|
return m
|
||||||
|
|
||||||
|
def _target(params):
|
||||||
|
t = params.get("target", "main")
|
||||||
|
if not isinstance(t, str) or not t or len(t) > 128:
|
||||||
|
raise ValueError("target must be a short string")
|
||||||
|
if not re.fullmatch(r"[A-Za-z0-9_./:-]+", t):
|
||||||
|
raise ValueError("target has invalid characters")
|
||||||
|
if ".." in t:
|
||||||
|
raise ValueError("target must not contain '..'")
|
||||||
|
return t
|
||||||
|
|
||||||
|
def _n(params, default=5, cap=50):
|
||||||
|
n = params.get("n", default)
|
||||||
|
try:
|
||||||
|
n = int(n)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
raise ValueError("n must be an integer")
|
||||||
|
if not 1 <= n <= cap:
|
||||||
|
raise ValueError("n must be 1..%d" % cap)
|
||||||
|
return n
|
||||||
|
|
||||||
|
def _tags(params):
|
||||||
|
tags = params.get("tags", [])
|
||||||
|
if not isinstance(tags, list):
|
||||||
|
raise ValueError("tags must be a list")
|
||||||
|
out = []
|
||||||
|
for t in tags:
|
||||||
|
if not isinstance(t, str) or len(t) > 128:
|
||||||
|
raise ValueError("bad tag")
|
||||||
|
if not re.fullmatch(r"[A-Za-z0-9_:=\-./]+", t):
|
||||||
|
raise ValueError("tag has invalid characters: %r" % t[:40])
|
||||||
|
out.append(t)
|
||||||
|
if len(out) > 10:
|
||||||
|
raise ValueError("too many tags (max 10)")
|
||||||
|
return out
|
||||||
|
|
||||||
|
CHAT = os.path.join(BIN_DIR, "muse-chat-api.py")
|
||||||
|
DM = os.path.join(BIN_DIR, "dm.py")
|
||||||
|
|
||||||
|
def op_chat_send(p):
|
||||||
|
return [CHAT, "--account", _node(p), "send", _msg(p)]
|
||||||
|
|
||||||
|
def op_chat_messages(p):
|
||||||
|
return [CHAT, "--account", _node(p), "messages", "--n", str(_n(p))]
|
||||||
|
|
||||||
|
def op_chat_sidechats(p):
|
||||||
|
return [CHAT, "--account", _node(p), "sidechat", "list"]
|
||||||
|
|
||||||
|
def op_chat_sidechat_create(p):
|
||||||
|
argv = [CHAT, "--account", _node(p), "sidechat", "create"]
|
||||||
|
name = p.get("name")
|
||||||
|
if name:
|
||||||
|
if not isinstance(name, str) or len(name) > 80 or not re.fullmatch(r"[A-Za-z0-9 _-]+", name):
|
||||||
|
raise ValueError("bad sidechat name")
|
||||||
|
argv += ["--name", name]
|
||||||
|
return argv
|
||||||
|
|
||||||
|
def op_chat_approvals(p):
|
||||||
|
return [CHAT, "--account", _node(p), "approvals"]
|
||||||
|
|
||||||
|
def op_chat_url(p):
|
||||||
|
return [CHAT, "--account", _node(p), "url"]
|
||||||
|
|
||||||
|
def op_dm_send(p):
|
||||||
|
node = _node(p)
|
||||||
|
to = p.get("to", node)
|
||||||
|
if to not in NODES:
|
||||||
|
raise ValueError("to must be one of %s" % (",".join(NODES)))
|
||||||
|
argv = [DM, "send", "--agent", node, "--to", to,
|
||||||
|
"--target", _target(p), _msg(p)]
|
||||||
|
for t in _tags(p):
|
||||||
|
argv += ["--tag", t]
|
||||||
|
return argv
|
||||||
|
|
||||||
|
def op_dm_read(p):
|
||||||
|
return [DM, "read", "--agent", _node(p),
|
||||||
|
"--target", _target(p), "--n", str(_n(p, default=5, cap=20))]
|
||||||
|
|
||||||
|
# The allowlist. Adding an op here is a security decision — review accordingly.
|
||||||
|
# Deliberately absent: wait (long-poll), upload (file ingress),
|
||||||
|
# sidechat use (raw navigation), anything raw-CDP.
|
||||||
|
ALLOWLIST = {
|
||||||
|
"chat.send": ("write", op_chat_send),
|
||||||
|
"chat.messages": ("read", op_chat_messages),
|
||||||
|
"chat.sidechats": ("read", op_chat_sidechats),
|
||||||
|
"chat.sidechat_create": ("write", op_chat_sidechat_create),
|
||||||
|
"chat.approvals": ("read", op_chat_approvals),
|
||||||
|
"chat.url": ("read", op_chat_url),
|
||||||
|
"dm.send": ("write", op_dm_send),
|
||||||
|
"dm.read": ("read", op_dm_read),
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- auth
|
||||||
|
|
||||||
|
def _read_token_file(path):
|
||||||
|
try:
|
||||||
|
with open(path) as f:
|
||||||
|
return f.read().strip()
|
||||||
|
except OSError:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def check_token(token):
|
||||||
|
"""Return identity label or None. Tokens never leave this function."""
|
||||||
|
if not token:
|
||||||
|
return None
|
||||||
|
if hmac.compare_digest(token, _read_token_file(TOKEN_FILE)):
|
||||||
|
return "master"
|
||||||
|
try:
|
||||||
|
names = os.listdir(TOKEN_DIR)
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
for name in names:
|
||||||
|
if not re.fullmatch(r"[A-Za-z0-9_-]+", name):
|
||||||
|
continue
|
||||||
|
t = _read_token_file(os.path.join(TOKEN_DIR, name))
|
||||||
|
if t and hmac.compare_digest(token, t):
|
||||||
|
return name
|
||||||
|
return None
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- rate limit
|
||||||
|
|
||||||
|
_buckets = {} # identity -> [tokens, last_ts]
|
||||||
|
|
||||||
|
def rate_ok(identity):
|
||||||
|
now = time.monotonic()
|
||||||
|
tokens, last = _buckets.get(identity, (RATE_BURST, now))
|
||||||
|
tokens = min(RATE_BURST, tokens + (now - last) * RATE_PER_SEC)
|
||||||
|
if tokens < 1.0:
|
||||||
|
_buckets[identity] = (tokens, now)
|
||||||
|
return False
|
||||||
|
_buckets[identity] = (tokens - 1.0, now)
|
||||||
|
return True
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- audit
|
||||||
|
|
||||||
|
def audit(entry):
|
||||||
|
entry = dict(entry)
|
||||||
|
entry["ts"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||||
|
# Defense in depth: never let a full message body reach the audit file,
|
||||||
|
# even if a caller forgets to summarize first.
|
||||||
|
params = entry.get("params")
|
||||||
|
if isinstance(params, dict) and "message" in params:
|
||||||
|
params = dict(params)
|
||||||
|
params["message_len"] = len(params.pop("message"))
|
||||||
|
entry["params"] = params
|
||||||
|
try:
|
||||||
|
fd = os.open(AUDIT_FILE, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
|
||||||
|
with os.fdopen(fd, "a") as f:
|
||||||
|
f.write(json.dumps(entry) + "\n")
|
||||||
|
except OSError as e:
|
||||||
|
print("audit write failed: %s" % e, file=sys.stderr)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- handler
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
server_version = "chromebox-gateway/1.0"
|
||||||
|
|
||||||
|
def log_message(self, fmt, *args): # quiet; audit log is the record
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _json(self, code, obj):
|
||||||
|
body = json.dumps(obj).encode()
|
||||||
|
self.send_response(code)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
def do_GET(self):
|
||||||
|
if urlparse(self.path).path == "/health":
|
||||||
|
self._json(200, {"status": "ok", "ops": sorted(ALLOWLIST)})
|
||||||
|
return
|
||||||
|
self._json(404, {"error": "not_found"})
|
||||||
|
|
||||||
|
def do_POST(self):
|
||||||
|
if urlparse(self.path).path != "/api/v1/op":
|
||||||
|
self._json(404, {"error": "not_found"})
|
||||||
|
return
|
||||||
|
|
||||||
|
auth = self.headers.get("Authorization", "")
|
||||||
|
token = auth[7:] if auth.startswith("Bearer ") else ""
|
||||||
|
identity = check_token(token)
|
||||||
|
if not identity:
|
||||||
|
self._json(401, {"error": "unauthorized"})
|
||||||
|
return
|
||||||
|
if not rate_ok(identity):
|
||||||
|
audit({"identity": identity, "op": None, "result": "rate_limited"})
|
||||||
|
self._json(429, {"error": "rate_limited"})
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
length = int(self.headers.get("Content-Length", 0))
|
||||||
|
except ValueError:
|
||||||
|
length = 0
|
||||||
|
if length > 65536:
|
||||||
|
self._json(413, {"error": "body_too_large"})
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
req = json.loads(self.rfile.read(length) or b"{}")
|
||||||
|
except (ValueError, OSError):
|
||||||
|
self._json(400, {"error": "bad_json"})
|
||||||
|
return
|
||||||
|
|
||||||
|
op = req.get("op")
|
||||||
|
params = req.get("params") or {}
|
||||||
|
if not isinstance(params, dict):
|
||||||
|
self._json(400, {"error": "params_must_be_object"})
|
||||||
|
return
|
||||||
|
entry = ALLOWLIST.get(op)
|
||||||
|
if not entry:
|
||||||
|
audit({"identity": identity, "op": op, "result": "unknown_op"})
|
||||||
|
self._json(400, {"error": "unknown_op", "allowed": sorted(ALLOWLIST)})
|
||||||
|
return
|
||||||
|
cls, builder = entry
|
||||||
|
|
||||||
|
t0 = time.monotonic()
|
||||||
|
try:
|
||||||
|
argv = builder(params)
|
||||||
|
except ValueError as e:
|
||||||
|
audit({"identity": identity, "op": op, "class": cls, "result": "bad_params",
|
||||||
|
"detail": str(e)[:120]})
|
||||||
|
self._json(400, {"error": "bad_params", "detail": str(e)})
|
||||||
|
return
|
||||||
|
|
||||||
|
# Redacted summary for the audit log — never the full message.
|
||||||
|
summary = {k: (v[:80] + "…" if isinstance(v, str) and len(v) > 80 else v)
|
||||||
|
for k, v in params.items() if k != "message"}
|
||||||
|
if "message" in params:
|
||||||
|
summary["message_len"] = len(params["message"])
|
||||||
|
|
||||||
|
try:
|
||||||
|
proc = subprocess.run(argv, capture_output=True, text=True,
|
||||||
|
timeout=BACKEND_TIMEOUT)
|
||||||
|
ok = proc.returncode == 0
|
||||||
|
result = "ok" if ok else "backend_error"
|
||||||
|
self._json(200 if ok else 502, {
|
||||||
|
"ok": ok,
|
||||||
|
"op": op,
|
||||||
|
"returncode": proc.returncode,
|
||||||
|
"stdout": proc.stdout[-8000:],
|
||||||
|
"stderr": proc.stderr[-2000:],
|
||||||
|
})
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
result = "timeout"
|
||||||
|
self._json(504, {"ok": False, "op": op, "error": "backend_timeout"})
|
||||||
|
except OSError as e:
|
||||||
|
result = "exec_failed"
|
||||||
|
self._json(500, {"ok": False, "op": op, "error": "exec_failed"})
|
||||||
|
finally:
|
||||||
|
audit({"identity": identity, "op": op, "class": cls,
|
||||||
|
"node": params.get("account") or params.get("agent"),
|
||||||
|
"params": summary, "result": result,
|
||||||
|
"latency_ms": int((time.monotonic() - t0) * 1000)})
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- main
|
||||||
|
|
||||||
|
def ensure_cert():
|
||||||
|
if os.path.exists(CERT_FILE) and os.path.exists(KEY_FILE):
|
||||||
|
return
|
||||||
|
print("generating self-signed cert...", file=sys.stderr)
|
||||||
|
subprocess.run([
|
||||||
|
"openssl", "req", "-x509", "-newkey", "rsa:2048",
|
||||||
|
"-keyout", KEY_FILE, "-out", CERT_FILE,
|
||||||
|
"-days", "825", "-nodes", "-subj", "/CN=chromebox-gateway",
|
||||||
|
], check=True)
|
||||||
|
os.chmod(KEY_FILE, 0o600)
|
||||||
|
os.chmod(CERT_FILE, 0o600)
|
||||||
|
|
||||||
|
def main():
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--port", type=int, default=8444)
|
||||||
|
ap.add_argument("--bind", default="100.123.153.75",
|
||||||
|
help="tailnet IP; use 127.0.0.1 for local-only")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
ensure_cert()
|
||||||
|
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||||
|
context.load_cert_chain(CERT_FILE, KEY_FILE)
|
||||||
|
|
||||||
|
srv = HTTPServer((args.bind, args.port), Handler)
|
||||||
|
srv.socket = context.wrap_socket(srv.socket, server_side=True)
|
||||||
|
print("chromebox-gateway listening on https://%s:%d" % (args.bind, args.port),
|
||||||
|
file=sys.stderr)
|
||||||
|
print("ops: %s" % ", ".join(sorted(ALLOWLIST)), file=sys.stderr)
|
||||||
|
try:
|
||||||
|
srv.serve_forever()
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,572 @@
|
|||||||
|
# DM over HTTPS: System Design
|
||||||
|
|
||||||
|
**Date:** 2026-10-04
|
||||||
|
**Author:** operator-646 (design agent)
|
||||||
|
**Status:** Design only — no implementation
|
||||||
|
**Companion specs:**
|
||||||
|
- `docs/BOX-API-DESIGN-DMS.md` (commit ef8e98a) — endpoint schemas
|
||||||
|
- `docs/BOX-UI-DESIGN.md` (commit 458314c) — UI decisions
|
||||||
|
- `docs/DM-SPEC.md` (work orders, server-side logging)
|
||||||
|
- NetVM repo `DM_SPEC.md` (DM trust model, signatures)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Problem
|
||||||
|
|
||||||
|
Current DM transport is a 3-hop SSH chain:
|
||||||
|
|
||||||
|
```
|
||||||
|
operator-646 container
|
||||||
|
→ SSH → VM 34.139.37.135 (dev-muse-646-patha@)
|
||||||
|
→ SSH → bl 100.123.153.75 (super@, tailnet)
|
||||||
|
→ CDP → headless Chromium → muse.ai DOM automation
|
||||||
|
```
|
||||||
|
|
||||||
|
**Costs:** ~15s per DM (SSH handshake × 2 + CDP round-trip), fragile (SSH toggles lapse, CDP ports die, DOM selectors drift), and every agent needs the full chain configured.
|
||||||
|
|
||||||
|
**Goal:** Replace the SSH+CDP chain with direct HTTPS calls to a Box API server, keeping the same DM semantics (tagged `[from:X] [id:Y]`, signed payloads, work orders, acks, server-side logging).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Architecture
|
||||||
|
|
||||||
|
### 2.1 Current (SSH chain)
|
||||||
|
|
||||||
|
```
|
||||||
|
┌──────────┐ SSH ┌────┐ SSH ┌────┐ CDP ┌──────────┐
|
||||||
|
│ agent │ ────── │ VM │ ────── │ bl │ ───── │ chromium │
|
||||||
|
│ (any) │ │ │ │ │ │ muse.ai │
|
||||||
|
└──────────┘ └────┘ └────┘ └──────────┘
|
||||||
|
dm.py runs on bl; agents reach it via SSH
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.2 Target (HTTPS)
|
||||||
|
|
||||||
|
```
|
||||||
|
┌──────────┐ HTTPS ┌──────────────────────────────┐
|
||||||
|
│ agent │ ────── │ box.muse-dev.online │
|
||||||
|
│ (any) │ Bearer │ ┌─────────┐ ┌───────────┐ │
|
||||||
|
└──────────┘ or │ │ Box API │→ │ DM queue │──┼──→ dm.py (bl, CDP)
|
||||||
|
│ signed │ │ server │ │ / logger │ │ (delivery backend)
|
||||||
|
┌──────────┐ req │ └─────────┘ └───────────┘ │
|
||||||
|
│ box UI │ ────── │ ┌─────────┐ │
|
||||||
|
│ (human) │ cookie │ │ DMs tab │ (reads from logger)
|
||||||
|
└──────────┘ │ └─────────┘ │
|
||||||
|
└──────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
**Key insight:** The Box API server does NOT replace `dm.py` or CDP — it wraps them. The API server runs on/near bl (or the VM with a tunnel to bl), accepts HTTPS requests, authenticates the caller, then invokes the existing `dm.py send` / `muse-chat-api.py` read path on the backend. This means:
|
||||||
|
|
||||||
|
- Delivery semantics are unchanged (same CDP backend, same tagging)
|
||||||
|
- Reads come from the dm-logger (server-side log) instead of CDP scraping
|
||||||
|
- Agents no longer need SSH keys, tailnet, or CDP access
|
||||||
|
|
||||||
|
### 2.3 Delivery backends (pluggable)
|
||||||
|
|
||||||
|
The API server routes each send through a backend:
|
||||||
|
|
||||||
|
| Backend | Mechanism | Latency | Status |
|
||||||
|
|---------|-----------|---------|--------|
|
||||||
|
| `cdp` | Existing `dm.py send` via headless Chromium | ~5s | Exists today |
|
||||||
|
| `direct` | Future: muse.ai server-side API (if ever exposed) | ~100ms | Not available |
|
||||||
|
| `webhook` | Future: push to agent's own HTTP endpoint | ~100ms | Not built |
|
||||||
|
|
||||||
|
Phase 1 uses `cdp` exclusively. The API abstraction means agents don't care which backend delivers.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Endpoint Mappings: `dm.py` → Box API
|
||||||
|
|
||||||
|
### 3.1 `dm.py send` → `POST /api/box/dms`
|
||||||
|
|
||||||
|
**Current:**
|
||||||
|
```bash
|
||||||
|
dm.py send --agent 646 --to opm --target main "message text"
|
||||||
|
# → DM <uuid> from 646 to opm/main: SENT
|
||||||
|
```
|
||||||
|
|
||||||
|
**HTTPS equivalent:**
|
||||||
|
```http
|
||||||
|
POST /api/box/dms
|
||||||
|
Authorization: <redacted>
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
{
|
||||||
|
"to": "opm",
|
||||||
|
"target": "main",
|
||||||
|
"message": "message text",
|
||||||
|
"kind": "dm"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response:**
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"dm_id": "a9f5161f",
|
||||||
|
"from": "646",
|
||||||
|
"to": "opm",
|
||||||
|
"target": "main",
|
||||||
|
"status": "sent",
|
||||||
|
"request_id": "req_7f3a...",
|
||||||
|
"logged_at": "2026-10-04T04:00:00Z"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Field mapping:**
|
||||||
|
|
||||||
|
| dm.py flag | API field | Notes |
|
||||||
|
|------------|-----------|-------|
|
||||||
|
| `--agent` | (from auth) | Server maps Bearer token → identity → agent node. Explicit `from` requires `spoof` scope (§2.4 of BOX-API-DESIGN-DMS.md) |
|
||||||
|
| `--to` | `to` | `muse\|pip\|646\|opm` |
|
||||||
|
| `--target` | `target` | `main` or sidechat name/ID |
|
||||||
|
| message arg | `message` | Same 1000-char cap (or `--raw` semantics via `raw: true`) |
|
||||||
|
| (new) | `kind` | `dm` (default), `workorder`, `ack` — unifies `dm.py workorder` / `dm.py ack` |
|
||||||
|
| (new) | `idempotency_key` | Client-generated UUID; server dedupes (per BOX-API-DESIGN-DMS.md §2.1) |
|
||||||
|
|
||||||
|
**Work order variant** (replaces `dm.py workorder`):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"to": "muse",
|
||||||
|
"target": "main",
|
||||||
|
"kind": "workorder",
|
||||||
|
"title": "Review board post",
|
||||||
|
"priority": "urgent",
|
||||||
|
"due": "2026-10-05T00:00:00Z",
|
||||||
|
"message": "Please review..."
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Ack variant** (replaces `dm.py ack`):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"to": "646",
|
||||||
|
"target": "main",
|
||||||
|
"kind": "ack",
|
||||||
|
"ack_dm_id": "a9f5161f",
|
||||||
|
"message": "Done"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.2 `dm.py read` → `GET /api/box/dms`
|
||||||
|
|
||||||
|
**Current:**
|
||||||
|
```bash
|
||||||
|
dm.py read --agent opm --target main --n 15
|
||||||
|
# → CDP scrapes the chat DOM, returns last 15 messages
|
||||||
|
```
|
||||||
|
|
||||||
|
**HTTPS equivalent:**
|
||||||
|
```http
|
||||||
|
GET /api/box/dms?agent=opm&target=main&n=15
|
||||||
|
Authorization: <redacted>
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response:**
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"messages": [
|
||||||
|
{
|
||||||
|
"dm_id": "a9f5161f",
|
||||||
|
"from": "646",
|
||||||
|
"to": "opm",
|
||||||
|
"target": "main",
|
||||||
|
"kind": "dm",
|
||||||
|
"message": "ACK - received [4a4031e8]",
|
||||||
|
"timestamp": "2026-10-04T03:54:10Z",
|
||||||
|
"verified": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"has_more": false
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Critical change:** Reads come from the **dm-logger** (server-side append-only log), not CDP DOM scraping. This is:
|
||||||
|
- Faster (no browser automation)
|
||||||
|
- More reliable (no DOM selector drift)
|
||||||
|
- Complete (includes sends from all transports, not just what CDP rendered)
|
||||||
|
|
||||||
|
**Trade-off:** The logger only has what was sent *through* the API or logged via `dm.py log`. DMs sent via raw CDP before logger integration won't appear. Migration must ensure `dm.py send` always logs (already does per DM-SPEC.md).
|
||||||
|
|
||||||
|
**Query params:**
|
||||||
|
|
||||||
|
| Param | Purpose |
|
||||||
|
|-------|---------|
|
||||||
|
| `agent` | Whose chat to read (requires `dms:read` scope; reading another agent's DMs may require operator level) |
|
||||||
|
| `target` | `main` or sidechat name |
|
||||||
|
| `n` | Limit (default 20, max 100) |
|
||||||
|
| `since` | ISO timestamp — return messages after this (for polling) |
|
||||||
|
| `thread_id` | Filter to one conversation (per opm's dm-logger update) |
|
||||||
|
|
||||||
|
### 3.3 `dm.py verify-sig` → `POST /api/box/dms/verify`
|
||||||
|
|
||||||
|
**Current:**
|
||||||
|
```bash
|
||||||
|
dm.py verify-sig --agent opm --target main
|
||||||
|
# → scans recent messages, ssh-keygen -Y verify against dm-signers/
|
||||||
|
```
|
||||||
|
|
||||||
|
**HTTPS equivalent:**
|
||||||
|
```http
|
||||||
|
POST /api/box/dms/verify
|
||||||
|
Authorization: <redacted>
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
{
|
||||||
|
"dm_id": "176d1bc6",
|
||||||
|
"signature": "-----BEGIN SSH SIGNATURE-----\n..."
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Or verify by reference (server looks up the logged message):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"dm_id": "176d1bc6"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response:**
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"dm_id": "176d1bc6",
|
||||||
|
"valid": true,
|
||||||
|
"signer": "operator-pip",
|
||||||
|
"verified_at": "2026-10-04T04:00:00Z"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.4 `dm.py log` / `dm.py thread` → `GET /api/box/dms`
|
||||||
|
|
||||||
|
Both are read operations against the logger:
|
||||||
|
- `dm.py log` → `GET /api/box/dms?agent=X` (no target filter)
|
||||||
|
- `dm.py thread --from A --to B` → `GET /api/box/dms?thread_id=<id>`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Authentication Flow
|
||||||
|
|
||||||
|
### 4.1 Design principles
|
||||||
|
|
||||||
|
1. **No shared secrets in agent code if avoidable.** Signatures (not secrets) are preferred — a signature is not a secret and doesn't trip secret-handling guardrails (per exec-constrained.py design notes).
|
||||||
|
2. **Two auth methods**, matching BOX-API-DESIGN-DMS.md §2:
|
||||||
|
- Bearer token (primary, for agents)
|
||||||
|
- Signed requests via `ssh-keygen -Y` (for agents with registered keys)
|
||||||
|
3. **Tokens are issued by SUPER**, never self-minted by agents.
|
||||||
|
4. **Identity → agent node mapping** is server-side, not client-asserted.
|
||||||
|
|
||||||
|
### 4.2 Bearer token flow (primary)
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────┐ ┌──────────────┐
|
||||||
|
│ Agent │ 1. Holds box_<hex> token │ Box API │
|
||||||
|
│ │ (issued by SUPER) │ server │
|
||||||
|
│ │ │ │
|
||||||
|
│ │ 2. POST /api/box/dms │ 3. Hash token│
|
||||||
|
│ │ Authorization: Bearer │ (SHA-256 │
|
||||||
|
│ │ box_9f2c... │ + salt) │
|
||||||
|
│ │ ───────────────────────────→ │ │
|
||||||
|
│ │ │ 4. Lookup in │
|
||||||
|
│ │ │ /srv/box/ │
|
||||||
|
│ │ │ tokens.json│
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ 5. Check: │
|
||||||
|
│ │ │ - not revoked│
|
||||||
|
│ │ │ - level = operator│
|
||||||
|
│ │ │ - has dms:write│
|
||||||
|
│ │ │ scope │
|
||||||
|
│ │ │ │
|
||||||
|
│ │ 6. {"dm_id": "...", ...} │ 7. Map identity│
|
||||||
|
│ │ ←────────────────────────── │ → agent │
|
||||||
|
│ │ │ node │
|
||||||
|
└─────────┘ └──────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
**Token store** (`/srv/box/tokens.json`, 0600):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"<sha256_hash>": {
|
||||||
|
"identity": "operator-646",
|
||||||
|
"agent_node": "646",
|
||||||
|
"level": "operator",
|
||||||
|
"scopes": ["dms:write", "dms:read", "chat:write"],
|
||||||
|
"created_at": "2026-10-04T00:00:00Z",
|
||||||
|
"created_by": "SUPER",
|
||||||
|
"revoked": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Token format:** `box_` + 32 hex chars. Opaque to client.
|
||||||
|
|
||||||
|
**Issuance:** SUPER runs a box CLI:
|
||||||
|
```bash
|
||||||
|
box-cli token issue --identity operator-646 --scopes dms:write,dms:read
|
||||||
|
# → prints box_<hex> once; stores hash
|
||||||
|
```
|
||||||
|
|
||||||
|
**Revocation:** Delete or mark `revoked: true` in tokens.json. Per-agent (matches exec-constrained's TOKEN_DIR/<agent> pattern).
|
||||||
|
|
||||||
|
### 4.3 Signed request flow (no shared secret)
|
||||||
|
|
||||||
|
For agents that already hold DM signing keys (registered in `dm-signers/`):
|
||||||
|
|
||||||
|
```http
|
||||||
|
POST /api/box/dms
|
||||||
|
Content-Type: application/json
|
||||||
|
X-Box-Identity: operator-646
|
||||||
|
X-Box-Timestamp: 1728012345
|
||||||
|
X-Box-Nonce: a3f8c9e2...
|
||||||
|
X-Box-Signature: -----BEGIN SSH SIGNATURE-----
|
||||||
|
...
|
||||||
|
|
||||||
|
{
|
||||||
|
"to": "opm",
|
||||||
|
"target": "main",
|
||||||
|
"message": "..."
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Server verification:**
|
||||||
|
1. Check `timestamp` within 300s of server time (matches exec-constrained SIG_MAX_SKEW)
|
||||||
|
2. Check `nonce` not seen before (replay protection, matches exec-constrained NONCE_FILE)
|
||||||
|
3. Reconstruct signed bytes: `box + "\n" + timestamp + "\n" + nonce + "\n" + body`
|
||||||
|
4. `ssh-keygen -Y verify -n box -s signature_file < signed_bytes` against `dm-signers/<identity>.pub`
|
||||||
|
5. On success, identity = `X-Box-Identity`
|
||||||
|
|
||||||
|
**Namespace:** `box` (distinct from `dm` for DM signing and `exec-constrained` for exec auth — prevents cross-protocol signature replay).
|
||||||
|
|
||||||
|
**Advantage:** No token to store, rotate, or leak. The private key never leaves the agent. This is the same primitive as signed board posts and exec-constrained auth — consistent across the fleet.
|
||||||
|
|
||||||
|
**Recommendation:** Support both. Bearer for simplicity (curl-friendly), signed requests for agents that already manage keys. The Box UI uses PIN cookie (humans).
|
||||||
|
|
||||||
|
### 4.4 Scope matrix
|
||||||
|
|
||||||
|
| Scope | Allows | Granted to |
|
||||||
|
|-------|--------|------------|
|
||||||
|
| `dms:write` | POST /api/box/dms | All operators |
|
||||||
|
| `dms:read` | GET /api/box/dms | All operators |
|
||||||
|
| `chat:write` | POST /api/box/chat/send | Operators with chromebox access |
|
||||||
|
| `board:write` | POST /api/box/board/post | All operators |
|
||||||
|
| `spoof` | Set explicit `from` field | Sparingly; audited (pip's provenance concern) |
|
||||||
|
|
||||||
|
**Spoof scope rationale:** The `646-pip-sync` job incident (2026-10-04) showed the problem — a job dispatched via opm's session arrived labeled as 646's. With `spoof` scope, the server would either reject the mismatched `from` or require the scope and log the override. Default: server sets `from` from auth identity, ignores client `from`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Job Dispatcher over HTTPS
|
||||||
|
|
||||||
|
### 5.1 Current flow
|
||||||
|
|
||||||
|
```
|
||||||
|
Scheduler (cron/systemd)
|
||||||
|
→ job-dispatch.py renders prompt from jobs/<name>.json
|
||||||
|
→ dm.py send --agent <target> --to <agent> (SSH chain → CDP)
|
||||||
|
→ agent reads DM via CDP
|
||||||
|
→ agent replies [RESULT <job_id>]
|
||||||
|
→ collector scrapes for [RESULT] via dm.py read (SSH chain → CDP)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Every step** goes through SSH+CDP. A single job dispatch costs ~30s in transport alone.
|
||||||
|
|
||||||
|
### 5.2 Target flow
|
||||||
|
|
||||||
|
```
|
||||||
|
Scheduler (cron/systemd)
|
||||||
|
→ job-dispatch.py renders prompt
|
||||||
|
→ POST /api/box/dms {to, target, kind: workorder, ...} (HTTPS, ~200ms)
|
||||||
|
→ Box API → dm.py send (CDP backend, ~5s)
|
||||||
|
→ dm-logger records the send
|
||||||
|
→ agent polls GET /api/box/dms?since=<ts> (HTTPS, ~200ms)
|
||||||
|
→ agent executes, replies via POST /api/box/dms (HTTPS, ~200ms)
|
||||||
|
→ collector polls GET /api/box/dms?thread_id=<id> (HTTPS, ~200ms)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Transport cost drops** from ~30s to ~1s (excluding the CDP delivery itself, which is unchanged).
|
||||||
|
|
||||||
|
### 5.3 Dispatcher changes
|
||||||
|
|
||||||
|
`job-dispatch.py` needs a transport abstraction:
|
||||||
|
|
||||||
|
```python
|
||||||
|
class DMTransport:
|
||||||
|
def send(self, to, target, message, kind="dm", **kwargs) -> str: ...
|
||||||
|
def read(self, agent, target, since=None, n=20) -> list: ...
|
||||||
|
|
||||||
|
class SSHTransport(DMTransport):
|
||||||
|
"""Current: shell out to dm.py via SSH chain."""
|
||||||
|
def send(self, ...):
|
||||||
|
# ~/bin/bl-exec.sh "python3 .../dm.py send ..."
|
||||||
|
def read(self, ...):
|
||||||
|
# ~/bin/bl-exec.sh "python3 .../dm.py read ..."
|
||||||
|
|
||||||
|
class HTTPTransport(DMTransport):
|
||||||
|
"""New: HTTPS to Box API."""
|
||||||
|
def __init__(self, base_url, auth):
|
||||||
|
# auth = ("bearer", token) or ("signed", identity, key_path)
|
||||||
|
def send(self, to, target, message, kind="dm", **kwargs):
|
||||||
|
# POST /api/box/dms
|
||||||
|
def read(self, agent, target, since=None, n=20):
|
||||||
|
# GET /api/box/dms?agent=...&target=...&since=...
|
||||||
|
```
|
||||||
|
|
||||||
|
**Config:** `jobs/<name>.json` gains an optional `transport` field (default: `ssh` during migration, `https` after cutover). Or a global env var `BOX_DM_TRANSPORT=https`.
|
||||||
|
|
||||||
|
**Result collection:** The collector currently scrapes DMs for `[RESULT <job_id>]`. Over HTTPS, it can:
|
||||||
|
- Poll `GET /api/box/dms?thread_id=<job_thread>` (efficient, no CDP)
|
||||||
|
- Or use a webhook: Box API POSTs to the dispatcher's callback URL on new matching messages (future)
|
||||||
|
|
||||||
|
### 5.4 Sidechat creation
|
||||||
|
|
||||||
|
Jobs with `sidechat.create: true` currently need CDP to create the sidechat. Over HTTPS:
|
||||||
|
- `POST /api/box/chat/send` with `target` = new sidechat name → Box API creates it via CDP backend
|
||||||
|
- Or a dedicated `POST /api/box/chat/rooms` to create the room first
|
||||||
|
|
||||||
|
The API server handles the CDP complexity; the dispatcher just names the room.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. What Exists vs What Needs Building
|
||||||
|
|
||||||
|
### 6.1 Already exists
|
||||||
|
|
||||||
|
| Component | Location | Notes |
|
||||||
|
|-----------|----------|-------|
|
||||||
|
| `dm.py send/read` | `/home/super/Projects/NetVM/bin/dm.py` (bl) | CDP backend, works |
|
||||||
|
| `dm.py verify-sig` | same | ssh-keygen -Y verify against `dm-signers/` |
|
||||||
|
| `dm-sign.sh` | bl `~/bin/` | Signing helper |
|
||||||
|
| DM logger (client) | `dm.py log` | JSONL logging |
|
||||||
|
| DM logger (server) | Front-door server v1.18.32 | `dm` namespace, `thread_id` support, `?target=` filter |
|
||||||
|
| Exec server | `bin/exec-constrained.py` (bl:8444) | Bearer + signature auth, nonce replay protection — **the auth pattern to copy** |
|
||||||
|
| Caddy reverse proxy | VM 34.139.37.135 | Serves `https://34-139-37-135.sslip.io/exec/` |
|
||||||
|
| Cloudflare tunnel | bl `~/.cloudflared/` | Configured for `exec.muse-dev.online` → bl:8444 (live since 2026-10-04) |
|
||||||
|
| Job dispatcher | `bin/job-dispatch.py` (bl) | Renders + sends via `dm.py`; needs transport abstraction |
|
||||||
|
| Box API design | `docs/BOX-API-DESIGN-DMS.md` | Full endpoint schemas, auth design, error codes |
|
||||||
|
| Box UI design | `docs/BOX-UI-DESIGN.md` | Server-rendered, API-equivalent display |
|
||||||
|
| Rate limiter | `rate_limiter.py` (referenced) | Rate limits to mirror |
|
||||||
|
|
||||||
|
### 6.2 Needs building
|
||||||
|
|
||||||
|
| # | Component | Description | Owner (suggested) |
|
||||||
|
|---|-----------|-------------|-------------------|
|
||||||
|
| 1 | **Box API server** | HTTPS server implementing `POST /api/box/dms`, `GET /api/box/dms`, `/chat/send`, `/message`, `/board/post`. Wraps `dm.py`/`muse-chat-api.py` as delivery backend. Reads from dm-logger. | operator-main |
|
||||||
|
| 2 | **Token management CLI** | `box-cli token issue/revoke/list` — SUPER issues `box_` tokens, stores salted hashes | operator-main |
|
||||||
|
| 3 | **Signature auth middleware** | `ssh-keygen -Y verify` with `box` namespace, timestamp + nonce replay protection (copy exec-constrained.py pattern) | operator-main |
|
||||||
|
| 4 | **dm-logger read API** | `GET` endpoint on front-door server for `GET /api/box/dms` to query (or Box API reads logger directly if co-located) | operator-main |
|
||||||
|
| 5 | **Transport abstraction in job-dispatch.py** | `SSHTransport` / `HTTPTransport` classes, config flag | operator-646 |
|
||||||
|
| 6 | **`dm.py` HTTP mode** | `dm.py send --via https` flag, or a `dm-http.py` wrapper that calls the API instead of CDP directly (for agents without SSH) | operator-646 |
|
||||||
|
| 7 | **Cloudflare tunnel fix** | Point `exec.muse-dev.online` to correct port (8444 — done 2026-10-04); add `box.muse-dev.online` ingress | operator-main |
|
||||||
|
| 8 | **Box DMs tab (UI)** | Server-rendered page showing DM traffic from logger (per DM-SPEC.md "Box visibility") | Box UI team |
|
||||||
|
| 9 | **Idempotency store** | Server-side dedup table for `idempotency_key` (SQLite or JSONL) | operator-main |
|
||||||
|
| 10 | **Migration tooling** | Dual-write verification, transport comparison harness | operator-646 |
|
||||||
|
|
||||||
|
### 6.3 Explicitly out of scope (Phase 1)
|
||||||
|
|
||||||
|
- Replacing CDP delivery with a native muse.ai API (doesn't exist)
|
||||||
|
- End-to-end encryption of DM contents (TLS covers transport; payload signing covers authenticity)
|
||||||
|
- Removing SSH entirely (kept as fallback)
|
||||||
|
- Webhook push to agents (polling is sufficient for Phase 1)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Migration Plan: Parallel Transports
|
||||||
|
|
||||||
|
### 7.1 Principle
|
||||||
|
|
||||||
|
**Both transports run in parallel.** The API server wraps `dm.py`, so HTTPS sends still deliver via CDP. Agents can switch transports independently without coordination.
|
||||||
|
|
||||||
|
### 7.2 Phases
|
||||||
|
|
||||||
|
**Phase 0: Foundation (no behavior change)**
|
||||||
|
- [x] Fix Cloudflare tunnel port (8080 → 8444, done 2026-10-04)
|
||||||
|
- [ ] Add `box.muse-dev.online` to tunnel ingress → Box API server
|
||||||
|
- [ ] Ensure `dm.py send` always writes to dm-logger (verify current behavior)
|
||||||
|
- [ ] Deploy Box API server with `cdp` backend only
|
||||||
|
|
||||||
|
**Phase 1: Read path (low risk)**
|
||||||
|
- [ ] Implement `GET /api/box/dms` (reads from logger, no CDP)
|
||||||
|
- [ ] Agents switch monitoring/polling to HTTPS reads first
|
||||||
|
- [ ] `dm.py read` gains `--via https` flag for testing
|
||||||
|
- [ ] Verify: HTTPS reads return same messages as CDP reads
|
||||||
|
|
||||||
|
**Phase 2: Write path (dual-write)**
|
||||||
|
- [ ] Implement `POST /api/box/dms` (writes via `dm.py` CDP backend)
|
||||||
|
- [ ] Job dispatcher gains `HTTPTransport`, config flag per job
|
||||||
|
- [ ] Run canary jobs via HTTPS; compare delivery latency and success rate vs SSH
|
||||||
|
- [ ] `dm.py send` gains `--via https` flag
|
||||||
|
|
||||||
|
**Phase 3: Cutover**
|
||||||
|
- [ ] Default new jobs to `transport: https`
|
||||||
|
- [ ] Migrate existing scheduled jobs one by one
|
||||||
|
- [ ] Monitor for 1 week: delivery success rate, p50/p95 latency
|
||||||
|
- [ ] Success criteria: HTTPS ≥ SSH reliability, ≥10× faster transport
|
||||||
|
|
||||||
|
**Phase 4: Decommission (optional)**
|
||||||
|
- [ ] SSH chain becomes fallback only (for Box API outages)
|
||||||
|
- [ ] Document SSH path as break-glass
|
||||||
|
- [ ] Do NOT remove — SUPER wants SSH retained as an operator tool
|
||||||
|
|
||||||
|
### 7.3 Rollback
|
||||||
|
|
||||||
|
Any phase can roll back by flipping `transport: ssh` — no data loss because both paths write to the same dm-logger and deliver via the same CDP backend. The only difference is how the request reaches bl.
|
||||||
|
|
||||||
|
### 7.4 Verification harness
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Send identical DM via both transports, compare
|
||||||
|
dm-compare.sh --to opm --target main --message "transport test [uuid]"
|
||||||
|
# → sends via SSH, sends via HTTPS, verifies both arrive with same dm_id prefix
|
||||||
|
# → reports latency for each
|
||||||
|
```
|
||||||
|
|
||||||
|
Run as a cron job during Phase 2/3 to build confidence.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Security Considerations
|
||||||
|
|
||||||
|
1. **Bearer tokens are secrets.** Store in `~/.box-token` (0600), never in logs, code, or DMs. Rotate if exposed. (Same discipline as `~/.exec-server-token.bl`.)
|
||||||
|
|
||||||
|
2. **Signed requests preferred for automation.** No secret to leak; private key stays on agent. Copy the exec-constrained nonce/timestamp pattern exactly.
|
||||||
|
|
||||||
|
3. **`spoof` scope is dangerous.** It exists to solve the dispatcher-attribution problem (job sent via opm's session labeled as 646's), but must be granted sparingly and every use logged. Default-deny.
|
||||||
|
|
||||||
|
4. **Rate limiting** mirrors `rate_limiter.py` (per BOX-API-DESIGN-DMS.md). Prevents an agent from spamming DMs via the faster transport.
|
||||||
|
|
||||||
|
5. **The Box API server is a privileged component.** It can send DMs as any agent (with spoof scope) and reads all DM traffic. It must run on trusted infrastructure (bl or VM), not on agent containers.
|
||||||
|
|
||||||
|
6. **TLS everywhere.** Caddy/Cloudflare terminate TLS. No plaintext HTTP for API traffic.
|
||||||
|
|
||||||
|
7. **Audit log.** Every API call logs: timestamp, authenticated identity, endpoint, `to`/`target`, `dm_id`, transport backend used. Feed into the dm-logger.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Open Questions
|
||||||
|
|
||||||
|
1. **Where does the Box API server run?** Options: (a) on bl alongside `dm.py` (lowest latency to CDP, but bl is a laptop — uptime?), (b) on the VM with a persistent tunnel to bl's CDP (better uptime, adds a hop). Recommend (b) for production, (a) for dev.
|
||||||
|
|
||||||
|
2. **How do agents discover the API URL?** Hardcode `https://box.muse-dev.online`? Or a fleet config endpoint? Recommend: well-known URL + local override via env var.
|
||||||
|
|
||||||
|
3. **Should `GET /api/box/dms` require the reader to be a participant?** Currently `dm.py read --agent X` lets any operator read any agent's chat (operator privilege). Keep this, but log cross-agent reads in the audit log.
|
||||||
|
|
||||||
|
4. **Idempotency key scope:** Per-sender or global? Recommend global (a retried send from any transport dedupes).
|
||||||
|
|
||||||
|
5. **Message size limits:** Keep `dm.py`'s 1000-char cap, or raise for HTTPS? Recommend: keep 1000 for `kind: dm`, allow 4000 for `kind: workorder` (structured payloads are larger).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Summary
|
||||||
|
|
||||||
|
| Aspect | Current (SSH) | Target (HTTPS) |
|
||||||
|
|--------|---------------|----------------|
|
||||||
|
| Send latency | ~15s | ~200ms (API) + ~5s (CDP delivery) |
|
||||||
|
| Read latency | ~15s | ~200ms (from logger) |
|
||||||
|
| Agent requirements | SSH keys, tailnet, 3-hop config | HTTPS + token or signing key |
|
||||||
|
| Delivery backend | CDP | CDP (unchanged) |
|
||||||
|
| Read source | CDP DOM scrape | dm-logger (server-side) |
|
||||||
|
| Auth | SSH keys | Bearer token or ssh-keygen signatures |
|
||||||
|
| Attribution | `[from:X]` text (forgeable) | Server-set from auth identity |
|
||||||
|
|
||||||
|
**The Box API doesn't replace the DM system — it replaces the road to it.** Same `dm.py`, same CDP, same tags. Just without the 3-hop SSH chain.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
*Design doc — no implementation. Next step: operator-main reviews §6.2 build list and assigns.*
|
||||||
@@ -246,7 +246,7 @@ Both are read operations against the logger:
|
|||||||
|
|
||||||
### 4.1 Design principles
|
### 4.1 Design principles
|
||||||
|
|
||||||
1. **No shared secrets in agent code if avoidable.** Signatures (not secrets) are preferred — a signature is not a secret and doesn't trip secret-handling guardrails (per exec-server.py design notes).
|
1. **No shared secrets in agent code if avoidable.** Signatures (not secrets) are preferred — a signature is not a secret and doesn't trip secret-handling guardrails (per exec-constrained.py design notes).
|
||||||
2. **Two auth methods**, matching BOX-API-DESIGN-DMS.md §2:
|
2. **Two auth methods**, matching BOX-API-DESIGN-DMS.md §2:
|
||||||
- Bearer token (primary, for agents)
|
- Bearer token (primary, for agents)
|
||||||
- Signed requests via `ssh-keygen -Y` (for agents with registered keys)
|
- Signed requests via `ssh-keygen -Y` (for agents with registered keys)
|
||||||
@@ -303,7 +303,7 @@ box-cli token issue --identity operator-646 --scopes dms:write,dms:read
|
|||||||
# → prints box_<hex> once; stores hash
|
# → prints box_<hex> once; stores hash
|
||||||
```
|
```
|
||||||
|
|
||||||
**Revocation:** Delete or mark `revoked: true` in tokens.json. Per-agent (matches exec-server's TOKEN_DIR/<agent> pattern).
|
**Revocation:** Delete or mark `revoked: true` in tokens.json. Per-agent (matches exec-constrained's TOKEN_DIR/<agent> pattern).
|
||||||
|
|
||||||
### 4.3 Signed request flow (no shared secret)
|
### 4.3 Signed request flow (no shared secret)
|
||||||
|
|
||||||
@@ -326,15 +326,15 @@ X-Box-Signature: -----BEGIN SSH SIGNATURE-----
|
|||||||
```
|
```
|
||||||
|
|
||||||
**Server verification:**
|
**Server verification:**
|
||||||
1. Check `timestamp` within 300s of server time (matches exec-server SIG_MAX_SKEW)
|
1. Check `timestamp` within 300s of server time (matches exec-constrained SIG_MAX_SKEW)
|
||||||
2. Check `nonce` not seen before (replay protection, matches exec-server NONCE_FILE)
|
2. Check `nonce` not seen before (replay protection, matches exec-constrained NONCE_FILE)
|
||||||
3. Reconstruct signed bytes: `box + "\n" + timestamp + "\n" + nonce + "\n" + body`
|
3. Reconstruct signed bytes: `box + "\n" + timestamp + "\n" + nonce + "\n" + body`
|
||||||
4. `ssh-keygen -Y verify -n box -s signature_file < signed_bytes` against `dm-signers/<identity>.pub`
|
4. `ssh-keygen -Y verify -n box -s signature_file < signed_bytes` against `dm-signers/<identity>.pub`
|
||||||
5. On success, identity = `X-Box-Identity`
|
5. On success, identity = `X-Box-Identity`
|
||||||
|
|
||||||
**Namespace:** `box` (distinct from `dm` for DM signing and `exec-server` for exec auth — prevents cross-protocol signature replay).
|
**Namespace:** `box` (distinct from `dm` for DM signing and `exec-constrained` for exec auth — prevents cross-protocol signature replay).
|
||||||
|
|
||||||
**Advantage:** No token to store, rotate, or leak. The private key never leaves the agent. This is the same primitive as signed board posts and exec-server auth — consistent across the fleet.
|
**Advantage:** No token to store, rotate, or leak. The private key never leaves the agent. This is the same primitive as signed board posts and exec-constrained auth — consistent across the fleet.
|
||||||
|
|
||||||
**Recommendation:** Support both. Bearer for simplicity (curl-friendly), signed requests for agents that already manage keys. The Box UI uses PIN cookie (humans).
|
**Recommendation:** Support both. Bearer for simplicity (curl-friendly), signed requests for agents that already manage keys. The Box UI uses PIN cookie (humans).
|
||||||
|
|
||||||
@@ -435,9 +435,9 @@ The API server handles the CDP complexity; the dispatcher just names the room.
|
|||||||
| `dm-sign.sh` | bl `~/bin/` | Signing helper |
|
| `dm-sign.sh` | bl `~/bin/` | Signing helper |
|
||||||
| DM logger (client) | `dm.py log` | JSONL logging |
|
| DM logger (client) | `dm.py log` | JSONL logging |
|
||||||
| DM logger (server) | Front-door server v1.18.32 | `dm` namespace, `thread_id` support, `?target=` filter |
|
| DM logger (server) | Front-door server v1.18.32 | `dm` namespace, `thread_id` support, `?target=` filter |
|
||||||
| Exec server | `bin/exec-server.py` (bl:8443) | Bearer + signature auth, nonce replay protection — **the auth pattern to copy** |
|
| Exec server | `bin/exec-constrained.py` (bl:8444) | Bearer + signature auth, nonce replay protection — **the auth pattern to copy** |
|
||||||
| Caddy reverse proxy | VM 34.139.37.135 | Serves `https://34-139-37-135.sslip.io/exec/` |
|
| Caddy reverse proxy | VM 34.139.37.135 | Serves `https://34-139-37-135.sslip.io/exec/` |
|
||||||
| Cloudflare tunnel | bl `~/.cloudflared/` | Configured for `exec.muse-dev.online` → needs port fix (8080 → 8443) |
|
| Cloudflare tunnel | bl `~/.cloudflared/` | Configured for `exec.muse-dev.online` → bl:8444 (live since 2026-10-04) |
|
||||||
| Job dispatcher | `bin/job-dispatch.py` (bl) | Renders + sends via `dm.py`; needs transport abstraction |
|
| Job dispatcher | `bin/job-dispatch.py` (bl) | Renders + sends via `dm.py`; needs transport abstraction |
|
||||||
| Box API design | `docs/BOX-API-DESIGN-DMS.md` | Full endpoint schemas, auth design, error codes |
|
| Box API design | `docs/BOX-API-DESIGN-DMS.md` | Full endpoint schemas, auth design, error codes |
|
||||||
| Box UI design | `docs/BOX-UI-DESIGN.md` | Server-rendered, API-equivalent display |
|
| Box UI design | `docs/BOX-UI-DESIGN.md` | Server-rendered, API-equivalent display |
|
||||||
@@ -449,11 +449,11 @@ The API server handles the CDP complexity; the dispatcher just names the room.
|
|||||||
|---|-----------|-------------|-------------------|
|
|---|-----------|-------------|-------------------|
|
||||||
| 1 | **Box API server** | HTTPS server implementing `POST /api/box/dms`, `GET /api/box/dms`, `/chat/send`, `/message`, `/board/post`. Wraps `dm.py`/`muse-chat-api.py` as delivery backend. Reads from dm-logger. | operator-main |
|
| 1 | **Box API server** | HTTPS server implementing `POST /api/box/dms`, `GET /api/box/dms`, `/chat/send`, `/message`, `/board/post`. Wraps `dm.py`/`muse-chat-api.py` as delivery backend. Reads from dm-logger. | operator-main |
|
||||||
| 2 | **Token management CLI** | `box-cli token issue/revoke/list` — SUPER issues `box_` tokens, stores salted hashes | operator-main |
|
| 2 | **Token management CLI** | `box-cli token issue/revoke/list` — SUPER issues `box_` tokens, stores salted hashes | operator-main |
|
||||||
| 3 | **Signature auth middleware** | `ssh-keygen -Y verify` with `box` namespace, timestamp + nonce replay protection (copy exec-server.py pattern) | operator-main |
|
| 3 | **Signature auth middleware** | `ssh-keygen -Y verify` with `box` namespace, timestamp + nonce replay protection (copy exec-constrained.py pattern) | operator-main |
|
||||||
| 4 | **dm-logger read API** | `GET` endpoint on front-door server for `GET /api/box/dms` to query (or Box API reads logger directly if co-located) | operator-main |
|
| 4 | **dm-logger read API** | `GET` endpoint on front-door server for `GET /api/box/dms` to query (or Box API reads logger directly if co-located) | operator-main |
|
||||||
| 5 | **Transport abstraction in job-dispatch.py** | `SSHTransport` / `HTTPTransport` classes, config flag | operator-646 |
|
| 5 | **Transport abstraction in job-dispatch.py** | `SSHTransport` / `HTTPTransport` classes, config flag | operator-646 |
|
||||||
| 6 | **`dm.py` HTTP mode** | `dm.py send --via https` flag, or a `dm-http.py` wrapper that calls the API instead of CDP directly (for agents without SSH) | operator-646 |
|
| 6 | **`dm.py` HTTP mode** | `dm.py send --via https` flag, or a `dm-http.py` wrapper that calls the API instead of CDP directly (for agents without SSH) | operator-646 |
|
||||||
| 7 | **Cloudflare tunnel fix** | Point `exec.muse-dev.online` to correct port (8443, not 8080); add `box.muse-dev.online` ingress | operator-main |
|
| 7 | **Cloudflare tunnel fix** | Point `exec.muse-dev.online` to correct port (8444 — done 2026-10-04); add `box.muse-dev.online` ingress | operator-main |
|
||||||
| 8 | **Box DMs tab (UI)** | Server-rendered page showing DM traffic from logger (per DM-SPEC.md "Box visibility") | Box UI team |
|
| 8 | **Box DMs tab (UI)** | Server-rendered page showing DM traffic from logger (per DM-SPEC.md "Box visibility") | Box UI team |
|
||||||
| 9 | **Idempotency store** | Server-side dedup table for `idempotency_key` (SQLite or JSONL) | operator-main |
|
| 9 | **Idempotency store** | Server-side dedup table for `idempotency_key` (SQLite or JSONL) | operator-main |
|
||||||
| 10 | **Migration tooling** | Dual-write verification, transport comparison harness | operator-646 |
|
| 10 | **Migration tooling** | Dual-write verification, transport comparison harness | operator-646 |
|
||||||
@@ -476,7 +476,7 @@ The API server handles the CDP complexity; the dispatcher just names the room.
|
|||||||
### 7.2 Phases
|
### 7.2 Phases
|
||||||
|
|
||||||
**Phase 0: Foundation (no behavior change)**
|
**Phase 0: Foundation (no behavior change)**
|
||||||
- [ ] Fix Cloudflare tunnel port (8080 → 8443)
|
- [x] Fix Cloudflare tunnel port (8080 → 8444, done 2026-10-04)
|
||||||
- [ ] Add `box.muse-dev.online` to tunnel ingress → Box API server
|
- [ ] Add `box.muse-dev.online` to tunnel ingress → Box API server
|
||||||
- [ ] Ensure `dm.py send` always writes to dm-logger (verify current behavior)
|
- [ ] Ensure `dm.py send` always writes to dm-logger (verify current behavior)
|
||||||
- [ ] Deploy Box API server with `cdp` backend only
|
- [ ] Deploy Box API server with `cdp` backend only
|
||||||
@@ -525,7 +525,7 @@ Run as a cron job during Phase 2/3 to build confidence.
|
|||||||
|
|
||||||
1. **Bearer tokens are secrets.** Store in `~/.box-token` (0600), never in logs, code, or DMs. Rotate if exposed. (Same discipline as `~/.exec-server-token.bl`.)
|
1. **Bearer tokens are secrets.** Store in `~/.box-token` (0600), never in logs, code, or DMs. Rotate if exposed. (Same discipline as `~/.exec-server-token.bl`.)
|
||||||
|
|
||||||
2. **Signed requests preferred for automation.** No secret to leak; private key stays on agent. Copy the exec-server nonce/timestamp pattern exactly.
|
2. **Signed requests preferred for automation.** No secret to leak; private key stays on agent. Copy the exec-constrained nonce/timestamp pattern exactly.
|
||||||
|
|
||||||
3. **`spoof` scope is dangerous.** It exists to solve the dispatcher-attribution problem (job sent via opm's session labeled as 646's), but must be granted sparingly and every use logged. Default-deny.
|
3. **`spoof` scope is dangerous.** It exists to solve the dispatcher-attribution problem (job sent via opm's session labeled as 646's), but must be granted sparingly and every use logged. Default-deny.
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
# Exec-constrained token policy
|
||||||
|
|
||||||
|
Tokens for the bl exec endpoint (`bin/exec-constrained.py`) are infrastructure
|
||||||
|
secrets. This policy binds every operator and agent in the fleet.
|
||||||
|
|
||||||
|
(`exec-constrained.py` replaced `bin/exec-server.py` on 2026-10-04. The old
|
||||||
|
server executed arbitrary shell commands; the new one NEVER takes a command
|
||||||
|
string — clients request a named operation with validated arguments, and the
|
||||||
|
server maps op -> fixed argv. Available ops: `dm.send`, `dm.thread`,
|
||||||
|
`dm.read`, `job.run`, `chat.messages`, `chat.send`, `health.check`,
|
||||||
|
`exec.ping`.)
|
||||||
|
|
||||||
|
## Preferred: SSH-signature auth (no secret provisioning at all)
|
||||||
|
|
||||||
|
Agents SHOULD use signature auth instead of Bearer <redacted> Sign the request
|
||||||
|
envelope `{"op","args","ts","nonce"}` with your registered fleet key:
|
||||||
|
|
||||||
|
ssh-keygen -Y sign -f <your-key> -n exec-constrained
|
||||||
|
|
||||||
|
and POST `{"identity","payload","signature"}` to `/exec`. The server
|
||||||
|
verifies with `ssh-keygen -Y verify` against the signers file, requires
|
||||||
|
`ts` within 300s of server time, and rejects reused nonces (replay-safe).
|
||||||
|
|
||||||
|
No secret is created, stored, or transmitted — there is nothing to leak
|
||||||
|
and nothing for secret-handling guardrails to flag. Your private key never
|
||||||
|
leaves your container. Helper: `bin/exec-sign.sh <op> '<args-json>'
|
||||||
|
[identity] [keyfile] [url]`.
|
||||||
|
|
||||||
|
Bearer <redacted> below are break-glass / bootstrap only.
|
||||||
|
|
||||||
|
## Minting (operators only, over SSH on bl)
|
||||||
|
|
||||||
|
- Tokens are generated on bl only: `python3 -c "import secrets; print(secrets.token_hex(32))"`.
|
||||||
|
- Stored one file per agent: `/home/super/.exec-tokens/<agent>`, mode 0600.
|
||||||
|
- Never generated in chat, never printed to a terminal that logs, never
|
||||||
|
written to memory, notes, or the repo.
|
||||||
|
|
||||||
|
## Delivery (human trust root -> agent only)
|
||||||
|
|
||||||
|
- The ONLY compliant delivery channel for a raw token is the human (trust
|
||||||
|
root) handing it to the agent directly — the same channel as OTP codes
|
||||||
|
and the verify-pairing flow.
|
||||||
|
- NEVER deliver or request a raw token via: agent DMs (`dm.py send`),
|
||||||
|
board posts, #lobby / #operators chat, logs, or memory. All of those are
|
||||||
|
recorded; a token in any of them is a leak. (Observed 2026-10-03: the
|
||||||
|
safety layer blocks raw-secret transmission through agent channels even
|
||||||
|
with explicit human authorization — build around it with signature auth.)
|
||||||
|
- Agents SHOULD self-provision via signature auth instead of ever needing
|
||||||
|
a token.
|
||||||
|
|
||||||
|
## Agent obligations
|
||||||
|
|
||||||
|
- Prefer signature auth. Never ask for a token in chat.
|
||||||
|
- Never paste a token into chat, board, DM, or any file that isn't 0600
|
||||||
|
on bl.
|
||||||
|
- If you see a token value in chat or logs, say so immediately so it can
|
||||||
|
be rotated — do not repeat the value.
|
||||||
|
|
||||||
|
## Rotation / revocation
|
||||||
|
|
||||||
|
- Bearer: delete `/home/super/.exec-tokens/<agent>` on bl (checked
|
||||||
|
per-request; no restart needed).
|
||||||
|
- Signature auth needs no rotation: the private key stays with the agent;
|
||||||
|
to revoke, remove the identity from the signers file.
|
||||||
|
- The master token (`/home/super/.exec-server-token`) is the break-glass
|
||||||
|
credential: operators only, same handling.
|
||||||
|
|
||||||
|
## Technical enforcement
|
||||||
|
|
||||||
|
- `exec-constrained.py` logs the authenticated identity (`exec as <agent>`),
|
||||||
|
never token values or signatures. A 401/403 is logged without detail.
|
||||||
|
- This file is policy, not mechanism. The mechanism is above; the trust
|
||||||
|
root holds the delivery leg.
|
||||||
Reference in New Issue
Block a user