feat(messaging): balanced TOOL parsing, DM shorthand, box.exec, tools.list

- response-harvester: extract [TOOL]/[EXEC] JSON args with balanced-brace scanning (']' and nesting inside args no longer truncate calls); add [DM {...}] shorthand mapping to dm.send; native aliases (dm, box, tools) plus arg-synonym normalization; formatters and expanded hints. - exec-constrained: new read-only box.exec op (27 allowlisted box-ctl reads) and tools.list op backed by --list-ops for dynamic discovery. - prompt_envelope: advertise dm.send/box.exec/tools.list in every timer DM; add dm_call builder. - lookup_engine + regex_patterns.json: canonical tool_call pattern accepts the DM engine, ']' in args, one nesting level. - tests/test_tool_calls.py: 38 tests; docs/INBAND-MESSAGING-SPEC.md: accepted decision record (Final).
2026-10-06 07:29:16 +00:00
parent 345eb09559
commit f2640397ed
7 changed files with 571 additions and 24 deletions
@@ -994,6 +994,67 @@ def _swarm_results_build(a):
    return [sys.executable, os.path.join(BIN_DIR, 'box-ctl.py'), 'swarm-results', a['swarm_id']]


+# box.exec allowlist: read-only box-ctl actions only (mirrors the read-only
+# subset of box-ctl.py IDEMPOTENT_ACTIONS). Actions with side-effecting
+# subverbs (main-loop enable/disable), path args (git-diff/git-log), or
+# complex argv shapes (job-next, policy-*, quality-validate, job-result)
+# are deliberately excluded.
+BOX_EXEC_NOARG_ACTIONS = frozenset({
+    'fleet-status', 'watchdog-alerts', 'relay-health', 'cdp-latency',
+    'chrome-errors', 'identity-audit', 'timer-list', 'job-list',
+    'vars-list', 'strat-list', 'loop-status', 'loop-health', 'loop-breaks',
+    'thread-list', 'dm-log', 'unread', 'swarm-list', 'quality-check',
+    'git-status',
+})
+BOX_EXEC_ONEARG_ACTIONS = frozenset({
+    'job-get', 'job-status', 'timer-status', 'vars-get', 'vars-history',
+    'strat-get', 'swarm-status', 'swarm-results',
+})
+_BOX_ARG_RE = re.compile(r'^[A-Za-z0-9][A-Za-z0-9/_.-]{0,127}$')
+
+
+def _box_exec_validate(raw):
+    if not isinstance(raw, dict):
+        raise OpError('args must be an object')
+    for k in raw:
+        if k not in {'action', 'arg', 'agent'}:
+            raise OpError(f'unknown arg: {k}')
+    action = raw.get('action')
+    if action in BOX_EXEC_NOARG_ACTIONS:
+        if raw.get('arg') is not None:
+            raise OpError(f'{action} takes no arg')
+        return {'action': action}
+    if action in BOX_EXEC_ONEARG_ACTIONS:
+        arg = raw.get('arg')
+        if arg is None:
+            return {'action': action}
+        if not isinstance(arg, str) or not _BOX_ARG_RE.fullmatch(arg):
+            raise OpError('arg must match safe token')
+        return {'action': action, 'arg': arg}
+    raise OpError(f'unknown or non-read-only box action: {action}')
+
+
+def _box_exec_build(a):
+    argv = [sys.executable, os.path.join(BIN_DIR, 'box-ctl.py'), a['action']]
+    if a.get('arg'):
+        argv.append(a['arg'])
+    return argv
+
+
+def _tools_list_validate(raw):
+    if not isinstance(raw, dict):
+        raise OpError('args must be an object')
+    for k in raw:
+        if k not in {'agent'}:
+            raise OpError(f'unknown arg: {k}')
+    return {}
+
+
+def _tools_list_build(a):
+    return [sys.executable, os.path.join(BIN_DIR, 'exec-constrained.py'),
+            '--list-ops']
+
+
 # op -> {validate, build, timeout, side_effecting, description}
 OPS = {
    'dm.send': {
@@ -1207,6 +1268,16 @@ OPS = {
        'timeout': 10, 'side_effecting': False,
        'desc': 'Canary no-op for watchdogs',
    },
+    'box.exec': {
+        'validate': _box_exec_validate, 'build': _box_exec_build,
+        'timeout': 60, 'side_effecting': False,
+        'desc': 'Call a read-only box-ctl action (fleet-status, dm-log, job-get, ...)',
+    },
+    'tools.list': {
+        'validate': _tools_list_validate, 'build': _tools_list_build,
+        'timeout': 15, 'side_effecting': False,
+        'desc': 'List all exec ops with descriptions (dynamic discovery)',
+    },
 }

 # identity -> set of ops. 'master' may invoke everything. Unknown identities
@@ -1495,6 +1566,19 @@ def main():
    ap.add_argument('--key-file',
                    default='/home/super/.exec-constrained-key.pem')
    ap.add_argument('--work-dir', default='/home/super')
+    ap.add_argument('--list-ops', action='store_true',
+                    help='Print the op registry as JSON and exit (backs tools.list)')
+    args = ap.parse_args()
+
+    if args.list_ops:
+        print(json.dumps({
+            'ok': True,
+            'ops': [{'op': k, 'desc': v.get('desc', ''),
+                     'side_effecting': bool(v.get('side_effecting', False)),
+                     'timeout': v.get('timeout', 30)}
+                    for k, v in sorted(OPS.items())],
+        }))
+        return
    args = ap.parse_args()

    TOKEN_FILE, TOKEN_DIR = args.token_file, args.token_dir
@@ -31,7 +31,7 @@ _COMPILED_PATTERNS: Dict[str, re.Pattern] = {}
 # Fallback hardcoded regexes in case files are missing or unreadable
 _FALLBACK_RESULT_RE = re.compile(r"\[RESULT\s+([A-Za-z0-9_/-]+)\]\s*(.*?)(?=\[RESULT\s|\Z)", re.S)
 _FALLBACK_VERB_RE = re.compile(r"\[(ACK|CLAIM|RESULT|DECLINE|NO-ACTION)\s+([A-Za-z0-9_/-]+)\]")
-_FALLBACK_TOOL_RE = re.compile(r"\[(TOOL|EXEC)\s+([a-zA-Z0-9_.-]+)\s+(\{.*?\})\]", re.S)
+_FALLBACK_TOOL_RE = re.compile(r"\[(TOOL|EXEC|DM)\s+(?:([a-zA-Z0-9_.-]+)\s+)?(\{([^{}]|\{[^{}]*\})*\})\]", re.S)
 _FALLBACK_CONTRACT_FOOTER = (
    "Reply: [ACK id] seen | [CLAIM id] mine | "
    "[RESULT id] done | [DECLINE id] | [NO-ACTION id]."
@@ -57,14 +57,22 @@ def pick_profile(job_name):


 def _tool(op, args):
-    # no ']' inside the JSON: the harvester's [TOOL ...] regex stops at the first one
+    # the harvester extracts JSON args with balanced-brace scanning, so
+    # ']' and nested objects/arrays inside args are safe
    return "[TOOL %s %s]" % (op, json.dumps(args, separators=(", ", ": ")))


 def spawn_call(job_id, job_name, profile):
    count, _, hint = PROFILES[profile]
-    # no brackets in the task text: the harvester's [TOOL ...] regex stops at the first ']'
    task = "Subagent for job %s (%s): %s." % (job_id, job_name, hint)
+
+
+def dm_call(to, target, message):
+    return "[DM %s]" % json.dumps(
+        {"to": to, "target": target, "message": message[:900]},
+        separators=(", ", ": "))
+
+
    return _tool("swarm.spawn", {"count": count, "task": task[:900], "label": (job_name or "job")[:60]})


@@ -111,7 +119,9 @@ def wrap(job_name, job_id, agent, target, rendered):
    bottom = (
        "\n--- End Task ---\n\n"
        f"Inspect tmux output: [TOOL tmux.capture {{\"session\": \"{session_name}\", \"lines\": 30}}]\n"
-        "Tools available: cron.create, cron.runs, health.check, swarm.spawn, swarm.list.\n"
+        "Tools: cron.create, cron.runs, health.check, swarm.spawn, swarm.list, dm.send, box.exec, tools.list.\n"
+        "Message a peer: [DM {\"to\": \"<agent>\", \"target\": \"<sidechat>\", \"message\": \"<text>\"}].\n"
+        "Query box: [TOOL box.exec {\"action\": \"<fleet-status|dm-log|job-get|...>\"}] \u2014 [TOOL tools.list {}] lists every op.\n"
    )
    if not has_result:
        bottom += f"When complete, report your verdict: [RESULT {job_id}] OK: <summary of actions>\n"
@@ -154,6 +154,12 @@ NATIVE_ALIASES = {
    "subagents.spawn": "swarm.spawn",
    "subagent.list": "swarm.list",
    "subagent.status": "swarm.status",
+    "dm": "dm.send",
+    "message.send": "dm.send",
+    "box": "box.exec",
+    "box.run": "box.exec",
+    "tools": "tools.list",
+    "tools.list": "tools.list",
 }


@@ -184,6 +190,23 @@ def normalize_native_call(op, args):
                    break
        if "count" not in args and "n" in args:
            args["count"] = args.pop("n")
+    elif op == "dm.send":
+        if "message" not in args:
+            for k in ("text", "body", "content", "msg"):
+                if k in args:
+                    args["message"] = args.pop(k)
+                    break
+        if "target" not in args:
+            for k in ("thread", "sidechat", "channel"):
+                if k in args:
+                    args["target"] = args.pop(k)
+                    break
+    elif op == "box.exec":
+        if "action" not in args:
+            for k in ("cmd", "verb", "command", "run"):
+                if k in args:
+                    args["action"] = args.pop(k)
+                    break
    elif op.startswith("tmux."):
        if "session" not in args:
            for k in ("name", "target", "s"):
@@ -198,24 +221,96 @@ def normalize_native_call(op, args):
    return op, args


+_TOOL_OPEN_RE = re.compile(r"\[(TOOL|EXEC)\s+([a-zA-Z0-9_.-]+)\s*")
+_DM_OPEN_RE = re.compile(r"\[DM\s+")
+
+
+def _extract_balanced_json(s, i):
+    """Extract one JSON object starting at s[i] == '{' (brace-aware, string-aware).
+
+    Returns (obj, end_index) with end_index just past the closing brace,
+    or (None, i) when no balanced object is present. Unlike a first-']'
+    regex this tolerates ']' (and nested objects/arrays) inside args.
+    """
+    if i >= len(s) or s[i] != "{":
+        return None, i
+    depth = 0
+    in_str = False
+    esc = False
+    for j in range(i, len(s)):
+        c = s[j]
+        if in_str:
+            if esc:
+                esc = False
+            elif c == "\\":
+                esc = True
+            elif c == '"':
+                in_str = False
+        elif c == '"':
+            in_str = True
+        elif c == "{":
+            depth += 1
+        elif c == "}":
+            depth -= 1
+            if depth == 0:
+                try:
+                    return json.loads(s[i:j + 1]), j + 1
+                except Exception:
+                    return None, i
+    return None, i
+
+
+def _scan_bracket_calls(text):
+    """Yield (op, args) for [TOOL op {...}] / [EXEC op {...}] / [DM {...}].
+
+    JSON args are extracted with balanced-brace scanning so ']' inside
+    strings, arrays, or nested objects no longer truncates the call.
+    Non-JSON tails keep the legacy first-']' behavior (raw passthrough).
+    """
+    out = []
+    spans = []
+    for m in _TOOL_OPEN_RE.finditer(text or ""):
+        spans.append((m.start(), "tool", m.group(2).strip(), m.end()))
+    for m in _DM_OPEN_RE.finditer(text or ""):
+        spans.append((m.start(), "dm", "dm.send", m.end()))
+    spans.sort()
+    for _, kind, op, pos in spans:
+        if pos < len(text) and text[pos] == "{":
+            args, _ = _extract_balanced_json(text, pos)
+            if args is None:
+                continue
+            if not isinstance(args, dict):
+                args = {"raw": args}
+        elif kind == "dm":
+            continue  # [DM ...] requires a JSON object; skip bare forms
+        else:
+            end = text.find("]", pos)
+            if end == -1:
+                continue
+            raw_args = text[pos:end].strip()
+            if not raw_args:
+                args = {}
+            else:
+                try:
+                    args = json.loads(raw_args)
+                    if not isinstance(args, dict):
+                        args = {"raw": args}
+                except Exception:
+                    args = {"raw": raw_args}
+        out.append((op, args))
+    return out
+
+
 def parse_tool_calls(text):
    """
    Extract structured tool/exec calls from assistant messages.
    Supports:
-      1. [TOOL <op> <json_args>] or [EXEC <op> <json_args>]
-      2. ```box / ```tool / ```exec JSON blocks
+      1. [TOOL <op> <json_args>] or [EXEC <op> <json_args>] (JSON may nest)
+      2. [DM <json_args>] shorthand for dm.send
+      3. ```box / ```tool / ```exec JSON blocks
    """
    calls = []
-    for m in re.finditer(r"\[(?:TOOL|EXEC)\s+([a-zA-Z0-9_.-]+)(?:\s+(.*?))?\]", text or ""):
-        op = m.group(1).strip()
-        raw_args = (m.group(2) or "").strip()
-        args = {}
-        if raw_args:
-            try:
-                args = json.loads(raw_args)
-            except Exception:
-                args = {"raw": raw_args}
-        calls.append((op, args))
+    calls.extend(_scan_bracket_calls(text))

    for m in re.finditer(r"```(?:box|tool|exec)\s*\n(.*?)```", text or "", re.DOTALL):
        block = m.group(1).strip()
@@ -311,6 +406,10 @@ def format_tool_result_for_chat(op, raw_output):
        data = json.loads(raw_output)
    except Exception:
        s = str(raw_output).strip()
+        if op == "box.exec":
+            if len(s) > 900:
+                s = s[:900] + "\n…(truncated, refine the call for detail)"
+            return f"box result:\n```\n{s}\n```"
        return s[:500] if len(s) > 500 else s

    if op == "health.check" and isinstance(data, dict) and "fleet" in data:
@@ -385,6 +484,26 @@ def format_tool_result_for_chat(op, raw_output):
        sw = data.get("swarm", {})
        return f"Swarm `{sw.get('swarm_id')}`: {sw.get('status')} ({sw.get('done', 0)}/{sw.get('count', 0)} slots completed)."

+    if op == "tools.list" and isinstance(data, dict):
+        ops = data.get("ops", [])
+        if not ops:
+            return "No tools registered."
+        ro = [o["op"] for o in ops if not o.get("side_effecting")]
+        se = [o["op"] for o in ops if o.get("side_effecting")]
+        lines = [f"{len(ops)} tools available via [TOOL <op> <args>]."]
+        lines.append("read-only: " + (", ".join(ro) if ro else "none"))
+        if se:
+            lines.append("side-effecting: " + ", ".join(se))
+        return "\n".join(lines)
+
+    if op == "box.exec" and isinstance(data, dict):
+        if data.get("ok") is False:
+            return f"box call failed: {data.get('error', 'unknown error')}"
+        out = json.dumps(data)
+        if len(out) > 900:
+            out = out[:900] + "\n…(truncated, refine the call for detail)"
+        return f"box result:\n```\n{out}\n```"
+
    # General fallback: compact JSON capped to 400 chars
    s = json.dumps(data)
    return s[:400] + "..." if len(s) > 400 else s
@@ -774,9 +893,12 @@ def process_messages(raw_messages, agent, thread_id, thread_name, last_wm, follo
                            thread_url = f"https://box.muse-dev.online/thread/{thread_id}"
                            tool_hint = (
                                f"[Runtime Context: {thread_url}]\n"
-                                f"Tools available: [TOOL <op> <args>] or curl -sk -X POST https://exec.muse-dev.online/exec\n"
+                                f"Tools: [TOOL <op> <args>] or curl -sk -X POST https://exec.muse-dev.online/exec\n"
+                                f"  • [TOOL tools.list {{}}] — discover every op dynamically\n"
+                                f"  • [TOOL swarm.spawn {{\"count\": 1, \"task\": \"<task>\"}}] — spawn subagents\n"
+                                f"  • [DM {{\"to\": \"<agent>\", \"target\": \"<sidechat>\", \"message\": \"<text>\"}}] — send a DM\n"
+                                f"  • [TOOL box.exec {{\"action\": \"fleet-status\"}}] — call box (read-only actions)\n"
                                f"  • [TOOL followup.create {{\"in_m\": 5, \"prompt\": \"<reminder>\"}}]\n"
-                                f"  • [TOOL swarm.spawn {{\"count\": 1, \"task\": \"<task>\"}}]\n"
                                f"  • [TOOL health.check {{}}]\n\n"
                                f"[Directive: Take next action or close with [RESULT <job_id>] <summary>]"
                            )