feat(retention): implement Piece 2 job archival, CLI wiring, and rotation driver

bin/box-ctl.py

@@ -540,15 +540,56 @@ def _job_summary(name, path):
     }
 
 
-def act_job_list():
+def act_job_list(include_archived=False):
     jobs = []
     if JOBS_DIR.exists():
         for p in sorted(JOBS_DIR.glob("*.json")):
             jobs.append(_job_summary(p.stem, p))
+        if include_archived:
+            arch_dir = JOBS_DIR / "archive"
+            if arch_dir.exists():
+                for p in sorted(arch_dir.glob("*.json")):
+                    s = _job_summary(p.stem, p)
+                    s["archived"] = True
+                    jobs.append(s)
     audit("job-list")
     out(True, jobs=jobs)
 
 
+def act_job_archive(name, force=False):
+    check_name(name)
+    script = BIN / "retention-archive-jobs.py"
+    cmd = [sys.executable, str(script), "archive", name, "--json"]
+    if force:
+        cmd.append("--force")
+    r = run(cmd)
+    try:
+        data = json.loads(r.stdout)
+    except Exception:
+        fail("ARCHIVE_FAILED", r.stderr or r.stdout)
+    if data.get("status") in ("archived", "dry_run"):
+        audit("job-archive", name)
+        out(True, **data)
+    else:
+        fail("ARCHIVE_FAILED", data.get("error") or data.get("status"))
+
+
+def act_job_unarchive(name):
+    check_name(name)
+    script = BIN / "retention-archive-jobs.py"
+    cmd = [sys.executable, str(script), "unarchive", name, "--json"]
+    r = run(cmd)
+    try:
+        data = json.loads(r.stdout)
+    except Exception:
+        fail("UNARCHIVE_FAILED", r.stderr or r.stdout)
+    if data.get("status") in ("unarchived", "dry_run"):
+        audit("job-unarchive", name)
+        out(True, **data)
+    else:
+        fail("UNARCHIVE_FAILED", data.get("error") or data.get("status"))
+
+
 def act_job_get(name):
     check_name(name)
     p = JOBS_DIR / f"{name}.json"
@@ -1821,6 +1862,176 @@ def act_ssh_info(name):
     out(True, **agent_md.get_ssh_info(name))
 
 
+SSH_CHECK_TIMEOUT = 25
+
+
+def build_ssh_sweep_script():
+    """Return the python3 probe script executed on the jump host.
+
+    Reads `kind:port` args (kind is `ssh` or `term`), connects to each
+    127.0.0.1:port, grabs the SSH banner or terminal HTTP status line,
+    and prints one JSON object: {"ports": {port: {...}}}.
+    """
+    return (
+        "import json, socket, sys, time\n"
+        "out = {}\n"
+        "for arg in sys.argv[1:]:\n"
+        "    try:\n"
+        "        kind, port_s = arg.split(':', 1)\n"
+        "        port = int(port_s)\n"
+        "    except ValueError:\n"
+        "        continue\n"
+        "    rec = {'open': False, 'latency_ms': None, 'banner': '', 'http': ''}\n"
+        "    t0 = time.time()\n"
+        "    try:\n"
+        "        s = socket.create_connection(('127.0.0.1', port), timeout=2.0)\n"
+        "    except Exception:\n"
+        "        out[str(port)] = rec\n"
+        "        continue\n"
+        "    rec['open'] = True\n"
+        "    rec['latency_ms'] = int((time.time() - t0) * 1000)\n"
+        "    try:\n"
+        "        s.settimeout(2.0)\n"
+        "        if kind == 'term':\n"
+        "            s.sendall(b'GET / HTTP/1.0\\r\\n\\r\\n')\n"
+        "            data = s.recv(128)\n"
+        "            rec['http'] = data.split(b'\\n')[0].decode('utf-8', 'replace').strip()[:64]\n"
+        "        else:\n"
+        "            data = s.recv(128)\n"
+        "            rec['banner'] = data.split(b'\\n')[0].decode('utf-8', 'replace').strip()[:64]\n"
+        "    except Exception:\n"
+        "        pass\n"
+        "    try:\n"
+        "        s.close()\n"
+        "    except Exception:\n"
+        "        pass\n"
+        "    out[str(port)] = rec\n"
+        "print(json.dumps({'ports': out}))\n"
+    )
+
+
+def parse_ssh_sweep_result(sweep, tunnels):
+    """Map sweep {port: probe} output onto {account: health} via TUNNEL_PORTS.
+
+    Unknown sweep ports are ignored; accounts with no sweep data keep
+    None (unknown) health. Always returns every account in tunnels.
+    """
+    ports = sweep.get("ports", {}) if isinstance(sweep, dict) else {}
+    by_port = {}
+    for acct, info in tunnels.items():
+        by_port[str(info.get("port"))] = (acct, "ssh")
+        by_port[str(info.get("terminal"))] = (acct, "term")
+
+    accounts = {}
+    for acct, info in tunnels.items():
+        accounts[acct] = {
+            "ssh_port": info.get("port"),
+            "ssh_up": None,
+            "ssh_latency_ms": None,
+            "ssh_banner": "",
+            "term_port": info.get("terminal"),
+            "term_up": None,
+            "term_latency_ms": None,
+            "term_http": "",
+            "container_user": info.get("user", "hatch"),
+        }
+    if not isinstance(ports, dict):
+        return accounts
+    for port_s, probe in ports.items():
+        slot = by_port.get(str(port_s))
+        if not slot or not isinstance(probe, dict):
+            continue
+        acct, kind = slot
+        ent = accounts.get(acct)
+        if ent is None:
+            continue
+        is_open = bool(probe.get("open"))
+        lat = probe.get("latency_ms")
+        try:
+            lat = int(lat) if lat is not None else None
+        except (TypeError, ValueError):
+            lat = None
+        if kind == "ssh":
+            ent["ssh_up"] = is_open
+            ent["ssh_latency_ms"] = lat if is_open else None
+            ent["ssh_banner"] = str(probe.get("banner") or "")[:64]
+        else:
+            ent["term_up"] = is_open
+            ent["term_latency_ms"] = lat if is_open else None
+            ent["term_http"] = str(probe.get("http") or "")[:64]
+    return accounts
+
+
+def run_vm_port_sweep(jump_host, operator_user, tunnels, timeout=SSH_CHECK_TIMEOUT):
+    """Run one SSH to the jump host sweeping all tunnel ports.
+
+    Returns (sweep_dict_or_None, error_str). sweep is the parsed
+    {"ports": {...}} payload; error is "" on success.
+    """
+    sweep_args = []
+    for _acct, info in tunnels.items():
+        sweep_args.append(f"ssh:{info.get('port')}")
+        sweep_args.append(f"term:{info.get('terminal')}")
+    cmd = [
+        "ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=8",
+        "-o", "StrictHostKeyChecking=no",
+    ]
+    identity = os.environ.get("SSH_IDENTITY_FILE", "")
+    if identity:
+        cmd += ["-o", "IdentitiesOnly=yes", "-i", identity]
+    cmd += [
+        f"{operator_user}@{jump_host}",
+        "python3", "-",
+    ] + sweep_args
+    try:
+        r = subprocess.run(cmd, input=build_ssh_sweep_script(),
+                           capture_output=True, text=True, timeout=timeout)
+    except subprocess.TimeoutExpired:
+        return None, f"jump host {jump_host} sweep timed out after {timeout}s"
+    except FileNotFoundError:
+        return None, "local ssh binary not found"
+    except Exception as e:
+        return None, f"ssh to {jump_host} failed: {e}"
+    if r.returncode != 0:
+        err = (r.stderr or "").strip().splitlines()
+        hint = err[-1][:160] if err else f"exit {r.returncode}"
+        return None, f"jump host {jump_host} unreachable: {hint}"
+    try:
+        sweep = json.loads(r.stdout)
+    except Exception:
+        return None, f"jump host {jump_host} returned unparseable sweep output"
+    if not isinstance(sweep, dict) or "ports" not in sweep:
+        return None, f"jump host {jump_host} returned malformed sweep output"
+    return sweep, ""
+
+
+def act_ssh_check():
+    """Probe all container reverse-tunnel ports from the jump host.
+
+    Single SSH connection, VM-side sweep. Always emits HTTP-200-style
+    ok:true with per-account health; jump failures surface as
+    jump_reachable:false (degraded-state data, not a fatal error).
+    """
+    import time as _time
+    import agent_md
+    audit("ssh-check")
+    t0 = _time.time()
+    tunnels = dict(agent_md.TUNNEL_PORTS)
+    jump_host = os.environ.get("SSH_JUMP_HOST", "34.139.37.135")
+    operator_user = os.environ.get("OPERATOR_USER", "super")
+    sweep, err = run_vm_port_sweep(jump_host, operator_user, tunnels)
+    wall_ms = int((_time.time() - t0) * 1000)
+    accounts = parse_ssh_sweep_result(sweep or {}, tunnels)
+    if err:
+        out(True, jump_host=jump_host, operator_user=operator_user,
+            jump_reachable=False, error=err, accounts=accounts,
+            checked_at=utcnow(), latency_ms=wall_ms)
+    else:
+        out(True, jump_host=jump_host, operator_user=operator_user,
+            jump_reachable=True, accounts=accounts,
+            checked_at=utcnow(), latency_ms=wall_ms)
+
+
 MD_MAX_READ = 64 * 1024
 MD_MAX_DIFF = 64 * 1024
 MD_MAX_LIST = 200
@@ -2252,12 +2463,13 @@ onboard actions:
   onboard-connects            consolidated active fleet and client onboard connects (read-only)
   (space-separated alias: onboard connects)
 
-ssh actions (space-separated alias: ssh mint|list|show|ports|info):
+ssh actions (space-separated alias: ssh mint|list|show|ports|info|check):
   ssh-mint <name> [--force]   mint a new SSH key
   ssh-list                    list minted keys
   ssh-show <name>             show key detail
   ssh-ports                   tunnel port inventory
-  ssh-info [name]             connection coordinates"""
+  ssh-info [name]             connection coordinates
+  ssh-check                   live tunnel health sweep via jump host"""
 
 
 def act_thread(op, agent, thread=None, title=None, limit=None, confirm=False):
@@ -3723,7 +3935,17 @@ def main(argv):
             else:
                 fail("BAD_NAME", "usage: timer list|status|create|delete|start|stop|enable|disable [...]")
     elif action == "job-list":
-        act_job_list()
+        include_archived = "--archived" in rest or "-a" in rest
+        act_job_list(include_archived=include_archived)
+    elif action == "job-archive":
+        if not rest or len(rest) > 2:
+            fail("BAD_NAME", "usage: job-archive <name> [--force]")
+        force = "--force" in rest[1:]
+        act_job_archive(rest[0], force=force)
+    elif action == "job-unarchive":
+        if len(rest) != 1:
+            fail("BAD_NAME", "usage: job-unarchive <name>")
+        act_job_unarchive(rest[0])
     elif action == "job-get":
         if len(rest) != 1:
             fail("BAD_NAME", "usage: job-get <name>")
@@ -3861,7 +4083,7 @@ def main(argv):
             fail("BAD_NAME", "usage: loop-resolve <dm_id> [note]")
         note = rest[1] if len(rest) > 1 else None
         act_loop_resolve(rest[0], note=note)
-    elif action in ("ssh", "ssh-mint", "ssh-list", "ssh-show", "ssh-ports", "ssh-info"):
+    elif action in ("ssh", "ssh-mint", "ssh-list", "ssh-show", "ssh-ports", "ssh-info", "ssh-check"):
         if action == "ssh-mint":
             if not rest:
                 fail("BAD_ARGS", "usage: ssh-mint <name> [--force]")
@@ -3876,9 +4098,11 @@ def main(argv):
             act_ssh_ports()
         elif action == "ssh-info":
             act_ssh_info(rest[0] if rest else None)
+        elif action == "ssh-check":
+            act_ssh_check()
         elif action == "ssh":
-            if not rest or rest[0] not in ("mint", "list", "show", "ports", "info"):
-                fail("BAD_NAME", "usage: ssh mint|list|show|ports|info [...]")
+            if not rest or rest[0] not in ("mint", "list", "show", "ports", "info", "check"):
+                fail("BAD_NAME", "usage: ssh mint|list|show|ports|info|check [...]")
             sub = rest[0]
             args = rest[1:]
             if sub == "mint":
@@ -3895,6 +4119,8 @@ def main(argv):
                 act_ssh_ports()
             elif sub == "info":
                 act_ssh_info(args[0] if args else None)
+            elif sub == "check":
+                act_ssh_check()
     elif action in ("md", "md-audit", "md-list", "md-read", "md-write", "md-diff", "md-inject-drive", "md-sync-all",
                       "md-amend", "md-append", "md-pull"):
         if action == "md-audit":