346 lines
15 KiB
Python
346 lines
15 KiB
Python
|
"""Tests for no-SSH agent development and communication streams.
|
||
|
|
|
||
|
|
Covers the second HTTPS expansion:
|
||
|
|
box-relay.sh (agent client) -> exec-constrained.py named ops
|
||
|
|
-> box-ctl.py backend verbs -> git / unittest / dm.py.
|
||
|
|
|
||
|
|
Live-socket round-trips are intentionally NOT covered here (loopback TCP is
|
||
|
|
unavailable in some sandboxes); instead we assert the exact argv each op
|
||
|
|
builds and execute the fast, side-effect-free argv directly. Live sends
|
||
|
|
(notify/ack) are NEVER executed here: only their validation-failure paths.
|
||
|
|
"""
|
||
|
|
import importlib.util
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
import unittest
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||
|
|
BOX_CTL = REPO_ROOT / "bin" / "box-ctl.py"
|
||
|
|
RELAY = REPO_ROOT / "bin" / "box-relay.sh"
|
||
|
|
|
||
|
|
|
||
|
|
def _load(name, relpath):
|
||
|
|
spec = importlib.util.spec_from_file_location(name, REPO_ROOT / relpath)
|
||
|
|
mod = importlib.util.module_from_spec(spec)
|
||
|
|
spec.loader.exec_module(mod)
|
||
|
|
return mod
|
||
|
|
|
||
|
|
|
||
|
|
exec_constrained = _load("exec_constrained_dev", "bin/exec-constrained.py")
|
||
|
|
|
||
|
|
|
||
|
|
def _box_ctl(*args):
|
||
|
|
return subprocess.run(
|
||
|
|
[sys.executable, str(BOX_CTL), *args],
|
||
|
|
capture_output=True, text=True, timeout=120)
|
||
|
|
|
||
|
|
|
||
|
|
class ExecGitOpsTests(unittest.TestCase):
|
||
|
|
def test_ops_registered_and_read_only(self):
|
||
|
|
for op in ("git.status", "git.diff", "git.log"):
|
||
|
|
self.assertIn(op, exec_constrained.OPS)
|
||
|
|
self.assertFalse(exec_constrained.OPS[op]["side_effecting"])
|
||
|
|
|
||
|
|
def test_default_perms_include_git_ops(self):
|
||
|
|
self.assertTrue(exec_constrained.permitted("some-unknown-identity", "git.status"))
|
||
|
|
self.assertTrue(exec_constrained.permitted("some-unknown-identity", "git.diff"))
|
||
|
|
self.assertTrue(exec_constrained.permitted("some-unknown-identity", "git.log"))
|
||
|
|
self.assertFalse(exec_constrained.permitted("exec-canary", "git.status"))
|
||
|
|
|
||
|
|
def test_git_status_validate(self):
|
||
|
|
v = exec_constrained.OPS["git.status"]["validate"]
|
||
|
|
self.assertEqual(v({}), {})
|
||
|
|
with self.assertRaises(exec_constrained.OpError):
|
||
|
|
v({"bogus": 1})
|
||
|
|
|
||
|
|
def test_git_diff_validate(self):
|
||
|
|
v = exec_constrained.OPS["git.diff"]["validate"]
|
||
|
|
self.assertEqual(v({}), {"path": None, "stat": False})
|
||
|
|
self.assertEqual(v({"path": "bin/dm.py", "stat": True}),
|
||
|
|
{"path": "bin/dm.py", "stat": True})
|
||
|
|
for bad in ("../x", "/abs/path", "a\x00b", ""):
|
||
|
|
with self.assertRaises(exec_constrained.OpError, msg=bad):
|
||
|
|
v({"path": bad})
|
||
|
|
|
||
|
|
def test_git_log_validate(self):
|
||
|
|
v = exec_constrained.OPS["git.log"]["validate"]
|
||
|
|
self.assertEqual(v({}), {"limit": 10, "path": None})
|
||
|
|
self.assertEqual(v({"limit": 3})["limit"], 3)
|
||
|
|
with self.assertRaises(exec_constrained.OpError):
|
||
|
|
v({"limit": 0})
|
||
|
|
with self.assertRaises(exec_constrained.OpError):
|
||
|
|
v({"limit": 51})
|
||
|
|
with self.assertRaises(exec_constrained.OpError):
|
||
|
|
v({"path": "../x"})
|
||
|
|
|
||
|
|
def test_build_argv_shapes(self):
|
||
|
|
status = exec_constrained.OPS["git.status"]
|
||
|
|
self.assertEqual(status["build"]({})[-1], "git-status")
|
||
|
|
diff = exec_constrained.OPS["git.diff"]
|
||
|
|
self.assertEqual(diff["build"]({"path": None, "stat": False})[-1], "git-diff")
|
||
|
|
argv = diff["build"]({"path": "bin/dm.py", "stat": True})
|
||
|
|
self.assertEqual(argv[-4:], ["git-diff", "--stat", "--path", "bin/dm.py"])
|
||
|
|
log = exec_constrained.OPS["git.log"]
|
||
|
|
argv = log["build"]({"limit": 3, "path": None})
|
||
|
|
self.assertEqual(argv[-3:], ["git-log", "--limit", "3"])
|
||
|
|
self.assertIsInstance(argv, list)
|
||
|
|
|
||
|
|
def test_git_status_built_argv_executes(self):
|
||
|
|
spec = exec_constrained.OPS["git.status"]
|
||
|
|
argv = spec["build"](spec["validate"]({}))
|
||
|
|
argv[0] = sys.executable # hermetic interpreter, same script + args
|
||
|
|
r = subprocess.run(argv, capture_output=True, text=True, timeout=60)
|
||
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
||
|
|
data = json.loads(r.stdout)
|
||
|
|
self.assertTrue(data["ok"])
|
||
|
|
self.assertIn("branch", data)
|
||
|
|
self.assertIsInstance(data["changes"], list)
|
||
|
|
|
||
|
|
|
||
|
|
class ExecTestsRunTests(unittest.TestCase):
|
||
|
|
def test_registered_and_side_effecting(self):
|
||
|
|
self.assertIn("tests.run", exec_constrained.OPS)
|
||
|
|
self.assertTrue(exec_constrained.OPS["tests.run"]["side_effecting"])
|
||
|
|
|
||
|
|
def test_known_identities_only(self):
|
||
|
|
# Executes repo code: excluded from the read-only default subset.
|
||
|
|
self.assertFalse(exec_constrained.permitted("some-unknown-identity", "tests.run"))
|
||
|
|
self.assertTrue(exec_constrained.permitted("operator-646", "tests.run"))
|
||
|
|
|
||
|
|
def test_validate(self):
|
||
|
|
v = exec_constrained.OPS["tests.run"]["validate"]
|
||
|
|
self.assertEqual(v({}), {"test": None, "filter": None})
|
||
|
|
self.assertEqual(v({"test": "tests.test_box_read_https"}),
|
||
|
|
{"test": "tests.test_box_read_https",
|
||
|
|
"filter": None})
|
||
|
|
self.assertEqual(v({"filter": "safepath"})["filter"], "safepath")
|
||
|
|
for bad in ("os", "tests..x", "tests/x", "tests.test-x", ""):
|
||
|
|
with self.assertRaises(exec_constrained.OpError, msg=bad):
|
||
|
|
v({"test": bad})
|
||
|
|
for bad in ("", "x" * 201, "a\nb"):
|
||
|
|
with self.assertRaises(exec_constrained.OpError, msg=repr(bad)):
|
||
|
|
v({"filter": bad})
|
||
|
|
|
||
|
|
def test_build_argv_shape(self):
|
||
|
|
b = exec_constrained.OPS["tests.run"]["build"]
|
||
|
|
self.assertEqual(b({"test": None, "filter": None})[-1], "tests-run")
|
||
|
|
argv = b({"test": "tests.test_box_read_https", "filter": None})
|
||
|
|
self.assertEqual(argv[-2:], ["tests-run", "tests.test_box_read_https"])
|
||
|
|
argv = b({"test": None, "filter": "safepath"})
|
||
|
|
self.assertEqual(argv[-3:], ["tests-run", "--filter", "safepath"])
|
||
|
|
|
||
|
|
|
||
|
|
class ExecCommsOpsTests(unittest.TestCase):
|
||
|
|
def test_registered_and_side_effecting(self):
|
||
|
|
for op in ("notify.send", "dm.ack"):
|
||
|
|
self.assertIn(op, exec_constrained.OPS)
|
||
|
|
self.assertTrue(exec_constrained.OPS[op]["side_effecting"])
|
||
|
|
|
||
|
|
def test_known_identities_only(self):
|
||
|
|
self.assertFalse(exec_constrained.permitted("some-unknown-identity", "notify.send"))
|
||
|
|
self.assertFalse(exec_constrained.permitted("some-unknown-identity", "dm.ack"))
|
||
|
|
self.assertTrue(exec_constrained.permitted("operator-646", "notify.send"))
|
||
|
|
self.assertTrue(exec_constrained.permitted("muse", "dm.ack"))
|
||
|
|
|
||
|
|
def test_notify_send_validate(self):
|
||
|
|
v = exec_constrained.OPS["notify.send"]["validate"]
|
||
|
|
self.assertEqual(v({"agent": "pip", "message": "hi"}),
|
||
|
|
{"agent": "pip", "message": "hi",
|
||
|
|
"sidechat": None, "sender": None})
|
||
|
|
with self.assertRaises(exec_constrained.OpError):
|
||
|
|
v({"agent": "nope", "message": "hi"})
|
||
|
|
with self.assertRaises(exec_constrained.OpError):
|
||
|
|
v({"agent": "pip", "message": "x" * 1001})
|
||
|
|
with self.assertRaises(exec_constrained.OpError):
|
||
|
|
v({"agent": "pip", "message": " "})
|
||
|
|
with self.assertRaises(exec_constrained.OpError):
|
||
|
|
v({"agent": "pip", "message": "hi", "sidechat": "a" * 65})
|
||
|
|
|
||
|
|
def test_dm_ack_validate(self):
|
||
|
|
v = exec_constrained.OPS["dm.ack"]["validate"]
|
||
|
|
good = v({"id": "bdf7beb6", "to": "pip", "sender": "opm"})
|
||
|
|
self.assertEqual(good["id"], "bdf7beb6")
|
||
|
|
self.assertFalse(good["allow_main_chat"])
|
||
|
|
with self.assertRaises(exec_constrained.OpError):
|
||
|
|
v({"id": "xyz!", "to": "pip", "sender": "opm"})
|
||
|
|
with self.assertRaises(exec_constrained.OpError):
|
||
|
|
v({"id": "bdf7beb6", "to": "nope", "sender": "opm"})
|
||
|
|
with self.assertRaises(exec_constrained.OpError):
|
||
|
|
v({"id": "bdf7beb6", "to": "pip"}) # sender required
|
||
|
|
|
||
|
|
def test_build_argv_shapes(self):
|
||
|
|
n = exec_constrained.OPS["notify.send"]
|
||
|
|
argv = n["build"]({"agent": "pip", "message": "hi",
|
||
|
|
"sidechat": None, "sender": None})
|
||
|
|
self.assertEqual(argv[-3:], ["notify", "pip", "hi"])
|
||
|
|
argv = n["build"]({"agent": "pip", "message": "hi",
|
||
|
|
"sidechat": "pip tasks", "sender": "opm"})
|
||
|
|
self.assertIn("--sidechat", argv)
|
||
|
|
self.assertIn("--sender", argv)
|
||
|
|
a = exec_constrained.OPS["dm.ack"]
|
||
|
|
argv = a["build"]({"id": "bdf7beb6", "to": "pip", "sender": "opm",
|
||
|
|
"sidechat": None, "allow_main_chat": False})
|
||
|
|
self.assertEqual(argv[-6:],
|
||
|
|
["ack", "bdf7beb6", "--to", "pip", "--sender", "opm"])
|
||
|
|
|
||
|
|
|
||
|
|
class BoxCtlGitTests(unittest.TestCase):
|
||
|
|
def test_git_status_live(self):
|
||
|
|
r = _box_ctl("git-status")
|
||
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
||
|
|
data = json.loads(r.stdout)
|
||
|
|
self.assertTrue(data["ok"])
|
||
|
|
self.assertTrue(data["branch"])
|
||
|
|
self.assertIsInstance(data["changes"], list)
|
||
|
|
|
||
|
|
def test_git_log_live(self):
|
||
|
|
r = _box_ctl("git-log", "--limit", "2")
|
||
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
||
|
|
data = json.loads(r.stdout)
|
||
|
|
self.assertTrue(data["ok"])
|
||
|
|
self.assertEqual(len(data["commits"]), 2)
|
||
|
|
self.assertIn("sha", data["commits"][0])
|
||
|
|
self.assertIn("subject", data["commits"][0])
|
||
|
|
|
||
|
|
def test_git_diff_stat_live(self):
|
||
|
|
r = _box_ctl("git-diff", "--stat")
|
||
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
||
|
|
data = json.loads(r.stdout)
|
||
|
|
self.assertTrue(data["ok"])
|
||
|
|
self.assertIn("diff", data)
|
||
|
|
|
||
|
|
def test_rejects_bad_path_and_limit(self):
|
||
|
|
for bad in ("../x", "/abs/path"):
|
||
|
|
r = _box_ctl("git-diff", "--path", bad)
|
||
|
|
self.assertNotEqual(r.returncode, 0)
|
||
|
|
self.assertEqual(json.loads(r.stdout)["code"], "BAD_NAME")
|
||
|
|
r = _box_ctl("git-log", "--limit", "0")
|
||
|
|
self.assertNotEqual(r.returncode, 0)
|
||
|
|
r = _box_ctl("git-log", "--limit", "51")
|
||
|
|
self.assertNotEqual(r.returncode, 0)
|
||
|
|
|
||
|
|
def test_quality_validate_git_verbs(self):
|
||
|
|
for args in (["git-status"], ["git-diff", "--stat"],
|
||
|
|
["git-diff", "--path", "bin/dm.py"],
|
||
|
|
["git-log", "--limit", "5"]):
|
||
|
|
r = _box_ctl("quality-validate", *args)
|
||
|
|
self.assertTrue(json.loads(r.stdout)["valid"], args)
|
||
|
|
r = _box_ctl("quality-validate", "git-diff", "--path", "../x")
|
||
|
|
self.assertFalse(json.loads(r.stdout)["valid"])
|
||
|
|
|
||
|
|
|
||
|
|
class BoxCtlTestsRunTests(unittest.TestCase):
|
||
|
|
def test_tests_run_single_module_live(self):
|
||
|
|
r = _box_ctl("tests-run", "tests.test_box_read_https")
|
||
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
||
|
|
data = json.loads(r.stdout)
|
||
|
|
self.assertTrue(data["ok"], data.get("output", "")[-2000:])
|
||
|
|
self.assertEqual(data["returncode"], 0)
|
||
|
|
|
||
|
|
def test_tests_run_discovery_importable(self):
|
||
|
|
# Full discover must import every test module. An impossible -k
|
||
|
|
# filter runs zero tests in seconds while still importing all of
|
||
|
|
# them, deterministically guarding the discover argv (a `-t .`
|
||
|
|
# regresses to ImportError here). Never asserts suite success:
|
||
|
|
# outage-sensitive tests may be red independently.
|
||
|
|
r = _box_ctl("tests-run", "--filter", "zzz_no_match_zzz")
|
||
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
||
|
|
data = json.loads(r.stdout)
|
||
|
|
self.assertIn("Ran 0 tests", data.get("output", ""))
|
||
|
|
self.assertNotIn("Traceback", data.get("output", ""))
|
||
|
|
|
||
|
|
def test_tests_run_survives_safepath_invoker(self):
|
||
|
|
# `python -m` drops CWD from sys.path under PYTHONSAFEPATH/-P;
|
||
|
|
# tests-run pins PYTHONPATH so it still resolves the tests package.
|
||
|
|
env = dict(os.environ)
|
||
|
|
env["PYTHONSAFEPATH"] = "1"
|
||
|
|
r = subprocess.run(
|
||
|
|
[sys.executable, str(BOX_CTL), "tests-run",
|
||
|
|
"tests.test_box_read_https"],
|
||
|
|
capture_output=True, text=True, timeout=120, env=env)
|
||
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
||
|
|
data = json.loads(r.stdout)
|
||
|
|
self.assertTrue(data["ok"], data.get("output", "")[-2000:])
|
||
|
|
|
||
|
|
def test_rejects_bad_module(self):
|
||
|
|
r = _box_ctl("tests-run", "os")
|
||
|
|
self.assertNotEqual(r.returncode, 0)
|
||
|
|
self.assertEqual(json.loads(r.stdout)["code"], "BAD_NAME")
|
||
|
|
r = _box_ctl("tests-run", "tests.nonexistent_xyz")
|
||
|
|
self.assertNotEqual(r.returncode, 0)
|
||
|
|
self.assertEqual(json.loads(r.stdout)["code"], "NOT_FOUND")
|
||
|
|
|
||
|
|
def test_quality_validate_tests_run(self):
|
||
|
|
r = _box_ctl("quality-validate", "tests-run")
|
||
|
|
self.assertTrue(json.loads(r.stdout)["valid"], r.stdout)
|
||
|
|
r = _box_ctl("quality-validate", "tests-run", "tests.test_box_read_https")
|
||
|
|
self.assertTrue(json.loads(r.stdout)["valid"], r.stdout)
|
||
|
|
r = _box_ctl("quality-validate", "tests-run", "--filter", "safepath")
|
||
|
|
self.assertTrue(json.loads(r.stdout)["valid"], r.stdout)
|
||
|
|
r = _box_ctl("quality-validate", "tests-run", "os")
|
||
|
|
self.assertFalse(json.loads(r.stdout)["valid"], r.stdout)
|
||
|
|
r = _box_ctl("quality-validate", "tests-run", "--filter")
|
||
|
|
self.assertFalse(json.loads(r.stdout)["valid"], r.stdout)
|
||
|
|
|
||
|
|
|
||
|
|
class BoxCtlAckTests(unittest.TestCase):
|
||
|
|
# Validation-failure paths only: a live ack would send a real DM.
|
||
|
|
|
||
|
|
def test_rejects_bad_id_and_agents(self):
|
||
|
|
r = _box_ctl("ack", "xyz!", "--to", "pip", "--sender", "opm")
|
||
|
|
self.assertNotEqual(r.returncode, 0)
|
||
|
|
self.assertEqual(json.loads(r.stdout)["code"], "BAD_NAME")
|
||
|
|
r = _box_ctl("ack", "bdf7beb6", "--to", "nope", "--sender", "opm")
|
||
|
|
self.assertNotEqual(r.returncode, 0)
|
||
|
|
self.assertEqual(json.loads(r.stdout)["code"], "BAD_NAME")
|
||
|
|
|
||
|
|
def test_requires_sender(self):
|
||
|
|
r = _box_ctl("ack", "bdf7beb6", "--to", "pip")
|
||
|
|
self.assertNotEqual(r.returncode, 0)
|
||
|
|
self.assertEqual(json.loads(r.stdout)["code"], "BAD_ARGS")
|
||
|
|
|
||
|
|
def test_quality_validate_ack(self):
|
||
|
|
r = _box_ctl("quality-validate", "ack", "bdf7beb6",
|
||
|
|
"--to", "pip", "--sender", "opm")
|
||
|
|
self.assertTrue(json.loads(r.stdout)["valid"], r.stdout)
|
||
|
|
r = _box_ctl("quality-validate", "ack", "xyz!",
|
||
|
|
"--to", "pip", "--sender", "opm")
|
||
|
|
self.assertFalse(json.loads(r.stdout)["valid"], r.stdout)
|
||
|
|
|
||
|
|
|
||
|
|
class BoxCtlNotifyValidationTests(unittest.TestCase):
|
||
|
|
# Failure paths only: act_notify validates before sending.
|
||
|
|
|
||
|
|
def test_rejects_unknown_agent(self):
|
||
|
|
r = _box_ctl("notify", "nope", "hi")
|
||
|
|
self.assertNotEqual(r.returncode, 0)
|
||
|
|
self.assertEqual(json.loads(r.stdout)["code"], "BAD_NAME")
|
||
|
|
|
||
|
|
def test_rejects_oversize_message(self):
|
||
|
|
r = _box_ctl("notify", "pip", "x" * 1001)
|
||
|
|
self.assertNotEqual(r.returncode, 0)
|
||
|
|
self.assertEqual(json.loads(r.stdout)["code"], "INVALID_JOB")
|
||
|
|
|
||
|
|
|
||
|
|
class BoxRelayDevTests(unittest.TestCase):
|
||
|
|
def test_relay_help_lists_dev_commands(self):
|
||
|
|
r = subprocess.run(["bash", str(RELAY), "help"],
|
||
|
|
capture_output=True, text=True, timeout=30)
|
||
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
||
|
|
for line in ("box git status", "box git diff", "box git log",
|
||
|
|
"box tests run", "box notify", "box dm ack"):
|
||
|
|
self.assertIn(line, r.stdout)
|
||
|
|
|
||
|
|
def test_relay_maps_dev_commands_to_ops(self):
|
||
|
|
text = RELAY.read_text()
|
||
|
|
for op in ("git.status", "git.diff", "git.log",
|
||
|
|
'"tests.run"', '"notify.send"', '"dm.ack"'):
|
||
|
|
self.assertIn(op, text)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
unittest.main()
|