ZZZ Digital Lab

Back to WaitLens

tests/test_post_sdp.py

"""Sequential synthetic pipeline tests; never call real SDP tools or a target."""
import argparse
import contextlib
import io
import json
from pathlib import Path
import shutil
import tempfile
import unittest
from unittest.mock import patch

from scripts import fake_sdp, post_sdp
from scripts.check_evidence import check
from waitlens.cli import main as cli
from waitlens.core import analyze, compare
from waitlens.report import render

ROOT = Path(__file__).resolve().parents[1]


def setUpModule():
    # A fresh source archive has no ignored tmp/ directory yet. Each test module
    # must prepare its own scratch parent instead of depending on discovery order.
    (ROOT / 'tmp').mkdir(exist_ok=True)


class PipelineTests(unittest.TestCase):
    def make_project(self, directory):
        for name in ['docs/tutorial-draft.md', 'target/waitlens_probe.c', 'target/capture.sh']:
            destination = directory / name
            destination.parent.mkdir(parents=True, exist_ok=True)
            shutil.copyfile(ROOT / name, destination)

    def exercise(self, project, factory=None):
        with patch.object(post_sdp, 'ROOT', project), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
            if factory:
                with patch.object(fake_sdp, 'create', factory):
                    code = post_sdp.main(['--dry-run'])
            else:
                code = post_sdp.main(['--dry-run'])
        runs = sorted((project / 'build').glob('dry-run-*'))
        return code, runs

    def test_fake_sdp_full_flow_and_real_gate_refusal(self):
        with tempfile.TemporaryDirectory(dir=ROOT / 'tmp') as temp:
            project = Path(temp)
            self.make_project(project)
            original = (project / 'docs/tutorial-draft.md').read_bytes()
            code, runs = self.exercise(project)
            self.assertEqual(code, 0)
            run = runs[0]
            state = json.loads((run / 'pipeline.json').read_text(encoding='utf-8'))
            self.assertEqual(state['status'], 'synthetic-control-flow-only')
            self.assertEqual(len(state['completed_stages']), 6)
            self.assertFalse(state['runtime_verified'])
            self.assertFalse(state['submission_ready'])
            self.assertFalse(state['target_capture_checked'])
            preview = (run / 'tutorial-preview.md').read_text(encoding='utf-8')
            self.assertNotIn('[[REAL OUTPUT REQUIRED:', preview)
            self.assertIn('SYNTHETIC PREVIEW', preview)
            self.assertEqual((project / 'docs/tutorial-draft.md').read_bytes(), original)
            self.assertFalse((run / 'tutorial-filled.md').exists())
            report = json.loads((run / 'comparison.json').read_text(encoding='utf-8'))
            self.assertEqual(report['before']['provenance'], 'synthetic')
            self.assertEqual(report['changes'][0]['after']['state'], 'REPLY')
            with self.assertRaisesRegex(ValueError, 'synthetic'):
                check(run / 'build', run / 'capture')
            self.assertTrue((run / 'fake-sdp/host/usr/bin/qcc.py').exists())
            _, reruns = self.exercise(project)
            self.assertEqual(len(reruns), 2)
            self.assertTrue((run / 'manifest.json').exists())

    def test_failures_stop_before_tutorial_and_retain_status(self):
        real_create = fake_sdp.create
        cases = {
            'compiler': ('qcc', 'import sys\nsys.exit(7)\n', 'qcc build failed'),
            'transport': ('gdb', 'import sys\nsys.exit(8)\n', 'GDB deploy/capture failed'),
            'missing-marker': ('append', "\n(capture / 'capture-complete.txt').unlink()\n", 'capture-complete'),
            'stale-marker': ('append', "\n(target / 'capture-complete.txt').write_text('old', encoding='utf-8')\nshutil.copyfile(target / 'capture-complete.txt', capture / 'capture-complete.txt')\n", 'stale'),
            'wrong-phase': ('append', "\n(capture / 'send.txt').write_bytes((capture / 'reply.txt').read_bytes())\n", 'expected SEND'),
            'returned-binary': ('append', "\n(capture / 'waitlens_probe').write_bytes(b'changed')\n", 'binary differs'),
        }
        for name, (tool, source, expected) in cases.items():
            with self.subTest(name=name), tempfile.TemporaryDirectory(dir=ROOT / 'tmp') as temp:
                project = Path(temp)
                self.make_project(project)
                def factory(path):
                    qcc, gdb = real_create(path)
                    destination = qcc if tool == 'qcc' else gdb
                    prefix = destination.read_text(encoding='utf-8') if tool == 'append' else ''
                    destination.write_text(prefix + source, encoding='utf-8')
                    return qcc, gdb
                code, runs = self.exercise(project, factory)
                self.assertEqual(code, 2)
                state = json.loads((runs[0] / 'pipeline.json').read_text(encoding='utf-8'))
                self.assertEqual(state['status'], 'failed')
                self.assertIn(expected, state['error'])
                self.assertNotIn('tutorial-fill', state['completed_stages'])
                self.assertFalse((runs[0] / 'tutorial-filled.md').exists())
                self.assertFalse((runs[0] / 'tutorial-preview.md').exists())

    def test_placeholder_contract_and_markdown_capture_inert(self):
        template = (ROOT / 'docs/tutorial-draft.md').read_text(encoding='utf-8')
        values = {slot: '```\n# capture heading\n```' for slot in post_sdp.SLOTS}
        filled = post_sdp.fill_tutorial(template, values)
        self.assertIn('````text\n```', filled)
        self.assertNotIn('[[REAL OUTPUT REQUIRED:', filled)
        with self.assertRaisesRegex(ValueError, 'contract'):
            post_sdp.fill_tutorial(template + '\n[[REAL OUTPUT REQUIRED: UNKNOWN]]', values)

    def test_endpoint_and_option_injection_refused(self):
        self.assertEqual(post_sdp.endpoint('192.168.56.101:8000'), '192.168.56.101:8000')
        for bad in ['8.8.8.8:8000', '127.0.0.1:0', '192.168.1.1:65536', 'host;cmd:8000', '0.0.0.0:8000']:
            with self.subTest(value=bad), self.assertRaises(argparse.ArgumentTypeError):
                post_sdp.endpoint(bad)
        with contextlib.redirect_stderr(io.StringIO()):
            for args in [[], ['--dry-run', '--target', '127.0.0.1:8000'], ['--dry-run', '--target-shell', '/bin/sh;echo']]:
                with self.subTest(args=args), self.assertRaises(SystemExit):
                    post_sdp.main(args)


class HostErgonomicsTests(unittest.TestCase):
    def test_out_prefix_and_json_stdout_are_composable(self):
        with tempfile.TemporaryDirectory(dir=ROOT / 'tmp') as temp:
            prefix = Path(temp) / 'report'
            out, err = io.StringIO(), io.StringIO()
            with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
                self.assertEqual(cli(['compare', str(ROOT / 'fixtures/send.txt'), str(ROOT / 'fixtures/reply.txt'), '--out', str(prefix)]), 0)
            self.assertEqual(out.getvalue(), '')
            self.assertIn('1 observed changes', err.getvalue())
            self.assertTrue(prefix.with_suffix('.html').exists())
            with contextlib.redirect_stderr(io.StringIO()):
                self.assertEqual(cli(['analyze', str(ROOT / 'fixtures/send.txt'), '--out', str(prefix), '--json', str(prefix)]), 2)
            with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()):
                self.assertEqual(cli(['analyze', str(ROOT / 'fixtures/send.txt')]), 0)
            self.assertEqual(json.loads(out.getvalue())['schema_version'], 1)

    def test_named_diagnostic_and_unchanged_report(self):
        with tempfile.TemporaryDirectory(dir=ROOT / 'tmp') as temp:
            invalid = Path(temp) / 'broken.txt'
            invalid.write_text('not a capture', encoding='utf-8')
            err = io.StringIO()
            with contextlib.redirect_stderr(err):
                self.assertEqual(cli(['analyze', str(invalid)]), 2)
            self.assertIn('broken.txt: line 1', err.getvalue())
            self.assertIn('--help', err.getvalue())
        before = analyze((ROOT / 'fixtures/send.txt').read_text(encoding='utf-8'))
        html = render(compare(before, before))
        self.assertIn('No changes observed', html)
        self.assertIn('Blocking between samples is unknown', html)


if __name__ == '__main__':
    unittest.main()