AFI parity: generate the matrix from conformance probes, not prose
The per-adapter parity table was hand-maintained prose. An adapter that
never wired a capability (FastAPI SSR, Axum WebSocket) got its gap
relabelled "Django-only" or "out of scope — use native equivalents," and
nothing went red. The de-scope was crystallized in five mutually-ratifying
sites: the README §Stack-extensions table, the AFI fixture docstring
("channels/forms/shapes aren't AFI-common"), the core registry's
extension-hook framing, the mizan-fastapi __init__ docstring, and a
"CSRF is Django-only" comment in two adapters' session endpoints.
Replace prose-parity with conformance-generated parity:
- tests/afi/manifest.py declares the AFI-common surface as data — one list
of capabilities, one of adapters. Applicability ("—") is derived from
transport, never typed.
- tests/afi/probes.py independently inspects each backend's source for the
artifact a capability requires (comment-stripped, backend-scoped). Green
means wired; a cell can't be set by editing a word.
- tests/afi/test_capability_parity.py asserts every (capability × applicable
adapter) pair is wired. 35 unwired gaps are now loud red TFDD tests, each
naming an owed binding. No xfail/skip.
- tests/afi/parity_table.py generates the README table from the probes;
`make parity-check` fails CI on any hand-edit, like the codegen byte-parity.
Purge the five de-scope sites. The IR byte-parity gate is unchanged and green.
`make test-afi` is now intentionally red on the 35 gaps — that board is the
owed parity work, itemized; a gap turns green by being wired, never described.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
141
tests/afi/parity_table.py
Normal file
141
tests/afi/parity_table.py
Normal file
@@ -0,0 +1,141 @@
|
||||
"""
|
||||
Generate the README parity table from the conformance probes.
|
||||
|
||||
The table in the README is *output*, not *input*. It is computed by running
|
||||
every probe over every adapter and rendering the result. An agent can no longer
|
||||
type "Django-only" into a cell, because no one types cells — `make parity-table`
|
||||
overwrites the block between the markers, and `--check` fails CI if the
|
||||
committed block has drifted from what the probes actually report (the same
|
||||
forcing function the codegen byte-parity tests already use).
|
||||
|
||||
Glyphs:
|
||||
✅ wired the probe found the artifact
|
||||
◑ partial declared or stubbed, not complete — counts as RED in the suite
|
||||
❌ gap AFI-common, owed, not wired
|
||||
— n/a the capability does not exist over this adapter's transport
|
||||
(derived from `manifest.applies`, never asserted)
|
||||
|
||||
Usage:
|
||||
python parity_table.py --write # regenerate the README block
|
||||
python parity_table.py --check # exit 1 if README block is stale
|
||||
python parity_table.py # print the block to stdout
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from manifest import ADAPTERS, CAPABILITIES, Tier, applies
|
||||
from probes import run_probe
|
||||
|
||||
README = Path(__file__).resolve().parents[2] / "README.md"
|
||||
|
||||
START = "<!-- MIZAN:PARITY:START — generated by tests/afi/parity_table.py; do not edit by hand -->"
|
||||
END = "<!-- MIZAN:PARITY:END -->"
|
||||
|
||||
_GLYPH = {"pass": "✅", "partial": "◑", "fail": "❌"}
|
||||
|
||||
|
||||
def _cell(cap, adapter) -> str:
|
||||
if not applies(cap, adapter):
|
||||
return "—"
|
||||
return _GLYPH[run_probe(cap.id, adapter).state]
|
||||
|
||||
|
||||
def _tier_table(tier: Tier) -> str:
|
||||
caps = [c for c in CAPABILITIES if c.tier is tier]
|
||||
if not caps:
|
||||
return ""
|
||||
header = "| Capability | " + " | ".join(a.title for a in ADAPTERS) + " |"
|
||||
sep = "|---|" + "|".join(":---:" for _ in ADAPTERS) + "|"
|
||||
rows = []
|
||||
for cap in caps:
|
||||
cells = " | ".join(_cell(cap, a) for a in ADAPTERS)
|
||||
rows.append(f"| {cap.title} | {cells} |")
|
||||
return f"### {tier.value}\n\n{header}\n{sep}\n" + "\n".join(rows)
|
||||
|
||||
|
||||
def _notes() -> str:
|
||||
noted = [c for c in CAPABILITIES if c.note]
|
||||
if not noted:
|
||||
return ""
|
||||
lines = ["**Notes**", ""]
|
||||
for c in noted:
|
||||
lines.append(f"- **{c.title}** — {c.note}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_block() -> str:
|
||||
"""The full generated parity block: legend, one table per tier, notes."""
|
||||
legend = (
|
||||
"Legend: ✅ wired · ◑ partial (declared/stubbed) · ❌ gap (AFI-common, owed) · "
|
||||
"— not applicable to this adapter's transport\n\n"
|
||||
"Every capability below is **AFI-common**: each adapter owes a binding, and a "
|
||||
"❌ is a gap on the owed-work board (`tests/afi/`), never a category. "
|
||||
"Backend-specific *bindings* of common capabilities (django-readers for Shapes, "
|
||||
"Django Forms for Forms) and genuinely Django-ecosystem features (allauth) are "
|
||||
"out of this matrix by design — see `tests/afi/manifest.py` for the line."
|
||||
)
|
||||
parts = [legend]
|
||||
for tier in (Tier.PROTOCOL_CORE, Tier.EDGE_CACHE, Tier.EXTENSION):
|
||||
table = _tier_table(tier)
|
||||
if table:
|
||||
parts.append(table)
|
||||
notes = _notes()
|
||||
if notes:
|
||||
parts.append(notes)
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
def _wrap(block: str) -> str:
|
||||
return f"{START}\n{block}\n{END}"
|
||||
|
||||
|
||||
def _splice(readme_text: str, block: str) -> str:
|
||||
if START not in readme_text or END not in readme_text:
|
||||
raise SystemExit(
|
||||
f"README markers not found. Add a block bounded by:\n {START}\n {END}\n"
|
||||
f"to {README} where the parity table should render."
|
||||
)
|
||||
pre = readme_text.split(START)[0]
|
||||
post = readme_text.split(END)[1]
|
||||
return pre + _wrap(block) + post
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
block = generate_block()
|
||||
mode = argv[1] if len(argv) > 1 else "--print"
|
||||
|
||||
if mode == "--print":
|
||||
print(block)
|
||||
return 0
|
||||
|
||||
text = README.read_text(encoding="utf-8")
|
||||
spliced = _splice(text, block)
|
||||
|
||||
if mode == "--write":
|
||||
if spliced != text:
|
||||
README.write_text(spliced, encoding="utf-8")
|
||||
print(f"Wrote parity table to {README}")
|
||||
else:
|
||||
print("Parity table already current.")
|
||||
return 0
|
||||
|
||||
if mode == "--check":
|
||||
if spliced != text:
|
||||
print(
|
||||
"README parity table is STALE. The committed table does not match what "
|
||||
"the conformance probes report.\nRun: make parity-table",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
print("README parity table is current.")
|
||||
return 0
|
||||
|
||||
print(f"Unknown mode: {mode}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
Reference in New Issue
Block a user