From 9494549861180c65c150bfec3cb43a9273d5ef3c Mon Sep 17 00:00:00 2001 From: Ryth Azhur Date: Thu, 30 Jul 2026 21:12:29 -0400 Subject: [PATCH] Fixes/upgrades --- .../python/pydantic_to_rust/__init__.py | 35 +++ .../python/pydantic_to_rust/emit.py | 189 ++++++++++++ .../python/pydantic_to_rust/idents.py | 36 +++ .../python/pydantic_to_rust/ir.py | 82 +++++ .../pydantic_to_rust/templates/default.rs.j2 | 29 ++ .../pydantic_to_rust/templates/struct.rs.j2 | 38 +++ .../pydantic_to_rust/templates/type.rs.j2 | 16 + .../python/pydantic_to_rust/walker.py | 285 ++++++++++++++++++ ...{run_decoru.py => run_pydantic_to_rust.py} | 32 +- protocol/mizan-codegen/src/config.rs | 8 +- protocol/mizan-codegen/src/fetch.rs | 133 +++++--- .../{decoru => pydantic_to_rust}/enum.rs.j2 | 0 .../{decoru => pydantic_to_rust}/schema.rs.j2 | 0 13 files changed, 823 insertions(+), 60 deletions(-) create mode 100644 protocol/mizan-codegen/python/pydantic_to_rust/__init__.py create mode 100644 protocol/mizan-codegen/python/pydantic_to_rust/emit.py create mode 100644 protocol/mizan-codegen/python/pydantic_to_rust/idents.py create mode 100644 protocol/mizan-codegen/python/pydantic_to_rust/ir.py create mode 100644 protocol/mizan-codegen/python/pydantic_to_rust/templates/default.rs.j2 create mode 100644 protocol/mizan-codegen/python/pydantic_to_rust/templates/struct.rs.j2 create mode 100644 protocol/mizan-codegen/python/pydantic_to_rust/templates/type.rs.j2 create mode 100644 protocol/mizan-codegen/python/pydantic_to_rust/walker.py rename protocol/mizan-codegen/scripts/{run_decoru.py => run_pydantic_to_rust.py} (60%) rename protocol/mizan-codegen/templates/{decoru => pydantic_to_rust}/enum.rs.j2 (100%) rename protocol/mizan-codegen/templates/{decoru => pydantic_to_rust}/schema.rs.j2 (100%) diff --git a/protocol/mizan-codegen/python/pydantic_to_rust/__init__.py b/protocol/mizan-codegen/python/pydantic_to_rust/__init__.py new file mode 100644 index 0000000..71ca33c --- /dev/null +++ b/protocol/mizan-codegen/python/pydantic_to_rust/__init__.py @@ -0,0 +1,35 @@ +"""Pydantic to Rust type conversion. + +Reads Pydantic class declarations and emits matching serde-derived Rust +structs, so one declaration backs both tiers. + +What it covers: scalars, `Optional`, `list`/`set`/`tuple`, `dict`, string +`Literal`s, unions, NewTypes, references to other models and enums, and +Pydantic defaults. What it does not: methods, validators, generics, memory +layout, and enum bodies — an enum lowers to a reference by name, leaving its +declaration to the caller. +""" + +from pydantic_to_rust.emit import emit_rust_struct +from pydantic_to_rust.idents import to_rust_ident, to_rust_variant_ident +from pydantic_to_rust.ir import ( + DefaultValue, + FieldDecl, + ModelDecl, + Primitive, + TypeShape, +) +from pydantic_to_rust.walker import lower_annotation, walk_pydantic_model + +__all__ = [ + "DefaultValue", + "FieldDecl", + "ModelDecl", + "Primitive", + "TypeShape", + "emit_rust_struct", + "lower_annotation", + "to_rust_ident", + "to_rust_variant_ident", + "walk_pydantic_model", +] diff --git a/protocol/mizan-codegen/python/pydantic_to_rust/emit.py b/protocol/mizan-codegen/python/pydantic_to_rust/emit.py new file mode 100644 index 0000000..7059c39 --- /dev/null +++ b/protocol/mizan-codegen/python/pydantic_to_rust/emit.py @@ -0,0 +1,189 @@ +"""`ModelDecl` -> Rust source text. + +This module classifies shapes and defaults; `templates/` spells the Rust. +Emitted structs are for serialization — plain serde derives, no `repr(C)`, +no padding, no alignment control. +""" + +from __future__ import annotations + +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader, StrictUndefined + +from pydantic_to_rust.idents import to_rust_ident +from pydantic_to_rust.ir import DefaultValue, ModelDecl, TypeShape + +_DEFAULT_DERIVES = ( + "Debug", + "Clone", + "::serde::Serialize", + "::serde::Deserialize", +) + +_TEMPLATES = Path(__file__).parent / "templates" + + +def _rust_string(value: str) -> str: + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + + +def _rust_f64(value: float) -> str: + """Always carry a decimal point, so `0.65` cannot read as an integer.""" + if value == int(value): + return f"{int(value)}.0" + return repr(float(value)) + + +def _environment() -> Environment: + env = Environment( + loader=FileSystemLoader(_TEMPLATES), + undefined=StrictUndefined, + keep_trailing_newline=True, + ) + env.filters["rust_string"] = _rust_string + env.filters["rust_f64"] = _rust_f64 + return env + + +_ENV = _environment() + + +def _classify_shape(shape: TypeShape) -> dict: + """One node per reachable `TypeShape` variant, for the type template.""" + if shape.primitive is not None: + return {"kind": "primitive", "primitive": shape.primitive.value} + if shape.ref is not None: + return {"kind": "ref", "name": shape.ref} + if shape.array_element is not None and shape.array_length is not None: + return { + "kind": "array", + "element": _classify_shape(shape.array_element), + "length": shape.array_length, + } + if shape.list_inner is not None: + return {"kind": "vec", "inner": _classify_shape(shape.list_inner)} + if shape.optional_inner is not None: + return {"kind": "option", "inner": _classify_shape(shape.optional_inner)} + if shape.enum_variants is not None: + # An inline `Literal[...]` has no name to emit. + return {"kind": "primitive", "primitive": "string"} + if shape.union_branches is not None: + return {"kind": "opaque"} + if shape.map_value is not None: + key = ( + _classify_shape(shape.map_key) + if shape.map_key is not None + else {"kind": "primitive", "primitive": "string"} + ) + return {"kind": "map", "key": key, "value": _classify_shape(shape.map_value)} + raise ValueError(f"no variant set on TypeShape: {shape}") + + +def _classify_value(value) -> dict: + """A JSON-shaped value, for the value template.""" + if value is None: + return {"kind": "unit"} + if isinstance(value, bool): + return {"kind": "bool", "value": value} + if isinstance(value, int): + return {"kind": "int", "value": value} + if isinstance(value, float): + return {"kind": "float", "value": value} + if isinstance(value, str): + return {"kind": "string", "value": value} + if isinstance(value, (list, tuple)): + if not value: + return {"kind": "empty_vec"} + return {"kind": "vec", "elements": [_classify_value(v) for v in value]} + if isinstance(value, dict): + if not value: + return {"kind": "empty_map"} + return { + "kind": "map", + "pairs": [ + {"key": k, "value": _classify_value(v)} for k, v in value.items() + ], + } + return {"kind": "unit"} + + +def _classify_default(shape: TypeShape, default: DefaultValue) -> dict: + """One node per default kind, for the default template.""" + if default.kind == "null": + if shape.optional_inner is not None: + return {"kind": "none"} + return {"kind": "unit"} + if default.kind == "boolean": + return {"kind": "bool", "value": default.literal} + if default.kind == "integer": + return {"kind": "int", "value": int(default.literal)} + if default.kind == "number": + return {"kind": "float", "value": default.literal} + if default.kind == "string": + return {"kind": "string", "value": default.literal} + if default.kind == "empty_seq": + return {"kind": "empty_vec"} + if default.kind == "empty_map": + return {"kind": "empty_map"} + if default.kind == "tuple": + return { + "kind": "array", + "elements": [_classify_value(v) for v in default.literal], + } + if default.kind == "enum_variant": + return { + "kind": "enum_variant", + "type": default.literal["type"], + "variant": default.literal["variant"], + } + if default.kind == "compound_model": + # Defer to the referenced type's own Default rather than replaying + # the factory's field values. + return {"kind": "delegate", "type": default.literal["type"]} + return {"kind": "unit"} + + +def emit_rust_struct( + model: ModelDecl, + *, + derives: tuple[str, ...] = _DEFAULT_DERIVES, + extra_attrs: tuple[str, ...] = (), +) -> str: + """Render one model as a struct, its default functions, and an `impl + Default`. Returns a single block; the caller joins blocks into a file.""" + fields = [] + default_fns = [] + for decl in model.fields: + shape = _classify_shape(decl.shape) + default_fn = None + if decl.default is not None: + default_fn = f"__default_{model.name}_{decl.name}" + default_fns.append( + { + "name": default_fn, + "type": shape, + "body": _classify_default(decl.shape, decl.default), + } + ) + fields.append( + { + "ident": to_rust_ident(decl.name), + "type": shape, + "default_fn": default_fn, + } + ) + + doc_lines = [] + if model.docstring: + doc_lines = [line.rstrip() for line in model.docstring.strip().splitlines()] + + return _ENV.get_template("struct.rs.j2").render( + name=model.name, + doc_lines=doc_lines, + derives=list(derives), + extra_attrs=list(extra_attrs), + fields=fields, + default_fns=default_fns, + ) diff --git a/protocol/mizan-codegen/python/pydantic_to_rust/idents.py b/protocol/mizan-codegen/python/pydantic_to_rust/idents.py new file mode 100644 index 0000000..23d57b9 --- /dev/null +++ b/protocol/mizan-codegen/python/pydantic_to_rust/idents.py @@ -0,0 +1,36 @@ +"""Rust identifier hygiene.""" + +from __future__ import annotations + +# Strict and reserved-for-future-use keywords from the Rust reference's +# keyword table, 2021 edition. +_RUST_KEYWORDS = frozenset({ + "as", "break", "const", "continue", "crate", "else", "enum", "extern", + "false", "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", + "move", "mut", "pub", "ref", "return", "self", "Self", "static", "struct", + "super", "trait", "true", "type", "unsafe", "use", "where", "while", + "async", "await", "dyn", + "abstract", "become", "box", "do", "final", "macro", "override", "priv", + "typeof", "unsized", "virtual", "yield", + "try", + "union", # contextual, but escaping it costs nothing +}) + + +def to_rust_ident(name: str) -> str: + """`name`, or `r#name` when it collides with a keyword. + + Field casing is left alone — the caller owns that convention. + """ + if name in _RUST_KEYWORDS: + return f"r#{name}" + return name + + +def to_rust_variant_ident(python_member_name: str) -> str: + """`SONNET` -> `Sonnet`, `MY_OPTION` -> `MyOption`. + + Enum declarations and enum-valued defaults are emitted from different + places and both call this; two implementations would disagree. + """ + return "".join(part.capitalize() for part in python_member_name.split("_") if part) diff --git a/protocol/mizan-codegen/python/pydantic_to_rust/ir.py b/protocol/mizan-codegen/python/pydantic_to_rust/ir.py new file mode 100644 index 0000000..1c57bf2 --- /dev/null +++ b/protocol/mizan-codegen/python/pydantic_to_rust/ir.py @@ -0,0 +1,82 @@ +"""The typed shapes a Pydantic model lowers to before Rust is emitted. + +`TypeShape` is deliberately isomorphic to Mizan's KDL `TypeShape`, so the +two can be converted without a lookup table. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Optional + + +class Primitive(str, Enum): + INTEGER = "integer" + NUMBER = "number" + BOOLEAN = "boolean" + STRING = "string" + + +@dataclass(frozen=True) +class TypeShape: + """A recursive type shape. Exactly one variant field is set.""" + + primitive: Optional[Primitive] = None + + int_width: Optional[int] = None + """8 / 16 / 32 / 64, or None when unspecified. Only read alongside + `primitive == INTEGER`, and only by emitters that distinguish widths.""" + + int_signed: Optional[bool] = None + """True for i8..i64, False for u8..u64, None when unspecified.""" + + float_width: Optional[int] = None + """16 / 32 / 64, or None when unspecified. Only meaningful alongside + `primitive == NUMBER`.""" + + ref: Optional[str] = None + list_inner: Optional["TypeShape"] = None + optional_inner: Optional["TypeShape"] = None + enum_variants: Optional[tuple[str, ...]] = None + union_branches: Optional[tuple["TypeShape", ...]] = None + + map_value: Optional["TypeShape"] = None + """`dict[K, V]` value shape.""" + + map_key: Optional["TypeShape"] = None + """Set only when the key is a NewType. Wire keys are always strings; + this keeps the typed identity on the Rust side.""" + + array_element: Optional["TypeShape"] = None + """Fixed-length element: `tuple[T, T, T]` lowers to element T, length 3, + and emits `[T; 3]`.""" + + array_length: Optional[int] = None + + +@dataclass(frozen=True) +class DefaultValue: + """A field default captured during introspection. `kind` discriminates + how the emitter renders `literal`.""" + + kind: str # integer | number | boolean | string | null | empty_seq | + # empty_map | tuple | enum_variant | compound_model | compound + literal: object + + +@dataclass(frozen=True) +class FieldDecl: + name: str + shape: TypeShape + required: bool + default: Optional[DefaultValue] = None + + +@dataclass(frozen=True) +class ModelDecl: + """One Pydantic class, lowered field by field.""" + + name: str + fields: tuple[FieldDecl, ...] = field(default_factory=tuple) + docstring: Optional[str] = None diff --git a/protocol/mizan-codegen/python/pydantic_to_rust/templates/default.rs.j2 b/protocol/mizan-codegen/python/pydantic_to_rust/templates/default.rs.j2 new file mode 100644 index 0000000..9453ab5 --- /dev/null +++ b/protocol/mizan-codegen/python/pydantic_to_rust/templates/default.rs.j2 @@ -0,0 +1,29 @@ +{#- A default's body: one Rust expression, no trailing semicolon. -#} +{%- macro default_body(node) -%} +{%- if node.kind == "none" -%}::std::option::Option::None +{%- elif node.kind == "unit" -%}::std::default::Default::default() +{%- elif node.kind == "bool" -%}{{ "true" if node.value else "false" }} +{%- elif node.kind == "int" -%}{{ node.value }} +{%- elif node.kind == "float" -%}{{ node.value | rust_f64 }} +{%- elif node.kind == "string" -%}String::from({{ node.value | rust_string }}) +{%- elif node.kind == "empty_vec" -%}::std::vec::Vec::new() +{%- elif node.kind == "empty_map" -%}::std::collections::BTreeMap::new() +{%- elif node.kind == "array" -%}[{% for element in node.elements %}{{ rust_value(element) }}{{ ", " if not loop.last }}{% endfor %}] +{%- elif node.kind == "enum_variant" -%}{{ node.type }}::{{ node.variant }} +{%- elif node.kind == "delegate" -%}<{{ node.type }} as ::std::default::Default>::default() +{%- endif -%} +{%- endmacro -%} + +{#- A JSON-shaped value whose typed shape is not carried alongside. -#} +{%- macro rust_value(node) -%} +{%- if node.kind == "unit" -%}::std::default::Default::default() +{%- elif node.kind == "bool" -%}{{ "true" if node.value else "false" }} +{%- elif node.kind == "int" -%}{{ node.value }}_i64 +{%- elif node.kind == "float" -%}{{ node.value | rust_f64 }} +{%- elif node.kind == "string" -%}String::from({{ node.value | rust_string }}) +{%- elif node.kind == "empty_vec" -%}::std::vec::Vec::new() +{%- elif node.kind == "empty_map" -%}::std::collections::BTreeMap::new() +{%- elif node.kind == "vec" -%}vec![{% for element in node.elements %}{{ rust_value(element) }}{{ ", " if not loop.last }}{% endfor %}] +{%- elif node.kind == "map" -%}[{% for pair in node.pairs %}(String::from({{ pair.key | rust_string }}), {{ rust_value(pair.value) }}){{ ", " if not loop.last }}{% endfor %}].into_iter().collect() +{%- endif -%} +{%- endmacro -%} diff --git a/protocol/mizan-codegen/python/pydantic_to_rust/templates/struct.rs.j2 b/protocol/mizan-codegen/python/pydantic_to_rust/templates/struct.rs.j2 new file mode 100644 index 0000000..ef4808a --- /dev/null +++ b/protocol/mizan-codegen/python/pydantic_to_rust/templates/struct.rs.j2 @@ -0,0 +1,38 @@ +{%- import "type.rs.j2" as types -%} +{%- import "default.rs.j2" as defaults -%} +{%- for line in doc_lines -%} +/// {{ line }} +{% endfor -%} +#[derive({{ derives | join(", ") }})] +{% for attr in extra_attrs -%} +{{ attr }} +{% endfor -%} +pub struct {{ name }} { +{%- for field in fields %} +{%- if field.default_fn %} + #[serde(default = "{{ field.default_fn }}")] +{%- endif %} + pub {{ field.ident }}: {{ types.rust_type(field.type) }}, +{%- endfor %} +} + +{% for fn in default_fns -%} +fn {{ fn.name }}() -> {{ types.rust_type(fn.type) }} { {{ defaults.default_body(fn.body) }} } +{% endfor -%} +{% if default_fns %} +{% endif -%} +{% if fields -%} +impl Default for {{ name }} { + fn default() -> Self { + Self { +{%- for field in fields %} +{%- if field.default_fn %} + {{ field.ident }}: {{ field.default_fn }}(), +{%- else %} + {{ field.ident }}: ::std::default::Default::default(), +{%- endif %} +{%- endfor %} + } + } +} +{% endif -%} diff --git a/protocol/mizan-codegen/python/pydantic_to_rust/templates/type.rs.j2 b/protocol/mizan-codegen/python/pydantic_to_rust/templates/type.rs.j2 new file mode 100644 index 0000000..20cb4e9 --- /dev/null +++ b/protocol/mizan-codegen/python/pydantic_to_rust/templates/type.rs.j2 @@ -0,0 +1,16 @@ +{#- Rust type spelling. `node` is the classified shape emit.py produces. -#} +{%- macro rust_type(node) -%} +{%- if node.kind == "primitive" -%} +{%- if node.primitive == "integer" -%}i64 +{%- elif node.primitive == "number" -%}f64 +{%- elif node.primitive == "boolean" -%}bool +{%- else -%}String +{%- endif -%} +{%- elif node.kind == "ref" -%}{{ node.name }} +{%- elif node.kind == "array" -%}[{{ rust_type(node.element) }}; {{ node.length }}] +{%- elif node.kind == "vec" -%}Vec<{{ rust_type(node.inner) }}> +{%- elif node.kind == "option" -%}Option<{{ rust_type(node.inner) }}> +{%- elif node.kind == "opaque" -%}::serde_json::Value +{%- elif node.kind == "map" -%}::std::collections::BTreeMap<{{ rust_type(node.key) }}, {{ rust_type(node.value) }}> +{%- endif -%} +{%- endmacro -%} diff --git a/protocol/mizan-codegen/python/pydantic_to_rust/walker.py b/protocol/mizan-codegen/python/pydantic_to_rust/walker.py new file mode 100644 index 0000000..e25fb3c --- /dev/null +++ b/protocol/mizan-codegen/python/pydantic_to_rust/walker.py @@ -0,0 +1,285 @@ +"""Pydantic class -> `ModelDecl`. + +Walks `model_fields`, lowers each annotation to a `TypeShape`, and captures +defaults. Annotations this does not recognize lower to `Primitive.STRING`, +matching what Mizan's own IR does with an unknown shape. +""" + +from __future__ import annotations + +import enum +import re +import sys +import types +from typing import Any, Literal, Union, get_args, get_origin + +from pydantic import BaseModel +from pydantic_core import PydanticUndefined + +from pydantic_to_rust.idents import to_rust_variant_ident +from pydantic_to_rust.ir import ( + DefaultValue, + FieldDecl, + ModelDecl, + Primitive, + TypeShape, +) + +# NewTypes named i8..i64 / u8..u64 / f16..f64, recognized only when the +# caller opts in. +_WIDTH_NEWTYPE_RE = re.compile(r"^([iuf])(8|16|32|64)$") + + +def walk_pydantic_model( + model: type[BaseModel], + *, + newtype_width_detection: bool = False, +) -> ModelDecl: + """Lower one Pydantic class. + + With `newtype_width_detection`, width-named NewTypes carry their width + and signedness. Off, every NewType lowers to a `ref`. + """ + fields: list[FieldDecl] = [] + for field_name, field_info in model.model_fields.items(): + required = field_info.is_required() + default: DefaultValue | None = None + if not required: + # A factory wins over a literal — it is the canonical Pydantic + # path for compound defaults. + factory = getattr(field_info, "default_factory", None) + if factory is not None: + default = _capture_factory_default(_call_factory(factory, field_name)) + else: + default = _capture_default(field_info.default) + fields.append( + FieldDecl( + name=field_name, + shape=lower_annotation( + field_info.annotation, + newtype_width_detection=newtype_width_detection, + ), + required=required, + default=default, + ) + ) + return ModelDecl( + name=model.__name__, + fields=tuple(fields), + docstring=(model.__doc__ or None), + ) + + +def _call_factory(factory, field_name: str): + """A factory that raises leaves the field with no captured default.""" + try: + return factory() + except Exception as exc: + print( + f"pydantic_to_rust: default_factory for {field_name!r} raised " + f"{type(exc).__name__}: {exc}; emitting no default", + file=sys.stderr, + ) + return None + + +def lower_annotation( + annotation: Any, + *, + newtype_width_detection: bool = False, +) -> TypeShape: + """Lower a bare annotation, for callers that do not have a whole model.""" + return _lower_annotation( + annotation, + newtype_width_detection=newtype_width_detection, + ) + + +def _capture_factory_default(value: Any) -> DefaultValue | None: + """Capture a factory's result. Empty collections map onto Rust's own + `Default`; a model instance is serialized so the emitter can name it.""" + if value is None: + return DefaultValue(kind="null", literal=None) + if isinstance(value, (list, tuple, set, frozenset)) and len(value) == 0: + return DefaultValue(kind="empty_seq", literal=None) + if isinstance(value, dict) and len(value) == 0: + return DefaultValue(kind="empty_map", literal=None) + if isinstance(value, BaseModel): + return DefaultValue( + kind="compound_model", + literal={"type": type(value).__name__, "fields": value.model_dump()}, + ) + if isinstance(value, bool): + return DefaultValue(kind="boolean", literal=value) + if isinstance(value, int): + return DefaultValue(kind="integer", literal=value) + if isinstance(value, float): + return DefaultValue(kind="number", literal=value) + if isinstance(value, str): + return DefaultValue(kind="string", literal=value) + return DefaultValue(kind="compound", literal=value) + + +def _capture_default(raw: Any) -> DefaultValue | None: + """Filter Pydantic's sentinels, then record the literal's kind.""" + if raw is None: + return DefaultValue(kind="null", literal=None) + if raw is PydanticUndefined or raw is ...: + return None + if isinstance(raw, enum.Enum): + # Store the Rust variant name, not the Python member name, so the + # default body and the enum declaration agree. + return DefaultValue( + kind="enum_variant", + literal={ + "type": type(raw).__name__, + "variant": to_rust_variant_ident(raw.name), + }, + ) + if isinstance(raw, bool): + return DefaultValue(kind="boolean", literal=raw) + if isinstance(raw, int): + return DefaultValue(kind="integer", literal=raw) + if isinstance(raw, float): + return DefaultValue(kind="number", literal=raw) + if isinstance(raw, str): + return DefaultValue(kind="string", literal=raw) + if isinstance(raw, tuple): + return DefaultValue(kind="tuple", literal=tuple(raw)) + return DefaultValue(kind="compound", literal=raw) + + +def _lower_annotation( + annotation: Any, + *, + newtype_width_detection: bool = False, +) -> TypeShape: + # NewType carries `__supertype__` and `__name__`, and keeps its named + # identity on the Rust side. + if hasattr(annotation, "__supertype__") and hasattr(annotation, "__name__"): + if newtype_width_detection: + match = _WIDTH_NEWTYPE_RE.match(annotation.__name__) + if match: + kind, bits = match.group(1), int(match.group(2)) + if kind == "f": + return TypeShape(primitive=Primitive.NUMBER, float_width=bits) + return TypeShape( + primitive=Primitive.INTEGER, + int_width=bits, + int_signed=(kind == "i"), + ) + return TypeShape(ref=annotation.__name__) + + inner, is_optional = _extract_optional(annotation) + if is_optional: + return TypeShape( + optional_inner=_lower_annotation( + inner, newtype_width_detection=newtype_width_detection + ) + ) + + origin = get_origin(annotation) + + if origin is Union or isinstance(annotation, types.UnionType): + branches = [a for a in get_args(annotation) if a is not type(None)] + if len(branches) > 1: + return TypeShape( + union_branches=tuple( + _lower_annotation( + b, newtype_width_detection=newtype_width_detection + ) + for b in branches + ) + ) + + # `tuple[T1..TN]` with no ellipsis becomes `[T; N]`, but only when every + # element shares a shape; mixed tuples fall through to the list path. + if origin is tuple: + args = get_args(annotation) + if args and args[-1] is not Ellipsis: + first = _lower_annotation( + args[0], newtype_width_detection=newtype_width_detection + ) + uniform = all( + _lower_annotation(a, newtype_width_detection=newtype_width_detection) + == first + for a in args[1:] + ) + if uniform: + return TypeShape(array_element=first, array_length=len(args)) + + element = _extract_list_element(annotation) + if element is not None: + return TypeShape( + list_inner=_lower_annotation( + element, newtype_width_detection=newtype_width_detection + ) + ) + + if origin is dict: + args = get_args(annotation) + if len(args) == 2: + key = _lower_annotation( + args[0], newtype_width_detection=newtype_width_detection + ) + value = _lower_annotation( + args[1], newtype_width_detection=newtype_width_detection + ) + return TypeShape( + map_value=value, + map_key=key if key.ref is not None else None, + ) + + if origin is Literal: + args = get_args(annotation) + if all(isinstance(a, str) for a in args): + return TypeShape(enum_variants=tuple(args)) + if len(args) == 1: + sole = args[0] + if isinstance(sole, bool): + return TypeShape(primitive=Primitive.BOOLEAN) + if isinstance(sole, int): + return TypeShape(primitive=Primitive.INTEGER) + if isinstance(sole, float): + return TypeShape(primitive=Primitive.NUMBER) + + # An Enum or model class is referenced by name; the caller emits or + # hand-writes the corresponding Rust declaration. + if isinstance(annotation, type) and issubclass(annotation, enum.Enum): + return TypeShape(ref=annotation.__name__) + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return TypeShape(ref=annotation.__name__) + + if annotation is int: + return TypeShape(primitive=Primitive.INTEGER) + if annotation is float: + return TypeShape(primitive=Primitive.NUMBER) + if annotation is bool: + return TypeShape(primitive=Primitive.BOOLEAN) + if annotation is str: + return TypeShape(primitive=Primitive.STRING) + + return TypeShape(primitive=Primitive.STRING) + + +def _extract_optional(ann: Any) -> tuple[Any, bool]: + """`(T, True)` for `Optional[T]` or `T | None`, else `(ann, False)`.""" + origin = get_origin(ann) + if origin is Union or isinstance(ann, types.UnionType): + args = get_args(ann) + non_none = [a for a in args if a is not type(None)] + if len(non_none) == 1 and type(None) in args: + return non_none[0], True + return ann, False + + +def _extract_list_element(ann: Any) -> Any | None: + """`T` for `list[T]`, `tuple[T, ...]`, `set[T]`, `frozenset[T]`.""" + origin = get_origin(ann) + if origin in (list, tuple, set, frozenset): + args = get_args(ann) + if origin is tuple and len(args) >= 2 and args[1] is Ellipsis: + return args[0] + if args: + return args[0] + return None diff --git a/protocol/mizan-codegen/scripts/run_decoru.py b/protocol/mizan-codegen/scripts/run_pydantic_to_rust.py similarity index 60% rename from protocol/mizan-codegen/scripts/run_decoru.py rename to protocol/mizan-codegen/scripts/run_pydantic_to_rust.py index c447b83..3b7f765 100644 --- a/protocol/mizan-codegen/scripts/run_decoru.py +++ b/protocol/mizan-codegen/scripts/run_pydantic_to_rust.py @@ -1,17 +1,8 @@ #!/usr/bin/env python3 -"""Introspect a Pydantic module and print its declarations as JSON. +"""Report a Pydantic module's declarations as JSON on stdout. -argv[1] is a JSON object: - - module: Python module to import (e.g. "claude_manage.schema") - - derives: derive identifiers decoru applies to every emitted struct - -stdout is a JSON object: - - enums: [{"name": , "variants": [, ...]}] - - structs: [, ...] as rendered by decoru - -decoru itself is scoped to BaseModel, so Enum subclasses are reported as -shapes for the caller to render; only their variant identifiers go through -decoru, which keeps them equal to the ones it bakes into field defaults. +Takes one argv JSON object of `module` and `derives`; returns `enums` as +name plus variant identifiers, and `structs` as rendered Rust source. """ import importlib @@ -21,9 +12,9 @@ import sys from enum import Enum from pathlib import Path -from pydantic import BaseModel # type: ignore[import-untyped] +from pydantic import BaseModel -from decoru import ( # type: ignore[import-untyped] +from pydantic_to_rust import ( emit_rust_struct, to_rust_variant_ident, walk_pydantic_model, @@ -35,8 +26,8 @@ def _declared_in(module, obj) -> bool: def discover_models(module) -> list[type[BaseModel]]: - """BaseModel subclasses declared in this module. Imported helpers are - skipped — only own-module declarations qualify.""" + """BaseModel subclasses declared here. An imported one is another + module's to emit.""" return [ obj for _, obj in inspect.getmembers(module, inspect.isclass) @@ -47,8 +38,8 @@ def discover_models(module) -> list[type[BaseModel]]: def discover_enums(module) -> list[type[Enum]]: - """Enum subclasses declared in this module. Filters out the - framework's own Enum class and anything imported from elsewhere.""" + """Enum subclasses declared here. Only variant identifiers are reported; + the caller renders the enum body.""" return [ obj for _, obj in inspect.getmembers(module, inspect.isclass) @@ -58,7 +49,7 @@ def discover_enums(module) -> list[type[Enum]]: def main() -> int: if len(sys.argv) < 2: - sys.stderr.write("run_decoru.py: missing JSON payload argument\n") + sys.stderr.write("run_pydantic_to_rust.py: missing JSON payload argument\n") return 2 payload = json.loads(sys.argv[1]) @@ -72,7 +63,8 @@ def main() -> int: models = discover_models(module) if not enums and not models: sys.stderr.write( - f"run_decoru.py: no Enum or BaseModel subclasses declared in {module_name!r}\n" + f"run_pydantic_to_rust.py: no Enum or BaseModel subclasses " + f"declared in {module_name!r}\n" ) return 3 diff --git a/protocol/mizan-codegen/src/config.rs b/protocol/mizan-codegen/src/config.rs index 17b43b8..a3fa1a1 100644 --- a/protocol/mizan-codegen/src/config.rs +++ b/protocol/mizan-codegen/src/config.rs @@ -177,8 +177,8 @@ pub struct RustSource { #[serde(default)] pub env: BTreeMap, - /// Pre-step run before the Cargo bin: decoru writes Rust types from a - /// Pydantic source module to `pydantic.output`. + /// Pre-step run before the Cargo bin: writes Rust types from a Pydantic + /// source module to `pydantic.output`. #[serde(default)] pub pydantic: Option, } @@ -226,8 +226,8 @@ pub struct PydanticPreStep { pub derives: Vec, /// Prelude inserted at the top of the generated file — the leading - /// comment plus `use` statements for referenced types decoru does not - /// itself produce. + /// comment plus `use` statements for referenced types the converter does + /// not itself produce. #[serde(default)] pub header: String, diff --git a/protocol/mizan-codegen/src/fetch.rs b/protocol/mizan-codegen/src/fetch.rs index a594b60..a18b274 100644 --- a/protocol/mizan-codegen/src/fetch.rs +++ b/protocol/mizan-codegen/src/fetch.rs @@ -5,14 +5,17 @@ //! //! - FastAPI: `python -m mizan_fastapi.ir ` //! - Django: `python manage.py export_mizan_ir` +//! //! - Rust: `cargo run --bin `, a consumer-side binary that //! force-links its `#[derive(Mizan)]` types and `#[mizan::client]` //! functions, then calls `mizan_core::build_ir()`. //! -//! The Rust source supports an optional `[source.rust.pydantic]` -//! pre-step: a Python helper reports the module's Pydantic and Enum -//! declarations, and this module renders them into the Rust file the +//! `[source.rust.pydantic]` adds a pre-step: a helper reports the module's +//! Pydantic and Enum declarations, which render into the Rust file the //! cargo bin then compiles against. +//! +//! That converter lives under `python/`, so the pre-step needs only an +//! interpreter carrying pydantic and jinja2. use std::collections::BTreeMap; use std::fs; @@ -32,7 +35,46 @@ use crate::ir::{parse_ir, MizanIR}; /// Bridge script handed to the configured Python interpreter when /// `[source.rust.pydantic]` is set. -const DECORU_BRIDGE_SCRIPT: &str = include_str!("../scripts/run_decoru.py"); +const BRIDGE_SCRIPT: &str = include_str!("../scripts/run_pydantic_to_rust.py"); + + +/// The `pydantic_to_rust` package, embedded so the pre-step needs only an +/// interpreter carrying pydantic and jinja2. Keys are paths relative to the +/// directory the bridge is materialized into. +const CONVERTER_SOURCES: &[(&str, &str)] = &[ + ( + "pydantic_to_rust/__init__.py", + include_str!("../python/pydantic_to_rust/__init__.py"), + ), + ( + "pydantic_to_rust/ir.py", + include_str!("../python/pydantic_to_rust/ir.py"), + ), + ( + "pydantic_to_rust/idents.py", + include_str!("../python/pydantic_to_rust/idents.py"), + ), + ( + "pydantic_to_rust/walker.py", + include_str!("../python/pydantic_to_rust/walker.py"), + ), + ( + "pydantic_to_rust/emit.py", + include_str!("../python/pydantic_to_rust/emit.py"), + ), + ( + "pydantic_to_rust/templates/struct.rs.j2", + include_str!("../python/pydantic_to_rust/templates/struct.rs.j2"), + ), + ( + "pydantic_to_rust/templates/type.rs.j2", + include_str!("../python/pydantic_to_rust/templates/type.rs.j2"), + ), + ( + "pydantic_to_rust/templates/default.rs.j2", + include_str!("../python/pydantic_to_rust/templates/default.rs.j2"), + ), +]; pub fn fetch_schema(config: &Config, config_dir: &Path) -> Result { @@ -182,69 +224,82 @@ fn run_rust(src: &RustSource, config_dir: &Path) -> Result { #[derive(Deserialize)] -struct DecoruDiscovery { - enums: Vec, +struct Discovery { + enums: Vec, structs: Vec, } #[derive(Deserialize)] -struct DecoruEnumSpec { +struct EnumSpec { name: String, variants: Vec, } #[derive(Template)] -#[template(path = "decoru/enum.rs.j2", escape = "none")] -struct DecoruEnumTemplate<'a> { +#[template(path = "pydantic_to_rust/enum.rs.j2", escape = "none")] +struct EnumTemplate<'a> { name: &'a str, derives: Vec, - variants: Vec>, + variants: Vec>, } -struct DecoruVariant<'a> { +struct EnumVariant<'a> { ident: &'a str, is_default: bool, } #[derive(Template)] -#[template(path = "decoru/schema.rs.j2", escape = "none")] -struct DecoruSchemaTemplate<'a> { +#[template(path = "pydantic_to_rust/schema.rs.j2", escape = "none")] +struct SchemaTemplate<'a> { header: &'a str, blocks: Vec, } -fn render_decoru_enum(spec: &DecoruEnumSpec, derives: &[String]) -> String { - // decoru puts `impl Default` on every struct it emits, so an enum-typed - // field with no Pydantic default still has to satisfy `T::default()`. - // The last member carries `#[default]` to keep the file compiling. +fn render_enum(spec: &EnumSpec, derives: &[String]) -> String { + // Every emitted struct carries `impl Default`, so an enum-typed field + // with no Pydantic default still has to satisfy `T::default()`. The last + // member takes `#[default]` to keep the file compiling. let last = spec.variants.len().saturating_sub(1); let variants = spec.variants.iter() .enumerate() - .map(|(i, ident)| DecoruVariant { ident, is_default: i == last }) + .map(|(i, ident)| EnumVariant { ident, is_default: i == last }) .collect(); let mut derives = derives.to_vec(); derives.push("Default".to_string()); - DecoruEnumTemplate { name: &spec.name, derives, variants } + EnumTemplate { name: &spec.name, derives, variants } .render() - .unwrap_or_else(|e| panic!("template decoru/enum.rs.j2: {e}")) + .unwrap_or_else(|e| panic!("template pydantic_to_rust/enum.rs.j2: {e}")) } -/// Materialize the bridge script so the interpreter can be handed a path -/// instead of a stdin pipe. -fn write_bridge_script() -> Result { - let path = std::env::temp_dir() - .join(format!("mizan-decoru-bridge-{}.py", std::process::id())); - fs::write(&path, DECORU_BRIDGE_SCRIPT) - .with_context(|| format!("writing decoru bridge to {}", path.display()))?; - Ok(path) +/// Write the bridge and the converter beside each other, returning the script +/// path. Python puts a script's own directory on `sys.path`, which is what +/// `from pydantic_to_rust import ...` resolves against. +fn materialize_bridge() -> Result { + let root = std::env::temp_dir() + .join(format!("mizan-pydantic-to-rust-{}", std::process::id())); + + for (relative, source) in CONVERTER_SOURCES { + let path = root.join(relative); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("mkdir {}", parent.display()))?; + } + fs::write(&path, source) + .with_context(|| format!("writing converter to {}", path.display()))?; + } + + let script = root.join("run_pydantic_to_rust.py"); + fs::write(&script, BRIDGE_SCRIPT) + .with_context(|| format!("writing bridge to {}", script.display()))?; + Ok(script) } @@ -263,27 +318,33 @@ fn run_pydantic_prestep(src: &PydanticPreStep, config_dir: &Path) -> Result<()> }) .to_string(); - let script_path = write_bridge_script()?; + let script_path = materialize_bridge()?; let command = interpreter(&src.command, &src.python); let mut args = command.args().to_vec(); args.push(script_path.to_string_lossy().into_owned()); args.push(payload); - let stdout = run_subprocess(command.program(), &args, &cwd, &src.env, "decoru bridge")?; + let stdout = run_subprocess( + command.program(), + &args, + &cwd, + &src.env, + "pydantic-to-rust bridge", + )?; - let discovery: DecoruDiscovery = serde_json::from_str(&stdout) - .context("decoding the decoru bridge's JSON report")?; + let discovery: Discovery = serde_json::from_str(&stdout) + .context("decoding the bridge's JSON report")?; let mut blocks: Vec = discovery.enums.iter() - .map(|spec| render_decoru_enum(spec, &src.derives)) + .map(|spec| render_enum(spec, &src.derives)) .collect(); let enum_count = blocks.len(); let struct_count = discovery.structs.len(); blocks.extend(discovery.structs); - let rendered = DecoruSchemaTemplate { header: &src.header, blocks } + let rendered = SchemaTemplate { header: &src.header, blocks } .render() - .unwrap_or_else(|e| panic!("template decoru/schema.rs.j2: {e}")); + .unwrap_or_else(|e| panic!("template pydantic_to_rust/schema.rs.j2: {e}")); if let Some(parent) = output_abs.parent() { fs::create_dir_all(parent) @@ -293,7 +354,7 @@ fn run_pydantic_prestep(src: &PydanticPreStep, config_dir: &Path) -> Result<()> .with_context(|| format!("write {}", output_abs.display()))?; eprintln!( - "[mizan] decoru: {enum_count} enum(s) + {struct_count} struct(s) -> {}", + "[mizan] pydantic: {enum_count} enum(s) + {struct_count} struct(s) -> {}", output_abs.display(), ); diff --git a/protocol/mizan-codegen/templates/decoru/enum.rs.j2 b/protocol/mizan-codegen/templates/pydantic_to_rust/enum.rs.j2 similarity index 100% rename from protocol/mizan-codegen/templates/decoru/enum.rs.j2 rename to protocol/mizan-codegen/templates/pydantic_to_rust/enum.rs.j2 diff --git a/protocol/mizan-codegen/templates/decoru/schema.rs.j2 b/protocol/mizan-codegen/templates/pydantic_to_rust/schema.rs.j2 similarity index 100% rename from protocol/mizan-codegen/templates/decoru/schema.rs.j2 rename to protocol/mizan-codegen/templates/pydantic_to_rust/schema.rs.j2