Fixes/upgrades

This commit is contained in:
2026-07-30 21:12:29 -04:00
parent e00b3a177e
commit 9494549861
13 changed files with 823 additions and 60 deletions

View File

@@ -0,0 +1,35 @@
"""Pydantic to Rust type conversion.
Reads Pydantic class declarations and emits matching serde-derived Rust
structs, so one declaration backs both tiers.
What it covers: scalars, `Optional`, `list`/`set`/`tuple`, `dict`, string
`Literal`s, unions, NewTypes, references to other models and enums, and
Pydantic defaults. What it does not: methods, validators, generics, memory
layout, and enum bodies — an enum lowers to a reference by name, leaving its
declaration to the caller.
"""
from pydantic_to_rust.emit import emit_rust_struct
from pydantic_to_rust.idents import to_rust_ident, to_rust_variant_ident
from pydantic_to_rust.ir import (
DefaultValue,
FieldDecl,
ModelDecl,
Primitive,
TypeShape,
)
from pydantic_to_rust.walker import lower_annotation, walk_pydantic_model
__all__ = [
"DefaultValue",
"FieldDecl",
"ModelDecl",
"Primitive",
"TypeShape",
"emit_rust_struct",
"lower_annotation",
"to_rust_ident",
"to_rust_variant_ident",
"walk_pydantic_model",
]

View File

@@ -0,0 +1,189 @@
"""`ModelDecl` -> Rust source text.
This module classifies shapes and defaults; `templates/` spells the Rust.
Emitted structs are for serialization — plain serde derives, no `repr(C)`,
no padding, no alignment control.
"""
from __future__ import annotations
from pathlib import Path
from jinja2 import Environment, FileSystemLoader, StrictUndefined
from pydantic_to_rust.idents import to_rust_ident
from pydantic_to_rust.ir import DefaultValue, ModelDecl, TypeShape
_DEFAULT_DERIVES = (
"Debug",
"Clone",
"::serde::Serialize",
"::serde::Deserialize",
)
_TEMPLATES = Path(__file__).parent / "templates"
def _rust_string(value: str) -> str:
escaped = value.replace("\\", "\\\\").replace('"', '\\"')
return f'"{escaped}"'
def _rust_f64(value: float) -> str:
"""Always carry a decimal point, so `0.65` cannot read as an integer."""
if value == int(value):
return f"{int(value)}.0"
return repr(float(value))
def _environment() -> Environment:
env = Environment(
loader=FileSystemLoader(_TEMPLATES),
undefined=StrictUndefined,
keep_trailing_newline=True,
)
env.filters["rust_string"] = _rust_string
env.filters["rust_f64"] = _rust_f64
return env
_ENV = _environment()
def _classify_shape(shape: TypeShape) -> dict:
"""One node per reachable `TypeShape` variant, for the type template."""
if shape.primitive is not None:
return {"kind": "primitive", "primitive": shape.primitive.value}
if shape.ref is not None:
return {"kind": "ref", "name": shape.ref}
if shape.array_element is not None and shape.array_length is not None:
return {
"kind": "array",
"element": _classify_shape(shape.array_element),
"length": shape.array_length,
}
if shape.list_inner is not None:
return {"kind": "vec", "inner": _classify_shape(shape.list_inner)}
if shape.optional_inner is not None:
return {"kind": "option", "inner": _classify_shape(shape.optional_inner)}
if shape.enum_variants is not None:
# An inline `Literal[...]` has no name to emit.
return {"kind": "primitive", "primitive": "string"}
if shape.union_branches is not None:
return {"kind": "opaque"}
if shape.map_value is not None:
key = (
_classify_shape(shape.map_key)
if shape.map_key is not None
else {"kind": "primitive", "primitive": "string"}
)
return {"kind": "map", "key": key, "value": _classify_shape(shape.map_value)}
raise ValueError(f"no variant set on TypeShape: {shape}")
def _classify_value(value) -> dict:
"""A JSON-shaped value, for the value template."""
if value is None:
return {"kind": "unit"}
if isinstance(value, bool):
return {"kind": "bool", "value": value}
if isinstance(value, int):
return {"kind": "int", "value": value}
if isinstance(value, float):
return {"kind": "float", "value": value}
if isinstance(value, str):
return {"kind": "string", "value": value}
if isinstance(value, (list, tuple)):
if not value:
return {"kind": "empty_vec"}
return {"kind": "vec", "elements": [_classify_value(v) for v in value]}
if isinstance(value, dict):
if not value:
return {"kind": "empty_map"}
return {
"kind": "map",
"pairs": [
{"key": k, "value": _classify_value(v)} for k, v in value.items()
],
}
return {"kind": "unit"}
def _classify_default(shape: TypeShape, default: DefaultValue) -> dict:
"""One node per default kind, for the default template."""
if default.kind == "null":
if shape.optional_inner is not None:
return {"kind": "none"}
return {"kind": "unit"}
if default.kind == "boolean":
return {"kind": "bool", "value": default.literal}
if default.kind == "integer":
return {"kind": "int", "value": int(default.literal)}
if default.kind == "number":
return {"kind": "float", "value": default.literal}
if default.kind == "string":
return {"kind": "string", "value": default.literal}
if default.kind == "empty_seq":
return {"kind": "empty_vec"}
if default.kind == "empty_map":
return {"kind": "empty_map"}
if default.kind == "tuple":
return {
"kind": "array",
"elements": [_classify_value(v) for v in default.literal],
}
if default.kind == "enum_variant":
return {
"kind": "enum_variant",
"type": default.literal["type"],
"variant": default.literal["variant"],
}
if default.kind == "compound_model":
# Defer to the referenced type's own Default rather than replaying
# the factory's field values.
return {"kind": "delegate", "type": default.literal["type"]}
return {"kind": "unit"}
def emit_rust_struct(
model: ModelDecl,
*,
derives: tuple[str, ...] = _DEFAULT_DERIVES,
extra_attrs: tuple[str, ...] = (),
) -> str:
"""Render one model as a struct, its default functions, and an `impl
Default`. Returns a single block; the caller joins blocks into a file."""
fields = []
default_fns = []
for decl in model.fields:
shape = _classify_shape(decl.shape)
default_fn = None
if decl.default is not None:
default_fn = f"__default_{model.name}_{decl.name}"
default_fns.append(
{
"name": default_fn,
"type": shape,
"body": _classify_default(decl.shape, decl.default),
}
)
fields.append(
{
"ident": to_rust_ident(decl.name),
"type": shape,
"default_fn": default_fn,
}
)
doc_lines = []
if model.docstring:
doc_lines = [line.rstrip() for line in model.docstring.strip().splitlines()]
return _ENV.get_template("struct.rs.j2").render(
name=model.name,
doc_lines=doc_lines,
derives=list(derives),
extra_attrs=list(extra_attrs),
fields=fields,
default_fns=default_fns,
)

View File

@@ -0,0 +1,36 @@
"""Rust identifier hygiene."""
from __future__ import annotations
# Strict and reserved-for-future-use keywords from the Rust reference's
# keyword table, 2021 edition.
_RUST_KEYWORDS = frozenset({
"as", "break", "const", "continue", "crate", "else", "enum", "extern",
"false", "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod",
"move", "mut", "pub", "ref", "return", "self", "Self", "static", "struct",
"super", "trait", "true", "type", "unsafe", "use", "where", "while",
"async", "await", "dyn",
"abstract", "become", "box", "do", "final", "macro", "override", "priv",
"typeof", "unsized", "virtual", "yield",
"try",
"union", # contextual, but escaping it costs nothing
})
def to_rust_ident(name: str) -> str:
"""`name`, or `r#name` when it collides with a keyword.
Field casing is left alone — the caller owns that convention.
"""
if name in _RUST_KEYWORDS:
return f"r#{name}"
return name
def to_rust_variant_ident(python_member_name: str) -> str:
"""`SONNET` -> `Sonnet`, `MY_OPTION` -> `MyOption`.
Enum declarations and enum-valued defaults are emitted from different
places and both call this; two implementations would disagree.
"""
return "".join(part.capitalize() for part in python_member_name.split("_") if part)

View File

@@ -0,0 +1,82 @@
"""The typed shapes a Pydantic model lowers to before Rust is emitted.
`TypeShape` is deliberately isomorphic to Mizan's KDL `TypeShape`, so the
two can be converted without a lookup table.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from enum import Enum
from typing import Optional
class Primitive(str, Enum):
INTEGER = "integer"
NUMBER = "number"
BOOLEAN = "boolean"
STRING = "string"
@dataclass(frozen=True)
class TypeShape:
"""A recursive type shape. Exactly one variant field is set."""
primitive: Optional[Primitive] = None
int_width: Optional[int] = None
"""8 / 16 / 32 / 64, or None when unspecified. Only read alongside
`primitive == INTEGER`, and only by emitters that distinguish widths."""
int_signed: Optional[bool] = None
"""True for i8..i64, False for u8..u64, None when unspecified."""
float_width: Optional[int] = None
"""16 / 32 / 64, or None when unspecified. Only meaningful alongside
`primitive == NUMBER`."""
ref: Optional[str] = None
list_inner: Optional["TypeShape"] = None
optional_inner: Optional["TypeShape"] = None
enum_variants: Optional[tuple[str, ...]] = None
union_branches: Optional[tuple["TypeShape", ...]] = None
map_value: Optional["TypeShape"] = None
"""`dict[K, V]` value shape."""
map_key: Optional["TypeShape"] = None
"""Set only when the key is a NewType. Wire keys are always strings;
this keeps the typed identity on the Rust side."""
array_element: Optional["TypeShape"] = None
"""Fixed-length element: `tuple[T, T, T]` lowers to element T, length 3,
and emits `[T; 3]`."""
array_length: Optional[int] = None
@dataclass(frozen=True)
class DefaultValue:
"""A field default captured during introspection. `kind` discriminates
how the emitter renders `literal`."""
kind: str # integer | number | boolean | string | null | empty_seq |
# empty_map | tuple | enum_variant | compound_model | compound
literal: object
@dataclass(frozen=True)
class FieldDecl:
name: str
shape: TypeShape
required: bool
default: Optional[DefaultValue] = None
@dataclass(frozen=True)
class ModelDecl:
"""One Pydantic class, lowered field by field."""
name: str
fields: tuple[FieldDecl, ...] = field(default_factory=tuple)
docstring: Optional[str] = None

View File

@@ -0,0 +1,29 @@
{#- A default's body: one Rust expression, no trailing semicolon. -#}
{%- macro default_body(node) -%}
{%- if node.kind == "none" -%}::std::option::Option::None
{%- elif node.kind == "unit" -%}::std::default::Default::default()
{%- elif node.kind == "bool" -%}{{ "true" if node.value else "false" }}
{%- elif node.kind == "int" -%}{{ node.value }}
{%- elif node.kind == "float" -%}{{ node.value | rust_f64 }}
{%- elif node.kind == "string" -%}String::from({{ node.value | rust_string }})
{%- elif node.kind == "empty_vec" -%}::std::vec::Vec::new()
{%- elif node.kind == "empty_map" -%}::std::collections::BTreeMap::new()
{%- elif node.kind == "array" -%}[{% for element in node.elements %}{{ rust_value(element) }}{{ ", " if not loop.last }}{% endfor %}]
{%- elif node.kind == "enum_variant" -%}{{ node.type }}::{{ node.variant }}
{%- elif node.kind == "delegate" -%}<{{ node.type }} as ::std::default::Default>::default()
{%- endif -%}
{%- endmacro -%}
{#- A JSON-shaped value whose typed shape is not carried alongside. -#}
{%- macro rust_value(node) -%}
{%- if node.kind == "unit" -%}::std::default::Default::default()
{%- elif node.kind == "bool" -%}{{ "true" if node.value else "false" }}
{%- elif node.kind == "int" -%}{{ node.value }}_i64
{%- elif node.kind == "float" -%}{{ node.value | rust_f64 }}
{%- elif node.kind == "string" -%}String::from({{ node.value | rust_string }})
{%- elif node.kind == "empty_vec" -%}::std::vec::Vec::new()
{%- elif node.kind == "empty_map" -%}::std::collections::BTreeMap::new()
{%- elif node.kind == "vec" -%}vec![{% for element in node.elements %}{{ rust_value(element) }}{{ ", " if not loop.last }}{% endfor %}]
{%- elif node.kind == "map" -%}[{% for pair in node.pairs %}(String::from({{ pair.key | rust_string }}), {{ rust_value(pair.value) }}){{ ", " if not loop.last }}{% endfor %}].into_iter().collect()
{%- endif -%}
{%- endmacro -%}

View File

@@ -0,0 +1,38 @@
{%- import "type.rs.j2" as types -%}
{%- import "default.rs.j2" as defaults -%}
{%- for line in doc_lines -%}
/// {{ line }}
{% endfor -%}
#[derive({{ derives | join(", ") }})]
{% for attr in extra_attrs -%}
{{ attr }}
{% endfor -%}
pub struct {{ name }} {
{%- for field in fields %}
{%- if field.default_fn %}
#[serde(default = "{{ field.default_fn }}")]
{%- endif %}
pub {{ field.ident }}: {{ types.rust_type(field.type) }},
{%- endfor %}
}
{% for fn in default_fns -%}
fn {{ fn.name }}() -> {{ types.rust_type(fn.type) }} { {{ defaults.default_body(fn.body) }} }
{% endfor -%}
{% if default_fns %}
{% endif -%}
{% if fields -%}
impl Default for {{ name }} {
fn default() -> Self {
Self {
{%- for field in fields %}
{%- if field.default_fn %}
{{ field.ident }}: {{ field.default_fn }}(),
{%- else %}
{{ field.ident }}: ::std::default::Default::default(),
{%- endif %}
{%- endfor %}
}
}
}
{% endif -%}

View File

@@ -0,0 +1,16 @@
{#- Rust type spelling. `node` is the classified shape emit.py produces. -#}
{%- macro rust_type(node) -%}
{%- if node.kind == "primitive" -%}
{%- if node.primitive == "integer" -%}i64
{%- elif node.primitive == "number" -%}f64
{%- elif node.primitive == "boolean" -%}bool
{%- else -%}String
{%- endif -%}
{%- elif node.kind == "ref" -%}{{ node.name }}
{%- elif node.kind == "array" -%}[{{ rust_type(node.element) }}; {{ node.length }}]
{%- elif node.kind == "vec" -%}Vec<{{ rust_type(node.inner) }}>
{%- elif node.kind == "option" -%}Option<{{ rust_type(node.inner) }}>
{%- elif node.kind == "opaque" -%}::serde_json::Value
{%- elif node.kind == "map" -%}::std::collections::BTreeMap<{{ rust_type(node.key) }}, {{ rust_type(node.value) }}>
{%- endif -%}
{%- endmacro -%}

View File

@@ -0,0 +1,285 @@
"""Pydantic class -> `ModelDecl`.
Walks `model_fields`, lowers each annotation to a `TypeShape`, and captures
defaults. Annotations this does not recognize lower to `Primitive.STRING`,
matching what Mizan's own IR does with an unknown shape.
"""
from __future__ import annotations
import enum
import re
import sys
import types
from typing import Any, Literal, Union, get_args, get_origin
from pydantic import BaseModel
from pydantic_core import PydanticUndefined
from pydantic_to_rust.idents import to_rust_variant_ident
from pydantic_to_rust.ir import (
DefaultValue,
FieldDecl,
ModelDecl,
Primitive,
TypeShape,
)
# NewTypes named i8..i64 / u8..u64 / f16..f64, recognized only when the
# caller opts in.
_WIDTH_NEWTYPE_RE = re.compile(r"^([iuf])(8|16|32|64)$")
def walk_pydantic_model(
model: type[BaseModel],
*,
newtype_width_detection: bool = False,
) -> ModelDecl:
"""Lower one Pydantic class.
With `newtype_width_detection`, width-named NewTypes carry their width
and signedness. Off, every NewType lowers to a `ref`.
"""
fields: list[FieldDecl] = []
for field_name, field_info in model.model_fields.items():
required = field_info.is_required()
default: DefaultValue | None = None
if not required:
# A factory wins over a literal — it is the canonical Pydantic
# path for compound defaults.
factory = getattr(field_info, "default_factory", None)
if factory is not None:
default = _capture_factory_default(_call_factory(factory, field_name))
else:
default = _capture_default(field_info.default)
fields.append(
FieldDecl(
name=field_name,
shape=lower_annotation(
field_info.annotation,
newtype_width_detection=newtype_width_detection,
),
required=required,
default=default,
)
)
return ModelDecl(
name=model.__name__,
fields=tuple(fields),
docstring=(model.__doc__ or None),
)
def _call_factory(factory, field_name: str):
"""A factory that raises leaves the field with no captured default."""
try:
return factory()
except Exception as exc:
print(
f"pydantic_to_rust: default_factory for {field_name!r} raised "
f"{type(exc).__name__}: {exc}; emitting no default",
file=sys.stderr,
)
return None
def lower_annotation(
annotation: Any,
*,
newtype_width_detection: bool = False,
) -> TypeShape:
"""Lower a bare annotation, for callers that do not have a whole model."""
return _lower_annotation(
annotation,
newtype_width_detection=newtype_width_detection,
)
def _capture_factory_default(value: Any) -> DefaultValue | None:
"""Capture a factory's result. Empty collections map onto Rust's own
`Default`; a model instance is serialized so the emitter can name it."""
if value is None:
return DefaultValue(kind="null", literal=None)
if isinstance(value, (list, tuple, set, frozenset)) and len(value) == 0:
return DefaultValue(kind="empty_seq", literal=None)
if isinstance(value, dict) and len(value) == 0:
return DefaultValue(kind="empty_map", literal=None)
if isinstance(value, BaseModel):
return DefaultValue(
kind="compound_model",
literal={"type": type(value).__name__, "fields": value.model_dump()},
)
if isinstance(value, bool):
return DefaultValue(kind="boolean", literal=value)
if isinstance(value, int):
return DefaultValue(kind="integer", literal=value)
if isinstance(value, float):
return DefaultValue(kind="number", literal=value)
if isinstance(value, str):
return DefaultValue(kind="string", literal=value)
return DefaultValue(kind="compound", literal=value)
def _capture_default(raw: Any) -> DefaultValue | None:
"""Filter Pydantic's sentinels, then record the literal's kind."""
if raw is None:
return DefaultValue(kind="null", literal=None)
if raw is PydanticUndefined or raw is ...:
return None
if isinstance(raw, enum.Enum):
# Store the Rust variant name, not the Python member name, so the
# default body and the enum declaration agree.
return DefaultValue(
kind="enum_variant",
literal={
"type": type(raw).__name__,
"variant": to_rust_variant_ident(raw.name),
},
)
if isinstance(raw, bool):
return DefaultValue(kind="boolean", literal=raw)
if isinstance(raw, int):
return DefaultValue(kind="integer", literal=raw)
if isinstance(raw, float):
return DefaultValue(kind="number", literal=raw)
if isinstance(raw, str):
return DefaultValue(kind="string", literal=raw)
if isinstance(raw, tuple):
return DefaultValue(kind="tuple", literal=tuple(raw))
return DefaultValue(kind="compound", literal=raw)
def _lower_annotation(
annotation: Any,
*,
newtype_width_detection: bool = False,
) -> TypeShape:
# NewType carries `__supertype__` and `__name__`, and keeps its named
# identity on the Rust side.
if hasattr(annotation, "__supertype__") and hasattr(annotation, "__name__"):
if newtype_width_detection:
match = _WIDTH_NEWTYPE_RE.match(annotation.__name__)
if match:
kind, bits = match.group(1), int(match.group(2))
if kind == "f":
return TypeShape(primitive=Primitive.NUMBER, float_width=bits)
return TypeShape(
primitive=Primitive.INTEGER,
int_width=bits,
int_signed=(kind == "i"),
)
return TypeShape(ref=annotation.__name__)
inner, is_optional = _extract_optional(annotation)
if is_optional:
return TypeShape(
optional_inner=_lower_annotation(
inner, newtype_width_detection=newtype_width_detection
)
)
origin = get_origin(annotation)
if origin is Union or isinstance(annotation, types.UnionType):
branches = [a for a in get_args(annotation) if a is not type(None)]
if len(branches) > 1:
return TypeShape(
union_branches=tuple(
_lower_annotation(
b, newtype_width_detection=newtype_width_detection
)
for b in branches
)
)
# `tuple[T1..TN]` with no ellipsis becomes `[T; N]`, but only when every
# element shares a shape; mixed tuples fall through to the list path.
if origin is tuple:
args = get_args(annotation)
if args and args[-1] is not Ellipsis:
first = _lower_annotation(
args[0], newtype_width_detection=newtype_width_detection
)
uniform = all(
_lower_annotation(a, newtype_width_detection=newtype_width_detection)
== first
for a in args[1:]
)
if uniform:
return TypeShape(array_element=first, array_length=len(args))
element = _extract_list_element(annotation)
if element is not None:
return TypeShape(
list_inner=_lower_annotation(
element, newtype_width_detection=newtype_width_detection
)
)
if origin is dict:
args = get_args(annotation)
if len(args) == 2:
key = _lower_annotation(
args[0], newtype_width_detection=newtype_width_detection
)
value = _lower_annotation(
args[1], newtype_width_detection=newtype_width_detection
)
return TypeShape(
map_value=value,
map_key=key if key.ref is not None else None,
)
if origin is Literal:
args = get_args(annotation)
if all(isinstance(a, str) for a in args):
return TypeShape(enum_variants=tuple(args))
if len(args) == 1:
sole = args[0]
if isinstance(sole, bool):
return TypeShape(primitive=Primitive.BOOLEAN)
if isinstance(sole, int):
return TypeShape(primitive=Primitive.INTEGER)
if isinstance(sole, float):
return TypeShape(primitive=Primitive.NUMBER)
# An Enum or model class is referenced by name; the caller emits or
# hand-writes the corresponding Rust declaration.
if isinstance(annotation, type) and issubclass(annotation, enum.Enum):
return TypeShape(ref=annotation.__name__)
if isinstance(annotation, type) and issubclass(annotation, BaseModel):
return TypeShape(ref=annotation.__name__)
if annotation is int:
return TypeShape(primitive=Primitive.INTEGER)
if annotation is float:
return TypeShape(primitive=Primitive.NUMBER)
if annotation is bool:
return TypeShape(primitive=Primitive.BOOLEAN)
if annotation is str:
return TypeShape(primitive=Primitive.STRING)
return TypeShape(primitive=Primitive.STRING)
def _extract_optional(ann: Any) -> tuple[Any, bool]:
"""`(T, True)` for `Optional[T]` or `T | None`, else `(ann, False)`."""
origin = get_origin(ann)
if origin is Union or isinstance(ann, types.UnionType):
args = get_args(ann)
non_none = [a for a in args if a is not type(None)]
if len(non_none) == 1 and type(None) in args:
return non_none[0], True
return ann, False
def _extract_list_element(ann: Any) -> Any | None:
"""`T` for `list[T]`, `tuple[T, ...]`, `set[T]`, `frozenset[T]`."""
origin = get_origin(ann)
if origin in (list, tuple, set, frozenset):
args = get_args(ann)
if origin is tuple and len(args) >= 2 and args[1] is Ellipsis:
return args[0]
if args:
return args[0]
return None