glm-5.3-flash
844c199fb8
- F1: materialize_plan_array (validate_bytes' packed path) now carries the zero-progress array guard the reader and legacy walker already had; validate_bytes no longer accepts an empty buffer against a stride-0 empty-struct-element array that SequentialReader rejects. Cross-consumer agreement test added (review #007 probe transcript). - F2: MAX_LENGTH = 2^26 cap on the maxLength annotation — the N2 dual-layer pattern (clean Schema parse error naming value+maximum, meta-schema "maximum": 67108864 so the published contract matches). Also closes the silent usize-overflow drop in parse_max_length. - docs/reviews/007-coverage-audit.md records the full audit: per-file numbers, all classifications, and the N3a dead-surface list deferred to the pre-release review. Verification: 477 lib + 78 integration tests green, clippy -D warnings clean, wasm32-unknown-unknown build green.
479 lines
17 KiB
Rust
479 lines
17 KiB
Rust
//! Schema layer: the `AlkTypeKind` enum, `Endian`/`VariableEncoding`
|
||
//! annotations, and shared constants.
|
||
//!
|
||
//! Under the BAST pivot the v0.1.0 custom-keyword accessors
|
||
//! (`get_alktype_kind*`, `normalize_refs`, `inline_union_variant_refs`,
|
||
//! `parse_*`, `resolve_ref*`, `DiscriminatorKind`) were removed — the
|
||
//! BAST parser ([`crate::bast`]) is the typed surface every engine
|
||
//! module walks. This module retains only the foundational types that
|
||
//! the layout/data-access/materialize/tunion/builder layers depend on:
|
||
//! the kind enum (with its BAST string mapping), the two annotation
|
||
//! enums, and the shared byte-layout constants.
|
||
|
||
use crate::error::AlkTypeError;
|
||
use std::fmt;
|
||
|
||
pub(crate) const U32_SIZE: usize = 4;
|
||
pub(crate) const DISCRIMINATOR_PATH: &str = "__discriminator";
|
||
|
||
/// Maximum declared array element count. Schemas are untrusted input
|
||
/// (AGENTS.md §3): every layout walker expands `count` into per-element
|
||
/// work (offset-map/layout entries, per-element reads), so an unbounded
|
||
/// count is a memory/CPU denial-of-service. The cap applies at parse
|
||
/// time, before any walker sees the array; 2^16 elements bounds the
|
||
/// per-array compile-time entry cost at ~10 MB.
|
||
pub(crate) const MAX_ARRAY_ELEMENTS: usize = 1 << 16;
|
||
|
||
/// Maximum computed byte size of a single array (`count × element
|
||
/// stride`). Bounds the arithmetic product independently of
|
||
/// [`MAX_ARRAY_ELEMENTS`] so multi-megabyte strides cannot combine with
|
||
/// a legal count into an unbounded layout request.
|
||
pub(crate) const MAX_ARRAY_BYTES: usize = 1 << 26;
|
||
|
||
/// Maximum declared `align` annotation (struct- or field-level).
|
||
/// Schemas are untrusted input (AGENTS.md §3): `align_up` rounds the
|
||
/// running offset by the full annotation, so an unbounded align lets a
|
||
/// one-field schema declare an exabyte-scale layout (`total_size = 2^63`
|
||
/// from `align: 2^62` — review #006 N2 probe). Honest layouts never
|
||
/// need more than page granularity (4096).
|
||
pub(crate) const MAX_ALIGN: usize = 4096;
|
||
|
||
/// Maximum declared `maxLength` annotation. Schemas are untrusted input
|
||
/// (AGENTS.md §3): a string/bytes reservation contributes its full
|
||
/// `maxLength` to `total_size`, so an unbounded value lets a one-field
|
||
/// schema declare a terabyte-scale layout (`total_size = 2^40` from
|
||
/// `maxLength: 1e12` — review #007 F2 probe). The cap matches
|
||
/// [`MAX_ARRAY_BYTES`]'s rationale (a single fixed reservation no
|
||
/// larger than the largest legal array).
|
||
pub(crate) const MAX_LENGTH: usize = 1 << 26;
|
||
|
||
/// The 18 BAST kinds recognized by the engine.
|
||
///
|
||
/// Each variant corresponds to a lowercase BAST kind string
|
||
/// ([`AlkTypeKind::to_bast_str`], e.g. `"uint32"` ↔ `Uint32`). The enum
|
||
/// provides compile-time exhaustiveness checking and integer
|
||
/// discriminant dispatch (jump table) instead of string comparison.
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||
pub enum AlkTypeKind {
|
||
Int8,
|
||
Int16,
|
||
Int32,
|
||
Int64,
|
||
Uint8,
|
||
Uint16,
|
||
Uint32,
|
||
Uint64,
|
||
Float32,
|
||
Float64,
|
||
Boolean,
|
||
Enum,
|
||
String,
|
||
Bytes,
|
||
Struct,
|
||
Union,
|
||
Array,
|
||
Record,
|
||
}
|
||
|
||
impl AlkTypeKind {
|
||
/// Fixed byte size, or `None` for variable-size / composite kinds.
|
||
pub fn type_size(self) -> Option<usize> {
|
||
match self {
|
||
AlkTypeKind::Float32 | AlkTypeKind::Int32 | AlkTypeKind::Uint32 | AlkTypeKind::Enum => {
|
||
Some(4)
|
||
}
|
||
AlkTypeKind::Float64 | AlkTypeKind::Int64 | AlkTypeKind::Uint64 => Some(8),
|
||
AlkTypeKind::Int8 | AlkTypeKind::Uint8 | AlkTypeKind::Boolean => Some(1),
|
||
AlkTypeKind::Int16 | AlkTypeKind::Uint16 => Some(2),
|
||
AlkTypeKind::String
|
||
| AlkTypeKind::Bytes
|
||
| AlkTypeKind::Struct
|
||
| AlkTypeKind::Union
|
||
| AlkTypeKind::Array
|
||
| AlkTypeKind::Record => None,
|
||
}
|
||
}
|
||
|
||
/// Natural alignment: 1 for u8/i8/bool, 2 for u16/i16, 4 for u32/i32/f32/enum,
|
||
/// 8 for u64/i64/f64, 4 for variable-length (u32 length prefix), 1 for
|
||
/// composites.
|
||
pub fn natural_alignment(self) -> usize {
|
||
match self {
|
||
AlkTypeKind::Int8 | AlkTypeKind::Uint8 | AlkTypeKind::Boolean => 1,
|
||
AlkTypeKind::Int16 | AlkTypeKind::Uint16 => 2,
|
||
AlkTypeKind::Int32
|
||
| AlkTypeKind::Uint32
|
||
| AlkTypeKind::Float32
|
||
| AlkTypeKind::Enum => 4,
|
||
AlkTypeKind::Float64 | AlkTypeKind::Int64 | AlkTypeKind::Uint64 => 8,
|
||
AlkTypeKind::String
|
||
| AlkTypeKind::Bytes
|
||
| AlkTypeKind::Record => 4,
|
||
AlkTypeKind::Struct | AlkTypeKind::Union | AlkTypeKind::Array => 1,
|
||
}
|
||
}
|
||
|
||
/// Returns `true` for fixed-size primitive kinds.
|
||
pub fn is_fixed_size(self) -> bool {
|
||
matches!(
|
||
self,
|
||
AlkTypeKind::Float32
|
||
| AlkTypeKind::Float64
|
||
| AlkTypeKind::Int8
|
||
| AlkTypeKind::Int16
|
||
| AlkTypeKind::Int32
|
||
| AlkTypeKind::Int64
|
||
| AlkTypeKind::Uint8
|
||
| AlkTypeKind::Uint16
|
||
| AlkTypeKind::Uint32
|
||
| AlkTypeKind::Uint64
|
||
| AlkTypeKind::Boolean
|
||
| AlkTypeKind::Enum
|
||
)
|
||
}
|
||
|
||
/// Returns `true` for kinds whose read/write functions need an `Endian` parameter.
|
||
pub fn needs_endian(self) -> bool {
|
||
matches!(
|
||
self,
|
||
AlkTypeKind::Int16
|
||
| AlkTypeKind::Int32
|
||
| AlkTypeKind::Int64
|
||
| AlkTypeKind::Uint16
|
||
| AlkTypeKind::Uint32
|
||
| AlkTypeKind::Uint64
|
||
| AlkTypeKind::Float32
|
||
| AlkTypeKind::Float64
|
||
| AlkTypeKind::Enum
|
||
| AlkTypeKind::String
|
||
| AlkTypeKind::Bytes
|
||
)
|
||
}
|
||
|
||
/// Returns `true` for composite kinds (Struct, Union, Array, Record).
|
||
pub fn is_composite(self) -> bool {
|
||
matches!(
|
||
self,
|
||
AlkTypeKind::Struct | AlkTypeKind::Union | AlkTypeKind::Array | AlkTypeKind::Record
|
||
)
|
||
}
|
||
|
||
/// Returns `true` for variable-length kinds (String, Bytes, Record).
|
||
pub fn is_variable_length(self) -> bool {
|
||
matches!(
|
||
self,
|
||
AlkTypeKind::String
|
||
| AlkTypeKind::Bytes
|
||
| AlkTypeKind::Record
|
||
)
|
||
}
|
||
|
||
/// The lowercase BAST kind string, e.g. `"uint32"` (D-BAST-002).
|
||
///
|
||
/// This is the string used in BAST documents (`{ "kind": "uint32" }`)
|
||
/// and the value the parser and validator dispatch on.
|
||
pub fn to_bast_str(self) -> &'static str {
|
||
match self {
|
||
AlkTypeKind::Int8 => "int8",
|
||
AlkTypeKind::Int16 => "int16",
|
||
AlkTypeKind::Int32 => "int32",
|
||
AlkTypeKind::Int64 => "int64",
|
||
AlkTypeKind::Uint8 => "uint8",
|
||
AlkTypeKind::Uint16 => "uint16",
|
||
AlkTypeKind::Uint32 => "uint32",
|
||
AlkTypeKind::Uint64 => "uint64",
|
||
AlkTypeKind::Float32 => "float32",
|
||
AlkTypeKind::Float64 => "float64",
|
||
AlkTypeKind::Boolean => "bool",
|
||
AlkTypeKind::String => "string",
|
||
AlkTypeKind::Bytes => "bytes",
|
||
AlkTypeKind::Enum => "enum",
|
||
AlkTypeKind::Struct => "struct",
|
||
AlkTypeKind::Union => "union",
|
||
AlkTypeKind::Array => "array",
|
||
AlkTypeKind::Record => "record",
|
||
}
|
||
}
|
||
|
||
/// Parse a lowercase BAST kind string into an `AlkTypeKind`
|
||
/// (D-BAST-002). Returns `AlkTypeError::Schema` for unknown strings.
|
||
///
|
||
/// This is the inverse of [`to_bast_str`](Self::to_bast_str).
|
||
pub fn from_bast_str(s: &str) -> Result<AlkTypeKind, AlkTypeError> {
|
||
match s {
|
||
"int8" => Ok(AlkTypeKind::Int8),
|
||
"int16" => Ok(AlkTypeKind::Int16),
|
||
"int32" => Ok(AlkTypeKind::Int32),
|
||
"int64" => Ok(AlkTypeKind::Int64),
|
||
"uint8" => Ok(AlkTypeKind::Uint8),
|
||
"uint16" => Ok(AlkTypeKind::Uint16),
|
||
"uint32" => Ok(AlkTypeKind::Uint32),
|
||
"uint64" => Ok(AlkTypeKind::Uint64),
|
||
"float32" => Ok(AlkTypeKind::Float32),
|
||
"float64" => Ok(AlkTypeKind::Float64),
|
||
"bool" => Ok(AlkTypeKind::Boolean),
|
||
"string" => Ok(AlkTypeKind::String),
|
||
"bytes" => Ok(AlkTypeKind::Bytes),
|
||
"enum" => Ok(AlkTypeKind::Enum),
|
||
"struct" => Ok(AlkTypeKind::Struct),
|
||
"union" => Ok(AlkTypeKind::Union),
|
||
"array" => Ok(AlkTypeKind::Array),
|
||
"record" => Ok(AlkTypeKind::Record),
|
||
other => Err(AlkTypeError::Schema(format!(
|
||
"unknown BAST kind: {other}"
|
||
))),
|
||
}
|
||
}
|
||
}
|
||
|
||
impl fmt::Display for AlkTypeKind {
|
||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||
f.write_str(self.to_bast_str())
|
||
}
|
||
}
|
||
|
||
/// Byte endianness for multi-byte integer and float fields.
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||
pub enum Endian {
|
||
Little,
|
||
Big,
|
||
}
|
||
|
||
/// The encoding strategy for a variable-length type.
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||
pub enum VariableEncoding {
|
||
/// `[length: u32][data]` — the default. Length prefix at a known offset,
|
||
/// variable data follows immediately.
|
||
LengthPrefixed,
|
||
/// `{offset: u32, length: u32}` pointing into a separate data region.
|
||
/// The metatensor blob tensor pattern.
|
||
OffsetIndirect,
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn type_size_fixed_kinds() {
|
||
assert_eq!(AlkTypeKind::Float32.type_size(), Some(4));
|
||
assert_eq!(AlkTypeKind::Float64.type_size(), Some(8));
|
||
assert_eq!(AlkTypeKind::Int8.type_size(), Some(1));
|
||
assert_eq!(AlkTypeKind::Int16.type_size(), Some(2));
|
||
assert_eq!(AlkTypeKind::Int32.type_size(), Some(4));
|
||
assert_eq!(AlkTypeKind::Int64.type_size(), Some(8));
|
||
assert_eq!(AlkTypeKind::Uint8.type_size(), Some(1));
|
||
assert_eq!(AlkTypeKind::Uint16.type_size(), Some(2));
|
||
assert_eq!(AlkTypeKind::Uint32.type_size(), Some(4));
|
||
assert_eq!(AlkTypeKind::Uint64.type_size(), Some(8));
|
||
assert_eq!(AlkTypeKind::Boolean.type_size(), Some(1));
|
||
assert_eq!(AlkTypeKind::Enum.type_size(), Some(4));
|
||
}
|
||
|
||
#[test]
|
||
fn type_size_variable_and_composite_kinds() {
|
||
for kind in [
|
||
AlkTypeKind::String,
|
||
AlkTypeKind::Bytes,
|
||
AlkTypeKind::Struct,
|
||
AlkTypeKind::Union,
|
||
AlkTypeKind::Array,
|
||
AlkTypeKind::Record,
|
||
] {
|
||
assert_eq!(kind.type_size(), None, "failed for {kind:?}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn natural_alignment_matches_spec() {
|
||
assert_eq!(AlkTypeKind::Int8.natural_alignment(), 1);
|
||
assert_eq!(AlkTypeKind::Uint8.natural_alignment(), 1);
|
||
assert_eq!(AlkTypeKind::Boolean.natural_alignment(), 1);
|
||
assert_eq!(AlkTypeKind::Int16.natural_alignment(), 2);
|
||
assert_eq!(AlkTypeKind::Uint16.natural_alignment(), 2);
|
||
assert_eq!(AlkTypeKind::Int32.natural_alignment(), 4);
|
||
assert_eq!(AlkTypeKind::Uint32.natural_alignment(), 4);
|
||
assert_eq!(AlkTypeKind::Float32.natural_alignment(), 4);
|
||
assert_eq!(AlkTypeKind::Enum.natural_alignment(), 4);
|
||
assert_eq!(AlkTypeKind::Float64.natural_alignment(), 8);
|
||
assert_eq!(AlkTypeKind::Int64.natural_alignment(), 8);
|
||
assert_eq!(AlkTypeKind::Uint64.natural_alignment(), 8);
|
||
assert_eq!(AlkTypeKind::String.natural_alignment(), 4);
|
||
assert_eq!(AlkTypeKind::Bytes.natural_alignment(), 4);
|
||
assert_eq!(AlkTypeKind::Record.natural_alignment(), 4);
|
||
assert_eq!(AlkTypeKind::Struct.natural_alignment(), 1);
|
||
assert_eq!(AlkTypeKind::Union.natural_alignment(), 1);
|
||
assert_eq!(AlkTypeKind::Array.natural_alignment(), 1);
|
||
}
|
||
|
||
#[test]
|
||
fn is_fixed_size_classifies_correctly() {
|
||
for kind in [
|
||
AlkTypeKind::Float32,
|
||
AlkTypeKind::Float64,
|
||
AlkTypeKind::Int8,
|
||
AlkTypeKind::Int16,
|
||
AlkTypeKind::Int32,
|
||
AlkTypeKind::Int64,
|
||
AlkTypeKind::Uint8,
|
||
AlkTypeKind::Uint16,
|
||
AlkTypeKind::Uint32,
|
||
AlkTypeKind::Uint64,
|
||
AlkTypeKind::Boolean,
|
||
AlkTypeKind::Enum,
|
||
] {
|
||
assert!(kind.is_fixed_size(), "expected fixed: {kind:?}");
|
||
}
|
||
for kind in [
|
||
AlkTypeKind::String,
|
||
AlkTypeKind::Bytes,
|
||
AlkTypeKind::Struct,
|
||
AlkTypeKind::Union,
|
||
AlkTypeKind::Array,
|
||
AlkTypeKind::Record,
|
||
] {
|
||
assert!(!kind.is_fixed_size(), "expected variable: {kind:?}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn needs_endian_classifies_correctly() {
|
||
for kind in [
|
||
AlkTypeKind::Int16,
|
||
AlkTypeKind::Int32,
|
||
AlkTypeKind::Int64,
|
||
AlkTypeKind::Uint16,
|
||
AlkTypeKind::Uint32,
|
||
AlkTypeKind::Uint64,
|
||
AlkTypeKind::Float32,
|
||
AlkTypeKind::Float64,
|
||
AlkTypeKind::Enum,
|
||
AlkTypeKind::String,
|
||
AlkTypeKind::Bytes,
|
||
] {
|
||
assert!(kind.needs_endian(), "expected needs_endian: {kind:?}");
|
||
}
|
||
for kind in [
|
||
AlkTypeKind::Int8,
|
||
AlkTypeKind::Uint8,
|
||
AlkTypeKind::Boolean,
|
||
AlkTypeKind::Struct,
|
||
AlkTypeKind::Union,
|
||
AlkTypeKind::Array,
|
||
AlkTypeKind::Record,
|
||
] {
|
||
assert!(!kind.needs_endian(), "expected not needs_endian: {kind:?}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn is_composite_classifies_correctly() {
|
||
for kind in [
|
||
AlkTypeKind::Struct,
|
||
AlkTypeKind::Union,
|
||
AlkTypeKind::Array,
|
||
AlkTypeKind::Record,
|
||
] {
|
||
assert!(kind.is_composite(), "expected composite: {kind:?}");
|
||
}
|
||
for kind in [
|
||
AlkTypeKind::Int8,
|
||
AlkTypeKind::Int16,
|
||
AlkTypeKind::Int32,
|
||
AlkTypeKind::Int64,
|
||
AlkTypeKind::Uint8,
|
||
AlkTypeKind::Uint16,
|
||
AlkTypeKind::Uint32,
|
||
AlkTypeKind::Uint64,
|
||
AlkTypeKind::Float32,
|
||
AlkTypeKind::Float64,
|
||
AlkTypeKind::Boolean,
|
||
AlkTypeKind::Enum,
|
||
AlkTypeKind::String,
|
||
AlkTypeKind::Bytes,
|
||
] {
|
||
assert!(!kind.is_composite(), "expected not composite: {kind:?}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn to_bast_str_covers_all_18_variants() {
|
||
let cases: &[(AlkTypeKind, &str)] = &[
|
||
(AlkTypeKind::Int8, "int8"),
|
||
(AlkTypeKind::Int16, "int16"),
|
||
(AlkTypeKind::Int32, "int32"),
|
||
(AlkTypeKind::Int64, "int64"),
|
||
(AlkTypeKind::Uint8, "uint8"),
|
||
(AlkTypeKind::Uint16, "uint16"),
|
||
(AlkTypeKind::Uint32, "uint32"),
|
||
(AlkTypeKind::Uint64, "uint64"),
|
||
(AlkTypeKind::Float32, "float32"),
|
||
(AlkTypeKind::Float64, "float64"),
|
||
(AlkTypeKind::Boolean, "bool"),
|
||
(AlkTypeKind::String, "string"),
|
||
(AlkTypeKind::Bytes, "bytes"),
|
||
(AlkTypeKind::Enum, "enum"),
|
||
(AlkTypeKind::Struct, "struct"),
|
||
(AlkTypeKind::Union, "union"),
|
||
(AlkTypeKind::Array, "array"),
|
||
(AlkTypeKind::Record, "record"),
|
||
];
|
||
assert_eq!(cases.len(), 18);
|
||
for (kind, expected) in cases {
|
||
assert_eq!(kind.to_bast_str(), *expected);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn from_bast_str_round_trips_all_variants() {
|
||
for kind in [
|
||
AlkTypeKind::Int8,
|
||
AlkTypeKind::Int16,
|
||
AlkTypeKind::Int32,
|
||
AlkTypeKind::Int64,
|
||
AlkTypeKind::Uint8,
|
||
AlkTypeKind::Uint16,
|
||
AlkTypeKind::Uint32,
|
||
AlkTypeKind::Uint64,
|
||
AlkTypeKind::Float32,
|
||
AlkTypeKind::Float64,
|
||
AlkTypeKind::Boolean,
|
||
AlkTypeKind::String,
|
||
AlkTypeKind::Bytes,
|
||
AlkTypeKind::Enum,
|
||
AlkTypeKind::Struct,
|
||
AlkTypeKind::Union,
|
||
AlkTypeKind::Array,
|
||
AlkTypeKind::Record,
|
||
] {
|
||
let s = kind.to_bast_str();
|
||
let parsed = AlkTypeKind::from_bast_str(s).expect("round-trip {s}");
|
||
assert_eq!(parsed, kind, "round-trip {s}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn from_bast_str_parses_lowercase_strings() {
|
||
assert!(matches!(
|
||
AlkTypeKind::from_bast_str("uint32"),
|
||
Ok(AlkTypeKind::Uint32)
|
||
));
|
||
assert!(matches!(
|
||
AlkTypeKind::from_bast_str("bool"),
|
||
Ok(AlkTypeKind::Boolean)
|
||
));
|
||
assert!(matches!(
|
||
AlkTypeKind::from_bast_str("struct"),
|
||
Ok(AlkTypeKind::Struct)
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn from_bast_str_rejects_unknown_strings() {
|
||
assert!(AlkTypeKind::from_bast_str("uint128").is_err());
|
||
assert!(AlkTypeKind::from_bast_str("Uint32").is_err());
|
||
assert!(AlkTypeKind::from_bast_str("AlkType:Uint32").is_err());
|
||
assert!(AlkTypeKind::from_bast_str("timestamp").is_err());
|
||
assert!(AlkTypeKind::from_bast_str("").is_err());
|
||
}
|
||
} |