diff --git a/docs/plans/030-compiled-forms.md b/docs/plans/030-compiled-forms.md index 63c2e04..9fdd1c9 100644 --- a/docs/plans/030-compiled-forms.md +++ b/docs/plans/030-compiled-forms.md @@ -526,7 +526,39 @@ warnings`. `cargo build --target wasm32-unknown-unknown --release`. --- -## Phase 5 — `OffsetMap` carries `LeafMeta` (ADR-012 §2b) +## Phase 5 — `OffsetMap` carries `LeafMeta` (ADR-012 §2b) — **DONE (2026-09-02)** + +> **Status: implemented.** Prerequisite first (review #005 M2): +> `Hash` added to `Endian`/`VariableEncoding` derives in `schema.rs` +> (additive; both are fieldless `Eq` enums). New public types +> `LeafMeta { kind, encoding, endian }` (`Copy + PartialEq + Eq + +> Hash`) and `OffsetEntry { range, meta }` (with `start()`/`end()` +> convenience accessors), both re-exported from `lib.rs`; `ByteRange` +> gained `Hash` (additive). Storage is `Vec<(String, OffsetEntry)>` +> (deferred decision 2 resolved: struct — `get` returns +> `Option<&OffsetEntry>`, `iter` yields `(&str, &OffsetEntry)`). +> `LeafMeta` is computed at `compute` time with **effective** endian +> threaded through the walk: container default → field override per +> field, propagated into nested-struct probes and array elements via +> the referring field (the same propagation the aligned materializer +> uses). **Parity note:** this replaces `engine.rs`'s +> `lookup_leaf_field` walk, which computed nested-struct defaults from +> the *nested struct's own* `endian` annotation — the two paths +> diverged whenever a nested struct declared `endian` and its +> referring field also declared one (the map now agrees with the +> aligned materializer and the packed `ReadPlan`; the old divergence +> was unreachable through `read_field` only when a nested annotation +> existed, and no test pinned it). `read_field`/`write_field` dispatch +> on the entry's `LeafMeta` — the `BastDoc` re-parse + +> `lookup_leaf_field`/`LeafFieldInfo` per access are gone (the last +> two M1 sites, engine.rs `read_field`/`write_field`). Behavior +> change: `read_field` on a path absent from the map (e.g. a +> whole-struct field path) now errors with `Offset` ("field not found +> in offset map") instead of `Access` ("does not support composite +> types") — the composite-path test already accepted either variant. +> `materialize_aligned`'s four `offset_map.get` call sites updated to +> `.range.start`. `alktty`/`alkcall` untouched (the bench never uses +> `OffsetMap::get`; alkcall has no dependency yet). **Goal:** Extend `OffsetMap`'s entries with `LeafMeta { kind, encoding, endian }` computed at `compute` time. `read_field`/`write_field` drop diff --git a/src/engine.rs b/src/engine.rs index 132a045..21be5f9 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -16,7 +16,7 @@ //! §"The AlkTypeEngine struct" and //! [overview.md](../../docs/architecture/overview.md). -use crate::bast::{BastDefKind, BastDoc, BastStruct, BastType}; +use crate::bast::{BastDefKind, BastDoc}; use crate::data_access; use crate::error::AlkTypeError; use crate::layout_builder::LayoutBuilder; @@ -333,14 +333,16 @@ impl AlkTypeEngine { /// Read a field from a buffer at its computed offset (aligned mode). /// - /// Looks up the field's byte range in the [`OffsetMap`] and reads the - /// appropriate type using the [`crate::data_access`] functions. Works - /// for fixed-size primitive kinds and length-prefixed `String`/ - /// `Bytes` fields. + /// Looks up the field's [`OffsetEntry`](crate::offset_map::OffsetEntry) in the [`OffsetMap`] and reads + /// the appropriate type using the [`crate::data_access`] functions, + /// dispatching on the entry's [`LeafMeta`](crate::offset_map::LeafMeta) (kind, encoding, effective + /// endian — computed at compile time, ADR-012 §2b). Works for + /// fixed-size primitive kinds and length-prefixed `String`/`Bytes` + /// fields. /// /// Returns an error if compiled in packed mode — use /// [`AlkTypeEngine::sequential_reader`] for packed mode. Also - /// returns an error for composite kinds (`Struct`, `Union`, `Array`, + /// returns an error for composite kinds (`Union`, `Array`, /// `Record`) — those are better handled via the layout-specific APIs. /// /// # Errors @@ -365,18 +367,14 @@ impl AlkTypeEngine { }); } }; - let range = offset_map + let entry = offset_map .get(field_path) .ok_or_else(|| AlkTypeError::Offset { field_path: field_path.to_string(), reason: "field not found in offset map".to_string(), })?; - let leaf = lookup_leaf_field(&self.doc, field_path).ok_or_else(|| AlkTypeError::Offset { - field_path: field_path.to_string(), - reason: "field schema not found in BAST tree or has no primitive kind".to_string(), - })?; - let kind = leaf.kind; - let endian = leaf.endian; + let (kind, encoding, endian) = (entry.meta.kind, entry.meta.encoding, entry.meta.endian); + let range = entry.range; match kind { AlkTypeKind::Int8 => { let v = data_access::read_i8(buffer, range.start, field_path)?; @@ -427,7 +425,7 @@ impl AlkTypeEngine { Ok(FieldValue::Enum(v)) } AlkTypeKind::String => { - let v = match leaf.encoding { + let v = match encoding { VariableEncoding::OffsetIndirect => { data_access::read_string_indirect(buffer, range.start, field_path, endian)? } @@ -438,7 +436,7 @@ impl AlkTypeEngine { Ok(FieldValue::String(v)) } AlkTypeKind::Bytes => { - let v = match leaf.encoding { + let v = match encoding { VariableEncoding::OffsetIndirect => { data_access::read_bytes_indirect(buffer, range.start, field_path, endian)? } @@ -465,10 +463,12 @@ impl AlkTypeEngine { /// Write a field to a buffer at its computed offset (aligned mode). /// - /// Looks up the field's byte range in the [`OffsetMap`] and writes the - /// appropriate type using the [`crate::data_access`] functions. Works - /// for fixed-size primitive kinds and length-prefixed `String`/ - /// `Bytes` fields. + /// Looks up the field's [`OffsetEntry`](crate::offset_map::OffsetEntry) in the [`OffsetMap`] and writes + /// the appropriate type using the [`crate::data_access`] functions, + /// dispatching on the entry's [`LeafMeta`](crate::offset_map::LeafMeta) (kind, encoding, effective + /// endian — computed at compile time, ADR-012 §2b). Works for + /// fixed-size primitive kinds and length-prefixed `String`/`Bytes` + /// fields. /// /// Returns an error if compiled in packed mode — use /// [`AlkTypeEngine::layout_builder`] for packed mode. Also returns @@ -497,17 +497,14 @@ impl AlkTypeEngine { }); } }; - let range = offset_map + let entry = offset_map .get(field_path) .ok_or_else(|| AlkTypeError::Offset { field_path: field_path.to_string(), reason: "field not found in offset map".to_string(), })?; - let leaf = lookup_leaf_field(&self.doc, field_path).ok_or_else(|| AlkTypeError::Offset { - field_path: field_path.to_string(), - reason: "field schema not found in BAST tree or has no primitive kind".to_string(), - })?; - let endian = leaf.endian; + let (encoding, endian) = (entry.meta.encoding, entry.meta.endian); + let range = entry.range; match value { FieldValue::I8(v) => data_access::write_i8(buffer, range.start, *v, field_path), FieldValue::I16(v) => { @@ -540,7 +537,7 @@ impl AlkTypeEngine { data_access::write_enum(buffer, range.start, *v, field_path, endian) } FieldValue::String(v) => { - if leaf.encoding == VariableEncoding::OffsetIndirect { + if encoding == VariableEncoding::OffsetIndirect { return Err(AlkTypeError::Access { field_path: field_path.to_string(), reason: "write_field cannot write offset-indirect fields; \ @@ -553,7 +550,7 @@ impl AlkTypeEngine { Ok(()) } FieldValue::Bytes(v) => { - if leaf.encoding == VariableEncoding::OffsetIndirect { + if encoding == VariableEncoding::OffsetIndirect { return Err(AlkTypeError::Access { field_path: field_path.to_string(), reason: "write_field cannot write offset-indirect fields; \ @@ -588,55 +585,6 @@ impl fmt::Debug for AlkTypeEngine { } } -/// The resolved leaf-field metadata needed by `read_field`/`write_field`: -/// the field's kind, its variable-length encoding, and its effective -/// endianness (field override, else the enclosing struct's default). -struct LeafFieldInfo { - kind: AlkTypeKind, - encoding: VariableEncoding, - endian: Endian, -} - -/// Walk the BAST typed tree to find the leaf field for a dotted field -/// path. Returns `None` if any segment is missing or the resolved type is -/// a `$ref` that can't be resolved. -/// -/// Splits `field_path` on `.` and descends into the root struct's -/// `fields` at each step, resolving `$ref`s via [`BastDoc::resolve_typeref`]. -/// Array element segments (`field[i]`) are not handled here — -/// `read_field`/`write_field` only address leaf fields. -fn lookup_leaf_field(doc: &BastDoc, field_path: &str) -> Option { - let root_def = doc.root_def(); - let mut current_struct: BastStruct = match root_def.kind() { - BastDefKind::Struct(s) => s.clone(), - _ => return None, - }; - let mut struct_endian = current_struct.endian(); - let segments: Vec<&str> = field_path.split('.').collect(); - let last = segments.len(); - for (i, segment) in segments.iter().enumerate() { - let field = current_struct.fields().iter().find(|f| f.name() == *segment)?; - let ty = field.ty(); - let resolved = doc.resolve_typeref(ty).ok()?; - let field_endian = field.effective_endian(struct_endian); - if i + 1 == last { - return Some(LeafFieldInfo { - kind: resolved.alk_kind(), - encoding: field.encoding(), - endian: field_endian, - }); - } - match &resolved { - BastType::Struct(s) => { - current_struct = s.clone(); - struct_endian = s.endian(); - } - _ => return None, - } - } - None -} - #[cfg(test)] mod tests { use super::*; @@ -1023,7 +971,7 @@ mod tests { } #[test] - fn lookup_leaf_field_walks_dotted_path() { + fn offset_map_entries_carry_leaf_meta() { let doc = json!({ "$defs": { "S": { @@ -1043,11 +991,12 @@ mod tests { } } }); - let d = BastDoc::new(&doc, "S").expect("doc"); - assert_eq!(lookup_leaf_field(&d, "header.magic").unwrap().kind, AlkTypeKind::Uint32); - assert_eq!(lookup_leaf_field(&d, "header.version").unwrap().kind, AlkTypeKind::Uint8); - assert!(lookup_leaf_field(&d, "header.missing").is_none()); - assert!(lookup_leaf_field(&d, "missing").is_none()); + let engine = AlkTypeEngine::compile(&doc, "S", LayoutMode::Aligned, None).expect("compile"); + let map = engine.offset_map().expect("aligned map"); + assert_eq!(map.get("header.magic").unwrap().meta.kind, AlkTypeKind::Uint32); + assert_eq!(map.get("header.version").unwrap().meta.kind, AlkTypeKind::Uint8); + assert!(map.get("header.missing").is_none()); + assert!(map.get("missing").is_none()); } #[test] diff --git a/src/lib.rs b/src/lib.rs index 5d17ddd..4a31c6a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -70,7 +70,7 @@ pub use builder::{Definitions, Discriminator, Schema}; pub use engine::{LayoutMode, AlkTypeEngine}; pub use error::AlkTypeError; pub use layout_builder::{FieldPosition, LayoutBuilder, PackedLayout}; -pub use offset_map::{ByteRange, OffsetMap}; +pub use offset_map::{ByteRange, LeafMeta, OffsetEntry, OffsetMap}; pub use read_plan::{ CompositePlan, DiscriminatorPlan, FieldPlan, ReadKind, ReadPlan, }; diff --git a/src/materialize.rs b/src/materialize.rs index 7601f07..d415ce3 100644 --- a/src/materialize.rs +++ b/src/materialize.rs @@ -856,11 +856,11 @@ fn materialize_struct_aligned( let kind = resolved.alk_kind(); let field_endian = field.effective_endian(endian); let value = if kind.is_fixed_size() { - let range = offset_map.get(&path).ok_or_else(|| AlkTypeError::Offset { + let entry = offset_map.get(&path).ok_or_else(|| AlkTypeError::Offset { field_path: path.clone(), reason: "field not found in offset map".to_string(), })?; - materialize_leaf_at(doc, &resolved, &path, range.start, field_endian, buffer)? + materialize_leaf_at(doc, &resolved, &path, entry.range.start, field_endian, buffer)? } else if kind == AlkTypeKind::Struct { if let BastType::Struct(s) = &resolved { materialize_struct_aligned(doc, s, &path, offset_map, field_endian, buffer)? @@ -878,11 +878,11 @@ fn materialize_struct_aligned( ))); } } else if kind == AlkTypeKind::Record { - let range = offset_map.get(&path).ok_or_else(|| AlkTypeError::Offset { + let entry = offset_map.get(&path).ok_or_else(|| AlkTypeError::Offset { field_path: path.clone(), reason: "field not found in offset map".to_string(), })?; - let mut offset = range.start; + let mut offset = entry.range.start; materialize_typeref_packed( doc, &resolved, @@ -892,11 +892,11 @@ fn materialize_struct_aligned( &mut offset, )? } else if kind.is_variable_length() { - let range = offset_map.get(&path).ok_or_else(|| AlkTypeError::Offset { + let entry = offset_map.get(&path).ok_or_else(|| AlkTypeError::Offset { field_path: path.clone(), reason: "field not found in offset map".to_string(), })?; - materialize_variable_aligned(doc, field, &resolved, &path, range, field_endian, buffer)? + materialize_variable_aligned(doc, field, &resolved, &path, &entry.range, field_endian, buffer)? } else { return Err(AlkTypeError::Schema(format!( "materialize: unsupported kind {kind} at {path} in aligned mode" @@ -924,11 +924,11 @@ fn materialize_array_aligned( let mut arr = Vec::with_capacity(count); for i in 0..count { let elem_path = format!("{field_path}[{i}]"); - let range = offset_map.get(&elem_path).ok_or_else(|| AlkTypeError::Offset { + let entry = offset_map.get(&elem_path).ok_or_else(|| AlkTypeError::Offset { field_path: elem_path.clone(), reason: "array element not found in offset map".to_string(), })?; - let value = materialize_leaf_at(doc, &resolved_elem, &elem_path, range.start, endian, buffer)?; + let value = materialize_leaf_at(doc, &resolved_elem, &elem_path, entry.range.start, endian, buffer)?; arr.push(value); } Ok(Value::Array(arr)) diff --git a/src/offset_map.rs b/src/offset_map.rs index 80cf609..81a67dc 100644 --- a/src/offset_map.rs +++ b/src/offset_map.rs @@ -16,7 +16,7 @@ use crate::bast::{ BastArray, BastDefKind, BastDoc, BastField, BastStruct, BastType, }; use crate::error::AlkTypeError; -use crate::schema::{AlkTypeKind, VariableEncoding}; +use crate::schema::{AlkTypeKind, Endian, VariableEncoding}; /// A byte range within a buffer. /// @@ -25,7 +25,7 @@ use crate::schema::{AlkTypeKind, VariableEncoding}; /// size in the static layout (for variable-length fields, this is the /// size of the length prefix, the `{offset, length}` pair, or the /// `maxLength` reservation — not the variable data itself). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct ByteRange { /// Inclusive start byte offset. pub start: usize, @@ -45,8 +45,48 @@ impl ByteRange { } } -/// A flat table of `(field_path, byte_range)` pairs computed from a BAST -/// document. +/// The resolved leaf metadata needed to read or write a field's bytes: +/// the field's kind, its variable-length encoding, and its effective +/// endianness (field override, else the enclosing struct/union default, +/// propagated the same way the aligned materializer propagates it — a +/// nested container inherits the *referring field's* effective endian). +/// +/// Computed at [`OffsetMap::compute`] time so `read_field`/`write_field` +/// don't re-walk the BAST tree per access (ADR-012 §2b). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct LeafMeta { + /// The leaf's type kind. + pub kind: AlkTypeKind, + /// The variable-length encoding strategy (meaningful for + /// `String`/`Bytes`/`Record` leaves). + pub encoding: VariableEncoding, + /// The effective endianness for reads/writes at this leaf. + pub endian: Endian, +} + +/// A field's offset-map entry: its byte range plus its [`LeafMeta`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct OffsetEntry { + /// The field's byte range in the static layout. + pub range: ByteRange, + /// The resolved leaf metadata (kind, encoding, effective endian). + pub meta: LeafMeta, +} + +impl OffsetEntry { + /// The entry's start byte offset. + pub fn start(&self) -> usize { + self.range.start + } + + /// The entry's exclusive end byte offset. + pub fn end(&self) -> usize { + self.range.end + } +} + +/// A flat table of `(field_path, OffsetEntry)` pairs computed from a +/// BAST document. /// /// Fields have fixed positions with natural alignment padding. /// Used for mmap-friendly formats (metatensor, safetensors) where random @@ -59,7 +99,7 @@ impl ByteRange { /// data lives outside the static layout. #[derive(Debug)] pub struct OffsetMap { - fields: Vec<(String, ByteRange)>, + fields: Vec<(String, OffsetEntry)>, total_size: usize, } @@ -93,24 +133,25 @@ impl OffsetMap { offset: 0, }; let struct_default_align = struct_node.align().unwrap_or(1).max(1); - let (total, _align) = ctx.compute_struct(struct_node, "", struct_default_align)?; + let endian = struct_node.endian(); + let (total, _align) = ctx.compute_struct(struct_node, "", struct_default_align, endian)?; Ok(Self { fields: ctx.fields, total_size: total, }) } - /// Look up a field's byte range by dotted path (e.g., `"header.version"`). + /// Look up a field's entry by dotted path (e.g., `"header.version"`). /// /// Returns `None` if no field with the given path was recorded. For /// TUnion byte-offset discriminators, the discriminator is recorded /// under the synthetic path `"__discriminator"` (qualified by the /// union field's path, e.g. `"payload.__discriminator"`). - pub fn get(&self, field_path: &str) -> Option<&ByteRange> { + pub fn get(&self, field_path: &str) -> Option<&OffsetEntry> { self.fields .iter() .find(|(path, _)| path == field_path) - .map(|(_, range)| range) + .map(|(_, entry)| entry) } /// The total size of the struct in bytes (including trailing alignment padding). @@ -118,12 +159,12 @@ impl OffsetMap { self.total_size } - /// Iterate over all `(field_path, byte_range)` pairs in insertion order. + /// Iterate over all `(field_path, OffsetEntry)` pairs in insertion order. /// /// Field order matches the BAST `fields` array order. Nested struct /// fields appear after their parent's path prefix. - pub fn iter(&self) -> impl Iterator { - self.fields.iter() + pub fn iter(&self) -> impl Iterator { + self.fields.iter().map(|(path, entry)| (path.as_str(), entry)) } } @@ -134,7 +175,7 @@ impl OffsetMap { /// keeps the recursive helper signatures small. struct ComputeCtx<'d> { doc: &'d BastDoc, - fields: Vec<(String, ByteRange)>, + fields: Vec<(String, OffsetEntry)>, offset: usize, } @@ -161,6 +202,7 @@ impl<'d> ComputeCtx<'d> { struct_node: &BastStruct, prefix: &str, parent_struct_align: usize, + endian: Endian, ) -> Result<(usize, usize), AlkTypeError> { let struct_default_align = struct_node.align().unwrap_or(parent_struct_align).max(1); let mut max_align: usize = 1; @@ -194,7 +236,7 @@ impl<'d> ComputeCtx<'d> { } } } - let layout = self.compute_field(field, &field_path, struct_default_align)?; + let layout = self.compute_field(field, &field_path, struct_default_align, endian)?; if layout.align > max_align { max_align = layout.align; } @@ -213,11 +255,13 @@ impl<'d> ComputeCtx<'d> { field: &BastField, field_path: &str, struct_default_align: usize, + container_endian: Endian, ) -> Result { let ty = field.ty(); let resolved = self.doc.resolve_typeref(ty)?; + let field_endian = field.effective_endian(container_endian); match &resolved { - BastType::Struct(s) => self.compute_struct_field(s, field, field_path, struct_default_align), + BastType::Struct(s) => self.compute_struct_field(s, field, field_path, struct_default_align, field_endian), BastType::Union(_) => Err(AlkTypeError::Offset { field_path: field_path.to_string(), reason: "TUnion is not supported in aligned static mode (ADR-008). \ @@ -226,21 +270,22 @@ impl<'d> ComputeCtx<'d> { a struct with an explicit discriminator field." .to_string(), }), - BastType::Array(a) => self.compute_array_field(a, field, field_path, struct_default_align), + BastType::Array(a) => self.compute_array_field(a, field, field_path, struct_default_align, field_endian), BastType::Record(_) => { - self.compute_variable_field(field, field_path, struct_default_align) + self.compute_variable_field(field, field_path, struct_default_align, resolved.alk_kind(), field_endian) } BastType::Primitive(k) if k.is_variable_length() => { - self.compute_variable_field(field, field_path, struct_default_align) + self.compute_variable_field(field, field_path, struct_default_align, *k, field_endian) } BastType::Primitive(k) => { - self.compute_fixed_field(*k, field, field_path, struct_default_align) + self.compute_fixed_field(*k, field, field_path, struct_default_align, field_endian) } BastType::Enum(_) => self.compute_fixed_field( AlkTypeKind::Enum, field, field_path, struct_default_align, + field_endian, ), BastType::Ref(_) => Err(AlkTypeError::Offset { field_path: field_path.to_string(), @@ -256,6 +301,7 @@ impl<'d> ComputeCtx<'d> { field: &BastField, field_path: &str, struct_default_align: usize, + endian: Endian, ) -> Result { let size = kind.type_size().ok_or_else(|| AlkTypeError::Offset { field_path: field_path.to_string(), @@ -271,7 +317,7 @@ impl<'d> ComputeCtx<'d> { field_path: field_path.to_string(), reason: format!("offset {start} + size {size} overflows usize"), })?; - self.push(field_path, start, start + size); + self.push(field_path, start, start + size, kind, field.encoding(), endian); Ok(FieldLayout { align }) } @@ -286,6 +332,7 @@ impl<'d> ComputeCtx<'d> { field: &BastField, field_path: &str, struct_default_align: usize, + endian: Endian, ) -> Result { let inner_parent_align = field.align().unwrap_or(struct_default_align); let mut probe = ComputeCtx { @@ -294,18 +341,22 @@ impl<'d> ComputeCtx<'d> { offset: 0, }; let (inner_total, inner_align) = - probe.compute_struct(struct_node, field_path, inner_parent_align)?; + probe.compute_struct(struct_node, field_path, inner_parent_align, endian)?; let natural = inner_align; let align = field_alignment(field, struct_default_align, natural); align_up(&mut self.offset, align); let struct_start = self.offset; - for (path, range) in probe.fields { + for (path, entry) in probe.fields { + let range = ByteRange { + start: struct_start + entry.range.start, + end: struct_start + entry.range.end, + }; self.fields.push(( path, - ByteRange { - start: struct_start + range.start, - end: struct_start + range.end, + OffsetEntry { + range, + meta: entry.meta, }, )); } @@ -325,6 +376,7 @@ impl<'d> ComputeCtx<'d> { field: &BastField, field_path: &str, struct_default_align: usize, + endian: Endian, ) -> Result { let element_ty = array.element(); let resolved_elem = self.doc.resolve_typeref(element_ty)?; @@ -351,6 +403,8 @@ impl<'d> ComputeCtx<'d> { align_up(&mut self.offset, array_align); let start = self.offset; + let elem_encoding = field.encoding(); + let elem_endian = field_endian_for_element(&resolved_elem, endian); for i in 0..count { let elem_start = start .checked_add( @@ -371,7 +425,7 @@ impl<'d> ComputeCtx<'d> { reason: format!("element end {elem_start} + {elem_size} overflows usize"), })?; let elem_path = format!("{field_path}[{i}]"); - self.push(&elem_path, elem_start, elem_end); + self.push(&elem_path, elem_start, elem_end, elem_kind, elem_encoding, elem_endian); } let array_size = count .checked_mul(stride) @@ -399,6 +453,8 @@ impl<'d> ComputeCtx<'d> { field: &BastField, field_path: &str, struct_default_align: usize, + kind: AlkTypeKind, + endian: Endian, ) -> Result { let encoding = field.encoding(); let max_length = field.max_length(); @@ -418,14 +474,23 @@ impl<'d> ComputeCtx<'d> { field_path: field_path.to_string(), reason: format!("offset {start} + size {size} overflows usize"), })?; - self.push(field_path, start, start + size); + self.push(field_path, start, start + size, kind, encoding, endian); Ok(FieldLayout { align }) } - /// Push a `(field_path, ByteRange)` pair onto the fields vec. - fn push(&mut self, path: &str, start: usize, end: usize) { - self.fields - .push((path.to_string(), ByteRange { start, end })); + /// Push an `OffsetEntry` onto the fields vec. + fn push(&mut self, path: &str, start: usize, end: usize, kind: AlkTypeKind, encoding: VariableEncoding, endian: Endian) { + self.fields.push(( + path.to_string(), + OffsetEntry { + range: ByteRange { start, end }, + meta: LeafMeta { + kind, + encoding, + endian, + }, + }, + )); } } @@ -437,6 +502,20 @@ fn field_variable_kind(field: &BastField) -> Option { } } +/// The effective endianness for an array element's reads/writes. The +/// element TypeRef doesn't carry a field-level override (BAST field +/// annotations live on the field, not the element), so the referring +/// field's effective endian applies — the same propagation the aligned +/// materializer uses. +fn field_endian_for_element(elem_ty: &BastType, field_endian: Endian) -> Endian { + match elem_ty { + BastType::Struct(s) => s.endian(), + BastType::Union(u) => u.endian(), + BastType::Enum(_) | BastType::Array(_) | BastType::Record(_) => field_endian, + BastType::Primitive(_) | BastType::Ref(_) => field_endian, + } +} + /// Resolve the field's alignment: field-level `align` annotation, /// then the struct default, then the natural alignment. fn field_alignment(field: &BastField, struct_default_align: usize, natural: usize) -> usize { @@ -512,8 +591,8 @@ mod tests { } }); let m = map(&root, "S"); - assert_eq!(m.get("flag"), Some(&ByteRange { start: 0, end: 1 })); - assert_eq!(m.get("id"), Some(&ByteRange { start: 4, end: 8 })); + assert_eq!(m.get("flag"), Some(&OffsetEntry { range: ByteRange { start: 0, end: 1 }, meta: LeafMeta { kind: AlkTypeKind::Uint8, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); + assert_eq!(m.get("id"), Some(&OffsetEntry { range: ByteRange { start: 4, end: 8 }, meta: LeafMeta { kind: AlkTypeKind::Uint32, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); assert_eq!(m.total_size(), 8); } @@ -531,8 +610,8 @@ mod tests { } }); let m = map(&root, "S"); - assert_eq!(m.get("a"), Some(&ByteRange { start: 0, end: 1 })); - assert_eq!(m.get("b"), Some(&ByteRange { start: 4, end: 8 })); + assert_eq!(m.get("a"), Some(&OffsetEntry { range: ByteRange { start: 0, end: 1 }, meta: LeafMeta { kind: AlkTypeKind::Uint8, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); + assert_eq!(m.get("b"), Some(&OffsetEntry { range: ByteRange { start: 4, end: 8 }, meta: LeafMeta { kind: AlkTypeKind::Uint32, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); } #[test] @@ -558,12 +637,12 @@ mod tests { } }); let m = map(&root, "S"); - assert_eq!(m.get("header.magic"), Some(&ByteRange { start: 0, end: 4 })); + assert_eq!(m.get("header.magic"), Some(&OffsetEntry { range: ByteRange { start: 0, end: 4 }, meta: LeafMeta { kind: AlkTypeKind::Uint32, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); assert_eq!( m.get("header.version"), - Some(&ByteRange { start: 4, end: 5 }) + Some(&OffsetEntry { range: ByteRange { start: 4, end: 5 }, meta: LeafMeta { kind: AlkTypeKind::Uint8, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } }) ); - assert_eq!(m.get("body"), Some(&ByteRange { start: 8, end: 12 })); + assert_eq!(m.get("body"), Some(&OffsetEntry { range: ByteRange { start: 8, end: 12 }, meta: LeafMeta { kind: AlkTypeKind::Uint32, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); assert_eq!(m.total_size(), 12); } @@ -580,9 +659,9 @@ mod tests { } }); let m = map(&root, "S"); - assert_eq!(m.get("vals[0]"), Some(&ByteRange { start: 0, end: 4 })); - assert_eq!(m.get("vals[1]"), Some(&ByteRange { start: 4, end: 8 })); - assert_eq!(m.get("vals[2]"), Some(&ByteRange { start: 8, end: 12 })); + assert_eq!(m.get("vals[0]"), Some(&OffsetEntry { range: ByteRange { start: 0, end: 4 }, meta: LeafMeta { kind: AlkTypeKind::Uint32, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); + assert_eq!(m.get("vals[1]"), Some(&OffsetEntry { range: ByteRange { start: 4, end: 8 }, meta: LeafMeta { kind: AlkTypeKind::Uint32, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); + assert_eq!(m.get("vals[2]"), Some(&OffsetEntry { range: ByteRange { start: 8, end: 12 }, meta: LeafMeta { kind: AlkTypeKind::Uint32, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); assert_eq!(m.total_size(), 12); } @@ -600,8 +679,8 @@ mod tests { } }); let m = map(&root, "S"); - assert_eq!(m.get("id"), Some(&ByteRange { start: 0, end: 4 })); - assert_eq!(m.get("name"), Some(&ByteRange { start: 4, end: 8 })); + assert_eq!(m.get("id"), Some(&OffsetEntry { range: ByteRange { start: 0, end: 4 }, meta: LeafMeta { kind: AlkTypeKind::Uint32, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); + assert_eq!(m.get("name"), Some(&OffsetEntry { range: ByteRange { start: 4, end: 8 }, meta: LeafMeta { kind: AlkTypeKind::String, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); assert_eq!(m.total_size(), 8); } @@ -619,8 +698,8 @@ mod tests { } }); let m = map(&root, "S"); - assert_eq!(m.get("id"), Some(&ByteRange { start: 0, end: 4 })); - assert_eq!(m.get("name"), Some(&ByteRange { start: 4, end: 260 })); + assert_eq!(m.get("id"), Some(&OffsetEntry { range: ByteRange { start: 0, end: 4 }, meta: LeafMeta { kind: AlkTypeKind::Uint32, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); + assert_eq!(m.get("name"), Some(&OffsetEntry { range: ByteRange { start: 4, end: 260 }, meta: LeafMeta { kind: AlkTypeKind::String, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); assert_eq!(m.total_size(), 260); } @@ -638,8 +717,8 @@ mod tests { } }); let m = map(&root, "S"); - assert_eq!(m.get("id"), Some(&ByteRange { start: 0, end: 4 })); - assert_eq!(m.get("blob"), Some(&ByteRange { start: 4, end: 12 })); + assert_eq!(m.get("id"), Some(&OffsetEntry { range: ByteRange { start: 0, end: 4 }, meta: LeafMeta { kind: AlkTypeKind::Uint32, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); + assert_eq!(m.get("blob"), Some(&OffsetEntry { range: ByteRange { start: 4, end: 12 }, meta: LeafMeta { kind: AlkTypeKind::String, encoding: VariableEncoding::OffsetIndirect, endian: Endian::Little } })); assert_eq!(m.total_size(), 12); } @@ -729,8 +808,8 @@ mod tests { } }); let m = map(&root, "S"); - assert_eq!(m.get("id"), Some(&ByteRange { start: 0, end: 4 })); - assert_eq!(m.get("name"), Some(&ByteRange { start: 4, end: 8 })); + assert_eq!(m.get("id"), Some(&OffsetEntry { range: ByteRange { start: 0, end: 4 }, meta: LeafMeta { kind: AlkTypeKind::Uint32, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); + assert_eq!(m.get("name"), Some(&OffsetEntry { range: ByteRange { start: 4, end: 8 }, meta: LeafMeta { kind: AlkTypeKind::String, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); } #[test] @@ -747,8 +826,8 @@ mod tests { } }); let m = map(&root, "S"); - assert_eq!(m.get("name"), Some(&ByteRange { start: 0, end: 256 })); - assert_eq!(m.get("id"), Some(&ByteRange { start: 256, end: 260 })); + assert_eq!(m.get("name"), Some(&OffsetEntry { range: ByteRange { start: 0, end: 256 }, meta: LeafMeta { kind: AlkTypeKind::String, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); + assert_eq!(m.get("id"), Some(&OffsetEntry { range: ByteRange { start: 256, end: 260 }, meta: LeafMeta { kind: AlkTypeKind::Uint32, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); } #[test] @@ -765,8 +844,8 @@ mod tests { } }); let m = map(&root, "S"); - assert_eq!(m.get("blob"), Some(&ByteRange { start: 0, end: 8 })); - assert_eq!(m.get("id"), Some(&ByteRange { start: 8, end: 12 })); + assert_eq!(m.get("blob"), Some(&OffsetEntry { range: ByteRange { start: 0, end: 8 }, meta: LeafMeta { kind: AlkTypeKind::String, encoding: VariableEncoding::OffsetIndirect, endian: Endian::Little } })); + assert_eq!(m.get("id"), Some(&OffsetEntry { range: ByteRange { start: 8, end: 12 }, meta: LeafMeta { kind: AlkTypeKind::Uint32, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); } #[test] @@ -783,7 +862,7 @@ mod tests { } }); let m = map(&root, "S"); - assert_eq!(m.get("flag"), Some(&ByteRange { start: 0, end: 1 })); + assert_eq!(m.get("flag"), Some(&OffsetEntry { range: ByteRange { start: 0, end: 1 }, meta: LeafMeta { kind: AlkTypeKind::Uint8, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); assert_eq!(m.total_size(), 16); } @@ -803,9 +882,9 @@ mod tests { } }); let m = map(&root, "S"); - assert_eq!(m.get("tag"), Some(&ByteRange { start: 0, end: 1 })); - assert_eq!(m.get("flag"), Some(&ByteRange { start: 16, end: 17 })); - assert_eq!(m.get("id"), Some(&ByteRange { start: 20, end: 24 })); + assert_eq!(m.get("tag"), Some(&OffsetEntry { range: ByteRange { start: 0, end: 1 }, meta: LeafMeta { kind: AlkTypeKind::Uint8, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); + assert_eq!(m.get("flag"), Some(&OffsetEntry { range: ByteRange { start: 16, end: 17 }, meta: LeafMeta { kind: AlkTypeKind::Uint8, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); + assert_eq!(m.get("id"), Some(&OffsetEntry { range: ByteRange { start: 20, end: 24 }, meta: LeafMeta { kind: AlkTypeKind::Uint32, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); assert_eq!(m.total_size(), 24); } @@ -824,8 +903,8 @@ mod tests { } }); let m = map(&root, "S"); - assert_eq!(m.get("a"), Some(&ByteRange { start: 0, end: 1 })); - assert_eq!(m.get("b"), Some(&ByteRange { start: 1, end: 5 })); + assert_eq!(m.get("a"), Some(&OffsetEntry { range: ByteRange { start: 0, end: 1 }, meta: LeafMeta { kind: AlkTypeKind::Uint8, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); + assert_eq!(m.get("b"), Some(&OffsetEntry { range: ByteRange { start: 1, end: 5 }, meta: LeafMeta { kind: AlkTypeKind::Uint32, encoding: VariableEncoding::LengthPrefixed, endian: Endian::Little } })); assert_eq!(m.total_size(), 8); } @@ -843,7 +922,7 @@ mod tests { } }); let m = map(&root, "S"); - let paths: Vec<&String> = m.iter().map(|(p, _)| p).collect(); + let paths: Vec<&str> = m.iter().map(|(p, _)| p).collect(); assert_eq!(paths, vec!["a", "b"]); } diff --git a/src/schema.rs b/src/schema.rs index d4242ac..761fdfa 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -202,14 +202,14 @@ impl fmt::Display for AlkTypeKind { } /// Byte endianness for multi-byte integer and float fields. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Endian { Little, Big, } /// The encoding strategy for a variable-length type. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum VariableEncoding { /// `[length: u32][data]` — the default. Length prefix at a known offset, /// variable data follows immediately. diff --git a/tests/poc_roundtrip.rs b/tests/poc_roundtrip.rs index aacdbe2..8fb78a8 100644 --- a/tests/poc_roundtrip.rs +++ b/tests/poc_roundtrip.rs @@ -44,29 +44,29 @@ fn fixed_size_round_trip_via_offset_map() -> Result<(), AlkTypeError> { let mut buffer = vec![0u8; offset_map.total_size()]; let id_range = offset_map.get("id").expect("id range"); - data_access::write_u32(&mut buffer, id_range.start, 42, "id", Endian::Little)?; + data_access::write_u32(&mut buffer, id_range.start(), 42, "id", Endian::Little)?; let score_range = offset_map.get("score").expect("score range"); - data_access::write_f32(&mut buffer, score_range.start, 1.5, "score", Endian::Little)?; + data_access::write_f32(&mut buffer, score_range.start(), 1.5, "score", Endian::Little)?; let flag_range = offset_map.get("flag").expect("flag range"); - data_access::write_u8(&mut buffer, flag_range.start, 1, "flag")?; + data_access::write_u8(&mut buffer, flag_range.start(), 1, "flag")?; let count_range = offset_map.get("count").expect("count range"); data_access::write_u16( &mut buffer, - count_range.start, + count_range.start(), 1000, "count", Endian::Little, )?; assert_eq!( - data_access::read_u32(&buffer, id_range.start, "id", Endian::Little)?, + data_access::read_u32(&buffer, id_range.start(), "id", Endian::Little)?, 42 ); - let score = data_access::read_f32(&buffer, score_range.start, "score", Endian::Little)?; + let score = data_access::read_f32(&buffer, score_range.start(), "score", Endian::Little)?; assert!((score - 1.5).abs() < 0.001, "score: {score}"); - assert_eq!(data_access::read_u8(&buffer, flag_range.start, "flag")?, 1); + assert_eq!(data_access::read_u8(&buffer, flag_range.start(), "flag")?, 1); assert_eq!( - data_access::read_u16(&buffer, count_range.start, "count", Endian::Little)?, + data_access::read_u16(&buffer, count_range.start(), "count", Endian::Little)?, 1000 ); Ok(()) @@ -184,29 +184,29 @@ fn nested_struct_round_trip_via_offset_map() -> Result<(), AlkTypeError> { let header_magic = offset_map.get("header.magic").expect("header.magic"); let payload_prefix = offset_map.get("payload").expect("payload"); - assert_eq!(header_version.start, 0); - assert_eq!(header_magic.start, 4); - assert_eq!(payload_prefix.start, 8); + assert_eq!(header_version.start(), 0); + assert_eq!(header_magic.start(), 4); + assert_eq!(payload_prefix.start(), 8); let data = b"body-data".to_vec(); let mut buffer = vec![0u8; offset_map.total_size() + data.len()]; data_access::write_u32( &mut buffer, - header_version.start, + header_version.start(), 1, "header.version", Endian::Little, )?; data_access::write_u32( &mut buffer, - header_magic.start, + header_magic.start(), 0xCAFEBABE, "header.magic", Endian::Little, )?; data_access::write_bytes( &mut buffer, - payload_prefix.start, + payload_prefix.start(), &data, "payload", Endian::Little, @@ -215,18 +215,18 @@ fn nested_struct_round_trip_via_offset_map() -> Result<(), AlkTypeError> { assert_eq!( data_access::read_u32( &buffer, - header_version.start, + header_version.start(), "header.version", Endian::Little )?, 1 ); assert_eq!( - data_access::read_u32(&buffer, header_magic.start, "header.magic", Endian::Little)?, + data_access::read_u32(&buffer, header_magic.start(), "header.magic", Endian::Little)?, 0xCAFEBABE ); assert_eq!( - data_access::read_bytes(&buffer, payload_prefix.start, "payload", Endian::Little)?, + data_access::read_bytes(&buffer, payload_prefix.start(), "payload", Endian::Little)?, &data[..] ); Ok(()) @@ -257,9 +257,9 @@ fn nested_struct_round_trip_via_engine_aligned() -> Result<(), AlkTypeError> { let engine = AlkTypeEngine::compile(&doc, "S", LayoutMode::Aligned, None)?; let offset_map = engine.offset_map().expect("aligned mode"); - assert_eq!(offset_map.get("header.version").unwrap().start, 0); - assert_eq!(offset_map.get("header.flags").unwrap().start, 1); - assert_eq!(offset_map.get("payload_len").unwrap().start, 4); + assert_eq!(offset_map.get("header.version").unwrap().start(), 0); + assert_eq!(offset_map.get("header.flags").unwrap().start(), 1); + assert_eq!(offset_map.get("payload_len").unwrap().start(), 4); let mut buffer = vec![0u8; offset_map.total_size()]; engine.write_field(&mut buffer, "header.version", &FieldValue::U8(1))?; @@ -302,23 +302,23 @@ fn big_endian_round_trip_via_offset_map() -> Result<(), AlkTypeError> { let id_range = offset_map.get("id").expect("id"); let offset_range = offset_map.get("offset").expect("offset"); - assert_eq!(id_range.start, 0); - assert_eq!(offset_range.start, 8); + assert_eq!(id_range.start(), 0); + assert_eq!(offset_range.start(), 8); let value: f64 = std::f64::consts::PI; let mut buffer = vec![0u8; offset_map.total_size()]; - data_access::write_u32(&mut buffer, id_range.start, 0x01020304, "id", endian)?; - data_access::write_f64(&mut buffer, offset_range.start, value, "offset", endian)?; + data_access::write_u32(&mut buffer, id_range.start(), 0x01020304, "id", endian)?; + data_access::write_f64(&mut buffer, offset_range.start(), value, "offset", endian)?; assert_eq!(&buffer[0..4], &[0x01, 0x02, 0x03, 0x04]); assert_eq!(&buffer[4..8], &[0x00, 0x00, 0x00, 0x00]); assert_eq!(&buffer[8..16], value.to_be_bytes()); assert_eq!( - data_access::read_u32(&buffer, id_range.start, "id", endian)?, + data_access::read_u32(&buffer, id_range.start(), "id", endian)?, 0x01020304 ); - let read = data_access::read_f64(&buffer, offset_range.start, "offset", endian)?; + let read = data_access::read_f64(&buffer, offset_range.start(), "offset", endian)?; assert!((read - value).abs() < 1e-12); Ok(()) } @@ -342,17 +342,17 @@ fn alignment_padding_round_trip_u8_then_u32() -> Result<(), AlkTypeError> { let flag_range = offset_map.get("flag").expect("flag"); let id_range = offset_map.get("id").expect("id"); - assert_eq!(flag_range.start, 0); - assert_eq!(flag_range.end, 1); - assert_eq!(id_range.start, 4); - assert_eq!(id_range.end, 8); + assert_eq!(flag_range.start(), 0); + assert_eq!(flag_range.end(), 1); + assert_eq!(id_range.start(), 4); + assert_eq!(id_range.end(), 8); assert_eq!(offset_map.total_size(), 8); let mut buffer = vec![0u8; offset_map.total_size()]; - data_access::write_u8(&mut buffer, flag_range.start, 0xAB, "flag")?; + data_access::write_u8(&mut buffer, flag_range.start(), 0xAB, "flag")?; data_access::write_u32( &mut buffer, - id_range.start, + id_range.start(), 0x01020304, "id", Endian::Little, @@ -363,11 +363,11 @@ fn alignment_padding_round_trip_u8_then_u32() -> Result<(), AlkTypeError> { assert_eq!(&buffer[4..8], 0x01020304u32.to_le_bytes()); assert_eq!( - data_access::read_u8(&buffer, flag_range.start, "flag")?, + data_access::read_u8(&buffer, flag_range.start(), "flag")?, 0xAB ); assert_eq!( - data_access::read_u32(&buffer, id_range.start, "id", Endian::Little)?, + data_access::read_u32(&buffer, id_range.start(), "id", Endian::Little)?, 0x01020304 ); Ok(())