Fix offset-indirect, field-level endian, and aligned materialization
Resolves review #003 findings M1, M2, L1, and L2. - offset-indirect (L1 + M2 arm): rework read_*_indirect to same-buffer absolute offsets (safetensors-style, per the metatensor model) and add write_*_indirect. Wire the encoding dispatch into engine.read_field/ write_field and the aligned materializer. Previously the encoding was laid out but never read back. - field-level endian override (M1): thread field.effective_endian through sequential_reader, engine.read_field/write_field, and materialize_struct_aligned. Previously a per-field endian override was silently ignored, misreading multi-byte values. - aligned-mode materialization (M2): materialize fixed-size arrays via their vals[i] offset-map entries and maxLength reservations as zero-padded fixed-size slices (trailing NULs trimmed). Previously both read garbage or errored. - dead endian param (L2): drop the ignored endian argument from materialize_packed/materialize_aligned; endianness is read from the root struct. Verification: cargo test --release (404 pass), cargo clippy --all-targets -- -D warnings (clean), cargo build --target wasm32-unknown-unknown --release (clean), cargo llvm-cov --release (89.60% lines).
This commit is contained in:
1 parent
ec73440c19
commit
e5c7cc1ca2
6 files changed
+612
-102
No files matched your search
+183
-19
@@ -23,7 +23,7 @@ use crate::error::AlkTypeError;
|
||||
use crate::layout_builder::LayoutBuilder;
|
||||
use crate::materialize;
|
||||
use crate::offset_map::OffsetMap;
|
||||
use crate::schema::{Endian, AlkTypeKind};
|
||||
use crate::schema::{AlkTypeKind, Endian, VariableEncoding};
|
||||
use crate::sequential_reader::{FieldValue, SequentialReader};
|
||||
use crate::validation;
|
||||
use serde_json::Value;
|
||||
@@ -282,9 +282,9 @@ impl AlkTypeEngine {
|
||||
pub fn validate_bytes(&self, buffer: &[u8]) -> Result<(), AlkTypeError> {
|
||||
let doc = BastDoc::new(&self.bast_doc, &self.root_name)?;
|
||||
let value = match &self.layout {
|
||||
Layout::Packed { .. } => materialize::materialize_packed(&doc, buffer, self.endian)?,
|
||||
Layout::Packed { .. } => materialize::materialize_packed(&doc, buffer)?,
|
||||
Layout::Aligned { offset_map } => {
|
||||
materialize::materialize_aligned(&doc, buffer, offset_map, self.endian)?
|
||||
materialize::materialize_aligned(&doc, buffer, offset_map)?
|
||||
}
|
||||
};
|
||||
bast_validation::validate_value(&doc, &value)
|
||||
@@ -331,11 +331,12 @@ impl AlkTypeEngine {
|
||||
reason: "field not found in offset map".to_string(),
|
||||
})?;
|
||||
let doc = BastDoc::new(&self.bast_doc, &self.root_name)?;
|
||||
let kind = lookup_field_kind(&doc, field_path).ok_or_else(|| AlkTypeError::Offset {
|
||||
let leaf = lookup_leaf_field(&doc, field_path).ok_or_else(|| AlkTypeError::Offset {
|
||||
field_path: field_path.to_string(),
|
||||
reason: "field schema not found in BAST tree or has no primitive kind".to_string(),
|
||||
})?;
|
||||
let endian = self.endian;
|
||||
let kind = leaf.kind;
|
||||
let endian = leaf.endian;
|
||||
match kind {
|
||||
AlkTypeKind::Int8 => {
|
||||
let v = data_access::read_i8(buffer, range.start, field_path)?;
|
||||
@@ -386,11 +387,25 @@ impl AlkTypeEngine {
|
||||
Ok(FieldValue::Enum(v))
|
||||
}
|
||||
AlkTypeKind::String => {
|
||||
let v = data_access::read_string(buffer, range.start, field_path, endian)?;
|
||||
let v = match leaf.encoding {
|
||||
VariableEncoding::OffsetIndirect => {
|
||||
data_access::read_string_indirect(buffer, range.start, field_path, endian)?
|
||||
}
|
||||
VariableEncoding::LengthPrefixed => {
|
||||
data_access::read_string(buffer, range.start, field_path, endian)?
|
||||
}
|
||||
};
|
||||
Ok(FieldValue::String(v))
|
||||
}
|
||||
AlkTypeKind::Bytes => {
|
||||
let v = data_access::read_bytes(buffer, range.start, field_path, endian)?;
|
||||
let v = match leaf.encoding {
|
||||
VariableEncoding::OffsetIndirect => {
|
||||
data_access::read_bytes_indirect(buffer, range.start, field_path, endian)?
|
||||
}
|
||||
VariableEncoding::LengthPrefixed => {
|
||||
data_access::read_bytes(buffer, range.start, field_path, endian)?
|
||||
}
|
||||
};
|
||||
Ok(FieldValue::Bytes(v))
|
||||
}
|
||||
AlkTypeKind::Timestamp => {
|
||||
@@ -452,7 +467,12 @@ impl AlkTypeEngine {
|
||||
field_path: field_path.to_string(),
|
||||
reason: "field not found in offset map".to_string(),
|
||||
})?;
|
||||
let endian = self.endian;
|
||||
let doc = BastDoc::new(&self.bast_doc, &self.root_name)?;
|
||||
let leaf = lookup_leaf_field(&doc, field_path).ok_or_else(|| AlkTypeError::Offset {
|
||||
field_path: field_path.to_string(),
|
||||
reason: "field schema not found in BAST tree or has no primitive kind".to_string(),
|
||||
})?;
|
||||
let endian = leaf.endian;
|
||||
match value {
|
||||
FieldValue::I8(v) => data_access::write_i8(buffer, range.start, *v, field_path),
|
||||
FieldValue::I16(v) => {
|
||||
@@ -485,10 +505,28 @@ impl AlkTypeEngine {
|
||||
data_access::write_enum(buffer, range.start, *v, field_path, endian)
|
||||
}
|
||||
FieldValue::String(v) => {
|
||||
if leaf.encoding == VariableEncoding::OffsetIndirect {
|
||||
return Err(AlkTypeError::Access {
|
||||
field_path: field_path.to_string(),
|
||||
reason: "write_field cannot write offset-indirect fields; \
|
||||
use data_access::write_string_indirect with the \
|
||||
offset map range and a data offset"
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
data_access::write_string(buffer, range.start, v, field_path, endian)?;
|
||||
Ok(())
|
||||
}
|
||||
FieldValue::Bytes(v) => {
|
||||
if leaf.encoding == VariableEncoding::OffsetIndirect {
|
||||
return Err(AlkTypeError::Access {
|
||||
field_path: field_path.to_string(),
|
||||
reason: "write_field cannot write offset-indirect fields; \
|
||||
use data_access::write_bytes_indirect with the \
|
||||
offset map range and a data offset"
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
data_access::write_bytes(buffer, range.start, v, field_path, endian)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -515,31 +553,49 @@ impl fmt::Debug for AlkTypeEngine {
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk the BAST typed tree to find the primitive/composite kind for a
|
||||
/// dotted field path. Returns `None` if any segment is missing or the
|
||||
/// resolved type is a `$ref` that can't be resolved.
|
||||
/// The resolved leaf-field metadata needed by `read_field`/`write_field`:
|
||||
/// the field's kind, its variable-length encoding, and its effective
|
||||
/// endianness (field override, else the enclosing struct's default).
|
||||
struct LeafFieldInfo {
|
||||
kind: AlkTypeKind,
|
||||
encoding: VariableEncoding,
|
||||
endian: Endian,
|
||||
}
|
||||
|
||||
/// Walk the BAST typed tree to find the leaf field for a dotted field
|
||||
/// path. Returns `None` if any segment is missing or the resolved type is
|
||||
/// a `$ref` that can't be resolved.
|
||||
///
|
||||
/// Splits `field_path` on `.` and descends into the root struct's
|
||||
/// `fields` at each step, resolving `$ref`s via [`BastDoc::resolve_typeref`].
|
||||
/// Array element segments (`field[i]`) are not handled here —
|
||||
/// `read_field`/`write_field` only address leaf fields.
|
||||
fn lookup_field_kind(doc: &BastDoc<'_>, field_path: &str) -> Option<AlkTypeKind> {
|
||||
fn lookup_leaf_field(doc: &BastDoc<'_>, field_path: &str) -> Option<LeafFieldInfo> {
|
||||
let root_def = doc.root_def();
|
||||
let mut current_struct: BastStruct<'_> = match root_def.kind() {
|
||||
BastDefKind::Struct(s) => s.clone(),
|
||||
_ => return None,
|
||||
};
|
||||
let mut struct_endian = current_struct.endian();
|
||||
let segments: Vec<&str> = field_path.split('.').collect();
|
||||
let last = segments.len();
|
||||
for (i, segment) in segments.iter().enumerate() {
|
||||
let field = current_struct.fields().iter().find(|f| f.name() == *segment)?;
|
||||
let ty = field.ty();
|
||||
let resolved = doc.resolve_typeref(ty).ok()?;
|
||||
let field_endian = field.effective_endian(struct_endian);
|
||||
if i + 1 == last {
|
||||
return Some(resolved.alk_kind());
|
||||
return Some(LeafFieldInfo {
|
||||
kind: resolved.alk_kind(),
|
||||
encoding: field.encoding(),
|
||||
endian: field_endian,
|
||||
});
|
||||
}
|
||||
match &resolved {
|
||||
BastType::Struct(s) => current_struct = s.clone(),
|
||||
BastType::Struct(s) => {
|
||||
current_struct = s.clone();
|
||||
struct_endian = s.endian();
|
||||
}
|
||||
_ => return None,
|
||||
}
|
||||
}
|
||||
@@ -867,7 +923,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lookup_field_kind_walks_dotted_path() {
|
||||
fn lookup_leaf_field_walks_dotted_path() {
|
||||
let doc = json!({
|
||||
"$defs": {
|
||||
"S": {
|
||||
@@ -888,10 +944,10 @@ mod tests {
|
||||
}
|
||||
});
|
||||
let d = BastDoc::new(&doc, "S").expect("doc");
|
||||
assert_eq!(lookup_field_kind(&d, "header.magic"), Some(AlkTypeKind::Uint32));
|
||||
assert_eq!(lookup_field_kind(&d, "header.version"), Some(AlkTypeKind::Uint8));
|
||||
assert_eq!(lookup_field_kind(&d, "header.missing"), None);
|
||||
assert_eq!(lookup_field_kind(&d, "missing"), None);
|
||||
assert_eq!(lookup_leaf_field(&d, "header.magic").unwrap().kind, AlkTypeKind::Uint32);
|
||||
assert_eq!(lookup_leaf_field(&d, "header.version").unwrap().kind, AlkTypeKind::Uint8);
|
||||
assert!(lookup_leaf_field(&d, "header.missing").is_none());
|
||||
assert!(lookup_leaf_field(&d, "missing").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -980,6 +1036,114 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_field_aligned_honors_field_level_endian_override() {
|
||||
let doc = json!({
|
||||
"$defs": {
|
||||
"S": {
|
||||
"kind": "struct",
|
||||
"endian": "big",
|
||||
"fields": [
|
||||
{ "name": "crc", "kind": "uint32", "endian": "little" }
|
||||
]
|
||||
}
|
||||
}
|
||||
});
|
||||
let engine = AlkTypeEngine::compile(&doc, "S", LayoutMode::Aligned, None).expect("compile");
|
||||
let buf = [0x04u8, 0x03, 0x02, 0x01];
|
||||
assert_eq!(
|
||||
engine.read_field(&buf, "crc").unwrap(),
|
||||
FieldValue::U32(0x01020304)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_field_aligned_honors_field_level_endian_override() {
|
||||
let doc = json!({
|
||||
"$defs": {
|
||||
"S": {
|
||||
"kind": "struct",
|
||||
"endian": "big",
|
||||
"fields": [
|
||||
{ "name": "crc", "kind": "uint32", "endian": "little" }
|
||||
]
|
||||
}
|
||||
}
|
||||
});
|
||||
let engine = AlkTypeEngine::compile(&doc, "S", LayoutMode::Aligned, None).expect("compile");
|
||||
let mut buf = [0u8; 4];
|
||||
engine
|
||||
.write_field(&mut buf, "crc", &FieldValue::U32(0x01020304))
|
||||
.unwrap();
|
||||
assert_eq!(buf, [0x04, 0x03, 0x02, 0x01]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_field_aligned_reads_offset_indirect_bytes() {
|
||||
let doc = json!({
|
||||
"$defs": {
|
||||
"S": {
|
||||
"kind": "struct",
|
||||
"fields": [
|
||||
{ "name": "blob", "kind": "bytes", "encoding": "offset-indirect" }
|
||||
]
|
||||
}
|
||||
}
|
||||
});
|
||||
let engine = AlkTypeEngine::compile(&doc, "S", LayoutMode::Aligned, None).expect("compile");
|
||||
let mut buf = vec![0u8; 8 + 3];
|
||||
buf[0..4].copy_from_slice(&8u32.to_le_bytes());
|
||||
buf[4..8].copy_from_slice(&3u32.to_le_bytes());
|
||||
buf[8..11].copy_from_slice(&[0xAA, 0xBB, 0xCC]);
|
||||
match engine.read_field(&buf, "blob").unwrap() {
|
||||
FieldValue::Bytes(b) => assert_eq!(b, &[0xAA, 0xBB, 0xCC]),
|
||||
other => panic!("expected Bytes, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_field_aligned_reads_offset_indirect_string() {
|
||||
let doc = json!({
|
||||
"$defs": {
|
||||
"S": {
|
||||
"kind": "struct",
|
||||
"fields": [
|
||||
{ "name": "name", "kind": "string", "encoding": "offset-indirect" }
|
||||
]
|
||||
}
|
||||
}
|
||||
});
|
||||
let engine = AlkTypeEngine::compile(&doc, "S", LayoutMode::Aligned, None).expect("compile");
|
||||
let mut buf = vec![0u8; 8 + 5];
|
||||
buf[0..4].copy_from_slice(&8u32.to_le_bytes());
|
||||
buf[4..8].copy_from_slice(&5u32.to_le_bytes());
|
||||
buf[8..13].copy_from_slice(b"hello");
|
||||
match engine.read_field(&buf, "name").unwrap() {
|
||||
FieldValue::String(s) => assert_eq!(s, "hello"),
|
||||
other => panic!("expected String, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_field_aligned_rejects_offset_indirect() {
|
||||
let doc = json!({
|
||||
"$defs": {
|
||||
"S": {
|
||||
"kind": "struct",
|
||||
"fields": [
|
||||
{ "name": "blob", "kind": "bytes", "encoding": "offset-indirect" }
|
||||
]
|
||||
}
|
||||
}
|
||||
});
|
||||
let engine = AlkTypeEngine::compile(&doc, "S", LayoutMode::Aligned, None).expect("compile");
|
||||
let mut buf = vec![0u8; 16];
|
||||
let err = engine
|
||||
.write_field(&mut buf, "blob", &FieldValue::Bytes(&[1, 2, 3]))
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, AlkTypeError::Access { .. }), "got {err:?}");
|
||||
}
|
||||
|
||||
// ----- validate_bytes tests (ADR-010) -----
|
||||
//
|
||||
// Note: step 4 wires the layout layer to BAST. The `validate_bytes`
|
||||
|
||||
Reference in new issue
Block a user