Fix offset-indirect, field-level endian, and aligned materialization

Resolves review #003 findings M1, M2, L1, and L2.

- offset-indirect (L1 + M2 arm): rework read_*_indirect to same-buffer
  absolute offsets (safetensors-style, per the metatensor model) and add
  write_*_indirect. Wire the encoding dispatch into engine.read_field/
  write_field and the aligned materializer. Previously the encoding was
  laid out but never read back.
- field-level endian override (M1): thread field.effective_endian through
  sequential_reader, engine.read_field/write_field, and
  materialize_struct_aligned. Previously a per-field endian override was
  silently ignored, misreading multi-byte values.
- aligned-mode materialization (M2): materialize fixed-size arrays via
  their vals[i] offset-map entries and maxLength reservations as
  zero-padded fixed-size slices (trailing NULs trimmed). Previously both
  read garbage or errored.
- dead endian param (L2): drop the ignored endian argument from
  materialize_packed/materialize_aligned; endianness is read from the
  root struct.

Verification: cargo test --release (404 pass), cargo clippy --all-targets
-- -D warnings (clean), cargo build --target wasm32-unknown-unknown
--release (clean), cargo llvm-cov --release (89.60% lines).

src/materialize.rs

@@ -29,7 +29,7 @@ use crate::bast::{
 };
 use crate::data_access;
 use crate::error::AlkTypeError;
-use crate::schema::{AlkTypeKind, Endian};
+use crate::schema::{AlkTypeKind, Endian, VariableEncoding};
 use serde_json::{Map, Value};
 
 const U32_SIZE: usize = 4;
@@ -38,13 +38,11 @@ const DISCRIMINATOR_KEY: &str = "__discriminator";
 /// Materialize a `Value` tree from `buffer` by walking the BAST root type
 /// in packed mode (sequential, from offset 0).
 ///
-/// The root type must be a struct. `endian` is the schema's endianness
-/// (parsed by the caller); a struct-level `endian` annotation overrides
-/// it for the root and all nested structs.
+/// The root type must be a struct. Endianness is read from the root
+/// struct's `endian` annotation (defaults to little-endian).
 pub fn materialize_packed(
     doc: &BastDoc<'_>,
     buffer: &[u8],
-    endian: Endian,
 ) -> Result<Value, AlkTypeError> {
     let root_def = doc.root_def();
     let struct_node = match root_def.kind() {
@@ -57,7 +55,6 @@ pub fn materialize_packed(
         }
     };
     let effective_endian = struct_node.endian();
-    let _ = endian;
     materialize_struct_packed(doc, struct_node, "", effective_endian, buffer, &mut 0)
 }
 
@@ -65,12 +62,12 @@ pub fn materialize_packed(
 /// in aligned mode (offsets from `offset_map`).
 ///
 /// The root type must be a struct. `offset_map` must have been computed
-/// from the same BAST document. `endian` is the schema's endianness.
+/// from the same BAST document. Endianness is read from the root struct's
+/// `endian` annotation (defaults to little-endian).
 pub fn materialize_aligned(
     doc: &BastDoc<'_>,
     buffer: &[u8],
     offset_map: &crate::offset_map::OffsetMap,
-    endian: Endian,
 ) -> Result<Value, AlkTypeError> {
     let root_def = doc.root_def();
     let struct_node = match root_def.kind() {
@@ -83,7 +80,6 @@ pub fn materialize_aligned(
         }
     };
     let effective_endian = struct_node.endian();
-    let _ = endian;
     materialize_struct_aligned(doc, struct_node, "", offset_map, effective_endian, buffer)
 }
 
@@ -456,8 +452,8 @@ fn number_from_f64(f: f64) -> Value {
 
 /// Aligned-mode materialization: walk the BAST typed tree, but read
 /// leaves at offsets from the `OffsetMap` and fall back to packed-style
-/// sequential walks for composites (Array/Union/Record) whose internal
-/// layout can't be precomputed into the static offset map.
+/// sequential walks for composites (Record) whose internal layout can't
+/// be precomputed into the static offset map.
 fn materialize_struct_aligned(
     doc: &BastDoc<'_>,
     struct_node: &BastStruct<'_>,
@@ -476,21 +472,30 @@ fn materialize_struct_aligned(
         let ty = field.ty();
         let resolved = doc.resolve_typeref(ty)?;
         let kind = resolved.alk_kind();
-        let value = if kind.is_fixed_size() || kind.is_variable_length() {
+        let field_endian = field.effective_endian(endian);
+        let value = if kind.is_fixed_size() {
             let range = offset_map.get(&path).ok_or_else(|| AlkTypeError::Offset {
                 field_path: path.clone(),
                 reason: "field not found in offset map".to_string(),
             })?;
-            materialize_leaf_at(doc, &resolved, &path, range.start, endian, buffer)?
+            materialize_leaf_at(doc, &resolved, &path, range.start, field_endian, buffer)?
         } else if kind == AlkTypeKind::Struct {
             if let BastType::Struct(s) = &resolved {
-                materialize_struct_aligned(doc, s, &path, offset_map, endian, buffer)?
+                materialize_struct_aligned(doc, s, &path, offset_map, field_endian, buffer)?
             } else {
                 return Err(AlkTypeError::Schema(format!(
                     "materialize: resolved kind Struct but type was {resolved} at {path}"
                 )));
             }
-        } else {
+        } else if kind == AlkTypeKind::Array {
+            if let BastType::Array(a) = &resolved {
+                materialize_array_aligned(doc, a, &path, offset_map, field_endian, buffer)?
+            } else {
+                return Err(AlkTypeError::Schema(format!(
+                    "materialize: resolved kind Array but type was {resolved} at {path}"
+                )));
+            }
+        } else if kind == AlkTypeKind::Record {
             let range = offset_map.get(&path).ok_or_else(|| AlkTypeError::Offset {
                 field_path: path.clone(),
                 reason: "field not found in offset map".to_string(),
@@ -501,16 +506,124 @@ fn materialize_struct_aligned(
                 &resolved,
                 field,
                 &path,
-                endian,
+                field_endian,
                 buffer,
                 &mut offset,
             )?
+        } else if kind.is_variable_length() {
+            let range = offset_map.get(&path).ok_or_else(|| AlkTypeError::Offset {
+                field_path: path.clone(),
+                reason: "field not found in offset map".to_string(),
+            })?;
+            materialize_variable_aligned(doc, field, &resolved, &path, range, field_endian, buffer)?
+        } else {
+            return Err(AlkTypeError::Schema(format!(
+                "materialize: unsupported kind {kind} at {path} in aligned mode"
+            )));
         };
         obj.insert(field.name().to_string(), value);
     }
     Ok(Value::Object(obj))
 }
 
+/// Materialize a fixed-size array in aligned mode by reading each element
+/// at its `vals[i]` offset-map entry. Array elements are always fixed-size
+/// in aligned mode (the offset map rejects variable-length elements).
+fn materialize_array_aligned(
+    doc: &BastDoc<'_>,
+    array: &BastArray<'_>,
+    field_path: &str,
+    offset_map: &crate::offset_map::OffsetMap,
+    endian: Endian,
+    buffer: &[u8],
+) -> Result<Value, AlkTypeError> {
+    let element_ty = array.element();
+    let resolved_elem = doc.resolve_typeref(element_ty)?;
+    let count = array.count();
+    let mut arr = Vec::with_capacity(count);
+    for i in 0..count {
+        let elem_path = format!("{field_path}[{i}]");
+        let range = offset_map.get(&elem_path).ok_or_else(|| AlkTypeError::Offset {
+            field_path: elem_path.clone(),
+            reason: "array element not found in offset map".to_string(),
+        })?;
+        let value = materialize_leaf_at(doc, &resolved_elem, &elem_path, range.start, endian, buffer)?;
+        arr.push(value);
+    }
+    Ok(Value::Array(arr))
+}
+
+/// Materialize a variable-length leaf (String/Bytes/Timestamp) in aligned
+/// mode, dispatching on the field's encoding strategy:
+///
+/// - `offset-indirect`: an 8-byte `{offset, length}` pair at `start`,
+///   pointing absolutely into the same buffer.
+/// - `maxLength` reservation: `end - start` bytes of zero-padded data;
+///   trailing NUL bytes are trimmed.
+/// - inline length-prefixed (default): a 4-byte length prefix at `start`.
+fn materialize_variable_aligned(
+    doc: &BastDoc<'_>,
+    field: &BastField<'_>,
+    ty: &BastType<'_>,
+    field_path: &str,
+    range: &crate::offset_map::ByteRange,
+    endian: Endian,
+    buffer: &[u8],
+) -> Result<Value, AlkTypeError> {
+    let start = range.start;
+    match field.encoding() {
+        VariableEncoding::OffsetIndirect => match ty {
+            BastType::Primitive(AlkTypeKind::String)
+            | BastType::Primitive(AlkTypeKind::Timestamp) => {
+                let s = data_access::read_string_indirect(buffer, start, field_path, endian)?;
+                Ok(Value::String(s.to_string()))
+            }
+            BastType::Primitive(AlkTypeKind::Bytes) => {
+                let b = data_access::read_bytes_indirect(buffer, start, field_path, endian)?;
+                let arr: Vec<Value> = b.iter().map(|&byte| Value::from(u32::from(byte))).collect();
+                Ok(Value::Array(arr))
+            }
+            _ => Err(AlkTypeError::Schema(format!(
+                "materialize: offset-indirect encoding on non-variable kind at {field_path}"
+            ))),
+        },
+        VariableEncoding::LengthPrefixed => {
+            if let Some(_max_len) = field.max_length() {
+                let data = buffer.get(start..range.end).ok_or_else(|| AlkTypeError::Access {
+                    field_path: field_path.to_string(),
+                    reason: format!("reservation [{start}..{}) unavailable", range.end),
+                })?;
+                let trimmed_len = data
+                    .iter()
+                    .rposition(|&b| b != 0)
+                    .map(|i| i + 1)
+                    .unwrap_or(0);
+                let trimmed = &data[..trimmed_len];
+                match ty {
+                    BastType::Primitive(AlkTypeKind::String)
+                    | BastType::Primitive(AlkTypeKind::Timestamp) => {
+                        let s = std::str::from_utf8(trimmed).map_err(|e| AlkTypeError::Access {
+                            field_path: field_path.to_string(),
+                            reason: format!("invalid UTF-8 in maxLength string: {e}"),
+                        })?;
+                        Ok(Value::String(s.to_string()))
+                    }
+                    BastType::Primitive(AlkTypeKind::Bytes) => {
+                        let arr: Vec<Value> =
+                            trimmed.iter().map(|&byte| Value::from(u32::from(byte))).collect();
+                        Ok(Value::Array(arr))
+                    }
+                    _ => Err(AlkTypeError::Schema(format!(
+                        "materialize: maxLength on non-variable kind at {field_path}"
+                    ))),
+                }
+            } else {
+                materialize_leaf_at(doc, ty, field_path, start, endian, buffer)
+            }
+        }
+    }
+}
+
 fn materialize_leaf_at(
     doc: &BastDoc<'_>,
     ty: &BastType<'_>,
@@ -569,9 +682,8 @@ mod tests {
     fn materialize_packed_strict(
         doc: &BastDoc<'_>,
         buffer: &[u8],
-        endian: Endian,
     ) -> Result<Value, AlkTypeError> {
-        materialize_packed(doc, buffer, endian)
+        materialize_packed(doc, buffer)
     }
 
     #[test]
@@ -590,7 +702,7 @@ mod tests {
         buf[4] = 0x41;
         buf[5] = 0x42;
         buf[6] = 0xC3;
-        let v = materialize_packed_strict(&doc, &buf, Endian::Little).expect("materialize");
+        let v = materialize_packed_strict(&doc, &buf).expect("materialize");
         assert_eq!(v["blob"], json!([65, 66, 195]));
     }
 
@@ -611,7 +723,7 @@ mod tests {
         buf[5] = 0xFE;
         buf[6] = 0x00;
         buf[7] = 0x80;
-        let v = materialize_packed_strict(&doc, &buf, Endian::Little).expect("materialize");
+        let v = materialize_packed_strict(&doc, &buf).expect("materialize");
         assert_eq!(v["raw"], json!([255, 254, 0, 128]));
     }
 
@@ -643,7 +755,7 @@ mod tests {
         buf[off] = b'a';
         off += 1;
         buf[off..off + 4].copy_from_slice(&20u32.to_le_bytes());
-        let v = materialize_packed_strict(&doc, &buf, Endian::Little).expect("materialize");
+        let v = materialize_packed_strict(&doc, &buf).expect("materialize");
         assert_eq!(v["counts"], json!({ "b": 10, "a": 20 }));
     }
 
@@ -662,7 +774,7 @@ mod tests {
         let doc = doc_from(&root, "S");
         let mut buf = vec![0u8; 4];
         buf[0..4].copy_from_slice(&0u32.to_le_bytes());
-        let v = materialize_packed_strict(&doc, &buf, Endian::Little).expect("materialize");
+        let v = materialize_packed_strict(&doc, &buf).expect("materialize");
         assert_eq!(v["counts"], json!({}));
     }
 
@@ -680,7 +792,7 @@ mod tests {
         });
         let doc = doc_from(&root, "S");
         let buf = [0u8; 2];
-        let err = materialize_packed_strict(&doc, &buf, Endian::Little).unwrap_err();
+        let err = materialize_packed_strict(&doc, &buf).unwrap_err();
         assert!(matches!(err, AlkTypeError::Access { .. }), "got {err:?}");
     }
 
@@ -714,7 +826,7 @@ mod tests {
         let mut buf = vec![0u8; 5];
         buf[0] = 5;
         buf[1..5].copy_from_slice(&42u32.to_le_bytes());
-        let v = materialize_packed_strict(&doc, &buf, Endian::Little).expect("materialize");
+        let v = materialize_packed_strict(&doc, &buf).expect("materialize");
         let payload = &v["payload"];
         assert_eq!(payload["__discriminator"], json!(5));
         assert_eq!(payload["id"], json!(42));
@@ -752,7 +864,7 @@ mod tests {
         buf[0..4].copy_from_slice(&4u32.to_le_bytes());
         buf[4..8].copy_from_slice(b"read");
         buf[8..12].copy_from_slice(&7u32.to_le_bytes());
-        let v = materialize_packed_strict(&doc, &buf, Endian::Little).expect("materialize");
+        let v = materialize_packed_strict(&doc, &buf).expect("materialize");
         let payload = &v["payload"];
         assert_eq!(payload["__discriminator"], json!("read"));
         assert_eq!(payload["type"], json!("read"));
@@ -793,7 +905,7 @@ mod tests {
         buf[4..8].copy_from_slice(b"read");
         buf[8..12].copy_from_slice(&7u32.to_le_bytes());
         buf[12..16].copy_from_slice(&99u32.to_le_bytes());
-        let v = materialize_packed_strict(&doc, &buf, Endian::Little).expect("materialize");
+        let v = materialize_packed_strict(&doc, &buf).expect("materialize");
         assert_eq!(v["trailer"], json!(99));
         assert_eq!(v["payload"]["n"], json!(7));
     }
@@ -826,7 +938,7 @@ mod tests {
         });
         let doc = doc_from(&root, "S");
         let buf = [99u8];
-        let err = materialize_packed_strict(&doc, &buf, Endian::Little).unwrap_err();
+        let err = materialize_packed_strict(&doc, &buf).unwrap_err();
         assert!(matches!(err, AlkTypeError::Access { .. }), "got {err:?}");
     }
 
@@ -860,7 +972,7 @@ mod tests {
         let mut buf = vec![0u8; 5];
         buf[0] = 5;
         buf[1..5].copy_from_slice(&42u32.to_le_bytes());
-        let v = materialize_packed_strict(&doc, &buf, Endian::Little).expect("materialize");
+        let v = materialize_packed_strict(&doc, &buf).expect("materialize");
         assert_eq!(v["payload"]["__discriminator"], json!(5));
         assert_eq!(v["payload"]["id"], json!(42));
     }
@@ -897,7 +1009,7 @@ mod tests {
         buf[2..4].copy_from_slice(&2u16.to_le_bytes());
         buf[4..6].copy_from_slice(&3u16.to_le_bytes());
         buf[6..8].copy_from_slice(&4u16.to_le_bytes());
-        let v = materialize_packed_strict(&doc, &buf, Endian::Little).expect("materialize");
+        let v = materialize_packed_strict(&doc, &buf).expect("materialize");
         assert_eq!(v["items"], json!([{ "x": 1, "y": 2 }, { "x": 3, "y": 4 }]));
     }
 
@@ -928,7 +1040,7 @@ mod tests {
         buf[0] = 1;
         buf[1] = 2;
         buf[2..6].copy_from_slice(&0x03030303u32.to_le_bytes());
-        let v = materialize_packed_strict(&doc, &buf, Endian::Little).expect("materialize");
+        let v = materialize_packed_strict(&doc, &buf).expect("materialize");
         assert_eq!(v["header"]["a"], json!(1));
         assert_eq!(v["header"]["b"], json!(2));
         assert_eq!(v["after"], json!(0x03030303u32));
@@ -950,8 +1062,135 @@ mod tests {
         });
         let doc = doc_from(&root, "S");
         let buf = [0u8, 0u8, 0u8, 42u8, 0u8, 0u8, 0u8, 7u8];
-        let v = materialize_packed_strict(&doc, &buf, Endian::Big).expect("materialize");
+        let v = materialize_packed_strict(&doc, &buf).expect("materialize");
         assert_eq!(v["channel_id"], json!(42));
         assert_eq!(v["length"], json!(7));
     }
+
+    // ----- Aligned-mode materialization (M2 fixes) ----------------------
+
+    fn materialize_aligned_strict(
+        doc: &BastDoc<'_>,
+        buffer: &[u8],
+    ) -> Result<Value, AlkTypeError> {
+        let offset_map = crate::offset_map::OffsetMap::compute(doc)?;
+        materialize_aligned(doc, buffer, &offset_map)
+    }
+
+    #[test]
+    fn materialize_aligned_fixed_size_array() {
+        let root = json!({
+            "$defs": {
+                "S": {
+                    "kind": "struct",
+                    "fields": [
+                        { "name": "vals", "kind": { "kind": "array", "element": "uint32", "count": 2 } }
+                    ]
+                }
+            }
+        });
+        let doc = doc_from(&root, "S");
+        let buf = [1u8, 0, 0, 0, 2, 0, 0, 0];
+        let v = materialize_aligned_strict(&doc, &buf).expect("materialize");
+        assert_eq!(v["vals"], json!([1, 2]));
+    }
+
+    #[test]
+    fn materialize_aligned_max_length_string_trims_padding() {
+        let root = json!({
+            "$defs": {
+                "S": {
+                    "kind": "struct",
+                    "fields": [
+                        { "name": "name", "kind": "string", "maxLength": 8 }
+                    ]
+                }
+            }
+        });
+        let doc = doc_from(&root, "S");
+        let mut buf = vec![0u8; 8];
+        buf[..5].copy_from_slice(b"hello");
+        let v = materialize_aligned_strict(&doc, &buf).expect("materialize");
+        assert_eq!(v["name"], json!("hello"));
+    }
+
+    #[test]
+    fn materialize_aligned_max_length_bytes_trims_padding() {
+        let root = json!({
+            "$defs": {
+                "S": {
+                    "kind": "struct",
+                    "fields": [
+                        { "name": "blob", "kind": "bytes", "maxLength": 8 }
+                    ]
+                }
+            }
+        });
+        let doc = doc_from(&root, "S");
+        let mut buf = vec![0u8; 8];
+        buf[..3].copy_from_slice(&[0xAA, 0xBB, 0xCC]);
+        let v = materialize_aligned_strict(&doc, &buf).expect("materialize");
+        assert_eq!(v["blob"], json!([0xAA, 0xBB, 0xCC]));
+    }
+
+    #[test]
+    fn materialize_aligned_offset_indirect_bytes() {
+        let root = json!({
+            "$defs": {
+                "S": {
+                    "kind": "struct",
+                    "fields": [
+                        { "name": "blob", "kind": "bytes", "encoding": "offset-indirect" }
+                    ]
+                }
+            }
+        });
+        let doc = doc_from(&root, "S");
+        let mut buf = vec![0u8; 8 + 3];
+        buf[0..4].copy_from_slice(&8u32.to_le_bytes());
+        buf[4..8].copy_from_slice(&3u32.to_le_bytes());
+        buf[8..11].copy_from_slice(&[0xAA, 0xBB, 0xCC]);
+        let v = materialize_aligned_strict(&doc, &buf).expect("materialize");
+        assert_eq!(v["blob"], json!([0xAA, 0xBB, 0xCC]));
+    }
+
+    #[test]
+    fn materialize_aligned_offset_indirect_string() {
+        let root = json!({
+            "$defs": {
+                "S": {
+                    "kind": "struct",
+                    "fields": [
+                        { "name": "name", "kind": "string", "encoding": "offset-indirect" }
+                    ]
+                }
+            }
+        });
+        let doc = doc_from(&root, "S");
+        let mut buf = vec![0u8; 8 + 5];
+        buf[0..4].copy_from_slice(&8u32.to_le_bytes());
+        buf[4..8].copy_from_slice(&5u32.to_le_bytes());
+        buf[8..13].copy_from_slice(b"hello");
+        let v = materialize_aligned_strict(&doc, &buf).expect("materialize");
+        assert_eq!(v["name"], json!("hello"));
+    }
+
+    #[test]
+    fn materialize_aligned_honors_field_level_endian_override() {
+        let root = json!({
+            "$defs": {
+                "S": {
+                    "kind": "struct",
+                    "endian": "big",
+                    "fields": [
+                        { "name": "crc", "kind": "uint32", "endian": "little" }
+                    ]
+                }
+            }
+        });
+        let doc = doc_from(&root, "S");
+        let buf = [0x04u8, 0x03, 0x02, 0x01];
+        let v = materialize_aligned_strict(&doc, &buf).expect("materialize");
+        assert_eq!(v["crc"], json!(0x01020304u32));
+    }
 }