From 2c4a4994dcea95e92d5d6faab8af2bdf2081f166 Mon Sep 17 00:00:00 2001 From: "glm-5.2" Date: Sun, 2 Aug 2026 05:59:12 +0000 Subject: [PATCH] Port alknet-typedef crate from alknet Copy the binary struct engine (src/, tests/) verbatim from alknet/crates/alknet-typedef and create a standalone Cargo.toml (workspace-inherited fields inlined). Port the architecture docs (specs, ADRs 095-102, OQs 069-071) from alknet's nested multi-crate layout to a flat single-crate layout, fixing relative link paths. Build, 295 tests, and clippy all pass clean. --- .gitignore | 3 + Cargo.lock | 949 +++++++++ Cargo.toml | 17 + docs/architecture/README.md | 110 ++ docs/architecture/data-access.md | 385 ++++ ...typedef-purpose-scope-jsonschema-engine.md | 173 ++ .../096-two-layout-modes-packed-vs-aligned.md | 137 ++ .../decisions/097-schema-annotations.md | 259 +++ .../098-error-handling-validation-strategy.md | 157 ++ .../099-int64-uint64-first-class-kinds.md | 114 ++ ...-inline-length-prefixed-in-aligned-mode.md | 112 ++ .../decisions/101-packed-mode-read-factory.md | 103 + .../102-reject-tunion-in-aligned-mode.md | 111 ++ docs/architecture/layout-engine.md | 360 ++++ docs/architecture/open-questions.md | 104 + docs/architecture/overview.md | 203 ++ ...rays-of-variable-length-element-structs.md | 20 + .../questions/070-no-std-alloc-support.md | 22 + ...071-builder-api-for-schema-construction.md | 26 + docs/architecture/schema-layer.md | 506 +++++ docs/architecture/validation.md | 335 ++++ src/data_access.rs | 632 ++++++ src/engine.rs | 754 ++++++++ src/error.rs | 45 + src/layout_builder.rs | 1705 +++++++++++++++++ src/lib.rs | 47 + src/macros.rs | 251 +++ src/offset_map.rs | 873 +++++++++ src/schema.rs | 812 ++++++++ src/sequential_reader.rs | 1543 +++++++++++++++ src/tunion.rs | 645 +++++++ src/validation.rs | 630 ++++++ tests/engine_integration.rs | 389 ++++ tests/error_paths.rs | 415 ++++ tests/poc_roundtrip.rs | 535 ++++++ tests/tunion_dispatch.rs | 323 ++++ 36 files changed, 13805 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 docs/architecture/README.md create mode 100644 docs/architecture/data-access.md create mode 100644 docs/architecture/decisions/095-alknet-typedef-purpose-scope-jsonschema-engine.md create mode 100644 docs/architecture/decisions/096-two-layout-modes-packed-vs-aligned.md create mode 100644 docs/architecture/decisions/097-schema-annotations.md create mode 100644 docs/architecture/decisions/098-error-handling-validation-strategy.md create mode 100644 docs/architecture/decisions/099-int64-uint64-first-class-kinds.md create mode 100644 docs/architecture/decisions/100-reject-non-final-inline-length-prefixed-in-aligned-mode.md create mode 100644 docs/architecture/decisions/101-packed-mode-read-factory.md create mode 100644 docs/architecture/decisions/102-reject-tunion-in-aligned-mode.md create mode 100644 docs/architecture/layout-engine.md create mode 100644 docs/architecture/open-questions.md create mode 100644 docs/architecture/overview.md create mode 100644 docs/architecture/questions/069-arrays-of-variable-length-element-structs.md create mode 100644 docs/architecture/questions/070-no-std-alloc-support.md create mode 100644 docs/architecture/questions/071-builder-api-for-schema-construction.md create mode 100644 docs/architecture/schema-layer.md create mode 100644 docs/architecture/validation.md create mode 100644 src/data_access.rs create mode 100644 src/engine.rs create mode 100644 src/error.rs create mode 100644 src/layout_builder.rs create mode 100644 src/lib.rs create mode 100644 src/macros.rs create mode 100644 src/offset_map.rs create mode 100644 src/schema.rs create mode 100644 src/sequential_reader.rs create mode 100644 src/tunion.rs create mode 100644 src/validation.rs create mode 100644 tests/engine_integration.rs create mode 100644 tests/error_paths.rs create mode 100644 tests/poc_roundtrip.rs create mode 100644 tests/tunion_dispatch.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ed2d55c --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +target/ +node_modules/ +.worktrees/ \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..f3bc1fe --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,949 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alknet-typedef" +version = "0.1.0" +dependencies = [ + "jsonschema", + "serde_json", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "fancy-regex" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "fraction" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872" +dependencies = [ + "lazy_static", + "num", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "jsonschema" +version = "0.46.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a699d3e77675e6aa4bfffe3b907c8b5f7ed3241f9965bffb25475ad4b08d05" +dependencies = [ + "ahash", + "bytecount", + "data-encoding", + "email_address", + "fancy-regex", + "fraction", + "getrandom", + "idna", + "itoa", + "jsonschema-regex", + "num-cmp", + "num-traits", + "percent-encoding", + "referencing", + "regex", + "serde", + "serde_json", + "unicode-general-category", + "uuid-simd", +] + +[[package]] +name = "jsonschema-regex" +version = "0.46.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbd1086b01b9349fd4ef9a07433965af64c8ce8159abe633a189e4ff817bd13" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "micromap" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "referencing" +version = "0.46.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fbf332a2f81899f6836f22c03da73dae8a664c32e3016b84692c23cddadc95d" +dependencies = [ + "ahash", + "fluent-uri", + "getrandom", + "hashbrown 0.16.1", + "itoa", + "micromap", + "parking_lot", + "percent-encoding", + "serde_json", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "indexmap", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..c78ccea --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "alknet-typedef" +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +description = "Binary struct engine: takes a JSON Schema with TypeDef:* custom keywords and produces an offset map, read/write functions, and validation" +repository = "https://git.alk.dev/alkdev/alktype" + +[lib] +name = "alknet_typedef" + +[features] +default = [] + +[dependencies] +jsonschema = { version = "0.46", default-features = false } +serde_json = { version = "1", features = ["preserve_order"] } \ No newline at end of file diff --git a/docs/architecture/README.md b/docs/architecture/README.md new file mode 100644 index 0000000..bb5b3d9 --- /dev/null +++ b/docs/architecture/README.md @@ -0,0 +1,110 @@ +--- +status: draft +last_updated: 2026-07-22 +--- + +# alknet-typedef + +The binary struct engine: a small Rust crate that takes a JSON Schema +with `TypeDef:*` custom keywords and produces an offset map, read/write +functions, and validation — all driven by the schema. The schema is the +format definition; the engine is generic. + +## Documents + +| Document | Status | Description | +|----------|--------|-------------| +| [overview.md](overview.md) | draft | Crate purpose, "schema is the format" principle, dependencies, consumers, scope boundaries | +| [schema-layer.md](schema-layer.md) | draft | The 19 `TypeDef:*` kinds, jsonschema custom keyword integration, TypeBox interop, schema annotations | +| [layout-engine.md](layout-engine.md) | draft | Offset computation, the two layout modes (packed sequential vs aligned static), alignment, endianness, variable-length handling | +| [data-access.md](data-access.md) | draft | Read/write functions, TUnion dispatch, field paths, zero-copy access, length-prefix reading | +| [validation.md](validation.md) | draft | Custom keyword validators for all 19 `TypeDef:*` kinds, `TypedefError`, load-time vs access-time validation, `TypedefEngine` | + +## Applicable ADRs + +| ADR | Title | Relevance | +|-----|-------|-----------| +| [095](decisions/095-alknet-typedef-purpose-scope-jsonschema-engine.md) | Purpose, Scope, and the jsonschema Engine | What the crate is/isn't; why jsonschema not a custom engine; "schema is the format" principle; scope boundaries | +| [096](decisions/096-two-layout-modes-packed-vs-aligned.md) | Two Layout Modes — Packed Sequential vs Aligned Static | The most important architectural finding; when to use each mode; `LayoutBuilder`/`SequentialReader` vs `OffsetMap` | +| [097](decisions/097-schema-annotations.md) | Schema Annotations — Endianness, Alignment, Encoding, TUnion Discriminators | Concrete JSON shapes for all schema-level annotations | +| [098](decisions/098-error-handling-validation-strategy.md) | Error Handling and Validation Strategy | `TypedefError` enum; load-time build, access-time check; field-path-carrying errors | +| [099](decisions/099-int64-uint64-first-class-kinds.md) | Int64/Uint64 as First-Class Kinds | 64-bit integers (SFTP offsets, metatensor data_offsets); JSON precision caveat | +| [100](decisions/100-reject-non-final-inline-length-prefixed-in-aligned-mode.md) | Reject Non-Final Inline Length-Prefixed Variable Fields in Aligned Mode | Prevents silent data corruption (inline variable data clobbering subsequent fields) | +| [101](decisions/101-packed-mode-read-factory.md) | Packed-Mode Read API — Engine as SequentialReader Factory | `engine.sequential_reader()` returns an owned reader, not a reference | +| [102](decisions/102-reject-tunion-in-aligned-mode.md) | Reject TUnion in Aligned Mode for v1 | Unions are the protocol pattern; aligned-mode union semantics were broken | + +## Relevant Open Questions + +| OQ | Title | Status | Relevance | +|----|-------|--------|-----------| +| OQ-069 | Arrays of variable-length-element structs | deferred(scope) | Requires lazy walking logic; blocked on a concrete consumer that needs it | +| OQ-070 | `no_std` + `alloc` support | deferred(scope) | Target `std` for v1; blocked on an embedded use case | +| OQ-071 | Builder API for schema construction | deferred(scope) | Schemas are authored in TypeBox or hand-written JSON for v1; blocked on a concrete need | + +## Key Design Principles + +1. **The schema is the format.** A JSON Schema with `TypeDef:*` custom + keywords is both the validation spec and the layout spec. No separate + format definition, no separate parser, no separate validator. One + schema, three uses: validate, compute offsets, access data. See + [overview.md](overview.md) and [ADR-095](decisions/095-alknet-typedef-purpose-scope-jsonschema-engine.md). + +2. **jsonschema is the validation engine, not a custom engine.** The + `jsonschema` crate (v0.46.5, Draft 2020-12) handles validation with + custom keyword support. The novel code is the offset computation, not + the validation. This eliminates ~14,000 lines of hand-rolled schema + engines (typebox-rs, alktype). See [schema-layer.md](schema-layer.md) + and [ADR-095](decisions/095-alknet-typedef-purpose-scope-jsonschema-engine.md). + +3. **Two layout modes for two use cases.** Packed sequential + (`LayoutBuilder`/`SequentialReader`) for protocol wire formats (SFTP, + channels, TTY). Aligned static (`OffsetMap`) for mmap-friendly formats + (metatensor). The consumer selects the mode; the schema is the same. + See [layout-engine.md](layout-engine.md) and + [ADR-096](decisions/096-two-layout-modes-packed-vs-aligned.md). + +4. **Variable-length types default to inline length-prefixing.** + `[length: u32][data]` is the universal pattern used by channels, SFTP, + TTY, and most binary protocols. Offset indirection (the metatensor + blob tensor pattern) is opt-in via the `encoding` annotation. See + [layout-engine.md](layout-engine.md) and + [ADR-097](decisions/097-schema-annotations.md). + +5. **TUnion supports both byte-offset and field-name discriminators.** + Byte-offset for protocol dispatch (SFTP type bytes, call protocol + event types). Field-name for the typedef.ts string pattern. See + [data-access.md](data-access.md) and + [ADR-097](decisions/097-schema-annotations.md). + +6. **Endianness is per-schema, default little-endian.** The engine reads + the `"endian"` annotation and byte-swaps accordingly. SFTP consumers + specify `"endian": "big"`. See [layout-engine.md](layout-engine.md) + and [ADR-097](decisions/097-schema-annotations.md). + +7. **Validation is opt-in, built once at load time.** The jsonschema + validator is compiled once at schema load time. Access-time validation + is a fast `is_valid()` check. High-throughput paths can skip + validation; security-sensitive paths can validate every frame. See + [validation.md](validation.md) and + [ADR-098](decisions/098-error-handling-validation-strategy.md). + +8. **Not a serialization framework.** The typedef engine is not a + general-purpose serde replacement. It operates on raw byte buffers at + computed offsets — no intermediate `Value` tree, no reflection, no + dynamic dispatch per field. For JSON data, use serde. For binary data + with a known schema, use typedef. See [overview.md](overview.md) and + [ADR-095](decisions/095-alknet-typedef-purpose-scope-jsonschema-engine.md). + +## References + +- `docs/research/alknet-typedef/findings.md` — POC results (26 tests + passing, two layout modes, TUnion dispatch, endianness) +- `docs/research/call-channels-unification/findings.md` §"alknet-typedef: + JSON Schema as the binary struct engine" — the origin of this research + thread +- `/workspace/@alkdev/typebox/example/typedef/typedef.ts` — the TypeBox + schema kinds (619 lines) +- `/workspace/jsonschema/` — the jsonschema crate (v0.46.5, Draft 2020-12) +- `/workspace/alknet-typedef-poc/` — the POC code (disposable) +- `/workspace/@alkimiadev/typebox-rs/` — prior attempt, replaced by typedef +- `/workspace/@alkimiadev/alktype/` — prior attempt, replaced by typedef diff --git a/docs/architecture/data-access.md b/docs/architecture/data-access.md new file mode 100644 index 0000000..be1c30a --- /dev/null +++ b/docs/architecture/data-access.md @@ -0,0 +1,385 @@ +--- +status: draft +last_updated: 2026-07-22 +--- + +# alknet-typedef — Data Access + +The data access layer: read/write functions, TUnion dispatch, field paths, +zero-copy access for fixed-size types, and length-prefix reading for +variable-length types. This is the consumer-facing API — given a compiled +`TypedefEngine` and a byte buffer, read and write fields at +schema-computed offsets. + +This document covers two layers: + +- **Primitive read/write functions** in the `data_access` module — + typed reads/writes at a caller-provided offset. These are the building + blocks used by the layout types (`OffsetMap`, `LayoutBuilder`, + `SequentialReader`) and the `TypedefEngine`. Each operates on a raw + byte buffer at a known offset and returns a `TypedefError::Access` + carrying the field path on bounds or encoding failures. +- **The `FieldValue` enum and the higher-level APIs** — + `TypedefEngine::read_field`/`write_field` (aligned mode) and + `SequentialReader::read_next`/`read_field` (packed mode) — which look + up a field's offset via the layout and dispatch to the primitive + functions, returning a unified `FieldValue<'a>`. + +## The `FieldValue` enum + +The higher-level read APIs return a single unified type — `FieldValue<'a>` +— so one method can read any field kind without the caller dispatching on +schema kind first. The variant carries the typed value; the lifetime +borrows from the input buffer for variable-length kinds (zero-copy). + +```rust +pub enum FieldValue<'a> { + I8(i8), I16(i16), I32(i32), I64(i64), + U8(u8), U16(u16), U32(u32), U64(u64), + F32(f32), F64(f64), + Bool(bool), + Enum(u32), // u32 index into the schema's "enum" array + String(&'a str), // borrows from the buffer + Bytes(&'a [u8]), // borrows from the buffer + Struct { start: usize, end: usize }, // consumer recurses with a fresh reader + Union { discriminator: String, variant_start: usize }, + Array { count: u32, element_start: usize, element_stride: usize }, +} +``` + +For composite kinds (`Struct`, `Union`, `Array`), `FieldValue` returns a +layout descriptor, not the decoded contents — the consumer recurses with +a fresh `SequentialReader` (or a sub-range read) scoped to the reported +byte range. `Array`'s `element_stride` is `0` for variable-length element +types, signalling the consumer must walk each element sequentially. + +## Read/Write Model + +The typedef engine operates on raw byte buffers (`&[u8]` for reading, +`&mut [u8]` for writing). There is no intermediate `Value` tree, no +reflection, no dynamic dispatch per field. The engine uses the offset map +(or `LayoutBuilder`/`SequentialReader`) to locate fields, then performs +typed access at the computed positions. + +### Higher-level read/write + +The `TypedefEngine` and `SequentialReader` provide the primary +consumer-facing read/write APIs. They look up a field's offset via the +layout and dispatch to the primitive `data_access` functions, returning +`FieldValue` (read) or accepting `&FieldValue` (write). + +```rust +impl TypedefEngine { + // Aligned mode: looks up the field's ByteRange in the OffsetMap, + // dispatches to the right data_access function by TypeDefKind. + // Returns TypedefError::Access if compiled in packed mode + // (use sequential_reader() for packed mode). + pub fn read_field<'a>(&self, buffer: &'a [u8], field_path: &str) + -> Result, TypedefError>; + pub fn write_field(&self, buffer: &mut [u8], field_path: &str, + value: &FieldValue<'_>) -> Result<(), TypedefError>; + + // Packed mode: returns an owned fresh SequentialReader (ADR-101). + // Each call returns a new reader with the cursor at position 0. + // The consumer owns the reader and drives read_next/read_field/reset. + pub fn sequential_reader(&self) -> Option; +} + +impl SequentialReader { + // Packed mode: walks the buffer field-by-field, reading length + // prefixes to find each field's position. read_field walks all + // preceding fields to reach the target. + pub fn read_next<'a>(&mut self, buffer: &'a [u8]) + -> Result)>, TypedefError>; + pub fn read_field<'a>(&mut self, buffer: &'a [u8], field_path: &str) + -> Result, TypedefError>; + pub fn reset(&mut self); + pub fn position(&self) -> usize; + pub fn endian(&self) -> Endian; +} +``` + +`read_field`/`write_field` on `TypedefEngine` work for the fixed-size +primitive kinds and the length-prefixed `String`/`Bytes`/`Timestamp` +fields. Composite kinds (`Struct`, `Union`, `Array`, `Record`) return a +`FieldValue` carrying a layout descriptor (byte range, variant start, +or array stride) for the consumer to recurse on — see §"FieldValue" above. + +For writing in packed mode, the consumer uses `LayoutBuilder::build` to +compute positions, then calls the primitive `data_access::write_*` +functions at the computed offsets. There is no packed-mode +`engine.write_field` — the layout depends on the actual data sizes, +which the builder consumes at `build` time. + +### Primitive read/write functions + +The `data_access` module exposes typed read/write functions for each +primitive kind. Each takes `field_path: &str` for error attribution +(produces a `TypedefError::Access` carrying the path on bounds or +encoding failures) and, for multi-byte types, an `Endian` parameter. + +### Fixed-size types + +Fixed-size types (`TFloat32`, `TInt32`, `TUint8`, `TEnum`, etc.) are +accessed via zero-copy reads of N bytes at the offset: + +```rust +// Read a u32 at a known offset, applying endianness. Bounds-checked. +fn read_u32(buffer: &[u8], offset: usize, field_path: &str, endian: Endian) + -> Result { + let bytes: [u8; 4] = read_array(buffer, offset, field_path)?; + Ok(match endian { + Endian::Little => u32::from_le_bytes(bytes), + Endian::Big => u32::from_be_bytes(bytes), + }) +} + +// Write a u32 at a known offset, applying endianness. Bounds-checked. +fn write_u32(buffer: &mut [u8], offset: usize, value: u32, + field_path: &str, endian: Endian) -> Result<(), TypedefError> { + let bytes = match endian { + Endian::Little => value.to_le_bytes(), + Endian::Big => value.to_be_bytes(), + }; + write_array(buffer, offset, bytes, field_path) +} +``` + +The engine applies endianness at access time based on the schema's +`"endian"` annotation (ADR-097). The offset computation is +endian-agnostic. The `read_array`/`write_array` helpers perform the +bounds check and produce `TypedefError::Access` with the field path on +failure. + +### TEnum access + +`TEnum` is a fixed-size type (4 bytes, `u32` index). Read/write delegates +to the `u32` primitives, applying the schema's endianness: + +```rust +pub fn read_enum(buffer: &[u8], offset: usize, field_path: &str, endian: Endian) + -> Result { + read_u32(buffer, offset, field_path, endian) +} +``` + +The consumer maps the `u32` index back to the enum's string values using +the schema's `"enum"` array (index 0 → first value, index 1 → second +value, etc.). The engine does not perform this mapping — it operates on +the raw `u32` index. The jsonschema validator checks that the index +corresponds to a valid enum value at the JSON level. + +### Variable-length types (inline length-prefixing) + +For variable-length types with inline length-prefixing (the default), +the `data_access` module provides `read_string`/`write_string`/ +`read_bytes`/`write_bytes`. Each takes `field_path: &str` for error +attribution and `endian` for the length prefix: + +```rust +// Read a length-prefixed string, borrowing from the buffer. +fn read_string<'a>(buffer: &'a [u8], offset: usize, + field_path: &str, endian: Endian) -> Result<&'a str, TypedefError>; + +// Write a length-prefixed string. Returns total bytes written (4 + data.len()). +fn write_string(buffer: &mut [u8], offset: usize, value: &str, + field_path: &str, endian: Endian) -> Result; + +// read_bytes / write_bytes have the same shape — raw bytes, no UTF-8 check. +``` + +The engine reads the 4-byte length prefix at the field's offset, then +slices the data that follows. For writing, the engine writes the length +prefix + data. `read_string` validates UTF-8 and returns a `&str` +borrowing from the input buffer (zero-copy); `read_bytes` returns a +`&[u8]` slice with no encoding check. + +In packed sequential mode, the `SequentialReader` uses the length prefix +to determine the position of the next field. In aligned static mode, the +`OffsetMap` records the position of the length prefix; the variable data +is accessed separately. + +### Variable-length types (offset indirection) + +For variable-length types with offset indirection (opt-in), the +`data_access` module provides `read_string_indirect`/`read_bytes_indirect`. +The 8-byte struct at `buffer[offset..offset+8]` is +`{ data_offset: u32, data_length: u32 }` (endian-aware); the actual +bytes live in a separate `data_region`: + +```rust +fn read_string_indirect<'a>(buffer: &'a [u8], offset: usize, + data_region: &'a [u8], field_path: &str, + endian: Endian) -> Result<&'a str, TypedefError>; +fn read_bytes_indirect<'a>(buffer: &'a [u8], offset: usize, + data_region: &'a [u8], field_path: &str, + endian: Endian) -> Result<&'a [u8], TypedefError>; +``` + +The field is a struct `{offset: u32, length: u32}` at a known position +in the `OffsetMap`. The consumer provides the data region separately; the +engine reads the offset and length, then slices the data region. + +## TUnion Dispatch + +The `tunion` module provides TUnion discriminator dispatch — reading the +discriminator value from a byte buffer, looking up the variant schema in +the union's `mapping`, and reporting the offset where the variant struct +begins. All reads go through the `data_access` primitives so bounds checks +and endianness handling are uniform with the rest of the engine. + +The result of dispatch is a `UnionDispatch` struct: + +```rust +pub struct UnionDispatch { + pub key: String, // mapping key (stringified disc value) + pub variant_offset: usize, // byte offset where the variant struct starts + pub discriminator_size: usize, // discriminator's byte size +} +``` + +After dispatch, the consumer calls `tunion::resolve_variant(union_schema, &dispatch.key)` +to get the variant schema, then reads the variant's fields at +`dispatch.variant_offset` using the normal `data_access` functions (or a +fresh `SequentialReader` scoped to the variant). + +### Byte-offset discriminator + +```rust +/// Read the discriminator value from a byte-offset TUnion. The discriminator +/// is a fixed-size integer (TypeDef:Uint8/Uint16/Uint32) at a known byte +/// offset. Returns the mapping key (stringified integer) and the variant +/// struct offset. +pub fn read_byte_discriminator( + buffer: &[u8], + union_schema: &Value, + endian: Endian, +) -> Result; +``` + +This is the SFTP `Packet` enum pattern — byte 0 is the type byte, bytes +1..N are the variant struct. The call protocol's 5 event types +(`call.requested` → 0x01, etc.) use the same pattern. The variant struct +starts at `offset + discriminator_size`. + +### Field-name discriminator + +```rust +/// Read the discriminator value from a field-name TUnion. The +/// discriminator is a named field within the struct — the consumer +/// provides the field's computed offset (from the OffsetMap or +/// LayoutBuilder). Supports TypeDef:String, Uint8, and Enum discriminator +/// fields. +pub fn read_field_discriminator( + buffer: &[u8], + union_schema: &Value, + disc_field_offset: usize, + endian: Endian, +) -> Result; +``` + +The discriminator is a named field within the struct. Its offset is +computed like any other field (the consumer passes it in as +`disc_field_offset`). The mapping keys are string values. After reading +the discriminator, the consumer looks up the variant schema and reads +the variant's fields starting at the end of the discriminator field. + +### Variant resolution + +```rust +/// Look up a variant schema from the union's mapping. Inline schemas +/// are returned directly. $ref pointers of the form "#/$defs/" +/// are resolved against the union schema's own $defs block. +pub fn resolve_variant<'a>(union_schema: &'a Value, key: &str) + -> Result<&'a Value, TypedefError>; + +/// Get the discriminator's byte size (1/2/4 for Uint8/16/32) for a +/// byte-offset TUnion. Field-name discriminators have no fixed size +/// and produce a TypedefError::Schema. +pub fn discriminator_size(union_schema: &Value) -> Result; +``` + +### TUnion in the layout engines + +The `LayoutBuilder` and `SequentialReader` also handle TUnion fields +inline during traversal (the consumer does not need to call the `tunion` +functions for a union field reached during a sequential walk). For +`LayoutBuilder`, the consumer supplies the discriminator value (byte-offset) +or variant index (field-name) in `var_sizes` under the synthetic key +`".__discriminator"` or `".__variant"`. For +`SequentialReader`, a union field yields +`FieldValue::Union { discriminator, variant_start }`. The standalone +`tunion` functions are for dispatch outside the layout walk — e.g., a +consumer that receives a bare union buffer and needs to identify the +variant before recursing. + +## Field Paths + +Fields are addressed by dotted paths: `"header.version"`, `"payload.data"`. +Both `OffsetMap` and `PackedLayout` store fully-qualified paths (nested +struct fields appear under their parent's path prefix). The higher-level +APIs (`TypedefEngine::read_field`/`write_field`, `SequentialReader::read_field`) +accept a field path, look up the byte range/position in the layout, and +dispatch to the primitive `data_access` function for the field's kind. + +For aligned-mode access, `TypedefEngine::read_field(&buffer, "header.version")` +returns `FieldValue` — it looks up the `ByteRange` in the `OffsetMap`, finds +the field's `TypeDef:*` kind in the schema, and calls the matching +`data_access::read_*` function. `write_field` is the mirror. Composite +kinds (`Struct`, `Union`, `Array`, `Record`) return a `FieldValue` +carrying a layout descriptor; the consumer recurses with a fresh reader +or sub-range read. + +For packed-mode access, `SequentialReader::read_field(&buffer, "c")` walks +all preceding fields to reach the target (sequential access is inherent +to packed layouts). `read_next` walks fields in declaration order. + +Nested structs produce nested field paths. The offset computation +propagates the field path prefix during recursion, so the `OffsetMap` +and `PackedLayout` contain entries like `"header.version"` and +`"header.magic"`. + +## Zero-Copy Access + +For fixed-size types, the engine provides zero-copy access — the consumer +gets a reference to the bytes in the buffer, not a copy. This is +important for performance-sensitive paths (metatensor tensor access, +high-throughput protocol parsing). + +For variable-length types with inline length-prefixing, the engine +returns a slice of the buffer — the string or byte array data is not +copied. The consumer gets a `&str` or `&[u8]` that borrows from the +input buffer. + +For offset-indirect types, the consumer provides the data region; the +engine returns a slice of that region. + +## Error Handling + +Read/write errors carry the field path for debugging. See +[ADR-098](decisions/098-error-handling-validation-strategy.md) and +[validation.md](validation.md) for the full error model. + +## Design Decisions + +| Decision | ADR | Summary | +|----------|-----|---------| +| Two layout modes | [ADR-096](decisions/096-two-layout-modes-packed-vs-aligned.md) | Determines whether offsets are fixed (OffsetMap) or sequential (SequentialReader) | +| Schema annotations | [ADR-097](decisions/097-schema-annotations.md) | Endianness, encoding, and TUnion discriminator shapes that control data access | +| Error handling | [ADR-098](decisions/098-error-handling-validation-strategy.md) | Field-path-carrying errors for read/write operations | + +## Open Questions + +See [open-questions.md](open-questions.md) for full details. + +- **OQ-069** (deferred(scope)): Arrays of variable-length-element structs + — affects the sequential walking logic for array access. + +## References + +- `docs/research/alknet-typedef/findings.md` §"POC Results" — POC 1 + (read/write round-trip) and POC 2 (SFTP byte-identical round-trip) +- [layout-engine.md](layout-engine.md) — offset computation that produces + the positions this layer reads/writes at +- [validation.md](validation.md) — validation that runs on the same + buffers diff --git a/docs/architecture/decisions/095-alknet-typedef-purpose-scope-jsonschema-engine.md b/docs/architecture/decisions/095-alknet-typedef-purpose-scope-jsonschema-engine.md new file mode 100644 index 0000000..93f1f5e --- /dev/null +++ b/docs/architecture/decisions/095-alknet-typedef-purpose-scope-jsonschema-engine.md @@ -0,0 +1,173 @@ +# ADR-095: alknet-typedef — Purpose, Scope, and the jsonschema Engine + +## Status +Accepted + +## Context + +Three threads in the codebase converge on the same pattern: a JSON Schema +describes the shape of binary data, and the binary data is the struct's +bytes at computed offsets. + +1. **typedef.ts** (`/workspace/@alkdev/typebox/example/typedef/typedef.ts`, + 619 lines) defines custom TypeBox schema kinds (`TFloat32`, `TStruct`, + `TUnion`, etc.) that carry binary layout semantics. These are registered + via `TypeRegistry.Set` with custom validators. + +2. **russh-sftp** has 29 packet types, each a struct with typed fields + (`Read { id: u32, handle: String, offset: u64, len: u32 }`). The wire + format is `[length: u32][type: u8][payload]` where payload is the + struct's serde bytes. The `Packet` enum dispatches on the type byte — + a tagged union of structs. Under the typedef lens, each packet is a + `TStruct`; the `Packet` enum is a `TUnion` with a byte-offset + discriminator. + +3. **metatensor** needs an offset map for mmap-friendly tensor access — + given a schema describing a model layout (ConvNet struct, tensor refs), + compute byte offsets for each field so the consumer can read tensor + data at known positions without parsing. + +The common pattern: **a JSON Schema with `TypeDef:*` custom keywords +describes the shape of binary data; the binary data is the struct's bytes +at computed offsets.** The schema is the format definition; the engine is +generic. + +Two prior attempts built their own jsonschema engines — the fatal flaw: + +- **typebox-rs** (`/workspace/@alkimiadev/typebox-rs/`, ~8,400 lines): + a full 26-variant `SchemaKind` enum, a custom `Value` type with typed + arrays, and a 912-line hand-written validator. +- **alktype** (`/workspace/@alkimiadev/alktype/`, ~5,600 lines): a + handler-registry pattern that also implements its own validation for + each type. + +The `jsonschema` crate (v0.46.5, Draft 2020-12) is already in the +workspace at `/workspace/jsonschema/`. It handles validation with custom +keyword support — the novel code is the offset computation, not the +validation. + +The call-channels-unification research +(`docs/research/call-channels-unification/findings.md` §"alknet-typedef: +JSON Schema as the binary struct engine") identified the convergence and +bumped typedef up in the timeline. The POC +(`docs/research/alknet-typedef/findings.md`, 26 tests passing) validated +the approach: a ~1,900-line Rust crate that takes a JSON Schema with +`TypeDef:*` custom keywords and produces an offset map, read/write +functions, and validation — all driven by the schema. + +## Decision + +**alknet-typedef is a small Rust crate that takes a JSON Schema with +`TypeDef:*` custom keywords and produces three capabilities:** + +1. **An offset map** — walks the schema, computes byte offsets for each + field based on type sizes, field order, and alignment. +2. **Read/write functions** — given a `&[u8]` buffer and a field path, + read the field's bytes at its offset (zero-copy for fixed-size types). + Given a `&mut [u8]` buffer, write a value at its offset. +3. **Validation** — via `jsonschema` custom keywords, validates that data + conforms to the schema's type constraints. The jsonschema validator + operates on `serde_json::Value` instances (JSON representations), not + raw byte buffers directly. A consumer that wants to validate a binary + buffer reads it into a `Value` tree via the data access layer, then + validates that `Value` against the jsonschema validator. + +**The heavy lifting is done by the `jsonschema` crate (validation) and +`serde_json` (schema parsing).** The novel code is the offset computation +— a recursive walk of the schema JSON that computes byte positions for +each field. The custom keyword implementations are ~10 lines each. + +**The schema is the format.** A JSON Schema with `TypeDef:Float32`, +`TypeDef:Struct`, `TypeDef:Union` etc. is both the validation spec and +the layout spec. No separate format definition, no separate parser, no +separate validator. One schema, three uses: validate, compute offsets, +access data. + +**The crate depends on `jsonschema` and `serde_json` (with +`preserve_order`).** No tokio, no platform deps. Compiles to +`wasm32-unknown-unknown` for browser use. The `jsonschema` crate's +`with_keyword("TypeDef:Float32", factory)` API is the integration point +for custom type kinds — each `TypeDef:*` kind maps to a custom keyword +validator in Rust. Same semantics as TypeBox's `TypeRegistry.Set`, same +JSON Schema wire format. + +**The crate targets `std` for v1.** The WASM target has `std` available +via `wasm-bindgen`. If embedded use cases emerge, `no_std` + `alloc` can +be added as a feature gate later — the engine's core (offset computation, +read/write) is already allocation-free. See OQ-070. + +## Consequences + +### Positive + +- **Eliminates ~14,000 lines of hand-rolled schema engines.** typebox-rs + and alktype are replaced by `jsonschema` + an offset map + ~50 lines of + custom keyword implementations. The codebase drops from "a port of + TypeBox" to "jsonschema + an offset map." +- **One schema, three uses.** The same JSON Schema validates, computes + offsets, and drives data access. No separate format definition, parser, + or validator per protocol. +- **Schema-driven, not code-driven.** Adding a new SFTP packet type is + adding a variant to the schema JSON, not writing a new Rust struct + + serde impl. The engine is generic; the schema is the configuration. +- **WASM-clean.** `serde_json` + `jsonschema` + byte manipulation. No + tokio, no platform deps. The same typedef schemas work in browser, + Node, Python (via `wasmtime-py`), Go (via `wazero`), and any other + WASM host. +- **TypeBox interop.** TypeBox modules render to standard JSON Schema + under `$defs`. That JSON feeds directly into `jsonschema::validator_for` + on the Rust side. Zero translation. The same schema validates in both + ecosystems. +- **Defense in depth.** Schema validation via jsonschema custom keywords — + a malformed binary payload can be read into a `Value` tree via the data + access layer and validated against the schema before any consumer + touches it. The `jsonschema` crate's compiled validators are fast enough + to run on every incoming frame. + +### Negative + +- **New dependency on `jsonschema`.** The crate is already in the + workspace but not yet used by any alknet crate. This is the first + consumer. +- **`serde_json` with `preserve_order` is required.** Field order is + load-bearing for binary layouts. The `preserve_order` feature adds a + small compile-time cost. +- **Schema authoring is external.** Schemas are authored in TypeBox (JS) + or hand-written JSON. The typedef engine consumes schemas; it does not + generate them. A builder API is deferred (OQ-071). + +## Scope Boundaries (What This Is Not) + +- **Not metatensor.** typedef is the binary struct *engine*. Metatensor + is a *format* (8-byte header + JSON header + binary data) that uses the + typedef engine for its offset computation and tensor access. +- **Not a Value system.** TypeBox's `Value.Diff`, `Value.Migrate`, + `Value.Convert` — schema evolution — is out of scope for v1. The engine + should not do anything that explicitly blocks adding a Value system + later. +- **Not a code generator.** typebox-rs's `codegen/` module is a separate + concern. The typedef engine consumes schemas; it does not generate them. +- **Not a schema builder.** The typedef engine does not provide a fluent + API for constructing schemas. Schemas are plain JSON. +- **Not a serialization framework.** The typedef engine is not a + general-purpose serde replacement. It operates on raw byte buffers at + computed offsets — no intermediate `Value` tree, no reflection, no + dynamic dispatch per field. For JSON data, use serde. For binary data + with a known schema, use typedef. + +## References + +- `docs/research/alknet-typedef/findings.md` — POC results (26 tests + passing, two layout modes, TUnion dispatch, endianness) +- `docs/research/call-channels-unification/findings.md` §"alknet-typedef: + JSON Schema as the binary struct engine" — the origin of this research + thread +- `/workspace/@alkdev/typebox/example/typedef/typedef.ts` — the TypeBox + schema kinds (619 lines) +- `/workspace/jsonschema/` — the jsonschema crate (v0.46.5, Draft 2020-12) +- `/workspace/alknet-typedef-poc/` — the POC code (disposable) +- [ADR-096](096-two-layout-modes-packed-vs-aligned.md) — the two layout + modes decision +- [ADR-097](097-schema-annotations.md) — schema annotation shapes +- [ADR-098](098-error-handling-validation-strategy.md) — error handling + and validation strategy diff --git a/docs/architecture/decisions/096-two-layout-modes-packed-vs-aligned.md b/docs/architecture/decisions/096-two-layout-modes-packed-vs-aligned.md new file mode 100644 index 0000000..7d1c85c --- /dev/null +++ b/docs/architecture/decisions/096-two-layout-modes-packed-vs-aligned.md @@ -0,0 +1,137 @@ +# ADR-096: Two Layout Modes — Packed Sequential vs Aligned Static + +## Status +Accepted + +## Context + +The POCs surfaced that protocols and mmap-friendly formats need different +layout strategies. POC 1 built an aligned `OffsetMap` with natural +alignment padding — correct for mmap-friendly formats (metatensor) but +wrong for protocol wire formats (SFTP, channels, TTY). POC 2 built a +`LayoutBuilder` and `SequentialReader` for packed sequential layouts — +correct for protocol wire formats but wrong for mmap-friendly formats. + +This is the most important architectural finding from the POCs. The +engine must support both modes; a single layout strategy cannot serve +both use cases. + +### Packed sequential layout (protocol wire formats) + +Protocols pack fields sequentially with no alignment padding. +Variable-length fields shift all subsequent fields. Writing requires +knowing actual data sizes upfront; reading walks the buffer sequentially, +reading length prefixes to determine positions. + +This is the layout used by SFTP (all strings and byte arrays are +length-prefixed inline), channels (`[channel_id: u32][size: u32][payload]`), +TTY (`[stream_type: u8][length: u32][payload]`), and most binary protocols. + +### Aligned static layout (mmap-friendly formats) + +Fields have fixed positions with natural alignment padding. +Variable-length fields get a 4-byte length prefix at a known offset; the +variable data is not included in the static layout. This enables +mmap-friendly random access — the consumer can read field N at a known +offset without parsing the fields before it. + +This is the layout used by metatensor (blob tensor pattern: index struct +in one region, blob data in another) and safetensors (header + aligned +tensor data). + +## Decision + +**The typedef engine supports two layout modes, selected by the consumer +at engine construction time:** + +### Mode 1: Packed sequential (`LayoutBuilder` / `SequentialReader`) + +For protocol wire formats. Fields are packed with no alignment padding. +Variable-length fields shift all subsequent fields. + +- **LayoutBuilder** — takes a schema and actual data sizes for + variable-length fields, computes byte positions for each field in a + packed layout. Used at write time when the consumer knows the data + sizes upfront. +- **SequentialReader** — walks a buffer field-by-field according to the + schema, reading length prefixes to determine variable-length data + positions. Used at read time when the consumer is parsing an incoming + frame. + +The `LayoutBuilder` and `SequentialReader` are the primary interface for +protocol consumers (SFTP, binary call frames, TTY negotiation). + +### Mode 2: Aligned static (`OffsetMap`) + +For mmap-friendly formats. Fields have fixed positions with natural +alignment padding. Variable-length fields get a 4-byte length prefix at +a known offset; the variable data is not included in the static layout. + +- **OffsetMap** — walks the schema once, computes fixed byte positions + for each field based on type sizes and alignment. The output is a flat + table of `(field_path, byte_range)` pairs. Used for both read and write + at known offsets. + +The `OffsetMap` is the primary interface for mmap consumers (metatensor). + +### Variable-length handling in each mode + +**Packed sequential mode:** Variable-length fields are inline +length-prefixed by default (`[length: u32][data]`). The `LayoutBuilder` +takes the actual data size to compute the length prefix value and the +position of subsequent fields. The `SequentialReader` reads the length +prefix to determine the data extent and the position of the next field. + +**Aligned static mode:** Variable-length fields get a 4-byte length +prefix at a known offset. The variable data lives outside the static +layout — either immediately after the fixed fields (inline +length-prefixing) or in a separate data region (offset indirection, the +metatensor blob tensor pattern). The `OffsetMap` records the position of +the length prefix (or the `{offset, length}` pair for offset-indirect +fields). + +### Default for variable-length types + +Inline length-prefixing (`[length: u32][data]`) is the default for all +variable-length types in both modes. This is the universal pattern used +by channels, SFTP, TTY, and most binary protocols. Offset indirection is +opt-in via the `encoding` annotation (see ADR-097). + +## Consequences + +### Positive + +- **One engine, two modes.** The same schema can be used in either mode. + A schema describing an SFTP packet can be consumed by a `SequentialReader` + (for parsing incoming frames) and a `LayoutBuilder` (for constructing + outgoing frames). A schema describing a metatensor layout can be + consumed by an `OffsetMap` (for mmap access). +- **Correct for both use cases.** Packed sequential mode produces + byte-identical output to hand-written protocol serialization (validated + by POC 2's russh-sftp round-trip tests). Aligned static mode produces + correct offsets for mmap-friendly access (validated by POC 1's + alignment tests). +- **No mode confusion.** The consumer explicitly selects the mode at + engine construction time. A protocol consumer never accidentally gets + alignment padding; an mmap consumer never accidentally gets + variable-length field shifting. + +### Negative + +- **Two APIs to learn.** Consumers must choose between + `LayoutBuilder`/`SequentialReader` and `OffsetMap`. The choice is + determined by the use case (protocol vs mmap), not by the schema. +- **Variable-length fields in packed mode require size foreknowledge.** + The `LayoutBuilder` needs actual data sizes for variable-length fields + to compute correct positions for subsequent fields. This is inherent + to packed layouts — the consumer must know the data sizes before + writing. + +## References + +- `docs/research/alknet-typedef/findings.md` §"POC Results" — POC 1 + (aligned OffsetMap) and POC 2 (packed LayoutBuilder/SequentialReader) +- [ADR-095](095-alknet-typedef-purpose-scope-jsonschema-engine.md) — + purpose and scope +- [ADR-097](097-schema-annotations.md) — schema annotations including + the `encoding` field for variable-length types diff --git a/docs/architecture/decisions/097-schema-annotations.md b/docs/architecture/decisions/097-schema-annotations.md new file mode 100644 index 0000000..b9eb2ce --- /dev/null +++ b/docs/architecture/decisions/097-schema-annotations.md @@ -0,0 +1,259 @@ +# ADR-097: Schema Annotations — Endianness, Alignment, Encoding, and TUnion Discriminators + +## Status +Accepted + +## Context + +The typedef engine needs concrete JSON shapes for schema-level +annotations that control binary layout behavior. The POCs validated the +semantics; this ADR pins the shapes. + +Four annotation categories need concrete shapes: + +1. **Endianness** — safetensors is little-endian, SFTP is big-endian. + The engine needs to know which to use. +2. **Alignment** — different backends have different alignment + requirements (wgpu: 256-byte, protocols: natural, mmap: page). +3. **Variable-length encoding** — inline length-prefixing vs offset + indirection for strings, byte arrays, and other variable-length types. +4. **TUnion discriminators** — byte-offset (protocol dispatch) vs + field-name (typedef.ts pattern). + +## Decision + +### 1. Endianness + +**Schema-level annotation with a default of little-endian.** + +```json +{ + "TypeDef:Struct": true, + "endian": "big", + "properties": { ... } +} +``` + +- `"endian": "little"` (default) — read/write in little-endian byte order. +- `"endian": "big"` — read/write in big-endian byte order. +- The annotation applies to the entire schema and all nested types. +- Mixed endianness within one schema is not supported (pathological; no + known protocol requires it). +- The default is little-endian, matching safetensors, wgpu, and most + modern formats. SFTP consumers specify `"endian": "big"`. + +### 2. Alignment + +**Both struct-level and field-level, with field-level overriding +struct-level.** + +```json +{ + "TypeDef:Struct": true, + "align": 256, + "properties": { + "header": { "TypeDef:Struct": true, "properties": { ... } }, + "weight": { "TypeDef:Float32": true, "align": 16 } + } +} +``` + +- Struct-level `"align"` sets the default alignment for all fields in + that struct. The struct's total size is rounded up to this alignment. +- Field-level `"align"` overrides the struct default for that specific + field. +- Default alignment (when no annotation is present): 1 for u8/bool, 2 + for u16/i16, 4 for u32/i32/f32, 8 for u64/i64/f64, max field alignment + for structs. +- Alignment is only meaningful in aligned static mode (ADR-096). In + packed sequential mode, alignment annotations are ignored — fields are + packed with no padding. + +### 3. Variable-length encoding + +**Three strategies for variable-length types, selected by the `encoding` +annotation and the standard JSON Schema `maxLength` keyword.** + +```json +// Strategy 1: Inline length-prefixing (default, shorthand) +{ "TypeDef:String": true } + +// Strategy 1: Explicit inline length-prefixing +{ "TypeDef:String": { "encoding": "length-prefixed" } } + +// Strategy 2: Fixed-size reservation (uses standard maxLength) +{ "TypeDef:String": true, "maxLength": 256 } + +// Strategy 3: Offset indirection (opt-in) +{ "TypeDef:String": { "encoding": "offset-indirect" } } +``` + +**Strategy 1: Inline length-prefixing (default).** The field's fixed +portion is a 4-byte length prefix at a computed offset. The variable data +follows immediately after. In packed sequential mode, the length prefix +determines the position of subsequent fields. In aligned static mode, the +length prefix is at a known offset; the variable data is not included in +the static layout. This is the universal pattern used by channels, SFTP, +TTY, and most binary protocols. + +**Strategy 2: Fixed-size reservation.** When a variable-length field +declares `maxLength` (a standard JSON Schema keyword), the engine reserves +`maxLength` bytes at a fixed offset in aligned static mode. Data shorter +than `maxLength` is zero-padded; data longer than `maxLength` is a +validation error. This makes the field fixed-size from the layout +perspective — subsequent fields have known, unchanging offsets. This is +the database `VARCHAR(N)` pattern and the metatensor struct-tensor +pattern for fields with known maximum sizes. In packed sequential mode, +`maxLength` is a validation constraint only — the engine still uses +inline length-prefixing (strategy 1). + +**Strategy 3: Offset indirection.** The field is a struct +`{offset: u32, length: u32}` that points into a separate data region. +This is the metatensor blob tensor pattern — the index struct lives in +one region, the blob data lives in another. The consumer provides the +data region separately. Enables mmap-friendly random access to +variable-length data without parsing length prefixes and without +reserving worst-case space. + +**Default strategy selection:** +- In packed sequential mode: always strategy 1 (inline length-prefixing). + `maxLength` is a validation constraint only. +- In aligned static mode: strategy 2 (fixed-size reservation) if + `maxLength` is declared; strategy 3 (offset indirection) if + `"encoding": "offset-indirect"` is declared; strategy 1 (inline + length-prefixing) otherwise. + +- `true` is a shorthand for the default (length-prefixed). This keeps + the common case concise and the override explicit. +- The `encoding` annotation and `maxLength` apply to all variable-length + types: `TypeDef:String`, `TypeDef:Bytes`, `TypeDef:Array`, + `TypeDef:Record`, `TypeDef:Timestamp`. + +### 3a. TRecord value type + +`TypeDef:Record` is a string-keyed map. The value type is declared via +the `"values"` property in the schema: + +```json +{ + "TypeDef:Record": true, + "values": { "TypeDef:Float32": true } +} +``` + +- `"values"` is a schema object declaring the `TypeDef:*` kind of all + values in the record. All values share the same type. +- The binary layout is a count-prefixed sequence of `(key, value)` pairs: + `[count: u32][key_len: u32][key_bytes][value]...` repeated `count` + times. Each key is a length-prefixed UTF-8 string. Each value is + encoded according to its declared `TypeDef:*` kind — a `Record` + value is 4 raw bytes; a `Record` value is itself a + length-prefixed string; a `Record` value is the struct's + fields laid out inline. There is **no separate `value_len` prefix** — + the value's size is determined by its kind (fixed-size kinds have a + known size; variable-length kinds carry their own length prefix). +- The count and key-length prefixes respect the schema's endianness. +- In aligned static mode with `maxLength`, the entire record is reserved + at `maxLength` bytes (zero-padded). + +### 4. TUnion discriminators + +**Two discriminator kinds: byte-offset (protocol dispatch) and +field-name (typedef.ts pattern).** + +#### Kind A: Byte-offset discriminator + +```json +{ + "TypeDef:Union": true, + "discriminator": { + "kind": "byte", + "offset": 0, + "type": "TypeDef:Uint8" + }, + "mapping": { + "1": { "$ref": "#/$defs/Init" }, + "3": { "$ref": "#/$defs/Open" }, + "5": { "$ref": "#/$defs/Read" }, + "6": { "$ref": "#/$defs/Write" }, + "101": { "$ref": "#/$defs/Status" } + } +} +``` + +- The discriminator is a fixed-size integer at a known byte offset. +- `"offset"` is the byte position of the discriminator within the union's + buffer. +- `"type"` is the `TypeDef:*` kind of the discriminator (typically + `TypeDef:Uint8` for protocol type bytes). +- The mapping keys are stringified integers (`"1"`, `"5"`, `"101"`). + The engine parses the key to match the discriminator value. +- The variant struct starts at `offset + discriminator_size`. +- This is the SFTP `Packet` enum pattern and the call protocol's event + type dispatch. + +#### Kind B: Field-name discriminator + +```json +{ + "TypeDef:Union": true, + "discriminator": { + "kind": "field", + "name": "type" + }, + "mapping": { + "read": { "$ref": "#/$defs/Read" }, + "write": { "$ref": "#/$defs/Write" } + } +} +``` + +- The discriminator is a named field within the struct. +- `"name"` is the field name that holds the discriminator value. +- The mapping keys are string values matching the discriminator field's + value. +- The discriminator field is just another field in the struct — its + offset is computed like any other field. +- This is the typedef.ts `TUnion` pattern. + +#### Mapping values + +Mapping values may be either inline schemas or `$ref` pointers. `$ref` +is cleaner for large unions (29 SFTP variants) but requires a `$defs` +section. Inline schemas are simpler for small unions (5 call protocol +event types). Both work. + +## Consequences + +### Positive + +- **Concrete, validated shapes.** All four annotation categories have + concrete JSON shapes that were validated by the POCs. +- **Sensible defaults.** Little-endian, natural alignment, inline + length-prefixing — the common case requires no annotations. +- **Explicit overrides.** Big-endian, custom alignment, offset + indirection — the uncommon case is explicit and self-documenting. +- **TUnion covers both protocol and typedef.ts patterns.** The + byte-offset discriminator handles SFTP type bytes and call protocol + event types. The field-name discriminator handles the typedef.ts string + pattern. No separate union type needed. + +### Negative + +- **Keyword value shape change.** `"TypeDef:String": true` (boolean) and + `"TypeDef:String": { "encoding": "length-prefixed" }` (object) are both + valid. The engine must handle both shapes. This is a minor parsing + concern — the POC already handles it. +- **Alignment annotations are mode-specific.** Alignment is only + meaningful in aligned static mode. In packed sequential mode, alignment + annotations are ignored. This is documented, not enforced — a consumer + that specifies alignment in packed mode gets no error, just no effect. + +## References + +- `docs/research/alknet-typedef/findings.md` §"Open Questions" — the + annotation shape questions this ADR resolves +- [ADR-095](095-alknet-typedef-purpose-scope-jsonschema-engine.md) — + purpose and scope +- [ADR-096](096-two-layout-modes-packed-vs-aligned.md) — the two layout + modes (alignment only meaningful in aligned static mode) diff --git a/docs/architecture/decisions/098-error-handling-validation-strategy.md b/docs/architecture/decisions/098-error-handling-validation-strategy.md new file mode 100644 index 0000000..0e3f35c --- /dev/null +++ b/docs/architecture/decisions/098-error-handling-validation-strategy.md @@ -0,0 +1,157 @@ +# ADR-098: Error Handling and Validation Strategy + +## Status +Accepted + +## Context + +The typedef engine operates in three phases, each with distinct error +conditions: + +1. **Schema parsing** — invalid JSON, missing required keywords, unknown + `TypeDef:*` kinds, malformed annotations. +2. **Offset computation** — field not found, type not supported for + offset computation, recursive schema depth exceeded. +3. **Read/write** — buffer too short, invalid UTF-8, value out of range + for the target type. +4. **Validation** — type constraint violations (range, UTF-8, field + presence, discriminator membership). + +The engine also needs a clear strategy for *when* validation happens: +once at schema load time (build the validator) vs repeatedly at access +time (validate each buffer). + +## Decision + +### Error type: `TypedefError` + +A single `TypedefError` enum with variants for each error category: + +```rust +pub enum TypedefError { + /// Schema parsing errors. + Schema(String), + /// Offset computation errors. + Offset { field_path: String, reason: String }, + /// Read/write errors. + Access { field_path: String, reason: String }, + /// Validation errors (delegated to jsonschema). + Validation(ValidationError<'static>), +} +``` + +- `Schema` — for invalid JSON, missing required keywords, unknown + `TypeDef:*` kinds. The error message describes the problem. +- `Offset` — for field-not-found, unsupported type for offset + computation, etc. Carries the field path for debugging. +- `Access` — for buffer-too-short, invalid UTF-8, value out of range. + Carries the field path for debugging. +- `Validation` — wraps `jsonschema`'s `ValidationError`. The + `jsonschema` crate already provides rich error messages with schema + paths; the typedef engine does not re-wrap or re-interpret them. + +The `Validation` variant uses `ValidationError<'static>` because the +validator is built once at schema load time and lives for the lifetime +of the `TypedefEngine`. The `'static` lifetime is correct — the validator +owns its schema reference. + +### Validation timing: load-time build, access-time check + +The jsonschema validator is built once at schema load time +(`validator_for(&schema)?`) and then called repeatedly +(`validator.is_valid(&instance)`). The typedef engine follows the same +pattern: + +1. **Load time:** Parse the schema JSON, build the offset map (or + `LayoutBuilder`/`SequentialReader`), build the jsonschema validator. + This is the `TypedefEngine::compile(schema: &Value) -> Result` constructor. +2. **Access time:** Use the compiled engine for repeated read/write + operations. Validation is opt-in per operation — the consumer calls + `engine.validate(buffer)` when validation is desired. + +The `TypedefEngine` struct is the compiled form of a schema: + +```rust +pub struct TypedefEngine { + offset_map: OffsetMap, // or LayoutBuilder/SequentialReader + validator: jsonschema::Validator, // compiled once at load time +} +``` + +### Custom keyword validators + +Each `TypeDef:*` kind gets a `Keyword` implementation registered via +`jsonschema::options().with_keyword(...)`. The validators check: + +- **Numeric types** (`TypeDef:Float32`, `TypeDef:Int8`, etc.): range + constraints (Int8: -128..127, Uint8: 0..255, etc.), finiteness for + floats. +- **`TypeDef:String`**: UTF-8 validity. +- **`TypeDef:Struct`**: field presence and types (delegated to + jsonschema's structural validation — the custom keyword only needs to + validate that the struct's fields match their declared `TypeDef:*` + kinds). +- **`TypeDef:Union`**: discriminator value membership in the mapping. +- **`TypeDef:Array`**: element type conformance. +- **`TypeDef:Boolean`**: value is `true` or `false`. +- **`TypeDef:Timestamp`**: RFC 3339 string format (the internet profile of ISO 8601). + +The `jsonschema` crate handles all the structural validation (object +properties, required fields, array items, enum values) — the custom +keywords only need to validate the leaf type constraints. Each custom +keyword implementation is ~10 lines. + +### Read/write errors carry field paths + +Read/write errors include the field path for debugging: + +```rust +// Example: reading a u32 from a buffer that's too short +Err(TypedefError::Access { + field_path: "header.version".to_string(), + reason: "buffer too short: need 4 bytes at offset 12, have 2".to_string(), +}) +``` + +This makes debugging binary format issues tractable — the error tells +you exactly which field failed and why. + +## Consequences + +### Positive + +- **Single error type.** Consumers handle one `TypedefError` enum, not + multiple error types from different engine phases. +- **Field-path-carrying errors.** Read/write errors include the field + path, making binary format debugging tractable. +- **Validation is opt-in.** The consumer decides when to validate. + High-throughput paths can skip validation; security-sensitive paths + can validate every frame. +- **jsonschema integration is clean.** The `ValidationError` is wrapped + as-is — no re-interpretation, no information loss. +- **Load-time build, access-time use.** The expensive work (schema + parsing, validator compilation, offset computation) happens once at + load time. Access-time operations are cheap (pointer casts, slice + operations, length-prefix reads). + +### Negative + +- **`ValidationError<'static>` lifetime.** The `'static` lifetime on the + `Validation` variant means the error cannot borrow from the buffer + being validated. This is correct (the validator owns its schema + reference) but may surprise readers who expect a shorter lifetime. +- **No error recovery.** The engine does not attempt to recover from + partial reads or writes. A buffer-too-short error on field N means + fields N+1.. are also unreadable. This is inherent to binary formats + — there is no "skip to next field" without a schema-driven parser. + +## References + +- `docs/research/alknet-typedef/findings.md` §"Open Questions" — error + handling strategy question (OQ 8) +- [ADR-095](095-alknet-typedef-purpose-scope-jsonschema-engine.md) — + purpose and scope +- [ADR-096](096-two-layout-modes-packed-vs-aligned.md) — the two layout + modes +- [ADR-097](097-schema-annotations.md) — schema annotations diff --git a/docs/architecture/decisions/099-int64-uint64-first-class-kinds.md b/docs/architecture/decisions/099-int64-uint64-first-class-kinds.md new file mode 100644 index 0000000..75d789c --- /dev/null +++ b/docs/architecture/decisions/099-int64-uint64-first-class-kinds.md @@ -0,0 +1,114 @@ +# ADR-099: Int64/Uint64 as First-Class Kinds + +## Status +Accepted + +## Context + +The typedef engine's kind set (ADR-095, ADR-097) tops out at 32-bit +integers. The POC included `u64` read/write primitives, and the +call-channels-unification research's own SFTP schema example uses +`"TypeDef:Uint64"` for the `offset` field (`Read`/`Write` packets have +`offset: u64`). Metatensor/safetensors `data_offsets` are also `u64`. + +A `TypeDef:Uint64` variant was added to the `TypeDefKind` enum during +implementation (the task decomposition correctly identified the gap), +but without an ADR the addition was half-finished: `type_size()` +returned `None`, the layout engines couldn't compute offsets for it, and +the validator didn't register a `TypeDef:Uint64` keyword. The variant +was then removed (commit `14d9cf2`) on the grounds that it was +unintended and a latent panic — but the underlying gap is real: SFTP and +metatensor, the two primary POC targets, both require 64-bit integers. + +The presumed reason 64-bit integers were left out of the original +specification is a JSON-level concern: `serde_json::Number` loses +precision past 2^53 when parsing from JSON text. This is a +*validation-layer* caveat, not a *layout-layer* one — the binary layout +is 8 raw bytes, and `from_le_bytes`/`from_be_bytes` work correctly for +the full `u64`/`i64` range. The validation concern is handled by +accepting integer-form JSON values (the `jsonschema` crate's +`as_i64`/`as_u64` methods handle the common range; values past 2^53 are +a JSON representation limitation, not a typedef limitation). + +## Decision + +**Add `TypeDef:Int64` and `TypeDef:Uint64` as first-class kinds.** + +Both are fixed-size (8 bytes), with natural alignment 8. They follow +the schema's endianness annotation like all other fixed-size types. +Read/write is via `data_access::read_i64`/`write_i64`/`read_u64`/ +`write_u64` (endian-aware, 8 bytes). + +### Kind table additions + +| Kind | TypeBox key | Rust type | Size | Alignment | +|------|-------------|-----------|------|-----------| +| `TInt64` | `TypeDef:Int64` | `i64` | 8 | 8 | +| `TUint64` | `TypeDef:Uint64` | `u64` | 8 | 8 | + +### Validation + +The custom keyword validators check: +- `TypeDef:Int64`: value must be an integer in `i64::MIN..=i64::MAX` + (`-9223372036854775808` to `9223372036854775807`). +- `TypeDef:Uint64`: value must be a non-negative integer in + `0..=u64::MAX` (`0` to `18446744073709551615`). + +The `jsonschema` crate's `as_i64`/`as_u64` handle the common range. +JSON numbers past 2^53 lose precision in the JSON representation — +this is a JSON limitation, not a typedef limitation. The binary +representation (8 raw bytes) is always exact. A consumer that needs +to validate the full 64-bit range from JSON should provide the value +as a JSON integer (which `serde_json` preserves for values up to +`u64::MAX`/`i64::MIN` when the `arbitrary_precision` feature is +enabled, or when the value fits in `i64`/`u64` without the feature). + +### `FieldValue` additions + +`FieldValue::I64(i64)` and `FieldValue::U64(u64)` are added to the +unified return type. The `SequentialReader`, `TypedefEngine::read_field`, +and `TypedefEngine::write_field` dispatch on the new kinds. + +### Kind count + +The engine now has **19** first-class kinds (17 + Int64 + Uint64). +`TypeDefKind::is_fixed_size()` returns `true` for both new kinds. +`type_size()` returns `Some(8)`. `natural_alignment()` returns `8`. +`needs_endian()` returns `true`. + +## Consequences + +### Positive + +- **Unblocks the two primary POC targets.** SFTP `Read`/`Write` packets + (`offset: u64`) and metatensor `data_offsets` (`u64`) are now + expressible in typedef schemas. +- **Completes the half-finished addition.** The `TypeDefKind` enum, + `data_access` primitives, and `FieldValue` variants for 64-bit + integers now have matching layout, validator, and engine support. +- **No new design surface.** Int64/Uint64 are fixed-size types that + follow all existing patterns (endianness, alignment, zero-copy + read/write). They are mechanical additions. + +### Negative + +- **JSON precision caveat.** Values past 2^53 lose precision in the + JSON representation (not in the binary representation). This is a + JSON limitation, not a typedef limitation, but it means the + validation layer cannot perfectly round-trip the full 64-bit range + through JSON `Number` without `arbitrary_precision`. In practice, + SFTP offsets and tensor data offsets are well within 2^53. +- **Two more kinds to maintain.** The kind table, validator + registration, `FieldValue` enum, and dispatch arms all grow by two + variants. This is the cost of completeness. + +## References + +- `docs/research/call-channels-unification/findings.md` §"russh-sftp" — + the SFTP schema with `"offset": { "TypeDef:Uint64": true }` +- `docs/research/alknet-typedef/findings.md` §"POC 1" — the POC included + u64 read/write +- [ADR-095](095-alknet-typedef-purpose-scope-jsonschema-engine.md) — + purpose and scope (the kind set) +- [ADR-097](097-schema-annotations.md) — schema annotations + (endianness applies to the new kinds) \ No newline at end of file diff --git a/docs/architecture/decisions/100-reject-non-final-inline-length-prefixed-in-aligned-mode.md b/docs/architecture/decisions/100-reject-non-final-inline-length-prefixed-in-aligned-mode.md new file mode 100644 index 0000000..41d5e9b --- /dev/null +++ b/docs/architecture/decisions/100-reject-non-final-inline-length-prefixed-in-aligned-mode.md @@ -0,0 +1,112 @@ +# ADR-100: Reject Non-Final Inline Length-Prefixed Variable Fields in Aligned Mode + +## Status +Accepted + +## Context + +The aligned static layout mode (ADR-096) is designed for mmap-friendly +formats: fields have fixed positions with natural alignment padding, +enabling random access by field path without parsing preceding fields. + +The spec (layout-engine.md) says variable-length fields in aligned mode +get a 4-byte length prefix at a known offset, and "the variable data +lives outside the static layout — either immediately after the fixed +fields (inline length-prefixing) or in a separate data region (offset +indirection)." + +The implementation has a bug: `OffsetMap::compute` reserves only 4 bytes +for an inline length-prefixed variable field (the length prefix), but +`TypedefEngine::write_field` for a `String`/`Bytes` field calls +`data_access::write_string` at `range.start`, which writes +`[4-byte length][data]` inline — clobbering every subsequent field. The +`read_field` path has the mirror behavior (reads inline), so the engine +is self-consistent but only works correctly when the variable field is +the last field in the struct (no subsequent field to clobber). + +Concretely, `{name: String, id: Uint32}` in aligned mode maps +`name → 0..4`, `id → 4..8`. Writing `"hello"` to `name` writes +`[5,0,0,0,h,e,l,l,o]` at offset 0, overwriting `id`'s range with +`hello`. All existing tests happen to put the variable field last, so +the bug is latent. + +The spec's "data region after fixed fields" model (where variable data +lives after all fixed fields) is the correct design for aligned mode, +but implementing it would require a two-region layout (fixed fields + +variable data region) with the `OffsetMap` tracking both the prefix +position and the data position. This is a significant design addition +for a use case that doesn't exist yet — real aligned-format consumers +(metatensor, safetensors) use `maxLength` reservation or +`offset-indirect` encoding for variable data, not inline +length-prefixing. + +## Decision + +**Reject non-final inline length-prefixed variable fields in aligned +static mode at `OffsetMap::compute` time.** + +A variable-length field (`TypeDef:String`, `TypeDef:Bytes`, +`TypeDef:Timestamp`, `TypeDef:Record`) in aligned static mode that uses +the default inline length-prefixing strategy (no `maxLength`, no +`offset-indirect`) must be the last field in its struct. If a non-final +inline length-prefixed variable field is encountered, +`OffsetMap::compute` returns `TypedefError::Offset` with a message +explaining that non-final variable fields in aligned mode require +`maxLength` (fixed-size reservation) or `"encoding": "offset-indirect"` +(offset indirection). + +This is a validation-time rejection (schema load time), not a runtime +check. The consumer learns about the problem when compiling the schema, +not when writing data. + +### What is NOT rejected + +- Inline length-prefixed variable fields that are the last field in + their struct — these are fine (no subsequent field to clobber). +- `maxLength` reservation and `offset-indirect` encoding in any + position — these make the field fixed-size from the layout + perspective (known size at a known offset), so they don't clobber. +- Inline length-prefixed variable fields in packed sequential mode — + packed mode doesn't have fixed offsets; variable fields shift + subsequent fields by design. + +## Consequences + +### Positive + +- **Eliminates a silent data-corruption bug.** A consumer that writes + a non-final string in aligned mode currently clobbers subsequent + fields with no error. After this fix, the schema is rejected at + compile time. +- **Matches real aligned-format usage.** mmap-friendly formats use + `maxLength` or `offset-indirect` for variable data; inline + length-prefixing in aligned mode is only meaningful as the last + field. +- **Simple to implement.** A single check in `compute_struct` (is this + variable field non-final and using inline length-prefixing? → reject). + No two-region layout needed. +- **Defers the two-region design without blocking consumers.** If a + future consumer needs inline length-prefixing in non-final position + in aligned mode, the two-region layout can be implemented then. The + rejection is reversible (remove the check, add the two-region logic). + +### Negative + +- **A schema that worked before (silently corrupting data) now fails + at compile time.** This is the correct behavior — the schema was + always broken, it just wasn't caught. +- **The "data region after fixed fields" model from the spec is not + implemented.** A consumer that wants inline variable data in a + non-final position must use packed mode or wait for the two-region + layout. This is acceptable for v1 — no current consumer needs it. + +## References + +- [ADR-096](096-two-layout-modes-packed-vs-aligned.md) — the two layout + modes (aligned static mode's variable-length handling) +- [ADR-097](097-schema-annotations.md) — the three variable-length + encoding strategies (`maxLength`, `offset-indirect`, inline + length-prefixing) +- `../layout-engine.md` §"Variable-length + fields in aligned mode" — the spec's "data region after fixed fields" + description \ No newline at end of file diff --git a/docs/architecture/decisions/101-packed-mode-read-factory.md b/docs/architecture/decisions/101-packed-mode-read-factory.md new file mode 100644 index 0000000..bed66ff --- /dev/null +++ b/docs/architecture/decisions/101-packed-mode-read-factory.md @@ -0,0 +1,103 @@ +# ADR-101: Packed-Mode Read API — Engine as SequentialReader Factory + +## Status +Accepted + +## Context + +`TypedefEngine` stores a `SequentialReader` inside its `Layout::Packed` +variant. The engine exposes it via +`engine.sequential_reader() -> Option<&SequentialReader>`. + +The problem: `SequentialReader`'s read methods (`read_next`, +`read_field`, `reset`) all take `&mut self` — they mutate the reader's +internal cursor (`field_index`, `position`). But the engine hands out +`&SequentialReader` (a shared reference), which cannot be used to call +`&mut self` methods. The accessor can only give the consumer +`position()` and `endian()` (the `&self` methods) — the actual read +API is unreachable. + +This makes the engine's packed read-side dead API. A consumer that +wants to read a packed buffer must construct their own +`SequentialReader::new(&schema)` from the schema, bypassing the engine +entirely. The stored reader is dead weight. + +Three options were considered: +1. **Factory method** — the engine provides a method that returns an + owned fresh `SequentialReader` (reconstructed from the stored + schema). The consumer owns the reader and drives it with `&mut self`. +2. **Interior mutability** — wrap the reader in `Mutex` or `RefCell` + so `&SequentialReader` can be upgraded to `&mut`. Adds overhead and + complexity for mutable cursor state that the consumer legitimately + wants to own. +3. **`sequential_reader_mut()`** — return `&mut SequentialReader`. + Requires `&mut self` on the engine, which is overly restrictive + (the consumer may share the engine across threads or hold it behind + an `Arc`). + +## Decision + +**The engine is a `SequentialReader` factory.** Replace +`sequential_reader() -> Option<&SequentialReader>` with +`sequential_reader() -> Option` — the method returns +an owned fresh reader, reconstructed from the stored schema. + +```rust +impl TypedefEngine { + /// Construct a fresh SequentialReader for packed-mode reads. + /// Returns None if compiled in aligned mode. + pub fn sequential_reader(&self) -> Option; +} +``` + +Each call returns a new reader with the cursor at position 0. The +consumer owns the reader and calls `read_next`/`read_field`/`reset` on +it directly. The engine still stores its own reader (used for schema +validation during construction), but no longer exposes it by +reference. + +The same applies to `LayoutBuilder`: `layout_builder()` returns +`Option<&LayoutBuilder>` which is fine — `LayoutBuilder::build` takes +`&self`, so the shared reference is usable. No change needed for the +write-side. + +### Cost + +`SequentialReader::new` clones the top-level struct's field schemas (a +`Vec<(String, Value)>` of the `properties` entries) and clones the +schema itself. This is cheap — a struct has a small number of fields +(SFTP's largest packet has 5). The construction cost is negligible +compared to the cost of reading a buffer. + +## Consequences + +### Positive + +- **The packed read API is now usable.** A consumer calls + `engine.sequential_reader()` to get an owned reader and drives it + directly. No dead API. +- **No interior mutability overhead.** The reader's mutable cursor + state is owned by the consumer, not shared through a lock. +- **Thread-safe engine.** The engine remains `Send + Sync` (it only + exposes `&self` methods). The reader is owned by the calling thread. +- **Simple.** One method signature change. The stored reader in + `Layout::Packed` can be removed (it was only used for schema + validation during construction, which is done by the time the + consumer calls `sequential_reader()`). + +### Negative + +- **Each call to `sequential_reader()` allocates a new reader.** The + cost is a `Vec` of field schemas + a schema clone. Acceptable for + the use case (one reader per buffer read). +- **The engine no longer holds a live reader.** If a future use case + needs to share a reader's cursor state across calls, the consumer + must manage that themselves. This is the correct separation — cursor + state is consumer-owned, not engine-owned. + +## References + +- [ADR-096](096-two-layout-modes-packed-vs-aligned.md) — packed + sequential mode (`SequentialReader` as the read-side) +- `../data-access.md` §"Higher-level + read/write" — the `SequentialReader` API \ No newline at end of file diff --git a/docs/architecture/decisions/102-reject-tunion-in-aligned-mode.md b/docs/architecture/decisions/102-reject-tunion-in-aligned-mode.md new file mode 100644 index 0000000..0d084c8 --- /dev/null +++ b/docs/architecture/decisions/102-reject-tunion-in-aligned-mode.md @@ -0,0 +1,111 @@ +# ADR-102: Reject TUnion in Aligned Mode for v1 + +## Status +Accepted + +## Context + +The aligned static layout mode (ADR-096) computes fixed byte positions +for each field, enabling random access by field path. `TUnion` in +aligned mode has three implementation problems: + +1. **No variant field offsets.** Only the `__discriminator` byte range + is recorded in the `OffsetMap`. Variant field offsets are not + available anywhere in aligned mode — the consumer must recompute + them by hand. This makes `TypedefEngine::read_field` on a union + variant field impossible. + +2. **`find_discriminator_field` takes the first variant's offset.** For + a field-name discriminator, the code probes the first variant that + contains the discriminator field and records that offset globally. + If variants order fields differently, the discriminator sits at + different offsets per variant and the recorded range is silently + wrong. The code should validate that the offset is identical across + all variants (or require the discriminator field to be first). + +3. **Byte-discriminator union total misaligns the variant.** The union + total is `disc_off + disc_size + variant_max_size`, but the variant + was probed from offset 0 with alignment. A `u8` discriminator + before a `u32`-bearing variant produces a variant region that + starts at an unaligned offset in a mode whose entire purpose is + alignment. + +The real question is whether `TUnion` in aligned mode is even needed. +The two consumer profiles are: + +- **Protocol consumers** (SFTP, call protocol event types): use packed + sequential mode. `TUnion` with byte-offset discriminators is the + core dispatch mechanism. This is well-supported. +- **mmap consumers** (metatensor, safetensors): use aligned static + mode. These formats are structs and arrays of structs — they don't + use tagged unions. A tensor file has a header struct with tensor + descriptors, not a "which variant is this?" dispatch. + +`TUnion` in aligned mode is a combination that no current or planned +consumer needs. Shipping broken semantics for an unused use case is +worse than rejecting it clearly. + +## Decision + +**Reject `TUnion` in aligned static mode for v1.** + +`OffsetMap::compute` returns `TypedefError::Offset` when it encounters +a `TypeDef:Union` field, with a message explaining that unions are not +supported in aligned mode and the consumer should use packed mode (or +restructure as a struct with an explicit discriminator field). + +This is a schema-load-time rejection. The consumer learns about the +problem when compiling the schema, not at runtime. + +### What is NOT rejected + +- `TUnion` in packed sequential mode — this is the core use case + (SFTP `Packet` dispatch, call protocol event types) and is fully + supported by `LayoutBuilder` and `SequentialReader`. +- `TStruct`, `TArray`, and all primitive kinds in aligned mode — these + are the mmap-format primitives and are fully supported. + +### Reversal + +This is a two-way door. If a future mmap-format consumer needs tagged +unions in aligned mode, the rejection can be lifted and the three +implementation problems fixed. The fix would require: +- Recording per-variant field offsets in the `OffsetMap` (which + variant's offsets to record when variants have different layouts?). +- Validating that field-name discriminators have identical offsets + across all variants. +- Aligning the variant region correctly after the byte discriminator. + +These are design questions that should be answered when the use case +arrives, not speculatively now. + +## Consequences + +### Positive + +- **No broken semantics shipped.** The three implementation problems + are removed from the API surface rather than silently producing + wrong offsets. +- **Clear scope boundary.** Aligned mode is for structs and arrays; + packed mode is for protocols (including union dispatch). The + consumer chooses the mode based on the use case. +- **Reversible.** When a real consumer needs aligned-mode unions, the + rejection is lifted and the design questions are worked through with + a concrete use case. + +### Negative + +- **A schema with a `TUnion` field cannot be compiled in aligned + mode.** A consumer that wants both aligned layout and union dispatch + must use packed mode or restructure. No current consumer needs this. +- **The aligned-mode union code in `offset_map.rs` is dead.** It can + be removed or left as a reference for when the rejection is lifted. + Removing it is cleaner. + +## References + +- [ADR-096](096-two-layout-modes-packed-vs-aligned.md) — the two layout + modes +- [ADR-097](097-schema-annotations.md) §4 — TUnion discriminators +- `../layout-engine.md` §"TUnion" — + aligned-mode union sizing \ No newline at end of file diff --git a/docs/architecture/layout-engine.md b/docs/architecture/layout-engine.md new file mode 100644 index 0000000..4855361 --- /dev/null +++ b/docs/architecture/layout-engine.md @@ -0,0 +1,360 @@ +--- +status: draft +last_updated: 2026-07-22 +--- + +# alknet-typedef — Layout Engine + +The layout engine: offset computation, the two layout modes (packed +sequential vs aligned static), alignment, endianness, and variable-length +field handling. This is the novel code — the recursive walk of the schema +JSON that computes byte positions for each field. + +## The Two Layout Modes + +The POCs surfaced that protocols and mmap-friendly formats need different +layout strategies. This is the most important architectural finding — +decided in [ADR-096](decisions/096-two-layout-modes-packed-vs-aligned.md). + +### Mode 1: Packed sequential (protocol wire formats) + +Fields are packed with no alignment padding. Variable-length fields shift +all subsequent fields. Used by SFTP, channels, TTY, and most binary +protocols. + +**Components:** + +- **`LayoutBuilder`** — constructed via `LayoutBuilder::new(schema)` (requires `TypeDef:Struct` at the top level), then `builder.build(&var_sizes) -> Result` where `var_sizes: &HashMap` maps variable-length field paths (and TUnion discriminator/variant keys) to their actual byte sizes. Used at write time when the consumer knows the data sizes upfront. The builder computes positions only; the consumer writes data via the [`data_access`](data-access.md) functions at the computed positions. +- **`SequentialReader`** — constructed via `SequentialReader::new(schema)`, then driven by `reader.read_next(&buffer) -> Result, TypedefError>` until `Ok(None)`, or `reader.read_field(&buffer, path)` to seek a single field (which walks all preceding fields to reach the target). `reader.reset()` rewinds to the start. Used at read time when the consumer is parsing an incoming frame. + +**How it works:** + +For a struct with fields `[u8, u32, string]` where the string is 10 bytes: + +``` +LayoutBuilder::build(var_sizes: {"payload": 10}): + field[0] u8: offset 0, size 1 + field[1] u32: offset 1, size 4 + field[2] string: offset 5, size 4 (length prefix) + 10 (data) + total: 19 + +SequentialReader::read_next (read): + read u8 at offset 0 + read u32 at offset 1 + read u32 length prefix at offset 5 → data_len + read string data at offset 9, length data_len + next field at offset 9 + data_len +``` + +There is no alignment padding. The `u32` at offset 1 is unaligned — this +is correct for protocol wire formats, which pack fields tightly. + +**Variable-length fields in packed mode:** + +The `LayoutBuilder` takes actual data sizes for variable-length fields +to compute correct positions for subsequent fields. The consumer must +know the data sizes before writing — this is inherent to packed layouts. + +The `SequentialReader` reads each field's length prefix to determine the +data extent and the position of the next field. The reader walks the +buffer sequentially; it cannot jump to field N without reading fields +0..N-1 first. + +### Mode 2: Aligned static (mmap-friendly formats) + +Fields have fixed positions with natural alignment padding. +Variable-length fields get a 4-byte length prefix at a known offset; the +variable data is not included in the static layout. Used by metatensor +and safetensors. + +**Component:** + +- **`OffsetMap`** — constructed via `OffsetMap::compute(schema) -> Result` (requires `TypeDef:Struct` at the top level). Walks the schema once, computes fixed byte positions for each field based on type sizes and alignment. The output is a flat table of `(field_path, byte_range)` pairs (see [Public Types](#public-types)). Used for both read and write at known offsets. + +**How it works:** + +For a struct with fields `[u8, u32, f32]` and natural alignment: + +``` +OffsetMap: + field[0] u8: offset 0, size 1 + field[1] u32: offset 4, size 4 (3 bytes padding after u8) + field[2] f32: offset 8, size 4 + total: 12 (struct aligned to 4) +``` + +The `u32` is aligned to offset 4 (its natural alignment). The consumer +can read `field[1]` at offset 4 without reading `field[0]` first — random +access by field path. + +**Variable-length fields in aligned mode:** + +Variable-length fields get a 4-byte length prefix at a known offset. The +variable data lives outside the static layout — either immediately after +the fixed fields (inline length-prefixing) or in a separate data region +(offset indirection). The `OffsetMap` records the position of the length +prefix (or the `{offset, length}` pair for offset-indirect fields). + +For inline length-prefixing, the variable data follows the fixed fields +but is not included in the `OffsetMap`'s field ranges. The consumer reads +the length prefix from the `OffsetMap`'s known offset, then slices the +data region. + +For offset indirection, the field is a struct `{offset: u32, length: u32}` +at a known position in the `OffsetMap`. The consumer reads the offset and +length, then slices the separate data region. + +### Inline length-prefixing in aligned mode — non-final field restriction + +Inline length-prefixed variable fields in aligned mode are only allowed +as the **last field** in their struct. A non-final inline +length-prefixed variable field is rejected at `OffsetMap::compute` time +with a `TypedefError::Offset` — the `OffsetMap` reserves only 4 bytes +(the length prefix), but `data_access::write_string` writes prefix + +data inline, which would clobber subsequent fields. Non-final variable +fields must use `maxLength` (fixed-size reservation) or +`"encoding": "offset-indirect"`. See +[ADR-100](decisions/100-reject-non-final-inline-length-prefixed-in-aligned-mode.md). + +## Offset Computation Algorithm + +The offset computation is a recursive walk of the schema JSON. The +algorithm is the same for both modes; the difference is whether alignment +padding is inserted between fields. + +### Fixed-size types + +For each fixed-size type, the algorithm: +1. Determines the type's byte size from the `TypeDef:*` kind. +2. In aligned mode: inserts padding to satisfy the type's alignment + (or the field's `align` annotation, or the struct's `align` default). +3. Records the field's `(start, end)` range. +4. Advances the current offset by the type's size. + +### Composite types + +**`TStruct`:** Recurse into the struct's `properties`. The inner fields +are computed relative to the struct's start offset. The struct's total +size is the sum of its fields' sizes (plus alignment padding in aligned +mode). The struct itself may have an `align` annotation that rounds up +its total size. + +**`TUnion`:** TUnion is supported in packed sequential mode only. In +aligned static mode, `OffsetMap::compute` rejects `TUnion` fields with +`TypedefError::Offset` — see +[ADR-102](decisions/102-reject-tunion-in-aligned-mode.md). Unions +are the protocol dispatch pattern (SFTP type bytes, call protocol event +types); mmap-friendly formats use structs and arrays, not tagged unions. + +In packed sequential mode, the discriminator occupies +`offset..offset + discriminator_size` bytes. For byte-offset +discriminators, the variant struct starts at `offset + discriminator_size`. +For field-name discriminators, the discriminator is just another field — +its offset is computed like any other field, and the variant struct +follows at the end of the discriminator field. + +Variant sizes depend on the actual sizes of variable-length fields within +each variant, which aren't known at schema time. The `LayoutBuilder` +takes the actual variant discriminator value and data sizes at write time, +computes the size of the selected variant, and uses that for the union's +total size. The `SequentialReader` reads the discriminator first, looks +up the variant schema, then reads the variant struct sequentially — it +doesn't need to know the union's total size upfront. + +**`TArray` of fixed-size elements:** Element stride = element size (plus +alignment padding in aligned mode). Element `i` starts at +`array_offset + i × stride`. The array's total size is `count × stride`. + +**`TArray` of variable-length-element structs:** Deferred for v1 +(OQ-069). + +### Variable-length types + +The typedef engine supports three strategies for variable-length types +(see [schema-layer.md](schema-layer.md) §Variable-length types and +[ADR-097](decisions/097-schema-annotations.md) §3 for the full +annotation shapes). + +**Strategy 1: Inline length-prefixing (default).** +1. Records the position of the 4-byte length prefix. +2. In aligned mode: the length prefix is aligned; the variable data is + not included in the static layout. +3. In packed mode: the `LayoutBuilder` takes the actual data size to + compute the length prefix value and the position of subsequent fields. + The `SequentialReader` reads the length prefix to determine the data + extent and the position of the next field. + +**Strategy 2: Fixed-size reservation (`maxLength`).** +1. In aligned static mode: reserves `maxLength` bytes at a fixed offset. + Data shorter than `maxLength` is zero-padded. Subsequent fields have + known, unchanging offsets — the field is fixed-size from the layout + perspective. This is the database `VARCHAR(N)` pattern. +2. In packed sequential mode: `maxLength` is a validation constraint + only. The engine uses strategy 1 (inline length-prefixing) because + protocols don't benefit from fixed-size reservation. + +**Strategy 3: Offset indirection (`"encoding": "offset-indirect"`).** +1. The field is a struct `{offset: u32, length: u32}`. +2. The `OffsetMap` records the position of this struct. +3. The consumer provides the data region separately. This is the + metatensor blob tensor pattern — the index struct lives in one region, + the blob data lives in another. + +### Nested structs and field paths + +Nested structs produce dotted field paths: `header.version`, +`header.magic`. The offset computation propagates the field path prefix +during recursion. Both `OffsetMap` and `PackedLayout` store fully-qualified +paths; the `iter()` method of each yields fields in schema `properties` +order, with nested struct fields appearing inline under their parent's +path prefix. + +### Endianness + +Endianness is per-schema (ADR-097). The offset computation is +endian-agnostic — it computes byte positions, not byte values. The +read/write functions apply endianness when converting between bytes and +typed values. The engine reads the `"endian"` annotation from the schema +and byte-swaps accordingly. All fixed-size types — including `TEnum` +(u32 index) — follow the schema's endianness. + +## Mode Selection + +The consumer selects the mode at engine construction time via the +`LayoutMode` enum, passed to `TypedefEngine::compile`: + +```rust +pub enum LayoutMode { + /// Packed sequential — for protocol wire formats (SFTP, channels, TTY). + Packed, + /// Aligned static — for mmap-friendly formats (metatensor, safetensors). + Aligned, +} +``` + +The choice is determined by the use case, not by the schema: + +- **Protocol consumer** (SFTP, binary call frames, TTY negotiation): + `LayoutMode::Packed` → uses `LayoutBuilder` for writing and + `SequentialReader` for reading. +- **mmap consumer** (metatensor): `LayoutMode::Aligned` → uses `OffsetMap` + for both reading and writing at known offsets. + +The same schema can be used in either mode. A schema describing an SFTP +packet can be consumed by a `SequentialReader` (for parsing incoming +frames) and a `LayoutBuilder` (for constructing outgoing frames). A schema +describing a metatensor layout can be consumed by an `OffsetMap` (for +mmap access). + +`TypedefEngine` exposes mode-appropriate accessors: `engine.offset_map()` +returns `Some(&OffsetMap)` in aligned mode and `None` in packed mode; +`engine.layout_builder()` returns `Some(&LayoutBuilder)` in packed mode +and `None` in aligned mode. `engine.sequential_reader()` returns +`Option` (an owned fresh reader, not a reference — the +reader has mutable cursor state that the consumer owns; see +[ADR-101](decisions/101-packed-mode-read-factory.md)) in packed +mode and `None` in aligned mode. See [validation.md](validation.md) +§"The TypedefEngine struct" for the engine API. + +## Public Types + +The layout engine produces three public types, one per layout component. +All are re-exported from the crate root. + +### `ByteRange` (aligned mode) + +```rust +pub struct ByteRange { + pub start: usize, // inclusive + pub end: usize, // exclusive +} +``` + +A half-open byte range produced by `OffsetMap::compute` for each field. +`end - start` is the field's byte size in the static layout (for +variable-length fields: the length prefix, the `{offset, length}` pair, +or the `maxLength` reservation — not the variable data). `ByteRange` +provides `len()` and `is_empty()`. + +### `FieldPosition` (packed mode) + +```rust +pub struct FieldPosition { + pub offset: usize, + pub size: usize, + pub kind: TypeDefKind, +} +``` + +A field's computed position in a packed layout, produced by +`LayoutBuilder::build`. For variable-length fields, `size` is `4` (the +length prefix); for fixed-size fields, `size` is the type's byte size. +`kind` records the field's `TypeDef:*` kind so the consumer can dispatch +to the correct `data_access` read/write function. + +### `PackedLayout` (packed mode) + +The result of `LayoutBuilder::build`: a map of `field_path → FieldPosition` +plus the total buffer size needed. + +```rust +impl PackedLayout { + pub fn get(&self, field_path: &str) -> Option<&FieldPosition>; + pub fn total_size(&self) -> usize; + pub fn iter(&self) -> impl Iterator; +} +``` + +`get` looks up a field by dotted path. For TUnion byte-offset +discriminators, the discriminator is recorded under the synthetic path +`".__discriminator"`. `iter` yields fields in layout order +(schema `properties` order, with nested struct fields appearing inline +under their parent's path prefix). + +### `OffsetMap` (aligned mode) + +A flat table of `(field_path, byte_range)` pairs computed from a schema. + +```rust +impl OffsetMap { + pub fn compute(schema: &Value) -> Result; + pub fn get(&self, field_path: &str) -> Option<&ByteRange>; + pub fn total_size(&self) -> usize; + pub fn iter(&self) -> impl Iterator; +} +``` + +`compute` requires a `TypeDef:Struct` at the top level. `total_size` +includes trailing alignment padding. `iter` yields fields in insertion +order (schema `properties` order, nested struct fields appearing inline). + +## Design Decisions + +| Decision | ADR | Summary | +|----------|-----|---------| +| Two layout modes | [ADR-096](decisions/096-two-layout-modes-packed-vs-aligned.md) | Packed sequential for protocols; aligned static for mmap formats | +| Schema annotations | [ADR-097](decisions/097-schema-annotations.md) | Endianness, alignment, encoding annotations that control layout behavior | +| Non-final inline variable fields | [ADR-100](decisions/100-reject-non-final-inline-length-prefixed-in-aligned-mode.md) | Rejected in aligned mode (would clobber subsequent fields); use `maxLength` or `offset-indirect` | +| Packed-mode read factory | [ADR-101](decisions/101-packed-mode-read-factory.md) | `engine.sequential_reader()` returns an owned fresh reader, not a reference | +| TUnion in aligned mode | [ADR-102](decisions/102-reject-tunion-in-aligned-mode.md) | Rejected for v1 (broken semantics; no current consumer needs it) | + +## Open Questions + +See [open-questions.md](open-questions.md) for full details. + +- **OQ-069** (deferred(scope)): Arrays of variable-length-element structs + — requires lazy walking logic; blocked on a concrete consumer that + needs it. + +## References + +- `docs/research/alknet-typedef/findings.md` §"POC Results" — POC 1 + (aligned OffsetMap) and POC 2 (packed LayoutBuilder/SequentialReader) +- [ADR-096](decisions/096-two-layout-modes-packed-vs-aligned.md) — + the two layout modes decision +- [ADR-097](decisions/097-schema-annotations.md) — schema + annotations +- [schema-layer.md](schema-layer.md) — the 17 TypeDef kinds and their + byte sizes +- [data-access.md](data-access.md) — read/write functions that use the + computed offsets diff --git a/docs/architecture/open-questions.md b/docs/architecture/open-questions.md new file mode 100644 index 0000000..e5f38d5 --- /dev/null +++ b/docs/architecture/open-questions.md @@ -0,0 +1,104 @@ +--- +status: draft +last_updated: 2026-07-22 +--- + +# Open Questions + +Each open question lives in its own file under [`questions/`](questions/), +named `NNN-slug.md` (mirroring the ADR convention). This file is the index: +theme-grouped tables for scannability, plus a cross-theme +[Deferred / Blocked](#deferred--blocked) section that surfaces the +safe-exit deferrals with their blocking conditions inline — so "what's +currently parked and why" is answerable at a glance. + +**Status values**: +- `open` — Needs to be resolved now. Has a clear path to resolution. +- `resolved` — Decided. The resolution is stated cleanly, without caveats about how it could be changed later. +- `deferred(scope)` — Cannot be resolved yet. The information is genuinely + missing — a crate spec, POC result, or use case that doesn't exist yet. + Has a concrete blocking condition. Not a failure — scope management. +- `deferred(unclear)` — Cannot be resolved yet. The pieces exist (decided + in other ADRs, existing types, existing patterns) but the composition + — how they fit together — isn't clear yet. Resolution requires + investigation (work through examples, maybe POC), not waiting. Has a + concrete investigation target and an impacts field. Not a failure — + honest uncertainty in a poorly-defined problem space. +- `partially resolved` — Some aspects decided, others deferred or open. +- `dissolved` — The question was reframed out of existence (e.g., superseded + by an ADR that retires the premise). Kept for reference. + +**Impacts field**: Every unresolved OQ (`open`, `deferred(scope)`, +`deferred(unclear)`, `partially resolved`) should have an `Impacts` +field stating what it blocks downstream. Be specific: "blocks the first +hub deployment because the hub dials workers" not "blocks the hub +crate." This is the triage signal that makes the deferral's urgency +visible. + +Door type classifications follow ADR-009 — they describe **reversal cost** (how expensive it is to undo), not urgency: +- **One-way door**: Reversal requires rewriting significant code or permanently closes a capability. Getting it wrong is expensive — requires ADR before implementation. +- **Two-way door**: Reversal is cheap or additive. Getting it wrong is recoverable — decide, implement, revert if needed. + +Door type is separate from whether a decision is made. A two-way door is a decision you make now and can revert later, not a decision to defer. + +## By Theme + +### Layout Engine + +| OQ | Title | Status | Door | Pri | +|----|-------|--------|------|-----| +| [OQ-069](questions/069-arrays-of-variable-length-element-structs.md) | Arrays of Variable-Length-Element Structs | deferred(scope) | two | low | + +### Platform Support + +| OQ | Title | Status | Door | Pri | +|----|-------|--------|------|-----| +| [OQ-070](questions/070-no-std-alloc-support.md) | `no_std` + `alloc` Support | deferred(scope) | two | low | + +### Schema Construction + +| OQ | Title | Status | Door | Pri | +|----|-------|--------|------|-----| +| [OQ-071](questions/071-builder-api-for-schema-construction.md) | Builder API for Schema Construction | deferred(scope) | two | med | + +## Deferred / Blocked + +The safe-exit visibility surface. These questions are parked because the +information needed to resolve them does not exist yet — each has a concrete +blocking condition. They are not failures; they are scope management. +This section exists so "what's currently blocking the architect" is +answerable at a glance, not by filtering the tables above. + +### OQ-069: Arrays of Variable-Length-Element Structs + +- **Blocked on**: A concrete consumer that needs arrays of structs with + variable-length fields, where the elements are interleaved + (`[fixed_0][str_0][fixed_1][str_1]...`) and the engine must walk + sequentially rather than use a fixed stride. The SFTP `Name` packet + has `Vec` where `File` contains strings, but SFTP serializes + this as a sequence of length-prefixed strings (the serde `SeqAccess` + pattern), not as an array of fixed-stride structs. Arrays of + fixed-size structs are fully supported. +- **Priority**: low +- **Full file**: [OQ-069](questions/069-arrays-of-variable-length-element-structs.md) + +### OQ-070: `no_std` + `alloc` Support + +- **Blocked on**: An embedded use case that requires `no_std` + `alloc` + (e.g., a microcontroller running Rust without `std`). The WASM target + has `std` available via `wasm-bindgen`. The engine's core (offset + computation, read/write) is already allocation-free; the `jsonschema` + dependency is the only `alloc` consumer. +- **Priority**: low +- **Full file**: [OQ-070](questions/070-no-std-alloc-support.md) + +### OQ-071: Builder API for Schema Construction + +- **Blocked on**: A concrete need for programmatic schema construction + in Rust. The current consumers (SFTP, metatensor, binary call frames, + TTY negotiation) all have schemas that can be hand-written or + generated from TypeBox. A builder API would be a fluent Rust API that + produces the same JSON Schema structure — it would sit on top of the + engine, not inside it. +- **Priority**: medium +- **Full file**: [OQ-071](questions/071-builder-api-for-schema-construction.md) \ No newline at end of file diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md new file mode 100644 index 0000000..94d5994 --- /dev/null +++ b/docs/architecture/overview.md @@ -0,0 +1,203 @@ +--- +status: draft +last_updated: 2026-07-22 +--- + +# alknet-typedef — Overview + +The binary struct engine: a small Rust crate that takes a JSON Schema +with `TypeDef:*` custom keywords and produces an offset map, read/write +functions, and validation — all driven by the schema. The schema is the +format definition; the engine is generic. + +This document covers the crate's purpose, the "schema is the format" +principle, its dependency edges, consumers, and scope boundaries. +Component details are in the sibling documents. + +## What + +`alknet-typedef` is a library crate that consumes JSON Schemas annotated +with `TypeDef:*` custom keywords (the same kinds defined in TypeBox's +`typedef.ts`, plus `TypeDef:Bytes`, `TypeDef:Int64`, and `TypeDef:Uint64` +as alknet-typedef additions) and produces three capabilities: + +1. **An offset map** — walks the schema, computes byte offsets for each + field based on type sizes, field order, and alignment. +2. **Read/write functions** — given a `&[u8]` buffer and a field path, + read the field's bytes at its offset (zero-copy for fixed-size types). + Given a `&mut [u8]` buffer, write a value at its offset. +3. **Validation** — via `jsonschema` custom keywords, validates that a + buffer's bytes match the schema's type constraints. + +The heavy lifting is done by the `jsonschema` crate (validation) and +`serde_json` (schema parsing). The novel code is the offset computation +— a recursive walk of the schema JSON that computes byte positions for +each field. The custom keyword implementations are small (a few lines +each, generated from shared macros — see [validation.md](validation.md)). + +The crate replaces two prior attempts that built their own jsonschema +engines — typebox-rs (~8,400 lines) and alktype (~5,600 lines) — with +`jsonschema` + an offset map + small custom keyword implementations. See +[ADR-095](decisions/095-alknet-typedef-purpose-scope-jsonschema-engine.md). + +## Why + +The crate's purpose is to be the binary struct engine for every alknet +component that reads or writes binary data at computed offsets. Instead +of per-protocol serde structs (russh-sftp's 29 packet types), per-handler +wire format code (TTY's 5-byte format parser), or per-format offset +computation (metatensor's tensor access), all of these become instances +of the same engine with different schemas. + +The guiding insight: + +> **The schema is the format.** A JSON Schema with `TypeDef:Float32`, +> `TypeDef:Struct`, `TypeDef:Union` etc. is both the validation spec and +> the layout spec. No separate format definition, no separate parser, no +> separate validator. One schema, three uses: validate, compute offsets, +> access data. + +This is the convergence of three threads identified in the +call-channels-unification research: the `typedef.ts` schema kinds from +TypeBox, the russh-sftp protocol packets, and the metatensor format. The +common pattern: a JSON Schema describes the shape of binary data, and +the binary data is the struct's bytes at computed offsets. + +The crate was bumped up in the timeline when the call-channels-unification +research surfaced that channels, TTY, and the binary call protocol are +all variations on the same wire-format family — `[discriminant][length][payload]`. +The typedef engine makes the "channels is call with a binary data plane" +unification concrete: the binary data plane's wire format is the call +protocol's own schema system, just binary-encoded. The `channel_open` +marker says "use binary framing"; the typedef engine says "here's how to +read/write the binary payload." + +## The "Schema Is the Format" Principle + +A JSON Schema with `TypeDef:*` custom keywords serves three roles +simultaneously: + +| Role | Mechanism | When | +|------|-----------|------| +| **Validation spec** | `jsonschema` custom keywords | Load time (build validator), access time (validate buffer) | +| **Layout spec** | Offset computation from type sizes + field order | Load time (build offset map) | +| **Data access** | Read/write at computed offsets | Access time (read field, write field) | + +No separate format definition, no separate parser, no separate validator. +The schema is the single source of truth for the binary format. Adding a +new field to a protocol is adding a property to the schema JSON — the +engine computes the new offsets automatically. + +This is the same principle as `#[repr(C)]` struct field access, but at +runtime from a portable JSON Schema instead of at compile-time from +language-specific annotations. The schema is the ABI contract. + +## Dependencies + +``` +alknet-typedef +├── jsonschema (v0.46.5, Draft 2020-12) — validation engine, custom keyword support +├── serde_json (with preserve_order) — schema parsing; field order is load-bearing +└── (no tokio, no platform deps) — WASM-clean by construction +``` + +`alknet-typedef` is dependency-light: `jsonschema` + `serde_json` only. +No tokio, no platform deps. Compiles to `wasm32-unknown-unknown` for +browser use. The `jsonschema` crate is already in the workspace at +`/workspace/jsonschema/` but not yet used by any alknet crate — typedef +is the first consumer. + +`serde_json` requires the `preserve_order` feature because field order +is load-bearing for binary layouts. The order of properties in the +schema JSON determines the order of fields in the binary struct. + +## Consumers + +| Consumer | Schema describes | Engine provides | +|----------|-----------------|-----------------| +| russh-sftp | 29 packet structs + Packet union (byte discriminator) | Read/write SFTP frames from bytes | +| metatensor | Model layout (ConvNet struct, tensor refs) | Offset map for mmap'd tensor access | +| binary call frames | `call.requested` / `call.responded` / etc. structs | Read/write binary call frames | +| TTY negotiation | `NegotiateRequest` / `NegotiateResponse` structs | Read/write TTY control frames | +| channels wire | `ChunkHeader { channel_id, length }` | Already trivial (8 bytes, no schema needed) | + +The russh-sftp case is the most instructive and the highest-value POC +target. The `Packet` enum's `TryFrom<&mut Bytes>` impl is a hand-written +dispatch on a type byte followed by serde deserialization. Under typedef, +the dispatch is `TUnion` with a byte-offset discriminator — the schema +says "byte 0 is the discriminator, bytes 1..N are the variant struct." +The engine reads the discriminator, looks up the variant schema, computes +offsets, reads fields. Same result, no per-packet-type code. + +## Scope Boundaries (What This Is Not) + +These boundaries are decided in [ADR-095](decisions/095-alknet-typedef-purpose-scope-jsonschema-engine.md). + +- **Not metatensor.** typedef is the binary struct *engine*. Metatensor + is a *format* (8-byte header + JSON header + binary data) that uses the + typedef engine for its offset computation and tensor access. +- **Not a Value system.** TypeBox's `Value.Diff`, `Value.Migrate`, + `Value.Convert` — schema evolution — is out of scope for v1. The engine + should not do anything that explicitly blocks adding a Value system + later. +- **Not a code generator.** typebox-rs's `codegen/` module is a separate + concern. The typedef engine consumes schemas; it does not generate them. +- **Not a schema builder.** The typedef engine does not provide a fluent + API for constructing schemas. Schemas are plain JSON — authored in + TypeBox, generated by ujsx components, or hand-written. A builder API + is deferred (OQ-071). +- **Not a serialization framework.** The typedef engine is not a + general-purpose serde replacement. It operates on raw byte buffers at + computed offsets — no intermediate `Value` tree, no reflection, no + dynamic dispatch per field. For JSON data, use serde. For binary data + with a known schema, use typedef. + +## Architecture (component pointers) + +- **[schema-layer.md](schema-layer.md)** — the 19 `TypeDef:*` kinds, + jsonschema custom keyword integration, TypeBox interop, schema + annotations (endianness, alignment, encoding, TUnion discriminators). +- **[layout-engine.md](layout-engine.md)** — offset computation, the two + layout modes (packed sequential vs aligned static), alignment, + endianness, variable-length field handling. +- **[data-access.md](data-access.md)** — read/write functions, TUnion + dispatch, field paths, zero-copy access for fixed-size types, + length-prefix reading for variable-length types. +- **[validation.md](validation.md)** — custom keyword validators for all + 19 `TypeDef:*` kinds, `TypedefError`, load-time vs access-time + validation, `TypedefEngine` as the compiled form of a schema. + +## Design Decisions + +| Decision | ADR | Summary | +|----------|-----|---------| +| Purpose, scope, and the jsonschema engine | [ADR-095](decisions/095-alknet-typedef-purpose-scope-jsonschema-engine.md) | What the crate is/isn't; why jsonschema not a custom engine; "schema is the format" principle; scope boundaries | +| Two layout modes | [ADR-096](decisions/096-two-layout-modes-packed-vs-aligned.md) | Packed sequential (`LayoutBuilder`/`SequentialReader`) for protocols; aligned static (`OffsetMap`) for mmap formats | +| Schema annotations | [ADR-097](decisions/097-schema-annotations.md) | Endianness (schema-level, default LE), alignment (struct + field-level), encoding (length-prefixed vs offset-indirect), TUnion discriminators (byte-offset vs field-name) | +| Error handling and validation | [ADR-098](decisions/098-error-handling-validation-strategy.md) | `TypedefError` enum; load-time build, access-time check; field-path-carrying errors; jsonschema `ValidationError` wrapping | +| Int64/Uint64 kinds | [ADR-099](decisions/099-int64-uint64-first-class-kinds.md) | 64-bit integers as first-class kinds (SFTP offsets, metatensor data_offsets) | +| Non-final inline variable fields | [ADR-100](decisions/100-reject-non-final-inline-length-prefixed-in-aligned-mode.md) | Rejected in aligned mode (would clobber subsequent fields) | +| Packed-mode read factory | [ADR-101](decisions/101-packed-mode-read-factory.md) | `engine.sequential_reader()` returns an owned fresh reader | +| TUnion in aligned mode | [ADR-102](decisions/102-reject-tunion-in-aligned-mode.md) | Rejected for v1 (broken semantics; no current consumer needs it) | + +## Open Questions + +See [open-questions.md](open-questions.md) for full details. + +- **OQ-069** (deferred(scope)): Arrays of variable-length-element structs. +- **OQ-070** (deferred(scope)): `no_std` + `alloc` support. +- **OQ-071** (deferred(scope)): Builder API for schema construction. + +## References + +- `docs/research/alknet-typedef/findings.md` — POC results (26 tests + passing, two layout modes, TUnion dispatch, endianness) +- `docs/research/call-channels-unification/findings.md` §"alknet-typedef: + JSON Schema as the binary struct engine" — the origin of this research + thread +- `/workspace/@alkdev/typebox/example/typedef/typedef.ts` — the TypeBox + schema kinds (619 lines) +- `/workspace/jsonschema/` — the jsonschema crate (v0.46.5, Draft 2020-12) +- `/workspace/alknet-typedef-poc/` — the POC code (disposable) +- `/workspace/@alkimiadev/typebox-rs/` — prior attempt, replaced by typedef +- `/workspace/@alkimiadev/alktype/` — prior attempt, replaced by typedef diff --git a/docs/architecture/questions/069-arrays-of-variable-length-element-structs.md b/docs/architecture/questions/069-arrays-of-variable-length-element-structs.md new file mode 100644 index 0000000..c0a1a8c --- /dev/null +++ b/docs/architecture/questions/069-arrays-of-variable-length-element-structs.md @@ -0,0 +1,20 @@ +# OQ-069: Arrays of variable-length-element structs + +- **Origin**: [../layout-engine.md](../layout-engine.md), + [../data-access.md](../data-access.md); + `docs/research/alknet-typedef/findings.md` §"Problem 3: Nested structs + and arrays of structs" +- **Status**: deferred(scope) +- **Door type**: Two-way (additive — the engine can add lazy walking + logic without changing the existing fixed-stride array support) +- **Priority**: low +- **Impacts**: Blocks any protocol with interleaved variable-length struct arrays (e.g., a protocol where each array element has a string field and elements are packed as `[fixed_0][str_0][fixed_1][str_1]...`). Does NOT block SFTP `Name` packet handling — SFTP serializes this as a sequence of length-prefixed strings (the serde `SeqAccess` pattern), not as an array of fixed-stride structs. Does NOT block any current consumer. +- **Blocked on**: A concrete consumer that needs arrays of structs with + variable-length fields, where the elements are interleaved + (`[fixed_0][str_0][fixed_1][str_1]...`) and the engine must walk + sequentially rather than use a fixed stride. +- **Resolution**: Not yet decidable. The mechanism (lazy sequential + walking of array elements, reading each element's length prefixes to + find the next element's start) is understood but not needed by any + current consumer. Arrays of fixed-size structs are fully supported. +- **Cross-references**: ADR-096, [layout-engine.md](../layout-engine.md) diff --git a/docs/architecture/questions/070-no-std-alloc-support.md b/docs/architecture/questions/070-no-std-alloc-support.md new file mode 100644 index 0000000..3ae8618 --- /dev/null +++ b/docs/architecture/questions/070-no-std-alloc-support.md @@ -0,0 +1,22 @@ +# OQ-070: `no_std` + `alloc` support + +- **Origin**: [../overview.md](../overview.md); + `docs/research/alknet-typedef/findings.md` §"Open Questions" (OQ 6) +- **Status**: deferred(scope) +- **Door type**: Two-way (additive — can be added as a feature gate + without changing the existing `std` API) +- **Priority**: low +- **Impacts**: Blocks bare-metal embedded targets (microcontrollers + running Rust without `std`). Does NOT block any current deployment + target. Does NOT block WASM — `wasm32-unknown-unknown` has `std` + available via `wasm-bindgen`; the crate is WASM-clean by construction + (no tokio, no platform deps, `jsonschema` builds for WASM with + `default-features = false`). +- **Blocked on**: An embedded use case that requires `no_std` + `alloc` + (e.g., a microcontroller running Rust without `std`). +- **Resolution**: Not yet decidable. Target `std` for v1. If embedded + use cases emerge, `no_std` + `alloc` can be added as a feature gate + later. The engine's core (offset computation, read/write) is already + allocation-free — it operates on `&[u8]` slices. The `jsonschema` + dependency is the only `alloc` consumer. +- **Cross-references**: ADR-095 diff --git a/docs/architecture/questions/071-builder-api-for-schema-construction.md b/docs/architecture/questions/071-builder-api-for-schema-construction.md new file mode 100644 index 0000000..b0e1e8c --- /dev/null +++ b/docs/architecture/questions/071-builder-api-for-schema-construction.md @@ -0,0 +1,26 @@ +# OQ-071: Builder API for schema construction + +- **Origin**: [../schema-layer.md](../schema-layer.md), + [../overview.md](../overview.md); + `docs/research/alknet-typedef/findings.md` (the builder API was noted + as the one detail not covered by the POCs) +- **Status**: deferred(scope) +- **Door type**: Two-way (additive — a builder API can be added without + changing the existing JSON-consumption path) +- **Priority**: medium +- **Impacts**: Blocks programmatic schema construction in Rust without a + JS toolchain. Any consumer that wants to build typedef schemas at + runtime from Rust code (rather than loading pre-authored JSON) must + construct the JSON manually or depend on TypeBox. Does NOT block any + current consumer — all v1 consumers (SFTP, metatensor, binary call + frames, TTY negotiation) use pre-authored schemas. +- **Blocked on**: A concrete need for programmatic schema construction + in Rust. The current consumers (SFTP, metatensor, binary call frames, + TTY negotiation) all have schemas that can be hand-written or generated + from TypeBox. +- **Resolution**: Not yet decidable. The builder API is important but + not needed for the initial consumers. The engine's JSON-consumption + path is the primary interface for v1. A builder API would be a fluent + Rust API that produces the same JSON Schema structure — it would sit + on top of the engine, not inside it. +- **Cross-references**: ADR-095, [schema-layer.md](../schema-layer.md) diff --git a/docs/architecture/schema-layer.md b/docs/architecture/schema-layer.md new file mode 100644 index 0000000..f77441c --- /dev/null +++ b/docs/architecture/schema-layer.md @@ -0,0 +1,506 @@ +--- +status: draft +last_updated: 2026-07-22 +--- + +# alknet-typedef — Schema Layer + +The schema layer: the 19 `TypeDef:*` custom type kinds, their mapping to +Rust types and byte sizes, the `jsonschema` custom keyword integration, +TypeBox interop, and the concrete JSON shapes for schema-level +annotations. + +## The 19 TypeDef Kinds + +These are the custom schema kinds defined in TypeBox's `typedef.ts` +(`/workspace/@alkdev/typebox/example/typedef/typedef.ts`, 619 lines) and +ported to Rust via `jsonschema` custom keywords. Each kind carries binary +layout semantics — a known byte size (for fixed-size types) or a known +encoding strategy (for variable-length types). + +| Kind | TypeBox key | Rust type | Size | Category | +|------|-------------|-----------|------|----------| +| `TFloat32` | `TypeDef:Float32` | `f32` | 4 | fixed | +| `TFloat64` | `TypeDef:Float64` | `f64` | 8 | fixed | +| `TInt8` | `TypeDef:Int8` | `i8` | 1 | fixed | +| `TInt16` | `TypeDef:Int16` | `i16` | 2 | fixed | +| `TInt32` | `TypeDef:Int32` | `i32` | 4 | fixed | +| `TInt64` | `TypeDef:Int64` | `i64` | 8 | fixed | +| `TUint8` | `TypeDef:Uint8` | `u8` | 1 | fixed | +| `TUint16` | `TypeDef:Uint16` | `u16` | 2 | fixed | +| `TUint32` | `TypeDef:Uint32` | `u32` | 4 | fixed | +| `TUint64` | `TypeDef:Uint64` | `u64` | 8 | fixed | +| `TBoolean` | `TypeDef:Boolean` | `bool` (0x00=false, 0x01=true) | 1 | fixed | +| `TString` | `TypeDef:String` | length-prefixed UTF-8 | variable | variable | +| `TBytes` | `TypeDef:Bytes` | length-prefixed raw bytes | variable | variable | +| `TStruct` | `TypeDef:Struct` | record of fields | sum of field sizes | composite | +| `TUnion` | `TypeDef:Union` | tagged union | discriminator + variant | composite | +| `TArray` | `TypeDef:Array` | repeated element | count × element size | composite | +| `TEnum` | `TypeDef:Enum` | u32 index into enum values | 4 (fixed) | fixed | +| `TRecord` | `TypeDef:Record` | count-prefixed sequence of (key, value) pairs | variable | variable | +| `TTimestamp` | `TypeDef:Timestamp` | length-prefixed RFC 3339 string | variable | variable | + +`TypeDef:Int64` and `TypeDef:Uint64` are alknet-typedef additions — +TypeBox's `typedef.ts` tops out at 32-bit integers. They are required by +the primary POC targets: SFTP `Read`/`Write` packets have `offset: u64`, +and metatensor `data_offsets` are `u64`. See +[ADR-099](decisions/099-int64-uint64-first-class-kinds.md). + +### The `TypeDefKind` enum + +The engine represents the 19 kinds as a Rust enum — `TypeDefKind` — with +one variant per kind (`TypeDefKind::Float32`, `TypeDefKind::Struct`, etc.). +The enum provides compile-time exhaustiveness checking and integer +discriminant dispatch (a jump table) instead of string comparison at +every field access. It is `pub` and re-exported from the crate root. + +```rust +pub enum TypeDefKind { + Int8, Int16, Int32, Int64, + Uint8, Uint16, Uint32, Uint64, + Float32, Float64, + Boolean, Enum, + String, Bytes, Timestamp, + Struct, Union, Array, Record, +} +``` + +The enum carries the kind's binary-layout metadata as inherent methods: + +| Method | Returns | Notes | +|--------|---------|-------| +| `as_str(self)` | `&'static str` | The JSON Schema keyword, e.g. `"TypeDef:Uint8"` | +| `type_size(self)` | `Option` | `Some(N)` for fixed-size kinds; `None` for variable/composite | +| `natural_alignment(self)` | `usize` | 1 for u8/i8/bool, 2 for u16/i16, 4 for u32/i32/f32/enum, 8 for u64/i64/f64, 4 for variable-length (the u32 length prefix), 1 for struct/union/array | +| `is_fixed_size(self)` | `bool` | True for the 12 fixed-size primitive kinds | +| `is_composite(self)` | `bool` | True for Struct, Union, Array, Record | +| `is_variable_length(self)` | `bool` | True for String, Bytes, Timestamp, Record | +| `needs_endian(self)` | `bool` | True for kinds whose read/write takes an `Endian` parameter | + +`TypeDefKind` implements `Display` (renders the keyword string) and +`FromStr` (parses the keyword string back into the variant, returning +`TypedefError::Schema` for unknown kinds). The layout engines and the +validator dispatch on the enum, not on strings. + +### Fixed-size types + +`TFloat32`, `TFloat64`, `TInt8`, `TInt16`, `TInt32`, `TUint8`, `TUint16`, +`TUint32`, `TBoolean`, and `TEnum` have known byte sizes. The offset +computation uses these sizes directly. Read/write is zero-copy pointer +cast for these types. + +**`TBoolean` byte representation:** `0x00` = false, `0x01` = true. Other +values are invalid and produce a `TypedefError::Access` on read. + +**`TEnum` binary representation:** A `u32` index into the enum's declared +values, in declaration order. The first declared value is index 0, the +second is index 1, etc. The enum's values are declared via the standard +JSON Schema `"enum"` keyword (e.g., `"enum": ["read", "write", "execute"]`). +The `TypeDef:Enum` custom keyword signals that the type is an enum for +layout purposes; the built-in `enum` keyword provides the value list. + +**Design note:** TypeBox's `TEnum` is a string enum (variable-length). The +typedef engine uses a `u32` index instead — a deliberate deviation from +TypeBox fidelity in favor of binary efficiency. Most enums have a small +number of variants (e.g., the call protocol's 5 event types); a `u32` +index is compact, fixed-size, and sufficient for any realistic enum. The +JSON representation (for validation) remains a string; the binary +representation is the `u32` index. +The `u32` index follows the schema's endianness annotation (ADR-097), like +all other fixed-size types. In little-endian mode the index is +`u32::from_le_bytes`; in big-endian mode it is `u32::from_be_bytes`. + +### Variable-length types + +`TString`, `TBytes`, `TRecord`, and `TTimestamp` have variable byte sizes. +The typedef engine supports three strategies for handling variable-length +types in binary layouts, selected by the `encoding` annotation and the +standard JSON Schema `maxLength` keyword: + +| Strategy | Encoding annotation | Layout behavior | Use case | +|----------|-------------------|-----------------|----------| +| **Inline length-prefixed** | `"length-prefixed"` (default) | `[length: u32][data]`; shifts subsequent fields in packed mode | Protocol wire formats (SFTP, channels, TTY) | +| **Fixed-size reservation** | (none — uses `maxLength`) | `[data: maxLength bytes]`, zero-padded; fixed offset in aligned mode | mmap-friendly formats where max size is known (database `VARCHAR(N)` pattern) | +| **Offset indirection** | `"offset-indirect"` | `{offset: u32, length: u32}` pointing into a separate data region | Blob tensors, metatensor variable-length data (the blob tensor pattern) | + +**Strategy 1: Inline length-prefixing (default).** The field's fixed +portion is a 4-byte length prefix at a computed offset. The variable data +follows immediately after. In packed sequential mode, the length prefix +determines the position of subsequent fields. In aligned static mode, the +length prefix is at a known offset; the variable data is not included in +the static layout. This is the universal pattern used by channels, SFTP, +TTY, and most binary protocols. + +**Strategy 2: Fixed-size reservation.** When a variable-length field +declares `maxLength` (a standard JSON Schema keyword), the engine reserves +`maxLength` bytes at a fixed offset in aligned static mode. Data shorter +than `maxLength` is zero-padded; data longer than `maxLength` is a +validation error. This makes the field fixed-size from the layout +perspective — subsequent fields have known, unchanging offsets. This is +the database `VARCHAR(N)` pattern and the metatensor struct-tensor +pattern for fields with known maximum sizes. + +In packed sequential mode, `maxLength` is a validation constraint only — +the engine still uses inline length-prefixing (strategy 1) because +protocols don't benefit from fixed-size reservation. + +**Strategy 3: Offset indirection.** The field is a struct +`{offset: u32, length: u32}` at a known position. The consumer provides +the data region separately; the engine reads the offset and length, then +slices the data region. This is the metatensor blob tensor pattern — the +index struct lives in one region, the blob data lives in another. Enables +mmap-friendly random access to variable-length data without parsing +length prefixes and without reserving worst-case space. + +**Default strategy selection:** +- In packed sequential mode: always strategy 1 (inline length-prefixing). + `maxLength` is a validation constraint only. +- In aligned static mode: strategy 2 (fixed-size reservation) if + `maxLength` is declared; strategy 3 (offset indirection) if + `"encoding": "offset-indirect"` is declared; strategy 1 (inline + length-prefixing) otherwise. + +**Length prefix endianness:** The 4-byte length prefix (strategies 1 and 3) +respects the schema's `"endian"` annotation (ADR-097). In little-endian +mode, the length is `u32::from_le_bytes`. In big-endian mode, the length +is `u32::from_be_bytes`. This ensures SFTP consumers (big-endian) have +consistent byte order for both field values and length prefixes. + +**`TBytes`:** Raw bytes — no UTF-8 constraint. The payload is `&[u8]`. +Otherwise identical to `TString` in layout (same three strategies). + +**Design note:** `TypeDef:Bytes` is an alknet-typedef addition — it does +not exist in TypeBox's `typedef.ts` (which defines 16 kinds). It is +included because raw byte arrays are a common binary protocol primitive +(SFTP data payloads, channels payloads, tensor data) and are semantically +distinct from UTF-8 strings. In the binary representation, TBytes is raw +bytes with no encoding (not base64, not hex). In the JSON representation +(for validation), TBytes is a string (JSON has no native byte type). + +**`TRecord`:** A string-keyed map. The value type is declared via the +schema's `"values"` property (e.g., `"values": { "TypeDef:Float32": true }`). +Binary layout is a count-prefixed sequence of `(key, value)` pairs: +`[count: u32][key_len: u32][key_bytes][value]...` repeated `count` times. +The count is the number of entries. Each key is a length-prefixed UTF-8 +string. Each value is encoded according to its declared `TypeDef:*` kind +— a `Record` value is 4 raw bytes; a `Record` value is +itself a length-prefixed string; a `Record` value is the struct's +fields laid out inline. There is **no separate `value_len` prefix** — +the value's size is determined by its kind (fixed-size kinds have a +known size; variable-length kinds carry their own length prefix). The +count and key-length prefixes respect the schema's endianness. In +aligned static mode with `maxLength`, the entire record is reserved at +`maxLength` bytes (zero-padded). + +**`TTimestamp`:** An RFC 3339 timestamp string (the internet profile of +ISO 8601). Stored as a length-prefixed UTF-8 string (strategy 1) or +fixed-size reservation (strategy 2 with `maxLength`). The data-access +layer treats timestamps as opaque length-prefixed strings — it does not +parse or validate the timestamp format. The jsonschema custom keyword +validator checks RFC 3339 conformance at the JSON level (see +[validation.md](validation.md)). + +`TArray` is variable-length when the element type is variable-length or +when the count is not known at schema time. For fixed-size element arrays +with a known count, the size is `element_size × count`. + +**`TArray` count declaration:** The array count is declared via the +standard JSON Schema `"minItems"` and `"maxItems"` keywords. When +`minItems == maxItems`, the array has a fixed count known at schema time. +When they differ or are absent, the count is variable and the array uses +a length-prefixed encoding: `[count: u32][element_0]...[element_N]`. +The count prefix respects the schema's endianness. + +### Composite types + +`TStruct` and `TUnion` are composite — their size is the sum of their +fields' sizes (plus alignment padding in aligned static mode). The offset +computation recurses into their properties. + +## Schema-Layer Public API + +The `schema` module exposes the foundational types and functions every +other module depends on. These are re-exported from the crate root. + +### `get_typedef_kind` vs `get_typedef_kind_loose` + +The engine recognizes a `TypeDef:*` kind on a schema node two ways, +because the keyword value may be either a boolean (`true`) or an +annotation object (`{ "encoding": "..." }`): + +| Function | Recognizes | Returns | +|----------|------------|---------| +| `get_typedef_kind(node) -> Option<&str>` | Boolean form only (`{ "TypeDef:String": true }`) | The keyword string, e.g. `"TypeDef:String"` | +| `get_typedef_kind_loose(node) -> Option<&str>` | Boolean form **and** object form | The keyword string | +| `get_typedef_kind_enum(node) -> Option` | Boolean form only | The parsed enum variant | +| `get_typedef_kind_loose_enum(node) -> Option` | Boolean form **and** object form | The parsed enum variant | + +The boolean-form-only functions are used by the validator factories +(which reject the object form as a schema error) and the top-level +kind-check in `OffsetMap::compute` / `LayoutBuilder::new` / `SequentialReader::new` +(which require `TypeDef:Struct` at the root). The "loose" variants are +used by the layout engines during field traversal, so that a variable- +length field with an `encoding` annotation (`{ "TypeDef:String": +{ "encoding": "offset-indirect" } }`) is still recognized as a `String`. + +### Annotation parsers + +Each schema-level annotation has a dedicated parser that reads it from a +`serde_json::Value` node and returns a sensible default when absent: + +| Function | Annotation | Default | +|----------|------------|---------| +| `parse_endian(node) -> Endian` | `"endian"` | `Endian::Little` | +| `parse_align(node) -> Option` | `"align"` | `None` | +| `parse_max_length(node) -> Option` | `"maxLength"` | `None` | +| `parse_encoding(keyword_value) -> VariableEncoding` | `"encoding"` (within the keyword's value object) | `VariableEncoding::LengthPrefixed` | +| `parse_discriminator(node) -> Result` | `"discriminator"` | (required — returns `TypedefError::Schema` if absent) | + +### Public enums + +```rust +pub enum Endian { Little, Big } +pub enum VariableEncoding { LengthPrefixed, OffsetIndirect } +pub enum DiscriminatorKind { + Byte { offset: usize, disc_type: TypeDefKind }, + Field { name: String }, +} +``` + +`DiscriminatorKind::Byte` carries the byte position (`offset`) and the +discriminator's `TypeDef:*` kind (`disc_type`, restricted to `Uint8`/ +`Uint16`/`Uint32`). `DiscriminatorKind::Field` carries the discriminator +field's name. See [data-access.md](data-access.md) §"TUnion Dispatch" for +how these drive dispatch. + +### `$ref` resolution and normalization + +| Function | Purpose | +|----------|---------| +| `normalize_refs(schema: &mut Value)` | Walks the schema; rewrites every `"$ref"` whose value is a bare name (no `#` prefix) to `"#/$defs/"`. Idempotent. Runs once at `TypedefEngine::compile` time. | +| `resolve_ref(root, ref_path) -> Option<&Value>` | Resolves a JSON Pointer `$ref` (e.g. `"#/$defs/Read"`) against the root schema. | +| `resolve_ref_or_inline(node, root) -> Option<&Value>` | If `node` has a `"$ref"`, resolves it against `root`; otherwise returns `node` itself (it's an inline schema). | + +`normalize_refs` bridges TypeBox's bare-name ref output and `jsonschema`'s +JSON Pointer requirement. The layout engines call `resolve_ref_or_inline` +on every `$ref`-bearing node they encounter during traversal. + +## jsonschema Custom Keyword Integration + +The `jsonschema` crate (v0.46.5, Draft 2020-12) supports custom keywords +via the `with_keyword` API. Each `TypeDef:*` kind is registered as a +custom keyword: + +```rust +let validator = jsonschema::options() + .with_keyword("TypeDef:Float32", factory) + .with_keyword("TypeDef:Int32", factory) + .with_keyword("TypeDef:Struct", factory) + // ... all 17 kinds + .build(&schema)?; +``` + +The factory closure receives the parent schema object, the keyword's +value, and the schema path — enabling cross-keyword awareness. The +`TypeDef:Struct` validator, for example, inspects the parent's +`properties` to validate each field against its declared `TypeDef:*` kind. + +Each custom keyword implementation is ~10 lines. The `jsonschema` crate +handles all structural validation (object properties, required fields, +array items, enum values) — the custom keywords only need to validate +the leaf type constraints. See [validation.md](validation.md) for the +validator implementations. + +This is the same pattern as TypeBox's `TypeRegistry.Set` on the JS side. +Same semantics, different language, same JSON Schema wire format. A +TypeBox schema serialized to JSON feeds into the typedef engine after a +single pre-processing step: normalizing `$ref` values (see below). + +## TypeBox Interop + +TypeBox modules render to standard JSON Schema under `$defs`. A TypeBox +schema like: + +```typescript +const TensorRef = Type.Object({ + dtype: Type.Union([Type.Literal("F32"), Type.Literal("I16")]), + shape: Type.Array(Type.Number()), + data_offsets: Type.Tuple([Type.Number(), Type.Number()]) +}); +``` + +serialized to JSON is a standard JSON Schema with `type: "object"`, +`properties`, and `required`. That JSON feeds into the typedef engine +after `$ref` normalization. The `TypeDef:*` custom keywords are added by +TypeBox's `TypeRegistry.Set` — they appear in the serialized JSON as +additional properties on the schema object. + +### `$ref` normalization + +TypeBox generates bare-name `$ref` values (e.g., `"$ref": "Read"`), +referencing sibling definitions within the same `$defs` block. The +`jsonschema` crate requires full JSON Pointer paths (e.g., +`"$ref": "#/$defs/Read"`). The typedef engine normalizes TypeBox-style +refs at schema load time via [`normalize_refs`](#ref-resolution-and-normalization) +— a ~20-line recursive walk that rewrites every bare-name `"$ref"` to +`"#/$defs/"`. The normalization is idempotent — full JSON Pointer +refs pass through unchanged. It runs once at `TypedefEngine::compile` +time, before the schema is passed to `jsonschema` or the offset +computation. + +**Verification:** The jsonschema crate (v0.46.5) rejects bare-name refs +with `Resource 'Read' is not present in a registry`. Full JSON Pointer +refs (`#/$defs/Read`) resolve correctly. The normalization step bridges +the gap between TypeBox's output and jsonschema's input. + +The typedef engine does not depend on TypeBox or any JS toolchain. It +consumes JSON — whether that JSON was authored in TypeBox, generated by +a ujsx component, or hand-written. The schema is the interface. + +## Schema Annotations + +Schema-level annotations control binary layout behavior. These are +decided in [ADR-097](decisions/097-schema-annotations.md). + +### Endianness + +Schema-level annotation with a default of little-endian: + +```json +{ "TypeDef:Struct": true, "endian": "big", "properties": { ... } } +``` + +- `"endian": "little"` (default) — read/write in little-endian byte order. +- `"endian": "big"` — read/write in big-endian byte order. +- Applies to the entire schema and all nested types. + +### Alignment + +Both struct-level and field-level, with field-level overriding: + +```json +{ + "TypeDef:Struct": true, + "align": 256, + "properties": { + "weight": { "TypeDef:Float32": true, "align": 16 } + } +} +``` + +- Struct-level `"align"` sets the default for all fields. +- Field-level `"align"` overrides the struct default. +- Default alignment: 1 for u8/i8/bool, 2 for u16/i16, 4 for u32/i32/f32/ + enum, 8 for u64/i64/f64, 4 for variable-length (the u32 length prefix), + 1 for struct/union/array. +- Only meaningful in aligned static mode (ADR-096). Ignored in packed + sequential mode. + +### Variable-length encoding + +The typedef engine supports three strategies for variable-length types +(see §Variable-length types above for full details). The strategy is +selected by the `encoding` annotation and the standard JSON Schema +`maxLength` keyword: + +```json +// Strategy 1: Inline length-prefixing (default, shorthand) +{ "TypeDef:String": true } + +// Strategy 1: Explicit inline length-prefixing +{ "TypeDef:String": { "encoding": "length-prefixed" } } + +// Strategy 2: Fixed-size reservation (uses standard maxLength) +{ "TypeDef:String": true, "maxLength": 256 } + +// Strategy 3: Offset indirection (opt-in) +{ "TypeDef:String": { "encoding": "offset-indirect" } } +``` + +- `"encoding": "length-prefixed"` (default) — 4-byte length prefix at + computed offset, variable data follows immediately. Used by protocol + wire formats. +- `maxLength` (standard JSON Schema keyword) — in aligned static mode, + reserves `maxLength` bytes at a fixed offset (zero-padded). Makes the + field fixed-size from the layout perspective. In packed sequential + mode, `maxLength` is a validation constraint only. +- `"encoding": "offset-indirect"` — field is a struct + `{offset: u32, length: u32}` pointing into a separate data region. + The consumer provides the data region separately. Used by metatensor + blob tensors. +- Applies to all variable-length types: `TypeDef:String`, `TypeDef:Bytes`, + `TypeDef:Array`, `TypeDef:Record`, `TypeDef:Timestamp`. + +### TUnion discriminators + +Two discriminator kinds: byte-offset (protocol dispatch) and field-name +(typedef.ts pattern). + +**Byte-offset discriminator** (SFTP type bytes, call protocol event types): + +```json +{ + "TypeDef:Union": true, + "discriminator": { + "kind": "byte", + "offset": 0, + "type": "TypeDef:Uint8" + }, + "mapping": { + "5": { "$ref": "#/$defs/Read" }, + "6": { "$ref": "#/$defs/Write" }, + "101": { "$ref": "#/$defs/Status" } + } +} +``` + +- `"offset"` — byte position of the discriminator. +- `"type"` — the `TypeDef:*` kind of the discriminator (typically + `TypeDef:Uint8`). +- Mapping keys are stringified integers. The variant struct starts at + `offset + discriminator_size`. + +**Field-name discriminator** (typedef.ts pattern): + +```json +{ + "TypeDef:Union": true, + "discriminator": { + "kind": "field", + "name": "type" + }, + "mapping": { + "read": { "$ref": "#/$defs/Read" }, + "write": { "$ref": "#/$defs/Write" } + } +} +``` + +- `"name"` — the field name holding the discriminator value. +- Mapping keys are string values matching the discriminator field's value. +- The discriminator field is just another field in the struct. + +Mapping values may be either inline schemas or `$ref` pointers. Both work. + +## Design Decisions + +| Decision | ADR | Summary | +|----------|-----|---------| +| Schema annotations | [ADR-097](decisions/097-schema-annotations.md) | Concrete JSON shapes for endianness, alignment, encoding, and TUnion discriminators | +| Int64/Uint64 kinds | [ADR-099](decisions/099-int64-uint64-first-class-kinds.md) | 64-bit integers as first-class kinds (required by SFTP offsets and metatensor data_offsets) | +| Purpose and scope | [ADR-095](decisions/095-alknet-typedef-purpose-scope-jsonschema-engine.md) | Why jsonschema not a custom engine; "schema is the format" principle | + +## Open Questions + +See [open-questions.md](open-questions.md) for full details. + +- **OQ-071** (deferred(scope)): Builder API for schema construction. + +## References + +- `/workspace/@alkdev/typebox/example/typedef/typedef.ts` — the TypeBox + schema kinds (619 lines) +- `/workspace/jsonschema/` — the jsonschema crate (v0.46.5, Draft 2020-12) +- [ADR-097](decisions/097-schema-annotations.md) — schema + annotation shapes +- [validation.md](validation.md) — custom keyword validator implementations diff --git a/docs/architecture/validation.md b/docs/architecture/validation.md new file mode 100644 index 0000000..dc8251f --- /dev/null +++ b/docs/architecture/validation.md @@ -0,0 +1,335 @@ +--- +status: draft +last_updated: 2026-07-22 +--- + +# alknet-typedef — Validation + +The validation layer: custom keyword validators for all 19 `TypeDef:*` +kinds, the `TypedefError` enum, load-time vs access-time validation +strategy, and the `TypedefEngine` as the compiled form of a schema. + +## Validation Strategy + +Validation is delegated to the `jsonschema` crate (v0.46.5, Draft +2020-12). The typedef engine does not implement its own validation — +it registers custom keyword validators for each `TypeDef:*` kind and +lets `jsonschema` handle the structural validation (object properties, +required fields, array items, enum values). + +The strategy is decided in [ADR-098](decisions/098-error-handling-validation-strategy.md): + +1. **Load time:** Parse the schema JSON, build the layout engine, build the + jsonschema validator. This is the `TypedefEngine::compile(schema)` constructor. +2. **Access time:** Use the compiled engine for repeated read/write + operations. Validation is opt-in per operation. + +### What validation validates + +The jsonschema validator operates on `serde_json::Value` instances — it +validates JSON representations of data, not raw byte buffers. This is +the correct separation of concerns: + +- **JSON validation** (jsonschema): validates that a JSON document + conforms to the schema. Used for validating hand-written schemas, + TypeBox output, JSON payloads, or the JSON representation of a binary + struct after deserialization. +- **Binary access validation** (data access layer): the read/write + functions perform type-level validation at access time — range checks + for integers, UTF-8 validity for strings, buffer bounds checking. + These return `TypedefError::Access` with field paths. + +The "schema is the format" principle means the same schema describes +both the JSON shape and the binary layout. The jsonschema validator +checks the JSON shape; the data access layer checks the binary layout. +A consumer that wants to validate a binary buffer end-to-end reads the +buffer into a `Value` tree via the data access layer, then validates +that `Value` against the jsonschema validator. This is a two-step +process, not a single `validate(buffer)` call. + +### The `TypedefEngine` struct + +The `TypedefEngine` is the compiled form of a schema. It supports both +layout modes (ADR-096) via an internal `Layout` enum: + +```rust +pub struct TypedefEngine { + layout: Layout, // packed or aligned (private enum) + validator: jsonschema::Validator, // compiled once at load time + endian: Endian, // parsed from the schema's "endian" annotation + schema: Value, // the normalized schema (refs resolved) +} + +// Private — the consumer selects via LayoutMode at compile time. +enum Layout { + Packed { builder: LayoutBuilder }, + Aligned { offset_map: OffsetMap }, +} +``` + +The consumer selects the mode at construction time via `LayoutMode` +(see [layout-engine.md](layout-engine.md) §"Mode Selection"). The `Layout` +enum is private — the engine exposes mode-appropriate accessors instead: + +```rust +impl TypedefEngine { + pub fn compile(schema: &mut Value, mode: LayoutMode) -> Result; + pub fn mode(&self) -> LayoutMode; + pub fn endian(&self) -> Endian; + pub fn offset_map(&self) -> Option<&OffsetMap>; // Some in aligned mode + pub fn layout_builder(&self) -> Option<&LayoutBuilder>; // Some in packed mode + pub fn sequential_reader(&self) -> Option; // owned fresh reader (ADR-101) +} +``` + +`compile` takes `&mut Value` because it normalizes `$ref` values in place +(via [`normalize_refs`](schema-layer.md#ref-resolution-and-normalization)) +before computing the layout and building the validator. The `schema` +field retains the normalized schema for `read_field`'s kind lookup and +for `sequential_reader()`'s factory construction. The validator is +mode-agnostic (it operates on `Value`, not raw bytes). + +The `Layout::Packed` variant stores only the `LayoutBuilder` (write-side). +The `SequentialReader` (read-side) is not stored — it has mutable cursor +state that the consumer owns, so `sequential_reader()` constructs a fresh +reader on each call (ADR-101). + +The `read_field`/`write_field` methods on `TypedefEngine` are the +aligned-mode data-access API — see [data-access.md](data-access.md) +§"Higher-level read/write". + +## Custom Keyword Validators + +Each `TypeDef:*` kind gets a `Keyword` implementation registered via +`jsonschema::options().with_keyword(...)`. The validators check leaf +type constraints; `jsonschema` handles all structural validation. + +### Numeric type validators + +**`TypeDef:Float32` / `TypeDef:Float64`:** +- Value must be a finite number. +- For `Float32`: value must be representable as `f32` (no precision loss + beyond `f32`'s mantissa). + +**`TypeDef:Int8` / `TypeDef:Int16` / `TypeDef:Int32`:** +- Value must be an integer within the type's range. +- Int8: -128..127, Int16: -32768..32767, Int32: -2147483648..2147483647. + +**`TypeDef:Uint8` / `TypeDef:Uint16` / `TypeDef:Uint32`:** +- Value must be a non-negative integer within the type's range. +- Uint8: 0..255, Uint16: 0..65535, Uint32: 0..4294967295. + +### String and binary validators + +**`TypeDef:String`:** +- Value must be a valid UTF-8 string. +- If `maxLength` is specified in the schema, the string's byte length + must not exceed it. + +**`TypeDef:Bytes`:** +- Value must be a string (JSON represents binary data as a string — JSON + has no native byte type). +- If `maxLength` is specified, the byte length must not exceed it. +- **Binary representation:** In the binary layout, `TBytes` is raw bytes + with no encoding (not base64, not hex). The JSON representation (for + validation) uses a string; the binary representation (for data access) + uses `&[u8]` directly. + +**`TypeDef:Enum`:** +- The `TypeDef:Enum` custom keyword signals that the type is an enum for + *layout* purposes (the engine needs to know it's a fixed-size u32 index, + not a variable-length string). The built-in `enum` keyword provides the + value list and handles value-membership validation. The custom keyword + validator is a no-op beyond the built-in check — it exists solely for + the layout engine to recognize the type. + +**`TypeDef:Timestamp`:** +- Value must be a valid RFC 3339 timestamp string (the internet profile + of ISO 8601, e.g., `"2026-07-20T15:30:00Z"`). + +### Composite type validators + +**`TypeDef:Struct`:** +- Value must be an object. +- Each property must match its declared `TypeDef:*` kind. +- Required fields must be present. +- The `jsonschema` crate's built-in `properties` and `required` keywords + handle the structural checks — the custom keyword only needs to + validate that each field's value matches its `TypeDef:*` kind. + +**`TypeDef:Union`:** +- The discriminator value must be one of the mapping keys. +- The variant struct must match the declared schema for that discriminator + value. + +**`TypeDef:Array`:** +- Value must be an array. +- Each element must match the array's declared element type. +- If `minItems`/`maxItems` is specified, the array length must be within + bounds. + +### Other validators + +**`TypeDef:Boolean`:** +- Value must be `true` or `false`. + +**`TypeDef:Record`:** +- Value must be an object. +- All values must match the record's declared value type (specified via + the `"values"` property in the schema, e.g., + `"values": { "TypeDef:Float32": true }`). + +### Validator implementation pattern + +Each custom keyword implementation is ~10 lines. Example for +`TypeDef:Float32`: + +```rust +struct Float32Validator; + +impl Keyword for Float32Validator { + fn validate<'i>(&self, instance: &'i Value) -> Result<(), ValidationError<'i>> { + match instance { + Value::Number(n) if n.as_f64().map_or(false, |f| f.is_finite()) => Ok(()), + _ => Err(ValidationError::custom("expected finite f32-compatible number")), + } + } + fn is_valid(&self, instance: &Value) -> bool { + instance.as_f64().map_or(false, |f| f.is_finite()) + } +} +``` + +Registration: + +```rust +let validator = jsonschema::options() + .with_keyword("TypeDef:Float32", |parent, value, path| { + Ok(Box::new(Float32Validator)) + }) + .build(&schema)?; +``` + +The factory closure receives the parent schema object, the keyword's +value, and the schema path. This enables cross-keyword awareness — for +example, a `TypeDef:Struct` validator can inspect the parent's +`properties` to validate each field against its declared `TypeDef:*` kind. + +## TypedefError + +A single `TypedefError` enum covers all error conditions across the +engine's three phases (schema parsing, offset computation, read/write) +plus validation. Decided in [ADR-098](decisions/098-error-handling-validation-strategy.md). + +```rust +pub enum TypedefError { + /// Schema parsing errors (invalid JSON, missing keywords, unknown TypeDef kinds). + Schema(String), + /// Offset computation errors (field not found, unsupported type). + Offset { field_path: String, reason: String }, + /// Read/write errors (buffer too short, invalid UTF-8, value out of range). + Access { field_path: String, reason: String }, + /// Validation errors (delegated to jsonschema). + Validation(ValidationError<'static>), +} +``` + +- **`Schema`** — for errors during `TypedefEngine::compile()`. Invalid + JSON, missing required keywords, unknown `TypeDef:*` kinds. +- **`Offset`** — for errors during offset computation. Field not found + in the schema, type not supported for offset computation, recursive + depth exceeded. Carries the field path. +- **`Access`** — for errors during read/write. Buffer too short, invalid + UTF-8 in a string field, value out of range for the target type. + Carries the field path. +- **`Validation`** — wraps `jsonschema`'s `ValidationError`. The + `'static` lifetime is correct — the validator owns its schema reference + and lives for the lifetime of the `TypedefEngine`. + +### Field-path-carrying errors + +Read/write and offset errors include the field path for debugging: + +```rust +Err(TypedefError::Access { + field_path: "header.version".to_string(), + reason: "buffer too short: need 4 bytes at offset 12, have 2".to_string(), +}) +``` + +This makes debugging binary format issues tractable — the error tells +you exactly which field failed and why. + +## Validation Timing + +### Load time: `TypedefEngine::compile()` + +The expensive work happens once at schema load time: +1. Normalize `$ref` values in the schema (`normalize_refs`). +2. Parse the schema's `"endian"` annotation. +3. Compute the layout (`LayoutBuilder`/`SequentialReader` for packed, `OffsetMap` for aligned). +4. Build the jsonschema validator (`jsonschema::options().with_keyword(...).build(&schema)?`). + +The result is a `TypedefEngine` that can be used for repeated operations. + +### Access time: `engine.validate_json(&Value)` / `engine.is_valid_json(&Value)` + +Validation is opt-in per operation. The consumer calls +`engine.validate_json(instance)` when validation is desired, or +`engine.is_valid_json(instance)` for a boolean check. The jsonschema +validator is already compiled — these are fast checks against the +compiled validator. + +```rust +pub fn validate_json(&self, instance: &Value) -> Result<(), TypedefError>; +pub fn is_valid_json(&self, instance: &Value) -> bool; +``` + +The argument is a `serde_json::Value` (the JSON representation of the +data), not a raw byte buffer — see §"What validation validates" above. +To validate a binary buffer end-to-end, the consumer reads it into a +`Value` tree via the data access layer, then validates that `Value`. + +High-throughput paths can skip validation. Security-sensitive paths +(parsing incoming frames from untrusted peers) can validate every frame. +The choice is the consumer's. + +## Relationship to Read/Write + +Validation and data access are independent operations on the same data. +The consumer can: + +1. Validate the JSON representation of a buffer to ensure it conforms to + the schema. +2. Read fields from the binary buffer at computed offsets. +3. Both — validate the JSON representation first, then read the binary + buffer (defense in depth). + +The engine does not couple validation and access. A consumer that trusts +its data source can skip validation and go straight to read/write. A +consumer that parses untrusted input can validate the JSON +representation first, then access the binary buffer. + +## Design Decisions + +| Decision | ADR | Summary | +|----------|-----|---------| +| Error handling and validation | [ADR-098](decisions/098-error-handling-validation-strategy.md) | `TypedefError` enum; load-time build, access-time check; field-path-carrying errors; jsonschema `ValidationError` wrapping | +| Purpose and scope | [ADR-095](decisions/095-alknet-typedef-purpose-scope-jsonschema-engine.md) | Why jsonschema not a custom engine | + +## Open Questions + +None specific to validation. The three typedef OQs (OQ-069, OQ-070, +OQ-071) are about layout, platform support, and schema construction — +not validation. + +## References + +- `docs/research/alknet-typedef/findings.md` §"Validation" — the POC's + custom keyword validators for all 17 kinds +- [ADR-098](decisions/098-error-handling-validation-strategy.md) — + error handling and validation strategy +- [schema-layer.md](schema-layer.md) — the 17 TypeDef kinds that the + validators check +- [data-access.md](data-access.md) — read/write functions that operate + on the same buffers diff --git a/src/data_access.rs b/src/data_access.rs new file mode 100644 index 0000000..b68a72b --- /dev/null +++ b/src/data_access.rs @@ -0,0 +1,632 @@ +//! Data access layer: primitive read/write functions for all 17 TypeDef +//! kinds with endianness support, bounds checking, and zero-copy access. +//! +//! These are the building blocks used by the layout types ([`crate::offset_map`], +//! [`crate::layout_builder`], [`crate::sequential_reader`]) and the +//! [`crate::engine::TypedefEngine`]. Each function operates on a raw byte +//! buffer at a caller-provided offset and returns a [`TypedefError::Access`] +//! carrying the field path on bounds or encoding failures. +//! +//! # Conventions +//! +//! - All multi-byte types respect the [`Endian`] parameter passed by the caller. +//! - Bounds checks ensure `buffer.len() >= offset + size`; failures produce +//! [`TypedefError::Access`] with a descriptive `reason`. +//! - Read functions for variable-length types return slices borrowing from +//! the input buffer — no allocation. +//! - No `unwrap()` / `expect()` on fallible operations. + +use crate::error::TypedefError; +use crate::schema::Endian; + +const U32_SIZE: usize = 4; + +fn check_bounds( + buffer_len: usize, + start: usize, + end: usize, + field_path: &str, +) -> Result<(), TypedefError> { + if end < start || buffer_len < end { + return Err(TypedefError::Access { + field_path: field_path.to_string(), + reason: format!( + "buffer bounds check failed: need bytes [{start}..{end}), buffer has {buffer_len}" + ), + }); + } + Ok(()) +} + +fn access_err(field_path: &str, reason: impl Into) -> TypedefError { + TypedefError::Access { + field_path: field_path.to_string(), + reason: reason.into(), + } +} + +pub(crate) fn read_array( + buffer: &[u8], + offset: usize, + field_path: &str, +) -> Result<[u8; N], TypedefError> { + let end = offset.checked_add(N).ok_or_else(|| { + access_err( + field_path, + format!("offset {offset} + size {N} overflows usize"), + ) + })?; + check_bounds(buffer.len(), offset, end, field_path)?; + let slice = buffer.get(offset..end).ok_or_else(|| { + access_err( + field_path, + format!( + "slice [{offset}..{end}) unavailable in buffer of length {}", + buffer.len() + ), + ) + })?; + slice.try_into().map_err(|_| { + access_err( + field_path, + format!("internal: try_into failed for {N}-byte slice"), + ) + }) +} + +pub(crate) fn write_array( + buffer: &mut [u8], + offset: usize, + bytes: [u8; N], + field_path: &str, +) -> Result<(), TypedefError> { + let end = offset.checked_add(N).ok_or_else(|| { + access_err( + field_path, + format!("offset {offset} + size {N} overflows usize"), + ) + })?; + check_bounds(buffer.len(), offset, end, field_path)?; + let dest = buffer.get_mut(offset..end).ok_or_else(|| { + access_err( + field_path, + format!("mutable slice [{offset}..{end}) unavailable"), + ) + })?; + dest.copy_from_slice(&bytes); + Ok(()) +} + +fn u32_from(bytes: [u8; U32_SIZE], endian: Endian) -> u32 { + match endian { + Endian::Little => u32::from_le_bytes(bytes), + Endian::Big => u32::from_be_bytes(bytes), + } +} + +fn u32_to(value: u32, endian: Endian) -> [u8; U32_SIZE] { + match endian { + Endian::Little => value.to_le_bytes(), + Endian::Big => value.to_be_bytes(), + } +} + +// --------------------------------------------------------------------------- +// Fixed-size read functions +// --------------------------------------------------------------------------- + +define_read_write_ne!(i8, read_i8, write_i8, 1, |bytes: [u8; 1]| bytes[0] as i8); +define_read_write_endian!(i16, read_i16, write_i16, 2); +define_read_write_endian!(i32, read_i32, write_i32, 4); +define_read_write_endian!(i64, read_i64, write_i64, 8); +define_read_write_ne!(u8, read_u8, write_u8, 1, |bytes: [u8; 1]| bytes[0]); +define_read_write_endian!(u16, read_u16, write_u16, 2); +define_read_write_endian!(u32, read_u32, write_u32, 4); +define_read_write_endian!(u64, read_u64, write_u64, 8); +define_read_write_endian!(f32, read_f32, write_f32, 4); +define_read_write_endian!(f64, read_f64, write_f64, 8); + +/// Read a `bool` at `offset` from `buffer`. +/// +/// `0x00` decodes to `false`, `0x01` decodes to `true`. Any other byte value +/// produces [`TypedefError::Access`] with a reason of the form +/// `"invalid boolean byte 0x02 at offset {offset}"`. +pub fn read_bool(buffer: &[u8], offset: usize, field_path: &str) -> Result { + let bytes: [u8; 1] = read_array(buffer, offset, field_path)?; + match bytes[0] { + 0x00 => Ok(false), + 0x01 => Ok(true), + other => Err(access_err( + field_path, + format!("invalid boolean byte 0x{other:02X} at offset {offset}"), + )), + } +} + +/// Write a `bool` `value` at `offset` into `buffer`. +/// +/// `false` is encoded as `0x00`, `true` as `0x01`. +pub fn write_bool( + buffer: &mut [u8], + offset: usize, + value: bool, + field_path: &str, +) -> Result<(), TypedefError> { + write_array( + buffer, + offset, + [if value { 0x01 } else { 0x00 }], + field_path, + ) +} + +/// Read a `TEnum` index (`u32`) at `offset` from `buffer`, applying `endian`. +/// +/// The caller maps the returned index to the schema's `"enum"` array entry. +pub fn read_enum( + buffer: &[u8], + offset: usize, + field_path: &str, + endian: Endian, +) -> Result { + read_u32(buffer, offset, field_path, endian) +} + +/// Write a `TEnum` index (`u32`) `value` at `offset` into `buffer`, applying `endian`. +pub fn write_enum( + buffer: &mut [u8], + offset: usize, + value: u32, + field_path: &str, + endian: Endian, +) -> Result<(), TypedefError> { + write_u32(buffer, offset, value, field_path, endian) +} + +// --------------------------------------------------------------------------- +// Variable-length read/write (inline length-prefixing) +// --------------------------------------------------------------------------- + +/// Read a length-prefixed UTF-8 string borrowing from `buffer`. +/// +/// Wire format: `[length: u32][UTF-8 bytes]`. The length prefix respects +/// `endian`. Returns a `&'a str` that borrows from the input buffer — no +/// allocation. Invalid UTF-8 produces [`TypedefError::Access`]. +pub fn read_string<'a>( + buffer: &'a [u8], + offset: usize, + field_path: &str, + endian: Endian, +) -> Result<&'a str, TypedefError> { + let bytes = read_bytes(buffer, offset, field_path, endian)?; + std::str::from_utf8(bytes).map_err(|e| { + access_err( + field_path, + format!("invalid UTF-8 in string at offset {offset}: {e}"), + ) + }) +} + +/// Read length-prefixed raw bytes borrowing from `buffer`. +/// +/// Wire format: `[length: u32][raw bytes]`. The length prefix respects +/// `endian`. Returns a `&'a [u8]` slice that borrows from the input buffer. +pub fn read_bytes<'a>( + buffer: &'a [u8], + offset: usize, + field_path: &str, + endian: Endian, +) -> Result<&'a [u8], TypedefError> { + let len_bytes: [u8; U32_SIZE] = read_array(buffer, offset, field_path)?; + let len = u32_from(len_bytes, endian) as usize; + let data_start = offset.checked_add(U32_SIZE).ok_or_else(|| { + access_err( + field_path, + format!("offset {offset} + {U32_SIZE} overflows usize"), + ) + })?; + let data_end = data_start.checked_add(len).ok_or_else(|| { + access_err( + field_path, + format!("data_start {data_start} + length {len} overflows usize"), + ) + })?; + check_bounds(buffer.len(), data_start, data_end, field_path)?; + Ok(&buffer[data_start..data_end]) +} + +/// Write a length-prefixed UTF-8 string into `buffer` at `offset`. +/// +/// Wire format: `[length: u32][UTF-8 bytes]`. The length prefix respects +/// `endian`. Returns the total number of bytes written +/// (`4 + value.len()`) so the caller can advance the cursor. +pub fn write_string( + buffer: &mut [u8], + offset: usize, + value: &str, + field_path: &str, + endian: Endian, +) -> Result { + write_bytes(buffer, offset, value.as_bytes(), field_path, endian) +} + +/// Write length-prefixed raw bytes into `buffer` at `offset`. +/// +/// Wire format: `[length: u32][raw bytes]`. The length prefix respects +/// `endian`. Returns the total number of bytes written (`4 + value.len()`). +pub fn write_bytes( + buffer: &mut [u8], + offset: usize, + value: &[u8], + field_path: &str, + endian: Endian, +) -> Result { + let data_len = value.len(); + let total = U32_SIZE.checked_add(data_len).ok_or_else(|| { + access_err( + field_path, + format!("prefix {U32_SIZE} + data length {data_len} overflows usize"), + ) + })?; + let end = offset.checked_add(total).ok_or_else(|| { + access_err( + field_path, + format!("offset {offset} + total {total} overflows usize"), + ) + })?; + check_bounds(buffer.len(), offset, end, field_path)?; + write_array(buffer, offset, u32_to(data_len as u32, endian), field_path)?; + let data_start = offset + U32_SIZE; + let dest = buffer.get_mut(data_start..end).ok_or_else(|| { + access_err( + field_path, + format!("mutable data slice [{data_start}..{end}) unavailable"), + ) + })?; + dest.copy_from_slice(value); + Ok(total) +} + +// --------------------------------------------------------------------------- +// Variable-length read (offset indirection) +// --------------------------------------------------------------------------- + +/// Read an offset-indirect string. +/// +/// The 8-byte struct at `buffer[offset..offset+8]` is +/// `{ data_offset: u32, data_length: u32 }` (endian-aware). The actual UTF-8 +/// bytes live in `data_region[data_offset..data_offset+data_length]`. Returns +/// a `&'a str` borrowing from `data_region`. Invalid UTF-8 produces +/// [`TypedefError::Access`]. +pub fn read_string_indirect<'a>( + buffer: &'a [u8], + offset: usize, + data_region: &'a [u8], + field_path: &str, + endian: Endian, +) -> Result<&'a str, TypedefError> { + let bytes = read_bytes_indirect(buffer, offset, data_region, field_path, endian)?; + std::str::from_utf8(bytes).map_err(|e| { + access_err( + field_path, + format!("invalid UTF-8 in offset-indirect string: {e}"), + ) + }) +} + +/// Read offset-indirect raw bytes. +/// +/// The 8-byte struct at `buffer[offset..offset+8]` is +/// `{ data_offset: u32, data_length: u32 }` (endian-aware). Returns a +/// `&'a [u8]` slice of `data_region[data_offset..data_offset+data_length]`. +pub fn read_bytes_indirect<'a>( + buffer: &'a [u8], + offset: usize, + data_region: &'a [u8], + field_path: &str, + endian: Endian, +) -> Result<&'a [u8], TypedefError> { + let struct_end = offset + .checked_add(8) + .ok_or_else(|| access_err(field_path, format!("offset {offset} + 8 overflows usize")))?; + check_bounds(buffer.len(), offset, struct_end, field_path)?; + let off_bytes: [u8; U32_SIZE] = buffer[offset..offset + U32_SIZE] + .try_into() + .map_err(|_| access_err(field_path, "internal: try_into failed for data_offset"))?; + let len_bytes: [u8; U32_SIZE] = buffer[offset + U32_SIZE..offset + 8] + .try_into() + .map_err(|_| access_err(field_path, "internal: try_into failed for data_length"))?; + let data_offset = u32_from(off_bytes, endian) as usize; + let data_length = u32_from(len_bytes, endian) as usize; + let data_end = data_offset.checked_add(data_length).ok_or_else(|| { + access_err( + field_path, + format!("data_offset {data_offset} + data_length {data_length} overflows usize"), + ) + })?; + check_bounds(data_region.len(), data_offset, data_end, field_path)?; + Ok(&data_region[data_offset..data_end]) +} + +#[cfg(test)] +mod tests { + use super::*; + + const LE: Endian = Endian::Little; + const BE: Endian = Endian::Big; + + #[test] + fn read_write_u8_round_trip() { + let mut buf = [0u8; 1]; + write_u8(&mut buf, 0, 0xAB, "f").unwrap(); + assert_eq!(read_u8(&buf, 0, "f").unwrap(), 0xAB); + } + + #[test] + fn read_write_i8_round_trip() { + let mut buf = [0u8; 1]; + write_i8(&mut buf, 0, -42, "f").unwrap(); + assert_eq!(read_i8(&buf, 0, "f").unwrap(), -42); + } + + #[test] + fn read_write_u16_endianness() { + let mut buf = [0u8; 2]; + write_u16(&mut buf, 0, 0x1234, "f", LE).unwrap(); + assert_eq!(buf, [0x34, 0x12]); + assert_eq!(read_u16(&buf, 0, "f", LE).unwrap(), 0x1234); + + write_u16(&mut buf, 0, 0x1234, "f", BE).unwrap(); + assert_eq!(buf, [0x12, 0x34]); + assert_eq!(read_u16(&buf, 0, "f", BE).unwrap(), 0x1234); + } + + #[test] + fn read_write_i16_endianness() { + let mut buf = [0u8; 2]; + write_i16(&mut buf, 0, -1, "f", LE).unwrap(); + assert_eq!(buf, [0xFF, 0xFF]); + assert_eq!(read_i16(&buf, 0, "f", LE).unwrap(), -1); + } + + #[test] + fn read_write_u32_endianness() { + let mut buf = [0u8; 4]; + write_u32(&mut buf, 0, 0x01020304, "f", LE).unwrap(); + assert_eq!(buf, [0x04, 0x03, 0x02, 0x01]); + assert_eq!(read_u32(&buf, 0, "f", LE).unwrap(), 0x01020304); + + write_u32(&mut buf, 0, 0x01020304, "f", BE).unwrap(); + assert_eq!(buf, [0x01, 0x02, 0x03, 0x04]); + assert_eq!(read_u32(&buf, 0, "f", BE).unwrap(), 0x01020304); + } + + #[test] + fn read_write_i32_endianness() { + let mut buf = [0u8; 4]; + write_i32(&mut buf, 0, i32::MIN, "f", BE).unwrap(); + assert_eq!(read_i32(&buf, 0, "f", BE).unwrap(), i32::MIN); + } + + #[test] + fn read_write_i64_endianness() { + let mut buf = [0u8; 8]; + write_i64(&mut buf, 0, i64::MIN, "f", BE).unwrap(); + assert_eq!(read_i64(&buf, 0, "f", BE).unwrap(), i64::MIN); + write_i64(&mut buf, 0, i64::MAX, "f", LE).unwrap(); + assert_eq!(read_i64(&buf, 0, "f", LE).unwrap(), i64::MAX); + } + + #[test] + fn read_write_u64_endianness() { + let mut buf = [0u8; 8]; + write_u64(&mut buf, 0, 0x0102030405060708, "f", LE).unwrap(); + assert_eq!(buf, [0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01]); + assert_eq!(read_u64(&buf, 0, "f", LE).unwrap(), 0x0102030405060708); + + write_u64(&mut buf, 0, 0x0102030405060708, "f", BE).unwrap(); + assert_eq!(buf, [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]); + assert_eq!(read_u64(&buf, 0, "f", BE).unwrap(), 0x0102030405060708); + } + + #[test] + fn read_write_f32_round_trip() { + let mut buf = [0u8; 4]; + let value: f32 = std::f32::consts::PI; + write_f32(&mut buf, 0, value, "f", LE).unwrap(); + let read = read_f32(&buf, 0, "f", LE).unwrap(); + assert!( + (read - value).abs() < 1e-6, + "le mismatch: {read} vs {value}" + ); + + write_f32(&mut buf, 0, value, "f", BE).unwrap(); + let read = read_f32(&buf, 0, "f", BE).unwrap(); + assert!( + (read - value).abs() < 1e-6, + "be mismatch: {read} vs {value}" + ); + } + + #[test] + fn read_write_f64_round_trip() { + let mut buf = [0u8; 8]; + let value: f64 = std::f64::consts::PI; + write_f64(&mut buf, 0, value, "f", LE).unwrap(); + assert_eq!(read_f64(&buf, 0, "f", LE).unwrap(), value); + + write_f64(&mut buf, 0, value, "f", BE).unwrap(); + assert_eq!(read_f64(&buf, 0, "f", BE).unwrap(), value); + } + + #[test] + fn read_write_bool_round_trip() { + let mut buf = [0u8; 1]; + write_bool(&mut buf, 0, false, "f").unwrap(); + assert_eq!(buf[0], 0x00); + assert!(!read_bool(&buf, 0, "f").unwrap()); + + write_bool(&mut buf, 0, true, "f").unwrap(); + assert_eq!(buf[0], 0x01); + assert!(read_bool(&buf, 0, "f").unwrap()); + } + + #[test] + fn read_bool_rejects_invalid_byte() { + let buf = [0x02u8]; + let err = read_bool(&buf, 0, "f").unwrap_err(); + match err { + TypedefError::Access { field_path, reason } => { + assert_eq!(field_path, "f"); + assert!(reason.contains("0x02"), "reason: {reason}"); + assert!(reason.contains("offset 0"), "reason: {reason}"); + } + other => panic!("expected Access, got {other:?}"), + } + } + + #[test] + fn read_write_enum_round_trip() { + let mut buf = [0u8; 4]; + write_enum(&mut buf, 0, 7, "f", LE).unwrap(); + assert_eq!(read_enum(&buf, 0, "f", LE).unwrap(), 7); + + write_enum(&mut buf, 0, 7, "f", BE).unwrap(); + assert_eq!(read_enum(&buf, 0, "f", BE).unwrap(), 7); + } + + #[test] + fn bounds_failure_returns_access_error() { + let buf = [0u8; 2]; + let err = read_u32(&buf, 0, "header.id", LE).unwrap_err(); + match err { + TypedefError::Access { field_path, reason } => { + assert_eq!(field_path, "header.id"); + assert!(reason.contains("bounds"), "reason: {reason}"); + } + other => panic!("expected Access, got {other:?}"), + } + } + + #[test] + fn write_bounds_failure_returns_access_error() { + let mut buf = [0u8; 2]; + let err = write_u32(&mut buf, 0, 1, "header.id", LE).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. })); + } + + #[test] + fn read_string_round_trip_and_zero_copy() { + let mut buf = vec![0u8; 32]; + let written = write_string(&mut buf, 0, "hello", "name", LE).unwrap(); + assert_eq!(written, 4 + 5); + let s = read_string(&buf, 0, "name", LE).unwrap(); + assert_eq!(s, "hello"); + assert!(std::ptr::eq(s.as_ptr(), buf.as_ptr().wrapping_add(4))); + } + + #[test] + fn read_string_be_length_prefix() { + let mut buf = vec![0u8; 16]; + write_string(&mut buf, 0, "abc", "name", BE).unwrap(); + assert_eq!(buf[0..4], [0x00, 0x00, 0x00, 0x03]); + assert_eq!(read_string(&buf, 0, "name", BE).unwrap(), "abc"); + } + + #[test] + fn read_bytes_round_trip_and_zero_copy() { + let mut buf = vec![0u8; 32]; + let payload = [0xAA, 0xBB, 0xCC, 0xDD]; + let written = write_bytes(&mut buf, 0, &payload, "data", LE).unwrap(); + assert_eq!(written, 4 + 4); + let bytes = read_bytes(&buf, 0, "data", LE).unwrap(); + assert_eq!(bytes, &payload[..]); + } + + #[test] + fn read_string_invalid_utf8() { + let mut buf = vec![0u8; 16]; + write_bytes(&mut buf, 0, &[0xFF, 0xFE, 0xFD], "name", LE).unwrap(); + let err = read_string(&buf, 0, "name", LE).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. })); + } + + #[test] + fn read_string_bounds_failure_on_prefix() { + let buf = [0u8; 2]; + let err = read_string(&buf, 0, "name", LE).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. })); + } + + #[test] + fn read_string_bounds_failure_on_data() { + let mut buf = vec![0u8; 6]; + let _ = write_bytes(&mut buf, 0, &[0x00; 32], "name", LE); + let len_bytes = (100u32).to_le_bytes(); + buf[0..4].copy_from_slice(&len_bytes); + let err = read_string(&buf, 0, "name", LE).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. })); + } + + #[test] + fn write_string_bounds_failure() { + let mut buf = vec![0u8; 4]; + let err = write_string(&mut buf, 0, "hello", "name", LE).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. })); + } + + #[test] + fn read_string_indirect_round_trip() { + let data_region = b"the quick brown fox"; + let mut index = [0u8; 8]; + write_u32(&mut index, 0, 4, "idx.off", LE).unwrap(); + write_u32(&mut index, 4, 11, "idx.len", LE).unwrap(); + let s = read_string_indirect(&index, 0, data_region, "msg", LE).unwrap(); + assert_eq!(s, "quick brown"); + } + + #[test] + fn read_bytes_indirect_round_trip() { + let data_region: &[u8] = b"HEADERbody-payloadTAIL"; + let mut index = [0u8; 8]; + write_u32(&mut index, 0, 6, "idx.off", BE).unwrap(); + write_u32(&mut index, 4, 12, "idx.len", BE).unwrap(); + let bytes = read_bytes_indirect(&index, 0, data_region, "blob", BE).unwrap(); + assert_eq!(bytes, b"body-payload"); + } + + #[test] + fn read_bytes_indirect_bounds_failure_on_index() { + let buf = [0u8; 4]; + let data_region = b"anything"; + let err = read_bytes_indirect(&buf, 0, data_region, "blob", LE).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. })); + } + + #[test] + fn read_bytes_indirect_bounds_failure_on_data_region() { + let mut buf = [0u8; 8]; + write_u32(&mut buf, 0, 100, "idx.off", LE).unwrap(); + write_u32(&mut buf, 4, 10, "idx.len", LE).unwrap(); + let data_region = b"too short"; + let err = read_bytes_indirect(&buf, 0, data_region, "blob", LE).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. })); + } + + #[test] + fn read_at_nonzero_offset() { + let mut buf = vec![0u8; 16]; + write_u32(&mut buf, 8, 0xDEADBEEF, "header.id", BE).unwrap(); + assert_eq!(read_u32(&buf, 8, "header.id", BE).unwrap(), 0xDEADBEEF); + } + + #[test] + fn write_bytes_zero_length() { + let mut buf = vec![0u8; 8]; + let written = write_bytes(&mut buf, 0, &[], "data", LE).unwrap(); + assert_eq!(written, 4); + assert_eq!(buf[0..4], [0, 0, 0, 0]); + let bytes = read_bytes(&buf, 0, "data", LE).unwrap(); + assert!(bytes.is_empty()); + } +} diff --git a/src/engine.rs b/src/engine.rs new file mode 100644 index 0000000..2abed79 --- /dev/null +++ b/src/engine.rs @@ -0,0 +1,754 @@ +//! `TypedefEngine` — the compiled form of a schema. +//! +//! Combines the layout engine (both packed and aligned modes) and the +//! jsonschema validator into a single struct. Built once at schema load +//! time via [`TypedefEngine::compile`]. Used for repeated read/write/ +//! validate operations at access time. +//! +//! See [validation.md](../../docs/architecture/crates/typedef/validation.md) +//! §"The TypedefEngine struct" and +//! [overview.md](../../docs/architecture/crates/typedef/overview.md). + +use crate::data_access; +use crate::error::TypedefError; +use crate::layout_builder::LayoutBuilder; +use crate::offset_map::OffsetMap; +use crate::schema::{self, get_typedef_kind_loose_enum, Endian, TypeDefKind}; +use crate::sequential_reader::{FieldValue, SequentialReader}; +use crate::validation; +use serde_json::Value; +use std::fmt; + +/// The layout mode selected at engine construction time. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LayoutMode { + /// Packed sequential — for protocol wire formats (SFTP, channels, TTY). + Packed, + /// Aligned static — for mmap-friendly formats (metatensor, safetensors). + Aligned, +} + +/// The layout strategy — packed sequential or aligned static. +/// +/// Carries the layout-specific handles needed for read/write access in +/// the selected mode. The consumer chooses the mode at construction time +/// via [`TypedefEngine::compile`]; the engine then exposes only the +/// APIs that make sense for that mode. +#[derive(Debug)] +enum Layout { + /// Packed sequential layout. The write-side is [`LayoutBuilder`]; the + /// read-side is a fresh [`SequentialReader`] constructed on demand + /// (ADR-101 — the reader has mutable cursor state that the consumer + /// owns, so the engine is a factory, not a holder). + Packed { + builder: LayoutBuilder, + }, + /// Aligned static layout. Field offsets are precomputed in an + /// [`OffsetMap`] for random access. + Aligned { offset_map: OffsetMap }, +} + +/// The compiled form of a typedef schema. Combines the layout engine +/// (both packed and aligned modes) and the jsonschema validator. +/// +/// Built once at schema load time via [`TypedefEngine::compile`]. +/// Used for repeated read/write/validate operations at access time. +/// +/// The consumer selects the layout mode at construction time. The engine +/// then exposes mode-appropriate accessors: [`TypedefEngine::offset_map`] +/// for aligned mode, [`TypedefEngine::layout_builder`] and +/// [`TypedefEngine::sequential_reader`] for packed mode. The +/// jsonschema validator is mode-agnostic and always available. +pub struct TypedefEngine { + layout: Layout, + validator: jsonschema::Validator, + endian: Endian, + schema: Value, +} + +impl TypedefEngine { + /// Compile a schema into a [`TypedefEngine`]. + /// + /// This is the expensive operation — it parses the schema, normalizes + /// `$ref` values, computes the layout, and builds the jsonschema + /// validator. Call once at load time; use the returned engine for + /// repeated operations. + /// + /// The `mode` parameter selects the layout strategy. The same schema + /// can be compiled in either mode. + /// + /// # Errors + /// + /// Returns [`TypedefError::Schema`] if the schema is malformed or the + /// underlying layout/validator construction fails. The error is + /// propagated from [`LayoutBuilder::new`], [`SequentialReader::new`], + /// [`OffsetMap::compute`], or [`validation::build_validator`]. + pub fn compile(schema: &mut Value, mode: LayoutMode) -> Result { + schema::normalize_refs(schema); + let endian = Endian::from_schema(schema); + let layout = match mode { + LayoutMode::Packed => { + let builder = LayoutBuilder::new(schema)?; + Layout::Packed { builder } + } + LayoutMode::Aligned => { + let offset_map = OffsetMap::compute(schema)?; + Layout::Aligned { offset_map } + } + }; + let validator = validation::build_validator(schema)?; + Ok(Self { + layout, + validator, + endian, + schema: schema.clone(), + }) + } + + /// The schema's endianness. + pub fn endian(&self) -> Endian { + self.endian + } + + /// The layout mode this engine was compiled with. + pub fn mode(&self) -> LayoutMode { + match self.layout { + Layout::Packed { .. } => LayoutMode::Packed, + Layout::Aligned { .. } => LayoutMode::Aligned, + } + } + + /// Access the aligned offset map. Returns `None` if compiled in + /// packed mode. + pub fn offset_map(&self) -> Option<&OffsetMap> { + match &self.layout { + Layout::Aligned { offset_map } => Some(offset_map), + Layout::Packed { .. } => None, + } + } + + /// Access the layout builder (write-side of packed mode). + /// Returns `None` if compiled in aligned mode. + pub fn layout_builder(&self) -> Option<&LayoutBuilder> { + match &self.layout { + Layout::Packed { builder, .. } => Some(builder), + Layout::Aligned { .. } => None, + } + } + + /// Construct a fresh [`SequentialReader`] for packed-mode reads + /// (ADR-101). Each call returns a new reader with the cursor at + /// position 0. The consumer owns the reader and calls + /// `read_next`/`read_field`/`reset` on it directly. + /// + /// Returns `None` if compiled in aligned mode. + pub fn sequential_reader(&self) -> Option { + match &self.layout { + Layout::Packed { .. } => SequentialReader::new(&self.schema).ok(), + Layout::Aligned { .. } => None, + } + } + + /// Validate a JSON value against the schema. The jsonschema validator + /// is already compiled — this is a fast check. + /// + /// Returns `Ok(())` if valid, `Err(TypedefError::Validation(...))` if + /// invalid. + pub fn validate_json(&self, instance: &Value) -> Result<(), TypedefError> { + self.validator + .validate(instance) + .map_err(|e| TypedefError::Validation(e.to_owned())) + } + + /// Check if a JSON value is valid against the schema. + pub fn is_valid_json(&self, instance: &Value) -> bool { + self.validator.is_valid(instance) + } + + /// Read a field from a buffer at its computed offset (aligned mode). + /// + /// Looks up the field's byte range in the [`OffsetMap`] and reads the + /// appropriate type using the [`crate::data_access`] functions. Works + /// for fixed-size primitive kinds and length-prefixed `String`/ + /// `Bytes`/`Timestamp` fields. + /// + /// Returns an error if compiled in packed mode — use + /// [`TypedefEngine::sequential_reader`] for packed mode. Also + /// returns an error for composite kinds (`Struct`, `Union`, `Array`, + /// `Record`) — those are better handled via the layout-specific APIs. + /// + /// # Errors + /// + /// - [`TypedefError::Access`] if compiled in packed mode. + /// - [`TypedefError::Offset`] if `field_path` is not in the offset map. + /// - [`TypedefError::Access`] for buffer-too-short or invalid data, + /// propagated from [`crate::data_access`]. + pub fn read_field<'a>( + &self, + buffer: &'a [u8], + field_path: &str, + ) -> Result, TypedefError> { + let offset_map = match &self.layout { + Layout::Aligned { offset_map } => offset_map, + Layout::Packed { .. } => { + return Err(TypedefError::Access { + field_path: field_path.to_string(), + reason: "read_field is only available in aligned mode; \ + use sequential_reader() for packed mode" + .to_string(), + }); + } + }; + let range = offset_map + .get(field_path) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "field not found in offset map".to_string(), + })?; + let field_schema = + lookup_field_schema(&self.schema, field_path).ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "field schema not found in schema tree".to_string(), + })?; + let kind = get_typedef_kind_loose_enum(field_schema).ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "field schema has no TypeDef:* kind".to_string(), + })?; + let endian = self.endian; + match kind { + TypeDefKind::Int8 => { + let v = data_access::read_i8(buffer, range.start, field_path)?; + Ok(FieldValue::I8(v)) + } + TypeDefKind::Int16 => { + let v = data_access::read_i16(buffer, range.start, field_path, endian)?; + Ok(FieldValue::I16(v)) + } + TypeDefKind::Int32 => { + let v = data_access::read_i32(buffer, range.start, field_path, endian)?; + Ok(FieldValue::I32(v)) + } + TypeDefKind::Int64 => { + let v = data_access::read_i64(buffer, range.start, field_path, endian)?; + Ok(FieldValue::I64(v)) + } + TypeDefKind::Uint8 => { + let v = data_access::read_u8(buffer, range.start, field_path)?; + Ok(FieldValue::U8(v)) + } + TypeDefKind::Uint16 => { + let v = data_access::read_u16(buffer, range.start, field_path, endian)?; + Ok(FieldValue::U16(v)) + } + TypeDefKind::Uint32 => { + let v = data_access::read_u32(buffer, range.start, field_path, endian)?; + Ok(FieldValue::U32(v)) + } + TypeDefKind::Uint64 => { + let v = data_access::read_u64(buffer, range.start, field_path, endian)?; + Ok(FieldValue::U64(v)) + } + TypeDefKind::Float32 => { + let v = data_access::read_f32(buffer, range.start, field_path, endian)?; + Ok(FieldValue::F32(v)) + } + TypeDefKind::Float64 => { + let v = data_access::read_f64(buffer, range.start, field_path, endian)?; + Ok(FieldValue::F64(v)) + } + TypeDefKind::Boolean => { + let v = data_access::read_bool(buffer, range.start, field_path)?; + Ok(FieldValue::Bool(v)) + } + TypeDefKind::Enum => { + let v = data_access::read_enum(buffer, range.start, field_path, endian)?; + Ok(FieldValue::Enum(v)) + } + TypeDefKind::String => { + let v = data_access::read_string(buffer, range.start, field_path, endian)?; + Ok(FieldValue::String(v)) + } + TypeDefKind::Bytes => { + let v = data_access::read_bytes(buffer, range.start, field_path, endian)?; + Ok(FieldValue::Bytes(v)) + } + TypeDefKind::Timestamp => { + let v = data_access::read_string(buffer, range.start, field_path, endian)?; + Ok(FieldValue::String(v)) + } + TypeDefKind::Struct => Ok(FieldValue::Struct { + start: range.start, + end: range.end, + }), + TypeDefKind::Union | TypeDefKind::Array | TypeDefKind::Record => { + Err(TypedefError::Access { + field_path: field_path.to_string(), + reason: "read_field does not support composite types; \ + use the layout-specific APIs" + .to_string(), + }) + } + } + } + + /// Write a field to a buffer at its computed offset (aligned mode). + /// + /// Looks up the field's byte range in the [`OffsetMap`] and writes the + /// appropriate type using the [`crate::data_access`] functions. Works + /// for fixed-size primitive kinds and length-prefixed `String`/ + /// `Bytes`/`Timestamp` fields. + /// + /// Returns an error if compiled in packed mode — use + /// [`TypedefEngine::layout_builder`] for packed mode. Also returns an + /// error for composite kinds (`Struct`, `Union`, `Array`, `Record`). + /// + /// # Errors + /// + /// - [`TypedefError::Access`] if compiled in packed mode. + /// - [`TypedefError::Offset`] if `field_path` is not in the offset map. + /// - [`TypedefError::Access`] for buffer-too-short or invalid data, + /// propagated from [`crate::data_access`]. + pub fn write_field( + &self, + buffer: &mut [u8], + field_path: &str, + value: &FieldValue<'_>, + ) -> Result<(), TypedefError> { + let offset_map = match &self.layout { + Layout::Aligned { offset_map } => offset_map, + Layout::Packed { .. } => { + return Err(TypedefError::Access { + field_path: field_path.to_string(), + reason: "write_field is only available in aligned mode; \ + use layout_builder() for packed mode" + .to_string(), + }); + } + }; + let range = offset_map + .get(field_path) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "field not found in offset map".to_string(), + })?; + let endian = self.endian; + match value { + FieldValue::I8(v) => data_access::write_i8(buffer, range.start, *v, field_path), + FieldValue::I16(v) => { + data_access::write_i16(buffer, range.start, *v, field_path, endian) + } + FieldValue::I32(v) => { + data_access::write_i32(buffer, range.start, *v, field_path, endian) + } + FieldValue::I64(v) => { + data_access::write_i64(buffer, range.start, *v, field_path, endian) + } + FieldValue::U8(v) => data_access::write_u8(buffer, range.start, *v, field_path), + FieldValue::U16(v) => { + data_access::write_u16(buffer, range.start, *v, field_path, endian) + } + FieldValue::U32(v) => { + data_access::write_u32(buffer, range.start, *v, field_path, endian) + } + FieldValue::U64(v) => { + data_access::write_u64(buffer, range.start, *v, field_path, endian) + } + FieldValue::F32(v) => { + data_access::write_f32(buffer, range.start, *v, field_path, endian) + } + FieldValue::F64(v) => { + data_access::write_f64(buffer, range.start, *v, field_path, endian) + } + FieldValue::Bool(v) => data_access::write_bool(buffer, range.start, *v, field_path), + FieldValue::Enum(v) => { + data_access::write_enum(buffer, range.start, *v, field_path, endian) + } + FieldValue::String(v) => { + data_access::write_string(buffer, range.start, v, field_path, endian)?; + Ok(()) + } + FieldValue::Bytes(v) => { + data_access::write_bytes(buffer, range.start, v, field_path, endian)?; + Ok(()) + } + FieldValue::Struct { .. } | FieldValue::Union { .. } | FieldValue::Array { .. } => { + Err(TypedefError::Access { + field_path: field_path.to_string(), + reason: "write_field does not support composite types; \ + use the layout-specific APIs" + .to_string(), + }) + } + } + } +} + +impl fmt::Debug for TypedefEngine { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TypedefEngine") + .field("layout", &self.layout) + .field("validator", &"") + .field("endian", &self.endian) + .field("schema", &self.schema) + .finish() + } +} + +/// Walk a schema tree to find the node for a dotted field path. +/// +/// Splits `field_path` on `.` and descends into `schema["properties"][segment]` +/// at each step. Returns `None` if any segment is missing or the schema is +/// not an object. Does not resolve `$ref` pointers — the engine stores the +/// normalized schema, and the aligned offset map only records paths for +/// inline fields, so refs at intermediate levels are not expected here. +fn lookup_field_schema<'a>(schema: &'a Value, field_path: &str) -> Option<&'a Value> { + let mut current = schema; + for segment in field_path.split('.') { + current = current.as_object()?.get("properties")?.get(segment)?; + } + Some(current) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn fixed_struct_schema() -> Value { + json!({ + "TypeDef:Struct": true, + "endian": "little", + "properties": { + "flag": { "TypeDef:Uint8": true }, + "id": { "TypeDef:Uint32": true }, + "score": { "TypeDef:Float32": true }, + "tag": { "TypeDef:String": true } + } + }) + } + + #[test] + fn compile_aligned_builds_offset_map() { + let mut schema = fixed_struct_schema(); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned).expect("compile"); + assert_eq!(engine.mode(), LayoutMode::Aligned); + assert!(engine.offset_map().is_some()); + assert!(engine.layout_builder().is_none()); + assert!(engine.sequential_reader().is_none()); + } + + #[test] + fn compile_packed_builds_builder_and_reader() { + let mut schema = fixed_struct_schema(); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Packed).expect("compile"); + assert_eq!(engine.mode(), LayoutMode::Packed); + assert!(engine.layout_builder().is_some()); + assert!(engine.sequential_reader().is_some()); + assert!(engine.offset_map().is_none()); + } + + #[test] + fn compile_normalizes_refs() { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { + "child": { "$ref": "Child" } + }, + "$defs": { + "Child": { + "TypeDef:Struct": true, + "properties": { "x": { "TypeDef:Uint8": true } } + } + } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Packed).expect("compile"); + assert_eq!( + engine.schema["properties"]["child"]["$ref"], + json!("#/$defs/Child") + ); + } + + #[test] + fn endian_parsed_from_schema() { + let mut schema = json!({ + "TypeDef:Struct": true, + "endian": "big", + "properties": { "id": { "TypeDef:Uint32": true } } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Packed).expect("compile"); + assert_eq!(engine.endian(), Endian::Big); + } + + #[test] + fn endian_defaults_to_little() { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { "id": { "TypeDef:Uint32": true } } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Packed).expect("compile"); + assert_eq!(engine.endian(), Endian::Little); + } + + #[test] + fn validate_json_accepts_valid_instance() { + let mut schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "id": { "TypeDef:Uint32": true, "type": "integer" } + }, + "required": ["id"] + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned).expect("compile"); + assert!(engine.validate_json(&json!({"id": 42})).is_ok()); + } + + #[test] + fn validate_json_rejects_invalid_instance() { + let mut schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "id": { "TypeDef:Uint32": true, "type": "integer" } + }, + "required": ["id"] + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned).expect("compile"); + let err = engine.validate_json(&json!({"id": -1})).unwrap_err(); + assert!(matches!(err, TypedefError::Validation(_)), "got {err:?}"); + } + + #[test] + fn is_valid_json_returns_bool() { + let mut schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "id": { "TypeDef:Uint32": true, "type": "integer" } + }, + "required": ["id"] + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned).expect("compile"); + assert!(engine.is_valid_json(&json!({"id": 42}))); + assert!(!engine.is_valid_json(&json!({"id": -1}))); + } + + #[test] + fn read_field_aligned_reads_fixed_fields() { + let mut schema = json!({ + "TypeDef:Struct": true, + "endian": "little", + "properties": { + "flag": { "TypeDef:Uint8": true }, + "id": { "TypeDef:Uint32": true } + } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned).expect("compile"); + let mut buf = vec![0u8; 8]; + buf[0] = 0xAB; + buf[4..8].copy_from_slice(&0x01020304u32.to_le_bytes()); + assert_eq!( + engine.read_field(&buf, "flag").unwrap(), + FieldValue::U8(0xAB) + ); + assert_eq!( + engine.read_field(&buf, "id").unwrap(), + FieldValue::U32(0x01020304) + ); + } + + #[test] + fn read_field_aligned_reads_string_length_prefixed() { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { + "name": { "TypeDef:String": true } + } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned).expect("compile"); + let mut buf = vec![0u8; 32]; + let len_bytes = 5u32.to_le_bytes(); + buf[0..4].copy_from_slice(&len_bytes); + buf[4..9].copy_from_slice(b"hello"); + assert_eq!( + engine.read_field(&buf, "name").unwrap(), + FieldValue::String("hello") + ); + } + + #[test] + fn read_field_returns_access_error_in_packed_mode() { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { "id": { "TypeDef:Uint32": true } } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Packed).expect("compile"); + let buf = [0u8; 4]; + let err = engine.read_field(&buf, "id").unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); + } + + #[test] + fn read_field_returns_offset_error_for_missing_field() { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { "id": { "TypeDef:Uint32": true } } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned).expect("compile"); + let buf = [0u8; 4]; + let err = engine.read_field(&buf, "missing").unwrap_err(); + assert!(matches!(err, TypedefError::Offset { .. }), "got {err:?}"); + } + + #[test] + fn read_field_returns_error_for_composite_types() { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { + "vals": { + "TypeDef:Array": true, + "items": { "TypeDef:Uint32": true } + } + } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned).expect("compile"); + let buf = [0u8; 8]; + let err = engine.read_field(&buf, "vals").unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); + } + + #[test] + fn write_field_aligned_writes_fixed_fields() { + let mut schema = json!({ + "TypeDef:Struct": true, + "endian": "little", + "properties": { + "flag": { "TypeDef:Uint8": true }, + "id": { "TypeDef:Uint32": true } + } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned).expect("compile"); + let mut buf = vec![0u8; 8]; + engine + .write_field(&mut buf, "flag", &FieldValue::U8(0xAB)) + .unwrap(); + engine + .write_field(&mut buf, "id", &FieldValue::U32(0x01020304)) + .unwrap(); + assert_eq!(buf[0], 0xAB); + assert_eq!(&buf[4..8], &0x01020304u32.to_le_bytes()); + assert_eq!( + engine.read_field(&buf, "flag").unwrap(), + FieldValue::U8(0xAB) + ); + assert_eq!( + engine.read_field(&buf, "id").unwrap(), + FieldValue::U32(0x01020304) + ); + } + + #[test] + fn write_field_round_trips_string() { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { + "name": { "TypeDef:String": true } + } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned).expect("compile"); + let mut buf = vec![0u8; 32]; + engine + .write_field(&mut buf, "name", &FieldValue::String("hello")) + .unwrap(); + assert_eq!( + engine.read_field(&buf, "name").unwrap(), + FieldValue::String("hello") + ); + } + + #[test] + fn write_field_returns_access_error_in_packed_mode() { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { "id": { "TypeDef:Uint32": true } } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Packed).expect("compile"); + let mut buf = [0u8; 4]; + let err = engine + .write_field(&mut buf, "id", &FieldValue::U32(1)) + .unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); + } + + #[test] + fn write_field_returns_offset_error_for_missing_field() { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { "id": { "TypeDef:Uint32": true } } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned).expect("compile"); + let mut buf = [0u8; 4]; + let err = engine + .write_field(&mut buf, "missing", &FieldValue::U32(1)) + .unwrap_err(); + assert!(matches!(err, TypedefError::Offset { .. }), "got {err:?}"); + } + + #[test] + fn write_field_returns_error_for_composite_value() { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { "id": { "TypeDef:Uint32": true } } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned).expect("compile"); + let mut buf = [0u8; 8]; + let err = engine + .write_field(&mut buf, "id", &FieldValue::Struct { start: 0, end: 4 }) + .unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); + } + + #[test] + fn compile_returns_schema_error_for_invalid_top_level() { + let mut schema = json!({ "type": "object", "properties": {} }); + let err = TypedefEngine::compile(&mut schema, LayoutMode::Packed).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); + } + + #[test] + fn debug_formats_without_panicking() { + let mut schema = fixed_struct_schema(); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned).expect("compile"); + let s = format!("{engine:?}"); + assert!(s.contains("TypedefEngine")); + assert!(s.contains("Aligned")); + } + + #[test] + fn lookup_field_schema_walks_dotted_path() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "header": { + "TypeDef:Struct": true, + "properties": { + "version": { "TypeDef:Uint8": true } + } + } + } + }); + let node = lookup_field_schema(&schema, "header.version").expect("found"); + assert_eq!(node, &json!({ "TypeDef:Uint8": true })); + assert!(lookup_field_schema(&schema, "header.missing").is_none()); + assert!(lookup_field_schema(&schema, "missing").is_none()); + } + + #[test] + fn typedef_kind_loose_recognizes_object_form() { + let node = json!({ "TypeDef:String": { "encoding": "offset-indirect" } }); + assert_eq!( + get_typedef_kind_loose_enum(&node), + Some(TypeDefKind::String) + ); + } +} diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..113419a --- /dev/null +++ b/src/error.rs @@ -0,0 +1,45 @@ +//! Error types for the typedef engine. +//! +//! Decided in ADR-098: a single `TypedefError` enum covers all error +//! conditions across the engine's three phases (schema parsing, offset +//! computation, read/write) plus validation. + +use std::fmt; + +/// Errors produced by the typedef engine across all phases. +#[derive(Debug)] +pub enum TypedefError { + /// Schema parsing errors — invalid JSON, missing required keywords, + /// unknown `TypeDef:*` kinds, malformed annotations. + Schema(String), + + /// Offset computation errors — field not found, type not supported + /// for offset computation, recursive depth exceeded. + Offset { field_path: String, reason: String }, + + /// Read/write errors — buffer too short, invalid UTF-8, value out + /// of range for the target type. + Access { field_path: String, reason: String }, + + /// Validation errors — delegated to the `jsonschema` crate. + /// The `'static` lifetime is correct: the validator owns its schema + /// reference and lives for the lifetime of the `TypedefEngine`. + Validation(jsonschema::ValidationError<'static>), +} + +impl fmt::Display for TypedefError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TypedefError::Schema(msg) => write!(f, "schema error: {msg}"), + TypedefError::Offset { field_path, reason } => { + write!(f, "offset error at {field_path}: {reason}") + } + TypedefError::Access { field_path, reason } => { + write!(f, "access error at {field_path}: {reason}") + } + TypedefError::Validation(inner) => write!(f, "validation error: {inner}"), + } + } +} + +impl std::error::Error for TypedefError {} diff --git a/src/layout_builder.rs b/src/layout_builder.rs new file mode 100644 index 0000000..efbc6a8 --- /dev/null +++ b/src/layout_builder.rs @@ -0,0 +1,1705 @@ +//! Packed sequential `LayoutBuilder` — Mode 1 write-side (ADR-096). +//! +//! Fields are packed with no alignment padding. Variable-length fields +//! shift all subsequent fields. The consumer provides actual data sizes +//! for variable-length fields; the builder computes byte positions for +//! each field. Used at write time when the consumer knows the data sizes +//! upfront. +//! +//! Per [layout-engine.md](../../docs/architecture/crates/typedef/layout-engine.md) +//! §"Mode 1: Packed sequential". +//! +//! # Layout rules +//! +//! - **Fixed-size fields**: recorded at the current offset with their +//! known byte size; the offset advances by the size. No alignment +//! padding is inserted (the `u32` at offset 1 is unaligned — correct +//! for protocol wire formats). +//! - **Variable-length fields** (`TypeDef:String`, `TypeDef:Bytes`, +//! `TypeDef:Timestamp`, `TypeDef:Record`): always inline +//! length-prefixed in packed mode. The 4-byte length prefix is +//! recorded at the current offset; the offset advances by `4 + +//! data_size` where `data_size` comes from `var_sizes` keyed by the +//! field's dotted path. +//! - **`TypeDef:Struct`**: recurses into `properties`, propagating the +//! field path prefix (e.g., `"header.version"`). No padding before, +//! between, or after the struct's fields. +//! - **`TypeDef:Array`** of fixed-size elements with a fixed count +//! (`minItems == maxItems`): element `i` at +//! `array_offset + i × element_size`. Each element is recorded as +//! `"[i]"`. +//! - **`TypeDef:Array`** with a variable count: a 4-byte count prefix +//! at the array's offset. The consumer provides the total element +//! data size in `var_sizes` under the array's field path; the builder +//! adds `4 + data_size`. +//! - **`TypeDef:Union`** with a byte-offset discriminator: the +//! discriminator is recorded at `".__discriminator"`. +//! The consumer provides the discriminator value as a `usize` in +//! `var_sizes` under `".__discriminator"`. The variant +//! struct is laid out starting at `union_offset + disc_offset + +//! disc_size`, with field paths prefixed by the union's path. +//! - **`TypeDef:Union`** with a field-name discriminator: the consumer +//! provides the 0-based variant index in `var_sizes` under +//! `".__variant"`. The selected variant struct is laid +//! out at the union's offset, with field paths prefixed by the +//! union's path. The discriminator field is a regular field within +//! the variant struct. + +use crate::error::TypedefError; +use crate::schema::{self, get_typedef_kind_loose_enum, resolve_ref_or_inline, DiscriminatorKind, Endian, TypeDefKind, DISCRIMINATOR_PATH, U32_SIZE}; +use serde_json::Value; +use std::collections::HashMap; + +const VARIANT_KEY: &str = "__variant"; + +/// A field position computed by the LayoutBuilder. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FieldPosition { + /// Byte offset of the field within the buffer. + pub offset: usize, + /// Byte size of the field (4 for length prefix of variable-length + /// fields, actual size for fixed-size fields). + pub size: usize, + /// The TypeDef kind of the field. + pub kind: TypeDefKind, +} + +/// The result of building a layout: a map of field_path → FieldPosition +/// and the total buffer size needed. +/// +/// Construct via [`LayoutBuilder::build`]. Fields are stored in layout +/// order (the order they appear in the schema's `properties`, with +/// nested struct fields appearing inline). +#[derive(Debug)] +pub struct PackedLayout { + fields: Vec<(String, FieldPosition)>, + total_size: usize, +} + +impl PackedLayout { + /// Look up a field's position by dotted path (e.g., `"header.version"`). + /// + /// Returns `None` if no field with the given path was recorded. For + /// TUnion byte-offset discriminators, the discriminator is recorded + /// under the synthetic path `".__discriminator"`. + pub fn get(&self, field_path: &str) -> Option<&FieldPosition> { + self.fields + .iter() + .find(|(path, _)| path == field_path) + .map(|(_, pos)| pos) + } + + /// The total buffer size needed to hold all fields. + pub fn total_size(&self) -> usize { + self.total_size + } + + /// Iterate over all `(field_path, position)` pairs in layout order. + /// + /// Field order matches the schema's `properties` order (preserved by + /// `serde_json`'s `preserve_order` feature). Nested struct fields + /// appear after their parent's path prefix. + pub fn iter(&self) -> impl Iterator { + self.fields.iter() + } +} + +/// Builds a packed sequential layout for protocol wire formats. +/// +/// Fields are packed with no alignment padding. Variable-length fields +/// shift all subsequent fields. The consumer provides actual data sizes +/// for variable-length fields to compute correct positions. +/// +/// Used at write time when the consumer knows the data sizes upfront. +/// The builder does not write data — it only computes positions. The +/// consumer uses the [`crate::data_access`] write functions at the +/// computed positions. +/// +/// # Example +/// +/// For a struct with fields `[u8, u32, string]` where the string is +/// 10 bytes: +/// +/// ```text +/// LayoutBuilder::build(var_sizes: {"payload": 10}): +/// field[0] u8: offset 0, size 1 +/// field[1] u32: offset 1, size 4 +/// field[2] string: offset 5, size 4 (length prefix) + 10 (data) = 14 +/// total: 19 +/// ``` +/// +/// There is no alignment padding. The `u32` at offset 1 is unaligned — +/// this is correct for protocol wire formats, which pack fields tightly. +#[derive(Debug)] +pub struct LayoutBuilder { + schema: Value, + endian: Endian, +} + +impl LayoutBuilder { + /// Create a new LayoutBuilder from a schema. + /// + /// The top-level schema must declare `TypeDef:Struct`. Endianness is + /// parsed via [`Endian::from_schema`] (defaults to little-endian). + /// + /// # Errors + /// + /// Returns [`TypedefError::Schema`] if the schema has no + /// `TypeDef:*` kind or the top-level kind is not `TypeDef:Struct`. + pub fn new(schema: &Value) -> Result { + let kind = schema::get_typedef_kind(schema) + .and_then(|s| s.parse::().ok()) + .ok_or_else(|| { + TypedefError::Schema("top-level schema has no TypeDef:* kind".to_string()) + })?; + if kind != TypeDefKind::Struct { + return Err(TypedefError::Schema(format!( + "LayoutBuilder requires a TypeDef:Struct at the top level, got {kind}" + ))); + } + let endian = Endian::from_schema(schema); + Ok(Self { + schema: schema.clone(), + endian, + }) + } + + /// Build the packed layout given actual data sizes for variable-length + /// fields. + /// + /// `var_sizes` maps field paths to their actual byte sizes (not + /// including the 4-byte length prefix — the builder adds that). + /// + /// For fixed-size fields, the size is known from the schema. For + /// variable-length fields, the size comes from `var_sizes`. For + /// TUnion, the consumer provides the discriminator value (byte-offset) + /// or variant index (field-name) plus the variant's field sizes. + /// + /// # Errors + /// + /// - [`TypedefError::Schema`] for malformed schemas (missing + /// `properties`, unknown kind, unresolvable `$ref`). + /// - [`TypedefError::Offset`] for missing variable-length field + /// sizes in `var_sizes`, missing discriminator values, or unknown + /// discriminator values. + pub fn build(&self, var_sizes: &HashMap) -> Result { + let mut ctx = BuildCtx { + root: &self.schema, + var_sizes, + fields: Vec::new(), + }; + let mut offset: usize = 0; + ctx.walk_struct(&self.schema, "", &mut offset)?; + Ok(PackedLayout { + fields: ctx.fields, + total_size: offset, + }) + } + + /// The endianness parsed from the schema. + pub fn endian(&self) -> Endian { + self.endian + } +} + +/// Mutable context threaded through the recursive layout computation. +struct BuildCtx<'a> { + root: &'a Value, + var_sizes: &'a HashMap, + fields: Vec<(String, FieldPosition)>, +} + +impl<'a> BuildCtx<'a> { + /// Recurse into a `TypeDef:Struct`, appending `(field_path, FieldPosition)` + /// pairs to `self.fields` and advancing `offset`. + /// + /// `prefix` is the dotted path prefix for nested fields (empty at the + /// top level). + fn walk_struct( + &mut self, + schema: &Value, + prefix: &str, + offset: &mut usize, + ) -> Result<(), TypedefError> { + let properties = schema + .as_object() + .and_then(|o| o.get("properties")) + .and_then(Value::as_object) + .ok_or_else(|| { + TypedefError::Schema("struct schema has no 'properties' object".to_string()) + })?; + + let field_schemas: Vec<(String, Value)> = properties + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + for (name, field_schema) in field_schemas { + let field_path = if prefix.is_empty() { + name + } else { + format!("{prefix}.{name}") + }; + self.walk_field(&field_schema, &field_path, offset)?; + } + Ok(()) + } + + /// Compute the layout for a single field, advancing `offset` and + /// appending any field paths to `self.fields`. + fn walk_field( + &mut self, + field_schema: &Value, + field_path: &str, + offset: &mut usize, + ) -> Result<(), TypedefError> { + let kind = get_typedef_kind_loose_enum(field_schema).ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "field schema has no TypeDef:* kind".to_string(), + })?; + + match kind { + TypeDefKind::Struct => self.walk_struct(field_schema, field_path, offset), + TypeDefKind::Union => self.walk_union(field_schema, field_path, offset), + TypeDefKind::Array => self.walk_array(field_schema, field_path, offset), + TypeDefKind::String + | TypeDefKind::Bytes + | TypeDefKind::Timestamp + | TypeDefKind::Record => { + self.walk_variable(field_path, offset, kind) + } + k if k.is_fixed_size() => { + let size = k.type_size().ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("type_size returned None for fixed kind {k}"), + })?; + let start = *offset; + *offset = start + .checked_add(size) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("offset {start} + size {size} overflows usize"), + })?; + self.push(field_path, start, size, k); + Ok(()) + } + _ => unreachable!("all TypeDefKind variants are covered above"), + } + } + + /// Compute the layout for a variable-length field (String/Bytes/ + /// Timestamp/Record). Always inline length-prefixed in packed mode. + fn walk_variable( + &mut self, + field_path: &str, + offset: &mut usize, + kind: TypeDefKind, + ) -> Result<(), TypedefError> { + let data_size = + self.var_sizes + .get(field_path) + .copied() + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "missing variable-length field size".to_string(), + })?; + let start = *offset; + let total = U32_SIZE + .checked_add(data_size) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("prefix {U32_SIZE} + data size {data_size} overflows usize"), + })?; + *offset = start + .checked_add(total) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("offset {start} + total {total} overflows usize"), + })?; + self.push(field_path, start, U32_SIZE, kind); + Ok(()) + } + + /// Compute the layout for a `TypeDef:Array` field. + fn walk_array( + &mut self, + field_schema: &Value, + field_path: &str, + offset: &mut usize, + ) -> Result<(), TypedefError> { + let obj = field_schema + .as_object() + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "array schema is not an object".to_string(), + })?; + + let items = obj.get("items").ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "TArray is missing 'items'".to_string(), + })?; + let element_schema = + resolve_ref_or_inline(items, self.root).ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "could not resolve TArray items schema".to_string(), + })?; + let elem_kind = get_typedef_kind_loose_enum(element_schema).ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "TArray element schema has no TypeDef:* kind".to_string(), + })?; + + if !elem_kind.is_fixed_size() { + return Err(TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!( + "TArray of variable-length element kind {elem_kind} is not supported (OQ-069)" + ), + }); + } + + let elem_size = elem_kind.type_size().ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("element kind {elem_kind} has no fixed size"), + })?; + + let min_items = obj + .get("minItems") + .and_then(Value::as_u64) + .map(|n| n as usize); + let max_items = obj + .get("maxItems") + .and_then(Value::as_u64) + .map(|n| n as usize); + let fixed_count = match (min_items, max_items) { + (Some(mn), Some(mx)) if mn == mx => Some(mn), + _ => None, + }; + + if let Some(count) = fixed_count { + let start = *offset; + for i in 0..count { + let elem_offset = start + .checked_add( + i.checked_mul(elem_size) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!( + "element index {i} × size {elem_size} overflows usize" + ), + })?, + ) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("element offset {start} + {i}×{elem_size} overflows usize"), + })?; + self.push( + &format!("{field_path}[{i}]"), + elem_offset, + elem_size, + elem_kind, + ); + } + let array_size = count + .checked_mul(elem_size) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("array size {count} × {elem_size} overflows usize"), + })?; + *offset = start + .checked_add(array_size) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("offset {start} + array size {array_size} overflows usize"), + })?; + Ok(()) + } else { + let data_size = + self.var_sizes + .get(field_path) + .copied() + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "missing variable-count array element data size".to_string(), + })?; + let start = *offset; + let total = U32_SIZE + .checked_add(data_size) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!( + "count prefix {U32_SIZE} + data size {data_size} overflows usize" + ), + })?; + *offset = start + .checked_add(total) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("offset {start} + total {total} overflows usize"), + })?; + self.push(field_path, start, U32_SIZE, TypeDefKind::Array); + Ok(()) + } + } + + /// Compute the layout for a `TypeDef:Union` field. + fn walk_union( + &mut self, + field_schema: &Value, + field_path: &str, + offset: &mut usize, + ) -> Result<(), TypedefError> { + let disc = schema::parse_discriminator(field_schema)?; + let mapping = field_schema + .as_object() + .and_then(|o| o.get("mapping")) + .and_then(Value::as_object) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "TUnion is missing 'mapping' object".to_string(), + })?; + + match disc { + DiscriminatorKind::Byte { + offset: disc_off, + disc_type, + } => self + .walk_byte_discriminator_union(field_path, offset, disc_type, disc_off, mapping), + DiscriminatorKind::Field { name: _ } => { + self.walk_field_discriminator_union(field_path, offset, mapping) + } + } + } + + /// Lay out a TUnion with a byte-offset discriminator. + fn walk_byte_discriminator_union( + &mut self, + field_path: &str, + offset: &mut usize, + disc_type: TypeDefKind, + disc_off: usize, + mapping: &serde_json::Map, + ) -> Result<(), TypedefError> { + let disc_size = disc_type.type_size().ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("discriminator type {disc_type} has no fixed size"), + })?; + + let disc_key = format!("{field_path}.{DISCRIMINATOR_PATH}"); + let disc_value = + self.var_sizes + .get(&disc_key) + .copied() + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("missing discriminator value at key '{disc_key}'"), + })?; + + let union_start = *offset; + let disc_abs_offset = + union_start + .checked_add(disc_off) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!( + "union offset {union_start} + disc offset {disc_off} overflows usize" + ), + })?; + self.push(&disc_key, disc_abs_offset, disc_size, disc_type); + + let variant_key = disc_value.to_string(); + let variant_schema = mapping + .get(&variant_key) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("unknown discriminator value: {variant_key}"), + })?; + let resolved = resolve_ref_or_inline(variant_schema, self.root).ok_or_else(|| { + TypedefError::Offset { + field_path: field_path.to_string(), + reason: "could not resolve TUnion variant schema ($ref not found)".to_string(), + } + })?; + let v_kind = get_typedef_kind_loose_enum(resolved).ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "TUnion variant schema has no TypeDef:* kind".to_string(), + })?; + if v_kind != TypeDefKind::Struct { + return Err(TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("TUnion variant must be TypeDef:Struct, got {v_kind}"), + }); + } + + let variant_start = + disc_abs_offset + .checked_add(disc_size) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!( + "variant start {disc_abs_offset} + disc size {disc_size} overflows usize" + ), + })?; + *offset = variant_start; + self.walk_struct(resolved, field_path, offset)?; + Ok(()) + } + + /// Lay out a TUnion with a field-name discriminator. + /// + /// The discriminator is a regular field within the variant struct. + /// The consumer selects the variant by 0-based index in + /// `var_sizes[".__variant"]`. + fn walk_field_discriminator_union( + &mut self, + field_path: &str, + offset: &mut usize, + mapping: &serde_json::Map, + ) -> Result<(), TypedefError> { + let variant_key = format!("{field_path}.{VARIANT_KEY}"); + let variant_index = + self.var_sizes + .get(&variant_key) + .copied() + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("missing variant index at key '{variant_key}'"), + })?; + let variant_entry = + mapping + .iter() + .nth(variant_index) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!( + "variant index {variant_index} out of range (mapping has {} entries)", + mapping.len() + ), + })?; + let variant_schema = variant_entry.1; + let resolved = resolve_ref_or_inline(variant_schema, self.root).ok_or_else(|| { + TypedefError::Offset { + field_path: field_path.to_string(), + reason: "could not resolve TUnion variant schema ($ref not found)".to_string(), + } + })?; + let v_kind = get_typedef_kind_loose_enum(resolved).ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "TUnion variant schema has no TypeDef:* kind".to_string(), + })?; + if v_kind != TypeDefKind::Struct { + return Err(TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("TUnion variant must be TypeDef:Struct, got {v_kind}"), + }); + } + self.walk_struct(resolved, field_path, offset)?; + Ok(()) + } + + /// Push a `(field_path, FieldPosition)` pair onto the fields vec. + fn push(&mut self, path: &str, offset: usize, size: usize, kind: TypeDefKind) { + self.fields.push(( + path.to_string(), + FieldPosition { + offset, + size, + kind, + }, + )); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn build(schema: &Value, var_sizes: &HashMap) -> PackedLayout { + LayoutBuilder::new(schema) + .expect("builder") + .build(var_sizes) + .expect("layout") + } + + fn var_sizes(pairs: &[(&str, usize)]) -> HashMap { + pairs.iter().map(|(k, v)| (k.to_string(), *v)).collect() + } + + #[test] + fn fixed_fields_packed_no_alignment_padding() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "a": { "TypeDef:Uint8": true }, + "b": { "TypeDef:Uint32": true }, + "c": { "TypeDef:Uint16": true } + } + }); + let layout = build(&schema, &var_sizes(&[])); + assert_eq!( + layout.get("a"), + Some(&FieldPosition { + offset: 0, + size: 1, + kind: TypeDefKind::Uint8 + }) + ); + assert_eq!( + layout.get("b"), + Some(&FieldPosition { + offset: 1, + size: 4, + kind: TypeDefKind::Uint32 + }) + ); + assert_eq!( + layout.get("c"), + Some(&FieldPosition { + offset: 5, + size: 2, + kind: TypeDefKind::Uint16 + }) + ); + assert_eq!(layout.total_size(), 7); + } + + #[test] + fn spec_example_u8_u32_string_total_19() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "flag": { "TypeDef:Uint8": true }, + "id": { "TypeDef:Uint32": true }, + "payload": { "TypeDef:String": true } + } + }); + let layout = build(&schema, &var_sizes(&[("payload", 10)])); + assert_eq!( + layout.get("flag"), + Some(&FieldPosition { + offset: 0, + size: 1, + kind: TypeDefKind::Uint8 + }) + ); + assert_eq!( + layout.get("id"), + Some(&FieldPosition { + offset: 1, + size: 4, + kind: TypeDefKind::Uint32 + }) + ); + assert_eq!( + layout.get("payload"), + Some(&FieldPosition { + offset: 5, + size: 4, + kind: TypeDefKind::String + }) + ); + assert_eq!(layout.total_size(), 19); + } + + #[test] + fn variable_length_field_shifts_subsequent_fields() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "name": { "TypeDef:String": true }, + "tail": { "TypeDef:Uint8": true } + } + }); + let layout = build(&schema, &var_sizes(&[("name", 5)])); + assert_eq!( + layout.get("name"), + Some(&FieldPosition { + offset: 0, + size: 4, + kind: TypeDefKind::String + }) + ); + assert_eq!( + layout.get("tail"), + Some(&FieldPosition { + offset: 9, + size: 1, + kind: TypeDefKind::Uint8 + }) + ); + assert_eq!(layout.total_size(), 10); + } + + #[test] + fn bytes_field_uses_var_sizes() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "blob": { "TypeDef:Bytes": true } + } + }); + let layout = build(&schema, &var_sizes(&[("blob", 3)])); + assert_eq!( + layout.get("blob"), + Some(&FieldPosition { + offset: 0, + size: 4, + kind: TypeDefKind::Bytes + }) + ); + assert_eq!(layout.total_size(), 7); + } + + #[test] + fn timestamp_field_uses_var_sizes() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "ts": { "TypeDef:Timestamp": true } + } + }); + let layout = build(&schema, &var_sizes(&[("ts", 20)])); + assert_eq!( + layout.get("ts"), + Some(&FieldPosition { + offset: 0, + size: 4, + kind: TypeDefKind::Timestamp + }) + ); + assert_eq!(layout.total_size(), 24); + } + + #[test] + fn record_field_uses_var_sizes() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "counts": { + "TypeDef:Record": true, + "values": { "TypeDef:Uint32": true } + } + } + }); + let layout = build(&schema, &var_sizes(&[("counts", 100)])); + assert_eq!( + layout.get("counts"), + Some(&FieldPosition { + offset: 0, + size: 4, + kind: TypeDefKind::Record + }) + ); + assert_eq!(layout.total_size(), 104); + } + + #[test] + fn missing_var_size_returns_offset_error() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "name": { "TypeDef:String": true } + } + }); + let builder = LayoutBuilder::new(&schema).expect("builder"); + let err = builder.build(&var_sizes(&[])).unwrap_err(); + match err { + TypedefError::Offset { field_path, reason } => { + assert_eq!(field_path, "name"); + assert!( + reason.contains("missing variable-length field size"), + "reason: {reason}" + ); + } + other => panic!("expected Offset, got {other:?}"), + } + } + + #[test] + fn nested_struct_dotted_paths() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "header": { + "TypeDef:Struct": true, + "properties": { + "magic": { "TypeDef:Uint32": true }, + "version": { "TypeDef:Uint8": true } + } + }, + "body": { "TypeDef:Uint32": true } + } + }); + let layout = build(&schema, &var_sizes(&[])); + assert_eq!( + layout.get("header.magic"), + Some(&FieldPosition { + offset: 0, + size: 4, + kind: TypeDefKind::Uint32 + }) + ); + assert_eq!( + layout.get("header.version"), + Some(&FieldPosition { + offset: 4, + size: 1, + kind: TypeDefKind::Uint8 + }) + ); + assert_eq!( + layout.get("body"), + Some(&FieldPosition { + offset: 5, + size: 4, + kind: TypeDefKind::Uint32 + }) + ); + assert_eq!(layout.total_size(), 9); + } + + #[test] + fn nested_struct_with_variable_field() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "header": { + "TypeDef:Struct": true, + "properties": { + "id": { "TypeDef:Uint8": true }, + "name": { "TypeDef:String": true } + } + }, + "tail": { "TypeDef:Uint8": true } + } + }); + let layout = build(&schema, &var_sizes(&[("header.name", 3)])); + assert_eq!( + layout.get("header.id"), + Some(&FieldPosition { + offset: 0, + size: 1, + kind: TypeDefKind::Uint8 + }) + ); + assert_eq!( + layout.get("header.name"), + Some(&FieldPosition { + offset: 1, + size: 4, + kind: TypeDefKind::String + }) + ); + assert_eq!( + layout.get("tail"), + Some(&FieldPosition { + offset: 8, + size: 1, + kind: TypeDefKind::Uint8 + }) + ); + assert_eq!(layout.total_size(), 9); + } + + #[test] + fn array_fixed_count_element_offsets() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "vals": { + "TypeDef:Array": true, + "items": { "TypeDef:Uint32": true }, + "minItems": 3, + "maxItems": 3 + } + } + }); + let layout = build(&schema, &var_sizes(&[])); + assert_eq!( + layout.get("vals[0]"), + Some(&FieldPosition { + offset: 0, + size: 4, + kind: TypeDefKind::Uint32 + }) + ); + assert_eq!( + layout.get("vals[1]"), + Some(&FieldPosition { + offset: 4, + size: 4, + kind: TypeDefKind::Uint32 + }) + ); + assert_eq!( + layout.get("vals[2]"), + Some(&FieldPosition { + offset: 8, + size: 4, + kind: TypeDefKind::Uint32 + }) + ); + assert_eq!(layout.total_size(), 12); + } + + #[test] + fn array_fixed_count_after_preceding_field() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "id": { "TypeDef:Uint8": true }, + "vals": { + "TypeDef:Array": true, + "items": { "TypeDef:Uint16": true }, + "minItems": 2, + "maxItems": 2 + } + } + }); + let layout = build(&schema, &var_sizes(&[])); + assert_eq!( + layout.get("id"), + Some(&FieldPosition { + offset: 0, + size: 1, + kind: TypeDefKind::Uint8 + }) + ); + assert_eq!( + layout.get("vals[0]"), + Some(&FieldPosition { + offset: 1, + size: 2, + kind: TypeDefKind::Uint16 + }) + ); + assert_eq!( + layout.get("vals[1]"), + Some(&FieldPosition { + offset: 3, + size: 2, + kind: TypeDefKind::Uint16 + }) + ); + assert_eq!(layout.total_size(), 5); + } + + #[test] + fn array_variable_count_uses_count_prefix() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "vals": { + "TypeDef:Array": true, + "items": { "TypeDef:Uint32": true } + } + } + }); + let layout = build(&schema, &var_sizes(&[("vals", 12)])); + assert_eq!( + layout.get("vals"), + Some(&FieldPosition { + offset: 0, + size: 4, + kind: TypeDefKind::Array + }) + ); + assert_eq!(layout.total_size(), 16); + } + + #[test] + fn array_variable_count_missing_size_is_offset_error() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "vals": { + "TypeDef:Array": true, + "items": { "TypeDef:Uint32": true } + } + } + }); + let builder = LayoutBuilder::new(&schema).expect("builder"); + let err = builder.build(&var_sizes(&[])).unwrap_err(); + assert!(matches!(err, TypedefError::Offset { .. })); + } + + #[test] + fn array_variable_length_element_is_not_supported() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "items": { + "TypeDef:Array": true, + "items": { "TypeDef:String": true } + } + } + }); + let builder = LayoutBuilder::new(&schema).expect("builder"); + let err = builder.build(&var_sizes(&[("items", 10)])).unwrap_err(); + match err { + TypedefError::Offset { reason, .. } => { + assert!(reason.contains("OQ-069"), "reason: {reason}"); + } + other => panic!("expected Offset, got {other:?}"), + } + } + + #[test] + fn union_byte_discriminator_sftp_pattern() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "payload": { + "TypeDef:Union": true, + "discriminator": { + "kind": "byte", + "offset": 0, + "type": "TypeDef:Uint8" + }, + "mapping": { + "5": { "$ref": "#/$defs/Read" }, + "6": { "$ref": "#/$defs/Write" } + } + } + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": { + "handle": { "TypeDef:Uint32": true }, + "length": { "TypeDef:Uint32": true } + } + }, + "Write": { + "TypeDef:Struct": true, + "properties": { + "handle": { "TypeDef:Uint32": true }, + "length": { "TypeDef:Uint32": true }, + "data": { "TypeDef:Uint32": true } + } + } + } + }); + let vs = var_sizes(&[("payload.__discriminator", 5)]); + let layout = build(&schema, &vs); + assert_eq!( + layout.get("payload.__discriminator"), + Some(&FieldPosition { + offset: 0, + size: 1, + kind: TypeDefKind::Uint8 + }) + ); + assert_eq!( + layout.get("payload.handle"), + Some(&FieldPosition { + offset: 1, + size: 4, + kind: TypeDefKind::Uint32 + }) + ); + assert_eq!( + layout.get("payload.length"), + Some(&FieldPosition { + offset: 5, + size: 4, + kind: TypeDefKind::Uint32 + }) + ); + assert_eq!(layout.total_size(), 9); + } + + #[test] + fn union_byte_discriminator_write_variant() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "payload": { + "TypeDef:Union": true, + "discriminator": { + "kind": "byte", + "offset": 0, + "type": "TypeDef:Uint8" + }, + "mapping": { + "5": { "$ref": "#/$defs/Read" }, + "6": { "$ref": "#/$defs/Write" } + } + } + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": { + "handle": { "TypeDef:Uint32": true }, + "length": { "TypeDef:Uint32": true } + } + }, + "Write": { + "TypeDef:Struct": true, + "properties": { + "handle": { "TypeDef:Uint32": true }, + "length": { "TypeDef:Uint32": true }, + "data": { "TypeDef:Uint32": true } + } + } + } + }); + let vs = var_sizes(&[("payload.__discriminator", 6)]); + let layout = build(&schema, &vs); + assert_eq!( + layout.get("payload.__discriminator"), + Some(&FieldPosition { + offset: 0, + size: 1, + kind: TypeDefKind::Uint8 + }) + ); + assert_eq!( + layout.get("payload.data"), + Some(&FieldPosition { + offset: 9, + size: 4, + kind: TypeDefKind::Uint32 + }) + ); + assert_eq!(layout.total_size(), 13); + } + + #[test] + fn union_byte_discriminator_with_variable_variant_field() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "packet": { + "TypeDef:Union": true, + "discriminator": { + "kind": "byte", + "offset": 0, + "type": "TypeDef:Uint8" + }, + "mapping": { + "5": { "$ref": "#/$defs/Read" } + } + } + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": { + "handle": { "TypeDef:Uint32": true }, + "path": { "TypeDef:String": true } + } + } + } + }); + let vs = var_sizes(&[("packet.__discriminator", 5), ("packet.path", 8)]); + let layout = build(&schema, &vs); + assert_eq!( + layout.get("packet.__discriminator"), + Some(&FieldPosition { + offset: 0, + size: 1, + kind: TypeDefKind::Uint8 + }) + ); + assert_eq!( + layout.get("packet.handle"), + Some(&FieldPosition { + offset: 1, + size: 4, + kind: TypeDefKind::Uint32 + }) + ); + assert_eq!( + layout.get("packet.path"), + Some(&FieldPosition { + offset: 5, + size: 4, + kind: TypeDefKind::String + }) + ); + assert_eq!(layout.total_size(), 17); + } + + #[test] + fn union_byte_discriminator_unknown_value_is_offset_error() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "payload": { + "TypeDef:Union": true, + "discriminator": { + "kind": "byte", + "offset": 0, + "type": "TypeDef:Uint8" + }, + "mapping": { + "5": { "$ref": "#/$defs/Read" } + } + } + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": { "x": { "TypeDef:Uint8": true } } + } + } + }); + let vs = var_sizes(&[("payload.__discriminator", 99)]); + let builder = LayoutBuilder::new(&schema).expect("builder"); + let err = builder.build(&vs).unwrap_err(); + match err { + TypedefError::Offset { reason, .. } => { + assert!(reason.contains("99"), "reason: {reason}"); + } + other => panic!("expected Offset, got {other:?}"), + } + } + + #[test] + fn union_byte_discriminator_missing_value_is_offset_error() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "payload": { + "TypeDef:Union": true, + "discriminator": { + "kind": "byte", + "offset": 0, + "type": "TypeDef:Uint8" + }, + "mapping": { + "5": { "$ref": "#/$defs/Read" } + } + } + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": { "x": { "TypeDef:Uint8": true } } + } + } + }); + let builder = LayoutBuilder::new(&schema).expect("builder"); + let err = builder.build(&var_sizes(&[])).unwrap_err(); + assert!(matches!(err, TypedefError::Offset { .. })); + } + + #[test] + fn union_field_name_discriminator() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "event": { + "TypeDef:Union": true, + "discriminator": { "kind": "field", "name": "type" }, + "mapping": { + "read": { "$ref": "#/$defs/Read" }, + "write": { "$ref": "#/$defs/Write" } + } + } + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": { + "type": { "TypeDef:Uint8": true }, + "handle": { "TypeDef:Uint32": true } + } + }, + "Write": { + "TypeDef:Struct": true, + "properties": { + "type": { "TypeDef:Uint8": true }, + "handle": { "TypeDef:Uint32": true }, + "length": { "TypeDef:Uint32": true } + } + } + } + }); + let vs = var_sizes(&[("event.__variant", 0)]); + let layout = build(&schema, &vs); + assert_eq!( + layout.get("event.type"), + Some(&FieldPosition { + offset: 0, + size: 1, + kind: TypeDefKind::Uint8 + }) + ); + assert_eq!( + layout.get("event.handle"), + Some(&FieldPosition { + offset: 1, + size: 4, + kind: TypeDefKind::Uint32 + }) + ); + assert_eq!(layout.total_size(), 5); + } + + #[test] + fn union_field_name_discriminator_write_variant() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "event": { + "TypeDef:Union": true, + "discriminator": { "kind": "field", "name": "type" }, + "mapping": { + "read": { "$ref": "#/$defs/Read" }, + "write": { "$ref": "#/$defs/Write" } + } + } + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": { + "type": { "TypeDef:Uint8": true }, + "handle": { "TypeDef:Uint32": true } + } + }, + "Write": { + "TypeDef:Struct": true, + "properties": { + "type": { "TypeDef:Uint8": true }, + "handle": { "TypeDef:Uint32": true }, + "length": { "TypeDef:Uint32": true } + } + } + } + }); + let vs = var_sizes(&[("event.__variant", 1)]); + let layout = build(&schema, &vs); + assert_eq!( + layout.get("event.length"), + Some(&FieldPosition { + offset: 5, + size: 4, + kind: TypeDefKind::Uint32 + }) + ); + assert_eq!(layout.total_size(), 9); + } + + #[test] + fn union_field_name_missing_variant_index_is_offset_error() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "event": { + "TypeDef:Union": true, + "discriminator": { "kind": "field", "name": "type" }, + "mapping": { + "read": { "$ref": "#/$defs/Read" } + } + } + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": { "type": { "TypeDef:Uint8": true } } + } + } + }); + let builder = LayoutBuilder::new(&schema).expect("builder"); + let err = builder.build(&var_sizes(&[])).unwrap_err(); + assert!(matches!(err, TypedefError::Offset { .. })); + } + + #[test] + fn union_field_name_variant_index_out_of_range_is_offset_error() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "event": { + "TypeDef:Union": true, + "discriminator": { "kind": "field", "name": "type" }, + "mapping": { + "read": { "$ref": "#/$defs/Read" } + } + } + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": { "type": { "TypeDef:Uint8": true } } + } + } + }); + let vs = var_sizes(&[("event.__variant", 5)]); + let builder = LayoutBuilder::new(&schema).expect("builder"); + let err = builder.build(&vs).unwrap_err(); + assert!(matches!(err, TypedefError::Offset { .. })); + } + + #[test] + fn iter_returns_fields_in_layout_order() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "a": { "TypeDef:Uint8": true }, + "b": { "TypeDef:Uint32": true }, + "c": { "TypeDef:Uint16": true } + } + }); + let layout = build(&schema, &var_sizes(&[])); + let paths: Vec<&str> = layout.iter().map(|(p, _)| p.as_str()).collect(); + assert_eq!(paths, vec!["a", "b", "c"]); + } + + #[test] + fn iter_includes_nested_struct_fields() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "header": { + "TypeDef:Struct": true, + "properties": { + "magic": { "TypeDef:Uint32": true }, + "version": { "TypeDef:Uint8": true } + } + }, + "body": { "TypeDef:Uint32": true } + } + }); + let layout = build(&schema, &var_sizes(&[])); + let paths: Vec<&str> = layout.iter().map(|(p, _)| p.as_str()).collect(); + assert_eq!(paths, vec!["header.magic", "header.version", "body"]); + } + + #[test] + fn get_returns_none_for_unknown_path() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "a": { "TypeDef:Uint8": true } + } + }); + let layout = build(&schema, &var_sizes(&[])); + assert!(layout.get("missing").is_none()); + } + + #[test] + fn endian_parsed_from_schema() { + let schema = json!({ + "TypeDef:Struct": true, + "endian": "big", + "properties": { + "id": { "TypeDef:Uint32": true } + } + }); + let builder = LayoutBuilder::new(&schema).expect("builder"); + assert_eq!(builder.endian(), Endian::Big); + } + + #[test] + fn endian_defaults_to_little() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "id": { "TypeDef:Uint32": true } + } + }); + let builder = LayoutBuilder::new(&schema).expect("builder"); + assert_eq!(builder.endian(), Endian::Little); + } + + #[test] + fn new_rejects_non_struct_top_level() { + let schema = json!({ "TypeDef:Uint32": true }); + let err = LayoutBuilder::new(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_))); + } + + #[test] + fn new_rejects_missing_typedef_kind() { + let schema = json!({ "type": "object", "properties": {} }); + let err = LayoutBuilder::new(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_))); + } + + #[test] + fn build_rejects_struct_without_properties() { + let schema = json!({ "TypeDef:Struct": true }); + let builder = LayoutBuilder::new(&schema).expect("builder"); + let err = builder.build(&var_sizes(&[])).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_))); + } + + #[test] + fn build_rejects_field_without_typedef_kind() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "a": { "type": "integer" } + } + }); + let builder = LayoutBuilder::new(&schema).expect("builder"); + let err = builder.build(&var_sizes(&[])).unwrap_err(); + assert!(matches!(err, TypedefError::Offset { .. })); + } + + #[test] + fn object_form_keyword_is_recognized() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "name": { "TypeDef:String": { "encoding": "length-prefixed" } } + } + }); + let layout = build(&schema, &var_sizes(&[("name", 5)])); + assert_eq!( + layout.get("name"), + Some(&FieldPosition { + offset: 0, + size: 4, + kind: TypeDefKind::String + }) + ); + assert_eq!(layout.total_size(), 9); + } + + #[test] + fn all_fixed_size_kinds_get_correct_sizes() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "i8": { "TypeDef:Int8": true }, + "i16": { "TypeDef:Int16": true }, + "i32": { "TypeDef:Int32": true }, + "u8": { "TypeDef:Uint8": true }, + "u16": { "TypeDef:Uint16": true }, + "u32": { "TypeDef:Uint32": true }, + "f32": { "TypeDef:Float32": true }, + "f64": { "TypeDef:Float64": true }, + "b": { "TypeDef:Boolean": true }, + "e": { "TypeDef:Enum": true } + } + }); + let layout = build(&schema, &var_sizes(&[])); + assert_eq!(layout.get("i8").unwrap().size, 1); + assert_eq!(layout.get("i16").unwrap().size, 2); + assert_eq!(layout.get("i32").unwrap().size, 4); + assert_eq!(layout.get("u8").unwrap().size, 1); + assert_eq!(layout.get("u16").unwrap().size, 2); + assert_eq!(layout.get("u32").unwrap().size, 4); + assert_eq!(layout.get("f32").unwrap().size, 4); + assert_eq!(layout.get("f64").unwrap().size, 8); + assert_eq!(layout.get("b").unwrap().size, 1); + assert_eq!(layout.get("e").unwrap().size, 4); + assert_eq!(layout.total_size(), 1 + 2 + 4 + 1 + 2 + 4 + 4 + 8 + 1 + 4); + } + + #[test] + fn empty_struct_produces_zero_size_layout() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": {} + }); + let layout = build(&schema, &var_sizes(&[])); + assert_eq!(layout.total_size(), 0); + assert_eq!(layout.iter().count(), 0); + } + + #[test] + fn inline_variant_schema_works_without_ref() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "packet": { + "TypeDef:Union": true, + "discriminator": { + "kind": "byte", + "offset": 0, + "type": "TypeDef:Uint8" + }, + "mapping": { + "5": { + "TypeDef:Struct": true, + "properties": { + "x": { "TypeDef:Uint32": true } + } + } + } + } + } + }); + let vs = var_sizes(&[("packet.__discriminator", 5)]); + let layout = build(&schema, &vs); + assert_eq!( + layout.get("packet.__discriminator"), + Some(&FieldPosition { + offset: 0, + size: 1, + kind: TypeDefKind::Uint8 + }) + ); + assert_eq!( + layout.get("packet.x"), + Some(&FieldPosition { + offset: 1, + size: 4, + kind: TypeDefKind::Uint32 + }) + ); + assert_eq!(layout.total_size(), 5); + } + + #[test] + fn union_nested_inside_struct_after_preceding_field() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "id": { "TypeDef:Uint8": true }, + "payload": { + "TypeDef:Union": true, + "discriminator": { + "kind": "byte", + "offset": 0, + "type": "TypeDef:Uint8" + }, + "mapping": { + "5": { "$ref": "#/$defs/Read" } + } + } + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": { + "handle": { "TypeDef:Uint32": true } + } + } + } + }); + let vs = var_sizes(&[("payload.__discriminator", 5)]); + let layout = build(&schema, &vs); + assert_eq!( + layout.get("id"), + Some(&FieldPosition { + offset: 0, + size: 1, + kind: TypeDefKind::Uint8 + }) + ); + assert_eq!( + layout.get("payload.__discriminator"), + Some(&FieldPosition { + offset: 1, + size: 1, + kind: TypeDefKind::Uint8 + }) + ); + assert_eq!( + layout.get("payload.handle"), + Some(&FieldPosition { + offset: 2, + size: 4, + kind: TypeDefKind::Uint32 + }) + ); + assert_eq!(layout.total_size(), 6); + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..73eaaa8 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,47 @@ +//! alknet-typedef: The binary struct engine. +//! +//! Takes a JSON Schema with `TypeDef:*` custom keywords and produces +//! an offset map, read/write functions, and validation — all driven +//! by the schema. The schema is the format definition; the engine is +//! generic. +//! +//! ## Architecture +//! +//! - **Schema layer** ([`schema`]): TypeDef kind detection, annotation +//! parsing, `$ref` normalization, endianness. +//! - **Layout engine** ([`offset_map`], [`layout_builder`], +//! [`sequential_reader`]): Two layout modes — aligned static for +//! mmap-friendly formats, packed sequential for protocol wire formats. +//! - **Data access** ([`data_access`]): Typed read/write at computed +//! offsets, zero-copy for fixed-size types. +//! - **TUnion dispatch** ([`tunion`]): Byte-offset and field-name +//! discriminator dispatch. +//! - **Validation** ([`validation`]): Custom keyword validators for all +//! 17 `TypeDef:*` kinds, delegated to the `jsonschema` crate. +//! - **Engine** ([`engine`]): `TypedefEngine` — the compiled form of a +//! schema, combining layout and validation. + +#[macro_use] +mod macros; +pub mod data_access; +pub mod engine; +pub mod error; +pub mod layout_builder; +pub mod offset_map; +pub mod schema; +pub mod sequential_reader; +pub mod tunion; +pub mod validation; + +pub use engine::{LayoutMode, TypedefEngine}; +pub use error::TypedefError; +pub use layout_builder::{FieldPosition, LayoutBuilder, PackedLayout}; +pub use offset_map::{ByteRange, OffsetMap}; +pub use schema::{ + get_typedef_kind_loose, get_typedef_kind_loose_enum, normalize_refs, parse_align, + parse_discriminator, parse_encoding, parse_endian, parse_max_length, resolve_ref, + resolve_ref_or_inline, DiscriminatorKind, Endian, TypeDefKind, VariableEncoding, +}; +pub use sequential_reader::{FieldValue, SequentialReader}; +pub use tunion::UnionDispatch; +pub use validation::build_validator; diff --git a/src/macros.rs b/src/macros.rs new file mode 100644 index 0000000..0711d2d --- /dev/null +++ b/src/macros.rs @@ -0,0 +1,251 @@ +//! Macros for generating repetitive code across the 17 TypeDef kinds. +//! +//! These macros eliminate boilerplate in validation, data access, and +//! dispatch. Each macro takes a compact specification and generates the +//! full implementation, ensuring consistency across all types. + +// --------------------------------------------------------------------------- +// Validation macros +// --------------------------------------------------------------------------- + +/// Generate a signed integer validator struct and its factory closure. +#[macro_export] +macro_rules! define_int_validator { + ($validator_struct:ident, $factory_fn:ident, $keyword:literal, $min:literal, $max:literal) => { + struct $validator_struct; + impl jsonschema::Keyword for $validator_struct { + fn validate<'i>( + &self, + instance: &'i serde_json::Value, + ) -> Result<(), jsonschema::ValidationError<'i>> { + match instance.as_i64() { + Some(n) if ($min..=$max).contains(&n) => Ok(()), + _ => Err(jsonschema::ValidationError::custom(concat!( + "expected an integer in range [", + stringify!($min), + ", ", + stringify!($max), + "]" + ))), + } + } + fn is_valid(&self, instance: &serde_json::Value) -> bool { + instance + .as_i64() + .is_some_and(|n| ($min..=$max).contains(&n)) + } + } + + fn $factory_fn<'a>( + _parent: &'a serde_json::Map, + value: &'a serde_json::Value, + _path: jsonschema::paths::Location, + ) -> Result, jsonschema::ValidationError<'a>> { + if value.as_bool() == Some(true) { + Ok(Box::new($validator_struct)) + } else { + Err(jsonschema::ValidationError::schema(concat!( + $keyword, + " must be set to true" + ))) + } + } + }; +} + +/// Generate an unsigned integer validator struct and its factory closure. +#[macro_export] +macro_rules! define_uint_validator { + ($validator_struct:ident, $factory_fn:ident, $keyword:literal, $max:literal) => { + struct $validator_struct; + impl jsonschema::Keyword for $validator_struct { + fn validate<'i>( + &self, + instance: &'i serde_json::Value, + ) -> Result<(), jsonschema::ValidationError<'i>> { + match instance.as_u64() { + Some(n) if n <= $max => Ok(()), + _ => Err(jsonschema::ValidationError::custom(concat!( + "expected an unsigned integer in range [0, ", + stringify!($max), + "]" + ))), + } + } + fn is_valid(&self, instance: &serde_json::Value) -> bool { + instance.as_u64().is_some_and(|n| n <= $max) + } + } + + fn $factory_fn<'a>( + _parent: &'a serde_json::Map, + value: &'a serde_json::Value, + _path: jsonschema::paths::Location, + ) -> Result, jsonschema::ValidationError<'a>> { + if value.as_bool() == Some(true) { + Ok(Box::new($validator_struct)) + } else { + Err(jsonschema::ValidationError::schema(concat!( + $keyword, + " must be set to true" + ))) + } + } + }; +} + +/// Generate a float validator struct and its factory closure. +#[macro_export] +macro_rules! define_float_validator { + ($validator_struct:ident, $factory_fn:ident, $keyword:literal, $error_msg:literal) => { + struct $validator_struct; + impl jsonschema::Keyword for $validator_struct { + fn validate<'i>( + &self, + instance: &'i serde_json::Value, + ) -> Result<(), jsonschema::ValidationError<'i>> { + match instance.as_f64() { + Some(f) if f.is_finite() => Ok(()), + _ => Err(jsonschema::ValidationError::custom($error_msg)), + } + } + fn is_valid(&self, instance: &serde_json::Value) -> bool { + instance.as_f64().is_some_and(|f| f.is_finite()) + } + } + + fn $factory_fn<'a>( + _parent: &'a serde_json::Map, + value: &'a serde_json::Value, + _path: jsonschema::paths::Location, + ) -> Result, jsonschema::ValidationError<'a>> { + if value.as_bool() == Some(true) { + Ok(Box::new($validator_struct)) + } else { + Err(jsonschema::ValidationError::schema(concat!( + $keyword, + " must be set to true" + ))) + } + } + }; +} + +/// Generate a simple type-check validator (object/array/boolean) and its factory. +#[macro_export] +macro_rules! define_type_validator { + ($validator_struct:ident, $factory_fn:ident, $keyword:literal, $check_method:ident, $error_msg:literal) => { + struct $validator_struct; + impl jsonschema::Keyword for $validator_struct { + fn validate<'i>( + &self, + instance: &'i serde_json::Value, + ) -> Result<(), jsonschema::ValidationError<'i>> { + if instance.$check_method() { + Ok(()) + } else { + Err(jsonschema::ValidationError::custom($error_msg)) + } + } + fn is_valid(&self, instance: &serde_json::Value) -> bool { + instance.$check_method() + } + } + + fn $factory_fn<'a>( + _parent: &'a serde_json::Map, + value: &'a serde_json::Value, + _path: jsonschema::paths::Location, + ) -> Result, jsonschema::ValidationError<'a>> { + if value.as_bool() == Some(true) { + Ok(Box::new($validator_struct)) + } else { + Err(jsonschema::ValidationError::schema(concat!( + $keyword, + " must be set to true" + ))) + } + } + }; +} + +// --------------------------------------------------------------------------- +// Data access macros +// --------------------------------------------------------------------------- + +/// Generate a pair of read/write functions for a fixed-size endian-sensitive type. +#[macro_export] +macro_rules! define_read_write_endian { + ($rust_ty:ty, $read_name:ident, $write_name:ident, $size:literal) => { + #[doc = concat!( + "Read a `", + stringify!($rust_ty), + "` at `offset` from `buffer`, applying `endian`." + )] + pub fn $read_name( + buffer: &[u8], + offset: usize, + field_path: &str, + endian: $crate::Endian, + ) -> Result<$rust_ty, $crate::TypedefError> { + let bytes: [u8; $size] = $crate::data_access::read_array(buffer, offset, field_path)?; + Ok(match endian { + $crate::Endian::Little => <$rust_ty>::from_le_bytes(bytes), + $crate::Endian::Big => <$rust_ty>::from_be_bytes(bytes), + }) + } + + #[doc = concat!( + "Write a `", + stringify!($rust_ty), + "` `value` at `offset` into `buffer`, applying `endian`." + )] + pub fn $write_name( + buffer: &mut [u8], + offset: usize, + value: $rust_ty, + field_path: &str, + endian: $crate::Endian, + ) -> Result<(), $crate::TypedefError> { + let bytes = match endian { + $crate::Endian::Little => value.to_le_bytes(), + $crate::Endian::Big => value.to_be_bytes(), + }; + $crate::data_access::write_array(buffer, offset, bytes, field_path) + } + }; +} + +/// Generate a pair of read/write functions for a fixed-size endian-insensitive type. +#[macro_export] +macro_rules! define_read_write_ne { + ($rust_ty:ty, $read_name:ident, $write_name:ident, $size:literal, $read_expr:expr) => { + #[doc = concat!( + "Read a `", + stringify!($rust_ty), + "` at `offset` from `buffer`." + )] + pub fn $read_name( + buffer: &[u8], + offset: usize, + field_path: &str, + ) -> Result<$rust_ty, $crate::TypedefError> { + let bytes: [u8; $size] = $crate::data_access::read_array(buffer, offset, field_path)?; + Ok($read_expr(bytes)) + } + + #[doc = concat!( + "Write a `", + stringify!($rust_ty), + "` `value` at `offset` into `buffer`." + )] + pub fn $write_name( + buffer: &mut [u8], + offset: usize, + value: $rust_ty, + field_path: &str, + ) -> Result<(), $crate::TypedefError> { + $crate::data_access::write_array(buffer, offset, value.to_ne_bytes(), field_path) + } + }; +} diff --git a/src/offset_map.rs b/src/offset_map.rs new file mode 100644 index 0000000..0df45ea --- /dev/null +++ b/src/offset_map.rs @@ -0,0 +1,873 @@ +//! Aligned static `OffsetMap` — Mode 2 of the two layout modes (ADR-096). +//! +//! Fields have fixed positions with natural alignment padding. +//! Variable-length fields get a 4-byte length prefix at a known offset; +//! the variable data is not included in the static layout. Used for +//! mmap-friendly formats (metatensor, safetensors). +//! +//! The offset computation is a recursive walk of the schema JSON. Nested +//! structs propagate field path prefixes (producing dotted paths like +//! `"header.version"`). Alignment padding is inserted before each field +//! to satisfy the field's alignment requirement (natural alignment by +//! default, overridable via the `"align"` annotation). + +use crate::error::TypedefError; +use crate::schema::{ + get_typedef_kind, get_typedef_kind_loose_enum, parse_align, parse_encoding, + parse_max_length, resolve_ref_or_inline, TypeDefKind, VariableEncoding, +}; +use serde_json::Value; + +/// A byte range within a buffer. +/// +/// Produced by [`OffsetMap::compute`] for each field in a schema. The +/// range is half-open: `start..end`. `end - start` is the field's byte +/// size in the static layout (for variable-length fields, this is the +/// size of the length prefix, the `{offset, length}` pair, or the +/// `maxLength` reservation — not the variable data itself). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ByteRange { + /// Inclusive start byte offset. + pub start: usize, + /// Exclusive end byte offset. + pub end: usize, +} + +impl ByteRange { + /// Byte length of the range (`end - start`). + pub fn len(&self) -> usize { + self.end - self.start + } + + /// True if the range covers zero bytes. + pub fn is_empty(&self) -> bool { + self.end == self.start + } +} + +/// A flat table of `(field_path, byte_range)` pairs computed from a schema. +/// +/// Fields have fixed positions with natural alignment padding. +/// Used for mmap-friendly formats (metatensor, safetensors) where random +/// access by field path is required — the consumer can read field N +/// without reading fields `0..N-1` first. +/// +/// Construct via [`OffsetMap::compute`]. Variable-length fields appear +/// in the table as their fixed-position portion only (length prefix, +/// `{offset, length}` pair, or `maxLength` reservation); the variable +/// data lives outside the static layout. +#[derive(Debug)] +pub struct OffsetMap { + fields: Vec<(String, ByteRange)>, + total_size: usize, +} + +impl OffsetMap { + /// Compute the offset map from a schema JSON value. + /// + /// Walks the schema recursively, computing byte positions for each + /// field based on type sizes, field order, and alignment. The + /// top-level schema must be a `TypeDef:Struct`. + /// + /// # Errors + /// + /// Returns [`TypedefError::Schema`] if the top-level schema is not a + /// `TypeDef:Struct` or has no `TypeDef:*` kind, or if the schema is + /// malformed (missing `properties`, unknown kind, etc.). + /// + /// Returns [`TypedefError::Offset`] for unsupported type combinations + /// encountered during the walk. + pub fn compute(schema: &Value) -> Result { + let kind = get_typedef_kind(schema) + .and_then(|s| s.parse::().ok()) + .ok_or_else(|| { + TypedefError::Schema("top-level schema has no TypeDef:* kind".to_string()) + })?; + if kind != TypeDefKind::Struct { + return Err(TypedefError::Schema(format!( + "OffsetMap::compute requires a TypeDef:Struct at the top level, got {kind}" + ))); + } + let mut ctx = ComputeCtx { + root: schema, + fields: Vec::new(), + offset: 0, + }; + let (total, _align) = ctx.compute_struct(schema, "", 1)?; + Ok(Self { + fields: ctx.fields, + total_size: total, + }) + } + + /// Look up a field's byte range by dotted path (e.g., `"header.version"`). + /// + /// Returns `None` if no field with the given path was recorded. For + /// TUnion byte-offset discriminators, the discriminator is recorded + /// under the synthetic path `"__discriminator"` (qualified by the + /// union field's path, e.g. `"payload.__discriminator"`). + pub fn get(&self, field_path: &str) -> Option<&ByteRange> { + self.fields + .iter() + .find(|(path, _)| path == field_path) + .map(|(_, range)| range) + } + + /// The total size of the struct in bytes (including trailing alignment padding). + pub fn total_size(&self) -> usize { + self.total_size + } + + /// Iterate over all `(field_path, byte_range)` pairs in insertion order. + /// + /// Field order matches the schema's `properties` order (preserved by + /// `serde_json`'s `preserve_order` feature). Nested struct fields + /// appear after their parent's path prefix. + pub fn iter(&self) -> impl Iterator { + self.fields.iter() + } +} + +/// Mutable context threaded through the recursive offset computation. +/// +/// Carries the running `offset`, the accumulating `fields` vec, and a +/// reference to the root schema for `$ref` resolution. Grouping these +/// keeps the recursive helper signatures small. +struct ComputeCtx<'a> { + root: &'a Value, + fields: Vec<(String, ByteRange)>, + offset: usize, +} + +/// Result of laying out a single field: its alignment. +struct FieldLayout { + align: usize, +} + +impl<'a> ComputeCtx<'a> { + /// Recurse into a `TypeDef:Struct`, appending `(field_path, ByteRange)` + /// pairs to `self.fields` and advancing `self.offset`. + /// + /// Returns `(total_size, alignment)` where `total_size` includes + /// trailing alignment padding and `alignment` is the struct's + /// effective alignment (its own `align` annotation, or the max of its + /// fields' alignments). + /// + /// `struct_schema` is the schema of the struct to walk. `prefix` is the + /// dotted path prefix for nested fields (empty at the top level). + /// `parent_struct_align` is the default alignment a field inherits + /// when it specifies neither its own `align` annotation nor a natural + /// alignment larger than the default. + fn compute_struct( + &mut self, + struct_schema: &Value, + prefix: &str, + parent_struct_align: usize, + ) -> Result<(usize, usize), TypedefError> { + let obj = struct_schema + .as_object() + .ok_or_else(|| TypedefError::Schema("struct schema is not an object".to_string()))?; + let properties = obj + .get("properties") + .and_then(|v| v.as_object()) + .ok_or_else(|| { + TypedefError::Schema("struct schema has no 'properties' object".to_string()) + })?; + + let struct_default_align = parse_align(struct_schema).unwrap_or(parent_struct_align); + let mut max_align: usize = 1; + let struct_start = self.offset; + + let field_schemas: Vec<(String, Value)> = properties + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + let field_count = field_schemas.len(); + for (i, (field_name, field_schema)) in field_schemas.iter().enumerate() { + let field_path = if prefix.is_empty() { + field_name.clone() + } else { + format!("{prefix}.{field_name}") + }; + // ADR-100: reject non-final inline length-prefixed variable fields. + // The OffsetMap reserves only 4 bytes (the length prefix), but + // data_access::write_string writes prefix+data inline — clobbering + // subsequent fields. Only allowed as the last field in the struct. + if i < field_count - 1 { + if let Some(kind) = get_typedef_kind_loose_enum(field_schema) { + if kind.is_variable_length() { + let keyword_value = field_schema + .as_object() + .and_then(|o| { + o.keys() + .find(|k| k.starts_with("TypeDef:")) + .and_then(|k| o.get(k)) + }) + .cloned() + .unwrap_or(Value::Bool(true)); + let encoding = parse_encoding(&keyword_value); + let max_length = parse_max_length(field_schema); + let is_inline_length_prefixed = + encoding == VariableEncoding::LengthPrefixed && max_length.is_none(); + if is_inline_length_prefixed { + return Err(TypedefError::Offset { + field_path: field_path.clone(), + reason: format!( + "non-final inline length-prefixed variable field \ + ({kind}) in aligned mode: the variable data would \ + clobber subsequent fields. Use `maxLength` \ + (fixed-size reservation) or \ + `\"encoding\": \"offset-indirect\"`, or move this \ + field to the last position in the struct. (ADR-100)" + ), + }); + } + } + } + } + let layout = self.compute_field(field_schema, &field_path, struct_default_align)?; + if layout.align > max_align { + max_align = layout.align; + } + } + + let effective_align = parse_align(struct_schema).unwrap_or(max_align).max(1); + align_up(&mut self.offset, effective_align); + let total = self.offset - struct_start; + Ok((total, effective_align)) + } + + /// Compute the layout for a single field, advancing `self.offset` + /// and appending any field paths to `self.fields`. + fn compute_field( + &mut self, + field_schema: &Value, + field_path: &str, + struct_default_align: usize, + ) -> Result { + let kind = get_typedef_kind_loose_enum(field_schema).ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "field schema has no TypeDef:* kind".to_string(), + })?; + + match kind { + TypeDefKind::Struct => { + self.compute_struct_field(field_schema, field_path, struct_default_align) + } + TypeDefKind::Union => Err(TypedefError::Offset { + field_path: field_path.to_string(), + reason: "TUnion is not supported in aligned static mode (ADR-102). \ + Unions are the protocol dispatch pattern — use packed sequential \ + mode (LayoutMode::Packed) for TUnion fields, or restructure as \ + a struct with an explicit discriminator field." + .to_string(), + }), + TypeDefKind::Array => { + self.compute_array_field(field_schema, field_path, struct_default_align) + } + TypeDefKind::String + | TypeDefKind::Bytes + | TypeDefKind::Record + | TypeDefKind::Timestamp => { + self.compute_variable_field(field_schema, field_path, struct_default_align) + } + k if k.is_fixed_size() => { + self.compute_fixed_field(k, field_schema, field_path, struct_default_align) + } + _ => unreachable!("all TypeDefKind variants are covered above"), + } + } + + /// Compute the layout for a fixed-size primitive field. + fn compute_fixed_field( + &mut self, + kind: TypeDefKind, + field_schema: &Value, + field_path: &str, + struct_default_align: usize, + ) -> Result { + let size = kind.type_size().ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("type_size returned None for fixed kind {kind}"), + })?; + let natural = kind.natural_alignment(); + let align = field_alignment(field_schema, struct_default_align, natural); + align_up(&mut self.offset, align); + let start = self.offset; + self.offset += size; + self.push(field_path, start, start + size); + Ok(FieldLayout { align }) + } + + /// Compute the layout for a nested `TypeDef:Struct` field. + /// + /// Probes the nested struct's layout at a temporary offset of 0 to + /// determine its total size and alignment, aligns the parent offset, + /// then shifts the nested fields to their final positions. + fn compute_struct_field( + &mut self, + field_schema: &Value, + field_path: &str, + struct_default_align: usize, + ) -> Result { + let inner_parent_align = parse_align(field_schema).unwrap_or(struct_default_align); + let mut probe = ComputeCtx { + root: self.root, + fields: Vec::new(), + offset: 0, + }; + let (inner_total, inner_align) = + probe.compute_struct(field_schema, field_path, inner_parent_align)?; + + let align = field_alignment(field_schema, struct_default_align, inner_align); + align_up(&mut self.offset, align); + let struct_start = self.offset; + for (path, range) in probe.fields { + self.fields.push(( + path, + ByteRange { + start: struct_start + range.start, + end: struct_start + range.end, + }, + )); + } + self.offset = struct_start + inner_total; + Ok(FieldLayout { align }) + } + + /// Compute the layout for a `TypeDef:Array` field. + fn compute_array_field( + &mut self, + field_schema: &Value, + field_path: &str, + struct_default_align: usize, + ) -> Result { + let obj = field_schema + .as_object() + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "array schema is not an object".to_string(), + })?; + + let items = obj.get("items").ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "TArray is missing 'items'".to_string(), + })?; + let element_schema = + resolve_ref_or_inline(items, self.root).ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "could not resolve TArray items schema".to_string(), + })?; + let elem_kind = get_typedef_kind(element_schema) + .and_then(|s| s.parse::().ok()) + .ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: "TArray element schema has no TypeDef:* kind".to_string(), + })?; + if !elem_kind.is_fixed_size() { + return Err(TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!( + "TArray of variable-length element kind {elem_kind} is not supported (OQ-069)" + ), + }); + } + + let elem_size = elem_kind.type_size().ok_or_else(|| TypedefError::Offset { + field_path: field_path.to_string(), + reason: format!("element kind {elem_kind} has no fixed size"), + })?; + let elem_natural = elem_kind.natural_alignment(); + let elem_align = field_alignment(element_schema, struct_default_align, elem_natural); + let stride = round_up(elem_size, elem_align); + + let min_items = obj + .get("minItems") + .and_then(|v| v.as_u64()) + .map(|n| n as usize); + let max_items = obj + .get("maxItems") + .and_then(|v| v.as_u64()) + .map(|n| n as usize); + let fixed_count = match (min_items, max_items) { + (Some(mn), Some(mx)) if mn == mx => Some(mn), + _ => None, + }; + + let array_align = field_alignment(field_schema, struct_default_align, elem_align); + + if let Some(count) = fixed_count { + align_up(&mut self.offset, array_align); + let start = self.offset; + for i in 0..count { + let elem_start = start + i * stride; + let elem_end = elem_start + elem_size; + let elem_path = format!("{field_path}[{i}]"); + self.push(&elem_path, elem_start, elem_end); + } + let array_size = count * stride; + self.offset = start + array_size; + Ok(FieldLayout { align: array_align }) + } else { + let count_prefix_align = array_align.max(4); + align_up(&mut self.offset, count_prefix_align); + let start = self.offset; + self.push(field_path, start, start + 4); + self.offset = start + 4; + Ok(FieldLayout { + align: count_prefix_align, + }) + } + } + + /// Compute the layout for a variable-length field (String/Bytes/Record/Timestamp). + /// + /// In aligned static mode, three strategies are supported: + /// - `maxLength` reservation: `maxLength` bytes at a fixed offset. + /// - `offset-indirect` encoding: an 8-byte `{offset: u32, length: u32}` pair. + /// - inline length-prefixing (default): a 4-byte length prefix. + fn compute_variable_field( + &mut self, + field_schema: &Value, + field_path: &str, + struct_default_align: usize, + ) -> Result { + let keyword_value = field_schema + .as_object() + .and_then(|o| { + o.keys() + .find(|k| k.starts_with("TypeDef:")) + .and_then(|k| o.get(k)) + }) + .cloned() + .unwrap_or(Value::Bool(true)); + let encoding = parse_encoding(&keyword_value); + let max_length = parse_max_length(field_schema); + + let (size, natural) = match (max_length, encoding) { + (Some(max_len), _) => (max_len, 1), + (None, VariableEncoding::OffsetIndirect) => (8, 4), + (None, VariableEncoding::LengthPrefixed) => (4, 4), + }; + + let align = field_alignment(field_schema, struct_default_align, natural); + align_up(&mut self.offset, align); + let start = self.offset; + self.offset += size; + self.push(field_path, start, start + size); + Ok(FieldLayout { align }) + } + + /// Push a `(field_path, ByteRange)` pair onto the fields vec. + fn push(&mut self, path: &str, start: usize, end: usize) { + self.fields + .push((path.to_string(), ByteRange { start, end })); + } +} + +/// Resolve the field's alignment: field-level `align` annotation, +/// then the struct default, then the natural alignment. +fn field_alignment(field_schema: &Value, struct_default_align: usize, natural: usize) -> usize { + if let Some(a) = parse_align(field_schema) { + return a.max(1); + } + struct_default_align.max(natural).max(1) +} + +/// Round `offset` up to the next multiple of `align`. No-op if `align <= 1`. +fn align_up(offset: &mut usize, align: usize) { + if align <= 1 { + return; + } + let rem = *offset % align; + if rem != 0 { + *offset += align - rem; + } +} + +/// Round `n` up to the next multiple of `align`. +fn round_up(n: usize, align: usize) -> usize { + if align <= 1 { + return n; + } + let rem = n % align; + if rem == 0 { + n + } else { + n + align - rem + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn map(schema: &Value) -> OffsetMap { + OffsetMap::compute(schema).expect("offset map computation") + } + + #[test] + fn simple_fixed_fields_natural_alignment() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "flag": { "TypeDef:Uint8": true }, + "id": { "TypeDef:Uint32": true } + } + }); + let m = map(&schema); + assert_eq!(m.get("flag"), Some(&ByteRange { start: 0, end: 1 })); + assert_eq!(m.get("id"), Some(&ByteRange { start: 4, end: 8 })); + assert_eq!(m.total_size(), 8); + } + + #[test] + fn u8_then_u32_three_bytes_padding() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "a": { "TypeDef:Uint8": true }, + "b": { "TypeDef:Uint32": true } + } + }); + let m = map(&schema); + assert_eq!(m.get("a"), Some(&ByteRange { start: 0, end: 1 })); + assert_eq!(m.get("b"), Some(&ByteRange { start: 4, end: 8 })); + } + + #[test] + fn nested_struct_dotted_paths() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "header": { + "TypeDef:Struct": true, + "properties": { + "magic": { "TypeDef:Uint32": true }, + "version": { "TypeDef:Uint8": true } + } + }, + "body": { "TypeDef:Uint32": true } + } + }); + let m = map(&schema); + assert_eq!(m.get("header.magic"), Some(&ByteRange { start: 0, end: 4 })); + assert_eq!( + m.get("header.version"), + Some(&ByteRange { start: 4, end: 5 }) + ); + assert_eq!(m.get("body"), Some(&ByteRange { start: 8, end: 12 })); + assert_eq!(m.total_size(), 12); + } + + #[test] + fn array_fixed_count_element_offsets() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "vals": { + "TypeDef:Array": true, + "items": { "TypeDef:Uint32": true }, + "minItems": 3, + "maxItems": 3 + } + } + }); + let m = map(&schema); + assert_eq!(m.get("vals[0]"), Some(&ByteRange { start: 0, end: 4 })); + assert_eq!(m.get("vals[1]"), Some(&ByteRange { start: 4, end: 8 })); + assert_eq!(m.get("vals[2]"), Some(&ByteRange { start: 8, end: 12 })); + assert_eq!(m.total_size(), 12); + } + + #[test] + fn array_variable_count_length_prefix() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "vals": { + "TypeDef:Array": true, + "items": { "TypeDef:Uint32": true } + } + } + }); + let m = map(&schema); + assert_eq!(m.get("vals"), Some(&ByteRange { start: 0, end: 4 })); + assert_eq!(m.total_size(), 4); + } + + #[test] + fn variable_string_length_prefix_at_known_offset() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "id": { "TypeDef:Uint32": true }, + "name": { "TypeDef:String": true } + } + }); + let m = map(&schema); + assert_eq!(m.get("id"), Some(&ByteRange { start: 0, end: 4 })); + assert_eq!(m.get("name"), Some(&ByteRange { start: 4, end: 8 })); + assert_eq!(m.total_size(), 8); + } + + #[test] + fn variable_string_max_length_reservation() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "id": { "TypeDef:Uint32": true }, + "name": { "TypeDef:String": true, "maxLength": 256 } + } + }); + let m = map(&schema); + assert_eq!(m.get("id"), Some(&ByteRange { start: 0, end: 4 })); + assert_eq!(m.get("name"), Some(&ByteRange { start: 4, end: 260 })); + assert_eq!(m.total_size(), 260); + } + + #[test] + fn variable_string_offset_indirect_eight_bytes() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "id": { "TypeDef:Uint32": true }, + "blob": { "TypeDef:String": { "encoding": "offset-indirect" } } + } + }); + let m = map(&schema); + assert_eq!(m.get("id"), Some(&ByteRange { start: 0, end: 4 })); + assert_eq!(m.get("blob"), Some(&ByteRange { start: 4, end: 12 })); + assert_eq!(m.total_size(), 12); + } + + #[test] + fn union_byte_discriminator_rejected_in_aligned_mode() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "payload": { + "TypeDef:Union": true, + "discriminator": { + "kind": "byte", + "offset": 0, + "type": "TypeDef:Uint8" + }, + "mapping": { + "5": { "$ref": "#/$defs/Read" }, + "6": { "$ref": "#/$defs/Write" } + } + } + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": { + "handle": { "TypeDef:Uint32": true }, + "length": { "TypeDef:Uint32": true } + } + }, + "Write": { + "TypeDef:Struct": true, + "properties": { + "handle": { "TypeDef:Uint32": true }, + "length": { "TypeDef:Uint32": true }, + "data": { "TypeDef:Uint32": true } + } + } + } + }); + let err = OffsetMap::compute(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Offset { .. }), "got {err:?}"); + let reason = match err { + TypedefError::Offset { reason, .. } => reason, + _ => unreachable!(), + }; + assert!(reason.contains("ADR-102"), "reason: {reason}"); + } + + #[test] + fn union_field_name_discriminator_rejected_in_aligned_mode() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "event": { + "TypeDef:Union": true, + "discriminator": { "kind": "field", "name": "type" }, + "mapping": { + "read": { "$ref": "#/$defs/Read" }, + "write": { "$ref": "#/$defs/Write" } + } + } + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": { + "type": { "TypeDef:Uint8": true }, + "handle": { "TypeDef:Uint32": true } + } + }, + "Write": { + "TypeDef:Struct": true, + "properties": { + "type": { "TypeDef:Uint8": true }, + "handle": { "TypeDef:Uint32": true }, + "length": { "TypeDef:Uint32": true } + } + } + } + }); + let err = OffsetMap::compute(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Offset { .. }), "got {err:?}"); + } + + #[test] + fn non_final_inline_string_rejected_in_aligned_mode() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "name": { "TypeDef:String": true }, + "id": { "TypeDef:Uint32": true } + } + }); + let err = OffsetMap::compute(&schema).unwrap_err(); + match err { + TypedefError::Offset { field_path, reason } => { + assert_eq!(field_path, "name"); + assert!(reason.contains("ADR-100"), "reason: {reason}"); + } + other => panic!("expected Offset, got {other:?}"), + } + } + + #[test] + fn final_inline_string_allowed_in_aligned_mode() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "id": { "TypeDef:Uint32": true }, + "name": { "TypeDef:String": true } + } + }); + let m = map(&schema); + assert_eq!(m.get("id"), Some(&ByteRange { start: 0, end: 4 })); + assert_eq!(m.get("name"), Some(&ByteRange { start: 4, end: 8 })); + } + + #[test] + fn non_final_maxlength_string_allowed_in_aligned_mode() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "name": { "TypeDef:String": true, "maxLength": 256 }, + "id": { "TypeDef:Uint32": true } + } + }); + let m = map(&schema); + assert_eq!(m.get("name"), Some(&ByteRange { start: 0, end: 256 })); + assert_eq!(m.get("id"), Some(&ByteRange { start: 256, end: 260 })); + } + + #[test] + fn non_final_offset_indirect_string_allowed_in_aligned_mode() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "blob": { "TypeDef:String": { "encoding": "offset-indirect" } }, + "id": { "TypeDef:Uint32": true } + } + }); + let m = map(&schema); + assert_eq!(m.get("blob"), Some(&ByteRange { start: 0, end: 8 })); + assert_eq!(m.get("id"), Some(&ByteRange { start: 8, end: 12 })); + } + + #[test] + fn struct_level_align_rounds_up_total() { + let schema = json!({ + "TypeDef:Struct": true, + "align": 16, + "properties": { + "flag": { "TypeDef:Uint8": true } + } + }); + let m = map(&schema); + assert_eq!(m.get("flag"), Some(&ByteRange { start: 0, end: 1 })); + assert_eq!(m.total_size(), 16); + } + + #[test] + fn field_level_align_overrides_struct_default() { + let schema = json!({ + "TypeDef:Struct": true, + "align": 1, + "properties": { + "tag": { "TypeDef:Uint8": true }, + "flag": { "TypeDef:Uint8": true, "align": 16 }, + "id": { "TypeDef:Uint32": true } + } + }); + let m = map(&schema); + assert_eq!(m.get("tag"), Some(&ByteRange { start: 0, end: 1 })); + assert_eq!(m.get("flag"), Some(&ByteRange { start: 16, end: 17 })); + assert_eq!(m.get("id"), Some(&ByteRange { start: 20, end: 24 })); + assert_eq!(m.total_size(), 24); + } + + #[test] + fn field_align_smaller_than_struct_default() { + let schema = json!({ + "TypeDef:Struct": true, + "align": 8, + "properties": { + "a": { "TypeDef:Uint8": true }, + "b": { "TypeDef:Uint32": true, "align": 1 } + } + }); + let m = map(&schema); + assert_eq!(m.get("a"), Some(&ByteRange { start: 0, end: 1 })); + assert_eq!(m.get("b"), Some(&ByteRange { start: 1, end: 5 })); + assert_eq!(m.total_size(), 8); + } + + #[test] + fn iter_returns_all_paths_in_order() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "a": { "TypeDef:Uint8": true }, + "b": { "TypeDef:Uint32": true } + } + }); + let m = map(&schema); + let paths: Vec<&String> = m.iter().map(|(p, _)| p).collect(); + assert_eq!(paths, vec!["a", "b"]); + } + + #[test] + fn compute_rejects_non_struct_top_level() { + let schema = + json!({ "TypeDef:Union": true, "discriminator": { "kind": "byte" }, "mapping": {} }); + let err = OffsetMap::compute(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_))); + } + + #[test] + fn compute_rejects_missing_typedef_kind() { + let schema = json!({ "type": "object", "properties": {} }); + let err = OffsetMap::compute(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_))); + } + + #[test] + fn byte_range_len_and_is_empty() { + let r = ByteRange { start: 4, end: 8 }; + assert_eq!(r.len(), 4); + assert!(!r.is_empty()); + let empty = ByteRange { start: 5, end: 5 }; + assert_eq!(empty.len(), 0); + assert!(empty.is_empty()); + } +} diff --git a/src/schema.rs b/src/schema.rs new file mode 100644 index 0000000..b64d01e --- /dev/null +++ b/src/schema.rs @@ -0,0 +1,812 @@ +//! Schema layer: TypeDef kind detection, annotation parsing, `$ref` +//! normalization, and the `Endian` enum. +//! +//! Per ADR-097 and the schema-layer spec. This module provides the +//! foundational types and functions that every other module depends on: +//! `TypeDef:*` kind detection, fixed byte-size lookups, natural alignment, +//! schema annotation parsing (`encoding`, `align`, `maxLength`, `endian`), +//! TUnion discriminator parsing, and `$ref` normalization from TypeBox +//! bare-name refs to JSON Pointer refs. + +use crate::error::TypedefError; +use serde_json::Value; +use std::fmt; +use std::str::FromStr; + +const TYPEDEF_PREFIX: &str = "TypeDef:"; + +pub(crate) const U32_SIZE: usize = 4; +pub(crate) const DISCRIMINATOR_PATH: &str = "__discriminator"; + +const BYTE_DISCRIMINATOR_TYPES: &[TypeDefKind] = &[ + TypeDefKind::Uint8, + TypeDefKind::Uint16, + TypeDefKind::Uint32, +]; + +/// The 19 `TypeDef:*` kinds recognized by the engine. +/// +/// Each variant corresponds to a `TypeDef:` JSON Schema keyword. +/// The enum provides compile-time exhaustiveness checking and integer +/// discriminant dispatch (jump table) instead of string comparison. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum TypeDefKind { + Int8, + Int16, + Int32, + Int64, + Uint8, + Uint16, + Uint32, + Uint64, + Float32, + Float64, + Boolean, + Enum, + String, + Bytes, + Struct, + Union, + Array, + Record, + Timestamp, +} + +impl TypeDefKind { + /// The JSON Schema keyword string, e.g. `"TypeDef:Int8"`. + pub fn as_str(self) -> &'static str { + match self { + TypeDefKind::Int8 => "TypeDef:Int8", + TypeDefKind::Int16 => "TypeDef:Int16", + TypeDefKind::Int32 => "TypeDef:Int32", + TypeDefKind::Int64 => "TypeDef:Int64", + TypeDefKind::Uint8 => "TypeDef:Uint8", + TypeDefKind::Uint16 => "TypeDef:Uint16", + TypeDefKind::Uint32 => "TypeDef:Uint32", + TypeDefKind::Uint64 => "TypeDef:Uint64", + TypeDefKind::Float32 => "TypeDef:Float32", + TypeDefKind::Float64 => "TypeDef:Float64", + TypeDefKind::Boolean => "TypeDef:Boolean", + TypeDefKind::Enum => "TypeDef:Enum", + TypeDefKind::String => "TypeDef:String", + TypeDefKind::Bytes => "TypeDef:Bytes", + TypeDefKind::Struct => "TypeDef:Struct", + TypeDefKind::Union => "TypeDef:Union", + TypeDefKind::Array => "TypeDef:Array", + TypeDefKind::Record => "TypeDef:Record", + TypeDefKind::Timestamp => "TypeDef:Timestamp", + } + } + + /// Fixed byte size, or `None` for variable-size / composite kinds. + pub fn type_size(self) -> Option { + match self { + TypeDefKind::Float32 | TypeDefKind::Int32 | TypeDefKind::Uint32 | TypeDefKind::Enum => { + Some(4) + } + TypeDefKind::Float64 | TypeDefKind::Int64 | TypeDefKind::Uint64 => Some(8), + TypeDefKind::Int8 | TypeDefKind::Uint8 | TypeDefKind::Boolean => Some(1), + TypeDefKind::Int16 | TypeDefKind::Uint16 => Some(2), + TypeDefKind::String + | TypeDefKind::Bytes + | TypeDefKind::Struct + | TypeDefKind::Union + | TypeDefKind::Array + | TypeDefKind::Record + | TypeDefKind::Timestamp => None, + } + } + + /// Natural alignment: 1 for u8/i8/bool, 2 for u16/i16, 4 for u32/i32/f32/enum, + /// 8 for u64/i64/f64, 4 for variable-length (u32 length prefix), 1 for + /// composites. + pub fn natural_alignment(self) -> usize { + match self { + TypeDefKind::Int8 | TypeDefKind::Uint8 | TypeDefKind::Boolean => 1, + TypeDefKind::Int16 | TypeDefKind::Uint16 => 2, + TypeDefKind::Int32 + | TypeDefKind::Uint32 + | TypeDefKind::Float32 + | TypeDefKind::Enum => 4, + TypeDefKind::Float64 | TypeDefKind::Int64 | TypeDefKind::Uint64 => 8, + TypeDefKind::String + | TypeDefKind::Bytes + | TypeDefKind::Record + | TypeDefKind::Timestamp => 4, + TypeDefKind::Struct | TypeDefKind::Union | TypeDefKind::Array => 1, + } + } + + /// Returns `true` for fixed-size primitive kinds. + pub fn is_fixed_size(self) -> bool { + matches!( + self, + TypeDefKind::Float32 + | TypeDefKind::Float64 + | TypeDefKind::Int8 + | TypeDefKind::Int16 + | TypeDefKind::Int32 + | TypeDefKind::Int64 + | TypeDefKind::Uint8 + | TypeDefKind::Uint16 + | TypeDefKind::Uint32 + | TypeDefKind::Uint64 + | TypeDefKind::Boolean + | TypeDefKind::Enum + ) + } + + /// Returns `true` for kinds whose read/write functions need an `Endian` parameter. + pub fn needs_endian(self) -> bool { + matches!( + self, + TypeDefKind::Int16 + | TypeDefKind::Int32 + | TypeDefKind::Int64 + | TypeDefKind::Uint16 + | TypeDefKind::Uint32 + | TypeDefKind::Uint64 + | TypeDefKind::Float32 + | TypeDefKind::Float64 + | TypeDefKind::Enum + | TypeDefKind::String + | TypeDefKind::Bytes + | TypeDefKind::Timestamp + ) + } + + /// Returns `true` for composite kinds (Struct, Union, Array, Record). + pub fn is_composite(self) -> bool { + matches!( + self, + TypeDefKind::Struct | TypeDefKind::Union | TypeDefKind::Array | TypeDefKind::Record + ) + } + + /// Returns `true` for variable-length kinds (String, Bytes, Timestamp, Record). + pub fn is_variable_length(self) -> bool { + matches!( + self, + TypeDefKind::String + | TypeDefKind::Bytes + | TypeDefKind::Timestamp + | TypeDefKind::Record + ) + } +} + +impl fmt::Display for TypeDefKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FromStr for TypeDefKind { + type Err = TypedefError; + + fn from_str(s: &str) -> Result { + match s { + "TypeDef:Int8" => Ok(TypeDefKind::Int8), + "TypeDef:Int16" => Ok(TypeDefKind::Int16), + "TypeDef:Int32" => Ok(TypeDefKind::Int32), + "TypeDef:Int64" => Ok(TypeDefKind::Int64), + "TypeDef:Uint8" => Ok(TypeDefKind::Uint8), + "TypeDef:Uint16" => Ok(TypeDefKind::Uint16), + "TypeDef:Uint32" => Ok(TypeDefKind::Uint32), + "TypeDef:Uint64" => Ok(TypeDefKind::Uint64), + "TypeDef:Float32" => Ok(TypeDefKind::Float32), + "TypeDef:Float64" => Ok(TypeDefKind::Float64), + "TypeDef:Boolean" => Ok(TypeDefKind::Boolean), + "TypeDef:Enum" => Ok(TypeDefKind::Enum), + "TypeDef:String" => Ok(TypeDefKind::String), + "TypeDef:Bytes" => Ok(TypeDefKind::Bytes), + "TypeDef:Struct" => Ok(TypeDefKind::Struct), + "TypeDef:Union" => Ok(TypeDefKind::Union), + "TypeDef:Array" => Ok(TypeDefKind::Array), + "TypeDef:Record" => Ok(TypeDefKind::Record), + "TypeDef:Timestamp" => Ok(TypeDefKind::Timestamp), + other => Err(TypedefError::Schema(format!( + "unknown TypeDef kind: {other}" + ))), + } + } +} + +/// Returns the `TypeDef:*` kind string if the schema node declares one. +/// Returns `None` if the node has no `TypeDef:*` keyword. +/// +/// A TypeDef kind is recognized when the schema object has a key starting +/// with `TypeDef:` whose value is `true`. (Object form with annotations +/// like `{ "encoding": "..." }` is handled by the annotation parsers, not +/// here — `get_typedef_kind` only checks for the presence of the keyword.) +pub fn get_typedef_kind(node: &Value) -> Option<&str> { + let obj = node.as_object()?; + for key in obj.keys() { + if key.starts_with(TYPEDEF_PREFIX) && obj.get(key) == Some(&Value::Bool(true)) { + return Some(key.as_str()); + } + } + None +} + +/// Returns the `TypeDefKind` enum variant if the schema node declares one +/// (boolean form only, like `get_typedef_kind`). +pub fn get_typedef_kind_enum(node: &Value) -> Option { + get_typedef_kind(node).and_then(|s| s.parse().ok()) +} + +/// Byte endianness for multi-byte integer and float fields. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Endian { + Little, + Big, +} + +impl Endian { + /// Parse from the schema's `"endian"` annotation. Defaults to `Little` + /// if the annotation is absent or unrecognized. + pub fn from_schema(schema: &Value) -> Self { + parse_endian(schema) + } +} + +/// The encoding strategy for a variable-length type. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VariableEncoding { + /// `[length: u32][data]` — the default. Length prefix at a known offset, + /// variable data follows immediately. + LengthPrefixed, + /// `{offset: u32, length: u32}` pointing into a separate data region. + /// The metatensor blob tensor pattern. + OffsetIndirect, +} + +/// Parse the `"encoding"` annotation from a variable-length type's keyword value. +/// +/// The keyword value may be `true` (shorthand for length-prefixed) or an +/// object with an `"encoding"` field. Defaults to `LengthPrefixed` when +/// absent or unrecognized. +pub fn parse_encoding(keyword_value: &Value) -> VariableEncoding { + match keyword_value { + Value::Bool(true) => VariableEncoding::LengthPrefixed, + Value::Object(obj) => { + let encoding = obj.get("encoding").and_then(Value::as_str); + match encoding { + Some("offset-indirect") => VariableEncoding::OffsetIndirect, + _ => VariableEncoding::LengthPrefixed, + } + } + _ => VariableEncoding::LengthPrefixed, + } +} + +/// Parse the `"align"` annotation from a schema node. Returns `None` if not +/// specified or not a non-negative integer. +pub fn parse_align(node: &Value) -> Option { + let n = node.as_object()?.get("align")?.as_u64()?; + Some(n as usize) +} + +/// Parse the `"maxLength"` annotation (standard JSON Schema keyword). +/// Returns `None` if not specified or not a non-negative integer. +pub fn parse_max_length(node: &Value) -> Option { + let n = node.as_object()?.get("maxLength")?.as_u64()?; + Some(n as usize) +} + +/// Parse the `"endian"` annotation. Defaults to `Little` if absent or +/// unrecognized. Operates on any node, not just the root. +pub fn parse_endian(node: &Value) -> Endian { + match node + .as_object() + .and_then(|o| o.get("endian")) + .and_then(Value::as_str) + { + Some("big") => Endian::Big, + _ => Endian::Little, + } +} + +/// The kind of TUnion discriminator. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DiscriminatorKind { + /// Byte-offset discriminator: a fixed-size integer at a known byte offset. + /// Mapping keys are stringified integers. Used by SFTP type bytes and + /// call protocol event types. + Byte { + /// Byte position of the discriminator within the union's buffer. + offset: usize, + /// The `TypeDef:*` kind of the discriminator (typically + /// `TypeDef:Uint8`). + disc_type: TypeDefKind, + }, + /// Field-name discriminator: a named field within the struct. Mapping keys + /// are string values matching the discriminator field's value. The + /// typedef.ts pattern. + Field { + /// The field name that holds the discriminator value. + name: String, + }, +} + +/// Parse the `"discriminator"` annotation from a TUnion schema node. +/// +/// Returns [`TypedefError::Schema`] for malformed discriminators (unknown +/// `kind`, missing required `name`, or an unsupported discriminator `type`). +pub fn parse_discriminator(node: &Value) -> Result { + let obj = node.as_object().ok_or_else(|| { + TypedefError::Schema("discriminator requires a schema object".to_string()) + })?; + let disc = obj.get("discriminator").ok_or_else(|| { + TypedefError::Schema("union is missing 'discriminator' annotation".to_string()) + })?; + let disc_obj = disc + .as_object() + .ok_or_else(|| TypedefError::Schema("'discriminator' must be an object".to_string()))?; + let kind = disc_obj + .get("kind") + .and_then(Value::as_str) + .ok_or_else(|| TypedefError::Schema("discriminator is missing 'kind' field".to_string()))?; + match kind { + "byte" => { + let offset = disc_obj.get("offset").and_then(Value::as_u64).unwrap_or(0) as usize; + let disc_type_str = disc_obj + .get("type") + .and_then(Value::as_str) + .unwrap_or("TypeDef:Uint8"); + let disc_type: TypeDefKind = disc_type_str.parse().map_err(|_| { + TypedefError::Schema(format!( + "discriminator 'type' must be one of {BYTE_DISCRIMINATOR_TYPES:?}, got {disc_type_str:?}" + )) + })?; + if !BYTE_DISCRIMINATOR_TYPES.contains(&disc_type) { + return Err(TypedefError::Schema(format!( + "discriminator 'type' must be one of {BYTE_DISCRIMINATOR_TYPES:?}, got {disc_type:?}" + ))); + } + Ok(DiscriminatorKind::Byte { + offset, + disc_type, + }) + } + "field" => { + let name = disc_obj + .get("name") + .and_then(Value::as_str) + .ok_or_else(|| { + TypedefError::Schema( + "field discriminator is missing required 'name' field".to_string(), + ) + })? + .to_string(); + Ok(DiscriminatorKind::Field { name }) + } + other => Err(TypedefError::Schema(format!( + "unknown discriminator 'kind': {other:?} (expected \"byte\" or \"field\")" + ))), + } +} + +/// Detect a `TypeDef:*` kind from a schema node, accepting either the +/// boolean form (`{ "TypeDef:String": true }`) or the object-annotation +/// form (`{ "TypeDef:String": { "encoding": "..." } }`). +/// +/// [`get_typedef_kind`] only recognizes the boolean form; layout computation +/// and engine dispatch also need to recognize the object form so that +/// variable-length encoding annotations don't hide the kind. +pub fn get_typedef_kind_loose(node: &Value) -> Option<&str> { + let obj = node.as_object()?; + for key in obj.keys() { + if key.starts_with(TYPEDEF_PREFIX) && obj.get(key).is_some_and(|v| !v.is_null()) { + return Some(key.as_str()); + } + } + None +} + +/// Like [`get_typedef_kind_loose`] but returns the parsed [`TypeDefKind`] enum. +pub fn get_typedef_kind_loose_enum(node: &Value) -> Option { + get_typedef_kind_loose(node).and_then(|s| s.parse().ok()) +} + +/// Resolve a `$ref` against the root schema, or return the inline schema. +/// +/// If `node` has a `"$ref"` key, parse the JSON Pointer and walk `root`. +/// Otherwise, return `node` itself (it's an inline schema). +pub fn resolve_ref_or_inline<'a>(node: &'a Value, root: &'a Value) -> Option<&'a Value> { + let obj = node.as_object()?; + if let Some(Value::String(ref_path)) = obj.get("$ref") { + return resolve_ref(root, ref_path); + } + Some(node) +} + +/// Resolve a JSON Pointer `$ref` (e.g., `"#/$defs/Read"`) against `root`. +pub fn resolve_ref<'a>(root: &'a Value, ref_path: &str) -> Option<&'a Value> { + let stripped = ref_path.strip_prefix('#').unwrap_or(ref_path); + let stripped = stripped.strip_prefix('/').unwrap_or(stripped); + if stripped.is_empty() { + return Some(root); + } + let mut current = root; + for segment in stripped.split('/') { + let decoded = segment.replace("~1", "/").replace("~0", "~"); + if let Ok(idx) = decoded.parse::() { + current = current.get(idx)?; + } else { + current = current.get(&decoded)?; + } + } + Some(current) +} + +/// Walk the schema tree. For every `"$ref"` whose value is a bare name +/// (no `#` prefix), rewrite it to `"#/$defs/"`. Full JSON Pointer refs +/// (starting with `#`) pass through unchanged. Idempotent. +pub fn normalize_refs(schema: &mut Value) { + normalize_refs_recursive(schema); +} + +fn normalize_refs_recursive(node: &mut Value) { + if let Value::Object(obj) = node { + if let Some(Value::String(ref s)) = obj.get("$ref") { + if !s.starts_with('#') && !s.is_empty() { + let new_ref = format!("#/$defs/{s}"); + if let Some(slot) = obj.get_mut("$ref") { + *slot = Value::String(new_ref); + } + } + } + for value in obj.values_mut() { + normalize_refs_recursive(value); + } + } else if let Value::Array(arr) = node { + for item in arr.iter_mut() { + normalize_refs_recursive(item); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn get_typedef_kind_detects_bool_keyword() { + let schema = json!({"TypeDef:Uint32": true}); + assert_eq!(get_typedef_kind(&schema), Some("TypeDef:Uint32")); + } + + #[test] + fn get_typedef_kind_ignores_object_keyword() { + let schema = json!({"TypeDef:String": {"encoding": "length-prefixed"}}); + assert_eq!(get_typedef_kind(&schema), None); + } + + #[test] + fn get_typedef_kind_none_for_plain_schema() { + let schema = json!({"type": "object", "properties": {}}); + assert_eq!(get_typedef_kind(&schema), None); + } + + #[test] + fn type_size_fixed_kinds() { + assert_eq!(TypeDefKind::Float32.type_size(), Some(4)); + assert_eq!(TypeDefKind::Float64.type_size(), Some(8)); + assert_eq!(TypeDefKind::Int8.type_size(), Some(1)); + assert_eq!(TypeDefKind::Int16.type_size(), Some(2)); + assert_eq!(TypeDefKind::Int32.type_size(), Some(4)); + assert_eq!(TypeDefKind::Int64.type_size(), Some(8)); + assert_eq!(TypeDefKind::Uint8.type_size(), Some(1)); + assert_eq!(TypeDefKind::Uint16.type_size(), Some(2)); + assert_eq!(TypeDefKind::Uint32.type_size(), Some(4)); + assert_eq!(TypeDefKind::Uint64.type_size(), Some(8)); + assert_eq!(TypeDefKind::Boolean.type_size(), Some(1)); + assert_eq!(TypeDefKind::Enum.type_size(), Some(4)); + } + + #[test] + fn type_size_variable_and_composite_kinds() { + for kind in [ + TypeDefKind::String, + TypeDefKind::Bytes, + TypeDefKind::Struct, + TypeDefKind::Union, + TypeDefKind::Array, + TypeDefKind::Record, + TypeDefKind::Timestamp, + ] { + assert_eq!(kind.type_size(), None, "failed for {kind}"); + } + } + + #[test] + fn type_size_unknown_kind_returns_none() { + assert!("TypeDef:Uint128".parse::().is_err()); + assert!("TypeDef:Int128".parse::().is_err()); + assert!("not-a-typedef".parse::().is_err()); + } + + #[test] + fn natural_alignment_matches_spec() { + assert_eq!(TypeDefKind::Int8.natural_alignment(), 1); + assert_eq!(TypeDefKind::Uint8.natural_alignment(), 1); + assert_eq!(TypeDefKind::Boolean.natural_alignment(), 1); + assert_eq!(TypeDefKind::Int16.natural_alignment(), 2); + assert_eq!(TypeDefKind::Uint16.natural_alignment(), 2); + assert_eq!(TypeDefKind::Int32.natural_alignment(), 4); + assert_eq!(TypeDefKind::Uint32.natural_alignment(), 4); + assert_eq!(TypeDefKind::Float32.natural_alignment(), 4); + assert_eq!(TypeDefKind::Enum.natural_alignment(), 4); + assert_eq!(TypeDefKind::Float64.natural_alignment(), 8); + assert_eq!(TypeDefKind::Int64.natural_alignment(), 8); + assert_eq!(TypeDefKind::Uint64.natural_alignment(), 8); + assert_eq!(TypeDefKind::String.natural_alignment(), 4); + assert_eq!(TypeDefKind::Bytes.natural_alignment(), 4); + assert_eq!(TypeDefKind::Record.natural_alignment(), 4); + assert_eq!(TypeDefKind::Timestamp.natural_alignment(), 4); + assert_eq!(TypeDefKind::Struct.natural_alignment(), 1); + assert_eq!(TypeDefKind::Union.natural_alignment(), 1); + assert_eq!(TypeDefKind::Array.natural_alignment(), 1); + } + + #[test] + fn is_fixed_size_classifies_correctly() { + for kind in [ + TypeDefKind::Float32, + TypeDefKind::Float64, + TypeDefKind::Int8, + TypeDefKind::Int16, + TypeDefKind::Int32, + TypeDefKind::Int64, + TypeDefKind::Uint8, + TypeDefKind::Uint16, + TypeDefKind::Uint32, + TypeDefKind::Uint64, + TypeDefKind::Boolean, + TypeDefKind::Enum, + ] { + assert!(kind.is_fixed_size(), "expected fixed: {kind}"); + } + for kind in [ + TypeDefKind::String, + TypeDefKind::Bytes, + TypeDefKind::Struct, + TypeDefKind::Union, + TypeDefKind::Array, + TypeDefKind::Record, + TypeDefKind::Timestamp, + ] { + assert!(!kind.is_fixed_size(), "expected variable: {kind}"); + } + } + + #[test] + fn endian_from_schema_defaults_to_little() { + assert_eq!(Endian::from_schema(&json!({})), Endian::Little); + assert_eq!( + Endian::from_schema(&json!({"endian": "little"})), + Endian::Little + ); + assert_eq!( + Endian::from_schema(&json!({"endian": "weird"})), + Endian::Little + ); + } + + #[test] + fn endian_from_schema_big() { + assert_eq!(Endian::from_schema(&json!({"endian": "big"})), Endian::Big); + } + + #[test] + fn parse_encoding_shorthand_true() { + assert_eq!( + parse_encoding(&json!(true)), + VariableEncoding::LengthPrefixed + ); + } + + #[test] + fn parse_encoding_object_length_prefixed() { + assert_eq!( + parse_encoding(&json!({"encoding": "length-prefixed"})), + VariableEncoding::LengthPrefixed + ); + } + + #[test] + fn parse_encoding_object_offset_indirect() { + assert_eq!( + parse_encoding(&json!({"encoding": "offset-indirect"})), + VariableEncoding::OffsetIndirect + ); + } + + #[test] + fn parse_encoding_unknown_defaults_to_length_prefixed() { + assert_eq!( + parse_encoding(&json!({"encoding": "weird"})), + VariableEncoding::LengthPrefixed + ); + assert_eq!(parse_encoding(&json!(42)), VariableEncoding::LengthPrefixed); + assert_eq!( + parse_encoding(&json!(null)), + VariableEncoding::LengthPrefixed + ); + } + + #[test] + fn parse_align_returns_value() { + assert_eq!(parse_align(&json!({"align": 256})), Some(256)); + assert_eq!(parse_align(&json!({"align": 0})), Some(0)); + } + + #[test] + fn parse_align_none_when_absent() { + assert_eq!(parse_align(&json!({})), None); + assert_eq!(parse_align(&json!({"align": "not-a-number"})), None); + } + + #[test] + fn parse_max_length_returns_value() { + assert_eq!(parse_max_length(&json!({"maxLength": 1024})), Some(1024)); + } + + #[test] + fn parse_max_length_none_when_absent() { + assert_eq!(parse_max_length(&json!({})), None); + assert_eq!(parse_max_length(&json!({"maxLength": "x"})), None); + } + + #[test] + fn parse_endian_alias_matches_from_schema() { + assert_eq!(parse_endian(&json!({"endian": "big"})), Endian::Big); + assert_eq!(parse_endian(&json!({})), Endian::Little); + } + + #[test] + fn parse_discriminator_byte_default_offset_and_type() { + let schema = json!({"discriminator": {"kind": "byte"}}); + let disc = parse_discriminator(&schema).expect("byte discriminator"); + assert_eq!( + disc, + DiscriminatorKind::Byte { + offset: 0, + disc_type: TypeDefKind::Uint8, + } + ); + } + + #[test] + fn parse_discriminator_byte_explicit() { + let schema = json!({ + "discriminator": {"kind": "byte", "offset": 4, "type": "TypeDef:Uint16"} + }); + let disc = parse_discriminator(&schema).expect("byte discriminator"); + assert_eq!( + disc, + DiscriminatorKind::Byte { + offset: 4, + disc_type: TypeDefKind::Uint16, + } + ); + } + + #[test] + fn parse_discriminator_field() { + let schema = json!({"discriminator": {"kind": "field", "name": "type"}}); + let disc = parse_discriminator(&schema).expect("field discriminator"); + assert_eq!( + disc, + DiscriminatorKind::Field { + name: "type".to_string() + } + ); + } + + #[test] + fn parse_discriminator_missing_discriminator_is_error() { + let schema = json!({"TypeDef:Union": true}); + assert!(matches!( + parse_discriminator(&schema), + Err(TypedefError::Schema(_)) + )); + } + + #[test] + fn parse_discriminator_field_missing_name_is_error() { + let schema = json!({"discriminator": {"kind": "field"}}); + assert!(matches!( + parse_discriminator(&schema), + Err(TypedefError::Schema(_)) + )); + } + + #[test] + fn parse_discriminator_unknown_kind_is_error() { + let schema = json!({"discriminator": {"kind": "magic"}}); + assert!(matches!( + parse_discriminator(&schema), + Err(TypedefError::Schema(_)) + )); + } + + #[test] + fn parse_discriminator_byte_invalid_type_is_error() { + let schema = json!({ + "discriminator": {"kind": "byte", "type": "TypeDef:Float32"} + }); + assert!(matches!( + parse_discriminator(&schema), + Err(TypedefError::Schema(_)) + )); + } + + #[test] + fn normalize_refs_rewrites_bare_name() { + let mut schema = json!({"$ref": "Read"}); + normalize_refs(&mut schema); + assert_eq!(schema, json!({"$ref": "#/$defs/Read"})); + } + + #[test] + fn normalize_refs_leaves_pointer_ref_unchanged() { + let mut schema = json!({"$ref": "#/$defs/Read"}); + normalize_refs(&mut schema); + assert_eq!(schema, json!({"$ref": "#/$defs/Read"})); + } + + #[test] + fn normalize_refs_is_idempotent() { + let mut schema = json!({"$ref": "Read"}); + normalize_refs(&mut schema); + normalize_refs(&mut schema); + assert_eq!(schema, json!({"$ref": "#/$defs/Read"})); + } + + #[test] + fn normalize_refs_walks_nested_objects() { + let mut schema = json!({ + "properties": { + "child": {"$ref": "Child"}, + "other": {"$ref": "#/$defs/Other"} + }, + "items": [ + {"$ref": "InArray"}, + {"foo": {"$ref": "Deep"}} + ] + }); + normalize_refs(&mut schema); + assert_eq!( + schema, + json!({ + "properties": { + "child": {"$ref": "#/$defs/Child"}, + "other": {"$ref": "#/$defs/Other"} + }, + "items": [ + {"$ref": "#/$defs/InArray"}, + {"foo": {"$ref": "#/$defs/Deep"}} + ] + }) + ); + } + + #[test] + fn normalize_refs_preserves_sibling_keys() { + let mut schema = json!({ + "$ref": "Read", + "typedef:annotation": "kept" + }); + normalize_refs(&mut schema); + assert_eq!( + schema, + json!({ + "$ref": "#/$defs/Read", + "typedef:annotation": "kept" + }) + ); + } +} diff --git a/src/sequential_reader.rs b/src/sequential_reader.rs new file mode 100644 index 0000000..8340bb3 --- /dev/null +++ b/src/sequential_reader.rs @@ -0,0 +1,1543 @@ +//! Packed sequential `SequentialReader` — Mode 1 read-side (ADR-096). +//! +//! Walks a buffer field-by-field according to the schema, reading length +//! prefixes to determine variable-length data positions. Used at read time +//! when the consumer is parsing an incoming frame. +//! +//! The reader is sequential — it cannot jump to field N without reading +//! fields 0..N-1 first. This is inherent to packed layouts where +//! variable-length fields shift subsequent fields. [`SequentialReader`] +//! uses the [`crate::data_access`] read functions for all typed reads +//! and applies the schema's endianness to every multi-byte value. + +use crate::data_access; +use crate::error::TypedefError; +use crate::schema::{self, get_typedef_kind_loose_enum, DiscriminatorKind, Endian, TypeDefKind, U32_SIZE}; +use serde_json::Value; + +/// A value read from a field during sequential traversal. +/// +/// Composite kinds ([`FieldValue::Struct`], [`FieldValue::Union`], +/// [`FieldValue::Array`]) return layout descriptors; the consumer +/// recurses with a fresh [`SequentialReader`] scoped to the +/// reported byte range. +#[derive(Debug, PartialEq)] +pub enum FieldValue<'a> { + /// `TypeDef:Int8`. + I8(i8), + /// `TypeDef:Int16`. + I16(i16), + /// `TypeDef:Int32`. + I32(i32), + /// `TypeDef:Int64`. + I64(i64), + /// `TypeDef:Uint8`. + U8(u8), + /// `TypeDef:Uint16`. + U16(u16), + /// `TypeDef:Uint32`. + U32(u32), + /// `TypeDef:Uint64`. + U64(u64), + /// `TypeDef:Float32`. + F32(f32), + /// `TypeDef:Float64`. + F64(f64), + /// `TypeDef:Boolean`. + Bool(bool), + /// `TypeDef:Enum` — `u32` index into the schema's `"enum"` array. + Enum(u32), + /// `TypeDef:String` — borrows from the input buffer. + String(&'a str), + /// `TypeDef:Bytes` — borrows from the input buffer. + Bytes(&'a [u8]), + /// `TypeDef:Struct` — the consumer recurses with a new + /// [`SequentialReader`] scoped to `start..end`. + Struct { + /// Inclusive start of the nested struct's byte range. + start: usize, + /// Exclusive end of the nested struct's byte range. + end: usize, + }, + /// `TypeDef:Union` — the consumer looks up the variant schema using + /// `discriminator` and recurses at `variant_start`. + Union { + /// Stringified discriminator value (mapping key). + discriminator: String, + /// Byte offset where the variant struct begins. + variant_start: usize, + }, + /// `TypeDef:Array` — the consumer iterates `count` elements of + /// stride `element_stride` starting at `element_start`. + Array { + /// Number of elements in the array. + count: u32, + /// Byte offset of the first element. + element_start: usize, + /// Byte distance between consecutive elements. `0` signals a + /// variable-length element type — the consumer must walk each + /// element sequentially. + element_stride: usize, + }, +} + +/// Walks a buffer field-by-field according to a schema, reading length +/// prefixes to determine variable-length data positions. Used at read +/// time when parsing incoming protocol frames. +/// +/// The reader is sequential — it cannot jump to field N without reading +/// fields 0..N-1 first. This is inherent to packed layouts where +/// variable-length fields shift subsequent fields. +/// +/// Construct with [`SequentialReader::new`], then drive with +/// [`SequentialReader::read_next`] until it returns `Ok(None)`. Use +/// [`SequentialReader::reset`] to walk the same buffer again, or +/// [`SequentialReader::read_field`] to seek a single field by name +/// (which walks all preceding fields to reach the target). +#[derive(Debug)] +pub struct SequentialReader { + schema: Value, + endian: Endian, + fields: Vec<(String, Value)>, + field_index: usize, + position: usize, +} + +impl SequentialReader { + /// Create a new `SequentialReader` from a top-level struct schema. + /// + /// The schema must declare `TypeDef:Struct` and have a `properties` + /// object. Endianness is parsed via [`Endian::from_schema`]. + /// + /// # Errors + /// + /// Returns [`TypedefError::Schema`] if the schema is not an object, + /// does not declare `TypeDef:Struct`, or has no `properties` object. + pub fn new(schema: &Value) -> Result { + let kind = schema::get_typedef_kind(schema) + .and_then(|s| s.parse::().ok()) + .ok_or_else(|| TypedefError::Schema("schema has no TypeDef:* kind".to_string()))?; + if kind != TypeDefKind::Struct { + return Err(TypedefError::Schema(format!( + "SequentialReader only supports TypeDef:Struct at the top level, got {kind}" + ))); + } + let properties = schema + .as_object() + .and_then(|obj| obj.get("properties")) + .and_then(Value::as_object) + .ok_or_else(|| { + TypedefError::Schema("struct schema has no properties object".to_string()) + })?; + let fields: Vec<(String, Value)> = properties + .iter() + .map(|(name, value)| (name.clone(), value.clone())) + .collect(); + let endian = Endian::from_schema(schema); + Ok(Self { + schema: schema.clone(), + endian, + fields, + field_index: 0, + position: 0, + }) + } + + /// Read the next field from `buffer` at the current position. + /// + /// Returns `Ok(Some((field_name, value)))` and advances the internal + /// position, or `Ok(None)` when all fields have been read. Variable- + /// length fields (`TypeDef:String`/`TypeDef:Bytes`) consume their + /// 4-byte length prefix plus the data; composite fields advance past + /// their computed byte range. + /// + /// # Errors + /// + /// Propagates [`TypedefError::Access`] from the underlying + /// [`crate::data_access`] reads when `buffer` is too short or + /// contains invalid data. + pub fn read_next<'a>( + &mut self, + buffer: &'a [u8], + ) -> Result)>, TypedefError> { + if self.field_index >= self.fields.len() { + return Ok(None); + } + let (name, value) = match self.read_field_at(buffer, self.field_index, self.position) { + Ok((value, new_position)) => { + self.position = new_position; + self.field_index += 1; + (self.fields[self.field_index - 1].0.clone(), value) + } + Err(e) => return Err(e), + }; + Ok(Some((name, value))) + } + + /// Read a specific field by name. This walks through all preceding + /// fields to reach the target (sequential access is inherent to + /// packed layouts). Resets the reader first; the cursor is left at + /// the position just past the target field. + /// + /// # Errors + /// + /// Returns [`TypedefError::Schema`] if `field_path` does not match + /// any top-level field. Propagates [`TypedefError::Access`] for + /// buffer-too-short or invalid data. + pub fn read_field<'a>( + &mut self, + buffer: &'a [u8], + field_path: &str, + ) -> Result, TypedefError> { + self.reset(); + let target_index = self + .fields + .iter() + .position(|(name, _)| name == field_path) + .ok_or_else(|| { + TypedefError::Schema(format!("field not found in struct: {field_path}")) + })?; + let mut position = 0usize; + let mut target_value: Option> = None; + for index in 0..=target_index { + let (value, new_position) = self.read_field_at(buffer, index, position)?; + position = new_position; + if index == target_index { + target_value = Some(value); + } + } + self.position = position; + self.field_index = target_index + 1; + target_value.ok_or_else(|| { + TypedefError::Schema(format!( + "internal: target field {field_path} not produced by loop" + )) + }) + } + + /// Reset the reader to the beginning of the buffer (position 0, + /// first field). + pub fn reset(&mut self) { + self.field_index = 0; + self.position = 0; + } + + /// The current byte position in the buffer. + pub fn position(&self) -> usize { + self.position + } + + /// The endianness used by this reader. + pub fn endian(&self) -> Endian { + self.endian + } + + /// The schema this reader was constructed from. + pub fn schema(&self) -> &Value { + &self.schema + } + + fn read_field_at<'a>( + &self, + buffer: &'a [u8], + index: usize, + offset: usize, + ) -> Result<(FieldValue<'a>, usize), TypedefError> { + let (_, field_schema) = self + .fields + .get(index) + .ok_or_else(|| TypedefError::Schema(format!("field index {index} out of range")))?; + let field_path = self.fields[index].0.as_str(); + read_field_value( + buffer, + &self.schema, + field_schema, + field_path, + offset, + self.endian, + ) + } +} + +/// Read a single field value at `offset` and return the value plus the +/// position just past the field. Field paths are used for error +/// attribution only — this helper does not recurse into nested structs. +/// +/// `root_schema` is the top-level schema used to resolve `$ref` pointers +/// found in nested union variants. +fn read_field_value<'a>( + buffer: &'a [u8], + root_schema: &Value, + field_schema: &Value, + field_path: &str, + offset: usize, + endian: Endian, +) -> Result<(FieldValue<'a>, usize), TypedefError> { + let kind = get_typedef_kind_loose_enum(field_schema).ok_or_else(|| { + TypedefError::Schema(format!( + "field {field_path} has no TypeDef:* kind: {field_schema}" + )) + })?; + + match kind { + TypeDefKind::Int8 => { + let v = data_access::read_i8(buffer, offset, field_path)?; + Ok((FieldValue::I8(v), offset + 1)) + } + TypeDefKind::Int16 => { + let v = data_access::read_i16(buffer, offset, field_path, endian)?; + Ok((FieldValue::I16(v), offset + 2)) + } + TypeDefKind::Int32 => { + let v = data_access::read_i32(buffer, offset, field_path, endian)?; + Ok((FieldValue::I32(v), offset + 4)) + } + TypeDefKind::Int64 => { + let v = data_access::read_i64(buffer, offset, field_path, endian)?; + Ok((FieldValue::I64(v), offset + 8)) + } + TypeDefKind::Uint8 => { + let v = data_access::read_u8(buffer, offset, field_path)?; + Ok((FieldValue::U8(v), offset + 1)) + } + TypeDefKind::Uint16 => { + let v = data_access::read_u16(buffer, offset, field_path, endian)?; + Ok((FieldValue::U16(v), offset + 2)) + } + TypeDefKind::Uint32 => { + let v = data_access::read_u32(buffer, offset, field_path, endian)?; + Ok((FieldValue::U32(v), offset + 4)) + } + TypeDefKind::Uint64 => { + let v = data_access::read_u64(buffer, offset, field_path, endian)?; + Ok((FieldValue::U64(v), offset + 8)) + } + TypeDefKind::Float32 => { + let v = data_access::read_f32(buffer, offset, field_path, endian)?; + Ok((FieldValue::F32(v), offset + 4)) + } + TypeDefKind::Float64 => { + let v = data_access::read_f64(buffer, offset, field_path, endian)?; + Ok((FieldValue::F64(v), offset + 8)) + } + TypeDefKind::Boolean => { + let v = data_access::read_bool(buffer, offset, field_path)?; + Ok((FieldValue::Bool(v), offset + 1)) + } + TypeDefKind::Enum => { + let v = data_access::read_enum(buffer, offset, field_path, endian)?; + Ok((FieldValue::Enum(v), offset + 4)) + } + TypeDefKind::String => { + let s = data_access::read_string(buffer, offset, field_path, endian)?; + let total = U32_SIZE + s.len(); + Ok((FieldValue::String(s), offset + total)) + } + TypeDefKind::Bytes => { + let b = data_access::read_bytes(buffer, offset, field_path, endian)?; + let total = U32_SIZE + b.len(); + Ok((FieldValue::Bytes(b), offset + total)) + } + TypeDefKind::Timestamp => { + let s = data_access::read_string(buffer, offset, field_path, endian)?; + let total = U32_SIZE + s.len(); + Ok((FieldValue::String(s), offset + total)) + } + TypeDefKind::Struct => { + let size = walk_struct_size(root_schema, field_schema, buffer, offset, endian)?; + let end = offset + .checked_add(size) + .ok_or_else(|| TypedefError::Access { + field_path: field_path.to_string(), + reason: format!("struct end {offset} + {size} overflows usize"), + })?; + Ok((FieldValue::Struct { start: offset, end }, end)) + } + TypeDefKind::Union => read_union_value( + buffer, + root_schema, + field_schema, + field_path, + offset, + endian, + ), + TypeDefKind::Array => read_array_value( + buffer, + root_schema, + field_schema, + field_path, + offset, + endian, + ), + TypeDefKind::Record => read_record_value( + buffer, + root_schema, + field_schema, + field_path, + offset, + endian, + ), + } +} + +/// Read a `TypeDef:Union` field: read the discriminator, return the +/// variant start offset, and advance past the entire union payload. +/// +/// For byte-offset discriminators the union occupies +/// `discriminator_size + variant_size` bytes. Because the variant is a +/// struct (or a ref to one) whose size depends on variable-length fields, +/// the variant size is computed by walking the variant struct. The +/// variant schema is resolved from the `"mapping"` table using the +/// stringified discriminator value. +/// +/// For field-name discriminators the discriminator is itself a +/// length-prefixed string field. The variant begins immediately after +/// the discriminator field and is sized by walking the variant struct. +fn read_union_value<'a>( + buffer: &'a [u8], + root_schema: &Value, + schema: &Value, + field_path: &str, + offset: usize, + endian: Endian, +) -> Result<(FieldValue<'a>, usize), TypedefError> { + let disc = schema::parse_discriminator(schema)?; + let mapping = schema + .as_object() + .and_then(|obj| obj.get("mapping")) + .and_then(Value::as_object) + .ok_or_else(|| TypedefError::Schema(format!("union {field_path} has no mapping object")))?; + + match disc { + DiscriminatorKind::Byte { + offset: disc_offset, + disc_type, + } => { + let abs_offset = + offset + .checked_add(disc_offset) + .ok_or_else(|| TypedefError::Access { + field_path: field_path.to_string(), + reason: format!( + "discriminator offset {offset} + {disc_offset} overflows usize" + ), + })?; + let (disc_value, disc_size) = + read_byte_discriminator(buffer, abs_offset, field_path, disc_type, endian)?; + let key = disc_value.to_string(); + let variant_schema = mapping.get(&key).ok_or_else(|| TypedefError::Access { + field_path: field_path.to_string(), + reason: format!("unknown union discriminator value: {key}"), + })?; + let variant_start = + abs_offset + .checked_add(disc_size) + .ok_or_else(|| TypedefError::Access { + field_path: field_path.to_string(), + reason: format!("variant start {abs_offset} + {disc_size} overflows usize"), + })?; + let variant_size = resolve_and_walk_variant( + root_schema, + schema, + variant_schema, + buffer, + variant_start, + endian, + field_path, + )?; + let end = + variant_start + .checked_add(variant_size) + .ok_or_else(|| TypedefError::Access { + field_path: field_path.to_string(), + reason: format!( + "union end {variant_start} + {variant_size} overflows usize" + ), + })?; + Ok(( + FieldValue::Union { + discriminator: key, + variant_start, + }, + end, + )) + } + DiscriminatorKind::Field { name } => { + let properties = schema + .as_object() + .and_then(|obj| obj.get("properties")) + .and_then(Value::as_object) + .ok_or_else(|| { + TypedefError::Schema(format!( + "field-name union {field_path} has no properties object" + )) + })?; + let disc_schema = properties.get(&name).ok_or_else(|| { + TypedefError::Schema(format!( + "union {field_path} has no discriminator field '{name}'" + )) + })?; + let (disc_value, after_disc) = + read_field_value(buffer, root_schema, disc_schema, field_path, offset, endian)?; + let key = discriminator_string_value(&disc_value, field_path)?; + let variant_schema = mapping.get(&key).ok_or_else(|| TypedefError::Access { + field_path: field_path.to_string(), + reason: format!("unknown union discriminator value: {key}"), + })?; + let variant_size = resolve_and_walk_variant( + root_schema, + schema, + variant_schema, + buffer, + after_disc, + endian, + field_path, + )?; + let end = after_disc + .checked_add(variant_size) + .ok_or_else(|| TypedefError::Access { + field_path: field_path.to_string(), + reason: format!("union end {after_disc} + {variant_size} overflows usize"), + })?; + Ok(( + FieldValue::Union { + discriminator: key, + variant_start: after_disc, + }, + end, + )) + } + } +} + +/// Read a byte-offset discriminator integer and return its value (as a +/// `u32`) plus its byte size. +fn read_byte_discriminator( + buffer: &[u8], + offset: usize, + field_path: &str, + disc_type: TypeDefKind, + endian: Endian, +) -> Result<(u32, usize), TypedefError> { + match disc_type { + TypeDefKind::Uint8 => { + let v = data_access::read_u8(buffer, offset, field_path)?; + Ok((v as u32, 1)) + } + TypeDefKind::Uint16 => { + let v = data_access::read_u16(buffer, offset, field_path, endian)?; + Ok((v as u32, 2)) + } + TypeDefKind::Uint32 => { + let v = data_access::read_u32(buffer, offset, field_path, endian)?; + Ok((v, 4)) + } + other => Err(TypedefError::Schema(format!( + "unsupported byte discriminator type: {other}" + ))), + } +} + +/// Stringify a field-name discriminator value. Only the common kinds +/// (String, Uint8/16/32, Enum) are supported — anything else is a schema +/// error. +fn discriminator_string_value( + value: &FieldValue<'_>, + field_path: &str, +) -> Result { + match value { + FieldValue::String(s) => Ok(s.to_string()), + FieldValue::U8(v) => Ok(v.to_string()), + FieldValue::U16(v) => Ok(v.to_string()), + FieldValue::U32(v) => Ok(v.to_string()), + FieldValue::Enum(v) => Ok(v.to_string()), + other => Err(TypedefError::Schema(format!( + "union {field_path} has unsupported field discriminator kind: {other:?}" + ))), + } +} + +/// Read a `TypeDef:Array` field: read the count (fixed via `minItems`/ +/// `maxItems` equality, or a 4-byte count prefix) and compute the +/// element stride. Fixed-size elements produce a non-zero stride so the +/// consumer can index directly; variable-length elements produce a +/// stride of `0` so the consumer must walk each element sequentially. +fn read_array_value<'a>( + buffer: &'a [u8], + root_schema: &Value, + schema: &Value, + field_path: &str, + offset: usize, + endian: Endian, +) -> Result<(FieldValue<'a>, usize), TypedefError> { + let obj = schema.as_object().ok_or_else(|| { + TypedefError::Schema(format!("array {field_path} schema is not an object")) + })?; + let items_schema = obj + .get("items") + .ok_or_else(|| TypedefError::Schema(format!("array {field_path} has no items schema")))?; + + let min = obj + .get("minItems") + .and_then(Value::as_u64) + .map(|n| n as u32); + let max = obj + .get("maxItems") + .and_then(Value::as_u64) + .map(|n| n as u32); + let fixed_count = matches!((min, max), (Some(a), Some(b)) if a == b); + let (count, element_start) = if fixed_count { + let count = min.ok_or_else(|| { + TypedefError::Schema(format!( + "array {field_path} declared fixed count but minItems is absent" + )) + })?; + (count, offset) + } else { + let count = data_access::read_u32(buffer, offset, field_path, endian)?; + (count, offset + U32_SIZE) + }; + + let element_kind = get_typedef_kind_loose_enum(items_schema).ok_or_else(|| { + TypedefError::Schema(format!( + "array {field_path} items schema has no TypeDef:* kind" + )) + })?; + + let element_stride = if element_kind.is_fixed_size() { + element_kind.type_size().unwrap_or(0) + } else { + 0 + }; + + let total = if element_stride == 0 { + walk_variable_array_size( + root_schema, + items_schema, + buffer, + element_start, + count, + endian, + field_path, + )? + } else { + (count as usize) + .checked_mul(element_stride) + .ok_or_else(|| TypedefError::Access { + field_path: field_path.to_string(), + reason: format!("array size {count} × stride {element_stride} overflows usize"), + })? + }; + + let end = element_start + .checked_add(total) + .ok_or_else(|| TypedefError::Access { + field_path: field_path.to_string(), + reason: format!("array end {element_start} + {total} overflows usize"), + })?; + + Ok(( + FieldValue::Array { + count, + element_start, + element_stride, + }, + end, + )) +} + +/// Walk `count` variable-length array elements starting at `offset` and +/// return the total byte size of the element data (excluding any count +/// prefix, which the caller has already accounted for). +fn walk_variable_array_size( + root_schema: &Value, + items_schema: &Value, + buffer: &[u8], + start: usize, + count: u32, + endian: Endian, + field_path: &str, +) -> Result { + let mut position = start; + for i in 0..count { + let element_path = format!("{field_path}[{i}]"); + let (_, new_position) = read_field_value( + buffer, + root_schema, + items_schema, + &element_path, + position, + endian, + )?; + if new_position < position { + return Err(TypedefError::Access { + field_path: element_path, + reason: format!("array element walked backwards: {position} → {new_position}"), + }); + } + position = new_position; + } + Ok(position - start) +} + +/// Read a `TypeDef:Record` field: `[count: u32]` followed by `count` +/// entries of `[key_len: u32][key_bytes][value]`. Returns the total size +/// consumed. The reader does not decode the entries — the consumer +/// recurses into the record's value schema. +fn read_record_value<'a>( + buffer: &'a [u8], + root_schema: &Value, + schema: &Value, + field_path: &str, + offset: usize, + endian: Endian, +) -> Result<(FieldValue<'a>, usize), TypedefError> { + let count = data_access::read_u32(buffer, offset, field_path, endian)?; + let value_schema = schema + .as_object() + .and_then(|obj| obj.get("values")) + .ok_or_else(|| TypedefError::Schema(format!("record {field_path} has no values schema")))?; + let mut position = offset + U32_SIZE; + for i in 0..count { + let entry_path = format!("{field_path}[{i}].key"); + let key = data_access::read_string(buffer, position, &entry_path, endian)?; + position += U32_SIZE + key.len(); + let value_path = format!("{field_path}[{i}].value"); + let (_, new_position) = read_field_value( + buffer, + root_schema, + value_schema, + &value_path, + position, + endian, + )?; + position = new_position; + } + Ok((FieldValue::Bytes(&buffer[offset..position]), position)) +} + +/// Resolve a variant schema and walk its size starting at +/// `variant_start`. Used by [`read_union_value`]. Inline schemas are +/// returned as-is; `$ref` pointers are resolved against the union +/// schema's own `$defs` block, then the root schema's `$defs` block. +fn resolve_and_walk_variant( + root_schema: &Value, + union_schema: &Value, + variant_schema: &Value, + buffer: &[u8], + variant_start: usize, + endian: Endian, + field_path: &str, +) -> Result { + let resolved = + resolve_variant_schema(root_schema, union_schema, variant_schema).ok_or_else(|| { + TypedefError::Schema(format!( + "union {field_path} variant could not be resolved: {variant_schema}" + )) + })?; + let kind = get_typedef_kind_loose_enum(resolved).ok_or_else(|| { + TypedefError::Schema(format!( + "union {field_path} variant has no TypeDef:* kind: {resolved}" + )) + })?; + match kind { + TypeDefKind::Struct => { + walk_struct_size(root_schema, resolved, buffer, variant_start, endian) + } + TypeDefKind::Union => { + let (_, end) = read_union_value( + buffer, + root_schema, + resolved, + field_path, + variant_start, + endian, + )?; + Ok(end - variant_start) + } + other => Err(TypedefError::Schema(format!( + "union {field_path} variant must be Struct or Union, got {other}" + ))), + } +} + +/// Resolve a variant schema. Inline schemas (objects with a `TypeDef:*` +/// kind) are returned directly. `$ref` pointers of the form +/// `#/$defs/` are resolved against `union_schema["$defs"]` first, +/// then `root_schema["$defs"]`. Returns `None` if the ref cannot be +/// resolved or the target is absent. +fn resolve_variant_schema<'a>( + root_schema: &'a Value, + union_schema: &'a Value, + variant: &'a Value, +) -> Option<&'a Value> { + let obj = variant.as_object()?; + if let Some(ref_value) = obj.get("$ref").and_then(Value::as_str) { + if !ref_value.starts_with("#/$defs/") { + return None; + } + let name = &ref_value["#/$defs/".len()..]; + for host in [union_schema, root_schema] { + if let Some(target) = host + .as_object() + .and_then(|o| o.get("$defs")) + .and_then(Value::as_object) + .and_then(|d| d.get(name)) + { + return Some(target); + } + } + return None; + } + if get_typedef_kind_loose_enum(variant).is_some() { + Some(variant) + } else { + None + } +} + +/// Walk the fields of a struct schema sequentially, reading length +/// prefixes for variable-length fields, and return the total byte size +/// of the struct starting at `offset`. Does not return field values — +/// only advances the cursor to compute the struct's end position. +/// +/// `root_schema` is the top-level schema used to resolve `$ref` pointers +/// found in nested union variants. +fn walk_struct_size( + root_schema: &Value, + schema: &Value, + buffer: &[u8], + offset: usize, + endian: Endian, +) -> Result { + let properties = schema + .as_object() + .and_then(|obj| obj.get("properties")) + .and_then(Value::as_object) + .ok_or_else(|| { + TypedefError::Schema("struct schema has no properties object".to_string()) + })?; + let mut position = offset; + for (name, field_schema) in properties.iter() { + let (_, new_position) = read_field_value( + buffer, + root_schema, + field_schema, + name.as_str(), + position, + endian, + )?; + if new_position < position { + return Err(TypedefError::Access { + field_path: name.clone(), + reason: format!("struct field walked backwards: {position} → {new_position}"), + }); + } + position = new_position; + } + Ok(position - offset) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const LE: Endian = Endian::Little; + const BE: Endian = Endian::Big; + + fn write_u32(buf: &mut [u8], offset: usize, value: u32, endian: Endian) { + let bytes = match endian { + Endian::Little => value.to_le_bytes(), + Endian::Big => value.to_be_bytes(), + }; + buf[offset..offset + 4].copy_from_slice(&bytes); + } + + fn write_string(buf: &mut [u8], offset: usize, value: &str, endian: Endian) -> usize { + let bytes = value.as_bytes(); + let total = 4 + bytes.len(); + write_u32(buf, offset, bytes.len() as u32, endian); + buf[offset + 4..offset + 4 + bytes.len()].copy_from_slice(bytes); + total + } + + #[test] + fn reads_fixed_size_fields_in_sequence() { + let schema = json!({ + "TypeDef:Struct": true, + "endian": "little", + "properties": { + "a": { "TypeDef:Uint8": true }, + "b": { "TypeDef:Uint32": true }, + "c": { "TypeDef:Uint16": true } + } + }); + let mut buf = vec![0u8; 16]; + buf[0] = 42; + write_u32(&mut buf, 1, 0x01020304, LE); + buf[5..7].copy_from_slice(&1000u16.to_le_bytes()); + + let mut reader = SequentialReader::new(&schema).expect("reader"); + assert_eq!(reader.position(), 0); + + let (name, value) = reader.read_next(&buf).unwrap().expect("field 0"); + assert_eq!(name, "a"); + assert_eq!(value, FieldValue::U8(42)); + assert_eq!(reader.position(), 1); + + let (name, value) = reader.read_next(&buf).unwrap().expect("field 1"); + assert_eq!(name, "b"); + assert_eq!(value, FieldValue::U32(0x01020304)); + assert_eq!(reader.position(), 5); + + let (name, value) = reader.read_next(&buf).unwrap().expect("field 2"); + assert_eq!(name, "c"); + assert_eq!(value, FieldValue::U16(1000)); + assert_eq!(reader.position(), 7); + + assert!(reader.read_next(&buf).unwrap().is_none()); + } + + #[test] + fn reads_variable_length_string_with_length_prefix() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "id": { "TypeDef:Uint8": true }, + "name": { "TypeDef:String": true }, + "tail": { "TypeDef:Uint8": true } + } + }); + let mut buf = vec![0u8; 32]; + buf[0] = 7; + let written = write_string(&mut buf, 1, "hello", LE); + let after = 1 + written; + buf[after] = 99; + + let mut reader = SequentialReader::new(&schema).unwrap(); + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "id"); + assert_eq!(value, FieldValue::U8(7)); + assert_eq!(reader.position(), 1); + + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "name"); + assert_eq!(value, FieldValue::String("hello")); + assert_eq!(reader.position(), after); + + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "tail"); + assert_eq!(value, FieldValue::U8(99)); + assert_eq!(reader.position(), after + 1); + + assert!(reader.read_next(&buf).unwrap().is_none()); + } + + #[test] + fn reads_bytes_field() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "blob": { "TypeDef:Bytes": true } + } + }); + let mut buf = vec![0u8; 16]; + let payload = [0xAA, 0xBB, 0xCC]; + write_u32(&mut buf, 0, 3, LE); + buf[4..7].copy_from_slice(&payload); + + let mut reader = SequentialReader::new(&schema).unwrap(); + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "blob"); + assert_eq!(value, FieldValue::Bytes(&payload[..])); + assert_eq!(reader.position(), 7); + } + + #[test] + fn respects_big_endian() { + let schema = json!({ + "TypeDef:Struct": true, + "endian": "big", + "properties": { + "id": { "TypeDef:Uint32": true } + } + }); + let mut buf = vec![0u8; 8]; + write_u32(&mut buf, 0, 0x01020304, BE); + let mut reader = SequentialReader::new(&schema).unwrap(); + let (_, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(value, FieldValue::U32(0x01020304)); + assert_eq!(reader.endian(), BE); + } + + #[test] + fn reset_rewinds_cursor() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "a": { "TypeDef:Uint8": true }, + "b": { "TypeDef:Uint8": true } + } + }); + let buf = [10u8, 20u8]; + let mut reader = SequentialReader::new(&schema).unwrap(); + let _ = reader.read_next(&buf).unwrap().unwrap(); + let _ = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(reader.position(), 2); + reader.reset(); + assert_eq!(reader.position(), 0); + assert_eq!(reader.field_index, 0); + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "a"); + assert_eq!(value, FieldValue::U8(10)); + } + + #[test] + fn read_field_walks_preceding_fields() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "a": { "TypeDef:Uint8": true }, + "b": { "TypeDef:Uint32": true }, + "c": { "TypeDef:Uint8": true } + } + }); + let mut buf = vec![0u8; 16]; + buf[0] = 1; + write_u32(&mut buf, 1, 0xDEADBEEF, LE); + buf[5] = 9; + + let mut reader = SequentialReader::new(&schema).unwrap(); + let value = reader.read_field(&buf, "c").unwrap(); + assert_eq!(value, FieldValue::U8(9)); + assert_eq!(reader.position(), 6); + + reader.reset(); + let value = reader.read_field(&buf, "b").unwrap(); + assert_eq!(value, FieldValue::U32(0xDEADBEEF)); + } + + #[test] + fn read_field_unknown_returns_schema_error() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { "a": { "TypeDef:Uint8": true } } + }); + let buf = [0u8; 4]; + let mut reader = SequentialReader::new(&schema).unwrap(); + let err = reader.read_field(&buf, "missing").unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); + } + + #[test] + fn buffer_too_short_returns_access_error() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "id": { "TypeDef:Uint32": true } + } + }); + let buf = [0u8; 2]; + let mut reader = SequentialReader::new(&schema).unwrap(); + let err = reader.read_next(&buf).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); + } + + #[test] + fn rejects_non_struct_top_level() { + let schema = json!({ "TypeDef:Uint32": true }); + let err = SequentialReader::new(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); + } + + #[test] + fn rejects_schema_without_typedef_kind() { + let schema = json!({ "type": "object", "properties": {} }); + let err = SequentialReader::new(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); + } + + #[test] + fn reads_all_fixed_size_kinds() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "i8": { "TypeDef:Int8": true }, + "i16": { "TypeDef:Int16": true }, + "i32": { "TypeDef:Int32": true }, + "i64": { "TypeDef:Int64": true }, + "u8": { "TypeDef:Uint8": true }, + "u16": { "TypeDef:Uint16": true }, + "u32": { "TypeDef:Uint32": true }, + "u64": { "TypeDef:Uint64": true }, + "f32": { "TypeDef:Float32": true }, + "f64": { "TypeDef:Float64": true }, + "b": { "TypeDef:Boolean": true }, + "e": { "TypeDef:Enum": true } + } + }); + let mut buf = vec![0u8; 80]; + buf[0] = 0x80; + buf[1..3].copy_from_slice(&(-1i16).to_le_bytes()); + buf[3..7].copy_from_slice(&(-5i32).to_le_bytes()); + buf[7..15].copy_from_slice(&(-9i64).to_le_bytes()); + buf[15] = 200; + buf[16..18].copy_from_slice(&0xBEEFu16.to_le_bytes()); + buf[18..22].copy_from_slice(&0xDEADBEEFu32.to_le_bytes()); + buf[22..30].copy_from_slice(&0x0102030405060708u64.to_le_bytes()); + buf[30..34].copy_from_slice(&std::f32::consts::PI.to_le_bytes()); + buf[34..42].copy_from_slice(&std::f64::consts::PI.to_le_bytes()); + buf[42] = 0x01; + buf[43..47].copy_from_slice(&7u32.to_le_bytes()); + + let mut reader = SequentialReader::new(&schema).unwrap(); + let (_, v) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(v, FieldValue::I8(-128)); + let (_, v) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(v, FieldValue::I16(-1)); + let (_, v) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(v, FieldValue::I32(-5)); + let (_, v) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(v, FieldValue::I64(-9)); + let (_, v) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(v, FieldValue::U8(200)); + let (_, v) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(v, FieldValue::U16(0xBEEF)); + let (_, v) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(v, FieldValue::U32(0xDEADBEEF)); + let (_, v) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(v, FieldValue::U64(0x0102030405060708)); + let (_, v) = reader.read_next(&buf).unwrap().unwrap(); + assert!(matches!(v, FieldValue::F32(x) if (x - std::f32::consts::PI).abs() < 1e-6)); + let (_, v) = reader.read_next(&buf).unwrap().unwrap(); + assert!(matches!(v, FieldValue::F64(x) if (x - std::f64::consts::PI).abs() < 1e-12)); + let (_, v) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(v, FieldValue::Bool(true)); + let (_, v) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(v, FieldValue::Enum(7)); + assert!(reader.read_next(&buf).unwrap().is_none()); + } + + #[test] + fn nested_struct_reports_byte_range() { + let nested = json!({ + "TypeDef:Struct": true, + "properties": { + "inner": { + "TypeDef:Struct": true, + "properties": { + "x": { "TypeDef:Uint8": true }, + "y": { "TypeDef:Uint16": true } + } + }, + "tail": { "TypeDef:Uint8": true } + } + }); + let mut buf = vec![0u8; 16]; + buf[0] = 1; + buf[1..3].copy_from_slice(&0x0203u16.to_le_bytes()); + buf[3] = 9; + + let mut reader = SequentialReader::new(&nested).unwrap(); + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "inner"); + match value { + FieldValue::Struct { start, end } => { + assert_eq!(start, 0); + assert_eq!(end, 3); + let inner_schema = &nested["properties"]["inner"]; + let inner_reader = SequentialReader::new(inner_schema).unwrap(); + let inner_end = + walk_struct_size(inner_schema, inner_schema, &buf, start, LE).unwrap(); + assert_eq!(inner_end, end - start); + let _ = inner_reader; + } + other => panic!("expected Struct, got {other:?}"), + } + assert_eq!(reader.position(), 3); + + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "tail"); + assert_eq!(value, FieldValue::U8(9)); + assert_eq!(reader.position(), 4); + } + + #[test] + fn byte_discriminator_union_reads_value() { + let variant = json!({ + "TypeDef:Struct": true, + "properties": { + "x": { "TypeDef:Uint8": true } + } + }); + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "packet": { + "TypeDef:Union": true, + "discriminator": { "kind": "byte", "offset": 0, "type": "TypeDef:Uint8" }, + "mapping": { "5": variant.clone() }, + "$defs": { "Read": variant.clone() } + } + } + }); + let mut buf = vec![0u8; 8]; + buf[0] = 5; + buf[1] = 42; + + let mut reader = SequentialReader::new(&schema).unwrap(); + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "packet"); + match value { + FieldValue::Union { + discriminator, + variant_start, + } => { + assert_eq!(discriminator, "5"); + assert_eq!(variant_start, 1); + } + other => panic!("expected Union, got {other:?}"), + } + assert_eq!(reader.position(), 2); + } + + #[test] + fn field_discriminator_union_reads_value() { + let variant = json!({ + "TypeDef:Struct": true, + "properties": { + "payload": { "TypeDef:Uint8": true } + } + }); + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "event": { + "TypeDef:Union": true, + "discriminator": { "kind": "field", "name": "type" }, + "properties": { + "type": { "TypeDef:String": true } + }, + "mapping": { "read": variant.clone() } + } + } + }); + let mut buf = vec![0u8; 32]; + let written = write_string(&mut buf, 0, "read", LE); + let after = written; + buf[after] = 7; + + let mut reader = SequentialReader::new(&schema).unwrap(); + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "event"); + match value { + FieldValue::Union { + discriminator, + variant_start, + } => { + assert_eq!(discriminator, "read"); + assert_eq!(variant_start, after); + } + other => panic!("expected Union, got {other:?}"), + } + assert_eq!(reader.position(), after + 1); + } + + #[test] + fn fixed_count_array_reads_count_inline() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "vals": { + "TypeDef:Array": true, + "minItems": 3, + "maxItems": 3, + "items": { "TypeDef:Uint8": true } + } + } + }); + let buf = [1u8, 2, 3]; + + let mut reader = SequentialReader::new(&schema).unwrap(); + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "vals"); + match value { + FieldValue::Array { + count, + element_start, + element_stride, + } => { + assert_eq!(count, 3); + assert_eq!(element_start, 0); + assert_eq!(element_stride, 1); + } + other => panic!("expected Array, got {other:?}"), + } + assert_eq!(reader.position(), 3); + } + + #[test] + fn variable_count_array_reads_count_prefix() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "vals": { + "TypeDef:Array": true, + "items": { "TypeDef:Uint16": true } + } + } + }); + let mut buf = vec![0u8; 16]; + write_u32(&mut buf, 0, 2, LE); + buf[4..6].copy_from_slice(&100u16.to_le_bytes()); + buf[6..8].copy_from_slice(&200u16.to_le_bytes()); + + let mut reader = SequentialReader::new(&schema).unwrap(); + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "vals"); + match value { + FieldValue::Array { + count, + element_start, + element_stride, + } => { + assert_eq!(count, 2); + assert_eq!(element_start, 4); + assert_eq!(element_stride, 2); + } + other => panic!("expected Array, got {other:?}"), + } + assert_eq!(reader.position(), 8); + } + + #[test] + fn variable_length_element_array_walks_sequentially() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "items": { + "TypeDef:Array": true, + "items": { "TypeDef:String": true } + } + } + }); + let mut buf = vec![0u8; 64]; + write_u32(&mut buf, 0, 2, LE); + let mut pos = 4; + pos += write_string(&mut buf, pos, "ab", LE); + pos += write_string(&mut buf, pos, "cdef", LE); + + let mut reader = SequentialReader::new(&schema).unwrap(); + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "items"); + match value { + FieldValue::Array { + count, + element_start, + element_stride, + } => { + assert_eq!(count, 2); + assert_eq!(element_start, 4); + assert_eq!(element_stride, 0); + } + other => panic!("expected Array, got {other:?}"), + } + assert_eq!(reader.position(), pos); + } + + #[test] + fn timestamp_field_reads_as_length_prefixed_string() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "ts": { "TypeDef:Timestamp": true } + } + }); + let mut buf = vec![0u8; 64]; + let stamp = "2026-07-20T15:30:00Z"; + let written = write_string(&mut buf, 0, stamp, LE); + + let mut reader = SequentialReader::new(&schema).unwrap(); + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "ts"); + assert_eq!(value, FieldValue::String(stamp)); + assert_eq!(reader.position(), written); + } + + #[test] + fn record_field_walks_entries() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "counts": { + "TypeDef:Record": true, + "values": { "TypeDef:Uint32": true } + } + } + }); + let mut buf = vec![0u8; 64]; + write_u32(&mut buf, 0, 2, LE); + let mut pos = 4; + pos += write_string(&mut buf, pos, "a", LE); + buf[pos..pos + 4].copy_from_slice(&1u32.to_le_bytes()); + pos += 4; + pos += write_string(&mut buf, pos, "bb", LE); + buf[pos..pos + 4].copy_from_slice(&2u32.to_le_bytes()); + pos += 4; + + let mut reader = SequentialReader::new(&schema).unwrap(); + let (name, _value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "counts"); + assert_eq!(reader.position(), pos); + } + + #[test] + fn union_unknown_discriminator_returns_access_error() { + let variant = json!({ + "TypeDef:Struct": true, + "properties": { "x": { "TypeDef:Uint8": true } } + }); + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "packet": { + "TypeDef:Union": true, + "discriminator": { "kind": "byte", "offset": 0, "type": "TypeDef:Uint8" }, + "mapping": { "5": variant }, + "$defs": { "Read": variant } + } + } + }); + let buf = [99u8, 0]; + let mut reader = SequentialReader::new(&schema).unwrap(); + let err = reader.read_next(&buf).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); + } + + #[test] + fn union_via_ref_resolves_variant() { + let read_variant = json!({ + "TypeDef:Struct": true, + "properties": { "len": { "TypeDef:Uint32": true } } + }); + let schema = json!({ + "TypeDef:Struct": true, + "$defs": { "Read": read_variant }, + "properties": { + "packet": { + "TypeDef:Union": true, + "discriminator": { "kind": "byte", "offset": 0, "type": "TypeDef:Uint8" }, + "mapping": { "5": { "$ref": "#/$defs/Read" } } + } + } + }); + let mut buf = vec![0u8; 16]; + buf[0] = 5; + write_u32(&mut buf, 1, 1234, LE); + + let mut reader = SequentialReader::new(&schema).unwrap(); + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "packet"); + match value { + FieldValue::Union { + discriminator, + variant_start, + } => { + assert_eq!(discriminator, "5"); + assert_eq!(variant_start, 1); + } + other => panic!("expected Union, got {other:?}"), + } + assert_eq!(reader.position(), 5); + } + + #[test] + fn union_via_ref_with_local_defs_resolves_variant() { + let read_variant = json!({ + "TypeDef:Struct": true, + "properties": { "len": { "TypeDef:Uint32": true } } + }); + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "packet": { + "TypeDef:Union": true, + "discriminator": { "kind": "byte", "offset": 0, "type": "TypeDef:Uint8" }, + "mapping": { "5": { "$ref": "#/$defs/Read" } }, + "$defs": { "Read": read_variant } + } + } + }); + let mut buf = vec![0u8; 16]; + buf[0] = 5; + write_u32(&mut buf, 1, 1234, LE); + + let mut reader = SequentialReader::new(&schema).unwrap(); + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "packet"); + match value { + FieldValue::Union { + discriminator, + variant_start, + } => { + assert_eq!(discriminator, "5"); + assert_eq!(variant_start, 1); + } + other => panic!("expected Union, got {other:?}"), + } + assert_eq!(reader.position(), 5); + } + + #[test] + fn empty_struct_returns_none_immediately() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": {} + }); + let buf = []; + let mut reader = SequentialReader::new(&schema).unwrap(); + assert!(reader.read_next(&buf).unwrap().is_none()); + assert_eq!(reader.position(), 0); + } + + #[test] + fn nested_struct_with_variable_field_computes_end() { + let nested = json!({ + "TypeDef:Struct": true, + "properties": { + "header": { + "TypeDef:Struct": true, + "properties": { + "id": { "TypeDef:Uint8": true }, + "name": { "TypeDef:String": true } + } + }, + "tail": { "TypeDef:Uint8": true } + } + }); + let mut buf = vec![0u8; 64]; + buf[0] = 1; + let written = write_string(&mut buf, 1, "abc", LE); + let header_end = 1 + written; + buf[header_end] = 7; + let expected_end = header_end + 1; + + let mut reader = SequentialReader::new(&nested).unwrap(); + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "header"); + match value { + FieldValue::Struct { start, end } => { + assert_eq!(start, 0); + assert_eq!(end, header_end); + } + other => panic!("expected Struct, got {other:?}"), + } + assert_eq!(reader.position(), header_end); + + let (name, value) = reader.read_next(&buf).unwrap().unwrap(); + assert_eq!(name, "tail"); + assert_eq!(value, FieldValue::U8(7)); + assert_eq!(reader.position(), expected_end); + } +} diff --git a/src/tunion.rs b/src/tunion.rs new file mode 100644 index 0000000..dd17505 --- /dev/null +++ b/src/tunion.rs @@ -0,0 +1,645 @@ +//! TUnion discriminator dispatch (ADR-097 §4). +//! +//! TUnion supports two discriminator kinds: byte-offset (protocol +//! dispatch, e.g., SFTP type bytes) and field-name (typedef.ts string +//! pattern). This module reads the discriminator value from a byte +//! buffer, looks up the variant schema in the union's `mapping`, and +//! reports the offset where the variant struct begins. +//! +//! All reads go through [`crate::data_access`] so bounds checks and +//! endianness handling are uniform with the rest of the engine. + +use crate::data_access::{read_enum, read_string, read_u16, read_u32, read_u8}; +use crate::error::TypedefError; +use crate::schema::{get_typedef_kind, parse_discriminator, DiscriminatorKind, Endian, TypeDefKind, DISCRIMINATOR_PATH, U32_SIZE}; +use serde_json::Value; + +const STRING_PREFIX_SIZE: usize = 4; + +/// The result of reading a TUnion discriminator. +#[derive(Debug, Clone)] +pub struct UnionDispatch { + /// The mapping key (stringified discriminator value for byte-offset, + /// string value for field-name). + pub key: String, + /// The byte offset where the variant struct starts. + pub variant_offset: usize, + /// The size of the discriminator in bytes. + pub discriminator_size: usize, +} + +/// Read the discriminator value from a byte-offset TUnion. +/// +/// The discriminator is a fixed-size integer at a known byte offset. +/// Returns the mapping key (as a string) and the variant struct offset. +/// +/// This is the SFTP `Packet` enum pattern — byte 0 is the type byte, +/// bytes 1..N are the variant struct. The call protocol's event type +/// dispatch uses the same pattern. +/// +/// # Errors +/// +/// - [`TypedefError::Schema`] if the discriminator annotation is missing +/// or malformed, or if the discriminator `type` is not one of +/// `TypeDef:Uint8` / `TypeDef:Uint16` / `TypeDef:Uint32`. +/// - [`TypedefError::Access`] if the buffer is too short to contain the +/// discriminator, or if the read value is not present in the union's +/// `mapping`. +pub fn read_byte_discriminator( + buffer: &[u8], + union_schema: &Value, + endian: Endian, +) -> Result { + let disc = parse_discriminator(union_schema)?; + let (offset, disc_type) = match disc { + DiscriminatorKind::Byte { offset, disc_type } => (offset, disc_type), + DiscriminatorKind::Field { .. } => { + return Err(TypedefError::Schema( + "read_byte_discriminator requires a byte-offset discriminator".to_string(), + )); + } + }; + + let (disc_value, discriminator_size) = match disc_type { + TypeDefKind::Uint8 => (u32::from(read_u8(buffer, offset, DISCRIMINATOR_PATH)?), 1), + TypeDefKind::Uint16 => ( + u32::from(read_u16(buffer, offset, DISCRIMINATOR_PATH, endian)?), + 2, + ), + TypeDefKind::Uint32 => (read_u32(buffer, offset, DISCRIMINATOR_PATH, endian)?, 4), + other => { + return Err(TypedefError::Schema(format!( + "unsupported byte discriminator type: {other}" + ))); + } + }; + + let key = disc_value.to_string(); + verify_mapping_key(union_schema, &key, DISCRIMINATOR_PATH, &key)?; + + let variant_offset = + offset + .checked_add(discriminator_size) + .ok_or_else(|| TypedefError::Access { + field_path: DISCRIMINATOR_PATH.to_string(), + reason: format!( + "offset {offset} + discriminator_size {discriminator_size} overflows usize" + ), + })?; + + Ok(UnionDispatch { + key, + variant_offset, + discriminator_size, + }) +} + +/// Read the discriminator value from a field-name TUnion. +/// +/// The discriminator is a named field within the struct — its offset +/// is computed like any other field. The consumer provides the +/// discriminator field's offset (from the OffsetMap or LayoutBuilder). +/// +/// This is the typedef.ts `TUnion` pattern — the discriminator is a +/// field like any other, and the mapping keys are string values. +/// +/// # Errors +/// +/// - [`TypedefError::Schema`] if the discriminator annotation is missing +/// or malformed, the discriminator field is not declared in +/// `properties`, the field has no `TypeDef:*` kind, or the field's +/// kind is not one of `TypeDef:String` / `TypeDef:Uint8` / +/// `TypeDef:Enum`. +/// - [`TypedefError::Access`] if the buffer is too short to contain the +/// discriminator field, or if the read value is not present in the +/// union's `mapping`. +pub fn read_field_discriminator( + buffer: &[u8], + union_schema: &Value, + disc_field_offset: usize, + endian: Endian, +) -> Result { + let disc = parse_discriminator(union_schema)?; + let name = match disc { + DiscriminatorKind::Field { name } => name, + DiscriminatorKind::Byte { .. } => { + return Err(TypedefError::Schema( + "read_field_discriminator requires a field-name discriminator".to_string(), + )); + } + }; + + let field_schema = union_schema + .get("properties") + .and_then(Value::as_object) + .and_then(|props| props.get(&name)) + .ok_or_else(|| { + TypedefError::Schema(format!( + "discriminator field '{name}' not found in union properties" + )) + })?; + + let kind = get_typedef_kind(field_schema) + .and_then(|s| s.parse::().ok()) + .ok_or_else(|| { + TypedefError::Schema(format!( + "discriminator field '{name}' has no TypeDef:* kind" + )) + })?; + + let (key, discriminator_field_size) = match kind { + TypeDefKind::String => { + let s = read_string(buffer, disc_field_offset, &name, endian)?; + let size = + STRING_PREFIX_SIZE + .checked_add(s.len()) + .ok_or_else(|| TypedefError::Access { + field_path: name.clone(), + reason: format!( + "string prefix {STRING_PREFIX_SIZE} + data length {} overflows usize", + s.len() + ), + })?; + (s.to_string(), size) + } + TypeDefKind::Uint8 => { + let v = read_u8(buffer, disc_field_offset, &name)?; + (v.to_string(), 1) + } + TypeDefKind::Enum => { + let v = read_enum(buffer, disc_field_offset, &name, endian)?; + (v.to_string(), U32_SIZE) + } + other => { + return Err(TypedefError::Schema(format!( + "unsupported discriminator field type: {other}" + ))); + } + }; + + verify_mapping_key(union_schema, &key, &name, &key)?; + + let variant_offset = disc_field_offset + .checked_add(discriminator_field_size) + .ok_or_else(|| TypedefError::Access { + field_path: name.clone(), + reason: format!( + "disc_field_offset {disc_field_offset} + discriminator_field_size {discriminator_field_size} overflows usize" + ), + })?; + + Ok(UnionDispatch { + key, + variant_offset, + discriminator_size: discriminator_field_size, + }) +} + +/// Look up a variant schema from the union's mapping. +/// +/// Returns the variant schema. Inline schemas are returned directly. +/// `$ref` pointers of the form `"#/$defs/"` are resolved against +/// the `union_schema`'s own `$defs` block (when the union schema is the +/// schema root). For nested unions whose `$defs` live on an ancestor, +/// the caller (typically `TypedefEngine::compile`) is expected to +/// resolve refs before reaching this function, or to inline the +/// variant schemas into the mapping at load time. +/// +/// # Errors +/// +/// - [`TypedefError::Schema`] if the union has no `mapping` object, the +/// `key` is not present, a `$ref` is malformed, or a `$ref` cannot be +/// resolved against the union schema's own `$defs`. +pub fn resolve_variant<'a>(union_schema: &'a Value, key: &str) -> Result<&'a Value, TypedefError> { + let mapping = union_schema + .get("mapping") + .and_then(Value::as_object) + .ok_or_else(|| TypedefError::Schema("union is missing 'mapping' object".to_string()))?; + + let variant = mapping + .get(key) + .ok_or_else(|| TypedefError::Schema(format!("unknown mapping key: {key}")))?; + + let ref_str = match variant.get("$ref").and_then(Value::as_str) { + Some(r) => r, + None => return Ok(variant), + }; + + let pointer = ref_str + .strip_prefix('#') + .ok_or_else(|| TypedefError::Schema(format!("unsupported $ref form: {ref_str}")))?; + + let resolved = resolve_json_pointer(union_schema, pointer).ok_or_else(|| { + TypedefError::Schema(format!( + "cannot resolve $ref {ref_str} against union schema; ensure refs are inlined or the union schema contains $defs" + )) + })?; + Ok(resolved) +} + +/// Get the discriminator size in bytes for a byte-offset discriminator. +/// +/// Returns 1 for `TypeDef:Uint8`, 2 for `TypeDef:Uint16`, and 4 for +/// `TypeDef:Uint32`. Field-name discriminators have no fixed size and +/// produce a [`TypedefError::Schema`]. +/// +/// # Errors +/// +/// - [`TypedefError::Schema`] if the discriminator annotation is +/// missing/malformed, the discriminator `type` is unsupported, or the +/// discriminator is a field-name discriminator. +pub fn discriminator_size(union_schema: &Value) -> Result { + let disc = parse_discriminator(union_schema)?; + match disc { + DiscriminatorKind::Byte { disc_type, .. } => match disc_type { + TypeDefKind::Uint8 => Ok(1), + TypeDefKind::Uint16 => Ok(2), + TypeDefKind::Uint32 => Ok(4), + other => Err(TypedefError::Schema(format!( + "unsupported byte discriminator type: {other}" + ))), + }, + DiscriminatorKind::Field { .. } => Err(TypedefError::Schema( + "field-name discriminator has no fixed size".to_string(), + )), + } +} + +fn verify_mapping_key( + union_schema: &Value, + key: &str, + field_path: &str, + raw_value: &str, +) -> Result<(), TypedefError> { + let in_mapping = union_schema + .get("mapping") + .and_then(Value::as_object) + .map(|m| m.contains_key(key)) + .unwrap_or(false); + if in_mapping { + Ok(()) + } else { + Err(TypedefError::Access { + field_path: field_path.to_string(), + reason: format!("unknown discriminator value: {raw_value}"), + }) + } +} + +fn resolve_json_pointer<'a>(root: &'a Value, pointer: &str) -> Option<&'a Value> { + if pointer.is_empty() { + return Some(root); + } + let trimmed = pointer.strip_prefix('/')?; + let mut current = root; + for unescaped in trimmed.split('/') { + let segment = unescape_json_pointer_token(unescaped)?; + current = current.get(&segment)?; + } + Some(current) +} + +fn unescape_json_pointer_token(token: &str) -> Option { + let mut out = String::with_capacity(token.len()); + let mut chars = token.chars(); + while let Some(c) = chars.next() { + match c { + '~' => match chars.next() { + Some('0') => out.push('~'), + Some('1') => out.push('/'), + _ => return None, + }, + other => out.push(other), + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const LE: Endian = Endian::Little; + const BE: Endian = Endian::Big; + + fn byte_union_schema(offset: usize, disc_type: &str) -> Value { + json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "byte", "offset": offset, "type": disc_type}, + "mapping": { + "5": {"TypeDef:Struct": true, "properties": {"id": {"TypeDef:Uint32": true}}}, + "6": {"TypeDef:Struct": true, "properties": {"len": {"TypeDef:Uint16": true}}} + } + }) + } + + fn field_union_schema(field_name: &str, field_kind: &str) -> Value { + let field_schema = match field_kind { + "TypeDef:Enum" => json!({ + "TypeDef:Enum": true, + "enum": ["read", "write"] + }), + _ => json!({field_kind: true}), + }; + let (key_a, key_b) = match field_kind { + "TypeDef:String" => ("read", "write"), + _ => ("0", "1"), + }; + json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "field", "name": field_name}, + "properties": { + field_name: field_schema + }, + "mapping": { + key_a: {"TypeDef:Struct": true, "properties": {"n": {"TypeDef:Uint32": true}}}, + key_b: {"TypeDef:Struct": true, "properties": {"m": {"TypeDef:Uint16": true}}} + } + }) + } + + #[test] + fn read_byte_discriminator_uint8_default_offset() { + let schema = byte_union_schema(0, "TypeDef:Uint8"); + let buf = [5u8, 0xAA, 0xBB, 0xCC]; + let d = read_byte_discriminator(&buf, &schema, LE).expect("read"); + assert_eq!(d.key, "5"); + assert_eq!(d.variant_offset, 1); + assert_eq!(d.discriminator_size, 1); + } + + #[test] + fn read_byte_discriminator_uint8_big_endian() { + let schema = byte_union_schema(0, "TypeDef:Uint8"); + let buf = [6u8]; + let d = read_byte_discriminator(&buf, &schema, BE).expect("read"); + assert_eq!(d.key, "6"); + assert_eq!(d.variant_offset, 1); + } + + #[test] + fn read_byte_discriminator_uint16_little_endian() { + let schema = byte_union_schema(2, "TypeDef:Uint16"); + let mut buf = vec![0u8; 4]; + buf[2..4].copy_from_slice(&5u16.to_le_bytes()); + let d = read_byte_discriminator(&buf, &schema, LE).expect("read"); + assert_eq!(d.key, "5"); + assert_eq!(d.variant_offset, 4); + assert_eq!(d.discriminator_size, 2); + } + + #[test] + fn read_byte_discriminator_uint16_big_endian() { + let schema = byte_union_schema(0, "TypeDef:Uint16"); + let buf = [0x00, 0x06, 0xAA, 0xBB]; + let d = read_byte_discriminator(&buf, &schema, BE).expect("read"); + assert_eq!(d.key, "6"); + assert_eq!(d.variant_offset, 2); + } + + #[test] + fn read_byte_discriminator_uint32_little_endian() { + let schema = byte_union_schema(0, "TypeDef:Uint32"); + let mut buf = vec![0u8; 8]; + buf[0..4].copy_from_slice(&5u32.to_le_bytes()); + let d = read_byte_discriminator(&buf, &schema, LE).expect("read"); + assert_eq!(d.key, "5"); + assert_eq!(d.variant_offset, 4); + assert_eq!(d.discriminator_size, 4); + } + + #[test] + fn read_byte_discriminator_uint32_big_endian() { + let schema = byte_union_schema(0, "TypeDef:Uint32"); + let mut buf = vec![0u8; 8]; + buf[0..4].copy_from_slice(&6u32.to_be_bytes()); + let d = read_byte_discriminator(&buf, &schema, BE).expect("read"); + assert_eq!(d.key, "6"); + assert_eq!(d.variant_offset, 4); + } + + #[test] + fn read_byte_discriminator_unknown_value_is_access_error() { + let schema = byte_union_schema(0, "TypeDef:Uint8"); + let buf = [99u8]; + let err = read_byte_discriminator(&buf, &schema, LE).unwrap_err(); + match err { + TypedefError::Access { field_path, reason } => { + assert_eq!(field_path, DISCRIMINATOR_PATH); + assert!(reason.contains("99"), "reason: {reason}"); + } + other => panic!("expected Access, got {other:?}"), + } + } + + #[test] + fn read_byte_discriminator_buffer_too_short_is_access_error() { + let schema = byte_union_schema(4, "TypeDef:Uint32"); + let buf = [0u8; 2]; + let err = read_byte_discriminator(&buf, &schema, LE).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. })); + } + + #[test] + fn read_byte_discriminator_field_kind_is_schema_error() { + let schema = field_union_schema("type", "TypeDef:String"); + let buf = [0u8; 16]; + let err = read_byte_discriminator(&buf, &schema, LE).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_))); + } + + #[test] + fn read_field_discriminator_string() { + let schema = field_union_schema("type", "TypeDef:String"); + let mut buf = vec![0u8; 32]; + let value = "read"; + let len_bytes = (value.len() as u32).to_le_bytes(); + buf[0..4].copy_from_slice(&len_bytes); + buf[4..4 + value.len()].copy_from_slice(value.as_bytes()); + let d = read_field_discriminator(&buf, &schema, 0, LE).expect("read"); + assert_eq!(d.key, "read"); + assert_eq!(d.variant_offset, 4 + value.len()); + assert_eq!(d.discriminator_size, 4 + value.len()); + } + + #[test] + fn read_field_discriminator_uint8() { + let schema = field_union_schema("type", "TypeDef:Uint8"); + let mut buf = vec![0u8; 8]; + buf[0] = 0; + let d = read_field_discriminator(&buf, &schema, 0, LE).expect("read"); + assert_eq!(d.key, "0"); + assert_eq!(d.variant_offset, 1); + assert_eq!(d.discriminator_size, 1); + } + + #[test] + fn read_field_discriminator_enum() { + let schema = field_union_schema("type", "TypeDef:Enum"); + let mut buf = vec![0u8; 8]; + buf[0..4].copy_from_slice(&0u32.to_le_bytes()); + let d = read_field_discriminator(&buf, &schema, 0, LE).expect("read"); + assert_eq!(d.key, "0"); + assert_eq!(d.variant_offset, 4); + assert_eq!(d.discriminator_size, 4); + } + + #[test] + fn read_field_discriminator_string_big_endian() { + let schema = field_union_schema("type", "TypeDef:String"); + let mut buf = vec![0u8; 32]; + let value = "write"; + let len_bytes = (value.len() as u32).to_be_bytes(); + buf[0..4].copy_from_slice(&len_bytes); + buf[4..4 + value.len()].copy_from_slice(value.as_bytes()); + let d = read_field_discriminator(&buf, &schema, 0, BE).expect("read"); + assert_eq!(d.key, "write"); + assert_eq!(d.variant_offset, 4 + value.len()); + } + + #[test] + fn read_field_discriminator_unknown_value_is_access_error() { + let schema = field_union_schema("type", "TypeDef:Uint8"); + let mut buf = vec![0u8; 8]; + buf[0] = 99; + let err = read_field_discriminator(&buf, &schema, 0, LE).unwrap_err(); + match err { + TypedefError::Access { field_path, reason } => { + assert_eq!(field_path, "type"); + assert!(reason.contains("99"), "reason: {reason}"); + } + other => panic!("expected Access, got {other:?}"), + } + } + + #[test] + fn read_field_discriminator_field_not_found_is_schema_error() { + let schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "field", "name": "missing"}, + "properties": {"other": {"TypeDef:Uint8": true}}, + "mapping": {"5": {"TypeDef:Struct": true}} + }); + let buf = [0u8; 4]; + let err = read_field_discriminator(&buf, &schema, 0, LE).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_))); + } + + #[test] + fn read_field_discriminator_no_typedef_kind_is_schema_error() { + let schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "field", "name": "type"}, + "properties": {"type": {"type": "string"}}, + "mapping": {"read": {"TypeDef:Struct": true}} + }); + let buf = [0u8; 4]; + let err = read_field_discriminator(&buf, &schema, 0, LE).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_))); + } + + #[test] + fn read_field_discriminator_unsupported_kind_is_schema_error() { + let schema = field_union_schema("type", "TypeDef:Float32"); + let buf = [0u8; 8]; + let err = read_field_discriminator(&buf, &schema, 0, LE).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_))); + } + + #[test] + fn read_field_discriminator_byte_kind_is_schema_error() { + let schema = byte_union_schema(0, "TypeDef:Uint8"); + let buf = [5u8]; + let err = read_field_discriminator(&buf, &schema, 0, LE).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_))); + } + + #[test] + fn resolve_variant_inline_schema() { + let schema = byte_union_schema(0, "TypeDef:Uint8"); + let variant = resolve_variant(&schema, "5").expect("resolve"); + assert_eq!( + variant.get("TypeDef:Struct").and_then(Value::as_bool), + Some(true) + ); + } + + #[test] + fn resolve_variant_ref_against_own_defs() { + let schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "byte"}, + "mapping": { + "5": {"$ref": "#/$defs/Read"} + }, + "$defs": { + "Read": {"TypeDef:Struct": true, "properties": {"id": {"TypeDef:Uint32": true}}} + } + }); + let variant = resolve_variant(&schema, "5").expect("resolve"); + assert_eq!( + variant.get("TypeDef:Struct").and_then(Value::as_bool), + Some(true) + ); + } + + #[test] + fn resolve_variant_unknown_key_is_schema_error() { + let schema = byte_union_schema(0, "TypeDef:Uint8"); + let err = resolve_variant(&schema, "999").unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_))); + } + + #[test] + fn resolve_variant_missing_mapping_is_schema_error() { + let schema = json!({"TypeDef:Union": true, "discriminator": {"kind": "byte"}}); + let err = resolve_variant(&schema, "5").unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_))); + } + + #[test] + fn resolve_variant_unresolvable_ref_is_schema_error() { + let schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "byte"}, + "mapping": { + "5": {"$ref": "#/$defs/Read"} + } + }); + let err = resolve_variant(&schema, "5").unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_))); + } + + #[test] + fn discriminator_size_uint8() { + let schema = byte_union_schema(0, "TypeDef:Uint8"); + assert_eq!(discriminator_size(&schema).unwrap(), 1); + } + + #[test] + fn discriminator_size_uint16() { + let schema = byte_union_schema(0, "TypeDef:Uint16"); + assert_eq!(discriminator_size(&schema).unwrap(), 2); + } + + #[test] + fn discriminator_size_uint32() { + let schema = byte_union_schema(0, "TypeDef:Uint32"); + assert_eq!(discriminator_size(&schema).unwrap(), 4); + } + + #[test] + fn discriminator_size_field_kind_is_schema_error() { + let schema = field_union_schema("type", "TypeDef:String"); + let err = discriminator_size(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_))); + } + + #[test] + fn discriminator_size_missing_discriminator_is_schema_error() { + let schema = json!({"TypeDef:Union": true}); + let err = discriminator_size(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_))); + } +} diff --git a/src/validation.rs b/src/validation.rs new file mode 100644 index 0000000..96feca9 --- /dev/null +++ b/src/validation.rs @@ -0,0 +1,630 @@ +//! Custom keyword validators for all 17 `TypeDef:*` kinds, registered +//! via `jsonschema::options().with_keyword(...)`. +//! +//! Per ADR-098: the `jsonschema` crate handles all structural validation; +//! the custom keywords only validate leaf type constraints. Each validator +//! is a small (~10 line) struct implementing [`jsonschema::Keyword`]. +//! +//! The factory closures reject schemas where the keyword is not set to +//! `true` (returning [`jsonschema::ValidationError::schema`]). A few +//! factories read parent context (e.g. `maxLength`) to pass into the +//! validator struct. + +use crate::error::TypedefError; +use jsonschema::{Keyword, ValidationError}; +use serde_json::{Map, Value}; + +/// Build a jsonschema validator with all 17 `TypeDef:*` custom keywords +/// registered. +/// +/// The returned validator can validate JSON representations of data +/// against the schema's type constraints. Structural validation +/// (`properties`, `required`, `items`, `enum`, ...) is handled by +/// jsonschema's built-in keywords; the custom keywords only check leaf +/// type constraints (range, finiteness, RFC 3339 shape, ...). +/// +/// # Errors +/// +/// Returns [`TypedefError::Schema`] if the schema is malformed or the +/// underlying jsonschema validator cannot be built. +pub fn build_validator(schema: &Value) -> Result { + jsonschema::options() + .with_keyword("TypeDef:Float32", float32_factory) + .with_keyword("TypeDef:Float64", float64_factory) + .with_keyword("TypeDef:Int8", int8_factory) + .with_keyword("TypeDef:Int16", int16_factory) + .with_keyword("TypeDef:Int32", int32_factory) + .with_keyword("TypeDef:Int64", int64_factory) + .with_keyword("TypeDef:Uint8", uint8_factory) + .with_keyword("TypeDef:Uint16", uint16_factory) + .with_keyword("TypeDef:Uint32", uint32_factory) + .with_keyword("TypeDef:Uint64", uint64_factory) + .with_keyword("TypeDef:Boolean", boolean_factory) + .with_keyword("TypeDef:String", string_factory) + .with_keyword("TypeDef:Bytes", bytes_factory) + .with_keyword("TypeDef:Enum", enum_factory) + .with_keyword("TypeDef:Struct", struct_factory) + .with_keyword("TypeDef:Union", union_factory) + .with_keyword("TypeDef:Array", array_factory) + .with_keyword("TypeDef:Record", record_factory) + .with_keyword("TypeDef:Timestamp", timestamp_factory) + .build(schema) + .map_err(|e| TypedefError::Schema(format!("validator build failed: {e}"))) +} + +// --------------------------------------------------------------------------- +// Numeric validators (generated via macros) +// --------------------------------------------------------------------------- + +define_int_validator!(Int8Validator, int8_factory, "TypeDef:Int8", -128, 127); +define_int_validator!(Int16Validator, int16_factory, "TypeDef:Int16", -32768, 32767); +define_int_validator!(Int32Validator, int32_factory, "TypeDef:Int32", -2147483648, 2147483647); +define_uint_validator!(Uint8Validator, uint8_factory, "TypeDef:Uint8", 255); +define_uint_validator!(Uint16Validator, uint16_factory, "TypeDef:Uint16", 65535); +define_uint_validator!(Uint32Validator, uint32_factory, "TypeDef:Uint32", 4294967295); + +// Int64/Uint64 use the full i64/u64 range, so the macro's `n <= $max` check +// is always true (clippy: "comparison useless due to type limits"). Write +// them directly — the validator just checks that the JSON value is an +// integer in the right range. +struct Int64Validator; +impl Keyword for Int64Validator { + fn validate<'i>(&self, instance: &'i Value) -> Result<(), ValidationError<'i>> { + match instance.as_i64() { + Some(_) => Ok(()), + None => Err(ValidationError::custom("expected an i64 integer")), + } + } + fn is_valid(&self, instance: &Value) -> bool { + instance.as_i64().is_some() + } +} + +fn int64_factory<'a>( + _parent: &'a Map, + value: &'a Value, + _path: jsonschema::paths::Location, +) -> Result, ValidationError<'a>> { + if value.as_bool() == Some(true) { + Ok(Box::new(Int64Validator)) + } else { + Err(ValidationError::schema("TypeDef:Int64 must be set to true")) + } +} + +struct Uint64Validator; +impl Keyword for Uint64Validator { + fn validate<'i>(&self, instance: &'i Value) -> Result<(), ValidationError<'i>> { + match instance.as_u64() { + Some(_) => Ok(()), + None => Err(ValidationError::custom("expected a u64 integer")), + } + } + fn is_valid(&self, instance: &Value) -> bool { + instance.as_u64().is_some() + } +} + +fn uint64_factory<'a>( + _parent: &'a Map, + value: &'a Value, + _path: jsonschema::paths::Location, +) -> Result, ValidationError<'a>> { + if value.as_bool() == Some(true) { + Ok(Box::new(Uint64Validator)) + } else { + Err(ValidationError::schema("TypeDef:Uint64 must be set to true")) + } +} +define_float_validator!( + Float32Validator, + float32_factory, + "TypeDef:Float32", + "expected a finite f32-compatible number" +); +define_float_validator!( + Float64Validator, + float64_factory, + "TypeDef:Float64", + "expected a finite f64 number" +); + +// --------------------------------------------------------------------------- +// String and binary validators (hand-written: need maxLength from parent) +// --------------------------------------------------------------------------- + +struct StringValidator { + max_length: Option, +} +impl Keyword for StringValidator { + fn validate<'i>(&self, instance: &'i Value) -> Result<(), ValidationError<'i>> { + match instance.as_str() { + Some(s) => { + if let Some(max) = self.max_length { + if s.len() > max { + return Err(ValidationError::custom(format!( + "string byte length {} exceeds maxLength {max}", + s.len() + ))); + } + } + Ok(()) + } + None => Err(ValidationError::custom("expected a string")), + } + } + fn is_valid(&self, instance: &Value) -> bool { + instance + .as_str() + .is_some_and(|s| self.max_length.is_none_or(|max| s.len() <= max)) + } +} + +struct BytesValidator { + max_length: Option, +} +impl Keyword for BytesValidator { + fn validate<'i>(&self, instance: &'i Value) -> Result<(), ValidationError<'i>> { + match instance.as_str() { + Some(s) => { + if let Some(max) = self.max_length { + if s.len() > max { + return Err(ValidationError::custom(format!( + "bytes length {} exceeds maxLength {max}", + s.len() + ))); + } + } + Ok(()) + } + None => Err(ValidationError::custom("expected a string for bytes")), + } + } + fn is_valid(&self, instance: &Value) -> bool { + instance + .as_str() + .is_some_and(|s| self.max_length.is_none_or(|max| s.len() <= max)) + } +} + +/// `TypeDef:Enum` is a layout marker — the built-in `enum` keyword +/// handles value-membership validation. The custom keyword exists solely +/// for the layout engine to recognize the type as a fixed-size u32 index. +struct EnumValidator; +impl Keyword for EnumValidator { + fn validate<'i>(&self, _instance: &'i Value) -> Result<(), ValidationError<'i>> { + Ok(()) + } + fn is_valid(&self, _instance: &Value) -> bool { + true + } +} + +struct TimestampValidator; +impl Keyword for TimestampValidator { + fn validate<'i>(&self, instance: &'i Value) -> Result<(), ValidationError<'i>> { + match instance.as_str() { + Some(s) if is_rfc3339_timestamp(s) => Ok(()), + _ => Err(ValidationError::custom( + "expected an RFC 3339 timestamp string", + )), + } + } + fn is_valid(&self, instance: &Value) -> bool { + instance.as_str().is_some_and(is_rfc3339_timestamp) + } +} + +/// Simple RFC 3339 / ISO 8601 datetime check: `YYYY-MM-DDTHH:MM:SS` +/// optionally followed by `Z` or a timezone offset. +fn is_rfc3339_timestamp(s: &str) -> bool { + let parts: Vec<&str> = s.splitn(2, 'T').collect(); + if parts.len() != 2 { + return false; + } + let date_parts: Vec<&str> = parts[0].split('-').collect(); + if date_parts.len() != 3 { + return false; + } + let time_part = parts[1]; + let time_clean = if let Some(pos) = time_part.find(['Z', '+']) { + &time_part[..pos] + } else if let Some(pos) = time_part.rfind('-') { + if pos >= 8 { + &time_part[..pos] + } else { + time_part + } + } else { + time_part + }; + let time_parts: Vec<&str> = time_clean.split(':').collect(); + if time_parts.len() < 2 || time_parts.len() > 3 { + return false; + } + date_parts[0].parse::().is_ok_and(|y| y > 0) + && date_parts[1] + .parse::() + .is_ok_and(|m| (1..=12).contains(&m)) + && date_parts[2] + .parse::() + .is_ok_and(|d| (1..=31).contains(&d)) + && time_parts[0].parse::().is_ok_and(|h| h <= 23) + && time_parts[1].parse::().is_ok_and(|m| m <= 59) +} + +// --------------------------------------------------------------------------- +// Composite validators (generated via macros) +// --------------------------------------------------------------------------- + +define_type_validator!(StructValidator, struct_factory, "TypeDef:Struct", is_object, "expected an object"); +define_type_validator!(UnionValidator, union_factory, "TypeDef:Union", is_object, "expected an object for union"); +define_type_validator!(ArrayValidator, array_factory, "TypeDef:Array", is_array, "expected an array"); +define_type_validator!(RecordValidator, record_factory, "TypeDef:Record", is_object, "expected an object for record"); +define_type_validator!(BooleanValidator, boolean_factory, "TypeDef:Boolean", is_boolean, "expected a boolean"); + +// --------------------------------------------------------------------------- +// Factory closures for non-macro-generated validators +// --------------------------------------------------------------------------- + +fn string_factory<'a>( + parent: &'a Map, + value: &'a Value, + _path: jsonschema::paths::Location, +) -> Result, ValidationError<'a>> { + if !value.is_boolean() && !value.is_object() { + return Err(ValidationError::schema( + "TypeDef:String must be set to true or an annotation object", + )); + } + let max_length = parent + .get("maxLength") + .and_then(Value::as_u64) + .map(|n| n as usize); + Ok(Box::new(StringValidator { max_length })) +} + +fn bytes_factory<'a>( + parent: &'a Map, + value: &'a Value, + _path: jsonschema::paths::Location, +) -> Result, ValidationError<'a>> { + if !value.is_boolean() && !value.is_object() { + return Err(ValidationError::schema( + "TypeDef:Bytes must be set to true or an annotation object", + )); + } + let max_length = parent + .get("maxLength") + .and_then(Value::as_u64) + .map(|n| n as usize); + Ok(Box::new(BytesValidator { max_length })) +} + +fn enum_factory<'a>( + _parent: &'a Map, + value: &'a Value, + _path: jsonschema::paths::Location, +) -> Result, ValidationError<'a>> { + if value.as_bool() == Some(true) { + Ok(Box::new(EnumValidator)) + } else { + Err(ValidationError::schema("TypeDef:Enum must be set to true")) + } +} + +fn timestamp_factory<'a>( + _parent: &'a Map, + value: &'a Value, + _path: jsonschema::paths::Location, +) -> Result, ValidationError<'a>> { + if value.as_bool() == Some(true) { + Ok(Box::new(TimestampValidator)) + } else { + Err(ValidationError::schema( + "TypeDef:Timestamp must be set to true", + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn validator_for(schema: &Value) -> jsonschema::Validator { + build_validator(schema).expect("validator should build") + } + + #[test] + fn validates_valid_struct_instance() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "id": { "TypeDef:Uint32": true, "type": "integer" }, + "score": { "TypeDef:Float32": true, "type": "number" }, + "flag": { "TypeDef:Uint8": true, "type": "integer" }, + "count": { "TypeDef:Uint16": true, "type": "integer" } + }, + "required": ["id", "score", "flag", "count"] + }); + let validator = validator_for(&schema); + let instance = json!({ + "id": 42, + "score": 3.5, + "flag": 1, + "count": 1000 + }); + assert!(validator.is_valid(&instance)); + } + + #[test] + fn rejects_uint32_out_of_range() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { "id": { "TypeDef:Uint32": true, "type": "integer" } }, + "required": ["id"] + }); + let validator = validator_for(&schema); + assert!(!validator.is_valid(&json!({"id": -1}))); + assert!(!validator.is_valid(&json!({"id": 5_000_000_000u64}))); + } + + #[test] + fn validates_int8_range() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { "val": { "TypeDef:Int8": true, "type": "integer" } }, + "required": ["val"] + }); + let validator = validator_for(&schema); + assert!(validator.is_valid(&json!({"val": 0}))); + assert!(validator.is_valid(&json!({"val": 127}))); + assert!(validator.is_valid(&json!({"val": -128}))); + assert!(!validator.is_valid(&json!({"val": 128}))); + assert!(!validator.is_valid(&json!({"val": -129}))); + } + + #[test] + fn validates_int16_and_int32_ranges() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "i16": { "TypeDef:Int16": true, "type": "integer" }, + "i32": { "TypeDef:Int32": true, "type": "integer" } + }, + "required": ["i16", "i32"] + }); + let validator = validator_for(&schema); + assert!(validator.is_valid(&json!({"i16": 32767, "i32": 2147483647}))); + assert!(validator.is_valid(&json!({"i16": -32768, "i32": -2147483648}))); + assert!(!validator.is_valid(&json!({"i16": 32768, "i32": 0}))); + assert!(!validator.is_valid(&json!({"i16": 0, "i32": 2147483648u64}))); + } + + #[test] + fn validates_uint16_and_uint32_ranges() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "u16": { "TypeDef:Uint16": true, "type": "integer" }, + "u32": { "TypeDef:Uint32": true, "type": "integer" } + }, + "required": ["u16", "u32"] + }); + let validator = validator_for(&schema); + assert!(validator.is_valid(&json!({"u16": 65535, "u32": 4294967295u64}))); + assert!(!validator.is_valid(&json!({"u16": 65536, "u32": 0}))); + assert!(!validator.is_valid(&json!({"u16": -1, "u32": 0}))); + } + + #[test] + fn validates_int64_range() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { "val": { "TypeDef:Int64": true, "type": "integer" } }, + "required": ["val"] + }); + let validator = validator_for(&schema); + assert!(validator.is_valid(&json!({"val": 0}))); + assert!(validator.is_valid(&json!({"val": 9223372036854775807i64}))); + assert!(validator.is_valid(&json!({"val": -9223372036854775808i64}))); + assert!(!validator.is_valid(&json!({"val": "x"}))); + } + + #[test] + fn validates_uint64_range() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { "val": { "TypeDef:Uint64": true, "type": "integer" } }, + "required": ["val"] + }); + let validator = validator_for(&schema); + assert!(validator.is_valid(&json!({"val": 0}))); + assert!(validator.is_valid(&json!({"val": 18446744073709551615u64}))); + assert!(!validator.is_valid(&json!({"val": -1}))); + assert!(!validator.is_valid(&json!({"val": "x"}))); + } + + #[test] + fn validates_float_finiteness() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "f32": { "TypeDef:Float32": true, "type": "number" }, + "f64": { "TypeDef:Float64": true, "type": "number" } + }, + "required": ["f32", "f64"] + }); + let validator = validator_for(&schema); + assert!(validator.is_valid(&json!({"f32": 3.5, "f64": 2.5}))); + assert!(validator.is_valid(&json!({"f32": 0, "f64": 0}))); + assert!(!validator.is_valid(&json!({"f32": "x", "f64": 0}))); + } + + #[test] + fn validates_boolean() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { "active": { "TypeDef:Boolean": true, "type": "boolean" } }, + "required": ["active"] + }); + let validator = validator_for(&schema); + assert!(validator.is_valid(&json!({"active": true}))); + assert!(validator.is_valid(&json!({"active": false}))); + assert!(!validator.is_valid(&json!({"active": "yes"}))); + } + + #[test] + fn validates_string_and_maxlength() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "name": { "TypeDef:String": true, "type": "string", "maxLength": 5 } + }, + "required": ["name"] + }); + let validator = validator_for(&schema); + assert!(validator.is_valid(&json!({"name": "hi"}))); + assert!(validator.is_valid(&json!({"name": "hello"}))); + assert!(!validator.is_valid(&json!({"name": "toolong"}))); + assert!(!validator.is_valid(&json!({"name": 42}))); + } + + #[test] + fn validates_bytes_and_maxlength() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "blob": { "TypeDef:Bytes": true, "type": "string", "maxLength": 4 } + }, + "required": ["blob"] + }); + let validator = validator_for(&schema); + assert!(validator.is_valid(&json!({"blob": "abcd"}))); + assert!(!validator.is_valid(&json!({"blob": "abcde"}))); + assert!(!validator.is_valid(&json!({"blob": 42}))); + } + + #[test] + fn enum_validator_is_noop_and_builtin_enum_handles_membership() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "status": { + "TypeDef:Enum": true, + "type": "string", + "enum": ["ok", "error", "pending"] + } + }, + "required": ["status"] + }); + let validator = validator_for(&schema); + assert!(validator.is_valid(&json!({"status": "ok"}))); + assert!(validator.is_valid(&json!({"status": "error"}))); + assert!(!validator.is_valid(&json!({"status": "unknown"}))); + } + + #[test] + fn validates_timestamp_rfc3339() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "created_at": { "TypeDef:Timestamp": true, "type": "string" } + }, + "required": ["created_at"] + }); + let validator = validator_for(&schema); + assert!(validator.is_valid(&json!({"created_at": "2026-07-20T15:30:00Z"}))); + assert!(validator.is_valid(&json!({"created_at": "2026-07-20T15:30:00"}))); + assert!(validator.is_valid(&json!({"created_at": "2026-07-20T15:30:00+02:00"}))); + assert!(!validator.is_valid(&json!({"created_at": "not-a-date"}))); + } + + #[test] + fn validates_array_type() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "items": { + "TypeDef:Array": true, + "type": "array", + "items": { "TypeDef:Uint8": true, "type": "integer" } + } + }, + "required": ["items"] + }); + let validator = validator_for(&schema); + assert!(validator.is_valid(&json!({"items": [1, 2, 3]}))); + assert!(!validator.is_valid(&json!({"items": "not-array"}))); + } + + #[test] + fn validates_record_type() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "counts": { + "TypeDef:Record": true, + "type": "object", + "additionalProperties": { "TypeDef:Uint32": true, "type": "integer" } + } + }, + "required": ["counts"] + }); + let validator = validator_for(&schema); + assert!(validator.is_valid(&json!({"counts": {"a": 1, "b": 2}}))); + assert!(!validator.is_valid(&json!({"counts": "not-object"}))); + } + + #[test] + fn validates_union_type() { + let schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "packet": { + "TypeDef:Union": true, + "type": "object", + "properties": { + "type": { "type": "string" } + }, + "required": ["type"] + } + }, + "required": ["packet"] + }); + let validator = validator_for(&schema); + assert!(validator.is_valid(&json!({"packet": {"type": "read"}}))); + assert!(!validator.is_valid(&json!({"packet": "not-object"}))); + } + + #[test] + fn build_validator_returns_schema_error_for_malformed_keyword() { + let schema = json!({"TypeDef:Uint32": "not-a-bool"}); + let err = build_validator(&schema).expect_err("should fail"); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); + } + + #[test] + fn build_validator_maps_build_error_to_typedef_error() { + let schema = json!([1, 2, 3]); + let err = build_validator(&schema).expect_err("schema must be an object"); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); + } +} diff --git a/tests/engine_integration.rs b/tests/engine_integration.rs new file mode 100644 index 0000000..0694e81 --- /dev/null +++ b/tests/engine_integration.rs @@ -0,0 +1,389 @@ +//! Integration tests for the `TypedefEngine` public API. +//! +//! Exercises the engine across both layout modes, the convenience +//! accessors, validation convenience methods, and the aligned-mode +//! `read_field` / `write_field` round-trip for the fixed-size primitive +//! kinds and length-prefixed `String` / `Bytes`. + +use alknet_typedef::*; +use serde_json::json; + +fn mixed_fixed_struct_schema() -> serde_json::Value { + json!({ + "TypeDef:Struct": true, + "endian": "little", + "properties": { + "flag": { "TypeDef:Uint8": true }, + "id": { "TypeDef:Uint32": true }, + "score": { "TypeDef:Float32": true }, + "tag": { "TypeDef:String": true } + } + }) +} + +#[test] +fn compile_aligned_builds_engine_with_offset_map() -> Result<(), TypedefError> { + let mut schema = mixed_fixed_struct_schema(); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned)?; + assert_eq!(engine.mode(), LayoutMode::Aligned); + assert!(engine.offset_map().is_some()); + assert!(engine.layout_builder().is_none()); + assert!(engine.sequential_reader().is_none()); + Ok(()) +} + +#[test] +fn compile_packed_builds_engine_with_builder_and_reader() -> Result<(), TypedefError> { + let mut schema = mixed_fixed_struct_schema(); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Packed)?; + assert_eq!(engine.mode(), LayoutMode::Packed); + assert!(engine.offset_map().is_none()); + assert!(engine.layout_builder().is_some()); + assert!(engine.sequential_reader().is_some()); + Ok(()) +} + +#[test] +fn compile_normalizes_bare_name_refs() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { + "child": { "$ref": "Child" } + }, + "$defs": { + "Child": { + "TypeDef:Struct": true, + "properties": { "x": { "TypeDef:Uint8": true } } + } + } + }); + let _engine = TypedefEngine::compile(&mut schema, LayoutMode::Packed)?; + assert_eq!( + schema["properties"]["child"]["$ref"], + json!("#/$defs/Child") + ); + Ok(()) +} + +#[test] +fn compile_leaves_full_pointer_refs_unchanged() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { + "child": { "$ref": "#/$defs/Child" } + }, + "$defs": { + "Child": { + "TypeDef:Struct": true, + "properties": { "x": { "TypeDef:Uint8": true } } + } + } + }); + let _engine = TypedefEngine::compile(&mut schema, LayoutMode::Packed)?; + assert_eq!( + schema["properties"]["child"]["$ref"], + json!("#/$defs/Child") + ); + Ok(()) +} + +#[test] +fn compile_returns_schema_error_when_no_typedef_kind() { + let mut schema = json!({ "type": "object", "properties": {} }); + let err = TypedefEngine::compile(&mut schema, LayoutMode::Aligned).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); +} + +#[test] +fn endian_parsed_from_schema_big() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "endian": "big", + "properties": { "id": { "TypeDef:Uint32": true } } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Packed)?; + assert_eq!(engine.endian(), Endian::Big); + Ok(()) +} + +#[test] +fn endian_defaults_to_little() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { "id": { "TypeDef:Uint32": true } } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Packed)?; + assert_eq!(engine.endian(), Endian::Little); + Ok(()) +} + +#[test] +fn validate_json_accepts_valid_instance() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "id": { "TypeDef:Uint32": true, "type": "integer" } + }, + "required": ["id"] + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned)?; + assert!(engine.validate_json(&json!({"id": 42})).is_ok()); + Ok(()) +} + +#[test] +fn validate_json_rejects_invalid_instance() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "id": { "TypeDef:Uint32": true, "type": "integer" } + }, + "required": ["id"] + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned)?; + let err = engine.validate_json(&json!({"id": -1})).unwrap_err(); + assert!(matches!(err, TypedefError::Validation(_)), "got {err:?}"); + Ok(()) +} + +#[test] +fn is_valid_json_returns_bool() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "type": "object", + "properties": { + "id": { "TypeDef:Uint32": true, "type": "integer" } + }, + "required": ["id"] + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned)?; + assert!(engine.is_valid_json(&json!({"id": 42}))); + assert!(!engine.is_valid_json(&json!({"id": -1}))); + Ok(()) +} + +#[test] +fn read_write_aligned_round_trips_all_fixed_size_kinds() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "endian": "little", + "properties": { + "i8": { "TypeDef:Int8": true }, + "u8": { "TypeDef:Uint8": true }, + "i16": { "TypeDef:Int16": true }, + "u16": { "TypeDef:Uint16": true }, + "i32": { "TypeDef:Int32": true }, + "u32": { "TypeDef:Uint32": true }, + "i64": { "TypeDef:Int64": true }, + "u64": { "TypeDef:Uint64": true }, + "f32": { "TypeDef:Float32": true }, + "f64": { "TypeDef:Float64": true }, + "b": { "TypeDef:Boolean": true }, + "e": { "TypeDef:Enum": true } + } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned)?; + let offset_map = engine.offset_map().expect("aligned mode has offset_map"); + let mut buffer = vec![0u8; offset_map.total_size()]; + + engine.write_field(&mut buffer, "i8", &FieldValue::I8(-127))?; + engine.write_field(&mut buffer, "u8", &FieldValue::U8(0xAB))?; + engine.write_field(&mut buffer, "i16", &FieldValue::I16(-32000))?; + engine.write_field(&mut buffer, "u16", &FieldValue::U16(0xBEEF))?; + engine.write_field(&mut buffer, "i32", &FieldValue::I32(-2_000_000_007))?; + engine.write_field(&mut buffer, "u32", &FieldValue::U32(0xDEADBEEF))?; + engine.write_field(&mut buffer, "i64", &FieldValue::I64(-9_000_000_000_000_000_000))?; + engine.write_field(&mut buffer, "u64", &FieldValue::U64(0x0102030405060708))?; + engine.write_field(&mut buffer, "f32", &FieldValue::F32(1.5))?; + engine.write_field(&mut buffer, "f64", &FieldValue::F64(2.5))?; + engine.write_field(&mut buffer, "b", &FieldValue::Bool(true))?; + engine.write_field(&mut buffer, "e", &FieldValue::Enum(7))?; + + assert_eq!(engine.read_field(&buffer, "i8")?, FieldValue::I8(-127)); + assert_eq!(engine.read_field(&buffer, "u8")?, FieldValue::U8(0xAB)); + assert_eq!(engine.read_field(&buffer, "i16")?, FieldValue::I16(-32000)); + assert_eq!(engine.read_field(&buffer, "u16")?, FieldValue::U16(0xBEEF)); + assert_eq!( + engine.read_field(&buffer, "i32")?, + FieldValue::I32(-2_000_000_007) + ); + assert_eq!( + engine.read_field(&buffer, "u32")?, + FieldValue::U32(0xDEADBEEF) + ); + assert_eq!( + engine.read_field(&buffer, "i64")?, + FieldValue::I64(-9_000_000_000_000_000_000) + ); + assert_eq!( + engine.read_field(&buffer, "u64")?, + FieldValue::U64(0x0102030405060708) + ); + assert_eq!(engine.read_field(&buffer, "f32")?, FieldValue::F32(1.5)); + assert_eq!(engine.read_field(&buffer, "f64")?, FieldValue::F64(2.5)); + assert_eq!(engine.read_field(&buffer, "b")?, FieldValue::Bool(true)); + assert_eq!(engine.read_field(&buffer, "e")?, FieldValue::Enum(7)); + Ok(()) +} + +#[test] +fn read_write_aligned_round_trips_string() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { + "name": { "TypeDef:String": true } + } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned)?; + let offset_map = engine.offset_map().expect("aligned mode has offset_map"); + let mut buffer = vec![0u8; offset_map.total_size() + 64]; + engine.write_field(&mut buffer, "name", &FieldValue::String("hello world"))?; + assert_eq!( + engine.read_field(&buffer, "name")?, + FieldValue::String("hello world") + ); + Ok(()) +} + +#[test] +fn read_write_aligned_round_trips_bytes() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { + "blob": { "TypeDef:Bytes": true } + } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned)?; + let offset_map = engine.offset_map().expect("aligned mode has offset_map"); + let payload = b"the quick brown fox".to_vec(); + let mut buffer = vec![0u8; offset_map.total_size() + payload.len()]; + engine.write_field(&mut buffer, "blob", &FieldValue::Bytes(&payload))?; + assert_eq!( + engine.read_field(&buffer, "blob")?, + FieldValue::Bytes(&payload) + ); + Ok(()) +} + +#[test] +fn read_field_returns_access_error_in_packed_mode() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { "id": { "TypeDef:Uint32": true } } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Packed)?; + let buffer = [0u8; 4]; + let err = engine.read_field(&buffer, "id").unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); + Ok(()) +} + +#[test] +fn write_field_returns_access_error_in_packed_mode() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { "id": { "TypeDef:Uint32": true } } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Packed)?; + let mut buffer = [0u8; 4]; + let err = engine + .write_field(&mut buffer, "id", &FieldValue::U32(1)) + .unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); + Ok(()) +} + +#[test] +fn read_field_returns_offset_error_for_missing_path() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { "id": { "TypeDef:Uint32": true } } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned)?; + let buffer = [0u8; 8]; + let err = engine.read_field(&buffer, "missing").unwrap_err(); + assert!(matches!(err, TypedefError::Offset { .. }), "got {err:?}"); + Ok(()) +} + +#[test] +fn write_field_returns_offset_error_for_missing_path() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { "id": { "TypeDef:Uint32": true } } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned)?; + let mut buffer = [0u8; 8]; + let err = engine + .write_field(&mut buffer, "missing", &FieldValue::U32(1)) + .unwrap_err(); + assert!(matches!(err, TypedefError::Offset { .. }), "got {err:?}"); + Ok(()) +} + +#[test] +fn read_field_returns_access_error_for_composite_types() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { + "vals": { + "TypeDef:Array": true, + "items": { "TypeDef:Uint32": true } + } + } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned)?; + let buffer = [0u8; 8]; + let err = engine.read_field(&buffer, "vals").unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); + Ok(()) +} + +#[test] +fn write_field_returns_access_error_for_composite_value() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { "id": { "TypeDef:Uint32": true } } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned)?; + let mut buffer = [0u8; 8]; + let err = engine + .write_field(&mut buffer, "id", &FieldValue::Struct { start: 0, end: 4 }) + .unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); + Ok(()) +} + +#[test] +fn read_field_aligned_reads_nested_struct_byte_range() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { + "header": { + "TypeDef:Struct": true, + "properties": { + "version": { "TypeDef:Uint8": true }, + "magic": { "TypeDef:Uint32": true } + } + } + } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned)?; + let offset_map = engine.offset_map().expect("aligned mode"); + let mut buffer = vec![0u8; offset_map.total_size()]; + + engine.write_field(&mut buffer, "header.version", &FieldValue::U8(3))?; + engine.write_field(&mut buffer, "header.magic", &FieldValue::U32(0xCAFEBABE))?; + + assert_eq!( + engine.read_field(&buffer, "header.version")?, + FieldValue::U8(3) + ); + assert_eq!( + engine.read_field(&buffer, "header.magic")?, + FieldValue::U32(0xCAFEBABE) + ); + Ok(()) +} diff --git a/tests/error_paths.rs b/tests/error_paths.rs new file mode 100644 index 0000000..00ab38a --- /dev/null +++ b/tests/error_paths.rs @@ -0,0 +1,415 @@ +//! Error path integration tests for `alknet-typedef`. +//! +//! Exercises the `TypedefError` variants across the crate: +//! `Access` (buffer too short, invalid UTF-8, invalid boolean byte, +//! unknown discriminator value), `Schema` (missing TypeDef kind, +//! malformed discriminator annotation), and `Offset` (missing +//! variable-length field size in `LayoutBuilder::build`). + +use alknet_typedef::data_access; +use alknet_typedef::tunion; +use alknet_typedef::*; +use serde_json::json; +use std::collections::HashMap; + +#[test] +fn read_u32_buffer_too_short_returns_access_error() { + let buffer = [0u8; 2]; + let err = data_access::read_u32(&buffer, 0, "header.id", Endian::Little).unwrap_err(); + match err { + TypedefError::Access { field_path, reason } => { + assert_eq!(field_path, "header.id"); + assert!(reason.contains("bounds"), "reason: {reason}"); + } + other => panic!("expected Access, got {other:?}"), + } +} + +#[test] +fn read_u16_buffer_too_short_returns_access_error() { + let buffer = [0u8; 1]; + let err = data_access::read_u16(&buffer, 0, "tag", Endian::Little).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn read_u64_buffer_too_short_returns_access_error() { + let buffer = [0u8; 4]; + let err = data_access::read_u64(&buffer, 0, "offset", Endian::Big).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn read_f32_buffer_too_short_returns_access_error() { + let buffer = [0u8; 2]; + let err = data_access::read_f32(&buffer, 0, "score", Endian::Little).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn read_f64_buffer_too_short_returns_access_error() { + let buffer = [0u8; 4]; + let err = data_access::read_f64(&buffer, 0, "score", Endian::Little).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn read_i32_buffer_too_short_returns_access_error() { + let buffer = [0u8; 2]; + let err = data_access::read_i32(&buffer, 0, "id", Endian::Little).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn read_bool_buffer_too_short_returns_access_error() { + let buffer: [u8; 0] = []; + let err = data_access::read_bool(&buffer, 0, "flag").unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn read_string_buffer_too_short_on_prefix_returns_access_error() { + let buffer = [0u8; 2]; + let err = data_access::read_string(&buffer, 0, "name", Endian::Little).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn read_string_buffer_too_short_on_data_returns_access_error() { + let mut buffer = vec![0u8; 6]; + buffer[0..4].copy_from_slice(&100u32.to_le_bytes()); + let err = data_access::read_string(&buffer, 0, "name", Endian::Little).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn read_bytes_buffer_too_short_on_data_returns_access_error() { + let mut buffer = vec![0u8; 5]; + buffer[0..4].copy_from_slice(&100u32.to_le_bytes()); + let err = data_access::read_bytes(&buffer, 0, "blob", Endian::Little).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn read_string_invalid_utf8_returns_access_error() { + let mut buffer = vec![0u8; 16]; + let invalid = [0xFFu8, 0xFE, 0xFD]; + let _ = data_access::write_bytes(&mut buffer, 0, &invalid, "name", Endian::Little); + let err = data_access::read_string(&buffer, 0, "name", Endian::Little).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn read_bool_invalid_byte_returns_access_error() { + let buffer = [0x02u8]; + let err = data_access::read_bool(&buffer, 0, "flag").unwrap_err(); + match err { + TypedefError::Access { field_path, reason } => { + assert_eq!(field_path, "flag"); + assert!(reason.contains("0x02"), "reason: {reason}"); + } + other => panic!("expected Access, got {other:?}"), + } +} + +#[test] +fn read_bool_zero_is_false() -> Result<(), TypedefError> { + let buffer = [0x00u8]; + assert!(!data_access::read_bool(&buffer, 0, "flag")?); + Ok(()) +} + +#[test] +fn read_bool_one_is_true() -> Result<(), TypedefError> { + let buffer = [0x01u8]; + assert!(data_access::read_bool(&buffer, 0, "flag")?); + Ok(()) +} + +#[test] +fn read_bool_three_is_access_error() { + let buffer = [0x03u8]; + let err = data_access::read_bool(&buffer, 0, "flag").unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn write_u32_buffer_too_short_returns_access_error() { + let mut buffer = [0u8; 2]; + let err = data_access::write_u32(&mut buffer, 0, 1, "id", Endian::Little).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn write_string_buffer_too_short_returns_access_error() { + let mut buffer = vec![0u8; 4]; + let err = + data_access::write_string(&mut buffer, 0, "hello", "name", Endian::Little).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn compile_missing_typedef_kind_returns_schema_error() { + let mut schema = json!({ "type": "object", "properties": {} }); + let err = TypedefEngine::compile(&mut schema, LayoutMode::Aligned).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); +} + +#[test] +fn offset_map_compute_missing_typedef_kind_returns_schema_error() { + let schema = json!({ "type": "object", "properties": {} }); + let err = OffsetMap::compute(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); +} + +#[test] +fn offset_map_compute_non_struct_top_level_returns_schema_error() { + let schema = json!({ "TypeDef:Uint32": true }); + let err = OffsetMap::compute(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); +} + +#[test] +fn layout_builder_new_missing_typedef_kind_returns_schema_error() { + let schema = json!({ "type": "object", "properties": {} }); + let err = LayoutBuilder::new(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); +} + +#[test] +fn layout_builder_new_non_struct_top_level_returns_schema_error() { + let schema = json!({ "TypeDef:Uint32": true }); + let err = LayoutBuilder::new(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); +} + +#[test] +fn parse_discriminator_missing_returns_schema_error() { + let schema = json!({"TypeDef:Union": true}); + let err = parse_discriminator(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); +} + +#[test] +fn parse_discriminator_field_missing_name_returns_schema_error() { + let schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "field"} + }); + let err = parse_discriminator(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); +} + +#[test] +fn parse_discriminator_unknown_kind_returns_schema_error() { + let schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "magic"} + }); + let err = parse_discriminator(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); +} + +#[test] +fn parse_discriminator_byte_invalid_type_returns_schema_error() { + let schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "byte", "type": "TypeDef:Float32"} + }); + let err = parse_discriminator(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); +} + +#[test] +fn read_byte_discriminator_unknown_value_returns_access_error() -> Result<(), TypedefError> { + let union_schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "byte", "type": "TypeDef:Uint8"}, + "mapping": {"5": {"TypeDef:Struct": true, "properties": {"x": {"TypeDef:Uint8": true}}}} + }); + let buffer = [99u8, 0x00, 0x00]; + let err = tunion::read_byte_discriminator(&buffer, &union_schema, Endian::Little).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); + Ok(()) +} + +#[test] +fn read_byte_discriminator_buffer_too_short_returns_access_error() -> Result<(), TypedefError> { + let union_schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "byte", "offset": 4, "type": "TypeDef:Uint32"}, + "mapping": {"5": {"TypeDef:Struct": true, "properties": {"x": {"TypeDef:Uint8": true}}}} + }); + let buffer = [0u8; 2]; + let err = tunion::read_byte_discriminator(&buffer, &union_schema, Endian::Little).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); + Ok(()) +} + +#[test] +fn read_field_discriminator_unknown_value_returns_access_error() -> Result<(), TypedefError> { + let union_schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "field", "name": "type"}, + "properties": {"type": {"TypeDef:Uint8": true}}, + "mapping": {"0": {"TypeDef:Struct": true, "properties": {"x": {"TypeDef:Uint8": true}}}} + }); + let mut buffer = vec![0u8; 8]; + buffer[0] = 99; + let err = + tunion::read_field_discriminator(&buffer, &union_schema, 0, Endian::Little).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); + Ok(()) +} + +#[test] +fn layout_builder_missing_var_size_returns_offset_error() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "name": { "TypeDef:String": true } + } + }); + let builder = LayoutBuilder::new(&schema).expect("builder"); + let empty: HashMap = HashMap::new(); + let err = builder.build(&empty).unwrap_err(); + match err { + TypedefError::Offset { field_path, reason } => { + assert_eq!(field_path, "name"); + assert!( + reason.contains("missing variable-length field size"), + "reason: {reason}" + ); + } + other => panic!("expected Offset, got {other:?}"), + } +} + +#[test] +fn layout_builder_missing_array_data_size_returns_offset_error() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "vals": { + "TypeDef:Array": true, + "items": { "TypeDef:Uint32": true } + } + } + }); + let builder = LayoutBuilder::new(&schema).expect("builder"); + let empty: HashMap = HashMap::new(); + let err = builder.build(&empty).unwrap_err(); + assert!(matches!(err, TypedefError::Offset { .. }), "got {err:?}"); +} + +#[test] +fn layout_builder_missing_discriminator_value_returns_offset_error() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "payload": { + "TypeDef:Union": true, + "discriminator": {"kind": "byte", "type": "TypeDef:Uint8"}, + "mapping": {"5": {"$ref": "#/$defs/Read"}} + } + }, + "$defs": { + "Read": {"TypeDef:Struct": true, "properties": {"x": {"TypeDef:Uint8": true}}} + } + }); + let builder = LayoutBuilder::new(&schema).expect("builder"); + let empty: HashMap = HashMap::new(); + let err = builder.build(&empty).unwrap_err(); + assert!(matches!(err, TypedefError::Offset { .. }), "got {err:?}"); +} + +#[test] +fn layout_builder_unknown_discriminator_value_returns_offset_error() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "payload": { + "TypeDef:Union": true, + "discriminator": {"kind": "byte", "type": "TypeDef:Uint8"}, + "mapping": {"5": {"$ref": "#/$defs/Read"}} + } + }, + "$defs": { + "Read": {"TypeDef:Struct": true, "properties": {"x": {"TypeDef:Uint8": true}}} + } + }); + let builder = LayoutBuilder::new(&schema).expect("builder"); + let mut vs = HashMap::new(); + vs.insert("payload.__discriminator".to_string(), 99); + let err = builder.build(&vs).unwrap_err(); + match err { + TypedefError::Offset { reason, .. } => { + assert!(reason.contains("99"), "reason: {reason}"); + } + other => panic!("expected Offset, got {other:?}"), + } +} + +#[test] +fn sequential_reader_buffer_too_short_returns_access_error() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "id": { "TypeDef:Uint32": true } + } + }); + let buffer = [0u8; 2]; + let mut reader = SequentialReader::new(&schema).unwrap(); + let err = reader.read_next(&buffer).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn sequential_reader_unknown_field_returns_schema_error() { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { "a": { "TypeDef:Uint8": true } } + }); + let buffer = [0u8; 4]; + let mut reader = SequentialReader::new(&schema).unwrap(); + let err = reader.read_field(&buffer, "missing").unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); +} + +#[test] +fn sequential_reader_new_non_struct_returns_schema_error() { + let schema = json!({ "TypeDef:Uint32": true }); + let err = SequentialReader::new(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); +} + +#[test] +fn read_string_indirect_data_region_too_short_returns_access_error() { + let mut index = [0u8; 8]; + let _ = data_access::write_u32(&mut index, 0, 100, "idx.off", Endian::Little); + let _ = data_access::write_u32(&mut index, 4, 10, "idx.len", Endian::Little); + let data_region = b"too short"; + let err = data_access::read_bytes_indirect(&index, 0, data_region, "blob", Endian::Little) + .unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn read_bytes_indirect_index_too_short_returns_access_error() { + let buffer = [0u8; 4]; + let data_region = b"anything"; + let err = data_access::read_bytes_indirect(&buffer, 0, data_region, "blob", Endian::Little) + .unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} + +#[test] +fn read_string_indirect_invalid_utf8_returns_access_error() { + let data_region: &[u8] = &[0xFF, 0xFE, 0xFD]; + let mut index = [0u8; 8]; + let _ = data_access::write_u32(&mut index, 0, 0, "idx.off", Endian::Little); + let _ = data_access::write_u32(&mut index, 4, 3, "idx.len", Endian::Little); + let err = data_access::read_string_indirect(&index, 0, data_region, "name", Endian::Little) + .unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); +} diff --git a/tests/poc_roundtrip.rs b/tests/poc_roundtrip.rs new file mode 100644 index 0000000..0e0cabf --- /dev/null +++ b/tests/poc_roundtrip.rs @@ -0,0 +1,535 @@ +//! POC round-trip tests adapted from `/workspace/alknet-typedef-poc/`. +//! +//! These tests re-validate the byte-identical round-trip behaviour that +//! the POC verified: fixed-size primitives, length-prefixed strings and +//! bytes, nested structs, big-endian, alignment padding, packed-layout +//! `LayoutBuilder` with `data_access` writes, and `SequentialReader` +//! walks. Each test writes values to a buffer at computed offsets and +//! reads them back, asserting both the values and (where applicable) +//! the byte positions. + +use alknet_typedef::data_access; +use alknet_typedef::tunion; +use alknet_typedef::*; +use serde_json::json; +use std::collections::HashMap; + +fn var_sizes(pairs: &[(&str, usize)]) -> HashMap { + pairs.iter().map(|(k, v)| (k.to_string(), *v)).collect() +} + +#[test] +fn fixed_size_round_trip_via_offset_map() -> Result<(), TypedefError> { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "id": { "TypeDef:Uint32": true }, + "score": { "TypeDef:Float32": true }, + "flag": { "TypeDef:Uint8": true }, + "count": { "TypeDef:Uint16": true } + } + }); + let offset_map = OffsetMap::compute(&schema)?; + let mut buffer = vec![0u8; offset_map.total_size()]; + + let id_range = offset_map.get("id").expect("id range"); + data_access::write_u32(&mut buffer, id_range.start, 42, "id", Endian::Little)?; + let score_range = offset_map.get("score").expect("score range"); + data_access::write_f32(&mut buffer, score_range.start, 1.5, "score", Endian::Little)?; + let flag_range = offset_map.get("flag").expect("flag range"); + data_access::write_u8(&mut buffer, flag_range.start, 1, "flag")?; + let count_range = offset_map.get("count").expect("count range"); + data_access::write_u16( + &mut buffer, + count_range.start, + 1000, + "count", + Endian::Little, + )?; + + assert_eq!( + data_access::read_u32(&buffer, id_range.start, "id", Endian::Little)?, + 42 + ); + let score = data_access::read_f32(&buffer, score_range.start, "score", Endian::Little)?; + assert!((score - 1.5).abs() < 0.001, "score: {score}"); + assert_eq!(data_access::read_u8(&buffer, flag_range.start, "flag")?, 1); + assert_eq!( + data_access::read_u16(&buffer, count_range.start, "count", Endian::Little)?, + 1000 + ); + Ok(()) +} + +#[test] +fn fixed_size_round_trip_via_engine_aligned() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "endian": "little", + "properties": { + "id": { "TypeDef:Uint32": true }, + "score": { "TypeDef:Float32": true }, + "flag": { "TypeDef:Uint8": true } + } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned)?; + let offset_map = engine.offset_map().expect("aligned mode has offset_map"); + let mut buffer = vec![0u8; offset_map.total_size()]; + + engine.write_field(&mut buffer, "id", &FieldValue::U32(42))?; + engine.write_field(&mut buffer, "score", &FieldValue::F32(1.5))?; + engine.write_field(&mut buffer, "flag", &FieldValue::U8(1))?; + + assert_eq!(engine.read_field(&buffer, "id")?, FieldValue::U32(42)); + let score = match engine.read_field(&buffer, "score")? { + FieldValue::F32(f) => f, + other => panic!("expected F32, got {other:?}"), + }; + assert!((score - 1.5).abs() < 0.001); + assert_eq!(engine.read_field(&buffer, "flag")?, FieldValue::U8(1)); + Ok(()) +} + +#[test] +fn string_round_trip_via_data_access() -> Result<(), TypedefError> { + let mut buffer = vec![0u8; 32]; + let written = data_access::write_string(&mut buffer, 0, "hello", "name", Endian::Little)?; + assert_eq!(written, 4 + 5); + assert_eq!(buffer[0..4], 5u32.to_le_bytes()); + assert_eq!(&buffer[4..9], b"hello"); + assert_eq!( + data_access::read_string(&buffer, 0, "name", Endian::Little)?, + "hello" + ); + Ok(()) +} + +#[test] +fn string_round_trip_via_engine_aligned() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { + "name": { "TypeDef:String": true } + } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned)?; + let offset_map = engine.offset_map().expect("aligned mode has offset_map"); + let mut buffer = vec![0u8; offset_map.total_size() + 64]; + engine.write_field(&mut buffer, "name", &FieldValue::String("hello"))?; + assert_eq!( + engine.read_field(&buffer, "name")?, + FieldValue::String("hello") + ); + Ok(()) +} + +#[test] +fn bytes_round_trip_via_data_access() -> Result<(), TypedefError> { + let payload = [0xAA, 0xBB, 0xCC, 0xDD]; + let mut buffer = vec![0u8; 32]; + let written = data_access::write_bytes(&mut buffer, 0, &payload, "data", Endian::Little)?; + assert_eq!(written, 4 + 4); + assert_eq!(buffer[0..4], 4u32.to_le_bytes()); + assert_eq!(&buffer[4..8], &payload); + assert_eq!( + data_access::read_bytes(&buffer, 0, "data", Endian::Little)?, + &payload[..] + ); + Ok(()) +} + +#[test] +fn nested_struct_round_trip_via_offset_map() -> Result<(), TypedefError> { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "header": { + "TypeDef:Struct": true, + "properties": { + "version": { "TypeDef:Uint32": true }, + "magic": { "TypeDef:Uint32": true } + } + }, + "payload": { "TypeDef:Bytes": true } + } + }); + let offset_map = OffsetMap::compute(&schema)?; + + let header_version = offset_map.get("header.version").expect("header.version"); + let header_magic = offset_map.get("header.magic").expect("header.magic"); + let payload_prefix = offset_map.get("payload").expect("payload"); + + assert_eq!(header_version.start, 0); + assert_eq!(header_magic.start, 4); + assert_eq!(payload_prefix.start, 8); + + let data = b"body-data".to_vec(); + let mut buffer = vec![0u8; offset_map.total_size() + data.len()]; + data_access::write_u32( + &mut buffer, + header_version.start, + 1, + "header.version", + Endian::Little, + )?; + data_access::write_u32( + &mut buffer, + header_magic.start, + 0xCAFEBABE, + "header.magic", + Endian::Little, + )?; + data_access::write_bytes( + &mut buffer, + payload_prefix.start, + &data, + "payload", + Endian::Little, + )?; + + assert_eq!( + data_access::read_u32( + &buffer, + header_version.start, + "header.version", + Endian::Little + )?, + 1 + ); + assert_eq!( + data_access::read_u32(&buffer, header_magic.start, "header.magic", Endian::Little)?, + 0xCAFEBABE + ); + assert_eq!( + data_access::read_bytes(&buffer, payload_prefix.start, "payload", Endian::Little)?, + &data[..] + ); + Ok(()) +} + +#[test] +fn nested_struct_round_trip_via_engine_aligned() -> Result<(), TypedefError> { + let mut schema = json!({ + "TypeDef:Struct": true, + "properties": { + "header": { + "TypeDef:Struct": true, + "properties": { + "version": { "TypeDef:Uint8": true }, + "flags": { "TypeDef:Uint8": true } + } + }, + "payload_len": { "TypeDef:Uint32": true } + } + }); + let engine = TypedefEngine::compile(&mut schema, LayoutMode::Aligned)?; + let offset_map = engine.offset_map().expect("aligned mode"); + + assert_eq!(offset_map.get("header.version").unwrap().start, 0); + assert_eq!(offset_map.get("header.flags").unwrap().start, 1); + assert_eq!(offset_map.get("payload_len").unwrap().start, 4); + + let mut buffer = vec![0u8; offset_map.total_size()]; + engine.write_field(&mut buffer, "header.version", &FieldValue::U8(1))?; + engine.write_field(&mut buffer, "header.flags", &FieldValue::U8(0x0F))?; + engine.write_field(&mut buffer, "payload_len", &FieldValue::U32(1024))?; + + assert_eq!( + engine.read_field(&buffer, "header.version")?, + FieldValue::U8(1) + ); + assert_eq!( + engine.read_field(&buffer, "header.flags")?, + FieldValue::U8(0x0F) + ); + assert_eq!( + engine.read_field(&buffer, "payload_len")?, + FieldValue::U32(1024) + ); + Ok(()) +} + +#[test] +fn big_endian_round_trip_via_offset_map() -> Result<(), TypedefError> { + let schema = json!({ + "TypeDef:Struct": true, + "endian": "big", + "properties": { + "id": { "TypeDef:Uint32": true }, + "offset": { "TypeDef:Float64": true } + } + }); + let offset_map = OffsetMap::compute(&schema)?; + let endian = Endian::from_schema(&schema); + assert_eq!(endian, Endian::Big); + + let id_range = offset_map.get("id").expect("id"); + let offset_range = offset_map.get("offset").expect("offset"); + + assert_eq!(id_range.start, 0); + assert_eq!(offset_range.start, 8); + + let value: f64 = std::f64::consts::PI; + let mut buffer = vec![0u8; offset_map.total_size()]; + data_access::write_u32(&mut buffer, id_range.start, 0x01020304, "id", endian)?; + data_access::write_f64(&mut buffer, offset_range.start, value, "offset", endian)?; + + assert_eq!(&buffer[0..4], &[0x01, 0x02, 0x03, 0x04]); + assert_eq!(&buffer[4..8], &[0x00, 0x00, 0x00, 0x00]); + assert_eq!(&buffer[8..16], value.to_be_bytes()); + + assert_eq!( + data_access::read_u32(&buffer, id_range.start, "id", endian)?, + 0x01020304 + ); + let read = data_access::read_f64(&buffer, offset_range.start, "offset", endian)?; + assert!((read - value).abs() < 1e-12); + Ok(()) +} + +#[test] +fn alignment_padding_round_trip_u8_then_u32() -> Result<(), TypedefError> { + let schema = json!({ + "TypeDef:Struct": true, + "properties": { + "flag": { "TypeDef:Uint8": true }, + "id": { "TypeDef:Uint32": true } + } + }); + let offset_map = OffsetMap::compute(&schema)?; + + let flag_range = offset_map.get("flag").expect("flag"); + let id_range = offset_map.get("id").expect("id"); + + assert_eq!(flag_range.start, 0); + assert_eq!(flag_range.end, 1); + assert_eq!(id_range.start, 4); + assert_eq!(id_range.end, 8); + assert_eq!(offset_map.total_size(), 8); + + let mut buffer = vec![0u8; offset_map.total_size()]; + data_access::write_u8(&mut buffer, flag_range.start, 0xAB, "flag")?; + data_access::write_u32( + &mut buffer, + id_range.start, + 0x01020304, + "id", + Endian::Little, + )?; + + assert_eq!(buffer[0], 0xAB); + assert_eq!(&buffer[1..4], &[0x00, 0x00, 0x00]); + assert_eq!(&buffer[4..8], 0x01020304u32.to_le_bytes()); + + assert_eq!( + data_access::read_u8(&buffer, flag_range.start, "flag")?, + 0xAB + ); + assert_eq!( + data_access::read_u32(&buffer, id_range.start, "id", Endian::Little)?, + 0x01020304 + ); + Ok(()) +} + +#[test] +fn packed_layout_round_trip_via_layout_builder() -> Result<(), TypedefError> { + let schema = json!({ + "TypeDef:Struct": true, + "endian": "little", + "properties": { + "flag": { "TypeDef:Uint8": true }, + "id": { "TypeDef:Uint32": true }, + "payload": { "TypeDef:String": true } + } + }); + let builder = LayoutBuilder::new(&schema)?; + let layout = builder.build(&var_sizes(&[("payload", 10)]))?; + + let flag_pos = layout.get("flag").expect("flag"); + let id_pos = layout.get("id").expect("id"); + let payload_pos = layout.get("payload").expect("payload"); + + assert_eq!(flag_pos.offset, 0); + assert_eq!(id_pos.offset, 1); + assert_eq!(payload_pos.offset, 5); + assert_eq!(layout.total_size(), 19); + + let payload_str = "ten bytes!"; + let payload_bytes = payload_str.as_bytes(); + assert_eq!(payload_bytes.len(), 10); + let mut buffer = vec![0u8; layout.total_size()]; + data_access::write_u8(&mut buffer, flag_pos.offset, 0xAB, "flag")?; + data_access::write_u32(&mut buffer, id_pos.offset, 0x01020304, "id", Endian::Little)?; + data_access::write_string( + &mut buffer, + payload_pos.offset, + payload_str, + "payload", + Endian::Little, + )?; + + assert_eq!(buffer[0], 0xAB); + assert_eq!(&buffer[1..5], 0x01020304u32.to_le_bytes()); + assert_eq!(&buffer[5..9], 10u32.to_le_bytes()); + assert_eq!(&buffer[9..19], payload_bytes); + + assert_eq!( + data_access::read_u8(&buffer, flag_pos.offset, "flag")?, + 0xAB + ); + assert_eq!( + data_access::read_u32(&buffer, id_pos.offset, "id", Endian::Little)?, + 0x01020304 + ); + assert_eq!( + data_access::read_string(&buffer, payload_pos.offset, "payload", Endian::Little)?, + payload_str + ); + Ok(()) +} + +#[test] +fn sequential_reader_round_trip_packed_buffer() -> Result<(), TypedefError> { + let schema = json!({ + "TypeDef:Struct": true, + "endian": "little", + "properties": { + "id": { "TypeDef:Uint8": true }, + "name": { "TypeDef:String": true }, + "tail": { "TypeDef:Uint8": true } + } + }); + let builder = LayoutBuilder::new(&schema)?; + let payload = "hello"; + let layout = builder.build(&var_sizes(&[("name", payload.len())]))?; + + let mut buffer = vec![0u8; layout.total_size()]; + data_access::write_u8(&mut buffer, 0, 7, "id")?; + data_access::write_string(&mut buffer, 1, payload, "name", Endian::Little)?; + let after = 1 + 4 + payload.len(); + data_access::write_u8(&mut buffer, after, 99, "tail")?; + + let mut reader = SequentialReader::new(&schema)?; + assert_eq!(reader.endian(), Endian::Little); + assert_eq!(reader.position(), 0); + + let (name, value) = reader.read_next(&buffer)?.expect("field 0"); + assert_eq!(name, "id"); + assert_eq!(value, FieldValue::U8(7)); + assert_eq!(reader.position(), 1); + + let (name, value) = reader.read_next(&buffer)?.expect("field 1"); + assert_eq!(name, "name"); + assert_eq!(value, FieldValue::String("hello")); + assert_eq!(reader.position(), after); + + let (name, value) = reader.read_next(&buffer)?.expect("field 2"); + assert_eq!(name, "tail"); + assert_eq!(value, FieldValue::U8(99)); + assert_eq!(reader.position(), after + 1); + + assert!(reader.read_next(&buffer)?.is_none()); + Ok(()) +} + +#[test] +fn sequential_reader_read_field_walks_preceding_fields() -> Result<(), TypedefError> { + let schema = json!({ + "TypeDef:Struct": true, + "endian": "little", + "properties": { + "a": { "TypeDef:Uint8": true }, + "b": { "TypeDef:Uint32": true }, + "c": { "TypeDef:Uint8": true } + } + }); + let mut buffer = vec![0u8; 16]; + data_access::write_u8(&mut buffer, 0, 1, "a")?; + data_access::write_u32(&mut buffer, 1, 0xDEADBEEF, "b", Endian::Little)?; + data_access::write_u8(&mut buffer, 5, 9, "c")?; + + let mut reader = SequentialReader::new(&schema)?; + let value = reader.read_field(&buffer, "c")?; + assert_eq!(value, FieldValue::U8(9)); + assert_eq!(reader.position(), 6); + + reader.reset(); + let value = reader.read_field(&buffer, "b")?; + assert_eq!(value, FieldValue::U32(0xDEADBEEF)); + Ok(()) +} + +#[test] +fn tunion_byte_offset_discriminator_dispatch() -> Result<(), TypedefError> { + let union_schema = json!({ + "TypeDef:Union": true, + "discriminator": { + "kind": "byte", + "offset": 0, + "type": "TypeDef:Uint8" + }, + "mapping": { + "5": { "$ref": "#/$defs/Read" }, + "6": { "$ref": "#/$defs/Write" } + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": { + "handle": { "TypeDef:Uint32": true }, + "length": { "TypeDef:Uint32": true } + } + }, + "Write": { + "TypeDef:Struct": true, + "properties": { + "handle": { "TypeDef:Uint32": true }, + "length": { "TypeDef:Uint32": true }, + "data": { "TypeDef:Uint32": true } + } + } + } + }); + let mut buffer = vec![0u8; 32]; + buffer[0] = 5; + data_access::write_u32(&mut buffer, 1, 0x01020304, "Read.handle", Endian::Big)?; + data_access::write_u32(&mut buffer, 5, 4096, "Read.length", Endian::Big)?; + + let dispatch = tunion::read_byte_discriminator(&buffer, &union_schema, Endian::Big)?; + assert_eq!(dispatch.key, "5"); + assert_eq!(dispatch.variant_offset, 1); + assert_eq!(dispatch.discriminator_size, 1); + + let variant = tunion::resolve_variant(&union_schema, &dispatch.key)?; + assert_eq!( + variant + .get("TypeDef:Struct") + .and_then(serde_json::Value::as_bool), + Some(true) + ); + Ok(()) +} + +#[test] +fn tunion_byte_offset_discriminator_size_lookup() -> Result<(), TypedefError> { + let u8_schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "byte", "type": "TypeDef:Uint8"}, + "mapping": {} + }); + let u16_schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "byte", "type": "TypeDef:Uint16"}, + "mapping": {} + }); + let u32_schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "byte", "type": "TypeDef:Uint32"}, + "mapping": {} + }); + assert_eq!(tunion::discriminator_size(&u8_schema)?, 1); + assert_eq!(tunion::discriminator_size(&u16_schema)?, 2); + assert_eq!(tunion::discriminator_size(&u32_schema)?, 4); + Ok(()) +} diff --git a/tests/tunion_dispatch.rs b/tests/tunion_dispatch.rs new file mode 100644 index 0000000..14cfbd4 --- /dev/null +++ b/tests/tunion_dispatch.rs @@ -0,0 +1,323 @@ +//! Integration tests for TUnion discriminator dispatch. +//! +//! Exercises both discriminator kinds end-to-end: byte-offset (SFTP +//! pattern) and field-name (typedef.ts pattern). Verifies that +//! `read_byte_discriminator` / `read_field_discriminator` produce the +//! correct mapping key and variant offset, that `resolve_variant` +//! follows `$ref` pointers, and that `discriminator_size` reports the +//! right fixed sizes. + +use alknet_typedef::data_access; +use alknet_typedef::tunion; +use alknet_typedef::{Endian, TypedefError}; +use serde_json::json; + +fn sftp_like_byte_union() -> serde_json::Value { + json!({ + "TypeDef:Union": true, + "discriminator": { + "kind": "byte", + "offset": 0, + "type": "TypeDef:Uint8" + }, + "mapping": { + "5": { "$ref": "#/$defs/Read" }, + "6": { "$ref": "#/$defs/Write" } + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": { + "handle": { "TypeDef:Uint32": true }, + "length": { "TypeDef:Uint32": true } + } + }, + "Write": { + "TypeDef:Struct": true, + "properties": { + "handle": { "TypeDef:Uint32": true }, + "length": { "TypeDef:Uint32": true }, + "data": { "TypeDef:Uint32": true } + } + } + } + }) +} + +#[test] +fn read_byte_discriminator_uint8_dispatches_to_read() -> Result<(), TypedefError> { + let union_schema = sftp_like_byte_union(); + let mut buffer = vec![0u8; 16]; + buffer[0] = 5; + data_access::write_u32(&mut buffer, 1, 0x01020304, "Read.handle", Endian::Big)?; + + let dispatch = tunion::read_byte_discriminator(&buffer, &union_schema, Endian::Big)?; + assert_eq!(dispatch.key, "5"); + assert_eq!(dispatch.variant_offset, 1); + assert_eq!(dispatch.discriminator_size, 1); + + let variant = tunion::resolve_variant(&union_schema, &dispatch.key)?; + assert_eq!( + variant + .get("TypeDef:Struct") + .and_then(serde_json::Value::as_bool), + Some(true) + ); + Ok(()) +} + +#[test] +fn read_byte_discriminator_uint8_dispatches_to_write() -> Result<(), TypedefError> { + let union_schema = sftp_like_byte_union(); + let mut buffer = vec![0u8; 16]; + buffer[0] = 6; + data_access::write_u32(&mut buffer, 1, 0xDEADBEEF, "Write.handle", Endian::Big)?; + + let dispatch = tunion::read_byte_discriminator(&buffer, &union_schema, Endian::Big)?; + assert_eq!(dispatch.key, "6"); + assert_eq!(dispatch.variant_offset, 1); + assert_eq!(dispatch.discriminator_size, 1); + + let variant = tunion::resolve_variant(&union_schema, &dispatch.key)?; + let props = variant + .get("properties") + .and_then(serde_json::Value::as_object) + .expect("variant has properties"); + assert!(props.contains_key("data")); + Ok(()) +} + +#[test] +fn read_byte_discriminator_uint16_little_endian() -> Result<(), TypedefError> { + let schema = json!({ + "TypeDef:Union": true, + "discriminator": { + "kind": "byte", + "offset": 2, + "type": "TypeDef:Uint16" + }, + "mapping": { + "5": {"TypeDef:Struct": true, "properties": {"id": {"TypeDef:Uint32": true}}} + } + }); + let mut buffer = vec![0u8; 16]; + buffer[2..4].copy_from_slice(&5u16.to_le_bytes()); + let dispatch = tunion::read_byte_discriminator(&buffer, &schema, Endian::Little)?; + assert_eq!(dispatch.key, "5"); + assert_eq!(dispatch.variant_offset, 4); + assert_eq!(dispatch.discriminator_size, 2); + Ok(()) +} + +#[test] +fn read_byte_discriminator_uint32_big_endian() -> Result<(), TypedefError> { + let schema = json!({ + "TypeDef:Union": true, + "discriminator": { + "kind": "byte", + "offset": 0, + "type": "TypeDef:Uint32" + }, + "mapping": { + "101": {"TypeDef:Struct": true, "properties": {"id": {"TypeDef:Uint32": true}}} + } + }); + let mut buffer = vec![0u8; 16]; + buffer[0..4].copy_from_slice(&101u32.to_be_bytes()); + let dispatch = tunion::read_byte_discriminator(&buffer, &schema, Endian::Big)?; + assert_eq!(dispatch.key, "101"); + assert_eq!(dispatch.variant_offset, 4); + assert_eq!(dispatch.discriminator_size, 4); + Ok(()) +} + +#[test] +fn read_byte_discriminator_unknown_value_returns_access_error() -> Result<(), TypedefError> { + let union_schema = sftp_like_byte_union(); + let buffer = [99u8, 0x00, 0x00, 0x00]; + let err = tunion::read_byte_discriminator(&buffer, &union_schema, Endian::Big).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); + Ok(()) +} + +#[test] +fn read_field_discriminator_string_dispatches_to_read() -> Result<(), TypedefError> { + let union_schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "field", "name": "type"}, + "properties": { + "type": { "TypeDef:String": true } + }, + "mapping": { + "read": {"$ref": "#/$defs/Read"}, + "write": {"$ref": "#/$defs/Write"} + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": { + "handle": { "TypeDef:Uint32": true }, + "length": { "TypeDef:Uint32": true } + } + }, + "Write": { + "TypeDef:Struct": true, + "properties": { + "handle": { "TypeDef:Uint32": true }, + "data": { "TypeDef:Bytes": true } + } + } + } + }); + let value = "read"; + let mut buffer = vec![0u8; 32]; + data_access::write_string(&mut buffer, 0, value, "type", Endian::Little)?; + let dispatch = tunion::read_field_discriminator(&buffer, &union_schema, 0, Endian::Little)?; + assert_eq!(dispatch.key, "read"); + assert_eq!(dispatch.variant_offset, 4 + value.len()); + assert_eq!(dispatch.discriminator_size, 4 + value.len()); + + let variant = tunion::resolve_variant(&union_schema, &dispatch.key)?; + assert_eq!( + variant + .get("TypeDef:Struct") + .and_then(serde_json::Value::as_bool), + Some(true) + ); + Ok(()) +} + +#[test] +fn read_field_discriminator_string_dispatches_to_write() -> Result<(), TypedefError> { + let union_schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "field", "name": "type"}, + "properties": { + "type": { "TypeDef:String": true } + }, + "mapping": { + "read": {"$ref": "#/$defs/Read"}, + "write": {"$ref": "#/$defs/Write"} + }, + "$defs": { + "Read": { + "TypeDef:Struct": true, + "properties": {"x": {"TypeDef:Uint8": true}} + }, + "Write": { + "TypeDef:Struct": true, + "properties": {"y": {"TypeDef:Uint16": true}} + } + } + }); + let value = "write"; + let mut buffer = vec![0u8; 32]; + data_access::write_string(&mut buffer, 0, value, "type", Endian::Little)?; + let dispatch = tunion::read_field_discriminator(&buffer, &union_schema, 0, Endian::Little)?; + assert_eq!(dispatch.key, "write"); + assert_eq!(dispatch.variant_offset, 4 + value.len()); + + let variant = tunion::resolve_variant(&union_schema, &dispatch.key)?; + let props = variant + .get("properties") + .and_then(serde_json::Value::as_object) + .expect("variant has properties"); + assert!(props.contains_key("y")); + assert!(!props.contains_key("x")); + Ok(()) +} + +#[test] +fn read_field_discriminator_uint8_field() -> Result<(), TypedefError> { + let union_schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "field", "name": "tag"}, + "properties": { + "tag": { "TypeDef:Uint8": true } + }, + "mapping": { + "0": {"TypeDef:Struct": true, "properties": {"a": {"TypeDef:Uint32": true}}}, + "1": {"TypeDef:Struct": true, "properties": {"b": {"TypeDef:Uint16": true}}} + } + }); + let mut buffer = vec![0u8; 8]; + buffer[0] = 0; + let dispatch = tunion::read_field_discriminator(&buffer, &union_schema, 0, Endian::Little)?; + assert_eq!(dispatch.key, "0"); + assert_eq!(dispatch.variant_offset, 1); + assert_eq!(dispatch.discriminator_size, 1); + + buffer[0] = 1; + let dispatch = tunion::read_field_discriminator(&buffer, &union_schema, 0, Endian::Little)?; + assert_eq!(dispatch.key, "1"); + Ok(()) +} + +#[test] +fn read_field_discriminator_unknown_value_returns_access_error() -> Result<(), TypedefError> { + let union_schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "field", "name": "tag"}, + "properties": { + "tag": { "TypeDef:Uint8": true } + }, + "mapping": { + "0": {"TypeDef:Struct": true, "properties": {"a": {"TypeDef:Uint32": true}}} + } + }); + let mut buffer = vec![0u8; 8]; + buffer[0] = 99; + let err = + tunion::read_field_discriminator(&buffer, &union_schema, 0, Endian::Little).unwrap_err(); + assert!(matches!(err, TypedefError::Access { .. }), "got {err:?}"); + Ok(()) +} + +#[test] +fn discriminator_size_returns_correct_values() -> Result<(), TypedefError> { + let u8_schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "byte", "type": "TypeDef:Uint8"}, + "mapping": {} + }); + let u16_schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "byte", "type": "TypeDef:Uint16"}, + "mapping": {} + }); + let u32_schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "byte", "type": "TypeDef:Uint32"}, + "mapping": {} + }); + assert_eq!(tunion::discriminator_size(&u8_schema)?, 1); + assert_eq!(tunion::discriminator_size(&u16_schema)?, 2); + assert_eq!(tunion::discriminator_size(&u32_schema)?, 4); + Ok(()) +} + +#[test] +fn discriminator_size_field_kind_returns_schema_error() { + let schema = json!({ + "TypeDef:Union": true, + "discriminator": {"kind": "field", "name": "type"}, + "properties": {"type": {"TypeDef:Uint8": true}}, + "mapping": {} + }); + let err = tunion::discriminator_size(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); +} + +#[test] +fn resolve_variant_returns_schema_error_for_unknown_key() { + let union_schema = sftp_like_byte_union(); + let err = tunion::resolve_variant(&union_schema, "999").unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); +} + +#[test] +fn parse_discriminator_missing_returns_schema_error() { + let schema = json!({"TypeDef:Union": true}); + let err = alknet_typedef::parse_discriminator(&schema).unwrap_err(); + assert!(matches!(err, TypedefError::Schema(_)), "got {err:?}"); +}