From aa96f8b98116ed5beb97a1a419953f121030776a Mon Sep 17 00:00:00 2001 From: "glm-5.3-flash" Date: Mon, 12 Oct 2026 08:47:55 +0000 Subject: [PATCH] =?UTF-8?q?feat(tests):=20the=20conformance=20suite's=20pg?= =?UTF-8?q?-lo=20column=20=E2=80=94=20the=20LO-medium=20SKIP-posture=20col?= =?UTF-8?q?umn=20green=20(wave=205)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tests/conformance/pg_lo_column.rs (new): the two pair-gated factories (DualTierMemPgLoFactory on all(mem, pg-lo) — the one-window-host rule's named test topology; DualTierPostgresPgLoFactory on all(postgres, pg-lo) — the consolidation posture, two schemas per open, carrying the dormancy row), a fresh dedicated database per open with the engine-owned schemas minted inside, teardown reaping tracked LOIDs (lo_unlink) then dropping the minted schemas then the database (WITH (FORCE)) with per-open drop-guard janitors for the abandoned paths, and the rows: the verbatim form-independent mirror (INV-1, INV-6, the two no-op abort shapes), the schedule-relative INV-2/3, the two-sweep exact counts over the full protection set across each factory's observed window host, the window-delay race, the holder-stages-only lease, INV-5's durable row, the three dispatch rows over the LO-medium placement oracle, the INV-1 extension (window-edge ranged reads + the planted-overclaim beyond-length clean failure), the virgin-store rows, and the dormancy row; structural isolation + panicking-row tests probing zero orphan residue (no kv-only rejection row — recorded; pg-lo is a large engine) - the LO-isolation corollary carried in the engine task's recorded per-test-database shape (the C7 lesson): zero LO residue on the shared harness database is structural, so the engine suite's shared_harness_db_holds_zero_lo_residue gate stays green alongside the column under --all-features - main.rs: the pg_lo_column gate (feature pg-lo, default-off) + the columns' doc paragraph - README: the pg-lo rows table in §Columns beyond the reference set, the SKIP-posture note naming the second landed column and the per-factory feature-pair gating, current state = five columns - store.rs: one mechanical gate fix — the PgOpts import was unused in the sqlite+pg-lo config (tripping clippy --features pg-lo -D warnings); the gate now names the configs that consume it Verification (harness server live 2026-10-12): --all-features against the server fully green (188 lib + 106 conformance + 4 git-interop; zero LO/schema/database residue after the run); default test green (69/19/3); --features "pg-lo,mem" and "postgres,pg-lo" green server-less (skipping cleanly) and --features pg-lo alone; clippy -D warnings green on default, pg-lo, pg-lo+mem, postgres+pg-lo, all-features; fmt --check green; cargo doc --no-deps clean. --- src/store.rs | 2 +- tasks/suite-column-pg-lo.md | 142 +- tests/conformance/README.md | 117 +- tests/conformance/main.rs | 9 +- tests/conformance/pg_lo_column.rs | 5201 +++++++++++++++++++++++++++++ 5 files changed, 5457 insertions(+), 14 deletions(-) create mode 100644 tests/conformance/pg_lo_column.rs diff --git a/src/store.rs b/src/store.rs index d430161..b2baa88 100644 --- a/src/store.rs +++ b/src/store.rs @@ -287,7 +287,7 @@ use crate::local::{LocalBackend, LocalCursor, LocalStaging}; #[cfg(feature = "mem")] use crate::mem::{MemBackend, MemCursor}; -#[cfg(any(feature = "postgres", feature = "pg-lo"))] +#[cfg(any(feature = "postgres", all(feature = "pg-lo", feature = "mem")))] use crate::pg_common::PgOpts; #[cfg(feature = "pg-lo")] use crate::pg_lo::{PgLoBackend, PgLoCursor, PgLoStaging}; diff --git a/tasks/suite-column-pg-lo.md b/tasks/suite-column-pg-lo.md index 1a750d7..2422cc0 100644 --- a/tasks/suite-column-pg-lo.md +++ b/tasks/suite-column-pg-lo.md @@ -1,7 +1,7 @@ --- id: suite-column-pg-lo name: Conformance suite — the pg-lo column (the LO-medium SKIP-posture column) -status: pending +status: completed depends_on: [store-pg-lo-construction] scope: moderate risk: medium @@ -94,24 +94,24 @@ feature = "pg-lo")`; the column's full run (acceptance) is ## Acceptance Criteria -- [ ] The pg-lo column green **against the harness server** (`pglo-poc`, +- [x] The pg-lo column green **against the harness server** (`pglo-poc`, :15432) under `--all-features`: the mirrored reference rows + the durable-schedule sweep rows on both factories (the pg-lo-host shape and the kv-host/dormancy shape) + the dispatch rows + the virgin-store row, every row ADR-stamped and traceable to the backlog -- [ ] The dormancy row asserts the consolidation shape's pg-lo schema +- [x] The dormancy row asserts the consolidation shape's pg-lo schema carries no `gc_window`; the mem-pglo rows assert the window on the pg-lo host -- [ ] Server-less runs skip cleanly (the reachability probe; the +- [x] Server-less runs skip cleanly (the reachability probe; the default-build gates stay green with the column skipped) -- [ ] Factory isolation + idempotent teardown hold (fresh schema per +- [x] Factory isolation + idempotent teardown hold (fresh schema per open; schemas dropped, LOs unlinked; **zero `pg_largeobject` residue** probed; teardown safe against a panicking row), structurally exercised -- [ ] README amendments recorded (the pg-lo rows table; the +- [x] README amendments recorded (the pg-lo rows table; the feature-pair gating note; current state) -- [ ] `cargo test` (default) green — the column compiles out; +- [x] `cargo test` (default) green — the column compiles out; `cargo test --features pg-lo` (and the mem/postgres pair variants) green server-less (skipping) and green against the server at acceptance; `cargo clippy --all-targets -- -D warnings` @@ -139,8 +139,132 @@ feature = "pg-lo")`; the column's full run (acceptance) is ## Notes -> To be filled by implementation agent +> Implemented in `tests/conformance/pg_lo_column.rs` (the two +> pair-gated factories, the shared harness/oracle/janitor parent, the +> row bodies over both factories, and the structural tests), +> `tests/conformance/main.rs` (the module gate + the columns' doc +> paragraph), `tests/conformance/README.md` (the amendments), and one +> mechanical line in `src/store.rs` (see below). Recorded design +> points, as resolved: +> +> - **The LO-isolation corollary carried in the engine task's recorded +> shape — per-open dedicated databases.** The column first followed +> the postgres column's shared-db schema pattern with tracked-LOID +> `lo_unlink` teardown, and the wave-5 engine-suite gate +> (`pg_lo::tests::shared_harness_db_holds_zero_lo_residue`) broke at +> acceptance: its permanent probe (zero `pg_largeobject` pages in the +> shared `blobs` database) runs concurrently with the suite and the +> column's LOs legitimately existed mid-run there. The task's +> alternative clause ("or the per-test-database shape if that is what +> the engine task recorded") resolved the fork: the engine task +> **did** record the per-test dedicated database shape, so the column +> carries it — `open()` mints a fresh dedicated database (unique per +> call), the engine-owned schemas are minted inside it (the alkstore +> pg-factory pattern's registry, housed on its own database), and +> teardown reaps the instance's LOs (`SELECT loid FROM +> lo_entries` → `lo_unlink`), drops exactly the minted schemas +> (`DROP SCHEMA IF EXISTS … CASCADE`), and drops the database +> (`WITH (FORCE)`), with a per-open drop-guard janitor (the crate's +> `DatabaseJanitor` pattern re-implemented in the column file — the +> crate's janitor is `#[cfg(test)]`-internal) backing the abandoned +> paths. The structural tests probe **zero orphan residue inside the +> dedicated database before it goes** (the companion table and the +> LO catalog agree exactly — the C1 shape), assert the recorded +> reaped LOIDs match the pre-captured set, and observe the database +> gone afterward; zero residue on the shared database is structural +> (no pg-lo pool ever opens against `blobs`). +> - **The consolidation factory mints two schemas** (the kv half and +> the pg-lo half, both inside the dedicated database) so the dormancy +> row observes genuine absence; the corollary's one-schema mint +> remains the engine tests' recorded alternative. +> - **The mem-pglo kv half is facade-observed** (the mem column's +> introspection shape: served reads and `has()` against the LO +> medium's genuine absence); the postgres-pglo kv half counts the kv +> schema's `blobs` rows (the postgres column's oracle shape). +> - **The INV-1 extension row** asserts ranged reads across the +> shipped window edge (`READ_CHUNK`'s 512 KiB, duplicated in the +> column file since the constant is engine-internal), the EOF +> clamping family (tail, at-EOF empty, `u64::MAX` offset clamp, +> zero-length, over-read), the ADR-006 range digests, and the +> **beyond-length clean failure** against a **planted overclaiming +> companion row** (`lo_create`/`lo_open`/`lowrite` + `lo_entries` +> insert through the oracle — the corrupted half-state healthy +> stores never hold): the read trusts the row (metadata-only) and +> fails with the unexpected-EOF-shaped `Error::Io`, and the +> executing sweep's delete reaps the planted LO. The identical- +> content legs dedup into one entry (CAS across the paths), so the +> cleanup stages exactly two. +> - **Row inventory** (both factories; the factory dimension is the +> run): the form-independent mirror (INV-1, INV-6, the two no-op +> abort shapes — verbatim from rows.rs) + INV-2/INV-3 (schedule- +> relative, both tiers' legs) + the two-sweep exact-count row, the +> window-delay race row, the holder-stages-only lease row, INV-5's +> durable row, the three dispatch rows, the INV-1 extension, the +> virgin-store row, the postgres-pglo-only dormancy row; **no +> kv-only rejection row** (no kv-only pg-lo constructor exists; +> recorded). The mem-pglo virgin row asserts the window on the +> pg-lo host (`gc_window` present-and-empty, `gc_pins` never). +> - **One mechanical `src/` fix** (out of scope but required by the +> gate): `src/store.rs`'s `use crate::pg_common::PgOpts` was gated +> `any(postgres, pg-lo)` and sat unused in the `sqlite`+`pg-lo` +> config (pg-lo alone over default features), tripping +> `cargo clippy --features pg-lo -D warnings`; the gate now names +> the configs that consume the import +> (`any(postgres, all(pg-lo, mem))` — no semantic change). +> - **Two harness-server residue cleanups** — found + removed (same +> discipline as the d69d9a3 finding): the shared `blobs` db carried +> the pre-rework runs' schemas (`abpglo_*`/the over-long first +> mints) + their orphaned LO pages (1032 catalog pages; the +> schema-pattern teardown dropped schemas, stranding LOs — exactly +> the C7 lesson motivating the dedicated-database shape); cleanup +> verified zero (`pg_largeobject`, schemas, and suites' databases). +> The first shared-db schema mints also grazed postgres' 63-char +> identifier limit (E42622) — the dedicated-db mint +> (`abpglosuite_…`/`abpglo_…`) stays well under it. +> - **The first rework run's failure was a miscount, not a bug**: the +> INV-1 extension row's two identical-content legs dedup into one +> entry, so the cleanup sweep stages two (one real + the planted), +> not three — fixed in the row's expectations and recorded here. ## Summary -> To be filled on completion \ No newline at end of file +> Completed 2026-10-12 (wave 5). The pg-lo column — the suite's +> second SKIP-posture column — landed in +> `tests/conformance/pg_lo_column.rs`: `DualTierMemPgLoFactory` (kv = +> mem + large = pg-lo, the one-window-host rule's named test +> topology, `all(mem, pg-lo)`) and `DualTierPostgresPgLoFactory` (the +> consolidation posture on one pg instance, `all(postgres, pg-lo)`; +> two schemas inside each open's dedicated database; the dormancy +> row asserting no `gc_window` in the pg-lo schema and the window on +> the kv host), each factory vending a fresh dedicated database per +> open with engine-owned schemas minted inside, teardown reaping +> tracked LOIDs → dropping the minted schemas → dropping the +> database `WITH (FORCE)`, plus per-open drop-guard janitors for the +> abandoned paths — zero LO residue on the shared `blobs` database +> structural (the engine suite's gate holds green alongside the +> column). Fifteen rows on the mem-pglo factory, sixteen on the +> postgres-pglo factory (its extra one being the dormancy row): the +> verbatim form-independent +> mirror (INV-1, INV-6, the no-op abort shapes), the +> schedule-relative INV-2/3, the two-sweep exact counts over the +> full protection set across the observed window host, the +> window-delay race, the holder-stages-only lease, INV-5's durable +> row, the three dispatch rows over the LO-medium oracle, the INV-1 +> extension (window-edge ranged reads + the planted-overclaim +> beyond-length clean failure), and the virgin-store row (the +> mem-pglo one asserting the window on the pg-lo host) — the +> dormancy row being the postgres-pglo factory's extra — plus the +> structural isolation/panicking-row tests probing zero orphan +> residue. No kv-only rejection row (recorded; pg-lo is a large +> engine). README recorded (the pg-lo rows table in §Columns beyond +> the reference set, the SKIP-posture note naming the second landed +> column and the per-factory feature-pair gating, current state = +> five columns). Verified (harness server live 2026-10-12): acceptance +> `cargo test --all-features` against the server fully green (188 lib +> + 106 conformance + 4 git-interop; zero LO/schema/database residue +> after the run); `cargo test` (default) green (69/19/3 — the column +> compiles out); `--features "pg-lo,mem"` and `--features +> "postgres,pg-lo"` green server-less (skipping cleanly) and under +> `--features pg-lo` alone; clippy `-D warnings` green on default, +> pg-lo, pg-lo+mem, postgres+pg-lo, and all-features; `cargo fmt +> --check` green; `cargo doc --no-deps` clean. \ No newline at end of file diff --git a/tests/conformance/README.md b/tests/conformance/README.md index 4b00a73..4813b24 100644 --- a/tests/conformance/README.md +++ b/tests/conformance/README.md @@ -336,6 +336,72 @@ connections fail closed on next use). | `kv_tier_misses_fall_through_to_the_large_tier_per_op` | ADR-012 §6.1 (fall-through per op); ADR-012 §6.5 ruling 5 (miss shapes); ADR-006; ADR-012 §6.3; ADR-007 §5 | | `virgin_store_reads_are_no_ops_on_a_fresh_schema` | store-api.md §Error model (virgin-store semantics); ADR-012 §6.5; ADR-007 §5 (the constructor always ensures the schema — the facade's fresh-schema form of the tolerance) | +**The pg-lo column's own rows** +(tests/conformance/pg_lo_column.rs, the second SKIP-posture column — +the **LO-medium column**, ADR-009, landed wave 5): the reference +inventory's form-independent rows mirror verbatim against a +dual-tier factory (INV-1's round-trip family over the content +matrix, INV-6's stat-discipline row — whose non-fetching discipline +rides ADR-009's companion-row probe on this medium — and the +`NoLivenessSources`/`ProtectFailed` abort shapes), and the column +grows the durable-schedule sweep family (§The schedule-relative +reading for SQL-backed columns; ADR-012 §6.7 — the same window delay +tuning, `0` — execute-on-next-sweep), the dispatch rows with the +placement oracle adapted to the LO medium (large entries are +companion rows + LOs — an independent connection counting +`lo_entries` and probing `pg_largeobject`; kv entries per the +factory's kv engine: the mem-pglo factory's kv half is in-process, +the mem introspection shape, the postgres-pglo factory's kv half +counts the kv schema's `blobs` rows), the INV-1 extension row (the +LO medium's ranged reads across the shipped window edge, EOF +clamping, and the beyond-length clean failure against a planted +overclaiming companion row), and the virgin-store row. **No +kv-only rejection row** — no kv-only pg-lo constructor exists (pg-lo +is a large engine; the postgres column hosts that row for the SQL +kv engines). + +The **two factories span feature pairs — the first column whose +factories need different feature sets** (a recorded design point): +`DualTierMemPgLoFactory` (*kv = mem* + *large = pg-lo*, the +one-window-host rule's named test topology — the durable-schedule +rows run with the window hosted on the **large engine**, +`gc_window` in the pg-lo schema) gates on +`all(feature = "mem", feature = "pg-lo")`; +`DualTierPostgresPgLoFactory` (*kv = postgres* + *large = pg-lo* on +one pg instance — the consolidation posture) gates on +`all(feature = "postgres", feature = "pg-lo")` and carries the +**dormancy row** (the pg-lo schema carries no `gc_window`; the +window lives on the kv schema — ADR-012 §5); the column's full run +(acceptance) is `--all-features` against the server. The factories +follow the alkstore pg-factory pattern with the **LO-isolation +corollary carried in the engine task's recorded shape** (the C7 +lesson): a fresh dedicated database per `open()` (unique per call, +tracked per-instance; engine schemas minted inside it), teardown +reaping the instance's LOs (`SELECT loid FROM lo_entries` → +`lo_unlink`), dropping exactly the minted schemas +(`DROP SCHEMA IF EXISTS … CASCADE`), and dropping the database +(`WITH (FORCE)`) — idempotent, panicking-row-safe, with a +drop-guard janitor per open backing the abandoned paths — so zero +LO residue on the shared harness database is structural (the engine +suite's `shared_harness_db_holds_zero_lo_residue` gate measures it), +and the structural teardown assertions probe zero orphan residue +inside the dedicated database before it goes. + +| Row (`pg_lo_column.rs`) | Traces to | +|---|---| +| `dedup_identical_content_is_one_pool_entry` | INV-2 (kv and large legs; counts read against the durable schedule; ADR-009 §2/§4 — the companion table, delete = lo_unlink + row delete) | +| `stage_hygiene_zero_residue_on_failure_exactly_one_entry_on_success` | INV-3 (the LO tier's real staged form — the held tx's failure paths roll back to no row, no LO pages; schedule-relative counts; ADR-009 §2) | +| `two_sweep_exact_counts_over_the_full_protection_set` | INV-4/INV-5 (the full protection set across two sweeps, both tiers; the window observed on its host — the pg-lo schema for mem-pglo, the kv schema for postgres-pglo; ADR-012 §5) | +| `a_put_landing_during_the_window_survives_the_executing_sweep` | INV-4 (the sweep-vs-put race across the window delay; execution-time re-arbitration; ADR-012 §5 — the host rule's shapes) | +| `the_contended_second_sweeper_aborts_as_data_and_the_holder_stages_only` | INV-4 (the `SweeperLock` abort shape — abort before staging; the holder's durable-form completion: stages only) | +| `list_correctness_is_observable_through_durable_schedule_outcomes` | INV-5 (exact counts, never list comparison) | +| `lo_windows_read_across_window_boundaries_and_a_short_body_fails_cleanly` | INV-1's LO-medium extension (ranged reads across the shipped window edge; EOF clamping incl. the `u64::MAX` clamp — the C7 trap not inherited; the beyond-length clean failure); store-api.md §Read path; ADR-009 §2/§3 (the companion table as authority; the `lo_get` windows); ADR-006; ADR-010 §1 | +| `routing_is_a_pure_function_of_length_across_put_paths_and_re_puts` | ADR-003 §Dispatch policy (purity/no-migration); ADR-009 §2 (the companion table); ADR-012 §5 (the topology); ADR-012 §6.3; ADR-012 §6.7; INV-7 facade-contract row | +| `threshold_boundary_lengths_route_deterministically` | ADR-003 §Dispatch policy (at/below/above, tuned + shipped-default thresholds); ADR-009 §2; ADR-012 §6.3 | +| `kv_tier_misses_fall_through_to_the_large_tier_per_op` | ADR-012 §6.1 (fall-through per op); ADR-012 §6.5 ruling 5 (miss shapes); ADR-006; ADR-009 §2/§3 (the `lo_get` windows serve the fall-through); ADR-012 §6.3 | +| `virgin_store_reads_are_no_ops_on_a_fresh_schema` | store-api.md §Error model (virgin-store semantics); ADR-012 §6.5; ADR-009 §2 (the companion-table bootstrap); ADR-012 §5 (the mem-pglo row asserts the window on the pg-lo host — `gc_window` present-and-empty, `gc_pins` never) | +| `the_window_lives_on_the_kv_schema_and_the_pg_lo_schema_stays_dormant` (postgres-pglo only) | the dormancy row: ADR-012 §5 (the one-window-host rule's kv-host arm — the pg-lo window clauses dormant, no `gc_window` in the pg-lo schema; the schedule observed on the kv host); ADR-008 §2; ADR-012 §6.7 | + ## Column plan **mem is the contract-reference engine** (ADR-011 §2): wave 2 lands @@ -345,12 +411,16 @@ column propagates as a requirement to all other columns (INV-7). The pg columns (postgres, pg-lo) additionally run the SKIP posture above — the **postgres column is the posture's first landed host** (server-less runs skip cleanly; its wave-4 acceptance is the green run against the -harness server); the fleet-form rows of INV-4 are engine-gated to the -postgres column and first run in wave 6. +harness server) and the **pg-lo column is the second** (landed wave 5; +its two factories gate on the `mem`+`pg-lo` and `postgres`+`pg-lo` +feature pairs respectively — the first column whose factories need +different feature sets — with the column's full run `--all-features` +against the harness server; the fleet-form rows of INV-4 are +engine-gated to the postgres column and first run in wave 6). ## Current state -The harness is green on **four columns** (waves 2–4). The **mem +The harness is green on **five columns** (waves 2–5). The **mem column** is the contract-reference column: the factory binding is the concrete `alkblobs::Store` facade (see §The factory contract — the binding demonstrator remains in `binding.rs`), the reference row set @@ -431,3 +501,44 @@ failure — and its acceptance is the green run **against the harness server** (dockerized `pglo-poc`, `postgres:16-alpine`, :15432, db `blobs`, verified live 2026-10-12; the alkstore harness's same server). + +The **pg-lo column** (the suite's **second SKIP-posture column** — +the LO-medium column, feature `pg-lo` **default-off**, ADR-009, its +two factories gating on the `mem`+`pg-lo` and `postgres`+`pg-lo` +feature pairs — the first column whose factories need different +feature sets, landed wave 5) mirrors the form-independent reference +rows (INV-1's round-trip family, INV-6's stat-discipline row riding +ADR-009's companion-row probe, and the `NoLivenessSources`/ +`ProtectFailed` abort shapes) and grows the durable-schedule sweep +rows (run on **both factories** — the window observed on its host: +the pg-lo schema for the mem-pglo factory per the one-window-host +rule's named test topology, the kv schema for the postgres-pglo +consolidation shape, which carries the **dormancy row**), the +dispatch rows with the placement oracle adapted to the LO medium +(companion rows + `pg_largeobject` probing for the large tier; the +mem kv half in-process, the postgres kv half's `blobs` rows), the +INV-1 extension row (the LO medium's ranged reads across the +shipped window edge and the beyond-length clean failure), and the +virgin-store rows — the recorded inventory is in §Columns beyond +the reference set. **No kv-only rejection row mirrors here** (no +kv-only pg-lo constructor exists; recorded). Connection settings +ride the environment (`ALKBLOBS_PG_HOST/PORT/USER/PASSWORD/DB` — +`ALKBLOBS_PG_HOST` gates; the rest default to the harness server's +values) and the runners gate behind the reachability probe — a skip +is never a failure — and its acceptance is the green run +**against the harness server under `--all-features`** (dockerized +`pglo-poc`, verified live 2026-10-12). Its factories declare the +column-tuned threshold and the test tuning of the window delay +(`0` — execute-on-next-sweep) and follow the alkstore pg-factory +pattern with the **LO-isolation corollary carried in the engine +task's recorded shape** (the C7 lesson): a fresh dedicated database +per `open()` (unique per call, engine schemas minted inside it, +tracked per-instance; drop-guard janitors backing the abandoned +paths), teardown reaping the instance's LOs (`SELECT loid FROM +lo_entries` → `lo_unlink`), dropping exactly the minted schemas, +and dropping the database (`WITH (FORCE)`) — idempotent, +panicking-row-safe — so zero LO residue on the shared database is +structural (the engine suite's +`shared_harness_db_holds_zero_lo_residue` gate measures it) and the +structural teardown assertions probe zero orphan residue inside the +dedicated database before it goes. diff --git a/tests/conformance/main.rs b/tests/conformance/main.rs index 0e801bf..9325dc3 100644 --- a/tests/conformance/main.rs +++ b/tests/conformance/main.rs @@ -33,7 +33,12 @@ //! columns); [`postgres_column`] — the suite's first SKIP-posture //! column (the fleet-capable-kv shape, `kv = postgres` + `large = //! `local`, feature `postgres` default-off), which mirrors the same -//! sets against the harness server and skips cleanly server-less. +//! sets against the harness server and skips cleanly server-less; +//! [`pg_lo_column`] — the suite's second SKIP-posture column (the +//! LO-medium column, feature `pg-lo` default-off, its two factories +//! spanning feature pairs `mem`+`pg-lo` and `postgres`+`pg-lo`), which +//! runs the same sets against the LO medium and skips cleanly +//! server-less. //! //! Items beyond the tests below are the later columns' surface, hence //! the scoped `#![allow(dead_code)]` — it shrinks as columns land. @@ -48,6 +53,8 @@ mod factory; mod local_column; #[cfg(feature = "mem")] mod mem_column; +#[cfg(feature = "pg-lo")] +mod pg_lo_column; #[cfg(feature = "postgres")] mod postgres_column; mod rows; diff --git a/tests/conformance/pg_lo_column.rs b/tests/conformance/pg_lo_column.rs new file mode 100644 index 0000000..4042223 --- /dev/null +++ b/tests/conformance/pg_lo_column.rs @@ -0,0 +1,5201 @@ +//! The **pg-lo column** — the suite's second SKIP-posture column: the +//! reference row set and the durable-schedule sweep rows run against +//! the pg-lo large tier (feature `pg-lo`, default-off, ADR-009). The +//! column grows **two factories spanning feature pairs — the first +//! column whose factories need different feature sets** (a recorded +//! design point): +//! +//! - **[`DualTierMemPgLoFactory`](mem_pglo::DualTierMemPgLoFactory)** — +//! `kv = mem` + `large = pg-lo`: the one-window-host rule's named +//! test topology (ADR-012 §5). The durable-schedule sweep rows run +//! with the window hosted on the **large engine** (`gc_window` in +//! the pg-lo schema) — the shape no earlier column exercises. Gates +//! on `all(feature = "mem", feature = "pg-lo")`. +//! - **[`DualTierPostgresPgLoFactory`](postgres_pglo::DualTierPostgresPgLoFactory)** — +//! `kv = postgres` + `large = pg-lo` on one pg instance: the +//! consolidation posture (ADR-009; the fleet-valid engine set's last +//! constructor). Mirrors the postgres column's inventory with the +//! large medium swapped to LOs, and carries the **dormancy row** +//! (the pg-lo schema carries no `gc_window`; the window lives on the +//! kv schema — ADR-012 §5). Gates on +//! `all(feature = "postgres", feature = "pg-lo")`. +//! +//! The SKIP posture (README §The SKIP posture) is a factory-owner +//! responsibility: the connection settings ride the environment +//! (`ALKBLOBS_PG_HOST/PORT/USER/PASSWORD/DB` — host required; the rest +//! default to the harness server's values), every runner gates behind +//! a **reachability probe**, and when no server is reachable the +//! column skips cleanly — a skip is not a failure. The acceptance run +//! is green **against the harness server** (dockerized `pglo-poc`, +//! `postgres:16-alpine`, :15432, db `blobs`) under `--all-features` — +//! a skip never substitutes for it. +//! +//! The factories (README §The factory contract): `open()` mints a +//! **fresh, dedicated database per call** carrying the engine-owned +//! schemas (the alkstore pg-factory pattern — a fresh engine-owned +//! schema per open, unique per call, tracked per-instance in a +//! registry — housed on its own database), and vends the concrete +//! [`alkblobs::Store`] facade over them (the mem-pglo factory's kv +//! tier is in-process mem, needing no schema of its own; the +//! postgres-pglo factory mints two — the kv half and the pg-lo half — +//! so the dormancy row observes genuine absence). **The LO-isolation +//! corollary carried in the engine task's recorded shape** (the C7 +//! lesson): Large Objects are **database-level**, not schema-level — +//! the schema pattern alone does not isolate them and a bare +//! `DROP SCHEMA` strands their pages in `pg_largeobject` — so each +//! open gets its own database, and teardown reaps the instance's LOs +//! (`SELECT loid FROM .lo_entries` — the companion +//! table, the contract authority — then `lo_unlink` each tracked +//! LOID), drops exactly the minted schemas (`DROP SCHEMA IF EXISTS … +//! CASCADE`, panicking-row-safe), and drops the dedicated database +//! (`WITH (FORCE)`), with a drop-guard janitor per open backing every +//! abandoned path. Zero LO residue on the shared harness database +//! (`blobs`) is therefore structural — no pg-lo pool ever opens +//! against it (the engine suite's +//! `shared_harness_db_holds_zero_lo_residue` gate measures exactly +//! this) — and the structural teardown assertions probe zero orphan +//! residue inside the dedicated database before it goes. +//! +//! The sweep rows are **schedule-relative on this column** (README +//! §The schedule-relative reading for SQL-backed columns, citing +//! ADR-012 §6.7): the durable form stages its candidates this sweep +//! and executes them by the next, so the exact counts are asserted +//! against the staged schedule — on the pg-lo host for the mem-pglo +//! factory, on the kv host for the postgres-pglo factory. The +//! form-independent rows mirror verbatim ([`crate::rows`]: the +//! round-trip family over the content matrix, the stat-discipline +//! row, and the `NoLivenessSources`/`ProtectFailed` abort shapes). +//! **No kv-only rejection row** — no kv-only pg-lo constructor exists +//! (pg-lo is a large engine; the postgres column hosts that row for +//! the SQL kv engines). +//! +//! The dual-tier dispatch rows mirror the postgres column's family +//! with the placement oracle adapted to the LO medium: large entries +//! are **companion rows + LOs** (an independent pg connection counting +//! `lo_entries` and probing `pg_largeobject` for the row's LOID), kv +//! entries per the factory's kv engine — the mem-pglo factory's kv +//! placement is in-process (the oracle's kv half follows the mem +//! column's introspection shape: served reads and absence from the +//! LO medium), the postgres-pglo factory's kv half counts the kv +//! schema's `blobs` rows. + +#![allow(dead_code)] + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use tokio_postgres::NoTls; + +use alkblobs::Error; + +/// The shipped LO read-window size — src/pg_lo.rs's crate-internal +/// `READ_CHUNK` constant (ADR-009 §3, POC #7 C2/C4's window shape; one +/// `lo_get` statement per window). The column's INV-1 extension row +/// straddles this edge — duplicated here because the constant is +/// engine-internal. +pub(crate) const LO_WINDOW: usize = 512 * 1024; + +/// The **test tuning of the window delay** (ADR-008 §2's legal +/// degenerate tuning): `0` — execute-on-next-sweep. The durable +/// schedule's staging sweep stages the dead set; the *next* sweep is +/// its executor, which is exactly the schedule the sweep rows assert +/// against. Both factories run at this tuning. +pub(crate) const WINDOW_DELAY: Duration = Duration::ZERO; + +// ------------------------------------------------------------------ +// The harness connection (the SKIP posture's gate) — the settings +// ride the environment, the runners probe reachability before their +// rows run, an unreachable server skips the column cleanly. +// ------------------------------------------------------------------ + +/// The harness connection settings ride the environment +/// (`ALKBLOBS_PG_HOST/PORT/USER/PASSWORD/DB` — the alkstore harness +/// convention). `ALKBLOBS_PG_HOST` set (non-empty) is the probe's +/// "configured" answer; port/user/password/db default to the harness +/// server's values (`pglo-poc`: :15432, `postgres`/`poc`, db +/// `blobs`). Absent → the caller skips. +pub(crate) fn harness_dsn() -> Option { + let host = std::env::var("ALKBLOBS_PG_HOST") + .ok() + .filter(|v| !v.is_empty())?; + let port = std::env::var("ALKBLOBS_PG_PORT") + .ok() + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| "15432".to_owned()); + let user = std::env::var("ALKBLOBS_PG_USER") + .ok() + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| "postgres".to_owned()); + let password = std::env::var("ALKBLOBS_PG_PASSWORD") + .ok() + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| "poc".to_owned()); + let db = std::env::var("ALKBLOBS_PG_DB") + .ok() + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| "blobs".to_owned()); + Some(format!( + "host={host} port={port} user={user} password={password} dbname={db}" + )) +} + +/// The skip note — the SKIP posture's clean-skip print, exactly once +/// per skipped test. +pub(crate) fn skip_note() { + eprintln!("skipping (no postgres harness server): set ALKBLOBS_PG_HOST/PORT/USER/PASSWORD/DB"); +} + +async fn admin_connect(dsn: &str) -> Result { + let (client, connection) = tokio_postgres::connect(dsn, NoTls) + .await + .map_err(|e| Error::Io(format!("postgres harness: {e}")))?; + tokio::spawn(async move { + let _ = connection.await; + }); + Ok(client) +} + +/// The server reachability probe: the SKIP posture's gate the runners +/// check before their rows run. One cheap admin connection per test; +/// teardown reconnects independently (no shared admin client +/// lifetime). +pub(crate) async fn harness_ready() -> bool { + let Some(dsn) = harness_dsn() else { + skip_note(); + return false; + }; + match tokio::time::timeout(Duration::from_secs(3), admin_connect(&dsn)).await { + Ok(Ok(_)) => true, + Ok(Err(e)) => { + eprintln!("skipping (harness server unreachable): {e}"); + false + } + Err(_) => { + eprintln!("skipping (harness server unreachable: connect timeout)"); + false + } + } +} + +// ------------------------------------------------------------------ +// The dedicated-database mint + the LO-isolation corollary's teardown +// arms. +// ------------------------------------------------------------------ + +/// The maintenance DSN — connections through the `postgres` +/// database that create and drop dedicated databases. +pub(crate) fn admin_dsn(base: &str) -> String { + let base = base + .rsplit_once("dbname=") + .expect("the harness DSN names a db") + .0; + format!("{base}dbname=postgres") +} + +/// The DSN scoped to one dedicated database — the store's and the +/// oracle's connection settings for that open. +pub(crate) fn db_dsn(base: &str, db: &str) -> String { + let base = base + .rsplit_once("dbname=") + .expect("the harness DSN names a db") + .0; + format!("{base}dbname={db}") +} + +/// A fresh dedicated-database name — unique per call, seeded per +/// (tag, pid, nanos, factory-internal sequence) so two factory +/// instances never share a database, racing opens included. +pub(crate) fn fresh_db_name(tag: &str, seq: &AtomicU64) -> String { + format!( + "abpglosuite_{}_{}_{:x}_{}", + tag, + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0), + seq.fetch_add(1, Ordering::Relaxed), + ) +} + +/// A fresh engine-owned schema name — unique per call, seeded per +/// (tag, pid, nanos, factory-internal sequence) so two factory +/// instances never share a schema name, racing opens included. The +/// mint stays well under postgres' 63-character identifier limit. +pub(crate) fn fresh_schema_name(tag: &str, seq: &AtomicU64) -> String { + format!( + "abpglo_{}_{}_{:x}_{}", + tag, + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0), + seq.fetch_add(1, Ordering::Relaxed), + ) +} + +/// Create a dedicated database (through the maintenance DSN). +pub(crate) async fn create_db(maintenance_dsn: &str, db: &str) -> Result<(), Error> { + let client = admin_connect(maintenance_dsn).await?; + client + .execute(&format!("CREATE DATABASE \"{db}\""), &[]) + .await + .map_err(|e| Error::Io(format!("postgres harness: {e}")))?; + Ok(()) +} + +/// Whether a dedicated database still exists server-side — the +/// structural teardown tests' instrument (`pg_database`). +pub(crate) async fn db_exists(maintenance_dsn: &str, db: &str) -> bool { + let client = admin_connect(maintenance_dsn) + .await + .expect("the oracle connection opens"); + let count: i64 = client + .query_one( + "SELECT count(*) FROM pg_database WHERE datname = $1", + &[&db], + ) + .await + .expect("the oracle query succeeds") + .get(0); + count > 0 +} + +/// Drop a dedicated database — `WITH (FORCE)` reaps any connection +/// the store's pool has not yet drained (postgres ≥ 13; the +/// panicking-row-safe arm; the same clause the engine suite's +/// janitor carries). +pub(crate) async fn drop_db(maintenance_dsn: &str, db: &str) -> Result<(), Error> { + let client = admin_connect(maintenance_dsn).await?; + client + .execute(&format!("DROP DATABASE \"{db}\" WITH (FORCE)"), &[]) + .await + .map_err(|e| Error::Io(format!("postgres harness: {e}")))?; + Ok(()) +} + +/// The zero-residue probe inside one dedicated database — the LO +/// catalog and the companion table agree exactly (no missing pages, +/// no orphans): the LO-isolation corollary's structural assertion, +/// the engine suite's C1 exact-count shape. +pub(crate) async fn zero_lo_residue(dsn: &str, schema: &str) -> bool { + let client = admin_connect(dsn) + .await + .expect("the oracle connection opens"); + let sql = format!( + "SELECT count(*) FROM (\ + SELECT loid FROM \"{schema}\".lo_entries \ + EXCEPT SELECT DISTINCT loid FROM pg_largeobject\ + ) missing, (\ + SELECT DISTINCT loid FROM pg_largeobject \ + EXCEPT SELECT loid FROM \"{schema}\".lo_entries\ + ) orphaned" + ); + let disagreements: i64 = client + .query_one(sql.as_str(), &[]) + .await + .expect("the oracle query succeeds") + .get(0); + disagreements == 0 +} + +/// Reap one schema's committed LOs before the schema goes away, +/// because Large Objects are database-level and a bare `DROP SCHEMA` +/// would strand their pages in `pg_largeobject`. The loids are read +/// from the companion table (the contract authority — never +/// `pg_largeobject`); each is unlinked best-effort so an idempotent +/// teardown is never torn by a missing oid; uncommitted staged pages +/// cannot exist (the held-tx rollback stories). Returns the reaped +/// LOIDs — the tracked-LOID list the structural tests assert against. +pub(crate) async fn reap_lo_schema(client: &tokio_postgres::Client, schema: &str) -> Vec { + let present = table_exists_conn(client, schema, "lo_entries").await; + if !present { + return Vec::new(); + } + let sql = format!("SELECT loid FROM \"{schema}\".lo_entries"); + let loids: Vec = client + .query(sql.as_str(), &[]) + .await + .expect("the loid enumeration succeeds") + .iter() + .map(|row| row.get::<_, u32>(0)) + .collect(); + for loid in &loids { + let _ = client.execute("SELECT lo_unlink($1)", &[loid]).await; + } + loids +} + +/// The drop-guard janitor — the engine suite's recorded +/// `DatabaseJanitor`-shaped pattern (`src/pg_common.rs`'s +/// test-support janitor, `#[cfg(test)]`-internal there): its drop +/// removes the dedicated database when *Drop* is the only exit — a +/// row that panics before teardown, a factory released without one. +/// Same Drop mechanics: no async teardown exists at Drop, so the drop +/// executes synchronously from a std thread over a one-shot +/// current-thread runtime; an unreachable harness server is tolerated. +pub(crate) struct DbJanitor { + maintenance_dsn: String, + db: String, +} + +impl DbJanitor { + /// Register the guard for one dedicated database. Constructed as + /// part of `open()`'s bookkeeping; it fires only if teardown never + /// consumed the record (teardown disarms it). Dropping an + /// already-dropped database is a tolerated no-op server-side. + pub(crate) fn new(maintenance_dsn: &str, db: &str) -> DbJanitor { + DbJanitor { + maintenance_dsn: maintenance_dsn.to_owned(), + db: db.to_owned(), + } + } +} + +impl Drop for DbJanitor { + fn drop(&mut self) { + let maintenance_dsn = self.maintenance_dsn.clone(); + let db = self.db.clone(); + let _ = std::thread::spawn(move || { + let Ok(rt) = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + else { + return; + }; + rt.block_on(async move { + let Ok((client, conn)) = tokio_postgres::connect(&maintenance_dsn, NoTls).await + else { + return; + }; + tokio::spawn(async move { + let _ = conn.await; + }); + let _ = client + .execute(&format!("DROP DATABASE \"{db}\" WITH (FORCE)"), &[]) + .await; + }); + }) + .join(); + } +} + +/// One open's tracked backing site: the dedicated database this open +/// minted, the engine-owned schemas minted inside it (the alkstore +/// pg-factory pattern's schema registry, housed on its own database), +/// and the drop-guard janitor that disposes of the database if +/// teardown never gets to it (the abandoned paths — a row that panics +/// before teardown releases the factory into a Drop that runs the +/// guards). Teardown consumes the record, reaps + drops explicitly, +/// and disarms the guard. +pub(crate) struct TrackedDb { + /// The dedicated database's name (the structural tests' + /// `pg_database` probe target). + pub(crate) name: String, + /// The db-scoped DSN — the store's connection settings and the + /// oracle's address into this database. + pub(crate) dsn: String, + /// Every engine-owned schema minted inside the database (the kv + /// and pg-lo halves, in mint order — the pg-factory pattern's + /// registry). + pub(crate) schemas: Vec, + /// The pg-lo halves — the schemas carrying `lo_entries`, whose + /// LOs teardown reaps. + pub(crate) lo_schemas: Vec, + /// The janitor disarmed by teardown; Drop runs it otherwise. + janitor: Option, +} + +impl TrackedDb { + /// Disarm the janitor (teardown took over the disposal). + pub(crate) fn disarm(&mut self) { + self.janitor = None; + } +} + +// ------------------------------------------------------------------ +// The medium oracle (the dispatch rows' instrument, adapted to the LO +// medium): an independent connection per probe — large entries are +// companion rows + LOs (the `lo_entries` count and the `pg_largeobject` +// pages behind each row's LOID), kv entries per the factory's kv +// engine, and the schedule itself is `gc_window` rows in the +// window-hosting schema (the pg-lo schema on the mem-pglo factory, +// the kv schema on the postgres-pglo factory). +// ------------------------------------------------------------------ + +async fn table_exists_conn(client: &tokio_postgres::Client, schema: &str, table: &str) -> bool { + let qualified = format!("\"{schema}\".\"{table}\""); + let row = client + .query_one("SELECT to_regclass($1) IS NOT NULL", &[&qualified]) + .await + .expect("the table probe succeeds"); + row.get(0) +} + +/// Rows in one of the store's schemas' tables — the kv tier's pool +/// (`blobs`), the durable window's staged candidates (`gc_window`), +/// or the pg-lo tier's companion table (`lo_entries`) — counted from +/// the oracle's own independent connection to the shared server. +pub(crate) async fn pg_table_rows(dsn: &str, schema: &str, table: &str) -> usize { + let client = admin_connect(dsn) + .await + .expect("the oracle connection opens"); + let sql = format!("SELECT COUNT(*) FROM \"{schema}\".{table}"); + let row = client + .query_one(sql.as_str(), &[]) + .await + .expect("the oracle query succeeds"); + let count: i64 = row.get(0); + count.max(0) as usize +} + +/// The pg-lo tier's companion-table residency — exactly one row per +/// committed large-tier entry (the contract authority). +pub(crate) async fn lo_entry_rows(dsn: &str, schema: &str) -> usize { + pg_table_rows(dsn, schema, "lo_entries").await +} + +/// The durable window's staged-candidate residency — the schedule +/// observer the durable rows assert against (staged rows land here; +/// the executing sweep's take empties them). +pub(crate) async fn window_rows(dsn: &str, schema: &str) -> usize { + pg_table_rows(dsn, schema, "gc_window").await +} + +/// The companion table's loids — the tracked-LOID list the residue +/// probes and placement assertions ride. +pub(crate) async fn companion_loids(dsn: &str, schema: &str) -> Vec { + let client = admin_connect(dsn) + .await + .expect("the oracle connection opens"); + let sql = format!("SELECT loid FROM \"{schema}\".lo_entries ORDER BY loid"); + client + .query(sql.as_str(), &[]) + .await + .expect("the oracle query succeeds") + .iter() + .map(|row| row.get::<_, u32>(0)) + .collect() +} + +/// The LOID behind one companion row — looked up by the entry's key +/// bytes. +pub(crate) async fn loid_for_key(dsn: &str, schema: &str, key_bytes: &[u8]) -> Option { + let client = admin_connect(dsn) + .await + .expect("the oracle connection opens"); + let sql = format!("SELECT loid FROM \"{schema}\".lo_entries WHERE key = $1"); + client + .query_opt(sql.as_str(), &[&key_bytes]) + .await + .expect("the oracle query succeeds") + .map(|row| row.get::<_, u32>(0)) +} + +/// The `pg_largeobject` pages behind one LOID — the LO-presence probe +/// the placement oracle and the zero-residue teardown assertions +/// ride (the catalog read is address-adjacent, never addressing: the +/// companion table remains the authority). +pub(crate) async fn loid_pages(dsn: &str, loid: u32) -> i64 { + let client = admin_connect(dsn) + .await + .expect("the oracle connection opens"); + let row = client + .query_one( + "SELECT count(*) FROM pg_largeobject WHERE loid = $1", + &[&loid], + ) + .await + .expect("the oracle query succeeds"); + row.get(0) +} + +/// Whether an engine-owned schema carries one of its tables — the +/// window-host assertions' instrument (`to_regclass`; the dormancy +/// row's genuine-absence probe). +pub(crate) async fn table_exists(dsn: &str, schema: &str, table: &str) -> bool { + let client = admin_connect(dsn) + .await + .expect("the oracle connection opens"); + table_exists_conn(&client, schema, table).await +} + +/// Whether the engine-owned schema still exists server-side — the +/// structural teardown tests' instrument (`pg_namespace`, the same +/// probe the alkstore factory shape uses). +pub(crate) async fn schema_exists(dsn: &str, schema: &str) -> bool { + let client = admin_connect(dsn) + .await + .expect("the oracle connection opens"); + let count: i64 = client + .query_one( + "SELECT count(*) FROM pg_namespace WHERE nspname = $1", + &[&schema], + ) + .await + .expect("the oracle query succeeds") + .get(0); + count > 0 +} + +// ------------------------------------------------------------------ +// The mem-pglo factory — the one-window-host rule's named test +// topology, feature pair `mem` + `pg-lo`. +// ------------------------------------------------------------------ + +#[cfg(all(feature = "mem", feature = "pg-lo"))] +mod mem_pglo { + use std::io::Cursor; + use std::sync::atomic::AtomicU64; + use std::sync::{Arc, Mutex}; + use std::time::{Duration, SystemTime}; + + use tokio_postgres::NoTls; + + use crate::factory::StoreFactory; + use crate::pg_lo_column::{ + admin_dsn, companion_loids, create_db, db_dsn, db_exists, drop_db, fresh_db_name, + harness_dsn, harness_ready, lo_entry_rows, loid_for_key, loid_pages, reap_lo_schema, + window_rows, DbJanitor, TrackedDb, LO_WINDOW, WINDOW_DELAY, + }; + use crate::rows::{ + self, patterned, slice_digest, Failing, LatchProtect, SetProtect, SetSource, + }; + use alkblobs::{Error, Key, PgOpts, Store, StoreMode, DEFAULT_DISPATCH_THRESHOLD}; + + /// The column's declared dual-tier threshold — small, so the + /// shared content matrix and the race fixtures cross it and both + /// tiers' bodies are exercised (sub-threshold rows land in the + /// in-process mem kv tier; over-threshold rows stage-commit into + /// LOs behind companion rows). + pub(crate) const TUNED_THRESHOLD: u64 = 1024; + + /// One vended backing site: the dedicated database this open + /// minted and the engine-owned pg-lo schema minted inside it. The + /// kv tier is in-process mem — its placement is facade-observed + /// (the mem introspection shape), the LO medium is the schema. + #[derive(Clone)] + pub(crate) struct Site { + /// The db-scoped DSN — the oracle's independent connections + /// address it; each open's database is private and gets + /// dropped whole at teardown (the LO-isolation corollary: LOs + /// are database-level). + dsn: String, + /// The engine-owned pg-lo schema this open minted — the LO + /// medium (the dispatch oracle's LO-side counters and the + /// durable window's row home, per the one-window-host rule). + schema: String, + /// The dedicated database's name — the structural teardown + /// tests' `pg_database` probe target. + db: String, + } + + impl Site { + /// The minted pg-lo schema — the LO medium's and the window's + /// schema-qualified probe target. + pub(crate) fn schema(&self) -> &str { + &self.schema + } + + /// The DSN the open minted — scoped to the dedicated database. + pub(crate) fn dsn(&self) -> &str { + &self.dsn + } + } + + /// The dispatch-coverage factory: `kv = mem` + `large = pg-lo` at + /// the threshold this column declares and the test tuning of the + /// window delay (see [`TUNED_THRESHOLD`] and `WINDOW_DELAY` — the + /// one-window-host rule's named test topology, the durable window + /// hosting on the large engine, ADR-012 §5). + /// + /// **Why a fresh instance per runner (not a shared static):** + /// teardown drops the dedicated databases *this instance* minted + /// (having reaped their LOs and dropped their schemas); a shared + /// instance's teardown would race concurrently running rows' + /// sites. Every runner constructs its own and owns its lifecycle; + /// a row that panics before teardown is still backed by the + /// per-open drop-guard janitors. + pub(crate) struct DualTierMemPgLoFactory { + dsn: String, + /// The declared dispatch threshold — the routing boundary this + /// column's rows run at (constructor-tunable; ADR-003). + /// Declared configuration: the dual-tier store never rejects, + /// so there is nothing to probe. + threshold: u64, + /// The declared window delay — the durable schedule's cadence + /// knob the sweep rows run at (the test tuning: + /// execute-on-next-sweep). + #[allow(dead_code)] + window_delay: Duration, + seq: AtomicU64, + /// Every dedicated database this instance minted, for its own + /// teardown and its drop-guard janitors. + dbs: Mutex>, + /// The LOIDs teardown reaped, for the structural tests' + /// assertions. + reaped_loids: Mutex>, + } + + impl DualTierMemPgLoFactory { + pub(crate) fn new(threshold: u64) -> DualTierMemPgLoFactory { + DualTierMemPgLoFactory { + dsn: harness_dsn().expect("the harness server is configured for the column"), + threshold, + window_delay: WINDOW_DELAY, + seq: AtomicU64::new( + (std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.subsec_nanos() as u64) + .unwrap_or(0)) + ^ ((std::process::id() as u64) << 8), + ), + dbs: Mutex::new(Vec::new()), + reaped_loids: Mutex::new(Vec::new()), + } + } + + /// Open + track the vended site: the shape the column's own + /// rows use (they observe the LO medium, so they need the site + /// alongside the store). + pub(crate) async fn open_tracked(&self) -> Result<(Store, Site), Error> { + let maintenance = admin_dsn(&self.dsn); + let db = fresh_db_name("mem", &self.seq); + create_db(&maintenance, &db).await?; + let dsn = db_dsn(&self.dsn, &db); + let schema = crate::pg_lo_column::fresh_schema_name("mem", &self.seq); + self.dbs + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(TrackedDb { + name: db.clone(), + dsn: dsn.clone(), + schemas: vec![schema.clone()], + lo_schemas: vec![schema.clone()], + janitor: Some(DbJanitor::new(&maintenance, &db)), + }); + let store = + Store::dual_tier_mem_pglo(&dsn, opts(schema.clone()), self.threshold, WINDOW_DELAY) + .await?; + Ok((store, Site { dsn, schema, db })) + } + + /// The dedicated databases this instance minted — names, + /// db-scoped DSNs, and the schemas minted inside — for the + /// structural tests' assertions. + pub(crate) fn tracked_dbs(&self) -> Vec<(String, String, Vec)> { + self.dbs + .lock() + .unwrap_or_else(|e| e.into_inner()) + .iter() + .map(|d| (d.name.clone(), d.dsn.clone(), d.schemas.clone())) + .collect() + } + + /// The LOIDs teardown reaped, for the structural tests' + /// tracked-LOID assertions. + pub(crate) fn reaped_loids(&self) -> Vec { + self.reaped_loids + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } + + /// The maintenance DSN the factory mints databases through. + pub(crate) fn dsn(&self) -> &str { + &self.dsn + } + } + + /// The factories' schema-name policy, in engine-opts form: the + /// constructor's `PgOpts` with exactly the factory's minted + /// schema and the ship defaults for the rest (pool size, + /// `synchronous_commit` — the alkstore factory's shape: only the + /// schema name is factory policy). + fn opts(schema: String) -> PgOpts { + PgOpts { + schema, + ..PgOpts::default() + } + } + + impl StoreFactory for DualTierMemPgLoFactory { + fn name(&self) -> &'static str { + "dual-tier mem+pg-lo" + } + + async fn open(&self) -> Result { + self.open_tracked().await.map(|(store, _)| store) + } + + async fn teardown(&self) -> Result<(), Error> { + let mut dbs = self + .dbs + .lock() + .unwrap_or_else(|e| e.into_inner()) + .drain(..) + .collect::>(); + for tracked in dbs.iter_mut() { + // The LO-isolation corollary's arms: reap the tracked + // LOIDs (via the companion table — the contract + // authority), drop exactly the minted schemas, then + // drop the dedicated database whole (`WITH (FORCE)` + // reaps any connection the abandoned store's pool has + // not drained). + let maintenance = admin_dsn(&self.dsn); + let mut reaped = Vec::new(); + if let Ok(client) = admin_connect_for_teardown(&tracked.dsn).await { + for schema in &tracked.lo_schemas { + reaped.extend(reap_lo_schema(&client, schema).await); + } + for schema in &tracked.schemas { + let sql = format!("DROP SCHEMA IF EXISTS \"{schema}\" CASCADE"); + let _ = client.batch_execute(&sql).await; + } + } + if !reaped.is_empty() { + self.reaped_loids + .lock() + .unwrap_or_else(|e| e.into_inner()) + .extend(reaped); + } + drop_db(&maintenance, &tracked.name).await?; + TrackedDb::disarm(tracked); + } + Ok(()) + } + } + + async fn admin_connect_for_teardown(dsn: &str) -> Result { + let (client, connection) = tokio_postgres::connect(dsn, NoTls) + .await + .map_err(|e| Error::Io(format!("postgres harness: {e}")))?; + tokio::spawn(async move { + let _ = connection.await; + }); + Ok(client) + } + + /// The skip gate every runner checks (the SKIP posture's + /// factory-owner responsibility). + async fn ready() -> bool { + harness_ready().await + } + + // -------------------------------------------------------------- + // The column's own rows — the durable-schedule sweep family, the + // schedule-relative readings, the dispatch forms, the INV-1 + // extension, and the virgin-store row. Every row takes this + // column's concrete factory (it observes the LO medium), is + // ADR-stamped per the convention, and traces its family. + // -------------------------------------------------------------- + + /// INV-2 on the durable schedule, over both tiers: identical + /// content put twice — through the same put path and across the + /// two paths — is one pool entry, and the re-put's `Pin` protects + /// exactly as a fresh put's does. The kv leg's single-pool-entry + /// is asserted on the facade plus the LO medium's genuine absence + /// (the mem introspection shape — the oracle's kv half); the large + /// leg's is one companion row behind one LO. One-pool-entry reads + /// through durable-schedule outcomes (the INV-5 posture): the pins + /// held stage nothing; once unpinned, sweep 1 stages exactly the + /// one entry and removes nothing, and its executor sweep deletes + /// exactly that staged set (README §The schedule-relative reading + /// for SQL-backed columns). + /// + /// Contract stamp: store-api.md §Invariants 2; ADR-003 (whole-value + /// put is CAS first-writer-wins; the routing function's tier + /// placement); ADR-005 (a re-put refcounts the existing entry + /// under the same arbitration lock); ADR-009 §2/§4 (the companion + /// table is the contract authority; delete = lo_unlink + row + /// delete); ADR-008 §2 (the staged re-arbitrated delete); ADR-012 + /// §5 (the window hosts on the pg-lo engine in this topology); + /// ADR-012 §6.7 (the executor is the next sweep — the schedule + /// these counts read against). + pub async fn dedup_identical_content_is_one_pool_entry(factory: &DualTierMemPgLoFactory) { + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + // The kv leg (in-process mem): both refcounts compose under + // the same lock, nothing ever lands on the LO medium. + let content = b"deduplicated content".to_vec(); + let canonical = Key::derive(&content, content.len() as u64); + let (key, first) = store + .put(Some(content.len() as u64), Cursor::new(content.as_slice())) + .await + .expect("the fresh put publishes"); + let (reput, second) = store + .put(Some(content.len() as u64), Cursor::new(content.as_slice())) + .await + .expect("the same-path re-put publishes"); + assert_eq!(key, canonical); + assert_eq!(reput, canonical, "one derivation, one address"); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(report.aborted, None); + assert_eq!( + report.candidates_staged, 0, + "one pool entry, both refcounts composing under the same lock" + ); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + 0, + "the kv-tier entries never land on the LO medium" + ); + drop(first); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + report.candidates_staged, 0, + "the re-put's pin protects exactly as a fresh put's" + ); + drop(second); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (report.candidates_staged, report.deleted), + (1, 0), + "exactly one pool entry all along — staged this sweep, not executed" + ); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (report.deleted, report.cancelled_by_arbitration), + (1, 0), + "the executing sweep deleted exactly the staged entry" + ); + assert!(!store.has(&canonical.as_bytes()).await.unwrap()); + + // The large leg: identical content through both put paths — + // one companion row behind one LO, both refcounts composing. + let cross = patterned(2 * TUNED_THRESHOLD as usize + 5, 167); + let cross_canonical = Key::derive(&cross, cross.len() as u64); + let (key, first) = store + .put(Some(cross.len() as u64), Cursor::new(cross.as_slice())) + .await + .expect("the known-length put publishes"); + let (reput, second) = store + .put(None, Cursor::new(cross.as_slice())) + .await + .expect("the unknown-length re-put publishes"); + assert_eq!(key, cross_canonical); + assert_eq!(reput, cross_canonical, "one pool entry across the paths"); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (report.aborted, report.candidates_staged), + (None, 0), + "both refcounts compose" + ); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + 1, + "one pool entry across both paths — one companion row" + ); + let loids = companion_loids(site.dsn(), site.schema()).await; + assert_eq!(loids.len(), 1, "one LO behind the companion row"); + assert!( + loid_pages(site.dsn(), loids[0]).await > 0, + "the LO holds the content's pages (never zero)" + ); + drop(second); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + report.candidates_staged, 0, + "the re-put's pin protects exactly as a fresh put's (other drop order)" + ); + drop(first); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (report.candidates_staged, report.deleted), + (1, 0), + "exactly one pool entry all along" + ); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(report.deleted, 1, "executed exactly the staged set"); + assert!(!store.has(&cross_canonical.as_bytes()).await.unwrap()); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + 0, + "the executed delete emptied the companion table" + ); + assert_eq!( + loid_pages(site.dsn(), loids[0]).await, + 0, + "delete is one tx: the LO's pages went with the row" + ); + assert_eq!( + window_rows(site.dsn(), site.schema()).await, + 0, + "the window holds nothing after both executed rounds (the pg-lo host)" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// INV-3 on the durable schedule: stage hygiene over the LO + /// medium's real staged form — every put-path failure leaves zero + /// residue (no companion rows, no LO pages: the held-put + /// transaction's every failure path converges on ROLLBACK) and + /// every success leaves exactly one committed entry, over both put + /// paths and both injectable failure modes (a mid-stream stream + /// failure and a declared-length mismatch), observed through + /// durable-schedule sweep outcomes: zero staged after every + /// failure, and a success staged exactly once, then executed + /// exactly once. + /// + /// Contract stamp: store-api.md §Invariants 3; ADR-010 §1 (the + /// staged-put contract: commit publishes, every failure path + /// discards); ADR-003 (stage-then-commit; the routing places this + /// content on the large tier); ADR-009 §2 (put = one tx — the + /// staged form is the held tx, so a failure's LO pages roll back + /// with it); ADR-012 §6.7 (the schedule-relative reading the + /// success counts follow). + pub async fn stage_hygiene_zero_residue_on_failure_exactly_one_entry_on_success( + factory: &DualTierMemPgLoFactory, + ) { + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + let content = patterned(2 * TUNED_THRESHOLD as usize + 17, 167); + let len = content.len() as u64; + let canonical = Key::derive(&content, len); + + // The known path's mid-stream failure — the held tx rolls + // back: no companion row, no LO pages. + let err = store + .put(Some(len), Failing::new(&content, 40)) + .await + .expect_err("the failed mid-stream put rejects"); + assert!( + matches!(err, Error::Io(_)), + "the stream failure surfaces as Io: {err}" + ); + // The unknown path's buffered failure (before the threshold + // overflow) and its staged failure (the held tx after the + // overflow) — both roll back to zero residue. + let err = store + .put(None, Failing::new(&content, 40)) + .await + .expect_err("the failed unknown-length put rejects before staging"); + assert!( + matches!(err, Error::Io(_)), + "the stream failure surfaces as Io: {err}" + ); + let err = store + .put(None, Failing::new(&content, TUNED_THRESHOLD as usize + 600)) + .await + .expect_err("the failed unknown-length put rejects mid-stage"); + assert!( + matches!(err, Error::Io(_)), + "the stream failure surfaces as Io: {err}" + ); + // Declared-length mismatches, over-fed and under-fed — both + // above the threshold, so the staged form began its LO and + // every failure path discards it. + let err = store + .put(Some(TUNED_THRESHOLD + 10), Cursor::new(content.as_slice())) + .await + .expect_err("the over-fed mismatch rejects"); + assert_eq!(err, Error::Verification, "declared-len mismatch rejects"); + let err = store + .put(Some(len + 17), Cursor::new(content.as_slice())) + .await + .expect_err("the under-fed mismatch rejects"); + assert_eq!(err, Error::Verification, "declared-len mismatch rejects"); + + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(report.aborted, None); + assert_eq!( + ( + report.candidates_staged, + report.deleted, + report.cancelled_by_arbitration + ), + (0, 0, 0), + "zero residue after every failure, both put paths, both failure modes" + ); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + 0, + "the companion table holds zero entries after every failure" + ); + assert_eq!( + window_rows(site.dsn(), site.schema()).await, + 0, + "the window staged nothing behind the failures" + ); + + // Success, known path: exactly one committed entry — one + // companion row behind one LO — staged once, executed once. + let (key, pin) = store + .put(Some(len), Cursor::new(content.as_slice())) + .await + .expect("the known-length put publishes"); + assert_eq!(key, canonical); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + 1, + "exactly one committed entry on success (one companion row)" + ); + let loid = loid_for_key(site.dsn(), site.schema(), &key.as_bytes()) + .await + .expect("the companion row carries its LOID"); + assert!( + loid_pages(site.dsn(), loid).await > 0, + "the committed LO holds the content's pages" + ); + drop(pin); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (report.candidates_staged, report.deleted), + (1, 0), + "exactly one committed entry staged on success" + ); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + report.deleted, 1, + "the executing sweep removed exactly that one entry" + ); + assert!(!store.has(&canonical.as_bytes()).await.unwrap()); + assert_eq!(loid_pages(site.dsn(), loid).await, 0); + + // Success, unknown path: the overflow staged into the held tx, + // the commit published — exactly one committed entry on the + // other put path too. + let (key, pin) = store + .put(None, Cursor::new(content.as_slice())) + .await + .expect("the unknown-length put publishes"); + assert_eq!(key, canonical); + drop(pin); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (report.candidates_staged, report.deleted), + (1, 0), + "exactly one committed entry staged on the other put path too" + ); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(report.deleted, 1, "and executed exactly that staged set"); + assert!(!store.has(&canonical.as_bytes()).await.unwrap()); + assert_eq!( + ( + lo_entry_rows(site.dsn(), site.schema()).await, + window_rows(site.dsn(), site.schema()).await + ), + (0, 0), + "the medium and the window are both back to zero" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// INV-4, the durable-schedule happy shape over both tiers: the + /// full protection set spanning both tiers (a source-live kv + /// entry, a pinned large-tier entry, a protect-named large-tier + /// entry) plus one dead entry per tier — sweep 1 stages exactly + /// the two dead candidates into the **pg-lo schema's `gc_window`** + /// (the window hosted on the large engine — the one-window-host + /// rule's named topology) and deletes nothing; sweep 2 executes + /// exactly that staged set, and the protected entries stay + /// byte-identical. The medium oracle distinguishes the tiers (the + /// companion table's rows behind LOs vs the kv tier's + /// facade-observed residency) and watches the schedule itself. + /// + /// Contract stamp: store-api.md §Invariants 4 (exact counts with + /// correct liveness) and §Invariants 5 (list correctness through + /// sweep outcomes); ADR-005 (three liveness marks; delete windows + /// with arbitrate-at-delete-time); ADR-008 §2 (the staged + /// re-arbitrated delete; the constructor-tunable delay); ADR-009 + /// §2/§4 (the companion table; GC-participating deletes); ADR-012 + /// §1 (sweeps are never errors); ADR-012 §5 (the one-window-host + /// rule's named topology — the window on the large engine); ADR-012 + /// §6.7 (the executor is the next sweep). + pub async fn two_sweep_exact_counts_over_the_full_protection_set( + factory: &DualTierMemPgLoFactory, + ) { + let threshold = factory.threshold; + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + let source = SetSource::register(&store); + let protect = Arc::new(SetProtect::default()); + store.register_protect_callback(protect.clone()); + + let source_live = patterned((threshold - 1) as usize, 191); + let (key_live, _) = store + .put( + Some(source_live.len() as u64), + Cursor::new(source_live.as_slice()), + ) + .await + .expect("the source-live entry publishes"); + source.add(key_live.clone()); + + let pinned = patterned(2 * threshold as usize + 7, 173); + let (key_pinned, pin_pinned) = store + .put(None, Cursor::new(pinned.as_slice())) + .await + .expect("the pinned entry publishes"); + + let protected = patterned(2 * threshold as usize + 11, 167); + let (key_protected, _) = store + .put( + Some(protected.len() as u64), + Cursor::new(protected.as_slice()), + ) + .await + .expect("the protect-named entry publishes"); + protect.set_keys(vec![key_protected.clone()]); + + let dead_kv = patterned((threshold - 3) as usize, 149); + let (key_dead_kv, pin) = store + .put(Some(dead_kv.len() as u64), Cursor::new(dead_kv.as_slice())) + .await + .expect("the kv-tier dead entry publishes"); + drop(pin); + let dead_large = patterned(2 * threshold as usize + 13, 151); + let (key_dead_large, pin) = store + .put(None, Cursor::new(dead_large.as_slice())) + .await + .expect("the large-tier dead entry publishes"); + drop(pin); + + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + 3, + "before the sweep, the LO medium holds exactly the three \ + large-tier entries (the kv tier's entries are in-process)" + ); + let dead_large_loid = loid_for_key(site.dsn(), site.schema(), &key_dead_large.as_bytes()) + .await + .expect("the dead large entry's companion row carries its LOID"); + assert!( + loid_pages(site.dsn(), dead_large_loid).await > 0, + "the LO medium holds the dead large entry's pages too" + ); + assert!( + store.has(&key_live.as_bytes()).await.unwrap() + && store.has(&key_dead_kv.as_bytes()).await.unwrap() + && store.has(&key_dead_large.as_bytes()).await.unwrap(), + "the pool spans both tiers and holds exactly its five entries" + ); + + let first = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(first.aborted, None); + assert_eq!( + ( + first.candidates_staged, + first.deleted, + first.cancelled_by_arbitration + ), + (2, 0, 0), + "sweep 1 staged exactly the two dead entries and deleted \ + nothing — staging is not executing" + ); + assert_eq!( + window_rows(site.dsn(), site.schema()).await, + 2, + "the staged set waits in the durable window — hosted on the \ + pg-lo schema (the one-window-host rule's large-engine arm)" + ); + for (key, content) in [ + (&key_live, &source_live), + (&key_pinned, &pinned), + (&key_protected, &protected), + ] { + let (len, blob) = store + .get(&key.as_bytes()) + .await + .expect("get succeeds") + .expect("the protected entry is present"); + assert_eq!( + blob.read_at(0, len).await.expect("the window reads"), + content.as_slice(), + "the protected entry is byte-identical after the staging sweep" + ); + } + assert!( + store.has(&key_live.as_bytes()).await.unwrap() + && store.has(&key_pinned.as_bytes()).await.unwrap() + && store.has(&key_protected.as_bytes()).await.unwrap(), + "the staged sweep deleted nothing: every entry intact" + ); + + let second = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(second.aborted, None); + assert_eq!( + second.candidates_staged, 0, + "nothing newly dead since the staging sweep" + ); + assert_eq!( + second.deleted + second.cancelled_by_arbitration, + 2, + "sweep 2 executed exactly the staged set" + ); + assert_eq!( + (second.deleted, second.cancelled_by_arbitration), + (2, 0), + "with correct liveness, every executed candidate deleted" + ); + assert!(!store.has(&key_dead_kv.as_bytes()).await.unwrap()); + assert!(!store.has(&key_dead_large.as_bytes()).await.unwrap()); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + 2, + "the executed deletes landed on both media exactly (the dead \ + large entry's companion row is gone)" + ); + assert_eq!( + loid_pages(site.dsn(), dead_large_loid).await, + 0, + "the dead large entry's LO pages are gone with it" + ); + assert_eq!( + window_rows(site.dsn(), site.schema()).await, + 0, + "the executed rows left the durable window" + ); + + drop(pin_pinned); + let third = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (third.candidates_staged, third.deleted), + (1, 0), + "with the pin dropped, the formerly pinned entry stages" + ); + let fourth = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + fourth.deleted, 1, + "and the next sweep executes it — exactly one deletion" + ); + assert!(!store.has(&key_pinned.as_bytes()).await.unwrap()); + assert!(store.has(&key_live.as_bytes()).await.unwrap()); + assert!(store.has(&key_protected.as_bytes()).await.unwrap()); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + 1, + "exactly the protect-named entry's companion row remains" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// INV-4, the race, in its durable-schedule form: a put landing + /// *during the window* survives the executing sweep — the re-put's + /// pin is minted (pin-before-publish) between the staging sweep + /// and the executing sweep, so the execution-time re-arbitration + /// cancels the pinned candidate while the unpinned twin deletes; + /// once the pin drops, the twin re-stages and executes exactly + /// once. Both put paths race in-body over both tiers (a kv twin + /// and a large twin), across the window delay the pg-lo host + /// carries. + /// + /// Contract stamp: store-api.md §Invariants 4; ADR-005 (pin-before- + /// publish ordering; the delete window closes the sweep-vs-put race); + /// ADR-008 §2 (the staged re-arbitrated delete — the race holds + /// across the window delay); ADR-009 §4 (GC-participating deletes + /// over the companion table); ADR-012 §5 (the window on the large + /// engine in this topology); ADR-012 §6.7 (execution-time + /// re-arbitration is the executor sweep's posture). + pub async fn a_put_landing_during_the_window_survives_the_executing_sweep( + factory: &DualTierMemPgLoFactory, + ) { + let threshold = factory.threshold; + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + let rounds: [(String, Vec, bool, bool, bool); 2] = [ + ( + "the kv twin (known-path re-put)".to_owned(), + patterned((threshold - 1) as usize, 191), + true, + true, + false, + ), + ( + "the large twin (unknown-path re-put)".to_owned(), + patterned(2 * threshold as usize + 5, 173), + false, + false, + true, + ), + ]; + for (index, (tag, content, original_known, reput_known, lands_large)) in + rounds.iter().enumerate() + { + let len = content.len() as u64; + let twin = Key::derive(content, len); + let witness = format!("the dead witness of round {index}").into_bytes(); + let witness_key = Key::derive(&witness, witness.len() as u64); + + let original = if *original_known { Some(len) } else { None }; + let (_, pin) = store + .put(original, Cursor::new(content.as_slice())) + .await + .expect("the twin publishes"); + drop(pin); + let (_, pin) = store + .put(None, Cursor::new(witness.as_slice())) + .await + .expect("the witness publishes"); + drop(pin); + + let staged = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + ( + staged.aborted, + staged.candidates_staged, + staged.deleted, + staged.cancelled_by_arbitration + ), + (None, 2, 0, 0), + "{tag}: the staging sweep stages exactly the two dead \ + twins and deletes nothing" + ); + + // The put lands during the window (the re-put dedups into + // the existing twin and refcounts its pin): protected + // across the delay, pin-before-publish. + let reput = if *reput_known { Some(len) } else { None }; + let (_, pin) = store + .put(reput, Cursor::new(content.as_slice())) + .await + .expect("the re-put publishes"); + let executed = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(executed.aborted, None, "{tag}"); + assert_eq!( + executed.candidates_staged, 0, + "{tag}: nothing newly dead since the staging sweep" + ); + assert_eq!( + executed.deleted + executed.cancelled_by_arbitration, + 2, + "{tag}: the executing sweep ran exactly the staged set" + ); + assert_eq!(executed.deleted, 1, "{tag}: the unpinned witness deleted"); + assert_eq!( + executed.cancelled_by_arbitration, 1, + "{tag}: the pinned twin's candidate cancelled at execution time" + ); + assert!( + !store.has(&witness_key.as_bytes()).await.unwrap(), + "{tag}: the unpinned twin is gone" + ); + let (got_len, blob) = store + .get(&twin.as_bytes()) + .await + .expect("get succeeds") + .expect("{tag}: the put landing during the window survived"); + assert_eq!(got_len, len, "{tag}"); + assert_eq!( + blob.read_at(0, len).await.expect("the window reads"), + content.as_slice(), + "{tag}: the surviving twin is byte-identical" + ); + drop(pin); + + let restaged = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (restaged.candidates_staged, restaged.deleted), + (1, 0), + "{tag}: the pin dropped — the twin re-stages" + ); + let executed = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + executed.deleted, 1, + "{tag}: the re-staged candidate executes exactly once" + ); + assert!(!store.has(&twin.as_bytes()).await.unwrap()); + assert_eq!( + window_rows(site.dsn(), site.schema()).await, + 0, + "{tag}: the window holds nothing after the executed round \ + (the pg-lo host)" + ); + if *lands_large { + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + 0, + "{tag}: the executed LO deletes reaped the medium exactly" + ); + } + } + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// INV-4, abort shape: the sweep is single-flight on the durable + /// form too — the in-process sweep lease (ADR-013 §2's single-node + /// analog). The contended second sweeper aborts with + /// `SweeperLock` in a completed `Ok(SweepReport)` — nothing + /// executed, nothing staged, nothing deleted (abort before + /// staging: form-independent) — while the lease holder's sweep + /// completes by **staging only**: its staged set waits in the + /// pg-lo schema's `gc_window` for the next sweep to execute (the + /// durable form's schedule-relative replacement for the reference + /// row's inline holder-completion leg). + /// + /// Contract stamp: store-api.md §Invariants 4; ADR-012 §1 (abort is + /// report data, never an error); ADR-013 §2 (the named exclusive + /// sweep lease — one sweeper per pool, holdership ends at release); + /// ADR-012 §5 (the window hosts on the large engine in this + /// topology); ADR-012 §6.7 (the executor is the next sweep — the + /// holder stages, it does not execute). + pub async fn the_contended_second_sweeper_aborts_as_data_and_the_holder_stages_only( + factory: &DualTierMemPgLoFactory, + ) { + let threshold = factory.threshold; + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + let (entered_tx, entered_rx) = std::sync::mpsc::channel::<()>(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + store.register_protect_callback(Arc::new(LatchProtect { + entered: entered_tx, + release: parking_lot::Mutex::new(Some(release_rx)), + })); + + let content = patterned(2 * threshold as usize + 9, 167); + let (key, pin) = store + .put(Some(content.len() as u64), Cursor::new(content.as_slice())) + .await + .expect("the dead witness publishes"); + drop(pin); + + // The lease holder sweeps on its own task (its protect consult + // blocks that task's thread mid-callback); this task contends + // and then releases the holder. + let store = Arc::new(store); + let holder = tokio::spawn({ + let store = Arc::clone(&store); + async move { store.sweep().await } + }); + + while entered_rx + .recv_timeout(std::time::Duration::from_millis(50)) + .is_err() + { + tokio::task::yield_now().await; + } + + let second = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + second.aborted, + Some(alkblobs::GcAbortCause::SweeperLock), + "the contended sweeper aborts as report data" + ); + assert_eq!( + ( + second.candidates_staged, + second.deleted, + second.cancelled_by_arbitration + ), + (0, 0, 0), + "the contended sweep executes, stages, and deletes nothing" + ); + assert!( + store.has(&key.as_bytes()).await.unwrap(), + "the contended sweep deleted nothing" + ); + assert_eq!( + window_rows(site.dsn(), site.schema()).await, + 0, + "the aborted sweep staged nothing into the window" + ); + release_tx.send(()).ok(); + + let report = holder + .await + .expect("the holder task joined") + .expect("sweeps are never errors"); + assert_eq!(report.aborted, None, "the lease holder completes"); + assert_eq!( + (report.candidates_staged, report.deleted), + (1, 0), + "the holder stages exactly the dead set — it does not execute \ + (the durable form's schedule)" + ); + assert_eq!( + window_rows(site.dsn(), site.schema()).await, + 1, + "the holder's staged set waits in the durable window" + ); + assert!( + store.has(&key.as_bytes()).await.unwrap(), + "the holder staged, did not execute: the entry is intact" + ); + + let executed = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + executed.deleted, 1, + "the next sweep is the executor of the staged set" + ); + assert!(!store.has(&key.as_bytes()).await.unwrap()); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + 0, + "the executed delete emptied the companion table" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// INV-5 on the durable schedule: `list()` correctness is + /// observable only through GC — the enumeration is complete and + /// well-formed in both directions, proven by exact + /// durable-schedule outcomes: the staging sweep stages exactly the + /// dead set (neither more nor fewer candidates), the executing + /// sweep deletes exactly that staged set, and a liveness flip + /// re-stages exactly the newly dead entry. Never a direct list + /// comparison. + /// + /// Contract stamp: store-api.md §Invariants 5; ADR-003 (`list()` is + /// complete and well-formed by contract — load-bearing for GC); + /// ADR-005 (the sweep's live-set diff rides `list()`); ADR-012 §5 + /// (the window hosts on the large engine in this topology); ADR-012 + /// §6.7 (the executor is the next sweep — the schedule these + /// counts read against). + pub async fn list_correctness_is_observable_through_durable_schedule_outcomes( + factory: &DualTierMemPgLoFactory, + ) { + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + let source = SetSource::register(&store); + + let mut keys = Vec::new(); + for (index, content) in [ + b"kept live by the source, entry one".to_vec(), + b"kept live by the source, entry two".to_vec(), + b"dead at the first staging, entry three".to_vec(), + b"dead at the first staging, entry four".to_vec(), + b"dead at the first staging, entry five".to_vec(), + ] + .into_iter() + .enumerate() + { + let declared = if index % 2 == 0 { + Some(content.len() as u64) + } else { + None + }; + let (key, pin) = store + .put(declared, Cursor::new(content.as_slice())) + .await + .expect("the entry publishes"); + drop(pin); + keys.push(key); + } + source.add(keys[0].clone()); + source.add(keys[1].clone()); + + let first = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(first.aborted, None); + assert_eq!( + (first.candidates_staged, first.deleted), + (3, 0), + "the enumeration staged exactly the three dead entries — \ + complete and well-formed in both directions" + ); + for key in &keys { + assert!( + store.has(&key.as_bytes()).await.unwrap(), + "staging deleted nothing: every enumerated entry is intact" + ); + } + assert_eq!( + window_rows(site.dsn(), site.schema()).await, + 3, + "the staged schedule holds the three candidates" + ); + + let second = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (second.candidates_staged, second.deleted), + (0, 3), + "the executing sweep removed exactly the staged set" + ); + assert!(store.has(&keys[0].as_bytes()).await.unwrap()); + assert!(store.has(&keys[1].as_bytes()).await.unwrap()); + for key in &keys[2..] { + assert!(!store.has(&key.as_bytes()).await.unwrap()); + } + + source.remove(&keys[1]); + let third = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (third.candidates_staged, third.deleted), + (1, 0), + "the newly dead entry is staged exactly" + ); + let fourth = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(fourth.deleted, 1, "and executed exactly by the next sweep"); + assert!(!store.has(&keys[1].as_bytes()).await.unwrap()); + assert!(store.has(&keys[0].as_bytes()).await.unwrap()); + + source.remove(&keys[0]); + let fifth = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(fifth.candidates_staged, 1, "the last entry stages exactly"); + let sixth = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(sixth.deleted, 1, "and executes exactly"); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// Dispatch row: **routing is a pure function of length** — at, + /// below, and over the declared threshold, the same content lands + /// on exactly the same tier through both put paths (same placement + /// every time; nothing moves, nothing duplicates), identical + /// content through both paths is one pool entry, and every read is + /// byte-identical from wherever that tier holds it. + /// + /// Tier placement is observed through the LO medium (the oracle's + /// LO half): large entries hold a companion row behind an LO; kv + /// entries hold nothing on the medium — the facade's served reads + /// and `has()` are the mem kv half's introspection shape (the mem + /// column's form; the kv tier is in-process, facade-invisible by + /// design). + /// + /// Contract stamp: store-api.md §Invariants 7 (facade-contract row); + /// ADR-003 §Dispatch policy (routing is a pure function of length; + /// no migration; CAS first-writer-wins makes re-puts one entry); + /// ADR-005 (re-put refcounts); ADR-009 §2 (the companion table is + /// the contract authority); ADR-012 §5 (this topology's named + /// shape); ADR-012 §6.3 (the dual-tier dispatch-coverage shape); + /// ADR-012 §6.7 (the schedule-relative cleanup the dedup legs read + /// against). + pub async fn routing_is_a_pure_function_of_length_across_put_paths_and_re_puts( + factory: &DualTierMemPgLoFactory, + ) { + let threshold = factory.threshold; + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + for content in [ + patterned((threshold - 1) as usize, 191), + patterned(threshold as usize, 173), + patterned((threshold + 1) as usize, 167), + patterned(4 * threshold as usize + 9, 151), + ] { + let len = content.len() as u64; + let canonical = Key::derive(&content, len); + let expected_companions = if len > threshold { 1 } else { 0 }; + + for (path_tag, declared) in [("the known path", Some(len)), ("the unknown path", None)] + { + let (key, pin) = store + .put(declared, Cursor::new(content.as_slice())) + .await + .expect("put publishes"); + assert_eq!( + key, canonical, + "the canonical derivation addresses the put on {path_tag}" + ); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + expected_companions, + "length {len} on {path_tag}: the placement is exactly the \ + routing function's answer and never moves or duplicates \ + (the LO medium holds {expected_companions} companion rows)" + ); + assert!( + store.has(&key.as_bytes()).await.unwrap(), + "length {len} on {path_tag}: the pool holds the entry (the \ + mem kv half's introspection shape: served reads, no \ + medium of its own)" + ); + let (got_len, blob) = store + .get(&key.as_bytes()) + .await + .expect("get succeeds") + .expect("the entry is present"); + assert_eq!( + got_len, len, + "the read serves the entry from its placed tier" + ); + assert_eq!( + blob.read_at(0, len).await.expect("the whole window reads"), + content, + "the entry is byte-identical from its placed tier across paths" + ); + drop(pin); + } + + let staged = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + staged.candidates_staged, 1, + "identical content through both put paths is one pool entry" + ); + let executed = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + executed.deleted, 1, + "the executing sweep removed exactly the one entry (durable \ + schedule: staged this sweep, executed by the next)" + ); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + 0, + "the LO medium is back to zero" + ); + } + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// Dispatch row: **threshold boundary lengths route + /// deterministically** — at/below/above the factory's declared + /// threshold, on both put paths, the placement is the routing + /// function's answer every time (the LO medium's companion rows + /// against the mem kv half's facade shape); and the + /// default-threshold boundary re-routes with the tuning (the + /// constructor-tuned threshold deterministically places the + /// default boundary's length on the tier its own threshold names). + /// Run against two factories by the runner — the column's tuned + /// threshold and the shipped default. The durable cleanup: the + /// staged sweep stages exactly the six entries' dead set and the + /// next sweep executes it. + /// + /// Contract stamp: store-api.md §Invariants 7 (facade-contract row); + /// ADR-003 §Dispatch policy (default 128 KiB at-or-below routes kv; + /// constructor-tunable without API change); ADR-009 §2 (the + /// companion table); ADR-012 §5 (this topology's named shape); + /// ADR-012 §6.3 (the dual-tier dispatch-coverage shape); ADR-012 + /// §6.7 (schedule-relative cleanup). + pub async fn threshold_boundary_lengths_route_deterministically( + factory: &DualTierMemPgLoFactory, + ) { + let threshold = factory.threshold; + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + let boundary = [ + (threshold - 1, 191u32, false), + (threshold, 173, false), + (threshold, 167, false), + (threshold + 1, 151, true), + (threshold + 1, 139, true), + ]; + let mut expected_companions = 0u64; + for (len, modulus, lands_large) in boundary { + let content = patterned(len as usize, modulus); + for declared in [Some(content.len() as u64), None] { + let (key, pin) = store + .put(declared, Cursor::new(content.as_slice())) + .await + .expect("put publishes"); + assert_eq!( + key, + Key::derive(&content, len), + "the canonical derivation addresses the put" + ); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await as u64, + expected_companions + if lands_large { 1 } else { 0 }, + "boundary length {len}: at/below the threshold lands kv \ + (nothing on the LO medium), over it lands large (one \ + companion row behind an LO), both put paths" + ); + drop(pin); + } + expected_companions += if lands_large { 1 } else { 0 }; + } + + // The default boundary re-routes with the constructor's + // tuning: the shipped default's length belongs to the tier + // this configuration's own threshold puts it on. + let at_default = patterned(DEFAULT_DISPATCH_THRESHOLD as usize, 199); + let default_lands_large = DEFAULT_DISPATCH_THRESHOLD > threshold; + let (key, pin) = store + .put( + Some(DEFAULT_DISPATCH_THRESHOLD), + Cursor::new(at_default.as_slice()), + ) + .await + .expect("the default-boundary put publishes"); + assert_eq!( + key, + Key::derive(&at_default, DEFAULT_DISPATCH_THRESHOLD), + "the canonical derivation addresses the put" + ); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await as u64, + expected_companions + if default_lands_large { 1 } else { 0 }, + "the default boundary's length routes to the tier this \ + configuration's tuning names (the LO medium's answer)" + ); + drop(pin); + + let staged = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + staged.candidates_staged, 6, + "each leg's content was exactly one pool entry" + ); + let executed = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + executed.deleted, 6, + "the executing sweep removed every entry, tier notwithstanding" + ); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + 0, + "the LO medium is back to zero" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// Dispatch row: **a kv-tier miss falls through to the large tier + /// on every key-addressed op** — content the routing function + /// placed on the large tier (one companion row behind an LO; the + /// LO medium and `pg_largeobject` hold the pages while the kv tier + /// misses — the kv half is in-process mem, facade-served) serves + /// `get`, `stat`, `has`, and `read_range` whole, exact, and + /// digest-correct; the mirror-position leg proves the kv tier is + /// the *first* consult (sub-threshold content, nothing on the LO + /// medium, reads still served); and both-tier misses resolve per + /// the error model's per-op shapes (reached here by the durable + /// schedule's staged-then-executed deletion of the subject entry). + /// + /// Contract stamp: store-api.md §Read path (fall-through consult + /// order); ADR-012 §6.1 (every key-addressed op consults kv first, + /// then large; the general rule); ADR-012 §6.5 ruling 5 (both-tier + /// miss shapes: `None` on get/stat, `Err(Missing)` on read_range); + /// ADR-006 (the slice digest); ADR-009 §2/§3 (the companion table + /// and the `lo_get` windows serving the fall-through); + /// ADR-012 §6.3 (the dispatch-coverage shape). + pub async fn kv_tier_misses_fall_through_to_the_large_tier_per_op( + factory: &DualTierMemPgLoFactory, + ) { + let threshold = factory.threshold; + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + // Over-threshold content — the kv tier misses, the large tier + // serves every op. Both put paths hold the shape. + for (content, declared) in [ + ( + patterned(3 * threshold as usize + 41, 181), + Some(3 * threshold + 41), + ), + (patterned(3 * threshold as usize + 43, 197), None), + ] { + let len = content.len() as u64; + let (key, pin) = store + .put(declared, Cursor::new(content.as_slice())) + .await + .expect("put publishes"); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + 1, + "the content lives on exactly the large tier — the kv tier \ + misses" + ); + let loid = loid_for_key(site.dsn(), site.schema(), &key.as_bytes()) + .await + .expect("the companion row carries its LOID"); + assert!( + loid_pages(site.dsn(), loid).await > 0, + "the LO medium holds the entry's pages" + ); + + let (got_len, blob) = store + .get(&key.as_bytes()) + .await + .expect("get succeeds") + .expect("the large tier served the kv miss"); + assert_eq!(got_len, len); + assert_eq!( + blob.read_at(0, len).await.expect("the whole window reads"), + content, + "the fall-through read is byte-identical" + ); + assert_eq!( + store.stat(&key.as_bytes()).await.expect("stat succeeds"), + Some(alkblobs::EntryMeta { + len, + algorithm: key.algorithm(), + }), + "stat falls through and carries the length plus algorithm" + ); + assert!( + store.has(&key.as_bytes()).await.unwrap(), + "has falls through" + ); + let mid = len / 2; + let (slice, digest) = store + .read_range(&key.as_bytes(), mid, 17) + .await + .expect("read_range falls through"); + assert_eq!( + slice, + content[(mid as usize)..(mid as usize + 17)], + "the range slice is exact" + ); + assert_eq!( + digest, + slice_digest(&slice), + "the slice digest is SHA-256 over exactly the returned bytes" + ); + + // Sweep the entry away (the pin dropped): the miss shapes + // now resolve per op — the large tier misses too. The + // durable schedule: staged this sweep, executed by the next. + drop(pin); + let staged = store.sweep().await.expect("sweeps are never errors"); + assert_eq!((staged.candidates_staged, staged.deleted), (1, 0)); + let executed = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + executed.deleted, 1, + "exactly the one entry died across the two sweeps" + ); + assert!( + store + .get(&key.as_bytes()) + .await + .expect("get succeeds") + .is_none(), + "a both-tier miss on get is Ok(None)" + ); + assert_eq!( + store.stat(&key.as_bytes()).await.expect("stat succeeds"), + None, + "a both-tier miss on stat is Ok(None)" + ); + assert!( + !store.has(&key.as_bytes()).await.unwrap(), + "a both-tier miss on has is false" + ); + assert_eq!( + store + .read_range(&key.as_bytes(), 0, 4) + .await + .expect_err("the both-tier miss errs"), + Error::Missing, + "a both-tier miss on read_range is Err(Missing)" + ); + } + + // The mirror-position leg: sub-threshold content — the kv tier + // is the first consult and serves before the large tier is + // reached (nothing ever lands on the LO medium). + let small = patterned((threshold - 1) as usize, 149); + let (key, pin) = store + .put(Some(small.len() as u64), Cursor::new(small.as_slice())) + .await + .expect("the sub-threshold put publishes"); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + 0, + "the kv tier holds the sub-threshold entry — the large tier \ + was never consulted on the write, never reached on the read" + ); + let (len, blob) = store + .get(&key.as_bytes()) + .await + .expect("get succeeds") + .expect("the kv tier served"); + assert_eq!(len, small.len() as u64); + assert_eq!( + blob.read_at(0, len).await.expect("the whole window reads"), + small + ); + + drop(pin); + + // Both-tier miss shapes for a key that never existed at all, + // per op, over the fresh both-tier store. + let absent = b"never put here".to_vec(); + let absent = Key::derive(&absent, absent.len() as u64); + assert!( + store + .get(&absent.as_bytes()) + .await + .expect("get succeeds") + .is_none(), + "never-put: get is Ok(None)" + ); + assert_eq!( + store.stat(&absent.as_bytes()).await.expect("stat succeeds"), + None, + "never-put: stat is Ok(None)" + ); + assert!( + !store.has(&absent.as_bytes()).await.unwrap(), + "never-put: has is false" + ); + assert_eq!( + store + .read_range(&absent.as_bytes(), 0, 4) + .await + .expect_err("never-put read_range errs"), + Error::Missing, + "never-put: read_range is Err(Missing)" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// INV-1's LO-medium extension: **the `lo_get` windows read + /// across window boundaries byte-exact, and a body short of its + /// claimed length fails cleanly** — the round-trip family's ranged + /// reads extended to the LO medium. Entries on the large tier + /// (both put paths) read whole through the multi-window assembly, + /// straddle the shipped window edge (one read crossing two + /// `lo_get` statements), read ranged slices with the ADR-006 + /// digest over exactly the returned bytes, clamp at EOF (tail, + /// at-EOF empty, a `u64::MAX` offset clamping in `u64` arithmetic, + /// zero-length, over-read), and — the beyond-length clean failure + /// — a companion row overclaiming its size (a corrupted half-state + /// healthy stores never hold, planted through the oracle) + /// surfaces as the unexpected-EOF-shaped `Error::Io`, never a + /// partial read or a panic. + /// + /// Contract stamp: store-api.md §Invariants 1 (the round-trip + /// family's ranged-read legs) and §Read path (the streaming + /// large-tier windows, pread-style clamping); ADR-009 §2/§3 (the + /// companion table is the authority — the cursor closes over the + /// row; descriptorless `lo_get` windows, the shipped read shape + /// and its unexpected-EOF failure); ADR-006 (the slice digest over + /// exactly the returned bytes); ADR-010 §1 (the large-tier cursor + /// body family). + pub async fn lo_windows_read_across_window_boundaries_and_a_short_body_fails_cleanly( + factory: &DualTierMemPgLoFactory, + ) { + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + let content = patterned(2 * LO_WINDOW + 100, 167); + let len = content.len() as u64; + for (path_tag, declared) in [("the known path", Some(len)), ("the unknown path", None)] { + let (key, pin) = store + .put(declared, Cursor::new(content.as_slice())) + .await + .expect("put publishes"); + assert_eq!( + key, + Key::derive(&content, len), + "the canonical derivation addresses the put on {path_tag}" + ); + let (_, blob) = store + .get(&key.as_bytes()) + .await + .expect("get succeeds") + .expect("the entry is present"); + + assert_eq!( + blob.read_at(0, len).await.expect("the whole window reads"), + content, + "the multi-window assembly reads whole and byte-exact on {path_tag}" + ); + assert_eq!( + blob.read_at(LO_WINDOW as u64 - 100, 300) + .await + .expect("the straddling read succeeds"), + content[(LO_WINDOW - 100)..(LO_WINDOW + 200)], + "a read straddling the window edge crosses two lo_get \ + statements byte-exact on {path_tag}" + ); + let (slice, digest) = blob + .read_range(LO_WINDOW as u64, 7) + .await + .expect("the ranged slice reads"); + assert_eq!( + slice, + content[LO_WINDOW..LO_WINDOW + 7], + "the ranged slice is exact on {path_tag}" + ); + assert_eq!( + digest, + slice_digest(&slice), + "the slice digest is SHA-256 over exactly the returned bytes" + ); + assert_eq!( + blob.read_at(len - 5, 100).await.expect("the tail reads"), + content[(len as usize - 5)..], + "the tail read clamps at the entry's end" + ); + assert_eq!( + blob.read_at(len, 8).await.expect("the EOF read succeeds"), + Vec::::new(), + "a read at EOF is empty, never an error" + ); + assert_eq!( + blob.read_at(u64::MAX, 5).await.expect("the offset reads"), + Vec::::new(), + "an overflowing u64 offset clamps in u64 arithmetic — the \ + C7 int4 trap not inherited" + ); + assert_eq!( + blob.read_at(0, 0).await.expect("the empty read succeeds"), + Vec::::new(), + "a zero-length read is empty" + ); + assert_eq!( + blob.read_at(0, len + 64) + .await + .expect("the over-read reads"), + content, + "an over-read clamps to the whole entry" + ); + + drop(pin); + } + + // The beyond-length clean failure: a companion row claiming + // more bytes than its LO holds (the corrupted half-state + // healthy stores never hold) planted through the oracle — the + // read window trusts the row (the contract authority, + // metadata-only) and fails cleanly when the LO comes up short. + let planted = b"the planted short body".to_vec(); + let planted_key = Key::derive(&planted, planted.len() as u64); + let claimed = (planted.len() + 4096) as i64; + let loid = { + let client = admin_connect_plant(site.dsn()).await; + client + .batch_execute("BEGIN") + .await + .expect("the plant opens"); + let loid: u32 = client + .query_one("SELECT lo_create(0) AS loid", &[]) + .await + .expect("lo_create mints the planted LO") + .get(0); + let fd: i32 = client + .query_one("SELECT lo_open($1, $2)", &[&loid, &INV_WRITE]) + .await + .expect("the planted LO opens for write") + .get(0); + client + .query_one("SELECT lowrite($1, $2)", &[&fd, &planted]) + .await + .expect("the planted partial body writes"); + let committed_at = SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as i64) + .unwrap_or(0); + client + .execute( + &format!( + "INSERT INTO \"{}\".lo_entries (key, loid, size, committed_at) \ + VALUES ($1, $2, $3, $4)", + site.schema() + ), + &[&planted_key.as_bytes(), &loid, &claimed, &committed_at], + ) + .await + .expect("the overclaiming companion row plants"); + client + .batch_execute("COMMIT") + .await + .expect("the plant commits"); + drop(client); + loid + }; + let (got_len, planted_blob) = store + .get(&planted_key.as_bytes()) + .await + .expect("get succeeds") + .expect( + "the planted entry is present (the companion row is the \ + authority, metadata-only)", + ); + assert_eq!( + got_len, claimed as u64, + "the cursor's length is the companion row's overclaim — never \ + an LO content probe" + ); + let err = planted_blob + .read_at(0, got_len) + .await + .expect_err("the short body fails cleanly beyond its length"); + assert!( + matches!(&err, Error::Io(m) if m.contains("short of")), + "the beyond-length read surfaces the unexpected-EOF clean \ + failure, never a partial read: {err}" + ); + + // The durable cleanup: the round-trip legs' identical content + // was exactly one pool entry (CAS across the paths), so that + // entry and the planted one stage exactly, and the executing + // sweep deletes exactly that staged set — the planted LO's + // pages too. + let staged = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (staged.aborted, staged.candidates_staged), + (None, 2), + "the round-trip entry (one across both paths) and the planted \ + one stage exactly" + ); + let executed = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(executed.deleted, 2, "the staged set executed exactly"); + assert_eq!( + ( + lo_entry_rows(site.dsn(), site.schema()).await, + window_rows(site.dsn(), site.schema()).await + ), + (0, 0), + "the medium and the window are back to zero" + ); + assert_eq!( + loid_pages(site.dsn(), loid).await, + 0, + "the executing sweep's lo_unlink reaped the planted LO" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// The plant's raw-LO flag — postgres Large Objects' `INV_WRITE` + /// mode bits, the same constant the engine's orphan test plants + /// with. + const INV_WRITE: i32 = 0x0002_0000; + + async fn admin_connect_plant(dsn: &str) -> tokio_postgres::Client { + let (client, connection) = tokio_postgres::connect(dsn, NoTls) + .await + .expect("the plant connection opens"); + tokio::spawn(async move { + let _ = connection.await; + }); + client + } + + /// Virgin-store row: **reads are no-ops on a fresh schema** — an + /// empty `list`, absent reads per op (`None` on get/stat, `false` + /// on has, `Err(Missing)` on read_range), and nothing on the fresh + /// medium itself (zero companion rows, and the window state per + /// the host rule: `gc_window` lives in the pg-lo schema — the + /// one-window-host rule's large-host arm — and holds zero rows; + /// `gc_pins` is never created by pg-lo). The constructor always + /// ensures the pg-lo schema, this row being the facade form. + /// + /// Contract stamp: store-api.md §Error model (virgin-store + /// semantics: reads on a fresh store see absent/empty, never + /// errors); ADR-012 §6.5 ruling 5 (the per-op miss shapes); ADR-009 + /// §2 (the companion-table bootstrap at the constructor); ADR-012 + /// §5 (the one-window-host rule's named topology — the window on + /// the pg-lo host, and pg-lo never carries fleet pins). + pub async fn virgin_store_reads_are_no_ops_on_a_fresh_schema(factory: &DualTierMemPgLoFactory) { + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + assert_eq!( + store.mode(), + StoreMode::DualTier, + "the column's factory is the dispatch-coverage shape" + ); + let dsn = site.dsn(); + let schema = site.schema(); + + let absent = b"nothing was ever put".to_vec(); + let absent = Key::derive(&absent, absent.len() as u64); + + assert_eq!( + store.list().await.expect("list succeeds"), + Vec::::new(), + "the virgin pool lists empty" + ); + assert!( + store + .get(&absent.as_bytes()) + .await + .expect("get succeeds") + .is_none(), + "absent get is Ok(None)" + ); + assert_eq!( + store.stat(&absent.as_bytes()).await.expect("stat succeeds"), + None, + "absent stat is Ok(None)" + ); + assert!( + !store.has(&absent.as_bytes()).await.unwrap(), + "absent has is false" + ); + assert_eq!( + store + .read_range(&absent.as_bytes(), 0, 4) + .await + .expect_err("absent read_range errs"), + Error::Missing, + "absent read_range is Err(Missing)" + ); + assert_eq!( + lo_entry_rows(dsn, schema).await, + 0, + "the fresh LO medium holds nothing (zero companion rows)" + ); + assert_eq!( + window_rows(dsn, schema).await, + 0, + "the fresh window holds no staged candidates" + ); + assert!( + crate::pg_lo_column::table_exists(dsn, schema, "gc_window").await, + "the window lives on the pg-lo host — the one-window-host \ + rule's large-host arm (ADR-012 §5)" + ); + assert!( + !crate::pg_lo_column::table_exists(dsn, schema, "gc_pins").await, + "gc_pins is never created by pg-lo (fleet pins live on the kv \ + host, ADR-012 §5)" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + // -------------------------------------------------------------- + // The runners — the column's tests: the form-independent reference + // rows mirrored verbatim against the dual-tier factory (INV-1, INV-6, + // and the two no-op abort shapes), the durable-schedule rows above, + // and the factory contract's structural shapes. Every runner gates + // behind the reachability probe — the SKIP posture: server-less + // runs skip cleanly, a skip is not a failure; acceptance is the + // green run against the harness server. + // -------------------------------------------------------------- + + #[tokio::test] + async fn mem_pglo_inv1_round_trip_puts_and_gets_are_byte_identical() { + if !ready().await { + return; + } + let factory = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + rows::round_trip_puts_and_gets_are_byte_identical(&factory).await; + } + + #[tokio::test] + async fn mem_pglo_inv2_dedup_identical_content_is_one_pool_entry() { + if !ready().await { + return; + } + let factory = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + dedup_identical_content_is_one_pool_entry(&factory).await; + } + + #[tokio::test] + async fn mem_pglo_inv3_stage_hygiene_zero_residue_one_entry_on_success() { + if !ready().await { + return; + } + let factory = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + stage_hygiene_zero_residue_on_failure_exactly_one_entry_on_success(&factory).await; + } + + #[tokio::test] + async fn mem_pglo_inv4_two_sweep_exact_counts_over_the_full_protection_set() { + if !ready().await { + return; + } + let factory = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + two_sweep_exact_counts_over_the_full_protection_set(&factory).await; + } + + #[tokio::test] + async fn mem_pglo_inv4_a_put_landing_during_the_window_survives_the_executing_sweep() { + if !ready().await { + return; + } + let factory = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + a_put_landing_during_the_window_survives_the_executing_sweep(&factory).await; + } + + #[tokio::test] + async fn mem_pglo_inv4_sweep_without_liveness_sources_aborts_and_deletes_nothing() { + if !ready().await { + return; + } + let factory = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + rows::sweep_without_liveness_sources_aborts_and_deletes_nothing(&factory).await; + } + + #[tokio::test] + async fn mem_pglo_inv4_sweep_with_a_failed_protect_callback_aborts_and_deletes_nothing() { + if !ready().await { + return; + } + let factory = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + rows::sweep_with_a_failed_protect_callback_aborts_and_deletes_nothing(&factory).await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn mem_pglo_inv4_the_contended_second_sweeper_aborts_as_data_and_the_holder_stages_only() + { + if !ready().await { + return; + } + let factory = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + the_contended_second_sweeper_aborts_as_data_and_the_holder_stages_only(&factory).await; + } + + #[tokio::test] + async fn mem_pglo_inv5_list_correctness_through_durable_schedule_outcomes() { + if !ready().await { + return; + } + let factory = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + list_correctness_is_observable_through_durable_schedule_outcomes(&factory).await; + } + + #[tokio::test] + async fn mem_pglo_inv6_stat_reports_len_and_algorithm_without_fetching_content() { + if !ready().await { + return; + } + let factory = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + rows::stat_reports_len_and_algorithm_without_fetching_content(&factory).await; + } + + #[tokio::test] + async fn mem_pglo_lo_windows_read_across_window_boundaries_and_a_short_body_fails_cleanly() { + if !ready().await { + return; + } + let factory = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + lo_windows_read_across_window_boundaries_and_a_short_body_fails_cleanly(&factory).await; + } + + #[tokio::test] + async fn mem_pglo_virgin_store_reads_are_no_ops_on_a_fresh_schema() { + if !ready().await { + return; + } + let factory = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + virgin_store_reads_are_no_ops_on_a_fresh_schema(&factory).await; + } + + #[tokio::test] + async fn dispatch_mem_pglo_routing_is_a_pure_function_of_length_across_put_paths_and_re_puts() { + if !ready().await { + return; + } + let factory = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + routing_is_a_pure_function_of_length_across_put_paths_and_re_puts(&factory).await; + } + + #[tokio::test] + async fn dispatch_mem_pglo_threshold_boundary_lengths_route_deterministically() { + if !ready().await { + return; + } + let tuned = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + threshold_boundary_lengths_route_deterministically(&tuned).await; + let default = DualTierMemPgLoFactory::new(DEFAULT_DISPATCH_THRESHOLD); + threshold_boundary_lengths_route_deterministically(&default).await; + } + + #[tokio::test] + async fn dispatch_mem_pglo_kv_tier_misses_fall_through_to_the_large_tier_per_op() { + if !ready().await { + return; + } + let factory = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + kv_tier_misses_fall_through_to_the_large_tier_per_op(&factory).await; + } + + // -------------------------------------------------------------- + // The factory contract's structural shapes (the postgres column's + // pattern, carried to the LO medium with the LO-isolation + // corollary's arms): fresh isolated opens (a fresh dedicated + // database with a fresh schema per open, nothing shared — two + // opens never observe one another) and teardown against a + // panicking row (the schemas really dropped, the LOs really + // unlinked — zero orphan residue probed inside the dedicated + // database before it goes, and the shared harness database + // untouched by construction). Not conformance rows — no backlog + // entry — the factory contract's own wiring-level assertions. + // -------------------------------------------------------------- + + /// Every `open()` mints a fresh, isolated database + schema pair: + /// no shared state between islands, observable on the facade *and* + /// on the fresh medium (each open's database and schema are + /// private and start empty). + #[tokio::test] + async fn mem_pglo_factory_open_vends_fresh_isolated_stores() { + if !ready().await { + return; + } + let factory = DualTierMemPgLoFactory::new(TUNED_THRESHOLD); + + let (a, site_a) = factory.open_tracked().await.expect("open vends a store"); + let (b, site_b) = factory.open_tracked().await.expect("open vends another"); + assert_ne!( + site_a.schema(), + site_b.schema(), + "two opens never share a schema" + ); + assert_ne!(site_a.db, site_b.db, "two opens never share a database"); + + let content_a = patterned(2 * TUNED_THRESHOLD as usize + 5, 167); + let content_b = patterned(TUNED_THRESHOLD as usize - 5, 191); + let (key_a, _) = a + .put( + Some(content_a.len() as u64), + Cursor::new(content_a.as_slice()), + ) + .await + .expect("island a's put publishes"); + let (key_b, _) = b + .put(None, Cursor::new(content_b.as_slice())) + .await + .expect("island b's put publishes"); + + assert!(a.has(&key_a.as_bytes()).await.unwrap()); + assert!( + !a.has(&key_b.as_bytes()).await.unwrap(), + "island a sees nothing of island b" + ); + assert!(b.has(&key_b.as_bytes()).await.unwrap()); + assert!( + !b.has(&key_a.as_bytes()).await.unwrap(), + "island b sees nothing of island a" + ); + assert_eq!( + lo_entry_rows(site_a.dsn(), site_a.schema()).await, + 1, + "island a's medium holds exactly its own entry (a's content routes large)" + ); + assert_eq!( + lo_entry_rows(site_b.dsn(), site_b.schema()).await, + 0, + "island b's medium holds exactly its own entry (b's content routes kv)" + ); + assert!( + crate::pg_lo_column::zero_lo_residue(site_a.dsn(), site_a.schema()).await, + "island a's committed content has zero orphan LO residue (the \ + LO catalog and the companion table agree exactly)" + ); + + let (c, site_c) = factory.open_tracked().await.expect("a third vends fresh"); + assert_eq!( + lo_entry_rows(site_c.dsn(), site_c.schema()).await, + 0, + "a third open's medium is virgin" + ); + assert!( + !c.has(&key_a.as_bytes()).await.unwrap(), + "a third open shares nothing with the first" + ); + assert!( + !c.has(&key_b.as_bytes()).await.unwrap(), + "a third open shares nothing with the second" + ); + + let dbs = factory.tracked_dbs(); + assert_eq!(dbs.len(), 3, "one minted database per open"); + assert_eq!( + dbs.iter() + .map(|(name, _, _)| name.clone()) + .collect::>() + .len(), + 3, + "no two opens share a database" + ); + + // The engine-owned schemas really exist inside their dedicated + // databases (the registry's minted list is not bookkeeping + // fiction). + for (_, dsn, schemas) in &dbs { + for schema in schemas { + assert!( + crate::pg_lo_column::schema_exists(dsn, schema).await, + "the minted schema exists inside its dedicated database: {schema}" + ); + } + } + + factory + .teardown() + .await + .expect("teardown reaps the LOs, drops the schemas, drops the databases"); + factory.teardown().await.expect("teardown repeats cleanly"); + let maintenance = admin_dsn(factory.dsn()); + for (name, _, _) in &dbs { + assert!( + !db_exists(&maintenance, name).await, + "teardown dropped the dedicated database: {name} (schemas \ + and all — the whole LO state goes with it)" + ); + } + } + + /// A failing assertion panics the row before teardown runs — and + /// teardown must make sense over the abandoned instance anyway: it + /// is idempotent against the panicked row's path, the LOs are + /// really unlinked (the tracked-LOID reap arm — the recorded + /// LOIDs match the pre-captured set) and the minted schemas really + /// dropped (gone with the dedicated database, **zero orphan + /// `pg_largeobject` residue** probed inside before it goes), and + /// the next open mints a fresh database on a fresh (empty) medium + /// (README §The factory contract). The shared harness database is + /// untouched by construction — no pg-lo pool ever opens against + /// it. + #[tokio::test] + async fn mem_pglo_factory_teardown_is_safe_against_a_panicking_row() { + if !ready().await { + return; + } + let content = patterned(2 * TUNED_THRESHOLD as usize + 7, 173); + let abandoned = Key::derive(&content, content.len() as u64); + let factory = Arc::new(DualTierMemPgLoFactory::new(TUNED_THRESHOLD)); + + let task = { + let factory = Arc::clone(&factory); + tokio::spawn(async move { + let (store, _) = factory.open_tracked().await.expect("open vends the store"); + let (_, pin) = store + .put(Some(content.len() as u64), Cursor::new(content.as_slice())) + .await + .expect("the put publishes"); + drop(pin); + panic!("a failing assertion panics the row (the factory contract's early exit)"); + }) + }; + assert!( + task.await.is_err(), + "the row's panic surfaced as its failure" + ); + + let dbs = factory.tracked_dbs(); + assert_eq!(dbs.len(), 1, "the panicked row minted one database"); + let (name, dsn, schemas) = &dbs[0]; + let loids = companion_loids(dsn, &schemas[0]).await; + assert!(!loids.is_empty(), "the panicked row's LO is tracked"); + assert!( + crate::pg_lo_column::zero_lo_residue(dsn, &schemas[0]).await, + "the abandoned run's committed content has zero orphan LO \ + residue (the LO catalog and the companion table agree \ + exactly)" + ); + + factory + .teardown() + .await + .expect("teardown makes sense over the abandoned instance"); + factory.teardown().await.expect("teardown is idempotent"); + assert!( + !db_exists(&admin_dsn(factory.dsn()), name).await, + "teardown dropped the panicked row's database: {name} (the \ + schemas and the whole LO state go with it)" + ); + let mut reaped = factory.reaped_loids(); + reaped.sort_unstable(); + let mut captured = loids.clone(); + captured.sort_unstable(); + assert_eq!( + reaped, captured, + "the teardown's reap arm unlinked exactly the tracked LOIDs" + ); + + let (store, site) = factory + .open_tracked() + .await + .expect("the next open still vends a store"); + assert!( + !store.has(&abandoned.as_bytes()).await.unwrap(), + "no residue from the panicked row — isolation is factory-owned" + ); + assert_eq!( + lo_entry_rows(site.dsn(), site.schema()).await, + 0, + "and its medium carries nothing of the abandoned run" + ); + factory + .teardown() + .await + .expect("teardown drops the last schema"); + } +} +// ------------------------------------------------------------------ +// The postgres-pglo factory — the consolidation posture, feature pair +// `postgres` + `pg-lo`: kv = postgres + large = pg-lo on one pg +// instance, the durable window hosted on the kv schema (the pg-lo +// window clauses dormant), and the placement oracle's kv half counted +// as rows in the kv schema's `blobs` table. +// ------------------------------------------------------------------ + +#[cfg(all(feature = "postgres", feature = "pg-lo"))] +mod postgres_pglo { + use std::io::Cursor; + use std::sync::atomic::AtomicU64; + use std::sync::{Arc, Mutex}; + use std::time::{Duration, SystemTime}; + + use tokio_postgres::NoTls; + + use crate::factory::StoreFactory; + use crate::pg_lo_column::{ + admin_dsn, companion_loids, create_db, db_dsn, db_exists, drop_db, fresh_db_name, + harness_dsn, harness_ready, lo_entry_rows, loid_for_key, loid_pages, pg_table_rows, + reap_lo_schema, window_rows, DbJanitor, TrackedDb, LO_WINDOW, WINDOW_DELAY, + }; + use crate::rows::{ + self, patterned, slice_digest, Failing, LatchProtect, SetProtect, SetSource, + }; + use alkblobs::{Error, Key, PgOpts, Store, StoreMode, DEFAULT_DISPATCH_THRESHOLD}; + + /// The column's declared dual-tier threshold — small, so the + /// shared content matrix and the race fixtures cross it and both + /// tiers' bodies are exercised (sub-threshold rows land as rows in + /// the kv schema's `blobs` table; over-threshold rows land as + /// companion rows + LOs in the pg-lo schema). + pub(crate) const TUNED_THRESHOLD: u64 = 1024; + + /// One vended backing site: the dedicated database this open + /// minted and the two engine-owned schemas minted inside it — the + /// kv half (`blobs` + the GC tables) and the pg-lo half + /// (`lo_entries`, window-clauses dormant) — on one pg instance. + #[derive(Clone)] + pub(crate) struct Site { + /// The db-scoped DSN — the oracle's independent connections + /// address it; each open's database is private and gets + /// dropped whole at teardown (the LO-isolation corollary: LOs + /// are database-level). + dsn: String, + /// The kv engine's schema (the pool table and, per the + /// one-window-host rule, the `gc_window` + `gc_pins` tables). + kv_schema: String, + /// The pg-lo engine's schema (the companion table; no window + /// clauses — the dormancy shape). + pglo_schema: String, + /// The dedicated database's name — the structural teardown + /// tests' `pg_database` probe target. + db: String, + } + + impl Site { + /// The kv tier's schema — the dispatch oracle's kv-side + /// counter (`blobs`) and the window host's `gc_window`. + pub(crate) fn kv_schema(&self) -> &str { + &self.kv_schema + } + + /// The pg-lo tier's schema — the LO medium's companion table + /// and the dormancy row's genuine-absence target. + pub(crate) fn pglo_schema(&self) -> &str { + &self.pglo_schema + } + + /// The DSN the open minted — scoped to the dedicated database. + pub(crate) fn dsn(&self) -> &str { + &self.dsn + } + } + + /// The dispatch-coverage factory: `kv = postgres` + `large = + /// pg-lo` on one pg instance (the consolidation posture, ADR-009) + /// at the threshold this column declares and the test tuning of + /// the window delay (see [`TUNED_THRESHOLD`] and `WINDOW_DELAY` — + /// the window lives on the kv schema; the pg-lo schema carries no + /// `gc_window`, the dormancy shape the column's row asserts). + /// + /// **Why a fresh instance per runner (not a shared static):** + /// teardown drops the dedicated databases *this instance* minted + /// (having reaped their LOs and dropped their schemas); a shared + /// instance's teardown would race concurrently running rows' + /// sites. Every runner constructs its own and owns its lifecycle; + /// a row that panics before teardown is still backed by the + /// per-open drop-guard janitors. + pub(crate) struct DualTierPostgresPgLoFactory { + dsn: String, + /// The declared dispatch threshold — the routing boundary this + /// column's rows run at (constructor-tunable; ADR-003). + /// Declared configuration: the dual-tier store never rejects, + /// so there is nothing to probe. + threshold: u64, + /// The declared window delay — the durable schedule's cadence + /// knob the sweep rows run at (the test tuning: + /// execute-on-next-sweep). + #[allow(dead_code)] + window_delay: Duration, + seq: AtomicU64, + /// Every dedicated database this instance minted, for its own + /// teardown and its drop-guard janitors. + dbs: Mutex>, + /// The LOIDs teardown reaped, for the structural tests' + /// assertions. + reaped_loids: Mutex>, + } + + impl DualTierPostgresPgLoFactory { + pub(crate) fn new(threshold: u64) -> DualTierPostgresPgLoFactory { + DualTierPostgresPgLoFactory { + dsn: harness_dsn().expect("the harness server is configured for the column"), + threshold, + window_delay: WINDOW_DELAY, + seq: AtomicU64::new( + (std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.subsec_nanos() as u64) + .unwrap_or(0)) + ^ ((std::process::id() as u64) << 8), + ), + dbs: Mutex::new(Vec::new()), + reaped_loids: Mutex::new(Vec::new()), + } + } + + /// Open + track the vended site: the shape the column's own + /// rows use (they observe both mediums, so they need the site + /// alongside the store). One dedicated database per open with + /// two minted schemas — the kv half and the pg-lo half — so + /// the dormancy row observes genuine absence. + pub(crate) async fn open_tracked(&self) -> Result<(Store, Site), Error> { + let maintenance = admin_dsn(&self.dsn); + let db = fresh_db_name("consolidation", &self.seq); + create_db(&maintenance, &db).await?; + let dsn = db_dsn(&self.dsn, &db); + let kv_schema = crate::pg_lo_column::fresh_schema_name("kv", &self.seq); + let pglo_schema = crate::pg_lo_column::fresh_schema_name("lo", &self.seq); + self.dbs + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(TrackedDb { + name: db.clone(), + dsn: dsn.clone(), + schemas: vec![kv_schema.clone(), pglo_schema.clone()], + lo_schemas: vec![pglo_schema.clone()], + janitor: Some(DbJanitor::new(&maintenance, &db)), + }); + let store = Store::dual_tier_postgres_pglo( + &dsn, + opts(kv_schema.clone()), + &dsn, + opts(pglo_schema.clone()), + self.threshold, + WINDOW_DELAY, + ) + .await?; + Ok(( + store, + Site { + dsn, + kv_schema, + pglo_schema, + db, + }, + )) + } + + /// The dedicated databases this instance minted — names, + /// db-scoped DSNs, and the schemas minted inside — for the + /// structural tests' assertions. + pub(crate) fn tracked_dbs(&self) -> Vec<(String, String, Vec)> { + self.dbs + .lock() + .unwrap_or_else(|e| e.into_inner()) + .iter() + .map(|d| (d.name.clone(), d.dsn.clone(), d.schemas.clone())) + .collect() + } + + /// The LOIDs teardown reaped, for the structural tests' + /// tracked-LOID assertions. + pub(crate) fn reaped_loids(&self) -> Vec { + self.reaped_loids + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } + + /// The maintenance DSN the factory mints databases through. + pub(crate) fn dsn(&self) -> &str { + &self.dsn + } + } + + /// The factories' schema-name policy, in engine-opts form: the + /// constructor's `PgOpts` with exactly the factory's minted schema + /// and the ship defaults for the rest (pool size, + /// `synchronous_commit` — the alkstore factory's shape: only the + /// schema name is factory policy). + fn opts(schema: String) -> PgOpts { + PgOpts { + schema, + ..PgOpts::default() + } + } + + impl StoreFactory for DualTierPostgresPgLoFactory { + fn name(&self) -> &'static str { + "dual-tier postgres+pg-lo" + } + + async fn open(&self) -> Result { + self.open_tracked().await.map(|(store, _)| store) + } + + async fn teardown(&self) -> Result<(), Error> { + let mut dbs = self + .dbs + .lock() + .unwrap_or_else(|e| e.into_inner()) + .drain(..) + .collect::>(); + for tracked in dbs.iter_mut() { + // The LO-isolation corollary's arms: reap the tracked + // LOIDs (via the companion table — the contract + // authority), drop exactly the minted schemas (kv and + // pg-lo halves), then drop the dedicated database + // whole (`WITH (FORCE)` reaps any connection the + // abandoned store's pool has not drained). + let maintenance = admin_dsn(&self.dsn); + let mut reaped = Vec::new(); + if let Ok(client) = admin_connect_for_teardown(&tracked.dsn).await { + for schema in &tracked.lo_schemas { + reaped.extend(reap_lo_schema(&client, schema).await); + } + for schema in &tracked.schemas { + let sql = format!("DROP SCHEMA IF EXISTS \"{schema}\" CASCADE"); + let _ = client.batch_execute(&sql).await; + } + } + if !reaped.is_empty() { + self.reaped_loids + .lock() + .unwrap_or_else(|e| e.into_inner()) + .extend(reaped); + } + drop_db(&maintenance, &tracked.name).await?; + TrackedDb::disarm(tracked); + } + Ok(()) + } + } + + async fn admin_connect_for_teardown(dsn: &str) -> Result { + let (client, connection) = tokio_postgres::connect(dsn, NoTls) + .await + .map_err(|e| Error::Io(format!("postgres harness: {e}")))?; + tokio::spawn(async move { + let _ = connection.await; + }); + Ok(client) + } + + /// The skip gate every runner checks (the SKIP posture's + /// factory-owner responsibility). + async fn ready() -> bool { + harness_ready().await + } + + // -------------------------------------------------------------- + // The column's own rows — the durable-schedule sweep family, the + // schedule-relative readings, the dispatch forms, the INV-1 + // extension, the dormancy row, and the virgin-store row. Every + // row takes this column's concrete factory (it observes the + // mediums), is ADR-stamped per the convention, and traces its + // family. + // -------------------------------------------------------------- + + /// INV-2 on the durable schedule, over both tiers: identical + /// content put twice — through the same put path and across the + /// two paths — is one pool entry, and the re-put's `Pin` protects + /// exactly as a fresh put's does. The kv leg's single-pool-entry + /// is one row in the kv schema; the large leg's is one companion + /// row behind one LO in the pg-lo schema. One-pool-entry reads + /// through durable-schedule outcomes (the INV-5 posture): the pins + /// held stage nothing; once unpinned, sweep 1 stages exactly the + /// one entry and removes nothing, and its executor sweep deletes + /// exactly that staged set (README §The schedule-relative reading + /// for SQL-backed columns). + /// + /// Contract stamp: store-api.md §Invariants 2; ADR-003 (whole-value + /// put is CAS first-writer-wins; the routing function's tier + /// placement); ADR-005 (a re-put refcounts the existing entry + /// under the same arbitration lock); ADR-007 §5 (postgres, the + /// consolidation shape's kv engine); ADR-009 §2/§4 (the companion + /// table is the contract authority; delete = lo_unlink + row + /// delete); ADR-008 §2 (the staged re-arbitrated delete); ADR-012 + /// §4/§5 (the consolidation posture; the kv host wins the window); + /// ADR-012 §6.7 (the executor is the next sweep — the schedule + /// these counts read against). + pub async fn dedup_identical_content_is_one_pool_entry(factory: &DualTierPostgresPgLoFactory) { + let threshold = factory.threshold; + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + // The kv leg: one row in the kv schema, both refcounts + // composing under the same lock. + let content = patterned((threshold - 1) as usize, 191); + let canonical = Key::derive(&content, content.len() as u64); + let (key, first) = store + .put(Some(content.len() as u64), Cursor::new(content.as_slice())) + .await + .expect("the fresh put publishes"); + let (reput, second) = store + .put(Some(content.len() as u64), Cursor::new(content.as_slice())) + .await + .expect("the same-path re-put publishes"); + assert_eq!(key, canonical); + assert_eq!(reput, canonical, "one derivation, one address"); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(report.aborted, None); + assert_eq!( + report.candidates_staged, 0, + "one pool entry, both refcounts composing under the same lock" + ); + assert_eq!( + pg_table_rows(site.dsn(), site.kv_schema(), "blobs").await, + 1, + "one pool entry, both refcounts — one row in the kv schema" + ); + assert_eq!( + lo_entry_rows(site.dsn(), site.pglo_schema()).await, + 0, + "the kv-tier entries never land on the LO medium" + ); + drop(first); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + report.candidates_staged, 0, + "the re-put's pin protects exactly as a fresh put's" + ); + drop(second); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (report.candidates_staged, report.deleted), + (1, 0), + "exactly one pool entry all along — staged this sweep, not executed" + ); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (report.deleted, report.cancelled_by_arbitration), + (1, 0), + "the executing sweep deleted exactly the staged entry" + ); + assert!(!store.has(&canonical.as_bytes()).await.unwrap()); + + // The large leg: identical content through both put paths — + // one companion row behind one LO, both refcounts composing. + let cross = patterned(2 * threshold as usize + 5, 167); + let cross_canonical = Key::derive(&cross, cross.len() as u64); + let (key, first) = store + .put(Some(cross.len() as u64), Cursor::new(cross.as_slice())) + .await + .expect("the known-length put publishes"); + let (reput, second) = store + .put(None, Cursor::new(cross.as_slice())) + .await + .expect("the unknown-length re-put publishes"); + assert_eq!(key, cross_canonical); + assert_eq!(reput, cross_canonical, "one pool entry across the paths"); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (report.aborted, report.candidates_staged), + (None, 0), + "both refcounts compose" + ); + assert_eq!( + lo_entry_rows(site.dsn(), site.pglo_schema()).await, + 1, + "one pool entry across both paths — one companion row" + ); + let loids = companion_loids(site.dsn(), site.pglo_schema()).await; + assert_eq!(loids.len(), 1, "one LO behind the companion row"); + assert!( + loid_pages(site.dsn(), loids[0]).await > 0, + "the LO holds the content's pages (never zero)" + ); + drop(second); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + report.candidates_staged, 0, + "the re-put's pin protects exactly as a fresh put's (other drop order)" + ); + drop(first); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (report.candidates_staged, report.deleted), + (1, 0), + "exactly one pool entry all along" + ); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(report.deleted, 1, "executed exactly the staged set"); + assert!(!store.has(&cross_canonical.as_bytes()).await.unwrap()); + assert_eq!( + lo_entry_rows(site.dsn(), site.pglo_schema()).await, + 0, + "the executed delete emptied the companion table" + ); + assert_eq!( + loid_pages(site.dsn(), loids[0]).await, + 0, + "delete is one tx: the LO's pages went with the row" + ); + assert_eq!( + window_rows(site.dsn(), site.kv_schema()).await, + 0, + "the window holds nothing after both executed rounds (the kv host)" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// INV-3 on the durable schedule: stage hygiene over both mediums' + /// forms — every put-path failure leaves zero residue (the kv tier + /// never touches its pool; the large tier's held-put transaction + /// converges on ROLLBACK: no companion rows, no LO pages) and every + /// success leaves exactly one committed entry, over both put paths + /// and both injectable failure modes (a mid-stream stream failure + /// and a declared-length mismatch), observed through + /// durable-schedule sweep outcomes: zero staged after every + /// failure, and a success staged exactly once, then executed + /// exactly once. + /// + /// Contract stamp: store-api.md §Invariants 3; ADR-010 §1 (the + /// staged-put contract: commit publishes, every failure path + /// discards); ADR-003 (stage-then-commit; the routing places each + /// leg's content on its tier); ADR-007 §5 (postgres, the kv + /// engine); ADR-009 §2 (put = one tx — the staged form is the held + /// tx, so a failure's LO pages roll back with it); ADR-012 §6.7 + /// (the schedule-relative reading the success counts follow). + pub async fn stage_hygiene_zero_residue_on_failure_exactly_one_entry_on_success( + factory: &DualTierPostgresPgLoFactory, + ) { + let threshold = factory.threshold; + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + // The kv tier's degenerate stage form (nothing stages outside + // a flush) — residue is the pool itself, the kv schema's rows. + let small = patterned((threshold - 1) as usize, 167); + let err = store + .put(Some(small.len() as u64), Failing::new(&small, 40)) + .await + .expect_err("the failed mid-stream put rejects"); + assert!( + matches!(err, Error::Io(_)), + "the stream failure surfaces as Io: {err}" + ); + let err = store + .put(None, Failing::new(&small, 40)) + .await + .expect_err("the failed unknown-length put rejects"); + assert!( + matches!(err, Error::Io(_)), + "the stream failure surfaces as Io: {err}" + ); + let err = store + .put(Some(100), Cursor::new(small.as_slice())) + .await + .expect_err("the over-fed mismatch rejects"); + assert_eq!(err, Error::Verification, "declared-len mismatch rejects"); + let err = store + .put(Some(400), Cursor::new(small.as_slice())) + .await + .expect_err("the under-fed mismatch rejects"); + assert_eq!(err, Error::Verification, "declared-len mismatch rejects"); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + ( + report.candidates_staged, + report.deleted, + report.cancelled_by_arbitration + ), + (0, 0, 0), + "zero residue after every kv failure, both put paths, both \ + failure modes" + ); + assert_eq!( + pg_table_rows(site.dsn(), site.kv_schema(), "blobs").await, + 0, + "the pool table holds zero entries after every kv failure" + ); + + let (key, pin) = store + .put(Some(small.len() as u64), Cursor::new(small.as_slice())) + .await + .expect("the known-length put publishes"); + drop(pin); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (report.candidates_staged, report.deleted), + (1, 0), + "exactly one committed kv entry staged on success" + ); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(report.deleted, 1, "and executed exactly that staged set"); + assert!(!store.has(&key.as_bytes()).await.unwrap()); + + let (_, pin) = store + .put(None, Cursor::new(small.as_slice())) + .await + .expect("the unknown-length put publishes"); + drop(pin); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (report.candidates_staged, report.deleted), + (1, 0), + "exactly one committed kv entry staged on the other put path too" + ); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(report.deleted, 1, "and executed exactly that staged set"); + + // The LO tier's real staged form — the held tx's failure paths + // roll back to no companion row, no LO pages. + let content = patterned(2 * threshold as usize + 17, 173); + let len = content.len() as u64; + let canonical = Key::derive(&content, len); + let err = store + .put(Some(len), Failing::new(&content, 40)) + .await + .expect_err("the failed mid-stream put rejects"); + assert!( + matches!(err, Error::Io(_)), + "the stream failure surfaces as Io: {err}" + ); + let err = store + .put(None, Failing::new(&content, threshold as usize + 600)) + .await + .expect_err("the failed unknown-length put rejects mid-stage"); + assert!( + matches!(err, Error::Io(_)), + "the stream failure surfaces as Io: {err}" + ); + let err = store + .put(Some(threshold + 10), Cursor::new(content.as_slice())) + .await + .expect_err("the over-fed mismatch rejects"); + assert_eq!(err, Error::Verification, "declared-len mismatch rejects"); + let err = store + .put(Some(len + 17), Cursor::new(content.as_slice())) + .await + .expect_err("the under-fed mismatch rejects"); + assert_eq!(err, Error::Verification, "declared-len mismatch rejects"); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + ( + report.candidates_staged, + report.deleted, + report.cancelled_by_arbitration + ), + (0, 0, 0), + "zero residue after every large failure, both put paths, both \ + failure modes" + ); + assert_eq!( + lo_entry_rows(site.dsn(), site.pglo_schema()).await, + 0, + "the companion table holds zero entries after every failure" + ); + assert_eq!( + window_rows(site.dsn(), site.kv_schema()).await, + 0, + "the window staged nothing behind the failures" + ); + + // Success, known path on the LO tier: exactly one committed + // entry — one companion row behind one LO — staged once, + // executed once. + let (key, pin) = store + .put(Some(len), Cursor::new(content.as_slice())) + .await + .expect("the known-length put publishes"); + assert_eq!(key, canonical); + let loid = loid_for_key(site.dsn(), site.pglo_schema(), &key.as_bytes()) + .await + .expect("the companion row carries its LOID"); + assert!( + loid_pages(site.dsn(), loid).await > 0, + "the committed LO holds the content's pages" + ); + drop(pin); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (report.candidates_staged, report.deleted), + (1, 0), + "exactly one committed entry staged on success" + ); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + report.deleted, 1, + "the executing sweep removed exactly that one entry" + ); + assert!(!store.has(&canonical.as_bytes()).await.unwrap()); + assert_eq!(loid_pages(site.dsn(), loid).await, 0); + + // Success, unknown path: the overflow staged into the held tx, + // the commit published — exactly one committed entry on the + // other put path too. + let (key, pin) = store + .put(None, Cursor::new(content.as_slice())) + .await + .expect("the unknown-length put publishes"); + assert_eq!(key, canonical); + drop(pin); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (report.candidates_staged, report.deleted), + (1, 0), + "exactly one committed entry staged on the other put path too" + ); + let report = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(report.deleted, 1, "and executed exactly that staged set"); + assert!(!store.has(&canonical.as_bytes()).await.unwrap()); + assert_eq!( + ( + lo_entry_rows(site.dsn(), site.pglo_schema()).await, + window_rows(site.dsn(), site.kv_schema()).await + ), + (0, 0), + "both mediums and the window are back to zero" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// INV-4, the durable-schedule happy shape over both tiers: the + /// full protection set spanning both tiers (a source-live kv + /// entry, a pinned large-tier entry, a protect-named large-tier + /// entry) plus one dead entry per tier — sweep 1 stages exactly + /// the two dead candidates into the **kv schema's `gc_window`** + /// (the window hosted on the kv engine per the one-window-host + /// rule) and deletes nothing; sweep 2 executes exactly that staged + /// set, and the protected entries stay byte-identical. The medium + /// oracle distinguishes the tiers (the kv schema's pool rows vs + /// the pg-lo schema's companion rows behind LOs) and watches the + /// schedule itself. + /// + /// Contract stamp: store-api.md §Invariants 4 (exact counts with + /// correct liveness) and §Invariants 5 (list correctness through + /// sweep outcomes); ADR-005 (three liveness marks; delete windows + /// with arbitrate-at-delete-time); ADR-008 §2 (the staged + /// re-arbitrated delete; the constructor-tunable delay); ADR-009 + /// §2/§4 (the companion table; GC-participating deletes); ADR-007 + /// §5 (postgres, the kv engine); ADR-012 §1 (sweeps are never + /// errors); ADR-012 §4/§5 (the consolidation posture; the kv host + /// wins the window); ADR-012 §6.7 (the executor is the next + /// sweep). + pub async fn two_sweep_exact_counts_over_the_full_protection_set( + factory: &DualTierPostgresPgLoFactory, + ) { + let threshold = factory.threshold; + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + let source = SetSource::register(&store); + let protect = Arc::new(SetProtect::default()); + store.register_protect_callback(protect.clone()); + + let source_live = patterned((threshold - 1) as usize, 191); + let (key_live, _) = store + .put( + Some(source_live.len() as u64), + Cursor::new(source_live.as_slice()), + ) + .await + .expect("the source-live entry publishes"); + source.add(key_live.clone()); + + let pinned = patterned(2 * threshold as usize + 7, 173); + let (key_pinned, pin_pinned) = store + .put(None, Cursor::new(pinned.as_slice())) + .await + .expect("the pinned entry publishes"); + + let protected = patterned(2 * threshold as usize + 11, 167); + let (key_protected, _) = store + .put( + Some(protected.len() as u64), + Cursor::new(protected.as_slice()), + ) + .await + .expect("the protect-named entry publishes"); + protect.set_keys(vec![key_protected.clone()]); + + let dead_kv = patterned((threshold - 3) as usize, 149); + let (key_dead_kv, pin) = store + .put(Some(dead_kv.len() as u64), Cursor::new(dead_kv.as_slice())) + .await + .expect("the kv-tier dead entry publishes"); + drop(pin); + let dead_large = patterned(2 * threshold as usize + 13, 151); + let (key_dead_large, pin) = store + .put(None, Cursor::new(dead_large.as_slice())) + .await + .expect("the large-tier dead entry publishes"); + drop(pin); + + assert_eq!( + ( + pg_table_rows(site.dsn(), site.kv_schema(), "blobs").await, + lo_entry_rows(site.dsn(), site.pglo_schema()).await + ), + (2, 3), + "before the sweep, both tiers hold their entries exactly (two \ + pool rows in the kv schema, three companion rows in the \ + pg-lo schema)" + ); + let dead_large_loid = + loid_for_key(site.dsn(), site.pglo_schema(), &key_dead_large.as_bytes()) + .await + .expect("the dead large entry's companion row carries its LOID"); + assert!( + loid_pages(site.dsn(), dead_large_loid).await > 0, + "the LO medium holds the dead large entry's pages too" + ); + + let first = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(first.aborted, None); + assert_eq!( + ( + first.candidates_staged, + first.deleted, + first.cancelled_by_arbitration + ), + (2, 0, 0), + "sweep 1 staged exactly the two dead entries and deleted \ + nothing — staging is not executing" + ); + assert_eq!( + window_rows(site.dsn(), site.kv_schema()).await, + 2, + "the staged set waits in the durable window — hosted on the \ + kv schema (the one-window-host rule's kv host)" + ); + for (key, content) in [ + (&key_live, &source_live), + (&key_pinned, &pinned), + (&key_protected, &protected), + ] { + let (len, blob) = store + .get(&key.as_bytes()) + .await + .expect("get succeeds") + .expect("the protected entry is present"); + assert_eq!( + blob.read_at(0, len).await.expect("the window reads"), + content.as_slice(), + "the protected entry is byte-identical after the staging sweep" + ); + } + assert!( + store.has(&key_live.as_bytes()).await.unwrap() + && store.has(&key_pinned.as_bytes()).await.unwrap() + && store.has(&key_protected.as_bytes()).await.unwrap(), + "the staged sweep deleted nothing: every entry intact" + ); + + let second = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(second.aborted, None); + assert_eq!( + second.candidates_staged, 0, + "nothing newly dead since the staging sweep" + ); + assert_eq!( + second.deleted + second.cancelled_by_arbitration, + 2, + "sweep 2 executed exactly the staged set" + ); + assert_eq!( + (second.deleted, second.cancelled_by_arbitration), + (2, 0), + "with correct liveness, every executed candidate deleted" + ); + assert!(!store.has(&key_dead_kv.as_bytes()).await.unwrap()); + assert!(!store.has(&key_dead_large.as_bytes()).await.unwrap()); + assert_eq!( + ( + pg_table_rows(site.dsn(), site.kv_schema(), "blobs").await, + lo_entry_rows(site.dsn(), site.pglo_schema()).await + ), + (1, 2), + "the executed deletes landed on both mediums exactly" + ); + assert_eq!( + loid_pages(site.dsn(), dead_large_loid).await, + 0, + "the dead large entry's LO pages are gone with it" + ); + assert_eq!( + window_rows(site.dsn(), site.kv_schema()).await, + 0, + "the executed rows left the durable window" + ); + + drop(pin_pinned); + let third = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (third.candidates_staged, third.deleted), + (1, 0), + "with the pin dropped, the formerly pinned entry stages" + ); + let fourth = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + fourth.deleted, 1, + "and the next sweep executes it — exactly one deletion" + ); + assert!(!store.has(&key_pinned.as_bytes()).await.unwrap()); + assert!(store.has(&key_live.as_bytes()).await.unwrap()); + assert!(store.has(&key_protected.as_bytes()).await.unwrap()); + assert_eq!( + lo_entry_rows(site.dsn(), site.pglo_schema()).await, + 1, + "exactly the protect-named entry's companion row remains" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// INV-4, the race, in its durable-schedule form: a put landing + /// *during the window* survives the executing sweep — the re-put's + /// pin is minted (pin-before-publish) between the staging sweep + /// and the executing sweep, so the execution-time re-arbitration + /// cancels the pinned candidate while the unpinned twin deletes; + /// once the pin drops, the twin re-stages and executes exactly + /// once. Both put paths race in-body over both tiers (a kv twin + /// and a large twin), across the window delay the kv host carries. + /// + /// Contract stamp: store-api.md §Invariants 4; ADR-005 (pin-before- + /// publish ordering; the delete window closes the sweep-vs-put race); + /// ADR-008 §2 (the staged re-arbitrated delete — the race holds + /// across the window delay); ADR-009 §4 (GC-participating deletes + /// over the companion table); ADR-012 §5 (the window on the kv + /// host in this consolidation shape); ADR-012 §6.7 (execution-time + /// re-arbitration is the executor sweep's posture). + pub async fn a_put_landing_during_the_window_survives_the_executing_sweep( + factory: &DualTierPostgresPgLoFactory, + ) { + let threshold = factory.threshold; + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + let rounds: [(String, Vec, bool, bool, bool); 2] = [ + ( + "the kv twin (known-path re-put)".to_owned(), + patterned((threshold - 1) as usize, 191), + true, + true, + false, + ), + ( + "the large twin (unknown-path re-put)".to_owned(), + patterned(2 * threshold as usize + 5, 173), + false, + false, + true, + ), + ]; + for (index, (tag, content, original_known, reput_known, lands_large)) in + rounds.iter().enumerate() + { + let len = content.len() as u64; + let twin = Key::derive(content, len); + let witness = format!("the dead witness of round {index}").into_bytes(); + let witness_key = Key::derive(&witness, witness.len() as u64); + + let original = if *original_known { Some(len) } else { None }; + let (_, pin) = store + .put(original, Cursor::new(content.as_slice())) + .await + .expect("the twin publishes"); + drop(pin); + let (_, pin) = store + .put(None, Cursor::new(witness.as_slice())) + .await + .expect("the witness publishes"); + drop(pin); + + let staged = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + ( + staged.aborted, + staged.candidates_staged, + staged.deleted, + staged.cancelled_by_arbitration + ), + (None, 2, 0, 0), + "{tag}: the staging sweep stages exactly the two dead \ + twins and deletes nothing" + ); + + // The put lands during the window (the re-put dedups into + // the existing twin and refcounts its pin): protected + // across the delay, pin-before-publish. + let reput = if *reput_known { Some(len) } else { None }; + let (_, pin) = store + .put(reput, Cursor::new(content.as_slice())) + .await + .expect("the re-put publishes"); + let executed = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(executed.aborted, None, "{tag}"); + assert_eq!( + executed.candidates_staged, 0, + "{tag}: nothing newly dead since the staging sweep" + ); + assert_eq!( + executed.deleted + executed.cancelled_by_arbitration, + 2, + "{tag}: the executing sweep ran exactly the staged set" + ); + assert_eq!(executed.deleted, 1, "{tag}: the unpinned witness deleted"); + assert_eq!( + executed.cancelled_by_arbitration, 1, + "{tag}: the pinned twin's candidate cancelled at execution time" + ); + assert!( + !store.has(&witness_key.as_bytes()).await.unwrap(), + "{tag}: the unpinned twin is gone" + ); + let (got_len, blob) = store + .get(&twin.as_bytes()) + .await + .expect("get succeeds") + .expect("{tag}: the put landing during the window survived"); + assert_eq!(got_len, len, "{tag}"); + assert_eq!( + blob.read_at(0, len).await.expect("the window reads"), + content.as_slice(), + "{tag}: the surviving twin is byte-identical" + ); + drop(pin); + + let restaged = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (restaged.candidates_staged, restaged.deleted), + (1, 0), + "{tag}: the pin dropped — the twin re-stages" + ); + let executed = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + executed.deleted, 1, + "{tag}: the re-staged candidate executes exactly once" + ); + assert!(!store.has(&twin.as_bytes()).await.unwrap()); + assert_eq!( + window_rows(site.dsn(), site.kv_schema()).await, + 0, + "{tag}: the window holds nothing after the executed round \ + (the kv host)" + ); + if *lands_large { + assert_eq!( + lo_entry_rows(site.dsn(), site.pglo_schema()).await, + 0, + "{tag}: the executed LO deletes reaped the medium exactly" + ); + } + } + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// INV-4, abort shape: the sweep is single-flight on the durable + /// form too — the in-process sweep lease (ADR-013 §2's single-node + /// analog). The contended second sweeper aborts with + /// `SweeperLock` in a completed `Ok(SweepReport)` — nothing + /// executed, nothing staged, nothing deleted (abort before + /// staging: form-independent) — while the lease holder's sweep + /// completes by **staging only**: its staged set waits in the + /// kv schema's `gc_window` for the next sweep to execute (the + /// durable form's schedule-relative replacement for the reference + /// row's inline holder-completion leg). + /// + /// Contract stamp: store-api.md §Invariants 4; ADR-012 §1 (abort is + /// report data, never an error); ADR-013 §2 (the named exclusive + /// sweep lease — one sweeper per pool, holdership ends at release); + /// ADR-012 §5 (the window hosts on the kv engine in this + /// consolidation shape); ADR-012 §6.7 (the executor is the next + /// sweep — the holder stages, it does not execute). + pub async fn the_contended_second_sweeper_aborts_as_data_and_the_holder_stages_only( + factory: &DualTierPostgresPgLoFactory, + ) { + let threshold = factory.threshold; + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + let (entered_tx, entered_rx) = std::sync::mpsc::channel::<()>(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + store.register_protect_callback(Arc::new(LatchProtect { + entered: entered_tx, + release: parking_lot::Mutex::new(Some(release_rx)), + })); + + let content = patterned(2 * threshold as usize + 9, 167); + let (key, pin) = store + .put(Some(content.len() as u64), Cursor::new(content.as_slice())) + .await + .expect("the dead witness publishes"); + drop(pin); + + // The lease holder sweeps on its own task (its protect consult + // blocks that task's thread mid-callback); this task contends + // and then releases the holder. + let store = Arc::new(store); + let holder = tokio::spawn({ + let store = Arc::clone(&store); + async move { store.sweep().await } + }); + + while entered_rx + .recv_timeout(std::time::Duration::from_millis(50)) + .is_err() + { + tokio::task::yield_now().await; + } + + let second = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + second.aborted, + Some(alkblobs::GcAbortCause::SweeperLock), + "the contended sweeper aborts as report data" + ); + assert_eq!( + ( + second.candidates_staged, + second.deleted, + second.cancelled_by_arbitration + ), + (0, 0, 0), + "the contended sweep executes, stages, and deletes nothing" + ); + assert!( + store.has(&key.as_bytes()).await.unwrap(), + "the contended sweep deleted nothing" + ); + assert_eq!( + window_rows(site.dsn(), site.kv_schema()).await, + 0, + "the aborted sweep staged nothing into the window" + ); + release_tx.send(()).ok(); + + let report = holder + .await + .expect("the holder task joined") + .expect("sweeps are never errors"); + assert_eq!(report.aborted, None, "the lease holder completes"); + assert_eq!( + (report.candidates_staged, report.deleted), + (1, 0), + "the holder stages exactly the dead set — it does not execute \ + (the durable form's schedule)" + ); + assert_eq!( + window_rows(site.dsn(), site.kv_schema()).await, + 1, + "the holder's staged set waits in the durable window" + ); + assert!( + store.has(&key.as_bytes()).await.unwrap(), + "the holder staged, did not execute: the entry is intact" + ); + + let executed = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + executed.deleted, 1, + "the next sweep is the executor of the staged set" + ); + assert!(!store.has(&key.as_bytes()).await.unwrap()); + assert_eq!( + lo_entry_rows(site.dsn(), site.pglo_schema()).await, + 0, + "the executed delete emptied the companion table" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// INV-5 on the durable schedule: `list()` correctness is + /// observable only through GC — the enumeration is complete and + /// well-formed in both directions, proven by exact + /// durable-schedule outcomes: the staging sweep stages exactly the + /// dead set (neither more nor fewer candidates), the executing + /// sweep deletes exactly that staged set, and a liveness flip + /// re-stages exactly the newly dead entry. Never a direct list + /// comparison. + /// + /// Contract stamp: store-api.md §Invariants 5; ADR-003 (`list()` is + /// complete and well-formed by contract — load-bearing for GC); + /// ADR-005 (the sweep's live-set diff rides `list()`); ADR-012 §5 + /// (the window hosts on the kv engine in this consolidation + /// shape); ADR-012 §6.7 (the executor is the next sweep — the + /// schedule these counts read against). + pub async fn list_correctness_is_observable_through_durable_schedule_outcomes( + factory: &DualTierPostgresPgLoFactory, + ) { + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + let source = SetSource::register(&store); + + let mut keys = Vec::new(); + for (index, content) in [ + b"kept live by the source, entry one".to_vec(), + b"kept live by the source, entry two".to_vec(), + b"dead at the first staging, entry three".to_vec(), + b"dead at the first staging, entry four".to_vec(), + b"dead at the first staging, entry five".to_vec(), + ] + .into_iter() + .enumerate() + { + let declared = if index % 2 == 0 { + Some(content.len() as u64) + } else { + None + }; + let (key, pin) = store + .put(declared, Cursor::new(content.as_slice())) + .await + .expect("the entry publishes"); + drop(pin); + keys.push(key); + } + source.add(keys[0].clone()); + source.add(keys[1].clone()); + + let first = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(first.aborted, None); + assert_eq!( + (first.candidates_staged, first.deleted), + (3, 0), + "the enumeration staged exactly the three dead entries — \ + complete and well-formed in both directions" + ); + for key in &keys { + assert!( + store.has(&key.as_bytes()).await.unwrap(), + "staging deleted nothing: every enumerated entry is intact" + ); + } + assert_eq!( + window_rows(site.dsn(), site.kv_schema()).await, + 3, + "the staged schedule holds the three candidates" + ); + + let second = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (second.candidates_staged, second.deleted), + (0, 3), + "the executing sweep removed exactly the staged set" + ); + assert!(store.has(&keys[0].as_bytes()).await.unwrap()); + assert!(store.has(&keys[1].as_bytes()).await.unwrap()); + for key in &keys[2..] { + assert!(!store.has(&key.as_bytes()).await.unwrap()); + } + + source.remove(&keys[1]); + let third = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (third.candidates_staged, third.deleted), + (1, 0), + "the newly dead entry is staged exactly" + ); + let fourth = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(fourth.deleted, 1, "and executed exactly by the next sweep"); + assert!(!store.has(&keys[1].as_bytes()).await.unwrap()); + assert!(store.has(&keys[0].as_bytes()).await.unwrap()); + + source.remove(&keys[0]); + let fifth = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(fifth.candidates_staged, 1, "the last entry stages exactly"); + let sixth = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(sixth.deleted, 1, "and executes exactly"); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// Dispatch row: **routing is a pure function of length** — at, + /// below, and over the declared threshold, the same content lands + /// on exactly the same tier through both put paths (same placement + /// every time; nothing moves, nothing duplicates), identical + /// content through both paths is one pool entry, and every read is + /// byte-identical from wherever that tier holds it. + /// + /// Tier placement is observed through both mediums (the oracle + /// adapted to the LO medium): kv entries hold a row in the kv + /// schema's `blobs` table and nothing in the pg-lo schema; large + /// entries hold a companion row behind an LO and no pool row — the + /// facade cannot and must not expose tier residency (it is + /// engine-blind). + /// + /// Contract stamp: store-api.md §Invariants 7 (facade-contract row); + /// ADR-003 §Dispatch policy (routing is a pure function of length; + /// no migration; CAS first-writer-wins makes re-puts one entry); + /// ADR-005 (re-put refcounts); ADR-007 §5 (postgres, the kv + /// engine); ADR-009 §2 (the companion table is the contract + /// authority); ADR-012 §4/§5 (the consolidation posture); ADR-012 + /// §6.3 (the dual-tier dispatch-coverage shape); ADR-012 §6.7 (the + /// schedule-relative cleanup the dedup legs read against). + pub async fn routing_is_a_pure_function_of_length_across_put_paths_and_re_puts( + factory: &DualTierPostgresPgLoFactory, + ) { + let threshold = factory.threshold; + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + for content in [ + patterned((threshold - 1) as usize, 191), + patterned(threshold as usize, 173), + patterned((threshold + 1) as usize, 167), + patterned(4 * threshold as usize + 9, 151), + ] { + let len = content.len() as u64; + let canonical = Key::derive(&content, len); + let expected_rows = if len > threshold { 0 } else { 1 }; + let expected_companions = if len > threshold { 1 } else { 0 }; + + for (path_tag, declared) in [("the known path", Some(len)), ("the unknown path", None)] + { + let (key, pin) = store + .put(declared, Cursor::new(content.as_slice())) + .await + .expect("put publishes"); + assert_eq!( + key, canonical, + "the canonical derivation addresses the put on {path_tag}" + ); + assert_eq!( + pg_table_rows(site.dsn(), site.kv_schema(), "blobs").await, + expected_rows, + "length {len} on {path_tag}: the placement is exactly the \ + routing function's answer and never moves or duplicates \ + (the kv medium holds {expected_rows} pool rows)" + ); + assert_eq!( + lo_entry_rows(site.dsn(), site.pglo_schema()).await, + expected_companions, + "length {len} on {path_tag}: the LO medium holds \ + {expected_companions} companion rows" + ); + let (got_len, blob) = store + .get(&key.as_bytes()) + .await + .expect("get succeeds") + .expect("the entry is present"); + assert_eq!( + got_len, len, + "the read serves the entry from its placed tier" + ); + assert_eq!( + blob.read_at(0, len).await.expect("the whole window reads"), + content, + "the entry is byte-identical from its placed tier across paths" + ); + drop(pin); + } + + let staged = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + staged.candidates_staged, 1, + "identical content through both put paths is one pool entry" + ); + let executed = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + executed.deleted, 1, + "the executing sweep removed exactly the one entry (durable \ + schedule: staged this sweep, executed by the next)" + ); + assert_eq!( + ( + pg_table_rows(site.dsn(), site.kv_schema(), "blobs").await, + lo_entry_rows(site.dsn(), site.pglo_schema()).await + ), + (0, 0), + "both mediums are back to zero" + ); + } + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// Dispatch row: **threshold boundary lengths route + /// deterministically** — at/below/above the factory's declared + /// threshold, on both put paths, the placement is the routing + /// function's answer every time (the mediums distinguish the kv + /// schema's rows from the pg-lo schema's companion rows); and the + /// default-threshold boundary re-routes with the tuning (the + /// constructor-tuned threshold deterministically places the + /// default boundary's length on the tier its own threshold names). + /// Run against two factories by the runner — the column's tuned + /// threshold and the shipped default. The durable cleanup: the + /// staged sweep stages exactly the six entries' dead set and the + /// next sweep executes it. + /// + /// Contract stamp: store-api.md §Invariants 7 (facade-contract row); + /// ADR-003 §Dispatch policy (default 128 KiB at-or-below routes kv; + /// constructor-tunable without API change); ADR-007 §5 (postgres, + /// the kv engine); ADR-009 §2 (the companion table); ADR-012 §6.3 + /// (the dual-tier dispatch-coverage shape); ADR-012 §6.7 + /// (schedule-relative cleanup). + pub async fn threshold_boundary_lengths_route_deterministically( + factory: &DualTierPostgresPgLoFactory, + ) { + let threshold = factory.threshold; + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + let boundary = [ + (threshold - 1, 191u32, false), + (threshold, 173, false), + (threshold, 167, false), + (threshold + 1, 151, true), + (threshold + 1, 139, true), + ]; + let mut expected_rows = 0u64; + let mut expected_companions = 0u64; + for (len, modulus, lands_large) in boundary { + let content = patterned(len as usize, modulus); + for declared in [Some(content.len() as u64), None] { + let (key, pin) = store + .put(declared, Cursor::new(content.as_slice())) + .await + .expect("put publishes"); + assert_eq!( + key, + Key::derive(&content, len), + "the canonical derivation addresses the put" + ); + assert_eq!( + pg_table_rows(site.dsn(), site.kv_schema(), "blobs").await as u64, + expected_rows + if lands_large { 0 } else { 1 }, + "boundary length {len}: at/below the threshold lands kv \ + (one row in the kv schema), over it lands large (a \ + companion row instead), both put paths" + ); + assert_eq!( + lo_entry_rows(site.dsn(), site.pglo_schema()).await as u64, + expected_companions + if lands_large { 1 } else { 0 }, + "boundary length {len}: the LO medium matches the routing answer" + ); + drop(pin); + } + if lands_large { + expected_companions += 1; + } else { + expected_rows += 1; + } + } + + // The default boundary re-routes with the constructor's + // tuning: the shipped default's length belongs to the tier + // this configuration's own threshold puts it on. + let at_default = patterned(DEFAULT_DISPATCH_THRESHOLD as usize, 199); + let default_lands_large = DEFAULT_DISPATCH_THRESHOLD > threshold; + let (key, pin) = store + .put( + Some(DEFAULT_DISPATCH_THRESHOLD), + Cursor::new(at_default.as_slice()), + ) + .await + .expect("the default-boundary put publishes"); + assert_eq!( + key, + Key::derive(&at_default, DEFAULT_DISPATCH_THRESHOLD), + "the canonical derivation addresses the put" + ); + assert_eq!( + pg_table_rows(site.dsn(), site.kv_schema(), "blobs").await as u64, + expected_rows + if default_lands_large { 0 } else { 1 }, + "the default boundary's length routes to the tier this \ + configuration's tuning names (the kv medium's answer)" + ); + assert_eq!( + lo_entry_rows(site.dsn(), site.pglo_schema()).await as u64, + expected_companions + if default_lands_large { 1 } else { 0 }, + "the default boundary's length routes to the tier this \ + configuration's tuning names (the LO medium's answer)" + ); + drop(pin); + + let staged = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + staged.candidates_staged, 6, + "each leg's content was exactly one pool entry" + ); + let executed = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + executed.deleted, 6, + "the executing sweep removed every entry, tier notwithstanding" + ); + assert_eq!( + ( + pg_table_rows(site.dsn(), site.kv_schema(), "blobs").await, + lo_entry_rows(site.dsn(), site.pglo_schema()).await + ), + (0, 0), + "both mediums are back to zero" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// Dispatch row: **a kv-tier miss falls through to the large tier + /// on every key-addressed op** — content the routing function + /// placed on the large tier (one companion row behind an LO; the + /// kv medium holds no pool row — the kv tier misses) serves `get`, + /// `stat`, `has`, and `read_range` whole, exact, and + /// digest-correct; the mirror-position leg proves the kv tier is + /// the *first* consult (sub-threshold content, one pool row and no + /// companion row, reads still served); and both-tier misses + /// resolve per the error model's per-op shapes (reached here by + /// the durable schedule's staged-then-executed deletion of the + /// subject entry). + /// + /// Contract stamp: store-api.md §Read path (fall-through consult + /// order); ADR-012 §6.1 (every key-addressed op consults kv first, + /// then large; the general rule); ADR-012 §6.5 ruling 5 (both-tier + /// miss shapes: `None` on get/stat, `Err(Missing)` on read_range); + /// ADR-006 (the slice digest); ADR-007 §5 (postgres, the kv + /// engine); ADR-009 §2/§3 (the companion table and the `lo_get` + /// windows serving the fall-through); ADR-012 §6.3 (the + /// dispatch-coverage shape). + pub async fn kv_tier_misses_fall_through_to_the_large_tier_per_op( + factory: &DualTierPostgresPgLoFactory, + ) { + let threshold = factory.threshold; + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + // Over-threshold content — the kv tier misses, the large tier + // serves every op. Both put paths hold the shape. + for (content, declared) in [ + ( + patterned(3 * threshold as usize + 41, 181), + Some(3 * threshold + 41), + ), + (patterned(3 * threshold as usize + 43, 197), None), + ] { + let len = content.len() as u64; + let (key, pin) = store + .put(declared, Cursor::new(content.as_slice())) + .await + .expect("put publishes"); + assert_eq!( + ( + pg_table_rows(site.dsn(), site.kv_schema(), "blobs").await, + lo_entry_rows(site.dsn(), site.pglo_schema()).await + ), + (0, 1), + "the content lives on exactly the large tier — the kv tier misses" + ); + let loid = loid_for_key(site.dsn(), site.pglo_schema(), &key.as_bytes()) + .await + .expect("the companion row carries its LOID"); + assert!( + loid_pages(site.dsn(), loid).await > 0, + "the LO medium holds the entry's pages" + ); + + let (got_len, blob) = store + .get(&key.as_bytes()) + .await + .expect("get succeeds") + .expect("the large tier served the kv miss"); + assert_eq!(got_len, len); + assert_eq!( + blob.read_at(0, len).await.expect("the whole window reads"), + content, + "the fall-through read is byte-identical" + ); + assert_eq!( + store.stat(&key.as_bytes()).await.expect("stat succeeds"), + Some(alkblobs::EntryMeta { + len, + algorithm: key.algorithm(), + }), + "stat falls through and carries the length plus algorithm" + ); + assert!( + store.has(&key.as_bytes()).await.unwrap(), + "has falls through" + ); + let mid = len / 2; + let (slice, digest) = store + .read_range(&key.as_bytes(), mid, 17) + .await + .expect("read_range falls through"); + assert_eq!( + slice, + content[(mid as usize)..(mid as usize + 17)], + "the range slice is exact" + ); + assert_eq!( + digest, + slice_digest(&slice), + "the slice digest is SHA-256 over exactly the returned bytes" + ); + + // Sweep the entry away (the pin dropped): the miss shapes + // now resolve per op — the large tier misses too. The + // durable schedule: staged this sweep, executed by the next. + drop(pin); + let staged = store.sweep().await.expect("sweeps are never errors"); + assert_eq!((staged.candidates_staged, staged.deleted), (1, 0)); + let executed = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + executed.deleted, 1, + "exactly the one entry died across the two sweeps" + ); + assert!( + store + .get(&key.as_bytes()) + .await + .expect("get succeeds") + .is_none(), + "a both-tier miss on get is Ok(None)" + ); + assert_eq!( + store.stat(&key.as_bytes()).await.expect("stat succeeds"), + None, + "a both-tier miss on stat is Ok(None)" + ); + assert!( + !store.has(&key.as_bytes()).await.unwrap(), + "a both-tier miss on has is false" + ); + assert_eq!( + store + .read_range(&key.as_bytes(), 0, 4) + .await + .expect_err("the both-tier miss errs"), + Error::Missing, + "a both-tier miss on read_range is Err(Missing)" + ); + } + + // The mirror-position leg: sub-threshold content — the kv tier + // is the first consult and serves before the large tier is + // reached (one pool row; nothing ever lands on the LO medium). + let small = patterned((threshold - 1) as usize, 149); + let (key, pin) = store + .put(Some(small.len() as u64), Cursor::new(small.as_slice())) + .await + .expect("the sub-threshold put publishes"); + assert_eq!( + ( + pg_table_rows(site.dsn(), site.kv_schema(), "blobs").await, + lo_entry_rows(site.dsn(), site.pglo_schema()).await + ), + (1, 0), + "the kv tier holds the sub-threshold entry — the large tier \ + was never consulted on the write, never reached on the read" + ); + let (len, blob) = store + .get(&key.as_bytes()) + .await + .expect("get succeeds") + .expect("the kv tier served"); + assert_eq!(len, small.len() as u64); + assert_eq!( + blob.read_at(0, len).await.expect("the whole window reads"), + small + ); + + drop(pin); + + // Both-tier miss shapes for a key that never existed at all, + // per op, over the fresh both-tier store. + let absent = b"never put here".to_vec(); + let absent = Key::derive(&absent, absent.len() as u64); + assert!( + store + .get(&absent.as_bytes()) + .await + .expect("get succeeds") + .is_none(), + "never-put: get is Ok(None)" + ); + assert_eq!( + store.stat(&absent.as_bytes()).await.expect("stat succeeds"), + None, + "never-put: stat is Ok(None)" + ); + assert!( + !store.has(&absent.as_bytes()).await.unwrap(), + "never-put: has is false" + ); + assert_eq!( + store + .read_range(&absent.as_bytes(), 0, 4) + .await + .expect_err("never-put read_range errs"), + Error::Missing, + "never-put: read_range is Err(Missing)" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// INV-1's LO-medium extension (the mem-pglo factory's twin row, + /// over the consolidation shape's kv+LO mediums): **the `lo_get` + /// windows read across window boundaries byte-exact, and a body + /// short of its claimed length fails cleanly** — entries on the + /// large tier (both put paths) read whole through the + /// multi-window assembly, straddle the shipped window edge, read + /// ranged slices with the ADR-006 digest, clamp at EOF (tail, + /// at-EOF empty, a `u64::MAX` offset clamping in `u64` arithmetic, + /// zero-length, over-read), and a companion row overclaiming its + /// size (planted through the oracle) surfaces as the + /// unexpected-EOF-shaped `Error::Io`, never a partial read or a + /// panic. + /// + /// Contract stamp: store-api.md §Invariants 1 (the round-trip + /// family's ranged-read legs) and §Read path (the streaming + /// large-tier windows, pread-style clamping); ADR-009 §2/§3 (the + /// companion table is the authority — the cursor closes over the + /// row; descriptorless `lo_get` windows, the shipped read shape + /// and its unexpected-EOF failure); ADR-006 (the slice digest over + /// exactly the returned bytes); ADR-010 §1 (the large-tier cursor + /// body family). + pub async fn lo_windows_read_across_window_boundaries_and_a_short_body_fails_cleanly( + factory: &DualTierPostgresPgLoFactory, + ) { + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + SetSource::register(&store); + + let content = patterned(2 * LO_WINDOW + 100, 167); + let len = content.len() as u64; + for (path_tag, declared) in [("the known path", Some(len)), ("the unknown path", None)] { + let (key, pin) = store + .put(declared, Cursor::new(content.as_slice())) + .await + .expect("put publishes"); + assert_eq!( + key, + Key::derive(&content, len), + "the canonical derivation addresses the put on {path_tag}" + ); + assert_eq!( + pg_table_rows(site.dsn(), site.kv_schema(), "blobs").await, + 0, + "the entries live on the LO medium — the kv tier misses" + ); + let (_, blob) = store + .get(&key.as_bytes()) + .await + .expect("get succeeds") + .expect("the entry is present"); + + assert_eq!( + blob.read_at(0, len).await.expect("the whole window reads"), + content, + "the multi-window assembly reads whole and byte-exact on {path_tag}" + ); + assert_eq!( + blob.read_at(LO_WINDOW as u64 - 100, 300) + .await + .expect("the straddling read succeeds"), + content[(LO_WINDOW - 100)..(LO_WINDOW + 200)], + "a read straddling the window edge crosses two lo_get \ + statements byte-exact on {path_tag}" + ); + let (slice, digest) = blob + .read_range(LO_WINDOW as u64, 7) + .await + .expect("the ranged slice reads"); + assert_eq!( + slice, + content[LO_WINDOW..LO_WINDOW + 7], + "the ranged slice is exact on {path_tag}" + ); + assert_eq!( + digest, + slice_digest(&slice), + "the slice digest is SHA-256 over exactly the returned bytes" + ); + assert_eq!( + blob.read_at(len - 5, 100).await.expect("the tail reads"), + content[(len as usize - 5)..], + "the tail read clamps at the entry's end" + ); + assert_eq!( + blob.read_at(len, 8).await.expect("the EOF read succeeds"), + Vec::::new(), + "a read at EOF is empty, never an error" + ); + assert_eq!( + blob.read_at(u64::MAX, 5).await.expect("the offset reads"), + Vec::::new(), + "an overflowing u64 offset clamps in u64 arithmetic — the \ + C7 int4 trap not inherited" + ); + assert_eq!( + blob.read_at(0, 0).await.expect("the empty read succeeds"), + Vec::::new(), + "a zero-length read is empty" + ); + assert_eq!( + blob.read_at(0, len + 64) + .await + .expect("the over-read reads"), + content, + "an over-read clamps to the whole entry" + ); + + drop(pin); + } + + // The beyond-length clean failure: a companion row claiming + // more bytes than its LO holds (the corrupted half-state + // healthy stores never hold) planted through the oracle — the + // read window trusts the row (the contract authority, + // metadata-only) and fails cleanly when the LO comes up short. + let planted = b"the planted short body".to_vec(); + let planted_key = Key::derive(&planted, planted.len() as u64); + let claimed = (planted.len() + 4096) as i64; + let loid = { + let client = admin_connect_plant(site.dsn()).await; + client + .batch_execute("BEGIN") + .await + .expect("the plant opens"); + let loid: u32 = client + .query_one("SELECT lo_create(0) AS loid", &[]) + .await + .expect("lo_create mints the planted LO") + .get(0); + let fd: i32 = client + .query_one("SELECT lo_open($1, $2)", &[&loid, &INV_WRITE]) + .await + .expect("the planted LO opens for write") + .get(0); + client + .query_one("SELECT lowrite($1, $2)", &[&fd, &planted]) + .await + .expect("the planted partial body writes"); + let committed_at = SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as i64) + .unwrap_or(0); + client + .execute( + &format!( + "INSERT INTO \"{}\".lo_entries (key, loid, size, committed_at) \ + VALUES ($1, $2, $3, $4)", + site.pglo_schema() + ), + &[&planted_key.as_bytes(), &loid, &claimed, &committed_at], + ) + .await + .expect("the overclaiming companion row plants"); + client + .batch_execute("COMMIT") + .await + .expect("the plant commits"); + drop(client); + loid + }; + let (got_len, planted_blob) = store + .get(&planted_key.as_bytes()) + .await + .expect("get succeeds") + .expect( + "the planted entry is present (the companion row is the \ + authority, metadata-only)", + ); + assert_eq!( + got_len, claimed as u64, + "the cursor's length is the companion row's overclaim — never \ + an LO content probe" + ); + let err = planted_blob + .read_at(0, got_len) + .await + .expect_err("the short body fails cleanly beyond its length"); + assert!( + matches!(&err, Error::Io(m) if m.contains("short of")), + "the beyond-length read surfaces the unexpected-EOF clean \ + failure, never a partial read: {err}" + ); + + // The durable cleanup: the round-trip legs' identical content + // was exactly one pool entry (CAS across the paths), so that + // entry and the planted one stage exactly, and the executing + // sweep deletes exactly that staged set — the planted LO's + // pages too. + let staged = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (staged.aborted, staged.candidates_staged), + (None, 2), + "the round-trip entry (one across both paths) and the planted \ + one stage exactly" + ); + let executed = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(executed.deleted, 2, "the staged set executed exactly"); + assert_eq!( + ( + pg_table_rows(site.dsn(), site.kv_schema(), "blobs").await, + lo_entry_rows(site.dsn(), site.pglo_schema()).await, + window_rows(site.dsn(), site.kv_schema()).await + ), + (0, 0, 0), + "both mediums and the window are back to zero" + ); + assert_eq!( + loid_pages(site.dsn(), loid).await, + 0, + "the executing sweep's lo_unlink reaped the planted LO" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// The plant's raw-LO flag — postgres Large Objects' `INV_WRITE` + /// mode bits, the same constant the engine's orphan test plants + /// with. + const INV_WRITE: i32 = 0x0002_0000; + + async fn admin_connect_plant(dsn: &str) -> tokio_postgres::Client { + let (client, connection) = tokio_postgres::connect(dsn, NoTls) + .await + .expect("the plant connection opens"); + tokio::spawn(async move { + let _ = connection.await; + }); + client + } + + /// The **dormancy row** — the consolidation posture's + /// schema-placement shape (the one-window-host rule's kv-host arm, + /// ADR-012 §5): the kv schema carries `blobs` and both GC tables + /// (`gc_pins` + `gc_window` — the fleet host's both-tables row), + /// the pg-lo schema carries only the companion table and **no + /// `gc_window`** (the pg-lo window clauses are dormant on the + /// dual-SQL instance), and the schedule itself is observable on + /// the kv host: a dead entry stages into the kv schema's + /// `gc_window`, never anywhere in the pg-lo schema. + /// + /// Contract stamp: store-api.md §Invariants 4 (the durable + /// schedule's host, read through its own staging outcome); + /// ADR-008 §2 (the durable window; the staged re-arbitrated + /// delete); ADR-012 §5 (the one-window-host rule: the kv engine + /// hosts the window on dual-SQL instances — the pg-lo window + /// clauses dormant; and pg-lo never carries fleet pins); ADR-012 + /// §6.7 (the executor is the next sweep); ADR-009 §2 (the + /// companion table's bootstrap). + pub async fn the_window_lives_on_the_kv_schema_and_the_pg_lo_schema_stays_dormant( + factory: &DualTierPostgresPgLoFactory, + ) { + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + assert_eq!( + store.mode(), + StoreMode::DualTier, + "the column's factory is the dispatch-coverage shape" + ); + let dsn = site.dsn(); + let kv = site.kv_schema(); + let lo = site.pglo_schema(); + + assert!( + crate::pg_lo_column::table_exists(dsn, kv, "blobs").await, + "the kv tier's pool table exists in the kv schema" + ); + assert!( + crate::pg_lo_column::table_exists(dsn, kv, "gc_window").await, + "the kv host wins the window: gc_window in the kv schema" + ); + assert!( + crate::pg_lo_column::table_exists(dsn, kv, "gc_pins").await, + "the fleet host's both-tables row: gc_pins in the kv schema" + ); + assert!( + crate::pg_lo_column::table_exists(dsn, lo, "lo_entries").await, + "the pg-lo tier's companion table exists in the pg-lo schema" + ); + assert!( + !crate::pg_lo_column::table_exists(dsn, lo, "gc_window").await, + "the pg-lo window clauses are dormant on the dual-SQL instance: \ + the pg-lo schema carries no gc_window" + ); + assert!( + !crate::pg_lo_column::table_exists(dsn, lo, "gc_pins").await, + "gc_pins is never created by pg-lo (fleet pins live on the kv host)" + ); + + // The schedule itself rides the kv host: a dead entry stages + // into the kv schema's gc_window and executes by the next + // sweep — nothing ever stages into the pg-lo schema (which + // carries no window at all). + SetSource::register(&store); + let content = patterned(2 * TUNED_THRESHOLD as usize + 21, 173); + let (key, pin) = store + .put(Some(content.len() as u64), Cursor::new(content.as_slice())) + .await + .expect("the dead candidate publishes"); + drop(pin); + let staged = store.sweep().await.expect("sweeps are never errors"); + assert_eq!( + (staged.aborted, staged.candidates_staged, staged.deleted), + (None, 1, 0), + "the staging sweep staged the dead candidate" + ); + assert_eq!( + window_rows(dsn, kv).await, + 1, + "the staged candidate waits in the kv schema's gc_window" + ); + let executed = store.sweep().await.expect("sweeps are never errors"); + assert_eq!(executed.deleted, 1, "the next sweep executed it"); + assert!( + !store.has(&key.as_bytes()).await.unwrap(), + "the executed delete landed" + ); + assert_eq!( + (window_rows(dsn, kv).await, lo_entry_rows(dsn, lo).await), + (0, 0), + "the window emptied and the companion table carries nothing of it" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + /// Virgin-store row: **reads are no-ops on a fresh schema** — an + /// empty `list`, absent reads per op (`None` on get/stat, `false` + /// on has, `Err(Missing)` on read_range), and nothing on the fresh + /// mediums themselves (zero pool rows in the kv schema, zero + /// companion rows in the pg-lo schema, and the window state per + /// the host rule: `gc_window` in the kv schema holds zero rows). + /// The constructors always ensure their schemas, this row being + /// the facade form. + /// + /// Contract stamp: store-api.md §Error model (virgin-store + /// semantics: reads on a fresh store see absent/empty, never + /// errors); ADR-012 §6.5 ruling 5 (the per-op miss shapes); ADR-007 + /// §5 (postgres, the schema-ensuring construction posture); + /// ADR-009 §2 (the companion-table bootstrap); ADR-012 §5 (the + /// one-window-host rule's kv host on dual-SQL instances). + pub async fn virgin_store_reads_are_no_ops_on_a_fresh_schema( + factory: &DualTierPostgresPgLoFactory, + ) { + let (store, site) = factory + .open_tracked() + .await + .expect("open vends a fresh, isolated store"); + assert_eq!( + store.mode(), + StoreMode::DualTier, + "the column's factory is the dispatch-coverage shape" + ); + let dsn = site.dsn(); + + let absent = b"nothing was ever put".to_vec(); + let absent = Key::derive(&absent, absent.len() as u64); + + assert_eq!( + store.list().await.expect("list succeeds"), + Vec::::new(), + "the virgin pool lists empty" + ); + assert!( + store + .get(&absent.as_bytes()) + .await + .expect("get succeeds") + .is_none(), + "absent get is Ok(None)" + ); + assert_eq!( + store.stat(&absent.as_bytes()).await.expect("stat succeeds"), + None, + "absent stat is Ok(None)" + ); + assert!( + !store.has(&absent.as_bytes()).await.unwrap(), + "absent has is false" + ); + assert_eq!( + store + .read_range(&absent.as_bytes(), 0, 4) + .await + .expect_err("absent read_range errs"), + Error::Missing, + "absent read_range is Err(Missing)" + ); + assert_eq!( + ( + pg_table_rows(dsn, site.kv_schema(), "blobs").await, + lo_entry_rows(dsn, site.pglo_schema()).await, + window_rows(dsn, site.kv_schema()).await + ), + (0, 0, 0), + "the fresh mediums hold nothing (pool, companion table, and \ + the kv-hosted window all empty)" + ); + + drop(store); + factory.teardown().await.expect("teardown is idempotent"); + } + + // -------------------------------------------------------------- + // The runners — the column's tests: the form-independent reference + // rows mirrored verbatim against the dual-tier factory (INV-1, INV-6, + // and the two no-op abort shapes), the durable-schedule rows above, + // and the factory contract's structural shapes. Every runner gates + // behind the reachability probe — the SKIP posture: server-less + // runs skip cleanly, a skip is not a failure; acceptance is the + // green run against the harness server. + // -------------------------------------------------------------- + + #[tokio::test] + async fn postgres_pglo_inv1_round_trip_puts_and_gets_are_byte_identical() { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + rows::round_trip_puts_and_gets_are_byte_identical(&factory).await; + } + + #[tokio::test] + async fn postgres_pglo_inv2_dedup_identical_content_is_one_pool_entry() { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + dedup_identical_content_is_one_pool_entry(&factory).await; + } + + #[tokio::test] + async fn postgres_pglo_inv3_stage_hygiene_zero_residue_one_entry_on_success() { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + stage_hygiene_zero_residue_on_failure_exactly_one_entry_on_success(&factory).await; + } + + #[tokio::test] + async fn postgres_pglo_inv4_two_sweep_exact_counts_over_the_full_protection_set() { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + two_sweep_exact_counts_over_the_full_protection_set(&factory).await; + } + + #[tokio::test] + async fn postgres_pglo_inv4_a_put_landing_during_the_window_survives_the_executing_sweep() { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + a_put_landing_during_the_window_survives_the_executing_sweep(&factory).await; + } + + #[tokio::test] + async fn postgres_pglo_inv4_sweep_without_liveness_sources_aborts_and_deletes_nothing() { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + rows::sweep_without_liveness_sources_aborts_and_deletes_nothing(&factory).await; + } + + #[tokio::test] + async fn postgres_pglo_inv4_sweep_with_a_failed_protect_callback_aborts_and_deletes_nothing() { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + rows::sweep_with_a_failed_protect_callback_aborts_and_deletes_nothing(&factory).await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn postgres_pglo_inv4_the_contended_second_sweeper_aborts_as_data_and_the_holder_stages_only( + ) { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + the_contended_second_sweeper_aborts_as_data_and_the_holder_stages_only(&factory).await; + } + + #[tokio::test] + async fn postgres_pglo_inv5_list_correctness_through_durable_schedule_outcomes() { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + list_correctness_is_observable_through_durable_schedule_outcomes(&factory).await; + } + + #[tokio::test] + async fn postgres_pglo_inv6_stat_reports_len_and_algorithm_without_fetching_content() { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + rows::stat_reports_len_and_algorithm_without_fetching_content(&factory).await; + } + + #[tokio::test] + async fn postgres_pglo_lo_windows_read_across_window_boundaries_and_a_short_body_fails_cleanly() + { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + lo_windows_read_across_window_boundaries_and_a_short_body_fails_cleanly(&factory).await; + } + + #[tokio::test] + async fn postgres_pglo_the_window_lives_on_the_kv_schema_and_the_pg_lo_schema_stays_dormant() { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + the_window_lives_on_the_kv_schema_and_the_pg_lo_schema_stays_dormant(&factory).await; + } + + #[tokio::test] + async fn postgres_pglo_virgin_store_reads_are_no_ops_on_a_fresh_schema() { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + virgin_store_reads_are_no_ops_on_a_fresh_schema(&factory).await; + } + + #[tokio::test] + async fn dispatch_postgres_pglo_routing_is_a_pure_function_of_length_across_put_paths_and_re_puts( + ) { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + routing_is_a_pure_function_of_length_across_put_paths_and_re_puts(&factory).await; + } + + #[tokio::test] + async fn dispatch_postgres_pglo_threshold_boundary_lengths_route_deterministically() { + if !ready().await { + return; + } + let tuned = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + threshold_boundary_lengths_route_deterministically(&tuned).await; + let default = DualTierPostgresPgLoFactory::new(DEFAULT_DISPATCH_THRESHOLD); + threshold_boundary_lengths_route_deterministically(&default).await; + } + + #[tokio::test] + async fn dispatch_postgres_pglo_kv_tier_misses_fall_through_to_the_large_tier_per_op() { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + kv_tier_misses_fall_through_to_the_large_tier_per_op(&factory).await; + } + + // -------------------------------------------------------------- + // The factory contract's structural shapes (the postgres column's + // pattern, carried to the LO medium with the LO-isolation + // corollary's arms): fresh isolated opens (a fresh dedicated + // database with two fresh schemas per open, nothing shared — two + // opens never observe one another) and teardown against a + // panicking row (the schemas really dropped, the LOs really + // unlinked — zero orphan residue probed inside the dedicated + // database before it goes, and the shared harness database + // untouched by construction). Not conformance rows — no backlog + // entry — the factory contract's own wiring-level assertions. + // -------------------------------------------------------------- + + /// Every `open()` mints a fresh, isolated database + schema pair: + /// no shared state between islands, observable on the facade *and* + /// on the fresh mediums (each open's kv and pg-lo schemas are + /// private and start empty). + #[tokio::test] + async fn postgres_pglo_factory_open_vends_fresh_isolated_stores() { + if !ready().await { + return; + } + let factory = DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD); + + let (a, site_a) = factory.open_tracked().await.expect("open vends a store"); + let (b, site_b) = factory.open_tracked().await.expect("open vends another"); + assert_ne!( + site_a.kv_schema(), + site_b.kv_schema(), + "two opens never share a kv schema" + ); + assert_ne!( + site_a.pglo_schema(), + site_b.pglo_schema(), + "two opens never share a pg-lo schema" + ); + assert_ne!(site_a.db, site_b.db, "two opens never share a database"); + + let content_a = patterned(2 * TUNED_THRESHOLD as usize + 5, 167); + let content_b = patterned(TUNED_THRESHOLD as usize - 5, 191); + let (key_a, _) = a + .put( + Some(content_a.len() as u64), + Cursor::new(content_a.as_slice()), + ) + .await + .expect("island a's put publishes"); + let (key_b, _) = b + .put(None, Cursor::new(content_b.as_slice())) + .await + .expect("island b's put publishes"); + + assert!(a.has(&key_a.as_bytes()).await.unwrap()); + assert!( + !a.has(&key_b.as_bytes()).await.unwrap(), + "island a sees nothing of island b" + ); + assert!(b.has(&key_b.as_bytes()).await.unwrap()); + assert!( + !b.has(&key_a.as_bytes()).await.unwrap(), + "island b sees nothing of island a" + ); + assert_eq!( + ( + pg_table_rows(site_a.dsn(), site_a.kv_schema(), "blobs").await, + lo_entry_rows(site_a.dsn(), site_a.pglo_schema()).await + ), + (0, 1), + "island a's mediums hold exactly its own entry (a's content routes large)" + ); + assert_eq!( + ( + pg_table_rows(site_b.dsn(), site_b.kv_schema(), "blobs").await, + lo_entry_rows(site_b.dsn(), site_b.pglo_schema()).await + ), + (1, 0), + "island b's mediums hold exactly its own entry (b's content routes kv)" + ); + assert!( + crate::pg_lo_column::zero_lo_residue(site_a.dsn(), site_a.pglo_schema()).await, + "island a's committed content has zero orphan LO residue (the \ + LO catalog and the companion table agree exactly)" + ); + + let (c, site_c) = factory.open_tracked().await.expect("a third vends fresh"); + assert_eq!( + ( + pg_table_rows(site_c.dsn(), site_c.kv_schema(), "blobs").await, + lo_entry_rows(site_c.dsn(), site_c.pglo_schema()).await + ), + (0, 0), + "a third open's mediums are virgin" + ); + assert!( + !c.has(&key_a.as_bytes()).await.unwrap(), + "a third open shares nothing with the first" + ); + assert!( + !c.has(&key_b.as_bytes()).await.unwrap(), + "a third open shares nothing with the second" + ); + + let dbs = factory.tracked_dbs(); + assert_eq!(dbs.len(), 3, "one minted database per open"); + assert_eq!( + dbs.iter() + .map(|(name, _, _)| name.clone()) + .collect::>() + .len(), + 3, + "no two opens share a database" + ); + for (_, _, schemas) in &dbs { + assert_eq!( + schemas.len(), + 2, + "one kv + one pg-lo minted schema per database" + ); + } + + // The engine-owned schemas really exist inside their dedicated + // databases (the registry's minted list is not bookkeeping + // fiction). + for (name, dsn, schemas) in &dbs { + for schema in schemas { + assert!( + crate::pg_lo_column::schema_exists(dsn, schema).await, + "the minted schema exists inside {name}: {schema}" + ); + } + } + + factory + .teardown() + .await + .expect("teardown reaps the LOs, drops the schemas, drops the databases"); + factory.teardown().await.expect("teardown repeats cleanly"); + let maintenance = admin_dsn(factory.dsn()); + for (name, _, _) in &dbs { + assert!( + !db_exists(&maintenance, name).await, + "teardown dropped the dedicated database: {name} (schemas \ + and all — the whole LO state goes with it)" + ); + } + } + + /// A failing assertion panics the row before teardown runs — and + /// teardown must make sense over the abandoned instance anyway: it + /// is idempotent against the panicked row's path, the LOs are + /// really unlinked (the tracked-LOID reap arm — the recorded + /// LOIDs match the pre-captured set) and the minted schemas really + /// dropped (gone with the dedicated database, **zero orphan + /// `pg_largeobject` residue** probed inside before it goes), and + /// the next open mints a fresh database on fresh (empty) mediums + /// (README §The factory contract). The shared harness database is + /// untouched by construction — no pg-lo pool ever opens against it. + #[tokio::test] + async fn postgres_pglo_factory_teardown_is_safe_against_a_panicking_row() { + if !ready().await { + return; + } + let content = patterned(2 * TUNED_THRESHOLD as usize + 7, 173); + let abandoned = Key::derive(&content, content.len() as u64); + let factory = Arc::new(DualTierPostgresPgLoFactory::new(TUNED_THRESHOLD)); + + let task = { + let factory = Arc::clone(&factory); + tokio::spawn(async move { + let (store, _) = factory.open_tracked().await.expect("open vends the store"); + let (_, pin) = store + .put(Some(content.len() as u64), Cursor::new(content.as_slice())) + .await + .expect("the put publishes"); + drop(pin); + panic!("a failing assertion panics the row (the factory contract's early exit)"); + }) + }; + assert!( + task.await.is_err(), + "the row's panic surfaced as its failure" + ); + + let dbs = factory.tracked_dbs(); + assert_eq!(dbs.len(), 1, "the panicked row minted one database"); + let (name, dsn, schemas) = &dbs[0]; + let lo_schema = { + let mut found = None; + for schema in schemas { + if crate::pg_lo_column::table_exists(dsn, schema, "lo_entries").await { + found = Some(schema); + break; + } + } + found.expect("the minted pg-lo half carries the companion table") + }; + let loids = companion_loids(dsn, lo_schema).await; + assert!(!loids.is_empty(), "the panicked row's LO is tracked"); + assert!( + crate::pg_lo_column::zero_lo_residue(dsn, lo_schema).await, + "the abandoned run's committed content has zero orphan LO \ + residue (the LO catalog and the companion table agree exactly)" + ); + + factory + .teardown() + .await + .expect("teardown makes sense over the abandoned instance"); + factory.teardown().await.expect("teardown is idempotent"); + assert!( + !db_exists(&admin_dsn(factory.dsn()), name).await, + "teardown dropped the panicked row's database: {name} (the \ + schemas and the whole LO state go with it)" + ); + let mut reaped = factory.reaped_loids(); + reaped.sort_unstable(); + let mut captured = loids.clone(); + captured.sort_unstable(); + assert_eq!( + reaped, captured, + "the teardown's reap arm unlinked exactly the tracked LOIDs" + ); + + let (store, site) = factory + .open_tracked() + .await + .expect("the next open still vends a store"); + assert!( + !store.has(&abandoned.as_bytes()).await.unwrap(), + "no residue from the panicked row — isolation is factory-owned" + ); + assert_eq!( + ( + pg_table_rows(site.dsn(), site.kv_schema(), "blobs").await, + lo_entry_rows(site.dsn(), site.pglo_schema()).await + ), + (0, 0), + "and its mediums carry nothing of the abandoned run" + ); + factory + .teardown() + .await + .expect("teardown drops the last database"); + } +}