From a48c12745a4376643ba5ef51cfd03651635359c2 Mon Sep 17 00:00:00 2001 From: "glm-5.3-flash" Date: Sat, 10 Oct 2026 11:00:26 +0000 Subject: [PATCH] =?UTF-8?q?pg-fix-scheduler-tick-seam=20=E2=80=94=20the=20?= =?UTF-8?q?pg=20scheduler's=20transient-fault=20tick=20seam=20lands,=20ret?= =?UTF-8?q?iring=20review=20003=20Finding=201=20(MEDIUM:=20the=20tick=20re?= =?UTF-8?q?try=20loop=20was=20dead=20code=20under=20test=20=E2=80=94=20the?= =?UTF-8?q?=20last=20uncovered=20hardening=20arm=20of=20the=20wave-4=20fix?= =?UTF-8?q?=20batch,=20the=20exact=20class=20review=20002's=20live=20bugs?= =?UTF-8?q?=20came=20from).=20The=20seam=20follows=20the=20sqlite-commit-e?= =?UTF-8?q?rror-arm=20pattern=20of=20record:=20a=20cfg(test)=20per-store?= =?UTF-8?q?=20counting=20fault=20arm=20(crate::seam::tick=5Ffault=20?= =?UTF-8?q?=E2=80=94=20Flag=20=3D=20Arc,=20born=20disarmed=20at?= =?UTF-8?q?=200,=20arm(n)=20adds,=20take()=20consumes=20one=20via=20CAS=20?= =?UTF-8?q?try=5Fupdate;=20a=20counting=20arm=20because=20a=20boolean=20ca?= =?UTF-8?q?nnot=20reach=20the=20exhaustion=20arm=20=E2=80=94=20arm(TICK=5F?= =?UTF-8?q?RETRIES+1)=20faults=20the=20loop's=20whole=20budget=20while=20a?= =?UTF-8?q?rm(1)=20leaves=20the=20later=20ticks=20real)=20feeding=20a=20fa?= =?UTF-8?q?bricated=20pool/database-shaped=20opaque=20Database=20error=20i?= =?UTF-8?q?nto=20the=20REAL=20retry=20loop=20=E2=80=94=20both=20tests=20dr?= =?UTF-8?q?ive=20the=20full=20run=5Fschedules=20=E2=86=92=20tick=5Fwith=5F?= =?UTF-8?q?retries=20=E2=86=92=20tick=5Fonce=20path=20end-to-end,=20no=20l?= =?UTF-8?q?oop=20replication.=20Plumbing=20per=20the=20house=20pattern:=20?= =?UTF-8?q?cfg-gated=20tick=5Ffault=20field=20on=20EngineCtx=20and=20PgSto?= =?UTF-8?q?re=20(engine=5Fctx()=20carries=20the=20clone;=20production=20bu?= =?UTF-8?q?ilds=20never=20materialize=20it),=20cfg-gated=20fault=20param?= =?UTF-8?q?=20on=20tick=5Fonce=20(the=20seam=20check=20sits=20before=20the?= =?UTF-8?q?=20real=20attempt=20=E2=80=94=20one=20attempt=20faults,=20never?= =?UTF-8?q?=20ticks)=20and=20tick=5Fwith=5Fretries,=20cfg-branched=20attem?= =?UTF-8?q?pt=20calls;=20PgStore::arm=5Ftick=5Ffault(faults)=20arming=20su?= =?UTF-8?q?rface=20+=20tick=5Ffault=5Fflag()=20observation=20surface=20(a?= =?UTF-8?q?=20Flag=20clone=20so=20the=20consumption=20pin=20survives=20the?= =?UTF-8?q?=20store=20moving=20into=20a=20runner=20task).=20Two=20arms=20p?= =?UTF-8?q?inned:=20(a)=20one=5Ffaulted=5Ftick=5Fis=5Fretried=5Fand=5Fthe?= =?UTF-8?q?=5Fjob=5Ffires=20=E2=80=94=20fault=20consumed=20on=20attempt=20?= =?UTF-8?q?1,=20the=20250=20ms=20backoff=20sleeps,=20the=20retried=20attem?= =?UTF-8?q?pt=20fires=20the=20backdated=20due=20boundary=20into=20the=20qu?= =?UTF-8?q?eue,=20exactly-one=20consumption,=20the=20runner=20survives=20t?= =?UTF-8?q?he=20hiccup=20to=20a=20clean=20stop=20Ok(());=20(b)=20tick=5Fre?= =?UTF-8?q?try=5Fexhaustion=5Fexits=5Ftyped=5Fand=5Fleaves=5Fthe=5Flock=5F?= =?UTF-8?q?to=5Flapse=20=E2=80=94=20the=20exhaustion=20exits=20typed=20(op?= =?UTF-8?q?aque=20Database,=20never=20LeadershipLost;=20the=20tick-phase?= =?UTF-8?q?=20context=20rides=20the=20source=20chain=20with=20the=20fabric?= =?UTF-8?q?ated=20fault=20at=20its=20bottom;=20wall-clock=20=E2=89=A5=201.?= =?UTF-8?q?7=20s=20proving=20the=20real=20backoff=20window),=20the=20leade?= =?UTF-8?q?rship=20lock=20row=20is=20left=20(not=20released)=20to=20lapse?= =?UTF-8?q?=20at=20its=20TTL=20(present,=20expiry=20bounded=20by=20the=20r?= =?UTF-8?q?un's=20TTL=20horizon),=20exact=20consumption,=20and=20the=20sto?= =?UTF-8?q?re=20remains=20fully=20usable=20after=20=E2=80=94=20the=20"runn?= =?UTF-8?q?er=20survives"=20pin=20as=20the=20module=20docs'=20contract=20s?= =?UTF-8?q?tates=20it.=20Replay-proofed=20live:=20with=20the=20seam's=20co?= =?UTF-8?q?nsumption=20disabled=20the=20exhaustion=20test=20cannot=20compl?= =?UTF-8?q?ete=20(the=20runner=20loops=20forever=20=E2=80=94=20no=20fault?= =?UTF-8?q?=20ever=20enters=20the=20budget;=20the=20mutated=20run's=20time?= =?UTF-8?q?out=20is=20itself=20the=20proof);=20the=20test=20is=20then=20bo?= =?UTF-8?q?unded=20(20=20s)=20so=20a=20future=20regression=20of=20this=20s?= =?UTF-8?q?hape=20fails,=20not=20hangs.=20LEADER=5FTTL=5FS=20widened=20pri?= =?UTF-8?q?vate=20=E2=86=92=20pub(crate)=20for=20the=20lock-left-to-TTL=20?= =?UTF-8?q?expiry=20pin.=20Verified:=20full=20pg=20column=20live=20vs=20th?= =?UTF-8?q?e=20harness=20(124=20lib=20+=2025=20suite=20+=209=20schema;=20n?= =?UTF-8?q?ew=20tests=20green=20solo=20=C3=973);=20cargo=20test=20--worksp?= =?UTF-8?q?ace=20green=20server-less,=20skips=20clean=20(core=2025=20+=20h?= =?UTF-8?q?arness=203,=20sqlite=20191=20+=2025);=20clippy=20--all-targets?= =?UTF-8?q?=20-D=20warnings=20clean;=20fmt=20clean;=20cargo-llvm-cov=20con?= =?UTF-8?q?firms=20the=20Finding-1=20sites=20(the=20retry/backoff/exhausti?= =?UTF-8?q?on-arm=20region=20+=20the=20tick=5Fonce=20seam=20arm)=20carry?= =?UTF-8?q?=20zero=20missed=20lines;=20taskgraph=20validate=20green=20(61?= =?UTF-8?q?=20tasks)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- alkstore-postgres/src/scheduler.rs | 49 +++- alkstore-postgres/src/seam.rs | 47 ++++ alkstore-postgres/src/store.rs | 27 ++ .../src/store/scheduler_tests.rs | 241 ++++++++++++++++++ tasks/pg-fix-scheduler-tick-seam.md | 110 +++++++- 5 files changed, 461 insertions(+), 13 deletions(-) diff --git a/alkstore-postgres/src/scheduler.rs b/alkstore-postgres/src/scheduler.rs index 6701691..153d36d 100644 --- a/alkstore-postgres/src/scheduler.rs +++ b/alkstore-postgres/src/scheduler.rs @@ -120,6 +120,8 @@ use crate::resolution::{ Stamps, now_unix, outbox_backing_queue_name, outbox_default_stamps, plain_queue_default_stamps, }; use crate::schema::{QualifiedTable, tables}; +#[cfg(test)] +use crate::seam::tick_fault; use crate::seam::{database_error, pg_error, pool_error}; use crate::tx::{encode_payload_bytes, enqueue_row, wake_tx}; @@ -132,7 +134,7 @@ const SCHEDULER_MAX_CATCHUP_FIRES: i64 = 64; /// renewed every loop iteration *and* across the sleeps (a runner /// that cannot renew has lost the lock, ADR-009 §4 — the lock /// machinery is the engine's own). -const LEADER_TTL_S: i64 = 10; +pub(crate) const LEADER_TTL_S: i64 = 10; /// Seconds between lease renewals during the sleep. const SLEEP_RENEW_EVERY_S: u64 = 2; @@ -300,6 +302,11 @@ pub(crate) struct EngineCtx { pub pool: Pool, pub schema: String, pub closed: Arc, + /// The tick-fault seam flag (the seam module's per-store scoping — + /// `born disarmed`, never armed in production builds, which do not + /// compile the field at all). + #[cfg(test)] + pub(crate) tick_fault: crate::seam::tick_fault::Flag, } impl EngineCtx { @@ -612,10 +619,13 @@ pub(crate) async fn tick(client: &tokio_postgres::Client, schema: &str, now: i64 /// The leader loop (see the module docs). pub(crate) fn run_schedules(stop: StopToken, ctx: EngineCtx) -> BoxedFuture<'static, Result<()>> { + #[cfg(test)] + let tick_fault = ctx.tick_fault.clone(); let EngineCtx { pool, schema, closed, + .. } = ctx; Box::pin(async move { if closed.load(Ordering::Acquire) { @@ -667,6 +677,9 @@ pub(crate) fn run_schedules(stop: StopToken, ctx: EngineCtx) -> BoxedFuture<'sta // before ticking: never fire on a stolen lease. return Err(Error::LeadershipLost); } + #[cfg(test)] + let soonest = tick_with_retries(&pool, &schema, Some(&tick_fault)).await?; + #[cfg(not(test))] let soonest = tick_with_retries(&pool, &schema).await?; if stop.is_cancelled() { let _ = lease.release().await; @@ -684,8 +697,22 @@ pub(crate) fn run_schedules(stop: StopToken, ctx: EngineCtx) -> BoxedFuture<'sta } /// One tick attempt over a fresh pool checkout (the loop body's -/// original shape, hoisted for the retry wrapper). -async fn tick_once(pool: &Pool, schema: &str) -> Result { +/// original shape, hoisted for the retry wrapper). Under `cfg(test)` +/// the tick-fault seam (see `crate::seam::tick_fault`) precedes the +/// real attempt: an armed store faults this attempt with the +/// fabricated transient error into the loop's retry arms; the flag is +/// consumed-on-use, the next attempt is real. +async fn tick_once( + pool: &Pool, + schema: &str, + #[cfg(test)] fault: Option<&tick_fault::Flag>, +) -> Result { + #[cfg(test)] + if let Some(flag) = fault + && tick_fault::take(flag) + { + return Err(tick_fault::fabricated_transient_error()); + } let conn = pool.get().await.map_err(pool_error)?; in_tx(conn, |client| { let schema = schema.to_string(); @@ -703,11 +730,20 @@ async fn tick_once(pool: &Pool, schema: &str) -> Result { /// the loss decision regardless). Each failed attempt logs (the /// engine's diagnostic posture); the exhausting exit wraps the last /// error with the tick-phase context in the source chain -/// ([`context_error`]). -async fn tick_with_retries(pool: &Pool, schema: &str) -> Result { +/// ([`context_error`]). The attempt call below is cfg-branched (the +/// house pattern — production builds compile the plain path). +async fn tick_with_retries( + pool: &Pool, + schema: &str, + #[cfg(test)] fault: Option<&tick_fault::Flag>, +) -> Result { let mut retries = 0u64; loop { - match tick_once(pool, schema).await { + #[cfg(test)] + let attempt = tick_once(pool, schema, fault).await; + #[cfg(not(test))] + let attempt = tick_once(pool, schema).await; + match attempt { Ok(soonest) => return Ok(soonest), Err(e) if retries < TICK_RETRIES => { retries += 1; @@ -792,6 +828,7 @@ pub(crate) fn new_outbox_handle(name: &str, ctx: EngineCtx) -> Result Error { pub(crate) fn database_error(message: impl Into) -> Error { Error::database(std::io::Error::other(message.into())) } + +/// The `cfg(test)` tick-fault seam (`pg-fix-scheduler-tick-seam`, +/// review 003 Finding 1): a per-store arm that, when set, substitutes +/// a fabricated pool/database-shaped error for one `tick_once` +/// attempt, feeding the *real* retry loop (`tick_with_retries`) — the +/// retry, its backoff sleeps, the exhaustion error, and the diagnostic +/// execute under test, never a replica (the +/// `sqlite-commit-error-arm` pattern). +#[cfg(test)] +pub(crate) mod tick_fault { + use std::sync::Arc; + use std::sync::atomic::{AtomicU64, Ordering}; + + use alkstore::Error; + + /// The remaining-fault count (born disarmed at 0; consumed-on-use; + /// parallel test stores never interfere — the commit-fault seam's + /// per-store scoping). + pub(crate) type Flag = Arc; + + pub(crate) fn disarmed() -> Flag { + Arc::new(AtomicU64::new(0)) + } + + pub(crate) fn arm(flag: &Flag, faults: u64) { + flag.fetch_add(faults, Ordering::Release); + } + + /// Consume one fault (the count semantics: arming `N` faults the + /// next `N` attempts, so the exhaustion arm reaches the typed + /// error while a 1-count arm leaves the later ticks real). + pub(crate) fn take(flag: &Flag) -> bool { + flag.try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| { + (remaining > 0).then(|| remaining - 1) + }) + .is_ok() + } + + /// A pool/database-shaped transient error (the taxonomy's opaque + /// fallback with the source chain — the shape a hiccup surfaces). + pub(crate) fn fabricated_transient_error() -> Error { + Error::database(std::io::Error::other( + "fabricated transient fault (cfg(test) seam): pool/database \ + unavailable for one tick attempt", + )) + } +} diff --git a/alkstore-postgres/src/store.rs b/alkstore-postgres/src/store.rs index 59f4deb..cd8017a 100644 --- a/alkstore-postgres/src/store.rs +++ b/alkstore-postgres/src/store.rs @@ -54,6 +54,11 @@ pub struct PgStore { #[cfg(test)] listener_application_name: String, closed: Arc, + /// The tick-fault seam's per-store flag (the `sqlite-commit-error- + /// arm` pattern — born disarmed, consumed-on-use; production + /// builds do not compile the field). + #[cfg(test)] + tick_fault: crate::seam::tick_fault::Flag, } impl std::fmt::Debug for PgStore { @@ -101,6 +106,24 @@ impl PgStore { &self.schema } + /// Arm the store's tick-fault seam — the next `faults` `tick_once` + /// attempts fault with the fabricated transient error into the + /// leader loop's real retry arms. Per-store scoping (`Arc` counter, + /// born disarmed, consumed-on-use) keeps parallel test stores + /// isolated. `#[cfg(test)]`-gated. + #[cfg(test)] + pub(crate) fn arm_tick_fault(&self, faults: u64) { + crate::seam::tick_fault::arm(&self.tick_fault, faults); + } + + /// The seam's flag (a `Flag` clone — the remaining-fault count is + /// loadable from it for the consumption-exactness pins even while + /// the store has moved into a runner). `#[cfg(test)]`-gated. + #[cfg(test)] + pub(crate) fn tick_fault_flag(&self) -> crate::seam::tick_fault::Flag { + self.tick_fault.clone() + } + /// The scheduler/outbox paths' wiring bundle (the pool, the /// schema, the closed flag). fn engine_ctx(&self) -> crate::scheduler::EngineCtx { @@ -108,6 +131,8 @@ impl PgStore { pool: self.pool.clone(), schema: self.schema.clone(), closed: self.closed.clone(), + #[cfg(test)] + tick_fault: self.tick_fault.clone(), } } @@ -224,6 +249,8 @@ pub(crate) async fn open_store(config: &str, opts: PgOpts) -> alkstore::Result

= 1 { + break c; + } + assert!( + tokio::time::Instant::now() < deadline, + "the retried tick never fired — the retry arm broke the runner" + ); + tokio::time::sleep(Duration::from_millis(200)).await; + }; + assert!(fired >= 1, "the retried attempt fired the due boundary"); + + // The seam consumed exactly its one fault (the runner's later + // ticks stayed real), and the runner survived the hiccup: the + // clean stop is `Ok(())`. + assert_eq!( + flag.load(Ordering::Acquire), + 0, + "exactly one fault armed must be exactly one consumed" + ); + stop.cancel(); + let outcome = tokio::time::timeout(Duration::from_secs(10), runner) + .await + .expect("clean stop must terminate promptly") + .unwrap(); + assert!( + matches!(outcome, Ok(())), + "the runner must survive the faulted tick: {outcome:?}" + ); + + drop_schema(&fx.admin, &fx.schema).await; +} + +/// Acceptance: arm (b), exhaustion — the seam faults the loop's whole +/// budget (`TICK_RETRIES + 1`: the initial attempt plus every retry), +/// the loop exhausts to the typed error shape (opaque `Database`; the +/// tick-phase context rides the source chain with the fabricated fault +/// at its bottom), the backoff-sleep arms execute for real, and the +/// exit strands nothing: the leadership lock row is left (not +/// released) to lapse at its TTL and the store remains fully usable — +/// the resilience posture the exhaustion exit owes. +#[tokio::test(flavor = "multi_thread")] +async fn tick_retry_exhaustion_exits_typed_and_leaves_the_lock_to_lapse() { + use std::sync::atomic::Ordering; + + let Some(fx) = Fixture::open(&instance_namer("so_exh")).await else { + eprintln!("skip: no harness server"); + return; + }; + + // Nothing is due ever (`@every 3600s`, one boundary out) — the + // exhausting exit is fault-driven, not boundary-driven. + fx.store + .schedule( + "slow", + "@every 3600s", + "exhaustq", + json!(1), + Default::default(), + ) + .await + .unwrap(); + + let runner_store = open_store(&harness_dsn().unwrap(), test_opts(&fx.schema)) + .await + .unwrap(); + runner_store.arm_tick_fault(crate::scheduler::TICK_RETRIES + 1); + let flag = runner_store.tick_fault_flag(); + + // The runner is driven in-line (bounded — a regression that never + // exhausts fails here, not a hang); its exit shape *is* the pin. + let started = std::time::Instant::now(); + let err = tokio::time::timeout( + Duration::from_secs(20), + runner_store.run_schedules(StopToken::new()), + ) + .await + .expect("the exhausting loop must exit within its backoff budget") + .unwrap_err(); + + // The backoff-sleep arms executed for real: the exhaustion took the + // full 250 ms → 500 ms → 1 s window (a fast-fail exit would not). + let elapsed = started.elapsed(); + assert!( + elapsed >= Duration::from_millis(1_700), + "the exhaustion must have slept the full backoff window (1.75 s), took {elapsed:?}" + ); + assert!( + elapsed < Duration::from_secs(20), + "the exhaustion must exit promptly after the budget, took {elapsed:?}" + ); + + // The typed exhaustion shape: opaque `Database` (never + // LeadershipLost, never a panic), the tick-phase context riding the + // source chain with the fabricated fault preserved at its bottom. + assert!( + matches!(err, Error::Database(..)), + "the exhausting loop must exit opaque Database: {err:?}" + ); + let mut link: &(dyn std::error::Error + 'static) = &err; + let mut saw_context = false; + let mut saw_fault = false; + loop { + let text = link.to_string(); + if text.contains("scheduler tick failed after 3 retries") + && text.contains("the leadership lock row is left to lapse at its TTL") + { + saw_context = true; + } + if text.contains("fabricated transient fault") { + saw_fault = true; + } + match link.source() { + Some(next) => link = next, + None => break, + } + } + assert!( + saw_context, + "the exhaustion context (tick phase, lock posture) must ride the source chain" + ); + assert!( + saw_fault, + "the fabricated transiency must stay at the chain's bottom (source preserved)" + ); + + // Exactly the four attempts consumed the four armed faults. + assert_eq!( + flag.load(Ordering::Acquire), + 0, + "the budget's fault count must be consumed exactly" + ); + + // The leadership lock row is left (not released) on the `Err` exit — + // its `expires_at` is the TTL's own horizon (renewed once at the + // loop's top, never renewed again past the exhaustion). + let row = fx + .admin + .query_one( + &format!( + "SELECT count(*), max(expires_at) FROM {} WHERE name = $1", + crate::schema::QualifiedTable { + schema: &fx.schema, + name: crate::schema::tables::LOCKS, + } + ), + &[&crate::scheduler::SCHEDULER_LEADER_LOCK], + ) + .await + .unwrap(); + let count: i64 = row.get(0); + let expires_at: i64 = row.get(1); + assert_eq!(count, 1, "the lock row is left to lapse, not released"); + assert!( + expires_at > now_unix(), + "the left-behind expiry is still live (inside its TTL)" + ); + assert!( + expires_at <= now_unix() + crate::scheduler::LEADER_TTL_S, + "the lock row was left to the TTL, not renewed past the exit" + ); + + // The store remains fully usable after exhaustion (the runner's + // exit strands nothing): a subsequent entry point works. + let after = runner_store + .schedule("after", "@every 1s", "postq", json!(1), Default::default()) + .await + .unwrap(); + assert_eq!(after.name, "after"); + + drop_schema(&fx.admin, &fx.schema).await; +} diff --git a/tasks/pg-fix-scheduler-tick-seam.md b/tasks/pg-fix-scheduler-tick-seam.md index b35f9bf..2372009 100644 --- a/tasks/pg-fix-scheduler-tick-seam.md +++ b/tasks/pg-fix-scheduler-tick-seam.md @@ -1,7 +1,7 @@ --- id: pg-fix-scheduler-tick-seam name: "pg scheduler — transient-fault tick seam: retry/exhaustion arms tested (review 003 Finding 1)" -status: pending +status: completed depends_on: [] scope: narrow risk: low @@ -42,13 +42,13 @@ disarmed, consumed-on-use. ## Acceptance Criteria -- [ ] The seam lands cfg(test)-gated; the real `tick_with_retries` +- [x] The seam lands cfg(test)-gated; the real `tick_with_retries` executes under test (no loop replication) -- [ ] Arms (a) and (b) pinned as above; the runner survives +- [x] Arms (a) and (b) pinned as above; the runner survives exhaustion -- [ ] `cargo test -p alkstore-postgres` green against the harness +- [x] `cargo test -p alkstore-postgres` green against the harness server; skips clean server-less -- [ ] Clippy `-D warnings`, fmt clean; workspace gates green +- [x] Clippy `-D warnings`, fmt clean; workspace gates green ## References @@ -61,8 +61,104 @@ disarmed, consumed-on-use. ## Notes -> To be filled by implementation agent +> Decisions of record the implementation made that the description +> didn't pin: + +- **The seam is a per-store *counting* fault arm, not a boolean.** A + boolean arm (the sqlite commit-fault shape) reaches only the + retry-once arm; exhaustion needs the fault to survive the loop's + whole budget. `crate::seam::tick_fault` holds + `Flag = Arc` — born disarmed at 0, `arm(n)` adds n, + `take` consumes one (CAS `try_update`, no negative underflow) — so + arm(1) gives the retry-then-success arm and arm(`TICK_RETRIES + 1`) + the exhaustion arm with one mechanism. Consumption-exactness is + pinned by loading the flag after the run (0 = exactly the armed + count consumed). +- **Plumbing follows the `sqlite-commit-error-arm` precedent + exactly** (the `pg-fix-forwarder-reconnect` house pattern): + `#[cfg(test)]` field on `EngineCtx` and `PgStore`; + `PgStore::engine_ctx()` carries the store's flag clone into every + scheduler ctx; `#[cfg(test)]` param on `tick_once` / `tick_with_retries` + (the seam check sits at `tick_once`'s top — one *attempt* faults, + never ticks) with cfg-branched attempt statements — production + builds compile the plain path and never materialize the field. + Arming/observation surfaces: `PgStore::arm_tick_fault(faults)` and + `PgStore::tick_fault_flag()` (a `Flag` clone so the consumption pin + survives the store moving into a runner task — a load is enough, + no extra accessor). Per-store `Arc` scoping keeps parallel test + stores isolated. +- **The fault enters the loop via the entry points it already calls** + — both tests drive the full real path (`run_schedules` → the loop's + `tick_with_retries` → `tick_once`), no loop or error-shape + replication anywhere. The fabricated error is pool/database-shaped + (`Error::database(io::Error)` — the taxonomy's opaque fallback; the + loop retries on any `Err`, so the shape is an honesty choice, not a + correctness need). +- **"The runner survives exhaustion" pinned as written.** The + exhausting loop's exit *is* the typed error (the module docs' + contract — exhaustion exits `Err`); "survives" is pinned as: no + panic, opaque `Database` (never `LeadershipLost`), the tick-phase + context riding the source chain with the fabricated fault at its + bottom, the leadership lock row left (not released) to lapse at its + TTL (present, expiry bounded by the TTL of the run — never renewed + past the exit), and the store fully usable after (subsequent entry + points work). Arm (a) carries the *hiccups-don't-kill* survival + property (fault → backoff → real tick fires the due boundary → + clean stop `Ok(())`). +- **Replay-proofed live** (the sqlite task's discipline): with the + seam's fault consumption disabled (`false && take(…)` probe), the + exhaustion test cannot complete — the runner loops forever on its + nothing-due schedule because no fault ever enters the budget (the + mutation run was killed on timeout, which is itself the proof the + seam is load-bearing); reverted, green. The exhaustion test was + then wrapped in a 20 s bounded wait so a future regression of this + shape fails instead of hanging. +- **The backoff-sleep arms are pinned by wall-clock**, not value: the + exhaustion run must take ≥ 1.7 s (the 250 ms → 500 ms → 1 s window; + a fast-fail exit would not). The retry/exhaustion arms' `eprintln` + diagnostics execute (lcov-verified) but are not value-pinned — + stderr capture isn't in the harness convention; the exhaustion + *context* message (chain-carried) is the pinned diagnostic shape. +- `LEADER_TTL_S` widened `private` → `pub(crate)` (the lock-left-to- + TTL expiry pin reads it; no production-path change). +- lcov after: the Finding-1 sites (the retry loop — retry arm, backoff + sleeps, exhaustion `context_error`, diagnostics — and the `tick_once` + seam arm) carry **zero missed lines**. ## Summary -> To be filled on completion \ No newline at end of file +> What landed, verified how: + +- **`alkstore-postgres/src/seam.rs`**: the `#[cfg(test)]` + `tick_fault` module — the per-store counting `Flag` + (`Arc`), `disarmed`/`arm`/`take` (consume-one CAS), and + the fabricated pool/database-shaped transient error, with the + pattern provenance (review 003 Finding 1 / + `sqlite-commit-error-arm`) in its doc comment. +- **`alkstore-postgres/src/scheduler.rs`**: cfg-gated `tick_fault` + field on `EngineCtx` (wildcard-safe destructures elsewhere); the + cfg-gated fault param on `tick_once` (seam check precedes the real + attempt) and `tick_with_retries`, attempt calls cfg-branched; + `run_schedules` threads the store's flag into the loop. The + production path never materializes the seam. +- **`alkstore-postgres/src/store.rs`**: cfg-gated `tick_fault` field + on `PgStore` (born `disarmed()`), `engine_ctx()` clones it into the + scheduler ctx, and the `arm_tick_fault` / `tick_fault_flag` + test-observation surfaces. +- **`alkstore-postgres/src/store/scheduler_tests.rs`**: two pins — + `one_faulted_tick_is_retried_and_the_job_fires` (arm a: faulted + attempt 1 → 250 ms backoff → the retried attempt fires the + backdated due boundary; exactly-one consumption; clean stop `Ok(())`) + and `tick_retry_exhaustion_exits_typed_and_leaves_the_lock_to_lapse` + (arm b: `TICK_RETRIES + 1` armed faults exhaust the budget; ≥ 1.7 s + real backoff window; opaque `Database` with the tick-phase context + and the fabricated fault in the source chain; exact consumption; the + leadership lock row left to its TTL lapse; the store usable after). +- **Gates**: `cargo build`; the full pg column live vs the harness + (124 lib + 25 suite + 9 schema; the two new tests green solo ×3 — + 4.2–4.4 s); `cargo test --workspace` server-less green (skips + clean: core 25 + harness 3; sqlite 191 + 25); `cargo clippy + --all-targets -- -D warnings`; `cargo fmt --check`. cargo-llvm-cov + confirms scheduler.rs's retry-loop region zero missed lines (the + crate stands at 90.73% scheduler line coverage; the residue is the + accepted defensive-arm posture). \ No newline at end of file