diff --git a/alkstore-postgres/src/scheduler.rs b/alkstore-postgres/src/scheduler.rs index 1f35da0..6701691 100644 --- a/alkstore-postgres/src/scheduler.rs +++ b/alkstore-postgres/src/scheduler.rs @@ -54,7 +54,14 @@ //! quarantine advance is an ordinary write). The quarantined row re- //! enters the due set on every subsequent tick (advanced strictly //! past now each time), so it never fires and never fails the -//! runner while its stored spec is broken. Wake-driven tick +//! runner while its stored spec is broken. Each firing schedule's +//! fire wake rides the tick tx ([`crate::tx::wake_tx`]): one +//! coalesced `pg_notify` per tick per schedule that fired ≥ 1, on +//! the fired queue's name — commit-atomic (a rolled-back tick +//! discards the wake with its fire rows) and best-effort (ADR-006; +//! the queues row's pinned re-poll safety net covers any gap) — +//! closing the recorded cross-engine fire-wake latency-parity gap +//! (SQLite's watcher wakes on those commits). Wake-driven tick //! advance: the runner parks on the sleep deadline but the loop's //! stop-select shortens every sleep slice to ≤ 1 s; the slices renew //! the lease and check the stop token only — the soonest is *not* @@ -114,7 +121,7 @@ use crate::resolution::{ }; use crate::schema::{QualifiedTable, tables}; use crate::seam::{database_error, pg_error, pool_error}; -use crate::tx::{encode_payload_bytes, enqueue_row}; +use crate::tx::{encode_payload_bytes, enqueue_row, wake_tx}; /// The contract-constant per-task per-tick catch-up cap (ADR-009 §4 — /// honker parity, not a knob): a downtime longer than 64 interval @@ -424,7 +431,12 @@ pub(crate) fn unschedule(name: &str, ctx: EngineCtx) -> BoxedFuture<'static, Res /// row lock that serializes rogue second tickers (layer (b)) — a /// second ticker re-reading under the lock sees the advanced /// `next_fire_at` and skips. pg's default isolation is exactly the -/// committed-snapshot+row-lock semantics the guarantee names. +/// committed-snapshot+row-lock semantics the guarantee names. Each +/// firing schedule also issues its wake — one coalesced `pg_notify` +/// per tick per schedule that fired ≥ 1, on the fired queue's name, +/// inside the same tx ([`crate::tx::wake_tx`]: commit-atomic — a +/// rolled-back tick discards the wake with its fire rows; best-effort, +/// ADR-006). /// Returns the soonest next fire across all schedules for the sleep /// deadline (0 = none registered). pub(crate) async fn tick(client: &tokio_postgres::Client, schema: &str, now: i64) -> Result { @@ -547,6 +559,21 @@ pub(crate) async fn tick(client: &tokio_postgres::Client, schema: &str, now: i64 next_fire_at = next_fire_at.saturating_add(interval); } + // The fire wake (the recorded cross-engine latency-parity gap + // closed): one coalesced wake per firing schedule per tick — + // after the fire loop, still inside the caller's `in_tx` frame, + // so NOTIFY's native transactional delivery makes it commit- + // atomic (a rolled-back tick discards the wake with its fire + // rows). Channel = the fired queue's name; empty payload; one + // per tick per schedule that fired ≥ 1 (not per fire) — the + // wakes are hint-only (ADR-006), and the SQLite watcher's + // cadence is the committed tick transaction, not the enqueued + // row; a quarantined row (the `continue` above) fired nothing + // and wakes nothing. Best-effort — a failed wake never fails + // the tick (ADR-006); the queues row's pinned re-poll safety + // net covers any gap. + wake_tx(client, &row.queue).await; + client .execute( &format!( diff --git a/alkstore-postgres/src/store/scheduler_tests.rs b/alkstore-postgres/src/store/scheduler_tests.rs index c17de1b..dfa007c 100644 --- a/alkstore-postgres/src/store/scheduler_tests.rs +++ b/alkstore-postgres/src/store/scheduler_tests.rs @@ -136,6 +136,32 @@ fn qopts() -> QueueOpts { } } +/// The forwarder's fanout subscription surface — the listener hears +/// what the server delivers; the listener-side probe for the fire-wake +/// assertions (commit-atomic delivery, coalescing, nothing-due +/// silence). Same shape as `tx_tests.rs`'s helper (the per-file +/// helper convention). +async fn listener_hears( + rx: &mut tokio::sync::broadcast::Receiver, + channel: &str, + within: Duration, +) -> bool { + let deadline = tokio::time::Instant::now() + within; + loop { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + if remaining.is_zero() { + return false; + } + match tokio::time::timeout(remaining, rx.recv()).await { + Ok(Ok(n)) if n.channel == channel => return true, + Ok(Ok(_)) => continue, + Ok(Err(tokio::sync::broadcast::error::RecvError::Lagged(_))) => continue, + Ok(Err(tokio::sync::broadcast::error::RecvError::Closed)) => return false, + Err(_) => return false, + } + } +} + /// The scheduler table row probe (raw admin SQL — the schedule /// registry is engine-internal). Returns (next_fire_at, spec, queue). async fn sched_row( @@ -1167,6 +1193,216 @@ async fn tampered_spec_row_is_quarantined_and_the_runner_survives() { drop_schema(&fx.admin, &fx.schema).await; } +/// Acceptance: the tick's fire path issues its `pg_notify` wake +/// commit-atomically (the recorded cross-engine fire-wake parity gap, +/// closed) — pinned by four arms, the +/// `tx_producer_wakes_are_commit_atomic_and_rollback_discards_them` +/// pattern at the tick's `in_tx` frame: +/// - pre-commit silence: the wake is statement-issued inside the +/// tick's open tx and delivers only at/after its COMMIT (commit- +/// atomic, not statement-atomic); +/// - coalesced to one wake per firing schedule per tick, not per fire +/// (two elapsed boundaries in one tick ⇒ one wake); +/// - a schedule with nothing due in the same tick (and a later tick +/// with nothing due at all) issues no wake; +/// - the end-to-end runner shape: a live `run_schedules` leader's fire +/// wake reaches a registered listener at the tick's commit. +/// +/// The direct-tick legs drive the engine's own tick through the +/// runner's own `in_tx` frame (the `rogue_concurrent_ticks` shape — +/// the pre-commit silence window needs the tick body observable before +/// its COMMIT, which the runner's loop holds internally). The rollback +/// arm is native (NOTIFY's transactional delivery — proven by +/// `notify_tx` and the tx-wake test); no tick-fault seam is built +/// here. +#[tokio::test(flavor = "multi_thread")] +async fn scheduler_fire_wakes_are_commit_atomic_coalesced_and_nothing_due_is_silent() { + let Some(fx) = Fixture::open(&instance_namer("so_wake")).await else { + eprintln!("skip: no harness server"); + return; + }; + + // `beat` fires two elapsed boundaries in the first tick (backdated + // exactly one interval into the past — no live boundary races the + // window); `later` sits an hour out (nothing due anywhere). + fx.store + .schedule("beat", "@every 1s", "fireq", json!(1), Default::default()) + .await + .unwrap(); + fx.store + .schedule( + "later", + "@every 3600s", + "quietq", + json!(1), + Default::default(), + ) + .await + .unwrap(); + let now = now_unix(); + fx.admin + .execute( + &format!( + "UPDATE {} SET next_fire_at = $1 WHERE name = 'beat'", + crate::schema::QualifiedTable { + schema: &fx.schema, + name: crate::schema::tables::SCHEDULE, + } + ), + &[&(now - 1)], + ) + .await + .unwrap(); + + // Listeners registered BEFORE any write rides (the tx-wake test's + // posture): the fired queue's channel and the nothing-due queue's. + fx.store.forwarder().register("fireq").await.unwrap(); + fx.store.forwarder().register("quietq").await.unwrap(); + let mut fire_rx = fx.store.forwarder().subscribe().unwrap(); + let mut quiet_rx = fx.store.forwarder().subscribe().unwrap(); + + // The tick through the runner's own `in_tx` frame: fire the two + // boundaries, then observe the pre-commit silence window inside + // the still-open tx (statement-issued, not delivered). + let schema = fx.schema.clone(); + let (silent_pre_commit, mut fire_rx) = { + let client = fx.store.pool().get().await.unwrap(); + crate::queue::in_tx(client, |client| { + let schema = schema.clone(); + Box::pin(async move { + crate::scheduler::tick(client, &schema, now_unix()).await?; + tokio::time::sleep(Duration::from_millis(300)).await; + let silent = + !listener_hears(&mut fire_rx, "fireq", Duration::from_millis(100)).await; + Ok((silent, fire_rx)) + }) + }) + .await + .unwrap() + }; + assert!( + silent_pre_commit, + "no wake may deliver before the tick's COMMIT" + ); + + // At/after the COMMIT the coalesced wake delivers: one wake for + // the firing schedule, not one per fire. + assert!( + listener_hears(&mut fire_rx, "fireq", Duration::from_secs(5)).await, + "a committed tick wakes the fired queue's channel" + ); + assert!( + !listener_hears(&mut fire_rx, "fireq", Duration::from_millis(400)).await, + "the wake is coalesced per firing schedule per tick, not per fire" + ); + assert!( + !listener_hears(&mut quiet_rx, "quietq", Duration::from_millis(300)).await, + "a schedule with nothing due in the same tick issues no wake" + ); + + // The coalescing arm's load-bearing premise: both elapsed + // boundaries really fired in that one tick. + let fired: i64 = fx + .admin + .query_one( + &format!( + "SELECT count(*) FROM {} WHERE queue = 'fireq'", + crate::schema::QualifiedTable { + schema: &fx.schema, + name: crate::schema::tables::JOB, + } + ), + &[], + ) + .await + .unwrap() + .get(0); + assert!(fired >= 2, "two elapsed boundaries fired, got {fired}"); + + // Pin `beat` an hour out (deterministic — the engine-internal + // boundary, backdated via the same admin-SQL posture as the + // catch-up tests), then the nothing-due tick: no wake inside the + // frame nor after its commit, on either channel. + fx.admin + .execute( + &format!( + "UPDATE {} SET next_fire_at = $1 WHERE name = 'beat'", + crate::schema::QualifiedTable { + schema: &fx.schema, + name: crate::schema::tables::SCHEDULE, + } + ), + &[&(now_unix() + 3600)], + ) + .await + .unwrap(); + let (both_silent, mut fire_rx, mut quiet_rx) = { + let client = fx.store.pool().get().await.unwrap(); + let schema = fx.schema.clone(); + crate::queue::in_tx(client, |client| { + let schema = schema.clone(); + Box::pin(async move { + crate::scheduler::tick(client, &schema, now_unix()).await?; + tokio::time::sleep(Duration::from_millis(300)).await; + let fire_silent = + !listener_hears(&mut fire_rx, "fireq", Duration::from_millis(100)).await; + let quiet_silent = + !listener_hears(&mut quiet_rx, "quietq", Duration::from_millis(100)).await; + Ok((fire_silent && quiet_silent, fire_rx, quiet_rx)) + }) + }) + .await + .unwrap() + }; + assert!( + both_silent, + "the nothing-due tick must wake nobody inside its frame" + ); + assert!( + !listener_hears(&mut fire_rx, "fireq", Duration::from_millis(300)).await, + "a tick with nothing due issues no wake (fireq)" + ); + assert!( + !listener_hears(&mut quiet_rx, "quietq", Duration::from_millis(300)).await, + "a tick with nothing due issues no wake (quietq)" + ); + + // The end-to-end runner leg: a live `run_schedules` leader's fire + // wake reaches a registered listener at the tick's commit — the + // original gap's headline (fires on pg visible to listeners). + fx.store.forwarder().register("runnerq").await.unwrap(); + let mut runner_rx = fx.store.forwarder().subscribe().unwrap(); + fx.store + .schedule( + "runner", + "@every 1s", + "runnerq", + json!(1), + Default::default(), + ) + .await + .unwrap(); + let stop = StopToken::new(); + let runner_store = open_store(&harness_dsn().unwrap(), test_opts(&fx.schema)) + .await + .unwrap(); + let runner_stop = stop.clone(); + let runner = tokio::spawn(async move { runner_store.run_schedules(runner_stop).await }); + assert!( + listener_hears(&mut runner_rx, "runnerq", Duration::from_secs(15)).await, + "a live runner's fire wake reaches a registered listener" + ); + + stop.cancel(); + let outcome = tokio::time::timeout(Duration::from_secs(10), runner) + .await + .expect("clean stop must terminate promptly") + .unwrap(); + assert!(matches!(outcome, Ok(())), "{outcome:?}"); + + drop_schema(&fx.admin, &fx.schema).await; +} + /// The pg engine's own `@every` parser's unit pins (server-less — /// ADR-012 §2's one-owner rule: this engine owns its grammar; the /// contract suite pins the twins' outcomes identical, wave 5). diff --git a/alkstore-postgres/src/tx.rs b/alkstore-postgres/src/tx.rs index 6de0334..343cd92 100644 --- a/alkstore-postgres/src/tx.rs +++ b/alkstore-postgres/src/tx.rs @@ -63,7 +63,10 @@ //! channel realization) **inside the caller's transaction** ([`wake_tx`]). //! NOTIFY's native transactional delivery makes the wake commit-atomic: //! it delivers at the caller's COMMIT, and a rollback discards the row -//! *with* its wake (the tx/no-ghosts rows' pin). Wakes are best-effort +//! *with* its wake (the tx/no-ghosts rows' pin). The scheduler's tick +//! fire path rides the same [`wake_tx`] (one coalesced wake per firing +//! schedule per tick, on the fired queue's name — the scheduler's +//! docs). Wakes are best-effort //! (ADR-006; the engine-wide posture — a wake must never fail a write): //! a failed statement is logged and swallowed, and since a failed //! statement aborts the tx server-side the truth still surfaces typed @@ -283,8 +286,10 @@ fn enqueue_row_sql(schema: &str) -> String { /// effort at the op (the engine-wide posture — a wake must never fail /// a write): a failed statement is logged and swallowed; because a /// failed statement aborts the tx server-side, the truth still -/// surfaces typed at COMMIT. -async fn wake_tx(client: &tokio_postgres::Client, channel: &str) { +/// surfaces typed at COMMIT. Shared by the tx producer paths and the +/// scheduler's tick fire path (the tick's in-tx fire wake — the same +/// commit-atomic semantics at the tick's own `in_tx` frame). +pub(crate) async fn wake_tx(client: &tokio_postgres::Client, channel: &str) { if let Err(wake_err) = client .query_one("SELECT pg_notify($1, '')", &[&channel]) .await diff --git a/docs/plans/implementation.md b/docs/plans/implementation.md index c41f6b8..2c0e335 100644 --- a/docs/plans/implementation.md +++ b/docs/plans/implementation.md @@ -3,7 +3,9 @@ status: draft last_updated: 2026-10-10 (ADR-024 — the mem engine adopted as wave 6, release readiness renumbered to wave 7; wave 6 decomposed — eleven tasks per the pg-wave rhythm with the tx overlay load-bearing. Waves -1–5 implemented and reviewed; engine specs stable.) +1–5 implemented and reviewed; engine specs stable. Wave-5 review +residue decomposed — pg-fix-scheduler-fire-wake; flake-watch carried +to wave 7.) --- # alkstore — Implementation plan @@ -364,7 +366,11 @@ the crate README (written last, honestly), publish prep, N-1's growth-posture review — and the **fuzzing decision** (the alksocks/alktty/alktunnels pattern), the one remaining decided-at-implementation deferral. Decomposes after wave 6's review -gate, with review 003's Findings 2–3 riding the pre-release pass. +gate, with review 003's Findings 2–3 riding the pre-release pass, and +the wave-5 gate's flake-watch flag carried +(`tasks/review-wave-5.md` §Notes 6 — the two unreproducible pg row +failures; if either reproduces, a dedicated investigative session +before any postulate-and-fix). ## Decided points @@ -496,6 +502,23 @@ decomposes. Specific gates: suite's third column as the born-pinned deliverable, the harness's async-sleep pre-task, and review 003 Finding 1 riding the window. See the Wave 6 section. +- **Wave-5 residue decomposition** (2026-10-10) — the wave-5 review's + outstanding items audited post-gate: the one recorded "for a later + task" gap (scheduler fire-wake latency parity) decomposed as + `pg-fix-scheduler-fire-wake` (pre-wave-6, standalone); the flake-watch + flag carried into the wave-7 section above; review 003's Finding 1 + already decomposed (`pg-fix-scheduler-tick-seam`, the wave-6 rider) + and Findings 2–3 already parked to wave 7. See the Wave 7 section for + the watch carriage. +- **Wave-5 residue closure** (2026-10-10) — the remaining + "recorded for a later task" item landed: `pg-fix-scheduler-fire-wake` + closed the scheduler fire-wake latency-parity gap (the tick's + in-tx fires now issue their commit-atomic `pg_notify` wake, one + coalesced wake per firing schedule per tick); the three gap records + (`review-wave-4-fixes`'s recorded-for-later note, `suite-scheduler-rows`'s + disposition note + Notes bullet, `review-wave-5` §Notes 4's audit row) + retired with pointers, keeping the mem engine's fire posture written + against a closed gap before wave 6. - **Wave-4 fix-batch decomposition** (2026-10-09) — the general review's findings decomposed into seven `pg-fix-*` tasks + a `review-wave-4-fixes` gate: Finding 1 → `pg-fix-forwarder-reconnect` diff --git a/tasks/pg-fix-scheduler-fire-wake.md b/tasks/pg-fix-scheduler-fire-wake.md new file mode 100644 index 0000000..3d52e69 --- /dev/null +++ b/tasks/pg-fix-scheduler-fire-wake.md @@ -0,0 +1,205 @@ +--- +id: pg-fix-scheduler-fire-wake +name: "pg scheduler — fire wakes: the tick's in-tx enqueue fires issue their pg_notify wake (the recorded latency-parity gap, closed)" +status: completed +depends_on: [] +scope: narrow +risk: low +impact: component +level: implementation +tags: [wave-5-aftermath, postgres-engine, scheduler, wake] +--- + +## Description + +Close the recorded cross-engine scheduler fire-wake latency-parity gap — +the one item the wave-5 review gate left standing "for a later task" +(`tasks/review-wave-5.md` §Notes 4, audit verdict: "the recorded gap +stands for a later task; accepted as-is"). Provenance chain: +`tasks/review-wave-4-fixes.md` §"Recorded for later" (the original note: +"Scheduler *fire* wakes (the tick's in-tx enqueue) issue no `pg_notify` +— SQLite's watcher wakes on those commits… if wave 5's cross-engine +equivalence rows demand the parity, it is a one-call `wake_tx` inside +the tick's tx"), dispositioned not-demanded by +`tasks/suite-scheduler-rows.md` §Notes (the suite rows observe fires via +claim polling; wake latency out of scope), and audited as-is by the +wave-5 gate. Correctness was never at stake — the queues row's pinned +re-poll safety net covers it on both engines — but the asymmetry stands: +a schedule fire on pg is invisible to a registered listener until the +listener's re-poll, while on SQLite the watcher wakes that queue's +listeners at the tick's commit. + +**Site**: `alkstore-postgres/src/scheduler.rs:518-548` — the tick's +one-firer-per-boundary loop calls `enqueue_row(client, …)` inside the +[`crate::queue::in_tx`] frame (ADR-009 §4's layer (b)) and issues no +wake. The fix is the recorded one-call shape, riding the +`pg-fix-tx-wake` pattern of record (`tasks/pg-fix-tx-wake.md`): +`tx::wake_tx` (currently private at `tx.rs:287` — widen to +`pub(crate)`) issued **inside the tick's `in_tx` frame**, so NOTIFY's +native transactional delivery makes the wake commit-atomic — a rolled- +back tick (crash mid-tick ⇒ the boundary refires) discards the wake +with its fire rows, exactly the tx/no-ghosts discipline the tx producer +paths already pin. + +**Fix constraints** (pinned): + +- Channel = the fired queue's name (the mechanism-name-is-the-channel + realization; `schedule()` rejects reserved names per ADR-021 §3, so a + fire's target can never be a derived outbox channel — the plain queue + name is always the right channel). +- Empty payload — `Wake { channel }` only (ADR-008 §3); best-effort + log-and-swallow posture (`wake_tx`'s existing stance; a wake must + never fail a tick — ADR-006). +- **Coalescing default: one wake per tick per schedule that fired ≥ 1** + (after the fire loop, still inside the `in_tx` body), not per-fire — + wakes are hint-only (ADR-006; no per-wake semantics exist to + preserve), and this matches the SQLite twin's shape (the substrate + watcher wakes per committed tick transaction, not per enqueued row). + Deviate to per-fire only with the reason recorded in Notes. +- No double-wake risk: the wake is applied at the tick call site, NOT + inside the shared `enqueue_row` (the `pg-fix-tx-wake` one-owner + posture — `enqueue_row` stays wake-free; the auto-commit + `Queue::enqueue` and the tx paths keep their own single wakes, + untouched). + +**Test shape** (engine-side, pg): a harness test in the scheduler tests +— a schedule registered against a queue, a listener registered on that +queue's channel (`forwarder().register`; `listener_hears`), the runner +ticks with the schedule due: the fire wake arrives at/after the tick's +commit (pre-commit silence window to pin commit-atomicity, the +`tx_producer_wakes_are_commit_atomic_and_rollback_discards_them` +pattern); a tick with nothing due issues no wake. The rollback arm is +native (NOTIFY's transactional delivery — already proven by `notify_tx` +and the tx-wake test); do not build a tick-fault seam for it here +(`pg-fix-scheduler-tick-seam`'s seam, if landed first, is the retry +loop's, not this site's — same file, disjoint site, either landing +order is fine). + +**Record updates** (the gap is recorded in three places — retire all +three): `tasks/review-wave-4-fixes.md`'s "Recorded for later" bullet +(marked retired by this fix), `tasks/suite-scheduler-rows.md` §Notes' +fire-wake-parity disposition (gap closed, pointer), and +`tasks/review-wave-5.md` §Notes 4's audit-table row (pointer to the +closing task). Add the review-rounds line to +`docs/plans/implementation.md`. + +**Ordering note**: no hard dependency on wave 6, but landing before +`mem-engine-scheduler-outbox` keeps the mem engine's fire posture +written against a closed gap (mem's in-memory wake routing is trivially +immediate; the suite's three-way story then starts symmetric). + +## Acceptance Criteria + +- [x] The tick's fire path issues its wake commit-atomically inside the + `in_tx` frame (channel = the fired queue's name, empty payload, + best-effort `wake_tx` posture); the coalescing decision recorded + in Notes (default: one wake per firing schedule per tick) +- [x] Pinned by test: a listener on the target queue's channel receives + the wake at/after the tick's commit (pre-commit silence window); + a tick with nothing due issues no wake +- [x] No wake regression: auto-commit `Queue::enqueue`, the tx producer + paths, and `enqueue_row` itself unchanged (no double-wake) +- [x] The three gap records retired with pointers + (`review-wave-4-fixes`, `suite-scheduler-rows` Notes, + `review-wave-5` §Notes 4) + the `implementation.md` review-rounds + line +- [x] `cargo test -p alkstore-postgres` green against the harness + server; skips clean server-less; clippy `-D warnings`; fmt clean; + workspace gates green + +## References + +- tasks/review-wave-4-fixes.md §"Recorded for later" (the original + recorded note, incl. the one-call shape) +- tasks/suite-scheduler-rows.md §Notes (the not-demanded disposition) +- tasks/review-wave-5.md §Notes 4 (the gate's audit row) +- tasks/pg-fix-tx-wake.md (the wake_tx pattern + commit-atomicity + precedent; `tx.rs:287`) +- docs/architecture/decisions/006-wake-and-delivery-contract.md (wakes + best-effort, hint-only) +- docs/architecture/decisions/009-scheduler-collapse.md §4 (the tick's + tx frame) +- docs/architecture/engine-postgres.md (pg_notify as the wake trigger) + +## Notes + +Fix-shape decisions of record (what the description left open): + +- **Coalescing: the default** — one wake per tick per schedule that + fired ≥ 1, issued after the fire loop (before the boundary-advance + UPDATE), inside the caller's `in_tx` frame. Not per-fire: wakes are + hint-only (ADR-006, no per-wake semantics exist to preserve), and + the SQLite substrate's watcher wakes per committed tick transaction, + not per enqueued row — the parity shape is the committed tick, not + the individual fire. The `for row in &due` loop naturally yields one + wake per firing schedule (multi-schedule ticks wake each schedule's + own channel once). A quarantined row (`continue`s before the fire + loop) fired nothing and wakes nothing. No per-fire deviation. +- **Wake placement & ownership**: the wake is issued at the tick call + site in `scheduler.rs::tick` via the shared `tx::wake_tx` (widened + private → `pub(crate)` exactly as the task pinned); `enqueue_row` + stays wake-free (no double-wake), the auto-commit `Queue::enqueue` + and the tx producer paths untouched. Channel = the fired queue's + name; empty payload (`Wake { channel }`, ADR-008 §3); best-effort — + a swallowed wake failure aborts the tx server-side so the truth + surfaces typed at COMMIT (and the tick rolls back with its fires). +- **No wake issued for a due row whose fire count is zero-guarded** — + no guard added: a non-quarantined due row fires ≥ 1 by construction + (the `while next_fire_at <= now` loop runs at least once), so the + unconditional post-loop wake is exactly "per schedule that fired + ≥ 1". +- **Test shape**: one harness test in `scheduler_tests.rs` + (`scheduler_fire_wakes_are_commit_atomic_coalesced_and_nothing_due_is_silent`), + four arms: (a) pre-commit silence — the tick driven directly through + the runner's own `in_tx` frame (the `rogue_concurrent_ticks` shape; + the silence window needs the tick body observable before its COMMIT, + which the runner's loop holds internally), listener silent + statement-side, hears at/after commit; (b) coalescing — backdated + one interval ⇒ ≥ 2 boundaries fire in the one tick, exactly one wake + (not per fire); (c) nothing-due silence — a same-tick not-due + schedule and a later all-not-due tick wake nobody (in-frame and + post-commit); (d) end-to-end runner leg — a live `run_schedules` + leader's fire wake reaches a registered listener. The direct-tick + drive is the runner's exact frame (`tick_once` = `in_tx` + `tick`); + the runner loop itself is separately exercised by the existing fire + tests and by arm (d). A local `listener_hears` helper copy rides the + per-file helper convention (`tx_tests.rs` keeps its own). The + rollback arm was not test-built here (native — NOTIFY's transactional + delivery, proven by `notify_tx` and the tx-wake test; the task + pinned no tick-fault seam for it). +- **Doc updates rode along**: `tx.rs`'s "The tx wakes" module section + and `wake_tx`'s doc now name the scheduler fire path as a shared + caller; `scheduler.rs`'s module docs and `tick`'s doc pin the + coalesced commit-atomic fire-wake semantics (the contract-relevant + text). +- **Flake-ledger occurrence (unrelated to this task's mechanism)**: + during the first full pg suite run this session, + `row_lock_ttl_expiry_and_reacquisition` (contract suite) failed + once — the wave-5 ledger's second recorded occurrence. Green on the + next two full runs plus six focused runs. Recorded in + `tasks/review-wave-5.md` §6 with the ledger's own conclusion: the + "fails again" trigger is met, a dedicated investigative session is + owed by the wave-7 watch carriage before any postulate-and-fix. Not + load-bearing here (locks, not scheduler/wakes; scheduler code + untouched by that row). + +## Summary + +**Landed**: the scheduler tick's commit-atomic fire wake — +`scheduler.rs::tick` now issues one coalesced `pg_notify` per firing +schedule per tick (channel = the fired queue's name, empty payload) +inside the tick's `in_tx` frame via the shared `tx::wake_tx` (widened +to `pub(crate)`); the new four-arm pinning test; the driver docs +(`tx.rs` wake section, `scheduler.rs` module + `tick` docs); and the +three gap records retired with pointers (`review-wave-4-fixes`'s +recorded-for-later note, `suite-scheduler-rows`'s gate disposition +note + Notes bullet, `review-wave-5` §Notes 4's audit row) plus the +`implementation.md` review-rounds line — the three-way (SQLite / pg / +mem-to-be) fire-wake story is symmetric ahead of wave 6. + +**Verified**: full pg lib suite 122/122 green against the harness +server (the new test deterministically green across three solo runs); +pg contract suite 25/25 and schema 9/9 green (one unrelated single-run +flake noted above, ledger-carried); workspace `cargo test` green +server-less (pg skips clean, incl. the new test); `cargo clippy +--all-targets -- -D warnings` clean; `cargo fmt --check` clean. \ No newline at end of file diff --git a/tasks/review-wave-4-fixes.md b/tasks/review-wave-4-fixes.md index 22f3529..c899b08 100644 --- a/tasks/review-wave-4-fixes.md +++ b/tasks/review-wave-4-fixes.md @@ -216,7 +216,12 @@ the truthful "no other path targets it" claim. twins; correctness is covered by the queues row's pinned re-poll safety net. Not in review 002's fix shape; if wave 5's cross-engine equivalence rows demand the parity, it is a one-call - `wake_tx` inside the tick's tx. + `wake_tx` inside the tick's tx. **[Retired 2026-10-10]** Closed by + `tasks/pg-fix-scheduler-fire-wake.md` — the tick now issues the + one-call `wake_tx` inside its own `in_tx` frame (one coalesced wake + per firing schedule per tick, commit-atomic; the disposition note + in `tasks/suite-scheduler-rows.md` and the wave-5 gate's audit row + carry the same retirement pointer). ## Summary diff --git a/tasks/review-wave-5.md b/tasks/review-wave-5.md index eddcd04..039faf2 100644 --- a/tasks/review-wave-5.md +++ b/tasks/review-wave-5.md @@ -174,7 +174,7 @@ Focused stress: 6 consecutive `--test-threads=6` runs of | M-1 retention-failure pin's location | `suite-queue-depth-rows` Notes | Engine-side (SQLite trigger seam; pg by frame structure) — sound; the pg leg rests on code-read, accepted with the row's doc text recording it | | Beyond-64 skip-forward leg | `suite-scheduler-rows` Notes | Engine-side twins exist on both engines (`catch_up_replays_up_to_the_cap_then_skips_forward`); suite pins the ≤64 in-band leg; row doc states it | | Backoff cap leg | `suite-opts-backoff-rows` Notes | Engine-side pins verified present (both engines' unit tests); row doc records the disposition | -| Scheduler fire-wake parity | `suite-scheduler-rows` Notes | Not demanded by the row shapes (claim-polling observation only); the recorded gap stands for a later task; accepted as-is | +| Scheduler fire-wake parity | `suite-scheduler-rows` Notes | Not demanded by the row shapes (claim-polling observation only); the recorded gap stands for a later task; accepted as-is — **closed post-gate** (2026-10-10) by `pg-fix-scheduler-fire-wake` (the tick's commit-atomic fire wake; the row's Notes and `review-wave-4-fixes` carry the retirement pointers) | | Lock-row divergence call | `suite-lock-rows` Notes | No divergence found — the row pins the convergence; nothing to fix | | SQLite reconnect-success test | `sqlite-commit-error-arm` Notes | Taken — `reconnect_success_resumes_wake_delivery` exists and pins the success arm's re-baseline + restored delivery | @@ -219,6 +219,19 @@ clean pg runs across both rows since the failures, all conditions covered. **Flagged for watch in the next wave window** (release readiness at gate time — since renumbered to wave 7, ADR-024). +*Ledger update (2026-10-10, from the `pg-fix-scheduler-fire-wake` +session):* `row_lock_ttl_expiry_and_reacquisition` failed once more — +a single occurrence in one full pg suite run (`cargo test +-p alkstore-postgres`, sequential, against the harness; the row's +failure message was lost to the run's output filtering), then green on +two subsequent full runs and six consecutive focused `row_lock_ttl` +runs. Unrelated mechanism to that task's change (scheduler fire wake; +locks untouched). Per this ledger's own prescription the row now +meets the "fails again" trigger — a dedicated investigative session +(instrumentation + focused repro loop) is owed by the wave-7 watch +carriage before any postulate-and-fix. Recorded here for the ledger's +honesty; not blocking the fire-wake task. + **7. Findings.** No code, suite-row, or stamp defects requiring change were found at gate time; all changes this gate made are doc-side (engine specs `draft` → `stable` with dated gate annotations; diff --git a/tasks/suite-scheduler-rows.md b/tasks/suite-scheduler-rows.md index 5ac5640..ec60e6a 100644 --- a/tasks/suite-scheduler-rows.md +++ b/tasks/suite-scheduler-rows.md @@ -59,7 +59,11 @@ defect): the pg tick's in-tx enqueue fires issue no `pg_notify` wake wake-latency-parity gap the re-poll safety net covers for correctness. If this row's shape demands the parity, the one-call `wake_tx` inside the tick's tx rides **this** task; otherwise record the row's -latency-posture in Notes and leave the engines as they are. +latency-posture in Notes and leave the engines as they are. **[Retired +2026-10-10]** The parity was not demanded here, but the gap itself is +now closed: `tasks/pg-fix-scheduler-fire-wake.md` landed the tick's +commit-atomic fire wake (one coalesced wake per firing schedule per +tick) — see the updated Notes bullet below. ## Acceptance Criteria @@ -102,14 +106,20 @@ latency-posture in Notes and leave the engines as they are. - **Fire-wake parity: not demanded by this row set; engines left as-is.** The suite rows observe fire outcomes through ordinary claim polling, never through wake latency, so the row shapes demand - no wake. The recorded cross-engine latency-parity gap stands (the pg + no wake. The recorded cross-engine latency-parity gap stood (the pg tick's in-tx enqueue fires no `pg_notify`; SQLite's watcher wakes on those commits — `tasks/review-wave-4-fixes.md`'s recorded-for-later note): correctness is covered by the queues row's pinned re-poll safety net on both engines; the row set here pins state outcomes with tolerances bounded well above the boundary resolution, so wake - latency is out of scope. If a later task demands the parity, the - one-call `wake_tx` inside the pg tick's tx is the shape to ride. + latency is out of scope. **[Retired 2026-10-10]** The gap itself is + closed by `tasks/pg-fix-scheduler-fire-wake.md`: the pg tick now + issues its commit-atomic `pg_notify` fire wake inside the tick's tx + (one coalesced wake per firing schedule per tick, channel = the + fired queue's name, best-effort per ADR-006), so both engines wake + on a firing tick's commit and the three-way suite story starts + symmetric. The row shapes here stay as landed — claim-polling + observation, wake latency out of scope — with no re-shape owed. - **Runner-driving posture details** (the extension's riders, beyond what the description pinned): the spawned runners share the row's single store handle via `Arc` clones — leadership is