pg: run_schedules' failed-renewal arm classifies as Database, never LeadershipLost (review 004 item 9's pg twin) — a lost lease (Ok(false)) keeps the respawn arm unchanged; a failed renewal (closed store / storage failure) propagates its opaque classification, matching the sqlite task's 2026-10-11 classification landed engine-neutral in ADR-009 §1 / core-contract §Errors (cited, not re-decided); module doc aligned; two tests pin both arms on the harness server (the sqlite twin's pin shapes); task -> completed

This commit is contained in:
glm-5.3-flash committed 2026-10-11 12:31:48 +00:00
1 parent c03c798e34
commit 608104ab13
3 files changed
+249 -10

No files matched your search

+30 -7
View File
@@ -35,7 +35,16 @@
//! its own lease by drift; a failed mid-sleep renewal is not acted
//! on at the sleep site — the loop's top-of-iteration renew owns
//! the loss decision). Acquire-time refusal is `Err(LeadershipLost)`
//! — the same matchable loss a mid-run loser returns. The tick:
//! — the same matchable loss a mid-run loser returns. A mid-run
//! top-of-iteration renewal's arms (the SQLite twin's shape, the
//! 2026-10-11 classification, review 004 item 9): a *refused*
//! renewal (`renew` → `Ok(false)` — the lease expired and was
//! re-acquired elsewhere) is the loss, `Err(LeadershipLost)`
//! return-before-tick (the respawn arm); a *failed* renewal is not
//! a loss and propagates its opaque `Database` classification
//! unchanged (the closed-store posture — respawning a leader
//! against a dead store is misdirected — and any other storage
//! failure). The tick:
//! one pool-checked-out transaction (the [`crate::queue::in_tx`]
//! frame), the schedule row's `SELECT … FOR UPDATE` inside the tick
//! tx as the row lock that serializes rogue second tickers (ADR-009
@@ -679,12 +688,26 @@ pub(crate) fn run_schedules(stop: StopToken, ctx: EngineCtx) -> BoxedFuture<'sta
let _ = lease.release().await;
return Ok(());
}
let renewed = lease.renew(LEADER_TTL_S).await.unwrap_or(false);
if !renewed {
// The lock was lost (expired and re-acquired elsewhere
// — exclusion lapses silently at TTL expiry). Return
// before ticking: never fire on a stolen lease.
return Err(Error::LeadershipLost);
let renewed = lease.renew(LEADER_TTL_S).await;
match renewed {
Ok(true) => {}
Ok(false) => {
// The lock was lost (expired and re-acquired
// elsewhere — exclusion lapses silently at TTL
// expiry). Return before ticking: never fire on a
// stolen lease.
return Err(Error::LeadershipLost);
}
Err(e) => {
// A failed renewal is not a leadership loss: the
// lock semantics were never reached. The closed-
// store arm (the engine-wide `Database` posture —
// respawning a leader against a dead store is
// misdirected, review 004 item 9) and every other
// renewal failure propagate their opaque
// classification unchanged.
return Err(e);
}
}
#[cfg(test)]
let soonest = tick_with_retries(&pool, &schema, Some(&tick_fault)).await?;
@@ -1713,3 +1713,171 @@ async fn tick_retry_exhaustion_exits_typed_and_leaves_the_lock_to_lapse() {
drop_schema(&fx.admin, &fx.schema).await;
}
/// Acceptance: the closed-store renewal arm reports the engine-wide
/// closed-store `Database` classification — never `LeadershipLost`
/// (review 004 item 9: flattening every renew `Err` into the loss arm
/// misdirected the documented respawn recipe at a dead store — an
/// operator respawns a leader against a store that is closed; the
/// lock semantics were never reached when the machinery itself
/// errors, the 2026-10-11 classification). The runner runs on the
/// fixture store's own closed flag and pool (the `engine_ctx` seam);
/// the runner closes mid-run — the next loop-top renewal sees the
/// closed flag and the classification propagates unchanged. The
/// SQLite twin's pin shape (`closed_store_run_reports_the_closed_
/// classification_not_leadership_lost`).
#[tokio::test(flavor = "multi_thread")]
async fn closed_store_run_reports_the_closed_classification_not_leadership_lost() {
let Some(fx) = Fixture::open(&instance_namer("so_cls")).await else {
eprintln!("skip: no harness server");
return;
};
fx.store
.schedule("beat", "@every 1s", "closedq", json!(1), Default::default())
.await
.unwrap();
let ctx = fx.store.engine_ctx();
let runner =
tokio::spawn(async move { crate::scheduler::run_schedules(StopToken::new(), ctx).await });
// Cross the renew arm at least once: the loop is demonstrably
// mid-run (one boundary fired) before the store closes.
let deadline = tokio::time::Instant::now() + Duration::from_secs(20);
loop {
let fired: i64 = fx
.admin
.query_one(
&format!(
"SELECT count(*) FROM {} WHERE queue = 'closedq'",
crate::schema::QualifiedTable {
schema: &fx.schema,
name: crate::schema::tables::JOB,
}
),
&[],
)
.await
.unwrap()
.get(0);
if fired >= 1 {
break;
}
assert!(
tokio::time::Instant::now() < deadline,
"the runner never fired the first boundary"
);
tokio::time::sleep(Duration::from_millis(200)).await;
}
fx.store.close();
// The exit is prompt (the sleep is at most one 1 s boundary wide;
// the next loop-top renewal sees the closed store).
let outcome = tokio::time::timeout(Duration::from_secs(10), runner)
.await
.expect("the closed-store exit must terminate promptly")
.unwrap();
match outcome {
Err(Error::Database(source)) => {
assert!(
source.to_string().contains("closed"),
"the closed-store arm carries the closed source chain: {source}"
);
}
Err(Error::LeadershipLost) => {
panic!("a closed store must never report LeadershipLost (review 004 item 9)")
}
other => {
panic!("the closed-store arm reports the closed Database classification, got {other:?}")
}
}
drop_schema(&fx.admin, &fx.schema).await;
}
/// Acceptance: the genuine mid-run leadership-loss arm keeps its
/// disposition — `Err(LeadershipLost)`. Deleting the leadership lock
/// row while the runner runs (the expired-and-re-acquired-elsewhere
/// shape — exclusion lapses silently at TTL expiry, ADR-008 §7) makes
/// the next loop-top renewal report the refused-lease outcome and the
/// loop returns the loss before any tick; the respawn recipe's arm is
/// unchanged by the classification. The SQLite twin's pin shape
/// (`mid_run_lease_loss_still_returns_leadership_lost`).
#[tokio::test(flavor = "multi_thread")]
async fn mid_run_lease_loss_still_returns_leadership_lost() {
let Some(fx) = Fixture::open(&instance_namer("so_lss")).await else {
eprintln!("skip: no harness server");
return;
};
fx.store
.schedule("beat", "@every 1s", "lossq", json!(1), Default::default())
.await
.unwrap();
let runner_store = open_store(&harness_dsn().unwrap(), test_opts(&fx.schema))
.await
.unwrap();
let runner = tokio::spawn(async move { runner_store.run_schedules(StopToken::new()).await });
// Demonstrably mid-run: leadership held, one boundary fired.
let deadline = tokio::time::Instant::now() + Duration::from_secs(20);
loop {
let fired: i64 = fx
.admin
.query_one(
&format!(
"SELECT count(*) FROM {} WHERE queue = 'lossq'",
crate::schema::QualifiedTable {
schema: &fx.schema,
name: crate::schema::tables::JOB,
}
),
&[],
)
.await
.unwrap()
.get(0);
if fired >= 1 {
break;
}
assert!(
tokio::time::Instant::now() < deadline,
"the runner never fired the first boundary"
);
tokio::time::sleep(Duration::from_millis(200)).await;
}
// The rogue deletion: the runner's lease row vanishes.
let deleted = fx
.admin
.execute(
&format!(
"DELETE FROM {} WHERE name = $1",
crate::schema::QualifiedTable {
schema: &fx.schema,
name: crate::schema::tables::LOCKS,
}
),
&[&crate::scheduler::SCHEDULER_LEADER_LOCK],
)
.await
.unwrap();
assert_eq!(
deleted, 1,
"the leadership lock row must exist to be deleted mid-run"
);
let outcome = tokio::time::timeout(Duration::from_secs(10), runner)
.await
.expect("the lease-loss exit must terminate promptly")
.unwrap();
assert!(
matches!(outcome, Err(Error::LeadershipLost)),
"a deleted lease mid-run must return LeadershipLost: {outcome:?}"
);
drop_schema(&fx.admin, &fx.schema).await;
}