sqlite: run_schedules' failed-renewal arm classifies as Database, never LeadershipLost (review 004 item 9) — a lost lease (Ok(false)) keeps the respawn arm unchanged; a failed renewal (closed store / storage failure) propagates its opaque classification, so respawning misdirects at a dead store no more; dated (2026-10-11) ADR-017 §2 class-1 annotation of the loss-vs-failure arm set in ADR-009 §1, core-contract §Errors, queues.md, engine-sqlite.md, core+sqlite doc comments; two tests pin both arms; pg twin recorded for follow-up; task -> completed

This commit is contained in:
glm-5.3-flash committed 2026-10-11 11:07:44 +00:00
1 parent acdef94058
commit 9a9ccf26e0
8 files changed
+281 -21

No files matched your search

+28 -8
View File
@@ -13,8 +13,14 @@
//! - `run_schedules(stop)` — the leader loop (ADR-009 §1): acquire the
//! leadership lock `__alkstore_scheduler` (the engine's own lock
//! machinery with a derived engine-internal TTL), then loop {renew —
//! on loss return `Err(LeadershipLost)` *before ticking*; tick; sleep
//! until the next due boundary or stop}. The tick's fire enqueues and
//! on a refused renewal return `Err(LeadershipLost)` *before
//! ticking* (the lease expired and re-acquired elsewhere; the
//! respawn arm); a *failed* renewal is not a loss and propagates its
//! opaque `Database` classification unchanged (the closed-store
//! posture — respawning a leader against a dead store is misdirected,
//! review 004 item 9 — and any other storage failure; 2026-10-11
//! classification); tick; sleep until the next due boundary or stop}.
//! The tick's fire enqueues and
//! the boundary advance commit in one `BEGIN IMMEDIATE` transaction —
//! writer serialization as the row lock on the schedule row (ADR-009
//! §4's layer (b): even a rogue second ticker observing the same
@@ -213,12 +219,26 @@ pub(crate) fn run_schedules(
let _ = lease.release().await;
return Ok(());
}
let renewed = lease.renew(LEADER_TTL_S).await.unwrap_or(false);
if !renewed {
// The lock was lost (expired and re-acquired elsewhere
// — exclusion lapses silently at TTL expiry). Return
// before ticking: never fire on a stolen lease.
return Err(Error::LeadershipLost);
let renewed = lease.renew(LEADER_TTL_S).await;
match renewed {
Ok(true) => {}
Ok(false) => {
// The lock was lost (expired and re-acquired
// elsewhere — exclusion lapses silently at TTL
// expiry). Return before ticking: never fire on a
// stolen lease.
return Err(Error::LeadershipLost);
}
Err(e) => {
// A failed renewal is not a leadership loss: the
// lock semantics were never reached. The closed-
// store arm (the engine-wide `Database` posture with
// the "the store is closed" source — respawning a
// leader against a dead store is misdirected, review
// 004 item 9) and every other renewal failure
// propagate their opaque classification unchanged.
return Err(e);
}
}
let soonest = {
let writer = writer.clone();
@@ -599,3 +599,145 @@ async fn failed_tick_commit_replenishes_the_writer_slot() {
drop(concrete);
cleanup(&dir);
}
/// Acceptance: the closed-store renewal arm reports the engine-wide
/// closed-store `Database` classification — never `LeadershipLost`
/// (review 004 item 9: flattening every renew `Err` into the loss arm
/// misdirected the documented respawn recipe at a dead store — an
/// operator respawns a leader against a store that is closed; the
/// lock semantics were never reached when the machinery itself
/// errors, 2026-10-11 classification). The runner closes mid-run; the
/// next loop-top renewal fails and the classification propagates
/// unchanged.
#[tokio::test(flavor = "multi_thread")]
async fn closed_store_run_reports_the_closed_classification_not_leadership_lost() {
let dir = temp_dir("sched-closed");
let concrete = open_store(dir.join("store.db").to_str().unwrap(), Default::default()).unwrap();
concrete
.schedule("beat", "@every 1s", "closedq", json!(1), Default::default())
.await
.unwrap();
let runner = tokio::spawn(crate::scheduler::run_schedules(
StopToken::new(),
concrete.writer.clone(),
concrete.closed.clone(),
concrete.db_path.to_string_lossy().into_owned(),
concrete.commit_fault.clone(),
));
// Cross the renew arm at least once: the loop is demonstrably
// mid-run (one boundary fired) before the store closes.
let deadline = tokio::time::Instant::now() + Duration::from_secs(20);
loop {
let fired: i64 = {
let conn = rusqlite::Connection::open(dir.join("store.db")).unwrap();
conn.query_row(
"SELECT count(*) FROM __alkstore_live WHERE queue = 'closedq'",
[],
|r| r.get(0),
)
.unwrap()
};
if fired >= 1 {
break;
}
assert!(
tokio::time::Instant::now() < deadline,
"the runner never fired the first boundary"
);
tokio::time::sleep(Duration::from_millis(100)).await;
}
concrete.close();
// The exit is prompt (the sleep is at most one 1 s boundary wide;
// the next loop-top renewal sees the closed store).
let outcome = tokio::time::timeout(Duration::from_secs(10), runner)
.await
.expect("the closed-store exit must terminate promptly")
.unwrap();
match outcome {
Err(Error::Database(source)) => {
assert!(
source.to_string().contains("closed"),
"the closed-store arm carries the closed source chain: {source}"
);
}
Err(Error::LeadershipLost) => {
panic!("a closed store must never report LeadershipLost (review 004 item 9)")
}
other => {
panic!("the closed-store arm reports the closed Database classification, got {other:?}")
}
}
drop(concrete);
cleanup(&dir);
}
/// Acceptance: the genuine mid-run leadership-loss arm keeps its
/// disposition — `Err(LeadershipLost)`. Deleting the lease row while
/// the runner runs (the expired-and-re-acquired-elsewhere shape —
/// exclusion lapses silently at TTL expiry, ADR-008 §7) makes the next
/// loop-top renewal report the refused-lease outcome and the loop
/// returns the loss before any tick; the respawn recipe's arm is
/// unchanged by the 2026-10-11 classification.
#[tokio::test(flavor = "multi_thread")]
async fn mid_run_lease_loss_still_returns_leadership_lost() {
let (dir, store) = open_test_store("sched-midrun-loss");
store
.schedule("beat", "@every 1s", "lossq", json!(1), Default::default())
.await
.unwrap();
let runner_store =
open_store(dir.join("store.db").to_str().unwrap(), Default::default()).unwrap();
let runner = tokio::spawn(async move { runner_store.run_schedules(StopToken::new()).await });
// Demonstrably mid-run: leadership held, one boundary fired.
let deadline = tokio::time::Instant::now() + Duration::from_secs(20);
loop {
let fired: i64 = {
let conn = rusqlite::Connection::open(dir.join("store.db")).unwrap();
conn.query_row(
"SELECT count(*) FROM __alkstore_live WHERE queue = 'lossq'",
[],
|r| r.get(0),
)
.unwrap()
};
if fired >= 1 {
break;
}
assert!(
tokio::time::Instant::now() < deadline,
"the runner never fired the first boundary"
);
tokio::time::sleep(Duration::from_millis(100)).await;
}
// The rogue deletion: the runner's lease row vanishes.
{
let conn = rusqlite::Connection::open(dir.join("store.db")).unwrap();
conn.execute(
"DELETE FROM __alkstore_locks WHERE name = ?1",
[crate::lock::SCHEDULER_LEADER_LOCK],
)
.unwrap();
}
let outcome = tokio::time::timeout(Duration::from_secs(10), runner)
.await
.expect("the lease-loss exit must terminate promptly")
.unwrap();
assert!(
matches!(outcome, Err(Error::LeadershipLost)),
"a deleted lease mid-run must return LeadershipLost: {outcome:?}"
);
drop(store);
cleanup(&dir);
}