sqlite: run_schedules' failed-renewal arm classifies as Database, never LeadershipLost (review 004 item 9) — a lost lease (Ok(false)) keeps the respawn arm unchanged; a failed renewal (closed store / storage failure) propagates its opaque classification, so respawning misdirects at a dead store no more; dated (2026-10-11) ADR-017 §2 class-1 annotation of the loss-vs-failure arm set in ADR-009 §1, core-contract §Errors, queues.md, engine-sqlite.md, core+sqlite doc comments; two tests pin both arms; pg twin recorded for follow-up; task -> completed
This commit is contained in:
1 parent
acdef94058
commit
9a9ccf26e0
8 files changed
+281
-21
No files matched your search
@@ -13,8 +13,14 @@
|
||||
//! - `run_schedules(stop)` — the leader loop (ADR-009 §1): acquire the
|
||||
//! leadership lock `__alkstore_scheduler` (the engine's own lock
|
||||
//! machinery with a derived engine-internal TTL), then loop {renew —
|
||||
//! on loss return `Err(LeadershipLost)` *before ticking*; tick; sleep
|
||||
//! until the next due boundary or stop}. The tick's fire enqueues and
|
||||
//! on a refused renewal return `Err(LeadershipLost)` *before
|
||||
//! ticking* (the lease expired and re-acquired elsewhere; the
|
||||
//! respawn arm); a *failed* renewal is not a loss and propagates its
|
||||
//! opaque `Database` classification unchanged (the closed-store
|
||||
//! posture — respawning a leader against a dead store is misdirected,
|
||||
//! review 004 item 9 — and any other storage failure; 2026-10-11
|
||||
//! classification); tick; sleep until the next due boundary or stop}.
|
||||
//! The tick's fire enqueues and
|
||||
//! the boundary advance commit in one `BEGIN IMMEDIATE` transaction —
|
||||
//! writer serialization as the row lock on the schedule row (ADR-009
|
||||
//! §4's layer (b): even a rogue second ticker observing the same
|
||||
@@ -213,12 +219,26 @@ pub(crate) fn run_schedules(
|
||||
let _ = lease.release().await;
|
||||
return Ok(());
|
||||
}
|
||||
let renewed = lease.renew(LEADER_TTL_S).await.unwrap_or(false);
|
||||
if !renewed {
|
||||
// The lock was lost (expired and re-acquired elsewhere
|
||||
// — exclusion lapses silently at TTL expiry). Return
|
||||
// before ticking: never fire on a stolen lease.
|
||||
return Err(Error::LeadershipLost);
|
||||
let renewed = lease.renew(LEADER_TTL_S).await;
|
||||
match renewed {
|
||||
Ok(true) => {}
|
||||
Ok(false) => {
|
||||
// The lock was lost (expired and re-acquired
|
||||
// elsewhere — exclusion lapses silently at TTL
|
||||
// expiry). Return before ticking: never fire on a
|
||||
// stolen lease.
|
||||
return Err(Error::LeadershipLost);
|
||||
}
|
||||
Err(e) => {
|
||||
// A failed renewal is not a leadership loss: the
|
||||
// lock semantics were never reached. The closed-
|
||||
// store arm (the engine-wide `Database` posture with
|
||||
// the "the store is closed" source — respawning a
|
||||
// leader against a dead store is misdirected, review
|
||||
// 004 item 9) and every other renewal failure
|
||||
// propagate their opaque classification unchanged.
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
let soonest = {
|
||||
let writer = writer.clone();
|
||||
|
||||
@@ -599,3 +599,145 @@ async fn failed_tick_commit_replenishes_the_writer_slot() {
|
||||
drop(concrete);
|
||||
cleanup(&dir);
|
||||
}
|
||||
|
||||
/// Acceptance: the closed-store renewal arm reports the engine-wide
|
||||
/// closed-store `Database` classification — never `LeadershipLost`
|
||||
/// (review 004 item 9: flattening every renew `Err` into the loss arm
|
||||
/// misdirected the documented respawn recipe at a dead store — an
|
||||
/// operator respawns a leader against a store that is closed; the
|
||||
/// lock semantics were never reached when the machinery itself
|
||||
/// errors, 2026-10-11 classification). The runner closes mid-run; the
|
||||
/// next loop-top renewal fails and the classification propagates
|
||||
/// unchanged.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn closed_store_run_reports_the_closed_classification_not_leadership_lost() {
|
||||
let dir = temp_dir("sched-closed");
|
||||
let concrete = open_store(dir.join("store.db").to_str().unwrap(), Default::default()).unwrap();
|
||||
|
||||
concrete
|
||||
.schedule("beat", "@every 1s", "closedq", json!(1), Default::default())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let runner = tokio::spawn(crate::scheduler::run_schedules(
|
||||
StopToken::new(),
|
||||
concrete.writer.clone(),
|
||||
concrete.closed.clone(),
|
||||
concrete.db_path.to_string_lossy().into_owned(),
|
||||
concrete.commit_fault.clone(),
|
||||
));
|
||||
|
||||
// Cross the renew arm at least once: the loop is demonstrably
|
||||
// mid-run (one boundary fired) before the store closes.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(20);
|
||||
loop {
|
||||
let fired: i64 = {
|
||||
let conn = rusqlite::Connection::open(dir.join("store.db")).unwrap();
|
||||
conn.query_row(
|
||||
"SELECT count(*) FROM __alkstore_live WHERE queue = 'closedq'",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
if fired >= 1 {
|
||||
break;
|
||||
}
|
||||
assert!(
|
||||
tokio::time::Instant::now() < deadline,
|
||||
"the runner never fired the first boundary"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
|
||||
concrete.close();
|
||||
|
||||
// The exit is prompt (the sleep is at most one 1 s boundary wide;
|
||||
// the next loop-top renewal sees the closed store).
|
||||
let outcome = tokio::time::timeout(Duration::from_secs(10), runner)
|
||||
.await
|
||||
.expect("the closed-store exit must terminate promptly")
|
||||
.unwrap();
|
||||
match outcome {
|
||||
Err(Error::Database(source)) => {
|
||||
assert!(
|
||||
source.to_string().contains("closed"),
|
||||
"the closed-store arm carries the closed source chain: {source}"
|
||||
);
|
||||
}
|
||||
Err(Error::LeadershipLost) => {
|
||||
panic!("a closed store must never report LeadershipLost (review 004 item 9)")
|
||||
}
|
||||
other => {
|
||||
panic!("the closed-store arm reports the closed Database classification, got {other:?}")
|
||||
}
|
||||
}
|
||||
|
||||
drop(concrete);
|
||||
cleanup(&dir);
|
||||
}
|
||||
|
||||
/// Acceptance: the genuine mid-run leadership-loss arm keeps its
|
||||
/// disposition — `Err(LeadershipLost)`. Deleting the lease row while
|
||||
/// the runner runs (the expired-and-re-acquired-elsewhere shape —
|
||||
/// exclusion lapses silently at TTL expiry, ADR-008 §7) makes the next
|
||||
/// loop-top renewal report the refused-lease outcome and the loop
|
||||
/// returns the loss before any tick; the respawn recipe's arm is
|
||||
/// unchanged by the 2026-10-11 classification.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn mid_run_lease_loss_still_returns_leadership_lost() {
|
||||
let (dir, store) = open_test_store("sched-midrun-loss");
|
||||
|
||||
store
|
||||
.schedule("beat", "@every 1s", "lossq", json!(1), Default::default())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let runner_store =
|
||||
open_store(dir.join("store.db").to_str().unwrap(), Default::default()).unwrap();
|
||||
let runner = tokio::spawn(async move { runner_store.run_schedules(StopToken::new()).await });
|
||||
|
||||
// Demonstrably mid-run: leadership held, one boundary fired.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(20);
|
||||
loop {
|
||||
let fired: i64 = {
|
||||
let conn = rusqlite::Connection::open(dir.join("store.db")).unwrap();
|
||||
conn.query_row(
|
||||
"SELECT count(*) FROM __alkstore_live WHERE queue = 'lossq'",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
if fired >= 1 {
|
||||
break;
|
||||
}
|
||||
assert!(
|
||||
tokio::time::Instant::now() < deadline,
|
||||
"the runner never fired the first boundary"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
|
||||
// The rogue deletion: the runner's lease row vanishes.
|
||||
{
|
||||
let conn = rusqlite::Connection::open(dir.join("store.db")).unwrap();
|
||||
conn.execute(
|
||||
"DELETE FROM __alkstore_locks WHERE name = ?1",
|
||||
[crate::lock::SCHEDULER_LEADER_LOCK],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let outcome = tokio::time::timeout(Duration::from_secs(10), runner)
|
||||
.await
|
||||
.expect("the lease-loss exit must terminate promptly")
|
||||
.unwrap();
|
||||
assert!(
|
||||
matches!(outcome, Err(Error::LeadershipLost)),
|
||||
"a deleted lease mid-run must return LeadershipLost: {outcome:?}"
|
||||
);
|
||||
|
||||
drop(store);
|
||||
cleanup(&dir);
|
||||
}
|
||||
Reference in new issue
Block a user