1082 lines
35 KiB
Rust
1082 lines
35 KiB
Rust
//! The scheduler mechanism's acceptance tests (`pg-engine-scheduler-outbox`):
|
||
//! the entry-point validation triad (InvalidSpec pre-storage with no
|
||
//! round trip, InvalidName/ReservedName on both the schedule name and
|
||
//! the queue argument — ADR-021 §3), upsert read-back, the leader loop
|
||
//! firing due boundaries into the named queue (claimable via the
|
||
//! ordinary queue surface, get_job-visible with the plain-queue
|
||
//! defaults + ScheduleOpts applied — ADR-020 §3), the leadership lock
|
||
//! held with a per-instance owner token (`__alkstore_scheduler`),
|
||
//! second-runner refusal with `Err(LeadershipLost)` and
|
||
//! one-firer-per-boundary, acquire-time loss before any tick, in-sleep
|
||
//! lease renewals holding across boundaries further out than the TTL,
|
||
//! the 64-cap catch-up + skip-forward with no in-window refire, the
|
||
//! tick's row-locked transactional advance, and clean-stop Ok(()).
|
||
//!
|
||
//! Harness convention (the schema/open tasks'): connection settings
|
||
//! ride the environment (`ALKSTORE_PG_HOST/PORT/USER/PASSWORD/DB`),
|
||
//! never hardcoded; tests without a reachable server skip cleanly so
|
||
//! the workspace gates stay green server-less. Isolation is a fresh
|
||
//! unique schema per test. Flake-fix lesson (the SQLite task's Notes):
|
||
//! catch-up tests drive from far-out horizons so no live boundary
|
||
//! races the assertion window.
|
||
|
||
use std::time::Duration;
|
||
|
||
use alkstore::{Error, JobState, QueueOpts, ScheduleOpts, StopToken, Store};
|
||
use serde_json::json;
|
||
|
||
use crate::store::{PgStore, open_store};
|
||
|
||
const ENV_HOST: &str = "ALKSTORE_PG_HOST";
|
||
const ENV_PORT: &str = "ALKSTORE_PG_PORT";
|
||
const ENV_USER: &str = "ALKSTORE_PG_USER";
|
||
const ENV_PASSWORD: &str = "ALKSTORE_PG_PASSWORD";
|
||
const ENV_DB: &str = "ALKSTORE_PG_DB";
|
||
|
||
fn harness_dsn() -> Option<String> {
|
||
let host = std::env::var(ENV_HOST).ok()?;
|
||
let port: u16 = std::env::var(ENV_PORT).ok()?.parse().ok()?;
|
||
let user = std::env::var(ENV_USER).ok()?;
|
||
let password = std::env::var(ENV_PASSWORD).ok()?;
|
||
let db = std::env::var(ENV_DB).unwrap_or_else(|_| "postgres".to_string());
|
||
Some(format!(
|
||
"host={host} port={port} user={user} password={password} dbname={db}"
|
||
))
|
||
}
|
||
|
||
fn instance_namer(tag: &str) -> impl Fn() -> String + use<'_> {
|
||
use std::sync::atomic::{AtomicU64, Ordering};
|
||
let counter = AtomicU64::new(0);
|
||
move || {
|
||
format!(
|
||
"{tag}_{}_{}_{}",
|
||
std::process::id(),
|
||
std::time::SystemTime::now()
|
||
.duration_since(std::time::UNIX_EPOCH)
|
||
.unwrap_or_default()
|
||
.as_nanos(),
|
||
counter.fetch_add(1, Ordering::SeqCst),
|
||
)
|
||
}
|
||
}
|
||
|
||
async fn harness_client() -> Option<tokio_postgres::Client> {
|
||
let (client, connection) = tokio_postgres::connect(&harness_dsn()?, tokio_postgres::NoTls)
|
||
.await
|
||
.ok()?;
|
||
tokio::spawn(async move {
|
||
let _ = connection.await;
|
||
});
|
||
Some(client)
|
||
}
|
||
|
||
async fn drop_schema(client: &tokio_postgres::Client, schema: &str) {
|
||
if schema == crate::schema::DEFAULT_SCHEMA {
|
||
return;
|
||
}
|
||
let sql = format!(
|
||
"DROP SCHEMA IF EXISTS {} CASCADE",
|
||
crate::schema::quote_identifier(schema)
|
||
);
|
||
let _ = client.batch_execute(&sql).await;
|
||
}
|
||
|
||
fn test_opts(schema: &str) -> crate::opts::PgOpts {
|
||
crate::opts::PgOpts {
|
||
schema: schema.to_string(),
|
||
..crate::opts::PgOpts::default()
|
||
}
|
||
}
|
||
|
||
/// Open a store on the harness server with a fresh unique schema —
|
||
/// `None` = no harness server (the skip posture). The returned store
|
||
/// closes on `Fixture` drop (the schema teardown included).
|
||
struct Fixture {
|
||
store: PgStore,
|
||
admin: tokio_postgres::Client,
|
||
schema: String,
|
||
}
|
||
|
||
impl Fixture {
|
||
async fn open(namer: &impl Fn() -> String) -> Option<Fixture> {
|
||
let dsn = harness_dsn()?;
|
||
let schema = namer();
|
||
let store = open_store(&dsn, test_opts(&schema)).await.unwrap();
|
||
let admin = harness_client().await?;
|
||
Some(Fixture {
|
||
store,
|
||
admin,
|
||
schema,
|
||
})
|
||
}
|
||
}
|
||
|
||
impl Drop for Fixture {
|
||
fn drop(&mut self) {
|
||
self.store.close();
|
||
}
|
||
}
|
||
|
||
fn now_unix() -> i64 {
|
||
std::time::SystemTime::now()
|
||
.duration_since(std::time::UNIX_EPOCH)
|
||
.unwrap()
|
||
.as_secs() as i64
|
||
}
|
||
|
||
fn qopts() -> QueueOpts {
|
||
QueueOpts {
|
||
visibility_timeout_s: 300,
|
||
max_attempts: 3,
|
||
backoff_base_s: 5,
|
||
dead_letter_retention_s: None,
|
||
}
|
||
}
|
||
|
||
/// The scheduler table row probe (raw admin SQL — the schedule
|
||
/// registry is engine-internal). Returns (next_fire_at, spec, queue).
|
||
async fn sched_row(
|
||
admin: &tokio_postgres::Client,
|
||
schema: &str,
|
||
name: &str,
|
||
) -> (i64, String, String) {
|
||
let row = admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT next_fire_at, spec, queue FROM {} WHERE name = $1",
|
||
crate::schema::QualifiedTable {
|
||
schema,
|
||
name: crate::schema::tables::SCHEDULE,
|
||
}
|
||
),
|
||
&[&name],
|
||
)
|
||
.await
|
||
.unwrap();
|
||
(row.get(0), row.get(1), row.get(2))
|
||
}
|
||
|
||
/// Acceptance: `schedule` validates at the entry point on all three
|
||
/// arguments — empty/reserved schedule name (InvalidName /
|
||
/// ReservedName), empty/reserved queue argument (same, ADR-021 §3),
|
||
/// and a non-`@every` or malformed spec → `InvalidSpec` **before
|
||
/// storage** (no round trip; nothing lands in the registry). The
|
||
/// pg-owned parser's exact accepted/rejected surface pins here too:
|
||
/// the four units, multi-digit counts, and the rejects (cron, unknown
|
||
/// unit, zero, float, bare `every`, unitless).
|
||
#[tokio::test(flavor = "multi_thread")]
|
||
async fn schedule_entry_point_validation_is_pinned_on_all_arguments() {
|
||
let Some(fx) = Fixture::open(&instance_namer("so_val")).await else {
|
||
eprintln!("skip: no harness server");
|
||
return;
|
||
};
|
||
|
||
let check = |err: Error, expect: &str, name: &str| match err {
|
||
Error::InvalidName { .. } => assert_eq!(expect, "empty", "{name}"),
|
||
Error::ReservedName { .. } => assert_eq!(expect, "reserved", "{name}"),
|
||
Error::InvalidSpec { spec } => {
|
||
assert_eq!(expect, "spec", "{name}");
|
||
assert!(
|
||
!spec.is_empty(),
|
||
"{name}: InvalidSpec carries the spec string"
|
||
);
|
||
}
|
||
other => panic!("{name} got the wrong error: {other}"),
|
||
};
|
||
|
||
check(
|
||
fx.store
|
||
.schedule("", "@every 1s", "work", json!(null), Default::default())
|
||
.await
|
||
.unwrap_err(),
|
||
"empty",
|
||
"empty schedule name",
|
||
);
|
||
check(
|
||
fx.store
|
||
.schedule(
|
||
"__alkstore_rogue",
|
||
"@every 1s",
|
||
"work",
|
||
json!(null),
|
||
Default::default(),
|
||
)
|
||
.await
|
||
.unwrap_err(),
|
||
"reserved",
|
||
"reserved schedule name",
|
||
);
|
||
check(
|
||
fx.store
|
||
.schedule("tick", "@every 1s", "", json!(null), Default::default())
|
||
.await
|
||
.unwrap_err(),
|
||
"empty",
|
||
"empty queue argument",
|
||
);
|
||
check(
|
||
fx.store
|
||
.schedule(
|
||
"tick",
|
||
"@every 1s",
|
||
"__alkstore_outbox:steal",
|
||
json!(null),
|
||
Default::default(),
|
||
)
|
||
.await
|
||
.unwrap_err(),
|
||
"reserved",
|
||
"reserved queue argument (ADR-021 §3 — no third write path into the backing queue)",
|
||
);
|
||
for bad in [
|
||
"0 9 * * *",
|
||
"@every 1x",
|
||
"@every 0s",
|
||
"every 5s",
|
||
"@every s",
|
||
"@every 1.5s",
|
||
"@every 5",
|
||
"* * * * *",
|
||
] {
|
||
check(
|
||
fx.store
|
||
.schedule("bad-spec", bad, "work", json!(null), Default::default())
|
||
.await
|
||
.unwrap_err(),
|
||
"spec",
|
||
&(format!("spec {bad:?}")),
|
||
);
|
||
}
|
||
|
||
// The rejected calls never reached storage (pre-storage, no round
|
||
// trip): the schedule table has no `bad-spec` row and the two
|
||
// `tick` attempts left nothing either.
|
||
let rows: i64 = fx
|
||
.admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT count(*) FROM {}",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::SCHEDULE,
|
||
}
|
||
),
|
||
&[],
|
||
)
|
||
.await
|
||
.unwrap()
|
||
.get(0);
|
||
assert_eq!(rows, 0, "rejected calls must not round-trip");
|
||
|
||
// The accepted grammar surface: the four units parse.
|
||
for (spec, _unit) in [
|
||
("@every 30s", 30),
|
||
("@every 5m", 300),
|
||
("@every 2h", 7200),
|
||
("@every 7d", 604800),
|
||
] {
|
||
let sched = fx
|
||
.store
|
||
.schedule(
|
||
&format!("u-{spec}"),
|
||
spec,
|
||
"work",
|
||
json!(1),
|
||
Default::default(),
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(sched.spec, spec);
|
||
}
|
||
|
||
drop_schema(&fx.admin, &fx.schema).await;
|
||
}
|
||
|
||
/// Acceptance: `schedule` upserts by name — a re-register replaces the
|
||
/// whole row (spec, queue, payload, opts and the next-fire reset),
|
||
/// read-back reflects the replacement; `unschedule` returns true then
|
||
/// false.
|
||
#[tokio::test(flavor = "multi_thread")]
|
||
async fn schedule_upsert_replaces_the_whole_row_and_unschedule_removes() {
|
||
let Some(fx) = Fixture::open(&instance_namer("so_ups")).await else {
|
||
eprintln!("skip: no harness server");
|
||
return;
|
||
};
|
||
|
||
let first = fx
|
||
.store
|
||
.schedule(
|
||
"sweep",
|
||
"@every 10s",
|
||
"old-queue",
|
||
json!({"v": 1}),
|
||
ScheduleOpts {
|
||
priority: 3,
|
||
max_attempts: Some(7),
|
||
expires: Some(120),
|
||
},
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(first.queue, "old-queue");
|
||
assert_eq!(first.opts.priority, 3);
|
||
|
||
let second = fx
|
||
.store
|
||
.schedule(
|
||
"sweep",
|
||
"@every 20m",
|
||
"new-queue",
|
||
json!({"v": 2}),
|
||
ScheduleOpts::default(),
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(second.spec, "@every 20m");
|
||
assert_eq!(second.queue, "new-queue");
|
||
assert_eq!(second.opts, ScheduleOpts::default());
|
||
|
||
// The stored row reflects the replacement (spec, queue, one row).
|
||
let (_, spec, queue) = sched_row(&fx.admin, &fx.schema, "sweep").await;
|
||
assert_eq!(spec, "@every 20m");
|
||
assert_eq!(queue, "new-queue");
|
||
let count: i64 = fx
|
||
.admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT count(*) FROM {} WHERE name = 'sweep'",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::SCHEDULE,
|
||
}
|
||
),
|
||
&[],
|
||
)
|
||
.await
|
||
.unwrap()
|
||
.get(0);
|
||
assert_eq!(count, 1, "upsert by name — one row");
|
||
|
||
// unregister + re-register is the pause shape (ADR-009 §1).
|
||
assert!(fx.store.unschedule("sweep").await.unwrap());
|
||
assert!(!fx.store.unschedule("sweep").await.unwrap());
|
||
|
||
// unschedule validates like the register path (a name-bearing
|
||
// entry point).
|
||
for bad in ["", "__alkstore_x"] {
|
||
let err = fx.store.unschedule(bad).await.unwrap_err();
|
||
assert!(
|
||
matches!(err, Error::InvalidName { .. } if bad.is_empty())
|
||
|| matches!(err, Error::ReservedName { .. }),
|
||
"unschedule({bad:?}) must reject at the entry point: {err:?}"
|
||
);
|
||
}
|
||
|
||
drop_schema(&fx.admin, &fx.schema).await;
|
||
}
|
||
|
||
/// Acceptance: the leader loop fires due boundaries into the named
|
||
/// queue — the fired jobs are ordinary queue work (get_job-visible
|
||
/// with the plain-queue defaults + ScheduleOpts applied over them,
|
||
/// ADR-020 §3), the leadership lock row is live under
|
||
/// `__alkstore_scheduler`, and a clean stop returns `Ok(())` promptly.
|
||
#[tokio::test(flavor = "multi_thread")]
|
||
async fn run_schedules_fires_boundaries_and_stop_is_clean() {
|
||
let Some(fx) = Fixture::open(&instance_namer("so_fire")).await else {
|
||
eprintln!("skip: no harness server");
|
||
return;
|
||
};
|
||
|
||
fx.store
|
||
.schedule(
|
||
"fast",
|
||
"@every 1s",
|
||
"work",
|
||
json!({"kind": "sweep"}),
|
||
ScheduleOpts {
|
||
priority: 5,
|
||
max_attempts: Some(9),
|
||
expires: None,
|
||
},
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
let stop = StopToken::new();
|
||
let runner_store = open_store(&harness_dsn().unwrap(), test_opts(&fx.schema))
|
||
.await
|
||
.unwrap();
|
||
let runner_stop = stop.clone();
|
||
let runner = tokio::spawn(async move { runner_store.run_schedules(runner_stop).await });
|
||
|
||
// Two boundaries at 1 s cadence (+ registration's first-fire grace
|
||
// of exactly one interval): wait for two fired jobs, bounded.
|
||
let deadline = tokio::time::Instant::now() + Duration::from_secs(25);
|
||
let mut fired_ids = Vec::new();
|
||
loop {
|
||
let q = fx.store.queue("work", qopts()).await.unwrap();
|
||
let page = q.claim_batch("probe", 10).await.unwrap();
|
||
for h in page {
|
||
fired_ids.push(h.job().id);
|
||
}
|
||
if fired_ids.len() >= 2 {
|
||
break;
|
||
}
|
||
assert!(
|
||
tokio::time::Instant::now() < deadline,
|
||
"no two boundary fires within the window"
|
||
);
|
||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||
}
|
||
|
||
// The stamp source pin (ADR-020 §3): the plain defaults 300/3/5/none
|
||
// with ScheduleOpts applied — priority 5 and max_attempts 9 override,
|
||
// visibility/backoff/retention take the defaults.
|
||
let q = fx.store.queue("work", qopts()).await.unwrap();
|
||
let job = q.get_job(fired_ids[0]).await.unwrap().unwrap();
|
||
assert_eq!(job.state, JobState::Processing, "our probe claimed it");
|
||
assert_eq!(job.priority, 5);
|
||
assert_eq!(job.max_attempts, 9);
|
||
assert_eq!(job.visibility_timeout_s, 300);
|
||
assert_eq!(job.backoff_base_s, 5);
|
||
assert_eq!(job.dead_letter_retention_s, None);
|
||
assert_eq!(
|
||
job.payload,
|
||
json!({"kind": "sweep"}).to_string().into_bytes()
|
||
);
|
||
assert_eq!(job.worker_id.as_deref(), Some("probe"));
|
||
|
||
// The leadership lock row exists under the reserved name with a
|
||
// live TTL (raw admin probe — the reserved name is unreachable by
|
||
// the handle surface by design).
|
||
{
|
||
let row = fx
|
||
.admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT count(*) FROM {} WHERE name = $1 AND expires_at > $2",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::LOCKS,
|
||
}
|
||
),
|
||
&[&crate::scheduler::SCHEDULER_LEADER_LOCK, &now_unix()],
|
||
)
|
||
.await
|
||
.unwrap();
|
||
let count: i64 = row.get(0);
|
||
assert_eq!(count, 1, "the leadership lock row must be live");
|
||
}
|
||
// The owner token is per-runner-instance (pid-stamped), not a
|
||
// shared constant — the wave-3 lesson; pinned as its shape.
|
||
{
|
||
let owner: String = fx
|
||
.admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT owner FROM {} WHERE name = $1",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::LOCKS,
|
||
}
|
||
),
|
||
&[&crate::scheduler::SCHEDULER_LEADER_LOCK],
|
||
)
|
||
.await
|
||
.unwrap()
|
||
.get(0);
|
||
assert!(
|
||
owner.contains(&std::process::id().to_string()),
|
||
"the owner token is per-instance (pid-stamped): {owner}"
|
||
);
|
||
}
|
||
|
||
// Clean stop: `Ok(())`, promptly (not after the full sleep).
|
||
stop.cancel();
|
||
let outcome = tokio::time::timeout(Duration::from_secs(10), runner)
|
||
.await
|
||
.expect("clean stop must terminate promptly")
|
||
.unwrap();
|
||
assert!(
|
||
matches!(outcome, Ok(())),
|
||
"clean stop returns Ok(()): {outcome:?}"
|
||
);
|
||
|
||
// The leadership lock released cleanly on stop.
|
||
let count: i64 = fx
|
||
.admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT count(*) FROM {} WHERE name = $1",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::LOCKS,
|
||
}
|
||
),
|
||
&[&crate::scheduler::SCHEDULER_LEADER_LOCK],
|
||
)
|
||
.await
|
||
.unwrap()
|
||
.get(0);
|
||
assert_eq!(count, 0, "clean stop releases the leadership lock");
|
||
|
||
drop_schema(&fx.admin, &fx.schema).await;
|
||
}
|
||
|
||
/// Acceptance: a second concurrent runner does not double-fire — the
|
||
/// leadership lock admits one leader; the loser returns
|
||
/// `Err(LeadershipLost)` at acquire (no loop, no ticks), the winner
|
||
/// fires each boundary exactly once (the band test: fired total stays
|
||
/// within the elapsed-boundary band).
|
||
#[tokio::test(flavor = "multi_thread")]
|
||
async fn second_runner_is_refused_and_boundaries_fire_once() {
|
||
let Some(fx) = Fixture::open(&instance_namer("so_snd")).await else {
|
||
eprintln!("skip: no harness server");
|
||
return;
|
||
};
|
||
|
||
fx.store
|
||
.schedule("beat", "@every 1s", "single", json!(1), Default::default())
|
||
.await
|
||
.unwrap();
|
||
|
||
// Runner A takes leadership first.
|
||
let stop_a = StopToken::new();
|
||
let runner_a_store = open_store(&harness_dsn().unwrap(), test_opts(&fx.schema))
|
||
.await
|
||
.unwrap();
|
||
let runner_a = tokio::spawn({
|
||
let stop = stop_a.clone();
|
||
async move { runner_a_store.run_schedules(stop).await }
|
||
});
|
||
|
||
// Runner B comes second: it must be refused (locked out) with
|
||
// `Err(LeadershipLost)` — start it once A has demonstrably taken
|
||
// the lock (wait for the first fired job).
|
||
let deadline = tokio::time::Instant::now() + Duration::from_secs(20);
|
||
loop {
|
||
let q = fx.store.queue("single", qopts()).await.unwrap();
|
||
if !q.claim_batch("probe", 10).await.unwrap().is_empty() {
|
||
break;
|
||
}
|
||
assert!(
|
||
tokio::time::Instant::now() < deadline,
|
||
"runner A never fired a boundary"
|
||
);
|
||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||
}
|
||
|
||
let runner_b_store = open_store(&harness_dsn().unwrap(), test_opts(&fx.schema))
|
||
.await
|
||
.unwrap();
|
||
let outcome_b = runner_b_store
|
||
.run_schedules(StopToken::new())
|
||
.await
|
||
.unwrap_err();
|
||
assert!(
|
||
matches!(outcome_b, Error::LeadershipLost),
|
||
"runner B must be refused with LeadershipLost: {outcome_b:?}"
|
||
);
|
||
|
||
// Let the leader run several more boundaries; the total fired count
|
||
// matches the elapsed boundaries within fire-window tolerance —
|
||
// with a second non-firing runner, one firer per boundary.
|
||
tokio::time::sleep(Duration::from_millis(2500)).await;
|
||
let fired_total: i64 = fx
|
||
.admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT count(*) FROM {} WHERE queue = 'single'",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::JOB,
|
||
}
|
||
),
|
||
&[],
|
||
)
|
||
.await
|
||
.unwrap()
|
||
.get(0);
|
||
assert!(
|
||
(2..=5).contains(&fired_total),
|
||
"boundary fires must stay within the elapsed-boundary band, got {fired_total}"
|
||
);
|
||
|
||
stop_a.cancel();
|
||
let outcome_a = tokio::time::timeout(Duration::from_secs(10), runner_a)
|
||
.await
|
||
.expect("clean stop must terminate promptly")
|
||
.unwrap();
|
||
assert!(matches!(outcome_a, Ok(())), "{outcome_a:?}");
|
||
|
||
drop_schema(&fx.admin, &fx.schema).await;
|
||
}
|
||
|
||
/// Acceptance: losing leadership mid-run returns `Err(LeadershipLost)`
|
||
/// *before any tick* — a rogue operator holding the leadership lock
|
||
/// (inserting the row directly, bypassing the engine surface) makes
|
||
/// the runner's acquire refuse; the 1-hour schedule's queue stays
|
||
/// untouched.
|
||
#[tokio::test(flavor = "multi_thread")]
|
||
async fn leadership_loss_returns_the_loss_error_before_ticking() {
|
||
let Some(fx) = Fixture::open(&instance_namer("so_loss")).await else {
|
||
eprintln!("skip: no harness server");
|
||
return;
|
||
};
|
||
|
||
fx.store
|
||
.schedule(
|
||
"slow",
|
||
"@every 3600s",
|
||
"lossq",
|
||
json!(1),
|
||
Default::default(),
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
// Steal the lock from under no one: a rogue operator (bypassing
|
||
// the engine surface — try_lock itself rejects the reserved name)
|
||
// inserts the leadership row directly; the runner starts, must
|
||
// observe LeadershipLost at acquire.
|
||
fx.admin
|
||
.execute(
|
||
&format!(
|
||
"INSERT INTO {} (name, owner, expires_at) VALUES ($1, 'rogue-operator', $2::bigint + 30)",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::LOCKS,
|
||
}
|
||
),
|
||
&[&crate::scheduler::SCHEDULER_LEADER_LOCK, &now_unix()],
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
let runner_store = open_store(&harness_dsn().unwrap(), test_opts(&fx.schema))
|
||
.await
|
||
.unwrap();
|
||
let outcome = runner_store.run_schedules(StopToken::new()).await;
|
||
assert!(
|
||
matches!(outcome, Err(Error::LeadershipLost)),
|
||
"a runner refused at acquire returns LeadershipLost: {outcome:?}"
|
||
);
|
||
|
||
// No tick ever ran: the 1-hour schedule's queue is untouched.
|
||
let fired: i64 = fx
|
||
.admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT count(*) FROM {} WHERE queue = 'lossq'",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::JOB,
|
||
}
|
||
),
|
||
&[],
|
||
)
|
||
.await
|
||
.unwrap()
|
||
.get(0);
|
||
assert_eq!(fired, 0, "a refused runner must never tick");
|
||
|
||
drop_schema(&fx.admin, &fx.schema).await;
|
||
}
|
||
|
||
/// Acceptance: in-sleep lease renewals hold across boundaries further
|
||
/// out than the TTL (10 s) — a runner sleeping on a 40 s boundary
|
||
/// keeps its lease live for the full drift (the renew-on-cadence
|
||
/// posture): after the boundary fires, the lock row's expires_at has
|
||
/// moved forward (the renewal landed), and the lock was never lost.
|
||
#[tokio::test(flavor = "multi_thread")]
|
||
async fn in_sleep_renewals_hold_across_boundaries_beyond_the_ttl() {
|
||
let Some(fx) = Fixture::open(&instance_namer("so_lease")).await else {
|
||
eprintln!("skip: no harness server");
|
||
return;
|
||
};
|
||
|
||
fx.store
|
||
.schedule(
|
||
"slowish",
|
||
"@every 40s",
|
||
"leaseq",
|
||
json!(1),
|
||
Default::default(),
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
let stop = StopToken::new();
|
||
let runner_store = open_store(&harness_dsn().unwrap(), test_opts(&fx.schema))
|
||
.await
|
||
.unwrap();
|
||
let runner_stop = stop.clone();
|
||
let runner = tokio::spawn(async move { runner_store.run_schedules(runner_stop).await });
|
||
|
||
// Registration's first fire is now + 40s (one interval) — the
|
||
// runner sleeps ~39 s across it, more than 3× the 10 s TTL. Probe
|
||
// the lease row's expires_at at start and again just before the
|
||
// boundary: it must have moved forward (the in-sleep renewals ran)
|
||
// and never lapsed.
|
||
tokio::time::sleep(Duration::from_secs(6)).await;
|
||
let expires_start: i64 = fx
|
||
.admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT expires_at FROM {} WHERE name = $1",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::LOCKS,
|
||
}
|
||
),
|
||
&[&crate::scheduler::SCHEDULER_LEADER_LOCK],
|
||
)
|
||
.await
|
||
.unwrap()
|
||
.get(0);
|
||
assert!(expires_start > now_unix(), "the lease is live");
|
||
|
||
// Sample the lease again ~30 s into the sleep — well past the TTL
|
||
// without an in-sleep renewal would have lapsed it.
|
||
tokio::time::sleep(Duration::from_secs(24)).await;
|
||
let expires_mid: i64 = fx
|
||
.admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT expires_at FROM {} WHERE name = $1",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::LOCKS,
|
||
}
|
||
),
|
||
&[&crate::scheduler::SCHEDULER_LEADER_LOCK],
|
||
)
|
||
.await
|
||
.unwrap()
|
||
.get(0);
|
||
let owner_alive: i64 = fx
|
||
.admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT count(*) FROM {} WHERE name = $1",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::LOCKS,
|
||
}
|
||
),
|
||
&[&crate::scheduler::SCHEDULER_LEADER_LOCK],
|
||
)
|
||
.await
|
||
.unwrap()
|
||
.get(0);
|
||
assert_eq!(owner_alive, 1, "the lease never lapsed mid-sleep");
|
||
assert!(
|
||
expires_mid > expires_start,
|
||
"the in-sleep renewals moved the lease forward ({expires_mid} > {expires_start})"
|
||
);
|
||
|
||
// The boundary still fires (the lease held through it).
|
||
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
|
||
let fired: i64 = loop {
|
||
let c: i64 = fx
|
||
.admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT count(*) FROM {} WHERE queue = 'leaseq'",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::JOB,
|
||
}
|
||
),
|
||
&[],
|
||
)
|
||
.await
|
||
.unwrap()
|
||
.get(0);
|
||
if c >= 1 {
|
||
break c;
|
||
}
|
||
assert!(
|
||
tokio::time::Instant::now() < deadline,
|
||
"the post-sleep boundary never fired (the lease must have held)"
|
||
);
|
||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||
};
|
||
assert!(fired >= 1);
|
||
|
||
stop.cancel();
|
||
let outcome = tokio::time::timeout(Duration::from_secs(10), runner)
|
||
.await
|
||
.expect("clean stop must terminate promptly")
|
||
.unwrap();
|
||
assert!(matches!(outcome, Ok(())), "{outcome:?}");
|
||
|
||
drop_schema(&fx.admin, &fx.schema).await;
|
||
}
|
||
|
||
/// Acceptance: catch-up cap + skip-forward are observable through the
|
||
/// runner (ADR-009 §4): a schedule with its fire horizon pushed far
|
||
/// into the past (200 boundary-widths out, far beyond any live
|
||
/// boundary racing the window — the SQLite task's flake-fix lesson)
|
||
/// replays at most the 64-fires cap and skips forward strictly past
|
||
/// now; the skipped boundaries never enqueue in-window.
|
||
#[tokio::test(flavor = "multi_thread")]
|
||
async fn catch_up_replays_up_to_the_cap_then_skips_forward() {
|
||
let Some(fx) = Fixture::open(&instance_namer("so_cap")).await else {
|
||
eprintln!("skip: no harness server");
|
||
return;
|
||
};
|
||
|
||
fx.store
|
||
.schedule(
|
||
"backlog",
|
||
"@every 1h",
|
||
"catchq",
|
||
json!("w"),
|
||
Default::default(),
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
// Push the fire horizon 200 boundary-widths into the past (200
|
||
// elapsed boundaries > the 64 cap) before the runner ever starts —
|
||
// a far-out horizon, no live boundary anywhere near the window.
|
||
// `@every 1h` keeps the post-cap boundary beyond the test window:
|
||
// the only fires possible here are the capped replay's own.
|
||
let now = now_unix();
|
||
fx.admin
|
||
.execute(
|
||
&format!(
|
||
"UPDATE {} SET next_fire_at = $1 WHERE name = 'backlog'",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::SCHEDULE,
|
||
}
|
||
),
|
||
&[&(now - 200 * 3600)],
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
let runner_store = open_store(&harness_dsn().unwrap(), test_opts(&fx.schema))
|
||
.await
|
||
.unwrap();
|
||
let stop = StopToken::new();
|
||
let runner_stop = stop.clone();
|
||
let runner = tokio::spawn(async move { runner_store.run_schedules(runner_stop).await });
|
||
|
||
// The first tick fires exactly 64, advances past now (the
|
||
// skip-forward lands at the next 1 h boundary — far beyond this
|
||
// window), and the skipped boundaries never enqueue.
|
||
let deadline = tokio::time::Instant::now() + Duration::from_secs(20);
|
||
let fired: i64 = loop {
|
||
let c: i64 = fx
|
||
.admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT count(*) FROM {} WHERE queue = 'catchq'",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::JOB,
|
||
}
|
||
),
|
||
&[],
|
||
)
|
||
.await
|
||
.unwrap()
|
||
.get(0);
|
||
let next: i64 = fx
|
||
.admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT next_fire_at FROM {} WHERE name = 'backlog'",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::SCHEDULE,
|
||
}
|
||
),
|
||
&[],
|
||
)
|
||
.await
|
||
.unwrap()
|
||
.get(0);
|
||
if c == 64 && next > now_unix() {
|
||
break c;
|
||
}
|
||
assert!(
|
||
tokio::time::Instant::now() < deadline,
|
||
"the cap never landed (fires={c}, next={next})"
|
||
);
|
||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||
};
|
||
assert_eq!(fired, 64, "exactly the contract-constant cap replays");
|
||
|
||
stop.cancel();
|
||
let outcome = tokio::time::timeout(Duration::from_secs(10), runner)
|
||
.await
|
||
.expect("clean stop must terminate promptly")
|
||
.unwrap();
|
||
assert!(matches!(outcome, Ok(())), "{outcome:?}");
|
||
|
||
// After the stop: the skipped boundaries never enqueue — and the
|
||
// skip-forward's next 1 h boundary cannot have fired in-window.
|
||
let fired_after: i64 = fx
|
||
.admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT count(*) FROM {} WHERE queue = 'catchq'",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::JOB,
|
||
}
|
||
),
|
||
&[],
|
||
)
|
||
.await
|
||
.unwrap()
|
||
.get(0);
|
||
assert_eq!(fired_after, 64, "the skipped boundaries never enqueue");
|
||
|
||
drop_schema(&fx.admin, &fx.schema).await;
|
||
}
|
||
|
||
/// Acceptance: the tick's row-locked transactional advance (ADR-009 §4's
|
||
/// layer (b)) — a rogue second ticker bypassing the leadership lock
|
||
/// cannot double-fire: an out-of-engine client issuing the same
|
||
/// row-locked tick shape serializes behind the FOR UPDATE row lock and
|
||
/// observes the advanced `next_fire_at`. Exercised at the engine's own
|
||
/// tick op driven twice concurrently from two pool connections over
|
||
/// one due schedule: exactly one fire.
|
||
#[tokio::test(flavor = "multi_thread")]
|
||
async fn rogue_concurrent_ticks_cannot_double_fire_one_boundary() {
|
||
let Some(fx) = Fixture::open(&instance_namer("so_rogue")).await else {
|
||
eprintln!("skip: no harness server");
|
||
return;
|
||
};
|
||
|
||
fx.store
|
||
.schedule("twin", "@every 1h", "rogueq", json!(1), Default::default())
|
||
.await
|
||
.unwrap();
|
||
|
||
// Force exactly one due boundary (no boundary race — `@every 1h`
|
||
// backdated 1 s pins exactly one elapsed boundary before either
|
||
// ticker runs; the post-fire next boundary is an hour out).
|
||
let now = now_unix();
|
||
fx.admin
|
||
.execute(
|
||
&format!(
|
||
"UPDATE {} SET next_fire_at = $1 WHERE name = 'twin'",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::SCHEDULE,
|
||
}
|
||
),
|
||
&[&(now - 1)],
|
||
)
|
||
.await
|
||
.unwrap();
|
||
|
||
// Two concurrent pool clients, each running the engine's own tick
|
||
// op (the same shape run_schedules runs — the rogue-ticker shape:
|
||
// no leadership lock in either hand).
|
||
let dsn = harness_dsn().unwrap();
|
||
let schema = fx.schema.clone();
|
||
let (a, b) = tokio::join!(
|
||
async {
|
||
let client = open_store(&dsn, test_opts(&schema))
|
||
.await
|
||
.unwrap()
|
||
.pool()
|
||
.get()
|
||
.await
|
||
.unwrap();
|
||
crate::queue::in_tx(client, |c| {
|
||
let schema = schema.clone();
|
||
Box::pin(async move { crate::scheduler::tick(c, &schema, now_unix()).await })
|
||
})
|
||
.await
|
||
},
|
||
async {
|
||
let client = open_store(&dsn, test_opts(&schema))
|
||
.await
|
||
.unwrap()
|
||
.pool()
|
||
.get()
|
||
.await
|
||
.unwrap();
|
||
crate::queue::in_tx(client, |c| {
|
||
let schema = schema.clone();
|
||
Box::pin(async move { crate::scheduler::tick(c, &schema, now_unix()).await })
|
||
})
|
||
.await
|
||
}
|
||
);
|
||
// Both ticks succeed; exactly one fired (the second observes the
|
||
// advanced boundary and skips).
|
||
a.unwrap();
|
||
b.unwrap();
|
||
let fired: i64 = fx
|
||
.admin
|
||
.query_one(
|
||
&format!(
|
||
"SELECT count(*) FROM {} WHERE queue = 'rogueq'",
|
||
crate::schema::QualifiedTable {
|
||
schema: &fx.schema,
|
||
name: crate::schema::tables::JOB,
|
||
}
|
||
),
|
||
&[],
|
||
)
|
||
.await
|
||
.unwrap()
|
||
.get(0);
|
||
assert_eq!(fired, 1, "the row-locked tick serializes: exactly one fire");
|
||
|
||
drop_schema(&fx.admin, &fx.schema).await;
|
||
}
|
||
|
||
/// The pg engine's own `@every` parser's unit pins (server-less —
|
||
/// ADR-012 §2's one-owner rule: this engine owns its grammar; the
|
||
/// contract suite pins the twins' outcomes identical, wave 5).
|
||
#[test]
|
||
fn parse_every_interval_pins_the_grammar_surface() {
|
||
use crate::scheduler::parse_every_interval;
|
||
for (spec, expected) in [
|
||
("@every 1s", 1),
|
||
("@every 30s", 30),
|
||
("@every 5m", 300),
|
||
("@every 2h", 7200),
|
||
("@every 7d", 604800),
|
||
("@every 100d", 8_640_000),
|
||
] {
|
||
assert_eq!(parse_every_interval(spec).unwrap(), expected, "{spec}");
|
||
}
|
||
for bad in [
|
||
"0 9 * * *",
|
||
"@every",
|
||
"@every ",
|
||
"@every 1x",
|
||
"@every 0s",
|
||
"@every -3m",
|
||
"every 5s",
|
||
"@every s",
|
||
"@every 1.5s",
|
||
"@every 5",
|
||
"@every 1s2x",
|
||
"* * * * *",
|
||
"@EVERY 5s",
|
||
] {
|
||
let err = parse_every_interval(bad).unwrap_err();
|
||
assert!(
|
||
matches!(err, Error::InvalidSpec { .. }),
|
||
"{bad:?} must reject InvalidSpec, got {err:?}"
|
||
);
|
||
}
|
||
// The pre-storage shape: the error carries the spec string.
|
||
match parse_every_interval("0 9 * * *").unwrap_err() {
|
||
Error::InvalidSpec { spec } => assert_eq!(spec, "0 9 * * *"),
|
||
_ => unreachable!("cron must map InvalidSpec"),
|
||
}
|
||
}
|