The queue mechanism (ADR-010 state machine over the in-memory job table, mem's third-owner equal-jitter curve + opts resolution) already landed implemented with the seam task's whole-engine scope (mem-engine-seam-tx Note 1); this task's work, per the disposition the notify/listen twin recorded, was reconciling the description's pin list against the existing suite and adding the missing engine-side unit pins — no production code changed, the realization verified correct as-found. Already pinned by the seam task (reconciled, recorded in Notes): lifecycle + stamps + resolved get_job values, claim ordering, extent clamps at entry, worker-id validation, the uniform validity predicate, late-heartbeat refusal + absolute-reset renewal, reclaim-eats-attempt + loser-ack-refuses, the pinned dead-letter default strings, dead-row get_job visibility, both-states sweep + retention + scoping, the tx stamp shapes, and the resolution legs + base-5 curve range/cap/spread. Added (6): the in-process claim-race atomicity pin (8 concurrent workers over 4 rows — every row handed exactly once through the guarded section, the description's claim-vs-claim bullet); the claim-vs-expiry boundary determinism pin (at now == deadline: the d < now reclaim predicate and the d >= now holder-op predicate are coherent — no reclaim, heartbeat/ack still valid; one second past: reclaimable with its attempt consumed; lapsed ids silently uncounted by ack_batch); the engine-side mirror of the suite's backoff_curve_equivalence row — base 2 s at [1,2]/[2,4]/[4,8] driven end-to-end through claim -> retry(None) -> get_job's run_at with the Some(5)-verbatim and Some(0)-ready-now override legs, exact under the frozen clock — both through the handle and pinned at backoff_delay_s itself (the cap leg stays in resolution.rs's base-5 pins per suite-opts-backoff-rows' recorded disposition); the D-12 pre-claim sweep pin (a lapsed row with its budget spent claims nothing and dead-letters with the exhaust string at the next claim, never strands); and unconditional cancel across both states (the holder's post-cancel ack refusing — the not-an-interrupt honesty). One test-authoring note recorded: heartbeat's absolute reset moves the deadline, so boundary legs stage on fresh rows to keep each observation exact. Task file: status completed, Notes (the seam-carried disposition + reconciliation inventory + the six adds + the heartbeat-observation note) + Summary filled. Verified: cargo test -p alkstore-mem 97/97 (91 + 6 new; new tests green solo x3), workspace cargo test green (core 25, suite-harness 3, mem 97, pg 124+25+9, SQLite 191+25), clippy --all-targets -D warnings clean, cargo fmt --check clean, cargo check --target wasm32-unknown-unknown -p alkstore-mem clean
265 lines
9.7 KiB
Rust
265 lines
9.7 KiB
Rust
//! The engine-side opts-resolution arithmetic (ADR-020 §1–§3) and the
|
||
//! derived stamp sets — mem's own implementation (ADR-012 §2's
|
||
//! one-owner rule, third owner; the other engines' copies are
|
||
//! references, not dependencies, and the suite pins three-way
|
||
//! equivalence).
|
||
//!
|
||
//! Every function takes the enqueue instant as a parameter — the one
|
||
//! clock read per enqueue is the caller's (`crate::clock`), and these
|
||
//! functions never read a clock of their own except the jitter draw's
|
||
//! subsecond entropy.
|
||
|
||
use std::collections::hash_map::RandomState;
|
||
use std::hash::BuildHasher;
|
||
use std::time::{SystemTime, UNIX_EPOCH};
|
||
|
||
use alkstore::{EnqueueOpts, QueueOpts};
|
||
|
||
/// The four §3a stamps a queue row carries (resolved at enqueue).
|
||
#[derive(Debug, Clone, Copy)]
|
||
pub(crate) struct Stamps {
|
||
pub(crate) max_attempts: i64,
|
||
pub(crate) visibility_timeout_s: i64,
|
||
pub(crate) backoff_base_s: i64,
|
||
pub(crate) dead_letter_retention_s: Option<i64>,
|
||
}
|
||
|
||
/// The pinned dead-letter default strings (row content, not error
|
||
/// variants — ADR-010 §4 / ADR-019 §3's annotation: `"max attempts
|
||
/// exceeded"` at exhaustion, `"failed"` for `fail(None)`, `"expired"`
|
||
/// on sweeps; a caller-supplied string carries verbatim).
|
||
pub(crate) const EXHAUSTED_ERROR: &str = "max attempts exceeded";
|
||
pub(crate) const FAILED_ERROR: &str = "failed";
|
||
pub(crate) const EXPIRED_ERROR: &str = "expired";
|
||
|
||
/// The backoff cap (ADR-010 §3 — a pinned contract constant, no knob).
|
||
const BACKOFF_CAP_S: i64 = 3600;
|
||
|
||
/// The plain queue's derived `QueueOpts` defaults (300/3/5/none —
|
||
/// ADR-010 §3): the stamp source for every no-handle-open shape
|
||
/// (`enqueue_tx`, scheduler boundary fires, the scheduler-outbox task's
|
||
/// auto-commit uses). Named so the derived-defaults formula has one
|
||
/// owner engine-wide.
|
||
pub(crate) fn plain_queue_default_stamps() -> Stamps {
|
||
let QueueOpts {
|
||
visibility_timeout_s,
|
||
max_attempts,
|
||
backoff_base_s,
|
||
dead_letter_retention_s,
|
||
} = QueueOpts::default();
|
||
Stamps {
|
||
max_attempts,
|
||
visibility_timeout_s,
|
||
backoff_base_s,
|
||
dead_letter_retention_s,
|
||
}
|
||
}
|
||
|
||
/// The outbox backing queue's derived `QueueOpts` set (60/5/5 —
|
||
/// ADR-010 §3 / ADR-014 §1). The stamp source for `outbox_enqueue_tx`
|
||
/// and `Outbox::enqueue`.
|
||
pub(crate) fn outbox_default_stamps() -> Stamps {
|
||
Stamps {
|
||
visibility_timeout_s: 60,
|
||
max_attempts: 5,
|
||
backoff_base_s: 5,
|
||
dead_letter_retention_s: None,
|
||
}
|
||
}
|
||
|
||
/// The derived backing-queue name for an outbox (the reserved
|
||
/// namespace, ADR-008 §4): unreachable by `enqueue_tx` by design
|
||
/// (ADR-014 §1) — outbox enqueue is the only path into it.
|
||
pub(crate) fn outbox_backing_queue_name(outbox: &str) -> String {
|
||
format!("{}outbox:{outbox}", alkstore::RESERVED_PREFIX)
|
||
}
|
||
|
||
/// The one per-enqueue stamp override (ADR-010 §3a):
|
||
/// `EnqueueOpts::max_attempts` is the only per-job override — the
|
||
/// other stamps always take the base's. One helper for every enqueue
|
||
/// shape (tx, handle, outbox, scheduler fire).
|
||
pub(crate) fn stamps_with_override(base: Stamps, opts: &EnqueueOpts) -> Stamps {
|
||
Stamps {
|
||
max_attempts: opts.max_attempts.unwrap_or(base.max_attempts),
|
||
..base
|
||
}
|
||
}
|
||
|
||
/// The resolved enqueue fields (ADR-020 §1/§2): `delay` set → ready
|
||
/// time `now + delay` (wins over `run_at`); else `run_at` literally;
|
||
/// else `now`. `expires` is relative seconds from the enqueue instant,
|
||
/// resolved to the absolute row `expires_at`; `None` = never.
|
||
pub(crate) struct ResolvedEnqueue {
|
||
pub(crate) run_at: i64,
|
||
pub(crate) expires_at: Option<i64>,
|
||
}
|
||
|
||
pub(crate) fn resolve_enqueue_opts(now: i64, opts: &EnqueueOpts) -> ResolvedEnqueue {
|
||
let run_at = match opts.delay {
|
||
Some(delay) => now.saturating_add(delay),
|
||
None => opts.run_at.unwrap_or(now),
|
||
};
|
||
let expires_at = opts.expires.map(|s| now.saturating_add(s));
|
||
ResolvedEnqueue { run_at, expires_at }
|
||
}
|
||
|
||
/// The equal-jitter exponential backoff (ADR-010 §3 — mem's own
|
||
/// implementation): `delay ∈ [base·2^(a−1)/2, base·2^(a−1)]` from the
|
||
/// job's stamped `backoff_base_s`, capped at 1 hour. The attempt index
|
||
/// `a` is the row's `attempts` at the retry (post-claim count,
|
||
/// reclaims included); `Some(d)` overrides pass through without this
|
||
/// function.
|
||
///
|
||
/// Integerized second-precision: the draw is uniform over the integers
|
||
/// in `[ceil(half), cap]` (inclusive both ends) — never below the
|
||
/// pinned lower bound, never above the cap. The jitter source is
|
||
/// std-only (the ADR-005 dependency posture — no `rand` dep): a fresh
|
||
/// [`RandomState`] hasher over the row id, the attempt count, and the
|
||
/// subsecond clock spreads the draw; the pinned property is the range.
|
||
pub(crate) fn backoff_delay_s(base_s: i64, attempts: i64, id: i64) -> i64 {
|
||
let exponent: u32 = (attempts.max(1) as u32).saturating_sub(1).min(30);
|
||
let doubling: i64 = base_s.saturating_mul(1i64 << exponent).max(1);
|
||
let cap = doubling.min(BACKOFF_CAP_S);
|
||
let half = ((cap + 1) / 2).max(1);
|
||
let span = (cap - half + 1).max(1);
|
||
let now_nanos = SystemTime::now()
|
||
.duration_since(UNIX_EPOCH)
|
||
.map(|d| d.subsec_nanos())
|
||
.unwrap_or(0);
|
||
let draw = RandomState::new().hash_one((id, attempts, now_nanos));
|
||
half + (draw % span as u64) as i64
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn stamp_sets_match_the_pinned_defaults() {
|
||
let plain = plain_queue_default_stamps();
|
||
assert_eq!(
|
||
(
|
||
plain.visibility_timeout_s,
|
||
plain.max_attempts,
|
||
plain.backoff_base_s,
|
||
plain.dead_letter_retention_s
|
||
),
|
||
(300, 3, 5, None)
|
||
);
|
||
let outbox = outbox_default_stamps();
|
||
assert_eq!(
|
||
(
|
||
outbox.visibility_timeout_s,
|
||
outbox.max_attempts,
|
||
outbox.backoff_base_s,
|
||
outbox.dead_letter_retention_s
|
||
),
|
||
(60, 5, 5, None)
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn override_touches_only_max_attempts() {
|
||
let base = Stamps {
|
||
max_attempts: 3,
|
||
visibility_timeout_s: 300,
|
||
backoff_base_s: 5,
|
||
dead_letter_retention_s: None,
|
||
};
|
||
let opts = EnqueueOpts {
|
||
max_attempts: Some(9),
|
||
..Default::default()
|
||
};
|
||
let stamped = stamps_with_override(base, &opts);
|
||
assert_eq!(stamped.max_attempts, 9);
|
||
assert_eq!(stamped.visibility_timeout_s, 300);
|
||
assert_eq!(stamped.backoff_base_s, 5);
|
||
assert_eq!(stamped.dead_letter_retention_s, None);
|
||
let plain = EnqueueOpts::default();
|
||
assert_eq!(stamps_with_override(base, &plain).max_attempts, 3);
|
||
}
|
||
|
||
#[test]
|
||
fn delay_wins_over_run_at_and_neither_resolves_to_now() {
|
||
let now = 1_000_000;
|
||
let both = EnqueueOpts {
|
||
delay: Some(10),
|
||
run_at: Some(now + 500),
|
||
..Default::default()
|
||
};
|
||
assert_eq!(resolve_enqueue_opts(now, &both).run_at, now + 10);
|
||
let absolute = EnqueueOpts {
|
||
run_at: Some(now + 500),
|
||
..Default::default()
|
||
};
|
||
assert_eq!(resolve_enqueue_opts(now, &absolute).run_at, now + 500);
|
||
let neither = EnqueueOpts::default();
|
||
assert_eq!(resolve_enqueue_opts(now, &neither).run_at, now);
|
||
let negative = EnqueueOpts {
|
||
delay: Some(-5),
|
||
..Default::default()
|
||
};
|
||
assert_eq!(
|
||
resolve_enqueue_opts(now, &negative).run_at,
|
||
now - 5,
|
||
"a negative delay is a ready time in the past — total, ready now"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn expires_resolves_relative_to_enqueue() {
|
||
let now = 1_000_000;
|
||
let opts = EnqueueOpts {
|
||
expires: Some(60),
|
||
..Default::default()
|
||
};
|
||
assert_eq!(resolve_enqueue_opts(now, &opts).expires_at, Some(now + 60));
|
||
let none = EnqueueOpts::default();
|
||
assert_eq!(resolve_enqueue_opts(now, &none).expires_at, None);
|
||
}
|
||
|
||
#[test]
|
||
fn backoff_base_two_lands_in_the_suite_rows_exact_bounds() {
|
||
// The `backoff_curve_equivalence` suite row's identical ranges
|
||
// (base 2 s: [1,2], [2,4], [4,8]) — the engine-side numeric pin
|
||
// of the equivalence the row asserts across engines.
|
||
for (attempts, lower, upper) in [(1i64, 1i64, 2i64), (2, 2, 4), (3, 4, 8)] {
|
||
for id in [1i64, 2, 42, 999_999] {
|
||
let delay = backoff_delay_s(2, attempts, id);
|
||
assert!(
|
||
(lower..=upper).contains(&delay),
|
||
"base 2 attempt {attempts} id {id}: delay {delay} outside [{lower}, {upper}]"
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn backoff_lands_in_the_pinned_equal_jitter_range() {
|
||
for attempts in [1i64, 2, 3, 5, 30] {
|
||
let expected_cap = 5i64
|
||
.saturating_mul(1i64 << (attempts - 1).min(30))
|
||
.min(3600);
|
||
let half = (expected_cap + 1) / 2;
|
||
for id in [1i64, 42, 777, 123_456] {
|
||
let delay = backoff_delay_s(5, attempts, id);
|
||
assert!(
|
||
(half..=expected_cap).contains(&delay),
|
||
"attempt {attempts} id {id}: delay {delay} outside [{half}, {expected_cap}]"
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn backoff_caps_at_one_hour_and_spreads() {
|
||
let draws: Vec<i64> = (0..400).map(|i| backoff_delay_s(5, 40, i)).collect();
|
||
assert!(
|
||
draws.iter().all(|d| (1800..=3600).contains(d)),
|
||
"capped range"
|
||
);
|
||
assert!(draws.windows(2).any(|w| w[0] != w[1]), "the jitter spreads");
|
||
assert!(draws.iter().any(|d| *d < 2700) && draws.iter().any(|d| *d > 2700));
|
||
}
|
||
}
|