tests: review 003 Findings 2-3 — pre-release dispositions

Finding 2(a) landed: a one-shot cfg(test) generation-top LISTEN-pause
seam on the pg forwarder (listen_pause in seam.rs) lets the LISTEN-window
death test kill the fresh reconnect generation's backend before its
snapshot LISTEN — the LISTEN then fails honestly (57P01 terminating) and
the production fall-through arm (listen_ok == false) runs unmodified.
The harness test pins the honest mid-reconnect transient, full recovery
(wake, re-issue recovery of the refused mid-gap registration), and the
exactly-one-wake differentiator; a server-less unit pin covers the flag.
Finding 2(b) + the defensive arms stay parked with the code-read as pin,
recorded in the task file with current line refs.

Finding 3 landed: the rollback_fault seam (commit-fault mechanics) feeds
SQLITE_FULL-shaped errors into both ROLLBACK failure drop paths —
SqliteTxHandle::drop and the cancelled op's ConnLease::drop — exercised
in one test (phases: open-handle drop; single-poll-then-drop mid-op
cancellation), pinning slot replenish via reopen, no ghosts, and the
store staying usable.

alkstore-sqlite/src/seam.rs

@@ -173,6 +173,24 @@ pub(crate) fn closed_store_error() -> Error {
 /// holds a flag per [`crate::SqliteStore`] (born disarmed), so
 /// parallel test stores cannot interfere; `take` disarms on read —
 /// exactly one commit faults, and the post-failure commits are real.
+/// The `cfg(test)` rollback-fault seam (review 003 Finding 3): the
+/// commit-fault mechanics pointed at the drop paths — when armed, one
+/// `ROLLBACK` is substituted with a `SQLITE_FULL`-shaped driver error,
+/// feeding the *real* rollback-failure drop arms (the tx handle's and
+/// the cancelled op's connection lease: the error printed, the
+/// unknown-state connection dropped, the writer slot replenished via
+/// `reopen`) rather than replicating their behavior. No PRAGMA-space
+/// probe was needed on this one: SQLite exposes no deterministic
+/// in-process failure shape for `ROLLBACK` itself (its reachable
+/// failure is an environment I/O error), so a seam substitution is
+/// the only honest route — the same conclusion the commit seam
+/// reached after probing. One-store scoping; disarms on first
+/// consumption.
+#[cfg(test)]
+pub(crate) mod rollback_fault {
+    pub(crate) use super::commit_fault::{Flag, arm, disarmed, failure, take};
+}
+
 #[cfg(test)]
 pub(crate) mod commit_fault {
     use std::sync::Arc;

alkstore-sqlite/src/store.rs

@@ -59,6 +59,14 @@ pub struct SqliteStore {
     /// arm. Dead in production builds.
     #[cfg(test)]
     commit_fault: crate::seam::commit_fault::Flag,
+    /// The store's rollback-fault arm (review 003 Finding 3) — born
+    /// disarmed; a test arms it through
+    /// [`SqliteStore::arm_rollback_fault`] so the next ROLLBACK on a
+    /// drop path (the handle's or the cancelled op's lease's) fails
+    /// through the real rollback-failure drop arms. Dead in production
+    /// builds.
+    #[cfg(test)]
+    rollback_fault: crate::seam::rollback_fault::Flag,
 }
 
 impl std::fmt::Debug for SqliteStore {
@@ -90,6 +98,18 @@ impl SqliteStore {
     pub(crate) fn arm_commit_fault(&self) {
         crate::seam::commit_fault::arm(&self.commit_fault)
     }
+
+    /// Arm the rollback-fault seam (review 003 Finding 3): the next
+    /// `ROLLBACK` on a drop path of this store (a transaction handle
+    /// dropped open, or a tx-op future dropped mid-op) fails with a
+    /// `SQLITE_FULL`-shaped driver error and replenishes the writer
+    /// slot through the real rollback-failure drop arm. One-store
+    /// scoping (disarms on first consumption) keeps parallel test
+    /// stores unaffected.
+    #[cfg(test)]
+    pub(crate) fn arm_rollback_fault(&self) {
+        crate::seam::rollback_fault::arm(&self.rollback_fault)
+    }
 }
 
 impl Drop for SqliteStore {
@@ -130,6 +150,8 @@ pub(crate) fn open_store(path: &str, opts: SqliteOpts) -> alkstore::Result<Sqlit
         closed: Arc::new(AtomicBool::new(false)),
         #[cfg(test)]
         commit_fault: crate::seam::commit_fault::disarmed(),
+        #[cfg(test)]
+        rollback_fault: crate::seam::rollback_fault::disarmed(),
     })
 }
 
@@ -143,7 +165,12 @@ impl Store for SqliteStore {
             dyn Fn() -> alkstore::Result<rusqlite::Connection> + Send + Sync,
         > = std::sync::Arc::new(move || crate::seam::open_writer_connection(&db_path));
         #[cfg(test)]
-        let fut = crate::tx::SqliteTxHandle::begin(writer, reopen, self.commit_fault.clone());
+        let fut = crate::tx::SqliteTxHandle::begin(
+            writer,
+            reopen,
+            self.commit_fault.clone(),
+            self.rollback_fault.clone(),
+        );
         #[cfg(not(test))]
         let fut = crate::tx::SqliteTxHandle::begin(writer, reopen);
         fut

alkstore-sqlite/src/store/tx_tests.rs

@@ -1267,6 +1267,113 @@ async fn failed_commit_replenishes_the_writer_slot() {
     cleanup(&dir);
 }
 
+/// Acceptance (review 003 Finding 3): failure of `ROLLBACK` **itself**
+/// on the drop paths — the arms the commit-fault test's machinery
+/// mirrors but never exercised. When the rollback fails, the unknown-
+/// state connection is dropped (its error printed), and the writer
+/// slot is replenished via `reopen` (the wave-3 fix's property, whose
+/// commit arm the commit-fault seam proves): the next `begin_tx`
+/// proceeds, the dropped connection's uncommitted writes leave no
+/// ghosts (SQLite rolls an uncommitted transaction back at connection
+/// close even though the `ROLLBACK` statement failed), and the store
+/// stays fully usable — the next handle commits cleanly and
+/// auto-commit ops work.
+///
+/// **Both drop arms in one test**: `SqliteTxHandle::drop`'s
+/// failed-`ROLLBACK` path (a handle dropped with an open transaction)
+/// and `ConnLease::drop`'s (a tx-op future dropped mid-op — its lease
+/// dies holding the connection, the mid-op-cancellation shape).
+///
+/// Injection is the engine's `cfg(test)` rollback-fault seam
+/// (seam.rs, the commit-fault seam's mechanics): a `SQLITE_FULL`
+/// -shaped driver error substituted for one `ROLLBACK` per arming —
+/// SQLite exposes no deterministic in-process failure shape for
+/// `ROLLBACK` itself (its reachable failure is an environment I/O
+/// error), so the seam feeds the production drop arms rather than
+/// replicating them. The flag is per-store and disarms on its first
+/// consumption, so parallel test stores cannot interfere.
+#[tokio::test(flavor = "multi_thread")]
+async fn failed_rollback_on_drop_replenishes_the_writer_slot() {
+    let dir = temp_dir("rollback-err");
+    let concrete = open_store(dir.join("store.db").to_str().unwrap(), Default::default()).unwrap();
+
+    // Arm 1: the handle drop's failed-`ROLLBACK` path.
+    concrete.arm_rollback_fault();
+    let mut tx = concrete.begin_tx().await.unwrap();
+    let id = tx
+        .enqueue_tx("q", EnqueueOpts::default(), serde_json::json!({"n": 1}))
+        .await
+        .unwrap();
+    assert!(
+        id > 0,
+        "the in-tx write landed before the failed-rollback drop"
+    );
+    drop(tx);
+
+    let next = tokio::time::timeout(std::time::Duration::from_secs(5), concrete.begin_tx())
+        .await
+        .expect(
+            "the slot must be grantable after the failed-rollback drop \
+             (replenished via reopen)",
+        )
+        .unwrap();
+    let mut next = next;
+    let ghost = next.get_job_tx("q", id).await.unwrap();
+    assert_eq!(
+        ghost, None,
+        "the rollback-failed drop left no ghost — the dropped connection's \
+         uncommitted transaction rolled back at connection close"
+    );
+    next.commit()
+        .await
+        .expect("the post-failure handle commits cleanly (fault disarmed)");
+
+    // Arm 2: the cancelled op's connection-lease drop path. Exactly
+    // one poll fires the op's `spawn_blocking` (the blocking join's
+    // first poll is always Pending — it just spawned), then the
+    // future's drop cancels the op mid-flight: the lease that die
+    // holding the connection runs the `ConnLease::drop` fault arm.
+    concrete.arm_rollback_fault();
+    let mut tx2 = concrete.begin_tx().await.unwrap();
+    let mut fut = tx2.enqueue_tx("q", EnqueueOpts::default(), serde_json::json!({"n": 2}));
+    let waker = std::task::Waker::noop();
+    let mut cx = std::task::Context::from_waker(waker);
+    assert!(
+        matches!(
+            std::pin::Pin::new(&mut fut).poll(&mut cx),
+            std::task::Poll::Pending
+        ),
+        "the op's first poll fires its blocking round trip and parks on it"
+    );
+    drop(fut);
+
+    let next2 = tokio::time::timeout(std::time::Duration::from_secs(5), concrete.begin_tx())
+        .await
+        .expect(
+            "the slot must be grantable after the cancelled op's lease drop \
+             (replenished via reopen)",
+        )
+        .unwrap();
+    let mut next2 = next2;
+    let ghost2 = next2.get_job_tx("q", 1).await.unwrap();
+    assert_eq!(
+        ghost2, None,
+        "no ghost from the cancelled op either — whatever the race between \
+         the blocking op and the drop, an uncommitted row never surfaces"
+    );
+    next2
+        .commit()
+        .await
+        .expect("the post-lease-drop handle commits cleanly");
+
+    concrete
+        .notify("probe", serde_json::json!({"after": "rollback-error"}))
+        .await
+        .expect("auto-commit ops work after both rollback-failure arms");
+    drop(concrete);
+    cleanup(&dir);
+}
+
 /// Acceptance: a tokio task cancelled *mid-op* (its future dropped
 /// while an op's blocking round trip is in flight) cannot strand the
 /// writer slot — the op's RAII connection lease replenishes the slot

alkstore-sqlite/src/tx.rs

@@ -52,6 +52,11 @@ pub struct SqliteTxHandle {
     /// production builds.
     #[cfg(test)]
     commit_fault: crate::seam::commit_fault::Flag,
+    /// This store's rollback-fault arm (review 003 Finding 3) — read
+    /// once per drop/lease by the test-only fault branches; dead in
+    /// production builds.
+    #[cfg(test)]
+    rollback_fault: crate::seam::rollback_fault::Flag,
 }
 
 impl std::fmt::Debug for SqliteTxHandle {
@@ -67,6 +72,7 @@ impl SqliteTxHandle {
         writer: Arc<Writer>,
         reopen: Arc<dyn Fn() -> Result<rusqlite::Connection> + Send + Sync>,
         #[cfg(test)] commit_fault: crate::seam::commit_fault::Flag,
+        #[cfg(test)] rollback_fault: crate::seam::rollback_fault::Flag,
     ) -> BoxedFuture<'static, Result<Box<dyn TxHandle + Send>>> {
         Box::pin(async move {
             let writer2 = writer.clone();
@@ -91,6 +97,8 @@ impl SqliteTxHandle {
                 reopen,
                 #[cfg(test)]
                 commit_fault,
+                #[cfg(test)]
+                rollback_fault,
             }) as _)
         })
     }
@@ -114,11 +122,15 @@ impl SqliteTxHandle {
         };
         let writer = self.writer.clone();
         let reopen = self.reopen.clone();
+        #[cfg(test)]
+        let rollback_fault = self.rollback_fault.clone();
         let fut = blocking(move || {
             let lease = ConnLease {
                 conn: Some(conn),
                 writer: writer.clone(),
                 reopen: reopen.clone(),
+                #[cfg(test)]
+                rollback_fault,
             };
             let out = match lease.conn() {
                 Some(conn) => f(conn),
@@ -148,6 +160,11 @@ struct ConnLease {
     conn: Option<rusqlite::Connection>,
     writer: std::sync::Arc<Writer>,
     reopen: Arc<dyn Fn() -> Result<rusqlite::Connection> + Send + Sync>,
+    /// The store's rollback-fault arm (review 003 Finding 3), read
+    /// once by the lease's test-only drop branch; dead in production
+    /// builds.
+    #[cfg(test)]
+    rollback_fault: crate::seam::rollback_fault::Flag,
 }
 
 impl ConnLease {
@@ -163,7 +180,15 @@ impl ConnLease {
 impl Drop for ConnLease {
     fn drop(&mut self) {
         if let Some(conn) = self.conn.take() {
-            if let Err(rollback) = conn.execute_batch("ROLLBACK") {
+            #[cfg(test)]
+            let rolled_back = if crate::seam::rollback_fault::take(&self.rollback_fault) {
+                Err(crate::seam::rollback_fault::failure())
+            } else {
+                conn.execute_batch("ROLLBACK")
+            };
+            #[cfg(not(test))]
+            let rolled_back = conn.execute_batch("ROLLBACK");
+            if let Err(rollback) = rolled_back {
                 eprintln!(
                     "alkstore: error: rollback on a cancelled transaction op \
                      left the connection in an unknown state: {rollback}"
@@ -434,7 +459,15 @@ impl TxHandle for SqliteTxHandle {
 impl Drop for SqliteTxHandle {
     fn drop(&mut self) {
         if let Some(conn) = self.conn.take() {
-            match conn.execute_batch("ROLLBACK") {
+            #[cfg(test)]
+            let rolled_back = if crate::seam::rollback_fault::take(&self.rollback_fault) {
+                Err(crate::seam::rollback_fault::failure())
+            } else {
+                conn.execute_batch("ROLLBACK")
+            };
+            #[cfg(not(test))]
+            let rolled_back = conn.execute_batch("ROLLBACK");
+            match rolled_back {
                 Ok(()) => self.writer.release(conn),
                 Err(rollback) => {
                     eprintln!(