pg: surface the forwarder's swallowed connect/poll failures (review 004 item 7's two narrow arms, pg-fix-forwarder-diagnosability) — output form per the ADR-025 decision of record: the reconnect arm's .ok() swallow at the loop's attempt closure becomes a tracing::warn! per failed connect (attempt index mapping the backoff state — 50 ms base doubling to the 2 s cap — plus the raw io-class connect error; reconnect_with_retry's generic signature and its two unit pins unchanged); poll_loop returns the fatal poll_message error instead of discarding it and the serve loop's poll-death arm surfaces the ending as a tracing::error! (fatal error, no-error stream close, and the join abort/panic variant — each naming the death and the stall-until-reconnect consequence); secret-free re-verified by grep — both new emission contents checked against the substrate (tokio-postgres 0.7.18 connect errors are Error::connect(io::Error), no DSN/user/password in Display; no subscriber-asserting test added, holding ADR-025's dev-dep stance); gates green: full crate suite against the harness server (135+25+9, all pre-existing forwarder/reconnect pins and the hardening task's pins green unchanged), workspace build/tests server-less, clippy -D warnings, fmt, doc gate, fuzz corpus replay 6/6; task -> completed
This commit is contained in:
1 parent
a9b6cf0c86
commit
321f579ac2
2 files changed
+162
-14
No files matched your search
@@ -753,7 +753,9 @@ impl Drop for FanoutRelease {
|
||||
/// ([`LISTEN_SETTLE_TIMEOUT`]) reaps-or-aborts the poll task → serve
|
||||
/// commands (each also bounded, the shutdown watch selected with the
|
||||
/// await) → reconnect via [`reconnect_with_retry`] (exponential
|
||||
/// backoff 50 ms → 2 s cap; failed connects never leave the arm). The
|
||||
/// backoff 50 ms → 2 s cap; failed connects never leave the arm, and
|
||||
/// each failure surfaces a `warn` event — attempt index, error class,
|
||||
/// no credential material, ADR-025's secret-free law). The
|
||||
/// loop-carried connection is *always present* at the generation's
|
||||
/// top by construction: the first iteration holds the handed-in
|
||||
/// connection, and only a successful connect re-binds it — there is
|
||||
@@ -1060,7 +1062,43 @@ async fn forwarder_loop(
|
||||
// The poll task ended = the connection died (a
|
||||
// fatal poll error or server close). The session
|
||||
// is already gone server-side; reconnect below.
|
||||
_ = &mut poll_task => {
|
||||
// The poll error is surfaced, not discarded (the
|
||||
// review 004 item 7 arm): an operator must see
|
||||
// the death and its class before the wake loss —
|
||||
// the error content is io/connection-class
|
||||
// material, never DSN/password material (the
|
||||
// secret-free law, ADR-025).
|
||||
poll_outcome = &mut poll_task => {
|
||||
match poll_outcome {
|
||||
Ok(Some(err)) => {
|
||||
tracing::error!(
|
||||
"alkstore-postgres: forwarder poll \
|
||||
ended with a fatal \
|
||||
notification-polling error; \
|
||||
notifications are stalled until \
|
||||
the reconnect re-issue: {err}"
|
||||
);
|
||||
}
|
||||
Ok(None) => {
|
||||
tracing::error!(
|
||||
"alkstore-postgres: forwarder poll \
|
||||
ended without an error (the \
|
||||
server closed the notification \
|
||||
stream); notifications are \
|
||||
stalled until the reconnect \
|
||||
re-issue"
|
||||
);
|
||||
}
|
||||
Err(join_err) => {
|
||||
tracing::error!(
|
||||
"alkstore-postgres: forwarder poll \
|
||||
task aborted or panicked; \
|
||||
notifications are stalled until \
|
||||
the reconnect re-issue: \
|
||||
{join_err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -1079,13 +1117,37 @@ async fn forwarder_loop(
|
||||
connected.store(false, Ordering::Release);
|
||||
let attempt = {
|
||||
let slot = reconnect_config.clone();
|
||||
let mut attempt_no: u64 = 0;
|
||||
move || {
|
||||
let guard = slot
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let config = guard.clone();
|
||||
drop(guard);
|
||||
async move { config.connect(NoTls).await.ok() }
|
||||
attempt_no += 1;
|
||||
let attempt_no = attempt_no;
|
||||
async move {
|
||||
// The connect failure is surfaced, not `.ok()`
|
||||
// swallowed (the review 004 item 7 arm): the
|
||||
// attempt index maps the live backoff state (the
|
||||
// next sleep doubles per attempt up to the cap).
|
||||
// The error content is connect/io-class — no DSN
|
||||
// or password material ever reaches the event (the
|
||||
// secret-free law, ADR-025).
|
||||
match config.connect(NoTls).await {
|
||||
Ok(pair) => Some(pair),
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
"alkstore-postgres: forwarder reconnect \
|
||||
connect attempt {attempt_no} failed; the \
|
||||
retry continues on the exponential \
|
||||
backoff (50 ms base doubling, 2 s cap): \
|
||||
{err}"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
let Some((next_client, next_connection)) =
|
||||
@@ -1115,8 +1177,14 @@ async fn forwarder_loop(
|
||||
/// The poll loop over one connection generation: `poll_message`
|
||||
/// forever, fanning notifications into the bounded broadcast. Returns
|
||||
/// when the connection dies (a fatal poll error or server close) —
|
||||
/// the forwarder loop reconnects.
|
||||
async fn poll_loop(mut connection: ListenerConnection, fanout: broadcast::Sender<RawNotification>) {
|
||||
/// the forwarder loop reconnects. The fatal poll error, when one
|
||||
/// exists, is returned so the loop's exit path surfaces it (the
|
||||
/// review 004 item 7 diagnosability arm — an operator must see the
|
||||
/// death instead of the discarded error before the wake loss).
|
||||
async fn poll_loop(
|
||||
mut connection: ListenerConnection,
|
||||
fanout: broadcast::Sender<RawNotification>,
|
||||
) -> Option<tokio_postgres::Error> {
|
||||
loop {
|
||||
match std::future::poll_fn(|cx| connection.poll_message(cx)).await {
|
||||
Some(Ok(tokio_postgres::AsyncMessage::Notification(n))) => {
|
||||
@@ -1131,7 +1199,8 @@ async fn poll_loop(mut connection: ListenerConnection, fanout: broadcast::Sender
|
||||
});
|
||||
}
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(_)) | None => return,
|
||||
Some(Err(err)) => return Some(err),
|
||||
None => return None,
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user