fix: pump_session deadlock when backend emits >64 chunks before exit (P1)

The stdout drainer started only after the pumps+exit join completed,
but the pumps share a bounded (64-slot) channel with it. Once the
channel filled, pump_stdout parked on send, the join never completed,
and the session hung with the client actively reading (review #003 P1;
reproduced empirically at N=62 ok / N=63 hang). Any real session (cat,
ls -R) exceeds 64 chunks instantly.

pump_session now spawns drain_chunks (owning writer_rx + client_write)
before the join; the exit chunk is still enqueued after the join and
writer_tx is dropped after it, so the single FIFO preserves the
exit-chunk-is-last invariant (ADR-005).

Tests (review #003 P13):
- backpressure regression: 80 chunks through a real drive_session,
  in-order delivery + sentinel + exit asserted, 10s per-read timeout
  so a regression fails instead of hanging (validated against the bug:
  fails with the old order reinstated)
- wire: MAX_CHUNK_LEN exact-boundary round trip (limit is inclusive)
- session: stream_type 0/3 from the server ignored mid-stream without
  desynchronizing framing or losing stdout/exit

Verification: cargo test 98 lib (default) / 141 --all-features,
clippy all-targets + wasm32 -D warnings, fmt --check, wasm check, doc
--no-deps all clean.
This commit is contained in:
glm-5.3-flash committed 2026-09-05 10:05:38 +00:00
1 parent da94d5e23f
commit 87c52e5e4d
3 files changed
+211 -6

No files matched your search

+112 -6
View File
@@ -422,9 +422,11 @@ where
/// Phase 3: the bidirectional pump. Three concurrent tasks plus a drainer.
///
/// Enforces the exit-chunk-is-last invariant (ADR-055): the adapter waits for
/// BOTH the stdout/stderr pumps to complete AND `exit_code` to resolve
/// before enqueueing the exit chunk.
/// Enforces the exit-chunk-is-last invariant (ADR-055): the exit chunk
/// is enqueued only after the stdout/stderr pumps have finished sending
/// AND `exit_code` has resolved. The drainer runs concurrently with the
/// pumps so the shared writer channel is always being consumed — see
/// `drain_chunks`.
async fn pump_session<W, R>(
client_write: W,
client_read: R,
@@ -434,7 +436,7 @@ where
W: AsyncWrite + Send + Unpin + 'static,
R: AsyncRead + Send + Unpin + 'static,
{
let (writer_tx, mut writer_rx) = mpsc::channel::<Chunk>(64);
let (writer_tx, writer_rx) = mpsc::channel::<Chunk>(64);
let TtyHandle {
stdin,
@@ -444,6 +446,14 @@ where
control,
} = handle;
// The drainer MUST be running while the pumps send: it is the only
// consumer of the writer channel, so starting it only after the
// pumps+exit join would let a backend producing more chunks than the
// channel capacity fill the channel, park the pumps' `send` calls,
// and deadlock the session before the join could complete (review
// #003 P1).
let drainer = tokio::spawn(drain_chunks(client_write, writer_rx));
let writer_tx_out = writer_tx.clone();
let stdout_pump = tokio::spawn(pump_stdout(stdout, writer_tx_out));
@@ -477,6 +487,22 @@ where
drop(writer_tx);
drop(input_pump);
let _ = drainer.await;
debug!("tty: session complete");
Ok(())
}
/// Drain the writer channel to the client in arrival order. Spawned
/// before the pumps start so chunk producers are never parked on a full
/// channel without an active consumer (review #003 P1). The single FIFO
/// preserves the exit-chunk-is-last invariant (ADR-055): the exit chunk
/// is enqueued only after every pump has finished sending. Ends when all
/// senders are dropped (normal close) or on a client write error (the
/// pumps' subsequent sends then fail and the pumps wind down).
async fn drain_chunks<W>(client_write: W, mut writer_rx: mpsc::Receiver<Chunk>)
where
W: AsyncWrite + Send + Unpin + 'static,
{
let mut chunk_writer = ChunkWriter::new(client_write);
while let Some(chunk) = writer_rx.recv().await {
if let Err(e) = chunk_writer.write_chunk(&chunk).await {
@@ -485,8 +511,6 @@ where
}
}
let _ = chunk_writer.into_inner().shutdown().await;
debug!("tty: session complete");
Ok(())
}
async fn send_exit_chunk(writer_tx: &mpsc::Sender<Chunk>, code: i32) {
@@ -1050,6 +1074,88 @@ mod tests {
let _ = session.await;
}
/// Backpressure regression (review #003 P1): a backend producing
/// more stdout chunks than the writer-channel capacity (64) must
/// not deadlock the session. Before the fix, the drainer started
/// only after the pumps+exit join, so the 65th chunk parked the
/// stdout pump's send, the join never completed, and the client
/// (actively reading) hung with nothing delivered. Empirically: 62
/// chunks passed, 63 hung. The drainer now runs concurrently, so
/// 80 chunks flow, the sentinel follows the last data chunk, and
/// the exit chunk is still last (ADR-055).
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn backpressure_more_chunks_than_channel_capacity() {
const N: usize = 80;
let (backend, _control, _cancel) = TestBackend::builder().build();
let backends = make_backends(backend.clone());
let (mut client, server) = make_client_and_server();
let identity = identity_with_scope(TTY_OPEN_SCOPE);
let session = tokio::spawn(async move {
drive_session_server(server, backends, None, identity).await;
});
client.write_negotiation(TEST_NEG).await;
let stdout_tx = backend.take_stdout_tx().await.expect("stdout tx");
let _ = backend.take_stderr_tx().await;
// Produce from a spawned task: with the pre-fix code the 65th
// send would park here forever (channel full, drainer not yet
// running), and the test's own awaits below would hang instead
// of observing the deadlock via the bounded client reads.
let producer = tokio::spawn(async move {
for i in 0..N {
let payload = format!("chunk-{i:03}");
if stdout_tx
.send(Bytes::copy_from_slice(payload.as_bytes()))
.await
.is_err()
{
break;
}
}
});
let exit_tx = backend.take_exit_tx().await.expect("exit tx");
exit_tx.send(Ok(0)).unwrap();
// Bounded: a regression must fail this test, not hang it.
let deadline = std::time::Duration::from_secs(10);
let mut data_chunks = 0usize;
loop {
let read = tokio::time::timeout(deadline, client.read_chunk())
.await
.expect("chunk read must not stall (P1 regression: deadlock)");
let (st, bytes) = read;
match st {
STREAM_STDOUT if !bytes.is_empty() => {
assert_eq!(
bytes.as_ref(),
format!("chunk-{data_chunks:03}").as_bytes(),
"chunks must arrive in order"
);
data_chunks += 1;
}
STREAM_STDOUT => {
assert_eq!(
data_chunks, N,
"sentinel must follow exactly {N} data chunks"
);
}
crate::wire::STREAM_CTRL_OUT => {
let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(v["type"], "exit");
assert_eq!(v["code"], 0);
break;
}
other => panic!("unexpected stream_type {other}"),
}
}
assert_eq!(data_chunks, N, "all {N} chunks must be delivered");
let _ = producer.await;
let _ = session.await;
}
#[tokio::test]
async fn stdin_eof_zero_length_chunk_closes_backend_stdin() {
let (backend, _control, _cancel) = TestBackend::builder().build();