feat(websocket): write-progress timeout on the WS write pump (WS-18)
- DEFAULT_WS_WRITE_TIMEOUT (60 s, WS-01 knob family): one outbound WS send that stays unsent past the window (peer stopped reading) evicts the connection — the send path was slot-bounded but time-unbounded - bound is per send call: a slow-but-draining peer resets it with every message emitted; only a fully stalled sink trips it - on timeout the pump signals the stream error (InvalidData naming the write stall) and ends WITHOUT a close frame — a clogged socket cannot receive one and the close send would park on it - test-support split helper with both knobs explicit backs the scaled eviction test (clogged duplex, eviction well inside the 5 s slack) Verification: scripts/verify.sh OK (343 passed), clippy -D warnings clean, fmt clean
This commit is contained in:
@@ -147,6 +147,14 @@ pub const WS_INTERNAL_ERROR: u16 = 1011;
|
|||||||
/// failed), not a protocol error.
|
/// failed), not a protocol error.
|
||||||
pub const WS_GOING_AWAY: u16 = 1001;
|
pub const WS_GOING_AWAY: u16 = 1001;
|
||||||
|
|
||||||
|
/// Default write-progress timeout for the WS write pump (WS-18): one
|
||||||
|
/// outbound WS send that stays unsent past this window (the peer
|
||||||
|
/// stopped reading) evicts the connection — the write-side analog of
|
||||||
|
/// the idle-read knob. The bound is per `send` call, not per
|
||||||
|
/// connection: a slow-but-draining peer resets it with every message
|
||||||
|
/// that gets out.
|
||||||
|
pub const DEFAULT_WS_WRITE_TIMEOUT: Duration = Duration::from_secs(60);
|
||||||
|
|
||||||
/// Default idle-read timeout for the WS pumps (WS-01, WS-13): a
|
/// Default idle-read timeout for the WS pumps (WS-01, WS-13): a
|
||||||
/// connection whose inbound stream completes **no chunk** within this
|
/// connection whose inbound stream completes **no chunk** within this
|
||||||
/// window is evicted — the deadline resets on demux progress (complete
|
/// window is evicted — the deadline resets on demux progress (complete
|
||||||
@@ -431,10 +439,30 @@ async fn fail_write_pump<M, S>(
|
|||||||
/// frame with the requested code and its per-cause reason (WS-15) —
|
/// frame with the requested code and its per-cause reason (WS-15) —
|
||||||
/// the code alone is ambiguous across causes (idle 1001 vs protocol
|
/// the code alone is ambiguous across causes (idle 1001 vs protocol
|
||||||
/// 1002 vs internal 1011 arms), so the pump never invents one.
|
/// 1002 vs internal 1011 arms), so the pump never invents one.
|
||||||
|
/// The write pump: drains the queued byte spans, parses the pending
|
||||||
|
/// bytes for complete chunks (8-byte header, length validated against
|
||||||
|
/// `MAX_CHUNK_LEN` — WS-04/HY-09), byte-caps the accumulator
|
||||||
|
/// (`PENDING_BUFFER_CAP` — WS-05), and emits one WS binary message per
|
||||||
|
/// `WS_MESSAGE_CAP` piece. Ends with a WS Close after the queue
|
||||||
|
/// drains; `CloseWith` requests (`WriteMsg::CloseWith`) map to a close
|
||||||
|
/// frame with the requested code and its per-cause reason (WS-15) —
|
||||||
|
/// the code alone is ambiguous across causes (idle 1001 vs protocol
|
||||||
|
/// 1002 vs internal 1011 arms), so the pump never invents one.
|
||||||
|
///
|
||||||
|
/// Write-progress timeout (WS-18): a peer that stops reading parks
|
||||||
|
/// the pump inside a WS send — the slot-bounded queue bounds memory,
|
||||||
|
/// but the stall was time-unbounded. When `write_timeout` is
|
||||||
|
/// `Some(window)`, every WS send must complete within the window:
|
||||||
|
/// exceeding it signals the stream error (naming the stall) and ends
|
||||||
|
/// the pump **without** a close frame — the peer cannot receive one
|
||||||
|
/// through a clogged socket, and the close send would park on it too.
|
||||||
|
/// The window is per send, so a slow-but-moving peer survives; only a
|
||||||
|
/// fully stalled sink trips it.
|
||||||
async fn run_write_pump<M, S>(
|
async fn run_write_pump<M, S>(
|
||||||
mut ws_sink: futures::stream::SplitSink<S, <M as WsFraming>::Msg>,
|
mut ws_sink: futures::stream::SplitSink<S, <M as WsFraming>::Msg>,
|
||||||
mut write_rx: futures_mpsc::Receiver<WriteMsg>,
|
mut write_rx: futures_mpsc::Receiver<WriteMsg>,
|
||||||
write_error_slot: WriteErrorSlot,
|
write_error_slot: WriteErrorSlot,
|
||||||
|
write_timeout: Option<Duration>,
|
||||||
) where
|
) where
|
||||||
S: futures::Sink<<M as WsFraming>::Msg> + Unpin,
|
S: futures::Sink<<M as WsFraming>::Msg> + Unpin,
|
||||||
M: WsFraming,
|
M: WsFraming,
|
||||||
@@ -443,9 +471,15 @@ async fn run_write_pump<M, S>(
|
|||||||
while let Some(msg) = write_rx.next().await {
|
while let Some(msg) = write_rx.next().await {
|
||||||
match msg {
|
match msg {
|
||||||
WriteMsg::CloseWith(code, reason) => {
|
WriteMsg::CloseWith(code, reason) => {
|
||||||
let _ = ws_sink
|
let close = <M as WsFraming>::close_message(code, reason);
|
||||||
.send(<M as WsFraming>::close_message(code, reason))
|
match write_timeout {
|
||||||
.await;
|
None => {
|
||||||
|
let _ = ws_sink.send(close).await;
|
||||||
|
}
|
||||||
|
Some(window) => {
|
||||||
|
let _ = tokio::time::timeout(window, ws_sink.send(close)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
WriteMsg::Bytes(b) => {
|
WriteMsg::Bytes(b) => {
|
||||||
@@ -474,10 +508,8 @@ async fn run_write_pump<M, S>(
|
|||||||
}
|
}
|
||||||
let chunk: Vec<u8> = pending.drain(..total).collect();
|
let chunk: Vec<u8> = pending.drain(..total).collect();
|
||||||
for piece in chunk.chunks(WS_MESSAGE_CAP) {
|
for piece in chunk.chunks(WS_MESSAGE_CAP) {
|
||||||
if ws_sink
|
if !send_bounded::<M, _>(&mut ws_sink, piece, write_timeout, &write_error_slot)
|
||||||
.send(<M as WsFraming>::binary_message(piece.to_vec()))
|
|
||||||
.await
|
.await
|
||||||
.is_err()
|
|
||||||
{
|
{
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -496,6 +528,45 @@ async fn run_write_pump<M, S>(
|
|||||||
let _ = ws_sink.close().await;
|
let _ = ws_sink.close().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// One WS send under the WS-18 write-progress bound: a send that
|
||||||
|
/// outlasts the window means the peer stopped reading, so the pump
|
||||||
|
/// signals the error slot (`InvalidData` on the `AsyncWrite` half) and
|
||||||
|
/// ends without a close frame — a clogged socket cannot receive one,
|
||||||
|
/// and the close send would park on the same stall. On send error, end
|
||||||
|
/// silently (the sink is already broken). Returns whether the pump
|
||||||
|
/// should continue.
|
||||||
|
async fn send_bounded<M, S>(
|
||||||
|
ws_sink: &mut futures::stream::SplitSink<S, <M as WsFraming>::Msg>,
|
||||||
|
piece: &[u8],
|
||||||
|
write_timeout: Option<Duration>,
|
||||||
|
slot: &WriteErrorSlot,
|
||||||
|
) -> bool
|
||||||
|
where
|
||||||
|
M: WsFraming,
|
||||||
|
S: futures::Sink<<M as WsFraming>::Msg> + Unpin,
|
||||||
|
{
|
||||||
|
match write_timeout {
|
||||||
|
None => ws_sink
|
||||||
|
.send(<M as WsFraming>::binary_message(piece.to_vec()))
|
||||||
|
.await
|
||||||
|
.is_ok(),
|
||||||
|
Some(window) => {
|
||||||
|
let send = ws_sink.send(<M as WsFraming>::binary_message(piece.to_vec()));
|
||||||
|
match tokio::time::timeout(window, send).await {
|
||||||
|
Ok(Ok(())) => true,
|
||||||
|
Ok(Err(_)) => false,
|
||||||
|
Err(_elapsed) => {
|
||||||
|
send_write_error(
|
||||||
|
slot,
|
||||||
|
"connection made no write progress past the write timeout",
|
||||||
|
);
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The `AsyncRead + AsyncWrite` view of a `WebSocket` handed to
|
/// The `AsyncRead + AsyncWrite` view of a `WebSocket` handed to
|
||||||
/// alkcall's channels machinery (demux/mux).
|
/// alkcall's channels machinery (demux/mux).
|
||||||
pub struct WsByteStream {
|
pub struct WsByteStream {
|
||||||
@@ -619,6 +690,7 @@ pub fn split_ws_to_bytes_idle(
|
|||||||
ws_sink,
|
ws_sink,
|
||||||
write_rx,
|
write_rx,
|
||||||
write_error_slot,
|
write_error_slot,
|
||||||
|
Some(crate::websocket::DEFAULT_WS_WRITE_TIMEOUT),
|
||||||
));
|
));
|
||||||
|
|
||||||
(
|
(
|
||||||
@@ -887,6 +959,7 @@ where
|
|||||||
ws_sink,
|
ws_sink,
|
||||||
write_rx,
|
write_rx,
|
||||||
write_error_slot,
|
write_error_slot,
|
||||||
|
Some(crate::websocket::DEFAULT_WS_WRITE_TIMEOUT),
|
||||||
));
|
));
|
||||||
|
|
||||||
(
|
(
|
||||||
@@ -909,11 +982,116 @@ where
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Test-only split with both knobs explicit (WS-18 acceptance: the
|
||||||
|
/// stall test scales the write window down; other tests keep the
|
||||||
|
/// default).
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn split_tungstenite_to_bytes_idle_with_write<S>(
|
||||||
|
socket: tokio_tungstenite::WebSocketStream<S>,
|
||||||
|
idle_timeout: Option<Duration>,
|
||||||
|
write_timeout: Option<Duration>,
|
||||||
|
) -> (WsByteStream, WsPumps)
|
||||||
|
where
|
||||||
|
S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static,
|
||||||
|
{
|
||||||
|
let (ws_sink, ws_stream) = socket.split();
|
||||||
|
let (read_tx, read_rx) = mpsc::channel::<Vec<u8>>(READ_SLOTS);
|
||||||
|
let (write_tx, write_rx) = futures_mpsc::channel::<WriteMsg>(WRITE_SLOTS);
|
||||||
|
let (write_error_slot, write_error_rx) = make_write_error_slot();
|
||||||
|
|
||||||
|
let read_eof = tokio::sync::watch::channel(false).0;
|
||||||
|
|
||||||
|
let write_tx_for_read = write_tx.clone();
|
||||||
|
let write_error_slot_for_read = Arc::clone(&write_error_slot);
|
||||||
|
let read_eof_for_task = read_eof.clone();
|
||||||
|
let read_task = tokio::spawn(run_read_pump::<TungsteniteFraming, _, _>(
|
||||||
|
ws_stream,
|
||||||
|
read_tx,
|
||||||
|
write_tx_for_read,
|
||||||
|
write_error_slot_for_read,
|
||||||
|
idle_timeout,
|
||||||
|
move || {
|
||||||
|
let _ = read_eof_for_task.send(true);
|
||||||
|
},
|
||||||
|
));
|
||||||
|
|
||||||
|
let write_task = tokio::spawn(run_write_pump::<TungsteniteFraming, _>(
|
||||||
|
ws_sink,
|
||||||
|
write_rx,
|
||||||
|
write_error_slot,
|
||||||
|
write_timeout,
|
||||||
|
));
|
||||||
|
|
||||||
|
(
|
||||||
|
WsByteStream {
|
||||||
|
read_rx,
|
||||||
|
read_buf: Vec::new(),
|
||||||
|
read_pos: 0,
|
||||||
|
eof: false,
|
||||||
|
write_tx: Some(write_tx),
|
||||||
|
write_open: true,
|
||||||
|
write_error: write_error_rx,
|
||||||
|
write_failed: false,
|
||||||
|
},
|
||||||
|
WsPumps {
|
||||||
|
read_task,
|
||||||
|
write_task,
|
||||||
|
read_eof,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
|
|
||||||
|
/// WS-18 acceptance: a peer that stops reading (the sink clogs —
|
||||||
|
/// nothing is drained from the duplex) parks the write pump inside
|
||||||
|
/// a WS send; the pump must end within the knob (+ slack), surface
|
||||||
|
/// the stream error naming the write stall, and fail the
|
||||||
|
/// `AsyncWrite` half with `InvalidData`. Scaled test: both knobs
|
||||||
|
/// idle-read `None` (so only the write knob can evict) and the
|
||||||
|
/// write window at 150 ms.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn tungstenite_write_stall_is_evicted_within_the_write_timeout() {
|
||||||
|
let (client_io, _server_io) = tokio::io::duplex(64);
|
||||||
|
let ws = tokio_tungstenite::WebSocketStream::from_raw_socket(
|
||||||
|
client_io,
|
||||||
|
tokio_tungstenite::tungstenite::protocol::Role::Client,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let knob = std::time::Duration::from_millis(150);
|
||||||
|
let (mut stream, _pumps) = split_tungstenite_to_bytes_idle_with_write(ws, None, Some(knob));
|
||||||
|
|
||||||
|
let mut chunk = vec![0u8; 8];
|
||||||
|
chunk[4..8].copy_from_slice(&64u32.to_be_bytes());
|
||||||
|
chunk.extend_from_slice(&[0u8; 64]);
|
||||||
|
stream.write_all(&chunk).await.expect("chunk written");
|
||||||
|
stream.flush().await.expect("flush");
|
||||||
|
|
||||||
|
let started = tokio::time::Instant::now();
|
||||||
|
let err = loop {
|
||||||
|
match stream.write_all(&[0u8; 8]).await {
|
||||||
|
Err(e) => break e,
|
||||||
|
Ok(_) => assert!(
|
||||||
|
started.elapsed() < std::time::Duration::from_secs(5),
|
||||||
|
"stream never failed while the sink stayed clogged"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
};
|
||||||
|
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||||
|
assert!(
|
||||||
|
err.to_string().contains("write timeout"),
|
||||||
|
"error names the violation: {err}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
started.elapsed() < std::time::Duration::from_secs(5),
|
||||||
|
"eviction must happen within the knob (+ slack), not hang"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/// An 8-byte bogus chunk header claiming `MAX_CHUNK_LEN + 1`
|
/// An 8-byte bogus chunk header claiming `MAX_CHUNK_LEN + 1`
|
||||||
/// payload bytes (WS-04/HY-09 acceptance: the parser must fail the
|
/// payload bytes (WS-04/HY-09 acceptance: the parser must fail the
|
||||||
/// stream loudly instead of silently waiting for ~4 GiB).
|
/// stream loudly instead of silently waiting for ~4 GiB).
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ pub mod upgrade;
|
|||||||
|
|
||||||
pub use byte_adapter::{
|
pub use byte_adapter::{
|
||||||
split_ws_to_bytes, split_ws_to_bytes_idle, WsByteStream, WsPumps, DEFAULT_WS_IDLE_TIMEOUT,
|
split_ws_to_bytes, split_ws_to_bytes_idle, WsByteStream, WsPumps, DEFAULT_WS_IDLE_TIMEOUT,
|
||||||
INBOUND_WS_FRAME_CAP, INBOUND_WS_MESSAGE_CAP, MAX_CHUNK_LEN, PENDING_BUFFER_CAP, WS_GOING_AWAY,
|
DEFAULT_WS_WRITE_TIMEOUT, INBOUND_WS_FRAME_CAP, INBOUND_WS_MESSAGE_CAP, MAX_CHUNK_LEN,
|
||||||
WS_MESSAGE_CAP, WS_PROTOCOL_ERROR,
|
PENDING_BUFFER_CAP, WS_GOING_AWAY, WS_MESSAGE_CAP, WS_PROTOCOL_ERROR,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[cfg(any(test, feature = "wss"))]
|
#[cfg(any(test, feature = "wss"))]
|
||||||
|
|||||||
Reference in New Issue
Block a user