Add TLS handshake timeout (review #010 C3)

tls_handshake_timeout_secs (default 10s, must be > 0) wraps
tls_acceptor.accept() in tokio::time::timeout so stalled handshakes
(slowloris / crawler slow-handshake vector) can no longer hold an FD +
semaphore permit indefinitely. Config plumbing through FullConfig /
StaticConfig / validation / reload diff + docs.
This commit is contained in:
glm-5.3-flash committed 2026-09-13 07:16:43 +00:00
1 parent 978a362536
commit 9e26295cf7
11 files changed
+352 -19

No files matched your search

+1
View File
@@ -116,6 +116,7 @@ Configuration uses TOML and is split into **static** (requires restart) and
| `admin_key_path` | `/etc/reverse-proxy/admin-key` | Path to admin Bearer token file (empty string to disable) | | `admin_key_path` | `/etc/reverse-proxy/admin-key` | Path to admin Bearer token file (empty string to disable) |
| `shutdown_timeout_secs` | `30` | Graceful shutdown timeout | | `shutdown_timeout_secs` | `30` | Graceful shutdown timeout |
| `connection_idle_timeout_secs` | `60` | Server-side idle timeout for client TLS connections (prevents FD exhaustion from abandoned connections) | | `connection_idle_timeout_secs` | `60` | Server-side idle timeout for client TLS connections (prevents FD exhaustion from abandoned connections) |
| `tls_handshake_timeout_secs` | `10` | Max seconds to complete the TLS handshake; stalled handshakes are closed (prevents slowloris FD/permit exhaustion) |
| `max_connections` | `1024` | Max concurrent client TLS connections (backpressure via semaphore) | | `max_connections` | `1024` | Max concurrent client TLS connections (backpressure via semaphore) |
| `logging.level` | `"info"` | Log level | | `logging.level` | `"info"` | Log level |
| `logging.format` | `"text"` | Log format (`"text"` or `"json"`) | | `logging.format` | `"text"` | Log format (`"text"` or `"json"`) |
+6
View File
@@ -91,6 +91,7 @@ Immutable after startup. Changes require a process restart.
| `admin_key_path` | `String` | Path to file containing the admin Bearer token (default: `/etc/reverse-proxy/admin-key`; empty string to disable admin endpoints; see ADR-028) | | `admin_key_path` | `String` | Path to file containing the admin Bearer token (default: `/etc/reverse-proxy/admin-key`; empty string to disable admin endpoints; see ADR-028) |
| `shutdown_timeout_secs` | `u64` | Maximum seconds to wait for in-flight requests during graceful shutdown (default: `30`) | | `shutdown_timeout_secs` | `u64` | Maximum seconds to wait for in-flight requests during graceful shutdown (default: `30`) |
| `connection_idle_timeout_secs` | `u64` | Server-side idle timeout for client TLS connections. Idle HTTP/2 connections are closed after this duration (with keep-alive pings at 15s intervals to detect dead peers). HTTP/1.1 connections are closed if the client doesn't send a complete request header within this duration. Prevents FD exhaustion from abandoned connections (default: `60`; must be > 0; see review #007 C1) | | `connection_idle_timeout_secs` | `u64` | Server-side idle timeout for client TLS connections. Idle HTTP/2 connections are closed after this duration (with keep-alive pings at 15s intervals to detect dead peers). HTTP/1.1 connections are closed if the client doesn't send a complete request header within this duration. Prevents FD exhaustion from abandoned connections (default: `60`; must be > 0; see review #007 C1) |
| `tls_handshake_timeout_secs` | `u64` | Maximum seconds a client may take to complete the TLS handshake. Stalled handshakes (e.g. crawlers/slowloris clients that open a TCP connection but never send a ClientHello) are closed after this duration, releasing the FD and connection slot. Without it, a stalled handshake holds an FD + connection semaphore permit indefinitely (default: `10`; must be > 0; see review #010 C3) |
| `max_connections` | `usize` | Maximum number of concurrent client TLS connections. When the limit is reached, new connections wait in the OS TCP backlog until a slot frees (default: `1024`; must be > 0; see review #007 C2) | | `max_connections` | `usize` | Maximum number of concurrent client TLS connections. When the limit is reached, new connections wait in the OS TCP backlog until a slot frees (default: `1024`; must be > 0; see review #007 C2) |
| `logging` | `LoggingConfig` | Logging configuration (see below) | | `logging` | `LoggingConfig` | Logging configuration (see below) |
@@ -185,6 +186,7 @@ Phase 2.
| `admin_key_path` | `String` | `/etc/reverse-proxy/admin-key` | No | | `admin_key_path` | `String` | `/etc/reverse-proxy/admin-key` | No |
| `shutdown_timeout_secs` | `u64` | `30` | No | | `shutdown_timeout_secs` | `u64` | `30` | No |
| `connection_idle_timeout_secs` | `u64` | `60` | No | | `connection_idle_timeout_secs` | `u64` | `60` | No |
| `tls_handshake_timeout_secs` | `u64` | `10` | No |
| `max_connections` | `usize` | `1024` | No | | `max_connections` | `usize` | `1024` | No |
| `logging.level` | `String` | `"info"` | No | | `logging.level` | `String` | `"info"` | No |
| `logging.format` | `String` | `"text"` | No | | `logging.format` | `String` | `"text"` | No |
@@ -313,6 +315,7 @@ certificate:
health_check_port = 9900 # Local health check (0 to disable) health_check_port = 9900 # Local health check (0 to disable)
admin_key_path = "/etc/reverse-proxy/admin-key" # Empty string to disable admin_key_path = "/etc/reverse-proxy/admin-key" # Empty string to disable
# connection_idle_timeout_secs = 60 # Server-side idle timeout (default: 60) # connection_idle_timeout_secs = 60 # Server-side idle timeout (default: 60)
# tls_handshake_timeout_secs = 10 # TLS handshake timeout (default: 10)
# max_connections = 1024 # Max concurrent TLS connections (default: 1024) # max_connections = 1024 # Max concurrent TLS connections (default: 1024)
[logging] [logging]
@@ -461,6 +464,9 @@ On startup, the config is validated:
review #007 C1. review #007 C1.
22. `max_connections` must be > 0. A zero value would deadlock the connection 22. `max_connections` must be > 0. A zero value would deadlock the connection
semaphore, preventing any client connection from being accepted. semaphore, preventing any client connection from being accepted.
23. `tls_handshake_timeout_secs` must be > 0. A zero value would immediately
kill every TLS handshake, preventing any client connection from
completing (review #010 C3).
On SIGHUP reload, the same validation applies. If the new config fails On SIGHUP reload, the same validation applies. If the new config fails
validation, the reload is rejected and the old config remains active. An error validation, the reload is rejected and the old config remains active. An error
+20 -17
View File
@@ -20,8 +20,9 @@ fixes:
C1: FIXED 2026-09-13 — accept-loop error classification + backoff + C1: FIXED 2026-09-13 — accept-loop error classification + backoff +
signature-keyed log de-duplication (src/server.rs). signature-keyed log de-duplication (src/server.rs).
- >- - >-
C3: OPEN — next priority. No TLS handshake timeout; stalled handshakes C3: FIXED 2026-09-13 — TLS handshake timeout (tls_handshake_timeout_secs,
hold an FD + a semaphore permit indefinitely (src/server.rs:370). default 10s) wraps tls_acceptor.accept() in src/server.rs; stalled
handshakes release FD + permit.
- >- - >-
C4: OPEN — connection semaphore is per-listener (src/server.rs:338), so C4: OPEN — connection semaphore is per-listener (src/server.rs:338), so
the effective cap is max_connections × listeners; sequence before C2 so the effective cap is max_connections × listeners; sequence before C2 so
@@ -177,14 +178,16 @@ Notes from implementation:
accept errors (axum 0.8.9 `src/serve/listener.rs` → accept errors (axum 0.8.9 `src/serve/listener.rs` →
`handle_accept_error`). No change needed there; the busy-spin existed `handle_accept_error`). No change needed there; the busy-spin existed
only in the custom HTTPS loop. only in the custom HTTPS loop.
- Two follow-up findings surfaced while fixing C1 (still open, tracked - Two follow-up findings surfaced while fixing C1 (tracked in
in "Recommended next steps"): "Recommended next steps"):
- **C3 (new)**: `tls_acceptor.accept()` has no timeout — a stalled TLS - ~~**C3 (new)**~~ — **FIXED 2026-09-13**: `tls_acceptor.accept()` had
handshake holds an FD *and* a semaphore permit indefinitely (the idle no timeout, so a stalled TLS handshake held an FD *and* a semaphore
watchdog only starts after the handshake completes). This is the permit indefinitely (the idle watchdog only starts after the
likely actual FD-exhaustion vector for slow/held crawler handshake completes). Fixed via `tls_handshake_timeout_secs`
(default 10s) wrapping the accept in `tokio::time::timeout`. This
was the likely actual FD-exhaustion vector for slow/held crawler
handshakes, and a slowloris amplifier. handshakes, and a slowloris amplifier.
- **C4 (new)**: `conn_sem` is per-listener (`main.rs` spawns one - **C4 (open)**: `conn_sem` is per-listener (`main.rs` spawns one
`serve_https_listener` per listener, each creating its own `serve_https_listener` per listener, each creating its own
semaphore), so the effective connection cap is semaphore), so the effective connection cap is
`max_connections × listeners`. Any C2 RLIMIT cross-check must `max_connections × listeners`. Any C2 RLIMIT cross-check must
@@ -262,7 +265,7 @@ observed limit).
Residual risk after mitigation: none identified for FD exhaustion at Residual risk after mitigation: none identified for FD exhaustion at
current traffic (peak concurrent connections observed ≪ 800); the code current traffic (peak concurrent connections observed ≪ 800); the code
findings C3/C4/C2 remain the durable fix (C1 landed 2026-09-13). findings C4/C2 remain the durable fix (C1 + C3 landed 2026-09-13).
## Traffic-analysis side note (from the same investigation) ## Traffic-analysis side note (from the same investigation)
@@ -288,13 +291,13 @@ behavior, which is exactly what made the EMFILE state reachable.
1. ~~Land C1 (accept-loop error backoff + log de-duplication)~~ — DONE 1. ~~Land C1 (accept-loop error backoff + log de-duplication)~~ — DONE
2026-09-13 (see Finding C1). 2026-09-13 (see Finding C1).
2. Land C3 (TLS handshake timeout) — bound stalled handshakes so they 2. ~~Land C3 (TLS handshake timeout)~~ — DONE 2026-09-13. New static config
cannot hold FD + permit indefinitely; directly closes the crawler `tls_handshake_timeout_secs` (default 10s, must be > 0) wraps
slow-handshake vector described under "Trigger conditions". **Next `tls_acceptor.accept()` in `tokio::time::timeout`
priority**: it is the likely actual trigger of the incident (stalled (`accept_tls_with_timeout()`, src/server.rs). On timeout the handshake
crawler handshakes under the pre-mitigation 1024 cap), it is the only future is dropped, releasing the TCP FD and the semaphore permit; the
finding that defends against a live attacker (slowloris amplifier), idle watchdog never needs to run for a stalled handshake. Closes the
and it does not interact with C2/C4 design decisions. crawler slow-handshake vector described under "Trigger conditions".
3. Land C4 (shared connection semaphore across listeners) so 3. Land C4 (shared connection semaphore across listeners) so
`max_connections` is a global cap rather than per-listener. Sequenced `max_connections` is a global cap rather than per-listener. Sequenced
before C2 because the RLIMIT cross-check's FD budget depends on the before C2 because the RLIMIT cross-check's FD budget depends on the
+3
View File
@@ -189,6 +189,9 @@ fn diff_static_config(old: &StaticConfig, new: &StaticConfig) -> Vec<String> {
if old.connection_idle_timeout_secs != new.connection_idle_timeout_secs { if old.connection_idle_timeout_secs != new.connection_idle_timeout_secs {
changes.push("connection_idle_timeout_secs".to_string()); changes.push("connection_idle_timeout_secs".to_string());
} }
if old.tls_handshake_timeout_secs != new.tls_handshake_timeout_secs {
changes.push("tls_handshake_timeout_secs".to_string());
}
if old.max_connections != new.max_connections { if old.max_connections != new.max_connections {
changes.push("max_connections".to_string()); changes.push("max_connections".to_string());
} }
+3
View File
@@ -51,6 +51,8 @@ pub struct FullConfig {
pub shutdown_timeout_secs: u64, pub shutdown_timeout_secs: u64,
#[serde(default = "static_config::default_connection_idle_timeout_secs")] #[serde(default = "static_config::default_connection_idle_timeout_secs")]
pub connection_idle_timeout_secs: u64, pub connection_idle_timeout_secs: u64,
#[serde(default = "static_config::default_tls_handshake_timeout_secs")]
pub tls_handshake_timeout_secs: u64,
#[serde(default = "static_config::default_max_connections")] #[serde(default = "static_config::default_max_connections")]
pub max_connections: usize, pub max_connections: usize,
#[serde(default)] #[serde(default)]
@@ -72,6 +74,7 @@ impl FullConfig {
admin_key_path: self.admin_key_path, admin_key_path: self.admin_key_path,
shutdown_timeout_secs: self.shutdown_timeout_secs, shutdown_timeout_secs: self.shutdown_timeout_secs,
connection_idle_timeout_secs: self.connection_idle_timeout_secs, connection_idle_timeout_secs: self.connection_idle_timeout_secs,
tls_handshake_timeout_secs: self.tls_handshake_timeout_secs,
max_connections: self.max_connections, max_connections: self.max_connections,
logging: self.logging, logging: self.logging,
}; };
+12
View File
@@ -13,6 +13,8 @@ pub struct StaticConfig {
pub shutdown_timeout_secs: u64, pub shutdown_timeout_secs: u64,
#[serde(default = "default_connection_idle_timeout_secs")] #[serde(default = "default_connection_idle_timeout_secs")]
pub connection_idle_timeout_secs: u64, pub connection_idle_timeout_secs: u64,
#[serde(default = "default_tls_handshake_timeout_secs")]
pub tls_handshake_timeout_secs: u64,
#[serde(default = "default_max_connections")] #[serde(default = "default_max_connections")]
pub max_connections: usize, pub max_connections: usize,
#[serde(default)] #[serde(default)]
@@ -35,6 +37,10 @@ pub fn default_connection_idle_timeout_secs() -> u64 {
60 60
} }
pub fn default_tls_handshake_timeout_secs() -> u64 {
10
}
pub fn default_max_connections() -> usize { pub fn default_max_connections() -> usize {
1024 1024
} }
@@ -213,6 +219,8 @@ upstream = "127.0.0.1:8080"
shutdown_timeout_secs: u64, shutdown_timeout_secs: u64,
#[serde(default = "default_connection_idle_timeout_secs")] #[serde(default = "default_connection_idle_timeout_secs")]
connection_idle_timeout_secs: u64, connection_idle_timeout_secs: u64,
#[serde(default = "default_tls_handshake_timeout_secs")]
tls_handshake_timeout_secs: u64,
#[serde(default = "default_max_connections")] #[serde(default = "default_max_connections")]
max_connections: usize, max_connections: usize,
#[serde(default)] #[serde(default)]
@@ -232,6 +240,7 @@ upstream = "127.0.0.1:8080"
assert_eq!(config.admin_key_path, "/etc/reverse-proxy/admin-key"); assert_eq!(config.admin_key_path, "/etc/reverse-proxy/admin-key");
assert_eq!(config.shutdown_timeout_secs, 30); assert_eq!(config.shutdown_timeout_secs, 30);
assert_eq!(config.connection_idle_timeout_secs, 60); assert_eq!(config.connection_idle_timeout_secs, 60);
assert_eq!(config.tls_handshake_timeout_secs, 10);
assert_eq!(config.max_connections, 1024); assert_eq!(config.max_connections, 1024);
assert_eq!(config.logging.level, "info"); assert_eq!(config.logging.level, "info");
assert_eq!(config.logging.format, "text"); assert_eq!(config.logging.format, "text");
@@ -315,6 +324,8 @@ acme_cache_dir = "/tmp/cache"
shutdown_timeout_secs: u64, shutdown_timeout_secs: u64,
#[serde(default = "default_connection_idle_timeout_secs")] #[serde(default = "default_connection_idle_timeout_secs")]
connection_idle_timeout_secs: u64, connection_idle_timeout_secs: u64,
#[serde(default = "default_tls_handshake_timeout_secs")]
tls_handshake_timeout_secs: u64,
#[serde(default = "default_max_connections")] #[serde(default = "default_max_connections")]
max_connections: usize, max_connections: usize,
#[serde(default)] #[serde(default)]
@@ -330,6 +341,7 @@ acme_cache_dir = "/tmp/cache"
assert_eq!(config.admin_key_path, "/etc/reverse-proxy/admin-key"); assert_eq!(config.admin_key_path, "/etc/reverse-proxy/admin-key");
assert_eq!(config.shutdown_timeout_secs, 30); assert_eq!(config.shutdown_timeout_secs, 30);
assert_eq!(config.connection_idle_timeout_secs, 60); assert_eq!(config.connection_idle_timeout_secs, 60);
assert_eq!(config.tls_handshake_timeout_secs, 10);
assert_eq!(config.max_connections, 1024); assert_eq!(config.max_connections, 1024);
assert_eq!(config.logging.level, "info"); assert_eq!(config.logging.level, "info");
assert_eq!(config.logging.format, "text"); assert_eq!(config.logging.format, "text");
+1
View File
@@ -23,6 +23,7 @@ pub fn test_static_config() -> StaticConfig {
admin_key_path: "/etc/reverse-proxy/admin-key".to_string(), admin_key_path: "/etc/reverse-proxy/admin-key".to_string(),
shutdown_timeout_secs: 30, shutdown_timeout_secs: 30,
connection_idle_timeout_secs: 60, connection_idle_timeout_secs: 60,
tls_handshake_timeout_secs: 10,
max_connections: 1024, max_connections: 1024,
logging: LoggingConfig::default(), logging: LoggingConfig::default(),
} }
+24
View File
@@ -79,6 +79,8 @@ pub enum ValidationError {
AdminKeyPathTraversal { path: String }, AdminKeyPathTraversal { path: String },
#[error("connection_idle_timeout_secs must be > 0, got {value}")] #[error("connection_idle_timeout_secs must be > 0, got {value}")]
ConnectionIdleTimeoutZero { value: u64 }, ConnectionIdleTimeoutZero { value: u64 },
#[error("tls_handshake_timeout_secs must be > 0, got {value}")]
TlsHandshakeTimeoutZero { value: u64 },
#[error("max_connections must be > 0, got {value}")] #[error("max_connections must be > 0, got {value}")]
MaxConnectionsZero { value: usize }, MaxConnectionsZero { value: usize },
} }
@@ -292,6 +294,12 @@ pub fn validate(
}); });
} }
if static_config.tls_handshake_timeout_secs == 0 {
errors.push(ValidationError::TlsHandshakeTimeoutZero {
value: static_config.tls_handshake_timeout_secs,
});
}
if static_config.max_connections == 0 { if static_config.max_connections == 0 {
errors.push(ValidationError::MaxConnectionsZero { errors.push(ValidationError::MaxConnectionsZero {
value: static_config.max_connections, value: static_config.max_connections,
@@ -393,6 +401,7 @@ mod tests {
admin_key_path: "/etc/reverse-proxy/admin-key".to_string(), admin_key_path: "/etc/reverse-proxy/admin-key".to_string(),
shutdown_timeout_secs: 30, shutdown_timeout_secs: 30,
connection_idle_timeout_secs: 60, connection_idle_timeout_secs: 60,
tls_handshake_timeout_secs: 10,
max_connections: 1024, max_connections: 1024,
logging: LoggingConfig::default(), logging: LoggingConfig::default(),
} }
@@ -431,6 +440,7 @@ mod tests {
admin_key_path: "/etc/reverse-proxy/admin-key".to_string(), admin_key_path: "/etc/reverse-proxy/admin-key".to_string(),
shutdown_timeout_secs: 30, shutdown_timeout_secs: 30,
connection_idle_timeout_secs: 60, connection_idle_timeout_secs: 60,
tls_handshake_timeout_secs: 10,
max_connections: 1024, max_connections: 1024,
logging: LoggingConfig::default(), logging: LoggingConfig::default(),
} }
@@ -1121,6 +1131,7 @@ mod tests {
admin_key_path: "/etc/reverse-proxy/admin-key".to_string(), admin_key_path: "/etc/reverse-proxy/admin-key".to_string(),
shutdown_timeout_secs: 30, shutdown_timeout_secs: 30,
connection_idle_timeout_secs: 60, connection_idle_timeout_secs: 60,
tls_handshake_timeout_secs: 10,
max_connections: 1024, max_connections: 1024,
logging: LoggingConfig::default(), logging: LoggingConfig::default(),
}; };
@@ -1340,6 +1351,19 @@ mod tests {
.any(|e| matches!(e, ValidationError::ConnectionIdleTimeoutZero { value: 0 }))); .any(|e| matches!(e, ValidationError::ConnectionIdleTimeoutZero { value: 0 })));
} }
#[test]
fn rule_tls_handshake_timeout_zero_rejected() {
let mut config = valid_static_config();
config.tls_handshake_timeout_secs = 0;
let dynamic = valid_dynamic_config();
let result = validate(&config, &dynamic, false);
assert!(result.is_err());
let errors = result.unwrap_err();
assert!(errors
.iter()
.any(|e| matches!(e, ValidationError::TlsHandshakeTimeoutZero { value: 0 })));
}
#[test] #[test]
fn rule_max_connections_zero_rejected() { fn rule_max_connections_zero_rejected() {
let mut config = valid_static_config(); let mut config = valid_static_config();
+3
View File
@@ -234,6 +234,9 @@ async fn run_server(loaded_config: cli::LoadedConfig, config_path: &str) -> Resu
std::time::Duration::from_secs( std::time::Duration::from_secs(
loaded_config.static_config.connection_idle_timeout_secs, loaded_config.static_config.connection_idle_timeout_secs,
), ),
std::time::Duration::from_secs(
loaded_config.static_config.tls_handshake_timeout_secs,
),
loaded_config.static_config.max_connections, loaded_config.static_config.max_connections,
)); ));
+176 -2
View File
@@ -108,6 +108,26 @@ async fn handle_accept_error(reporter: &AcceptErrorReporter, e: std::io::Error)
} }
} }
pub const DEFAULT_TLS_HANDSHAKE_TIMEOUT_SECS: u64 = 10;
async fn accept_tls_with_timeout(
tls_acceptor: TlsAcceptor,
tcp_stream: tokio::net::TcpStream,
timeout: Duration,
) -> Result<tokio_rustls::server::TlsStream<tokio::net::TcpStream>, AcceptTlsError> {
match tokio::time::timeout(timeout, tls_acceptor.accept(tcp_stream)).await {
Ok(Ok(stream)) => Ok(stream),
Ok(Err(e)) => Err(AcceptTlsError::Handshake(e)),
Err(_) => Err(AcceptTlsError::Timeout),
}
}
#[derive(Debug)]
pub enum AcceptTlsError {
Handshake(std::io::Error),
Timeout,
}
pub struct InFlightCounter { pub struct InFlightCounter {
count: AtomicUsize, count: AtomicUsize,
} }
@@ -325,6 +345,7 @@ async fn idle_watchdog(idle_state: Arc<IdleState>, timeout: Duration) {
} }
} }
#[allow(clippy::too_many_arguments)]
pub async fn serve_https_listener( pub async fn serve_https_listener(
tcp_listener: TcpListener, tcp_listener: TcpListener,
tls_acceptor: TlsAcceptor, tls_acceptor: TlsAcceptor,
@@ -332,6 +353,7 @@ pub async fn serve_https_listener(
mut shutdown_rx: tokio::sync::watch::Receiver<bool>, mut shutdown_rx: tokio::sync::watch::Receiver<bool>,
in_flight: Arc<InFlightCounter>, in_flight: Arc<InFlightCounter>,
connection_idle_timeout: Duration, connection_idle_timeout: Duration,
tls_handshake_timeout: Duration,
max_connections: usize, max_connections: usize,
) { ) {
let local_addr = tcp_listener.local_addr(); let local_addr = tcp_listener.local_addr();
@@ -367,14 +389,30 @@ pub async fn serve_https_listener(
let _guard = InFlightGuard::new(in_flight.clone()); let _guard = InFlightGuard::new(in_flight.clone());
let _permit = permit; let _permit = permit;
let tls_stream = match tls_acceptor.accept(tcp_stream).await { let tls_result = match accept_tls_with_timeout(
tls_acceptor,
tcp_stream,
tls_handshake_timeout,
)
.await
{
Ok(stream) => stream, Ok(stream) => stream,
Err(e) => { Err(AcceptTlsError::Timeout) => {
warn!(
remote_addr = %remote_addr,
timeout_secs = tls_handshake_timeout.as_secs(),
"TLS handshake timeout; closing stalled handshake"
);
return;
}
Err(AcceptTlsError::Handshake(e)) => {
warn!(error = %e, "TLS handshake failed"); warn!(error = %e, "TLS handshake failed");
return; return;
} }
}; };
let tls_stream = tls_result;
let alpn = tls_stream.get_ref().1.alpn_protocol(); let alpn = tls_stream.get_ref().1.alpn_protocol();
let is_h2 = alpn == Some(b"h2"); let is_h2 = alpn == Some(b"h2");
@@ -590,6 +628,142 @@ mod tests {
assert_eq!(reports.len(), 2); assert_eq!(reports.len(), 2);
} }
#[tokio::test(start_paused = true)]
async fn accept_tls_with_timeout_times_out_on_stalled_handshake() {
use tokio::io::AsyncReadExt;
use tokio::net::TcpListener;
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let client = tokio::spawn(async move {
let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
let mut stream = stream;
let mut buf = [0u8; 16];
let _ = stream.read(&mut buf).await;
});
let (tcp_stream, _remote_addr) = listener.accept().await.unwrap();
let tls_acceptor = test_tls_acceptor();
let start = tokio::time::Instant::now();
let result =
accept_tls_with_timeout(tls_acceptor, tcp_stream, Duration::from_secs(3)).await;
client.abort();
assert!(matches!(result, Err(AcceptTlsError::Timeout)));
assert_eq!(start.elapsed(), Duration::from_secs(3));
}
#[tokio::test]
async fn accept_tls_with_timeout_completes_real_handshake() {
use tokio::net::TcpListener;
use tokio_rustls::TlsConnector;
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let client_config = test_client_tls_config();
let connector = TlsConnector::from(client_config);
let server_name =
rustls::pki_types::ServerName::try_from("test.local".to_string()).unwrap();
let client = tokio::spawn(async move {
let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
connector.connect(server_name, stream).await
});
let (tcp_stream, _remote_addr) = listener.accept().await.unwrap();
let tls_acceptor = test_tls_acceptor();
let result = accept_tls_with_timeout(
tls_acceptor,
tcp_stream,
Duration::from_secs(5),
)
.await;
assert!(result.is_ok(), "real handshake should complete: {result:?}");
client.abort();
}
fn test_tls_acceptor() -> TlsAcceptor {
use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer};
let mut params = rcgen::CertificateParams::new(vec!["test.local".to_string()]).unwrap();
params.distinguished_name = rcgen::DistinguishedName::new();
params
.distinguished_name
.push(rcgen::DnType::CommonName, "test.local");
let key_pair = rcgen::KeyPair::generate().unwrap();
let cert = params.self_signed(&key_pair).unwrap();
let cert_der = cert.der().clone();
let key_der = key_pair.serialize_der();
let private_key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(key_der));
let config = rustls::ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(vec![cert_der], private_key)
.unwrap();
TlsAcceptor::from(std::sync::Arc::new(config))
}
fn test_client_tls_config() -> std::sync::Arc<rustls::ClientConfig> {
let config = rustls::ClientConfig::builder()
.dangerous()
.with_custom_certificate_verifier(std::sync::Arc::new(UnverifiedCert))
.with_no_client_auth();
std::sync::Arc::new(config)
}
#[derive(Debug)]
struct UnverifiedCert;
impl rustls::client::danger::ServerCertVerifier for UnverifiedCert {
fn verify_server_cert(
&self,
_end_entity: &rustls::pki_types::CertificateDer<'_>,
_intermediates: &[rustls::pki_types::CertificateDer<'_>],
_server_name: &rustls::pki_types::ServerName<'_>,
_ocsp_response: &[u8],
_now: rustls::pki_types::UnixTime,
) -> Result<rustls::client::danger::ServerCertVerified, rustls::Error> {
Ok(rustls::client::danger::ServerCertVerified::assertion())
}
fn verify_tls12_signature(
&self,
_message: &[u8],
_cert: &rustls::pki_types::CertificateDer<'_>,
_dss: &rustls::DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
}
fn verify_tls13_signature(
&self,
_message: &[u8],
_cert: &rustls::pki_types::CertificateDer<'_>,
_dss: &rustls::DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
}
fn supported_verify_schemes(&self) -> Vec<rustls::SignatureScheme> {
vec![
rustls::SignatureScheme::RSA_PKCS1_SHA256,
rustls::SignatureScheme::ECDSA_NISTP256_SHA256,
rustls::SignatureScheme::RSA_PSS_SHA256,
rustls::SignatureScheme::RSA_PKCS1_SHA384,
rustls::SignatureScheme::ECDSA_NISTP384_SHA384,
rustls::SignatureScheme::RSA_PSS_SHA384,
rustls::SignatureScheme::RSA_PKCS1_SHA512,
rustls::SignatureScheme::RSA_PSS_SHA512,
rustls::SignatureScheme::ED25519,
]
}
}
#[tokio::test(start_paused = true)] #[tokio::test(start_paused = true)]
async fn handle_accept_error_sleeps_on_resource_errors() { async fn handle_accept_error_sleeps_on_resource_errors() {
let reporter = AcceptErrorReporter::new(); let reporter = AcceptErrorReporter::new();
+103
View File
@@ -1163,6 +1163,19 @@ mod idle_timeout_tests {
Arc<InFlightCounter>, Arc<InFlightCounter>,
tokio::task::JoinHandle<()>, tokio::task::JoinHandle<()>,
tokio::sync::watch::Sender<bool>, tokio::sync::watch::Sender<bool>,
) {
start_test_https_server_with_timeouts(idle_timeout, Duration::from_secs(10), upstream).await
}
async fn start_test_https_server_with_timeouts(
idle_timeout: Duration,
handshake_timeout: Duration,
upstream: String,
) -> (
std::net::SocketAddr,
Arc<InFlightCounter>,
tokio::task::JoinHandle<()>,
tokio::sync::watch::Sender<bool>,
) { ) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap(); let addr = listener.local_addr().unwrap();
@@ -1181,6 +1194,7 @@ mod idle_timeout_tests {
shutdown_rx, shutdown_rx,
in_flight_clone, in_flight_clone,
idle_timeout, idle_timeout,
handshake_timeout,
1024, 1024,
) )
.await; .await;
@@ -1436,4 +1450,93 @@ mod idle_timeout_tests {
let _ = upstream.shutdown_tx.send(()); let _ = upstream.shutdown_tx.send(());
} }
// Reproducer for review #010 C3: a client that opens a TCP connection but
// never sends a TLS ClientHello must not hold its FD + semaphore permit
// indefinitely. The server must close the stalled handshake after
// tls_handshake_timeout and release the permit.
#[tokio::test]
async fn stalled_tls_handshake_closed_after_timeout_and_permit_released() {
let handshake_timeout = Duration::from_millis(300);
let upstream = helpers::http_test_helper::TestUpstream::spawn_ok().await;
let upstream_addr = format!("127.0.0.1:{}", upstream.addr.port());
let (addr, in_flight, _handle, _shutdown_tx) =
start_test_https_server_with_timeouts(
Duration::from_secs(60),
handshake_timeout,
upstream_addr,
)
.await;
let stalled = tokio::net::TcpStream::connect(addr).await.unwrap();
// Hold the connection open without sending anything (stalled handshake).
// Wait past the handshake timeout.
tokio::time::sleep(handshake_timeout + Duration::from_millis(500)).await;
// The server must have dropped the stalled connection: reading from it
// should yield EOF (or an error), not hang.
let mut stalled = stalled;
let mut buf = [0u8; 16];
let read_result =
tokio::time::timeout(Duration::from_secs(2), stalled.read(&mut buf)).await;
let closed = match read_result {
Err(_) => panic!("stalled handshake was not closed by the server"),
Ok(Ok(0)) => true,
Ok(Ok(_)) => false,
Ok(Err(e)) if e.kind() == std::io::ErrorKind::UnexpectedEof => true,
Ok(Err(_)) => true,
};
assert!(
closed,
"server should close stalled TLS handshake after timeout"
);
// The semaphore permit and in-flight guard must have been released.
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
in_flight.count(),
0,
"in-flight count should return to 0 after stalled handshake closed"
);
let _ = upstream.shutdown_tx.send(());
}
// A real TLS handshake must complete comfortably within the timeout —
// the timeout must only kill stalled handshakes, not legitimate clients.
#[tokio::test]
async fn real_handshake_completes_within_handshake_timeout() {
let handshake_timeout = Duration::from_millis(300);
let upstream = helpers::http_test_helper::TestUpstream::spawn_ok().await;
let upstream_addr = format!("127.0.0.1:{}", upstream.addr.port());
let (addr, in_flight, _handle, _shutdown_tx) =
start_test_https_server_with_timeouts(
Duration::from_secs(60),
handshake_timeout,
upstream_addr,
)
.await;
let mut tls_stream = connect_tls(addr).await;
tls_stream
.write_all(b"GET / HTTP/1.1\r\nHost: test.local\r\nConnection: keep-alive\r\n\r\n")
.await
.unwrap();
let mut buf = vec![0u8; 4096];
let n = tokio::time::timeout(Duration::from_secs(2), tls_stream.read(&mut buf))
.await
.expect("timeout waiting for response")
.expect("read error");
let response = String::from_utf8_lossy(&buf[..n]);
assert!(
response.starts_with("HTTP/1.1 200 OK"),
"expected a real 200 round-trip, got: {response}"
);
assert_eq!(in_flight.count(), 1, "connection should remain in-flight");
let _ = upstream.shutdown_tx.send(());
}
} }