From 33aedd8ec497955e65dbdd5126d2e873afa8a97b Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Wed, 8 Jul 2026 16:48:56 -0400 Subject: [PATCH] crashes: Capture glibc's abort diagnostic in crash reports When glibc aborts the process itself (malloc integrity checks like "free(): invalid pointer", assertion failures, stack smashing reports), it records the diagnostic in the private __abort_msg global before raising SIGABRT. Resolve that symbol's address at startup and send it to the crash-handler process, which reads the message out of the crashed process's memory with process_vm_readv while the client is still parked in its signal handler. The message is attached to crash uploads as a searchable abort_message tag plus a full-text context. The crashed process does no work itself: in exactly these crashes the crashed thread may hold the allocator's arena lock, so it cannot safely allocate or format anything. --- Cargo.lock | 1 + crates/crashes/Cargo.toml | 3 + crates/crashes/src/crashes.rs | 103 ++++++++++++++++++++++++++++++++++ crates/zed/src/reliability.rs | 15 +++++ 4 files changed, 122 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 3f71a7000d7..1ea00eb5112 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4345,6 +4345,7 @@ version = "0.1.0" dependencies = [ "async-process", "crash-handler", + "libc", "log", "mach2 0.5.0", "minidumper", diff --git a/crates/crashes/Cargo.toml b/crates/crashes/Cargo.toml index f8b898112c1..1b118097cb4 100644 --- a/crates/crashes/Cargo.toml +++ b/crates/crashes/Cargo.toml @@ -16,6 +16,9 @@ serde_json.workspace = true system_specs.workspace = true zstd.workspace = true +[target.'cfg(target_os = "linux")'.dependencies] +libc.workspace = true + [target.'cfg(target_os = "macos")'.dependencies] mach2.workspace = true diff --git a/crates/crashes/src/crashes.rs b/crates/crashes/src/crashes.rs index 28217496cad..eef3ab4055b 100644 --- a/crates/crashes/src/crashes.rs +++ b/crates/crashes/src/crashes.rs @@ -138,6 +138,17 @@ where info!("crash signal handlers installed"); send_crash_server_message(&client, CrashServerMessage::Init(crash_init)); + #[cfg(all(target_os = "linux", target_env = "gnu"))] + if let Some(address) = abort_message_address() { + send_crash_server_message( + &client, + CrashServerMessage::AbortMessageLocation(AbortMessageLocation { + pid: process::id(), + address, + }), + ); + } + #[cfg(target_os = "linux")] handler.set_ptracer(Some(_crash_handler.id())); @@ -170,6 +181,7 @@ pub struct CrashServer { panic_info: Mutex>, active_gpu: Mutex>, user_info: Mutex>, + abort_message_location: Mutex>, has_connection: Arc, logs_dir: PathBuf, } @@ -179,11 +191,27 @@ pub struct CrashInfo { pub init: InitCrashHandler, pub panic: Option, pub minidump_error: Option, + /// The diagnostic the C runtime recorded before aborting the process, e.g. + /// glibc's "free(): invalid pointer". Only present when the crash was a + /// runtime-initiated abort rather than a signal like SIGSEGV or a panic. + #[serde(default)] + pub abort_message: Option, pub gpus: Vec, pub active_gpu: Option, pub user_info: Option, } +/// Where to find the C runtime's abort diagnostic in the crashed process's +/// memory. Sent by the client at startup so that after a crash the server can +/// recover the message with `process_vm_readv`; the crashed process itself +/// can't safely do this work, since its heap may be corrupt and its allocator +/// locks may be held by the crashed thread. +#[derive(Debug, Deserialize, Serialize, Clone, Copy)] +pub struct AbortMessageLocation { + pub pid: u32, + pub address: u64, +} + #[derive(Debug, Deserialize, Serialize, Clone)] pub struct InitCrashHandler { pub session_id: String, @@ -233,6 +261,68 @@ enum CrashServerMessage { Panic(CrashPanic), GPUInfo(GpuSpecs), UserInfo(UserInfo), + AbortMessageLocation(AbortMessageLocation), +} + +/// glibc records the diagnostic it prints just before aborting (malloc integrity +/// failures like "free(): invalid pointer", assertion failures, stack-smashing +/// reports) in the private global `__abort_msg`, specifically so it can be +/// recovered post-mortem. Resolve its address here, in a safe context at startup. +/// The symbol is only exported at the GLIBC_PRIVATE version, which plain `dlsym` +/// won't resolve, and it has no stability guarantee, so a null result (e.g. musl, +/// or a future glibc removing it) just disables this diagnostic. +#[cfg(all(target_os = "linux", target_env = "gnu"))] +fn abort_message_address() -> Option { + let ptr = unsafe { + libc::dlvsym( + libc::RTLD_DEFAULT, + c"__abort_msg".as_ptr(), + c"GLIBC_PRIVATE".as_ptr(), + ) + }; + std::ptr::NonNull::new(ptr).map(|ptr| ptr.as_ptr() as u64) +} + +/// Read the crashed process's abort diagnostic. `__abort_msg` points to a +/// `struct abort_msg_s { unsigned int size; char msg[]; }` that glibc allocates +/// with mmap so that it stays intact even when the heap is corrupt. +#[cfg(target_os = "linux")] +fn read_abort_message(location: AbortMessageLocation) -> Option { + const MAX_MESSAGE_LEN: u32 = 4096; + + let pointer_bytes = read_process_memory(location.pid, location.address, size_of::())?; + let message_address = u64::from_ne_bytes(pointer_bytes.try_into().ok()?); + if message_address == 0 { + return None; + } + let size_bytes = read_process_memory(location.pid, message_address, size_of::())?; + let size = u32::from_ne_bytes(size_bytes.try_into().ok()?).min(MAX_MESSAGE_LEN); + if size == 0 { + return None; + } + let message_bytes = read_process_memory( + location.pid, + message_address + size_of::() as u64, + size as usize, + )?; + let message = String::from_utf8_lossy(&message_bytes).trim().to_string(); + (!message.is_empty()).then_some(message) +} + +#[cfg(target_os = "linux")] +fn read_process_memory(pid: u32, address: u64, len: usize) -> Option> { + let mut buffer = vec![0u8; len]; + let local = libc::iovec { + iov_base: buffer.as_mut_ptr().cast(), + iov_len: len, + }; + let remote = libc::iovec { + iov_base: address as *mut libc::c_void, + iov_len: len, + }; + let bytes_read = + unsafe { libc::process_vm_readv(pid as libc::pid_t, &local, 1, &remote, 1, 0) }; + (bytes_read == len as isize).then_some(buffer) } impl minidumper::ServerHandler for CrashServer { @@ -281,6 +371,14 @@ impl minidumper::ServerHandler for CrashServer { } }; + // The crashed process is still alive at this point: it stays parked in + // its signal handler until the server acknowledges the dump request, + // which happens after this callback returns. + #[cfg(target_os = "linux")] + let abort_message = (*self.abort_message_location.lock()).and_then(read_abort_message); + #[cfg(not(target_os = "linux"))] + let abort_message = None; + let crash_info = CrashInfo { init: self .initialization_params @@ -289,6 +387,7 @@ impl minidumper::ServerHandler for CrashServer { .expect("not initialized"), panic: self.panic_info.lock().clone(), minidump_error, + abort_message, active_gpu: self.active_gpu.lock().clone(), gpus, user_info: self.user_info.lock().clone(), @@ -320,6 +419,9 @@ impl minidumper::ServerHandler for CrashServer { CrashServerMessage::UserInfo(user_info) => { self.user_info.lock().replace(user_info); } + CrashServerMessage::AbortMessageLocation(location) => { + self.abort_message_location.lock().replace(location); + } } } @@ -523,6 +625,7 @@ pub fn crash_server(socket: &Path, logs_dir: PathBuf) { initialization_params: Mutex::default(), panic_info: Mutex::default(), user_info: Mutex::default(), + abort_message_location: Mutex::default(), has_connection, active_gpu: Mutex::default(), logs_dir, diff --git a/crates/zed/src/reliability.rs b/crates/zed/src/reliability.rs index b4a06b2629d..00917069d20 100644 --- a/crates/zed/src/reliability.rs +++ b/crates/zed/src/reliability.rs @@ -220,6 +220,21 @@ async fn upload_minidump( if let Some(minidump_error) = metadata.minidump_error.clone() { form = form.text("minidump_error", minidump_error); } + if let Some(abort_message) = metadata.abort_message.as_ref() { + // Sentry tag values are limited to 200 characters on a single line, so + // put a searchable prefix in the tag (which grouping rules also match + // on) and the full message in a context. + let tag: String = abort_message + .lines() + .next() + .unwrap_or_default() + .chars() + .take(200) + .collect(); + form = form + .text("sentry[tags][abort_message]", tag) + .text("sentry[contexts][abort][message]", abort_message.clone()); + } if let Some(is_staff) = &metadata .user_info