Skip to content

Handle large DiagnosticsIPC ProcessInfo payloads gracefully #129994

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tommcdon merged 1 commit into from
Jun 30, 2026
+96 −44
Merged

Handle large DiagnosticsIPC ProcessInfo payloads gracefully #129994

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 18 additions & 11 deletions src/native/eventpipe/ds-dump-protocol.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,19 +118,20 @@ dump_protocol_generate_core_dump_response_fini(
}

static
uint16_t
bool
dump_protocol_generate_core_dump_response_get_size (
DiagnosticsGenerateCoreDumpResponsePayload *payload)
DiagnosticsGenerateCoreDumpResponsePayload *payload,
uint16_t *size_out)
{
EP_ASSERT (payload != NULL);
EP_ASSERT (size_out != NULL);

size_t size = sizeof(payload->error);

size += sizeof(uint32_t);
size += (payload->error_message != NULL) ? (ep_rt_utf16_string_len (payload->error_message) + 1) * sizeof(ep_char16_t) : 0;

EP_ASSERT (size <= UINT16_MAX);
return (uint16_t)size;
return ds_ipc_payload_size_try_narrow (size, size_out);
}

static
Expand All @@ -146,7 +147,11 @@ dump_protocol_generate_core_dump_response_flatten (
EP_ASSERT (buffer != NULL);
EP_ASSERT (*buffer != NULL);
EP_ASSERT (size != NULL);
EP_ASSERT (dump_protocol_generate_core_dump_response_get_size (response) == *size);
#ifdef EP_CHECKED_BUILD
uint16_t expected_size;
EP_ASSERT (dump_protocol_generate_core_dump_response_get_size (response, &expected_size));
EP_ASSERT (expected_size == *size);
#endif

bool success = true;

Expand Down Expand Up @@ -177,12 +182,14 @@ dump_protocol_generate_core_dump_response (

dump_protocol_generate_core_dump_response_init(&payload, error, errorText);

bool result = ds_ipc_message_initialize_buffer (
&message,
ds_ipc_header_get_generic_error (),
&payload,
dump_protocol_generate_core_dump_response_get_size(&payload),
dump_protocol_generate_core_dump_response_flatten);
uint16_t payload_size;
bool result = dump_protocol_generate_core_dump_response_get_size (&payload, &payload_size) &&
ds_ipc_message_initialize_buffer (
&message,
ds_ipc_header_get_generic_error (),
&payload,
payload_size,
dump_protocol_generate_core_dump_response_flatten);

if (result)
ds_ipc_message_send (&message, stream);
Expand Down
84 changes: 56 additions & 28 deletions src/native/eventpipe/ds-process-protocol.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
*/

static
uint16_t
process_info_payload_get_size (DiagnosticsProcessInfoPayload *payload);
bool
process_info_payload_get_size (DiagnosticsProcessInfoPayload *payload, uint16_t *size_out);

static
bool
Expand All @@ -27,8 +27,8 @@ process_info_payload_flatten (
uint16_t *size);

static
uint16_t
env_info_payload_get_size (DiagnosticsEnvironmentInfoPayload *payload);
bool
env_info_payload_get_size (DiagnosticsEnvironmentInfoPayload *payload, uint16_t *size_out);

static
uint32_t
Expand Down Expand Up @@ -112,8 +112,8 @@ process_protocol_helper_unknown_command (
*/

static
uint16_t
process_info_payload_get_size (DiagnosticsProcessInfoPayload *payload)
bool
process_info_payload_get_size (DiagnosticsProcessInfoPayload *payload, uint16_t *size_out)
{
// see IPC spec @ https://github.com/dotnet/diagnostics/blob/main/documentation/design-docs/ipc-protocol.md
// for definition of serialization format
Expand All @@ -125,6 +125,7 @@ process_info_payload_get_size (DiagnosticsProcessInfoPayload *payload)
// LPCWSTR Arch; -> 4 bytes + strlen * sizeof(WCHAR)

EP_ASSERT (payload != NULL);
EP_ASSERT (size_out != NULL);

size_t size = 0;
size += sizeof(payload->process_id);
Expand All @@ -142,8 +143,7 @@ process_info_payload_get_size (DiagnosticsProcessInfoPayload *payload)
size += (payload->arch != NULL) ?
(ep_rt_utf16_string_len (payload->arch) + 1) * sizeof(ep_char16_t) : 0;

EP_ASSERT (size <= UINT16_MAX);
return (uint16_t)size;
return ds_ipc_payload_size_try_narrow (size, size_out);
}

static
Expand All @@ -159,7 +159,11 @@ process_info_payload_flatten (
EP_ASSERT (buffer != NULL);
EP_ASSERT (*buffer != NULL);
EP_ASSERT (size != NULL);
EP_ASSERT (process_info_payload_get_size (process_info) == *size);
#ifdef EP_CHECKED_BUILD
uint16_t expected_size;
EP_ASSERT (process_info_payload_get_size (process_info, &expected_size));
EP_ASSERT (expected_size == *size);
#endif

// see IPC spec @ https://github.com/dotnet/diagnostics/blob/main/documentation/design-docs/ipc-protocol.md
// for definition of serialization format
Expand Down Expand Up @@ -226,8 +230,8 @@ ds_process_info_payload_fini (DiagnosticsProcessInfoPayload *payload)
*/

static
uint16_t
process_info_2_payload_get_size (DiagnosticsProcessInfo2Payload *payload)
bool
process_info_2_payload_get_size (DiagnosticsProcessInfo2Payload *payload, uint16_t *size_out)
{
// see IPC spec @ https://github.com/dotnet/diagnostics/blob/main/documentation/design-docs/ipc-protocol.md
// for definition of serialization format
Expand All @@ -241,6 +245,7 @@ process_info_2_payload_get_size (DiagnosticsProcessInfo2Payload *payload)
// LPCWSTR clr_product_version; -> 4 bytes + strlen * sizeof(WCHAR)

EP_ASSERT (payload != NULL);
EP_ASSERT (size_out != NULL);

size_t size = 0;
size += sizeof(payload->process_id);
Expand All @@ -266,8 +271,7 @@ process_info_2_payload_get_size (DiagnosticsProcessInfo2Payload *payload)
size += (payload->clr_product_version != NULL) ?
(ep_rt_utf16_string_len (payload->clr_product_version) + 1) * sizeof(ep_char16_t) : 0;

EP_ASSERT (size <= UINT16_MAX);
return (uint16_t)size;
return ds_ipc_payload_size_try_narrow (size, size_out);
}

static
Expand All @@ -283,7 +287,11 @@ process_info_2_payload_flatten (
EP_ASSERT (buffer != NULL);
EP_ASSERT (*buffer != NULL);
EP_ASSERT (size != NULL);
EP_ASSERT (process_info_2_payload_get_size (process_info) == *size);
#ifdef EP_CHECKED_BUILD
uint16_t expected_size;
EP_ASSERT (process_info_2_payload_get_size (process_info, &expected_size));
EP_ASSERT (expected_size == *size);
#endif

// see IPC spec @ https://github.com/dotnet/diagnostics/blob/main/documentation/design-docs/ipc-protocol.md
// for definition of serialization format
Expand Down Expand Up @@ -363,8 +371,8 @@ ds_process_info_2_payload_fini (DiagnosticsProcessInfo2Payload *payload)
*/

static
uint16_t
process_info_3_payload_get_size (DiagnosticsProcessInfo3Payload *payload)
bool
process_info_3_payload_get_size (DiagnosticsProcessInfo3Payload *payload, uint16_t *size_out)
{
// see IPC spec @ https://github.com/dotnet/diagnostics/blob/main/documentation/design-docs/ipc-protocol.md
// for definition of serialization format
Expand All @@ -380,6 +388,7 @@ process_info_3_payload_get_size (DiagnosticsProcessInfo3Payload *payload)
// LPCWSTR portable_rid; -> 4 bytes + strlen * sizeof(WCHAR)

EP_ASSERT (payload != NULL);
EP_ASSERT (size_out != NULL);

size_t size = 0;
size += sizeof(payload->version);
Expand Down Expand Up @@ -410,8 +419,7 @@ process_info_3_payload_get_size (DiagnosticsProcessInfo3Payload *payload)
size += (payload->portable_rid != NULL) ?
(ep_rt_utf16_string_len (payload->portable_rid) + 1) * sizeof(ep_char16_t) : 0;

EP_ASSERT (size <= UINT16_MAX);
return (uint16_t)size;
return ds_ipc_payload_size_try_narrow (size, size_out);
}

static
Expand All @@ -427,7 +435,11 @@ process_info_3_payload_flatten (
EP_ASSERT (buffer != NULL);
EP_ASSERT (*buffer != NULL);
EP_ASSERT (size != NULL);
EP_ASSERT (process_info_3_payload_get_size (process_info) == *size);
#ifdef EP_CHECKED_BUILD
uint16_t expected_size;
EP_ASSERT (process_info_3_payload_get_size (process_info, &expected_size));
EP_ASSERT (expected_size == *size);
#endif

// see IPC spec @ https://github.com/dotnet/diagnostics/blob/main/documentation/design-docs/ipc-protocol.md
// for definition of serialization format
Expand Down Expand Up @@ -523,17 +535,17 @@ ds_process_info_3_payload_fini (DiagnosticsProcessInfo3Payload *payload)
*/

static
uint16_t
env_info_payload_get_size (DiagnosticsEnvironmentInfoPayload *payload)
bool
env_info_payload_get_size (DiagnosticsEnvironmentInfoPayload *payload, uint16_t *size_out)
{
EP_ASSERT (payload != NULL);
EP_ASSERT (size_out != NULL);

size_t size = 0;
size += sizeof (payload->incoming_bytes);
size += sizeof (payload->future);

EP_ASSERT (size <= UINT16_MAX);
return (uint16_t)size;
return ds_ipc_payload_size_try_narrow (size, size_out);
}

static
Expand Down Expand Up @@ -568,7 +580,11 @@ env_info_payload_flatten (
EP_ASSERT (buffer != NULL);
EP_ASSERT (*buffer != NULL);
EP_ASSERT (size != NULL);
EP_ASSERT (env_info_payload_get_size (env_info) == *size);
#ifdef EP_CHECKED_BUILD
uint16_t expected_size;
EP_ASSERT (env_info_payload_get_size (env_info, &expected_size));
EP_ASSERT (expected_size == *size);
#endif

// see IPC spec @ https://github.com/dotnet/diagnostics/blob/main/documentation/design-docs/ipc-protocol.md
// for definition of serialization format
Expand Down Expand Up @@ -685,11 +701,14 @@ process_protocol_helper_get_process_info (
ds_ipc_advertise_cookie_v1_get ());
ep_raise_error_if_nok (process_info_payload != NULL);

uint16_t payload_size;
ep_raise_error_if_nok (process_info_payload_get_size (process_info_payload, &payload_size));

ep_raise_error_if_nok (ds_ipc_message_initialize_buffer (
message,
ds_ipc_header_get_generic_success (),
(void *)process_info_payload,
process_info_payload_get_size (process_info_payload),
payload_size,
process_info_payload_flatten));
Comment thread
tommcdon marked this conversation as resolved.

ep_raise_error_if_nok (ds_ipc_message_send (message, stream));
Expand Down Expand Up @@ -755,11 +774,14 @@ process_protocol_helper_get_process_info_2 (
clr_product_version);
ep_raise_error_if_nok (process_info_2_payload != NULL);

uint16_t payload_size;
ep_raise_error_if_nok (process_info_2_payload_get_size (process_info_2_payload, &payload_size));

ep_raise_error_if_nok (ds_ipc_message_initialize_buffer (
message,
ds_ipc_header_get_generic_success (),
(void *)process_info_2_payload,
process_info_2_payload_get_size (process_info_2_payload),
payload_size,
process_info_2_payload_flatten));

ep_raise_error_if_nok (ds_ipc_message_send (message, stream));
Expand Down Expand Up @@ -832,11 +854,14 @@ process_protocol_helper_get_process_info_3 (
portable_rid);
ep_raise_error_if_nok (process_info_3_payload != NULL);

uint16_t payload_size;
ep_raise_error_if_nok (process_info_3_payload_get_size (process_info_3_payload, &payload_size));

ep_raise_error_if_nok (ds_ipc_message_initialize_buffer (
message,
ds_ipc_header_get_generic_success (),
(void *)process_info_3_payload,
process_info_3_payload_get_size (process_info_3_payload),
payload_size,
process_info_3_payload_flatten));

ep_raise_error_if_nok (ds_ipc_message_send (message, stream));
Expand Down Expand Up @@ -877,11 +902,14 @@ process_protocol_helper_get_process_env (
env_info_payload = ds_env_info_payload_init (&payload);
ep_raise_error_if_nok (env_info_payload);

uint16_t payload_size;
ep_raise_error_if_nok (env_info_payload_get_size (env_info_payload, &payload_size));

ep_raise_error_if_nok (ds_ipc_message_initialize_buffer (
message,
ds_ipc_header_get_generic_success (),
(void *)env_info_payload,
env_info_payload_get_size (env_info_payload),
payload_size,
env_info_payload_flatten));

ep_raise_error_if_nok (ds_ipc_message_send (message, stream));
Expand Down
11 changes: 6 additions & 5 deletions src/native/eventpipe/ds-protocol.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,13 +276,14 @@ ipc_message_flatten (

bool result = true;
uint8_t *buffer = NULL;
uint16_t remaining_len = 0;

uint16_t total_len = 0;
EP_ASSERT (UINT16_MAX >= sizeof (DiagnosticsIpcHeader) + payload_len);
total_len += sizeof (DiagnosticsIpcHeader) + payload_len;
size_t total_len = (size_t)sizeof (DiagnosticsIpcHeader) + (size_t)payload_len;
EP_ASSERT (total_len <= UINT16_MAX);
ep_raise_error_if_nok (total_len <= UINT16_MAX);
Comment thread
tommcdon marked this conversation as resolved.

uint16_t remaining_len = total_len;
message->size = total_len;
remaining_len = (uint16_t)total_len;
message->size = (uint16_t)total_len;

buffer = ep_rt_byte_array_alloc (message->size);
ep_raise_error_if_nok (buffer != NULL);
Expand Down
16 changes: 16 additions & 0 deletions src/native/eventpipe/ds-protocol.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,22 @@ DS_DEFINE_GETTER_ARRAY_REF(DiagnosticsIpcHeader *, ipc_header, uint8_t *, const
DS_DEFINE_GETTER(DiagnosticsIpcHeader *, ipc_header, uint8_t, commandset) // ds_ipc_header_get_commandset
DS_DEFINE_GETTER(DiagnosticsIpcHeader *, ipc_header, uint8_t, commandid) // ds_ipc_header_get_commandid

// Validates that a payload size fits within the uint16_t message-size field
// (which must also accommodate the header) and writes the narrowed value to
// size_out. Returns true on success, false if the payload is too large.
static
inline
bool
ds_ipc_payload_size_try_narrow (size_t size, uint16_t *size_out)
{
EP_ASSERT (size_out != NULL);
EP_ASSERT (size <= UINT16_MAX - sizeof (DiagnosticsIpcHeader));
if (size > UINT16_MAX - sizeof (DiagnosticsIpcHeader))
return false;
*size_out = (uint16_t)size;
return true;
}
Comment thread
tommcdon marked this conversation as resolved.

/*
* DiagnosticsIpcMessage
*/
Expand Down
Loading