From 1b0cdf7bdcdb392c04ff9df5015f89530f45886d Mon Sep 17 00:00:00 2001 From: Shizuo Fujita Date: Mon, 6 Jul 2026 10:36:59 +0900 Subject: [PATCH 1/3] Raise StackError instead of crashing on deeply nested recursive extensions read_raw_body_begin() ignored the return value of _msgpack_unpacker_stack_push() on the recursive-extension path. Recursive extensions re-enter msgpack_unpacker_read() through the user proc, consuming a full C stack frame per nesting level. Once the unpacker stack was already at MSGPACK_UNPACKER_STACK_CAPACITY, the failed push was silently discarded and the recursion continued unbounded, exhausting the C stack and crashing the VM with SIGSEGV instead of raising StackError like every other container type. Return PRIMITIVE_STACK_TOO_DEEP when the push fails so a maliciously deep recursive-extension payload raises MessagePack::StackError. This is reachable from MessagePack.unpack / Factory#load on attacker-controlled bytes. Co-Authored-By: Claude Opus 4.8 --- ext/msgpack/unpacker.c | 9 ++++++++- spec/factory_spec.rb | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/ext/msgpack/unpacker.c b/ext/msgpack/unpacker.c index b2651cb0..4b94a6bb 100644 --- a/ext/msgpack/unpacker.c +++ b/ext/msgpack/unpacker.c @@ -378,7 +378,14 @@ static inline int read_raw_body_begin(msgpack_unpacker_t* uk, int raw_type) reset_head_byte(uk); uk->reading_raw_remaining = 0; - _msgpack_unpacker_stack_push(uk, STACK_TYPE_RECURSIVE, 1, Qnil); + if(_msgpack_unpacker_stack_push(uk, STACK_TYPE_RECURSIVE, 1, Qnil) < 0) { + /* Recursive extensions re-enter msgpack_unpacker_read() through the + * user proc, consuming a full C stack frame per nesting level. If we + * ignore a failed push (stack already at MSGPACK_UNPACKER_STACK_CAPACITY) + * the recursion continues unbounded and exhausts the C stack (SIGSEGV) + * instead of raising StackError like every other container type. */ + return PRIMITIVE_STACK_TOO_DEEP; + } int raised; obj = protected_proc_call(proc, 1, &uk->self, &raised); msgpack_unpacker_stack_pop(uk); diff --git a/spec/factory_spec.rb b/spec/factory_spec.rb index 25bea66f..c089c65c 100644 --- a/spec/factory_spec.rb +++ b/spec/factory_spec.rb @@ -661,6 +661,23 @@ class << Symbol GC.stress = false end end + + it 'raises StackError instead of crashing on deeply nested recursive extensions' do + recursive_type = Struct.new(:payload) + factory = MessagePack::Factory.new + factory.register_type(0x01, + recursive_type, + packer: ->(obj, packer) { packer.write(obj.payload) }, + unpacker: ->(unpacker) { recursive_type.new(unpacker.read) }, + recursive: true, + ) + + obj = 42 + 1000.times { obj = recursive_type.new(obj) } + payload = factory.dump(obj) + + expect { factory.load(payload) }.to raise_error(MessagePack::StackError) + end end describe 'memsize' do From 752ad7f71789d6e210c160513d84dafb5a2c0a80 Mon Sep 17 00:00:00 2001 From: Shizuo Fujita Date: Mon, 6 Jul 2026 10:57:50 +0900 Subject: [PATCH 2/3] Skip recursive-extension StackError spec on JRuby The MessagePack::StackError behavior verified by this spec is specific to the C extension, which caps nesting at MSGPACK_UNPACKER_STACK_CAPACITY. The Java (JRuby) decoder has no equivalent guard and recurses until the JVM stack overflows, so the spec cannot assert MessagePack::StackError there. Co-Authored-By: Claude Opus 4.8 --- spec/factory_spec.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spec/factory_spec.rb b/spec/factory_spec.rb index c089c65c..5efb235b 100644 --- a/spec/factory_spec.rb +++ b/spec/factory_spec.rb @@ -663,6 +663,11 @@ class << Symbol end it 'raises StackError instead of crashing on deeply nested recursive extensions' do + # The C extension caps nesting at MSGPACK_UNPACKER_STACK_CAPACITY and raises + # MessagePack::StackError. The Java (JRuby) decoder has no such guard and + # recurses until the JVM stack overflows, so this behavior is CRuby-specific. + skip if IS_JRUBY + recursive_type = Struct.new(:payload) factory = MessagePack::Factory.new factory.register_type(0x01, From cf586e97741195c303ed5402c7d2494ff67eaf16 Mon Sep 17 00:00:00 2001 From: Shizuo Fujita Date: Mon, 6 Jul 2026 11:08:12 +0900 Subject: [PATCH 3/3] Build deep recursive-extension spec payload without deep recursion Packing a 200+ level nested recursive extension recurses through the packer proc once per level, which overflows the C stack on platforms with a small default stack (Windows CI raised SystemStackError during Factory#dump before the assertion could run). Construct the nested type-0x01 extension payload iteratively from the inside out instead, so only the unpack path under test exercises deep nesting. The loaded payload still nests past MSGPACK_UNPACKER_STACK_CAPACITY (200 levels) and raises MessagePack::StackError on the C extension. Co-Authored-By: Claude Opus 4.8 --- spec/factory_spec.rb | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/spec/factory_spec.rb b/spec/factory_spec.rb index 5efb235b..b8fefee7 100644 --- a/spec/factory_spec.rb +++ b/spec/factory_spec.rb @@ -677,10 +677,26 @@ class << Symbol recursive: true, ) - obj = 42 - 1000.times { obj = recursive_type.new(obj) } - payload = factory.dump(obj) + # Build the nested type 0x01 extension payload from the inside out so the + # construction itself uses no deep recursion (packing 200+ levels would + # overflow the C stack on platforms with a small stack, e.g. Windows). + wrap_ext = ->(payload) do + len = payload.bytesize + header = + if len == 1 then [0xd4, 0x01].pack("CC") + elsif len <= 0xff then [0xc7, len, 0x01].pack("CCC") + elsif len <= 0xffff then [0xc8, len, 0x01].pack("CnC") + else [0xc9, len, 0x01].pack("CNC") + end + (header + payload).b + end + + payload = "\x2a".b # a single MessagePack integer (42) + 200.times { payload = wrap_ext.call(payload) } # deeper than the 128 stack cap + # Before the fix, read_raw_body_begin ignored a failed stack push for + # recursive extensions, so nesting past MSGPACK_UNPACKER_STACK_CAPACITY + # recursed unbounded in C and crashed the VM (SIGSEGV) rather than raising. expect { factory.load(payload) }.to raise_error(MessagePack::StackError) end end