diff --git a/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 b/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 index c42f016ca9..3833abeae8 100644 --- a/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 +++ b/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64 @@ -1,14 +1,17 @@ bench_ref_from_bytes_dynamic_padding: test dil, 3 jne .LBB5_1 - movabs rax, 9223372036854775804 - and rax, rsi - cmp rax, 9 - setb al test sil, 3 - setne cl + setne al + cmp rsi, 9 + setb cl or cl, al - jne .LBB5_1 + je .LBB5_3 +.LBB5_1: + xor edi, edi + mov rax, rdi + ret +.LBB5_3: add rsi, -9 movabs rcx, -6148914691236517205 mov rax, rsi @@ -16,7 +19,3 @@ bench_ref_from_bytes_dynamic_padding: shr rdx mov rax, rdi ret -.LBB5_1: - xor edi, edi - mov rax, rdi - ret diff --git a/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca b/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca index b020ff13e1..1652c2c040 100644 --- a/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca +++ b/zerocopy/benches/ref_from_bytes_dynamic_padding.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 2000 -Total Cycles: 673 -Total uOps: 2100 +Instructions: 1800 +Total Cycles: 607 +Total uOps: 1900 Dispatch Width: 4 -uOps Per Cycle: 3.12 +uOps Per Cycle: 3.13 IPC: 2.97 -Block RThroughput: 5.3 +Block RThroughput: 4.8 Instruction Info: @@ -20,14 +20,15 @@ Instruction Info: [1] [2] [3] [4] [5] [6] Instructions: 1 1 0.33 test dil, 3 1 1 1.00 jne .LBB5_1 - 1 1 0.33 movabs rax, 9223372036854775804 - 1 1 0.33 and rax, rsi - 1 1 0.33 cmp rax, 9 - 1 1 0.50 setb al 1 1 0.33 test sil, 3 - 1 1 0.50 setne cl + 1 1 0.50 setne al + 1 1 0.33 cmp rsi, 9 + 1 1 0.50 setb cl 1 1 0.33 or cl, al - 1 1 1.00 jne .LBB5_1 + 1 1 1.00 je .LBB5_3 + 1 0 0.25 xor edi, edi + 1 1 0.33 mov rax, rdi + 1 1 1.00 U ret 1 1 0.33 add rsi, -9 1 1 0.33 movabs rcx, -6148914691236517205 1 1 0.33 mov rax, rsi @@ -35,9 +36,6 @@ Instruction Info: 1 1 0.50 shr rdx 1 1 0.33 mov rax, rdi 1 1 1.00 U ret - 1 0 0.25 xor edi, edi - 1 1 0.33 mov rax, rdi - 1 1 1.00 U ret Resources: @@ -53,27 +51,25 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 6.66 6.66 - 6.68 - - + - - 6.00 6.00 - 6.00 - - Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.66 0.33 - 0.01 - - test dil, 3 - - - - - - 1.00 - - jne .LBB5_1 - - - 0.33 0.67 - - - - movabs rax, 9223372036854775804 - - - 1.00 - - - - - and rax, rsi - - - 0.67 - - 0.33 - - cmp rax, 9 - - - 0.34 - - 0.66 - - setb al - - - - 1.00 - - - - test sil, 3 - - - 0.33 - - 0.67 - - setne cl - - - 1.00 - - - - - or cl, al + - - 0.50 0.49 - 0.01 - - test dil, 3 - - - - - 1.00 - - jne .LBB5_1 - - - - 1.00 - - - - add rsi, -9 - - - - 0.99 - 0.01 - - movabs rcx, -6148914691236517205 - - - - 0.34 - 0.66 - - mov rax, rsi + - - 0.49 0.01 - 0.50 - - test sil, 3 + - - 1.00 - - - - - setne al + - - - 1.00 - - - - cmp rsi, 9 + - - 0.51 - - 0.49 - - setb cl + - - 0.50 - - 0.50 - - or cl, al + - - - - - 1.00 - - je .LBB5_3 + - - - - - - - - xor edi, edi + - - - 0.51 - 0.49 - - mov rax, rdi + - - - - - 1.00 - - ret + - - - 0.99 - 0.01 - - add rsi, -9 + - - - 1.00 - - - - movabs rcx, -6148914691236517205 + - - 0.51 0.49 - - - - mov rax, rsi - - 1.00 1.00 - - - - mul rcx - - 1.00 - - - - - shr rdx - - - - 0.66 - 0.34 - - mov rax, rdi - - - - - - 1.00 - - ret - - - - - - - - - xor edi, edi - - - 0.33 0.67 - - - - mov rax, rdi + - - 0.49 0.51 - - - - mov rax, rdi - - - - - 1.00 - - ret diff --git a/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 b/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 index 87bf910afc..17d5f1d8e4 100644 --- a/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 +++ b/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64 @@ -5,12 +5,13 @@ bench_ref_from_bytes_dynamic_size: mov rdx, rsi cmp rsi, 4 setb cl - or sil, cl - test sil, 1 - jne .LBB5_1 - add rdx, -4 - shr rdx - ret + or cl, dl + test cl, 1 + je .LBB5_3 .LBB5_1: xor eax, eax ret +.LBB5_3: + add rdx, -4 + shr rdx + ret diff --git a/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca b/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca index 41c86a6337..bbbdaa0ab8 100644 --- a/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca +++ b/zerocopy/benches/ref_from_bytes_dynamic_size.x86-64.mca @@ -24,14 +24,14 @@ Instruction Info: 1 1 0.33 mov rdx, rsi 1 1 0.33 cmp rsi, 4 1 1 0.50 setb cl - 1 1 0.33 or sil, cl - 1 1 0.33 test sil, 1 - 1 1 1.00 jne .LBB5_1 + 1 1 0.33 or cl, dl + 1 1 0.33 test cl, 1 + 1 1 1.00 je .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret 1 1 0.33 add rdx, -4 1 1 0.50 shr rdx 1 1 1.00 U ret - 1 0 0.25 xor eax, eax - 1 1 1.00 U ret Resources: @@ -54,14 +54,14 @@ Resource pressure by instruction: - - 0.33 0.66 - 0.01 - - mov rax, rdi - - 0.67 0.33 - - - - test al, 1 - - - - - 1.00 - - jne .LBB5_1 - - - 0.66 0.34 - - - - mov rdx, rsi + - - 0.33 0.34 - 0.33 - - mov rdx, rsi - - 0.33 0.66 - 0.01 - - cmp rsi, 4 - - 1.00 - - - - - setb cl - - - - 1.00 - - - - or sil, cl - - - - 1.00 - - - - test sil, 1 - - - - - - 1.00 - - jne .LBB5_1 - - - 0.66 0.34 - - - - add rdx, -4 - - - 0.67 - - 0.33 - - shr rdx - - - - - - 1.00 - - ret + - - - 1.00 - - - - or cl, dl + - - - 1.00 - - - - test cl, 1 + - - - - - 1.00 - - je .LBB5_3 - - - - - - - - xor eax, eax - - - - - 1.00 - - ret + - - 0.66 0.34 - - - - add rdx, -4 + - - 1.00 - - - - - shr rdx + - - - - - 1.00 - - ret diff --git a/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 b/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 index 0782220491..d68f40b002 100644 --- a/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 +++ b/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64 @@ -1,15 +1,16 @@ bench_try_ref_from_bytes_dynamic_padding: - movabs rax, 9223372036854775804 - and rax, rsi - cmp rax, 9 - setae al - mov ecx, esi - or ecx, edi - test cl, 3 - sete cl - and cl, al - cmp cl, 1 + test dil, 3 jne .LBB5_1 + test sil, 3 + setne al + cmp rsi, 9 + setb cl + or cl, al + je .LBB5_3 +.LBB5_1: + xor eax, eax + ret +.LBB5_3: add rsi, -9 movabs rcx, -6148914691236517205 mov rax, rsi @@ -19,6 +20,3 @@ bench_try_ref_from_bytes_dynamic_padding: cmp word ptr [rdi], -16192 cmove rax, rdi ret -.LBB5_1: - xor eax, eax - ret diff --git a/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca b/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca index ed0586281f..e0c1ffee32 100644 --- a/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca +++ b/zerocopy/benches/try_ref_from_bytes_dynamic_padding.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 2200 -Total Cycles: 742 -Total uOps: 2500 +Instructions: 1900 +Total Cycles: 641 +Total uOps: 2200 Dispatch Width: 4 -uOps Per Cycle: 3.37 +uOps Per Cycle: 3.43 IPC: 2.96 -Block RThroughput: 6.3 +Block RThroughput: 5.5 Instruction Info: @@ -18,17 +18,16 @@ Instruction Info: [6]: HasSideEffects (U) [1] [2] [3] [4] [5] [6] Instructions: - 1 1 0.33 movabs rax, 9223372036854775804 - 1 1 0.33 and rax, rsi - 1 1 0.33 cmp rax, 9 - 1 1 0.50 setae al - 1 1 0.33 mov ecx, esi - 1 1 0.33 or ecx, edi - 1 1 0.33 test cl, 3 - 1 1 0.50 sete cl - 1 1 0.33 and cl, al - 1 1 0.33 cmp cl, 1 + 1 1 0.33 test dil, 3 1 1 1.00 jne .LBB5_1 + 1 1 0.33 test sil, 3 + 1 1 0.50 setne al + 1 1 0.33 cmp rsi, 9 + 1 1 0.50 setb cl + 1 1 0.33 or cl, al + 1 1 1.00 je .LBB5_3 + 1 0 0.25 xor eax, eax + 1 1 1.00 U ret 1 1 0.33 add rsi, -9 1 1 0.33 movabs rcx, -6148914691236517205 1 1 0.33 mov rax, rsi @@ -38,8 +37,6 @@ Instruction Info: 2 6 0.50 * cmp word ptr [rdi], -16192 2 2 0.67 cmove rax, rdi 1 1 1.00 U ret - 1 0 0.25 xor eax, eax - 1 1 1.00 U ret Resources: @@ -55,29 +52,26 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 7.32 7.33 - 7.35 0.50 0.50 + - - 6.33 6.33 - 6.34 0.50 0.50 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - - 0.99 - 0.01 - - movabs rax, 9223372036854775804 - - - 0.02 0.98 - - - - and rax, rsi - - - 0.67 0.32 - 0.01 - - cmp rax, 9 - - - 0.99 - - 0.01 - - setae al - - - 0.01 0.34 - 0.65 - - mov ecx, esi - - - 0.32 0.02 - 0.66 - - or ecx, edi - - - 0.32 0.67 - 0.01 - - test cl, 3 - - - 0.33 - - 0.67 - - sete cl - - - 0.99 - - 0.01 - - and cl, al - - - 0.98 0.01 - 0.01 - - cmp cl, 1 + - - 0.66 0.33 - 0.01 - - test dil, 3 - - - - - 1.00 - - jne .LBB5_1 - - - 0.01 0.99 - - - - add rsi, -9 - - - 0.01 0.35 - 0.64 - - movabs rcx, -6148914691236517205 - - - - 0.98 - 0.02 - - mov rax, rsi - - - 1.00 1.00 - - - - mul rcx - - - 1.00 - - - - - shr rdx + - - 0.33 0.67 - - - - test sil, 3 + - - 0.99 - - 0.01 - - setne al + - - 0.66 0.34 - - - - cmp rsi, 9 + - - 0.99 - - 0.01 - - setb cl + - - 0.33 0.33 - 0.34 - - or cl, al + - - - - - 1.00 - - je .LBB5_3 - - - - - - - - xor eax, eax - - - 0.01 0.34 - 0.65 0.50 0.50 cmp word ptr [rdi], -16192 - - - 0.66 0.34 - 1.00 - - cmove rax, rdi - - - - - 1.00 - - ret + - - - 1.00 - - - - add rsi, -9 + - - 0.33 0.66 - 0.01 - - movabs rcx, -6148914691236517205 + - - 0.33 0.67 - - - - mov rax, rsi + - - 1.00 1.00 - - - - mul rcx + - - 0.02 - - 0.98 - - shr rdx - - - - - - - - xor eax, eax + - - 0.67 0.33 - - 0.50 0.50 cmp word ptr [rdi], -16192 + - - 0.02 1.00 - 0.98 - - cmove rax, rdi - - - - - 1.00 - - ret diff --git a/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 b/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 index ab0336e767..79b1462588 100644 --- a/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 +++ b/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64 @@ -1,20 +1,17 @@ bench_try_ref_from_bytes_dynamic_size: + mov rax, rdi + test al, 1 + jne .LBB5_1 + mov rdx, rsi cmp rsi, 4 - setae al - mov ecx, esi - or ecx, edi + setb cl + or cl, dl test cl, 1 - sete cl - and cl, al - cmp cl, 1 jne .LBB5_1 - lea rdx, [rsi - 4] + cmp word ptr [rax], -16192 + jne .LBB5_1 + add rdx, -4 shr rdx - movzx ecx, word ptr [rdi] - xor eax, eax - cmp ecx, 49344 - cmovne rdx, rsi - cmove rax, rdi ret .LBB5_1: xor eax, eax diff --git a/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca b/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca index c0baeab83a..66f57a7ff9 100644 --- a/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca +++ b/zerocopy/benches/try_ref_from_bytes_dynamic_size.x86-64.mca @@ -1,12 +1,12 @@ Iterations: 100 -Instructions: 1900 -Total Cycles: 607 -Total uOps: 2100 +Instructions: 1600 +Total Cycles: 506 +Total uOps: 1700 Dispatch Width: 4 -uOps Per Cycle: 3.46 -IPC: 3.13 -Block RThroughput: 5.3 +uOps Per Cycle: 3.36 +IPC: 3.16 +Block RThroughput: 5.0 Instruction Info: @@ -18,22 +18,19 @@ Instruction Info: [6]: HasSideEffects (U) [1] [2] [3] [4] [5] [6] Instructions: + 1 1 0.33 mov rax, rdi + 1 1 0.33 test al, 1 + 1 1 1.00 jne .LBB5_1 + 1 1 0.33 mov rdx, rsi 1 1 0.33 cmp rsi, 4 - 1 1 0.50 setae al - 1 1 0.33 mov ecx, esi - 1 1 0.33 or ecx, edi + 1 1 0.50 setb cl + 1 1 0.33 or cl, dl 1 1 0.33 test cl, 1 - 1 1 0.50 sete cl - 1 1 0.33 and cl, al - 1 1 0.33 cmp cl, 1 1 1 1.00 jne .LBB5_1 - 1 1 0.50 lea rdx, [rsi - 4] + 2 6 0.50 * cmp word ptr [rax], -16192 + 1 1 1.00 jne .LBB5_1 + 1 1 0.33 add rdx, -4 1 1 0.50 shr rdx - 1 5 0.50 * movzx ecx, word ptr [rdi] - 1 0 0.25 xor eax, eax - 1 1 0.33 cmp ecx, 49344 - 2 2 0.67 cmovne rdx, rsi - 2 2 0.67 cmove rax, rdi 1 1 1.00 U ret 1 0 0.25 xor eax, eax 1 1 1.00 U ret @@ -52,26 +49,23 @@ Resources: Resource pressure per iteration: [0] [1] [2] [3] [4] [5] [6.0] [6.1] - - - 5.99 5.99 - 6.02 0.50 0.50 + - - 4.98 4.99 - 5.03 0.50 0.50 Resource pressure by instruction: [0] [1] [2] [3] [4] [5] [6.0] [6.1] Instructions: - - - 0.03 0.96 - 0.01 - - cmp rsi, 4 - - - 1.00 - - - - - setae al - - - 0.96 0.04 - - - - mov ecx, esi - - - 0.03 0.96 - 0.01 - - or ecx, edi - - - 0.03 0.96 - 0.01 - - test cl, 1 - - - 0.05 - - 0.95 - - sete cl - - - 0.01 0.03 - 0.96 - - and cl, al - - - - 0.02 - 0.98 - - cmp cl, 1 + - - 0.98 0.01 - 0.01 - - mov rax, rdi + - - 0.01 0.99 - - - - test al, 1 - - - - - 1.00 - - jne .LBB5_1 - - - 0.97 0.03 - - - - lea rdx, [rsi - 4] + - - 0.99 0.01 - - - - mov rdx, rsi + - - 0.01 0.98 - 0.01 - - cmp rsi, 4 + - - 1.00 - - - - - setb cl + - - - 1.00 - - - - or cl, dl + - - - 1.00 - - - - test cl, 1 + - - - - - 1.00 - - jne .LBB5_1 + - - 0.98 0.01 - 0.01 0.50 0.50 cmp word ptr [rax], -16192 + - - - - - 1.00 - - jne .LBB5_1 + - - 0.01 0.99 - - - - add rdx, -4 - - 1.00 - - - - - shr rdx - - - - - - - 0.50 0.50 movzx ecx, word ptr [rdi] - - - - - - - - - xor eax, eax - - - - 0.99 - 0.01 - - cmp ecx, 49344 - - - 0.96 1.00 - 0.04 - - cmovne rdx, rsi - - - 0.95 1.00 - 0.05 - - cmove rax, rdi - - - - - 1.00 - - ret - - - - - - - - xor eax, eax - - - - - 1.00 - - ret diff --git a/zerocopy/src/layout.rs b/zerocopy/src/layout.rs index a2ac64a1f2..06dbde3ae1 100644 --- a/zerocopy/src/layout.rs +++ b/zerocopy/src/layout.rs @@ -92,11 +92,13 @@ impl SizeInfo { #[cfg_attr(test, derive(Debug))] #[allow(missing_debug_implementations)] pub enum CastType { + /// Cast the entire byte range, rejecting leftover bytes. + Exact, Prefix, Suffix, } -#[cfg_attr(test, derive(Debug))] +#[cfg_attr(test, derive(Debug, PartialEq, Eq))] pub(crate) enum MetadataCastError { Alignment, Size, @@ -554,30 +556,34 @@ impl DstLayout { /// be ignored). `split_at` is the index at which to split the memory region /// in order for the prefix (suffix) to contain the result of the cast, and /// in order for the remaining suffix (prefix) to contain the leftover - /// bytes. + /// bytes. An exact cast consumes the entire region and returns + /// `split_at == bytes_len`. /// - /// There are three conditions under which a cast can fail: + /// A cast can fail if: /// - The smallest possible value for the type is larger than the provided /// memory region - /// - A prefix cast is requested, and `addr` does not satisfy `self`'s - /// alignment requirement + /// - A prefix or exact cast is requested, and `addr` does not satisfy + /// `self`'s alignment requirement /// - A suffix cast is requested, and `addr + bytes_len` does not satisfy /// `self`'s alignment requirement (as a consequence, since all instances /// of the type are a multiple of its alignment, no size for the type will /// result in a starting address which is properly aligned) + /// - An exact cast is requested, and `bytes_len` is not a valid size for + /// the type /// /// # Safety /// /// The caller may assume that this implementation is correct, and may rely /// on that assumption for the soundness of their code. In particular, the /// caller may assume that, if `validate_cast_and_convert_metadata` returns - /// `Some((elems, split_at))`, then: + /// `Ok((elems, split_at))`, then: /// - A pointer to the type (for dynamically sized types, this includes /// `elems` as its pointer metadata) describes an object of size `size <= /// bytes_len` - /// - If this is a prefix cast: + /// - If this is a prefix or exact cast: /// - `addr` satisfies `self`'s alignment /// - `size == split_at` + /// - If this is an exact cast, `size == bytes_len` /// - If this is a suffix cast: /// - `split_at == bytes_len - size` /// - `addr + split_at` satisfies `self`'s alignment @@ -646,10 +652,8 @@ impl DstLayout { // Alignment checks go in their own block to avoid introducing variables // into the top-level scope. { - // We check alignment for `addr` (for prefix casts) or `addr + - // bytes_len` (for suffix casts). For a prefix cast, the correctness - // of this check is trivial - `addr` is the address the object will - // live at. + // Prefix and exact casts place the object at `addr`. Suffix casts + // instead check alignment of `addr + bytes_len`. // // For a suffix cast, we know that all valid sizes for the type are // a multiple of the alignment (and by safety precondition, we know @@ -659,7 +663,7 @@ impl DstLayout { // address for a suffix cast (`addr + bytes_len`) is not aligned, // then no valid start address will be aligned either. let offset = match cast_type { - CastType::Prefix => 0, + CastType::Prefix | CastType::Exact => 0, CastType::Suffix => bytes_len, }; @@ -673,6 +677,51 @@ impl DstLayout { } } + if let CastType::Exact = cast_type { + match size_info { + SizeInfo::Sized { size } => { + return if bytes_len == size { + Ok((0, size)) + } else { + Err(MetadataCastError::Size) + }; + } + SizeInfo::SliceDst(TrailingSliceLayout { offset, elem_size }) => { + // An exact cast must consume an aligned number of bytes; + // there is no need to round down to find a fitting prefix. + #[allow(clippy::arithmetic_side_effects)] + if bytes_len % self.align.get() != 0 { + return Err(MetadataCastError::Size); + } + let available = match bytes_len.checked_sub(offset) { + Some(available) => available, + None => return Err(MetadataCastError::Size), + }; + + // Let N = bytes_len, O = offset, E = elem_size, and + // A = align. Division gives N - O = elems * E + r, so + // the unpadded size is N - r. Since N is a multiple of A, + // rounding N - r up to A gives N exactly when r < A. + // Otherwise, at least A bytes remain, so the cast fails. + // A larger element count exceeds N before padding; a + // smaller count cannot produce a larger padded size. + // Thus, this count is maximal, and no other count can + // make an exact cast succeed when this one fails. + // + // The checked subtraction ensures N >= O, and E is + // non-zero, so none of these operations can overflow or + // divide by zero. The returned size is N itself. + #[allow(clippy::arithmetic_side_effects)] + let elems = available / elem_size.get(); + #[allow(clippy::arithmetic_side_effects)] + if available % elem_size.get() >= self.align.get() { + return Err(MetadataCastError::Size); + } + return Ok((elems, bytes_len)); + } + } + } + let (elems, self_bytes) = match size_info { SizeInfo::Sized { size } => { if size > bytes_len { @@ -730,7 +779,7 @@ impl DstLayout { __const_debug_assert!(self_bytes <= bytes_len); let split_at = match cast_type { - CastType::Prefix => self_bytes, + CastType::Prefix | CastType::Exact => self_bytes, // Guaranteed not to underflow: // - In the `Sized` branch, only returns `size` if `size <= // bytes_len`. @@ -1432,7 +1481,7 @@ mod tests { (@generate_elem_size _) => { 1..8 }; (@generate_align _) => { [1, 2, 4, 8, 16] }; (@generate_opt_usize _) => { [None].into_iter().chain((0..8).map(Some).into_iter()) }; - (@generate_cast_type _) => { [CastType::Prefix, CastType::Suffix] }; + (@generate_cast_type _) => { [CastType::Exact, CastType::Prefix, CastType::Suffix] }; (@generate_cast_type $variant:ident) => { [CastType::$variant] }; // Some expressions need to be wrapped in parentheses in order to be // valid `tt`s (required by the top match pattern). See the comment @@ -1459,9 +1508,25 @@ mod tests { Ok(Err(MetadataCastError::Size)) ); - // addr is unaligned for prefix cast - test!(layout(_, [2]).validate(ODDS, _, Prefix), Ok(Err(MetadataCastError::Alignment))); + // The start address must be aligned for prefix and exact casts. test!(layout(_, [2]).validate(ODDS, _, Prefix), Ok(Err(MetadataCastError::Alignment))); + test!(layout(_, [2]).validate(ODDS, _, Exact), Ok(Err(MetadataCastError::Alignment))); + + // Exact casts reject both undersized and oversized buffers. + test!(layout([4], [2]).validate([0], [3, 5], Exact), Ok(Err(MetadataCastError::Size))); + test!(layout([4], [2]).validate([0], [4], Exact), Ok(Ok((0, 4)))); + test!(layout([0], [4]).validate([0], [0], Exact), Ok(Ok((0, 0)))); + test!(layout([0], [4]).validate([0], [4], Exact), Ok(Err(MetadataCastError::Size))); + + // A remainder smaller than the alignment is trailing padding. A whole + // alignment unit of unused space cannot be consumed as padding. + test!(layout(([5], [12]), [4]).validate([0], [20], Exact), Ok(Ok((1, 20)))); + test!( + layout(([4], [12]), [4]).validate([0], [20], Exact), + Ok(Err(MetadataCastError::Size)) + ); + // Small elements may also fit in the space otherwise used as padding. + test!(layout(([5], [1]), [4]).validate([0], [8], Exact), Ok(Ok((3, 8)))); // addr is aligned, but end of buffer is unaligned for suffix cast test!(layout(_, [2]).validate(EVENS, ODDS, Suffix), Ok(Err(MetadataCastError::Alignment))); @@ -1506,9 +1571,28 @@ mod tests { use core::convert::TryFrom as _; let wide = |n: usize| u128::try_from(n).unwrap(); - if let Ok((elems, split_at)) = - layout.validate_cast_and_convert_metadata(addr, bytes_len, cast_type) - { + let result = layout.validate_cast_and_convert_metadata(addr, bytes_len, cast_type); + if let CastType::Exact = cast_type { + // Check completeness and error precedence against a prefix + // cast followed by a leftover check. The exact path uses a + // remainder test instead of reconstructing the padded size. + let expected = match layout.validate_cast_and_convert_metadata( + addr, + bytes_len, + CastType::Prefix, + ) { + Ok((elems, split_at)) => { + if split_at == bytes_len { + Ok((elems, split_at)) + } else { + Err(MetadataCastError::Size) + } + } + Err(err) => Err(err), + }; + assert_eq!(result, expected, "{:?}, {}, {}", layout, addr, bytes_len); + } + if let Ok((elems, split_at)) = result { let (size_info, align) = (layout.size_info, layout.align); let debug_str = format!( "layout({:?}, {}).validate_cast_and_convert_metadata({}, {}, {:?}) => ({}, {})", @@ -1546,9 +1630,12 @@ mod tests { // `validate_cast_and_convert_metadata`. assert!(resulting_size <= wide(bytes_len), "{}", debug_str); match cast_type { - CastType::Prefix => { + CastType::Prefix | CastType::Exact => { assert_eq!(addr % align, 0, "{}", debug_str); assert_eq!(resulting_size, wide(split_at), "{}", debug_str); + if let CastType::Exact = cast_type { + assert_eq!(split_at, bytes_len, "{}", debug_str); + } } CastType::Suffix => { assert_eq!( @@ -1561,6 +1648,11 @@ mod tests { } } } else { + if let CastType::Exact = cast_type { + // Failure, including a size mismatch, was checked against + // the prefix cast above. + return; + } let min_size = match layout.size_info { SizeInfo::Sized { size } => size, SizeInfo::SliceDst(TrailingSliceLayout { offset, .. }) => { @@ -1573,7 +1665,7 @@ mod tests { let insufficient_bytes = bytes_len < min_size; // 2. performing the cast would misalign type: let base = match cast_type { - CastType::Prefix => 0, + CastType::Prefix | CastType::Exact => 0, CastType::Suffix => bytes_len, }; let misaligned = (base + addr) % layout.align != 0; @@ -1591,8 +1683,13 @@ mod tests { let layouts = itertools::iproduct!(size_infos, [1, 2, 4, 8, 16, 32]) .filter(|(size_info, align)| !matches!(size_info, SizeInfo::Sized { size } if size % align != 0)) .map(|(size_info, align)| layout(size_info, align)); - itertools::iproduct!(layouts, 0..8, 0..8, [CastType::Prefix, CastType::Suffix]) - .for_each(validate_behavior); + itertools::iproduct!( + layouts, + 0..8, + 0..8, + [CastType::Exact, CastType::Prefix, CastType::Suffix] + ) + .for_each(validate_behavior); // Exercise remainders below, at, and above the alignment, as well as // sizes near integer limits. No allocation is needed: this method @@ -1625,7 +1722,7 @@ mod tests { layouts, [0usize, 1, 31], lengths, - [CastType::Prefix, CastType::Suffix] + [CastType::Exact, CastType::Prefix, CastType::Suffix] ) .filter(|(_, addr, len, _)| addr.checked_add(*len).is_some()) .for_each(validate_behavior); diff --git a/zerocopy/src/lib.rs b/zerocopy/src/lib.rs index 944f177c0b..1b919737bd 100644 --- a/zerocopy/src/lib.rs +++ b/zerocopy/src/lib.rs @@ -7996,6 +7996,7 @@ mod tests { for cast_type in [CastType::Prefix, CastType::Suffix] { let (expected, expected_rest) = match cast_type { + CastType::Exact => unreachable!(), CastType::Prefix => (&bytes[..3], &bytes[3..]), CastType::Suffix => (&bytes[1..], &bytes[..1]), }; @@ -8030,6 +8031,7 @@ mod tests { let storage = Align::<[bool; 9], AU64>::new([false; 9]); for cast_type in [CastType::Prefix, CastType::Suffix] { let source = match cast_type { + CastType::Exact => unreachable!(), CastType::Prefix => &storage.t[1..], CastType::Suffix => &storage.t[..], }; @@ -8057,6 +8059,7 @@ mod tests { for cast_type in [CastType::Prefix, CastType::Suffix] { let (expected, expected_rest) = match cast_type { + CastType::Exact => unreachable!(), CastType::Prefix => (&bytes[..3], &bytes[3..]), CastType::Suffix => (&bytes[1..], &bytes[..1]), }; diff --git a/zerocopy/src/pointer/inner.rs b/zerocopy/src/pointer/inner.rs index 023df07d04..19982dc0ec 100644 --- a/zerocopy/src/pointer/inner.rs +++ b/zerocopy/src/pointer/inner.rs @@ -544,21 +544,27 @@ impl<'a> PtrInner<'a, [u8]> { /// the cast will only succeed if it would produce an object with the given /// metadata. /// - /// Returns `None` if the resulting `U` would be invalidly-aligned, if no + /// Returns `Err` if the resulting `U` would be invalidly-aligned, if no /// `U` can fit in `self`, or if the provided pointer metadata describes an - /// invalid instance of `U`. On success, returns a pointer to the - /// largest-possible `U` which fits in `self`. + /// invalid instance of `U`. If metadata is not provided, returns a pointer + /// to the largest-possible `U` which fits in `self`. An exact cast also + /// fails if the resulting `U` would not consume all of `self`. /// /// # Safety /// /// The caller may assume that this implementation is correct, and may rely /// on that assumption for the soundness of their code. In particular, the - /// caller may assume that, if `try_cast_into` returns `Some((ptr, + /// caller may assume that, if `try_cast_into` returns `Ok((ptr, /// remainder))`, then `ptr` and `remainder` refer to non-overlapping byte /// ranges within `self`, and that `ptr` and `remainder` entirely cover /// `self`. Finally: - /// - If this is a prefix cast, `ptr` has the same address as `self`. + /// - If this is a prefix or exact cast, `ptr` has the same address as + /// `self`. /// - If this is a suffix cast, `remainder` has the same address as `self`. + /// - If this is an exact cast, `ptr` covers all of `self` and `remainder` + /// is empty. + /// + /// On error, the returned error contains `self`. #[inline] pub fn try_cast_into( self, @@ -603,7 +609,7 @@ impl<'a> PtrInner<'a, [u8]> { let (l_slice, r_slice) = unsafe { self.split_at_unchecked(split_at) }; let (target, remainder) = match cast_type { - CastType::Prefix => (l_slice, r_slice), + CastType::Prefix | CastType::Exact => (l_slice, r_slice), CastType::Suffix => (r_slice, l_slice), }; diff --git a/zerocopy/src/pointer/ptr.rs b/zerocopy/src/pointer/ptr.rs index dcf47b29dd..53327d433f 100644 --- a/zerocopy/src/pointer/ptr.rs +++ b/zerocopy/src/pointer/ptr.rs @@ -21,7 +21,7 @@ use crate::{ invariant::*, transmute::{MutationCompatible, SizeEq, TransmuteFromPtr}, }, - AlignmentError, CastError, CastType, KnownLayout, SizeError, TryFromBytes, ValidityError, + AlignmentError, CastError, CastType, KnownLayout, TryFromBytes, ValidityError, }; /// Module used to gate access to [`Ptr`]'s fields. @@ -1115,22 +1115,29 @@ mod _casts { /// then the cast will only succeed if it would produce an object with /// the given metadata. /// - /// Returns `None` if the resulting `U` would be invalidly-aligned, if + /// Returns `Err` if the resulting `U` would be invalidly-aligned, if /// no `U` can fit in `self`, or if the provided pointer metadata - /// describes an invalid instance of `U`. On success, returns a pointer - /// to the largest-possible `U` which fits in `self`. + /// describes an invalid instance of `U`. If metadata is not provided, + /// returns a pointer to the largest-possible `U` which fits in `self`. + /// An exact cast also fails if the resulting `U` would not consume all + /// of `self`. /// /// # Safety /// /// The caller may assume that this implementation is correct, and may /// rely on that assumption for the soundness of their code. In /// particular, the caller may assume that, if `try_cast_into` returns - /// `Some((ptr, remainder))`, then `ptr` and `remainder` refer to + /// `Ok((ptr, remainder))`, then `ptr` and `remainder` refer to /// non-overlapping byte ranges within `self`, and that `ptr` and /// `remainder` entirely cover `self`. Finally: - /// - If this is a prefix cast, `ptr` has the same address as `self`. + /// - If this is a prefix or exact cast, `ptr` has the same address as + /// `self`. /// - If this is a suffix cast, `remainder` has the same address as /// `self`. + /// - If this is an exact cast, `ptr` covers all of `self` and + /// `remainder` is empty. + /// + /// On error, the returned error contains `self`. #[inline(always)] pub fn try_cast_into( self, @@ -1213,24 +1220,9 @@ mod _casts { U: 'a + ?Sized + KnownLayout + Read, [u8]: Read, { - // SAFETY: The provided closure returns the only copy of `slf`. - unsafe { - self.try_with_unchecked( - #[inline(always)] - |slf| match slf.try_cast_into(CastType::Prefix, meta) { - Ok((slf, remainder)) => { - if remainder.is_empty() { - Ok(slf) - } else { - Err(CastError::Size(SizeError::<_, U>::new(()))) - } - } - Err(err) => Err(err.map_src( - #[inline(always)] - |_slf| (), - )), - }, - ) + match self.try_cast_into(CastType::Exact, meta) { + Ok((ptr, _)) => Ok(ptr), + Err(err) => Err(err), } } } @@ -1462,7 +1454,7 @@ mod tests { } for meta in metas.clone().into_iter() { - for cast_type in [CastType::Prefix, CastType::Suffix] { + for cast_type in [CastType::Exact, CastType::Prefix, CastType::Suffix] { if let Ok((slf, remaining)) = Ptr::from_ref(bytes) .try_cast_into::(cast_type, meta) { @@ -1475,12 +1467,17 @@ mod tests { #[allow(unstable_name_collisions)] let remaining_addr = remaining.as_inner().as_ptr().addr(); match cast_type { - CastType::Prefix => { + CastType::Prefix | CastType::Exact => { assert_eq!(remaining_addr, bytes_addr + len) } CastType::Suffix => assert_eq!(remaining_addr, bytes_addr), } + if let CastType::Exact = cast_type { + assert_eq!(len, bytes.len()); + assert!(remaining.is_empty()); + } + if let Some(want) = meta { let got = KnownLayout::pointer_to_metadata(slf.as_inner().as_ptr()); @@ -1565,7 +1562,8 @@ mod tests { let ptr = Ptr::from_ref(&bytes[..]); let res = ptr.try_cast_into::<$ty, BecauseImmutable>(CastType::Prefix, Some($elems)); - if let Some(expect) = $expect { + let expect: Option<<$ty as KnownLayout>::PointerMetadata> = $expect; + if let Some(expect) = expect { let (ptr, _) = res.unwrap(); assert_eq!(KnownLayout::pointer_to_metadata(ptr.as_inner().as_ptr()), expect); } else { @@ -1606,6 +1604,57 @@ mod tests { test!(Dst, 8, usize::MAX - 8 + 1, None); } + #[test] + fn test_try_cast_into_no_leftover_explicit_count() { + #[derive(KnownLayout, Immutable)] + #[repr(C, align(4))] + struct PaddedDst { + header: [u8; 9], + tail: [[u8; 3]], + } + + #[derive(KnownLayout, Immutable)] + #[repr(C, align(4))] + struct ZstDst { + header: [u8; 9], + tail: [()], + } + + let storage = crate::util::testutil::Align::<_, AU64>::new([0u8; 16]); + let bytes = &storage.t[..12]; + // Both counts have size 12 after padding. Preserve the requested + // count even when a larger count would also fit. + for count in [0, 1] { + let ptr = Ptr::from_ref(bytes) + .try_cast_into_no_leftover::(Some(count)) + .unwrap(); + assert_eq!(ptr.len(), count); + } + // Explicit metadata permits ZST elements and must not enter the + // inferred-count calculation, which rejects such elements. + for count in [0, usize::MAX] { + let ptr = Ptr::from_ref(bytes) + .try_cast_into_no_leftover::(Some(count)) + .unwrap(); + assert_eq!(ptr.len(), count); + } + + // Wrong sizes and overflowing counts must return the original slice. + for (bytes, count) in [ + (&storage.t[..8], 0), + (&storage.t[..16], 1), + (&storage.t[..12], 2), + (&storage.t[..12], usize::MAX), + ] { + let err = Ptr::from_ref(bytes) + .try_cast_into_no_leftover::(Some(count)) + .unwrap_err(); + assert!(matches!(err, CastError::Size(_))); + let recovered = err.into_src().as_ref(); + assert!(core::ptr::eq(recovered, bytes)); + } + } + #[test] fn test_try_cast_into_no_leftover_restores_original_slice() { let bytes = [0u8; 4];