From 9c474204f16bbb3940174ef8be51812d9698b3a1 Mon Sep 17 00:00:00 2001 From: Ossian O'Reilly Date: Thu, 13 Aug 2026 15:56:58 -0700 Subject: [PATCH 1/7] Add bitmap-significance-split coder with GPU-optimized two-level kernel Introduces kernel 2 (coding) for the bitmap-significance-split encode: flat per-block fixed-width, per-line width, and two-level occupancy variants, plus validation/CR/throughput tests (test_bitmap_code_hip) and kernel-1 validation (test_bitmap_encode_hip, test_bitmap_vs_cpu_rle) with build targets. Adds waveletBitmapCodeTwoLevelOptKernel: a byte-for-byte-identical, faster two-level coder found by a polyopt LLM-agent campaign. Levers: 1024 threads (one z-line/thread), wave64 exclusive scans as DPP row shifts (no ds_bpermute), the two width-independent scans fused into one dual scan, a branch-free per-width group packer, values staged in an LDS image drained with 16-byte non-temporal stores, and single-ballot occupancy writes. Validated on gfx950 (MI355x): output is byte-exact vs the reference two-level kernel across sizes and quantization; the coding kernel is ~2.5-5.3x faster (3.7x at 256^3, 0.054 -> 0.014 ms). Uses ~132 KB LDS so it targets the larger LDS on gfx950; the original waveletBitmapCodeTwoLevelKernel remains the portable path for gfx90a. Build the gfx950 kernel with HIP_ARCH=gfx950. --- hip/hipWaveletBitmap.h | 785 +++++++++++++++++++++++++++++++ makefile | 12 + tests/test_bitmap_code_hip.cpp | 316 +++++++++++++ tests/test_bitmap_encode_hip.cpp | 187 ++++++++ tests/test_bitmap_vs_cpu_rle.cpp | 322 +++++++++++++ 5 files changed, 1622 insertions(+) create mode 100644 hip/hipWaveletBitmap.h create mode 100644 tests/test_bitmap_code_hip.cpp create mode 100644 tests/test_bitmap_encode_hip.cpp create mode 100644 tests/test_bitmap_vs_cpu_rle.cpp diff --git a/hip/hipWaveletBitmap.h b/hip/hipWaveletBitmap.h new file mode 100644 index 0000000..996bb9c --- /dev/null +++ b/hip/hipWaveletBitmap.h @@ -0,0 +1,785 @@ +// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Use of this source code is governed by an MIT-style license that can be +// found in the LICENSE file or at https://opensource.org/licenses/MIT. + +#ifndef HIPWAVELET_BITMAP_H +#define HIPWAVELET_BITMAP_H + +// PROTOTYPE: significance-map (bitmap) + packed-value split of the fused +// wavelet+quantize encode. This is kernel 1 of a two-kernel design: +// +// kernel 1 (this file): wavelet ZYX + quantize -> per block: +// [ 4096 B bitmap : 1024 x uint32, one significance word per z-line ] +// [ packed int32 nonzero quantized values, in (x_off,tid,z) order ] +// block_sizes[bid] = 4096 + nnz(block)*4 +// per-z-line nnz = __popc(bitmap word) (no separate metadata needed) +// +// kernel 2 (coding, separate): consumes bitmap + packed values and emits +// the final coded stream (fixed-width / width-class / bit-plane). +// +// Phases 1-3 (load, Z-transform, Y+X transform) are identical to +// waveletRLEFusedKernel in hipWaveletRLE.h, so the quantized nonzero set and +// values are bit-for-bit identical to the RLE path (used for validation). + +#include +#include +#include "ds79.h" +#include "hipWaveletRLE.h" // wrle_float4_vec, WRLE_LDS_BYTES, ds79 helpers + +// One significance word (32 bits, one per z) per z-line; 4*256 z-lines/block. +static constexpr int WBMP_BITMAP_WORDS = 1024; +static constexpr int WBMP_BITMAP_BYTES = WBMP_BITMAP_WORDS * 4; // 4096 +// Worst case (all 32768 coefficients nonzero) value region. +static constexpr int WBMP_MAX_VAL_BYTES = 32768 * 4; // 131072 +static constexpr long WBMP_SLOT_BYTES = WBMP_BITMAP_BYTES + WBMP_MAX_VAL_BYTES; // 135168 + +__launch_bounds__(256, 2) +__global__ void waveletBitmapFusedKernel( + const float* __restrict__ input, + unsigned char* __restrict__ output, + size_t* __restrict__ block_sizes, + float scale, + int ldimx, int ldimxy, + const double* __restrict__ d_rms, + float* __restrict__ d_mulfac_out) +{ + constexpr int PLANES = 32; + constexpr int BATCH = 8; + constexpr int SLC = 2; + constexpr int NTHREADS = 256; + using BlockScan = rocprim::block_scan; + + __shared__ union { + float wavelet[BATCH * 1024]; + typename BlockScan::storage_type scan; + } lds; + + int tid = threadIdx.x; + int xg = tid % 8; + int yr = tid / 8; + + float mulfac; + if (d_rms != nullptr) { + float rms = (float)*d_rms; + float product = rms * scale; + mulfac = (product > 0.0f && __builtin_isfinite(1.0f / product)) + ? (1.0f / product) : 1.0f; + if (tid == 0 && blockIdx.x == 0 && blockIdx.y == 0 && blockIdx.z == 0) { + if (d_mulfac_out) *d_mulfac_out = mulfac; + } + } else { + mulfac = scale; + } + + const float* block_base = input + (size_t)blockIdx.z * 32 * ldimxy; + + int gx = blockIdx.x * 32 + xg * 4; + int gy = blockIdx.y * 32 + yr; + uint32_t byte_off = (gx + gy * ldimx) * (uint32_t)sizeof(float); + + // ---- Phase 1: Load 32 planes from global ---- + wrle_float4_vec regs[PLANES]; + #pragma unroll + for (int p = 0; p < PLANES; p++) { + auto rsrc = __builtin_amdgcn_make_buffer_rsrc( + const_cast(block_base + (long)p * ldimxy), + 0, -1, 0x00027000); + regs[p] = __builtin_bit_cast(wrle_float4_vec, + __builtin_amdgcn_raw_buffer_load_b128(rsrc, byte_off, 0, SLC)); + } + + // ---- Phase 2: Z-transform in registers ---- + ds79_forward_f4_scalar_tmp(regs, PLANES); + + // ---- Phase 3: Y+X transform in LDS (batches of 8) ---- + for (int pb = 0; pb < PLANES; pb += BATCH) { + for (int dp = 0; dp < BATCH; dp++) { + wrle_float4_vec v = regs[pb + dp]; + int x0 = xg * 4; + lds.wavelet[dp * 1024 + (x0+0) * 32 + (yr ^ (x0+0))] = v[0]; + lds.wavelet[dp * 1024 + (x0+1) * 32 + (yr ^ (x0+1))] = v[1]; + lds.wavelet[dp * 1024 + (x0+2) * 32 + (yr ^ (x0+2))] = v[2]; + lds.wavelet[dp * 1024 + (x0+3) * 32 + (yr ^ (x0+3))] = v[3]; + } + __syncthreads(); + + int pl = tid / 32; + int pos = tid % 32; + + float line[32]; + for (int y = 0; y < 32; y++) + line[y] = lds.wavelet[pl * 1024 + pos * 32 + (y ^ pos)]; + ds79_forward_reg32(line); + for (int y = 0; y < 32; y++) + lds.wavelet[pl * 1024 + pos * 32 + (y ^ pos)] = line[y]; + __syncthreads(); + + for (int x = 0; x < 32; x++) + line[x] = lds.wavelet[pl * 1024 + x * 32 + (pos ^ x)]; + ds79_forward_reg32(line); + for (int x = 0; x < 32; x++) + lds.wavelet[pl * 1024 + x * 32 + (pos ^ x)] = line[x]; + __syncthreads(); + + for (int dp = 0; dp < BATCH; dp++) { + wrle_float4_vec v; + int x0 = xg * 4; + v[0] = lds.wavelet[dp * 1024 + (x0+0) * 32 + (yr ^ (x0+0))]; + v[1] = lds.wavelet[dp * 1024 + (x0+1) * 32 + (yr ^ (x0+1))]; + v[2] = lds.wavelet[dp * 1024 + (x0+2) * 32 + (yr ^ (x0+2))]; + v[3] = lds.wavelet[dp * 1024 + (x0+3) * 32 + (yr ^ (x0+3))]; + regs[pb + dp] = v; + } + __syncthreads(); + } + + // ---- Phase 4: Quantize -> bitmap + packed nonzero values ---- + // Block layout: [4096B bitmap] [packed int32 values] + int bid = blockIdx.x + blockIdx.y * gridDim.x + blockIdx.z * gridDim.x * gridDim.y; + unsigned char* block_out = output + (long)bid * WBMP_SLOT_BYTES; + uint32_t* bitmap_out = reinterpret_cast(block_out); + int32_t* val_out = reinterpret_cast(block_out + WBMP_BITMAP_BYTES); + + int block_val_base = 0; + for (int x_off = 0; x_off < 4; ++x_off) { + // Build 32-bit significance mask for this z-line. + uint32_t mask = 0; + #pragma unroll + for (int z = 0; z < 32; ++z) { + int ival = (int)(mulfac * regs[z][x_off]); + if (ival != 0) mask |= (1u << z); + } + int nnz = __popc(mask); + bitmap_out[x_off * 256 + tid] = mask; + + // Value-stream offset = prefix sum of per-z-line nnz across the block. + int my_off, pass_total; + BlockScan().exclusive_scan(nnz, my_off, 0, pass_total, lds.scan); + __syncthreads(); + + // Scatter this z-line's nonzero values into the packed region, in + // z order. rank(z) = popcount of set bits below z. + int base = block_val_base + my_off; + #pragma unroll + for (int z = 0; z < 32; ++z) { + if (mask & (1u << z)) { + int ival = (int)(mulfac * regs[z][x_off]); + int rank = __popc(mask & ((1u << z) - 1)); + val_out[base + rank] = ival; + } + } + + block_val_base += pass_total; + __syncthreads(); + } + + if (tid == 0) + block_sizes[bid] = (size_t)WBMP_BITMAP_BYTES + (size_t)block_val_base * 4; +} + +// Coded block stride: [4096B bitmap][4B width header][nnz * W bytes], W<=4. +static constexpr long WBMP_CODE_SLOT_BYTES = + WBMP_BITMAP_BYTES + 8 + WBMP_MAX_VAL_BYTES; // 135176 + +// Minimum signed byte width to hold [-maxabs, maxabs]. +__host__ __device__ __forceinline__ int wbmp_width_bytes(int maxabs) { + if (maxabs <= 0x7f) return 1; + if (maxabs <= 0x7fff) return 2; + if (maxabs <= 0x7fffff) return 3; + return 4; +} + +// --------------------------------------------------------------------------- +// Kernel 2 (coding): per-block fixed-width packing of the value stream. +// Reads kernel-1 output [bitmap + packed int32]; writes [bitmap + W + W-byte +// values]. One workgroup per block. Bitmap is copied verbatim. +// coded block bytes = 4096 + 4 + nnz*W +// --------------------------------------------------------------------------- +__launch_bounds__(256, 4) +__global__ void waveletBitmapCodeKernel( + const unsigned char* __restrict__ scratch1, + const size_t* __restrict__ block_sizes1, + unsigned char* __restrict__ out, + size_t* __restrict__ block_sizes2) +{ + int bid = blockIdx.x; + int tid = threadIdx.x; + + const unsigned char* blk_in = scratch1 + (long)bid * WBMP_SLOT_BYTES; + const uint32_t* bmp_in = reinterpret_cast(blk_in); + const int32_t* vals_in = reinterpret_cast(blk_in + WBMP_BITMAP_BYTES); + int nnz = (int)((block_sizes1[bid] - WBMP_BITMAP_BYTES) / 4); + + unsigned char* blk_out = out + (long)bid * WBMP_CODE_SLOT_BYTES; + uint32_t* bmp_out = reinterpret_cast(blk_out); + + __shared__ int s_max; + if (tid == 0) s_max = 0; + __syncthreads(); + + int local = 0; + for (int i = tid; i < nnz; i += 256) { + int v = vals_in[i]; + int a = v < 0 ? -v : v; + if (a > local) local = a; + } + atomicMax(&s_max, local); + + // Copy bitmap verbatim (coalesced) while the reduction settles. + for (int i = tid; i < WBMP_BITMAP_WORDS; i += 256) + bmp_out[i] = bmp_in[i]; + __syncthreads(); + + int W = wbmp_width_bytes(s_max); + + if (tid == 0) { + reinterpret_cast(blk_out + WBMP_BITMAP_BYTES)[0] = W; + block_sizes2[bid] = (size_t)WBMP_BITMAP_BYTES + 4 + (size_t)nnz * W; + } + + unsigned char* vout = blk_out + WBMP_BITMAP_BYTES + 4; + for (int i = tid; i < nnz; i += 256) { + unsigned uv = (unsigned)vals_in[i]; + long base = (long)i * W; + #pragma unroll + for (int b = 0; b < 4; ++b) + if (b < W) vout[base + b] = (unsigned char)(uv >> (8 * b)); + } +} + +inline hipError_t hipWaveletBitmapCode( + const unsigned char* scratch1, + const size_t* block_sizes1, + unsigned char* out, + size_t* block_sizes2, + int nblocks, + hipStream_t stream = 0) +{ + waveletBitmapCodeKernel<<>>( + scratch1, block_sizes1, out, block_sizes2); + return hipGetLastError(); +} + +// --------------------------------------------------------------------------- +// Kernel 2 (coding), per-LINE width: one width per z-line instead of per +// block, eliminating the block-wide padding waste. Width table is 2 bits +// per z-line (code = W-1), 1024 lines -> 256 B/block. +// Coded block: [4096B bitmap][256B width table][packed variable-width values] +// bytes = 4096 + 256 + sum_line(nnz_line * W_line) +// Each thread persistently owns 4 z-lines (x_off=0..3, fixed tid), so per-line +// nnz/W/offsets stay in registers; two block-scans give input(nnz) and output +// (nnz*W) prefix sums. +// --------------------------------------------------------------------------- +static constexpr int WBMP_WTAB_BYTES = 256; // 2 bits * 1024 lines +static constexpr long WBMP_PL_SLOT_BYTES = + WBMP_BITMAP_BYTES + WBMP_WTAB_BYTES + WBMP_MAX_VAL_BYTES; + +__device__ __forceinline__ int wbmp_line_width(const int32_t* v, int n) { + int mx = 0; + for (int k = 0; k < n; ++k) { int a = v[k]; a = a < 0 ? -a : a; if (a > mx) mx = a; } + return wbmp_width_bytes(mx); +} + +__launch_bounds__(256, 4) +__global__ void waveletBitmapCodePerLineKernel( + const unsigned char* __restrict__ scratch1, + const size_t* __restrict__ block_sizes1, + unsigned char* __restrict__ out, + size_t* __restrict__ block_sizes2) +{ + constexpr int NTHREADS = 256; + using BlockScan = rocprim::block_scan; + __shared__ typename BlockScan::storage_type scan; + + int bid = blockIdx.x, tid = threadIdx.x; + const unsigned char* blk_in = scratch1 + (long)bid * WBMP_SLOT_BYTES; + const uint32_t* bmp_in = reinterpret_cast(blk_in); + const int32_t* vals_in = reinterpret_cast(blk_in + WBMP_BITMAP_BYTES); + + unsigned char* blk_out = out + (long)bid * WBMP_PL_SLOT_BYTES; + uint32_t* bmp_out = reinterpret_cast(blk_out); + uint32_t* wtab = reinterpret_cast(blk_out + WBMP_BITMAP_BYTES); + unsigned char* vout = blk_out + WBMP_BITMAP_BYTES + WBMP_WTAB_BYTES; + + for (int i = tid; i < WBMP_BITMAP_WORDS; i += NTHREADS) bmp_out[i] = bmp_in[i]; + for (int i = tid; i < WBMP_WTAB_BYTES / 4; i += NTHREADS) wtab[i] = 0; + __syncthreads(); + + int nnz[4], W[4], in_off[4], out_off[4]; + int in_base = 0, out_base = 0; + + // Phase 1: per-line nnz, input offsets (scan over nnz), per-line width. + for (int x = 0; x < 4; ++x) { + int nz = __popc(bmp_in[x * 256 + tid]); + nnz[x] = nz; + int off, tot; + BlockScan().exclusive_scan(nz, off, 0, tot, scan); + __syncthreads(); + in_off[x] = in_base + off; + in_base += tot; + W[x] = nz ? wbmp_line_width(vals_in + in_off[x], nz) : 1; + } + // Phase 2: output byte offsets (scan over nnz*W). + for (int x = 0; x < 4; ++x) { + int ob = nnz[x] * W[x]; + int off, tot; + BlockScan().exclusive_scan(ob, off, 0, tot, scan); + __syncthreads(); + out_off[x] = out_base + off; + out_base += tot; + } + // Phase 3: write 2-bit width table + pack values at per-line width. + for (int x = 0; x < 4; ++x) { + int L = x * 256 + tid; + if (nnz[x] > 0) + atomicOr(&wtab[L >> 4], (uint32_t)(W[x] - 1) << ((L & 15) * 2)); + long base = out_off[x]; + for (int k = 0; k < nnz[x]; ++k) { + unsigned uv = (unsigned)vals_in[in_off[x] + k]; + long p = base + (long)k * W[x]; + #pragma unroll + for (int b = 0; b < 4; ++b) + if (b < W[x]) vout[p + b] = (unsigned char)(uv >> (8 * b)); + } + } + if (tid == 0) + block_sizes2[bid] = (size_t)WBMP_BITMAP_BYTES + WBMP_WTAB_BYTES + (size_t)out_base; +} + +inline hipError_t hipWaveletBitmapCodePerLine( + const unsigned char* scratch1, + const size_t* block_sizes1, + unsigned char* out, + size_t* block_sizes2, + int nblocks, + hipStream_t stream = 0) +{ + waveletBitmapCodePerLineKernel<<>>( + scratch1, block_sizes1, out, block_sizes2); + return hipGetLastError(); +} + +// --------------------------------------------------------------------------- +// Kernel 2 (coding), TWO-LEVEL occupancy + per-line width. Replaces the flat +// 4096 B/block significance bitmap with a 128 B line-occupancy mask (1 bit per +// z-line) followed by only the nonempty lines' data. All region bases are +// recoverable at decode from popcount(occupancy). +// Coded block: +// [128B occupancy][4B * n_ne masks][2b * n_ne widths][per-line values] +// n_ne = popcount(occupancy) +// bytes = 128 + 4*n_ne + ceil(2*n_ne/8) + sum_line(nnz*W) +// --------------------------------------------------------------------------- +static constexpr int WBMP_OCC_BYTES = 128; // 1024 bits, 1 per z-line +static constexpr long WBMP_TL_SLOT_BYTES = + WBMP_OCC_BYTES + WBMP_BITMAP_BYTES + WBMP_WTAB_BYTES + WBMP_MAX_VAL_BYTES; + +__launch_bounds__(256, 4) +__global__ void waveletBitmapCodeTwoLevelKernel( + const unsigned char* __restrict__ scratch1, + const size_t* __restrict__ block_sizes1, + unsigned char* __restrict__ out, + size_t* __restrict__ block_sizes2) +{ + constexpr int NTHREADS = 256; + using BlockScan = rocprim::block_scan; + __shared__ typename BlockScan::storage_type scan; + + int bid = blockIdx.x, tid = threadIdx.x; + const unsigned char* blk_in = scratch1 + (long)bid * WBMP_SLOT_BYTES; + const uint32_t* bmp_in = reinterpret_cast(blk_in); + const int32_t* vals_in = reinterpret_cast(blk_in + WBMP_BITMAP_BYTES); + + unsigned char* blk_out = out + (long)bid * WBMP_TL_SLOT_BYTES; + uint32_t* occ = reinterpret_cast(blk_out); // 32 words + + for (int i = tid; i < WBMP_OCC_BYTES / 4; i += NTHREADS) occ[i] = 0; + __syncthreads(); + + int mask[4], nnz[4], W[4], occb[4], in_off[4], occ_rank[4], val_off[4]; + + // Phase 1: mask, nnz, occupancy, input offsets (scan nnz), per-line width. + int in_base = 0; + for (int x = 0; x < 4; ++x) { + uint32_t m = bmp_in[x * 256 + tid]; + mask[x] = (int)m; + int nz = __popc(m); + nnz[x] = nz; + occb[x] = m ? 1 : 0; + int off, tot; + BlockScan().exclusive_scan(nz, off, 0, tot, scan); + __syncthreads(); + in_off[x] = in_base + off; + in_base += tot; + W[x] = nz ? wbmp_line_width(vals_in + in_off[x], nz) : 1; + } + // Phase 2a: nonempty rank (scan occupancy) -> total_ne. + int ne_base = 0; + for (int x = 0; x < 4; ++x) { + int off, tot; + BlockScan().exclusive_scan(occb[x], off, 0, tot, scan); + __syncthreads(); + occ_rank[x] = ne_base + off; + ne_base += tot; + } + int total_ne = ne_base; + // Phase 2b: value byte offsets (scan nnz*W). + int vb_base = 0; + for (int x = 0; x < 4; ++x) { + int ob = nnz[x] * W[x]; + int off, tot; + BlockScan().exclusive_scan(ob, off, 0, tot, scan); + __syncthreads(); + val_off[x] = vb_base + off; + vb_base += tot; + } + int total_val = vb_base; + + long masks_base = WBMP_OCC_BYTES; // 128, 4-aligned + long wtab_base = masks_base + 4L * total_ne; // 4-aligned + long vals_base = wtab_base + (2L * total_ne + 7) / 8; + uint32_t* masks = reinterpret_cast(blk_out + masks_base); + uint32_t* wtab = reinterpret_cast(blk_out + wtab_base); + + // Zero the width-table words (2 bits per nonempty line). + int wtab_words = (2 * total_ne + 31) / 32; + for (int i = tid; i < wtab_words; i += NTHREADS) wtab[i] = 0; + __syncthreads(); + + // Phase 3: write occupancy, masks, widths, packed values (nonempty lines). + for (int x = 0; x < 4; ++x) { + if (!occb[x]) continue; + int L = x * 256 + tid; + atomicOr(&occ[L >> 5], 1u << (L & 31)); + int r = occ_rank[x]; + masks[r] = (uint32_t)mask[x]; + atomicOr(&wtab[r >> 4], (uint32_t)(W[x] - 1) << ((r & 15) * 2)); + long p = vals_base + val_off[x]; + for (int k = 0; k < nnz[x]; ++k) { + unsigned uv = (unsigned)vals_in[in_off[x] + k]; + long q = p + (long)k * W[x]; + #pragma unroll + for (int b = 0; b < 4; ++b) + if (b < W[x]) blk_out[q + b] = (unsigned char)(uv >> (8 * b)); + } + } + if (tid == 0) block_sizes2[bid] = (size_t)(vals_base + total_val); +} + +inline hipError_t hipWaveletBitmapCodeTwoLevel( + const unsigned char* scratch1, + const size_t* block_sizes1, + unsigned char* out, + size_t* block_sizes2, + int nblocks, + hipStream_t stream = 0) +{ + waveletBitmapCodeTwoLevelKernel<<>>( + scratch1, block_sizes1, out, block_sizes2); + return hipGetLastError(); +} + +// --------------------------------------------------------------------------- +// Kernel 2 (coding), TWO-LEVEL occupancy + per-line width -- OPTIMIZED. +// +// Byte-for-byte identical output to waveletBitmapCodeTwoLevelKernel (occupancy, +// masks, 2-bit width table and packed values in the same layout and order), so +// the existing decoder is unchanged. It is faster because of six levers found by +// the polyopt round-3 campaign (worker continue_1_s2, mean 1.59x on gfx950; the +// coding kernel `op` phase alone ~5x): +// - one workgroup of 1024 threads, one z-line per thread (vs 256 threads x 4) +// - wave64 exclusive scans as DPP row shifts, not ds_bpermute/LDS crossbar +// - the two width-independent scans (input offset, nonempty rank) fused into +// one dual block scan +// - a branch-free per-width group packer (whole-dword stores, ragged tail) +// - the packed values staged in an LDS image of the value region, then drained +// to global with 16-byte non-temporal stores (one instruction per ~1 KB) +// - occupancy written from a single ballot per warp +// +// LDS budget is WBMP_MAX_VAL_BYTES + ~1.2 KB (~132 KB), so this kernel targets +// gfx950 (CDNA4) and its larger LDS; the original kernel above remains the +// portable path for gfx90a and earlier. The unused tail of each output slot is +// left untouched exactly as in the original -- block_sizes2[bid] records the +// exact coded length and the decoder reads only [0, used). +// --------------------------------------------------------------------------- +namespace wbmp_opt { + +typedef unsigned int v4u __attribute__((ext_vector_type(4))); + +static constexpr int WBMP_OPT_THREADS = 1024; +static constexpr int WBMP_OPT_WARP = 64; +static constexpr int WBMP_OPT_NWARPS = WBMP_OPT_THREADS / WBMP_OPT_WARP; // 16 +// Worst case a coded block can pack is WBMP_MAX_VAL_BYTES; the extra 16 absorbs +// the offset that keeps an aligned 16-byte block of the image an aligned 16-byte +// block of the slot. No input can overflow it, so there is no fallback path. +static constexpr int WBMP_OPT_VBUF = WBMP_MAX_VAL_BYTES + 16; + +__device__ __forceinline__ void st32(unsigned char* p, unsigned u) { + __builtin_memcpy(p, &u, 4); +} + +// Wave64 inclusive add scan as six DPP row shifts. __shfl_up lowers to +// ds_bpermute_b32 on gfx9 -- an LDS crossbar round trip per step; the DPP form +// is six plain VALU adds with a modifier and touches no LDS. +template +__device__ __forceinline__ int dpp_add(int x) { + return x + __builtin_amdgcn_update_dpp(0, x, CTRL, RMASK, 0xf, false); +} + +__device__ __forceinline__ int winc(int v) { + v = dpp_add<0x111, 0xf>(v); /* row_shr:1 */ + v = dpp_add<0x112, 0xf>(v); /* row_shr:2 */ + v = dpp_add<0x114, 0xf>(v); /* row_shr:4 */ + v = dpp_add<0x118, 0xf>(v); /* row_shr:8 */ + v = dpp_add<0x142, 0xa>(v); /* row_bcast:15 -> rows 1,3 */ + v = dpp_add<0x143, 0xc>(v); /* row_bcast:31 -> rows 2,3 */ + return v; +} + +__device__ __forceinline__ int block_exscan(int v, int& total, int* warp_part) { + const int tid = threadIdx.x; + const int lane = tid & (WBMP_OPT_WARP - 1); + const int warp = tid >> 6; + const int wincl = winc(v); + const int wsum = __builtin_amdgcn_readlane(wincl, WBMP_OPT_WARP - 1); + if (lane == WBMP_OPT_WARP - 1) warp_part[warp] = wsum; + __syncthreads(); + if (warp == 0) { + const int s = (lane < WBMP_OPT_NWARPS) ? warp_part[lane] : 0; + const int sincl = winc(s); + if (lane < WBMP_OPT_NWARPS) warp_part[lane] = sincl; + } + __syncthreads(); + total = warp_part[WBMP_OPT_NWARPS - 1]; + const int wprefix = (warp == 0) ? 0 : warp_part[warp - 1]; + return wprefix + wincl - v; +} + +__device__ __forceinline__ void block_exscan2(int v0, int v1, int& ex0, int& ex1, int& total0, + int& total1, int* warp_part) { + const int tid = threadIdx.x; + const int lane = tid & (WBMP_OPT_WARP - 1); + const int warp = tid >> 6; + const int w0incl = winc(v0); + const int w1incl = winc(v1); + const int w0sum = __builtin_amdgcn_readlane(w0incl, WBMP_OPT_WARP - 1); + const int w1sum = __builtin_amdgcn_readlane(w1incl, WBMP_OPT_WARP - 1); + if (lane == WBMP_OPT_WARP - 1) { + warp_part[warp] = w0sum; + warp_part[warp + WBMP_OPT_NWARPS] = w1sum; + } + __syncthreads(); + if (warp == 0) { + const int s0 = (lane < WBMP_OPT_NWARPS) ? warp_part[lane] : 0; + const int s1 = (lane < WBMP_OPT_NWARPS) ? warp_part[lane + WBMP_OPT_NWARPS] : 0; + const int s0incl = winc(s0); + const int s1incl = winc(s1); + if (lane < WBMP_OPT_NWARPS) { + warp_part[lane] = s0incl; + warp_part[lane + WBMP_OPT_NWARPS] = s1incl; + } + } + __syncthreads(); + total0 = warp_part[WBMP_OPT_NWARPS - 1]; + total1 = warp_part[2 * WBMP_OPT_NWARPS - 1]; + const int w0prefix = (warp == 0) ? 0 : warp_part[warp - 1]; + const int w1prefix = (warp == 0) ? 0 : warp_part[warp + WBMP_OPT_NWARPS - 1]; + ex0 = w0prefix + w0incl - v0; + ex1 = w1prefix + w1incl - v1; +} + +template +__device__ __forceinline__ void pack_line_w(unsigned char* out, long p, const int32_t* val, + int base, int nz) { + long q = p; + int k = 0; + if (W == 1) { + for (; k + 4 <= nz; k += 4) { + const unsigned a = (unsigned)__ldg(val + base + k); + const unsigned b = (unsigned)__ldg(val + base + k + 1); + const unsigned c = (unsigned)__ldg(val + base + k + 2); + const unsigned d = (unsigned)__ldg(val + base + k + 3); + st32(out + q, (a & 0xffu) | ((b & 0xffu) << 8) | ((c & 0xffu) << 16) | (d << 24)); + q += 4; + } + } else if (W == 2) { + for (; k + 2 <= nz; k += 2) { + const unsigned a = (unsigned)__ldg(val + base + k); + const unsigned b = (unsigned)__ldg(val + base + k + 1); + st32(out + q, (a & 0xffffu) | (b << 16)); + q += 4; + } + } else if (W == 3) { + for (; k + 4 <= nz; k += 4) { + const unsigned a = (unsigned)__ldg(val + base + k); + const unsigned b = (unsigned)__ldg(val + base + k + 1); + const unsigned c = (unsigned)__ldg(val + base + k + 2); + const unsigned d = (unsigned)__ldg(val + base + k + 3); + st32(out + q + 0, (a & 0xffffffu) | (b << 24)); + st32(out + q + 4, ((b >> 8) & 0xffffu) | (c << 16)); + st32(out + q + 8, ((c >> 16) & 0xffu) | (d << 8)); + q += 12; + } + } else { + for (; k < nz; ++k) { + st32(out + q, (unsigned)__ldg(val + base + k)); + q += 4; + } + } + for (; k < nz; ++k) { + const unsigned u = (unsigned)__ldg(val + base + k); +#pragma unroll + for (int b = 0; b < W; ++b) out[q + b] = (unsigned char)(u >> (8 * b)); + q += W; + } +} + +__device__ __forceinline__ void pack_line(unsigned char* out, long p, const int32_t* val, + int base, int nz, int W) { + switch (W) { + case 1: pack_line_w<1>(out, p, val, base, nz); break; + case 2: pack_line_w<2>(out, p, val, base, nz); break; + case 3: pack_line_w<3>(out, p, val, base, nz); break; + default: pack_line_w<4>(out, p, val, base, nz); break; + } +} + +} // namespace wbmp_opt + +__launch_bounds__(wbmp_opt::WBMP_OPT_THREADS) +__global__ void waveletBitmapCodeTwoLevelOptKernel( + const unsigned char* __restrict__ scratch1, + const size_t* __restrict__ block_sizes1, + unsigned char* __restrict__ out, + size_t* __restrict__ block_sizes2) +{ + using namespace wbmp_opt; + (void)block_sizes1; // recomputed from the bitmap, kept for signature parity + __shared__ int warp_part[2 * WBMP_OPT_NWARPS]; + __shared__ unsigned char wtab8[WBMP_BITMAP_WORDS]; + __shared__ unsigned char vbuf[WBMP_OPT_VBUF]; + + const int bid = blockIdx.x; + const int tid = threadIdx.x; + const int lane = tid & (WBMP_OPT_WARP - 1); + const int warp = tid >> 6; + + const unsigned char* blk_in = scratch1 + (long)bid * WBMP_SLOT_BYTES; + const uint32_t* bmp = reinterpret_cast(blk_in); + const int32_t* val = reinterpret_cast(blk_in + WBMP_BITMAP_BYTES); + unsigned char* blk_out = out + (long)bid * WBMP_TL_SLOT_BYTES; + + const uint32_t m = __ldg(bmp + tid); + const int mask = (int)m; + const int nz = __popc(m); + const int occb = m ? 1 : 0; + + // First block scan carries both quantities that do not depend on the width: + // the input offset (over nnz) and the nonempty rank (over occupancy). + int tot_nz, tot_ne, in_off, occ_rank; + block_exscan2(nz, occb, in_off, occ_rank, tot_nz, tot_ne, warp_part); + (void)tot_nz; + + const long masks_base = WBMP_OCC_BYTES; + const long wtab_base = masks_base + 4L * tot_ne; + const long vals_base = wtab_base + (2L * tot_ne + 7) / 8; + + uint32_t* occ = reinterpret_cast(blk_out); + uint32_t* masks = reinterpret_cast(blk_out + masks_base); + + const unsigned long long occ_ballot = __ballot(occb); + if (lane == 31) occ[warp * 2] = (uint32_t)occ_ballot; + if (lane == 63) occ[warp * 2 + 1] = (uint32_t)(occ_ballot >> 32); + if (occb) masks[occ_rank] = (uint32_t)mask; + + int mx = 0; +#pragma unroll 4 + for (int k = 0; k < nz; ++k) { + const int a = __ldg(val + in_off + k); + mx |= (a < 0 ? -a : a); + } + const int W = nz ? wbmp_width_bytes(mx) : 1; + + int tot_val; + const int val_off = block_exscan(nz * W, tot_val, warp_part); + const long used = vals_base + tot_val; + + const long g0 = vals_base & ~15L; + const int lo = (int)(vals_base - g0); + + if (occb) { + wtab8[occ_rank] = (unsigned char)(W - 1); + pack_line(vbuf, (long)lo + val_off, val, in_off, nz, W); + } + __syncthreads(); + + // Drain the image: 16-byte aligned interior with non-temporal stores, the at + // most fifteen ragged bytes at either end byte-wise. + { + const long hi = (long)lo + tot_val; + const long abeg = ((long)lo + 15) & ~15L; + const long aend = hi & ~15L; + const long hend = abeg < hi ? abeg : hi; + const long tbeg = abeg > aend ? abeg : aend; + for (long i = (long)lo + tid; i < hend; i += WBMP_OPT_THREADS) blk_out[g0 + i] = vbuf[i]; + for (long i = abeg + 16L * tid; i + 16 <= aend; i += 16L * WBMP_OPT_THREADS) { + v4u v = *reinterpret_cast(vbuf + i); + __builtin_nontemporal_store(v, reinterpret_cast(blk_out + g0 + i)); + } + for (long i = tbeg + tid; i < hi; i += WBMP_OPT_THREADS) blk_out[g0 + i] = vbuf[i]; + } + + const long wtab_bytes = (2L * tot_ne + 7) / 8; + const int full_words = (int)(wtab_bytes >> 2); + uint32_t* wtab = reinterpret_cast(blk_out + wtab_base); + for (int wi = tid; wi <= full_words; wi += WBMP_OPT_THREADS) { + uint32_t w = 0; + const int base = wi * 16; +#pragma unroll + for (int j = 0; j < 16; ++j) { + if (base + j < tot_ne) w |= (uint32_t)wtab8[base + j] << (j * 2); + } + if (wi < full_words) { + wtab[wi] = w; + } else { + unsigned char* edge = blk_out + wtab_base + 4L * full_words; + for (int j = 0; j < (int)(wtab_bytes & 3); ++j) edge[j] = (unsigned char)(w >> (8 * j)); + } + } + + if (tid == 0) block_sizes2[bid] = (size_t)used; +} + +inline hipError_t hipWaveletBitmapCodeTwoLevelOpt( + const unsigned char* scratch1, + const size_t* block_sizes1, + unsigned char* out, + size_t* block_sizes2, + int nblocks, + hipStream_t stream = 0) +{ + waveletBitmapCodeTwoLevelOptKernel<<>>( + scratch1, block_sizes1, out, block_sizes2); + return hipGetLastError(); +} + +// Launch helper mirroring hipWaveletRLEFused. output must have +// nblocks * WBMP_SLOT_BYTES bytes; block_sizes has nblocks entries. +inline hipError_t hipWaveletBitmapFused( + const float* input, + unsigned char* output, + size_t* block_sizes, + float scale, + int nx, int ny, int nz, + int ldimx, int ldimxy, + const double* d_rms = nullptr, + float* d_mulfac_out = nullptr, + hipStream_t stream = 0) +{ + dim3 grid((nx + 31) / 32, (ny + 31) / 32, (nz + 31) / 32); + waveletBitmapFusedKernel<<>>( + input, output, block_sizes, scale, ldimx, ldimxy, + d_rms, d_mulfac_out); + return hipGetLastError(); +} + +#endif // HIPWAVELET_BITMAP_H diff --git a/makefile b/makefile index a3610e4..13d07d9 100644 --- a/makefile +++ b/makefile @@ -96,6 +96,10 @@ test_quantize_rle: tests/test_quantize_rle.cpp hip/quantize_rle_ref.h Run_Length test_zline_cr_benchmark: tests/test_zline_cr_benchmark.cpp hip/quantize_rle_ref.h libcvxcompress.$(LIB_EXT) | $(BUILDDIR) $(CXX) $(CFLAGS) $(TFLAG) -I. -Ihip -Itests tests/test_zline_cr_benchmark.cpp -L. -lcvxcompress '-Wl,-rpath,$$ORIGIN/..' -o $(BUILDDIR)/test_zline_cr_benchmark +# Bitmap vs production full-block RLE, four-bucket byte breakdown (CPU-only) +test_bitmap_vs_cpu_rle: tests/test_bitmap_vs_cpu_rle.cpp Run_Length_Encode_Slow.hxx Run_Length_Escape_Codes.hxx Wavelet_Transform_Fast.hxx Block_Copy.hxx libcvxcompress.$(LIB_EXT) | $(BUILDDIR) + $(CXX) $(CFLAGS) $(TFLAG) -I. -Ihip -Itests tests/test_bitmap_vs_cpu_rle.cpp -L. -lcvxcompress '-Wl,-rpath,$$ORIGIN/..' -o $(BUILDDIR)/test_bitmap_vs_cpu_rle + # GPU quantize+RLE encode test (validates against CPU reference) test_quantize_rle_hip: tests/test_quantize_rle_hip.cpp hip/quantize_rle_ref.h Run_Length_Escape_Codes.hxx hip/hipQuantizeRLE.h | $(BUILDDIR) $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -I. -Ihip -Itests tests/test_quantize_rle_hip.cpp -lm -o $(BUILDDIR)/test_quantize_rle_hip @@ -128,6 +132,14 @@ test_compress_api_hip: tests/test_compress_api_hip.cpp hip/hipCompress.cpp hip/h test_compress_2d_hip: tests/test_compress_2d_hip.cpp hip/hipCompress.cpp hip/hipCompress.h hip/hipBlockCopy.h hip/hipWaveletRLE.h hip/hipWaveletRLEInverse.h hip/hipWaveletRLE2D.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_compress_2d_hip.cpp hip/hipCompress.cpp -lm -o $(BUILDDIR)/test_compress_2d_hip +# Bitmap-significance-split encode (kernel 1) validation vs RLE ground truth +test_bitmap_encode_hip: tests/test_bitmap_encode_hip.cpp hip/hipWaveletBitmap.h hip/hipWaveletRLE.h hip/quantize_rle_ref.h Run_Length_Escape_Codes.hxx hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) + $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_bitmap_encode_hip.cpp -lm -o $(BUILDDIR)/test_bitmap_encode_hip + +# Bitmap-significance-split coding (kernel 2): correctness + CR + throughput +test_bitmap_code_hip: tests/test_bitmap_code_hip.cpp hip/hipWaveletBitmap.h hip/hipWaveletRLE.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) + $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_bitmap_code_hip.cpp -lm -o $(BUILDDIR)/test_bitmap_code_hip + # Async pipeline example (for profiling) example_async_pipeline: tests/example_async_pipeline.cpp hip/hipCompress.cpp hip/hipCompress.h hip/hipBlockCopy.h hip/hipWaveletRLE.h hip/hipWaveletRLEInverse.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/example_async_pipeline.cpp hip/hipCompress.cpp -lm -o $(BUILDDIR)/example_async_pipeline diff --git a/tests/test_bitmap_code_hip.cpp b/tests/test_bitmap_code_hip.cpp new file mode 100644 index 0000000..3360296 --- /dev/null +++ b/tests/test_bitmap_code_hip.cpp @@ -0,0 +1,316 @@ +// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Use of this source code is governed by an MIT-style license that can be +// found in the LICENSE file or at https://opensource.org/licenses/MIT. +// +// Kernel 2 (coding) prototype: per-block fixed-width packing of the bitmap +// value stream. Validates correctness (unpacked values == kernel-1 packed +// values), and reports compressed size (CR vs the RLE path at matched +// quantization) plus per-kernel throughput. + +#define DS79_INCLUDE_REG32 +#include "hipWaveletRLE.h" +#include "hipWaveletBitmap.h" + +#include +#include +#include +#include +#include +#include + +#define HIPCHECK(cmd) do { \ + hipError_t _e = (cmd); \ + if (_e != hipSuccess) { \ + printf("HIP error %s at %s:%d\n", hipGetErrorString(_e), __FILE__, __LINE__); \ + return 1; \ + } \ +} while (0) + +static inline int unpack_val(const unsigned char* p, int W) { + unsigned u = 0; + for (int b = 0; b < W; ++b) u |= (unsigned)p[b] << (8 * b); + unsigned sign = 1u << (8 * W - 1); + if (u & sign) u |= ~((sign << 1) - 1); // sign-extend (no-op for W=4) + return (int)u; +} + +static float time_kernel(void (*launch)(void*), void* ctx, int iters) { + hipEvent_t a, b; + hipEventCreate(&a); hipEventCreate(&b); + launch(ctx); // warmup + hipDeviceSynchronize(); + hipEventRecord(a); + for (int i = 0; i < iters; ++i) launch(ctx); + hipEventRecord(b); + hipEventSynchronize(b); + float ms = 0; hipEventElapsedTime(&ms, a, b); + hipEventDestroy(a); hipEventDestroy(b); + return ms / iters; // ms per launch +} + +struct Ctx { + const float* d_in; unsigned char* d_rle; size_t* d_rle_sizes; + unsigned char* d_bmp; size_t* d_bmp_sizes; + unsigned char* d_code; size_t* d_code_sizes; + unsigned char* d_codepl; size_t* d_codepl_sizes; + unsigned char* d_codetl; size_t* d_codetl_sizes; + unsigned char* d_codetlopt; size_t* d_codetlopt_sizes; + int NX, NY, NZ, ldimx, ldimxy, nbx, nby, nbz, nblocks; + float mulfac; +}; + +static void launch_rle(void* p) { + Ctx* c = (Ctx*)p; + dim3 grid(c->nbx, c->nby, c->nbz); + waveletRLEFusedKernel<<>>( + c->d_in, c->d_rle, c->d_rle_sizes, c->mulfac, c->ldimx, c->ldimxy, nullptr, nullptr); +} +static void launch_bmp(void* p) { + Ctx* c = (Ctx*)p; + dim3 grid(c->nbx, c->nby, c->nbz); + waveletBitmapFusedKernel<<>>( + c->d_in, c->d_bmp, c->d_bmp_sizes, c->mulfac, c->ldimx, c->ldimxy, nullptr, nullptr); +} +static void launch_code(void* p) { + Ctx* c = (Ctx*)p; + waveletBitmapCodeKernel<<nblocks, dim3(256)>>>( + c->d_bmp, c->d_bmp_sizes, c->d_code, c->d_code_sizes); +} +static void launch_codepl(void* p) { + Ctx* c = (Ctx*)p; + waveletBitmapCodePerLineKernel<<nblocks, dim3(256)>>>( + c->d_bmp, c->d_bmp_sizes, c->d_codepl, c->d_codepl_sizes); +} +static void launch_codetl(void* p) { + Ctx* c = (Ctx*)p; + waveletBitmapCodeTwoLevelKernel<<nblocks, dim3(256)>>>( + c->d_bmp, c->d_bmp_sizes, c->d_codetl, c->d_codetl_sizes); +} +static void launch_codetlopt(void* p) { + Ctx* c = (Ctx*)p; + waveletBitmapCodeTwoLevelOptKernel<<nblocks, dim3(wbmp_opt::WBMP_OPT_THREADS)>>>( + c->d_bmp, c->d_bmp_sizes, c->d_codetlopt, c->d_codetlopt_sizes); +} + +int main(int argc, char** argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + const int NX = (argc > 1) ? atoi(argv[1]) : 128; + const int NY = NX, NZ = NX; + const float mulfac = (argc > 2) ? (float)atof(argv[2]) : 20.0f; + const int iters = (argc > 3) ? atoi(argv[3]) : 100; + + if (NX % 32 != 0) { printf("NX must be multiple of 32\n"); return 1; } + + Ctx c; + c.NX = NX; c.NY = NY; c.NZ = NZ; c.mulfac = mulfac; + c.ldimx = NX; c.ldimxy = NX * NY; + c.nbx = NX / 32; c.nby = NY / 32; c.nbz = NZ / 32; + c.nblocks = c.nbx * c.nby * c.nbz; + const size_t nelem = (size_t)NX * NY * NZ; + + printf("bitmap-code test: %dx%dx%d nblocks=%d mulfac=%.3f iters=%d\n", + NX, NY, NZ, c.nblocks, mulfac, iters); + + std::vector h_in(nelem); + for (size_t i = 0; i < nelem; ++i) { + int x = (int)(i % NX); + int y = (int)((i / NX) % NY); + int z = (int)(i / ((size_t)NX * NY)); + float s = sinf(0.11f * x) * cosf(0.07f * y) * sinf(0.05f * z); + float n = 0.15f * (float)(((i * 1103515245u + 12345u) >> 16) & 0x7fff) / 32768.0f; + h_in[i] = 0.5f * s + n - 0.075f; + } + + HIPCHECK(hipMalloc(&c.d_in, nelem * sizeof(float))); + HIPCHECK(hipMemcpy((void*)c.d_in, h_in.data(), nelem * sizeof(float), hipMemcpyHostToDevice)); + + const long rle_stride = 4L * WRLE_LDS_BYTES; + HIPCHECK(hipMalloc(&c.d_rle, (size_t)c.nblocks * rle_stride)); + HIPCHECK(hipMalloc(&c.d_rle_sizes, c.nblocks * sizeof(size_t))); + HIPCHECK(hipMalloc(&c.d_bmp, (size_t)c.nblocks * WBMP_SLOT_BYTES)); + HIPCHECK(hipMalloc(&c.d_bmp_sizes, c.nblocks * sizeof(size_t))); + HIPCHECK(hipMalloc(&c.d_code, (size_t)c.nblocks * WBMP_CODE_SLOT_BYTES)); + HIPCHECK(hipMalloc(&c.d_code_sizes, c.nblocks * sizeof(size_t))); + HIPCHECK(hipMalloc(&c.d_codepl, (size_t)c.nblocks * WBMP_PL_SLOT_BYTES)); + HIPCHECK(hipMalloc(&c.d_codepl_sizes, c.nblocks * sizeof(size_t))); + HIPCHECK(hipMalloc(&c.d_codetl, (size_t)c.nblocks * WBMP_TL_SLOT_BYTES)); + HIPCHECK(hipMalloc(&c.d_codetl_sizes, c.nblocks * sizeof(size_t))); + HIPCHECK(hipMalloc(&c.d_codetlopt, (size_t)c.nblocks * WBMP_TL_SLOT_BYTES)); + HIPCHECK(hipMalloc(&c.d_codetlopt_sizes, c.nblocks * sizeof(size_t))); + + // Run the pipeline once for correctness + sizes. + launch_rle(&c); launch_bmp(&c); HIPCHECK(hipDeviceSynchronize()); + launch_code(&c); launch_codepl(&c); launch_codetl(&c); launch_codetlopt(&c); + HIPCHECK(hipDeviceSynchronize()); + HIPCHECK(hipGetLastError()); + + // ---- Correctness: unpack coded values, compare to kernel-1 int32 ---- + std::vector h_bmp((size_t)c.nblocks * WBMP_SLOT_BYTES); + std::vector h_code((size_t)c.nblocks * WBMP_CODE_SLOT_BYTES); + std::vector h_bmp_sizes(c.nblocks), h_code_sizes(c.nblocks), h_rle_sizes(c.nblocks); + HIPCHECK(hipMemcpy(h_bmp.data(), c.d_bmp, h_bmp.size(), hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_code.data(), c.d_code, h_code.size(), hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_bmp_sizes.data(), c.d_bmp_sizes, c.nblocks*sizeof(size_t), hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_code_sizes.data(), c.d_code_sizes, c.nblocks*sizeof(size_t), hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_rle_sizes.data(), c.d_rle_sizes, c.nblocks*sizeof(size_t), hipMemcpyDeviceToHost)); + + long mismatches = 0; + long w_hist[5] = {0,0,0,0,0}; + for (int bid = 0; bid < c.nblocks; ++bid) { + const unsigned char* bin = h_bmp.data() + (long)bid * WBMP_SLOT_BYTES; + const int32_t* vin = reinterpret_cast(bin + WBMP_BITMAP_BYTES); + int nnz = (int)((h_bmp_sizes[bid] - WBMP_BITMAP_BYTES) / 4); + + const unsigned char* cin = h_code.data() + (long)bid * WBMP_CODE_SLOT_BYTES; + int W = reinterpret_cast(cin + WBMP_BITMAP_BYTES)[0]; + const unsigned char* vpk = cin + WBMP_BITMAP_BYTES + 4; + if (W >= 1 && W <= 4) ++w_hist[W]; + + long exp = WBMP_BITMAP_BYTES + 4 + (long)nnz * W; + if ((long)h_code_sizes[bid] != exp) { ++mismatches; if (mismatches<10) + printf(" block %d size %ld != %ld\n", bid, (long)h_code_sizes[bid], exp); } + + for (int i = 0; i < nnz; ++i) { + int got = unpack_val(vpk + (long)i * W, W); + if (got != vin[i]) { if (mismatches<20) + printf(" block %d val %d: %d != %d (W=%d)\n", bid, i, got, vin[i], W); + ++mismatches; } + } + } + + // ---- Per-line coding validation (unpack vs kernel-1 int32) ---- + std::vector h_codepl((size_t)c.nblocks * WBMP_PL_SLOT_BYTES); + std::vector h_codepl_sizes(c.nblocks); + HIPCHECK(hipMemcpy(h_codepl.data(), c.d_codepl, h_codepl.size(), hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_codepl_sizes.data(), c.d_codepl_sizes, c.nblocks*sizeof(size_t), hipMemcpyDeviceToHost)); + for (int bid = 0; bid < c.nblocks; ++bid) { + const unsigned char* bin = h_bmp.data() + (long)bid * WBMP_SLOT_BYTES; + const int32_t* vin = reinterpret_cast(bin + WBMP_BITMAP_BYTES); + const unsigned char* cin = h_codepl.data() + (long)bid * WBMP_PL_SLOT_BYTES; + const uint32_t* bmp = reinterpret_cast(cin); + const uint32_t* wtab = reinterpret_cast(cin + WBMP_BITMAP_BYTES); + const unsigned char* vpk = cin + WBMP_BITMAP_BYTES + WBMP_WTAB_BYTES; + long idx = 0, ob = 0; + for (int x = 0; x < 4; ++x) + for (int tid = 0; tid < 256; ++tid) { + int L = x*256+tid; + int nnz = __builtin_popcount(bmp[L]); + int W = ((wtab[L>>4] >> ((L&15)*2)) & 3) + 1; + for (int k = 0; k < nnz; ++k) { + int got = unpack_val(vpk + ob, W); ob += W; + if (got != vin[idx]) { if (mismatches<20) + printf(" [PL] block %d line %d k %d: %d != %d (W=%d)\n", bid, L, k, got, vin[idx], W); + ++mismatches; } + ++idx; + } + } + long exp = WBMP_BITMAP_BYTES + WBMP_WTAB_BYTES + ob; + if ((long)h_codepl_sizes[bid] != exp) { ++mismatches; if (mismatches<25) + printf(" [PL] block %d size %ld != %ld\n", bid, (long)h_codepl_sizes[bid], exp); } + } + + // ---- Two-level occupancy coding validation (decode from occupancy) ---- + std::vector h_codetl((size_t)c.nblocks * WBMP_TL_SLOT_BYTES); + std::vector h_codetl_sizes(c.nblocks); + HIPCHECK(hipMemcpy(h_codetl.data(), c.d_codetl, h_codetl.size(), hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_codetl_sizes.data(), c.d_codetl_sizes, c.nblocks*sizeof(size_t), hipMemcpyDeviceToHost)); + for (int bid = 0; bid < c.nblocks; ++bid) { + const unsigned char* bin = h_bmp.data() + (long)bid * WBMP_SLOT_BYTES; + const int32_t* vin = reinterpret_cast(bin + WBMP_BITMAP_BYTES); + const unsigned char* cin = h_codetl.data() + (long)bid * WBMP_TL_SLOT_BYTES; + const uint32_t* occ = reinterpret_cast(cin); + int n_ne = 0; + for (int w = 0; w < 32; ++w) n_ne += __builtin_popcount(occ[w]); + const uint32_t* masks = reinterpret_cast(cin + WBMP_OCC_BYTES); + long wtab_base = WBMP_OCC_BYTES + 4L*n_ne; + const uint32_t* wtab = reinterpret_cast(cin + wtab_base); + long vals_base = wtab_base + (2L*n_ne + 7)/8; + const unsigned char* vpk = cin + vals_base; + long idx = 0, vo = 0; int rank = 0; + for (int L = 0; L < 1024; ++L) { + if (!(occ[L>>5] & (1u << (L&31)))) continue; + uint32_t m = masks[rank]; + int W = ((wtab[rank>>4] >> ((rank&15)*2)) & 3) + 1; + int nnz = __builtin_popcount(m); + for (int k = 0; k < nnz; ++k) { + int got = unpack_val(vpk + vo, W); vo += W; + if (got != vin[idx]) { if (mismatches<20) + printf(" [TL] block %d line %d k %d: %d != %d (W=%d)\n", bid, L, k, got, vin[idx], W); + ++mismatches; } + ++idx; + } + ++rank; + } + long exp = vals_base + vo; + if ((long)h_codetl_sizes[bid] != exp) { ++mismatches; if (mismatches<25) + printf(" [TL] block %d size %ld != %ld\n", bid, (long)h_codetl_sizes[bid], exp); } + } + + // ---- Optimized two-level: byte-exact vs reference two-level + decode ---- + std::vector h_codetlopt((size_t)c.nblocks * WBMP_TL_SLOT_BYTES); + std::vector h_codetlopt_sizes(c.nblocks); + HIPCHECK(hipMemcpy(h_codetlopt.data(), c.d_codetlopt, h_codetlopt.size(), hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_codetlopt_sizes.data(), c.d_codetlopt_sizes, c.nblocks*sizeof(size_t), hipMemcpyDeviceToHost)); + long tlopt_mismatch = 0; + for (int bid = 0; bid < c.nblocks; ++bid) { + // Size must match the reference two-level exactly. + if (h_codetlopt_sizes[bid] != h_codetl_sizes[bid]) { ++tlopt_mismatch; if (tlopt_mismatch<10) + printf(" [TLopt] block %d size %ld != ref %ld\n", bid, + (long)h_codetlopt_sizes[bid], (long)h_codetl_sizes[bid]); continue; } + // The used prefix [0, used) must be byte-for-byte identical to the + // reference kernel; the unused tail is intentionally left untouched. + const unsigned char* a = h_codetlopt.data() + (long)bid * WBMP_TL_SLOT_BYTES; + const unsigned char* b = h_codetl.data() + (long)bid * WBMP_TL_SLOT_BYTES; + long used = (long)h_codetlopt_sizes[bid]; + if (memcmp(a, b, used) != 0) { + ++tlopt_mismatch; + for (long i = 0; i < used && tlopt_mismatch < 20; ++i) + if (a[i] != b[i]) { printf(" [TLopt] block %d byte %ld: %u != %u\n", + bid, i, a[i], b[i]); ++tlopt_mismatch; break; } + } + } + mismatches += tlopt_mismatch; + printf("two-level opt vs reference: %s (%ld byte/size mismatches)\n", + tlopt_mismatch == 0 ? "byte-exact" : "MISMATCH", tlopt_mismatch); + + // ---- Size / CR at matched quantization ---- + long hdr = 8 + 8L*c.nblocks + 4; + long rle_total = hdr, bmp_i32_total = hdr, code_total = hdr, codepl_total = hdr, codetl_total = hdr; + for (int bid = 0; bid < c.nblocks; ++bid) { + rle_total += (long)h_rle_sizes[bid]; + bmp_i32_total += (long)h_bmp_sizes[bid]; + code_total += (long)h_code_sizes[bid]; + codepl_total += (long)h_codepl_sizes[bid]; + codetl_total += (long)h_codetl_sizes[bid]; + } + double raw = (double)nelem * 4.0; + printf("W histogram (bytes/value): 1B=%ld 2B=%ld 3B=%ld 4B=%ld\n", + w_hist[1], w_hist[2], w_hist[3], w_hist[4]); + printf("sizes (bytes): RLE=%ld bmp+int32=%ld bmp+fixedW=%ld bmp+perline=%ld bmp+twolevel=%ld\n", + rle_total, bmp_i32_total, code_total, codepl_total, codetl_total); + printf("CR: RLE=%.3f bmp+fixedW=%.3f bmp+perline=%.3f bmp+twolevel=%.3f\n", + raw/rle_total, raw/code_total, raw/codepl_total, raw/codetl_total); + printf("vs GPU-RLE size ratio: fixedW=%.3f perline=%.3f twolevel=%.3f (<1 = smaller)\n", + (double)code_total / rle_total, (double)codepl_total / rle_total, + (double)codetl_total / rle_total); + + // ---- Throughput (raw input volume / kernel time) ---- + float t_rle = time_kernel(launch_rle, &c, iters); + float t_bmp = time_kernel(launch_bmp, &c, iters); + float t_code = time_kernel(launch_code, &c, iters); + float t_codepl = time_kernel(launch_codepl, &c, iters); + float t_codetl = time_kernel(launch_codetl, &c, iters); + float t_codetlopt = time_kernel(launch_codetlopt, &c, iters); + double gbraw = raw / 1e9; + printf("throughput (raw GB/s): RLE_fused=%.1f bmp_fused=%.1f codepl=%.1f codetl=%.1f codetl_opt=%.1f bmp+codetl_opt=%.1f\n", + gbraw/(t_rle/1e3), gbraw/(t_bmp/1e3), gbraw/(t_codepl/1e3), gbraw/(t_codetl/1e3), + gbraw/(t_codetlopt/1e3), gbraw/((t_bmp+t_codetlopt)/1e3)); + printf("kernel ms: RLE_fused=%.3f bmp_fused=%.3f code=%.3f codepl=%.3f codetl=%.3f codetl_opt=%.3f\n", + t_rle, t_bmp, t_code, t_codepl, t_codetl, t_codetlopt); + printf("two-level coder speedup (codetl / codetl_opt): %.3fx\n", t_codetl / t_codetlopt); + + if (mismatches != 0) { printf("mismatches=%ld\nFAIL\n", mismatches); return 1; } + printf("PASS\n"); + return 0; +} diff --git a/tests/test_bitmap_encode_hip.cpp b/tests/test_bitmap_encode_hip.cpp new file mode 100644 index 0000000..7bf5f42 --- /dev/null +++ b/tests/test_bitmap_encode_hip.cpp @@ -0,0 +1,187 @@ +// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Use of this source code is governed by an MIT-style license that can be +// found in the LICENSE file or at https://opensource.org/licenses/MIT. +// +// Validation for kernel 1 of the bitmap-significance-split prototype. +// +// Strategy: the bitmap kernel and the RLE kernel share identical wavelet +// transform + identical quantization (int)(mulfac*coeff). So the set of +// nonzero coefficients and their integer values MUST match bit-for-bit. +// We run both kernels on the same input, decode the RLE stream on the host +// (ground truth), scatter the bitmap+packed values back to a dense grid, and +// require exact equality over all 32768 coefficients per block. + +#define DS79_INCLUDE_REG32 +#include "hipWaveletRLE.h" +#include "hipWaveletBitmap.h" +#include "quantize_rle_ref.h" + +#include +#include +#include +#include +#include +#include + +#define HIPCHECK(cmd) do { \ + hipError_t _e = (cmd); \ + if (_e != hipSuccess) { \ + printf("HIP error %s at %s:%d\n", hipGetErrorString(_e), __FILE__, __LINE__); \ + return 1; \ + } \ +} while (0) + +// Canonical per-coefficient index shared by both kernels: (x_off, tid, z). +static inline int coeff_index(int x_off, int tid, int z) { + return (x_off * 256 + tid) * 32 + z; +} + +int main(int argc, char** argv) +{ + const int NX = (argc > 1) ? atoi(argv[1]) : 64; + const int NY = NX, NZ = NX; + const float mulfac = (argc > 2) ? (float)atof(argv[2]) : 8.0f; + + if (NX % 32 != 0) { printf("NX must be multiple of 32\n"); return 1; } + + const int nbx = NX / 32, nby = NY / 32, nbz = NZ / 32; + const int nblocks = nbx * nby * nbz; + const int ldimx = NX, ldimxy = NX * NY; + const size_t nelem = (size_t)NX * NY * NZ; + const int COEFFS_PER_BLOCK = 4 * 256 * 32; // 32768 + + printf("bitmap-encode test: %dx%dx%d nblocks=%d mulfac=%.3f\n", + NX, NY, NZ, nblocks, mulfac); + + // ---- Deterministic input (mix of smooth + noise so we get both zeros + // and nonzeros after transform+quantize) ---- + std::vector h_in(nelem); + for (size_t i = 0; i < nelem; ++i) { + int x = (int)(i % NX); + int y = (int)((i / NX) % NY); + int z = (int)(i / ((size_t)NX * NY)); + float s = sinf(0.11f * x) * cosf(0.07f * y) * sinf(0.05f * z); + float n = 0.15f * (float)(((i * 1103515245u + 12345u) >> 16) & 0x7fff) / 32768.0f; + h_in[i] = 0.5f * s + n - 0.075f; + } + + float* d_in = nullptr; + HIPCHECK(hipMalloc(&d_in, nelem * sizeof(float))); + HIPCHECK(hipMemcpy(d_in, h_in.data(), nelem * sizeof(float), hipMemcpyHostToDevice)); + + // ---- Run RLE kernel (ground truth) ---- + const long rle_stride = 4L * WRLE_LDS_BYTES; + unsigned char* d_rle = nullptr; + size_t* d_rle_sizes = nullptr; + HIPCHECK(hipMalloc(&d_rle, (size_t)nblocks * rle_stride)); + HIPCHECK(hipMalloc(&d_rle_sizes, nblocks * sizeof(size_t))); + { + dim3 grid(nbx, nby, nbz); + waveletRLEFusedKernel<<>>( + d_in, d_rle, d_rle_sizes, mulfac, ldimx, ldimxy, nullptr, nullptr); + HIPCHECK(hipGetLastError()); + } + + // ---- Run bitmap kernel ---- + unsigned char* d_bmp = nullptr; + size_t* d_bmp_sizes = nullptr; + HIPCHECK(hipMalloc(&d_bmp, (size_t)nblocks * WBMP_SLOT_BYTES)); + HIPCHECK(hipMalloc(&d_bmp_sizes, nblocks * sizeof(size_t))); + HIPCHECK(hipWaveletBitmapFused(d_in, d_bmp, d_bmp_sizes, mulfac, + NX, NY, NZ, ldimx, ldimxy)); + HIPCHECK(hipDeviceSynchronize()); + + // ---- Copy results to host ---- + std::vector h_rle((size_t)nblocks * rle_stride); + std::vector h_bmp((size_t)nblocks * WBMP_SLOT_BYTES); + std::vector h_bmp_sizes(nblocks); + HIPCHECK(hipMemcpy(h_rle.data(), d_rle, h_rle.size(), hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_bmp.data(), d_bmp, h_bmp.size(), hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_bmp_sizes.data(), d_bmp_sizes, nblocks * sizeof(size_t), + hipMemcpyDeviceToHost)); + + // ---- Cross-check per block ---- + long total_nonzero = 0, total_coeffs = 0; + long mismatches = 0, hot_values = 0; + std::vector truth(COEFFS_PER_BLOCK); + std::vector recon(COEFFS_PER_BLOCK); + + for (int bid = 0; bid < nblocks; ++bid) { + // (a) Ground truth: decode the RLE stream. + const unsigned char* block = h_rle.data() + (long)bid * rle_stride; + const unsigned char* meta = block; + const unsigned char* rle = block + WRLE_META_PER_BLOCK; + int stream_offset = 0; + for (int x_off = 0; x_off < 4; ++x_off) { + int off = 0; + for (int tid = 0; tid < 256; ++tid) { + int my_bytes = meta[x_off * 256 + tid]; + int q[32]; + int got = decode_zline(rle + stream_offset + off, my_bytes, q, 32); + if (got != 32) { + printf(" block %d x_off %d tid %d: decoded %d != 32\n", + bid, x_off, tid, got); + return 1; + } + for (int z = 0; z < 32; ++z) { + truth[coeff_index(x_off, tid, z)] = q[z]; + if (q[z] != 0 && (q[z] > (1 << 23) || q[z] < -(1 << 23))) + ++hot_values; + } + off += my_bytes; + } + stream_offset += off; + } + + // (b) Reconstruct dense grid from bitmap + packed values. + const uint32_t* bmp = reinterpret_cast( + h_bmp.data() + (long)bid * WBMP_SLOT_BYTES); + const int32_t* vals = reinterpret_cast( + h_bmp.data() + (long)bid * WBMP_SLOT_BYTES + WBMP_BITMAP_BYTES); + long idx = 0, blk_nnz = 0; + for (int x_off = 0; x_off < 4; ++x_off) { + for (int tid = 0; tid < 256; ++tid) { + uint32_t mask = bmp[x_off * 256 + tid]; + blk_nnz += __builtin_popcount(mask); + for (int z = 0; z < 32; ++z) { + if (mask & (1u << z)) recon[coeff_index(x_off, tid, z)] = vals[idx++]; + else recon[coeff_index(x_off, tid, z)] = 0; + } + } + } + + // (c) block_sizes consistency: 4096 + nnz*4. + long exp_size = WBMP_BITMAP_BYTES + blk_nnz * 4; + if ((long)h_bmp_sizes[bid] != exp_size) { + printf(" block %d: block_size %ld != expected %ld (nnz=%ld)\n", + bid, (long)h_bmp_sizes[bid], exp_size, blk_nnz); + ++mismatches; + } + + // (d) exact coefficient equality. + for (int i = 0; i < COEFFS_PER_BLOCK; ++i) { + ++total_coeffs; + if (truth[i] != 0) ++total_nonzero; + if (truth[i] != recon[i]) { + if (mismatches < 20) + printf(" block %d coeff %d: truth %d != recon %d\n", + bid, i, truth[i], recon[i]); + ++mismatches; + } + } + } + + double nz_frac = total_coeffs ? (double)total_nonzero / total_coeffs : 0.0; + printf("coeffs=%ld nonzero=%ld (%.1f%%) hot(|v|>2^23)=%ld mismatches=%ld\n", + total_coeffs, total_nonzero, 100.0 * nz_frac, hot_values, mismatches); + if (hot_values > 0) + printf(" NOTE: hot values hit RLE VLESC4 float round-trip; lower mulfac " + "to keep |ival|<2^23 for an exact cross-check.\n"); + + hipFree(d_in); hipFree(d_rle); hipFree(d_rle_sizes); + hipFree(d_bmp); hipFree(d_bmp_sizes); + + if (mismatches != 0) { printf("FAIL\n"); return 1; } + printf("PASS\n"); + return 0; +} diff --git a/tests/test_bitmap_vs_cpu_rle.cpp b/tests/test_bitmap_vs_cpu_rle.cpp new file mode 100644 index 0000000..d078704 --- /dev/null +++ b/tests/test_bitmap_vs_cpu_rle.cpp @@ -0,0 +1,322 @@ +// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Use of this source code is governed by an MIT-style license that can be +// found in the LICENSE file or at https://opensource.org/licenses/MIT. +// +// CPU-only comparison of the production full-block RLE (Run_Length_Encode_Slow, +// with 8x packed escape codes) against the bitmap-significance-split, on +// identical CPU-wavelet-transformed, identically-quantized 32^3 blocks. +// +// Emits a four-bucket byte breakdown for both coders: +// RLE : zero-run structure | escape-tag overhead | value payload +// bitmap : significance (4096B) | width header | value payload (+padding) +// +// The instrumented RLE clone reproduces the exact control flow of +// Run_Length_Encode_Slow; its total is asserted against the real encoder. + +#include +#include +#include +#include + +#define MY_AVX_DEFINED +#define SIMDE_ENABLE_NATIVE_ALIASES +#include "simde/x86/avx512.h" + +#include "Block_Copy.hxx" +#include "Wavelet_Transform_Fast.hxx" +#include "Run_Length_Encode_Slow.hxx" +#include "Run_Length_Escape_Codes.hxx" + +// --------------------------------------------------------------------------- +// Instrumented clone of Run_Length_Encode_Slow (AVX / TMJ_AVX_RLE path). +// Adds byte-category counters at every write site. Byte format identical. +// --------------------------------------------------------------------------- +struct RleBuckets { long zero = 0, tag = 0, payload = 0; }; + +static inline void ib_rle(int& rle, char* dst, int& bp, RleBuckets& b) { + if (rle > 0) { + if (rle == 1) { dst[bp++] = (char)0; b.zero += 1; } + else if (rle < 256) { + int v = (RLESC1 & 0xFF) | ((rle & 0xFF) << 8); + *((short*)(dst + bp)) = (short)v; bp += 2; b.zero += 2; + } else { + int v = (RLESC3 & 0xFF) | ((rle & 0xFFFFFF) << 8); + *((int*)(dst + bp)) = v; bp += 4; b.zero += 4; + } + rle = 0; + } +} + +static inline void ib_word(int i, int zeros, int* esc, int* pay, int* nb, + int& rle, char* dst, int& bp, RleBuckets& b) { + if (zeros & (1 << i)) { ++rle; return; } + ib_rle(rle, dst, bp, b); + long rval = (long)esc[i] | ((long)pay[i] << 8); + *((long*)(dst + bp)) = rval; + int n = nb[i]; bp += n; + if (n == 1) b.payload += 1; // byte value carries no tag + else { b.tag += 1; b.payload += (n - 1); } // esc + (n-1) payload bytes +} + +static inline int ib_count_true(__m256 predicate) { + __m128i sum = _mm_hadd_epi32(_mm256_castsi256_si128(_mm256_castps_si256(predicate)), + _mm256_extractf128_si256(_mm256_castps_si256(predicate), 1)); + sum = _mm_hadd_epi32(sum, sum); + sum = _mm_hadd_epi32(sum, sum); + return -_mm_extract_epi32(sum, 0); +} + +// Returns total bytes; fills buckets. Mirrors Run_Length_Encode_Slow exactly. +static int rle_breakdown(float scale, float* vals, int num, char* dst, RleBuckets& b) { + int rle = 0, bp = 0; + __m256 _mm_scale = _mm256_set1_ps(scale); + __m256 _mm_byte_lo = _mm256_cvtepi32_ps(_mm256_set1_epi32(VLESC2)); + __m256 _mm_byte_hi = _mm256_cvtepi32_ps(_mm256_set1_epi32(RLESC3)); + __m256 _mm_short_lo = _mm256_cvtepi32_ps(_mm256_set1_epi32(-32768)); + __m256 _mm_short_hi = _mm256_cvtepi32_ps(_mm256_set1_epi32(32767)); + __m256 _mm_i3_lo = _mm256_cvtepi32_ps(_mm256_set1_epi32(-8388608)); + __m256 _mm_i3_hi = _mm256_cvtepi32_ps(_mm256_set1_epi32(8388607)); + for (int i = 0; i < num; i += 8) { + __m256 fvals = _mm256_mul_ps(_mm_scale, _mm256_load_ps(vals + i)); + __m256i ivals = _mm256_cvttps_epi32(fvals); + __m256 fivals = _mm256_cvtepi32_ps(ivals); + __m256 is_zero = _mm256_cmp_ps(fivals, _mm256_setzero_ps(), 0); + int zeros = _mm256_movemask_ps(is_zero); + if (zeros == 255) { rle += 8; continue; } + + __m256 is_byte = _mm256_and_ps(_mm256_cmp_ps(fivals, _mm_byte_lo, 30), + _mm256_cmp_ps(fivals, _mm_byte_hi, 17)); + if (zeros == 0 && _mm256_movemask_ps(is_byte) == 255) { + ib_rle(rle, dst, bp, b); + bp += 8; b.payload += 8; // 8 raw bytes, no tag + continue; + } + int num_bytes = ib_count_true(is_byte); + __m256 is_short = _mm256_and_ps(_mm256_cmp_ps(fivals, _mm_short_lo, 29), + _mm256_cmp_ps(fivals, _mm_short_hi, 18)); + if (zeros == 0 && _mm256_movemask_ps(is_short) == 255 && + (num_bytes + (8 - num_bytes) * 3) > 17) { + ib_rle(rle, dst, bp, b); + bp += 17; b.tag += 1; b.payload += 16; // VLESC2_8x + continue; + } + int num_shorts = ib_count_true(is_short); + __m256 is_i3 = _mm256_and_ps(_mm256_cmp_ps(fivals, _mm_i3_lo, 29), + _mm256_cmp_ps(fivals, _mm_i3_hi, 18)); + if (zeros == 0 && _mm256_movemask_ps(is_i3) == 255 && + (num_bytes + (num_shorts - num_bytes) * 3 + (8 - num_shorts) * 4) > 25) { + ib_rle(rle, dst, bp, b); + bp += 25; b.tag += 1; b.payload += 24; // VLESC3_8x + continue; + } + is_i3 = _mm256_andnot_ps(is_short, is_i3); + is_short = _mm256_andnot_ps(is_byte, is_short); + is_byte = _mm256_andnot_ps(is_zero, is_byte); + __m256 is_not_float = _mm256_or_ps(is_zero, _mm256_or_ps(is_byte, _mm256_or_ps(is_short, is_i3))); + + __m256 esc = _mm256_and_ps(is_byte, _mm256_and_ps(_mm256_castsi256_ps(_mm256_set1_epi32(0xFF)), + _mm256_castsi256_ps(ivals))); + esc = _mm256_or_ps(esc, _mm256_and_ps(is_short, _mm256_castsi256_ps(_mm256_set1_epi32(VLESC2 & 0xFF)))); + esc = _mm256_or_ps(esc, _mm256_and_ps(is_i3, _mm256_castsi256_ps(_mm256_set1_epi32(VLESC3 & 0xFF)))); + esc = _mm256_or_ps(esc, _mm256_andnot_ps(is_not_float, _mm256_castsi256_ps(_mm256_set1_epi32(VLESC4 & 0xFF)))); + __m256 payload = _mm256_and_ps(_mm256_or_ps(is_short, is_i3), _mm256_castsi256_ps(ivals)); + payload = _mm256_or_ps(payload, _mm256_andnot_ps(is_not_float, fvals)); + __m256 nbytes = _mm256_and_ps(is_byte, _mm256_castsi256_ps(_mm256_set1_epi32(1))); + nbytes = _mm256_or_ps(nbytes, _mm256_and_ps(is_short, _mm256_castsi256_ps(_mm256_set1_epi32(3)))); + nbytes = _mm256_or_ps(nbytes, _mm256_and_ps(is_i3, _mm256_castsi256_ps(_mm256_set1_epi32(4)))); + nbytes = _mm256_or_ps(nbytes, _mm256_andnot_ps(is_not_float, _mm256_castsi256_ps(_mm256_set1_epi32(5)))); + + int* p_esc = (int*)(&esc); + int* p_payload = (int*)(&payload); + int* p_nbytes = (int*)(&nbytes); + for (int k = 0; k < 8; ++k) + ib_word(k, zeros, p_esc, p_payload, p_nbytes, rle, dst, bp, b); + } + ib_rle(rle, dst, bp, b); + return bp; +} + +// Bitmap fixed-width per-value minimal byte width (no in-band escape collision). +static inline int bmp_width(int maxabs) { + if (maxabs <= 0x7f) return 1; + if (maxabs <= 0x7fff) return 2; + if (maxabs <= 0x7fffff) return 3; + return 4; +} + +int main(int argc, char** argv) +{ + const int NX = (argc > 1) ? atoi(argv[1]) : 128; + const int NY = NX, NZ = NX; + const float mulfac = (argc > 2) ? (float)atof(argv[2]) : 20.0f; + const int bx = 32, by = 32, bz = 32, bsz = bx * by * bz; + if (NX % 32) { printf("NX must be multiple of 32\n"); return 1; } + + const int nbx = NX/32, nby = NY/32, nbz = NZ/32, nblocks = nbx*nby*nbz; + const size_t nelem = (size_t)NX*NY*NZ; + printf("bitmap-vs-CPU-RLE: %dx%dx%d nblocks=%d mulfac=%.3f\n", NX,NY,NZ,nblocks,mulfac); + + float* vol = (float*)malloc(nelem * sizeof(float)); + for (size_t i = 0; i < nelem; ++i) { + int x=(int)(i%NX), y=(int)((i/NX)%NY), z=(int)(i/((size_t)NX*NY)); + float s = sinf(0.11f*x)*cosf(0.07f*y)*sinf(0.05f*z); + float n = 0.15f*(float)(((i*1103515245u+12345u)>>16)&0x7fff)/32768.0f; + vol[i] = 0.5f*s + n - 0.075f; + } + + float* block; posix_memalign((void**)&block, 64, sizeof(float)*bsz); + float* tmp; posix_memalign((void**)&tmp, 64, sizeof(float)*bx*8); + unsigned long* comp; posix_memalign((void**)&comp, 64, sizeof(float)*bsz*2); + char* scratch = (char*)malloc(bsz*5); + + // RLE accumulators + RleBuckets rle_b; long rle_total = 0; + // bitmap accumulators + long bmp_sig = 0, bmp_hdr = 0, bmp_val = 0, bmp_val_min = 0; + long bmp_pl_pay = 0; // per-line-width payload + long total_nnz = 0; + long total_nonempty = 0; // z-lines with >=1 nonzero (mask != 0) + long bmp_rle_runtok = 0; // zero-word-RLE control-token bytes + long mism = 0; + long w_hist[5] = {0}; + + for (int ibz=0; ibzmaxabs) maxabs=a; + val_min += bmp_width(a); + } + } + int W = bmp_width(maxabs); + if (nnz==0) W=0; else w_hist[W]++; + total_nnz += nnz; + bmp_sig += 4096; + bmp_hdr += 4; + bmp_val += (long)nnz * W; + bmp_val_min += val_min; + + // per-line width: one W per z-line (fixed (ix,iy), z inner). + // block raster: block[iz*xy + iy*bx + ix], xy = bx*by. + // Also gather bitmap-compression stats: occupancy (mask!=0 per line) + // and zero-word-RLE run tokens over raster line order. + int xy = bx * by; + int prev = -1, nzero_runs = 0, nnz_runs = 0, blk_nonempty = 0; + for (int iy=0; iylmax) lmax=a; } + } + int Wl = lnnz ? bmp_width(lmax) : 0; + bmp_pl_pay += (long)lnnz * Wl; + int occ = lnnz > 0 ? 1 : 0; + blk_nonempty += occ; + if (occ != prev) { if (occ) ++nnz_runs; else ++nzero_runs; prev = occ; } + } + total_nonempty += blk_nonempty; + // zero-run token = 2 B (count up to 1024), nonzero-run token = 1 B (len). + bmp_rle_runtok += (long)nzero_runs * 2 + (long)nnz_runs * 1; + } + + long hdr = 8 + 8L*nblocks + 4; + long rle_all = rle_b.zero + rle_b.tag + rle_b.payload + hdr; + long bmp_all = bmp_sig + bmp_hdr + bmp_val + hdr; + double raw = (double)nelem*4.0; + + printf("\n--- CPU full-block RLE (proper path, 8x codes) ---\n"); + printf(" zero-run structure : %10ld B (%.1f%%)\n", rle_b.zero, 100.0*rle_b.zero/rle_all); + printf(" escape-tag overhead: %10ld B (%.1f%%)\n", rle_b.tag, 100.0*rle_b.tag/rle_all); + printf(" value payload : %10ld B (%.1f%%)\n", rle_b.payload, 100.0*rle_b.payload/rle_all); + printf(" header : %10ld B\n", hdr); + printf(" TOTAL : %10ld B CR=%.3f\n", rle_all, raw/rle_all); + + printf("\n--- bitmap + per-block fixed width ---\n"); + printf(" significance bitmap: %10ld B (%.1f%%) [%d B/block, flat]\n", + bmp_sig, 100.0*bmp_sig/bmp_all, 4096); + printf(" nnz metadata : %10ld B (derived from popcount)\n", 0L); + printf(" width header : %10ld B\n", bmp_hdr); + printf(" value payload : %10ld B (%.1f%%) [minimal=%ld, padding=%ld]\n", + bmp_val, 100.0*bmp_val/bmp_all, bmp_val_min, bmp_val - bmp_val_min); + printf(" header : %10ld B\n", hdr); + printf(" TOTAL : %10ld B CR=%.3f\n", bmp_all, raw/bmp_all); + + // per-line width variant: sig 4096/blk + 2-bit width table (1024 lines*2b + // = 256 B/blk) + per-line-width payload. + long bmp_pl_wtab = 256L * nblocks; + long bmp_pl_all = bmp_sig + bmp_pl_wtab + bmp_pl_pay + hdr; + printf("\n--- bitmap + PER-LINE width (2-bit width table) ---\n"); + printf(" significance bitmap: %10ld B (%.1f%%)\n", bmp_sig, 100.0*bmp_sig/bmp_pl_all); + printf(" width table (2b/line): %8ld B (%.1f%%) [256 B/block]\n", + bmp_pl_wtab, 100.0*bmp_pl_wtab/bmp_pl_all); + printf(" value payload : %10ld B (%.1f%%) [minimal=%ld, padding=%ld]\n", + bmp_pl_pay, 100.0*bmp_pl_pay/bmp_pl_all, bmp_val_min, bmp_pl_pay - bmp_val_min); + printf(" TOTAL : %10ld B CR=%.3f vs RLE ratio=%.3f\n", + bmp_pl_all, raw/bmp_pl_all, (double)bmp_pl_all/rle_all); + + // --- Compressed-bitmap variants (both keep per-line width payload) --- + // Width table now covers only NONEMPTY lines (2 bits each); empty lines + // carry no value and no width. + long wtab_ne = (2L * total_nonempty + 7) / 8; + + // (A) two-level: 128 B/block line-occupancy bitmap + 4 B per nonempty mask. + long tl_sig = 128L * nblocks + 4L * total_nonempty; + long tl_all = tl_sig + wtab_ne + bmp_pl_pay + hdr; + printf("\n--- bitmap TWO-LEVEL occupancy + per-line width ---\n"); + printf(" occupancy(128B/blk)+masks: %ld B [%ld occ + %ld masks]\n", + tl_sig, 128L*nblocks, 4L*total_nonempty); + printf(" width table (2b/nonempty): %ld B\n", wtab_ne); + printf(" value payload : %10ld B [minimal=%ld]\n", bmp_pl_pay, bmp_val_min); + printf(" TOTAL : %10ld B CR=%.3f vs RLE ratio=%.3f\n", + tl_all, raw/tl_all, (double)tl_all/rle_all); + + // (B) zero-word RLE: run tokens + 4 B per nonempty mask. + long rl_sig = bmp_rle_runtok + 4L * total_nonempty; + long rl_all = rl_sig + wtab_ne + bmp_pl_pay + hdr; + printf("\n--- bitmap ZERO-WORD RLE + per-line width ---\n"); + printf(" runtokens+masks : %10ld B [%ld tokens + %ld masks]\n", + rl_sig, bmp_rle_runtok, 4L*total_nonempty); + printf(" width table (2b/nonempty): %ld B\n", wtab_ne); + printf(" value payload : %10ld B\n", bmp_pl_pay); + printf(" TOTAL : %10ld B CR=%.3f vs RLE ratio=%.3f\n", + rl_all, raw/rl_all, (double)rl_all/rle_all); + printf(" nonempty z-lines : %ld / %ld (%.1f%%, %.1f/block)\n", + total_nonempty, 1024L*nblocks, 100.0*total_nonempty/(1024.0*nblocks), + (double)total_nonempty/nblocks); + + printf("\nnonzero = %ld (%.1f%%) W hist: 1B=%ld 2B=%ld 3B=%ld 4B=%ld\n", + total_nnz, 100.0*total_nnz/((double)nblocks*bsz), + w_hist[1],w_hist[2],w_hist[3],w_hist[4]); + printf("bitmap/RLE size ratio = %.3f (<1 = bitmap smaller)\n", (double)bmp_all/rle_all); + printf("structure: RLE(zero+tag)=%ld vs bitmap(sig)=%ld\n", + rle_b.zero+rle_b.tag, bmp_sig); + printf("payload: RLE=%ld vs bitmap=%ld (bitmap minimal=%ld)\n", + rle_b.payload, bmp_val, bmp_val_min); + + free(vol); free(block); free(tmp); free(comp); free(scratch); + if (mism) { printf("\nInstrumented total mismatch in %ld blocks\nFAIL\n", mism); return 1; } + printf("\nPASS (instrumented RLE total matches real encoder)\n"); + return 0; +} From 55547ec62217127ee6cfdf887d47bf007e77e3ad Mon Sep 17 00:00:00 2001 From: Ossian O'Reilly Date: Fri, 14 Aug 2026 15:45:56 -0700 Subject: [PATCH 2/7] Octree significance codec + full decode pipeline; fix missing unroll flag Fix the missing -mllvm -unroll-threshold=10000 on the test_bitmap_octree_hip and test_wavelet_buffer_hip build rules. Without it the ds79 Z-transform's tmp[32] + dynamic mirror-index loops spilled to scratch; adding it keeps them in registers and raises the forward/inverse transform from ~460 GB/s to ~2.3-2.4 TB/s (4x), which dominates full encode/decode. Add the octree significance coder (DFS reference, parallel level-major, fused kernel-2) and a full decode pipeline: stage-A (coded stream -> bitmap+int32, byte-exact inverse of the fused encoder) and stage-B (bitmap+values -> dequant + inverse wavelet). Round-trip is byte-exact vs kernel-1 and the reconstructed field is bit-identical to the RLE decode. Benchmark vs RLE on real panels (512^3, scale 8): octree CR 244x vs 36.8x (s1000) / 60x vs 26x (s3000) at identical distortion; full decode 1.6-1.9x faster; encode within ~5-9%. --- hip/hipWaveletOctree.h | 1001 ++++++++++++++++++++++++++++++ makefile | 10 +- tests/test_bitmap_octree_hip.cpp | 378 +++++++++++ 3 files changed, 1388 insertions(+), 1 deletion(-) create mode 100644 hip/hipWaveletOctree.h create mode 100644 tests/test_bitmap_octree_hip.cpp diff --git a/hip/hipWaveletOctree.h b/hip/hipWaveletOctree.h new file mode 100644 index 0000000..5ffd22c --- /dev/null +++ b/hip/hipWaveletOctree.h @@ -0,0 +1,1001 @@ +// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Use of this source code is governed by an MIT-style license that can be +// found in the LICENSE file or at https://opensource.org/licenses/MIT. +// +// PROTOTYPE: octree significance coder (kernel-2 variant). Replaces the flat +// 4096 B/block significance bitmap (or the two-level occupancy+masks) with a +// full 3D octree of the 32^3 significance set: quadtree in (x,y) x bisection in +// z, depth 5 (node sizes 32,16,8,4,2), 8 bits (one per child) per non-empty +// node, serialized in a fixed DFS pre-order. A per-block 1-byte mode tag falls +// back to the flat bitmap when the octree would be larger (dense blocks). +// +// The encode/decode core is a single __host__ __device__ function with an +// explicit depth-5 stack, so the CPU reference and the GPU kernel produce +// byte-for-byte identical streams by construction. Both operate on the 32x32 +// z-line masks produced by waveletBitmapFusedKernel (kernel 1). +// +// Coded block layout: [1B mode][significance][2b/nonempty-line width table] +// [per-line packed values] +// mode 0 (flat) : significance = 1024 uint32 masks in L order (4096 B) +// mode 2 (octree) : significance = variable-length octree stream +// The width table and value payload are identical to the two-level coder, so +// only the significance representation is new here. + +#ifndef HIPWAVELET_OCTREE_H +#define HIPWAVELET_OCTREE_H + +#include +#include "hipWaveletBitmap.h" // WBMP_* layout constants + +// Worst-case (fully dense 32^3) octree node count = 4096+512+64+8+1 = 4681. +static constexpr int WOCT_MAX_SIG_BYTES = 4681; +static constexpr int WOCT_FLAT_SIG_BYTES = WBMP_BITMAP_BYTES; // 4096 +// 4-byte mode header keeps the significance region 4-aligned (the flat variant +// stores 1024 uint32 masks; unaligned uint32 access breaks on device). +static constexpr int WOCT_HDR_BYTES = 4; +// Coded slot: header + max significance + width table + max packed values, +// rounded up to 16 B so every per-block slot (and thus the 4-aligned +// significance region) stays aligned for uint32 flat masks / vectorized stores. +static constexpr long WOCT_SLOT_RAW = + WOCT_HDR_BYTES + (WOCT_MAX_SIG_BYTES > WOCT_FLAT_SIG_BYTES ? WOCT_MAX_SIG_BYTES : WOCT_FLAT_SIG_BYTES) + + WBMP_WTAB_BYTES + WBMP_MAX_VAL_BYTES; +static constexpr long WOCT_SLOT_BYTES = (WOCT_SLOT_RAW + 15) & ~15L; + +// Fused kernel-2 coded slot: [4B mode][significance][<=3B pad to 4-align the +// width table][2b/nonempty-line width table][per-line packed values]. +// sig = octree stream (<=WOCT_MAX_SIG_BYTES) or flat masks (4096 B) +// values = same per-line variable-width payload as the two-level coder +static constexpr long WOCT_CODE_SLOT_RAW = + WOCT_HDR_BYTES + WOCT_MAX_SIG_BYTES + 3 + WBMP_WTAB_BYTES + WBMP_MAX_VAL_BYTES; +static constexpr long WOCT_CODE_SLOT_BYTES = (WOCT_CODE_SLOT_RAW + 15) & ~15L; + +// Kernel-1 bitmap word order L=x_off*256+tid maps to spatial line (ix,iy): +// tid = L & 255 ; xg = tid & 7 ; yr = tid >> 3 ; ix = xg*4 + (L>>8) ; iy = yr +__host__ __device__ __forceinline__ int woct_L_to_spatial(int L) { + int x_off = L >> 8, tid = L & 255, xg = tid & 7, yr = tid >> 3; + return yr * 32 + (xg * 4 + x_off); // spatial index iy*32+ix +} + +// True if any significant voxel lies in the [ox,ox+cs) x [oy,oy+cs) x [oz,oz+cs) +// subcube of the block, given the 32 spatial z-line masks m[iy*32+ix]. +__host__ __device__ __forceinline__ bool +woct_subcube_any(const uint32_t* m, int ox, int oy, int oz, int cs) { + uint32_t zmask = (cs >= 32) ? 0xFFFFFFFFu + : (uint32_t)((((uint64_t)1 << cs) - 1) << oz); + for (int iy = oy; iy < oy + cs; ++iy) + for (int ix = ox; ix < ox + cs; ++ix) + if (m[iy * 32 + ix] & zmask) return true; + return false; +} + +// 8-bit child-occupancy of a node of the given size at (px,py,pz). child index +// c = cx | cy<<1 | cz<<2 ; child subcube side = side/2. +__host__ __device__ __forceinline__ int +woct_child_occ(const uint32_t* m, int px, int py, int pz, int side) { + int cs = side >> 1, b = 0; + for (int c = 0; c < 8; ++c) { + int cx = c & 1, cy = (c >> 1) & 1, cz = (c >> 2) & 1; + if (woct_subcube_any(m, px + cx * cs, py + cy * cs, pz + cz * cs, cs)) b |= (1 << c); + } + return b; +} + +// Explicit depth-<=5 DFS stack kept as parallel scalar arrays (no struct +// references) so the device compiler serializes it correctly. +// Encode the 32^3 significance (spatial z-line masks) into a DFS-preorder octree +// byte stream. Returns the number of bytes written (0 for an empty block). +__host__ __device__ __forceinline__ int +woct_encode(const uint32_t* m, unsigned char* out) { + int rootb = woct_child_occ(m, 0, 0, 0, 32); + if (rootb == 0) return 0; // empty block -> zero-length stream + int px[8], py[8], pz[8], side[8], bb[8], ch[8]; + int sp = 0, pos = 0; + out[pos++] = (unsigned char)rootb; + px[0]=0; py[0]=0; pz[0]=0; side[0]=32; bb[0]=rootb; ch[0]=0; sp = 1; + while (sp > 0) { + int i = sp - 1; + int cs = side[i] >> 1; + int found = -1; + if (cs > 1) { // size-2 nodes carry voxels, no node children + for (int c = ch[i]; c < 8; ++c) + if (bb[i] & (1 << c)) { found = c; break; } + } + if (found >= 0) { + ch[i] = found + 1; + int cx = found & 1, cy = (found >> 1) & 1, cz = (found >> 2) & 1; + int ox = px[i] + cx * cs, oy = py[i] + cy * cs, oz = pz[i] + cz * cs; + int b = woct_child_occ(m, ox, oy, oz, cs); + out[pos++] = (unsigned char)b; + px[sp]=ox; py[sp]=oy; pz[sp]=oz; side[sp]=cs; bb[sp]=b; ch[sp]=0; ++sp; + } else { + --sp; + } + } + return pos; +} + +// Inverse of woct_encode: reconstruct the spatial z-line masks m[iy*32+ix] +// (which the caller must pre-zero). sigbytes==0 means an empty block. +__host__ __device__ __forceinline__ void +woct_decode(const unsigned char* in, int sigbytes, uint32_t* m) { + if (sigbytes == 0) return; + int px[8], py[8], pz[8], side[8], bb[8], ch[8]; + int sp = 0, pos = 0; + px[0]=0; py[0]=0; pz[0]=0; side[0]=32; bb[0]=in[pos++]; ch[0]=0; sp = 1; + while (sp > 0) { + int i = sp - 1; + int cs = side[i] >> 1; + int found = -1; + for (int c = ch[i]; c < 8; ++c) + if (bb[i] & (1 << c)) { found = c; break; } + if (found < 0) { --sp; continue; } + ch[i] = found + 1; + int cx = found & 1, cy = (found >> 1) & 1, cz = (found >> 2) & 1; + int ox = px[i] + cx * cs, oy = py[i] + cy * cs, oz = pz[i] + cz * cs; + if (cs == 1) { // voxel leaf of a size-2 node + m[oy * 32 + ox] |= (1u << oz); + } else { + px[sp]=ox; py[sp]=oy; pz[sp]=oz; side[sp]=cs; bb[sp]=in[pos++]; ch[sp]=0; ++sp; + } + } +} + +// =========================================================================== +// Level-major (parallel-friendly) octree format. +// +// The DFS pre-order above is inherently serial to both emit and parse. The +// level-major layout below encodes the *same node set* (so the byte count, and +// thus the compression ratio, is byte-for-byte identical to woct_encode) but in +// breadth-first order: all size-16 node bytes, then all size-8, size-4, size-2, +// each level's present nodes listed in canonical linear index (x fastest). +// That order is producible/consumable in parallel via per-level prefix sums, +// which the GPU kernels below exploit. These host references define the +// canonical stream and serve as the byte-exact oracle for the parallel kernels. +// +// Node grids (occupancy): g16[2^3], g8[4^3], g4[8^3], g2[16^3]; strides x=1, +// y=dim, z=dim^2. voxel(x,y,z) = (m[y*32+x]>>z)&1. +// --------------------------------------------------------------------------- + +// Occupancy grids for the four interior levels (1 = subcube contains a voxel). +struct WoctGrids { unsigned char g16[8], g8[64], g4[512], g2[4096]; int rootocc; }; + +static inline void woct_build_grids(const uint32_t* m, WoctGrids& g) { + // g2: OR over each 2x2x2 voxel cube. + for (int k = 0; k < 16; ++k) for (int j = 0; j < 16; ++j) for (int i = 0; i < 16; ++i) { + int occ = 0; + for (int c = 0; c < 8 && !occ; ++c) { + int x = 2*i + (c&1), y = 2*j + ((c>>1)&1), z = 2*k + ((c>>2)&1); + occ = (m[y*32+x] >> z) & 1u; + } + g.g2[k*256 + j*16 + i] = (unsigned char)occ; + } + // g4,g8,g16: OR over each 2x2x2 subcube of the finer grid. + auto pool = [](const unsigned char* fine, int fdim, unsigned char* coarse, int cdim) { + for (int k = 0; k < cdim; ++k) for (int j = 0; j < cdim; ++j) for (int i = 0; i < cdim; ++i) { + int occ = 0; + for (int c = 0; c < 8 && !occ; ++c) { + int x = 2*i + (c&1), y = 2*j + ((c>>1)&1), z = 2*k + ((c>>2)&1); + occ = fine[z*fdim*fdim + y*fdim + x]; + } + coarse[k*cdim*cdim + j*cdim + i] = (unsigned char)occ; + } + }; + pool(g.g2, 16, g.g4, 8); + pool(g.g4, 8, g.g8, 4); + pool(g.g8, 4, g.g16, 2); + g.rootocc = 0; + for (int n = 0; n < 8; ++n) g.rootocc |= g.g16[n]; +} + +// 8-bit child-occupancy byte of node (Ix,Iy,Iz) whose children live in the +// finer grid `fine` of dimension fdim (child at 2I+c). +static inline int woct_byte_from_grid(const unsigned char* fine, int fdim, + int Ix, int Iy, int Iz) { + int b = 0; + for (int c = 0; c < 8; ++c) { + int x = 2*Ix + (c&1), y = 2*Iy + ((c>>1)&1), z = 2*Iz + ((c>>2)&1); + if (fine[z*fdim*fdim + y*fdim + x]) b |= (1 << c); + } + return b; +} + +// Level-major encode. Returns bytes written (== woct_encode's count). +static inline int woct_encode_lm(const uint32_t* m, unsigned char* out) { + WoctGrids g; woct_build_grids(m, g); + if (!g.rootocc) return 0; + int pos = 0; + // root byte = size-16 children occupancy. + out[pos++] = (unsigned char)woct_byte_from_grid(g.g16, 2, 0, 0, 0); + // size-16 nodes -> bytes over g8. + for (int n = 0; n < 8; ++n) if (g.g16[n]) { + int Ix = n&1, Iy = (n>>1)&1, Iz = (n>>2)&1; + out[pos++] = (unsigned char)woct_byte_from_grid(g.g8, 4, Ix, Iy, Iz); + } + // size-8 nodes -> bytes over g4. + for (int n = 0; n < 64; ++n) if (g.g8[n]) { + int Ix = n&3, Iy = (n>>2)&3, Iz = (n>>4)&3; + out[pos++] = (unsigned char)woct_byte_from_grid(g.g4, 8, Ix, Iy, Iz); + } + // size-4 nodes -> bytes over g2. + for (int n = 0; n < 512; ++n) if (g.g4[n]) { + int Ix = n&7, Iy = (n>>3)&7, Iz = (n>>6)&7; + out[pos++] = (unsigned char)woct_byte_from_grid(g.g2, 16, Ix, Iy, Iz); + } + // size-2 nodes -> voxel bytes (children are the voxels themselves). + for (int n = 0; n < 4096; ++n) if (g.g2[n]) { + int Ix = n&15, Iy = (n>>4)&15, Iz = (n>>8)&15, b = 0; + for (int c = 0; c < 8; ++c) { + int x = 2*Ix + (c&1), y = 2*Iy + ((c>>1)&1), z = 2*Iz + ((c>>2)&1); + if ((m[y*32+x] >> z) & 1u) b |= (1 << c); + } + out[pos++] = (unsigned char)b; + } + return pos; +} + +// Inverse of woct_encode_lm. Caller pre-zeros m. +static inline void woct_decode_lm(const unsigned char* in, int sigbytes, uint32_t* m) { + if (sigbytes == 0) return; + unsigned char g16[8]={0}, g8[64]={0}, g4[512]={0}, g2[4096]={0}; + int pos = 0; + // root -> size-16 present set. + { int b = in[pos++]; for (int c = 0; c < 8; ++c) if (b & (1< size-8 present set. + for (int n = 0; n < 8; ++n) if (g16[n]) { + int Ix = n&1, Iy = (n>>1)&1, Iz = (n>>2)&1, b = in[pos++]; + for (int c = 0; c < 8; ++c) if (b & (1<>1)&1), z = 2*Iz+((c>>2)&1); + g8[z*16 + y*4 + x] = 1; + } + } + // size-8 bytes -> size-4 present set. + for (int n = 0; n < 64; ++n) if (g8[n]) { + int Ix = n&3, Iy = (n>>2)&3, Iz = (n>>4)&3, b = in[pos++]; + for (int c = 0; c < 8; ++c) if (b & (1<>1)&1), z = 2*Iz+((c>>2)&1); + g4[z*64 + y*8 + x] = 1; + } + } + // size-4 bytes -> size-2 present set. + for (int n = 0; n < 512; ++n) if (g4[n]) { + int Ix = n&7, Iy = (n>>3)&7, Iz = (n>>6)&7, b = in[pos++]; + for (int c = 0; c < 8; ++c) if (b & (1<>1)&1), z = 2*Iz+((c>>2)&1); + g2[z*256 + y*16 + x] = 1; + } + } + // size-2 bytes -> voxels. + for (int n = 0; n < 4096; ++n) if (g2[n]) { + int Ix = n&15, Iy = (n>>4)&15, Iz = (n>>8)&15, b = in[pos++]; + for (int c = 0; c < 8; ++c) if (b & (1<>1)&1), z = 2*Iz+((c>>2)&1); + m[y*32+x] |= (1u << z); + } + } +} + +// --------------------------------------------------------------------------- +// GPU kernel: per-block octree significance encode with flat fallback. +// Reads kernel-1 output [4096B bitmap][packed int32]; writes +// out[bid*WOCT_SLOT_BYTES] = [1B mode][significance...] and records the +// significance length in sig_sizes[bid]. (Width table + values are appended by +// a separate pass reusing the two-level packer; omitted from this prototype, +// which validates the significance round-trip.) +// +// The volume gather is parallel; the DFS serialization is done by thread 0 so +// the stream is byte-identical to the host reference (prototype: not yet a +// parallel serializer). +// --------------------------------------------------------------------------- +__global__ void waveletOctreeSigEncodeKernel( + const unsigned char* __restrict__ scratch1, + unsigned char* __restrict__ out, + size_t* __restrict__ sig_sizes) +{ + int bid = blockIdx.x, tid = threadIdx.x; + const uint32_t* bmp = reinterpret_cast(scratch1 + (long)bid * WBMP_SLOT_BYTES); + + __shared__ uint32_t masks_sp[1024]; + for (int L = tid; L < 1024; L += blockDim.x) masks_sp[woct_L_to_spatial(L)] = bmp[L]; + __syncthreads(); + + if (tid == 0) { + unsigned char* blk = out + (long)bid * WOCT_SLOT_BYTES; + int ob = woct_encode(masks_sp, blk + WOCT_HDR_BYTES); + if (ob < WOCT_FLAT_SIG_BYTES) { + blk[0] = 2; // octree + sig_sizes[bid] = (size_t)ob; + } else { + uint32_t* flat = reinterpret_cast(blk + WOCT_HDR_BYTES); + for (int L = 0; L < 1024; ++L) flat[L] = bmp[L]; // flat masks in L order + blk[0] = 0; // flat + sig_sizes[bid] = (size_t)WOCT_FLAT_SIG_BYTES; + } + } +} + +// GPU kernel: decode significance -> reconstruct 1024 z-line masks in L order +// into out_masks[bid*1024 + L] for byte-exact comparison against the bitmap. +__global__ void waveletOctreeSigDecodeKernel( + const unsigned char* __restrict__ coded, + const size_t* __restrict__ sig_sizes, + uint32_t* __restrict__ out_masks) +{ + int bid = blockIdx.x, tid = threadIdx.x; + const unsigned char* blk = coded + (long)bid * WOCT_SLOT_BYTES; + + __shared__ uint32_t masks_sp[1024]; + for (int L = tid; L < 1024; L += blockDim.x) masks_sp[L] = 0; + __syncthreads(); + + if (tid == 0) { + int mode = blk[0]; + if (mode == 0) { + const uint32_t* flat = reinterpret_cast(blk + WOCT_HDR_BYTES); + for (int L = 0; L < 1024; ++L) masks_sp[woct_L_to_spatial(L)] = flat[L]; + } else { + woct_decode(blk + WOCT_HDR_BYTES, (int)sig_sizes[bid], masks_sp); + } + } + __syncthreads(); + + for (int L = tid; L < 1024; L += blockDim.x) + out_masks[(long)bid * 1024 + L] = masks_sp[woct_L_to_spatial(L)]; +} + +// =========================================================================== +// Parallel level-major octree kernels (1024 threads/block). +// +// Both kernels build the occupancy pyramid (o16/o8/o4/o2) cooperatively in LDS, +// then use per-level inclusive prefix sums to place / locate each present node's +// byte in the level-major stream defined by woct_encode_lm. The byte count (and +// thus CR) is identical to the DFS serial coder; only the emission order and the +// degree of parallelism differ. WOCT_PAR_THREADS must be 1024. +// =========================================================================== +static constexpr int WOCT_PAR_THREADS = 1024; + +// In-place inclusive Hillis-Steele scan of a[0..N) using 1024 threads (up to 4 +// elements/thread; N<=4096). Two syncs/step make it safe in place. +__device__ __forceinline__ void woct_incl_scan(int* a, int N) { + int tid = threadIdx.x; + for (int off = 1; off < N; off <<= 1) { + int v0=0,v1=0,v2=0,v3=0; + int i0=tid,i1=tid+1024,i2=tid+2048,i3=tid+3072; + if (i0=off) v0=a[i0-off]; + if (i1=off) v1=a[i1-off]; + if (i2=off) v2=a[i2-off]; + if (i3=off) v3=a[i3-off]; + __syncthreads(); + if (i0r[0..N) (lidx order), N in {8,64,512,4096}. +// Uses a blocked layout (thread t owns the contiguous chunk [t*C, t*C+C), C=N/1024 +// rounded up) so a single wave64 DPP block scan (wbmp_opt::block_exscan) replaces +// the ~log2(N) Hillis-Steele passes. r[n] holds the inclusive prefix at n exactly +// as woct_incl_scan, so downstream (rank = r[n]-occ[n], count = total) is +// unchanged. Returns the total (== popcount of occ). warp_part is 2*16 ints. +__device__ __forceinline__ int +woct_incl_scan_fast(const int* occ, int* r, int N, int* warp_part) { + int tid = threadIdx.x; + int C = (N + 1023) >> 10; // 1 (N<=1024) or 4 (N==4096) + int base = tid * C; + int l0=0,l1=0,l2=0,l3=0,lsum=0; + if (C == 1) { + if (base < N) { lsum = occ[base]; l0 = lsum; } + } else { // C == 4 + if (base+0 < N) { lsum += occ[base+0]; l0 = lsum; } + if (base+1 < N) { lsum += occ[base+1]; l1 = lsum; } + if (base+2 < N) { lsum += occ[base+2]; l2 = lsum; } + if (base+3 < N) { lsum += occ[base+3]; l3 = lsum; } + } + int total; + int ex = wbmp_opt::block_exscan(lsum, total, warp_part); + if (C == 1) { + if (base < N) r[base] = ex + l0; + } else { + if (base+0 < N) r[base+0] = ex + l0; + if (base+1 < N) r[base+1] = ex + l1; + if (base+2 < N) r[base+2] = ex + l2; + if (base+3 < N) r[base+3] = ex + l3; + } + __syncthreads(); // protect warp_part reuse + publish r + return total; +} + +// Shared occupancy pyramid + rank buffers reused by both parallel kernels. +struct WoctShared { + uint32_t masks_sp[1024]; + int o2[4096], o4[512], o8[64], o16[8]; // occupancy (0/1) + int r2[4096], r4[512], r8[64], r16[8]; // inclusive scans of occupancy +}; + +// Cooperatively build o2/o4/o8/o16 from masks_sp. Requires __syncthreads() by +// caller before use; issues its own syncs between levels. +__device__ __forceinline__ void woct_build_pyramid(WoctShared& s) { + int tid = threadIdx.x; + for (int n = tid; n < 4096; n += 1024) { + int Ix=n&15, Iy=(n>>4)&15, Iz=(n>>8)&15, occ=0; + for (int c=0;c<8 && !occ;++c){ + int x=2*Ix+(c&1), y=2*Iy+((c>>1)&1), z=2*Iz+((c>>2)&1); + occ = (s.masks_sp[y*32+x] >> z) & 1u; + } + s.o2[n]=occ; + } + __syncthreads(); + for (int n = tid; n < 512; n += 1024) { // o4 from o2 (df=16) + int Ix=n&7, Iy=(n>>3)&7, Iz=(n>>6)&7, occ=0; + for (int c=0;c<8 && !occ;++c){ + int x=2*Ix+(c&1), y=2*Iy+((c>>1)&1), z=2*Iz+((c>>2)&1); + occ = s.o2[z*256 + y*16 + x]; + } + s.o4[n]=occ; + } + __syncthreads(); + for (int n = tid; n < 64; n += 1024) { // o8 from o4 (df=8) + int Ix=n&3, Iy=(n>>2)&3, Iz=(n>>4)&3, occ=0; + for (int c=0;c<8 && !occ;++c){ + int x=2*Ix+(c&1), y=2*Iy+((c>>1)&1), z=2*Iz+((c>>2)&1); + occ = s.o4[z*64 + y*8 + x]; + } + s.o8[n]=occ; + } + __syncthreads(); + for (int n = tid; n < 8; n += 1024) { // o16 from o8 (df=4) + int Ix=n&1, Iy=(n>>1)&1, Iz=(n>>2)&1, occ=0; + for (int c=0;c<8 && !occ;++c){ + int x=2*Ix+(c&1), y=2*Iy+((c>>1)&1), z=2*Iz+((c>>2)&1); + occ = s.o8[z*16 + y*4 + x]; + } + s.o16[n]=occ; + } + __syncthreads(); +} + +// Parallel level-major encode: byte-for-byte identical stream to woct_encode_lm. +__global__ void waveletOctreeSigEncodeParKernel( + const unsigned char* __restrict__ scratch1, + unsigned char* __restrict__ out, + size_t* __restrict__ sig_sizes) +{ + int bid = blockIdx.x, tid = threadIdx.x; + const uint32_t* bmp = reinterpret_cast(scratch1 + (long)bid * WBMP_SLOT_BYTES); + unsigned char* blk = out + (long)bid * WOCT_SLOT_BYTES; + + __shared__ WoctShared s; + __shared__ int n16,n8,n4,n2; + for (int L = tid; L < 1024; L += 1024) s.masks_sp[woct_L_to_spatial(L)] = bmp[L]; + __syncthreads(); + + woct_build_pyramid(s); + + // Root occupancy: any size-16 node present. + int rootocc = 0; + for (int i=0;i<8;++i) rootocc |= s.o16[i]; + if (!rootocc) { // empty block -> zero-length stream + if (tid==0){ blk[0]=2; sig_sizes[bid]=0; } + return; + } + + // Per-level inclusive scans -> counts. + for (int i=tid;i<8; i+=1024) s.r16[i]=s.o16[i]; + for (int i=tid;i<64; i+=1024) s.r8[i] =s.o8[i]; + for (int i=tid;i<512; i+=1024) s.r4[i] =s.o4[i]; + for (int i=tid;i<4096;i+=1024) s.r2[i] =s.o2[i]; + __syncthreads(); + woct_incl_scan(s.r16,8); + woct_incl_scan(s.r8,64); + woct_incl_scan(s.r4,512); + woct_incl_scan(s.r2,4096); + if (tid==0){ n16=s.r16[7]; n8=s.r8[63]; n4=s.r4[511]; n2=s.r2[4095]; } + __syncthreads(); + + int oct_total = 1 + n16 + n8 + n4 + n2; + if (oct_total >= WOCT_FLAT_SIG_BYTES) { // dense -> flat fallback + uint32_t* flat = reinterpret_cast(blk + WOCT_HDR_BYTES); + for (int L=tid; L<1024; L+=1024) flat[L]=bmp[L]; + if (tid==0){ blk[0]=0; sig_sizes[bid]=(size_t)WOCT_FLAT_SIG_BYTES; } + return; + } + + unsigned char* sig = blk + WOCT_HDR_BYTES; + int base16=1, base8=1+n16, base4=1+n16+n8, base2=1+n16+n8+n4; + if (tid==0){ + blk[0]=2; sig_sizes[bid]=(size_t)oct_total; + int rb=0; for (int c=0;c<8;++c) if (s.o16[c]) rb|=(1< bytes over o8. + for (int n=tid;n<8;n+=1024) if (s.o16[n]) { + int Ix=n&1,Iy=(n>>1)&1,Iz=(n>>2)&1,b=0; + for (int c=0;c<8;++c){int x=2*Ix+(c&1),y=2*Iy+((c>>1)&1),z=2*Iz+((c>>2)&1); + if (s.o8[z*16+y*4+x]) b|=(1< bytes over o4. + for (int n=tid;n<64;n+=1024) if (s.o8[n]) { + int Ix=n&3,Iy=(n>>2)&3,Iz=(n>>4)&3,b=0; + for (int c=0;c<8;++c){int x=2*Ix+(c&1),y=2*Iy+((c>>1)&1),z=2*Iz+((c>>2)&1); + if (s.o4[z*64+y*8+x]) b|=(1< bytes over o2. + for (int n=tid;n<512;n+=1024) if (s.o4[n]) { + int Ix=n&7,Iy=(n>>3)&7,Iz=(n>>6)&7,b=0; + for (int c=0;c<8;++c){int x=2*Ix+(c&1),y=2*Iy+((c>>1)&1),z=2*Iz+((c>>2)&1); + if (s.o2[z*256+y*16+x]) b|=(1< voxel bytes. + for (int n=tid;n<4096;n+=1024) if (s.o2[n]) { + int Ix=n&15,Iy=(n>>4)&15,Iz=(n>>8)&15,b=0; + for (int c=0;c<8;++c){int x=2*Ix+(c&1),y=2*Iy+((c>>1)&1),z=2*Iz+((c>>2)&1); + if ((s.masks_sp[y*32+x]>>z)&1u) b|=(1<(sig); + for (int L=tid; L<1024; L+=1024) s.masks_sp[woct_L_to_spatial(L)]=flat[L]; + __syncthreads(); + for (int L=tid; L<1024; L+=1024) out_masks[(long)bid*1024+L]=s.masks_sp[woct_L_to_spatial(L)]; + return; + } + if (sig_sizes[bid]==0) { // empty block + for (int L=tid; L<1024; L+=1024) out_masks[(long)bid*1024+L]=0; + return; + } + + // root byte -> o16 present set. + if (tid==0){ int b=sig[0]; for (int c=0;c<8;++c) if (b&(1< o8. + for (int i=tid;i<8;i+=1024) s.r16[i]=s.o16[i]; + __syncthreads(); + woct_incl_scan(s.r16,8); + if (tid==0) n16=s.r16[7]; + __syncthreads(); + for (int n=tid;n<8;n+=1024) if (s.o16[n]) { + int Ix=n&1,Iy=(n>>1)&1,Iz=(n>>2)&1; + int b=sig[1 + (s.r16[n]-s.o16[n])]; + for (int c=0;c<8;++c) if (b&(1<>1)&1),z=2*Iz+((c>>2)&1); s.o8[z*16+y*4+x]=1; } + } + __syncthreads(); + // level-8: scan o8, expand -> o4. + for (int i=tid;i<64;i+=1024) s.r8[i]=s.o8[i]; + __syncthreads(); + woct_incl_scan(s.r8,64); + if (tid==0) n8=s.r8[63]; + __syncthreads(); + int base8=1+n16; + for (int n=tid;n<64;n+=1024) if (s.o8[n]) { + int Ix=n&3,Iy=(n>>2)&3,Iz=(n>>4)&3; + int b=sig[base8 + (s.r8[n]-s.o8[n])]; + for (int c=0;c<8;++c) if (b&(1<>1)&1),z=2*Iz+((c>>2)&1); s.o4[z*64+y*8+x]=1; } + } + __syncthreads(); + // level-4: scan o4, expand -> o2. + for (int i=tid;i<512;i+=1024) s.r4[i]=s.o4[i]; + __syncthreads(); + woct_incl_scan(s.r4,512); + if (tid==0) n4=s.r4[511]; + __syncthreads(); + int base4=1+n16+n8; + for (int n=tid;n<512;n+=1024) if (s.o4[n]) { + int Ix=n&7,Iy=(n>>3)&7,Iz=(n>>6)&7; + int b=sig[base4 + (s.r4[n]-s.o4[n])]; + for (int c=0;c<8;++c) if (b&(1<>1)&1),z=2*Iz+((c>>2)&1); s.o2[z*256+y*16+x]=1; } + } + __syncthreads(); + // level-2: scan o2, expand -> voxels (atomicOr: nodes share column words). + for (int i=tid;i<4096;i+=1024) s.r2[i]=s.o2[i]; + __syncthreads(); + woct_incl_scan(s.r2,4096); + __syncthreads(); + int base2=1+n16+n8+n4; + for (int n=tid;n<4096;n+=1024) if (s.o2[n]) { + int Ix=n&15,Iy=(n>>4)&15,Iz=(n>>8)&15; + int b=sig[base2 + (s.r2[n]-s.o2[n])]; + for (int c=0;c<8;++c) if (b&(1<>1)&1),z=2*Iz+((c>>2)&1); + atomicOr((unsigned int*)&s.masks_sp[y*32+x], 1u<4][2b/ne width table][per-line values] +// +// The significance region is byte-identical to waveletOctreeSigEncodeParKernel; +// the width table + value payload are byte-identical to the two-level coder +// (same nonempty-line order, per-line widths and packing). block_sizes2[bid] +// records the exact coded length. Values are streamed straight to global +// (byte stores); the significance and value regions are disjoint by construction +// (values start at the 4-aligned end of the significance region). +// =========================================================================== +__launch_bounds__(1024) +__global__ void waveletOctreeCodeParKernel( + const unsigned char* __restrict__ scratch1, + unsigned char* __restrict__ out, + size_t* __restrict__ block_sizes2, + size_t* __restrict__ sig_sizes) +{ + using namespace wbmp_opt; + int bid = blockIdx.x, tid = threadIdx.x; + const unsigned char* blk_in = scratch1 + (long)bid * WBMP_SLOT_BYTES; + const uint32_t* bmp = reinterpret_cast(blk_in); + const int32_t* val = reinterpret_cast(blk_in + WBMP_BITMAP_BYTES); + unsigned char* blk_out = out + (long)bid * WOCT_CODE_SLOT_BYTES; + unsigned char* sig = blk_out + WOCT_HDR_BYTES; + + __shared__ WoctShared s; + __shared__ int warp_part[2 * 16]; + + uint32_t m = bmp[tid]; // L-order word for this z-line + s.masks_sp[woct_L_to_spatial(tid)] = m; // spatial order for the octree + __syncthreads(); + + // ---- octree significance pyramid + per-level wave64 DPP scans ---- + woct_build_pyramid(s); + int rootocc = 0; + #pragma unroll + for (int i=0;i<8;++i) rootocc |= s.o16[i]; + int n16 = woct_incl_scan_fast(s.o16, s.r16, 8, warp_part); + int n8 = woct_incl_scan_fast(s.o8, s.r8, 64, warp_part); + int n4 = woct_incl_scan_fast(s.o4, s.r4, 512, warp_part); + int n2 = woct_incl_scan_fast(s.o2, s.r2, 4096, warp_part); + int oct_total = 1 + n16 + n8 + n4 + n2; + int mode = (rootocc && oct_total < WOCT_FLAT_SIG_BYTES) ? 2 : (rootocc ? 0 : 2); + int sig_bytes = (mode==2) ? (rootocc ? oct_total : 0) : WOCT_FLAT_SIG_BYTES; + if (tid==0){ blk_out[0] = (unsigned char)mode; sig_sizes[bid] = (size_t)sig_bytes; } + + if (mode==2 && rootocc) { // octree significance + int base16=1, base8=1+n16, base4=1+n16+n8, base2=1+n16+n8+n4; + if (tid==0){ int rb=0; for(int c=0;c<8;++c) if(s.o16[c]) rb|=(1<>1)&1,Iz=(n>>2)&1,b=0; + for (int c=0;c<8;++c){int x=2*Ix+(c&1),y=2*Iy+((c>>1)&1),z=2*Iz+((c>>2)&1); + if (s.o8[z*16+y*4+x]) b|=(1<>2)&3,Iz=(n>>4)&3,b=0; + for (int c=0;c<8;++c){int x=2*Ix+(c&1),y=2*Iy+((c>>1)&1),z=2*Iz+((c>>2)&1); + if (s.o4[z*64+y*8+x]) b|=(1<>3)&7,Iz=(n>>6)&7,b=0; + for (int c=0;c<8;++c){int x=2*Ix+(c&1),y=2*Iy+((c>>1)&1),z=2*Iz+((c>>2)&1); + if (s.o2[z*256+y*16+x]) b|=(1<>4)&15,Iz=(n>>8)&15,b=0; + for (int c=0;c<8;++c){int x=2*Ix+(c&1),y=2*Iy+((c>>1)&1),z=2*Iz+((c>>2)&1); + if ((s.masks_sp[y*32+x]>>z)&1u) b|=(1<(sig); + for (int L=tid; L<1024; L+=1024) flat[L]=bmp[L]; + } + __syncthreads(); + + long wtab_base = ((long)WOCT_HDR_BYTES + sig_bytes + 3) & ~3L; + + // ---- per-line width table + packed values (two-level layout) ---- + int nz = __popc(m), occb = m ? 1 : 0; + int in_off, occ_rank, tot_nz, tot_ne; + block_exscan2(nz, occb, in_off, occ_rank, tot_nz, tot_ne, warp_part); + (void)tot_nz; + int mx = 0; + for (int k=0;k(blk_out + wtab_base); + int wtab_words = (2 * tot_ne + 31) / 32; + for (int i=tid;i> 4], (uint32_t)(W - 1) << ((occ_rank & 15) * 2)); + long p = vals_base + val_off; + for (int k=0;k> (8*b)); + } + } + if (tid==0) block_sizes2[bid] = (size_t)(vals_base + tot_val); +} + +inline hipError_t hipWaveletOctreeCode( + const unsigned char* scratch1, unsigned char* out, size_t* block_sizes2, + size_t* sig_sizes, int nblocks, hipStream_t stream = 0) +{ + waveletOctreeCodeParKernel<<>>(scratch1, out, block_sizes2, sig_sizes); + return hipGetLastError(); +} + +// =========================================================================== +// FULL DECODE, stage A (1024 threads): fused octree coded stream -> kernel-1 +// scratch layout [4096B bitmap][packed int32 values]. Exact inverse of the +// fused encoder's value packing, so the output is byte-identical to the +// original kernel-1 output. +// =========================================================================== +__launch_bounds__(1024) +__global__ void waveletOctreeDecodeToBitmapKernel( + const unsigned char* __restrict__ coded, + const size_t* __restrict__ sig_sizes, + unsigned char* __restrict__ scratch1_out, + size_t* __restrict__ block_sizes) +{ + using namespace wbmp_opt; + int bid = blockIdx.x, tid = threadIdx.x; + const unsigned char* blk = coded + (long)bid * WOCT_CODE_SLOT_BYTES; + const unsigned char* sig = blk + WOCT_HDR_BYTES; + unsigned char* out = scratch1_out + (long)bid * WBMP_SLOT_BYTES; + uint32_t* bmp_out = reinterpret_cast(out); + int32_t* val_out = reinterpret_cast(out + WBMP_BITMAP_BYTES); + + __shared__ WoctShared s; + __shared__ int warp_part[2 * 16]; + + s.masks_sp[tid] = 0; + for (int i=tid;i<4096;i+=1024) s.o2[i]=0; + for (int i=tid;i<512; i+=1024) s.o4[i]=0; + for (int i=tid;i<64; i+=1024) s.o8[i]=0; + for (int i=tid;i<8; i+=1024) s.o16[i]=0; + __syncthreads(); + + int mode = blk[0]; + int sig_bytes = (int)sig_sizes[bid]; + + if (mode == 0) { // flat masks (L order) + const uint32_t* flat = reinterpret_cast(sig); + s.masks_sp[woct_L_to_spatial(tid)] = flat[tid]; + __syncthreads(); + } else if (sig_bytes > 0) { // octree decode -> masks_sp + if (tid==0){ int b=sig[0]; for (int c=0;c<8;++c) if (b&(1<>1)&1,Iz=(n>>2)&1; int b=sig[1 + (s.r16[n]-s.o16[n])]; + for (int c=0;c<8;++c) if (b&(1<>1)&1),z=2*Iz+((c>>2)&1); s.o8[z*16+y*4+x]=1; } + } + __syncthreads(); + int n8 = woct_incl_scan_fast(s.o8, s.r8, 64, warp_part); + int base8 = 1 + n16; + for (int n=tid;n<64;n+=1024) if (s.o8[n]) { + int Ix=n&3,Iy=(n>>2)&3,Iz=(n>>4)&3; int b=sig[base8 + (s.r8[n]-s.o8[n])]; + for (int c=0;c<8;++c) if (b&(1<>1)&1),z=2*Iz+((c>>2)&1); s.o4[z*64+y*8+x]=1; } + } + __syncthreads(); + int n4 = woct_incl_scan_fast(s.o4, s.r4, 512, warp_part); + int base4 = 1 + n16 + n8; + for (int n=tid;n<512;n+=1024) if (s.o4[n]) { + int Ix=n&7,Iy=(n>>3)&7,Iz=(n>>6)&7; int b=sig[base4 + (s.r4[n]-s.o4[n])]; + for (int c=0;c<8;++c) if (b&(1<>1)&1),z=2*Iz+((c>>2)&1); s.o2[z*256+y*16+x]=1; } + } + __syncthreads(); + woct_incl_scan_fast(s.o2, s.r2, 4096, warp_part); + int base2 = 1 + n16 + n8 + n4; + for (int n=tid;n<4096;n+=1024) if (s.o2[n]) { + int Ix=n&15,Iy=(n>>4)&15,Iz=(n>>8)&15; int b=sig[base2 + (s.r2[n]-s.o2[n])]; + for (int c=0;c<8;++c) if (b&(1<>1)&1),z=2*Iz+((c>>2)&1); + atomicOr((unsigned int*)&s.masks_sp[y*32+x], 1u< masks stay zero. + + uint32_t maskL = s.masks_sp[woct_L_to_spatial(tid)]; + bmp_out[tid] = maskL; // bitmap in L order + + // ---- value unpack (reverse of the fused encoder) ---- + long wtab_base = ((long)WOCT_HDR_BYTES + sig_bytes + 3) & ~3L; + int nz = __popc(maskL), occb = maskL ? 1 : 0; + int in_off, occ_rank, tot_nz, tot_ne; + block_exscan2(nz, occb, in_off, occ_rank, tot_nz, tot_ne, warp_part); + __syncthreads(); // protect warp_part reuse + const uint32_t* wtab = reinterpret_cast(blk + wtab_base); + int W = occb ? (int)((wtab[occ_rank >> 4] >> ((occ_rank & 15) * 2)) & 3u) + 1 : 1; + int val_off, tot_val; + val_off = block_exscan(nz * W, tot_val, warp_part); + (void)tot_val; + long vals_base = wtab_base + (2L * tot_ne + 7) / 8; + if (occb) { + const unsigned char* vp = blk + vals_base + val_off; + int sh = 32 - 8*W; + for (int k=0;k> sh; // sign-extend from W bytes + } + } + if (tid==0) block_sizes[bid] = (size_t)WBMP_BITMAP_BYTES + (size_t)tot_nz * 4; +} + +inline hipError_t hipWaveletOctreeDecodeToBitmap( + const unsigned char* coded, const size_t* sig_sizes, + unsigned char* scratch1_out, size_t* block_sizes, int nblocks, hipStream_t stream = 0) +{ + waveletOctreeDecodeToBitmapKernel<<>>( + coded, sig_sizes, scratch1_out, block_sizes); + return hipGetLastError(); +} + +// =========================================================================== +// FULL DECODE, stage B (256 threads): kernel-1 scratch layout +// [4096B bitmap][packed int32] -> dequantize + inverse wavelet ZYX -> wavefield. +// The inverse transform mirrors waveletRLEInverseFusedKernel; only the front-end +// (RLE decode) is swapped for the bitmap+value reconstruction, which reverses +// waveletBitmapFusedKernel's Phase 4. Requires DS79_INCLUDE_REG32. +// =========================================================================== +__launch_bounds__(256, 2) +__global__ void waveletBitmapInverseFusedKernel( + const unsigned char* __restrict__ scratch1, + float* __restrict__ output, + float inv_scale, int ldimx, int ldimxy) +{ + constexpr int PLANES=32, BATCH=8, SLC=2, NTHREADS=256; + using BlockScan = rocprim::block_scan; + __shared__ union { float wavelet[BATCH*1024]; typename BlockScan::storage_type scan; } lds; + + int tid=threadIdx.x, xg=tid%8, yr=tid/8; + int bid = blockIdx.x + blockIdx.y*gridDim.x + blockIdx.z*gridDim.x*gridDim.y; + const unsigned char* blk = scratch1 + (long)bid*WBMP_SLOT_BYTES; + const uint32_t* bmp = reinterpret_cast(blk); + const int32_t* vals = reinterpret_cast(blk + WBMP_BITMAP_BYTES); + + ds79_float4_vec regs[PLANES]; + + // ---- Phase 1: reconstruct dequantized coefficients (reverse fwd Phase 4) ---- + int block_val_base = 0; + for (int x_off=0; x_off<4; ++x_off){ + uint32_t mask = bmp[x_off*256 + tid]; + int nnz = __popc(mask); + int my_off, pass_total; + BlockScan().exclusive_scan(nnz, my_off, 0, pass_total, lds.scan); + __syncthreads(); + int base = block_val_base + my_off; + #pragma unroll + for (int z=0; z<32; ++z){ + float f = 0.0f; + if (mask & (1u<>>(scratch1, output, inv_scale, ldimx, ldimxy); + return hipGetLastError(); +} + +inline hipError_t hipWaveletOctreeSigEncode( + const unsigned char* scratch1, unsigned char* out, size_t* sig_sizes, + int nblocks, int threads = 256, hipStream_t stream = 0) +{ + waveletOctreeSigEncodeKernel<<>>(scratch1, out, sig_sizes); + return hipGetLastError(); +} +inline hipError_t hipWaveletOctreeSigEncodePar( + const unsigned char* scratch1, unsigned char* out, size_t* sig_sizes, + int nblocks, hipStream_t stream = 0) +{ + waveletOctreeSigEncodeParKernel<<>>(scratch1, out, sig_sizes); + return hipGetLastError(); +} +inline hipError_t hipWaveletOctreeSigDecodePar( + const unsigned char* coded, const size_t* sig_sizes, uint32_t* out_masks, + int nblocks, hipStream_t stream = 0) +{ + waveletOctreeSigDecodeParKernel<<>>(coded, sig_sizes, out_masks); + return hipGetLastError(); +} +inline hipError_t hipWaveletOctreeSigDecode( + const unsigned char* coded, const size_t* sig_sizes, uint32_t* out_masks, + int nblocks, int threads = 256, hipStream_t stream = 0) +{ + waveletOctreeSigDecodeKernel<<>>(coded, sig_sizes, out_masks); + return hipGetLastError(); +} + +#endif // HIPWAVELET_OCTREE_H diff --git a/makefile b/makefile index 13d07d9..fc3336b 100644 --- a/makefile +++ b/makefile @@ -86,7 +86,7 @@ hip/%.o: hip/%.cpp # Buffer-instruction wavelet kernel test test_wavelet_buffer_hip: tests/test_wavelet_buffer_hip.cpp hip/hipWaveletTransformBuffer.cpp | $(BUILDDIR) - $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -save-temps=obj -DBUILDDIR=\"$(BUILDDIR)\" -I. -Ihip -Itests -lrocrand -fopenmp tests/test_wavelet_buffer_hip.cpp hip/hipWaveletTransformBuffer.cpp $(HIPLDFLAGS) -o $(BUILDDIR)/test_wavelet_buffer_hip + $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -save-temps=obj -DBUILDDIR=\"$(BUILDDIR)\" -I. -Ihip -Itests -lrocrand -fopenmp tests/test_wavelet_buffer_hip.cpp hip/hipWaveletTransformBuffer.cpp $(HIPLDFLAGS) -o $(BUILDDIR)/test_wavelet_buffer_hip # Quantize + RLE z-line unit test (CPU-only, no HIP) test_quantize_rle: tests/test_quantize_rle.cpp hip/quantize_rle_ref.h Run_Length_Escape_Codes.hxx | $(BUILDDIR) @@ -100,6 +100,10 @@ test_zline_cr_benchmark: tests/test_zline_cr_benchmark.cpp hip/quantize_rle_ref. test_bitmap_vs_cpu_rle: tests/test_bitmap_vs_cpu_rle.cpp Run_Length_Encode_Slow.hxx Run_Length_Escape_Codes.hxx Wavelet_Transform_Fast.hxx Block_Copy.hxx libcvxcompress.$(LIB_EXT) | $(BUILDDIR) $(CXX) $(CFLAGS) $(TFLAG) -I. -Ihip -Itests tests/test_bitmap_vs_cpu_rle.cpp -L. -lcvxcompress '-Wl,-rpath,$$ORIGIN/..' -o $(BUILDDIR)/test_bitmap_vs_cpu_rle +# Rate-distortion scale sweep on a real seismic panel (CPU-only) +test_bitmap_rd_panel: tests/test_bitmap_rd_panel.cpp Run_Length_Encode_Slow.hxx Wavelet_Transform_Fast.hxx Block_Copy.hxx libcvxcompress.$(LIB_EXT) | $(BUILDDIR) + $(CXX) $(CFLAGS) $(TFLAG) -I. -Ihip -Itests tests/test_bitmap_rd_panel.cpp -L. -lcvxcompress '-Wl,-rpath,$$ORIGIN/..' -o $(BUILDDIR)/test_bitmap_rd_panel + # GPU quantize+RLE encode test (validates against CPU reference) test_quantize_rle_hip: tests/test_quantize_rle_hip.cpp hip/quantize_rle_ref.h Run_Length_Escape_Codes.hxx hip/hipQuantizeRLE.h | $(BUILDDIR) $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -I. -Ihip -Itests tests/test_quantize_rle_hip.cpp -lm -o $(BUILDDIR)/test_quantize_rle_hip @@ -140,6 +144,10 @@ test_bitmap_encode_hip: tests/test_bitmap_encode_hip.cpp hip/hipWaveletBitmap.h test_bitmap_code_hip: tests/test_bitmap_code_hip.cpp hip/hipWaveletBitmap.h hip/hipWaveletRLE.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_bitmap_code_hip.cpp -lm -o $(BUILDDIR)/test_bitmap_code_hip +# Octree significance coder (kernel 2 variant): round-trip + CR + throughput +test_bitmap_octree_hip: tests/test_bitmap_octree_hip.cpp hip/hipWaveletOctree.h hip/hipWaveletBitmap.h hip/hipWaveletRLE.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) + $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_bitmap_octree_hip.cpp -lm -o $(BUILDDIR)/test_bitmap_octree_hip + # Async pipeline example (for profiling) example_async_pipeline: tests/example_async_pipeline.cpp hip/hipCompress.cpp hip/hipCompress.h hip/hipBlockCopy.h hip/hipWaveletRLE.h hip/hipWaveletRLEInverse.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/example_async_pipeline.cpp hip/hipCompress.cpp -lm -o $(BUILDDIR)/example_async_pipeline diff --git a/tests/test_bitmap_octree_hip.cpp b/tests/test_bitmap_octree_hip.cpp new file mode 100644 index 0000000..e2a431a --- /dev/null +++ b/tests/test_bitmap_octree_hip.cpp @@ -0,0 +1,378 @@ +// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Use of this source code is governed by an MIT-style license that can be +// found in the LICENSE file or at https://opensource.org/licenses/MIT. +// +// Octree significance coder prototype validation: +// (1) GPU-encoded octree bytes are byte-for-byte identical to the shared +// host reference (woct_encode). +// (2) GPU decode reconstructs the 1024 z-line masks byte-exactly (round-trip). +// (3) full-codec CR (octree significance + two-level width table + values) vs +// CPU/GPU RLE and the two-level coder, at matched quantization. +// (4) octree encode/decode throughput. +// Uses a --panel loader (real seismic field) or a synthetic field fallback. + +#define DS79_INCLUDE_REG32 +#include "hipWaveletRLE.h" +#include "hipWaveletRLEInverse.h" +#include "hipWaveletBitmap.h" +#include "hipWaveletOctree.h" + +#include +#include +#include +#include +#include +#include +#include + +#define HIPCHECK(cmd) do { hipError_t _e=(cmd); if(_e!=hipSuccess){ \ + printf("HIP error %s at %s:%d\n",hipGetErrorString(_e),__FILE__,__LINE__); return 1;}}while(0) + +static float time_kernel(void (*launch)(void*), void* ctx, int iters) { + hipEvent_t a,b; hipEventCreate(&a); hipEventCreate(&b); + launch(ctx); hipDeviceSynchronize(); + hipEventRecord(a); + for (int i=0;inbx,c->nby,c->nbz); + waveletRLEFusedKernel<<>>(c->d_in,c->d_rle,c->d_rle_sizes,c->mulfac,c->ldimx,c->ldimxy,nullptr,nullptr); } +static void launch_bmp(void* p){ Ctx*c=(Ctx*)p; dim3 g(c->nbx,c->nby,c->nbz); + waveletBitmapFusedKernel<<>>(c->d_in,c->d_bmp,c->d_bmp_sizes,c->mulfac,c->ldimx,c->ldimxy,nullptr,nullptr); } +static void launch_codetl(void* p){ Ctx*c=(Ctx*)p; + waveletBitmapCodeTwoLevelKernel<<nblocks,dim3(256)>>>(c->d_bmp,c->d_bmp_sizes,c->d_codetl,c->d_codetl_sizes); } +static void launch_oct_enc(void* p){ Ctx*c=(Ctx*)p; + waveletOctreeSigEncodeKernel<<nblocks,dim3(c->enc_threads)>>>(c->d_bmp,c->d_oct,c->d_oct_sizes); } +static void launch_oct_dec(void* p){ Ctx*c=(Ctx*)p; + waveletOctreeSigDecodeKernel<<nblocks,dim3(c->enc_threads)>>>(c->d_oct,c->d_oct_sizes,c->d_dec_masks); } +static void launch_octp_enc(void* p){ Ctx*c=(Ctx*)p; + waveletOctreeSigEncodeParKernel<<nblocks,dim3(WOCT_PAR_THREADS)>>>(c->d_bmp,c->d_octp,c->d_octp_sizes); } +static void launch_octp_dec(void* p){ Ctx*c=(Ctx*)p; + waveletOctreeSigDecodeParKernel<<nblocks,dim3(WOCT_PAR_THREADS)>>>(c->d_octp,c->d_octp_sizes,c->d_decp_masks); } +static void launch_octc(void* p){ Ctx*c=(Ctx*)p; + waveletOctreeCodeParKernel<<nblocks,dim3(WOCT_PAR_THREADS)>>>(c->d_bmp,c->d_octc,c->d_octc_sizes,c->d_octc_sig_sizes); } +static void launch_octA(void* p){ Ctx*c=(Ctx*)p; // decode stage A: coded -> bitmap+values + waveletOctreeDecodeToBitmapKernel<<nblocks,dim3(WOCT_PAR_THREADS)>>>(c->d_octc,c->d_octc_sig_sizes,c->d_bmp2,c->d_bmp2_sizes); } +static void launch_octB(void* p){ Ctx*c=(Ctx*)p; // decode stage B: bitmap+values -> field + dim3 g(c->nbx,c->nby,c->nbz); + waveletBitmapInverseFusedKernel<<>>(c->d_bmp2,c->d_field_oct,1.0f/c->mulfac,c->ldimx,c->ldimxy); } +static void launch_rle_inv(void* p){ Ctx*c=(Ctx*)p; dim3 g(c->nbx,c->nby,c->nbz); + waveletRLEInverseFusedKernel<<>>(c->d_rle,c->d_rle_sizes,nullptr,c->d_field_rle,1.0f/c->mulfac,c->ldimx,c->ldimxy,0); } + +static bool load_center_crop(const std::string& path,int gnz,int gny,int gnx, + int cz,int cy,int cx,std::vector& out){ + FILE* f=std::fopen(path.c_str(),"rb"); if(!f){printf("open fail %s\n",path.c_str());return false;} + std::vector full((size_t)gnz*gny*gnx); + size_t r=std::fread(full.data(),sizeof(float),full.size(),f); std::fclose(f); + if(r!=full.size()){printf("short read %s\n",path.c_str());return false;} + int oz=(gnz-cz)/2,oy=(gny-cy)/2,ox=(gnx-cx)/2; + out.resize((size_t)cz*cy*cx); + for(int k=0;k0){ float inv=(float)(1.0/rms); for(size_t i=0;i h_in(nelem); + if(!panel.empty()){ + if(!load_center_crop(panel,512,512,512,NX,NX,NX,h_in)) return 1; + } else { + for(size_t i=0;i>16)&0x7fff)/32768.0f; + h_in[i]=0.5f*s+n-0.075f;} + } + printf("octree-code test: %d^3 nblocks=%d scale=%.3f iters=%d enc_threads=%d\n", + NX,c.nblocks,mulfac,iters,enc_threads); + + HIPCHECK(hipMalloc(&c.d_in,nelem*sizeof(float))); + HIPCHECK(hipMemcpy((void*)c.d_in,h_in.data(),nelem*sizeof(float),hipMemcpyHostToDevice)); + const long rle_stride=4L*WRLE_LDS_BYTES; + HIPCHECK(hipMalloc(&c.d_rle,(size_t)c.nblocks*rle_stride)); + HIPCHECK(hipMalloc(&c.d_rle_sizes,c.nblocks*sizeof(size_t))); + HIPCHECK(hipMalloc(&c.d_bmp,(size_t)c.nblocks*WBMP_SLOT_BYTES)); + HIPCHECK(hipMalloc(&c.d_bmp_sizes,c.nblocks*sizeof(size_t))); + HIPCHECK(hipMalloc(&c.d_codetl,(size_t)c.nblocks*WBMP_TL_SLOT_BYTES)); + HIPCHECK(hipMalloc(&c.d_codetl_sizes,c.nblocks*sizeof(size_t))); + HIPCHECK(hipMalloc(&c.d_oct,(size_t)c.nblocks*WOCT_SLOT_BYTES)); + HIPCHECK(hipMalloc(&c.d_oct_sizes,c.nblocks*sizeof(size_t))); + HIPCHECK(hipMalloc(&c.d_octp,(size_t)c.nblocks*WOCT_SLOT_BYTES)); + HIPCHECK(hipMalloc(&c.d_octp_sizes,c.nblocks*sizeof(size_t))); + HIPCHECK(hipMalloc(&c.d_octc,(size_t)c.nblocks*WOCT_CODE_SLOT_BYTES)); + HIPCHECK(hipMalloc(&c.d_octc_sizes,c.nblocks*sizeof(size_t))); + HIPCHECK(hipMalloc(&c.d_octc_sig_sizes,c.nblocks*sizeof(size_t))); + HIPCHECK(hipMalloc(&c.d_bmp2,(size_t)c.nblocks*WBMP_SLOT_BYTES)); + HIPCHECK(hipMalloc(&c.d_bmp2_sizes,c.nblocks*sizeof(size_t))); + HIPCHECK(hipMalloc(&c.d_field_oct,nelem*sizeof(float))); + HIPCHECK(hipMalloc(&c.d_field_rle,nelem*sizeof(float))); + HIPCHECK(hipMalloc(&c.d_dec_masks,(size_t)c.nblocks*1024*sizeof(uint32_t))); + HIPCHECK(hipMalloc(&c.d_decp_masks,(size_t)c.nblocks*1024*sizeof(uint32_t))); + + launch_rle(&c); launch_bmp(&c); HIPCHECK(hipDeviceSynchronize()); + launch_codetl(&c); launch_oct_enc(&c); HIPCHECK(hipDeviceSynchronize()); + launch_oct_dec(&c); HIPCHECK(hipDeviceSynchronize()); + launch_octp_enc(&c); HIPCHECK(hipDeviceSynchronize()); + launch_octp_dec(&c); launch_octc(&c); HIPCHECK(hipDeviceSynchronize()); + launch_octA(&c); HIPCHECK(hipDeviceSynchronize()); // decode stage A + launch_octB(&c); launch_rle_inv(&c); HIPCHECK(hipDeviceSynchronize()); // full decodes + HIPCHECK(hipGetLastError()); + + std::vector h_bmp((size_t)c.nblocks*WBMP_SLOT_BYTES); + std::vector h_oct((size_t)c.nblocks*WOCT_SLOT_BYTES); + std::vector h_octp((size_t)c.nblocks*WOCT_SLOT_BYTES); + std::vector h_octc((size_t)c.nblocks*WOCT_CODE_SLOT_BYTES); + std::vector h_codetl((size_t)c.nblocks*WBMP_TL_SLOT_BYTES); + std::vector h_bmp_sizes(c.nblocks),h_oct_sizes(c.nblocks),h_octp_sizes(c.nblocks),h_octc_sizes(c.nblocks); + std::vector h_rle_sizes(c.nblocks),h_codetl_sizes(c.nblocks); + std::vector h_dec((size_t)c.nblocks*1024),h_decp((size_t)c.nblocks*1024); + HIPCHECK(hipMemcpy(h_bmp.data(),c.d_bmp,h_bmp.size(),hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_oct.data(),c.d_oct,h_oct.size(),hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_octp.data(),c.d_octp,h_octp.size(),hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_octc.data(),c.d_octc,h_octc.size(),hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_codetl.data(),c.d_codetl,h_codetl.size(),hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_bmp_sizes.data(),c.d_bmp_sizes,c.nblocks*sizeof(size_t),hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_oct_sizes.data(),c.d_oct_sizes,c.nblocks*sizeof(size_t),hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_octp_sizes.data(),c.d_octp_sizes,c.nblocks*sizeof(size_t),hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_octc_sizes.data(),c.d_octc_sizes,c.nblocks*sizeof(size_t),hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_rle_sizes.data(),c.d_rle_sizes,c.nblocks*sizeof(size_t),hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_codetl_sizes.data(),c.d_codetl_sizes,c.nblocks*sizeof(size_t),hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_dec.data(),c.d_dec_masks,h_dec.size()*sizeof(uint32_t),hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_decp.data(),c.d_decp_masks,h_decp.size()*sizeof(uint32_t),hipMemcpyDeviceToHost)); + std::vector h_bmp2((size_t)c.nblocks*WBMP_SLOT_BYTES); + std::vector h_bmp2_sizes(c.nblocks); + std::vector h_field_oct(nelem), h_field_rle(nelem); + HIPCHECK(hipMemcpy(h_bmp2.data(),c.d_bmp2,h_bmp2.size(),hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_bmp2_sizes.data(),c.d_bmp2_sizes,c.nblocks*sizeof(size_t),hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_field_oct.data(),c.d_field_oct,nelem*sizeof(float),hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_field_rle.data(),c.d_field_rle,nelem*sizeof(float),hipMemcpyDeviceToHost)); + + // ---- decode validation: stage-A bytes vs kernel-1 (exact round-trip gate) ---- + long decA_mism=0; + for(int bid=0; bidfield_maxdiff) field_maxdiff=d; if(fo!=fr) ++field_mism; + sse_oct+=(fo-xin)*(fo-xin); sse_rle+=(fr-xin)*(fr-xin); sig2+=xin*xin; + } + double rel_l2_oct=std::sqrt(sse_oct/sig2), rel_l2_rle=std::sqrt(sse_rle/sig2); + + // ---- (1) GPU encode == host reference, (2) round-trip masks ---- + long enc_mism=0, rt_mism=0, host_rt_mism=0; + long lm_rt_mism=0, lm_cnt_mism=0; // level-major reference checks + long par_enc_mism=0, par_rt_mism=0; // parallel GPU vs host LM + round-trip + long fus_sig_mism=0, fus_wtab_mism=0, fus_val_mism=0, fus_size_mism=0; // fused kernel-2 + long fus_block_total=0; // sum of fused per-block coded bytes + long mode_flat=0, mode_oct=0; + long oct_sig_total=0; + std::vector ref((size_t)WOCT_MAX_SIG_BYTES); + std::vector reflm((size_t)WOCT_MAX_SIG_BYTES); + std::vector masks_sp(1024), masks_rt(1024); + for(int bid=0; bid(h_bmp.data()+(long)bid*WBMP_SLOT_BYTES); + for(int L=0;L<1024;++L) masks_sp[woct_L_to_spatial(L)] = bmp[L]; + int hb = woct_encode(masks_sp.data(), ref.data()); + // pure-host encode->decode round-trip (isolates algorithm from device) + for(int i=0;i<1024;++i) masks_rt[i]=0; + woct_decode(ref.data(), hb, masks_rt.data()); + for(int i=0;i<1024;++i) if(masks_rt[i]!=masks_sp[i]){ if(host_rt_mism<5) + printf(" [hostRT] blk %d sp %d dec %u != %u (octbytes=%d)\n",bid,i,masks_rt[i],masks_sp[i],hb); ++host_rt_mism; break; } + // level-major host reference: same byte count as DFS + round-trip. + int hblm = woct_encode_lm(masks_sp.data(), reflm.data()); + if(hblm != hb){ if(lm_cnt_mism<5) printf(" [lmCnt] blk %d lm %d != dfs %d\n",bid,hblm,hb); ++lm_cnt_mism; } + for(int i=0;i<1024;++i) masks_rt[i]=0; + woct_decode_lm(reflm.data(), hblm, masks_rt.data()); + for(int i=0;i<1024;++i) if(masks_rt[i]!=masks_sp[i]){ if(lm_rt_mism<5) + printf(" [lmRT] blk %d sp %d dec %u != %u (lmbytes=%d)\n",bid,i,masks_rt[i],masks_sp[i],hblm); ++lm_rt_mism; break; } + // parallel GPU level-major encode: byte-exact vs host LM reference. + const unsigned char* blkp = h_octp.data()+(long)bid*WOCT_SLOT_BYTES; + int pmode = blkp[0]; long psz = (long)h_octp_sizes[bid]; + if(pmode==2){ + if(psz != hblm){ if(par_enc_mism<10) printf(" [parEnc] blk %d sig %ld != lm %d\n",bid,psz,hblm); ++par_enc_mism; } + else if(memcmp(blkp+WOCT_HDR_BYTES, reflm.data(), hblm)!=0){ if(par_enc_mism<10) printf(" [parEnc] blk %d bytes differ\n",bid); ++par_enc_mism; } + } else { // flat fallback: octree stream must actually be >= flat size + if(hblm < WOCT_FLAT_SIG_BYTES && par_enc_mism<10){ printf(" [parEnc] blk %d flat but lm=%d < flat\n",bid,hblm); ++par_enc_mism; } + } + // parallel round-trip masks (independent decode kernel). + const uint32_t* dmp = &h_decp[(long)bid*1024]; + for(int L=0;L<1024;++L) if(dmp[L]!=bmp[L]){ if(par_rt_mism<10) + printf(" [parRT] blk %d mode %d L %d dec %u != %u\n",bid,pmode,L,dmp[L],bmp[L]); ++par_rt_mism; break; } + // ---- fused kernel-2: sig region vs host LM/flat; wtab+values vs two-level ---- + int ne=0; for(int L=0;L<1024;++L) if(bmp[L]) ++ne; + const unsigned char* blkc = h_octc.data()+(long)bid*WOCT_CODE_SLOT_BYTES; + int fmode = blkc[0]; + long fsig = (fmode==2) ? hblm : WOCT_FLAT_SIG_BYTES; // hblm==0 => empty + // significance region + if(fmode==2){ + if(hblm>0 && memcmp(blkc+WOCT_HDR_BYTES, reflm.data(), hblm)!=0){ + if(fus_sig_mism<10) printf(" [fusSig] blk %d octree bytes differ\n",bid); ++fus_sig_mism; } + } else { + const uint32_t* fflat = reinterpret_cast(blkc+WOCT_HDR_BYTES); + for(int L=0;L<1024;++L) if(fflat[L]!=bmp[L]){ if(fus_sig_mism<10) + printf(" [fusSig] blk %d flat L %d %u!=%u\n",bid,L,fflat[L],bmp[L]); ++fus_sig_mism; break; } + } + // width table + values vs the two-level reference regions + long tl_wtab_off = WBMP_OCC_BYTES + 4L*ne; + long tl_wtab_len = (2L*ne + 7)/8; + long tl_vals_off = tl_wtab_off + tl_wtab_len; + long val_len = (long)h_codetl_sizes[bid] - tl_vals_off; + const unsigned char* codetl = h_codetl.data()+(long)bid*WBMP_TL_SLOT_BYTES; + long wtab_base = (WOCT_HDR_BYTES + fsig + 3) & ~3L; + long vals_base = wtab_base + tl_wtab_len; + if(tl_wtab_len>0 && memcmp(blkc+wtab_base, codetl+tl_wtab_off, tl_wtab_len)!=0){ + if(fus_wtab_mism<10) printf(" [fusWtab] blk %d differ (ne=%d)\n",bid,ne); ++fus_wtab_mism; } + if(val_len>0 && memcmp(blkc+vals_base, codetl+tl_vals_off, val_len)!=0){ + if(fus_val_mism<10) printf(" [fusVal] blk %d differ (len=%ld)\n",bid,val_len); ++fus_val_mism; } + long fexp = vals_base + val_len; + if((long)h_octc_sizes[bid]!=fexp){ if(fus_size_mism<10) + printf(" [fusSize] blk %d got %ld exp %ld\n",bid,(long)h_octc_sizes[bid],fexp); ++fus_size_mism; } + fus_block_total += (long)h_octc_sizes[bid]; + const unsigned char* blk = h_oct.data()+(long)bid*WOCT_SLOT_BYTES; + int mode = blk[0]; + long gsz = (long)h_oct_sizes[bid]; + if(mode==2){ + ++mode_oct; oct_sig_total += gsz; + if(gsz != hb){ if(enc_mism<10) printf(" [enc] blk %d sig %ld != ref %d\n",bid,gsz,hb); ++enc_mism; } + else if(memcmp(blk+WOCT_HDR_BYTES, ref.data(), hb)!=0){ if(enc_mism<10) printf(" [enc] blk %d bytes differ\n",bid); ++enc_mism; } + } else { + ++mode_flat; oct_sig_total += WOCT_FLAT_SIG_BYTES; + } + // round-trip: decoded L-order masks must equal the bitmap words + const uint32_t* dm = &h_dec[(long)bid*1024]; + for(int L=0;L<1024;++L) if(dm[L]!=bmp[L]){ + if(rt_mism<10){ + // host-decode the GPU-written bytes for this block to localize + for(int i=0;i<1024;++i) masks_rt[i]=0; + if(mode==2) woct_decode(blk+WOCT_HDR_BYTES,(int)gsz,masks_rt.data()); + else for(int Lx=0;Lx<1024;++Lx) masks_rt[woct_L_to_spatial(Lx)]= + reinterpret_cast(blk+WOCT_HDR_BYTES)[Lx]; + int hostgpu_ok = (masks_rt[woct_L_to_spatial(L)]==bmp[L]); + printf(" [rt] blk %d mode %d sig %ld L %d dec %u != %u host(gpubytes)=%s\n", + bid,mode,gsz,L,dm[L],bmp[L], hostgpu_ok?"match":"MISMATCH"); + } + ++rt_mism; break; } + } + + // ---- (3) full-codec size / CR (octree sig + two-level wtab+payload) ---- + long hdr = 8 + 8L*c.nblocks + 4; + long rle_total=hdr, tl_total=hdr, oct_total=hdr; + for(int bid=0;bid(h_bmp.data()+(long)bid*WBMP_SLOT_BYTES); + int ne=0; for(int L=0;L<1024;++L) if(bmp[L]) ++ne; + long wtab_payload = (long)h_codetl_sizes[bid] - WBMP_OCC_BYTES - 4L*ne; + long sig = (h_oct.data()[(long)bid*WOCT_SLOT_BYTES]==2) ? (long)h_oct_sizes[bid] : WOCT_FLAT_SIG_BYTES; + oct_total += WOCT_HDR_BYTES + sig + wtab_payload; + } + double raw=(double)nelem*4.0; + printf("modes: octree=%ld (%.1f%%) flat=%ld | octree sig bytes total=%ld\n", + mode_oct,100.0*mode_oct/c.nblocks,mode_flat,oct_sig_total); + long fused_total = hdr + fus_block_total; // measured fused kernel-2 output + printf("sizes (B): RLE=%ld twolevel=%ld octree(est)=%ld fused(measured)=%ld\n", + rle_total,tl_total,oct_total,fused_total); + printf("CR: RLE=%.3f twolevel=%.3f octree=%.3f fused=%.3f\n", + raw/rle_total,raw/tl_total,raw/oct_total,raw/fused_total); + printf("octree/RLE size ratio = %.3f (<1 = octree smaller)\n",(double)oct_total/rle_total); + printf("octree/twolevel size ratio = %.3f\n",(double)oct_total/tl_total); + + // ---- (4) throughput ---- + float t_enc=time_kernel(launch_oct_enc,&c,iters); + float t_dec=time_kernel(launch_oct_dec,&c,iters); + float t_penc=time_kernel(launch_octp_enc,&c,iters); + float t_pdec=time_kernel(launch_octp_dec,&c,iters); + float t_tl =time_kernel(launch_codetl,&c,iters); + float t_fus=time_kernel(launch_octc,&c,iters); + // full-pipeline enc/dec: RLE (single fused kernel) vs octree (k1 + fused k2 enc; + // stage-A + stage-B dec) + float t_rle_enc=time_kernel(launch_rle,&c,iters); + float t_k1 =time_kernel(launch_bmp,&c,iters); + float t_rle_dec=time_kernel(launch_rle_inv,&c,iters); + float t_decA =time_kernel(launch_octA,&c,iters); + float t_decB =time_kernel(launch_octB,&c,iters); + double gbraw=raw/1e9; + float oct_enc_full=t_k1+t_fus, oct_dec_full=t_decA+t_decB; + printf("FULL PIPELINE (raw GB/s):\n"); + printf(" encode: RLE=%.1f (%.3fms) octree=%.1f (k1 %.3f + k2 %.3f = %.3fms) speedup=%.2fx\n", + gbraw/(t_rle_enc/1e3),t_rle_enc, gbraw/(oct_enc_full/1e3),t_k1,t_fus,oct_enc_full, t_rle_enc/oct_enc_full); + printf(" decode: RLE=%.1f (%.3fms) octree=%.1f (A %.3f + B %.3f = %.3fms) speedup=%.2fx\n", + gbraw/(t_rle_dec/1e3),t_rle_dec, gbraw/(oct_dec_full/1e3),t_decA,t_decB,oct_dec_full, t_rle_dec/oct_dec_full); + printf("throughput (raw GB/s): serial_enc=%.2f serial_dec=%.2f | PAR_enc=%.2f PAR_dec=%.2f | FUSED=%.2f | twolevel_enc=%.2f\n", + gbraw/(t_enc/1e3), gbraw/(t_dec/1e3), gbraw/(t_penc/1e3), gbraw/(t_pdec/1e3), gbraw/(t_fus/1e3), gbraw/(t_tl/1e3)); + printf("kernel ms: serial_enc=%.3f serial_dec=%.3f | PAR_enc=%.3f PAR_dec=%.3f | FUSED=%.3f | twolevel_enc=%.3f\n", + t_enc,t_dec,t_penc,t_pdec,t_fus,t_tl); + printf("parallel speedup vs serial: enc=%.1fx dec=%.1fx | fused vs twolevel: %.2fx\n", + t_enc/t_penc, t_dec/t_pdec, t_tl/t_fus); + + long total_mism = enc_mism + rt_mism; + printf("encode byte-exact vs host: %s (%ld) round-trip masks: %s (%ld) host-only RT: %s (%ld)\n", + enc_mism?"MISMATCH":"OK", enc_mism, rt_mism?"MISMATCH":"OK", rt_mism, + host_rt_mism?"MISMATCH":"OK", host_rt_mism); + printf("level-major host ref: byte-count vs DFS: %s (%ld) round-trip masks: %s (%ld)\n", + lm_cnt_mism?"MISMATCH":"OK", lm_cnt_mism, lm_rt_mism?"MISMATCH":"OK", lm_rt_mism); + printf("parallel GPU: byte-exact vs host LM: %s (%ld) round-trip masks: %s (%ld)\n", + par_enc_mism?"MISMATCH":"OK", par_enc_mism, par_rt_mism?"MISMATCH":"OK", par_rt_mism); + printf("fused kernel-2: sig=%s(%ld) wtab=%s(%ld) values=%s(%ld) size=%s(%ld)\n", + fus_sig_mism?"MISMATCH":"OK",fus_sig_mism, fus_wtab_mism?"MISMATCH":"OK",fus_wtab_mism, + fus_val_mism?"MISMATCH":"OK",fus_val_mism, fus_size_mism?"MISMATCH":"OK",fus_size_mism); + printf("decode: stage-A round-trip (bytes vs kernel-1): %s (%ld)\n", decA_mism?"MISMATCH":"OK", decA_mism); + printf("distortion vs original (rel_l2): octree=%.4e RLE=%.4e | octree-vs-RLE field maxdiff=%.3e (%ld voxels, codec f32-escape delta)\n", + rel_l2_oct, rel_l2_rle, field_maxdiff, field_mism); + total_mism += lm_cnt_mism + lm_rt_mism + par_enc_mism + par_rt_mism + + fus_sig_mism + fus_wtab_mism + fus_val_mism + fus_size_mism + + decA_mism; + if(total_mism){ printf("FAIL\n"); return 1; } + printf("PASS\n"); + return 0; +} From 1cfe67c3b8cbb0618419d09052472fcc0c4446bd Mon Sep 17 00:00:00 2001 From: Ossian O'Reilly Date: Mon, 17 Aug 2026 11:58:26 -0700 Subject: [PATCH 3/7] Wire octree codec into hipCompress public API Add HIP_COMPRESS_KERNEL_OCTREE as a first-class API kernel (3D only), end-to-end on the plan/scan/compact infrastructure: - Encode: k1 (bitmap+values) -> k2 (octree code) -> 4-align coded sizes -> exclusive scan -> woctCompactKernel writing a self-contained header [nb][nmf][offsets][sig_sizes][mulfac]. - Decode: header-addressed stage A (locates blocks via offsets, reads significance length from header, publishes 1/mulfac on device) -> dev-scale stage B; no host readback. - Refactor stage-A/stage-B into shared __device__ bodies so the validated prototype kernels keep byte-identical signatures. - 4-align each block in the packed stream so uint32 width-table / flat-mask reads stay aligned (the fixed-stride prototype masked this via 16B slots). - Plan gains octree scratch/sig/inv_scale buffers; MaxOutputSize and the header-size helper account for the octree layout. Add test_octree_round_trip and an octree case to bench_throughput. Validated on MI355x (gfx950): 38/38 API tests pass; at 512^3 octree CR 213.0 vs RLE 36.8, encode 0.91x RLE, decode 1.56x faster. --- hip/hipCompress.cpp | 73 +++++++++++++-- hip/hipCompress.h | 7 ++ hip/hipWaveletOctree.h | 160 +++++++++++++++++++++++++++++--- makefile | 2 +- tests/test_compress_api_hip.cpp | 81 ++++++++++++++-- 5 files changed, 292 insertions(+), 31 deletions(-) diff --git a/hip/hipCompress.cpp b/hip/hipCompress.cpp index dae243e..d3068e9 100644 --- a/hip/hipCompress.cpp +++ b/hip/hipCompress.cpp @@ -8,6 +8,8 @@ #include "hipWaveletRLE.h" #include "hipWaveletRLEInverse.h" #include "hipWaveletRLE2D.h" +#include "hipWaveletBitmap.h" +#include "hipWaveletOctree.h" #include "hipBlockCopy.h" #include @@ -90,11 +92,16 @@ hipError_t hipCompressCreatePlan(hipCompressPlan** plan, int nx, int ny, int nz, p->last_error = HIP_COMPRESS_ERROR_HIP_RUNTIME; if (is_2d) { + if (kernel == HIP_COMPRESS_KERNEL_OCTREE) // octree is 3D only + PLAN_ERROR(p, HIP_COMPRESS_ERROR_INVALID_DIMENSIONS, hipErrorInvalidValue); p->num_blocks = (nx / 32) * (ny / 32); p->scratch_slot_stride = WRLE2D_SLOT_BYTES; } else { p->num_blocks = (nx / 32) * (ny / 32) * (nz / 32); - p->scratch_slot_stride = 4L * WRLE_LDS_BYTES; + // Octree: d_scratch holds the kernel-1 bitmap+values layout (encode: k1 + // output; decode: stage-A output). RLE: fixed-stride RLE slots. + p->scratch_slot_stride = (kernel == HIP_COMPRESS_KERNEL_OCTREE) + ? (size_t)WBMP_SLOT_BYTES : 4L * WRLE_LDS_BYTES; } int nb = p->num_blocks; @@ -105,6 +112,13 @@ hipError_t hipCompressCreatePlan(hipCompressPlan** plan, int nx, int ny, int nz, HIPCHECK_PLAN(p, hipMalloc(&p->d_block_sizes, nb * sizeof(size_t))); HIPCHECK_PLAN(p, hipMalloc(&p->d_block_offsets, nb * sizeof(size_t))); + if (kernel == HIP_COMPRESS_KERNEL_OCTREE) { + HIPCHECK_PLAN(p, hipMalloc(&p->d_octree_coded, + (long)nb * WOCT_CODE_SLOT_BYTES)); + HIPCHECK_PLAN(p, hipMalloc(&p->d_octree_sig_sizes, nb * sizeof(size_t))); + HIPCHECK_PLAN(p, hipMalloc(&p->d_inv_scale, sizeof(float))); + } + p->scan_temp_bytes = 0; hipError_t err = hipWaveletRLECompactScanTempSize(nb, &p->scan_temp_bytes); if (err != hipSuccess) { p->last_error = HIP_COMPRESS_ERROR_HIP_RUNTIME; return err; } @@ -143,6 +157,9 @@ hipError_t hipCompressDestroyPlan(hipCompressPlan* plan) (void)hipFree(plan->d_scan_temp); (void)hipFree(plan->d_partial_sums); (void)hipFree(plan->d_rms); + (void)hipFree(plan->d_octree_coded); + (void)hipFree(plan->d_octree_sig_sizes); + (void)hipFree(plan->d_inv_scale); if (plan->h_staging) (void)hipHostFree(plan->h_staging); if (plan->ready_event) (void)hipEventDestroy(plan->ready_event); free(plan); @@ -172,11 +189,13 @@ hipError_t hipCompress( const int ldimx = nx; const int nb = plan->num_blocks; const int num_mulfacs = 1; - const int hdr_size = hipCompressHeaderSize(nb, num_mulfacs); + const int hdr_size = (plan->kernel == HIP_COMPRESS_KERNEL_OCTREE) + ? hipOctreeHeaderSize(nb, num_mulfacs) + : hipCompressHeaderSize(nb, num_mulfacs); hipStream_t s = user_stream; hipStream_t aux = plan->aux_stream; - // 1. Fused wavelet + quantize + RLE → scratch (user_stream) + // 1. Fused wavelet + quantize + encode → scratch (user_stream) if (plan->is_2d) { int nbx = nx / 32, nby = ny / 32; dim3 grid((nbx + WRLE2D_TILES_PER_WG - 1) / WRLE2D_TILES_PER_WG, nby); @@ -187,7 +206,22 @@ hipError_t hipCompress( } else { const int ldimxy = nx * ny; dim3 grid((nx + 31) / 32, (ny + 31) / 32, (nz + 31) / 32); - if (plan->kernel == HIP_COMPRESS_KERNEL_SEGRLE) { + if (plan->kernel == HIP_COMPRESS_KERNEL_OCTREE) { + // k1: wavelet ZYX + quantize → [bitmap][packed int32] in d_scratch. + waveletBitmapFusedKernel<<>>( + d_input, plan->d_scratch, plan->d_block_sizes, + scale, ldimx, ldimxy, + d_rms, plan->d_mulfac); + // k2: octree significance + width table + packed values → coded + // slots. d_block_sizes := coded length; d_octree_sig_sizes := + // significance length (both per block, for scan + header). + waveletOctreeCodeParKernel<<>>( + plan->d_scratch, plan->d_octree_coded, + plan->d_block_sizes, plan->d_octree_sig_sizes); + // 4-align each coded length so packed blocks keep uint32 reads + // (width table / flat masks) aligned in the compacted stream. + waveletOctreeCodeParAlignSizes(plan->d_block_sizes, nb, s); + } else if (plan->kernel == HIP_COMPRESS_KERNEL_SEGRLE) { waveletSegRLEFusedKernel<<>>( d_input, plan->d_scratch, plan->d_block_sizes, scale, ldimx, ldimxy, @@ -217,6 +251,11 @@ hipError_t hipCompress( plan->d_block_sizes, plan->d_block_offsets, d_output, nb, num_mulfacs, plan->d_mulfac, plan->scratch_slot_stride); + } else if (plan->kernel == HIP_COMPRESS_KERNEL_OCTREE) { + woctCompactKernel<<>>( + plan->d_octree_coded, d_output + hdr_size, + plan->d_block_sizes, plan->d_block_offsets, plan->d_octree_sig_sizes, + d_output, nb, num_mulfacs, plan->d_mulfac); } else { wrleCompactKernel<<>>( plan->d_scratch, d_output + hdr_size, @@ -248,7 +287,9 @@ hipError_t hipCompressSynchronize( const int nx = plan->nx, ny = plan->ny, nz = plan->nz; const int nb = plan->num_blocks; - const int hdr_size = hipCompressHeaderSize(nb, 1); + const int hdr_size = (plan->kernel == HIP_COMPRESS_KERNEL_OCTREE) + ? hipOctreeHeaderSize(nb, 1) + : hipCompressHeaderSize(nb, 1); size_t total_payload = plan->h_staging[0] + plan->h_staging[1]; long total_bytes = (long)hdr_size + (long)total_payload; @@ -430,7 +471,16 @@ hipError_t hipDecompress( } else { const int ldimxy = nx * ny; dim3 grid((nx + 31) / 32, (ny + 31) / 32, (nz + 31) / 32); - if (plan->kernel == HIP_COMPRESS_KERNEL_SEGRLE) { + if (plan->kernel == HIP_COMPRESS_KERNEL_OCTREE) { + const int nb = plan->num_blocks; + // stage A: compacted coded stream → kernel-1 scratch layout; also + // publishes inv_scale = 1/mulfac (device) for stage B. + waveletOctreeDecodeToBitmapHdrKernel<<>>( + d_input, plan->d_scratch, plan->d_inv_scale); + // stage B: dequantize + inverse wavelet ZYX → wavefield. + waveletBitmapInverseFusedDevKernel<<>>( + plan->d_scratch, d_output, plan->d_inv_scale, ldimx, ldimxy); + } else if (plan->kernel == HIP_COMPRESS_KERNEL_SEGRLE) { waveletSegRLEInverseFusedKernel<<>>( d_input, nullptr, nullptr, d_output, 0.0f, ldimx, ldimxy, 1); @@ -467,8 +517,15 @@ hipError_t hipCompressMaxOutputSize(const hipCompressPlan* plan, size_t* size) return hipErrorInvalidValue; } plan->last_error = HIP_COMPRESS_SUCCESS; - int hdr_size = hipCompressHeaderSize(plan->num_blocks, 1); - size_t raw = (size_t)hdr_size + (size_t)plan->num_blocks * plan->scratch_slot_stride; + size_t raw; + if (plan->kernel == HIP_COMPRESS_KERNEL_OCTREE) { + // Worst case: octree header + every block at its coded-slot upper bound. + int hdr_size = hipOctreeHeaderSize(plan->num_blocks, 1); + raw = (size_t)hdr_size + (size_t)plan->num_blocks * WOCT_CODE_SLOT_BYTES; + } else { + int hdr_size = hipCompressHeaderSize(plan->num_blocks, 1); + raw = (size_t)hdr_size + (size_t)plan->num_blocks * plan->scratch_slot_stride; + } // Include the 8B round-up slack applied to compressed_length so a buffer // sized from this value can always hold the padded stream. *size = (raw + 7) & ~(size_t)7; diff --git a/hip/hipCompress.h b/hip/hipCompress.h index 2665a5f..3c66a8f 100644 --- a/hip/hipCompress.h +++ b/hip/hipCompress.h @@ -32,6 +32,7 @@ const char* hipCompressErrorString(hipCompressError_t err); enum hipCompressKernel { HIP_COMPRESS_KERNEL_ZLINE = 0, // parallel z-line RLE (per-block metadata) HIP_COMPRESS_KERNEL_SEGRLE = 1, // segment-aligned RLE (no metadata overhead) + HIP_COMPRESS_KERNEL_OCTREE = 2, // octree significance coder (3D only) }; struct hipCompressPlan { @@ -48,6 +49,12 @@ struct hipCompressPlan { void* d_scan_temp; size_t scan_temp_bytes; + // Octree kernel only (nullptr otherwise): intermediate coded slots, per-block + // significance sizes, and the device inv_scale published for stage-B decode. + unsigned char* d_octree_coded; + size_t* d_octree_sig_sizes; + float* d_inv_scale; + double* d_partial_sums; int max_copy_blocks; double* d_rms; diff --git a/hip/hipWaveletOctree.h b/hip/hipWaveletOctree.h index 5ffd22c..f94c006 100644 --- a/hip/hipWaveletOctree.h +++ b/hip/hipWaveletOctree.h @@ -756,24 +756,92 @@ inline hipError_t hipWaveletOctreeCode( return hipGetLastError(); } +// Round each per-block coded length up to `align` bytes. In the compacted +// stream the blocks are packed back-to-back, so this keeps every block start +// (and thus its uint32 width-table / flat-mask reads) aligned. Overhead is +// >>(sizes, nb, 4); + return hipGetLastError(); +} + +// Compaction: copies the variable-length coded blocks from the fixed-stride +// (WOCT_CODE_SLOT_BYTES) intermediate into a tightly packed payload using the +// exclusive-scan offsets, and writes the self-contained octree header (block +// offsets + per-block significance sizes + mulfac). dst points to the payload +// region (after the header). Mirrors wrleCompactKernel; the extra sig-size +// table is what lets the decoder recover each block's significance length. +__global__ void woctCompactKernel( + const unsigned char* __restrict__ src, + unsigned char* __restrict__ dst, + const size_t* __restrict__ block_sizes, + const size_t* __restrict__ offsets, + const size_t* __restrict__ sig_sizes, + unsigned char* __restrict__ hdr, + int num_blocks, + int num_mulfacs, + const float* __restrict__ d_mulfac) +{ + int bid = blockIdx.x; + int tid = threadIdx.x; + size_t size = block_sizes[bid]; + size_t dst_off = offsets[bid]; + size_t src_off = (size_t)bid * WOCT_CODE_SLOT_BYTES; + + if (hdr != nullptr && tid == 0) { + ((size_t*)(hdr + 8))[bid] = offsets[bid]; + ((uint32_t*)(hdr + 8 + 8L * num_blocks))[bid] = (uint32_t)sig_sizes[bid]; + if (bid == 0) { + ((int*)hdr)[0] = num_blocks; + ((int*)hdr)[1] = num_mulfacs; + float* mf_dst = (float*)(hdr + 8 + 12L * num_blocks); + for (int i = 0; i < num_mulfacs; ++i) + mf_dst[i] = d_mulfac[i]; + } + } + + for (size_t i = tid * 4; i < size; i += blockDim.x * 4) { + unsigned val; + __builtin_memcpy(&val, src + src_off + i, 4); + size_t remain = size - i; + if (remain >= 4) { + __builtin_memcpy(dst + dst_off + i, &val, 4); + } else { + for (size_t b = 0; b < remain; ++b) + dst[dst_off + i + b] = (unsigned char)(val >> (b * 8)); + } + } +} + // =========================================================================== // FULL DECODE, stage A (1024 threads): fused octree coded stream -> kernel-1 // scratch layout [4096B bitmap][packed int32 values]. Exact inverse of the // fused encoder's value packing, so the output is byte-identical to the // original kernel-1 output. // =========================================================================== -__launch_bounds__(1024) -__global__ void waveletOctreeDecodeToBitmapKernel( - const unsigned char* __restrict__ coded, - const size_t* __restrict__ sig_sizes, - unsigned char* __restrict__ scratch1_out, - size_t* __restrict__ block_sizes) +// Shared decode body: reconstruct the kernel-1 scratch layout +// [4096B bitmap][packed int32 values] for one block from its coded bytes. +// blk — pointer to the block's coded bytes ([4B mode][sig][pad][wtab][values]) +// sig_bytes — significance-region length (octree stream length, or 4096 flat) +// out — this block's WBMP_SLOT_BYTES scratch slot +// bsz_this — optional: receives WBMP_BITMAP_BYTES + nnz*4 (may be null) +// Addressing is caller-supplied so the same body serves both the fixed-stride +// prototype kernel and the header-addressed (compacted stream) API kernel. +__device__ __forceinline__ void woct_decode_block_to_scratch( + const unsigned char* __restrict__ blk, int sig_bytes, + unsigned char* __restrict__ out, size_t* __restrict__ bsz_this) { using namespace wbmp_opt; - int bid = blockIdx.x, tid = threadIdx.x; - const unsigned char* blk = coded + (long)bid * WOCT_CODE_SLOT_BYTES; + int tid = threadIdx.x; const unsigned char* sig = blk + WOCT_HDR_BYTES; - unsigned char* out = scratch1_out + (long)bid * WBMP_SLOT_BYTES; uint32_t* bmp_out = reinterpret_cast(out); int32_t* val_out = reinterpret_cast(out + WBMP_BITMAP_BYTES); @@ -788,7 +856,6 @@ __global__ void waveletOctreeDecodeToBitmapKernel( __syncthreads(); int mode = blk[0]; - int sig_bytes = (int)sig_sizes[bid]; if (mode == 0) { // flat masks (L order) const uint32_t* flat = reinterpret_cast(sig); @@ -857,7 +924,54 @@ __global__ void waveletOctreeDecodeToBitmapKernel( val_out[in_off + k] = (int)(uv << sh) >> sh; // sign-extend from W bytes } } - if (tid==0) block_sizes[bid] = (size_t)WBMP_BITMAP_BYTES + (size_t)tot_nz * 4; + if (tid==0 && bsz_this) *bsz_this = (size_t)WBMP_BITMAP_BYTES + (size_t)tot_nz * 4; +} + +// Prototype kernel: fixed-stride coded slots + per-block sig-size array. +__launch_bounds__(1024) +__global__ void waveletOctreeDecodeToBitmapKernel( + const unsigned char* __restrict__ coded, + const size_t* __restrict__ sig_sizes, + unsigned char* __restrict__ scratch1_out, + size_t* __restrict__ block_sizes) +{ + int bid = blockIdx.x; + woct_decode_block_to_scratch( + coded + (long)bid * WOCT_CODE_SLOT_BYTES, (int)sig_sizes[bid], + scratch1_out + (long)bid * WBMP_SLOT_BYTES, &block_sizes[bid]); +} + +// Octree self-contained stream header: +// [int nb][int nmf][size_t offsets[nb]][uint32 sig_sizes[nb]][float mulfac[nmf]] +__host__ __device__ inline int hipOctreeHeaderSize(int num_blocks, int num_mulfacs) { + return 8 + 12 * num_blocks + 4 * num_mulfacs; +} + +// API decode kernel: locates each block in the compacted stream via the header +// offset table, reads its significance length from the header sig-size table, +// reconstructs the kernel-1 scratch layout, and (block 0) publishes +// inv_scale = 1/mulfac for stage B. +__launch_bounds__(1024) +__global__ void waveletOctreeDecodeToBitmapHdrKernel( + const unsigned char* __restrict__ input, + unsigned char* __restrict__ scratch1_out, + float* __restrict__ inv_scale_out) +{ + int bid = blockIdx.x, tid = threadIdx.x; + const int* hdr = reinterpret_cast(input); + int num_blocks = hdr[0]; + int num_mulfacs = hdr[1]; + const size_t* offsets = reinterpret_cast(input + 8); + const uint32_t* sig_u32 = reinterpret_cast(input + 8 + 8L * num_blocks); + const float* mulfacs = reinterpret_cast(input + 8 + 12L * num_blocks); + const unsigned char* data_base = input + hipOctreeHeaderSize(num_blocks, num_mulfacs); + + if (bid == 0 && tid == 0 && inv_scale_out) + *inv_scale_out = 1.0f / mulfacs[0]; + + woct_decode_block_to_scratch( + data_base + offsets[bid], (int)sig_u32[bid], + scratch1_out + (long)bid * WBMP_SLOT_BYTES, nullptr); } inline hipError_t hipWaveletOctreeDecodeToBitmap( @@ -876,8 +990,7 @@ inline hipError_t hipWaveletOctreeDecodeToBitmap( // (RLE decode) is swapped for the bitmap+value reconstruction, which reverses // waveletBitmapFusedKernel's Phase 4. Requires DS79_INCLUDE_REG32. // =========================================================================== -__launch_bounds__(256, 2) -__global__ void waveletBitmapInverseFusedKernel( +__device__ __forceinline__ void woct_bitmap_inverse_body( const unsigned char* __restrict__ scratch1, float* __restrict__ output, float inv_scale, int ldimx, int ldimxy) @@ -960,6 +1073,27 @@ __global__ void waveletBitmapInverseFusedKernel( } } +// Stage B with a host-scalar inv_scale (prototype / test path). +__launch_bounds__(256, 2) +__global__ void waveletBitmapInverseFusedKernel( + const unsigned char* __restrict__ scratch1, + float* __restrict__ output, + float inv_scale, int ldimx, int ldimxy) +{ + woct_bitmap_inverse_body(scratch1, output, inv_scale, ldimx, ldimxy); +} + +// Stage B with a device inv_scale pointer (API path): reads 1/mulfac published +// by the header-addressed decode kernel, so no host readback is needed. +__launch_bounds__(256, 2) +__global__ void waveletBitmapInverseFusedDevKernel( + const unsigned char* __restrict__ scratch1, + float* __restrict__ output, + const float* __restrict__ inv_scale, int ldimx, int ldimxy) +{ + woct_bitmap_inverse_body(scratch1, output, *inv_scale, ldimx, ldimxy); +} + inline hipError_t hipWaveletBitmapInverseFused( const unsigned char* scratch1, float* output, float inv_scale, int nx, int ny, int nz, int ldimx, int ldimxy, hipStream_t stream=0) diff --git a/makefile b/makefile index fc3336b..eeda9bc 100644 --- a/makefile +++ b/makefile @@ -129,7 +129,7 @@ test_wavelet_rle_fused_hip: tests/test_wavelet_rle_fused_hip.cpp hip/hipWaveletR $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -save-temps=obj -I. -Ihip -Itests tests/test_wavelet_rle_fused_hip.cpp hip/hipWaveletTransformBuffer.cpp -L. -lcvxcompress -lm -o $(BUILDDIR)/test_wavelet_rle_fused_hip # hipCompress public API test -test_compress_api_hip: tests/test_compress_api_hip.cpp hip/hipCompress.cpp hip/hipCompress.h hip/hipBlockCopy.h hip/hipWaveletRLE.h hip/hipWaveletRLEInverse.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc libcvxcompress.$(LIB_EXT) | $(BUILDDIR) +test_compress_api_hip: tests/test_compress_api_hip.cpp hip/hipCompress.cpp hip/hipCompress.h hip/hipBlockCopy.h hip/hipWaveletRLE.h hip/hipWaveletRLEInverse.h hip/hipWaveletBitmap.h hip/hipWaveletOctree.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc libcvxcompress.$(LIB_EXT) | $(BUILDDIR) $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_compress_api_hip.cpp hip/hipCompress.cpp -L. -lcvxcompress '-Wl,-rpath,$$ORIGIN/..' -lm -o $(BUILDDIR)/test_compress_api_hip # 2D compression test diff --git a/tests/test_compress_api_hip.cpp b/tests/test_compress_api_hip.cpp index f766a13..9fc93c9 100644 --- a/tests/test_compress_api_hip.cpp +++ b/tests/test_compress_api_hip.cpp @@ -182,6 +182,64 @@ static bool test_round_trip() return pass; } +static bool test_octree_round_trip() +{ + printf("Test: Octree compress/decompress round-trip\n"); + const int N = 128, total = N * N * N; + const float scale = 5e-2f; + + hipCompressPlan* oct = nullptr; + HIPCHECK(hipCompressCreatePlan(&oct, N, N, N, 0, HIP_COMPRESS_KERNEL_OCTREE)); + hipCompressPlan* rle = nullptr; + HIPCHECK(hipCompressCreatePlan(&rle, N, N, N, 0, HIP_COMPRESS_KERNEL_ZLINE)); + + float* d_input = nullptr; + float* d_output = nullptr; + unsigned char* d_comp = nullptr; + unsigned char* d_comp_rle = nullptr; + HIPCHECK(hipMalloc(&d_input, total * sizeof(float))); + HIPCHECK(hipMalloc(&d_output, total * sizeof(float))); + size_t comp_size = 0; + HIPCHECK(hipCompressMaxOutputSize(oct, &comp_size)); + HIPCHECK(hipMalloc(&d_comp, comp_size)); + size_t comp_size_rle = 0; + HIPCHECK(hipCompressMaxOutputSize(rle, &comp_size_rle)); + HIPCHECK(hipMalloc(&d_comp_rle, comp_size_rle)); + + int threads = 256, blocks = (total + threads - 1) / threads; + initSinKernel<<>>(d_input, N, N, N, 20.0f, 20.0f, 20.0f); + HIPCHECK(hipDeviceSynchronize()); + + long len = 0, len_rle = 0; + float cr = 0, cr_rle = 0; + HIPCHECK(compressWithAutoRMS(scale, d_input, d_comp, &len, &cr, oct)); + HIPCHECK(compressWithAutoRMS(scale, d_input, d_comp_rle, &len_rle, &cr_rle, rle)); + printf(" octree CR=%.2f (%ld B) RLE CR=%.2f (%ld B) gain=%.2fx\n", + cr, len, cr_rle, len_rle, cr_rle > 0 ? cr / cr_rle : 0.0f); + + HIPCHECK(hipDecompress(d_comp, d_output, oct, 0)); + + std::vector h_in(total), h_out(total); + HIPCHECK(hipMemcpy(h_in.data(), d_input, total * sizeof(float), hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_out.data(), d_output, total * sizeof(float), hipMemcpyDeviceToHost)); + + float rms = hostRMS(h_in.data(), total); + float max_err = maxAbsError(h_in.data(), h_out.data(), total); + printf(" decompress: max_err=%.6e, rms=%.6e, rel_max_err=%.6e\n", + max_err, rms, max_err / rms); + + // Correctness gate: bounded reconstruction + a real compression gain. The + // octree-vs-RLE CR delta is a benchmark property (printed above), not a + // correctness invariant, so it is not part of the pass condition. + bool pass = (max_err < rms) && (cr > 1.0f) && (len > 0); + printf(" octree round-trip: %s\n", pass ? "PASS" : "FAIL"); + + hipFree(d_input); hipFree(d_output); hipFree(d_comp); hipFree(d_comp_rle); + hipCompressDestroyPlan(oct); + hipCompressDestroyPlan(rle); + return pass; +} + static bool test_cr_vs_cpu() { printf("Test 3: CR matches CPU (within z-line gap)\n"); @@ -1664,13 +1722,14 @@ static bool test_wavelet_dims_helper() return pass; } -static void bench_grid_size(int nx, int ny, int nz, float scale) +static void bench_grid_size(int nx, int ny, int nz, float scale, + hipCompressKernel kernel, const char* label) { long total = (long)nx * ny * nz; float data_MB = (float)total * sizeof(float) / (1024.0f * 1024.0f); hipCompressPlan* plan = nullptr; - HIPCHECK(hipCompressCreatePlan(&plan, nx, ny, nz, 0)); + HIPCHECK(hipCompressCreatePlan(&plan, nx, ny, nz, 0, kernel)); float* d_input = nullptr; float* d_output = nullptr; @@ -1715,8 +1774,8 @@ static void bench_grid_size(int nx, int ny, int nz, float scale) float fwd_bw = data_MB / fwd_ms * 1000.0f / 1024.0f; float inv_bw = data_MB / inv_ms * 1000.0f / 1024.0f; - printf(" %4dx%4dx%4d %7.1f %5.1f:1 %8.3f %8.1f %8.3f %8.1f %5.2fx\n", - nx, ny, nz, data_MB, cr, fwd_ms, fwd_bw, inv_ms, inv_bw, inv_ms / fwd_ms); + printf(" %4dx%4dx%4d %7s %7.1f %6.1f:1 %8.3f %8.1f %8.3f %8.1f %5.2fx\n", + nx, ny, nz, label, data_MB, cr, fwd_ms, fwd_bw, inv_ms, inv_bw, inv_ms / fwd_ms); hipEventDestroy(t0); hipEventDestroy(t1); hipEventDestroy(t2); hipFree(d_input); hipFree(d_output); hipFree(d_compressed); @@ -1727,8 +1786,8 @@ static void bench_throughput() { const float scale = 5e-2f; printf("Benchmark: API throughput, scale=%.0e, sin(40x)sin(40y)sin(40z)\n", scale); - printf(" %16s %7s %5s %8s %8s %8s %8s %5s\n", - "grid", "MB", "CR", "fwd(ms)", "fwd GB/s", "inv(ms)", "inv GB/s", "ratio"); + printf(" %16s %7s %7s %6s %8s %8s %8s %8s %5s\n", + "grid", "codec", "MB", "CR", "fwd(ms)", "fwd GB/s", "inv(ms)", "inv GB/s", "ratio"); int sizes[][3] = { {352, 416, 320}, @@ -1741,8 +1800,11 @@ static void bench_throughput() {512, 512, 512}, }; - for (auto& s : sizes) - bench_grid_size(s[0], s[1], s[2], scale); + // RLE (z-line) vs octree on the API path, same input and quantization. + for (auto& s : sizes) { + bench_grid_size(s[0], s[1], s[2], scale, HIP_COMPRESS_KERNEL_ZLINE, "rle"); + bench_grid_size(s[0], s[1], s[2], scale, HIP_COMPRESS_KERNEL_OCTREE, "octree"); + } } static void generateRadialSinc(float* vol, int nx, int ny, int nz, @@ -2955,9 +3017,10 @@ int main(int argc, char** argv) printf("=== hipCompress API Tests ===\n\n"); - int passed = 0, total = 37; + int passed = 0, total = 38; if (test_plan_lifecycle()) ++passed; if (test_round_trip()) ++passed; + if (test_octree_round_trip()) ++passed; if (test_cr_vs_cpu()) ++passed; if (test_varying_scale()) ++passed; if (test_multiple_cycles()) ++passed; From fcfc9a993154a2820adfca2c612ce761a18767f2 Mon Sep 17 00:00:00 2001 From: Ossian O'Reilly Date: Wed, 19 Aug 2026 12:56:56 -0700 Subject: [PATCH 4/7] Add PFOR value coding + quadtree codec; auto-select default by dimensionality Replace the per-line width-table value coder in the octree (3D) and new quadtree (2D) significance codecs with a per-block PFOR (patched frame-of-reference) layout: a fixed W_lo base stream plus an exception bitmap and W_hi patch stream. All streams stay fixed-width and byte-aligned, so decode throughput is unchanged while compression ratio improves (~+5.6-10.4% at 512^3 for ~1-3% encode cost; near-0% decode). Add HIP_COMPRESS_KERNEL_AUTO and make it the plan default: resolves to QUADTREE for 2D (nz == 1) and OCTREE for 3D at plan creation, with ZLINE as the fallback for configurations the structured coders cannot handle. ZLINE remains the encode-throughput hedge for encode-bound callers. makefile: HIP_ARCH now accepts a space-separated list to build a fat binary (e.g. "gfx942 gfx950"). Validated: test_compress_api_hip 39/39 and test_compress_2d_hip 6/6 on gfx942 (MI300X) and gfx950 (MI355X). --- HIP_API.md | 78 ++- OCTREE_PFOR_THROUGHPUT.md | 135 +++++ hip/hipCompress.cpp | 143 ++++- hip/hipCompress.h | 32 +- hip/hipWaveletBitmap.h | 146 +++++ hip/hipWaveletOctree.h | 158 +++-- hip/hipWaveletQuadtree2D.h | 608 ++++++++++++++++++++ hip/hipWaveletRLE.h | 47 +- makefile | 46 +- tests/bench_quadtree_vs_cvx_2d.cpp | 462 +++++++++++++++ tests/test_bitmap_octree_hip.cpp | 42 +- tests/test_bitmap_rd_panel.cpp | 388 +++++++++++++ tests/test_compress_2d_hip.cpp | 83 +++ tests/test_compress_api_hip.cpp | 79 ++- tests/test_quant_flip_amplification_hip.cpp | 349 +++++++++++ 15 files changed, 2669 insertions(+), 127 deletions(-) create mode 100644 OCTREE_PFOR_THROUGHPUT.md create mode 100644 hip/hipWaveletQuadtree2D.h create mode 100644 tests/bench_quadtree_vs_cvx_2d.cpp create mode 100644 tests/test_bitmap_rd_panel.cpp create mode 100644 tests/test_quant_flip_amplification_hip.cpp diff --git a/HIP_API.md b/HIP_API.md index 4383a08..581eae8 100644 --- a/HIP_API.md +++ b/HIP_API.md @@ -187,12 +187,76 @@ The `scale` argument to `hipCompress` controls the quality/compression tradeoff: ### Kernel Variants -| Variant | Enum | Description | -|---------|------|-------------| -| Z-line | `HIP_COMPRESS_KERNEL_ZLINE` | Parallel z-line RLE with per-block metadata (default) | -| Seg-RLE | `HIP_COMPRESS_KERNEL_SEGRLE` | Segment-aligned RLE, no metadata overhead (unoptimized) | +| Variant | Enum | Dims | Description | +|---------|------|------|-------------| +| Auto | `HIP_COMPRESS_KERNEL_AUTO` | 2D/3D | **Default.** Dimensionality-selected: resolves to quadtree for 2D (`nz == 1`) and octree for 3D at plan creation, falling back to z-line for configurations the structured coders cannot handle | +| Z-line | `HIP_COMPRESS_KERNEL_ZLINE` | 2D/3D | Parallel z-line RLE with per-block metadata. Highest encode throughput; the throughput hedge for encode-bound paths (e.g. per-timestep RTM checkpoint spilling) | +| Seg-RLE | `HIP_COMPRESS_KERNEL_SEGRLE` | 3D | Segment-aligned RLE, no metadata overhead (unoptimized) | +| Octree | `HIP_COMPRESS_KERNEL_OCTREE` | 3D | Octree significance coder with per-block PFOR value coding. Best compression ratio and fastest decode; recommended for storage/archival | +| Quadtree | `HIP_COMPRESS_KERNEL_QUADTREE` | 2D | Quadtree significance coder with per-block PFOR value coding -- the 2D counterpart of octree | +| Two-level | `HIP_COMPRESS_KERNEL_TWOLEVEL` | 3D | Two-level occupancy + per-line width coder. Higher encode throughput than octree at a lower ratio; a hedge for compute/bandwidth-bound, rewrite-heavy paths | + +The codec is selected **at runtime, per plan** via the last argument of +`hipCompressCreatePlan` — it is an ordinary function parameter, so switching +between codecs requires **no recompilation** of the library or the application: -Select at plan creation: `hipCompressCreatePlan(&plan, nx, ny, nz, aux, HIP_COMPRESS_KERNEL_SEGRLE)`. +```cpp +// default (kernel arg omitted) → auto: octree for 3D, quadtree for 2D +hipCompressCreatePlan(&plan_def, nx, ny, nz, aux); + +// storage-oriented volume → octree (best ratio) — explicit form of the 3D default +hipCompressCreatePlan(&plan_oct, nx, ny, nz, aux, HIP_COMPRESS_KERNEL_OCTREE); + +// encode-bound / high-rate path → z-line (fastest encode) +hipCompressCreatePlan(&plan_zl, nx, ny, nz, aux, HIP_COMPRESS_KERNEL_ZLINE); +``` + +The codec is bound to the plan (internal buffer sizes and stream header layout +differ per codec), so the switching granularity is "which plan you create"; an +existing plan's codec cannot be changed in place. Octree and two-level are 3D +only, quadtree is 2D only — requesting one for the wrong dimensionality fails +plan creation with `HIP_COMPRESS_ERROR_INVALID_DIMENSIONS`. `AUTO` avoids this +by resolving to the dimensionality-appropriate coder at plan creation. + +**Choosing a codec.** The `AUTO` default maximizes compression ratio and decode +throughput and is the right choice for storage/archival and read-heavy paths. +For **encode-bound** paths that compress on a hot loop — e.g. per-timestep RTM +checkpoint spilling — prefer `ZLINE`, which has the highest encode throughput. +The octree/quadtree encode cost over z-line is small at production grid sizes +(~1–3% at 512³) but grows at small grids where the per-block histogram, scans, +and PFOR bookkeeping are not amortized (see `OCTREE_PFOR_THROUGHPUT.md`). + +**Memory footprint.** Octree/quadtree allocate a larger per-block scratch stride +(`WOCT_CODE_SLOT_BYTES` ~140 KB/block plus the bitmap scratch) than z-line; +these are global HBM buffers (not LDS). Negligible on MI300X/MI355X but worth +noting for very large volumes. + +#### Two-level: architecture-selected encoder + +The two-level codec has two encoders that emit a **byte-identical** stream: + +- an LDS-staged **opt** encoder (~131 KB LDS) that requires CDNA4 (gfx950, + e.g. MI355x), and +- a **portable** encoder used on every other architecture (gfx942/MI300x, + gfx90a, …). + +The encoder is chosen automatically at plan creation from the device +architecture (`gcnArchName`); no user action is required. Because the streams +are identical, a volume encoded on one GPU decodes correctly on any other, and +the single-arch-independent decoder runs everywhere. + +For this automatic selection to reach the fast path, the library `.so` must +have been built with the target architecture(s) included — build once as a fat +binary covering your deployment GPUs and no per-user recompile is ever needed: + +```bash +make libhipcvxcompress.so HIP_ARCH="gfx90a gfx942 gfx950" +``` + +`HIP_ARCH` accepts a space-separated list; each architecture is expanded to a +`--offload-arch=` flag. The opt encoder is compiled only into the gfx950 device +image (a no-op stub elsewhere), so the fat binary carries both, and the HIP +loader plus the runtime dispatch pick the correct one for the GPU in use. ### Two-Stream Model @@ -237,11 +301,13 @@ hip/ hipWaveletRLEInverse.h Fused inverse RLE + wavelet kernels hipRLEDecode.h Z-line RLE decoder hipSegmentedRLE.h Segment-aligned RLE encode/decode + hipWaveletBitmap.h Bitmap significance split + two-level coder/decoder + hipWaveletOctree.h Octree significance coder + shared inverse stage ds79.h DS 7/9 wavelet filter coefficients and transforms ds79_reg32.inc Unrolled forward wavelet (32-point) us79_reg32.inc Unrolled inverse wavelet (32-point) tests/ - test_compress_api_hip.cpp API test suite (37 tests + benchmarks) + test_compress_api_hip.cpp API test suite (39 tests + benchmarks) example_async_pipeline.cpp Async overlap example ``` diff --git a/OCTREE_PFOR_THROUGHPUT.md b/OCTREE_PFOR_THROUGHPUT.md new file mode 100644 index 0000000..59b2cdd --- /dev/null +++ b/OCTREE_PFOR_THROUGHPUT.md @@ -0,0 +1,135 @@ +# Octree + PFOR vs RLE: throughput across sparsity + +GPU encode/decode throughput of the **previous RLE codec** (`HIP_COMPRESS_KERNEL_ZLINE`, +the default z-line RLE) versus the **octree significance + per-block PFOR value +coder**, swept across coefficient sparsity at matched fidelity. + +## Method + +- **Data**: `solver_steps_512_marmousi_h20/snapshot_u_001000.raw`, a 128³ crop at + origin `(z0=0, y0=192, x0=192)` — a wavefront region (~33% of voxels above + 1e-3·max in the raw field). Normalized to unit RMS. +- **Sparsity axis**: quantization multiplier `--scale` (mulfac). Higher mulfac → + finer quantization → more nonzero coefficients → *denser* (lower CR). Fused CR + is used as the density proxy. +- **Matched fidelity**: both codecs share the wavelet transform + quantization, so + `vol_rel_l2` is identical at every point (verified per row below). +- **Harness**: `tests/test_bitmap_octree_hip.cpp`, `--nx 128 --iters 100`, raw GB/s + over the f32 volume. Encode octree = k1 (wavelet) + k2 (PFOR); decode octree = + stage-A (PFOR value unpack) + stage-B (inverse wavelet). +- **Value round-trip** validated format-agnostically (`decode: stage-A round-trip + vs kernel-1 = OK`) at every point. + +Command: + +```bash +for MF in 1 2 5 10 20 40 80; do + ./build/test_bitmap_octree_hip --panel $PANEL --nx 128 --z0 0 --y0 192 --x0 192 \ + --scale $MF --iters 100 +done +``` + +## MI300X (gfx942) + +Full-pipeline throughput (raw GB/s); `oct` = octree+PFOR. + +| mulfac | rel_l2 | fused CR | enc RLE | enc oct | enc oct/RLE | dec RLE | dec oct | dec oct/RLE | +|-------:|-------:|---------:|--------:|--------:|:-----------:|--------:|--------:|:-----------:| +| 1 | 1.32e-1 | 166 | 167.6 | 172.4 | 1.03× | 97.1 | 190.3 | 1.96× | +| 2 | 8.22e-2 | 98 | 162.1 | 165.2 | 1.02× | 90.4 | 166.5 | 1.84× | +| 5 | 4.25e-2 | 53 | 155.1 | 151.2 | 0.97× | 80.2 | 151.6 | 1.89× | +| 10 | 2.51e-2 | 35 | 151.9 | 141.0 | 0.93× | 76.6 | 141.9 | 1.85× | +| 20 | 1.45e-2 | 25 | 148.9 | 134.1 | 0.90× | 73.4 | 136.2 | 1.86× | +| 40 | 8.13e-3 | 18 | 146.0 | 122.4 | 0.84× | 69.8 | 129.8 | 1.86× | +| 80 | 4.46e-3 | 14 | 142.5 | 113.5 | 0.80× | 67.8 | 124.2 | 1.83× | + +Isolated value-coder kernels (raw GB/s): `FUSED` = k2 PFOR encode, `PAR_dec` = +stage-A PFOR decode. + +| mulfac | fused CR | FUSED (k2 enc) | PAR_dec (stage-A) | +|-------:|---------:|---------------:|------------------:| +| 1 | 166 | 369.9 | 755.4 | +| 2 | 98 | 339.5 | 747.6 | +| 5 | 53 | 286.2 | 740.7 | +| 10 | 35 | 253.5 | 735.0 | +| 20 | 25 | 238.6 | 728.8 | +| 40 | 18 | 215.8 | 725.1 | +| 80 | 14 | 203.6 | 725.2 | + +### Trends + +- **Decode**: octree+PFOR is **1.8–2.0× faster than RLE at every sparsity**, and + the ratio is nearly flat. RLE decode is the bottleneck (serial run expansion, + 97→68 GB/s as density rises). +- **Encode**: octree's edge is largest at nnz≈0 (1.03× at CR 166) and erodes + monotonically to 0.80× at the densest point — as expected, since PFOR must code + every nonzero while RLE encode is already bandwidth-bound (167→143, density-flat). + Encode crossover is around CR≈60 (mulfac≈5). +- **PFOR decode is density-insensitive**: stage-A only drops 755→725 GB/s (−4%) + across a ~12× CR range, because every stream is fixed-width / byte-aligned (no + per-nonzero branching). Encode-k2 scales with nnz (370→204) — the histogram + + exception scan + mask cost. That asymmetry is intentional: spend a little on + encode to keep decode fast and flat. + +Net: octree+PFOR wins decode decisively at all sparsities and wins encode only in +the sparse regime; in the dense regime it trades encode throughput for the CR gain +(and the large decode advantage). + +## MI355X (gfx950) + +Same data, crop, and sweep; CR and `rel_l2` are identical to MI300X (same +algorithm). Full-pipeline throughput (raw GB/s); `oct` = octree+PFOR. + +| mulfac | rel_l2 | fused CR | enc RLE | enc oct | enc oct/RLE | dec RLE | dec oct | dec oct/RLE | +|-------:|-------:|---------:|--------:|--------:|:-----------:|--------:|--------:|:-----------:| +| 1 | 1.32e-1 | 166 | 184.6 | 184.1 | 1.00× | 109.5 | 207.3 | 1.89× | +| 2 | 8.22e-2 | 98 | 177.2 | 175.8 | 0.99× | 101.0 | 181.8 | 1.80× | +| 5 | 4.25e-2 | 53 | 169.2 | 161.0 | 0.95× | 89.7 | 166.3 | 1.85× | +| 10 | 2.51e-2 | 35 | 165.8 | 152.3 | 0.92× | 84.4 | 156.4 | 1.85× | +| 20 | 1.45e-2 | 25 | 162.3 | 142.9 | 0.88× | 81.0 | 150.3 | 1.85× | +| 40 | 8.13e-3 | 18 | 159.7 | 135.0 | 0.85× | 76.9 | 143.4 | 1.86× | +| 80 | 4.46e-3 | 14 | 155.8 | 126.0 | 0.81× | 75.2 | 137.1 | 1.82× | + +Isolated value-coder kernels (raw GB/s): `FUSED` = k2 PFOR encode, `PAR_dec` = +stage-A PFOR decode. + +| mulfac | fused CR | FUSED (k2 enc) | PAR_dec (stage-A) | +|-------:|---------:|---------------:|------------------:| +| 1 | 166 | 387.6 | 823.2 | +| 2 | 98 | 352.1 | 809.6 | +| 5 | 53 | 298.5 | 804.8 | +| 10 | 35 | 270.0 | 801.0 | +| 20 | 25 | 244.6 | 796.6 | +| 40 | 18 | 229.3 | 792.6 | +| 80 | 14 | 211.9 | 789.6 | + +### Trends + +Identical qualitative behavior to MI300X, uniformly faster (higher clocks/BW): + +- **Decode**: octree+PFOR is **1.8–1.9× faster than RLE at every sparsity**; ratio + nearly flat. RLE decode 110→75 GB/s as density rises. +- **Encode**: octree edge highest at nnz≈0 (1.00× at CR 166), eroding to 0.81× at + the densest point; crossover near CR≈55 (mulfac≈5). RLE encode density-flat + (185→156). +- **PFOR decode is density-insensitive**: stage-A 823→790 GB/s (−4%) across the CR + range; encode-k2 scales with nnz (388→212). + +### MI355X vs MI300X + +MI355X is ~1.1–1.3× faster on every metric at matched work; the RLE-vs-octree +relationship (decode win everywhere, encode win only when sparse) is unchanged. + +| metric (CR 35 / mulfac 10) | MI300X | MI355X | × | +|----------------------------|-------:|-------:|-----:| +| octree decode (full) | 141.9 | 156.4 | 1.10 | +| octree encode (full) | 141.0 | 152.3 | 1.08 | +| PFOR decode (stage-A) | 735.0 | 801.0 | 1.09 | +| PFOR encode (k2) | 253.5 | 270.0 | 1.07 | + +## Provenance + +- Hardware: MI300X (gfx942, TheraC16), MI355X (gfx950, TheraC79); ROCm 7.2.1. +- Build: `make test_bitmap_octree_hip HIP_ARCH=`. +- All rows: `decode: stage-A round-trip vs kernel-1 = OK` (lossless PFOR recode). +- Date: 2026-08-18. diff --git a/hip/hipCompress.cpp b/hip/hipCompress.cpp index d3068e9..cf9c990 100644 --- a/hip/hipCompress.cpp +++ b/hip/hipCompress.cpp @@ -10,6 +10,7 @@ #include "hipWaveletRLE2D.h" #include "hipWaveletBitmap.h" #include "hipWaveletOctree.h" +#include "hipWaveletQuadtree2D.h" #include "hipBlockCopy.h" #include @@ -84,6 +85,17 @@ hipError_t hipCompressCreatePlan(hipCompressPlan** plan, int nx, int ny, int nz, if ((long)nx * (long)ny * (long)sizeof(float) > (1L << 32)) PLAN_ERROR(p, HIP_COMPRESS_ERROR_PLANE_TOO_LARGE, hipErrorInvalidValue); + // Resolve the dimensionality-selected default to a concrete codec: quadtree + // for 2D, octree for 3D. Dimensions are already validated as 32-multiples + // above (shared by every codec), so the structured coders always apply for + // valid dims; ZLINE remains the fallback for configurations they cannot + // handle, guarding future constraints so the default flip never breaks + // callers. Downstream logic only ever sees the resolved concrete kernel. + if (kernel == HIP_COMPRESS_KERNEL_AUTO) { + kernel = is_2d ? HIP_COMPRESS_KERNEL_QUADTREE + : HIP_COMPRESS_KERNEL_OCTREE; + } + p->kernel = kernel; p->nx = nx; p->ny = ny; p->nz = nz; p->is_2d = is_2d; @@ -92,15 +104,25 @@ hipError_t hipCompressCreatePlan(hipCompressPlan** plan, int nx, int ny, int nz, p->last_error = HIP_COMPRESS_ERROR_HIP_RUNTIME; if (is_2d) { - if (kernel == HIP_COMPRESS_KERNEL_OCTREE) // octree is 3D only + // Octree and two-level significance coders are 3D only. + if (kernel == HIP_COMPRESS_KERNEL_OCTREE || + kernel == HIP_COMPRESS_KERNEL_TWOLEVEL) PLAN_ERROR(p, HIP_COMPRESS_ERROR_INVALID_DIMENSIONS, hipErrorInvalidValue); p->num_blocks = (nx / 32) * (ny / 32); - p->scratch_slot_stride = WRLE2D_SLOT_BYTES; + // Quadtree: d_scratch holds the per-block int32 32x32 grid (encode: k1 + // output; decode: stage-A output). RLE: fixed-stride RLE slots. + p->scratch_slot_stride = (kernel == HIP_COMPRESS_KERNEL_QUADTREE) + ? (size_t)WQT2D_GRID_BYTES : (size_t)WRLE2D_SLOT_BYTES; } else { + // Quadtree significance coder is 2D only. + if (kernel == HIP_COMPRESS_KERNEL_QUADTREE) + PLAN_ERROR(p, HIP_COMPRESS_ERROR_INVALID_DIMENSIONS, hipErrorInvalidValue); p->num_blocks = (nx / 32) * (ny / 32) * (nz / 32); - // Octree: d_scratch holds the kernel-1 bitmap+values layout (encode: k1 - // output; decode: stage-A output). RLE: fixed-stride RLE slots. - p->scratch_slot_stride = (kernel == HIP_COMPRESS_KERNEL_OCTREE) + // Octree / two-level: d_scratch holds the kernel-1 bitmap+values layout + // (encode: k1 output; decode: stage-A output). RLE: fixed-stride slots. + bool bitmap_split = (kernel == HIP_COMPRESS_KERNEL_OCTREE || + kernel == HIP_COMPRESS_KERNEL_TWOLEVEL); + p->scratch_slot_stride = bitmap_split ? (size_t)WBMP_SLOT_BYTES : 4L * WRLE_LDS_BYTES; } @@ -117,6 +139,22 @@ hipError_t hipCompressCreatePlan(hipCompressPlan** plan, int nx, int ny, int nz, (long)nb * WOCT_CODE_SLOT_BYTES)); HIPCHECK_PLAN(p, hipMalloc(&p->d_octree_sig_sizes, nb * sizeof(size_t))); HIPCHECK_PLAN(p, hipMalloc(&p->d_inv_scale, sizeof(float))); + } else if (kernel == HIP_COMPRESS_KERNEL_QUADTREE) { + HIPCHECK_PLAN(p, hipMalloc(&p->d_octree_coded, + (long)nb * WQT2D_CODE_SLOT_BYTES)); + HIPCHECK_PLAN(p, hipMalloc(&p->d_octree_sig_sizes, nb * sizeof(size_t))); + HIPCHECK_PLAN(p, hipMalloc(&p->d_inv_scale, sizeof(float))); + } else if (kernel == HIP_COMPRESS_KERNEL_TWOLEVEL) { + HIPCHECK_PLAN(p, hipMalloc(&p->d_octree_coded, + (long)nb * WBMP_TL_SLOT_BYTES)); + HIPCHECK_PLAN(p, hipMalloc(&p->d_inv_scale, sizeof(float))); + // Select the encoder by device arch: the LDS-staged opt kernel needs + // gfx950's 160 KB LDS; every other arch uses the portable encoder. + int dev = 0; + hipDeviceProp_t prop; + if (hipGetDevice(&dev) == hipSuccess && + hipGetDeviceProperties(&prop, dev) == hipSuccess) + p->tl_use_opt = (strstr(prop.gcnArchName, "gfx950") != nullptr); } p->scan_temp_bytes = 0; @@ -189,7 +227,11 @@ hipError_t hipCompress( const int ldimx = nx; const int nb = plan->num_blocks; const int num_mulfacs = 1; - const int hdr_size = (plan->kernel == HIP_COMPRESS_KERNEL_OCTREE) + // OCTREE and QUADTREE use the self-contained header with a per-block + // significance-size table; the RLE / two-level paths use the compact header. + const bool octree_hdr = (plan->kernel == HIP_COMPRESS_KERNEL_OCTREE || + plan->kernel == HIP_COMPRESS_KERNEL_QUADTREE); + const int hdr_size = octree_hdr ? hipOctreeHeaderSize(nb, num_mulfacs) : hipCompressHeaderSize(nb, num_mulfacs); hipStream_t s = user_stream; @@ -199,10 +241,23 @@ hipError_t hipCompress( if (plan->is_2d) { int nbx = nx / 32, nby = ny / 32; dim3 grid((nbx + WRLE2D_TILES_PER_WG - 1) / WRLE2D_TILES_PER_WG, nby); - waveletRLE2DFusedKernel<<>>( - d_input, plan->d_scratch, plan->d_block_sizes, - scale, ldimx, nbx, - d_rms, plan->d_mulfac); + if (plan->kernel == HIP_COMPRESS_KERNEL_QUADTREE) { + // k1: 2D wavelet + quantize → int32 32x32 grid in d_scratch. + waveletQuadtree2DForwardKernel<<>>( + d_input, reinterpret_cast(plan->d_scratch), + scale, ldimx, nbx, d_rms, plan->d_mulfac); + // k2: quadtree significance + width table + packed values → coded + // slots. d_block_sizes := coded length; d_octree_sig_sizes := + // significance length (both per block, for scan + header). + waveletQuadtree2DCodeKernel<<>>( + reinterpret_cast(plan->d_scratch), plan->d_octree_coded, + plan->d_block_sizes, plan->d_octree_sig_sizes); + } else { + waveletRLE2DFusedKernel<<>>( + d_input, plan->d_scratch, plan->d_block_sizes, + scale, ldimx, nbx, + d_rms, plan->d_mulfac); + } } else { const int ldimxy = nx * ny; dim3 grid((nx + 31) / 32, (ny + 31) / 32, (nz + 31) / 32); @@ -221,6 +276,22 @@ hipError_t hipCompress( // 4-align each coded length so packed blocks keep uint32 reads // (width table / flat masks) aligned in the compacted stream. waveletOctreeCodeParAlignSizes(plan->d_block_sizes, nb, s); + } else if (plan->kernel == HIP_COMPRESS_KERNEL_TWOLEVEL) { + // k1: wavelet ZYX + quantize → [bitmap][packed int32] in d_scratch. + waveletBitmapFusedKernel<<>>( + d_input, plan->d_scratch, plan->d_block_sizes, + scale, ldimx, ldimxy, + d_rms, plan->d_mulfac); + // k2: two-level occupancy + width table + packed values → coded + // slots. Arch-selected encoder; both emit the identical stream. + if (plan->tl_use_opt) + waveletBitmapCodeTwoLevelOptKernel<<>>( + plan->d_scratch, nullptr, plan->d_octree_coded, plan->d_block_sizes); + else + waveletBitmapCodeTwoLevelKernel<<>>( + plan->d_scratch, nullptr, plan->d_octree_coded, plan->d_block_sizes); + // 4-align each coded length (uint32 occupancy/mask/width reads). + waveletOctreeCodeParAlignSizes(plan->d_block_sizes, nb, s); } else if (plan->kernel == HIP_COMPRESS_KERNEL_SEGRLE) { waveletSegRLEFusedKernel<<>>( d_input, plan->d_scratch, plan->d_block_sizes, @@ -245,7 +316,12 @@ hipError_t hipCompress( HIPCHECK_PLAN(plan, hipStreamWaitEvent(aux, plan->ready_event, 0)); // 4. Compact + write header on aux_stream - if (plan->is_2d) { + if (plan->is_2d && plan->kernel == HIP_COMPRESS_KERNEL_QUADTREE) { + wqt2dCompactKernel<<>>( + plan->d_octree_coded, d_output + hdr_size, + plan->d_block_sizes, plan->d_block_offsets, plan->d_octree_sig_sizes, + d_output, nb, num_mulfacs, plan->d_mulfac); + } else if (plan->is_2d) { wrle2DCompactKernel<<>>( plan->d_scratch, d_output + hdr_size, plan->d_block_sizes, plan->d_block_offsets, @@ -256,6 +332,11 @@ hipError_t hipCompress( plan->d_octree_coded, d_output + hdr_size, plan->d_block_sizes, plan->d_block_offsets, plan->d_octree_sig_sizes, d_output, nb, num_mulfacs, plan->d_mulfac); + } else if (plan->kernel == HIP_COMPRESS_KERNEL_TWOLEVEL) { + wtlCompactKernel<<>>( + plan->d_octree_coded, d_output + hdr_size, + plan->d_block_sizes, plan->d_block_offsets, + d_output, nb, num_mulfacs, plan->d_mulfac); } else { wrleCompactKernel<<>>( plan->d_scratch, d_output + hdr_size, @@ -287,7 +368,9 @@ hipError_t hipCompressSynchronize( const int nx = plan->nx, ny = plan->ny, nz = plan->nz; const int nb = plan->num_blocks; - const int hdr_size = (plan->kernel == HIP_COMPRESS_KERNEL_OCTREE) + const bool octree_hdr = (plan->kernel == HIP_COMPRESS_KERNEL_OCTREE || + plan->kernel == HIP_COMPRESS_KERNEL_QUADTREE); + const int hdr_size = octree_hdr ? hipOctreeHeaderSize(nb, 1) : hipCompressHeaderSize(nb, 1); size_t total_payload = plan->h_staging[0] + plan->h_staging[1]; @@ -465,9 +548,21 @@ hipError_t hipDecompress( if (plan->is_2d) { int nbx = nx / 32, nby = ny / 32; dim3 grid((nbx + WRLE2D_TILES_PER_WG - 1) / WRLE2D_TILES_PER_WG, nby); - waveletRLE2DInverseFusedKernel<<>>( - d_input, nullptr, nullptr, - d_output, 0.0f, ldimx, nbx, 1); + if (plan->kernel == HIP_COMPRESS_KERNEL_QUADTREE) { + const int nb = plan->num_blocks; + // stage A: compacted quadtree stream → int32 32x32 grid scratch; + // also publishes inv_scale = 1/mulfac (device) for stage B. + waveletQuadtree2DDecodeHdrKernel<<>>( + d_input, reinterpret_cast(plan->d_scratch), plan->d_inv_scale); + // stage B: dequantize + inverse 2D wavelet → wavefield. + waveletQuadtree2DInverseKernel<<>>( + reinterpret_cast(plan->d_scratch), d_output, + plan->d_inv_scale, ldimx, nbx); + } else { + waveletRLE2DInverseFusedKernel<<>>( + d_input, nullptr, nullptr, + d_output, 0.0f, ldimx, nbx, 1); + } } else { const int ldimxy = nx * ny; dim3 grid((nx + 31) / 32, (ny + 31) / 32, (nz + 31) / 32); @@ -480,6 +575,15 @@ hipError_t hipDecompress( // stage B: dequantize + inverse wavelet ZYX → wavefield. waveletBitmapInverseFusedDevKernel<<>>( plan->d_scratch, d_output, plan->d_inv_scale, ldimx, ldimxy); + } else if (plan->kernel == HIP_COMPRESS_KERNEL_TWOLEVEL) { + const int nb = plan->num_blocks; + // stage A: compacted two-level stream → kernel-1 scratch layout; + // also publishes inv_scale = 1/mulfac (device) for stage B. + waveletBitmapTwoLevelDecodeHdrKernel<<>>( + d_input, plan->d_scratch, plan->d_inv_scale); + // stage B: dequantize + inverse wavelet ZYX → wavefield. + waveletBitmapInverseFusedDevKernel<<>>( + plan->d_scratch, d_output, plan->d_inv_scale, ldimx, ldimxy); } else if (plan->kernel == HIP_COMPRESS_KERNEL_SEGRLE) { waveletSegRLEInverseFusedKernel<<>>( d_input, nullptr, nullptr, @@ -522,6 +626,15 @@ hipError_t hipCompressMaxOutputSize(const hipCompressPlan* plan, size_t* size) // Worst case: octree header + every block at its coded-slot upper bound. int hdr_size = hipOctreeHeaderSize(plan->num_blocks, 1); raw = (size_t)hdr_size + (size_t)plan->num_blocks * WOCT_CODE_SLOT_BYTES; + } else if (plan->kernel == HIP_COMPRESS_KERNEL_QUADTREE) { + // Worst case: octree-style header + every block at the quadtree slot bound. + int hdr_size = hipOctreeHeaderSize(plan->num_blocks, 1); + raw = (size_t)hdr_size + (size_t)plan->num_blocks * WQT2D_CODE_SLOT_BYTES; + } else if (plan->kernel == HIP_COMPRESS_KERNEL_TWOLEVEL) { + // Worst case: RLE-style header + every block at the two-level slot bound + // (larger than the WBMP_SLOT_BYTES scratch stride by the occupancy mask). + int hdr_size = hipCompressHeaderSize(plan->num_blocks, 1); + raw = (size_t)hdr_size + (size_t)plan->num_blocks * WBMP_TL_SLOT_BYTES; } else { int hdr_size = hipCompressHeaderSize(plan->num_blocks, 1); raw = (size_t)hdr_size + (size_t)plan->num_blocks * plan->scratch_slot_stride; diff --git a/hip/hipCompress.h b/hip/hipCompress.h index 3c66a8f..7a6d4ed 100644 --- a/hip/hipCompress.h +++ b/hip/hipCompress.h @@ -30,9 +30,20 @@ hipCompressError_t hipCompressGetLastError(const hipCompressPlan* plan); const char* hipCompressErrorString(hipCompressError_t err); enum hipCompressKernel { - HIP_COMPRESS_KERNEL_ZLINE = 0, // parallel z-line RLE (per-block metadata) - HIP_COMPRESS_KERNEL_SEGRLE = 1, // segment-aligned RLE (no metadata overhead) - HIP_COMPRESS_KERNEL_OCTREE = 2, // octree significance coder (3D only) + HIP_COMPRESS_KERNEL_ZLINE = 0, // parallel z-line RLE (per-block metadata) + HIP_COMPRESS_KERNEL_SEGRLE = 1, // segment-aligned RLE (no metadata overhead) + HIP_COMPRESS_KERNEL_OCTREE = 2, // octree significance coder (3D only) + HIP_COMPRESS_KERNEL_TWOLEVEL = 3, // two-level occupancy+width coder (3D only). + // Encoder is arch-selected: the LDS-staged + // opt kernel on gfx950, the portable kernel + // on gfx942/gfx90a (byte-identical stream). + HIP_COMPRESS_KERNEL_QUADTREE = 4, // quadtree significance coder (2D only) -- + // the 2D counterpart of OCTREE. + HIP_COMPRESS_KERNEL_AUTO = 5, // dimensionality-selected default: resolves + // to QUADTREE for 2D (nz == 1) and OCTREE + // for 3D at plan creation, falling back to + // ZLINE for configurations the structured + // coders cannot handle. }; struct hipCompressPlan { @@ -49,12 +60,21 @@ struct hipCompressPlan { void* d_scan_temp; size_t scan_temp_bytes; - // Octree kernel only (nullptr otherwise): intermediate coded slots, per-block - // significance sizes, and the device inv_scale published for stage-B decode. + // Octree / two-level / quadtree kernels only (nullptr otherwise): + // d_octree_coded is the fixed-stride intermediate coded buffer (octree slot + // for OCTREE, two-level slot for TWOLEVEL, quadtree slot for QUADTREE); + // d_octree_sig_sizes is the per-block significance-size table (OCTREE and + // QUADTREE; nullptr for TWOLEVEL -- its stream is self-describing); + // d_inv_scale is the device inv_scale published for stage-B decode. unsigned char* d_octree_coded; size_t* d_octree_sig_sizes; float* d_inv_scale; + // TWOLEVEL only: true on gfx950 to launch the LDS-staged opt encoder, false + // elsewhere to launch the portable encoder. Set once at plan creation from + // the device arch; both encoders emit the identical two-level stream. + bool tl_use_opt; + double* d_partial_sums; int max_copy_blocks; double* d_rms; @@ -89,7 +109,7 @@ hipError_t hipCompressCreatePlan( hipCompressPlan** plan, int nx, int ny, int nz, hipStream_t aux_stream, - hipCompressKernel kernel = HIP_COMPRESS_KERNEL_ZLINE); + hipCompressKernel kernel = HIP_COMPRESS_KERNEL_AUTO); hipError_t hipCompressDestroyPlan(hipCompressPlan* plan); diff --git a/hip/hipWaveletBitmap.h b/hip/hipWaveletBitmap.h index 996bb9c..a840cd2 100644 --- a/hip/hipWaveletBitmap.h +++ b/hip/hipWaveletBitmap.h @@ -652,6 +652,14 @@ __global__ void waveletBitmapCodeTwoLevelOptKernel( unsigned char* __restrict__ out, size_t* __restrict__ block_sizes2) { +// This kernel stages the whole per-block value payload in LDS +// (WBMP_OPT_VBUF ~= 131 KB), which fits only CDNA4 (gfx950, 160 KB LDS). On +// every other target (gfx942/gfx90a: 64 KB LDS, and the host pass) it compiles +// to an empty stub so the translation unit builds portably; the host dispatch +// (hipCompress) only ever launches it on gfx950 and falls back to +// waveletBitmapCodeTwoLevelKernel elsewhere. Both kernels emit byte-identical +// two-level streams, so the decoder is arch-independent. +#if defined(__gfx950__) using namespace wbmp_opt; (void)block_sizes1; // recomputed from the bitmap, kept for signature parity __shared__ int warp_part[2 * WBMP_OPT_NWARPS]; @@ -747,6 +755,9 @@ __global__ void waveletBitmapCodeTwoLevelOptKernel( } if (tid == 0) block_sizes2[bid] = (size_t)used; +#else + (void)scratch1; (void)block_sizes1; (void)out; (void)block_sizes2; +#endif // __gfx950__ } inline hipError_t hipWaveletBitmapCodeTwoLevelOpt( @@ -762,6 +773,141 @@ inline hipError_t hipWaveletBitmapCodeTwoLevelOpt( return hipGetLastError(); } +// =========================================================================== +// TWO-LEVEL codec: compaction + full decode for the public hipCompress API. +// =========================================================================== +// Compaction for the two-level coder: copies each variable-length coded block +// from the fixed-stride (WBMP_TL_SLOT_BYTES) intermediate into a tightly packed +// payload using the exclusive-scan offsets, and writes the self-contained +// RLE-style header (block offsets + mulfac). The two-level format is fully +// self-describing -- popcount(occupancy) recovers every region base -- so unlike +// the octree stream it needs no per-block significance-size table; the header is +// exactly hipCompressHeaderSize(nb, nmf). dst points to the payload (after the +// header). Mirrors woctCompactKernel / wrleCompactKernel. +__global__ void wtlCompactKernel( + const unsigned char* __restrict__ src, + unsigned char* __restrict__ dst, + const size_t* __restrict__ block_sizes, + const size_t* __restrict__ offsets, + unsigned char* __restrict__ hdr, + int num_blocks, + int num_mulfacs, + const float* __restrict__ d_mulfac) +{ + int bid = blockIdx.x; + int tid = threadIdx.x; + size_t size = block_sizes[bid]; + size_t dst_off = offsets[bid]; + size_t src_off = (size_t)bid * WBMP_TL_SLOT_BYTES; + + if (hdr != nullptr && tid == 0) { + ((size_t*)(hdr + 8))[bid] = offsets[bid]; + if (bid == 0) { + ((int*)hdr)[0] = num_blocks; + ((int*)hdr)[1] = num_mulfacs; + float* mf_dst = (float*)(hdr + 8 + 8L * num_blocks); + for (int i = 0; i < num_mulfacs; ++i) + mf_dst[i] = d_mulfac[i]; + } + } + + for (size_t i = tid * 4; i < size; i += blockDim.x * 4) { + unsigned val; + __builtin_memcpy(&val, src + src_off + i, 4); + size_t remain = size - i; + if (remain >= 4) { + __builtin_memcpy(dst + dst_off + i, &val, 4); + } else { + for (size_t b = 0; b < remain; ++b) + dst[dst_off + i + b] = (unsigned char)(val >> (b * 8)); + } + } +} + +// Two-level decode (stage A): reconstructs the kernel-1 scratch layout +// [4096B bitmap (L order)][packed int32 values] for one block from its coded +// bytes [128B occupancy][4B*n_ne masks][2b/nonempty-line widths][values]. +// Exact inverse of the two-level coder's packing, so the output is byte- +// identical to the waveletBitmapFusedKernel (kernel-1) output and feeds the +// shared inverse-wavelet stage B unchanged. One workgroup of 1024 threads per +// block, one z-line (L index) per thread. Portable: only wave64 DPP block +// scans, no arch-specific LDS budget (~128 B shared), so it runs on gfx90a/ +// gfx942/gfx950 alike regardless of which encoder produced the stream. +// blk — pointer to the block's coded bytes (4-byte aligned) +// out — this block's WBMP_SLOT_BYTES scratch slot +__device__ __forceinline__ void wtl_decode_block_to_scratch( + const unsigned char* __restrict__ blk, + unsigned char* __restrict__ out) +{ + using namespace wbmp_opt; + const int tid = threadIdx.x; // L index in [0,1024) + __shared__ int warp_part[2 * WBMP_OPT_NWARPS]; + + const uint32_t* occ = reinterpret_cast(blk); + uint32_t* bmp_out = reinterpret_cast(out); + int32_t* val_out = reinterpret_cast(out + WBMP_BITMAP_BYTES); + + // Occupancy bit for this line and its nonempty rank (exclusive scan). + const int occb = (occ[tid >> 5] >> (tid & 31)) & 1; + int tot_ne; + const int occ_rank = block_exscan(occb, tot_ne, warp_part); + __syncthreads(); // protect warp_part reuse + + const long masks_base = WBMP_OCC_BYTES; + const long wtab_base = masks_base + 4L * tot_ne; + const long vals_base = wtab_base + (2L * tot_ne + 7) / 8; + const uint32_t* masks = reinterpret_cast(blk + masks_base); + const uint32_t* wtab = reinterpret_cast(blk + wtab_base); + + const uint32_t maskL = occb ? masks[occ_rank] : 0u; + bmp_out[tid] = maskL; // bitmap in L order + const int nz = __popc(maskL); + const int W = occb ? (int)((wtab[occ_rank >> 4] >> ((occ_rank & 15) * 2)) & 3u) + 1 : 1; + + // Scratch int32 offset (scan nnz) and coded value byte offset (scan nnz*W). + int in_off, val_off, tot_nz, tot_val; + block_exscan2(nz, nz * W, in_off, val_off, tot_nz, tot_val, warp_part); + (void)tot_nz; (void)tot_val; + + if (occb) { + const unsigned char* vp = blk + vals_base + val_off; + const int sh = 32 - 8 * W; + for (int k = 0; k < nz; ++k) { + unsigned uv = 0; + #pragma unroll + for (int b = 0; b < 4; ++b) + if (b < W) uv |= (unsigned)vp[(long)k * W + b] << (8 * b); + val_out[in_off + k] = (int)(uv << sh) >> sh; // sign-extend from W bytes + } + } +} + +// API decode kernel: locates each block in the compacted stream via the header +// offset table, reconstructs the kernel-1 scratch layout, and (block 0) +// publishes inv_scale = 1/mulfac for stage B. Header is the RLE-style layout +// [int nb][int nmf][size_t offsets[nb]][float mulfac[nmf]]. +__launch_bounds__(wbmp_opt::WBMP_OPT_THREADS) +__global__ void waveletBitmapTwoLevelDecodeHdrKernel( + const unsigned char* __restrict__ input, + unsigned char* __restrict__ scratch1_out, + float* __restrict__ inv_scale_out) +{ + const int bid = blockIdx.x, tid = threadIdx.x; + const int* hdr = reinterpret_cast(input); + const int num_blocks = hdr[0]; + const int num_mulfacs = hdr[1]; + const size_t* offsets = reinterpret_cast(input + 8); + const float* mulfacs = reinterpret_cast(input + 8 + 8L * num_blocks); + const unsigned char* data_base = input + 8 + 8L * num_blocks + 4L * num_mulfacs; + + if (bid == 0 && tid == 0 && inv_scale_out) + *inv_scale_out = 1.0f / mulfacs[0]; + + wtl_decode_block_to_scratch( + data_base + offsets[bid], + scratch1_out + (long)bid * WBMP_SLOT_BYTES); +} + // Launch helper mirroring hipWaveletRLEFused. output must have // nblocks * WBMP_SLOT_BYTES bytes; block_sizes has nblocks entries. inline hipError_t hipWaveletBitmapFused( diff --git a/hip/hipWaveletOctree.h b/hip/hipWaveletOctree.h index f94c006..46648a3 100644 --- a/hip/hipWaveletOctree.h +++ b/hip/hipWaveletOctree.h @@ -41,12 +41,12 @@ static constexpr long WOCT_SLOT_RAW = + WBMP_WTAB_BYTES + WBMP_MAX_VAL_BYTES; static constexpr long WOCT_SLOT_BYTES = (WOCT_SLOT_RAW + 15) & ~15L; -// Fused kernel-2 coded slot: [4B mode][significance][<=3B pad to 4-align the -// width table][2b/nonempty-line width table][per-line packed values]. -// sig = octree stream (<=WOCT_MAX_SIG_BYTES) or flat masks (4096 B) -// values = same per-line variable-width payload as the two-level coder +// Fused kernel-2 coded slot: [4B mode][significance][<=3B pad to 4-align][per-block +// PFOR value payload]. PFOR region = [Wbyte+3 pad][exception mask: <=(32768/32)*4 +// =4096 B][base + patch: <= nnz*W_hi <= WBMP_MAX_VAL_BYTES]. +// sig = octree stream (<=WOCT_MAX_SIG_BYTES) or flat masks (4096 B) static constexpr long WOCT_CODE_SLOT_RAW = - WOCT_HDR_BYTES + WOCT_MAX_SIG_BYTES + 3 + WBMP_WTAB_BYTES + WBMP_MAX_VAL_BYTES; + WOCT_HDR_BYTES + WOCT_MAX_SIG_BYTES + 3 + 4 + ((32768 + 31) / 32) * 4 + WBMP_MAX_VAL_BYTES; static constexpr long WOCT_CODE_SLOT_BYTES = (WOCT_CODE_SLOT_RAW + 15) & ~15L; // Kernel-1 bitmap word order L=x_off*256+tid maps to spatial line (ix,iy): @@ -716,36 +716,78 @@ __global__ void waveletOctreeCodeParKernel( } __syncthreads(); - long wtab_base = ((long)WOCT_HDR_BYTES + sig_bytes + 3) & ~3L; - - // ---- per-line width table + packed values (two-level layout) ---- - int nz = __popc(m), occb = m ? 1 : 0; - int in_off, occ_rank, tot_nz, tot_ne; - block_exscan2(nz, occb, in_off, occ_rank, tot_nz, tot_ne, warp_part); - (void)tot_nz; - int mx = 0; - for (int k=0;k(blk_out + wtab_base); - int wtab_words = (2 * tot_ne + 31) / 32; - for (int i=tid;i coalesced GPU decode. + long val_base = ((long)WOCT_HDR_BYTES + sig_bytes + 3) & ~3L; + int nz = __popc(m); + int in_off, occ_rank_u, tot_nz, tot_ne_u; + block_exscan2(nz, m ? 1 : 0, in_off, occ_rank_u, tot_nz, tot_ne_u, warp_part); + (void)occ_rank_u; (void)tot_ne_u; + + // per-line width histogram (local counts, then block reduce) + int lc1=0,lc2=0,lc3=0,lc4=0; + for (int k=0;k Wlo) + if (2 > Wlo) e_line += lc2; + if (3 > Wlo) e_line += lc3; + if (4 > Wlo) e_line += lc4; + __syncthreads(); // protect warp_part reuse + int ex_base, tot_ex; + ex_base = block_exscan(e_line, tot_ex, warp_part); + + uint32_t* mp = reinterpret_cast(blk_out + mask_base); + for (int i=tid;i> 4], (uint32_t)(W - 1) << ((occ_rank & 15) * 2)); - long p = vals_base + val_off; + if (nz) { + int local_ex = 0; for (int k=0;k> (8*b)); + for (int b=0;b<4;++b) if (b> (8*b)); + int a = v<0?-v:v; + if (wbmp_width_bytes(a) > Wlo){ + atomicOr(&mp[g>>5], 1u << (g & 31)); + long pq = patch_base + (long)(ex_base + local_ex) * (Whi - Wlo); + for (int b=0;b> (8*(Wlo+b))); + ++local_ex; + } } } - if (tid==0) block_sizes2[bid] = (size_t)(vals_base + tot_val); + if (tid==0) block_sizes2[bid] = (size_t)(patch_base + (long)tot_ex * (Whi - Wlo)); } inline hipError_t hipWaveletOctreeCode( @@ -902,26 +944,46 @@ __device__ __forceinline__ void woct_decode_block_to_scratch( uint32_t maskL = s.masks_sp[woct_L_to_spatial(tid)]; bmp_out[tid] = maskL; // bitmap in L order - // ---- value unpack (reverse of the fused encoder) ---- - long wtab_base = ((long)WOCT_HDR_BYTES + sig_bytes + 3) & ~3L; - int nz = __popc(maskL), occb = maskL ? 1 : 0; - int in_off, occ_rank, tot_nz, tot_ne; - block_exscan2(nz, occb, in_off, occ_rank, tot_nz, tot_ne, warp_part); + // ---- per-block PFOR value unpack (reverse of the fused encoder) ---- + long val_base = ((long)WOCT_HDR_BYTES + sig_bytes + 3) & ~3L; + int nz = __popc(maskL); + int in_off, occ_rank_u, tot_nz, tot_ne_u; + block_exscan2(nz, maskL ? 1 : 0, in_off, occ_rank_u, tot_nz, tot_ne_u, warp_part); + (void)occ_rank_u; (void)tot_ne_u; __syncthreads(); // protect warp_part reuse - const uint32_t* wtab = reinterpret_cast(blk + wtab_base); - int W = occb ? (int)((wtab[occ_rank >> 4] >> ((occ_rank & 15) * 2)) & 3u) + 1 : 1; - int val_off, tot_val; - val_off = block_exscan(nz * W, tot_val, warp_part); - (void)tot_val; - long vals_base = wtab_base + (2L * tot_ne + 7) / 8; - if (occb) { - const unsigned char* vp = blk + vals_base + val_off; - int sh = 32 - 8*W; - for (int k=0;k> sh; // sign-extend from W bytes + int nnz = tot_nz; + if (nnz > 0) { + int Wlo = blk[val_base] & 0xF, Whi = (blk[val_base] >> 4) & 0xF; + int have_mask = (Wlo < Whi); + long mask_base = val_base + 4; + int mask_words = have_mask ? (nnz + 31) / 32 : 0; + long base_base = mask_base + (long)mask_words * 4; + long patch_base = base_base + (long)nnz * Wlo; + const uint32_t* mp = reinterpret_cast(blk + mask_base); + + int e_line = 0; // local exceptions in this line's rank range + if (have_mask) for (int k=0;k>5]>>(g&31))&1u) ++e_line; } + int ex_base, tot_ex; + ex_base = block_exscan(e_line, tot_ex, warp_part); + (void)tot_ex; + + int shlo = 32 - 8*Wlo, shhi = 32 - 8*Whi; + if (nz) { + int local_ex = 0; + for (int k=0;k>5]>>(g&31))&1u); + if (exc){ + long pq = patch_base + (long)(ex_base + local_ex) * (Whi - Wlo); + for (int b=0;b> shhi; + ++local_ex; + } else { + val_out[in_off + k] = (int)(uv << shlo) >> shlo; + } + } } } if (tid==0 && bsz_this) *bsz_this = (size_t)WBMP_BITMAP_BYTES + (size_t)tot_nz * 4; diff --git a/hip/hipWaveletQuadtree2D.h b/hip/hipWaveletQuadtree2D.h new file mode 100644 index 0000000..c65bdac --- /dev/null +++ b/hip/hipWaveletQuadtree2D.h @@ -0,0 +1,608 @@ +// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Use of this source code is governed by an MIT-style license that can be +// found in the LICENSE file or at https://opensource.org/licenses/MIT. +// +// 2D quadtree significance coder -- the 2D counterpart of the 3D octree +// (hipWaveletOctree.h). A 32x32 wavelet block's significance set is encoded as +// a quadtree: 2x2 subdivision, depth 5 (node sides 32,16,8,4,2), 4 bits (one per +// child) per non-empty node, nibble-packed two nodes per byte, serialized in +// DFS pre-order. A per-block 1-byte mode tag falls back to the flat 32-uint32 +// row bitmap when the quadtree would be larger (dense blocks). +// +// Pipeline (mirrors the octree split; 2D blocks are tiny so the significance / +// value passes run one workgroup per block, mostly serial in thread 0): +// encode: k1 forward (2D wavelet + quantize -> int32 32x32 grid scratch) +// k2 code (grid -> [mode][significance][per-block PFOR values]) +// compact (pack coded blocks + octree-style self-contained header) +// decode: stage A (coded stream -> int32 grid scratch; publishes inv_scale) +// stage B (grid -> dequantize + inverse 2D wavelet -> volume) +// +// The forward/inverse transform kernels are byte-for-byte the 2D DS 7/9 transform +// of hipWaveletRLE2D.h with the RLE tail replaced by an int32-grid read/write, so +// the reconstructed field matches the RLE 2D path bit-for-bit at equal scale. + +#ifndef HIPWAVELET_QUADTREE_2D_H +#define HIPWAVELET_QUADTREE_2D_H + +#include +#include "ds79.h" + +// Tile batching (identical to hipWaveletRLE2D.h so the transform is unchanged). +static constexpr int WQT2D_TILES_PER_WG = 32; +static constexpr int WQT2D_BATCH = 8; + +// Per-block scratch: a 32x32 int32 grid of quantized wavelet coefficients. +static constexpr int WQT2D_GRID_INTS = 32 * 32; // 1024 +static constexpr long WQT2D_GRID_BYTES = WQT2D_GRID_INTS * 4L; // 4096 + +// Coded-block regions. +static constexpr int WQT2D_HDR_BYTES = 1; // mode: 0 flat, 2 quadtree +static constexpr int WQT2D_FLAT_SIG = 128; // 32 uint32 row masks (little-endian) +// Worst-case (fully dense) quadtree node count 1+4+16+64+256 = 341, nibble-packed +// two nodes per byte -> ceil(341/2) = 171 bytes (transient; flat caps at 128). +static constexpr int WQT2D_MAX_SIG = 171; +// Value region is per-block PFOR: [Wbyte][base nnz*W_lo][mask ceil(nnz/8) if +// W_lo= 32) ? 0xFFFFFFFFu + : (uint32_t)((((uint32_t)1 << cs) - 1) << ox); + for (int iy = oy; iy < oy + cs; ++iy) + if (m[iy] & xmask) return true; + return false; +} + +// 4-bit child occupancy of the node at (px,py) with the given side. Child +// index c = cx | cy<<1 ; child subquad side = side/2. +__host__ __device__ __forceinline__ int +wqt2d_child_occ(const uint32_t* m, int px, int py, int side) { + int cs = side >> 1, b = 0; + for (int c = 0; c < 4; ++c) { + int cx = c & 1, cy = (c >> 1) & 1; + if (wqt2d_sub_any(m, px + cx * cs, py + cy * cs, cs)) b |= (1 << c); + } + return b; +} + +// Encode the 32x32 significance into a DFS-preorder quadtree nibble stream (one +// 4-bit occupancy per non-empty node, two nodes packed per byte: node 2k in the +// low nibble of byte k, node 2k+1 in the high nibble). Returns bytes written +// (0 for an empty block). Explicit depth-<=5 stack as parallel scalar arrays. +__host__ __device__ __forceinline__ int +wqt2d_encode(const uint32_t* m, unsigned char* out) { + int rootb = wqt2d_child_occ(m, 0, 0, 32); + if (rootb == 0) return 0; + int px[6], py[6], side[6], bb[6], ch[6]; + int sp = 0, nc = 0; // nc: node index (nibble position) +#define WQT2D_PUT_NIB(v) do { int _v = (v) & 0xF; \ + if (nc & 1) out[nc >> 1] |= (unsigned char)(_v << 4); \ + else out[nc >> 1] = (unsigned char)_v; ++nc; } while (0) + WQT2D_PUT_NIB(rootb); + px[0] = 0; py[0] = 0; side[0] = 32; bb[0] = rootb; ch[0] = 0; sp = 1; + while (sp > 0) { + int i = sp - 1; + int cs = side[i] >> 1; + int found = -1; + if (cs > 1) { // size-2 nodes carry cells, no node children + for (int c = ch[i]; c < 4; ++c) + if (bb[i] & (1 << c)) { found = c; break; } + } + if (found >= 0) { + ch[i] = found + 1; + int cx = found & 1, cy = (found >> 1) & 1; + int ox = px[i] + cx * cs, oy = py[i] + cy * cs; + int b = wqt2d_child_occ(m, ox, oy, cs); + WQT2D_PUT_NIB(b); + px[sp] = ox; py[sp] = oy; side[sp] = cs; bb[sp] = b; ch[sp] = 0; ++sp; + } else { + --sp; + } + } +#undef WQT2D_PUT_NIB + return (nc + 1) >> 1; // packed byte count +} + +// Inverse of wqt2d_encode: reconstruct row masks m[0..31] (caller pre-zeros). +// Reads one nibble per node in the same DFS order (self-terminating by tree +// structure); sigbytes is only used to short-circuit the empty block. +__host__ __device__ __forceinline__ void +wqt2d_decode(const unsigned char* in, int sigbytes, uint32_t* m) { + if (sigbytes == 0) return; + int px[6], py[6], side[6], bb[6], ch[6]; + int sp = 0, nc = 0; // nc: node index (nibble position) + int rootb = in[0] & 0xF; nc = 1; + px[0] = 0; py[0] = 0; side[0] = 32; bb[0] = rootb; ch[0] = 0; sp = 1; + while (sp > 0) { + int i = sp - 1; + int cs = side[i] >> 1; + int found = -1; + for (int c = ch[i]; c < 4; ++c) + if (bb[i] & (1 << c)) { found = c; break; } + if (found < 0) { --sp; continue; } + ch[i] = found + 1; + int cx = found & 1, cy = (found >> 1) & 1; + int ox = px[i] + cx * cs, oy = py[i] + cy * cs; + if (cs == 1) { // cell leaf of a size-2 node + m[oy] |= (1u << ox); + } else { + int b = (nc & 1) ? (in[nc >> 1] >> 4) : (in[nc >> 1] & 0xF); ++nc; + px[sp] = ox; py[sp] = oy; side[sp] = cs; bb[sp] = b; ch[sp] = 0; ++sp; + } + } +} + +// --------------------------------------------------------------------------- +// Per-block PFOR value coder (host + device, byte-exact by construction). The +// significant coefficients (nonzeros of m[], enumerated row-major, x ascending) +// are stored as three fixed-width streams so decode has no variable-bit chain: +// +// [Wbyte] low nibble = W_lo (base width), high nibble = W_hi (max width) +// [base] low W_lo bytes of every nonzero (little-endian), nnz*W_lo bytes +// [mask] 1 exception bit per nonzero (set iff width > W_lo); present only +// when W_lo < W_hi; ceil(nnz/8) bytes +// [patch] high (W_hi-W_lo) bytes of each exception, n_exc*(W_hi-W_lo) bytes +// +// W_lo is chosen to minimise total bytes (incl. mask). When W_lo==W_hi there +// are no exceptions, so mask and patch are omitted (the sparse-block fast path). +// nnz is not stored; the decoder recovers it from the significance map. +// --------------------------------------------------------------------------- + +// Encode grid[] nonzeros (mask m[]) into out; returns bytes written (0 if empty). +__host__ __device__ __forceinline__ int +wqt2d_pfor_encode(const int* g, const uint32_t* m, unsigned char* out) { + int cnt[5] = {0, 0, 0, 0, 0}, nnz = 0, Whi = 0; + for (int r = 0; r < 32; ++r) { uint32_t mm = m[r]; if (!mm) continue; + for (int c = 0; c < 32; ++c) if (mm & (1u << c)) { + int v = g[r * 32 + c]; unsigned a = (unsigned)(v < 0 ? -v : v); + int w = wqt2d_width_bytes(a); ++cnt[w]; ++nnz; if (w > Whi) Whi = w; + } + } + if (nnz == 0) return 0; + + int maskbytes = (nnz + 7) >> 3, Wlo = Whi; long best = -1; + for (int wl = 1; wl <= Whi; ++wl) { + int nexc = 0; for (int w = wl + 1; w <= 4; ++w) nexc += cnt[w]; + long cost = (long)nnz * wl + (long)nexc * (Whi - wl) + (wl < Whi ? maskbytes : 0); + if (best < 0 || cost < best) { best = cost; Wlo = wl; } + } + + out[0] = (unsigned char)((Whi << 4) | Wlo); + int base_off = 1, have_mask = (Wlo < Whi); + int mask_off = base_off + nnz * Wlo; + int patch_off = mask_off + (have_mask ? maskbytes : 0); + if (have_mask) for (int i = 0; i < maskbytes; ++i) out[mask_off + i] = 0; + + int i = 0, ex = 0; + for (int r = 0; r < 32; ++r) { uint32_t mm = m[r]; if (!mm) continue; + for (int c = 0; c < 32; ++c) if (mm & (1u << c)) { + int v = g[r * 32 + c]; unsigned uv = (unsigned)v; + for (int b = 0; b < Wlo; ++b) out[base_off + (long)i * Wlo + b] = (unsigned char)(uv >> (8 * b)); + unsigned a = (unsigned)(v < 0 ? -v : v); + if (wqt2d_width_bytes(a) > Wlo) { // exception + out[mask_off + (i >> 3)] |= (unsigned char)(1u << (i & 7)); + for (int b = 0; b < Whi - Wlo; ++b) + out[patch_off + (long)ex * (Whi - Wlo) + b] = (unsigned char)(uv >> (8 * (Wlo + b))); + ++ex; + } + ++i; + } + } + return patch_off + ex * (Whi - Wlo); +} + +// Inverse of wqt2d_pfor_encode: fill grid g[] (caller pre-zeros) from the mask m[]. +__host__ __device__ __forceinline__ void +wqt2d_pfor_decode(const unsigned char* in, const uint32_t* m, int* g) { + int nnz = 0; + for (int r = 0; r < 32; ++r) { uint32_t mm = m[r]; while (mm) { nnz += (int)(mm & 1u); mm >>= 1; } } + if (nnz == 0) return; + + int Wlo = in[0] & 0xF, Whi = (in[0] >> 4) & 0xF; + int base_off = 1, have_mask = (Wlo < Whi); + int mask_off = base_off + nnz * Wlo; + int patch_off = mask_off + (have_mask ? ((nnz + 7) >> 3) : 0); + int shlo = 32 - 8 * Wlo, shhi = 32 - 8 * Whi; + + int i = 0, ex = 0; + for (int r = 0; r < 32; ++r) { uint32_t mm = m[r]; if (!mm) continue; + for (int c = 0; c < 32; ++c) if (mm & (1u << c)) { + unsigned uv = 0; + for (int b = 0; b < Wlo; ++b) uv |= (unsigned)in[base_off + (long)i * Wlo + b] << (8 * b); + int exc = have_mask && ((in[mask_off + (i >> 3)] >> (i & 7)) & 1); + if (exc) { + for (int b = 0; b < Whi - Wlo; ++b) + uv |= (unsigned)in[patch_off + (long)ex * (Whi - Wlo) + b] << (8 * (Wlo + b)); + g[r * 32 + c] = (int)(uv << shhi) >> shhi; // sign-extend from W_hi + ++ex; + } else { + g[r * 32 + c] = (int)(uv << shlo) >> shlo; // sign-extend from W_lo + } + ++i; + } + } +} + +// =========================================================================== +// k1 forward: 2D DS 7/9 wavelet + quantize -> int32 32x32 grid scratch. +// Grid/tile layout identical to waveletRLE2DFusedKernel; only the tail differs. +// =========================================================================== +__launch_bounds__(256, 2) +__global__ void waveletQuadtree2DForwardKernel( + const float* __restrict__ input, + int* __restrict__ grid_out, + float scale, + int ldimx, + int nbx, + const double* __restrict__ d_rms, + float* __restrict__ d_mulfac_out) +{ + constexpr int SLC = 2; + __shared__ float wavelet[WQT2D_BATCH * 1024]; + + int tid = threadIdx.x; + + float mulfac; + if (d_rms != nullptr) { + float rms = (float)*d_rms; + float product = rms * scale; + mulfac = (product > 0.0f && __builtin_isfinite(1.0f / product)) + ? (1.0f / product) : 1.0f; + if (tid == 0 && blockIdx.x == 0 && blockIdx.y == 0 && d_mulfac_out) + *d_mulfac_out = mulfac; + } else { + mulfac = scale; + } + + int wg_tile_x0 = blockIdx.x * WQT2D_TILES_PER_WG; + + for (int batch = 0; batch < 4; ++batch) { + int batch_tile_x0 = wg_tile_x0 + batch * WQT2D_BATCH; + + // ---- Load 8 tiles → LDS ---- + int xg = tid % 8, yr = tid / 8; + for (int b = 0; b < WQT2D_BATCH; ++b) { + int tile_bx = batch_tile_x0 + b; + int gx = tile_bx * 32 + xg * 4; + int gy = blockIdx.y * 32 + yr; + int x0 = xg * 4; + if (tile_bx < nbx) { + uint32_t byte_off = (gx + gy * ldimx) * (uint32_t)sizeof(float); + auto rsrc = __builtin_amdgcn_make_buffer_rsrc( + const_cast(input), 0, -1, 0x00027000); + auto v = __builtin_bit_cast( + __attribute__((__vector_size__(4 * sizeof(float)))) float, + __builtin_amdgcn_raw_buffer_load_b128(rsrc, byte_off, 0, SLC)); + wavelet[b * 1024 + (x0+0) * 32 + (yr ^ (x0+0))] = v[0]; + wavelet[b * 1024 + (x0+1) * 32 + (yr ^ (x0+1))] = v[1]; + wavelet[b * 1024 + (x0+2) * 32 + (yr ^ (x0+2))] = v[2]; + wavelet[b * 1024 + (x0+3) * 32 + (yr ^ (x0+3))] = v[3]; + } else { + wavelet[b * 1024 + (x0+0) * 32 + (yr ^ (x0+0))] = 0.0f; + wavelet[b * 1024 + (x0+1) * 32 + (yr ^ (x0+1))] = 0.0f; + wavelet[b * 1024 + (x0+2) * 32 + (yr ^ (x0+2))] = 0.0f; + wavelet[b * 1024 + (x0+3) * 32 + (yr ^ (x0+3))] = 0.0f; + } + } + __syncthreads(); + + // ---- Y-transform in LDS ---- + { + int blk = tid / 32, pos = tid % 32; + float line[32]; + for (int y = 0; y < 32; y++) + line[y] = wavelet[blk * 1024 + pos * 32 + (y ^ pos)]; + ds79_forward_reg32(line); + for (int y = 0; y < 32; y++) + wavelet[blk * 1024 + pos * 32 + (y ^ pos)] = line[y]; + } + __syncthreads(); + + // ---- Transposed readback + X-transform in registers ---- + int blk = tid / 32; + int row = tid % 32; + int tile_bx = batch_tile_x0 + blk; + bool active = (tile_bx < nbx); + + float xline[32]; + if (active) { + for (int x = 0; x < 32; x++) + xline[x] = wavelet[blk * 1024 + x * 32 + (row ^ x)]; + ds79_forward_reg32(xline); + } else { + for (int x = 0; x < 32; x++) xline[x] = 0.0f; + } + __syncthreads(); + + // ---- Quantize → int32 grid ---- + if (active) { + int global_bid = tile_bx + blockIdx.y * nbx; + int* g = grid_out + (size_t)global_bid * WQT2D_GRID_INTS + row * 32; + #pragma unroll + for (int x = 0; x < 32; x++) g[x] = (int)(mulfac * xline[x]); + } + __syncthreads(); + } +} + +// =========================================================================== +// k2 code: int32 grid -> [mode][significance][row-width table][packed values]. +// One workgroup per block; the significance + value passes run in thread 0 +// (a 32x32 block is tiny), everything is byte-addressed so no alignment padding +// is required. block_sizes[bid] := total coded length; sig_sizes[bid] := +// significance-region length (for the self-contained header, like the octree). +// =========================================================================== +__launch_bounds__(WQT2D_CODE_THREADS) +__global__ void waveletQuadtree2DCodeKernel( + const int* __restrict__ grid, + unsigned char* __restrict__ out, + size_t* __restrict__ block_sizes, + size_t* __restrict__ sig_sizes) +{ + int bid = blockIdx.x, tid = threadIdx.x; + __shared__ int g[WQT2D_GRID_INTS]; + __shared__ uint32_t m[32]; + + const int* gin = grid + (size_t)bid * WQT2D_GRID_INTS; + for (int i = tid; i < WQT2D_GRID_INTS; i += blockDim.x) g[i] = gin[i]; + __syncthreads(); + for (int r = tid; r < 32; r += blockDim.x) { + uint32_t mm = 0; + for (int c = 0; c < 32; ++c) if (g[r * 32 + c] != 0) mm |= (1u << c); + m[r] = mm; + } + __syncthreads(); + + if (tid == 0) { + unsigned char* blk = out + (size_t)bid * WQT2D_CODE_SLOT_BYTES; + int ob = wqt2d_encode(m, blk + WQT2D_HDR_BYTES); + int siglen; + if (ob < WQT2D_FLAT_SIG) { // quadtree wins (includes empty ob==0) + blk[0] = 2; + siglen = ob; + } else { // flat 32 uint32 row masks (little-endian) + unsigned char* f = blk + WQT2D_HDR_BYTES; + for (int r = 0; r < 32; ++r) { + uint32_t mm = m[r]; + #pragma unroll + for (int b = 0; b < 4; ++b) f[r * 4 + b] = (unsigned char)(mm >> (8 * b)); + } + blk[0] = 0; + siglen = WQT2D_FLAT_SIG; + } + sig_sizes[bid] = (size_t)siglen; + + int vlen = wqt2d_pfor_encode(g, m, blk + WQT2D_HDR_BYTES + siglen); + block_sizes[bid] = (size_t)(WQT2D_HDR_BYTES + siglen + vlen); + } +} + +// Compaction: copy variable-length coded blocks from the fixed-stride +// intermediate into a tightly packed payload and write the self-contained +// header [int nb][int nmf][size_t offsets[nb]][uint32 sig_sizes[nb]][float mf[nmf]] +// (== hipOctreeHeaderSize). Mirrors woctCompactKernel with the 2D slot stride. +__global__ void wqt2dCompactKernel( + const unsigned char* __restrict__ src, + unsigned char* __restrict__ dst, + const size_t* __restrict__ block_sizes, + const size_t* __restrict__ offsets, + const size_t* __restrict__ sig_sizes, + unsigned char* __restrict__ hdr, + int num_blocks, + int num_mulfacs, + const float* __restrict__ d_mulfac) +{ + int bid = blockIdx.x, tid = threadIdx.x; + size_t size = block_sizes[bid]; + size_t dst_off = offsets[bid]; + size_t src_off = (size_t)bid * WQT2D_CODE_SLOT_BYTES; + + if (hdr != nullptr && tid == 0) { + ((size_t*)(hdr + 8))[bid] = offsets[bid]; + ((uint32_t*)(hdr + 8 + 8L * num_blocks))[bid] = (uint32_t)sig_sizes[bid]; + if (bid == 0) { + ((int*)hdr)[0] = num_blocks; + ((int*)hdr)[1] = num_mulfacs; + float* mf_dst = (float*)(hdr + 8 + 12L * num_blocks); + for (int i = 0; i < num_mulfacs; ++i) mf_dst[i] = d_mulfac[i]; + } + } + + for (size_t i = tid * 4; i < size; i += blockDim.x * 4) { + unsigned val; + __builtin_memcpy(&val, src + src_off + i, 4); + size_t remain = size - i; + if (remain >= 4) { + __builtin_memcpy(dst + dst_off + i, &val, 4); + } else { + for (size_t b = 0; b < remain; ++b) + dst[dst_off + i + b] = (unsigned char)(val >> (b * 8)); + } + } +} + +// =========================================================================== +// Decode stage A: compacted coded stream -> int32 grid scratch. Locates each +// block via the header offset table, reads its significance length from the +// header, reconstructs the 32x32 quantized grid, and (block 0) publishes +// inv_scale = 1/mulfac (device) for stage B. +// =========================================================================== +__launch_bounds__(WQT2D_CODE_THREADS) +__global__ void waveletQuadtree2DDecodeHdrKernel( + const unsigned char* __restrict__ input, + int* __restrict__ grid_out, + float* __restrict__ inv_scale_out) +{ + int bid = blockIdx.x, tid = threadIdx.x; + const int* hdr = reinterpret_cast(input); + int num_blocks = hdr[0]; + int num_mulfacs = hdr[1]; + const size_t* offsets = reinterpret_cast(input + 8); + const uint32_t* sig_u32 = reinterpret_cast(input + 8 + 8L * num_blocks); + const float* mulfacs = reinterpret_cast(input + 8 + 12L * num_blocks); + const unsigned char* data_base = input + 8 + 12L * num_blocks + 4L * num_mulfacs; + + if (bid == 0 && tid == 0 && inv_scale_out) + *inv_scale_out = 1.0f / mulfacs[0]; + + __shared__ int g[WQT2D_GRID_INTS]; + __shared__ uint32_t m[32]; + + for (int i = tid; i < WQT2D_GRID_INTS; i += blockDim.x) g[i] = 0; + for (int r = tid; r < 32; r += blockDim.x) m[r] = 0; + __syncthreads(); + + const unsigned char* blk = data_base + offsets[bid]; + int siglen = (int)sig_u32[bid]; + + if (tid == 0) { + int mode = blk[0]; + if (mode == 0) { + const unsigned char* f = blk + WQT2D_HDR_BYTES; + for (int r = 0; r < 32; ++r) { + uint32_t mm = 0; + #pragma unroll + for (int b = 0; b < 4; ++b) mm |= (uint32_t)f[r * 4 + b] << (8 * b); + m[r] = mm; + } + } else { + wqt2d_decode(blk + WQT2D_HDR_BYTES, siglen, m); + } + + wqt2d_pfor_decode(blk + WQT2D_HDR_BYTES + siglen, m, g); + } + __syncthreads(); + + int* gout = grid_out + (size_t)bid * WQT2D_GRID_INTS; + for (int i = tid; i < WQT2D_GRID_INTS; i += blockDim.x) gout[i] = g[i]; +} + +// =========================================================================== +// Decode stage B: int32 grid -> dequantize + inverse 2D DS 7/9 wavelet. +// Transform is byte-for-byte waveletRLE2DInverseFusedKernel with the RLE decode +// replaced by an int32-grid read scaled by inv_scale (read from device). +// =========================================================================== +__launch_bounds__(256, 2) +__global__ void waveletQuadtree2DInverseKernel( + const int* __restrict__ grid, + float* __restrict__ output, + const float* __restrict__ inv_scale_dev, + int ldimx, + int nbx) +{ + constexpr int SLC = 2; + constexpr int MACRO_BATCHES = 2; + using f4vec = __attribute__((__vector_size__(4 * sizeof(float)))) float; + + __shared__ float wavelet[WQT2D_BATCH * 1024]; + + int tid = threadIdx.x; + int wg_tile_x0 = blockIdx.x * WQT2D_TILES_PER_WG; + int blk = tid / 32; + int row = tid % 32; + + float inv_scale = inv_scale_dev ? *inv_scale_dev : 1.0f; + + for (int mp = 0; mp < 4 / MACRO_BATCHES; ++mp) { + + float decoded[MACRO_BATCHES][32]; + + // ---- Grid read + X-inverse ---- + for (int b = 0; b < MACRO_BATCHES; ++b) { + int batch = mp * MACRO_BATCHES + b; + int batch_tile_x0 = wg_tile_x0 + batch * WQT2D_BATCH; + int tile_bx = batch_tile_x0 + blk; + int global_bid = tile_bx + blockIdx.y * nbx; + bool active = (tile_bx < nbx); + + if (active) { + const int* g = grid + (size_t)global_bid * WQT2D_GRID_INTS + row * 32; + #pragma unroll + for (int x = 0; x < 32; x++) decoded[b][x] = (float)g[x] * inv_scale; + us79_inverse_reg32(decoded[b]); + } else { + #pragma unroll + for (int x = 0; x < 32; x++) decoded[b][x] = 0.0f; + } + } + __syncthreads(); + + // ---- Y-inverse + store ---- + for (int b = 0; b < MACRO_BATCHES; ++b) { + int batch = mp * MACRO_BATCHES + b; + int batch_tile_x0 = wg_tile_x0 + batch * WQT2D_BATCH; + + for (int x = 0; x < 32; x++) + wavelet[blk * 1024 + x * 32 + (row ^ x)] = decoded[b][x]; + __syncthreads(); + + { + int pl = tid / 32; + int pos = tid % 32; + float line[32]; + for (int y = 0; y < 32; y++) + line[y] = wavelet[pl * 1024 + pos * 32 + (y ^ pos)]; + us79_inverse_reg32(line); + for (int y = 0; y < 32; y++) + wavelet[pl * 1024 + pos * 32 + (y ^ pos)] = line[y]; + } + __syncthreads(); + + f4vec store_regs[WQT2D_BATCH]; + { + int xg = tid % 8, yr = tid / 8, x0 = xg * 4; + #pragma unroll + for (int s = 0; s < WQT2D_BATCH; ++s) { + store_regs[s][0] = wavelet[s * 1024 + (x0+0) * 32 + (yr ^ (x0+0))]; + store_regs[s][1] = wavelet[s * 1024 + (x0+1) * 32 + (yr ^ (x0+1))]; + store_regs[s][2] = wavelet[s * 1024 + (x0+2) * 32 + (yr ^ (x0+2))]; + store_regs[s][3] = wavelet[s * 1024 + (x0+3) * 32 + (yr ^ (x0+3))]; + } + } + __syncthreads(); + + { + int xg = tid % 8, yr = tid / 8; + #pragma unroll + for (int s = 0; s < WQT2D_BATCH; ++s) { + int tb = batch_tile_x0 + s; + if (tb >= nbx) continue; + int gx = tb * 32 + xg * 4; + int gy = blockIdx.y * 32 + yr; + uint32_t byte_off = (gx + gy * ldimx) * (uint32_t)sizeof(float); + auto rsrc = __builtin_amdgcn_make_buffer_rsrc( + output, 0, -1, 0x00027000); + auto vi = __builtin_bit_cast( + __attribute__((__vector_size__(4 * sizeof(int)))) int, store_regs[s]); + __builtin_amdgcn_raw_buffer_store_b128(vi, rsrc, byte_off, 0, SLC); + } + } + __syncthreads(); + } + } +} + +#endif // HIPWAVELET_QUADTREE_2D_H diff --git a/hip/hipWaveletRLE.h b/hip/hipWaveletRLE.h index 0d171f9..81090a1 100644 --- a/hip/hipWaveletRLE.h +++ b/hip/hipWaveletRLE.h @@ -125,7 +125,8 @@ __global__ void waveletRLEFusedKernel( float scale, int ldimx, int ldimxy, const double* __restrict__ d_rms, - float* __restrict__ d_mulfac_out) + float* __restrict__ d_mulfac_out, + float* __restrict__ d_coef_out = nullptr) { constexpr int PLANES = 32; constexpr int BATCH = 8; @@ -257,6 +258,22 @@ __global__ void waveletRLEFusedKernel( block_total += pass_total; } + // ---- Optional: dump pre-quant wavelet coefficients (debug/validation) ---- + // regs still holds the Phase-3 ZYX coefficients (the RLE pass reads them as + // const). Write them in the same [gx,gy,z] layout as the input, before any + // quantization. Zero cost when d_coef_out == nullptr. + if (d_coef_out) { + #pragma unroll + for (int p = 0; p < PLANES; p++) { + size_t base = (size_t)(blockIdx.z * 32 + p) * ldimxy + + (size_t)gy * ldimx + gx; + d_coef_out[base + 0] = regs[p][0]; + d_coef_out[base + 1] = regs[p][1]; + d_coef_out[base + 2] = regs[p][2]; + d_coef_out[base + 3] = regs[p][3]; + } + } + if (tid == 0) block_sizes[bid] = WRLE_META_PER_BLOCK + block_total; } @@ -272,7 +289,26 @@ inline hipError_t hipWaveletRLEFused( dim3 grid((nx + 31) / 32, (ny + 31) / 32, (nz + 31) / 32); waveletRLEFusedKernel<<>>( input, output, block_sizes, scale, ldimx, ldimxy, - nullptr, nullptr); + nullptr, nullptr, nullptr); + return hipGetLastError(); +} + +// Debug/validation: run the production fused kernel and additionally write the +// pre-quant ZYX wavelet coefficients to d_coef_out (same layout as input). +// Uses mulfac = scale directly (d_rms == nullptr), matching hipWaveletRLEFused. +inline hipError_t hipWaveletRLEFusedDumpCoef( + const float* input, + unsigned char* output, + size_t* block_sizes, + float* d_coef_out, + float scale, + int nx, int ny, int nz, + int ldimx, int ldimxy) +{ + dim3 grid((nx + 31) / 32, (ny + 31) / 32, (nz + 31) / 32); + waveletRLEFusedKernel<<>>( + input, output, block_sizes, scale, ldimx, ldimxy, + nullptr, nullptr, d_coef_out); return hipGetLastError(); } @@ -296,12 +332,15 @@ __global__ void waveletRLEFusedSaddrKernel( float scale, int ldimx, int ldimxy, const double* __restrict__ d_rms, - float* __restrict__ d_mulfac_out) + float* __restrict__ d_mulfac_out, + float* __restrict__ d_coef_out = nullptr) // kept for signature parity with + // waveletRLEFusedKernel (unused) { constexpr int PLANES = 32; constexpr int BATCH = 8; constexpr int NTHREADS = 256; using BlockScan = rocprim::block_scan; + (void)d_coef_out; __shared__ union { float wavelet[BATCH * 1024]; @@ -519,7 +558,7 @@ inline hipError_t hipWaveletRLEFusedCompact( dim3 grid((nx + 31) / 32, (ny + 31) / 32, (nz + 31) / 32); waveletRLEFusedKernel<<>>( input, scratch, block_sizes, scale, ldimx, ldimxy, - nullptr, nullptr); + nullptr, nullptr, nullptr); hipError_t err = rocprim::exclusive_scan( scan_temp, scan_temp_bytes, diff --git a/makefile b/makefile index eeda9bc..e782abd 100644 --- a/makefile +++ b/makefile @@ -24,7 +24,12 @@ BUILDDIR ?= build OBJECTS=CvxCompress.o Wavelet_Transform_Slow.o Wavelet_Transform_Fast.o Run_Length_Encode_Slow.o Block_Copy.o Read_Raw_Volume.o HIPCC ?= hipcc +# HIP_ARCH may be a space-separated list to build a fat binary that runs on +# several GPUs, e.g. HIP_ARCH="gfx942 gfx950" for an MI300x + MI355x binary. +# The two-level opt encoder is only emitted for gfx950; other arches get a +# no-op stub and the portable encoder is dispatched at runtime. HIP_ARCH ?= gfx90a +HIP_OFFLOAD = $(foreach a,$(HIP_ARCH),--offload-arch=$(a)) HIPCFLAGS = -O2 -std=c++17 -fopenmp HIPLDFLAGS = -lm @@ -82,11 +87,16 @@ libhipcvxcompress.$(LIB_EXT) : $(HIP_OBJECTS) $(HIPCC) -shared $(HIPLDFLAGS) -o libhipcvxcompress.$(LIB_EXT) $(HIP_OBJECTS) hip/%.o: hip/%.cpp - $(HIPCC) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -fPIC -c $(HIPCFLAGS) hip/$*.cpp -o $@ + $(HIPCC) $(HIP_OFFLOAD) -mllvm -unroll-threshold=10000 -I. -Ihip -fPIC -c $(HIPCFLAGS) hip/$*.cpp -o $@ # Buffer-instruction wavelet kernel test test_wavelet_buffer_hip: tests/test_wavelet_buffer_hip.cpp hip/hipWaveletTransformBuffer.cpp | $(BUILDDIR) - $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -save-temps=obj -DBUILDDIR=\"$(BUILDDIR)\" -I. -Ihip -Itests -lrocrand -fopenmp tests/test_wavelet_buffer_hip.cpp hip/hipWaveletTransformBuffer.cpp $(HIPLDFLAGS) -o $(BUILDDIR)/test_wavelet_buffer_hip + $(HIPCC) $(HIPCFLAGS) $(HIP_OFFLOAD) -mllvm -unroll-threshold=10000 -save-temps=obj -DBUILDDIR=\"$(BUILDDIR)\" -I. -Ihip -Itests -lrocrand -fopenmp tests/test_wavelet_buffer_hip.cpp hip/hipWaveletTransformBuffer.cpp $(HIPLDFLAGS) -o $(BUILDDIR)/test_wavelet_buffer_hip + +# Quantization flip-amplification test: production fused kernel (pre-quant coef +# dump) vs CPU ds79 reference + flip analysis. Header-only fused path. +test_quant_flip_amplification_hip: tests/test_quant_flip_amplification_hip.cpp hip/hipWaveletRLE.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc hip/ds79_f4_reg32.inc Run_Length_Escape_Codes.hxx | $(BUILDDIR) + $(HIPCC) $(HIPCFLAGS) $(HIP_OFFLOAD) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_quant_flip_amplification_hip.cpp $(HIPLDFLAGS) -o $(BUILDDIR)/test_quant_flip_amplification_hip # Quantize + RLE z-line unit test (CPU-only, no HIP) test_quantize_rle: tests/test_quantize_rle.cpp hip/quantize_rle_ref.h Run_Length_Escape_Codes.hxx | $(BUILDDIR) @@ -106,51 +116,55 @@ test_bitmap_rd_panel: tests/test_bitmap_rd_panel.cpp Run_Length_Encode_Slow.hxx # GPU quantize+RLE encode test (validates against CPU reference) test_quantize_rle_hip: tests/test_quantize_rle_hip.cpp hip/quantize_rle_ref.h Run_Length_Escape_Codes.hxx hip/hipQuantizeRLE.h | $(BUILDDIR) - $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -I. -Ihip -Itests tests/test_quantize_rle_hip.cpp -lm -o $(BUILDDIR)/test_quantize_rle_hip + $(HIPCC) $(HIPCFLAGS) $(HIP_OFFLOAD) -I. -Ihip -Itests tests/test_quantize_rle_hip.cpp -lm -o $(BUILDDIR)/test_quantize_rle_hip # GPU quantize+RLE encode performance benchmark test_quantize_rle_perf_hip: tests/test_quantize_rle_perf_hip.cpp Run_Length_Escape_Codes.hxx | $(BUILDDIR) - $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -save-temps=obj -I. -Ihip -Itests tests/test_quantize_rle_perf_hip.cpp -lm -o $(BUILDDIR)/test_quantize_rle_perf_hip + $(HIPCC) $(HIPCFLAGS) $(HIP_OFFLOAD) -save-temps=obj -I. -Ihip -Itests tests/test_quantize_rle_perf_hip.cpp -lm -o $(BUILDDIR)/test_quantize_rle_perf_hip # Fused inverse (decode + inverse wavelet) test test_inverse_fused_hip: tests/test_inverse_fused_hip.cpp hip/hipWaveletRLEInverse.h hip/hipWaveletRLE.h hip/hipRLEDecode.h hip/hipWaveletTransformBuffer.cpp hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) - $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_inverse_fused_hip.cpp hip/hipWaveletTransformBuffer.cpp -lm -o $(BUILDDIR)/test_inverse_fused_hip + $(HIPCC) $(HIPCFLAGS) $(HIP_OFFLOAD) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_inverse_fused_hip.cpp hip/hipWaveletTransformBuffer.cpp -lm -o $(BUILDDIR)/test_inverse_fused_hip # Z-line RLE decoder unit test test_rle_decode_hip: tests/test_rle_decode_hip.cpp hip/hipRLEDecode.h hip/quantize_rle_ref.h Run_Length_Escape_Codes.hxx | $(BUILDDIR) - $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -I. -Ihip -Itests tests/test_rle_decode_hip.cpp -lm -o $(BUILDDIR)/test_rle_decode_hip + $(HIPCC) $(HIPCFLAGS) $(HIP_OFFLOAD) -I. -Ihip -Itests tests/test_rle_decode_hip.cpp -lm -o $(BUILDDIR)/test_rle_decode_hip # Inverse wavelet transform unit test test_inverse_wavelet_hip: tests/test_inverse_wavelet_hip.cpp hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) - $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_inverse_wavelet_hip.cpp -lm -o $(BUILDDIR)/test_inverse_wavelet_hip + $(HIPCC) $(HIPCFLAGS) $(HIP_OFFLOAD) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_inverse_wavelet_hip.cpp -lm -o $(BUILDDIR)/test_inverse_wavelet_hip # Fused wavelet+RLE kernel test and benchmark test_wavelet_rle_fused_hip: tests/test_wavelet_rle_fused_hip.cpp hip/hipWaveletRLE.h hip/hipWaveletRLEInverse.h hip/hipRLEDecode.h hip/hipQuantizeRLE.h hip/hipWaveletTransformBuffer.cpp libcvxcompress.$(LIB_EXT) hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) - $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -save-temps=obj -I. -Ihip -Itests tests/test_wavelet_rle_fused_hip.cpp hip/hipWaveletTransformBuffer.cpp -L. -lcvxcompress -lm -o $(BUILDDIR)/test_wavelet_rle_fused_hip + $(HIPCC) $(HIPCFLAGS) $(HIP_OFFLOAD) -mllvm -unroll-threshold=10000 -save-temps=obj -I. -Ihip -Itests tests/test_wavelet_rle_fused_hip.cpp hip/hipWaveletTransformBuffer.cpp -L. -lcvxcompress -lm -o $(BUILDDIR)/test_wavelet_rle_fused_hip # hipCompress public API test -test_compress_api_hip: tests/test_compress_api_hip.cpp hip/hipCompress.cpp hip/hipCompress.h hip/hipBlockCopy.h hip/hipWaveletRLE.h hip/hipWaveletRLEInverse.h hip/hipWaveletBitmap.h hip/hipWaveletOctree.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc libcvxcompress.$(LIB_EXT) | $(BUILDDIR) - $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_compress_api_hip.cpp hip/hipCompress.cpp -L. -lcvxcompress '-Wl,-rpath,$$ORIGIN/..' -lm -o $(BUILDDIR)/test_compress_api_hip +test_compress_api_hip: tests/test_compress_api_hip.cpp hip/hipCompress.cpp hip/hipCompress.h hip/hipBlockCopy.h hip/hipWaveletRLE.h hip/hipWaveletRLEInverse.h hip/hipWaveletBitmap.h hip/hipWaveletOctree.h hip/hipWaveletQuadtree2D.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc libcvxcompress.$(LIB_EXT) | $(BUILDDIR) + $(HIPCC) $(HIPCFLAGS) $(HIP_OFFLOAD) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_compress_api_hip.cpp hip/hipCompress.cpp -L. -lcvxcompress '-Wl,-rpath,$$ORIGIN/..' -lm -o $(BUILDDIR)/test_compress_api_hip # 2D compression test -test_compress_2d_hip: tests/test_compress_2d_hip.cpp hip/hipCompress.cpp hip/hipCompress.h hip/hipBlockCopy.h hip/hipWaveletRLE.h hip/hipWaveletRLEInverse.h hip/hipWaveletRLE2D.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) - $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_compress_2d_hip.cpp hip/hipCompress.cpp -lm -o $(BUILDDIR)/test_compress_2d_hip +test_compress_2d_hip: tests/test_compress_2d_hip.cpp hip/hipCompress.cpp hip/hipCompress.h hip/hipBlockCopy.h hip/hipWaveletRLE.h hip/hipWaveletRLEInverse.h hip/hipWaveletRLE2D.h hip/hipWaveletQuadtree2D.h hip/hipWaveletBitmap.h hip/hipWaveletOctree.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) + $(HIPCC) $(HIPCFLAGS) $(HIP_OFFLOAD) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_compress_2d_hip.cpp hip/hipCompress.cpp -lm -o $(BUILDDIR)/test_compress_2d_hip # Bitmap-significance-split encode (kernel 1) validation vs RLE ground truth test_bitmap_encode_hip: tests/test_bitmap_encode_hip.cpp hip/hipWaveletBitmap.h hip/hipWaveletRLE.h hip/quantize_rle_ref.h Run_Length_Escape_Codes.hxx hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) - $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_bitmap_encode_hip.cpp -lm -o $(BUILDDIR)/test_bitmap_encode_hip + $(HIPCC) $(HIPCFLAGS) $(HIP_OFFLOAD) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_bitmap_encode_hip.cpp -lm -o $(BUILDDIR)/test_bitmap_encode_hip # Bitmap-significance-split coding (kernel 2): correctness + CR + throughput test_bitmap_code_hip: tests/test_bitmap_code_hip.cpp hip/hipWaveletBitmap.h hip/hipWaveletRLE.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) - $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_bitmap_code_hip.cpp -lm -o $(BUILDDIR)/test_bitmap_code_hip + $(HIPCC) $(HIPCFLAGS) $(HIP_OFFLOAD) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_bitmap_code_hip.cpp -lm -o $(BUILDDIR)/test_bitmap_code_hip # Octree significance coder (kernel 2 variant): round-trip + CR + throughput test_bitmap_octree_hip: tests/test_bitmap_octree_hip.cpp hip/hipWaveletOctree.h hip/hipWaveletBitmap.h hip/hipWaveletRLE.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) - $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_bitmap_octree_hip.cpp -lm -o $(BUILDDIR)/test_bitmap_octree_hip + $(HIPCC) $(HIPCFLAGS) $(HIP_OFFLOAD) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/test_bitmap_octree_hip.cpp -lm -o $(BUILDDIR)/test_bitmap_octree_hip + +# 2D quadtree (GPU) vs CvxCompress (CPU) rate-distortion comparison on real data +bench_quadtree_vs_cvx_2d: tests/bench_quadtree_vs_cvx_2d.cpp hip/hipCompress.cpp hip/hipCompress.h hip/hipWaveletQuadtree2D.h hip/hipWaveletRLE2D.h hip/hipBlockCopy.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc CvxCompress.hxx libcvxcompress.$(LIB_EXT) | $(BUILDDIR) + $(HIPCC) $(HIPCFLAGS) $(HIP_OFFLOAD) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/bench_quadtree_vs_cvx_2d.cpp hip/hipCompress.cpp -L. -lcvxcompress '-Wl,-rpath,$$ORIGIN/..' -lm -o $(BUILDDIR)/bench_quadtree_vs_cvx_2d # Async pipeline example (for profiling) example_async_pipeline: tests/example_async_pipeline.cpp hip/hipCompress.cpp hip/hipCompress.h hip/hipBlockCopy.h hip/hipWaveletRLE.h hip/hipWaveletRLEInverse.h hip/ds79.h hip/us79_reg32.inc hip/ds79_reg32.inc | $(BUILDDIR) - $(HIPCC) $(HIPCFLAGS) --offload-arch=$(HIP_ARCH) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/example_async_pipeline.cpp hip/hipCompress.cpp -lm -o $(BUILDDIR)/example_async_pipeline + $(HIPCC) $(HIPCFLAGS) $(HIP_OFFLOAD) -mllvm -unroll-threshold=10000 -I. -Ihip -Itests tests/example_async_pipeline.cpp hip/hipCompress.cpp -lm -o $(BUILDDIR)/example_async_pipeline clean: rm -f *.o diff --git a/tests/bench_quadtree_vs_cvx_2d.cpp b/tests/bench_quadtree_vs_cvx_2d.cpp new file mode 100644 index 0000000..c63d761 --- /dev/null +++ b/tests/bench_quadtree_vs_cvx_2d.cpp @@ -0,0 +1,462 @@ +// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Use of this source code is governed by an MIT-style license that can be +// found in the LICENSE file or at https://opensource.org/licenses/MIT. +// +// Rate-distortion comparison of the GPU 2D quadtree codec vs the CPU CvxCompress +// reference codec, on REAL 2D seismic data (a z-slice of a 512^3 RTM snapshot). +// +// CvxCompress is a 3D codec (block sizes >= 8 in z), so to obtain its behaviour +// on genuine 2D data we compress a z-constant volume: NZ copies of the same +// slice. Its z-wavelet detail bands are then identically zero, so the stream +// encodes only the 2D content, and the fair 2D ratio is CR_2D = ratio / NZ +// (equivalently raw_one_slice / compressed_bytes). Both codecs share the same +// Antonini 7/9 wavelet and truncation quantizer, but their scale->threshold maps +// differ, so we sweep each independently and compare CR at matched distortion +// vol_rel_l2 = sqrt( sum (x-xhat)^2 / sum x^2 ) (spatial domain, 2D slice). + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hipCompress.h" +#include "CvxCompress.hxx" +#include "ds79.h" + +#define HIPCHECK(cmd) do { \ + hipError_t e = (cmd); \ + if (e != hipSuccess) { \ + fprintf(stderr, "HIP error %s at %s:%d\n", hipGetErrorString(e), __FILE__, __LINE__); \ + exit(1); \ + } \ +} while(0) + +// Read a single z-slice (ny x nx, x fastest) from a raw x-fastest fp32 volume +// of grid (gnz,gny,gnx) at z index zc. +static bool load_zslice(const std::string& path, int gnz, int gny, int gnx, + int zc, int nx, int ny, std::vector& out) +{ + if (zc < 0 || zc >= gnz || nx > gnx || ny > gny) return false; + FILE* f = std::fopen(path.c_str(), "rb"); + if (!f) { std::fprintf(stderr, "open fail %s\n", path.c_str()); return false; } + out.resize((size_t)nx * ny); + // center-crop nx x ny out of the gny x gnx slice + int ox = (gnx - nx) / 2, oy = (gny - ny) / 2; + long base = (long)zc * gny * gnx; + bool ok = true; + for (int j = 0; j < ny && ok; ++j) { + long off = (base + (long)(oy + j) * gnx + ox) * (long)sizeof(float); + if (std::fseek(f, off, SEEK_SET) != 0) { ok = false; break; } + if (std::fread(&out[(size_t)j * nx], sizeof(float), nx, f) != (size_t)nx) ok = false; + } + std::fclose(f); + return ok; +} + +// Read a 3D crop (cz,cy,cx; x fastest) from a raw x-fastest fp32 volume +// (gnz,gny,gnx) at origin (z0,y0,x0), into out (z,y,x order, x fastest). +static bool load_crop3d(const std::string& path, int gnz, int gny, int gnx, + int z0, int y0, int x0, int cz, int cy, int cx, + std::vector& out) +{ + if (z0 < 0 || y0 < 0 || x0 < 0 || z0+cz > gnz || y0+cy > gny || x0+cx > gnx) return false; + FILE* f = std::fopen(path.c_str(), "rb"); + if (!f) { std::fprintf(stderr, "open fail %s\n", path.c_str()); return false; } + out.resize((size_t)cz * cy * cx); + bool ok = true; + for (int k = 0; k < cz && ok; ++k) + for (int j = 0; j < cy && ok; ++j) { + long off = ((long)(z0+k) * gny * gnx + (long)(y0+j) * gnx + x0) * (long)sizeof(float); + if (std::fseek(f, off, SEEK_SET) != 0) { ok = false; break; } + float* dst = &out[((size_t)k * cy + j) * cx]; + if (std::fread(dst, sizeof(float), cx, f) != (size_t)cx) ok = false; + } + std::fclose(f); + return ok; +} + +static double rel_l2(const float* a, const float* b, size_t n) +{ + double num = 0.0, den = 0.0; + for (size_t i = 0; i < n; ++i) { + double d = (double)a[i] - (double)b[i]; + num += d * d; den += (double)a[i] * (double)a[i]; + } + return den > 0.0 ? std::sqrt(num / den) : 0.0; +} + +// ---- GPU 2D quadtree: compress+decompress one slice, return CR and rel_l2 ---- +static void gpu_quadtree_rd(const std::vector& slice, int nx, int ny, + float scale, double* cr_out, double* err_out) +{ + const size_t total = (size_t)nx * ny; + hipCompressPlan* plan = nullptr; + HIPCHECK(hipCompressCreatePlan(&plan, nx, ny, 1, 0, HIP_COMPRESS_KERNEL_QUADTREE)); + + float* d_in = nullptr; float* d_out = nullptr; unsigned char* d_comp = nullptr; + size_t comp_cap = 0; + HIPCHECK(hipCompressMaxOutputSize(plan, &comp_cap)); + HIPCHECK(hipMalloc(&d_in, total * sizeof(float))); + HIPCHECK(hipMalloc(&d_out, total * sizeof(float))); + HIPCHECK(hipMalloc(&d_comp, comp_cap)); + HIPCHECK(hipMemcpy(d_in, slice.data(), total * sizeof(float), hipMemcpyHostToDevice)); + + HIPCHECK(hipComputeRMS(d_in, nx, nx, 0, 0, 0, nx, ny, 1, plan->d_rms, plan, 0)); + long clen = 0; float cr = 0.0f; + HIPCHECK(hipCompress(scale, plan->d_rms, d_in, d_comp, plan, 0)); + HIPCHECK(hipCompressSynchronize(plan, &clen, &cr)); + HIPCHECK(hipDecompress(d_comp, d_out, plan, 0)); + HIPCHECK(hipDeviceSynchronize()); + + std::vector rec(total); + HIPCHECK(hipMemcpy(rec.data(), d_out, total * sizeof(float), hipMemcpyDeviceToHost)); + + *cr_out = (double)(total * sizeof(float)) / (double)clen; + *err_out = rel_l2(slice.data(), rec.data(), total); + + hipFree(d_in); hipFree(d_out); hipFree(d_comp); + hipCompressDestroyPlan(plan); +} + +// ---- CPU CvxCompress on a z-constant NZ-deep volume, isolating 2D behaviour --- +static void cpu_cvx_rd(const std::vector& slice, int nx, int ny, int NZ, + float scale, double* cr_out, double* err_out) +{ + const size_t plane = (size_t)nx * ny; + const size_t nelem = plane * NZ; + std::vector vol(nelem); + for (int z = 0; z < NZ; ++z) + std::memcpy(&vol[(size_t)z * plane], slice.data(), plane * sizeof(float)); + + std::vector comp(nelem); // generous: raw-sized + CvxCompress cvx; + long clen = 0; + float ratio = cvx.Compress(scale, vol.data(), nx, ny, NZ, 32, 32, 32, + /*use_local_RMS=*/false, comp.data(), clen); + + int nx2 = 0, ny2 = 0, nz2 = 0; + float* rec = cvx.Decompress(nx2, ny2, nz2, comp.data(), clen); + + // 2D ratio: NZ identical slices compress to ~one slice's cost. + *cr_out = (double)ratio / (double)NZ; + *err_out = rec ? rel_l2(slice.data(), rec, plane) : -1.0; // z=0 plane + if (rec) free(rec); +} + +// ---- GPU 3D octree: compress+decompress a crop, return CR and rel_l2 -------- +static void gpu_octree_rd(const std::vector& vol, int nx, int ny, int nz, + float scale, double* cr_out, double* err_out) +{ + const size_t total = (size_t)nx * ny * nz; + hipCompressPlan* plan = nullptr; + HIPCHECK(hipCompressCreatePlan(&plan, nx, ny, nz, 0, HIP_COMPRESS_KERNEL_OCTREE)); + + float* d_in = nullptr; float* d_out = nullptr; unsigned char* d_comp = nullptr; + size_t comp_cap = 0; + HIPCHECK(hipCompressMaxOutputSize(plan, &comp_cap)); + HIPCHECK(hipMalloc(&d_in, total * sizeof(float))); + HIPCHECK(hipMalloc(&d_out, total * sizeof(float))); + HIPCHECK(hipMalloc(&d_comp, comp_cap)); + HIPCHECK(hipMemcpy(d_in, vol.data(), total * sizeof(float), hipMemcpyHostToDevice)); + + HIPCHECK(hipComputeRMS(d_in, nx, nx*ny, 0, 0, 0, nx, ny, nz, plan->d_rms, plan, 0)); + long clen = 0; float cr = 0.0f; + HIPCHECK(hipCompress(scale, plan->d_rms, d_in, d_comp, plan, 0)); + HIPCHECK(hipCompressSynchronize(plan, &clen, &cr)); + HIPCHECK(hipDecompress(d_comp, d_out, plan, 0)); + HIPCHECK(hipDeviceSynchronize()); + + std::vector rec(total); + HIPCHECK(hipMemcpy(rec.data(), d_out, total * sizeof(float), hipMemcpyDeviceToHost)); + *cr_out = (double)(total * sizeof(float)) / (double)clen; + *err_out = rel_l2(vol.data(), rec.data(), total); + + hipFree(d_in); hipFree(d_out); hipFree(d_comp); + hipCompressDestroyPlan(plan); +} + +// ---- CPU CvxCompress on a real 3D crop, return CR and rel_l2 ---------------- +static void cpu_cvx_rd_3d(const std::vector& vol, int nx, int ny, int nz, + float scale, double* cr_out, double* err_out) +{ + const size_t nelem = (size_t)nx * ny * nz; + std::vector work(vol); // Compress may modify in place + std::vector comp(nelem + 1024); + CvxCompress cvx; + long clen = 0; + float ratio = cvx.Compress(scale, work.data(), nx, ny, nz, 32, 32, 32, + /*use_local_RMS=*/false, comp.data(), clen); + int nx2 = 0, ny2 = 0, nz2 = 0; + float* rec = cvx.Decompress(nx2, ny2, nz2, comp.data(), clen); + *cr_out = (double)ratio; + *err_out = rec ? rel_l2(vol.data(), rec, nelem) : -1.0; + if (rec) free(rec); +} + +// Local host copies of the quadtree serial helpers (the kernel header is only +// compiled in hipCompress.cpp's TU; including it here would duplicate kernels). +static inline int qt_width(unsigned a) { + if (a <= 0x7fu) return 1; if (a <= 0x7fffu) return 2; + if (a <= 0x7fffffu) return 3; return 4; +} +static inline bool qt_sub_any(const uint32_t* m, int ox, int oy, int cs) { + uint32_t xmask = (cs>=32)?0xFFFFFFFFu:(uint32_t)((((uint32_t)1<>1,b=0; for(int c=0;c<4;++c){int cx=c&1,cy=(c>>1)&1; + if(qt_sub_any(m,px+cx*cs,py+cy*cs,cs)) b|=(1<0){int i=sp-1,cs=side[i]>>1,found=-1; + if(cs>1) for(int c=ch[i];c<4;++c) if(bb[i]&(1<=0){ch[i]=found+1;int cx=found&1,cy=(found>>1)&1; + int ox=px[i]+cx*cs,oy=py[i]+cy*cs,b=qt_child_occ(m,ox,oy,cs);++pos; + px[sp]=ox;py[sp]=oy;side[sp]=cs;bb[sp]=b;ch[sp]=0;++sp; + } else --sp; + } return pos; +} + +// Host 2D DS 7/9 forward on a 32x32 block (row-major q[iy*32+ix]); separable and +// linear, so row-then-col matches the GPU's col-then-row result. +static void host_block_forward(float* b /*32x32*/) { + float line[32]; + for (int iy = 0; iy < 32; ++iy) { // rows (x) + for (int ix = 0; ix < 32; ++ix) line[ix] = b[iy*32+ix]; + ds79_forward(line, 32); + for (int ix = 0; ix < 32; ++ix) b[iy*32+ix] = line[ix]; + } + for (int ix = 0; ix < 32; ++ix) { // cols (y) + for (int iy = 0; iy < 32; ++iy) line[iy] = b[iy*32+ix]; + ds79_forward(line, 32); + for (int iy = 0; iy < 32; ++iy) b[iy*32+ix] = line[iy]; + } +} + +// CPU byte-breakdown of the 2D quadtree stream on the real slice, at matched +// fidelity vs CvxCompress (same mulfac => same quantized ints => same rel_l2). +static void breakdown_2d(const std::vector& slice, int nx, int ny, + const std::vector& scales) +{ + double ss = 0.0; for (float v : slice) ss += (double)v * v; + double rms = std::sqrt(ss / slice.size()); + const int nbx = nx/32, nby = ny/32, nb = nbx*nby; + const long hdr = 8 + 12L*nb + 4; // octree-style header + const double raw = (double)nx*ny*4.0; + + // All schemes share the SAME quantization (fixed mulfac) => matched fidelity; + // only the value-region byte cost differs. Significance is nibble-packed (as + // shipped). Each scheme's own width-table / exception overhead is included. + // row : one byte-width per nonempty 32-cell row (current codec) + // g8/g4: one byte-width per nonempty 8- / 4-cell group (finer fixed width) + // pfor : bimodal patched fixed width - base W_lo for all + high bytes for the + // few exceptions + 1 exception-bit per nonzero + 1 byte (W_lo,W_hi) + // pv : ideal per-value width (lower bound; no table) + std::printf("\n2D value-coder model (per-block; all schemes matched fidelity; sig=nibble):\n"); + std::printf("%-7s %-6s | %-7s %-7s %-7s %-7s %-7s | %-6s %-6s %-6s %-6s\n", + "scale","nnz%","CR_row","CR_g8","CR_g4","CR_pfor","CR_pv", + "g8/row","g4/row","pf/row","pv/row"); + + for (float sc : scales) { + float mulfac = (rms>0)? (float)(1.0/(rms*sc)) : 1.0f; + long tnnz=0, t_sigN=0; + long t_vrow=0, t_vg8=0, t_vg4=0, t_vpfor=0, t_vpv=0; + for (int by=0; by0 && nb2<128) sigN=nb2; } + + long vrow=0, vg8=0, vg4=0, vpv=0; + int ne_row=0, ne_g8=0, ne_g4=0; + int wcnt[5]={0,0,0,0,0}; int nnz_blk=0, Whi=0; // for PFOR + for (int iy=0; iy<32; ++iy){ uint32_t mm=m[iy]; if(!mm) continue; ++ne_row; + unsigned mxa=0; int cnt=0; + for (int ix=0; ix<32; ++ix) if(mm&(1u<mxa)mxa=a; ++cnt; ++tnnz; ++nnz_blk; + vpv += w; ++wcnt[w]; if(w>Whi)Whi=w; } + vrow += (long)cnt * qt_width(mxa); + for (int g=0; g<4; ++g){ unsigned gm=0; int gc=0; + for (int ix=g*8; ixgm)gm=a; ++gc; } + if(gc){ ++ne_g8; vg8 += (long)gc*qt_width(gm); } } + for (int g=0; g<8; ++g){ unsigned gm=0; int gc=0; + for (int ix=g*4; ixgm)gm=a; ++gc; } + if(gc){ ++ne_g4; vg4 += (long)gc*qt_width(gm); } } + } + // PFOR: minimise nnz*W_lo + n_exc(W_lo)*(W_hi-W_lo) over W_lo in [1,W_hi] + long vpfor=0; + if (nnz_blk){ long best=-1; + for (int wlo=1; wlo<=Whi; ++wlo){ int nexc=0; for(int w=wlo+1; w<=4; ++w) nexc+=wcnt[w]; + long b=(long)nnz_blk*wlo + (long)nexc*(Whi-wlo); if(best<0||b gpu_scales = {1e-2f, 2e-2f, 5e-2f, 1e-1f, 2e-1f, 4e-1f}; + std::vector cpu_scales = {1e-3f, 2e-3f, 5e-3f, 1e-2f, 2e-2f, 5e-2f}; + + auto parse_list = [](const char* s, std::vector& v) { + v.clear(); char b[4096]; std::strncpy(b, s, 4095); b[4095] = '\0'; + for (char* t = std::strtok(b, ","); t; t = std::strtok(nullptr, ",")) v.push_back((float)atof(t)); + }; + for (int i = 1; i < argc; ++i) { + if (!std::strcmp(argv[i], "--panel") && i+1 < argc) panel = argv[++i]; + else if (!std::strcmp(argv[i], "--mode") && i+1 < argc) mode = argv[++i]; + else if (!std::strcmp(argv[i], "--pdims") && i+3 < argc) { gnz=atoi(argv[++i]); gny=atoi(argv[++i]); gnx=atoi(argv[++i]); } + else if (!std::strcmp(argv[i], "--nx") && i+1 < argc) nx = atoi(argv[++i]); + else if (!std::strcmp(argv[i], "--ny") && i+1 < argc) ny = atoi(argv[++i]); + else if (!std::strcmp(argv[i], "--z") && i+1 < argc) zc = atoi(argv[++i]); + else if (!std::strcmp(argv[i], "--nz") && i+1 < argc) NZ = atoi(argv[++i]); + else if (!std::strcmp(argv[i], "--crop") && i+6 < argc) { + x0=atoi(argv[++i]); y0=atoi(argv[++i]); z0=atoi(argv[++i]); + cx=atoi(argv[++i]); cy=atoi(argv[++i]); cz=atoi(argv[++i]); + } + else if (!std::strcmp(argv[i], "--gpu-scales") && i+1 < argc) parse_list(argv[++i], gpu_scales); + else if (!std::strcmp(argv[i], "--cpu-scales") && i+1 < argc) parse_list(argv[++i], cpu_scales); + } + if (panel.empty()) { + std::fprintf(stderr, "Usage: %s --panel FILE.raw [--mode 2d|3d] [--pdims NZ NY NX]\n" + " 2d: [--nx N --ny N] [--z ZC] [--nz NZstack]\n" + " 3d: [--crop X0 Y0 Z0 CX CY CZ]\n", argv[0]); + return 1; + } + + if (mode == "3d") { + if (cx % 32 || cy % 32 || cz % 32) { std::fprintf(stderr, "3D crop dims must be multiples of 32\n"); return 1; } + std::vector vol; + if (!load_crop3d(panel, gnz, gny, gnx, z0, y0, x0, cz, cy, cx, vol)) { + std::fprintf(stderr, "failed to load 3D crop\n"); return 1; + } + double ss = 0.0; for (float v : vol) ss += (double)v * v; + std::printf("panel=%s 3D crop origin=(%d,%d,%d) dims=%dx%dx%d rms=%.3e\n", + panel.c_str(), x0, y0, z0, cx, cy, cz, std::sqrt(ss / vol.size())); + + std::printf("\nGPU 3D octree:\n%-10s %-12s %-10s\n", "scale", "vol_rel_l2", "CR"); + std::vector> gpu_rd; + for (float sc : gpu_scales) { + double cr, err; gpu_octree_rd(vol, cx, cy, cz, sc, &cr, &err); + std::printf("%-10.4g %-12.4e %-10.3f\n", sc, err, cr); + gpu_rd.push_back({err, cr}); + } + std::printf("\nCPU CvxCompress (3D):\n%-10s %-12s %-10s\n", "scale", "vol_rel_l2", "CR"); + std::vector> cpu_rd; + for (float sc : cpu_scales) { + double cr, err; cpu_cvx_rd_3d(vol, cx, cy, cz, sc, &cr, &err); + std::printf("%-10.4g %-12.4e %-10.3f\n", sc, err, cr); + cpu_rd.push_back({err, cr}); + } + auto interp = [](std::vector> rd, double t)->double { + std::sort(rd.begin(), rd.end()); + if (t < rd.front().first || t > rd.back().first) return -1.0; + for (size_t i = 1; i < rd.size(); ++i) if (t <= rd[i].first) { + double e0=rd[i-1].first,e1=rd[i].first,c0=rd[i-1].second,c1=rd[i].second; + double u=(std::log(t)-std::log(e0))/(std::log(e1)-std::log(e0)); + return std::exp(std::log(c0)+u*(std::log(c1)-std::log(c0))); + } + return -1.0; + }; + std::printf("\nCR at matched fidelity (log-log interpolated):\n%-12s %-13s %-13s %-10s\n", + "vol_rel_l2", "GPU_oct_CR", "CPU_cvx_CR", "oct/cvx"); + for (double tgt : {2e-3, 5e-3, 1e-2, 2e-2, 5e-2, 1e-1}) { + double g = interp(gpu_rd, tgt), c = interp(cpu_rd, tgt); + if (g > 0 && c > 0) std::printf("%-12.3e %-13.3f %-13.3f %-10.3f\n", tgt, g, c, g / c); + else std::printf("%-12.3e %-13s %-13s %-10s\n", tgt, g>0?"-":"oob", c>0?"-":"oob", "-"); + } + return 0; + } + + if (nx % 32 || ny % 32) { std::fprintf(stderr, "nx,ny must be multiples of 32\n"); return 1; } + if (zc < 0) zc = gnz / 2; + + std::vector slice; + if (!load_zslice(panel, gnz, gny, gnx, zc, nx, ny, slice)) { + std::fprintf(stderr, "failed to load slice\n"); return 1; + } + + // slice RMS (for context) + double ss = 0.0; for (float v : slice) ss += (double)v * v; + double srms = std::sqrt(ss / slice.size()); + std::printf("panel=%s slice z=%d crop=%dx%d rms=%.3e (CPU codec: z-constant NZ=%d, 32^3 blocks)\n", + panel.c_str(), zc, nx, ny, srms, NZ); + + if (mode == "bd2d") { breakdown_2d(slice, nx, ny, gpu_scales); return 0; } + + std::printf("\nGPU 2D quadtree:\n%-10s %-12s %-10s\n", "scale", "vol_rel_l2", "CR"); + std::vector> gpu_rd; // (err, cr) + for (float sc : gpu_scales) { + double cr, err; gpu_quadtree_rd(slice, nx, ny, sc, &cr, &err); + std::printf("%-10.4g %-12.4e %-10.3f\n", sc, err, cr); + gpu_rd.push_back({err, cr}); + } + + std::printf("\nCPU CvxCompress (2D-isolated):\n%-10s %-12s %-10s\n", "scale", "vol_rel_l2", "CR"); + std::vector> cpu_rd; + for (float sc : cpu_scales) { + double cr, err; cpu_cvx_rd(slice, nx, ny, NZ, sc, &cr, &err); + std::printf("%-10.4g %-12.4e %-10.3f\n", sc, err, cr); + cpu_rd.push_back({err, cr}); + } + + // CR at matched distortion: interpolate each codec's CR(err) in log-log at a + // few shared rel_l2 targets that both curves bracket. + auto interp_cr = [](std::vector> rd, double target)->double { + std::sort(rd.begin(), rd.end()); + if (target < rd.front().first || target > rd.back().first) return -1.0; + for (size_t i = 1; i < rd.size(); ++i) { + if (target <= rd[i].first) { + double e0 = rd[i-1].first, e1 = rd[i].first; + double c0 = rd[i-1].second, c1 = rd[i].second; + double t = (std::log(target) - std::log(e0)) / (std::log(e1) - std::log(e0)); + return std::exp(std::log(c0) + t * (std::log(c1) - std::log(c0))); + } + } + return -1.0; + }; + std::printf("\nCR at matched fidelity (log-log interpolated):\n%-12s %-12s %-12s %-10s\n", + "vol_rel_l2", "GPU_qt_CR", "CPU_cvx_CR", "qt/cvx"); + for (double tgt : {1e-3, 2e-3, 5e-3, 1e-2, 2e-2}) { + double g = interp_cr(gpu_rd, tgt), c = interp_cr(cpu_rd, tgt); + if (g > 0 && c > 0) + std::printf("%-12.3e %-12.3f %-12.3f %-10.3f\n", tgt, g, c, g / c); + else + std::printf("%-12.3e %-12s %-12s %-10s\n", tgt, + g > 0 ? "-" : "oob", c > 0 ? "-" : "oob", "-"); + } + return 0; +} diff --git a/tests/test_bitmap_octree_hip.cpp b/tests/test_bitmap_octree_hip.cpp index e2a431a..bb66c85 100644 --- a/tests/test_bitmap_octree_hip.cpp +++ b/tests/test_bitmap_octree_hip.cpp @@ -80,12 +80,13 @@ static void launch_rle_inv(void* p){ Ctx*c=(Ctx*)p; dim3 g(c->nbx,c->nby,c->nbz) waveletRLEInverseFusedKernel<<>>(c->d_rle,c->d_rle_sizes,nullptr,c->d_field_rle,1.0f/c->mulfac,c->ldimx,c->ldimxy,0); } static bool load_center_crop(const std::string& path,int gnz,int gny,int gnx, - int cz,int cy,int cx,std::vector& out){ + int cz,int cy,int cx,std::vector& out, + int z0=-1,int y0=-1,int x0=-1){ FILE* f=std::fopen(path.c_str(),"rb"); if(!f){printf("open fail %s\n",path.c_str());return false;} std::vector full((size_t)gnz*gny*gnx); size_t r=std::fread(full.data(),sizeof(float),full.size(),f); std::fclose(f); if(r!=full.size()){printf("short read %s\n",path.c_str());return false;} - int oz=(gnz-cz)/2,oy=(gny-cy)/2,ox=(gnx-cx)/2; + int oz=(z0>=0)?z0:(gnz-cz)/2,oy=(y0>=0)?y0:(gny-cy)/2,ox=(x0>=0)?x0:(gnx-cx)/2; out.resize((size_t)cz*cy*cx); for(int k=0;k h_in(nelem); if(!panel.empty()){ - if(!load_center_crop(panel,512,512,512,NX,NX,NX,h_in)) return 1; + if(!load_center_crop(panel,512,512,512,NX,NX,NX,h_in,z0,y0,x0)) return 1; } else { for(size_t i=0;i0 && memcmp(blkc+wtab_base, codetl+tl_wtab_off, tl_wtab_len)!=0){ - if(fus_wtab_mism<10) printf(" [fusWtab] blk %d differ (ne=%d)\n",bid,ne); ++fus_wtab_mism; } - if(val_len>0 && memcmp(blkc+vals_base, codetl+tl_vals_off, val_len)!=0){ - if(fus_val_mism<10) printf(" [fusVal] blk %d differ (len=%ld)\n",bid,val_len); ++fus_val_mism; } - long fexp = vals_base + val_len; - if((long)h_octc_sizes[bid]!=fexp){ if(fus_size_mism<10) - printf(" [fusSize] blk %d got %ld exp %ld\n",bid,(long)h_octc_sizes[bid],fexp); ++fus_size_mism; } + // NOTE: the fused kernel-2 now uses a per-block PFOR value coder (not the + // legacy per-line width table). Its value round-trip is validated + // format-agnostically by the stage-A decode gate (decA_mism) below, so the + // old wtab/values/size byte-exact comparisons vs the two-level reference no + // longer apply. Only the measured coded size is kept (for the fused CR). + (void)fsig; fus_block_total += (long)h_octc_sizes[bid]; const unsigned char* blk = h_oct.data()+(long)bid*WOCT_SLOT_BYTES; int mode = blk[0]; @@ -363,15 +358,14 @@ int main(int argc,char** argv){ lm_cnt_mism?"MISMATCH":"OK", lm_cnt_mism, lm_rt_mism?"MISMATCH":"OK", lm_rt_mism); printf("parallel GPU: byte-exact vs host LM: %s (%ld) round-trip masks: %s (%ld)\n", par_enc_mism?"MISMATCH":"OK", par_enc_mism, par_rt_mism?"MISMATCH":"OK", par_rt_mism); - printf("fused kernel-2: sig=%s(%ld) wtab=%s(%ld) values=%s(%ld) size=%s(%ld)\n", - fus_sig_mism?"MISMATCH":"OK",fus_sig_mism, fus_wtab_mism?"MISMATCH":"OK",fus_wtab_mism, - fus_val_mism?"MISMATCH":"OK",fus_val_mism, fus_size_mism?"MISMATCH":"OK",fus_size_mism); + printf("fused kernel-2 (PFOR value coder): sig=%s(%ld) values: round-trip via decA gate below\n", + fus_sig_mism?"MISMATCH":"OK",fus_sig_mism); + (void)fus_wtab_mism; (void)fus_val_mism; (void)fus_size_mism; printf("decode: stage-A round-trip (bytes vs kernel-1): %s (%ld)\n", decA_mism?"MISMATCH":"OK", decA_mism); printf("distortion vs original (rel_l2): octree=%.4e RLE=%.4e | octree-vs-RLE field maxdiff=%.3e (%ld voxels, codec f32-escape delta)\n", rel_l2_oct, rel_l2_rle, field_maxdiff, field_mism); total_mism += lm_cnt_mism + lm_rt_mism + par_enc_mism + par_rt_mism - + fus_sig_mism + fus_wtab_mism + fus_val_mism + fus_size_mism - + decA_mism; + + fus_sig_mism + decA_mism; if(total_mism){ printf("FAIL\n"); return 1; } printf("PASS\n"); return 0; diff --git a/tests/test_bitmap_rd_panel.cpp b/tests/test_bitmap_rd_panel.cpp new file mode 100644 index 0000000..b21e9b6 --- /dev/null +++ b/tests/test_bitmap_rd_panel.cpp @@ -0,0 +1,388 @@ +// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Use of this source code is governed by an MIT-style license that can be +// found in the LICENSE file or at https://opensource.org/licenses/MIT. +// +// Rate-distortion sweep on a REAL seismic wavefield panel (e.g. the Marmousi +// RTM snapshots used in the GTAC paper). For each quantization scale we report, +// on identically CPU-wavelet-transformed / identically-quantized 32^3 blocks: +// +// distortion : vol_rel_l2 = sqrt( sum (x-xhat)^2 / sum x^2 ), measured in the +// SPATIAL domain via the real inverse wavelet transform of the +// truncation-quantized coefficients q/scale (q = (int)(scale*c)). +// rate : compression ratio CR = raw_fp32_bytes / compressed_bytes for +// - CPU Cvx full-block RLE (ground truth Run_Length_Encode_Slow) +// - bitmap TWO-LEVEL occupancy + per-line width (the new codec) +// - bitmap per-line width and bitmap fixed width (reference) +// +// Quantization is identical across all codecs, so the distortion column is shared +// and the CR columns are directly comparable at matched fidelity => clean R-D +// curves. Same header/CR basis as test_bitmap_vs_cpu_rle.cpp. + +#include +#include +#include +#include +#include +#include + +#define MY_AVX_DEFINED +#define SIMDE_ENABLE_NATIVE_ALIASES +#include "simde/x86/avx512.h" + +#include "Block_Copy.hxx" +#include "Wavelet_Transform_Fast.hxx" +#include "Run_Length_Encode_Slow.hxx" + +// per-value minimal signed byte width (no in-band escape collision) +static inline int bmp_width(int maxabs) { + if (maxabs <= 0x7f) return 1; + if (maxabs <= 0x7fff) return 2; + if (maxabs <= 0x7fffff) return 3; + return 4; +} + +// 1-D octree (bisection) cost of a 32-bit z-mask: number of non-empty internal +// nodes over sizes {32,16,8,4,2}; the encoder stores 2 bits per such node +// (its two children's occupancy), descending only into non-empty subtrees. +// bits = 2 * bisect_nodes(m). Caller guarantees m != 0. +static inline int bisect_nodes(unsigned m) { + int n = 1; // size-32 root (active line => non-empty) + for (int j=0;j<2; ++j) if (m & (0xFFFFu << (16*j))) ++n; + for (int j=0;j<4; ++j) if (m & (0xFFu << (8*j))) ++n; + for (int j=0;j<8; ++j) if (m & (0xFu << (4*j))) ++n; + for (int j=0;j<16;++j) if (m & (0x3u << (2*j))) ++n; + return n; +} + +// number of contiguous 1-runs in a 32-bit mask (contiguity probe) +static inline int mask_runs(unsigned m) { + int runs=0; unsigned prev=0; + for (int b=0;b<32;++b){ unsigned cur=(m>>b)&1u; if (cur & ~prev) ++runs; prev=cur; } + return runs; +} + +// nonzeros-per-active-line histogram bins: 1,2,3,4,5-6,7-8,9-12,13-16,17-32 +static const int NKB = 9; +static inline int khist_bin(int k) { + if (k<=4) return k-1; + if (k<=6) return 4; + if (k<=8) return 5; + if (k<=12) return 6; + if (k<=16) return 7; + return 8; +} + +// Load a raw x-fastest fp32 volume of grid (gnz,gny,gnx), center-crop an +// (cz,cy,cx) sub-volume into out (z,y,x order, x fastest). +static bool load_center_crop(const std::string& path, int gnz, int gny, int gnx, + int cz, int cy, int cx, std::vector& out) { + FILE* f = std::fopen(path.c_str(), "rb"); + if (!f) { std::fprintf(stderr, "open fail %s\n", path.c_str()); return false; } + std::vector full((size_t)gnz*gny*gnx); + size_t r = std::fread(full.data(), sizeof(float), full.size(), f); + std::fclose(f); + if (r != full.size()) { std::fprintf(stderr, "short read %s (got %zu need %zu)\n", + path.c_str(), r, full.size()); return false; } + int oz = (gnz - cz) / 2, oy = (gny - cy) / 2, ox = (gnx - cx) / 2; + if (oz < 0 || oy < 0 || ox < 0) { std::fprintf(stderr, "crop > grid\n"); return false; } + out.resize((size_t)cz*cy*cx); + for (int k = 0; k < cz; ++k) + for (int j = 0; j < cy; ++j) { + const float* src = &full[((size_t)(oz+k)*gny + (oy+j))*gnx + ox]; + float* dst = &out[((size_t)k*cy + j)*cx]; + std::memcpy(dst, src, (size_t)cx*sizeof(float)); + } + return true; +} + +int main(int argc, char** argv) +{ + std::string panel, out_csv; + int gnz = 512, gny = 512, gnx = 512; // grid of the raw file + int NX = 512; // cropped cube edge (mult of 32) + // scale is applied to the RMS-normalized volume (see below), so it is + // interpretable as an RMS-relative quantizer step, matching CvxCompress. + std::vector scales = {4.f,8.f,16.f,32.f,64.f,128.f,256.f,512.f}; + + for (int i = 1; i < argc; ++i) { + if (!std::strcmp(argv[i],"--panel") && i+1 vol; + if (!load_center_crop(panel, gnz, gny, gnx, NZ, NY, NX, vol)) return 1; + + // Normalize the cropped volume to unit RMS so that `scale` is an + // RMS-relative quantizer step (raw wavefield amplitudes are ~1e-8). + // vol_rel_l2 is a ratio and thus invariant to this uniform scaling. + double ss = 0.0; for (size_t i=0;i 0) { float inv=(float)(1.0/rms); for (size_t i=0;i sse(S,0.0); + std::vector rle_total(S,0), tl_nnz(S,0), tl_nonempty(S,0), tl_pl_pay(S,0); + std::vector fx_val(S,0); // fixed per-block-width payload + std::vector bisect_bits(S,0); // 1-D octree z-mask total bits + std::vector runs_tot(S,0); // total z-runs over active lines + std::vector khist(S*NKB,0); // nonzeros-per-active-line histogram + // full 3D octree (quadtree-xy x bisection-z) significance, per-block adaptive + std::vector oct_bits(S,0); // full-octree significance total bits + std::vector sig_oct(S,0), sig_best(S,0); // significance bytes (sum of per-block) + std::vector mode_win(S*4,0); // per-block winner: flat/2lvl/zbis/oct + double sig = 0.0; // signal energy (scale-independent) + + for (int ibz=0; ibzmaxabs)maxabs=a; } } + int W = nnz? bmp_width(maxabs):0; + tl_nnz[s]+=nnz; + fx_val[s]+=(long)nnz*W; + + // rate: per-line width + occupancy (two-level), plus z-mask + // structure probes (bisection cost, run count, k-histogram) and the + // full 32^3 significance for the 3D octree cost model. + int blk_nonempty=0; long blk_bisbits=0; + unsigned lines[1024]; // z-mask per (x,y) line: lines[iy*32+ix] + for (int iy=0; iylmax)lmax=a; } + } + lines[iy*32+ix]=mask; + if (lnnz) { + tl_pl_pay[s]+=(long)lnnz*bmp_width(lmax); ++blk_nonempty; + long bb=2L*bisect_nodes(mask); blk_bisbits+=bb; bisect_bits[s]+=bb; + runs_tot[s]+=mask_runs(mask); + khist[s*NKB+khist_bin(lnnz)]++; + } + } + tl_nonempty[s]+=blk_nonempty; + + // Full 3D octree of the 32^3 significance: 8 bits per non-empty node + // of size {32,16,8,4,2}. Built bottom-up as a pyramid; count[level] + // = number of non-empty nodes at that level. z is pooled in pairs at + // each level (bisection-z), xy is 2x2 pooled (quadtree-xy). + unsigned L2[256]; // 16x16 nodes, 16-bit z-occ (size-2) + int c2=0,c4=0,c8=0,c16=0,c32=0; + auto poolz=[&](unsigned oo)->unsigned { // pool adjacent z-pairs + unsigned r=0; for (int g=0; oo; ++g, oo>>=2) if (oo&3u) r|=(1u<0)? std::sqrt(sse[s]/sig) : 0.0; + double nnz_pct = 100.0*tl_nnz[s]/((double)nblocks*bsz); + + long rle_all = rle_total[s] + hdr; + + long wtab_ne = (2L*tl_nonempty[s] + 7)/8; // 2b width / nonempty line + long tl_sig = 128L*nblocks + 4L*tl_nonempty[s]; // occupancy + per-line masks + long tl_all = tl_sig + wtab_ne + tl_pl_pay[s] + hdr; + + long pl_wtab = 256L*nblocks; // 2b width / all 1024 lines + long pl_all = 4096L*nblocks + pl_wtab + tl_pl_pay[s] + hdr; + + long fx_all = 4096L*nblocks + 4L*nblocks + fx_val[s] + hdr; // sig + 4B width hdr/blk + + double rle_cr = raw/rle_all, tl_cr = raw/tl_all, pl_cr = raw/pl_all, fx_cr = raw/fx_all; + + std::printf("%-7.3f %-8.3f %-11.4e %-9.3f %-9.3f %-9.3f %-9.3f %-8.3f\n", + scales[s], nnz_pct, vol_rel_l2, rle_cr, tl_cr, pl_cr, fx_cr, + (double)tl_all/rle_all); + + // exact structural split + 1-D bisection z-mask replacement + long occ_b = 128L*nblocks; + long mask_b = 4L*tl_nonempty[s]; + long mbis_b = (bisect_bits[s] + 7)/8; // bisection masks + long tl_bis_all = occ_b + mbis_b + wtab_ne + tl_pl_pay[s] + hdr; + double avg_k = tl_nonempty[s]? (double)tl_nnz[s]/tl_nonempty[s] : 0.0; + double avg_runs = tl_nonempty[s]? (double)runs_tot[s]/tl_nonempty[s] : 0.0; + + long oct_all = sig_oct[s] + wtab_ne + tl_pl_pay[s] + hdr; + long best_all = sig_best[s] + wtab_ne + tl_pl_pay[s] + hdr; + double mtot = (double)nblocks; + + if (csv) std::fprintf(csv, + "%.4f,%.4f,%.6e,%ld,%.4f,%ld,%.4f,%ld,%.4f,%ld,%.4f,%.4f," + "%ld,%ld,%ld,%ld,%ld,%.3f,%.3f,%ld,%ld,%.4f,%.4f," + "%ld,%ld,%.4f,%.4f,%ld,%ld,%.4f,%.4f,%.2f,%.2f,%.2f,%.2f\n", + scales[s], nnz_pct, vol_rel_l2, rle_all, rle_cr, + tl_all, tl_cr, pl_all, pl_cr, fx_all, fx_cr, (double)tl_all/rle_all, + occ_b, mask_b, wtab_ne, tl_pl_pay[s], tl_nonempty[s], avg_k, avg_runs, + mbis_b, tl_bis_all, raw/tl_bis_all, (double)tl_bis_all/rle_all, + sig_oct[s], oct_all, raw/oct_all, (double)oct_all/rle_all, + sig_best[s], best_all, raw/best_all, (double)best_all/rle_all, + 100.0*mode_win[s*4+0]/mtot, 100.0*mode_win[s*4+1]/mtot, + 100.0*mode_win[s*4+2]/mtot, 100.0*mode_win[s*4+3]/mtot); + } + + // structure breakdown + 1-D bisection z-mask, per scale + std::printf("\n%-7s %-9s %-7s %-7s | %-10s %-10s %-10s %-10s | %-9s %-9s %-8s\n", + "scale","nonempty","avg_k","avgruns","occ_B","mask_B","wtab_B","payld_B", + "maskbisB","2lvlbisCR","bis/RLE"); + for (int s=0; s0? 100.0*khist[s*NKB+b]/tot : 0.0); + std::printf("\n"); + } + + // full 3D octree + per-block adaptive best-significance codec + std::printf("\n%-7s %-9s | %-11s %-11s %-11s | %-8s %-8s %-8s %-8s | %-8s %-8s\n", + "scale","nnz%","2lvl_sigB","octree_sigB","best_sigB", + "RLE_CR","2lvl_CR","oct_CR","best_CR","oct/RLE","best/RLE"); + for (int s=0; s>>(d_input, NX, NY, 24.0f, 24.0f); + HIPCHECK(hipDeviceSynchronize()); + + long len_rle = 0, len_qt = 0; + float cr_rle = 0, cr_qt = 0; + HIPCHECK(compressWithAutoRMS2D(scale, d_input, d_comp_rle, &len_rle, &cr_rle, p_rle)); + HIPCHECK(compressWithAutoRMS2D(scale, d_input, d_comp_qt, &len_qt, &cr_qt, p_qt)); + printf(" RLE : CR=%.2f, %ld bytes\n", cr_rle, len_rle); + printf(" quadtree : CR=%.2f, %ld bytes\n", cr_qt, len_qt); + + HIPCHECK(hipDecompress(d_comp_rle, d_out_rle, p_rle, 0)); + HIPCHECK(hipDecompress(d_comp_qt, d_out_qt, p_qt, 0)); + HIPCHECK(hipDeviceSynchronize()); + + std::vector h_in(total), h_rle(total), h_qt(total); + HIPCHECK(hipMemcpy(h_in.data(), d_input, total * sizeof(float), hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_rle.data(), d_out_rle, total * sizeof(float), hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_qt.data(), d_out_qt, total * sizeof(float), hipMemcpyDeviceToHost)); + + float rms = hostRMS(h_in.data(), total); + float qt_err = maxAbsError(h_in.data(), h_qt.data(), total); + float vs_rle = maxAbsError(h_rle.data(), h_qt.data(), total); + printf(" quadtree vs input: max_err=%.6e (rms=%.6e, rel=%.6e)\n", qt_err, rms, qt_err / rms); + printf(" quadtree vs RLE : max_err=%.6e (float-epsilon of RLE path)\n", vs_rle); + + // The quadtree path shares the RLE transform/quantizer, so it must match the + // RLE reconstruction to within float rounding (the two inverse kernels are the + // same algorithm but the compiler contracts FP independently) -- i.e. far + // below the quantization error. Require the codecs agree to <1e-4 abs. + bool pass = (qt_err < rms) && (cr_qt > 1.0f) && (vs_rle < 1e-4f); + printf(" quadtree 2D: %s\n", pass ? "PASS" : "FAIL"); + + hipFree(d_input); hipFree(d_out_rle); hipFree(d_out_qt); + hipFree(d_comp_rle); hipFree(d_comp_qt); + hipCompressDestroyPlan(p_rle); + hipCompressDestroyPlan(p_qt); + return pass; +} + int main() { printf("=== 2D Compression Tests ===\n\n"); @@ -348,6 +429,8 @@ int main() printf("\n"); total++; if (test_round_trip_with_copy_2d()) passed++; printf("\n"); + total++; if (test_quadtree_2d_round_trip()) passed++; + printf("\n"); printf("=== Results: %d/%d passed ===\n", passed, total); return (passed == total) ? 0 : 1; diff --git a/tests/test_compress_api_hip.cpp b/tests/test_compress_api_hip.cpp index 9fc93c9..feedefe 100644 --- a/tests/test_compress_api_hip.cpp +++ b/tests/test_compress_api_hip.cpp @@ -97,8 +97,9 @@ static bool test_plan_lifecycle() if (err == hipSuccess) { printf(" FAIL: should reject nx=100\n"); hipCompressDestroyPlan(plan); return false; } printf(" reject non-32-multiple: PASS\n"); - // MaxOutputSize - err = hipCompressCreatePlan(&plan, 64, 64, 64, 0); + // MaxOutputSize (ZLINE bound formula -- pin the codec so the expected value + // is codec-specific and independent of the dimensionality-selected default) + err = hipCompressCreatePlan(&plan, 64, 64, 64, 0, HIP_COMPRESS_KERNEL_ZLINE); if (err != hipSuccess) { printf(" FAIL: create 64^3 plan\n"); return false; } size_t max_sz = 0; HIPCHECK(hipCompressMaxOutputSize(plan, &max_sz)); @@ -240,6 +241,64 @@ static bool test_octree_round_trip() return pass; } +static bool test_twolevel_round_trip() +{ + printf("Test: Two-level compress/decompress round-trip\n"); + const int N = 128, total = N * N * N; + const float scale = 5e-2f; + + hipCompressPlan* tl = nullptr; + HIPCHECK(hipCompressCreatePlan(&tl, N, N, N, 0, HIP_COMPRESS_KERNEL_TWOLEVEL)); + hipCompressPlan* rle = nullptr; + HIPCHECK(hipCompressCreatePlan(&rle, N, N, N, 0, HIP_COMPRESS_KERNEL_ZLINE)); + printf(" encoder: %s (gfx950 opt path %s)\n", + tl->tl_use_opt ? "opt" : "portable", + tl->tl_use_opt ? "active" : "fallback"); + + float* d_input = nullptr; + float* d_output = nullptr; + unsigned char* d_comp = nullptr; + unsigned char* d_comp_rle = nullptr; + HIPCHECK(hipMalloc(&d_input, total * sizeof(float))); + HIPCHECK(hipMalloc(&d_output, total * sizeof(float))); + size_t comp_size = 0; + HIPCHECK(hipCompressMaxOutputSize(tl, &comp_size)); + HIPCHECK(hipMalloc(&d_comp, comp_size)); + size_t comp_size_rle = 0; + HIPCHECK(hipCompressMaxOutputSize(rle, &comp_size_rle)); + HIPCHECK(hipMalloc(&d_comp_rle, comp_size_rle)); + + int threads = 256, blocks = (total + threads - 1) / threads; + initSinKernel<<>>(d_input, N, N, N, 20.0f, 20.0f, 20.0f); + HIPCHECK(hipDeviceSynchronize()); + + long len = 0, len_rle = 0; + float cr = 0, cr_rle = 0; + HIPCHECK(compressWithAutoRMS(scale, d_input, d_comp, &len, &cr, tl)); + HIPCHECK(compressWithAutoRMS(scale, d_input, d_comp_rle, &len_rle, &cr_rle, rle)); + printf(" two-level CR=%.2f (%ld B) RLE CR=%.2f (%ld B) gain=%.2fx\n", + cr, len, cr_rle, len_rle, cr_rle > 0 ? cr / cr_rle : 0.0f); + + HIPCHECK(hipDecompress(d_comp, d_output, tl, 0)); + + std::vector h_in(total), h_out(total); + HIPCHECK(hipMemcpy(h_in.data(), d_input, total * sizeof(float), hipMemcpyDeviceToHost)); + HIPCHECK(hipMemcpy(h_out.data(), d_output, total * sizeof(float), hipMemcpyDeviceToHost)); + + float rms = hostRMS(h_in.data(), total); + float max_err = maxAbsError(h_in.data(), h_out.data(), total); + printf(" decompress: max_err=%.6e, rms=%.6e, rel_max_err=%.6e\n", + max_err, rms, max_err / rms); + + bool pass = (max_err < rms) && (cr > 1.0f) && (len > 0); + printf(" two-level round-trip: %s\n", pass ? "PASS" : "FAIL"); + + hipFree(d_input); hipFree(d_output); hipFree(d_comp); hipFree(d_comp_rle); + hipCompressDestroyPlan(tl); + hipCompressDestroyPlan(rle); + return pass; +} + static bool test_cr_vs_cpu() { printf("Test 3: CR matches CPU (within z-line gap)\n"); @@ -247,9 +306,10 @@ static bool test_cr_vs_cpu() const float scale = 5e-2f; const int bx = 32, by = 32, bz = 32; - // GPU compress + // GPU compress -- this test measures the z-line codec's CR against the CPU + // reference, so pin ZLINE explicitly (the default is now dimensionality-selected). hipCompressPlan* plan = nullptr; - HIPCHECK(hipCompressCreatePlan(&plan, N, N, N, 0)); + HIPCHECK(hipCompressCreatePlan(&plan, N, N, N, 0, HIP_COMPRESS_KERNEL_ZLINE)); float* d_input = nullptr; unsigned char* d_compressed = nullptr; @@ -1800,10 +1860,12 @@ static void bench_throughput() {512, 512, 512}, }; - // RLE (z-line) vs octree on the API path, same input and quantization. + // RLE (z-line) vs octree vs two-level on the API path, same input and + // quantization. Two-level uses the arch-selected encoder (opt on gfx950). for (auto& s : sizes) { - bench_grid_size(s[0], s[1], s[2], scale, HIP_COMPRESS_KERNEL_ZLINE, "rle"); - bench_grid_size(s[0], s[1], s[2], scale, HIP_COMPRESS_KERNEL_OCTREE, "octree"); + bench_grid_size(s[0], s[1], s[2], scale, HIP_COMPRESS_KERNEL_ZLINE, "rle"); + bench_grid_size(s[0], s[1], s[2], scale, HIP_COMPRESS_KERNEL_OCTREE, "octree"); + bench_grid_size(s[0], s[1], s[2], scale, HIP_COMPRESS_KERNEL_TWOLEVEL, "twolvl"); } } @@ -3017,10 +3079,11 @@ int main(int argc, char** argv) printf("=== hipCompress API Tests ===\n\n"); - int passed = 0, total = 38; + int passed = 0, total = 39; if (test_plan_lifecycle()) ++passed; if (test_round_trip()) ++passed; if (test_octree_round_trip()) ++passed; + if (test_twolevel_round_trip()) ++passed; if (test_cr_vs_cpu()) ++passed; if (test_varying_scale()) ++passed; if (test_multiple_cycles()) ++passed; diff --git a/tests/test_quant_flip_amplification_hip.cpp b/tests/test_quant_flip_amplification_hip.cpp new file mode 100644 index 0000000..9baaa27 --- /dev/null +++ b/tests/test_quant_flip_amplification_hip.cpp @@ -0,0 +1,349 @@ +// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Use of this source code is governed by an MIT-style license that can be +// found in the LICENSE file or at https://opensource.org/licenses/MIT. + +// --------------------------------------------------------------------------- +// Quantization flip-amplification test. +// +// Hypothesis under test (CPU vs GPU gradient divergence): +// The CPU (production AVX ds79) and GPU (ds79 on device) forward wavelet +// transforms differ only at fp32-rounding level (call it dc). That tiny +// coefficient difference is then AMPLIFIED by quantization: with +// mulfac = 1/(rms*scale) and ival = (int)(mulfac*coef), the quant step in +// decoded units is Delta = rms*scale. A coefficient sitting near a bin +// edge truncates to integers that differ by one between CPU and GPU, so the +// decoded value jumps by a full Delta regardless of how small dc is. Since +// Delta grows linearly with `scale`, the same fixed dc produces a larger +// decoded perturbation as compression coarsens. +// +// This test proves the chain end-to-end at the coefficient level: +// Stage 1 measure dc = ||c_cpu - c_gpu|| / ||c_cpu|| (expect ~fp32) +// Stage 2 control: identical coefficients -> zero flips at every scale +// Stage 3 quantize c_cpu vs c_gpu over a scale sweep and show +// (a) flips appear and the decoded error is composed of +/- k*Delta +// steps (the flip signature), +// (b) flip fraction ~ mean|dc|/(rms*scale) -> falls with scale, +// (c) decoded ||error|| grows with scale. +// +// The GPU transform here is hipWaveletTransformBufferZYX, which shares the +// ds79_forward_* routines with the production fused kernel (hipWaveletRLE.h); +// the quantization formula matches Run_Length_Encode_Slow.cpp / hipWaveletRLE.h +// exactly ((int) truncation toward zero of mulfac*coef). +// +// Build: make test_quant_flip_amplification_hip HIP_ARCH=gfx942 +// Run: ./build/test_quant_flip_amplification_hip [NX NY NZ] +// --------------------------------------------------------------------------- + +#include +#include +#include +#include +#include +#include +#include +#include +#include "hipWaveletRLE.h" // production fused kernel + hipWaveletRLEFusedDumpCoef + +#define HIPCHECK(cmd) do { \ + hipError_t err = (cmd); \ + if (err != hipSuccess) { \ + fprintf(stderr, "HIP error at %s:%d: %s\n", __FILE__, __LINE__, hipGetErrorString(err)); \ + exit(1); \ + } \ +} while(0) + +// --------------------------------------------------------------------------- +// CPU reference: Ds79 with stride (from Wavelet_Transform_Slow.cpp, identical +// to the copy used in test_wavelet_buffer_hip.cpp). +// --------------------------------------------------------------------------- +#define al0 8.526986790094000e-001f +#define al1 3.774028556126500e-001f +#define al2 -1.106244044184200e-001f +#define al3 -2.384946501938001e-002f +#define al4 3.782845550699501e-002f +#define ah0 7.884856164056601e-001f +#define ah1 -4.180922732222101e-001f +#define ah2 -4.068941760955800e-002f +#define ah3 6.453888262893799e-002f + +static inline int MIRR(int val, int dim) { + val = val < 0 ? -val : val; + val = (val >= dim) ? (2*dim-2-val) : val; + val = val < 0 ? -val : val; + val = (val >= dim) ? (2*dim-2-val) : val; + return val; +} + +static void Ds79(float* p_in, float* p_tmp, int stride, int dim) { + for (int n = dim; n >= 2; n = n - n/2) { + for (int i = 0; i < n; ++i) p_tmp[i] = p_in[i*stride]; + int nh = n / 2; + int nl = n - nh; + for (int ix = 0; ix < nl; ++ix) { + int i0 = 2*ix; + int im1 = MIRR(i0-1,n), ip1 = MIRR(i0+1,n); + int im2 = MIRR(i0-2,n), ip2 = MIRR(i0+2,n); + int im3 = MIRR(i0-3,n), ip3 = MIRR(i0+3,n); + int im4 = MIRR(i0-4,n), ip4 = MIRR(i0+4,n); + p_in[ix*stride] = al0*p_tmp[i0] + + al1*(p_tmp[im1]+p_tmp[ip1]) + + al2*(p_tmp[im2]+p_tmp[ip2]) + + al3*(p_tmp[im3]+p_tmp[ip3]) + + al4*(p_tmp[im4]+p_tmp[ip4]); + } + for (int ix = 0; ix < nh; ++ix) { + int i0 = 2*ix + 1; + int im1 = MIRR(i0-1,n), ip1 = MIRR(i0+1,n); + int im2 = MIRR(i0-2,n), ip2 = MIRR(i0+2,n); + int im3 = MIRR(i0-3,n), ip3 = MIRR(i0+3,n); + p_in[(nl+ix)*stride] = ah0*p_tmp[i0] + + ah1*(p_tmp[im1]+p_tmp[ip1]) + + ah2*(p_tmp[im2]+p_tmp[ip2]) + + ah3*(p_tmp[im3]+p_tmp[ip3]); + } + } +} + +static void cpu_wavelet_forward_z(float* data, int nx, int ny, int nz, int bz) { + float tmp[256]; + #pragma omp parallel for collapse(3) firstprivate(tmp) schedule(static) + for (int bzi = 0; bzi < nz/bz; ++bzi) + for (int gy = 0; gy < ny; ++gy) + for (int gx = 0; gx < nx; ++gx) + Ds79(data + bzi*bz*(size_t)(nx*ny) + gy*nx + gx, + tmp, nx*ny, bz); +} + +static void cpu_wavelet_forward_y(float* data, int nx, int ny, int nz, int by) { + float tmp[256]; + #pragma omp parallel for collapse(3) firstprivate(tmp) schedule(static) + for (int byi = 0; byi < ny/by; ++byi) + for (int gz = 0; gz < nz; ++gz) + for (int gx = 0; gx < nx; ++gx) + Ds79(data + byi*by*(size_t)nx + gz*(size_t)(nx*ny) + gx, + tmp, nx, by); +} + +static void cpu_wavelet_forward_x(float* data, int nx, int ny, int nz, int bx) { + float tmp[256]; + #pragma omp parallel for collapse(3) firstprivate(tmp) schedule(static) + for (int bxi = 0; bxi < nx/bx; ++bxi) + for (int gz = 0; gz < nz; ++gz) + for (int gy = 0; gy < ny; ++gy) + Ds79(data + bxi*bx + gy*(size_t)nx + gz*(size_t)(nx*ny), + tmp, 1, bx); +} + +// --------------------------------------------------------------------------- +// Deterministic, smooth-ish test field: superposition of a few sinusoids plus +// a small broadband component. Gives a realistic spread of wavelet-coefficient +// magnitudes (lots of near-zero high-band coefficients, a few large ones), so +// the near-bin-edge population is representative of real snapshots. +// --------------------------------------------------------------------------- +static void make_field(std::vector& F, int nx, int ny, int nz) { + F.resize((size_t)nx*ny*nz); + const double kPi = 3.14159265358979323846; + for (int k = 0; k < nz; ++k) { + double z = (double)k / nz; + for (int j = 0; j < ny; ++j) { + double y = (double)j / ny; + for (int i = 0; i < nx; ++i) { + double x = (double)i / nx; + double v = 1.0 * sin(2*kPi*3*x) * sin(2*kPi*2*y) * cos(2*kPi*1*z) + + 0.5 * sin(2*kPi*11*x + 0.7) * cos(2*kPi*7*y) + + 0.25 * cos(2*kPi*19*x) * sin(2*kPi*13*z + 0.3); + // small deterministic broadband ripple + unsigned h = (unsigned)(i*73856093 ^ j*19349663 ^ k*83492791); + double noise = ((h % 2048) / 2048.0 - 0.5) * 0.02; + F[(size_t)k*nx*ny + (size_t)j*nx + i] = (float)(v + noise); + } + } + } +} + +static double raw_rms(const std::vector& F) { + double acc = 0.0; + for (float v : F) { double d = (double)v; acc += d*d; } + return sqrt(acc / (double)F.size()); +} + +// Production quantization: truncation toward zero of mulfac*coef. +static inline int quantize(float coef, float mulfac) { + return (int)(mulfac * coef); +} + +// --------------------------------------------------------------------------- +static bool run_case(int NX, int NY, int NZ) { + const int BX = 32, BY = 32, BZ = 32; + if (NX % BX || NY % BY || NZ % BZ) { + std::printf(" dims must be multiples of 32; skipping %dx%dx%d\n", NX, NY, NZ); + return false; + } + const size_t N = (size_t)NX * NY * NZ; + const size_t BYTES = N * sizeof(float); + + std::printf("\n==================================================================\n"); + std::printf("Case %dx%dx%d (%zu samples, block 32^3)\n", NX, NY, NZ, N); + std::printf("==================================================================\n"); + + // ---- input field (host, deterministic) ---- + std::vector F; + make_field(F, NX, NY, NZ); + const double rms = raw_rms(F); + std::printf(" raw field RMS = %.6e\n", rms); + + // ---- GPU forward ZYX via the PRODUCTION fused kernel ---- + // hipWaveletRLEFusedDumpCoef runs the exact production waveletRLEFusedKernel + // (Phase 1-3 wavelet + Phase 4 quant/RLE) and additionally writes the + // pre-quant ZYX coefficients to d_coef. The RLE outputs are discarded. + // The 'scale' arg is used directly as mulfac here (d_rms path unused); the + // dumped coefficients are pre-quant and independent of it. + const int nbx = NX/32, nby = NY/32, nbz = NZ/32; + const int nblocks = nbx * nby * nbz; + const size_t scratch_bytes = (size_t)nblocks * 4 * WRLE_LDS_BYTES; + + float* d_in = nullptr; + float* d_coef = nullptr; + unsigned char* d_scratch = nullptr; + size_t* d_bsz = nullptr; + HIPCHECK(hipMalloc(&d_in, BYTES)); + HIPCHECK(hipMalloc(&d_coef, BYTES)); + HIPCHECK(hipMalloc(&d_scratch, scratch_bytes)); + HIPCHECK(hipMalloc(&d_bsz, (size_t)nblocks * sizeof(size_t))); + HIPCHECK(hipMemcpy(d_in, F.data(), BYTES, hipMemcpyHostToDevice)); + + HIPCHECK(hipWaveletRLEFusedDumpCoef( + d_in, d_scratch, d_bsz, d_coef, + /*scale(mulfac)=*/1.0f, NX, NY, NZ, NX, NX*NY)); + HIPCHECK(hipDeviceSynchronize()); + + std::vector c_gpu(N); + HIPCHECK(hipMemcpy(c_gpu.data(), d_coef, BYTES, hipMemcpyDeviceToHost)); + HIPCHECK(hipFree(d_in)); + HIPCHECK(hipFree(d_coef)); + HIPCHECK(hipFree(d_scratch)); + HIPCHECK(hipFree(d_bsz)); + + // ---- CPU forward ZYX (same layout / block order) ---- + std::vector c_cpu(F); + cpu_wavelet_forward_z(c_cpu.data(), NX, NY, NZ, BZ); + cpu_wavelet_forward_y(c_cpu.data(), NX, NY, NZ, BY); + cpu_wavelet_forward_x(c_cpu.data(), NX, NY, NZ, BX); + + // ---- Stage 1: coefficient difference dc ---- + double num = 0.0, den = 0.0, maxabs = 0.0, sum_abs = 0.0; + for (size_t i = 0; i < N; ++i) { + double a = (double)c_cpu[i], b = (double)c_gpu[i]; + double d = a - b; + num += d*d; den += a*a; + double ad = fabs(d); + if (ad > maxabs) maxabs = ad; + sum_abs += ad; + } + double dc_relL2 = den > 0 ? sqrt(num/den) : 0.0; + double mean_abs_dc = sum_abs / (double)N; + std::printf("\n Stage 1 - coefficient difference (pre-quant):\n"); + std::printf(" ||c_cpu - c_gpu|| / ||c_cpu|| = %.3e\n", dc_relL2); + std::printf(" max|dc| = %.3e mean|dc| = %.3e\n", maxabs, mean_abs_dc); + bool stage1_ok = (dc_relL2 < 1e-4); // fp32-rounding level + std::printf(" -> %s (coefficients agree at fp32-rounding level)\n", + stage1_ok ? "OK" : "UNEXPECTEDLY LARGE"); + + // ---- Stage 2: identical-coefficient control (quant determinism) ---- + // Feeding identical coefficients to the quantizer must yield zero flips at + // every scale; this isolates the drift to dc, not quant nondeterminism. + bool control_ok = true; + const double scales[] = {1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2, 1e-1, 3e-1}; + const int NS = (int)(sizeof(scales)/sizeof(scales[0])); + for (int s = 0; s < NS && control_ok; ++s) { + float mulfac = (float)(1.0 / (rms * scales[s])); + for (size_t i = 0; i < N; ++i) { + if (quantize(c_cpu[i], mulfac) != quantize(c_cpu[i], mulfac)) { control_ok = false; break; } + } + } + std::printf("\n Stage 2 - identical-coefficient control: %s (0 flips expected)\n", + control_ok ? "PASS" : "FAIL"); + + // ---- Stage 3: quantize c_cpu vs c_gpu over a scale sweep ---- + std::printf("\n Stage 3 - flip amplification (quantize c_cpu vs c_gpu):\n"); + std::printf(" %-8s %-10s %-11s %-12s %-12s %-10s %-8s\n", + "scale", "Delta", "flip_frac", "dec_relL2", "dec_L2abs", + "predict", "steps"); + bool stage3_ok = true; + double prev_flip_frac = -1.0; + double prev_dec_L2 = -1.0; + bool flips_grow = true, fracs_fall = true; + for (int s = 0; s < NS; ++s) { + double scale = scales[s]; + double Delta = rms * scale; // decoded units per level + float mulfac = (float)(1.0 / Delta); + + long long flips = 0; + long long h0 = 0, hp1 = 0, hm1 = 0, hp2 = 0, hm2 = 0, hbig = 0; + double dnum = 0.0, dden = 0.0; // decoded relL2 (Delta cancels) + bool steps_exact = true; + for (size_t i = 0; i < N; ++i) { + int qc = quantize(c_cpu[i], mulfac); + int qg = quantize(c_gpu[i], mulfac); + int d = qc - qg; + if (d != 0) ++flips; + switch (d) { + case 0: ++h0; break; + case 1: ++hp1; break; + case -1: ++hm1; break; + case 2: ++hp2; break; + case -2: ++hm2; break; + default: ++hbig; break; + } + dnum += (double)d * (double)d; + dden += (double)qc * (double)qc; + + // Flip signature: decoded error must be an exact integer multiple + // of Delta. Reconstruct as floats and check. + double err = (double)qc * Delta - (double)qg * Delta; + double levels = err / Delta; + if (fabs(levels - std::round(levels)) > 1e-3) steps_exact = false; + } + double flip_frac = (double)flips / (double)N; + double dec_relL2 = dden > 0 ? sqrt(dnum/dden) : 0.0; + double dec_L2abs = sqrt(dnum) * Delta; // absolute decoded L2 error + // predicted absolute decoded L2 ~ sqrt(mean|dc| * Delta) * sqrt(N) + double predict = sqrt(mean_abs_dc * Delta) * sqrt((double)N); + + std::printf(" %-8.0e %-10.3e %-11.3e %-12.3e %-12.3e %-10.3e %-8s\n", + scale, Delta, flip_frac, dec_relL2, dec_L2abs, predict, + steps_exact ? "exact" : "NONINT"); + + if (!steps_exact) stage3_ok = false; + if (prev_flip_frac >= 0.0) { + if (flip_frac > prev_flip_frac + 1e-12) fracs_fall = false; + if (dec_L2abs < prev_dec_L2 - 1e-9) flips_grow = false; + } + prev_flip_frac = flip_frac; + prev_dec_L2 = dec_L2abs; + // per-scale histogram of q differences + std::printf(" q-diff histogram: 0=%lld +1=%lld -1=%lld +2=%lld -2=%lld |>2|=%lld\n", + h0, hp1, hm1, hp2, hm2, hbig); + } + + std::printf("\n Trend: flip fraction falls with scale = %s ; " + "decoded L2 grows with scale = %s\n", + fracs_fall ? "yes" : "no", flips_grow ? "yes" : "no"); + + bool ok = stage1_ok && control_ok && stage3_ok; + std::printf("\n CASE %s\n", ok ? "PASS" : "FAIL"); + return ok; +} + +int main(int argc, char** argv) { + std::printf("=== Quantization flip-amplification test (CPU ds79 vs GPU ds79) ===\n"); + bool all = true; + if (argc >= 4) { + all &= run_case(atoi(argv[1]), atoi(argv[2]), atoi(argv[3])); + } else { + all &= run_case(128, 128, 128); + all &= run_case(256, 128, 64); + } + std::printf("\n%s\n", all ? "ALL PASS" : "SOME FAIL"); + return all ? 0 : 1; +} From 2f508abd3501b8b24282d3ef20958227ce61e1af Mon Sep 17 00:00:00 2001 From: Ossian O'Reilly Date: Wed, 19 Aug 2026 13:09:12 -0700 Subject: [PATCH 5/7] Bump AMD copyright year to 2026; fill header gap; refresh docs Bump all AMD source-header and doc-footer copyrights in CvxCompress from 2025 to 2026. Add the missing MIT header to tests/test_compress_2d_hip.cpp. Refresh HIP_API.md and README.md for the current state: auto-select default (octree 3D / quadtree 2D), 2D support, gfx950/MI355X, and ROCm 7.2.1. --- HIP_API.md | 28 ++-- OCTREE_PFOR_THROUGHPUT.md | 135 -------------------- README.md | 7 +- hip/ds79.h | 2 +- hip/ds79_f4_reg32.inc | 2 +- hip/ds79_reg32.inc | 2 +- hip/hipBlockCopy.h | 2 +- hip/hipCompress.cpp | 2 +- hip/hipCompress.h | 2 +- hip/hipQuantizeRLE.h | 2 +- hip/hipRLEDecode.h | 2 +- hip/hipSegmentedRLE.h | 2 +- hip/hipWaveletBitmap.h | 2 +- hip/hipWaveletOctree.h | 2 +- hip/hipWaveletQuadtree2D.h | 2 +- hip/hipWaveletRLE.h | 2 +- hip/hipWaveletRLE2D.h | 2 +- hip/hipWaveletRLEInverse.h | 2 +- hip/hipWaveletTransformBlocked.cpp | 2 +- hip/hipWaveletTransformBlocked.h | 2 +- hip/hipWaveletTransformBuffer.cpp | 2 +- hip/hipWaveletTransformBuffer.h | 2 +- hip/quantize_rle_ref.h | 2 +- hip/us79_reg32.inc | 2 +- tests/bench_quadtree_vs_cvx_2d.cpp | 2 +- tests/example_async_pipeline.cpp | 2 +- tests/hip_test_helpers.h | 2 +- tests/test_bitmap_code_hip.cpp | 2 +- tests/test_bitmap_encode_hip.cpp | 2 +- tests/test_bitmap_octree_hip.cpp | 2 +- tests/test_bitmap_rd_panel.cpp | 2 +- tests/test_bitmap_vs_cpu_rle.cpp | 2 +- tests/test_compress_2d_hip.cpp | 4 + tests/test_compress_api_hip.cpp | 2 +- tests/test_inverse_fused_hip.cpp | 2 +- tests/test_inverse_wavelet_hip.cpp | 2 +- tests/test_quant_flip_amplification_hip.cpp | 2 +- tests/test_quantize_rle.cpp | 2 +- tests/test_quantize_rle_hip.cpp | 2 +- tests/test_quantize_rle_perf_hip.cpp | 2 +- tests/test_rle_decode_hip.cpp | 2 +- tests/test_seg_rle_minimal.cpp | 2 +- tests/test_wavelet_buffer_hip.cpp | 2 +- tests/test_wavelet_rle_fused_hip.cpp | 2 +- tests/test_zline_cr_benchmark.cpp | 2 +- 45 files changed, 64 insertions(+), 192 deletions(-) delete mode 100644 OCTREE_PFOR_THROUGHPUT.md diff --git a/HIP_API.md b/HIP_API.md index 581eae8..bdadf52 100644 --- a/HIP_API.md +++ b/HIP_API.md @@ -6,11 +6,13 @@ > format. Compression ratios differ from the CPU reference due to different > block tiling strategies. -GPU-accelerated lossy compression for 3D floating-point volumes on AMD Instinct -GPUs (MI200, MI300). Targets seismic imaging workloads where wavefield snapshots -must be stored and retrieved at GPU memory bandwidth. +GPU-accelerated lossy compression for 2D and 3D floating-point volumes on AMD +Instinct GPUs (MI200, MI300, MI355). Targets seismic imaging workloads where +wavefield snapshots must be stored and retrieved at GPU memory bandwidth. -Single fused kernel: wavelet transform (DS 7/9) → quantization → RLE encoding. +Single fused kernel: wavelet transform (DS 7/9) → quantization → significance/RLE +coding. The coder is selectable per plan (see Kernel Variants); the default +resolves to octree for 3D and quadtree for 2D. Error norms match the CPU reference (CvxCompress) to floating-point rounding. ### Performance (MI300X vs 128-core EPYC 9554, AVX, best thread count) @@ -27,14 +29,14 @@ time rather than end-to-end latency. ## Requirements -- ROCm 7.x (`module load rocm/7.2.0`) -- AMD GPU: gfx90a (MI200) or gfx942 (MI300X) +- ROCm 7.x (`module load rocm/7.2.1`) +- AMD GPU: gfx90a (MI200), gfx942 (MI300X), or gfx950 (MI355X) - C++17, `hipcc`, `rocprim` ## Building ```bash -module load rocm/7.2.0 +module load rocm/7.2.1 # Build the CPU reference library (needed by tests) make libcvxcompress.so @@ -71,9 +73,9 @@ All functions are declared in [`hip/hipCompress.h`](hip/hipCompress.h). | Function | Description | |----------|-------------| -| `hipCompress` | Wavelet + quantize + RLE encode → self-contained compressed stream (async) | +| `hipCompress` | Wavelet + quantize + encode (per-plan codec) → self-contained compressed stream (async) | | `hipCompressSynchronize` | Block until compress completes, retrieve compressed length and CR | -| `hipDecompress` | RLE decode + inverse wavelet → wavelet buffer (single kernel, async) | +| `hipDecompress` | Decode (per-plan codec) + inverse wavelet → wavelet buffer (single kernel, async) | ### Utilities @@ -224,7 +226,7 @@ For **encode-bound** paths that compress on a hot loop — e.g. per-timestep RTM checkpoint spilling — prefer `ZLINE`, which has the highest encode throughput. The octree/quadtree encode cost over z-line is small at production grid sizes (~1–3% at 512³) but grows at small grids where the per-block histogram, scans, -and PFOR bookkeeping are not amortized (see `OCTREE_PFOR_THROUGHPUT.md`). +and PFOR bookkeeping are not amortized. **Memory footprint.** Octree/quadtree allocate a larger per-block scratch stride (`WOCT_CODE_SLOT_BYTES` ~140 KB/block plus the bitmap scratch) than z-line; @@ -260,8 +262,8 @@ loader plus the runtime dispatch pick the correct one for the GPU in use. ### Two-Stream Model -- **`user_stream`**: passed to each API call. Wavelet transform and RLE encoding - run here. The stream is free immediately after `hipCompress` returns. +- **`user_stream`**: passed to each API call. The wavelet transform and value + encoding run here. The stream is free immediately after `hipCompress` returns. - **`aux_stream`**: owned by the user, passed at plan creation. Compaction, header writing, and D2H readback run here. Shared across plans. @@ -313,5 +315,5 @@ tests/ ## License -Copyright (C) 2025 Advanced Micro Devices, Inc. Licensed under the +Copyright (C) 2026 Advanced Micro Devices, Inc. Licensed under the [MIT License](https://opensource.org/licenses/MIT). diff --git a/OCTREE_PFOR_THROUGHPUT.md b/OCTREE_PFOR_THROUGHPUT.md deleted file mode 100644 index 59b2cdd..0000000 --- a/OCTREE_PFOR_THROUGHPUT.md +++ /dev/null @@ -1,135 +0,0 @@ -# Octree + PFOR vs RLE: throughput across sparsity - -GPU encode/decode throughput of the **previous RLE codec** (`HIP_COMPRESS_KERNEL_ZLINE`, -the default z-line RLE) versus the **octree significance + per-block PFOR value -coder**, swept across coefficient sparsity at matched fidelity. - -## Method - -- **Data**: `solver_steps_512_marmousi_h20/snapshot_u_001000.raw`, a 128³ crop at - origin `(z0=0, y0=192, x0=192)` — a wavefront region (~33% of voxels above - 1e-3·max in the raw field). Normalized to unit RMS. -- **Sparsity axis**: quantization multiplier `--scale` (mulfac). Higher mulfac → - finer quantization → more nonzero coefficients → *denser* (lower CR). Fused CR - is used as the density proxy. -- **Matched fidelity**: both codecs share the wavelet transform + quantization, so - `vol_rel_l2` is identical at every point (verified per row below). -- **Harness**: `tests/test_bitmap_octree_hip.cpp`, `--nx 128 --iters 100`, raw GB/s - over the f32 volume. Encode octree = k1 (wavelet) + k2 (PFOR); decode octree = - stage-A (PFOR value unpack) + stage-B (inverse wavelet). -- **Value round-trip** validated format-agnostically (`decode: stage-A round-trip - vs kernel-1 = OK`) at every point. - -Command: - -```bash -for MF in 1 2 5 10 20 40 80; do - ./build/test_bitmap_octree_hip --panel $PANEL --nx 128 --z0 0 --y0 192 --x0 192 \ - --scale $MF --iters 100 -done -``` - -## MI300X (gfx942) - -Full-pipeline throughput (raw GB/s); `oct` = octree+PFOR. - -| mulfac | rel_l2 | fused CR | enc RLE | enc oct | enc oct/RLE | dec RLE | dec oct | dec oct/RLE | -|-------:|-------:|---------:|--------:|--------:|:-----------:|--------:|--------:|:-----------:| -| 1 | 1.32e-1 | 166 | 167.6 | 172.4 | 1.03× | 97.1 | 190.3 | 1.96× | -| 2 | 8.22e-2 | 98 | 162.1 | 165.2 | 1.02× | 90.4 | 166.5 | 1.84× | -| 5 | 4.25e-2 | 53 | 155.1 | 151.2 | 0.97× | 80.2 | 151.6 | 1.89× | -| 10 | 2.51e-2 | 35 | 151.9 | 141.0 | 0.93× | 76.6 | 141.9 | 1.85× | -| 20 | 1.45e-2 | 25 | 148.9 | 134.1 | 0.90× | 73.4 | 136.2 | 1.86× | -| 40 | 8.13e-3 | 18 | 146.0 | 122.4 | 0.84× | 69.8 | 129.8 | 1.86× | -| 80 | 4.46e-3 | 14 | 142.5 | 113.5 | 0.80× | 67.8 | 124.2 | 1.83× | - -Isolated value-coder kernels (raw GB/s): `FUSED` = k2 PFOR encode, `PAR_dec` = -stage-A PFOR decode. - -| mulfac | fused CR | FUSED (k2 enc) | PAR_dec (stage-A) | -|-------:|---------:|---------------:|------------------:| -| 1 | 166 | 369.9 | 755.4 | -| 2 | 98 | 339.5 | 747.6 | -| 5 | 53 | 286.2 | 740.7 | -| 10 | 35 | 253.5 | 735.0 | -| 20 | 25 | 238.6 | 728.8 | -| 40 | 18 | 215.8 | 725.1 | -| 80 | 14 | 203.6 | 725.2 | - -### Trends - -- **Decode**: octree+PFOR is **1.8–2.0× faster than RLE at every sparsity**, and - the ratio is nearly flat. RLE decode is the bottleneck (serial run expansion, - 97→68 GB/s as density rises). -- **Encode**: octree's edge is largest at nnz≈0 (1.03× at CR 166) and erodes - monotonically to 0.80× at the densest point — as expected, since PFOR must code - every nonzero while RLE encode is already bandwidth-bound (167→143, density-flat). - Encode crossover is around CR≈60 (mulfac≈5). -- **PFOR decode is density-insensitive**: stage-A only drops 755→725 GB/s (−4%) - across a ~12× CR range, because every stream is fixed-width / byte-aligned (no - per-nonzero branching). Encode-k2 scales with nnz (370→204) — the histogram + - exception scan + mask cost. That asymmetry is intentional: spend a little on - encode to keep decode fast and flat. - -Net: octree+PFOR wins decode decisively at all sparsities and wins encode only in -the sparse regime; in the dense regime it trades encode throughput for the CR gain -(and the large decode advantage). - -## MI355X (gfx950) - -Same data, crop, and sweep; CR and `rel_l2` are identical to MI300X (same -algorithm). Full-pipeline throughput (raw GB/s); `oct` = octree+PFOR. - -| mulfac | rel_l2 | fused CR | enc RLE | enc oct | enc oct/RLE | dec RLE | dec oct | dec oct/RLE | -|-------:|-------:|---------:|--------:|--------:|:-----------:|--------:|--------:|:-----------:| -| 1 | 1.32e-1 | 166 | 184.6 | 184.1 | 1.00× | 109.5 | 207.3 | 1.89× | -| 2 | 8.22e-2 | 98 | 177.2 | 175.8 | 0.99× | 101.0 | 181.8 | 1.80× | -| 5 | 4.25e-2 | 53 | 169.2 | 161.0 | 0.95× | 89.7 | 166.3 | 1.85× | -| 10 | 2.51e-2 | 35 | 165.8 | 152.3 | 0.92× | 84.4 | 156.4 | 1.85× | -| 20 | 1.45e-2 | 25 | 162.3 | 142.9 | 0.88× | 81.0 | 150.3 | 1.85× | -| 40 | 8.13e-3 | 18 | 159.7 | 135.0 | 0.85× | 76.9 | 143.4 | 1.86× | -| 80 | 4.46e-3 | 14 | 155.8 | 126.0 | 0.81× | 75.2 | 137.1 | 1.82× | - -Isolated value-coder kernels (raw GB/s): `FUSED` = k2 PFOR encode, `PAR_dec` = -stage-A PFOR decode. - -| mulfac | fused CR | FUSED (k2 enc) | PAR_dec (stage-A) | -|-------:|---------:|---------------:|------------------:| -| 1 | 166 | 387.6 | 823.2 | -| 2 | 98 | 352.1 | 809.6 | -| 5 | 53 | 298.5 | 804.8 | -| 10 | 35 | 270.0 | 801.0 | -| 20 | 25 | 244.6 | 796.6 | -| 40 | 18 | 229.3 | 792.6 | -| 80 | 14 | 211.9 | 789.6 | - -### Trends - -Identical qualitative behavior to MI300X, uniformly faster (higher clocks/BW): - -- **Decode**: octree+PFOR is **1.8–1.9× faster than RLE at every sparsity**; ratio - nearly flat. RLE decode 110→75 GB/s as density rises. -- **Encode**: octree edge highest at nnz≈0 (1.00× at CR 166), eroding to 0.81× at - the densest point; crossover near CR≈55 (mulfac≈5). RLE encode density-flat - (185→156). -- **PFOR decode is density-insensitive**: stage-A 823→790 GB/s (−4%) across the CR - range; encode-k2 scales with nnz (388→212). - -### MI355X vs MI300X - -MI355X is ~1.1–1.3× faster on every metric at matched work; the RLE-vs-octree -relationship (decode win everywhere, encode win only when sparse) is unchanged. - -| metric (CR 35 / mulfac 10) | MI300X | MI355X | × | -|----------------------------|-------:|-------:|-----:| -| octree decode (full) | 141.9 | 156.4 | 1.10 | -| octree encode (full) | 141.0 | 152.3 | 1.08 | -| PFOR decode (stage-A) | 735.0 | 801.0 | 1.09 | -| PFOR encode (k2) | 253.5 | 270.0 | 1.07 | - -## Provenance - -- Hardware: MI300X (gfx942, TheraC16), MI355X (gfx950, TheraC79); ROCm 7.2.1. -- Build: `make test_bitmap_octree_hip HIP_ARCH=`. -- All rows: `decode: stage-A round-trip vs kernel-1 = OK` (lossless PFOR recode). -- Date: 2026-08-18. diff --git a/README.md b/README.md index 3a5bf33..0587bd3 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,13 @@ https://library.seg.org/doi/pdf/10.1190/1.1826518 > - Optimizations are ongoing > - Not tested in an integrated production setting > - No backward compatibility guarantee for the compressed bitstream format -> - AMD GPUs only (ROCm 7.x, gfx90a/gfx942) +> - AMD GPUs only (ROCm 7.x, gfx90a/gfx942/gfx950) > - Compression ratios differ from the CPU reference due to different block tiling strategies See [HIP_API.md](HIP_API.md) for the AMD GPU port of this library targeting -MI200/MI300X via HIP. It implements a fused wavelet + quantization + RLE pipeline -in single GPU kernels, achieving 14–27x speedup over the fully-parallelized CPU +MI200/MI300X/MI355X via HIP. It implements a fused wavelet + quantization + +significance/RLE coding pipeline in single GPU kernels (default: octree for 3D, +quadtree for 2D), achieving 14–27x speedup over the fully-parallelized CPU reference (128-core EPYC 9554, AVX, best thread count) with matching error norms (to floating-point rounding). Includes API reference, usage examples, and async pipeline integration. diff --git a/hip/ds79.h b/hip/ds79.h index 13d83da..11447e1 100644 --- a/hip/ds79.h +++ b/hip/ds79.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/ds79_f4_reg32.inc b/hip/ds79_f4_reg32.inc index 213c875..e1bc3ee 100644 --- a/hip/ds79_f4_reg32.inc +++ b/hip/ds79_f4_reg32.inc @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/ds79_reg32.inc b/hip/ds79_reg32.inc index cce17b8..e9a0328 100644 --- a/hip/ds79_reg32.inc +++ b/hip/ds79_reg32.inc @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/hipBlockCopy.h b/hip/hipBlockCopy.h index 93ce222..15d14b1 100644 --- a/hip/hipBlockCopy.h +++ b/hip/hipBlockCopy.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/hipCompress.cpp b/hip/hipCompress.cpp index cf9c990..aaa024b 100644 --- a/hip/hipCompress.cpp +++ b/hip/hipCompress.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/hipCompress.h b/hip/hipCompress.h index 7a6d4ed..8f7ebcb 100644 --- a/hip/hipCompress.h +++ b/hip/hipCompress.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/hipQuantizeRLE.h b/hip/hipQuantizeRLE.h index 7691d60..f582a3e 100644 --- a/hip/hipQuantizeRLE.h +++ b/hip/hipQuantizeRLE.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/hipRLEDecode.h b/hip/hipRLEDecode.h index d2199f4..3346718 100644 --- a/hip/hipRLEDecode.h +++ b/hip/hipRLEDecode.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/hipSegmentedRLE.h b/hip/hipSegmentedRLE.h index f6abb48..ce4ab3b 100644 --- a/hip/hipSegmentedRLE.h +++ b/hip/hipSegmentedRLE.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/hipWaveletBitmap.h b/hip/hipWaveletBitmap.h index a840cd2..7e202a9 100644 --- a/hip/hipWaveletBitmap.h +++ b/hip/hipWaveletBitmap.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/hipWaveletOctree.h b/hip/hipWaveletOctree.h index 46648a3..f62e48f 100644 --- a/hip/hipWaveletOctree.h +++ b/hip/hipWaveletOctree.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. // diff --git a/hip/hipWaveletQuadtree2D.h b/hip/hipWaveletQuadtree2D.h index c65bdac..b94c405 100644 --- a/hip/hipWaveletQuadtree2D.h +++ b/hip/hipWaveletQuadtree2D.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. // diff --git a/hip/hipWaveletRLE.h b/hip/hipWaveletRLE.h index 81090a1..5edc878 100644 --- a/hip/hipWaveletRLE.h +++ b/hip/hipWaveletRLE.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/hipWaveletRLE2D.h b/hip/hipWaveletRLE2D.h index 9e9230e..8dc3d5d 100644 --- a/hip/hipWaveletRLE2D.h +++ b/hip/hipWaveletRLE2D.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/hipWaveletRLEInverse.h b/hip/hipWaveletRLEInverse.h index 708815e..bd966ad 100644 --- a/hip/hipWaveletRLEInverse.h +++ b/hip/hipWaveletRLEInverse.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/hipWaveletTransformBlocked.cpp b/hip/hipWaveletTransformBlocked.cpp index 239a280..2a3477c 100644 --- a/hip/hipWaveletTransformBlocked.cpp +++ b/hip/hipWaveletTransformBlocked.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/hipWaveletTransformBlocked.h b/hip/hipWaveletTransformBlocked.h index 13af505..bfc29a3 100644 --- a/hip/hipWaveletTransformBlocked.h +++ b/hip/hipWaveletTransformBlocked.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/hipWaveletTransformBuffer.cpp b/hip/hipWaveletTransformBuffer.cpp index f09d019..5102215 100644 --- a/hip/hipWaveletTransformBuffer.cpp +++ b/hip/hipWaveletTransformBuffer.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/hipWaveletTransformBuffer.h b/hip/hipWaveletTransformBuffer.h index 48a56f0..ded9a38 100644 --- a/hip/hipWaveletTransformBuffer.h +++ b/hip/hipWaveletTransformBuffer.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/quantize_rle_ref.h b/hip/quantize_rle_ref.h index 255215b..f0ba457 100644 --- a/hip/quantize_rle_ref.h +++ b/hip/quantize_rle_ref.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/hip/us79_reg32.inc b/hip/us79_reg32.inc index 2add087..0a06f72 100644 --- a/hip/us79_reg32.inc +++ b/hip/us79_reg32.inc @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/tests/bench_quadtree_vs_cvx_2d.cpp b/tests/bench_quadtree_vs_cvx_2d.cpp index c63d761..48efd05 100644 --- a/tests/bench_quadtree_vs_cvx_2d.cpp +++ b/tests/bench_quadtree_vs_cvx_2d.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. // diff --git a/tests/example_async_pipeline.cpp b/tests/example_async_pipeline.cpp index 238bc17..a9a0c22 100644 --- a/tests/example_async_pipeline.cpp +++ b/tests/example_async_pipeline.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/tests/hip_test_helpers.h b/tests/hip_test_helpers.h index 5f52c95..4d30522 100644 --- a/tests/hip_test_helpers.h +++ b/tests/hip_test_helpers.h @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/tests/test_bitmap_code_hip.cpp b/tests/test_bitmap_code_hip.cpp index 3360296..1809392 100644 --- a/tests/test_bitmap_code_hip.cpp +++ b/tests/test_bitmap_code_hip.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. // diff --git a/tests/test_bitmap_encode_hip.cpp b/tests/test_bitmap_encode_hip.cpp index 7bf5f42..d0dbd2e 100644 --- a/tests/test_bitmap_encode_hip.cpp +++ b/tests/test_bitmap_encode_hip.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. // diff --git a/tests/test_bitmap_octree_hip.cpp b/tests/test_bitmap_octree_hip.cpp index bb66c85..106d7b9 100644 --- a/tests/test_bitmap_octree_hip.cpp +++ b/tests/test_bitmap_octree_hip.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. // diff --git a/tests/test_bitmap_rd_panel.cpp b/tests/test_bitmap_rd_panel.cpp index b21e9b6..f6565cc 100644 --- a/tests/test_bitmap_rd_panel.cpp +++ b/tests/test_bitmap_rd_panel.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. // diff --git a/tests/test_bitmap_vs_cpu_rle.cpp b/tests/test_bitmap_vs_cpu_rle.cpp index d078704..1370698 100644 --- a/tests/test_bitmap_vs_cpu_rle.cpp +++ b/tests/test_bitmap_vs_cpu_rle.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. // diff --git a/tests/test_compress_2d_hip.cpp b/tests/test_compress_2d_hip.cpp index 2f24ec3..d55c1ef 100644 --- a/tests/test_compress_2d_hip.cpp +++ b/tests/test_compress_2d_hip.cpp @@ -1,3 +1,7 @@ +// Copyright (C) 2026 Advanced Micro Devices, Inc. +// Use of this source code is governed by an MIT-style license that can be +// found in the LICENSE file or at https://opensource.org/licenses/MIT. + // 2D compression round-trip test. // Tests hipCompress API with nz=1 (2D mode): // 1. Plan lifecycle with nz=1 diff --git a/tests/test_compress_api_hip.cpp b/tests/test_compress_api_hip.cpp index feedefe..066ceb0 100644 --- a/tests/test_compress_api_hip.cpp +++ b/tests/test_compress_api_hip.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/tests/test_inverse_fused_hip.cpp b/tests/test_inverse_fused_hip.cpp index d60ce3f..36b4f9a 100644 --- a/tests/test_inverse_fused_hip.cpp +++ b/tests/test_inverse_fused_hip.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/tests/test_inverse_wavelet_hip.cpp b/tests/test_inverse_wavelet_hip.cpp index 28280f2..e5abcde 100644 --- a/tests/test_inverse_wavelet_hip.cpp +++ b/tests/test_inverse_wavelet_hip.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/tests/test_quant_flip_amplification_hip.cpp b/tests/test_quant_flip_amplification_hip.cpp index 9baaa27..cb405a5 100644 --- a/tests/test_quant_flip_amplification_hip.cpp +++ b/tests/test_quant_flip_amplification_hip.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/tests/test_quantize_rle.cpp b/tests/test_quantize_rle.cpp index 314f80b..c8777c0 100644 --- a/tests/test_quantize_rle.cpp +++ b/tests/test_quantize_rle.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/tests/test_quantize_rle_hip.cpp b/tests/test_quantize_rle_hip.cpp index b495f85..dfaed7e 100644 --- a/tests/test_quantize_rle_hip.cpp +++ b/tests/test_quantize_rle_hip.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/tests/test_quantize_rle_perf_hip.cpp b/tests/test_quantize_rle_perf_hip.cpp index 2396285..7f681c5 100644 --- a/tests/test_quantize_rle_perf_hip.cpp +++ b/tests/test_quantize_rle_perf_hip.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/tests/test_rle_decode_hip.cpp b/tests/test_rle_decode_hip.cpp index de56359..275dab7 100644 --- a/tests/test_rle_decode_hip.cpp +++ b/tests/test_rle_decode_hip.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/tests/test_seg_rle_minimal.cpp b/tests/test_seg_rle_minimal.cpp index d9b3453..83f83ab 100644 --- a/tests/test_seg_rle_minimal.cpp +++ b/tests/test_seg_rle_minimal.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/tests/test_wavelet_buffer_hip.cpp b/tests/test_wavelet_buffer_hip.cpp index 289b0f2..6f34de7 100644 --- a/tests/test_wavelet_buffer_hip.cpp +++ b/tests/test_wavelet_buffer_hip.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/tests/test_wavelet_rle_fused_hip.cpp b/tests/test_wavelet_rle_fused_hip.cpp index e90e44e..3f0b372 100644 --- a/tests/test_wavelet_rle_fused_hip.cpp +++ b/tests/test_wavelet_rle_fused_hip.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. diff --git a/tests/test_zline_cr_benchmark.cpp b/tests/test_zline_cr_benchmark.cpp index d22935f..0e4d54d 100644 --- a/tests/test_zline_cr_benchmark.cpp +++ b/tests/test_zline_cr_benchmark.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Advanced Micro Devices, Inc. +// Copyright (C) 2026 Advanced Micro Devices, Inc. // Use of this source code is governed by an MIT-style license that can be // found in the LICENSE file or at https://opensource.org/licenses/MIT. From 6841bc83422db2977fac364cc11d1848ac1e6ebe Mon Sep 17 00:00:00 2001 From: Ossian O'Reilly Date: Wed, 19 Aug 2026 15:59:00 -0700 Subject: [PATCH 6/7] Collapse codec plane I/O to a single 64-bit global-addressing path Replace the per-plane buffer-instruction loads/stores (32-bit intra-plane offset, 4 GB/plane cap) with global nontemporal load/store using size_t byte offsets, via new hip/hipPlaneIO.h. Applied across RLE/segrle forward, RLE inverse, bitmap, octree inverse, RLE2D, quadtree2D, and block-copy. - Add an alignment-tolerant float4 (aligned(4)) helper for the block-copy source-array fast paths, where arbitrary user ldimx/x0 can yield 4-byte-but-not-16-byte aligned offsets; codec/wavelet-side paths keep the 16-aligned helper (32-multiple dims guarantee alignment). - Relax the plan guard from nx*ny*4 <= 2^32 (4 GB) to nx*ny < 2^31 elements (~8 GB/plane); the remaining bound is the int plane stride. Update the CreatePlan/Copy* error-path tests to the new contract. - Remove the experimental saddr/saddr64 A-B kernels and scratch bench; the production kernels are now the single saddr64 path. Update the rle-fused test to the sole production launcher. Validated on gfx942 and gfx950: API, 2D, rle-fused, bitmap-encode, and bitmap-octree suites pass. --- hip/hipBlockCopy.h | 93 ++++++------- hip/hipCompress.cpp | 13 +- hip/hipPlaneIO.h | 83 ++++++++++++ hip/hipWaveletBitmap.h | 12 +- hip/hipWaveletOctree.h | 9 +- hip/hipWaveletQuadtree2D.h | 19 +-- hip/hipWaveletRLE.h | 196 ++------------------------- hip/hipWaveletRLE2D.h | 19 +-- hip/hipWaveletRLEInverse.h | 24 ++-- tests/test_compress_api_hip.cpp | 27 ++-- tests/test_wavelet_rle_fused_hip.cpp | 9 +- 11 files changed, 182 insertions(+), 322 deletions(-) create mode 100644 hip/hipPlaneIO.h diff --git a/hip/hipBlockCopy.h b/hip/hipBlockCopy.h index 15d14b1..60bc1bb 100644 --- a/hip/hipBlockCopy.h +++ b/hip/hipBlockCopy.h @@ -7,6 +7,7 @@ #include #include +#include "hipPlaneIO.h" using bcopy_float4_vec = __attribute__((__vector_size__(4 * sizeof(float)))) float; using bcopy_int4_vec = __attribute__((__vector_size__(4 * sizeof(int)))) int; @@ -47,23 +48,32 @@ __global__ void copyToWaveletKernelOpt( bool y_in_range = (gy < ey); - uint32_t src_row_byte = ((y0 + gy) * ldimx + x0 + gx) * (uint32_t)sizeof(float); + size_t src_row_byte = ((size_t)(y0 + gy) * ldimx + x0 + gx) * sizeof(float); + bool src_x_all_in = (gx + 3 < ex); // ---- Phase 1: Load ZPB planes (or zero-fill beyond extraction) ---- + // Global loads have no OOB clamp (unlike the former buffer path), so the + // ragged x edge is read lane-by-lane, guarded by the extraction window. bcopy_float4_vec regs[BCOPY_ZPB]; constexpr bcopy_float4_vec zero_vec = {0.0f, 0.0f, 0.0f, 0.0f}; #pragma unroll for (int dz = 0; dz < BCOPY_ZPB; ++dz) { int iz = z_start + dz; - if (y_in_range && iz < ez) { - auto plane_rsrc = __builtin_amdgcn_make_buffer_rsrc( - const_cast(d_src + (long)(z0 + iz) * ldimxy), - 0, -1, 0x00027000); + if (!(y_in_range && iz < ez) || gx >= ex) { + regs[dz] = zero_vec; + continue; + } + const float* plane = d_src + (size_t)(z0 + iz) * ldimxy; + if (src_x_all_in) { regs[dz] = __builtin_bit_cast(bcopy_float4_vec, - __builtin_amdgcn_raw_buffer_load_b128( - plane_rsrc, src_row_byte, 0, SLC)); + hipPlaneLoadF4u(plane, src_row_byte)); } else { - regs[dz] = zero_vec; + bcopy_float4_vec v = zero_vec; + if (gx + 0 < ex) v[0] = hipPlaneLoadScalarNT(plane, src_row_byte + 0); + if (gx + 1 < ex) v[1] = hipPlaneLoadScalarNT(plane, src_row_byte + 4); + if (gx + 2 < ex) v[2] = hipPlaneLoadScalarNT(plane, src_row_byte + 8); + if (gx + 3 < ex) v[3] = hipPlaneLoadScalarNT(plane, src_row_byte + 12); + regs[dz] = v; } } @@ -84,18 +94,14 @@ __global__ void copyToWaveletKernelOpt( // ---- Phase 3: Store ZPB planes (skip planes beyond wnz) ---- if constexpr (DO_COPY) { - uint32_t dst_byte = (gx + gy * wnx) * (uint32_t)sizeof(float); + // Wavelet dst is 32-padded (wnx,wny multiples of 32) → always full tiles. + size_t dst_byte = ((size_t)gy * wnx + gx) * sizeof(float); #pragma unroll for (int dz = 0; dz < BCOPY_ZPB; ++dz) { - if (z_start + dz < wnz) { - auto plane_rsrc = __builtin_amdgcn_make_buffer_rsrc( - d_dst + (long)(z_start + dz) * wnx * wny, - 0, -1, 0x00027000); - auto vi = __builtin_bit_cast(bcopy_int4_vec, regs[dz]); - __builtin_amdgcn_raw_buffer_store_b128( - vi, plane_rsrc, dst_byte, 0, SLC); - } + if (z_start + dz < wnz) + hipPlaneStoreNT( + d_dst + (size_t)(z_start + dz) * wnx * wny, dst_byte, regs[dz]); } } @@ -173,59 +179,46 @@ __global__ void copyFromWaveletKernelOpt( if (gy >= ey) return; if (gx >= ex) return; - uint32_t src_byte = (gx + gy * wnx) * (uint32_t)sizeof(float); - long wav_plane = (long)wnx * wny; + // Wavelet src is 32-padded and the float4 never crosses a 32-tile boundary + // (xg*4+3 <= 31 < wnx), so the vector load is always fully in bounds. + size_t src_byte = ((size_t)gy * wnx + gx) * sizeof(float); + size_t wav_plane = (size_t)wnx * wny; constexpr bcopy_float4_vec zero_vec_from = {0.0f, 0.0f, 0.0f, 0.0f}; bcopy_float4_vec regs[BCOPY_ZPB]; #pragma unroll for (int dz = 0; dz < BCOPY_ZPB; ++dz) { - if (z_start + dz < wnz) { - auto plane_rsrc = __builtin_amdgcn_make_buffer_rsrc( - const_cast(d_src + (long)(z_start + dz) * wav_plane), - 0, -1, 0x00027000); - regs[dz] = __builtin_bit_cast(bcopy_float4_vec, - __builtin_amdgcn_raw_buffer_load_b128( - plane_rsrc, src_byte, 0, SLC)); - } else { + if (z_start + dz < wnz) + regs[dz] = hipPlaneLoadNT( + d_src + (size_t)(z_start + dz) * wav_plane, src_byte); + else regs[dz] = zero_vec_from; - } } - uint32_t dst_byte = ((y0 + gy) * ldimx + x0 + gx) * (uint32_t)sizeof(float); + size_t dst_byte = ((size_t)(y0 + gy) * ldimx + x0 + gx) * sizeof(float); bool x_all_in = (gx + 3 < ex); if (x_all_in) { #pragma unroll for (int dz = 0; dz < BCOPY_ZPB; ++dz) { int iz = z_start + dz; - if (iz < ez) { - auto plane_rsrc = __builtin_amdgcn_make_buffer_rsrc( - d_dst + (long)(z0 + iz) * ldimxy, - 0, -1, 0x00027000); - auto vi = __builtin_bit_cast(bcopy_int4_vec, regs[dz]); - __builtin_amdgcn_raw_buffer_store_b128( - vi, plane_rsrc, dst_byte, 0, SLC); - } + if (iz < ez) + hipPlaneStoreF4u( + d_dst + (size_t)(z0 + iz) * ldimxy, dst_byte, + __builtin_bit_cast(hip_float4_u, regs[dz])); } } else { + // Ragged x edge: store lane-by-lane, guarded by the extraction window + // (global stores have no OOB clamp). #pragma unroll for (int dz = 0; dz < BCOPY_ZPB; ++dz) { int iz = z_start + dz; if (iz < ez) { - auto plane_rsrc = __builtin_amdgcn_make_buffer_rsrc( - d_dst + (long)(z0 + iz) * ldimxy, - 0, -1, 0x00027000); - float e0 = regs[dz][0], e1 = regs[dz][1]; - float e2 = regs[dz][2], e3 = regs[dz][3]; - if (gx + 0 < ex) __builtin_amdgcn_raw_buffer_store_b32( - __builtin_bit_cast(int, e0), plane_rsrc, dst_byte + 0, 0, SLC); - if (gx + 1 < ex) __builtin_amdgcn_raw_buffer_store_b32( - __builtin_bit_cast(int, e1), plane_rsrc, dst_byte + 4, 0, SLC); - if (gx + 2 < ex) __builtin_amdgcn_raw_buffer_store_b32( - __builtin_bit_cast(int, e2), plane_rsrc, dst_byte + 8, 0, SLC); - if (gx + 3 < ex) __builtin_amdgcn_raw_buffer_store_b32( - __builtin_bit_cast(int, e3), plane_rsrc, dst_byte + 12, 0, SLC); + float* plane = d_dst + (size_t)(z0 + iz) * ldimxy; + if (gx + 0 < ex) hipPlaneStoreScalarNT(plane, dst_byte + 0, regs[dz][0]); + if (gx + 1 < ex) hipPlaneStoreScalarNT(plane, dst_byte + 4, regs[dz][1]); + if (gx + 2 < ex) hipPlaneStoreScalarNT(plane, dst_byte + 8, regs[dz][2]); + if (gx + 3 < ex) hipPlaneStoreScalarNT(plane, dst_byte + 12, regs[dz][3]); } } } diff --git a/hip/hipCompress.cpp b/hip/hipCompress.cpp index aaa024b..83575aa 100644 --- a/hip/hipCompress.cpp +++ b/hip/hipCompress.cpp @@ -53,7 +53,7 @@ const char* hipCompressErrorString(hipCompressError_t err) case HIP_COMPRESS_ERROR_MEMORY_ALLOCATION: return "memory allocation failed"; case HIP_COMPRESS_ERROR_INVALID_SCALE: return "scale must be > 0 and finite"; case HIP_COMPRESS_ERROR_EXTRACTION_DIMS_MISMATCH: return "extraction wavelet dims must equal plan dims"; - case HIP_COMPRESS_ERROR_PLANE_TOO_LARGE: return "single plane exceeds 4 GB (nx * ny * 4 > 2^32)"; + case HIP_COMPRESS_ERROR_PLANE_TOO_LARGE: return "single plane too large (nx * ny must be < 2^31 elements)"; case HIP_COMPRESS_ERROR_HIP_RUNTIME: return "internal HIP runtime error"; default: return "unknown error"; } @@ -82,7 +82,10 @@ hipError_t hipCompressCreatePlan(hipCompressPlan** plan, int nx, int ny, int nz, PLAN_ERROR(p, HIP_COMPRESS_ERROR_NOT_MULTIPLE_OF_32, hipErrorInvalidValue); } - if ((long)nx * (long)ny * (long)sizeof(float) > (1L << 32)) + // Intra-plane addressing uses 64-bit byte offsets, so a plane is no longer + // capped at 4 GB. The remaining bound is the int plane stride (nx*ny) used + // by the kernels: the element count must stay below 2^31 (~8 GB per plane). + if ((long)nx * (long)ny > (long)INT_MAX) PLAN_ERROR(p, HIP_COMPRESS_ERROR_PLANE_TOO_LARGE, hipErrorInvalidValue); // Resolve the dimensionality-selected default to a concrete codec: quadtree @@ -417,9 +420,6 @@ hipError_t hipCopyToWaveletLayout( if (ex < 32 || ey < 32 || ez < 32) PLAN_ERROR(plan, HIP_COMPRESS_ERROR_WINDOW_TOO_SMALL, hipErrorInvalidValue); } - if ((long)ldimxy * (long)sizeof(float) > (1L << 32)) - PLAN_ERROR(plan, HIP_COMPRESS_ERROR_PLANE_TOO_LARGE, hipErrorInvalidValue); - int wnx = hipCompressWaveletDim(ex); int wny = hipCompressWaveletDim(ey); int wnz = plan->is_2d ? 1 : hipCompressWaveletDim(ez); @@ -508,9 +508,6 @@ hipError_t hipCopyFromWaveletLayout( if (ex < 32 || ey < 32 || ez < 32) PLAN_ERROR(plan, HIP_COMPRESS_ERROR_WINDOW_TOO_SMALL, hipErrorInvalidValue); } - if ((long)ldimxy * (long)sizeof(float) > (1L << 32)) - PLAN_ERROR(plan, HIP_COMPRESS_ERROR_PLANE_TOO_LARGE, hipErrorInvalidValue); - int wnx = hipCompressWaveletDim(ex); int wny = hipCompressWaveletDim(ey); int wnz = plan->is_2d ? 1 : hipCompressWaveletDim(ez); diff --git a/hip/hipPlaneIO.h b/hip/hipPlaneIO.h new file mode 100644 index 0000000..d773786 --- /dev/null +++ b/hip/hipPlaneIO.h @@ -0,0 +1,83 @@ +// Copyright (C) 2026 Advanced Micro Devices, Inc. +// Use of this source code is governed by an MIT-style license that can be +// found in the LICENSE file or at https://opensource.org/licenses/MIT. +// +// Single global-memory plane-addressing path for the wavefield/wavelet buffers. +// +// The codec kernels used to read/write planes with buffer instructions +// (make_buffer_rsrc + raw_buffer_load/store_*), whose intra-plane byte offset +// is 32-bit -- capping a single plane at 4 GB (nx*ny*4 <= 2^32). A/B +// benchmarking (buffer vs global "saddr" load, uint32 vs size_t offset) showed +// the global path matches or beats buffer on both gfx942 and gfx950 with equal +// or lower register pressure, and that widening the offset to 64 bits is free. +// So every plane access now uses a 64-bit base pointer plus a size_t byte +// offset, lifting the 4 GB per-plane limit. Loads/stores are nontemporal to +// preserve the previous SLC (streaming) cache hint. +// +// Note: buffer instructions also provided free out-of-bounds clamping via +// num_records. Global loads/stores do not, so any kernel that could touch a +// ragged (non-32-multiple) edge must bound-check explicitly. The fused codec +// kernels operate only on 32-aligned full tiles (the public API mandates +// 32-multiple dims), so they never go out of bounds; the block-copy kernels, +// which handle arbitrary extraction windows, carry explicit per-lane guards. + +#pragma once + +#include +#include + +// Vector load of V (e.g. float4 / int4) from base + byte_off, nontemporal. +template +__device__ __forceinline__ +V hipPlaneLoadNT(const float* base, size_t byte_off) +{ + const uint8_t* p = reinterpret_cast(base) + byte_off; + return __builtin_nontemporal_load(reinterpret_cast(p)); +} + +// Vector store of V to base + byte_off, nontemporal. +template +__device__ __forceinline__ +void hipPlaneStoreNT(float* base, size_t byte_off, V v) +{ + uint8_t* p = reinterpret_cast(base) + byte_off; + __builtin_nontemporal_store(v, reinterpret_cast(p)); +} + +// Alignment-tolerant float4 load/store. Block-copy touches the caller's array +// with arbitrary strides/origins (ldimx, x0), so an intra-plane byte offset is +// only guaranteed 4-byte aligned (everything is elem*sizeof(float)), not 16. +// A 128-bit global load needs just 4-byte element alignment, so declaring the +// vector aligned(4) avoids the natural-16B assumption of hipPlaneLoadNT +// (which faults on, e.g., x0=10 -> offset 8 mod 16). The fused codec kernels +// keep the 16-aligned path (nx is a 32-multiple, so offsets are 16-aligned). +using hip_float4_u = float __attribute__((ext_vector_type(4), __aligned__(4))); + +__device__ __forceinline__ +hip_float4_u hipPlaneLoadF4u(const float* base, size_t byte_off) +{ + const uint8_t* p = reinterpret_cast(base) + byte_off; + return __builtin_nontemporal_load(reinterpret_cast(p)); +} + +__device__ __forceinline__ +void hipPlaneStoreF4u(float* base, size_t byte_off, hip_float4_u v) +{ + uint8_t* p = reinterpret_cast(base) + byte_off; + __builtin_nontemporal_store(v, reinterpret_cast(p)); +} + +// Scalar float load/store for ragged-edge lanes. +__device__ __forceinline__ +float hipPlaneLoadScalarNT(const float* base, size_t byte_off) +{ + const uint8_t* p = reinterpret_cast(base) + byte_off; + return __builtin_nontemporal_load(reinterpret_cast(p)); +} + +__device__ __forceinline__ +void hipPlaneStoreScalarNT(float* base, size_t byte_off, float v) +{ + uint8_t* p = reinterpret_cast(base) + byte_off; + __builtin_nontemporal_store(v, reinterpret_cast(p)); +} diff --git a/hip/hipWaveletBitmap.h b/hip/hipWaveletBitmap.h index 7e202a9..69a7cf3 100644 --- a/hip/hipWaveletBitmap.h +++ b/hip/hipWaveletBitmap.h @@ -75,18 +75,14 @@ __global__ void waveletBitmapFusedKernel( int gx = blockIdx.x * 32 + xg * 4; int gy = blockIdx.y * 32 + yr; - uint32_t byte_off = (gx + gy * ldimx) * (uint32_t)sizeof(float); + size_t byte_off = ((size_t)gy * ldimx + gx) * sizeof(float); // ---- Phase 1: Load 32 planes from global ---- wrle_float4_vec regs[PLANES]; #pragma unroll - for (int p = 0; p < PLANES; p++) { - auto rsrc = __builtin_amdgcn_make_buffer_rsrc( - const_cast(block_base + (long)p * ldimxy), - 0, -1, 0x00027000); - regs[p] = __builtin_bit_cast(wrle_float4_vec, - __builtin_amdgcn_raw_buffer_load_b128(rsrc, byte_off, 0, SLC)); - } + for (int p = 0; p < PLANES; p++) + regs[p] = hipPlaneLoadNT( + block_base + (size_t)p * ldimxy, byte_off); // ---- Phase 2: Z-transform in registers ---- ds79_forward_f4_scalar_tmp(regs, PLANES); diff --git a/hip/hipWaveletOctree.h b/hip/hipWaveletOctree.h index f62e48f..f2b697f 100644 --- a/hip/hipWaveletOctree.h +++ b/hip/hipWaveletOctree.h @@ -1126,13 +1126,10 @@ __device__ __forceinline__ void woct_bitmap_inverse_body( float* block_base = output + (size_t)blockIdx.z*32*ldimxy; int gx = blockIdx.x*32 + xg*4; int gy = blockIdx.y*32 + yr; - uint32_t byte_off = (gx + gy*ldimx)*(uint32_t)sizeof(float); + size_t byte_off = ((size_t)gy*ldimx + gx)*sizeof(float); #pragma unroll - for (int p=0;p(block_base + (size_t)p*ldimxy, byte_off, regs[p]); } // Stage B with a host-scalar inv_scale (prototype / test path). diff --git a/hip/hipWaveletQuadtree2D.h b/hip/hipWaveletQuadtree2D.h index b94c405..d4c9b83 100644 --- a/hip/hipWaveletQuadtree2D.h +++ b/hip/hipWaveletQuadtree2D.h @@ -26,6 +26,7 @@ #include #include "ds79.h" +#include "hipPlaneIO.h" // Tile batching (identical to hipWaveletRLE2D.h so the transform is unchanged). static constexpr int WQT2D_TILES_PER_WG = 32; @@ -290,12 +291,10 @@ __global__ void waveletQuadtree2DForwardKernel( int gy = blockIdx.y * 32 + yr; int x0 = xg * 4; if (tile_bx < nbx) { - uint32_t byte_off = (gx + gy * ldimx) * (uint32_t)sizeof(float); - auto rsrc = __builtin_amdgcn_make_buffer_rsrc( - const_cast(input), 0, -1, 0x00027000); - auto v = __builtin_bit_cast( - __attribute__((__vector_size__(4 * sizeof(float)))) float, - __builtin_amdgcn_raw_buffer_load_b128(rsrc, byte_off, 0, SLC)); + size_t byte_off = ((size_t)gy * ldimx + gx) * sizeof(float); + auto v = hipPlaneLoadNT< + __attribute__((__vector_size__(4 * sizeof(float)))) float>( + input, byte_off); wavelet[b * 1024 + (x0+0) * 32 + (yr ^ (x0+0))] = v[0]; wavelet[b * 1024 + (x0+1) * 32 + (yr ^ (x0+1))] = v[1]; wavelet[b * 1024 + (x0+2) * 32 + (yr ^ (x0+2))] = v[2]; @@ -592,12 +591,8 @@ __global__ void waveletQuadtree2DInverseKernel( if (tb >= nbx) continue; int gx = tb * 32 + xg * 4; int gy = blockIdx.y * 32 + yr; - uint32_t byte_off = (gx + gy * ldimx) * (uint32_t)sizeof(float); - auto rsrc = __builtin_amdgcn_make_buffer_rsrc( - output, 0, -1, 0x00027000); - auto vi = __builtin_bit_cast( - __attribute__((__vector_size__(4 * sizeof(int)))) int, store_regs[s]); - __builtin_amdgcn_raw_buffer_store_b128(vi, rsrc, byte_off, 0, SLC); + size_t byte_off = ((size_t)gy * ldimx + gx) * sizeof(float); + hipPlaneStoreNT(output, byte_off, store_regs[s]); } } __syncthreads(); diff --git a/hip/hipWaveletRLE.h b/hip/hipWaveletRLE.h index 5edc878..91d17d8 100644 --- a/hip/hipWaveletRLE.h +++ b/hip/hipWaveletRLE.h @@ -12,12 +12,13 @@ // // Eliminates the intermediate global write+read between wavelet and RLE. // 32 KB LDS reused across phases: wavelet Y/X, then RLE scan+compact. -// Zero scratch. Buffer instructions for loads. Occupancy 2 on gfx942. +// Zero scratch. Global (size_t offset) plane loads. Occupancy 2 on gfx942. #include #include #include #include "ds79.h" +#include "hipPlaneIO.h" #include "Run_Length_Escape_Codes.hxx" using wrle_float4_vec = ds79_float4_vec; @@ -161,19 +162,14 @@ __global__ void waveletRLEFusedKernel( int gx = blockIdx.x * 32 + xg * 4; int gy = blockIdx.y * 32 + yr; - uint32_t byte_off = (gx + gy * ldimx) * (uint32_t)sizeof(float); + size_t byte_off = ((size_t)gy * ldimx + gx) * sizeof(float); // ---- Phase 1: Load 32 planes from global ---- wrle_float4_vec regs[PLANES]; #pragma unroll - for (int p = 0; p < PLANES; p++) { - auto rsrc = __builtin_amdgcn_make_buffer_rsrc( - const_cast(block_base + (long)p * ldimxy), - 0, -1, 0x00027000); - regs[p] = __builtin_bit_cast(wrle_float4_vec, - __builtin_amdgcn_raw_buffer_load_b128( - rsrc, byte_off, 0, SLC)); - } + for (int p = 0; p < PLANES; p++) + regs[p] = hipPlaneLoadNT( + block_base + (size_t)p * ldimxy, byte_off); // ---- Phase 2: Z-transform in registers ---- ds79_forward_f4_scalar_tmp(regs, PLANES); @@ -312,173 +308,6 @@ inline hipError_t hipWaveletRLEFusedDumpCoef( return hipGetLastError(); } -// --------------------------------------------------------------------------- -// saddr variant: uses global_load_dwordx4 with SGPR base + VGPR offset -// instead of buffer instructions. Everything else is identical. -// --------------------------------------------------------------------------- - -__device__ __forceinline__ -wrle_float4_vec wrle_saddr_load_nt(const float* base, uint32_t byte_off) { - const uint8_t* p = reinterpret_cast(base); - return __builtin_nontemporal_load( - reinterpret_cast(p + byte_off)); -} - -__launch_bounds__(256, 2) -__global__ void waveletRLEFusedSaddrKernel( - const float* __restrict__ input, - unsigned char* __restrict__ output, - size_t* __restrict__ block_sizes, - float scale, - int ldimx, int ldimxy, - const double* __restrict__ d_rms, - float* __restrict__ d_mulfac_out, - float* __restrict__ d_coef_out = nullptr) // kept for signature parity with - // waveletRLEFusedKernel (unused) -{ - constexpr int PLANES = 32; - constexpr int BATCH = 8; - constexpr int NTHREADS = 256; - using BlockScan = rocprim::block_scan; - (void)d_coef_out; - - __shared__ union { - float wavelet[BATCH * 1024]; - typename BlockScan::storage_type scan; - unsigned char compact[WRLE_LDS_BYTES]; - } lds; - - int tid = threadIdx.x; - int xg = tid % 8; - int yr = tid / 8; - - float mulfac; - if (d_rms != nullptr) { - float rms = (float)*d_rms; - float product = rms * scale; - mulfac = (product > 0.0f && __builtin_isfinite(1.0f / product)) - ? (1.0f / product) : 1.0f; - if (tid == 0 && blockIdx.x == 0 && blockIdx.y == 0 && blockIdx.z == 0) { - if (d_mulfac_out) *d_mulfac_out = mulfac; - } - } else { - mulfac = scale; - } - - const float* block_base = input + (size_t)blockIdx.z * 32 * ldimxy; - int gx = blockIdx.x * 32 + xg * 4; - int gy = blockIdx.y * 32 + yr; - uint32_t xy_byte = (uint32_t)(gx + gy * ldimx) * (uint32_t)sizeof(float); - uint32_t byte_stride = (uint32_t)ldimxy * (uint32_t)sizeof(float); - - // ---- Phase 1: Load 32 planes from global (saddr) ---- - wrle_float4_vec regs[PLANES]; - #pragma unroll - for (int p = 0; p < PLANES; p++) - regs[p] = wrle_saddr_load_nt(block_base, xy_byte + (uint32_t)p * byte_stride); - - // ---- Phase 2: Z-transform in registers ---- - ds79_forward_f4_scalar_tmp(regs, PLANES); - - // ---- Phase 3: Y+X transform in LDS (batches of 8) ---- - for (int pb = 0; pb < PLANES; pb += BATCH) { - for (int dp = 0; dp < BATCH; dp++) { - wrle_float4_vec v = regs[pb + dp]; - int x0 = xg * 4; - lds.wavelet[dp * 1024 + (x0+0) * 32 + (yr ^ (x0+0))] = v[0]; - lds.wavelet[dp * 1024 + (x0+1) * 32 + (yr ^ (x0+1))] = v[1]; - lds.wavelet[dp * 1024 + (x0+2) * 32 + (yr ^ (x0+2))] = v[2]; - lds.wavelet[dp * 1024 + (x0+3) * 32 + (yr ^ (x0+3))] = v[3]; - } - __syncthreads(); - - int pl = tid / 32; - int pos = tid % 32; - - float line[32]; - for (int y = 0; y < 32; y++) - line[y] = lds.wavelet[pl * 1024 + pos * 32 + (y ^ pos)]; - ds79_forward_reg32(line); - for (int y = 0; y < 32; y++) - lds.wavelet[pl * 1024 + pos * 32 + (y ^ pos)] = line[y]; - __syncthreads(); - - for (int x = 0; x < 32; x++) - line[x] = lds.wavelet[pl * 1024 + x * 32 + (pos ^ x)]; - ds79_forward_reg32(line); - for (int x = 0; x < 32; x++) - lds.wavelet[pl * 1024 + x * 32 + (pos ^ x)] = line[x]; - __syncthreads(); - - for (int dp = 0; dp < BATCH; dp++) { - wrle_float4_vec v; - int x0 = xg * 4; - v[0] = lds.wavelet[dp * 1024 + (x0+0) * 32 + (yr ^ (x0+0))]; - v[1] = lds.wavelet[dp * 1024 + (x0+1) * 32 + (yr ^ (x0+1))]; - v[2] = lds.wavelet[dp * 1024 + (x0+2) * 32 + (yr ^ (x0+2))]; - v[3] = lds.wavelet[dp * 1024 + (x0+3) * 32 + (yr ^ (x0+3))]; - regs[pb + dp] = v; - } - __syncthreads(); - } - - // ---- Phase 4: Quantize + RLE encode (two-pass, compacted) ---- - // Block layout: [1024B zline_meta] [RLE data] - int bid = blockIdx.x + blockIdx.y * gridDim.x + blockIdx.z * gridDim.x * gridDim.y; - unsigned char* block_out = output + (long)bid * 4 * WRLE_LDS_BYTES; - unsigned char* meta_out = block_out; - unsigned char* rle_out = block_out + WRLE_META_PER_BLOCK; - int block_total = 0; - - for (int x_off = 0; x_off < 4; ++x_off) { - int my_count = wrle_zline(regs, x_off, mulfac, nullptr); - - meta_out[x_off * 256 + tid] = (unsigned char)my_count; - - int my_offset, pass_total; - BlockScan().exclusive_scan(my_count, my_offset, 0, pass_total, lds.scan); - __syncthreads(); - - wrle_zline(regs, x_off, mulfac, lds.compact + my_offset); - __syncthreads(); - - int aligned_total = (pass_total + 3) & ~3; - for (int off = tid * 4; off < aligned_total; off += NTHREADS * 4) { - unsigned val = 0; - if (off < pass_total) { - val = (unsigned)lds.compact[off]; - if (off+1 < pass_total) val |= (unsigned)lds.compact[off+1] << 8; - if (off+2 < pass_total) val |= (unsigned)lds.compact[off+2] << 16; - if (off+3 < pass_total) val |= (unsigned)lds.compact[off+3] << 24; - } - if (off < aligned_total) { - unsigned* dst32 = (unsigned*)(rle_out + block_total + off); - *dst32 = val; - } - } - __syncthreads(); - block_total += pass_total; - } - - if (tid == 0) - block_sizes[bid] = WRLE_META_PER_BLOCK + block_total; -} - -inline hipError_t hipWaveletRLEFusedSaddr( - const float* input, - unsigned char* output, - size_t* block_sizes, - float scale, - int nx, int ny, int nz, - int ldimx, int ldimxy) -{ - dim3 grid((nx + 31) / 32, (ny + 31) / 32, (nz + 31) / 32); - waveletRLEFusedSaddrKernel<<>>( - input, output, block_sizes, scale, ldimx, ldimxy, - nullptr, nullptr); - return hipGetLastError(); -} - // --------------------------------------------------------------------------- // Output compaction: copies variable-length blocks from fixed-stride layout // to a tightly packed buffer using precomputed prefix-sum offsets. @@ -628,19 +457,14 @@ __global__ void waveletSegRLEFusedKernel( int gx = blockIdx.x * 32 + xg * 4; int gy = blockIdx.y * 32 + yr; - uint32_t byte_off = (gx + gy * ldimx) * (uint32_t)sizeof(float); + size_t byte_off = ((size_t)gy * ldimx + gx) * sizeof(float); // ---- Phase 1: Load 32 planes from global ---- wrle_float4_vec regs[PLANES]; #pragma unroll - for (int p = 0; p < PLANES; p++) { - auto rsrc = __builtin_amdgcn_make_buffer_rsrc( - const_cast(block_base + (long)p * ldimxy), - 0, -1, 0x00027000); - regs[p] = __builtin_bit_cast(wrle_float4_vec, - __builtin_amdgcn_raw_buffer_load_b128( - rsrc, byte_off, 0, SLC)); - } + for (int p = 0; p < PLANES; p++) + regs[p] = hipPlaneLoadNT( + block_base + (size_t)p * ldimxy, byte_off); // ---- Phase 2: Z-transform in registers ---- ds79_forward_f4_scalar_tmp(regs, PLANES); diff --git a/hip/hipWaveletRLE2D.h b/hip/hipWaveletRLE2D.h index 8dc3d5d..4ef0034 100644 --- a/hip/hipWaveletRLE2D.h +++ b/hip/hipWaveletRLE2D.h @@ -19,6 +19,7 @@ #include #include #include "ds79.h" +#include "hipPlaneIO.h" #include "Run_Length_Escape_Codes.hxx" #include "hipRLEDecode.h" @@ -162,12 +163,10 @@ __global__ void waveletRLE2DFusedKernel( int gy = blockIdx.y * 32 + yr; int x0 = xg * 4; if (tile_bx < nbx) { - uint32_t byte_off = (gx + gy * ldimx) * (uint32_t)sizeof(float); - auto rsrc = __builtin_amdgcn_make_buffer_rsrc( - const_cast(input), 0, -1, 0x00027000); - auto v = __builtin_bit_cast( - __attribute__((__vector_size__(4 * sizeof(float)))) float, - __builtin_amdgcn_raw_buffer_load_b128(rsrc, byte_off, 0, SLC)); + size_t byte_off = ((size_t)gy * ldimx + gx) * sizeof(float); + auto v = hipPlaneLoadNT< + __attribute__((__vector_size__(4 * sizeof(float)))) float>( + input, byte_off); lds.wavelet[b * 1024 + (x0+0) * 32 + (yr ^ (x0+0))] = v[0]; lds.wavelet[b * 1024 + (x0+1) * 32 + (yr ^ (x0+1))] = v[1]; lds.wavelet[b * 1024 + (x0+2) * 32 + (yr ^ (x0+2))] = v[2]; @@ -401,12 +400,8 @@ __global__ void waveletRLE2DInverseFusedKernel( if (tb >= nbx) continue; int gx = tb * 32 + xg * 4; int gy = blockIdx.y * 32 + yr; - uint32_t byte_off = (gx + gy * ldimx) * (uint32_t)sizeof(float); - auto rsrc = __builtin_amdgcn_make_buffer_rsrc( - output, 0, -1, 0x00027000); - auto vi = __builtin_bit_cast( - __attribute__((__vector_size__(4 * sizeof(int)))) int, store_regs[s]); - __builtin_amdgcn_raw_buffer_store_b128(vi, rsrc, byte_off, 0, SLC); + size_t byte_off = ((size_t)gy * ldimx + gx) * sizeof(float); + hipPlaneStoreNT(output, byte_off, store_regs[s]); } } __syncthreads(); diff --git a/hip/hipWaveletRLEInverse.h b/hip/hipWaveletRLEInverse.h index bd966ad..a439e3c 100644 --- a/hip/hipWaveletRLEInverse.h +++ b/hip/hipWaveletRLEInverse.h @@ -145,16 +145,12 @@ __global__ void waveletRLEInverseFusedKernel( float* block_base = output + (size_t)blockIdx.z * 32 * ldimxy; int gx = blockIdx.x * 32 + xg * 4; int gy = blockIdx.y * 32 + yr; - uint32_t byte_off = (gx + gy * ldimx) * (uint32_t)sizeof(float); + size_t byte_off = ((size_t)gy * ldimx + gx) * sizeof(float); #pragma unroll - for (int p = 0; p < PLANES; p++) { - auto rsrc = __builtin_amdgcn_make_buffer_rsrc( - block_base + (long)p * ldimxy, - 0, -1, 0x00027000); - auto v = __builtin_bit_cast(__attribute__((__vector_size__(16))) int, regs[p]); - __builtin_amdgcn_raw_buffer_store_b128(v, rsrc, byte_off, 0, SLC); - } + for (int p = 0; p < PLANES; p++) + hipPlaneStoreNT( + block_base + (size_t)p * ldimxy, byte_off, regs[p]); } inline hipError_t hipWaveletRLEInverseFusedFixedStride( @@ -280,16 +276,12 @@ __global__ void waveletSegRLEInverseFusedKernel( float* block_base_out = output + (size_t)blockIdx.z * 32 * ldimxy; int gx = blockIdx.x * 32 + xg * 4; int gy = blockIdx.y * 32 + yr; - uint32_t byte_off = (gx + gy * ldimx) * (uint32_t)sizeof(float); + size_t byte_off = ((size_t)gy * ldimx + gx) * sizeof(float); #pragma unroll - for (int p = 0; p < PLANES; p++) { - auto rsrc = __builtin_amdgcn_make_buffer_rsrc( - block_base_out + (long)p * ldimxy, - 0, -1, 0x00027000); - auto v = __builtin_bit_cast(__attribute__((__vector_size__(16))) int, regs[p]); - __builtin_amdgcn_raw_buffer_store_b128(v, rsrc, byte_off, 0, SLC); - } + for (int p = 0; p < PLANES; p++) + hipPlaneStoreNT( + block_base_out + (size_t)p * ldimxy, byte_off, regs[p]); } #endif // HIPWAVELET_RLE_INVERSE_H diff --git a/tests/test_compress_api_hip.cpp b/tests/test_compress_api_hip.cpp index 066ceb0..4de2bb6 100644 --- a/tests/test_compress_api_hip.cpp +++ b/tests/test_compress_api_hip.cpp @@ -2461,7 +2461,10 @@ static bool test_error_codes() } { hipCompressPlan* plan = nullptr; - hipError_t err = hipCompressCreatePlan(&plan, 32800, 32768, 32, 0); + // 64-bit intra-plane offsets lifted the old 4 GB cap; the remaining + // bound is the int plane stride nx*ny (< 2^31 elements). 65536*32768 + // = 2^31 > INT_MAX, so the guard still rejects (before any hipMalloc). + hipError_t err = hipCompressCreatePlan(&plan, 65536, 32768, 32, 0); if (!check_error("CreatePlan plane too large", plan, err, hipErrorInvalidValue, HIP_COMPRESS_ERROR_PLANE_TOO_LARGE)) pass = false; @@ -2706,24 +2709,10 @@ static bool test_error_codes() } // --- Plane too large --- - { - hipError_t err = hipCopyToWaveletLayout( - d_buf, 32768, 1073741825, - 0, 0, 0, - 128, 128, 128, - d_buf, nullptr, plan, 0); - if (!check_error("CopyTo plane too large", plan, err, - hipErrorInvalidValue, HIP_COMPRESS_ERROR_PLANE_TOO_LARGE)) - pass = false; - } - { - hipError_t err = hipCopyFromWaveletLayout( - d_buf, d_buf, 32768, 1073741825, - 0, 0, 0, 128, 128, 128, plan, 0); - if (!check_error("CopyFrom plane too large", plan, err, - hipErrorInvalidValue, HIP_COMPRESS_ERROR_PLANE_TOO_LARGE)) - pass = false; - } + // The former CopyTo/CopyFrom "plane too large" (4 GB) rejections were + // removed: intra-plane addressing is now 64-bit, so source strides are + // bounded only by the int stride type (up to INT_MAX elements) and are no + // longer size-capped here. // --- Decompress errors --- { diff --git a/tests/test_wavelet_rle_fused_hip.cpp b/tests/test_wavelet_rle_fused_hip.cpp index 3f0b372..5472d04 100644 --- a/tests/test_wavelet_rle_fused_hip.cpp +++ b/tests/test_wavelet_rle_fused_hip.cpp @@ -314,10 +314,10 @@ void bench(int NX, int NY, int NZ, float scale, int warmup, int runs) printf(" unfused %7.3f ms %7.1f GB/s CR %.1f:1\n", ms_unfused, bw_u, cr_u); // Fused (no compaction) - FusedLauncher launchers[] = { hipWaveletRLEFused, hipWaveletRLEFusedSaddr }; - const char* names[] = { "fused-buf ", "fused-saddr " }; + FusedLauncher launchers[] = { hipWaveletRLEFused }; + const char* names[] = { "fused " }; - for (int k = 0; k < 2; ++k) { + for (int k = 0; k < 1; ++k) { for (int i = 0; i < warmup; ++i) HIPCHECK(launchers[k](d_raw, d_out, d_sizes, scale, NX, NY, NZ, ldimx, ldimxy)); HIPCHECK(hipDeviceSynchronize()); @@ -588,8 +588,7 @@ int main() float scales[] = {0.01f, 0.1f, 1.0f, 10.0f, 100.0f}; struct { FusedLauncher fn; const char* tag; } variants[] = { - { hipWaveletRLEFused, "buffer" }, - { hipWaveletRLEFusedSaddr, "saddr" }, + { hipWaveletRLEFused, "fused" }, }; printf("-- Correctness (fused) --\n"); From 15bbf35c99bff0aced970f15ba3c1d47468f46b5 Mon Sep 17 00:00:00 2001 From: Ossian O'Reilly Date: Thu, 20 Aug 2026 12:24:45 -0700 Subject: [PATCH 7/7] Add HIPCOMPRESS_DEBUG env-var diagnostics to hipCompress Opt-in, cached read of HIPCOMPRESS_DEBUG (0=silent). Level 1 prints the resolved scheme (with AUTO tag), dims, and block count at plan creation plus compressed bytes and CR at synchronize; level 2 adds device-memory detail. Zero output and a single cached branch when unset. --- hip/hipCompress.cpp | 62 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/hip/hipCompress.cpp b/hip/hipCompress.cpp index 83575aa..f0e18ad 100644 --- a/hip/hipCompress.cpp +++ b/hip/hipCompress.cpp @@ -31,6 +31,34 @@ } \ } while(0) +// Opt-in diagnostics: set HIPCOMPRESS_DEBUG=1 for scheme + result lines, +// HIPCOMPRESS_DEBUG=2 to also dump device-memory allocation detail. The +// environment is read once and cached, so a disabled build pays only a single +// predictable branch per instrumented call and prints nothing. +static int hipCompressDebugLevel() +{ + static int lvl = -1; + if (lvl < 0) { + const char* e = getenv("HIPCOMPRESS_DEBUG"); + lvl = (e && *e) ? atoi(e) : 0; + if (lvl < 0) lvl = 0; + } + return lvl; +} + +static const char* hipCompressKernelName(hipCompressKernel k) +{ + switch (k) { + case HIP_COMPRESS_KERNEL_ZLINE: return "ZLINE"; + case HIP_COMPRESS_KERNEL_SEGRLE: return "SEGRLE"; + case HIP_COMPRESS_KERNEL_OCTREE: return "OCTREE"; + case HIP_COMPRESS_KERNEL_TWOLEVEL: return "TWOLEVEL"; + case HIP_COMPRESS_KERNEL_QUADTREE: return "QUADTREE"; + case HIP_COMPRESS_KERNEL_AUTO: return "AUTO"; + default: return "UNKNOWN"; + } +} + hipCompressError_t hipCompressGetLastError(const hipCompressPlan* plan) { if (!plan) return HIP_COMPRESS_ERROR_NULL_PLAN; @@ -94,6 +122,7 @@ hipError_t hipCompressCreatePlan(hipCompressPlan** plan, int nx, int ny, int nz, // valid dims; ZLINE remains the fallback for configurations they cannot // handle, guarding future constraints so the default flip never breaks // callers. Downstream logic only ever sees the resolved concrete kernel. + const hipCompressKernel requested = kernel; if (kernel == HIP_COMPRESS_KERNEL_AUTO) { kernel = is_2d ? HIP_COMPRESS_KERNEL_QUADTREE : HIP_COMPRESS_KERNEL_OCTREE; @@ -185,6 +214,30 @@ hipError_t hipCompressCreatePlan(hipCompressPlan** plan, int nx, int ny, int nz, HIPCHECK_PLAN(p, hipStreamSynchronize(aux_stream)); p->last_error = HIP_COMPRESS_SUCCESS; + + if (hipCompressDebugLevel() >= 1) { + fprintf(stderr, + "[hipCompress] plan: scheme=%s%s dims=%dx%dx%d (%s) blocks=%d\n", + hipCompressKernelName(kernel), + requested == HIP_COMPRESS_KERNEL_AUTO ? " (AUTO)" : "", + nx, ny, nz, is_2d ? "2D" : "3D", nb); + if (hipCompressDebugLevel() >= 2) { + long coded_bytes = 0; + if (kernel == HIP_COMPRESS_KERNEL_OCTREE) + coded_bytes = (long)nb * WOCT_CODE_SLOT_BYTES; + else if (kernel == HIP_COMPRESS_KERNEL_QUADTREE) + coded_bytes = (long)nb * WQT2D_CODE_SLOT_BYTES; + else if (kernel == HIP_COMPRESS_KERNEL_TWOLEVEL) + coded_bytes = (long)nb * WBMP_TL_SLOT_BYTES; + double mib = 1.0 / (1024.0 * 1024.0); + fprintf(stderr, + "[hipCompress] mem: scratch=%.1f MiB coded=%.1f MiB " + "scan_temp=%.1f MiB slot_stride=%zuB\n", + (double)scratch_size * mib, (double)coded_bytes * mib, + (double)p->scan_temp_bytes * mib, p->scratch_slot_stride); + } + } + return hipSuccess; } @@ -393,6 +446,15 @@ hipError_t hipCompressSynchronize( *compression_ratio = (float)((long)total * sizeof(float)) / (float)total_bytes; } + if (hipCompressDebugLevel() >= 1) { + long total = (long)nx * ny * nz; + double raw = (double)total * (double)sizeof(float); + fprintf(stderr, + "[hipCompress] result: scheme=%s bytes=%ld raw=%.0f CR=%.2f\n", + hipCompressKernelName(plan->kernel), total_bytes, raw, + raw / (double)total_bytes); + } + plan->compress_pending = false; return hipSuccess; }