From e40f9c1526e128e2edba5b6aa02cfb686981de54 Mon Sep 17 00:00:00 2001 From: Herman Semenoff Date: Sat, 29 Aug 2026 17:07:19 +0300 Subject: [PATCH] Optimize encoding throughput across quality presets - Split partition averages, error squaring, and endpoint evaluation loops into unmasked SIMD fast paths and tails - Simplify ideal weight decimation by eliminating redundant error scale multiplies when constant - Hoist dot products and simplify weight difference calculations in weight realignment and ideal color finding - Hoist quantization table lookups in color quantization - Enable BMI/BMI2/LZCNT instructions for AVX2 builds --- Source/astcenc_averages_and_directions.cpp | 290 +++++++++++++++--- Source/astcenc_color_quantize.cpp | 14 +- Source/astcenc_compress_symbolic.cpp | 30 +- Source/astcenc_find_best_partitioning.cpp | 25 +- .../astcenc_ideal_endpoints_and_weights.cpp | 178 +++++++---- Source/astcenc_pick_best_endpoint_format.cpp | 83 ++++- Source/cmake_core.cmake | 4 +- 7 files changed, 486 insertions(+), 138 deletions(-) diff --git a/Source/astcenc_averages_and_directions.cpp b/Source/astcenc_averages_and_directions.cpp index d1dea24ec..4f52c3a78 100644 --- a/Source/astcenc_averages_and_directions.cpp +++ b/Source/astcenc_averages_and_directions.cpp @@ -63,14 +63,27 @@ static void compute_partition_averages_rgb( { vfloatacc pp_avg_rgb[3] {}; - vint lane_id = vint::lane_id(); - for (size_t i = 0; i < texel_count; i += ASTCENC_SIMD_WIDTH) + size_t texel_count_simd = round_down_to_simd_multiple_vla(texel_count); + for (size_t i = 0; i < texel_count_simd; i += ASTCENC_SIMD_WIDTH) { vint texel_partition(pi.partition_of_texel + i); + vmask p0_mask = texel_partition == vint(0); - vmask lane_mask = lane_id < vint_from_size(texel_count); - lane_id += vint(ASTCENC_SIMD_WIDTH); + vfloat data_r = loada(blk.data_r + i); + haccumulate(pp_avg_rgb[0], data_r, p0_mask); + vfloat data_g = loada(blk.data_g + i); + haccumulate(pp_avg_rgb[1], data_g, p0_mask); + + vfloat data_b = loada(blk.data_b + i); + haccumulate(pp_avg_rgb[2], data_b, p0_mask); + } + + if (texel_count_simd < texel_count) + { + size_t i = texel_count_simd; + vint texel_partition(pi.partition_of_texel + i); + vmask lane_mask = vint::lane_id() < vint_from_size(texel_count - texel_count_simd); vmask p0_mask = lane_mask & (texel_partition == vint(0)); vfloat data_r = loada(blk.data_r + i); @@ -99,13 +112,32 @@ static void compute_partition_averages_rgb( { vfloatacc pp_avg_rgb[2][3] {}; - vint lane_id = vint::lane_id(); - for (size_t i = 0; i < texel_count; i += ASTCENC_SIMD_WIDTH) + size_t texel_count_simd = round_down_to_simd_multiple_vla(texel_count); + for (size_t i = 0; i < texel_count_simd; i += ASTCENC_SIMD_WIDTH) { vint texel_partition(pi.partition_of_texel + i); - vmask lane_mask = lane_id < vint_from_size(texel_count); - lane_id += vint(ASTCENC_SIMD_WIDTH); + vmask p0_mask = texel_partition == vint(0); + vmask p1_mask = texel_partition == vint(1); + + vfloat data_r = loada(blk.data_r + i); + haccumulate(pp_avg_rgb[0][0], data_r, p0_mask); + haccumulate(pp_avg_rgb[1][0], data_r, p1_mask); + + vfloat data_g = loada(blk.data_g + i); + haccumulate(pp_avg_rgb[0][1], data_g, p0_mask); + haccumulate(pp_avg_rgb[1][1], data_g, p1_mask); + + vfloat data_b = loada(blk.data_b + i); + haccumulate(pp_avg_rgb[0][2], data_b, p0_mask); + haccumulate(pp_avg_rgb[1][2], data_b, p1_mask); + } + + if (texel_count_simd < texel_count) + { + size_t i = texel_count_simd; + vint texel_partition(pi.partition_of_texel + i); + vmask lane_mask = vint::lane_id() < vint_from_size(texel_count - texel_count_simd); vmask p0_mask = lane_mask & (texel_partition == vint(0)); vmask p1_mask = lane_mask & (texel_partition == vint(1)); @@ -144,13 +176,36 @@ static void compute_partition_averages_rgb( // For 4 partitions scan results for partition 0/1/2, compute partition 3 vfloatacc pp_avg_rgb[3][3] {}; - vint lane_id = vint::lane_id(); - for (size_t i = 0; i < texel_count; i += ASTCENC_SIMD_WIDTH) + size_t texel_count_simd = round_down_to_simd_multiple_vla(texel_count); + for (size_t i = 0; i < texel_count_simd; i += ASTCENC_SIMD_WIDTH) { vint texel_partition(pi.partition_of_texel + i); - vmask lane_mask = lane_id < vint_from_size(texel_count); - lane_id += vint(ASTCENC_SIMD_WIDTH); + vmask p0_mask = texel_partition == vint(0); + vmask p1_mask = texel_partition == vint(1); + vmask p2_mask = texel_partition == vint(2); + + vfloat data_r = loada(blk.data_r + i); + haccumulate(pp_avg_rgb[0][0], data_r, p0_mask); + haccumulate(pp_avg_rgb[1][0], data_r, p1_mask); + haccumulate(pp_avg_rgb[2][0], data_r, p2_mask); + + vfloat data_g = loada(blk.data_g + i); + haccumulate(pp_avg_rgb[0][1], data_g, p0_mask); + haccumulate(pp_avg_rgb[1][1], data_g, p1_mask); + haccumulate(pp_avg_rgb[2][1], data_g, p2_mask); + + vfloat data_b = loada(blk.data_b + i); + haccumulate(pp_avg_rgb[0][2], data_b, p0_mask); + haccumulate(pp_avg_rgb[1][2], data_b, p1_mask); + haccumulate(pp_avg_rgb[2][2], data_b, p2_mask); + } + + if (texel_count_simd < texel_count) + { + size_t i = texel_count_simd; + vint texel_partition(pi.partition_of_texel + i); + vmask lane_mask = vint::lane_id() < vint_from_size(texel_count - texel_count_simd); vmask p0_mask = lane_mask & (texel_partition == vint(0)); vmask p1_mask = lane_mask & (texel_partition == vint(1)); @@ -234,14 +289,30 @@ static void compute_partition_averages_rgba( { vfloat4 pp_avg_rgba[4] {}; - vint lane_id = vint::lane_id(); - for (size_t i = 0; i < texel_count; i += ASTCENC_SIMD_WIDTH) + size_t texel_count_simd = round_down_to_simd_multiple_vla(texel_count); + for (size_t i = 0; i < texel_count_simd; i += ASTCENC_SIMD_WIDTH) { vint texel_partition(pi.partition_of_texel + i); + vmask p0_mask = texel_partition == vint(0); + + vfloat data_r = loada(blk.data_r + i); + haccumulate(pp_avg_rgba[0], data_r, p0_mask); - vmask lane_mask = lane_id < vint_from_size(texel_count); - lane_id += vint(ASTCENC_SIMD_WIDTH); + vfloat data_g = loada(blk.data_g + i); + haccumulate(pp_avg_rgba[1], data_g, p0_mask); + vfloat data_b = loada(blk.data_b + i); + haccumulate(pp_avg_rgba[2], data_b, p0_mask); + + vfloat data_a = loada(blk.data_a + i); + haccumulate(pp_avg_rgba[3], data_a, p0_mask); + } + + if (texel_count_simd < texel_count) + { + size_t i = texel_count_simd; + vint texel_partition(pi.partition_of_texel + i); + vmask lane_mask = vint::lane_id() < vint_from_size(texel_count - texel_count_simd); vmask p0_mask = lane_mask & (texel_partition == vint(0)); vfloat data_r = loada(blk.data_r + i); @@ -274,13 +345,36 @@ static void compute_partition_averages_rgba( { vfloat4 pp_avg_rgba[2][4] {}; - vint lane_id = vint::lane_id(); - for (size_t i = 0; i < texel_count; i += ASTCENC_SIMD_WIDTH) + size_t texel_count_simd = round_down_to_simd_multiple_vla(texel_count); + for (size_t i = 0; i < texel_count_simd; i += ASTCENC_SIMD_WIDTH) { vint texel_partition(pi.partition_of_texel + i); - vmask lane_mask = lane_id < vint_from_size(texel_count); - lane_id += vint(ASTCENC_SIMD_WIDTH); + vmask p0_mask = texel_partition == vint(0); + vmask p1_mask = texel_partition == vint(1); + + vfloat data_r = loada(blk.data_r + i); + haccumulate(pp_avg_rgba[0][0], data_r, p0_mask); + haccumulate(pp_avg_rgba[1][0], data_r, p1_mask); + + vfloat data_g = loada(blk.data_g + i); + haccumulate(pp_avg_rgba[0][1], data_g, p0_mask); + haccumulate(pp_avg_rgba[1][1], data_g, p1_mask); + + vfloat data_b = loada(blk.data_b + i); + haccumulate(pp_avg_rgba[0][2], data_b, p0_mask); + haccumulate(pp_avg_rgba[1][2], data_b, p1_mask); + + vfloat data_a = loada(blk.data_a + i); + haccumulate(pp_avg_rgba[0][3], data_a, p0_mask); + haccumulate(pp_avg_rgba[1][3], data_a, p1_mask); + } + + if (texel_count_simd < texel_count) + { + size_t i = texel_count_simd; + vint texel_partition(pi.partition_of_texel + i); + vmask lane_mask = vint::lane_id() < vint_from_size(texel_count - texel_count_simd); vmask p0_mask = lane_mask & (texel_partition == vint(0)); vmask p1_mask = lane_mask & (texel_partition == vint(1)); @@ -325,13 +419,41 @@ static void compute_partition_averages_rgba( // For 4 partitions scan results for partition 0/1/2, compute partition 3 vfloat4 pp_avg_rgba[3][4] {}; - vint lane_id = vint::lane_id(); - for (size_t i = 0; i < texel_count; i += ASTCENC_SIMD_WIDTH) + size_t texel_count_simd = round_down_to_simd_multiple_vla(texel_count); + for (size_t i = 0; i < texel_count_simd; i += ASTCENC_SIMD_WIDTH) { vint texel_partition(pi.partition_of_texel + i); - vmask lane_mask = lane_id < vint_from_size(texel_count); - lane_id += vint(ASTCENC_SIMD_WIDTH); + vmask p0_mask = texel_partition == vint(0); + vmask p1_mask = texel_partition == vint(1); + vmask p2_mask = texel_partition == vint(2); + + vfloat data_r = loada(blk.data_r + i); + haccumulate(pp_avg_rgba[0][0], data_r, p0_mask); + haccumulate(pp_avg_rgba[1][0], data_r, p1_mask); + haccumulate(pp_avg_rgba[2][0], data_r, p2_mask); + + vfloat data_g = loada(blk.data_g + i); + haccumulate(pp_avg_rgba[0][1], data_g, p0_mask); + haccumulate(pp_avg_rgba[1][1], data_g, p1_mask); + haccumulate(pp_avg_rgba[2][1], data_g, p2_mask); + + vfloat data_b = loada(blk.data_b + i); + haccumulate(pp_avg_rgba[0][2], data_b, p0_mask); + haccumulate(pp_avg_rgba[1][2], data_b, p1_mask); + haccumulate(pp_avg_rgba[2][2], data_b, p2_mask); + + vfloat data_a = loada(blk.data_a + i); + haccumulate(pp_avg_rgba[0][3], data_a, p0_mask); + haccumulate(pp_avg_rgba[1][3], data_a, p1_mask); + haccumulate(pp_avg_rgba[2][3], data_a, p2_mask); + } + + if (texel_count_simd < texel_count) + { + size_t i = texel_count_simd; + vint texel_partition(pi.partition_of_texel + i); + vmask lane_mask = vint::lane_id() < vint_from_size(texel_count - texel_count_simd); vmask p0_mask = lane_mask & (texel_partition == vint(0)); vmask p1_mask = lane_mask & (texel_partition == vint(1)); @@ -771,13 +893,63 @@ void compute_error_squared_rgba( vfloat ew_b(blk.channel_weight.lane<2>()); vfloat ew_a(blk.channel_weight.lane<3>()); - // This implementation over-shoots, but this is safe as we initialize the texel_indexes - // array to extend the last value. This means min/max are not impacted, but we need to mask - // out the dummy values when we compute the line weighting. - vint lane_ids = vint::lane_id(); - for (size_t i = 0; i < texel_count; i += ASTCENC_SIMD_WIDTH) + size_t texel_count_simd = round_down_to_simd_multiple_vla(texel_count); + for (size_t i = 0; i < texel_count_simd; i += ASTCENC_SIMD_WIDTH) + { + const uint8_t* texel_idxs = texel_indexes + i; + + vfloat data_r = gatherf_byte_inds(blk.data_r, texel_idxs); + vfloat data_g = gatherf_byte_inds(blk.data_g, texel_idxs); + vfloat data_b = gatherf_byte_inds(blk.data_b, texel_idxs); + vfloat data_a = gatherf_byte_inds(blk.data_a, texel_idxs); + + vfloat uncor_param = (data_r * l_uncor_bs0) + + (data_g * l_uncor_bs1) + + (data_b * l_uncor_bs2) + + (data_a * l_uncor_bs3); + + uncor_loparamv = min(uncor_param, uncor_loparamv); + uncor_hiparamv = max(uncor_param, uncor_hiparamv); + + vfloat uncor_dist0 = (l_uncor_amod0 - data_r) + + (uncor_param * l_uncor_bs0); + vfloat uncor_dist1 = (l_uncor_amod1 - data_g) + + (uncor_param * l_uncor_bs1); + vfloat uncor_dist2 = (l_uncor_amod2 - data_b) + + (uncor_param * l_uncor_bs2); + vfloat uncor_dist3 = (l_uncor_amod3 - data_a) + + (uncor_param * l_uncor_bs3); + + vfloat uncor_err = (ew_r * uncor_dist0 * uncor_dist0) + + (ew_g * uncor_dist1 * uncor_dist1) + + (ew_b * uncor_dist2 * uncor_dist2) + + (ew_a * uncor_dist3 * uncor_dist3); + + haccumulate(uncor_errorsumv, uncor_err); + + // Process samechroma data + vfloat samec_param = (data_r * l_samec_bs0) + + (data_g * l_samec_bs1) + + (data_b * l_samec_bs2) + + (data_a * l_samec_bs3); + + vfloat samec_dist0 = samec_param * l_samec_bs0 - data_r; + vfloat samec_dist1 = samec_param * l_samec_bs1 - data_g; + vfloat samec_dist2 = samec_param * l_samec_bs2 - data_b; + vfloat samec_dist3 = samec_param * l_samec_bs3 - data_a; + + vfloat samec_err = (ew_r * samec_dist0 * samec_dist0) + + (ew_g * samec_dist1 * samec_dist1) + + (ew_b * samec_dist2 * samec_dist2) + + (ew_a * samec_dist3 * samec_dist3); + + haccumulate(samec_errorsumv, samec_err); + } + + if (texel_count_simd < texel_count) { - vmask mask = lane_ids < vint_from_size(texel_count); + size_t i = texel_count_simd; + vmask mask = vint::lane_id() < vint_from_size(texel_count - texel_count_simd); const uint8_t* texel_idxs = texel_indexes + i; vfloat data_r = gatherf_byte_inds(blk.data_r, texel_idxs); @@ -826,8 +998,6 @@ void compute_error_squared_rgba( + (ew_a * samec_dist3 * samec_dist3); haccumulate(samec_errorsumv, samec_err, mask); - - lane_ids += vint(ASTCENC_SIMD_WIDTH); } // Turn very small numbers and NaNs into a small number @@ -885,13 +1055,55 @@ void compute_error_squared_rgb( vfloat ew_g(blk.channel_weight.lane<1>()); vfloat ew_b(blk.channel_weight.lane<2>()); - // This implementation over-shoots, but this is safe as we initialize the weights array - // to extend the last value. This means min/max are not impacted, but we need to mask - // out the dummy values when we compute the line weighting. - vint lane_ids = vint::lane_id(); - for (size_t i = 0; i < texel_count; i += ASTCENC_SIMD_WIDTH) + size_t texel_count_simd = round_down_to_simd_multiple_vla(texel_count); + for (size_t i = 0; i < texel_count_simd; i += ASTCENC_SIMD_WIDTH) { - vmask mask = lane_ids < vint_from_size(texel_count); + const uint8_t* texel_idxs = texel_indexes + i; + + vfloat data_r = gatherf_byte_inds(blk.data_r, texel_idxs); + vfloat data_g = gatherf_byte_inds(blk.data_g, texel_idxs); + vfloat data_b = gatherf_byte_inds(blk.data_b, texel_idxs); + + vfloat uncor_param = (data_r * l_uncor_bs0) + + (data_g * l_uncor_bs1) + + (data_b * l_uncor_bs2); + + uncor_loparamv = min(uncor_param, uncor_loparamv); + uncor_hiparamv = max(uncor_param, uncor_hiparamv); + + vfloat uncor_dist0 = (l_uncor_amod0 - data_r) + + (uncor_param * l_uncor_bs0); + vfloat uncor_dist1 = (l_uncor_amod1 - data_g) + + (uncor_param * l_uncor_bs1); + vfloat uncor_dist2 = (l_uncor_amod2 - data_b) + + (uncor_param * l_uncor_bs2); + + vfloat uncor_err = (ew_r * uncor_dist0 * uncor_dist0) + + (ew_g * uncor_dist1 * uncor_dist1) + + (ew_b * uncor_dist2 * uncor_dist2); + + haccumulate(uncor_errorsumv, uncor_err); + + // Process samechroma data + vfloat samec_param = (data_r * l_samec_bs0) + + (data_g * l_samec_bs1) + + (data_b * l_samec_bs2); + + vfloat samec_dist0 = samec_param * l_samec_bs0 - data_r; + vfloat samec_dist1 = samec_param * l_samec_bs1 - data_g; + vfloat samec_dist2 = samec_param * l_samec_bs2 - data_b; + + vfloat samec_err = (ew_r * samec_dist0 * samec_dist0) + + (ew_g * samec_dist1 * samec_dist1) + + (ew_b * samec_dist2 * samec_dist2); + + haccumulate(samec_errorsumv, samec_err); + } + + if (texel_count_simd < texel_count) + { + size_t i = texel_count_simd; + vmask mask = vint::lane_id() < vint_from_size(texel_count - texel_count_simd); const uint8_t* texel_idxs = texel_indexes + i; vfloat data_r = gatherf_byte_inds(blk.data_r, texel_idxs); @@ -932,8 +1144,6 @@ void compute_error_squared_rgb( + (ew_b * samec_dist2 * samec_dist2); haccumulate(samec_errorsumv, samec_err, mask); - - lane_ids += vint(ASTCENC_SIMD_WIDTH); } // Turn very small numbers and NaNs into a small number diff --git a/Source/astcenc_color_quantize.cpp b/Source/astcenc_color_quantize.cpp index 4c015a68a..60113626a 100644 --- a/Source/astcenc_color_quantize.cpp +++ b/Source/astcenc_color_quantize.cpp @@ -90,10 +90,11 @@ static inline vint4 quant_color3( vint4 value ) { vint4 index = value * 2 + 1; + const uint8_t* table = color_unquant_to_uquant_tables[quant_level - QUANT_6]; return vint4( - color_unquant_to_uquant_tables[quant_level - QUANT_6][index.lane<0>()], - color_unquant_to_uquant_tables[quant_level - QUANT_6][index.lane<1>()], - color_unquant_to_uquant_tables[quant_level - QUANT_6][index.lane<2>()], + table[index.lane<0>()], + table[index.lane<1>()], + table[index.lane<2>()], 0); } @@ -146,10 +147,11 @@ static inline vint4 quant_color3( vmask4 mask = residual >= vfloat4(-0.1f); index = select(index, index + 1, mask); + const uint8_t* table = color_unquant_to_uquant_tables[quant_level - QUANT_6]; return vint4( - color_unquant_to_uquant_tables[quant_level - QUANT_6][index.lane<0>()], - color_unquant_to_uquant_tables[quant_level - QUANT_6][index.lane<1>()], - color_unquant_to_uquant_tables[quant_level - QUANT_6][index.lane<2>()], + table[index.lane<0>()], + table[index.lane<1>()], + table[index.lane<2>()], 0); } diff --git a/Source/astcenc_compress_symbolic.cpp b/Source/astcenc_compress_symbolic.cpp index a53d749e4..f94f1cbd9 100644 --- a/Source/astcenc_compress_symbolic.cpp +++ b/Source/astcenc_compress_symbolic.cpp @@ -278,20 +278,24 @@ static bool realign_weights_decimated( for (unsigned int te_idx = 0; te_idx < texels_to_evaluate; te_idx++) { unsigned int texel = di.weight_texels_tr[te_idx][we_idx]; - float tw_base = di.texel_contrib_for_weight[te_idx][we_idx]; - float weight_base = (uq_weightsf[di.texel_weights_tr[0][texel]] * di.texel_weight_contribs_float_tr[0][texel] - + uq_weightsf[di.texel_weights_tr[1][texel]] * di.texel_weight_contribs_float_tr[1][texel]) - + (uq_weightsf[di.texel_weights_tr[2][texel]] * di.texel_weight_contribs_float_tr[2][texel] - + uq_weightsf[di.texel_weights_tr[3][texel]] * di.texel_weight_contribs_float_tr[3][texel]); + float weight_base; + if (di.max_texel_weight_count <= 2) + { + weight_base = uq_weightsf[di.texel_weights_tr[0][texel]] * di.texel_weight_contribs_float_tr[0][texel] + + uq_weightsf[di.texel_weights_tr[1][texel]] * di.texel_weight_contribs_float_tr[1][texel]; + } + else + { + weight_base = (uq_weightsf[di.texel_weights_tr[0][texel]] * di.texel_weight_contribs_float_tr[0][texel] + + uq_weightsf[di.texel_weights_tr[1][texel]] * di.texel_weight_contribs_float_tr[1][texel]) + + (uq_weightsf[di.texel_weights_tr[2][texel]] * di.texel_weight_contribs_float_tr[2][texel] + + uq_weightsf[di.texel_weights_tr[3][texel]] * di.texel_weight_contribs_float_tr[3][texel]); + } - // Ideally this is integer rounded, but IQ gain it isn't worth the overhead - // float weight = astc::flt_rd(weight_base + 0.5f); - // float weight_down = astc::flt_rd(weight_base + 0.5f + uqw_diff_down * tw_base) - weight; - // float weight_up = astc::flt_rd(weight_base + 0.5f + uqw_diff_up * tw_base) - weight; - float weight_down = weight_base + uqw_diff_down * tw_base - weight_base; - float weight_up = weight_base + uqw_diff_up * tw_base - weight_base; + float step_down = uqw_diff_down * tw_base; + float step_up = uqw_diff_up * tw_base; unsigned int partition = pi.partition_of_texel[texel]; vfloat4 color_offset = offset[partition]; @@ -301,8 +305,8 @@ static bool realign_weights_decimated( vfloat4 orig_color = blk.texel(texel); vfloat4 color_diff = color - orig_color; - vfloat4 color_down_diff = color_diff + color_offset * weight_down; - vfloat4 color_up_diff = color_diff + color_offset * weight_up; + vfloat4 color_down_diff = color_diff + color_offset * step_down; + vfloat4 color_up_diff = color_diff + color_offset * step_up; error_basev += color_diff * color_diff; error_downv += color_down_diff * color_down_diff; diff --git a/Source/astcenc_find_best_partitioning.cpp b/Source/astcenc_find_best_partitioning.cpp index 8989a568d..4e5c0c485 100644 --- a/Source/astcenc_find_best_partitioning.cpp +++ b/Source/astcenc_find_best_partitioning.cpp @@ -626,7 +626,7 @@ unsigned int find_best_partition_candidates( uncor_lines[j].a = pm.avg; uncor_lines[j].b = normalize_safe(pm.dir, unit4()); - uncor_plines[j].amod = uncor_lines[j].a - uncor_lines[j].b * dot(uncor_lines[j].a, uncor_lines[j].b); + uncor_plines[j].amod = uncor_lines[j].a - uncor_lines[j].b * dot_s(uncor_lines[j].a, uncor_lines[j].b); uncor_plines[j].bs = uncor_lines[j].b; samec_lines[j].a = vfloat4::zero(); @@ -660,13 +660,9 @@ unsigned int find_best_partition_candidates( for (unsigned int j = 0; j < partition_count; j++) { float tpp = static_cast(pi.partition_texel_count[j]); - vfloat4 error_weights(tpp * weight_imprecision_estim); - - vfloat4 uncor_vector = uncor_lines[j].b * line_lengths[j]; - vfloat4 samec_vector = samec_lines[j].b * line_lengths[j]; - - uncor_error += dot_s(uncor_vector * uncor_vector, error_weights); - samec_error += dot_s(samec_vector * samec_vector, error_weights); + float imprecision_error = (line_lengths[j] * line_lengths[j]) * (tpp * weight_imprecision_estim); + uncor_error += imprecision_error; + samec_error += imprecision_error; } insert_result(requested_candidates, uncor_error, partition, uncor_best_errors, uncor_best_partitions); @@ -697,7 +693,7 @@ unsigned int find_best_partition_candidates( pl.samec_line.a = vfloat4::zero(); pl.samec_line.b = normalize_safe(pm.avg, unit3()); - pl.uncor_pline.amod = pl.uncor_line.a - pl.uncor_line.b * dot3(pl.uncor_line.a, pl.uncor_line.b); + pl.uncor_pline.amod = pl.uncor_line.a - pl.uncor_line.b * dot3_s(pl.uncor_line.a, pl.uncor_line.b); pl.uncor_pline.bs = pl.uncor_line.b; pl.samec_pline.amod = vfloat4::zero(); @@ -726,15 +722,10 @@ unsigned int find_best_partition_candidates( for (unsigned int j = 0; j < partition_count; j++) { partition_lines3& pl = plines[j]; - float tpp = static_cast(pi.partition_texel_count[j]); - vfloat4 error_weights(tpp * weight_imprecision_estim); - - vfloat4 uncor_vector = pl.uncor_line.b * pl.line_length; - vfloat4 samec_vector = pl.samec_line.b * pl.line_length; - - uncor_error += dot3_s(uncor_vector * uncor_vector, error_weights); - samec_error += dot3_s(samec_vector * samec_vector, error_weights); + float imprecision_error = (pl.line_length * pl.line_length) * (tpp * weight_imprecision_estim); + uncor_error += imprecision_error; + samec_error += imprecision_error; } insert_result(requested_candidates, uncor_error, partition, uncor_best_errors, uncor_best_partitions); diff --git a/Source/astcenc_ideal_endpoints_and_weights.cpp b/Source/astcenc_ideal_endpoints_and_weights.cpp index bd2e4ba28..d39f31cd7 100644 --- a/Source/astcenc_ideal_endpoints_and_weights.cpp +++ b/Source/astcenc_ideal_endpoints_and_weights.cpp @@ -427,12 +427,16 @@ static void compute_ideal_colors_and_weights_3_comp( float lowparam { 1e10f }; float highparam { -1e10f }; + float base_param = dot3_s(line.a, line.b); + float bx = line.b.lane<0>(); + float by = line.b.lane<1>(); + float bz = line.b.lane<2>(); + unsigned int partition_texel_count = pi.partition_texel_count[i]; for (unsigned int j = 0; j < partition_texel_count; j++) { unsigned int tix = pi.texels_of_partition[i][j]; - vfloat4 point = vfloat3(data_vr[tix], data_vg[tix], data_vb[tix]); - float param = dot3_s(point - line.a, line.b); + float param = (data_vr[tix] * bx + data_vg[tix] * by + data_vb[tix] * bz) - base_param; ei.weights[tix] = param; lowparam = astc::min(param, lowparam); @@ -501,7 +505,7 @@ static void compute_ideal_colors_and_weights_3_comp( // Zero initialize any SIMD over-fetch size_t texel_count_simd = round_up_to_simd_multiple_vla(texel_count); - for (size_t i = texel_count; i < texel_count_simd; i++) + for (unsigned int i = texel_count; i < texel_count_simd; i++) { ei.weights[i] = 0.0f; ei.weight_error_scale[i] = 0.0f; @@ -522,16 +526,15 @@ static void compute_ideal_colors_and_weights_4_comp( const partition_info& pi, endpoints_and_weights& ei ) { - const float error_weight = hadd_s(blk.channel_weight) / 4.0f; - unsigned int partition_count = pi.partition_count; - - unsigned int texel_count = blk.texel_count; - promise(texel_count > 0); + size_t texel_count = blk.texel_count; + size_t texel_count_simd = round_up_to_simd_multiple_vla(texel_count); promise(partition_count > 0); + promise(texel_count > 0); - partition_metrics pms[BLOCK_MAX_PARTITIONS]; + float error_weight = hadd_s(blk.channel_weight) * (1.0f / 4.0f); + partition_metrics pms[BLOCK_MAX_PARTITIONS]; compute_avgs_and_dirs_4_comp(pi, blk, pms); bool is_constant_wes { true }; @@ -549,12 +552,17 @@ static void compute_ideal_colors_and_weights_4_comp( float lowparam { 1e10f }; float highparam { -1e10f }; + float base_param = dot_s(line.a, line.b); + float bx = line.b.lane<0>(); + float by = line.b.lane<1>(); + float bz = line.b.lane<2>(); + float ba = line.b.lane<3>(); + unsigned int partition_texel_count = pi.partition_texel_count[i]; for (unsigned int j = 0; j < partition_texel_count; j++) { unsigned int tix = pi.texels_of_partition[i][j]; - vfloat4 point = blk.texel(tix); - float param = dot_s(point - line.a, line.b); + float param = (blk.data_r[tix] * bx + blk.data_g[tix] * by + blk.data_b[tix] * bz + blk.data_a[tix] * ba) - base_param; ei.weights[tix] = param; lowparam = astc::min(param, lowparam); @@ -598,7 +606,6 @@ static void compute_ideal_colors_and_weights_4_comp( } // Zero initialize any SIMD over-fetch - size_t texel_count_simd = round_up_to_simd_multiple_vla(texel_count); for (size_t i = texel_count; i < texel_count_simd; i++) { ei.weights[i] = 0.0f; @@ -870,38 +877,62 @@ void compute_ideal_weights_for_decimation( // Compute an initial average for each decimated weight bool constant_wes = ei.is_constant_weight_error_scale; - vfloat weight_error_scale(ei.weight_error_scale[0]); // This overshoots - this is OK as we initialize the array tails in the // decimation table structures to safe values ... - for (unsigned int i = 0; i < weight_count; i += ASTCENC_SIMD_WIDTH) + if (constant_wes) { - // Start with a small value to avoid div-by-zero later - vfloat weight_weight(1e-10f); - vfloat initial_weight = vfloat::zero(); + for (unsigned int i = 0; i < weight_count; i += ASTCENC_SIMD_WIDTH) + { + // Start with a small value to avoid div-by-zero later + vfloat weight_weight(1e-10f); + vfloat initial_weight = vfloat::zero(); - // Accumulate error weighting of all the texels using this weight - vint weight_texel_count(di.weight_texel_count + i); - unsigned int max_texel_count = hmax_s(weight_texel_count); - promise(max_texel_count > 0); + // Accumulate error weighting of all the texels using this weight + vint weight_texel_count(di.weight_texel_count + i); + unsigned int max_texel_count = hmax_s(weight_texel_count); + promise(max_texel_count > 0); - for (unsigned int j = 0; j < max_texel_count; j++) + for (unsigned int j = 0; j < max_texel_count; j++) + { + const uint8_t* texel = di.weight_texels_tr[j] + i; + vfloat weight = loada(di.weights_texel_contribs_tr[j] + i); + + weight_weight += weight; + initial_weight += gatherf_byte_inds(ei.weights, texel) * weight; + } + + storea(initial_weight / weight_weight, dec_weight_ideal_value + i); + } + } + else + { + vfloat weight_error_scale(ei.weight_error_scale[0]); + for (unsigned int i = 0; i < weight_count; i += ASTCENC_SIMD_WIDTH) { - const uint8_t* texel = di.weight_texels_tr[j] + i; - vfloat weight = loada(di.weights_texel_contribs_tr[j] + i); + // Start with a small value to avoid div-by-zero later + vfloat weight_weight(1e-10f); + vfloat initial_weight = vfloat::zero(); - if (!constant_wes) + // Accumulate error weighting of all the texels using this weight + vint weight_texel_count(di.weight_texel_count + i); + unsigned int max_texel_count = hmax_s(weight_texel_count); + promise(max_texel_count > 0); + + for (unsigned int j = 0; j < max_texel_count; j++) { + const uint8_t* texel = di.weight_texels_tr[j] + i; + vfloat weight = loada(di.weights_texel_contribs_tr[j] + i); weight_error_scale = gatherf_byte_inds(ei.weight_error_scale, texel); - } - vfloat contrib_weight = weight * weight_error_scale; + vfloat contrib_weight = weight * weight_error_scale; - weight_weight += contrib_weight; - initial_weight += gatherf_byte_inds(ei.weights, texel) * contrib_weight; - } + weight_weight += contrib_weight; + initial_weight += gatherf_byte_inds(ei.weights, texel) * contrib_weight; + } - storea(initial_weight / weight_weight, dec_weight_ideal_value + i); + storea(initial_weight / weight_weight, dec_weight_ideal_value + i); + } } // Populate the interpolated weight grid based on the initial average @@ -930,43 +961,78 @@ void compute_ideal_weights_for_decimation( constexpr float stepsize = 0.25f; constexpr float chd_scale = -WEIGHTS_TEXEL_SUM; - for (unsigned int i = 0; i < weight_count; i += ASTCENC_SIMD_WIDTH) + if (constant_wes) { - vfloat weight_val = loada(dec_weight_ideal_value + i); + for (unsigned int i = 0; i < weight_count; i += ASTCENC_SIMD_WIDTH) + { + vfloat weight_val = loada(dec_weight_ideal_value + i); + + // Accumulate error weighting of all the texels using this weight + // Start with a small value to avoid div-by-zero later + vfloat error_change0(1e-10f); + vfloat error_change1(0.0f); + + // Accumulate error weighting of all the texels using this weight + vint weight_texel_count(di.weight_texel_count + i); + unsigned int max_texel_count = hmax_s(weight_texel_count); + promise(max_texel_count > 0); - // Accumulate error weighting of all the texels using this weight - // Start with a small value to avoid div-by-zero later - vfloat error_change0(1e-10f); - vfloat error_change1(0.0f); + for (unsigned int j = 0; j < max_texel_count; j++) + { + const uint8_t* texel = di.weight_texels_tr[j] + i; + vfloat contrib_weight = loada(di.weights_texel_contribs_tr[j] + i); + + vfloat old_weight = gatherf_byte_inds(infilled_weights, texel); + vfloat ideal_weight = gatherf_byte_inds(ei.weights, texel); - // Accumulate error weighting of all the texels using this weight - vint weight_texel_count(di.weight_texel_count + i); - unsigned int max_texel_count = hmax_s(weight_texel_count); - promise(max_texel_count > 0); + error_change0 += contrib_weight * contrib_weight; + error_change1 += (old_weight - ideal_weight) * contrib_weight; + } - for (unsigned int j = 0; j < max_texel_count; j++) + vfloat step = (error_change1 * chd_scale) / error_change0; + step = clamp(-stepsize, stepsize, step); + + // Update the weight; note this can store negative values + storea(weight_val + step, dec_weight_ideal_value + i); + } + } + else + { + vfloat weight_error_scale(ei.weight_error_scale[0]); + for (unsigned int i = 0; i < weight_count; i += ASTCENC_SIMD_WIDTH) { - const uint8_t* texel = di.weight_texels_tr[j] + i; - vfloat contrib_weight = loada(di.weights_texel_contribs_tr[j] + i); + vfloat weight_val = loada(dec_weight_ideal_value + i); + + // Accumulate error weighting of all the texels using this weight + // Start with a small value to avoid div-by-zero later + vfloat error_change0(1e-10f); + vfloat error_change1(0.0f); - if (!constant_wes) + // Accumulate error weighting of all the texels using this weight + vint weight_texel_count(di.weight_texel_count + i); + unsigned int max_texel_count = hmax_s(weight_texel_count); + promise(max_texel_count > 0); + + for (unsigned int j = 0; j < max_texel_count; j++) { + const uint8_t* texel = di.weight_texels_tr[j] + i; + vfloat contrib_weight = loada(di.weights_texel_contribs_tr[j] + i); weight_error_scale = gatherf_byte_inds(ei.weight_error_scale, texel); - } - vfloat scale = weight_error_scale * contrib_weight; - vfloat old_weight = gatherf_byte_inds(infilled_weights, texel); - vfloat ideal_weight = gatherf_byte_inds(ei.weights, texel); + vfloat scale = weight_error_scale * contrib_weight; + vfloat old_weight = gatherf_byte_inds(infilled_weights, texel); + vfloat ideal_weight = gatherf_byte_inds(ei.weights, texel); - error_change0 += contrib_weight * scale; - error_change1 += (old_weight - ideal_weight) * scale; - } + error_change0 += contrib_weight * scale; + error_change1 += (old_weight - ideal_weight) * scale; + } - vfloat step = (error_change1 * chd_scale) / error_change0; - step = clamp(-stepsize, stepsize, step); + vfloat step = (error_change1 * chd_scale) / error_change0; + step = clamp(-stepsize, stepsize, step); - // Update the weight; note this can store negative values - storea(weight_val + step, dec_weight_ideal_value + i); + // Update the weight; note this can store negative values + storea(weight_val + step, dec_weight_ideal_value + i); + } } } diff --git a/Source/astcenc_pick_best_endpoint_format.cpp b/Source/astcenc_pick_best_endpoint_format.cpp index 485785def..ae5956e37 100644 --- a/Source/astcenc_pick_best_endpoint_format.cpp +++ b/Source/astcenc_pick_best_endpoint_format.cpp @@ -120,13 +120,88 @@ static void compute_error_squared_rgb_single_partition( vfloat l_bs1(l_pline.bs.lane<1>()); vfloat l_bs2(l_pline.bs.lane<2>()); - vint lane_ids = vint::lane_id(); - for (unsigned int i = 0; i < texel_count; i += ASTCENC_SIMD_WIDTH) + size_t texel_count_simd = round_down_to_simd_multiple_vla(texel_count); + for (unsigned int i = 0; i < texel_count_simd; i += ASTCENC_SIMD_WIDTH) { const uint8_t* tix = texel_indexes + i; - vmask mask = lane_ids < vint(texel_count); - lane_ids += vint(ASTCENC_SIMD_WIDTH); + // Compute the error that arises from just ditching alpha + vfloat data_a = gatherf_byte_inds(blk.data_a, tix); + vfloat alpha_diff = data_a - default_a; + alpha_diff = alpha_diff * alpha_diff; + + haccumulate(a_drop_errv, alpha_diff); + + vfloat data_r = gatherf_byte_inds(blk.data_r, tix); + vfloat data_g = gatherf_byte_inds(blk.data_g, tix); + vfloat data_b = gatherf_byte_inds(blk.data_b, tix); + + // Compute uncorrelated error + vfloat param = data_r * uncor_bs0 + + data_g * uncor_bs1 + + data_b * uncor_bs2; + + vfloat dist0 = (uncor_amod0 + param * uncor_bs0) - data_r; + vfloat dist1 = (uncor_amod1 + param * uncor_bs1) - data_g; + vfloat dist2 = (uncor_amod2 + param * uncor_bs2) - data_b; + + vfloat error = dist0 * dist0 * ews.lane<0>() + + dist1 * dist1 * ews.lane<1>() + + dist2 * dist2 * ews.lane<2>(); + + haccumulate(uncor_errv, error); + + // Compute same chroma error - no "amod", it's always zero + param = data_r * samec_bs0 + + data_g * samec_bs1 + + data_b * samec_bs2; + + dist0 = (param * samec_bs0) - data_r; + dist1 = (param * samec_bs1) - data_g; + dist2 = (param * samec_bs2) - data_b; + + error = dist0 * dist0 * ews.lane<0>() + + dist1 * dist1 * ews.lane<1>() + + dist2 * dist2 * ews.lane<2>(); + + haccumulate(samec_errv, error); + + // Compute rgbl error + param = data_r * rgbl_bs0 + + data_g * rgbl_bs1 + + data_b * rgbl_bs2; + + dist0 = (rgbl_amod0 + param * rgbl_bs0) - data_r; + dist1 = (rgbl_amod1 + param * rgbl_bs1) - data_g; + dist2 = (rgbl_amod2 + param * rgbl_bs2) - data_b; + + error = dist0 * dist0 * ews.lane<0>() + + dist1 * dist1 * ews.lane<1>() + + dist2 * dist2 * ews.lane<2>(); + + haccumulate(rgbl_errv, error); + + // Compute luma error - no "amod", it's always zero + param = data_r * l_bs0 + + data_g * l_bs1 + + data_b * l_bs2; + + dist0 = (param * l_bs0) - data_r; + dist1 = (param * l_bs1) - data_g; + dist2 = (param * l_bs2) - data_b; + + error = dist0 * dist0 * ews.lane<0>() + + dist1 * dist1 * ews.lane<1>() + + dist2 * dist2 * ews.lane<2>(); + + haccumulate(l_errv, error); + } + + if (texel_count_simd < texel_count) + { + unsigned int i = static_cast(texel_count_simd); + const uint8_t* tix = texel_indexes + i; + vmask mask = vint::lane_id() < vint(texel_count - texel_count_simd); // Compute the error that arises from just ditching alpha vfloat data_a = gatherf_byte_inds(blk.data_a, tix); diff --git a/Source/cmake_core.cmake b/Source/cmake_core.cmake index daf06f486..1c4372218 100644 --- a/Source/cmake_core.cmake +++ b/Source/cmake_core.cmake @@ -421,8 +421,8 @@ macro(astcenc_set_properties ASTCENC_TARGET_NAME ASTCENC_VENEER_TYPE) target_compile_options(${ASTCENC_TARGET_NAME} PRIVATE $<${is_msvc_fe}:/arch:AVX2> - $<${is_clangcl}:-mavx2 -mpopcnt -mf16c> - $<${is_gnu_fe}:-mavx2 -mpopcnt -mf16c> + $<${is_clangcl}:-mavx2 -mpopcnt -mf16c -mbmi -mbmi2 -mlzcnt> + $<${is_gnu_fe}:-mavx2 -mpopcnt -mf16c -mbmi -mbmi2 -mlzcnt> $<${is_gnu_fe}:-Wno-unused-command-line-argument>) endif()