From f02a7a7a592c758a6a1a47ad9ab54c1371c762ba Mon Sep 17 00:00:00 2001 From: prudvi Date: Thu, 16 Jul 2026 11:10:17 +0530 Subject: [PATCH 1/4] strip out non-teddy engines --- CMakeLists.txt | 24 ++++++------ src/fdr/fdr.c | 4 +- src/hwlm/hwlm.c | 12 ++++-- src/hwlm/hwlm_build.cpp | 86 +++++++++++++++++++++++++++++++++-------- src/hwlm/hwlm_build.h | 20 +++++----- src/nfa/nfa_rev_api.h | 4 +- 6 files changed, 106 insertions(+), 44 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 80492d1ef..601a91b02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -626,9 +626,9 @@ set (hs_exec_SRCS src/hwlm/hwlm.c src/hwlm/hwlm.h src/hwlm/hwlm_internal.h - src/hwlm/noodle_engine.c - src/hwlm/noodle_engine.h - src/hwlm/noodle_internal.h + #src/hwlm/noodle_engine.c + #src/hwlm/noodle_engine.h + #src/hwlm/noodle_internal.h src/nfa/accel.c src/nfa/accel.h src/nfa/castle.c @@ -771,14 +771,14 @@ SET (hs_compile_SRCS src/compiler/expression_info.h src/fdr/engine_description.cpp src/fdr/engine_description.h - src/fdr/fdr_compile.cpp - src/fdr/fdr_compile.h - src/fdr/fdr_compile_internal.h + #src/fdr/fdr_compile.cpp + #src/fdr/fdr_compile.h + #src/fdr/fdr_compile_internal.h src/fdr/fdr_compile_util.cpp src/fdr/fdr_confirm_compile.cpp src/fdr/fdr_confirm.h - src/fdr/fdr_engine_description.cpp - src/fdr/fdr_engine_description.h + #src/fdr/fdr_engine_description.cpp + #src/fdr/fdr_engine_description.h src/fdr/fdr_internal.h src/fdr/flood_compile.cpp src/fdr/teddy_compile.cpp @@ -791,9 +791,9 @@ SET (hs_compile_SRCS src/hwlm/hwlm_internal.h src/hwlm/hwlm_literal.cpp src/hwlm/hwlm_literal.h - src/hwlm/noodle_build.cpp - src/hwlm/noodle_build.h - src/hwlm/noodle_internal.h + #src/hwlm/noodle_build.cpp + #src/hwlm/noodle_build.h + #src/hwlm/noodle_internal.h src/nfa/accel.h src/nfa/accel_dfa_build_strat.cpp src/nfa/accel_dfa_build_strat.h @@ -1136,7 +1136,7 @@ SET (hs_compile_SRCS set(hs_dump_SRCS src/scratch_dump.cpp src/scratch_dump.h - src/fdr/fdr_dump.cpp + #src/fdr/fdr_dump.cpp src/hwlm/hwlm_dump.cpp src/hwlm/hwlm_dump.h src/nfa/accel_dump.cpp diff --git a/src/fdr/fdr.c b/src/fdr/fdr.c index d33756d35..3c0cc6096 100644 --- a/src/fdr/fdr.c +++ b/src/fdr/fdr.c @@ -42,6 +42,7 @@ /** \brief number of bytes processed in each iteration */ #define ITER_BYTES 16 +#if 0 /* FDR engine removed - Teddy-only mode */ /** \brief total zone buffer size */ #define ZONE_TOTAL_SIZE 64 @@ -788,6 +789,7 @@ hwlm_error_t fdr_engine_exec(const struct FDR *fdr, return HWLM_SUCCESS; } +#endif /* FDR engine removed */ #if defined(HAVE_AVX2) #define ONLY_AVX2(func) func @@ -800,7 +802,7 @@ typedef hwlm_error_t (*FDRFUNCTYPE)(const struct FDR *fdr, hwlm_group_t control); static const FDRFUNCTYPE funcs[] = { - fdr_engine_exec, + /* fdr_engine_exec, */ NULL, /* FDR removed - Teddy-only mode */ NULL, /* old: fast teddy */ NULL, /* old: fast teddy */ ONLY_AVX2(fdr_exec_fat_teddy_msks1), diff --git a/src/hwlm/hwlm.c b/src/hwlm/hwlm.c index 8cf585a98..745f67ed8 100644 --- a/src/hwlm/hwlm.c +++ b/src/hwlm/hwlm.c @@ -31,7 +31,7 @@ */ #include "hwlm.h" #include "hwlm_internal.h" -#include "noodle_engine.h" +//#include "noodle_engine.h" /* Teddy-only mode: noodle removed */ #include "scratch.h" #include "ue2common.h" #include "fdr/fdr.h" @@ -182,11 +182,14 @@ hwlm_error_t hwlmExec(const struct HWLM *t, const u8 *buf, size_t len, assert(start < len); + /* if (t->type == HWLM_ENGINE_NOOD) { DEBUG_PRINTF("calling noodExec\n"); return noodExec(HWLM_C_DATA(t), buf, len, start, cb, scratch); } + */ + /* Teddy-only mode: always use fdrExec (dispatches to Teddy) */ assert(t->type == HWLM_ENGINE_FDR); const union AccelAux *aa = &t->accel0; if ((groups & ~t->accel1_groups) == 0) { @@ -194,7 +197,7 @@ hwlm_error_t hwlmExec(const struct HWLM *t, const u8 *buf, size_t len, aa = &t->accel1; } do_accel_block(aa, buf, len, &start); - DEBUG_PRINTF("calling frankie (groups=%08llx, start=%zu)\n", groups, start); + DEBUG_PRINTF("calling teddy via fdrExec (groups=%08llx, start=%zu)\n", groups, start); return fdrExec(HWLM_C_DATA(t), buf, len, start, cb, scratch, groups); } @@ -217,6 +220,7 @@ hwlm_error_t hwlmExecStreaming(const struct HWLM *t, size_t len, size_t start, assert(start < len); + /* if (t->type == HWLM_ENGINE_NOOD) { DEBUG_PRINTF("calling noodExec\n"); // If we've been handed a start offset, we can use a block mode scan at @@ -228,7 +232,9 @@ hwlm_error_t hwlmExecStreaming(const struct HWLM *t, size_t len, size_t start, scratch); } } + */ + /* Teddy-only mode: always use fdrExecStreaming (dispatches to Teddy) */ assert(t->type == HWLM_ENGINE_FDR); const union AccelAux *aa = &t->accel0; if ((groups & ~t->accel1_groups) == 0) { @@ -236,7 +242,7 @@ hwlm_error_t hwlmExecStreaming(const struct HWLM *t, size_t len, size_t start, aa = &t->accel1; } do_accel_streaming(aa, hbuf, hlen, buf, len, &start); - DEBUG_PRINTF("calling frankie (groups=%08llx, start=%zu)\n", groups, start); + DEBUG_PRINTF("calling teddy via fdrExecStreaming (groups=%08llx, start=%zu)\n", groups, start); return fdrExecStreaming(HWLM_C_DATA(t), hbuf, hlen, buf, len, start, cb, scratch, groups); } diff --git a/src/hwlm/hwlm_build.cpp b/src/hwlm/hwlm_build.cpp index 1b3328152..b0ab87dfa 100644 --- a/src/hwlm/hwlm_build.cpp +++ b/src/hwlm/hwlm_build.cpp @@ -36,13 +36,17 @@ #include "hwlm.h" #include "hwlm_internal.h" #include "hwlm_literal.h" -#include "noodle_engine.h" -#include "noodle_build.h" +//#include "noodle_engine.h" /* Teddy-only mode: noodle removed */ +//#include "noodle_build.h" /* Teddy-only mode: noodle removed */ #include "scratch.h" #include "ue2common.h" -#include "fdr/fdr_compile.h" -#include "fdr/fdr_compile_internal.h" -#include "fdr/fdr_engine_description.h" +//#include "fdr/fdr_compile.h" /* Teddy-only mode: FDR compile removed */ +//#include "fdr/fdr_compile_internal.h" /* Teddy-only mode: FDR compile removed */ +//#include "fdr/fdr_engine_description.h" /* Teddy-only mode: FDR engine desc removed */ +#include "fdr/fdr.h" +#include "fdr/fdr_internal.h" /* for struct FDR and fdr->size */ +#include "fdr/fdr_compile_internal.h" /* for HINT_INVALID */ +#include "fdr/teddy_compile.h" #include "fdr/teddy_engine_description.h" #include "util/compile_context.h" #include "util/compile_error.h" @@ -60,13 +64,15 @@ namespace ue2 { HWLMProto::HWLMProto(u8 engType_in, vector lits_in) : engType(engType_in), lits(move(lits_in)) {} +/* HWLMProto::HWLMProto(u8 engType_in, - unique_ptr eng_in, - vector lits_in, - map> bucketToLits_in, - bool make_small_in) - : engType(engType_in), fdrEng(move(eng_in)), lits(move(lits_in)), - bucketToLits(move(bucketToLits_in)), make_small(make_small_in) {} + unique_ptr eng_in, + vector lits_in, + map> bucketToLits_in, + bool make_small_in) + : engType(engType_in), fdrEng(move(eng_in)), lits(move(lits_in)), + bucketToLits(move(bucketToLits_in)), make_small(make_small_in) {} +*/ HWLMProto::HWLMProto(u8 engType_in, unique_ptr eng_in, @@ -103,6 +109,7 @@ bool everyoneHasGroups(const vector &lits) { } #endif +/* static bool isNoodleable(const vector &lits, const CompileContext &cc) { @@ -117,15 +124,17 @@ bool isNoodleable(const vector &lits, return true; } +*/ bytecode_ptr hwlmBuild(const HWLMProto &proto, const CompileContext &cc, UNUSED hwlm_group_t expected_groups) { size_t engSize = 0; shared_ptr eng; - const auto &lits = proto.lits; - DEBUG_PRINTF("building table with %zu strings\n", lits.size()); + DEBUG_PRINTF("building table with %zu strings\n", proto.lits.size()); + /* + const auto &lits = proto.lits; if (proto.engType == HWLM_ENGINE_NOOD) { DEBUG_PRINTF("build noodle table\n"); const hwlmLiteral &lit = lits.front(); @@ -142,6 +151,17 @@ bytecode_ptr hwlmBuild(const HWLMProto &proto, const CompileContext &cc, } eng = move(fdr); } + */ + + /* Teddy-only mode: always build Teddy table */ + DEBUG_PRINTF("building teddy table (teddy-only mode)\n"); + if (proto.teddyEng) { + auto fdr = teddyBuildTable(proto, cc.grey); + if (fdr) { + engSize = fdr.size(); + } + eng = move(fdr); + } if (!eng) { return nullptr; @@ -199,6 +219,7 @@ hwlmBuildProto(vector &lits, bool make_small, assert(everyoneHasGroups(lits)); + /* if (isNoodleable(lits, cc)) { DEBUG_PRINTF("build noodle table\n"); proto = ue2::make_unique(HWLM_ENGINE_NOOD, lits); @@ -210,6 +231,17 @@ hwlmBuildProto(vector &lits, bool make_small, return nullptr; } } + */ + + /* Teddy-only mode: always build Teddy */ + DEBUG_PRINTF("building teddy (teddy-only mode)\n"); + proto = teddyBuildProtoHinted(HWLM_ENGINE_FDR, lits, make_small, + HINT_INVALID, cc.target_info); + if (!proto) { + throw CompileError("Pattern set exceeds Teddy engine capacity " + "(max 96 decomposed literals). Reduce pattern count " + "or avoid large character classes/alternations."); + } return proto; } @@ -217,6 +249,7 @@ hwlmBuildProto(vector &lits, bool make_small, size_t hwlmSize(const HWLM *h) { size_t engSize = 0; + /* switch (h->type) { case HWLM_ENGINE_NOOD: engSize = noodSize((const noodTable *)HWLM_C_DATA(h)); @@ -225,6 +258,11 @@ size_t hwlmSize(const HWLM *h) { engSize = fdrSize((const FDR *)HWLM_C_DATA(h)); break; } + */ + + assert(h->type == HWLM_ENGINE_FDR); + const struct FDR *fdr = (const struct FDR *)HWLM_C_DATA(h); + engSize = fdr->size; if (!engSize) { return 0; @@ -234,15 +272,18 @@ size_t hwlmSize(const HWLM *h) { } size_t hwlmFloodProneSuffixLen(size_t numLiterals, const CompileContext &cc) { - const size_t NO_LIMIT = ~(size_t)0; + UNUSED const size_t NO_LIMIT = ~(size_t)0; + /* // NOTE: this function contains a number of magic numbers which are // conservative estimates of flood-proneness based on internal details of // the various literal engines that fall under the HWLM aegis. If you // change those engines, you might need to change this function too. + */ DEBUG_PRINTF("%zu literals\n", numLiterals); + /* if (cc.grey.allowNoodle && numLiterals <= 1) { DEBUG_PRINTF("noodle\n"); return NO_LIMIT; @@ -259,11 +300,22 @@ size_t hwlmFloodProneSuffixLen(size_t numLiterals, const CompileContext &cc) { } } - // TODO: we had thought we could push this value up to 9, but it seems that - // hurts performance on floods in some FDR models. Super-conservative for - // now. DEBUG_PRINTF("fdr\n"); return 3; + */ + + /* Teddy-only mode */ + if (numLiterals <= 48) { + DEBUG_PRINTF("teddy\n"); + return 3; + } + if (cc.target_info.has_avx2() && numLiterals <= 96) { + DEBUG_PRINTF("avx2 teddy\n"); + return 3; + } + + DEBUG_PRINTF("teddy fallback\n"); + return 3; } } // namespace ue2 diff --git a/src/hwlm/hwlm_build.h b/src/hwlm/hwlm_build.h index 91f227dce..db456d85c 100644 --- a/src/hwlm/hwlm_build.h +++ b/src/hwlm/hwlm_build.h @@ -46,7 +46,7 @@ struct HWLM; namespace ue2 { -class FDREngineDescription; +/* class FDREngineDescription; */ /* Teddy-only mode: removed */ class TeddyEngineDescription; struct CompileContext; struct Grey; @@ -54,14 +54,14 @@ struct Grey; /** \brief Class representing a literal matcher prototype. */ struct HWLMProto { /** - * \brief Engine type to distinguish noodle from FDR and Teddy. + * \brief Engine type (always HWLM_ENGINE_FDR in teddy-only mode). */ u8 engType; - /** - * \brief FDR engine description. - */ + /* + \brief FDR engine description. std::unique_ptr fdrEng; + */ /** * \brief Teddy engine description. @@ -74,7 +74,7 @@ struct HWLMProto { std::vector lits; /** - * \brief Bucket assignment info in FDR and Teddy + * \brief Bucket assignment info in Teddy */ std::map> bucketToLits; @@ -85,10 +85,12 @@ struct HWLMProto { HWLMProto(u8 engType_in, std::vector lits_in); + /* HWLMProto(u8 engType_in, std::unique_ptr eng_in, - std::vector lits_in, - std::map> bucketToLits_in, - bool make_small_in); + std::vector lits_in, + std::map> bucketToLits_in, + bool make_small_in); + */ HWLMProto(u8 engType_in, std::unique_ptr eng_in, std::vector lits_in, diff --git a/src/nfa/nfa_rev_api.h b/src/nfa/nfa_rev_api.h index ea9bc411c..7fab4c2de 100644 --- a/src/nfa/nfa_rev_api.h +++ b/src/nfa/nfa_rev_api.h @@ -104,7 +104,7 @@ size_t nfaRevAccel_i(const struct NFA *nfa, const u8 *buffer, size_t length) { break; case ACCEL_RDEOD: DEBUG_PRINTF("ACCEL_RDEOD\n"); - if (length < nfa->rAccelOffset + 1) { + if (length < (size_t)nfa->rAccelOffset + 1) { break; } if (unaligned_load_u16(buffer + length - nfa->rAccelOffset) != @@ -114,7 +114,7 @@ size_t nfaRevAccel_i(const struct NFA *nfa, const u8 *buffer, size_t length) { break; case ACCEL_RDEOD_NOCASE: DEBUG_PRINTF("ACCEL_RDEOD_NOCASE\n"); - if (length < nfa->rAccelOffset + 1) { + if (length < (size_t)nfa->rAccelOffset + 1) { break; } if ((unaligned_load_u16(buffer + length - nfa->rAccelOffset) & From 90dc4383407952501711b22f2048eb41c85a390c Mon Sep 17 00:00:00 2001 From: prudvi Date: Thu, 16 Jul 2026 11:14:46 +0530 Subject: [PATCH 2/4] revamp unit test cases for teddy engine validation --- unit/CMakeLists.txt | 2 +- unit/hyperscan/arg_checks.cpp | 55 ++++++++++++++++++++++++--------- unit/hyperscan/identical.cpp | 6 ++-- unit/hyperscan/literals.cpp | 4 +-- unit/hyperscan/single.cpp | 5 ++- unit/hyperscan/som_overflow.cpp | 29 +++++++++++------ 6 files changed, 69 insertions(+), 32 deletions(-) diff --git a/unit/CMakeLists.txt b/unit/CMakeLists.txt index 2d6cbbb5f..6ed9539d0 100644 --- a/unit/CMakeLists.txt +++ b/unit/CMakeLists.txt @@ -107,7 +107,7 @@ set(unit_internal_SOURCES internal/nfagraph_repeat.cpp internal/nfagraph_util.cpp internal/nfagraph_width.cpp - internal/noodle.cpp + #internal/noodle.cpp internal/pack_bits.cpp internal/parser.cpp internal/partial.cpp diff --git a/unit/hyperscan/arg_checks.cpp b/unit/hyperscan/arg_checks.cpp index 6a2fa650d..1dd2ab1ce 100644 --- a/unit/hyperscan/arg_checks.cpp +++ b/unit/hyperscan/arg_checks.cpp @@ -28,6 +28,10 @@ #include "config.h" +#include +#include +#include + #include "gtest/gtest.h" #include "hs.h" #include "database.h" @@ -57,36 +61,59 @@ int singleHandler(unsigned id, unsigned long long from, namespace /* anonymous */ { +static void makeDatabaseWritable(hs_database *db) { + if (!db) { + return; + } + long ps_raw = sysconf(_SC_PAGESIZE); + if (ps_raw <= 0) { + return; + } + size_t ps = (size_t)ps_raw; + if ((ps & (ps - 1)) != 0) { + return; + } + if ((uintptr_t)db & (ps - 1)) { + return; + } + size_t db_len = sizeof(struct hs_database) + db->length; + if (db_len > SIZE_MAX - ps) { + return; + } + size_t rounded = (db_len + ps - 1) & ~(ps - 1); + if (rounded < db_len || rounded == 0) { + return; + } + int ret = mprotect(db, rounded, PROT_READ | PROT_WRITE); + (void)ret; +} + // Break the magic number of the given database. void breakDatabaseMagic(hs_database *db) { + makeDatabaseWritable(db); // database magic should be 0xdbdb at the start - size_t db_len = sizeof(struct hs_database) + db->length; - hs_db_unprotect(db, db_len); ASSERT_TRUE(memcmp("\xdb\xdb", db, 2) == 0); *(char *)db = 0xdc; } // Break the version number of the given database. void breakDatabaseVersion(hs_database *db) { + makeDatabaseWritable(db); // database version is the second u32 - size_t db_len = sizeof(struct hs_database) + db->length; - hs_db_unprotect(db, db_len); *((char *)db + 4) += 1; } // Break the platform data of the given database. void breakDatabasePlatform(hs_database *db) { + makeDatabaseWritable(db); // database platform is an aligned u64a 16 bytes in - size_t db_len = sizeof(struct hs_database) + db->length; - hs_db_unprotect(db, db_len); memset((char *)db + 16, 0xff, 8); } // Break the alignment of the bytecode for the given database. void breakDatabaseBytecode(hs_database *db) { + makeDatabaseWritable(db); // bytecode ptr is a u32 at offsetof(hs_database, bytecode) - size_t db_len = sizeof(struct hs_database) + db->length; - hs_db_unprotect(db, db_len); unsigned int *bytecode = (unsigned int *)((char *)db + offsetof(struct hs_database, bytecode)); ASSERT_NE(0U, *bytecode); ASSERT_EQ(0U, (size_t)((char *)db + *bytecode) % 16U); @@ -925,7 +952,7 @@ TEST(HyperscanArgChecks, ScanBlockBrokenDatabaseMagic) { // teardown err = hs_free_scratch(scratch); ASSERT_EQ(HS_SUCCESS, err); - hs_db_free(db, db_len1); + hs_free_database(db); } // hs_scan: Call with a database with broken version @@ -1131,7 +1158,7 @@ TEST(HyperscanArgChecks, ScanVectorBrokenDatabaseMagic) { // teardown err = hs_free_scratch(scratch); ASSERT_EQ(HS_SUCCESS, err); - hs_db_free(db, db_len2); + hs_free_database(db); } // hs_scan_vector: Call with a database with broken version @@ -1408,7 +1435,7 @@ TEST(HyperscanArgChecks, AllocScratchBadDatabaseMagic) { ASSERT_EQ(HS_INVALID, err); // teardown - hs_db_free(db, db_len3); + hs_free_database(db); } // hs_alloc_scratch: Call with broken database version @@ -1490,7 +1517,7 @@ TEST(HyperscanArgChecks, AllocScratchBadDatabaseCRC) { ASSERT_EQ(HS_SUCCESS, err); // for want of a better case, corrupt the "middle byte" of the database. - hs_db_unprotect(db, len); + makeDatabaseWritable(db); char *mid = (char *)db + len/2; *mid += 17; @@ -1575,14 +1602,14 @@ TEST(HyperscanArgChecks, StreamSizeBogusDatabase) { ASSERT_EQ(HS_SUCCESS, err); ASSERT_LT(0U, len); - hs_db_unprotect(db, len); + makeDatabaseWritable(db); memset(db, 0xf0, len); size_t sz; err = hs_stream_size(db, &sz); ASSERT_EQ(HS_INVALID, err); - hs_db_free(db, len); + hs_free_database(db); } // hs_stream_size: Call with a block-mode database diff --git a/unit/hyperscan/identical.cpp b/unit/hyperscan/identical.cpp index cbeb02422..528698d6a 100644 --- a/unit/hyperscan/identical.cpp +++ b/unit/hyperscan/identical.cpp @@ -47,7 +47,7 @@ TEST_P(IdenticalTest, Block) { const PatternInfo &info = GetParam(); std::vector patterns; - for (unsigned i = 0; i < 100; i++) { + for (unsigned i = 0; i < 96; i++) { patterns.push_back(pattern(info.expr, info.flags, i)); } @@ -85,7 +85,7 @@ TEST_P(IdenticalTest, Stream) { const PatternInfo &info = GetParam(); std::vector patterns; - for (unsigned i = 0; i < 100; i++) { + for (unsigned i = 0; i < 96; i++) { patterns.push_back(pattern(info.expr, info.flags, i)); } @@ -133,7 +133,7 @@ TEST_P(IdenticalTest, Vectored) { const PatternInfo &info = GetParam(); std::vector patterns; - for (unsigned i = 0; i < 100; i++) { + for (unsigned i = 0; i < 96; i++) { patterns.push_back(pattern(info.expr, info.flags, i)); } diff --git a/unit/hyperscan/literals.cpp b/unit/hyperscan/literals.cpp index 86bd317cd..64dda3a0d 100644 --- a/unit/hyperscan/literals.cpp +++ b/unit/hyperscan/literals.cpp @@ -235,11 +235,11 @@ static const unsigned test_modes[] = {HS_MODE_BLOCK, HS_MODE_STREAM, static const unsigned test_flags[] = {0, HS_FLAG_SINGLEMATCH, HS_FLAG_SOM_LEFTMOST}; -static const unsigned test_sizes[] = {1, 10, 100, 500, 10000}; +static const unsigned test_sizes[] = {1, 10, 50, 96}; static const pair test_bounds[] = {{3u, 10u}, {10u, 100u}}; INSTANTIATE_TEST_CASE_P(LiteralTest, HyperscanLiteralTest, Combine(ValuesIn(test_modes), ValuesIn(test_flags), ValuesIn(test_sizes), ValuesIn(test_bounds), - Bool())); + Values(false))); diff --git a/unit/hyperscan/single.cpp b/unit/hyperscan/single.cpp index bbe116a0e..6732fb33b 100644 --- a/unit/hyperscan/single.cpp +++ b/unit/hyperscan/single.cpp @@ -244,9 +244,8 @@ class HyperscanTestRuntime size_t origSize; err = hs_database_size(db, &origSize); ASSERT_EQ(HS_SUCCESS, err); - hs_db_unprotect(db, origSize); - memset(db, 0xff, origSize); - hs_db_free(db, origSize); + hs_free_database(db); + db = nullptr; // relocate to 16 different alignments, ensuring that we can // deserialize from any string diff --git a/unit/hyperscan/som_overflow.cpp b/unit/hyperscan/som_overflow.cpp index e4848afd8..0d53e2de8 100644 --- a/unit/hyperscan/som_overflow.cpp +++ b/unit/hyperscan/som_overflow.cpp @@ -52,22 +52,29 @@ #include #include -// HS_PLATFORM_ALL allows the database to run on any platform -#ifndef HS_PLATFORM_ALL -#define HS_PLATFORM_ALL (0xabcdef11abcdef00ULL) -#endif - namespace { /** * Helper: compute HMAC-SHA256 over the bytecode portion of a database, - * using the same key as Hyperscan's db_check_integrity. + * and also the header HMAC, using the same key as Hyperscan's + * db_check_integrity / db_check_header_integrity. */ static void compute_db_hmac(hs_database_t *db) { + // Bytecode HMAC char *bytecode = (char *)db + db->bytecode; unsigned int hmac_len = 32; HMAC(EVP_sha256(), HS_DB_HMAC_KEY, sizeof(HS_DB_HMAC_KEY), (const unsigned char *)bytecode, db->length, db->hmac, &hmac_len); + + // Header HMAC over (magic, version, length, platform) = 20 bytes + u8 buf[4 + 4 + 4 + 8]; + memcpy(buf, &db->magic, 4); + memcpy(buf + 4, &db->version, 4); + memcpy(buf + 8, &db->length, 4); + memcpy(buf + 12, &db->platform, 8); + hmac_len = 32; + HMAC(EVP_sha256(), HS_DB_HMAC_KEY, sizeof(HS_DB_HMAC_KEY), + buf, sizeof(buf), db->hmac_hdr, &hmac_len); } /** @@ -101,7 +108,7 @@ static hs_database_t *make_forged_db(u32 som_location_count, db->version = HS_DB_VERSION; db->length = static_cast(rose_size); db->bytecode = static_cast(bytecode_offset); - db->platform = HS_PLATFORM_ALL; + db->platform = hs_current_platform; struct RoseEngine *rose = reinterpret_cast(static_cast(mem) + bytecode_offset); @@ -117,14 +124,18 @@ static hs_database_t *make_forged_db(u32 som_location_count, rose->somLocationFatbitSize = rt_fatbit_size(som_location_count); // Set other fatbit fields to consistent values. - // rt_fatbit_size(0) returns 8 (minimum fatbit struct size), so - // even with zero counts we must set sizes >= 8 to pass validation. + // rt_fatbit_size(0) returns MIN_FAT_SIZE (32), so + // even with zero counts we must set sizes >= 32 to pass validation. rose->queueCount = 0; rose->activeQueueArraySize = rt_fatbit_size(0); rose->handledKeyCount = 0; rose->handledKeyFatbitSize = rt_fatbit_size(0); rose->delay_count = 0; rose->delay_fatbit_size = rt_fatbit_size(0); + rose->anchored_count = 0; + rose->anchored_fatbit_size = rt_fatbit_size(0); + rose->dkeyCount = 0; + rose->dkeyLogSize = rt_fatbit_size(0); // Ensure literal matcher offsets are 0 (returns NULL, skips lit path) rose->fmatcherOffset = 0; From 2e8052d63a23bf1deaaf79b23ea16117efd8681a Mon Sep 17 00:00:00 2001 From: prudvi Date: Fri, 17 Jul 2026 16:24:33 +0530 Subject: [PATCH 3/4] updated teddy-only compile flag --- CMakeLists.txt | 97 +++++++++++++++++-------- src/fdr/fdr.c | 4 +- src/hwlm/hwlm.c | 18 ++--- src/hwlm/hwlm_build.cpp | 124 +++++++++++++++----------------- src/hwlm/hwlm_build.h | 20 +++--- src/nfa/nfa_rev_api.h | 14 ++-- unit/CMakeLists.txt | 2 +- unit/hyperscan/arg_checks.cpp | 55 ++++---------- unit/hyperscan/identical.cpp | 6 +- unit/hyperscan/literals.cpp | 4 +- unit/hyperscan/single.cpp | 5 +- unit/hyperscan/som_overflow.cpp | 29 +++----- 12 files changed, 187 insertions(+), 191 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 601a91b02..9f2892fff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required (VERSION 2.8.11) +cmake_minimum_required (VERSION 3.5) project (hyperscan C CXX) set (HS_MAJOR_VERSION 5) @@ -76,14 +76,10 @@ include_directories(SYSTEM ${OPENSSL_INCLUDE_DIR}) include (${CMAKE_MODULE_PATH}/boost.cmake) # -- make this work? set(python_ADDITIONAL_VERSIONS 2.7 2.6) -find_package(PythonInterp) +find_package(Python3 REQUIRED COMPONENTS Interpreter) find_program(RAGEL ragel) -if(PYTHONINTERP_FOUND) - set(PYTHON ${PYTHON_EXECUTABLE}) -else() - message(FATAL_ERROR "No python interpreter found") -endif() +set(PYTHON ${Python3_EXECUTABLE}) # allow for reproducible builds - python for portability if (DEFINED ENV{SOURCE_DATE_EPOCH}) @@ -117,6 +113,18 @@ endif() option(BUILD_SHARED_LIBS "Build shared libs instead of static" OFF) option(BUILD_STATIC_AND_SHARED "Build shared libs as well as static" OFF) +option(TEDDY_ONLY "Build a Teddy-only library and skip the other literal engines" OFF) + +if (TEDDY_ONLY) + add_definitions(-DHS_TEDDY_ONLY) + set(HS_LIB_BASENAME hs_teddy) + set(HS_PC_NAME libhs_teddy) + set(HS_PKGCONFIG_FILE libhs_teddy.pc) +else() + set(HS_LIB_BASENAME hs) + set(HS_PC_NAME libhs) + set(HS_PKGCONFIG_FILE libhs.pc) +endif() if (BUILD_STATIC_AND_SHARED OR BUILD_SHARED_LIBS) message(STATUS "Building shared libraries") @@ -505,12 +513,16 @@ if (CORRECT_PCRE_VERSION AND PCRE_BUILD_SOURCE AND BUILD_STATIC_LIBS) set(BUILD_CHIMERA TRUE) endif() -add_subdirectory(unit) +if (NOT TEDDY_ONLY) + add_subdirectory(unit) +endif() if (EXISTS ${CMAKE_SOURCE_DIR}/tools/CMakeLists.txt) add_subdirectory(tools) endif() -if (EXISTS ${CMAKE_SOURCE_DIR}/chimera/CMakeLists.txt AND BUILD_CHIMERA) - add_subdirectory(chimera) +if (NOT TEDDY_ONLY) + if (EXISTS ${CMAKE_SOURCE_DIR}/chimera/CMakeLists.txt AND BUILD_CHIMERA) + add_subdirectory(chimera) + endif() endif() endif() @@ -528,8 +540,8 @@ if (NOT WIN32) endforeach() set(PRIVATE_LIBS "${PRIVATE_LIBS} -lcrypto") - configure_file(libhs.pc.in libhs.pc @ONLY) # only replace @ quoted vars - install(FILES ${CMAKE_BINARY_DIR}/libhs.pc + configure_file(libhs.pc.in ${HS_PKGCONFIG_FILE} @ONLY) # only replace @ quoted vars + install(FILES ${CMAKE_BINARY_DIR}/${HS_PKGCONFIG_FILE} DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") endif() @@ -558,12 +570,16 @@ if (CORRECT_PCRE_VERSION AND PCRE_BUILD_SOURCE AND BUILD_STATIC_LIBS) set(BUILD_CHIMERA TRUE) endif() -add_subdirectory(unit) +if (NOT TEDDY_ONLY) + add_subdirectory(unit) +endif() if (EXISTS ${CMAKE_SOURCE_DIR}/tools/CMakeLists.txt) add_subdirectory(tools) endif() -if (EXISTS ${CMAKE_SOURCE_DIR}/chimera/CMakeLists.txt AND BUILD_CHIMERA) - add_subdirectory(chimera) +if (NOT TEDDY_ONLY) + if (EXISTS ${CMAKE_SOURCE_DIR}/chimera/CMakeLists.txt AND BUILD_CHIMERA) + add_subdirectory(chimera) + endif() endif() endif() @@ -626,9 +642,9 @@ set (hs_exec_SRCS src/hwlm/hwlm.c src/hwlm/hwlm.h src/hwlm/hwlm_internal.h - #src/hwlm/noodle_engine.c - #src/hwlm/noodle_engine.h - #src/hwlm/noodle_internal.h + src/hwlm/noodle_engine.c + src/hwlm/noodle_engine.h + src/hwlm/noodle_internal.h src/nfa/accel.c src/nfa/accel.h src/nfa/castle.c @@ -745,6 +761,13 @@ set (hs_exec_SRCS src/database.h ) +if (TEDDY_ONLY) + list(REMOVE_ITEM hs_exec_SRCS + src/hwlm/noodle_engine.c + src/hwlm/noodle_engine.h + src/hwlm/noodle_internal.h) +endif() + set (hs_exec_avx2_SRCS src/fdr/teddy_avx2.c src/util/masked_move.c @@ -771,14 +794,14 @@ SET (hs_compile_SRCS src/compiler/expression_info.h src/fdr/engine_description.cpp src/fdr/engine_description.h - #src/fdr/fdr_compile.cpp - #src/fdr/fdr_compile.h - #src/fdr/fdr_compile_internal.h + src/fdr/fdr_compile.cpp + src/fdr/fdr_compile.h + src/fdr/fdr_compile_internal.h src/fdr/fdr_compile_util.cpp src/fdr/fdr_confirm_compile.cpp src/fdr/fdr_confirm.h - #src/fdr/fdr_engine_description.cpp - #src/fdr/fdr_engine_description.h + src/fdr/fdr_engine_description.cpp + src/fdr/fdr_engine_description.h src/fdr/fdr_internal.h src/fdr/flood_compile.cpp src/fdr/teddy_compile.cpp @@ -791,9 +814,9 @@ SET (hs_compile_SRCS src/hwlm/hwlm_internal.h src/hwlm/hwlm_literal.cpp src/hwlm/hwlm_literal.h - #src/hwlm/noodle_build.cpp - #src/hwlm/noodle_build.h - #src/hwlm/noodle_internal.h + src/hwlm/noodle_build.cpp + src/hwlm/noodle_build.h + src/hwlm/noodle_internal.h src/nfa/accel.h src/nfa/accel_dfa_build_strat.cpp src/nfa/accel_dfa_build_strat.h @@ -1133,10 +1156,21 @@ SET (hs_compile_SRCS src/util/verify_types.h ) +if (TEDDY_ONLY) + list(REMOVE_ITEM hs_compile_SRCS + src/fdr/fdr_compile.cpp + src/fdr/fdr_compile.h + src/fdr/fdr_engine_description.cpp + src/fdr/fdr_engine_description.h + src/hwlm/noodle_build.cpp + src/hwlm/noodle_build.h + src/hwlm/noodle_internal.h) +endif() + set(hs_dump_SRCS src/scratch_dump.cpp src/scratch_dump.h - #src/fdr/fdr_dump.cpp + src/fdr/fdr_dump.cpp src/hwlm/hwlm_dump.cpp src/hwlm/hwlm_dump.h src/nfa/accel_dump.cpp @@ -1185,6 +1219,11 @@ if (DUMP_SUPPORT) set(hs_compile_SRCS ${hs_compile_SRCS} ${hs_dump_SRCS}) endif() +if (TEDDY_ONLY) + list(REMOVE_ITEM hs_dump_SRCS + src/fdr/fdr_dump.cpp) +endif() + # we group things by sublibraries, specifying shared and static and then # choose which ones to build @@ -1212,6 +1251,7 @@ if (NOT FAT_RUNTIME) src/hs_valid_platform.c $ $) + set_target_properties(hs PROPERTIES OUTPUT_NAME ${HS_LIB_BASENAME}) endif (BUILD_STATIC_LIBS) if (BUILD_STATIC_AND_SHARED OR BUILD_SHARED_LIBS) @@ -1290,6 +1330,7 @@ else (FAT_RUNTIME) $ $ ${RUNTIME_LIBS}) + set_target_properties(hs PROPERTIES OUTPUT_NAME ${HS_LIB_BASENAME}) endif (BUILD_STATIC_LIBS) @@ -1421,7 +1462,7 @@ if (BUILD_STATIC_AND_SHARED OR BUILD_SHARED_LIBS) add_dependencies(hs_shared ragel_Parser) target_link_libraries(hs_shared OpenSSL::Crypto) set_target_properties(hs_shared PROPERTIES - OUTPUT_NAME hs + OUTPUT_NAME ${HS_LIB_BASENAME} VERSION ${LIB_VERSION} SOVERSION ${LIB_SOVERSION} MACOSX_RPATH ON) diff --git a/src/fdr/fdr.c b/src/fdr/fdr.c index 3c0cc6096..d33756d35 100644 --- a/src/fdr/fdr.c +++ b/src/fdr/fdr.c @@ -42,7 +42,6 @@ /** \brief number of bytes processed in each iteration */ #define ITER_BYTES 16 -#if 0 /* FDR engine removed - Teddy-only mode */ /** \brief total zone buffer size */ #define ZONE_TOTAL_SIZE 64 @@ -789,7 +788,6 @@ hwlm_error_t fdr_engine_exec(const struct FDR *fdr, return HWLM_SUCCESS; } -#endif /* FDR engine removed */ #if defined(HAVE_AVX2) #define ONLY_AVX2(func) func @@ -802,7 +800,7 @@ typedef hwlm_error_t (*FDRFUNCTYPE)(const struct FDR *fdr, hwlm_group_t control); static const FDRFUNCTYPE funcs[] = { - /* fdr_engine_exec, */ NULL, /* FDR removed - Teddy-only mode */ + fdr_engine_exec, NULL, /* old: fast teddy */ NULL, /* old: fast teddy */ ONLY_AVX2(fdr_exec_fat_teddy_msks1), diff --git a/src/hwlm/hwlm.c b/src/hwlm/hwlm.c index 745f67ed8..d103df331 100644 --- a/src/hwlm/hwlm.c +++ b/src/hwlm/hwlm.c @@ -31,7 +31,9 @@ */ #include "hwlm.h" #include "hwlm_internal.h" -//#include "noodle_engine.h" /* Teddy-only mode: noodle removed */ +#if !defined(HS_TEDDY_ONLY) +#include "noodle_engine.h" +#endif #include "scratch.h" #include "ue2common.h" #include "fdr/fdr.h" @@ -182,14 +184,13 @@ hwlm_error_t hwlmExec(const struct HWLM *t, const u8 *buf, size_t len, assert(start < len); - /* +#if !defined(HS_TEDDY_ONLY) if (t->type == HWLM_ENGINE_NOOD) { DEBUG_PRINTF("calling noodExec\n"); return noodExec(HWLM_C_DATA(t), buf, len, start, cb, scratch); } - */ +#endif - /* Teddy-only mode: always use fdrExec (dispatches to Teddy) */ assert(t->type == HWLM_ENGINE_FDR); const union AccelAux *aa = &t->accel0; if ((groups & ~t->accel1_groups) == 0) { @@ -197,7 +198,7 @@ hwlm_error_t hwlmExec(const struct HWLM *t, const u8 *buf, size_t len, aa = &t->accel1; } do_accel_block(aa, buf, len, &start); - DEBUG_PRINTF("calling teddy via fdrExec (groups=%08llx, start=%zu)\n", groups, start); + DEBUG_PRINTF("calling frankie (groups=%08llx, start=%zu)\n", groups, start); return fdrExec(HWLM_C_DATA(t), buf, len, start, cb, scratch, groups); } @@ -220,7 +221,7 @@ hwlm_error_t hwlmExecStreaming(const struct HWLM *t, size_t len, size_t start, assert(start < len); - /* +#if !defined(HS_TEDDY_ONLY) if (t->type == HWLM_ENGINE_NOOD) { DEBUG_PRINTF("calling noodExec\n"); // If we've been handed a start offset, we can use a block mode scan at @@ -232,9 +233,8 @@ hwlm_error_t hwlmExecStreaming(const struct HWLM *t, size_t len, size_t start, scratch); } } - */ +#endif - /* Teddy-only mode: always use fdrExecStreaming (dispatches to Teddy) */ assert(t->type == HWLM_ENGINE_FDR); const union AccelAux *aa = &t->accel0; if ((groups & ~t->accel1_groups) == 0) { @@ -242,7 +242,7 @@ hwlm_error_t hwlmExecStreaming(const struct HWLM *t, size_t len, size_t start, aa = &t->accel1; } do_accel_streaming(aa, hbuf, hlen, buf, len, &start); - DEBUG_PRINTF("calling teddy via fdrExecStreaming (groups=%08llx, start=%zu)\n", groups, start); + DEBUG_PRINTF("calling frankie (groups=%08llx, start=%zu)\n", groups, start); return fdrExecStreaming(HWLM_C_DATA(t), hbuf, hlen, buf, len, start, cb, scratch, groups); } diff --git a/src/hwlm/hwlm_build.cpp b/src/hwlm/hwlm_build.cpp index b0ab87dfa..766c81431 100644 --- a/src/hwlm/hwlm_build.cpp +++ b/src/hwlm/hwlm_build.cpp @@ -36,17 +36,17 @@ #include "hwlm.h" #include "hwlm_internal.h" #include "hwlm_literal.h" -//#include "noodle_engine.h" /* Teddy-only mode: noodle removed */ -//#include "noodle_build.h" /* Teddy-only mode: noodle removed */ +#include "noodle_engine.h" +#include "noodle_build.h" #include "scratch.h" #include "ue2common.h" -//#include "fdr/fdr_compile.h" /* Teddy-only mode: FDR compile removed */ -//#include "fdr/fdr_compile_internal.h" /* Teddy-only mode: FDR compile removed */ -//#include "fdr/fdr_engine_description.h" /* Teddy-only mode: FDR engine desc removed */ -#include "fdr/fdr.h" -#include "fdr/fdr_internal.h" /* for struct FDR and fdr->size */ -#include "fdr/fdr_compile_internal.h" /* for HINT_INVALID */ +#include "fdr/fdr_internal.h" +#if defined(HS_TEDDY_ONLY) #include "fdr/teddy_compile.h" +#endif +#include "fdr/fdr_compile.h" +#include "fdr/fdr_compile_internal.h" +#include "fdr/fdr_engine_description.h" #include "fdr/teddy_engine_description.h" #include "util/compile_context.h" #include "util/compile_error.h" @@ -61,18 +61,20 @@ using namespace std; namespace ue2 { +#if defined(HS_TEDDY_ONLY) +static constexpr size_t TEDDY_ONLY_MAX_DECOMPOSED_EXPRESSIONS = 96; +#endif + HWLMProto::HWLMProto(u8 engType_in, vector lits_in) : engType(engType_in), lits(move(lits_in)) {} -/* HWLMProto::HWLMProto(u8 engType_in, - unique_ptr eng_in, - vector lits_in, - map> bucketToLits_in, - bool make_small_in) - : engType(engType_in), fdrEng(move(eng_in)), lits(move(lits_in)), - bucketToLits(move(bucketToLits_in)), make_small(make_small_in) {} -*/ + unique_ptr eng_in, + vector lits_in, + map> bucketToLits_in, + bool make_small_in) + : engType(engType_in), fdrEng(move(eng_in)), lits(move(lits_in)), + bucketToLits(move(bucketToLits_in)), make_small(make_small_in) {} HWLMProto::HWLMProto(u8 engType_in, unique_ptr eng_in, @@ -109,7 +111,7 @@ bool everyoneHasGroups(const vector &lits) { } #endif -/* +#if !defined(HS_TEDDY_ONLY) static bool isNoodleable(const vector &lits, const CompileContext &cc) { @@ -124,17 +126,27 @@ bool isNoodleable(const vector &lits, return true; } -*/ +#endif bytecode_ptr hwlmBuild(const HWLMProto &proto, const CompileContext &cc, UNUSED hwlm_group_t expected_groups) { size_t engSize = 0; shared_ptr eng; +#if defined(HS_TEDDY_ONLY) DEBUG_PRINTF("building table with %zu strings\n", proto.lits.size()); - - /* +#else const auto &lits = proto.lits; + DEBUG_PRINTF("building table with %zu strings\n", lits.size()); +#endif + +#if defined(HS_TEDDY_ONLY) + auto teddy = teddyBuildTable(proto, cc.grey); + if (teddy) { + engSize = teddy.size(); + eng = move(teddy); + } +#else if (proto.engType == HWLM_ENGINE_NOOD) { DEBUG_PRINTF("build noodle table\n"); const hwlmLiteral &lit = lits.front(); @@ -151,17 +163,7 @@ bytecode_ptr hwlmBuild(const HWLMProto &proto, const CompileContext &cc, } eng = move(fdr); } - */ - - /* Teddy-only mode: always build Teddy table */ - DEBUG_PRINTF("building teddy table (teddy-only mode)\n"); - if (proto.teddyEng) { - auto fdr = teddyBuildTable(proto, cc.grey); - if (fdr) { - engSize = fdr.size(); - } - eng = move(fdr); - } +#endif if (!eng) { return nullptr; @@ -187,6 +189,12 @@ hwlmBuildProto(vector &lits, bool make_small, assert(!lits.empty()); dumpLits(lits); +#if defined(HS_TEDDY_ONLY) + if (lits.size() > TEDDY_ONLY_MAX_DECOMPOSED_EXPRESSIONS) { + throw CompileError("Teddy-only build supports at most 96 decomposed expressions."); + } +#endif + // Check that we haven't exceeded the maximum number of literals. if (lits.size() > cc.grey.limitLiteralCount) { throw ResourceLimitError(); @@ -219,7 +227,13 @@ hwlmBuildProto(vector &lits, bool make_small, assert(everyoneHasGroups(lits)); - /* +#if defined(HS_TEDDY_ONLY) + proto = teddyBuildProtoHinted(HWLM_ENGINE_FDR, lits, make_small, + HINT_INVALID, cc.target_info); + if (!proto) { + throw CompileError("Unable to generate Teddy-only literal matcher proto."); + } +#else if (isNoodleable(lits, cc)) { DEBUG_PRINTF("build noodle table\n"); proto = ue2::make_unique(HWLM_ENGINE_NOOD, lits); @@ -231,17 +245,7 @@ hwlmBuildProto(vector &lits, bool make_small, return nullptr; } } - */ - - /* Teddy-only mode: always build Teddy */ - DEBUG_PRINTF("building teddy (teddy-only mode)\n"); - proto = teddyBuildProtoHinted(HWLM_ENGINE_FDR, lits, make_small, - HINT_INVALID, cc.target_info); - if (!proto) { - throw CompileError("Pattern set exceeds Teddy engine capacity " - "(max 96 decomposed literals). Reduce pattern count " - "or avoid large character classes/alternations."); - } +#endif return proto; } @@ -249,7 +253,11 @@ hwlmBuildProto(vector &lits, bool make_small, size_t hwlmSize(const HWLM *h) { size_t engSize = 0; - /* +#if defined(HS_TEDDY_ONLY) + if (h->type == HWLM_ENGINE_FDR) { + engSize = ((const FDR *)HWLM_C_DATA(h))->size; + } +#else switch (h->type) { case HWLM_ENGINE_NOOD: engSize = noodSize((const noodTable *)HWLM_C_DATA(h)); @@ -258,11 +266,7 @@ size_t hwlmSize(const HWLM *h) { engSize = fdrSize((const FDR *)HWLM_C_DATA(h)); break; } - */ - - assert(h->type == HWLM_ENGINE_FDR); - const struct FDR *fdr = (const struct FDR *)HWLM_C_DATA(h); - engSize = fdr->size; +#endif if (!engSize) { return 0; @@ -272,18 +276,15 @@ size_t hwlmSize(const HWLM *h) { } size_t hwlmFloodProneSuffixLen(size_t numLiterals, const CompileContext &cc) { - UNUSED const size_t NO_LIMIT = ~(size_t)0; + const size_t NO_LIMIT = ~(size_t)0; - /* // NOTE: this function contains a number of magic numbers which are // conservative estimates of flood-proneness based on internal details of // the various literal engines that fall under the HWLM aegis. If you // change those engines, you might need to change this function too. - */ DEBUG_PRINTF("%zu literals\n", numLiterals); - /* if (cc.grey.allowNoodle && numLiterals <= 1) { DEBUG_PRINTF("noodle\n"); return NO_LIMIT; @@ -300,22 +301,11 @@ size_t hwlmFloodProneSuffixLen(size_t numLiterals, const CompileContext &cc) { } } + // TODO: we had thought we could push this value up to 9, but it seems that + // hurts performance on floods in some FDR models. Super-conservative for + // now. DEBUG_PRINTF("fdr\n"); return 3; - */ - - /* Teddy-only mode */ - if (numLiterals <= 48) { - DEBUG_PRINTF("teddy\n"); - return 3; - } - if (cc.target_info.has_avx2() && numLiterals <= 96) { - DEBUG_PRINTF("avx2 teddy\n"); - return 3; - } - - DEBUG_PRINTF("teddy fallback\n"); - return 3; } } // namespace ue2 diff --git a/src/hwlm/hwlm_build.h b/src/hwlm/hwlm_build.h index db456d85c..91f227dce 100644 --- a/src/hwlm/hwlm_build.h +++ b/src/hwlm/hwlm_build.h @@ -46,7 +46,7 @@ struct HWLM; namespace ue2 { -/* class FDREngineDescription; */ /* Teddy-only mode: removed */ +class FDREngineDescription; class TeddyEngineDescription; struct CompileContext; struct Grey; @@ -54,14 +54,14 @@ struct Grey; /** \brief Class representing a literal matcher prototype. */ struct HWLMProto { /** - * \brief Engine type (always HWLM_ENGINE_FDR in teddy-only mode). + * \brief Engine type to distinguish noodle from FDR and Teddy. */ u8 engType; - /* - \brief FDR engine description. + /** + * \brief FDR engine description. + */ std::unique_ptr fdrEng; - */ /** * \brief Teddy engine description. @@ -74,7 +74,7 @@ struct HWLMProto { std::vector lits; /** - * \brief Bucket assignment info in Teddy + * \brief Bucket assignment info in FDR and Teddy */ std::map> bucketToLits; @@ -85,12 +85,10 @@ struct HWLMProto { HWLMProto(u8 engType_in, std::vector lits_in); - /* HWLMProto(u8 engType_in, std::unique_ptr eng_in, - std::vector lits_in, - std::map> bucketToLits_in, - bool make_small_in); - */ + std::vector lits_in, + std::map> bucketToLits_in, + bool make_small_in); HWLMProto(u8 engType_in, std::unique_ptr eng_in, std::vector lits_in, diff --git a/src/nfa/nfa_rev_api.h b/src/nfa/nfa_rev_api.h index 7fab4c2de..6c0c908cd 100644 --- a/src/nfa/nfa_rev_api.h +++ b/src/nfa/nfa_rev_api.h @@ -104,23 +104,29 @@ size_t nfaRevAccel_i(const struct NFA *nfa, const u8 *buffer, size_t length) { break; case ACCEL_RDEOD: DEBUG_PRINTF("ACCEL_RDEOD\n"); - if (length < (size_t)nfa->rAccelOffset + 1) { + { + const size_t offset = nfa->rAccelOffset; + if (length < offset + 1) { break; } - if (unaligned_load_u16(buffer + length - nfa->rAccelOffset) != + if (unaligned_load_u16(buffer + length - offset) != nfa->rAccelData.dc) { return 0; } + } break; case ACCEL_RDEOD_NOCASE: DEBUG_PRINTF("ACCEL_RDEOD_NOCASE\n"); - if (length < (size_t)nfa->rAccelOffset + 1) { + { + const size_t offset = nfa->rAccelOffset; + if (length < offset + 1) { break; } - if ((unaligned_load_u16(buffer + length - nfa->rAccelOffset) & + if ((unaligned_load_u16(buffer + length - offset) & DOUBLE_CASE_CLEAR) != nfa->rAccelData.dc) { return 0; } + } break; default: assert(!"not here"); diff --git a/unit/CMakeLists.txt b/unit/CMakeLists.txt index 6ed9539d0..2d6cbbb5f 100644 --- a/unit/CMakeLists.txt +++ b/unit/CMakeLists.txt @@ -107,7 +107,7 @@ set(unit_internal_SOURCES internal/nfagraph_repeat.cpp internal/nfagraph_util.cpp internal/nfagraph_width.cpp - #internal/noodle.cpp + internal/noodle.cpp internal/pack_bits.cpp internal/parser.cpp internal/partial.cpp diff --git a/unit/hyperscan/arg_checks.cpp b/unit/hyperscan/arg_checks.cpp index 1dd2ab1ce..6a2fa650d 100644 --- a/unit/hyperscan/arg_checks.cpp +++ b/unit/hyperscan/arg_checks.cpp @@ -28,10 +28,6 @@ #include "config.h" -#include -#include -#include - #include "gtest/gtest.h" #include "hs.h" #include "database.h" @@ -61,59 +57,36 @@ int singleHandler(unsigned id, unsigned long long from, namespace /* anonymous */ { -static void makeDatabaseWritable(hs_database *db) { - if (!db) { - return; - } - long ps_raw = sysconf(_SC_PAGESIZE); - if (ps_raw <= 0) { - return; - } - size_t ps = (size_t)ps_raw; - if ((ps & (ps - 1)) != 0) { - return; - } - if ((uintptr_t)db & (ps - 1)) { - return; - } - size_t db_len = sizeof(struct hs_database) + db->length; - if (db_len > SIZE_MAX - ps) { - return; - } - size_t rounded = (db_len + ps - 1) & ~(ps - 1); - if (rounded < db_len || rounded == 0) { - return; - } - int ret = mprotect(db, rounded, PROT_READ | PROT_WRITE); - (void)ret; -} - // Break the magic number of the given database. void breakDatabaseMagic(hs_database *db) { - makeDatabaseWritable(db); // database magic should be 0xdbdb at the start + size_t db_len = sizeof(struct hs_database) + db->length; + hs_db_unprotect(db, db_len); ASSERT_TRUE(memcmp("\xdb\xdb", db, 2) == 0); *(char *)db = 0xdc; } // Break the version number of the given database. void breakDatabaseVersion(hs_database *db) { - makeDatabaseWritable(db); // database version is the second u32 + size_t db_len = sizeof(struct hs_database) + db->length; + hs_db_unprotect(db, db_len); *((char *)db + 4) += 1; } // Break the platform data of the given database. void breakDatabasePlatform(hs_database *db) { - makeDatabaseWritable(db); // database platform is an aligned u64a 16 bytes in + size_t db_len = sizeof(struct hs_database) + db->length; + hs_db_unprotect(db, db_len); memset((char *)db + 16, 0xff, 8); } // Break the alignment of the bytecode for the given database. void breakDatabaseBytecode(hs_database *db) { - makeDatabaseWritable(db); // bytecode ptr is a u32 at offsetof(hs_database, bytecode) + size_t db_len = sizeof(struct hs_database) + db->length; + hs_db_unprotect(db, db_len); unsigned int *bytecode = (unsigned int *)((char *)db + offsetof(struct hs_database, bytecode)); ASSERT_NE(0U, *bytecode); ASSERT_EQ(0U, (size_t)((char *)db + *bytecode) % 16U); @@ -952,7 +925,7 @@ TEST(HyperscanArgChecks, ScanBlockBrokenDatabaseMagic) { // teardown err = hs_free_scratch(scratch); ASSERT_EQ(HS_SUCCESS, err); - hs_free_database(db); + hs_db_free(db, db_len1); } // hs_scan: Call with a database with broken version @@ -1158,7 +1131,7 @@ TEST(HyperscanArgChecks, ScanVectorBrokenDatabaseMagic) { // teardown err = hs_free_scratch(scratch); ASSERT_EQ(HS_SUCCESS, err); - hs_free_database(db); + hs_db_free(db, db_len2); } // hs_scan_vector: Call with a database with broken version @@ -1435,7 +1408,7 @@ TEST(HyperscanArgChecks, AllocScratchBadDatabaseMagic) { ASSERT_EQ(HS_INVALID, err); // teardown - hs_free_database(db); + hs_db_free(db, db_len3); } // hs_alloc_scratch: Call with broken database version @@ -1517,7 +1490,7 @@ TEST(HyperscanArgChecks, AllocScratchBadDatabaseCRC) { ASSERT_EQ(HS_SUCCESS, err); // for want of a better case, corrupt the "middle byte" of the database. - makeDatabaseWritable(db); + hs_db_unprotect(db, len); char *mid = (char *)db + len/2; *mid += 17; @@ -1602,14 +1575,14 @@ TEST(HyperscanArgChecks, StreamSizeBogusDatabase) { ASSERT_EQ(HS_SUCCESS, err); ASSERT_LT(0U, len); - makeDatabaseWritable(db); + hs_db_unprotect(db, len); memset(db, 0xf0, len); size_t sz; err = hs_stream_size(db, &sz); ASSERT_EQ(HS_INVALID, err); - hs_free_database(db); + hs_db_free(db, len); } // hs_stream_size: Call with a block-mode database diff --git a/unit/hyperscan/identical.cpp b/unit/hyperscan/identical.cpp index 528698d6a..cbeb02422 100644 --- a/unit/hyperscan/identical.cpp +++ b/unit/hyperscan/identical.cpp @@ -47,7 +47,7 @@ TEST_P(IdenticalTest, Block) { const PatternInfo &info = GetParam(); std::vector patterns; - for (unsigned i = 0; i < 96; i++) { + for (unsigned i = 0; i < 100; i++) { patterns.push_back(pattern(info.expr, info.flags, i)); } @@ -85,7 +85,7 @@ TEST_P(IdenticalTest, Stream) { const PatternInfo &info = GetParam(); std::vector patterns; - for (unsigned i = 0; i < 96; i++) { + for (unsigned i = 0; i < 100; i++) { patterns.push_back(pattern(info.expr, info.flags, i)); } @@ -133,7 +133,7 @@ TEST_P(IdenticalTest, Vectored) { const PatternInfo &info = GetParam(); std::vector patterns; - for (unsigned i = 0; i < 96; i++) { + for (unsigned i = 0; i < 100; i++) { patterns.push_back(pattern(info.expr, info.flags, i)); } diff --git a/unit/hyperscan/literals.cpp b/unit/hyperscan/literals.cpp index 64dda3a0d..86bd317cd 100644 --- a/unit/hyperscan/literals.cpp +++ b/unit/hyperscan/literals.cpp @@ -235,11 +235,11 @@ static const unsigned test_modes[] = {HS_MODE_BLOCK, HS_MODE_STREAM, static const unsigned test_flags[] = {0, HS_FLAG_SINGLEMATCH, HS_FLAG_SOM_LEFTMOST}; -static const unsigned test_sizes[] = {1, 10, 50, 96}; +static const unsigned test_sizes[] = {1, 10, 100, 500, 10000}; static const pair test_bounds[] = {{3u, 10u}, {10u, 100u}}; INSTANTIATE_TEST_CASE_P(LiteralTest, HyperscanLiteralTest, Combine(ValuesIn(test_modes), ValuesIn(test_flags), ValuesIn(test_sizes), ValuesIn(test_bounds), - Values(false))); + Bool())); diff --git a/unit/hyperscan/single.cpp b/unit/hyperscan/single.cpp index 6732fb33b..bbe116a0e 100644 --- a/unit/hyperscan/single.cpp +++ b/unit/hyperscan/single.cpp @@ -244,8 +244,9 @@ class HyperscanTestRuntime size_t origSize; err = hs_database_size(db, &origSize); ASSERT_EQ(HS_SUCCESS, err); - hs_free_database(db); - db = nullptr; + hs_db_unprotect(db, origSize); + memset(db, 0xff, origSize); + hs_db_free(db, origSize); // relocate to 16 different alignments, ensuring that we can // deserialize from any string diff --git a/unit/hyperscan/som_overflow.cpp b/unit/hyperscan/som_overflow.cpp index 0d53e2de8..e4848afd8 100644 --- a/unit/hyperscan/som_overflow.cpp +++ b/unit/hyperscan/som_overflow.cpp @@ -52,29 +52,22 @@ #include #include +// HS_PLATFORM_ALL allows the database to run on any platform +#ifndef HS_PLATFORM_ALL +#define HS_PLATFORM_ALL (0xabcdef11abcdef00ULL) +#endif + namespace { /** * Helper: compute HMAC-SHA256 over the bytecode portion of a database, - * and also the header HMAC, using the same key as Hyperscan's - * db_check_integrity / db_check_header_integrity. + * using the same key as Hyperscan's db_check_integrity. */ static void compute_db_hmac(hs_database_t *db) { - // Bytecode HMAC char *bytecode = (char *)db + db->bytecode; unsigned int hmac_len = 32; HMAC(EVP_sha256(), HS_DB_HMAC_KEY, sizeof(HS_DB_HMAC_KEY), (const unsigned char *)bytecode, db->length, db->hmac, &hmac_len); - - // Header HMAC over (magic, version, length, platform) = 20 bytes - u8 buf[4 + 4 + 4 + 8]; - memcpy(buf, &db->magic, 4); - memcpy(buf + 4, &db->version, 4); - memcpy(buf + 8, &db->length, 4); - memcpy(buf + 12, &db->platform, 8); - hmac_len = 32; - HMAC(EVP_sha256(), HS_DB_HMAC_KEY, sizeof(HS_DB_HMAC_KEY), - buf, sizeof(buf), db->hmac_hdr, &hmac_len); } /** @@ -108,7 +101,7 @@ static hs_database_t *make_forged_db(u32 som_location_count, db->version = HS_DB_VERSION; db->length = static_cast(rose_size); db->bytecode = static_cast(bytecode_offset); - db->platform = hs_current_platform; + db->platform = HS_PLATFORM_ALL; struct RoseEngine *rose = reinterpret_cast(static_cast(mem) + bytecode_offset); @@ -124,18 +117,14 @@ static hs_database_t *make_forged_db(u32 som_location_count, rose->somLocationFatbitSize = rt_fatbit_size(som_location_count); // Set other fatbit fields to consistent values. - // rt_fatbit_size(0) returns MIN_FAT_SIZE (32), so - // even with zero counts we must set sizes >= 32 to pass validation. + // rt_fatbit_size(0) returns 8 (minimum fatbit struct size), so + // even with zero counts we must set sizes >= 8 to pass validation. rose->queueCount = 0; rose->activeQueueArraySize = rt_fatbit_size(0); rose->handledKeyCount = 0; rose->handledKeyFatbitSize = rt_fatbit_size(0); rose->delay_count = 0; rose->delay_fatbit_size = rt_fatbit_size(0); - rose->anchored_count = 0; - rose->anchored_fatbit_size = rt_fatbit_size(0); - rose->dkeyCount = 0; - rose->dkeyLogSize = rt_fatbit_size(0); // Ensure literal matcher offsets are 0 (returns NULL, skips lit path) rose->fmatcherOffset = 0; From 1b6e5a4137beb753ada3398ba9b06ca2cfcb68a1 Mon Sep 17 00:00:00 2001 From: prudvi Date: Fri, 17 Jul 2026 16:26:20 +0530 Subject: [PATCH 4/4] updated the lib name for teddy only build --- libhs.pc.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libhs.pc.in b/libhs.pc.in index fed4db454..bd28a1a61 100644 --- a/libhs.pc.in +++ b/libhs.pc.in @@ -3,9 +3,9 @@ exec_prefix=@CMAKE_INSTALL_PREFIX@ libdir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@ includedir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@ -Name: libhs +Name: @HS_PC_NAME@ Description: Intel(R) Hyperscan Library Version: @HS_VERSION@ -Libs: -L${libdir} -lhs +Libs: -L${libdir} -l@HS_LIB_BASENAME@ Libs.private: @PRIVATE_LIBS@ Cflags: -I${includedir}/hs