diff --git a/CMakeLists.txt b/CMakeLists.txt index 80492d1ef..9f2892fff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required (VERSION 2.8.11) +cmake_minimum_required (VERSION 3.5) project (hyperscan C CXX) set (HS_MAJOR_VERSION 5) @@ -76,14 +76,10 @@ include_directories(SYSTEM ${OPENSSL_INCLUDE_DIR}) include (${CMAKE_MODULE_PATH}/boost.cmake) # -- make this work? set(python_ADDITIONAL_VERSIONS 2.7 2.6) -find_package(PythonInterp) +find_package(Python3 REQUIRED COMPONENTS Interpreter) find_program(RAGEL ragel) -if(PYTHONINTERP_FOUND) - set(PYTHON ${PYTHON_EXECUTABLE}) -else() - message(FATAL_ERROR "No python interpreter found") -endif() +set(PYTHON ${Python3_EXECUTABLE}) # allow for reproducible builds - python for portability if (DEFINED ENV{SOURCE_DATE_EPOCH}) @@ -117,6 +113,18 @@ endif() option(BUILD_SHARED_LIBS "Build shared libs instead of static" OFF) option(BUILD_STATIC_AND_SHARED "Build shared libs as well as static" OFF) +option(TEDDY_ONLY "Build a Teddy-only library and skip the other literal engines" OFF) + +if (TEDDY_ONLY) + add_definitions(-DHS_TEDDY_ONLY) + set(HS_LIB_BASENAME hs_teddy) + set(HS_PC_NAME libhs_teddy) + set(HS_PKGCONFIG_FILE libhs_teddy.pc) +else() + set(HS_LIB_BASENAME hs) + set(HS_PC_NAME libhs) + set(HS_PKGCONFIG_FILE libhs.pc) +endif() if (BUILD_STATIC_AND_SHARED OR BUILD_SHARED_LIBS) message(STATUS "Building shared libraries") @@ -505,12 +513,16 @@ if (CORRECT_PCRE_VERSION AND PCRE_BUILD_SOURCE AND BUILD_STATIC_LIBS) set(BUILD_CHIMERA TRUE) endif() -add_subdirectory(unit) +if (NOT TEDDY_ONLY) + add_subdirectory(unit) +endif() if (EXISTS ${CMAKE_SOURCE_DIR}/tools/CMakeLists.txt) add_subdirectory(tools) endif() -if (EXISTS ${CMAKE_SOURCE_DIR}/chimera/CMakeLists.txt AND BUILD_CHIMERA) - add_subdirectory(chimera) +if (NOT TEDDY_ONLY) + if (EXISTS ${CMAKE_SOURCE_DIR}/chimera/CMakeLists.txt AND BUILD_CHIMERA) + add_subdirectory(chimera) + endif() endif() endif() @@ -528,8 +540,8 @@ if (NOT WIN32) endforeach() set(PRIVATE_LIBS "${PRIVATE_LIBS} -lcrypto") - configure_file(libhs.pc.in libhs.pc @ONLY) # only replace @ quoted vars - install(FILES ${CMAKE_BINARY_DIR}/libhs.pc + configure_file(libhs.pc.in ${HS_PKGCONFIG_FILE} @ONLY) # only replace @ quoted vars + install(FILES ${CMAKE_BINARY_DIR}/${HS_PKGCONFIG_FILE} DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") endif() @@ -558,12 +570,16 @@ if (CORRECT_PCRE_VERSION AND PCRE_BUILD_SOURCE AND BUILD_STATIC_LIBS) set(BUILD_CHIMERA TRUE) endif() -add_subdirectory(unit) +if (NOT TEDDY_ONLY) + add_subdirectory(unit) +endif() if (EXISTS ${CMAKE_SOURCE_DIR}/tools/CMakeLists.txt) add_subdirectory(tools) endif() -if (EXISTS ${CMAKE_SOURCE_DIR}/chimera/CMakeLists.txt AND BUILD_CHIMERA) - add_subdirectory(chimera) +if (NOT TEDDY_ONLY) + if (EXISTS ${CMAKE_SOURCE_DIR}/chimera/CMakeLists.txt AND BUILD_CHIMERA) + add_subdirectory(chimera) + endif() endif() endif() @@ -745,6 +761,13 @@ set (hs_exec_SRCS src/database.h ) +if (TEDDY_ONLY) + list(REMOVE_ITEM hs_exec_SRCS + src/hwlm/noodle_engine.c + src/hwlm/noodle_engine.h + src/hwlm/noodle_internal.h) +endif() + set (hs_exec_avx2_SRCS src/fdr/teddy_avx2.c src/util/masked_move.c @@ -1133,6 +1156,17 @@ SET (hs_compile_SRCS src/util/verify_types.h ) +if (TEDDY_ONLY) + list(REMOVE_ITEM hs_compile_SRCS + src/fdr/fdr_compile.cpp + src/fdr/fdr_compile.h + src/fdr/fdr_engine_description.cpp + src/fdr/fdr_engine_description.h + src/hwlm/noodle_build.cpp + src/hwlm/noodle_build.h + src/hwlm/noodle_internal.h) +endif() + set(hs_dump_SRCS src/scratch_dump.cpp src/scratch_dump.h @@ -1185,6 +1219,11 @@ if (DUMP_SUPPORT) set(hs_compile_SRCS ${hs_compile_SRCS} ${hs_dump_SRCS}) endif() +if (TEDDY_ONLY) + list(REMOVE_ITEM hs_dump_SRCS + src/fdr/fdr_dump.cpp) +endif() + # we group things by sublibraries, specifying shared and static and then # choose which ones to build @@ -1212,6 +1251,7 @@ if (NOT FAT_RUNTIME) src/hs_valid_platform.c $ $) + set_target_properties(hs PROPERTIES OUTPUT_NAME ${HS_LIB_BASENAME}) endif (BUILD_STATIC_LIBS) if (BUILD_STATIC_AND_SHARED OR BUILD_SHARED_LIBS) @@ -1290,6 +1330,7 @@ else (FAT_RUNTIME) $ $ ${RUNTIME_LIBS}) + set_target_properties(hs PROPERTIES OUTPUT_NAME ${HS_LIB_BASENAME}) endif (BUILD_STATIC_LIBS) @@ -1421,7 +1462,7 @@ if (BUILD_STATIC_AND_SHARED OR BUILD_SHARED_LIBS) add_dependencies(hs_shared ragel_Parser) target_link_libraries(hs_shared OpenSSL::Crypto) set_target_properties(hs_shared PROPERTIES - OUTPUT_NAME hs + OUTPUT_NAME ${HS_LIB_BASENAME} VERSION ${LIB_VERSION} SOVERSION ${LIB_SOVERSION} MACOSX_RPATH ON) diff --git a/libhs.pc.in b/libhs.pc.in index fed4db454..bd28a1a61 100644 --- a/libhs.pc.in +++ b/libhs.pc.in @@ -3,9 +3,9 @@ exec_prefix=@CMAKE_INSTALL_PREFIX@ libdir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@ includedir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@ -Name: libhs +Name: @HS_PC_NAME@ Description: Intel(R) Hyperscan Library Version: @HS_VERSION@ -Libs: -L${libdir} -lhs +Libs: -L${libdir} -l@HS_LIB_BASENAME@ Libs.private: @PRIVATE_LIBS@ Cflags: -I${includedir}/hs diff --git a/src/hwlm/hwlm.c b/src/hwlm/hwlm.c index 8cf585a98..d103df331 100644 --- a/src/hwlm/hwlm.c +++ b/src/hwlm/hwlm.c @@ -31,7 +31,9 @@ */ #include "hwlm.h" #include "hwlm_internal.h" +#if !defined(HS_TEDDY_ONLY) #include "noodle_engine.h" +#endif #include "scratch.h" #include "ue2common.h" #include "fdr/fdr.h" @@ -182,10 +184,12 @@ hwlm_error_t hwlmExec(const struct HWLM *t, const u8 *buf, size_t len, assert(start < len); +#if !defined(HS_TEDDY_ONLY) if (t->type == HWLM_ENGINE_NOOD) { DEBUG_PRINTF("calling noodExec\n"); return noodExec(HWLM_C_DATA(t), buf, len, start, cb, scratch); } +#endif assert(t->type == HWLM_ENGINE_FDR); const union AccelAux *aa = &t->accel0; @@ -217,6 +221,7 @@ hwlm_error_t hwlmExecStreaming(const struct HWLM *t, size_t len, size_t start, assert(start < len); +#if !defined(HS_TEDDY_ONLY) if (t->type == HWLM_ENGINE_NOOD) { DEBUG_PRINTF("calling noodExec\n"); // If we've been handed a start offset, we can use a block mode scan at @@ -228,6 +233,7 @@ hwlm_error_t hwlmExecStreaming(const struct HWLM *t, size_t len, size_t start, scratch); } } +#endif assert(t->type == HWLM_ENGINE_FDR); const union AccelAux *aa = &t->accel0; diff --git a/src/hwlm/hwlm_build.cpp b/src/hwlm/hwlm_build.cpp index 1b3328152..766c81431 100644 --- a/src/hwlm/hwlm_build.cpp +++ b/src/hwlm/hwlm_build.cpp @@ -40,6 +40,10 @@ #include "noodle_build.h" #include "scratch.h" #include "ue2common.h" +#include "fdr/fdr_internal.h" +#if defined(HS_TEDDY_ONLY) +#include "fdr/teddy_compile.h" +#endif #include "fdr/fdr_compile.h" #include "fdr/fdr_compile_internal.h" #include "fdr/fdr_engine_description.h" @@ -57,6 +61,10 @@ using namespace std; namespace ue2 { +#if defined(HS_TEDDY_ONLY) +static constexpr size_t TEDDY_ONLY_MAX_DECOMPOSED_EXPRESSIONS = 96; +#endif + HWLMProto::HWLMProto(u8 engType_in, vector lits_in) : engType(engType_in), lits(move(lits_in)) {} @@ -103,6 +111,7 @@ bool everyoneHasGroups(const vector &lits) { } #endif +#if !defined(HS_TEDDY_ONLY) static bool isNoodleable(const vector &lits, const CompileContext &cc) { @@ -117,15 +126,27 @@ bool isNoodleable(const vector &lits, return true; } +#endif bytecode_ptr hwlmBuild(const HWLMProto &proto, const CompileContext &cc, UNUSED hwlm_group_t expected_groups) { size_t engSize = 0; shared_ptr eng; +#if defined(HS_TEDDY_ONLY) + DEBUG_PRINTF("building table with %zu strings\n", proto.lits.size()); +#else const auto &lits = proto.lits; DEBUG_PRINTF("building table with %zu strings\n", lits.size()); +#endif +#if defined(HS_TEDDY_ONLY) + auto teddy = teddyBuildTable(proto, cc.grey); + if (teddy) { + engSize = teddy.size(); + eng = move(teddy); + } +#else if (proto.engType == HWLM_ENGINE_NOOD) { DEBUG_PRINTF("build noodle table\n"); const hwlmLiteral &lit = lits.front(); @@ -142,6 +163,7 @@ bytecode_ptr hwlmBuild(const HWLMProto &proto, const CompileContext &cc, } eng = move(fdr); } +#endif if (!eng) { return nullptr; @@ -167,6 +189,12 @@ hwlmBuildProto(vector &lits, bool make_small, assert(!lits.empty()); dumpLits(lits); +#if defined(HS_TEDDY_ONLY) + if (lits.size() > TEDDY_ONLY_MAX_DECOMPOSED_EXPRESSIONS) { + throw CompileError("Teddy-only build supports at most 96 decomposed expressions."); + } +#endif + // Check that we haven't exceeded the maximum number of literals. if (lits.size() > cc.grey.limitLiteralCount) { throw ResourceLimitError(); @@ -199,6 +227,13 @@ hwlmBuildProto(vector &lits, bool make_small, assert(everyoneHasGroups(lits)); +#if defined(HS_TEDDY_ONLY) + proto = teddyBuildProtoHinted(HWLM_ENGINE_FDR, lits, make_small, + HINT_INVALID, cc.target_info); + if (!proto) { + throw CompileError("Unable to generate Teddy-only literal matcher proto."); + } +#else if (isNoodleable(lits, cc)) { DEBUG_PRINTF("build noodle table\n"); proto = ue2::make_unique(HWLM_ENGINE_NOOD, lits); @@ -210,6 +245,7 @@ hwlmBuildProto(vector &lits, bool make_small, return nullptr; } } +#endif return proto; } @@ -217,6 +253,11 @@ hwlmBuildProto(vector &lits, bool make_small, size_t hwlmSize(const HWLM *h) { size_t engSize = 0; +#if defined(HS_TEDDY_ONLY) + if (h->type == HWLM_ENGINE_FDR) { + engSize = ((const FDR *)HWLM_C_DATA(h))->size; + } +#else switch (h->type) { case HWLM_ENGINE_NOOD: engSize = noodSize((const noodTable *)HWLM_C_DATA(h)); @@ -225,6 +266,7 @@ size_t hwlmSize(const HWLM *h) { engSize = fdrSize((const FDR *)HWLM_C_DATA(h)); break; } +#endif if (!engSize) { return 0; diff --git a/src/nfa/nfa_rev_api.h b/src/nfa/nfa_rev_api.h index ea9bc411c..6c0c908cd 100644 --- a/src/nfa/nfa_rev_api.h +++ b/src/nfa/nfa_rev_api.h @@ -104,23 +104,29 @@ size_t nfaRevAccel_i(const struct NFA *nfa, const u8 *buffer, size_t length) { break; case ACCEL_RDEOD: DEBUG_PRINTF("ACCEL_RDEOD\n"); - if (length < nfa->rAccelOffset + 1) { + { + const size_t offset = nfa->rAccelOffset; + if (length < offset + 1) { break; } - if (unaligned_load_u16(buffer + length - nfa->rAccelOffset) != + if (unaligned_load_u16(buffer + length - offset) != nfa->rAccelData.dc) { return 0; } + } break; case ACCEL_RDEOD_NOCASE: DEBUG_PRINTF("ACCEL_RDEOD_NOCASE\n"); - if (length < nfa->rAccelOffset + 1) { + { + const size_t offset = nfa->rAccelOffset; + if (length < offset + 1) { break; } - if ((unaligned_load_u16(buffer + length - nfa->rAccelOffset) & + if ((unaligned_load_u16(buffer + length - offset) & DOUBLE_CASE_CLEAR) != nfa->rAccelData.dc) { return 0; } + } break; default: assert(!"not here");