diff --git a/Makefile b/Makefile index 857d2ee..ce511d0 100644 --- a/Makefile +++ b/Makefile @@ -4,27 +4,21 @@ CC := gcc # Number of random text expressions to generate, for random testing NRAND_TESTS := 1000 -PYTHON != if (python --version 2>&1 | grep -q 'Python 2\..*'); then \ - echo 'python'; \ - elif command -v python2 >/dev/null 2>&1; then \ - echo 'python2'; \ - else \ - echo 'Error: no compatible python version found.' >&2; \ - exit 1; \ - fi +PYTHON := python # Flags to pass to compiler CFLAGS := -O3 -Wall -Wextra -std=c99 -I. all: - @$(CC) $(CFLAGS) re.c tests/test1.c -o tests/test1 - @$(CC) $(CFLAGS) re.c tests/test2.c -o tests/test2 - @$(CC) $(CFLAGS) re.c tests/test_rand.c -o tests/test_rand - @$(CC) $(CFLAGS) re.c tests/test_rand_neg.c -o tests/test_rand_neg - @$(CC) $(CFLAGS) re.c tests/test_compile.c -o tests/test_compile + @$(CC) $(CFLAGS) re.c tests/test1.c -o tests/test1 + @$(CC) $(CFLAGS) re.c tests/test2.c -o tests/test2 + @$(CC) $(CFLAGS) re.c tests/test_rand.c -o tests/test_rand + @$(CC) $(CFLAGS) re.c tests/test_rand_neg.c -o tests/test_rand_neg + @$(CC) $(CFLAGS) re.c tests/test_compile.c -o tests/test_compile + @$(CC) $(CFLAGS) re.c tests/test_end_anchor.c -o tests/test_end_anchor clean: - @rm -f tests/test1 tests/test2 tests/test_rand tests/test_compile + @rm -f tests/test1 tests/test2 tests/test_rand tests/test_compile tests/test_end_anchor @#@$(foreach test_bin,$(TEST_BINS), rm -f $(test_bin) ; ) @rm -f a.out @rm -f *.o @@ -37,6 +31,49 @@ test: all @./tests/test1 @echo Testing handling of invalid regex patterns @./tests/test_compile + @echo Compiling patterns in both Python and C and verifying the results are the same: + @echo + @$(PYTHON) ./scripts/regex_test_compile.py \\d+\\w?\\D\\d + @$(PYTHON) ./scripts/regex_test_compile.py \\s+[a-zA-Z0-9?]* + @$(PYTHON) ./scripts/regex_test_compile.py \\w*\\d?\\w\\? + @$(PYTHON) ./scripts/regex_test_compile.py [^\\d]+\\\\?\\s + @$(PYTHON) ./scripts/regex_test_compile.py [^\\w][^-1-4] + @$(PYTHON) ./scripts/regex_test_compile.py [^\\w] + @$(PYTHON) ./scripts/regex_test_compile.py [^1-4] + @$(PYTHON) ./scripts/regex_test_compile.py [^-1-4] + @$(PYTHON) ./scripts/regex_test_compile.py [^\\d]+\\s?[\\w]* + @$(PYTHON) ./scripts/regex_test_compile.py a+b*[ac]*.+.*.[\\.]. + @$(PYTHON) ./scripts/regex_test_compile.py a?b[ac*]*.?[\\]+[?]? + @$(PYTHON) ./scripts/regex_test_compile.py [1-5-]+[-1-2]-[-] + @$(PYTHON) ./scripts/regex_test_compile.py [-1-3]-[-]+ + @$(PYTHON) ./scripts/regex_test_compile.py [1-5]+[-1-2]-[\\-] + @$(PYTHON) ./scripts/regex_test_compile.py [-1-2]* + @$(PYTHON) ./scripts/regex_test_compile.py \\s?[a-fKL098]+-? + @$(PYTHON) ./scripts/regex_test_compile.py [\\-]* + @$(PYTHON) ./scripts/regex_test_compile.py [\\\\]+ + @$(PYTHON) ./scripts/regex_test_compile.py [0-9a-fA-F]+ + @$(PYTHON) ./scripts/regex_test_compile.py [1379][2468][abcdef] + @$(PYTHON) ./scripts/regex_test_compile.py [012345-9]?[0123-789] + @$(PYTHON) ./scripts/regex_test_compile.py [012345-9] + @$(PYTHON) ./scripts/regex_test_compile.py [0-56789] + @$(PYTHON) ./scripts/regex_test_compile.py [abc-zABC-Z] + @$(PYTHON) ./scripts/regex_test_compile.py [a\d]?1234 + @$(PYTHON) ./scripts/regex_test_compile.py .*123faerdig + @$(PYTHON) ./scripts/regex_test_compile.py .?\\w+jsj$ + @$(PYTHON) ./scripts/regex_test_compile.py [?to][+to][?ta][*ta] + @$(PYTHON) ./scripts/regex_test_compile.py \\d+ + @$(PYTHON) ./scripts/regex_test_compile.py [a-z]+ + @$(PYTHON) ./scripts/regex_test_compile.py \\s+[a-zA-Z0-9?]* + @$(PYTHON) ./scripts/regex_test_compile.py \\w + @$(PYTHON) ./scripts/regex_test_compile.py \\d + @$(PYTHON) ./scripts/regex_test_compile.py [\\d] + @$(PYTHON) ./scripts/regex_test_compile.py [^\\d] + @$(PYTHON) ./scripts/regex_test_compile.py [^-1-4] + @$(PYTHON) ./scripts/regex_test_compile.py \\x01[^\\xff][^ + @$(PYTHON) ./scripts/regex_test_compile.py \\x01[^\\xff][\ + @echo + @echo + @echo @echo Testing patterns against $(NRAND_TESTS) random strings matching the Python implementation and comparing: @echo @$(PYTHON) ./scripts/regex_test.py \\d+\\w?\\D\\d $(NRAND_TESTS) @@ -101,9 +138,20 @@ test: all @$(PYTHON) ./scripts/regex_test_neg.py [012345-9] $(NRAND_TESTS) @$(PYTHON) ./scripts/regex_test_neg.py [0-56789] $(NRAND_TESTS) @$(PYTHON) ./scripts/regex_test_neg.py .*123faerdig $(NRAND_TESTS) + @$(PYTHON) ./scripts/regex_test_neg.py a^ $(NRAND_TESTS) @echo @echo @./tests/test2 @echo @echo + @echo + @echo + @./tests/test_end_anchor + @echo + @echo + +CBMC := cbmc +# unwindset: loop max MAX_REGEXP_OBJECTS patterns +verify: + $(CBMC) -DCPROVER --unwindset 8 --unwind 16 --depth 16 --bounds-check --pointer-check --memory-leak-check --div-by-zero-check --signed-overflow-check --unsigned-overflow-check --pointer-overflow-check --conversion-check --undefined-shift-check --enum-range-check $(CBMC_ARGS) re.c diff --git a/README.md b/README.md index 0a2be86..d74f46a 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,6 @@ int re_match(const char* pattern, const char* text, int* matchlength); ### Supported regex-operators The following features / regex-operators are supported by this library. -NOTE: inverted character classes are buggy - see the test harness for concrete examples. - - `.` Dot, matches any character - `^` Start anchor, matches beginning of string @@ -63,7 +61,7 @@ NOTE: inverted character classes are buggy - see the test harness for concrete e - `[abc]` Character class, match if one of {'a', 'b', 'c'} - `[^abc]` Inverted class, match if NOT one of {'a', 'b', 'c'} - `[a-zA-Z]` Character ranges, the character set of the ranges { a-z | A-Z } - - `\s` Whitespace, \t \f \r \n \v and spaces + - `\s` Whitespace, '\t' '\f' '\r' '\n' '\v' and spaces - `\S` Non-whitespace - `\w` Alphanumeric, [a-zA-Z0-9_] - `\W` Non-alphanumeric @@ -90,7 +88,7 @@ int match_length; /* Standard null-terminated C-string to search: */ const char* string_to_search = "ahem.. 'hello world !' .."; -/* Compile a simple regular expression using character classes, meta-char and greedy + non-greedy quantifiers: */ +/* Compile a simple regular expression using character classes, meta-char and greedy quantifiers: */ re_t pattern = re_compile("[Hh]ello [Ww]orld\\s*[!]?"); /* Check if the regex matches the text: */ @@ -104,10 +102,15 @@ if (match_idx != -1) For more usage examples I encourage you to look at the code in the `tests`-folder. ### TODO -- Fix the implementation of inverted character classes. -- Fix implementation of branches (`|`), and see if that can lead us closer to groups as well, e.g. `(a|b)+`. +- Fix implementation of branches (`|`) (see the branch), and add groups as well, e.g. `(a|b)+`. +- `re_match_capture()` with groups. - Add `example.c` that demonstrates usage. - Add `tests/test_perf.c` for performance and time measurements. +- Add optional multibyte support (e.g. UTF-8). On non-wchar systems roll our own. +- Word boundary: \b \B +- non-greedy, lazy quantifiers (??, +?, *?, {n,m}?) +- case-insensitive option or API. `re_matchi()` +- '.' may not match '\r' nor '\n', unless a single-line option is given. - Testing: Improve pattern rejection testing. ### FAQ @@ -118,6 +121,3 @@ For more usage examples I encourage you to look at the code in the `tests`-folde ### License All material in this repository is in the public domain. - - - diff --git a/formal_verification.md b/formal_verification.md index 46fc9ee..a36bb45 100644 --- a/formal_verification.md +++ b/formal_verification.md @@ -140,3 +140,8 @@ sys 9m34.654s klee@780432c1aaae0:~$ ``` +---- + +For the formal verifier CBMC just call make verify. +This verifier is much faster and better than klee. +https://www.cprover.org/cbmc/ diff --git a/re.c b/re.c index 20d1474..1957e94 100644 --- a/re.c +++ b/re.c @@ -15,7 +15,7 @@ * '+' Plus, match one or more (greedy) * '?' Question, match zero or one (non-greedy) * '[abc]' Character class, match if one of {'a', 'b', 'c'} - * '[^abc]' Inverted class, match if NOT one of {'a', 'b', 'c'} -- NOTE: feature is currently broken! + * '[^abc]' Inverted class, match if NOT one of {'a', 'b', 'c'} * '[a-zA-Z]' Character ranges, the character set of the ranges { a-z | A-Z } * '\s' Whitespace, \t \f \r \n \v and spaces * '\S' Non-whitespace @@ -23,6 +23,7 @@ * '\W' Non-alphanumeric * '\d' Digits, [0-9] * '\D' Non-digits + * '|' Branch, matches either the preceding or following pattern * * */ @@ -35,30 +36,31 @@ /* Definitions: */ -#define MAX_REGEXP_OBJECTS 30 /* Max number of regex symbols in expression. */ -#define MAX_CHAR_CLASS_LEN 40 /* Max length of character-class buffer in. */ +#define MAX_REGEXP_LEN 70 /* Max number of bytes for a regex. */ -enum { UNUSED, DOT, BEGIN, END, QUESTIONMARK, STAR, PLUS, CHAR, CHAR_CLASS, INV_CHAR_CLASS, DIGIT, NOT_DIGIT, ALPHA, NOT_ALPHA, WHITESPACE, NOT_WHITESPACE, /* BRANCH */ }; +enum regex_type_e { UNUSED, DOT, BEGIN, END, QUESTIONMARK, STAR, PLUS, CHAR, CHAR_CLASS, INV_CHAR_CLASS, DIGIT, NOT_DIGIT, ALPHA, NOT_ALPHA, WHITESPACE, NOT_WHITESPACE, /* BRANCH */ }; typedef struct regex_t { - unsigned char type; /* CHAR, STAR, etc. */ - union - { - unsigned char ch; /* the character itself */ - unsigned char* ccl; /* OR a pointer to characters in class */ - } u; + unsigned char type; /* CHAR, STAR, etc. */ + unsigned char data_len; + unsigned char data[0]; } regex_t; +static re_t getnext(regex_t* pattern) +{ + return (re_t)(((unsigned char*)pattern) + 2 + pattern->data_len); +} + /* Private function declarations: */ static int matchpattern(regex_t* pattern, const char* text, int* matchlength); -static int matchcharclass(char c, const char* str); -static int matchstar(regex_t p, regex_t* pattern, const char* text, int* matchlength); -static int matchplus(regex_t p, regex_t* pattern, const char* text, int* matchlength); -static int matchone(regex_t p, char c); +static int matchcharclass(char c, unsigned char len, const char* str); +static int matchstar(regex_t *p, regex_t* pattern, const char* text, int* matchlength); +static int matchplus(regex_t *p, regex_t* pattern, const char* text, int* matchlength); +static int matchone(regex_t* p, char c); static int matchdigit(char c); static int matchalpha(char c); static int matchwhitespace(char c); @@ -80,9 +82,9 @@ int re_matchp(re_t pattern, const char* text, int* matchlength) *matchlength = 0; if (pattern != 0) { - if (pattern[0].type == BEGIN) + if (pattern->type == BEGIN) { - return ((matchpattern(&pattern[1], text, matchlength)) ? 0 : -1); + return ((matchpattern(getnext(pattern), text, matchlength)) ? 0 : -1); } else { @@ -99,6 +101,10 @@ int re_matchp(re_t pattern, const char* text, int* matchlength) return idx; } + + // Reset match length for the next starting point + *matchlength = 0; + } while (*text++ != '\0'); } @@ -106,33 +112,37 @@ int re_matchp(re_t pattern, const char* text, int* matchlength) return -1; } +static int min(int a, int b) +{ + return (a <= b) ? a : b; +} + re_t re_compile(const char* pattern) { - /* The sizes of the two static arrays below substantiates the static RAM usage of this module. - MAX_REGEXP_OBJECTS is the max number of symbols in the expression. - MAX_CHAR_CLASS_LEN determines the size of buffer for chars in all char-classes in the expression. */ - static regex_t re_compiled[MAX_REGEXP_OBJECTS]; - static unsigned char ccl_buf[MAX_CHAR_CLASS_LEN]; - int ccl_bufidx = 1; + /* The size of this static array substantiates the static RAM usage of this module. + MAX_REGEXP_LEN is the max number number of bytes in the expression. */ + static unsigned char re_data[MAX_REGEXP_LEN]; char c; /* current char in pattern */ int i = 0; /* index into pattern */ - int j = 0; /* index into re_compiled */ + int j = 0; /* index into re_data */ - while (pattern[i] != '\0' && (j+1 < MAX_REGEXP_OBJECTS)) + while (pattern[i] != '\0' && (j+3 < MAX_REGEXP_LEN)) { c = pattern[i]; + regex_t *re_compiled = (regex_t*)(re_data+j); + re_compiled->data_len = 0; switch (c) { /* Meta-characters: */ - case '^': { re_compiled[j].type = BEGIN; } break; - case '$': { re_compiled[j].type = END; } break; - case '.': { re_compiled[j].type = DOT; } break; - case '*': { re_compiled[j].type = STAR; } break; - case '+': { re_compiled[j].type = PLUS; } break; - case '?': { re_compiled[j].type = QUESTIONMARK; } break; -/* case '|': { re_compiled[j].type = BRANCH; } break; <-- not working properly */ + case '^': { re_compiled->type = BEGIN; } break; + case '$': { re_compiled->type = END; } break; + case '.': { re_compiled->type = DOT; } break; + case '*': { re_compiled->type = STAR; } break; + case '+': { re_compiled->type = PLUS; } break; + case '?': { re_compiled->type = QUESTIONMARK; } break; +/* case '|': { re_compiled->type = BRANCH; } break; <-- not working properly */ /* Escaped character-classes (\s \w ...): */ case '\\': @@ -145,41 +155,38 @@ re_t re_compile(const char* pattern) switch (pattern[i]) { /* Meta-character: */ - case 'd': { re_compiled[j].type = DIGIT; } break; - case 'D': { re_compiled[j].type = NOT_DIGIT; } break; - case 'w': { re_compiled[j].type = ALPHA; } break; - case 'W': { re_compiled[j].type = NOT_ALPHA; } break; - case 's': { re_compiled[j].type = WHITESPACE; } break; - case 'S': { re_compiled[j].type = NOT_WHITESPACE; } break; - - /* Escaped character, e.g. '.' or '$' */ + case 'd': { re_compiled->type = DIGIT; } break; + case 'D': { re_compiled->type = NOT_DIGIT; } break; + case 'w': { re_compiled->type = ALPHA; } break; + case 'W': { re_compiled->type = NOT_ALPHA; } break; + case 's': { re_compiled->type = WHITESPACE; } break; + case 'S': { re_compiled->type = NOT_WHITESPACE; } break; + + /* Escaped character, e.g. '.', '$' or '\\' */ default: { - re_compiled[j].type = CHAR; - re_compiled[j].u.ch = pattern[i]; + re_compiled->type = CHAR; + re_compiled->data_len = 1; + re_compiled->data[0] = pattern[i]; } break; } } - /* '\\' as last char in pattern -> invalid regular expression. */ -/* + /* '\\' as last char without previous \\ -> invalid regular expression. */ else { - re_compiled[j].type = CHAR; - re_compiled[j].ch = pattern[i]; + return 0; } -*/ } break; /* Character class: */ case '[': { - /* Remember where the char-buffer starts. */ - int buf_begin = ccl_bufidx; + int char_limit = min(0xff, MAX_REGEXP_LEN - j - 4); // 4 for this object and UNUSED at the minimum /* Look-ahead to determine if negated */ if (pattern[i+1] == '^') { - re_compiled[j].type = INV_CHAR_CLASS; + re_compiled->type = INV_CHAR_CLASS; i += 1; /* Increment i to avoid including '^' in the char-buffer */ if (pattern[i+1] == 0) /* incomplete pattern, missing non-zero char after '^' */ { @@ -188,7 +195,7 @@ re_t re_compile(const char* pattern) } else { - re_compiled[j].type = CHAR_CLASS; + re_compiled->type = CHAR_CLASS; } /* Copy characters inside [..] to buffer */ @@ -197,7 +204,7 @@ re_t re_compile(const char* pattern) { if (pattern[i] == '\\') { - if (ccl_bufidx >= MAX_CHAR_CLASS_LEN - 1) + if (re_compiled->data_len >= char_limit) { //fputs("exceeded internal buffer!\n", stderr); return 0; @@ -206,69 +213,78 @@ re_t re_compile(const char* pattern) { return 0; } - ccl_buf[ccl_bufidx++] = pattern[i++]; + re_compiled->data[re_compiled->data_len++] = pattern[i++]; } - else if (ccl_bufidx >= MAX_CHAR_CLASS_LEN) + // TODO: I think this "else if" is a bug, should just be "if" + else if (re_compiled->data_len >= char_limit) { //fputs("exceeded internal buffer!\n", stderr); return 0; } - ccl_buf[ccl_bufidx++] = pattern[i]; + re_compiled->data[re_compiled->data_len++] = pattern[i]; } - if (ccl_bufidx >= MAX_CHAR_CLASS_LEN) + if (re_compiled->data_len >= char_limit) { /* Catches cases such as [00000000000000000000000000000000000000][ */ //fputs("exceeded internal buffer!\n", stderr); return 0; } - /* Null-terminate string end */ - ccl_buf[ccl_bufidx++] = 0; - re_compiled[j].u.ccl = &ccl_buf[buf_begin]; } break; + case '\0': // EOL + return 0; + /* Other characters: */ default: { - re_compiled[j].type = CHAR; - re_compiled[j].u.ch = c; + re_compiled->type = CHAR; + re_compiled->data_len = 1; + re_compiled->data[0] = (unsigned char)c; } break; } - /* no buffer-out-of-bounds access on invalid patterns - see https://github.com/kokke/tiny-regex-c/commit/1a279e04014b70b0695fba559a7c05d55e6ee90b */ - if (pattern[i] == 0) - { - return 0; - } - i += 1; - j += 1; + j += 2 + re_compiled->data_len; + } + if (j + 1 >= MAX_REGEXP_LEN) { + //fputs("exceeded internal buffer!\n", stderr); + return 0; } /* 'UNUSED' is a sentinel used to indicate end-of-pattern */ - re_compiled[j].type = UNUSED; + re_data[j] = UNUSED; + re_data[j+1] = 0; - return (re_t) re_compiled; + return (re_t) re_data; } void re_print(regex_t* pattern) { - const char* types[] = { "UNUSED", "DOT", "BEGIN", "END", "QUESTIONMARK", "STAR", "PLUS", "CHAR", "CHAR_CLASS", "INV_CHAR_CLASS", "DIGIT", "NOT_DIGIT", "ALPHA", "NOT_ALPHA", "WHITESPACE", "NOT_WHITESPACE", "BRANCH" }; + const char *const types[] = { "UNUSED", "DOT", "BEGIN", "END", "QUESTIONMARK", "STAR", "PLUS", "CHAR", "CHAR_CLASS", "INV_CHAR_CLASS", "DIGIT", "NOT_DIGIT", "ALPHA", "NOT_ALPHA", "WHITESPACE", "NOT_WHITESPACE" /*, "BRANCH" */ }; - int i; int j; char c; - for (i = 0; i < MAX_REGEXP_OBJECTS; ++i) + + if (!pattern) + return; + for (;; pattern = getnext(pattern)) { - if (pattern[i].type == UNUSED) + if (pattern->type == UNUSED) { break; } - printf("type: %s", types[pattern[i].type]); - if (pattern[i].type == CHAR_CLASS || pattern[i].type == INV_CHAR_CLASS) + if (pattern->type <= NOT_WHITESPACE) + printf("type: %s", types[pattern->type]); + else + printf("invalid type: %d", pattern->type); + + if (pattern->type == CHAR_CLASS || pattern->type == INV_CHAR_CLASS) { printf(" ["); - for (j = 0; j < MAX_CHAR_CLASS_LEN; ++j) + if (pattern->type == INV_CHAR_CLASS) + printf("^"); + for (j = 0; j < pattern->data_len; ++j) { - c = pattern[i].u.ccl[j]; + c = pattern->data[j]; if ((c == '\0') || (c == ']')) { break; @@ -277,9 +293,9 @@ void re_print(regex_t* pattern) } printf("]"); } - else if (pattern[i].type == CHAR) + else if (pattern->type == CHAR) { - printf(" '%c'", pattern[i].u.ch); + printf(" '%c'", pattern->data[0]); } printf("\n"); } @@ -290,15 +306,15 @@ void re_print(regex_t* pattern) /* Private functions: */ static int matchdigit(char c) { - return isdigit(c); + return isdigit((unsigned char)c); } static int matchalpha(char c) { - return isalpha(c); + return isalpha((unsigned char)c); } static int matchwhitespace(char c) { - return isspace(c); + return isspace((unsigned char)c); } static int matchalphanum(char c) { @@ -342,32 +358,38 @@ static int matchmetachar(char c, const char* str) } } -static int matchcharclass(char c, const char* str) +static int matchcharclass(char c, unsigned char len, const char* str) { - do + if (str[0] == '-' && c == '-') { + return 1; + } + + for(unsigned char i = 0; i < len; i++) { - if (matchrange(c, str)) + if (matchrange(c, &str[i])) { return 1; } - else if (str[0] == '\\') + else if (str[i] == '\\') { /* Escape-char: increment str-ptr and match on next char */ - str += 1; - if (matchmetachar(c, str)) + i++; + if (matchmetachar(c, &str[i])) { return 1; } - else if ((c == str[0]) && !ismetachar(c)) + else if ((c == str[i]) && !ismetachar(c)) { return 1; } } - else if (c == str[0]) + else if (c == str[i]) { if (c == '-') { - return ((str[-1] == '\0') || (str[1] == '\0')); + if ((str[i-1] == '\0') || (i == len - 1)) + return 1; + // else continue } else { @@ -375,70 +397,52 @@ static int matchcharclass(char c, const char* str) } } } - while (*str++ != '\0'); return 0; } -static int matchone(regex_t p, char c) +static int matchone(regex_t* p, char c) { - switch (p.type) + switch (p->type) { case DOT: return matchdot(c); - case CHAR_CLASS: return matchcharclass(c, (const char*)p.u.ccl); - case INV_CHAR_CLASS: return !matchcharclass(c, (const char*)p.u.ccl); + case CHAR_CLASS: return matchcharclass(c, p->data_len, (const char*)p->data); + case INV_CHAR_CLASS: return !matchcharclass(c, p->data_len, (const char*)p->data); case DIGIT: return matchdigit(c); case NOT_DIGIT: return !matchdigit(c); case ALPHA: return matchalphanum(c); case NOT_ALPHA: return !matchalphanum(c); case WHITESPACE: return matchwhitespace(c); case NOT_WHITESPACE: return !matchwhitespace(c); - default: return (p.u.ch == c); + case BEGIN: return 0; + default: return (p->data[0] == c); } } -static int matchstar(regex_t p, regex_t* pattern, const char* text, int* matchlength) +static inline int matchstar(regex_t* p, regex_t* pattern, const char* text, int* matchlength) { - int prelen = *matchlength; - const char* prepoint = text; - while ((text[0] != '\0') && matchone(p, *text)) - { - text++; - (*matchlength)++; - } - while (text >= prepoint) - { - if (matchpattern(pattern, text--, matchlength)) - return 1; - (*matchlength)--; - } - - *matchlength = prelen; - return 0; + return matchplus(p, pattern, text, matchlength) || matchpattern(pattern, text, matchlength); } -static int matchplus(regex_t p, regex_t* pattern, const char* text, int* matchlength) +static int matchplus(regex_t* p, regex_t* pattern, const char* text, int* matchlength) { const char* prepoint = text; while ((text[0] != '\0') && matchone(p, *text)) { text++; - (*matchlength)++; } - while (text > prepoint) + for (; text > prepoint; text--) { - if (matchpattern(pattern, text--, matchlength)) + if (matchpattern(pattern, text, matchlength)) { + *matchlength += text - prepoint; return 1; - (*matchlength)--; + } } - return 0; } -static int matchquestion(regex_t p, regex_t* pattern, const char* text, int* matchlength) +static int matchquestion(regex_t *p, regex_t* pattern, const char* text, int* matchlength) { - if (p.type == UNUSED) - return 1; if (matchpattern(pattern, text, matchlength)) return 1; if (*text && matchone(p, *text++)) @@ -452,77 +456,47 @@ static int matchquestion(regex_t p, regex_t* pattern, const char* text, int* mat return 0; } - -#if 0 - -/* Recursive matching */ -static int matchpattern(regex_t* pattern, const char* text, int *matchlength) -{ - int pre = *matchlength; - if ((pattern[0].type == UNUSED) || (pattern[1].type == QUESTIONMARK)) - { - return matchquestion(pattern[1], &pattern[2], text, matchlength); - } - else if (pattern[1].type == STAR) - { - return matchstar(pattern[0], &pattern[2], text, matchlength); - } - else if (pattern[1].type == PLUS) - { - return matchplus(pattern[0], &pattern[2], text, matchlength); - } - else if ((pattern[0].type == END) && pattern[1].type == UNUSED) - { - return text[0] == '\0'; - } - else if ((text[0] != '\0') && matchone(pattern[0], text[0])) - { - (*matchlength)++; - return matchpattern(&pattern[1], text+1); - } - else - { - *matchlength = pre; - return 0; - } -} - -#else - /* Iterative matching */ static int matchpattern(regex_t* pattern, const char* text, int* matchlength) { int pre = *matchlength; - do + while (1) { - if ((pattern[0].type == UNUSED) || (pattern[1].type == QUESTIONMARK)) + if (pattern->type == UNUSED) { - return matchquestion(pattern[0], &pattern[2], text, matchlength); + return 1; + } + regex_t* next_pattern = getnext(pattern); + if (next_pattern->type == QUESTIONMARK) + { + return matchquestion(pattern, getnext(next_pattern), text, matchlength); } - else if (pattern[1].type == STAR) + else if (next_pattern->type == STAR) { - return matchstar(pattern[0], &pattern[2], text, matchlength); + return matchstar(pattern, getnext(next_pattern), text, matchlength); } - else if (pattern[1].type == PLUS) + else if (next_pattern->type == PLUS) { - return matchplus(pattern[0], &pattern[2], text, matchlength); + return matchplus(pattern, getnext(next_pattern), text, matchlength); } - else if ((pattern[0].type == END) && pattern[1].type == UNUSED) + else if ((pattern->type == END) && next_pattern->type == UNUSED) { return (text[0] == '\0'); } /* Branching is not working properly - else if (pattern[1].type == BRANCH) + else if (pattern->type == BRANCH) { - return (matchpattern(pattern, text) || matchpattern(&pattern[2], text)); + return (matchpattern(pattern, text) || matchpattern(getnext(next_pattern), text)); } */ (*matchlength)++; + if (text[0] == '\0') + break; + if (!matchone(pattern, *text++)) + break; + pattern = next_pattern; } - while ((text[0] != '\0') && matchone(*pattern++, *text++)); *matchlength = pre; return 0; } - -#endif diff --git a/scripts/exrex.py b/scripts/exrex.py deleted file mode 100755 index 9357748..0000000 --- a/scripts/exrex.py +++ /dev/null @@ -1,311 +0,0 @@ -#!/usr/bin/env python - -# This file is part of exrex. -# -# exrex is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# exrex is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with exrex. If not, see < http://www.gnu.org/licenses/ >. -# -# (C) 2012- by Adam Tauber, - -try: - from future_builtins import map, range -except: - pass -from re import match, sre_parse -from itertools import product, chain, tee -from random import choice,randint -import string - -__all__ = ('generate', 'CATEGORIES', 'count', 'parse', 'getone') - -CATEGORIES = {'category_space' : sorted(sre_parse.WHITESPACE) - ,'category_digit' : sorted(sre_parse.DIGITS) - ,'category_not_digit' : [chr(x) for x in range(32, 123) if - match('\D', chr(x))] - ,'category_any' : [chr(x) for x in range(32, 123)] - ,'category_word' : sorted( frozenset(string.ascii_letters + string.digits + "_") ) - ,'category_not_word' : [chr(x) for x in range(32, 123) if - match('\W', chr(x))] - } - -def comb(g, i): - for c in g: - g2,i = tee(i) - for c2 in g2: - yield c+c2 - -def mappend(g, c): - for cc in g: - yield cc+c - -def _in(d): - ret = [] - neg = False - for i in d: - if i[0] == 'range': - subs = map(chr, range(i[1][0], i[1][1]+1)) - if neg: - for char in subs: - try: - ret.remove(char) - except: - pass - else: - ret.extend(subs) - elif i[0] == 'literal': - if neg: - try: - ret.remove(chr(i[1])) - except: - pass - else: - ret.append(chr(i[1])) - elif i[0] == 'category': - subs = CATEGORIES.get(i[1], ['']) - if neg: - for char in subs: - try: - ret.remove(char) - except: - pass - else: - ret.extend(subs) - elif i[0] == 'negate': - ret = list(CATEGORIES['category_any']) - neg = True - return ret - - -def prods(orig, ran, items): - for o in orig: - for r in ran: - for s in product(items, repeat=r): - yield o+''.join(s) - -def ggen(g1, f, *args, **kwargs): - for a in g1: - g2 = f(*args, **kwargs) - if isinstance(g2, int): - yield g2 - else: - for b in g2: - yield a+b - -def _gen(d, limit=20, count=False): - """docstring for _gen""" - ret = [''] - strings = 0 - for i in d: - if i[0] == 'in': - subs = _in(i[1]) - if count: - strings = (strings or 1) * len(subs) - ret = comb(ret, subs) - elif i[0] == 'literal': - ret = mappend(ret, chr(i[1])) - elif i[0] == 'category': - subs = CATEGORIES.get(i[1], ['']) - if count: - strings = (strings or 1) * len(subs) - ret = comb(ret, subs) - elif i[0] == 'any': - subs = CATEGORIES['category_any'] - if count: - strings = (strings or 1) * len(subs) - ret = comb(ret, subs) - elif i[0] == 'max_repeat': - chars = filter(None, _gen(list(i[1][2]), limit)) - if i[1][1]+1 - i[1][0] >= limit: - ran = range(i[1][0], i[1][0]+limit) - else: - ran = range(i[1][0], i[1][1]+1) - if count: - for i in ran: - strings += pow(len(chars), i) - ret = prods(ret, ran, chars) - elif i[0] == 'branch': - subs = list(chain.from_iterable(_gen(list(x), limit) for x in i[1][1])) - if count: - strings = (strings or 1) * (len(subs) or 1) - ret = comb(ret, subs) - elif i[0] == 'subpattern': - if count: - strings = (strings or 1) * (sum(ggen([0], _gen, i[1][1], limit=limit, count=True)) or 1) - ret = ggen(ret, _gen, i[1][1], limit=limit, count=False) - # ignore ^ and $ - elif i[0] == 'at': - continue - elif i[0] == 'not_literal': - subs = list(CATEGORIES['category_any']) - subs.remove(chr(i[1])) - if count: - strings = (strings or 1) * len(subs) - ret = comb(ret, subs) - elif i[0] == 'assert': - print i[1][1] - continue - else: - #print('[!] cannot handle expression ' + repr(i)) - raise Exception('[!] cannot handle expression ' + repr(i)) - - if count: - return strings - - return ret - -def _randone(d, limit=20): - """docstring for _randone""" - ret = '' - for i in d: - if i[0] == 'in': - ret += choice(_in(i[1])) - elif i[0] == 'literal': - ret += chr(i[1]) - elif i[0] == 'category': - ret += choice(CATEGORIES.get(i[1], [''])) - elif i[0] == 'any': - ret += choice(CATEGORIES['category_any']) - elif i[0] == 'max_repeat': - chars = filter(None, _gen(list(i[1][2]), limit)) - if i[1][1]+1 - i[1][0] >= limit: - min,max = i[1][0], i[1][0]+limit - else: - min,max = i[1][0], i[1][1] - for _ in range(randint(min, max)): - ret += choice(chars) - elif i[0] == 'branch': - ret += choice(list(chain.from_iterable(_gen(list(x), limit) for x in i[1][1]))) - elif i[0] == 'subpattern': - ret += _randone(i[1][1], limit) - elif i[0] == 'at': - continue - elif i[0] == 'not_literal': - c=list(CATEGORIES['category_any']) - c.remove(chr(i[1])) - ret += choice(c) - else: - #print('[!] cannot handle expression "%s"' % str(i)) - raise Exception('[!] cannot handle expression "%s"' % str(i)) - - return ret - - -def parse(s): - """Regular expression parser - :param s: Regular expression - :type s: str - :rtype: list - """ - r = sre_parse.parse(s) - return list(r) - -def generate(s, limit=20): - """Creates a generator that generates all matching strings to a given regular expression - :param s: Regular expression - :type s: str - :param limit: Range limit - :type limit: int - :returns: string generator object - """ - return _gen(parse(s), limit) - -def count(s, limit=20): - """Counts all matching strings to a given regular expression - :param s: Regular expression - :type s: str - :param limit: Range limit - :type limit: int - :rtype: int - :returns: number of matching strings - """ - return _gen(parse(s), limit, count=True) - -def getone(regex_string, limit=20): - """Returns a random matching string to a given regular expression - """ - return _randone(parse(regex_string), limit) - -def argparser(): - import argparse - from sys import stdout - argp = argparse.ArgumentParser(description='exrex - regular expression string generator') - argp.add_argument('-o', '--output' - ,help = 'Output file - default is STDOUT' - ,metavar = 'FILE' - ,default = stdout - ,type = argparse.FileType('w') - ) - argp.add_argument('-l', '--limit' - ,help = 'Max limit for range size - default is 20' - ,default = 20 - ,action = 'store' - ,type = int - ,metavar = 'N' - ) - argp.add_argument('-c', '--count' - ,help = 'Count matching strings' - ,default = False - ,action = 'store_true' - ) - argp.add_argument('-r', '--random' - ,help = 'Returns a random string that matches to the regex' - ,default = False - ,action = 'store_true' - ) - argp.add_argument('-d', '--delimiter' - ,help = 'Delimiter - default is \\n' - ,default = '\n' - ) - argp.add_argument('-v', '--verbose' - ,action = 'store_true' - ,help = 'Verbose mode' - ,default = False - ) - argp.add_argument('regex' - ,metavar = 'REGEX' - ,help = 'REGEX string' - ) - return vars(argp.parse_args()) - -def __main__(): - from sys import exit, stderr - # 'as(d|f)qw(e|r|s)[a-zA-Z]{2,3}' - # 'as(QWE|Z([XC]|Y|U)V){2,3}asdf' - # '.?' - # '.+' - # 'asdf.{1,4}qwer{2,5}' - # 'a(b)?(c)?(d)?' - # 'a[b][c][d]?[e]? - args = argparser() - if args['verbose']: - args['output'].write('%r%s' % (parse(args['regex'], limit=args['limit']), args['delimiter'])) - if args['count']: - args['output'].write('%d%s' % (count(args['regex'], limit=args['limit']), args['delimiter'])) - exit(0) - if args['random']: - args['output'].write('%s%s' % (getone(args['regex'], limit=args['limit']), args['delimiter'])) - exit(0) - try: - g = generate(args['regex'], args['limit']) - except Exception, e: - print >> stderr, '[!] Error: ', e - exit(1) - for s in g: - try: - args['output'].write(s+args['delimiter']) - except: - break - -if __name__ == '__main__': - __main__() - diff --git a/scripts/regex_compile.py b/scripts/regex_compile.py new file mode 100644 index 0000000..0056dc4 --- /dev/null +++ b/scripts/regex_compile.py @@ -0,0 +1,179 @@ +from enum import Enum + +MAX_REGEXP_LEN = 1024 + +class RegexType(Enum): + UNUSED = 0 + DOT = 1 + BEGIN = 2 + END = 3 + QUESTIONMARK = 4 + STAR = 5 + PLUS = 6 + CHAR = 7 + CHAR_CLASS = 8 + INV_CHAR_CLASS = 9 + DIGIT = 10 + NOT_DIGIT = 11 + ALPHA = 12 + NOT_ALPHA = 13 + WHITESPACE = 14 + NOT_WHITESPACE = 15 + +class RegexSegment: + def __init__(self, r_type, data_len=0, data=None): + self.type = r_type + self.data_len = data_len + self.data = data if data else [] + + def to_bytes(self): + return bytes([self.type.value, self.data_len] + self.data) + + # def to_hex(self): + # if self.type in (RegexType.CHAR_CLASS, RegexType.INV_CHAR_CLASS): + # s = f"\\x{'[':02x}" + # e = f"\\x{']':02x}" + # content = ''.join(['\\x{0:02x}'.format(d) for d in self.data]) + # return s + f'\\x{"^":02x}' if self.type == RegexType.INV_CHAR_CLASS else '' + content + e + # else: + # return f"\\x{self.data[0]:02x}" + + def __str__(self): + if self.type in (RegexType.CHAR_CLASS, RegexType.INV_CHAR_CLASS): + content = ''.join(['\\x{0:02x}'.format(d) for d in self.data]) + return f"type: {self.type.name} [{'^' if self.type == RegexType.INV_CHAR_CLASS else ''}{content}]" + elif self.type == RegexType.CHAR: + return f"type: {self.type.name} '\\x{self.data[0]:02x}'" + else: + return f"type: {self.type.name}" + + +def to_buffer(segments): + # Create the flat memory buffer + buffer = bytearray() + for segment in segments: + buffer.extend(segment.to_bytes()) + return buffer + +# def to_hex(segments): +# pattern = "" +# for segment in segments: +# pattern += segment.to_hex() +# return pattern + +def print_buffer(segments): + print(to_buffer(segments)) + + +class MiniRegexCompiler: + MAX_REGEXP_LEN = 70 # Max number of bytes for a regex + + def compile(self, pattern): + segments = [] + i = 0 + + while i < len(pattern): + c = pattern[i] + if c == '.': + segments.append(RegexSegment(RegexType.DOT)) + elif c == '^': + segments.append(RegexSegment(RegexType.BEGIN)) + elif c == '$': + segments.append(RegexSegment(RegexType.END)) + elif c == '*': + segments.append(RegexSegment(RegexType.STAR)) + elif c == '+': + segments.append(RegexSegment(RegexType.PLUS)) + elif c == '?': + segments.append(RegexSegment(RegexType.QUESTIONMARK)) + elif c == '|': + raise Exception("Unsupported") + elif c == '\\': + i += 1 + if i < len(pattern): + escaped_segment = self.handle_escape(pattern[i]) + if escaped_segment: + segments.append(escaped_segment) + else: + return None # Invalid regex + else: + return None # Invalid regex + elif c == '[': + char_limit = MAX_REGEXP_LEN - 4 #min(0xff, MAX_REGEXP_LEN - j - 4) + i += 1 + if i < len(pattern) and pattern[i] == '^': + segment = RegexSegment(RegexType.INV_CHAR_CLASS) + i += 1 + if i >= len(pattern): + return None + else: + segment = RegexSegment(RegexType.CHAR_CLASS) + + while i < len(pattern) and pattern[i] != ']': + if pattern[i] == '\\': + i += 1 + if i < len(pattern): + self.add_escaped_char(segment, pattern[i]) + else: + return None # Invalid regex + elif segment.data_len >= char_limit: + return None + else: + segment.data.append(ord(pattern[i])) + segment.data_len += 1 + i += 1 + if segment.data_len >= char_limit: + return None + + segments.append(segment) + elif c == '\0': + return None + else: + segments.append(RegexSegment(RegexType.CHAR, 1, [ord(c)])) + + i += 1 + + if len(segments) * 3 > self.MAX_REGEXP_LEN: # Rough check, as each segment can have different lengths + return None # Exceeded internal buffer + + return segments + + def handle_escape(self, char): + if char == 'd': + return RegexSegment(RegexType.DIGIT) + elif char == 'D': + return RegexSegment(RegexType.NOT_DIGIT) + elif char == 'w': + return RegexSegment(RegexType.ALPHA) + elif char == 'W': + return RegexSegment(RegexType.NOT_ALPHA) + elif char == 's': + return RegexSegment(RegexType.WHITESPACE) + elif char == 'S': + return RegexSegment(RegexType.NOT_WHITESPACE) + elif char in {'.', '^', '$', '*', '+', '?', '[', ']', '\\'}: # TODO: add '|' + return RegexSegment(RegexType.CHAR, 1, [ord(char)]) + else: + return None # Invalid escape sequence + + def add_escaped_char(self, segment, char): + segment.data.append(ord('\\')) # Add the escape character + segment.data_len += 1 + segment.data.append(ord(char)) + segment.data_len += 1 + +def main(): + # Usage example + pattern = "t*31elJ)_?~*DF_ac]*.+.*.[\\.]." + compiler = MiniRegexCompiler() + segments = compiler.compile(pattern) + compiled_pattern = to_buffer(segments) + if compiled_pattern: + print("Compiled pattern:", compiled_pattern) + for segment in segments: + print(segment) + else: + print("Invalid regex pattern") + +if __name__ == "__main__": + main() diff --git a/scripts/regex_test.py b/scripts/regex_test.py index 4fa98de..c29e8f5 100755 --- a/scripts/regex_test.py +++ b/scripts/regex_test.py @@ -1,77 +1,75 @@ #!/usr/bin/env python """ - This program generates random text that matches a given regex-pattern. - The pattern is given via sys.argv and the generated text is passed to - the binary 'tests/test_rand' to check if the generated text also matches - the regex-pattern in the C implementation. - The exit-code of the testing program, is used to determine test success. +This python2 program generates random text that matches a given regex-pattern. +The pattern is given via sys.argv and the generated text is passed to +the binary 'tests/test_rand' to check if the generated text also matches +the regex-pattern in the C implementation. +The exit-code of the testing program, is used to determine test success. - This script is called by the Makefile when doing 'make test' +This script is called by the Makefile when doing 'make test' """ - -import re import sys -import exrex +import rstr from subprocess import call +from utils import get_executable_name -prog = "./tests/test_rand" +prog = get_executable_name("./tests/test_rand") if len(sys.argv) < 2: - print("") - print("usage: %s pattern [nrepeat]" % sys.argv[0]) - print(" where [nrepeat] is optional") - print("") - sys.exit(-1) + print("") + print("usage: %s pattern [nrepeat]" % sys.argv[0]) + print(" where [nrepeat] is optional") + print("") + sys.exit(-1) own_prog = sys.argv[0] pattern = sys.argv[1] if len(sys.argv) > 2: - ntests = int(sys.argv[2]) + ntests = int(sys.argv[2]) else: - ntests = 10 + ntests = 10 nfails = 0 repeats = ntests try: - repeats = int(sys.argv[2]) + repeats = int(sys.argv[2]) except: - pass + pass r = 50 while r < 0: - try: - g = exrex.generate(pattern) - break - except: - pass + try: + g = rstr.xeger(pattern) + break + except: + pass sys.stdout.write("%-35s" % (" pattern '%s': " % pattern)) while repeats >= 0: - try: - repeats -= 1 - example = exrex.getone(pattern) - #print("%s %s %s" % (prog, pattern, example)) - ret = call([prog, "\"%s\"" % pattern, "\"%s\"" % example]) - if ret != 0: - escaped = repr(example) # escapes special chars for better printing - print(" FAIL : doesn't match %s as expected [%s]." % (escaped, ", ".join([("0x%02x" % ord(e)) for e in example]) )) - nfails += 1 - - except: - #import traceback - #print("EXCEPTION!") - #raw_input(traceback.format_exc()) - ntests -= 1 - repeats += 1 - #nfails += 1 + try: + repeats -= 1 + example = rstr.xeger(pattern) + # print(f'{prog} "{pattern}" "{example}"') + ret = call([prog, f"'{pattern}'", f"'{example}'"], shell=False) + if ret != 0: + escaped = repr(example) # escapes special chars for better printing + print(f" FAIL: {pattern} doesn't match {example}") + nfails += 1 + + except: + # import traceback + # print("EXCEPTION!") + # input(traceback.format_exc()) + ntests -= 1 + repeats += 1 + # nfails += 1 sys.stdout.write("%4d/%d tests succeeded \n" % (ntests - nfails, ntests)) -#print("") - +# print("") diff --git a/scripts/regex_test_compile.py b/scripts/regex_test_compile.py new file mode 100644 index 0000000..f3f2401 --- /dev/null +++ b/scripts/regex_test_compile.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +""" +This python program verifies `scripts/regex_compile.py` generates the same compiled regex as re.c. +The pattern is given via sys.argv and the compiled hex pattern is passed to +the binary 'tests/test_compile' to check if the compiled hex pattern is +the same as the one compiled in the C code. +The exit code of the testing program determines test success. + +This script is called by the Makefile when doing 'make test'. +""" + +import binascii +import subprocess +import sys + +from regex_compile import MiniRegexCompiler, to_buffer +from utils import get_executable_name + +prog = get_executable_name("./tests/test_compile") + +if len(sys.argv) < 2: + print(f"\nusage: {sys.argv[0]} pattern\n\n") + sys.exit(-1) + +pattern = sys.argv[1] + +print(f" pattern '{pattern}': ", end='') + +compiler = MiniRegexCompiler() +segments = compiler.compile(pattern) +hex_pattern = '' +if segments: + compiled_pattern = to_buffer(segments) + hex_pattern = binascii.hexlify(compiled_pattern) + +ret = subprocess.call([prog, pattern, hex_pattern], shell=False) +if ret != 0: + print("Compiled pattern:", compiled_pattern) + for segment in segments: + print(segment) + print(f" FAIL: {pattern}") + print(hex_pattern) + sys.exit(1) +else: + print("SUCCEED") diff --git a/scripts/regex_test_neg.py b/scripts/regex_test_neg.py index c3daad6..b5ec6ad 100755 --- a/scripts/regex_test_neg.py +++ b/scripts/regex_test_neg.py @@ -1,82 +1,91 @@ #!/usr/bin/env python """ - This program generates random text that matches a given regex-pattern. - The pattern is given via sys.argv and the generated text is passed to - the binary 'tests/test_rand' to check if the generated text also matches - the regex-pattern in the C implementation. - The exit-code of the testing program, is used to determine test success. +This program generates random text that matches a given regex-pattern. +The pattern is given via sys.argv and the generated text is passed to +the binary 'tests/test_rand' to check if the generated text also matches +the regex-pattern in the C implementation. +The exit-code of the testing program, is used to determine test success. - This script is called by the Makefile when doing 'make test' +This script is called by the Makefile when doing 'make test' """ - import re import sys import string import random from subprocess import call +from utils import get_executable_name + -prog = "./tests/test_rand_neg" +prog = get_executable_name("./tests/test_rand_neg") if len(sys.argv) < 2: - print("") - print("usage: %s pattern [nrepeat]" % sys.argv[0]) - print(" where [nrepeat] is optional") - print("") - sys.exit(-1) + print("") + print("usage: %s pattern [nrepeat]" % sys.argv[0]) + print(" where [nrepeat] is optional") + print("") + sys.exit(-1) own_prog = sys.argv[0] pattern = sys.argv[1] if len(sys.argv) > 2: - ntests = int(sys.argv[2]) + ntests = int(sys.argv[2]) else: - ntests = 10 + ntests = 10 nfails = 0 repeats = ntests try: - repeats = int(sys.argv[2]) + repeats = int(sys.argv[2]) except: - pass + pass sys.stdout.write("%-35s" % (" pattern '%s': " % pattern)) - - def gen_no_match(pattern, minlen=1, maxlen=50, maxattempts=500): - nattempts = 0 - while True: - nattempts += 1 - ret = "".join([random.choice(string.printable) for i in range(random.Random().randint(minlen, maxlen))]) - if re.findall(pattern, ret) == []: - return ret - if nattempts >= maxattempts: - raise Exception("Could not generate string that did not match the regex pattern '%s' after %d attempts" % (pattern, nattempts)) - + nattempts = 0 + while True: + nattempts += 1 + ret = "".join( + [ + random.choice(string.printable) + for i in range(random.Random().randint(minlen, maxlen)) + ] + ) + if re.findall(pattern, ret) == []: + return ret + if nattempts >= maxattempts: + raise Exception( + "Could not generate string that did not match the regex pattern '%s' after %d attempts" + % (pattern, nattempts) + ) while repeats >= 0: - try: - repeats -= 1 - example = gen_no_match(pattern) - #print("%s %s %s" % (prog, pattern, example)) - ret = call([prog, "\"%s\"" % pattern, "\"%s\"" % example]) - if ret != 0: - escaped = repr(example) # escapes special chars for better printing - print(" FAIL : matches %s unexpectedly [%s]." % (escaped, ", ".join([("0x%02x" % ord(e)) for e in example]) )) - nfails += 1 - - except: - #import traceback - #print("EXCEPTION!") - #raw_input(traceback.format_exc()) - ntests -= 1 - repeats += 1 - #nfails += 1 + try: + repeats -= 1 + example = gen_no_match(pattern) + # print("%s %s %s" % (prog, pattern, example)) + ret = call([prog, f"'{pattern}'", f"'{example}'"], shell=False) + if ret != 0: + escaped = repr(example) # escapes special chars for better printing + print( + " FAIL : matches %s unexpectedly [%s]." + % (escaped, ", ".join([("0x%02x" % ord(e)) for e in example])) + ) + nfails += 1 + + except: + import traceback + print("EXCEPTION!") + input(traceback.format_exc()) + ntests -= 1 + repeats += 1 + nfails += 1 sys.stdout.write("%4d/%d tests succeeded \n" % (ntests - nfails, ntests)) -#print("") +# print("") diff --git a/scripts/regex_test_precompiled.py b/scripts/regex_test_precompiled.py new file mode 100644 index 0000000..fcd7752 --- /dev/null +++ b/scripts/regex_test_precompiled.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +""" +This python2 program generates random text that matches a given regex-pattern. +The pattern is given via sys.argv and the generated text is passed to +the binary 'tests/test_rand' to check if the generated text also matches +the regex-pattern in the C implementation. +The exit-code of the testing program, is used to determine test success. + +This script is called by the Makefile when doing 'make test' +""" + +import binascii +import subprocess +import sys +import rstr +from subprocess import call + +from regex_compile import MiniRegexCompiler +from utils import get_executable_name + +prog = get_executable_name("./tests/test_compile") + +if len(sys.argv) < 2: + print("") + print("usage: %s pattern [nrepeat]" % sys.argv[0]) + print(" where [nrepeat] is optional") + print("") + sys.exit(-1) + +own_prog = sys.argv[0] +pattern = sys.argv[1] +if len(sys.argv) > 2: + ntests = int(sys.argv[2]) +else: + ntests = 10 +nfails = 0 +repeats = ntests + + +try: + repeats = int(sys.argv[2]) +except: + pass + +r = 50 +while r < 0: + try: + g = rstr.xeger(pattern) + break + except: + pass + + +sys.stdout.write("%-35s" % (" pattern '%s': " % pattern)) + + +while repeats >= 0: + try: + repeats -= 1 + example = rstr.xeger(pattern) + # print(f'{prog} "{pattern}" "{example}"') + compiler = MiniRegexCompiler() + compiled_pattern, segments = compiler.compile(pattern) + hex_pattern = binascii.hexlify(compiled_pattern) + + ret = subprocess.call([prog, pattern, hex_pattern], shell=False) + if ret != 0: + print("Compiled pattern:", compiled_pattern) + for segment in segments: + print(segment) + + escaped = repr(example) # escapes special chars for better printing + print(f" FAIL: {pattern} doesn't match {example}") + nfails += 1 + print(hex_pattern) + exit() + + except: + import traceback + print("EXCEPTION!") + input(traceback.format_exc()) + ntests -= 1 + repeats += 1 + nfails += 1 + +sys.stdout.write("%4d/%d tests succeeded \n" % (ntests - nfails, ntests)) +# print("") diff --git a/scripts/utils.py b/scripts/utils.py new file mode 100644 index 0000000..f64cbd3 --- /dev/null +++ b/scripts/utils.py @@ -0,0 +1,11 @@ +import os + + +def get_executable_name(path: str) -> str: + """ + Adds .exe extension to the path if running on Windows and the path does not already end with .exe + """ + if os.name == "nt": # Check if the OS is Windows + if not path.lower().endswith(".exe"): + path += ".exe" + return path diff --git a/tests/test1.c b/tests/test1.c index 5fdfe74..292d0c1 100644 --- a/tests/test1.c +++ b/tests/test1.c @@ -4,13 +4,13 @@ #include #include +//#include #include "re.h" #define OK ((char*) 1) #define NOK ((char*) 0) - char* test_vector[][4] = { { OK, "\\d", "5", (char*) 1 }, @@ -36,6 +36,8 @@ char* test_vector[][4] = { OK, "[abc]", "1c2", (char*) 1 }, { NOK, "[abc]", "1C2", (char*) 0 }, { OK, "[1-5]+", "0123456789", (char*) 5 }, + { OK, "[1-5-]+", "123-", (char*) 4 }, + { OK, "[1-5-]+[-1-2]-[-]", "13132231--353444-511--", (char *) 22 }, { OK, "[.2]", "1C2", (char*) 1 }, { OK, "a*$", "Xaa", (char*) 2 }, { OK, "a*$", "Xaa", (char*) 2 }, @@ -75,20 +77,23 @@ char* test_vector[][4] = { OK, "[Hh]ello [Ww]orld\\s*[!]?", "Hello world! ", (char*) 11 }, { OK, "[Hh]ello [Ww]orld\\s*[!]?", "Hello world !", (char*) 13 }, { OK, "[Hh]ello [Ww]orld\\s*[!]?", "hello World !", (char*) 15 }, - { NOK, "\\d\\d?:\\d\\d?:\\d\\d?", "a:0", (char*) 0 }, /* Failing test case reported in https://github.com/kokke/tiny-regex-c/issues/12 */ -/* + { NOK, "\\d\\d?:\\d\\d?:\\d\\d?", "a:0", (char*) 0 }, { OK, "[^\\w][^-1-4]", ")T", (char*) 2 }, { OK, "[^\\w][^-1-4]", ")^", (char*) 2 }, { OK, "[^\\w][^-1-4]", "*)", (char*) 2 }, { OK, "[^\\w][^-1-4]", "!.", (char*) 2 }, { OK, "[^\\w][^-1-4]", " x", (char*) 2 }, { OK, "[^\\w][^-1-4]", "$b", (char*) 2 }, -*/ { OK, ".?bar", "real_bar", (char*) 4 }, { NOK, ".?bar", "real_foo", (char*) 0 }, { NOK, "X?Y", "Z", (char*) 0 }, { OK, "[a-z]+\nbreak", "blahblah\nbreak", (char*) 14 }, { OK, "[a-z\\s]+\nbreak", "bla bla \nbreak", (char*) 14 }, + { NOK, "a\\", "a\\", (char*) 0 }, + { NOK, "\\", "\\", (char*) 0 }, + { OK, "\\\\", "\\", (char*) 1 }, + // no multibyte support yet + //{ OK, "\\w+", "Çüéâ", (char*) 4 }, }; @@ -101,16 +106,21 @@ int main() int should_fail; int length; int correctlen; - size_t ntests = sizeof(test_vector) / sizeof(*test_vector); - size_t nfailed = 0; - size_t i; + unsigned long nvector_tests = sizeof(test_vector) / sizeof(*test_vector); + unsigned long ntests = nvector_tests + 1; + unsigned long nfailed = 0; + unsigned long i; - for (i = 0; i < ntests; ++i) + for (i = 0; i < nvector_tests; ++i) { pattern = test_vector[i][1]; text = test_vector[i][2]; should_fail = (test_vector[i][0] == NOK); + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpointer-to-int-cast" correctlen = (int)(test_vector[i][3]); +#pragma GCC diagnostic pop int m = re_match(pattern, text, &length); @@ -141,6 +151,21 @@ int main() } } + // regression test for unhandled BEGIN in the middle of an expression + // we need to test text strings with all possible values for the second + // byte because re.c was matching it against an uninitalized value, so + // it could be anything + pattern = "a^"; + for (i = 0; i < 255; i++) { + char text_buf[] = { 'a', i, '\0' }; + int m = re_match(pattern, text_buf, &length); + if (m != -1) { + fprintf(stderr, "[%lu/%lu]: pattern '%s' matched '%s' unexpectedly", ntests, ntests, pattern, text_buf); + nfailed += 1; + break; + } + } + // printf("\n"); printf("%lu/%lu tests succeeded.\n", ntests - nfailed, ntests); printf("\n"); diff --git a/tests/test2.c b/tests/test2.c index 723e262..2302a72 100644 --- a/tests/test2.c +++ b/tests/test2.c @@ -2066,7 +2066,7 @@ int main() size_t bufsize = sizeof(buf) - 1; int i; int dummy = 0; - size_t bufsizes[ntests]; + unsigned long bufsizes[ntests]; char old; for (i = ntests-1; i >= 0; --i) diff --git a/tests/test_compile.c b/tests/test_compile.c index 2a7b4d0..23c4155 100644 --- a/tests/test_compile.c +++ b/tests/test_compile.c @@ -5,17 +5,67 @@ This file tests two bug patterns reported by @DavidKorczynski in https://github. */ #include +#include +#include #include /* for NULL */ #include "re.h" +void hexdump(const unsigned char *data, size_t size) { + for (size_t i = 0; i < size; ++i) { + printf("\\x%02x", data[i]); + } + printf("\n"); +} -int main() -{ - /* Test 1: inverted set without a closing ']' */ - assert(re_compile("\\\x01[^\\\xff][^") == NULL); +int hex_to_int(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } else if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } else if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } else { + return -1; + } +} - /* Test 2: set with an incomplete escape sequence and without a closing ']' */ - assert(re_compile("\\\x01[^\\\xff][\\") == NULL); +// Function to convert a hex string to a byte array +unsigned char *hex_to_bytes(const char *hex, size_t *length) { + size_t len = strlen(hex); + if (len % 2 != 0) { + return NULL; // Invalid hex string + } + *length = len / 2; + unsigned char *bytes = malloc(*length); + for (size_t i = 0; i < *length; i++) { + int high = hex_to_int(hex[2 * i]); + int low = hex_to_int(hex[2 * i + 1]); + if (high == -1 || low == -1) { + free(bytes); + return NULL; // Invalid hex character + } + bytes[i] = (high << 4) | low; + } + return bytes; +} + +int main(int argc, char** argv) +{ + if (argc == 3) + { + size_t pattern_len; + re_t compiled_pattern = NULL; + if(argv[2] != NULL){ + compiled_pattern = (re_t)hex_to_bytes(argv[2], &pattern_len); + } + //hexdump(compiled_pattern, pattern_len); + //hexdump(re_compile(argv[1]), pattern_len); + assert(0 == memcmp(compiled_pattern, re_compile(argv[1]), pattern_len)); + } + else + { + printf("\nUsage: %s \n", argv[0]); + } return 0; } diff --git a/tests/test_end_anchor.c b/tests/test_end_anchor.c new file mode 100644 index 0000000..1809f7c --- /dev/null +++ b/tests/test_end_anchor.c @@ -0,0 +1,20 @@ +#include +#include +#include "re.h" + +int main() { + + const char *text = "table football"; + const char *pattern = "l$"; + int index,len; + + index = re_match(pattern, text, &len); + + if (index==13 && len==1) { + return 0; + } else { + printf("ERROR! index=%d len=%d \n",index,len); + return -1; + } + +}