diff --git a/doc/pcretest.1 b/doc/pcretest.1 index 2d6d0e4..92640da 100644 --- a/doc/pcretest.1 +++ b/doc/pcretest.1 @@ -679,8 +679,8 @@ recognized: after a successful match (number less than 32) .\" JOIN \eCname call pcre[16|32]_copy_named_substring() for substring - "name" after a successful match (name terminated - by next non alphanumeric character) + "name" after a successful match (name termin- + ated by next non alphanumeric character) .\" JOIN \eC+ show the current captured substrings at callout time @@ -701,8 +701,8 @@ recognized: after a successful match (number less than 32) .\" JOIN \eGname call pcre[16|32]_get_named_substring() for substring - "name" after a successful match (name terminated - by next non-alphanumeric character) + "name" after a successful match (name termin- + ated by next non-alphanumeric character) .\" JOIN \eJdd set up a JIT stack of dd kilobytes maximum (any number of digits) diff --git a/pcre_compile.c b/pcre_compile.c index 087bf2a..4d3b313 100644 --- a/pcre_compile.c +++ b/pcre_compile.c @@ -4645,10 +4645,9 @@ for (;; ptr++) goto FAILED; } - /* If in \Q...\E, check for the end; if not, we have a literal. Otherwise an - isolated \E is ignored. */ + /* If in \Q...\E, check for the end; if not, we have a literal */ - if (c != CHAR_NULL) + if (inescq && c != CHAR_NULL) { if (c == CHAR_BACKSLASH && ptr[1] == CHAR_E) { @@ -4656,7 +4655,7 @@ for (;; ptr++) ptr++; continue; } - else if (inescq) + else { if (previous_callout != NULL) { @@ -4671,27 +4670,18 @@ for (;; ptr++) } goto NORMAL_CHAR; } - - /* Check for the start of a \Q...\E sequence. We must do this here rather - than later in case it is immediately followed by \E, which turns it into a - "do nothing" sequence. */ - - if (c == CHAR_BACKSLASH && ptr[1] == CHAR_Q) - { - inescq = TRUE; - ptr++; - continue; - } + /* Control does not reach here. */ } - /* In extended mode, skip white space and comments. */ + /* In extended mode, skip white space and comments. We need a loop in order + to check for more white space and more comments after a comment. */ if ((options & PCRE_EXTENDED) != 0) { - const pcre_uchar *wscptr = ptr; - while (MAX_255(c) && (cd->ctypes[c] & ctype_space) != 0) c = *(++ptr); - if (c == CHAR_NUMBER_SIGN) + for (;;) { + while (MAX_255(c) && (cd->ctypes[c] & ctype_space) != 0) c = *(++ptr); + if (c != CHAR_NUMBER_SIGN) break; ptr++; while (*ptr != CHAR_NULL) { @@ -4705,33 +4695,8 @@ for (;; ptr++) if (utf) FORWARDCHAR(ptr); #endif } + c = *ptr; /* Either NULL or the char after a newline */ } - - /* If we skipped any characters, restart the loop. Otherwise, we didn't see - a comment. */ - - if (ptr > wscptr) - { - ptr--; - continue; - } - } - - /* Skip over (?# comments. We need to do this here because we want to know if - the next thing is a quantifier, and these comments may come between an item - and its quantifier. */ - - if (c == CHAR_LEFT_PARENTHESIS && ptr[1] == CHAR_QUESTION_MARK && - ptr[2] == CHAR_NUMBER_SIGN) - { - ptr += 3; - while (*ptr != CHAR_NULL && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++; - if (*ptr == CHAR_NULL) - { - *errorcodeptr = ERR18; - goto FAILED; - } - continue; } /* See if the next thing is a quantifier. */ @@ -5081,22 +5046,20 @@ for (;; ptr++) ptr = tempptr + 1; continue; - /* For the other POSIX classes (ascii, cntrl, xdigit) we are going - to fall through to the non-UCP case and build a bit map for - characters with code points less than 256. If we are in a negated - POSIX class, characters with code points greater than 255 must - either all match or all not match. In the special case where we - have not yet generated any xclass data, and this is the final item - in the overall class, we need do nothing: later on, the opcode + /* For the other POSIX classes (ascii, xdigit) we are going to fall + through to the non-UCP case and build a bit map for characters with + code points less than 256. If we are in a negated POSIX class + within a non-negated overall class, characters with code points + greater than 255 must all match. In the special case where we have + not yet generated any xclass data, and this is the final item in + the overall class, we need do nothing: later on, the opcode OP_NCLASS will be used to indicate that characters greater than 255 are acceptable. If we have already seen an xclass item or one may follow (we have to assume that it might if this is not the end of - the class), explicitly list all wide codepoints, which will then - either not match or match, depending on whether the class is or is - not negated. */ + the class), explicitly match all wide codepoints. */ default: - if (local_negate && + if (!negate_class && local_negate && (xclass || tempptr[2] != CHAR_RIGHT_SQUARE_BRACKET)) { *class_uchardata++ = XCL_RANGE; @@ -6566,6 +6529,21 @@ for (;; ptr++) case CHAR_LEFT_PARENTHESIS: ptr++; + /* First deal with comments. Putting this code right at the start ensures + that comments have no bad side effects. */ + + if (ptr[0] == CHAR_QUESTION_MARK && ptr[1] == CHAR_NUMBER_SIGN) + { + ptr += 2; + while (*ptr != CHAR_NULL && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++; + if (*ptr == CHAR_NULL) + { + *errorcodeptr = ERR18; + goto FAILED; + } + continue; + } + /* Now deal with various "verbs" that can be introduced by '*'. */ if (ptr[0] == CHAR_ASTERISK && (ptr[1] == ':' @@ -7280,7 +7258,7 @@ for (;; ptr++) issue is fixed "properly" in PCRE2. As PCRE1 is now in maintenance only mode, we finesse the bug by allowing more memory always. */ - *lengthptr += 4 + 4*LINK_SIZE; + *lengthptr += 2 + 2*LINK_SIZE; /* It is even worse than that. The current reference may be to an existing named group with a different number (so apparently not @@ -7607,15 +7585,39 @@ for (;; ptr++) newoptions = (options | set) & (~unset); /* If the options ended with ')' this is not the start of a nested - group with option changes, so the options change at this level. + group with option changes, so the options change at this level. If this + item is right at the start of the pattern, the options can be + abstracted and made external in the pre-compile phase, and ignored in + the compile phase. This can be helpful when matching -- for instance in + caseless checking of required bytes. + + If the code pointer is not (cd->start_code + 1 + LINK_SIZE), we are + definitely *not* at the start of the pattern because something has been + compiled. In the pre-compile phase, however, the code pointer can have + that value after the start, because it gets reset as code is discarded + during the pre-compile. However, this can happen only at top level - if + we are within parentheses, the starting BRA will still be present. At + any parenthesis level, the length value can be used to test if anything + has been compiled at that level. Thus, a test for both these conditions + is necessary to ensure we correctly detect the start of the pattern in + both phases. + If we are not at the pattern start, reset the greedy defaults and the case value for firstchar and reqchar. */ if (*ptr == CHAR_RIGHT_PARENTHESIS) { - greedy_default = ((newoptions & PCRE_UNGREEDY) != 0); - greedy_non_default = greedy_default ^ 1; - req_caseopt = ((newoptions & PCRE_CASELESS) != 0)? REQ_CASELESS:0; + if (code == cd->start_code + 1 + LINK_SIZE && + (lengthptr == NULL || *lengthptr == 2 + 2*LINK_SIZE)) + { + cd->external_options = newoptions; + } + else + { + greedy_default = ((newoptions & PCRE_UNGREEDY) != 0); + greedy_non_default = greedy_default ^ 1; + req_caseopt = ((newoptions & PCRE_CASELESS) != 0)? REQ_CASELESS:0; + } /* Change options at this level, and pass them back for use in subsequent branches. */ @@ -7894,6 +7896,16 @@ for (;; ptr++) c = ec; else { + if (escape == ESC_Q) /* Handle start of quoted string */ + { + if (ptr[1] == CHAR_BACKSLASH && ptr[2] == CHAR_E) + ptr += 2; /* avoid empty string */ + else inescq = TRUE; + continue; + } + + if (escape == ESC_E) continue; /* Perl ignores an orphan \E */ + /* For metasequences that actually match a character, we disable the setting of a first character if it hasn't already been set. */ diff --git a/pcre_get.c b/pcre_get.c index cdd2abc..8094b34 100644 --- a/pcre_get.c +++ b/pcre_get.c @@ -250,7 +250,6 @@ Arguments: code the compiled regex stringname the name of the capturing substring ovector the vector of matched substrings - stringcount number of captured substrings Returns: the number of the first that is set, or the number of the last one if none are set, @@ -259,16 +258,13 @@ Returns: the number of the first that is set, #if defined COMPILE_PCRE8 static int -get_first_set(const pcre *code, const char *stringname, int *ovector, - int stringcount) +get_first_set(const pcre *code, const char *stringname, int *ovector) #elif defined COMPILE_PCRE16 static int -get_first_set(const pcre16 *code, PCRE_SPTR16 stringname, int *ovector, - int stringcount) +get_first_set(const pcre16 *code, PCRE_SPTR16 stringname, int *ovector) #elif defined COMPILE_PCRE32 static int -get_first_set(const pcre32 *code, PCRE_SPTR32 stringname, int *ovector, - int stringcount) +get_first_set(const pcre32 *code, PCRE_SPTR32 stringname, int *ovector) #endif { const REAL_PCRE *re = (const REAL_PCRE *)code; @@ -299,7 +295,7 @@ if (entrysize <= 0) return entrysize; for (entry = (pcre_uchar *)first; entry <= (pcre_uchar *)last; entry += entrysize) { int n = GET2(entry, 0); - if (n < stringcount && ovector[n*2] >= 0) return n; + if (ovector[n*2] >= 0) return n; } return GET2(entry, 0); } @@ -406,7 +402,7 @@ pcre32_copy_named_substring(const pcre32 *code, PCRE_SPTR32 subject, PCRE_UCHAR32 *buffer, int size) #endif { -int n = get_first_set(code, stringname, ovector, stringcount); +int n = get_first_set(code, stringname, ovector); if (n <= 0) return n; #if defined COMPILE_PCRE8 return pcre_copy_substring(subject, ovector, stringcount, n, buffer, size); @@ -461,10 +457,7 @@ pcre_uchar **stringlist; pcre_uchar *p; for (i = 0; i < double_count; i += 2) - { - size += sizeof(pcre_uchar *) + IN_UCHARS(1); - if (ovector[i+1] > ovector[i]) size += IN_UCHARS(ovector[i+1] - ovector[i]); - } + size += sizeof(pcre_uchar *) + IN_UCHARS(ovector[i+1] - ovector[i] + 1); stringlist = (pcre_uchar **)(PUBL(malloc))(size); if (stringlist == NULL) return PCRE_ERROR_NOMEMORY; @@ -480,7 +473,7 @@ p = (pcre_uchar *)(stringlist + stringcount + 1); for (i = 0; i < double_count; i += 2) { - int len = (ovector[i+1] > ovector[i])? (ovector[i+1] - ovector[i]) : 0; + int len = ovector[i+1] - ovector[i]; memcpy(p, subject + ovector[i], IN_UCHARS(len)); *stringlist++ = p; p += len; @@ -626,7 +619,7 @@ pcre32_get_named_substring(const pcre32 *code, PCRE_SPTR32 subject, PCRE_SPTR32 *stringptr) #endif { -int n = get_first_set(code, stringname, ovector, stringcount); +int n = get_first_set(code, stringname, ovector); if (n <= 0) return n; #if defined COMPILE_PCRE8 return pcre_get_substring(subject, ovector, stringcount, n, stringptr); diff --git a/pcregrep.c b/pcregrep.c index cd53c64..64986b0 100644 --- a/pcregrep.c +++ b/pcregrep.c @@ -2437,7 +2437,7 @@ return options; static char * ordin(int n) { -static char buffer[14]; +static char buffer[8]; char *p = buffer; sprintf(p, "%d", n); while (*p != 0) p++; diff --git a/pcreposix.c b/pcreposix.c index dcc13ef..f024423 100644 --- a/pcreposix.c +++ b/pcreposix.c @@ -364,7 +364,6 @@ start location rather than being passed as a PCRE "starting offset". */ if ((eflags & REG_STARTEND) != 0) { - if (pmatch == NULL) return REG_INVARG; so = pmatch[0].rm_so; eo = pmatch[0].rm_eo; } diff --git a/testdata/testinput2 b/testdata/testinput2 index 967a241..e2e520f 100644 --- a/testdata/testinput2 +++ b/testdata/testinput2 @@ -4217,22 +4217,4 @@ backtracking verbs. --/ /a[[:punct:]b]/BZ -/L(?#(|++)(?J:(?)(?))(?)/ - \O\CC - -/(?=a\K)/ - ring bpattingbobnd $ 1,oern cou \rb\L - /-- End of testinput2 --/ diff --git a/testdata/testinput6 b/testdata/testinput6 index a178d3d..aeb62a0 100644 --- a/testdata/testinput6 +++ b/testdata/testinput6 @@ -1553,13 +1553,4 @@ \x{200} \x{37e} -/[^[:^ascii:]\d]/8W - a - ~ - 0 - \a - \x{7f} - \x{389} - \x{20ac} - /-- End of testinput6 --/ diff --git a/testdata/testinput7 b/testdata/testinput7 index 00b9738..e411a4b 100644 --- a/testdata/testinput7 +++ b/testdata/testinput7 @@ -853,8 +853,4 @@ of case for anything other than the ASCII letters. --/ /a[b[:punct:]]/8WBZ -/L(?#(|++a)...(?P=a)bbb(?P>a)d/BM -Memory allocation (code space): 93 +Memory allocation (code space): 77 ------------------------------------------------------------------ 0 24 Bra 2 5 CBra 1 diff --git a/testdata/testoutput11-32 b/testdata/testoutput11-32 index cdbda74..57e5da0 100644 --- a/testdata/testoutput11-32 +++ b/testdata/testoutput11-32 @@ -231,7 +231,7 @@ Memory allocation (code space): 155 ------------------------------------------------------------------ /(?Pa)...(?P=a)bbb(?P>a)d/BM -Memory allocation (code space): 189 +Memory allocation (code space): 157 ------------------------------------------------------------------ 0 24 Bra 2 5 CBra 1 diff --git a/testdata/testoutput11-8 b/testdata/testoutput11-8 index cb37896..748548a 100644 --- a/testdata/testoutput11-8 +++ b/testdata/testoutput11-8 @@ -231,7 +231,7 @@ Memory allocation (code space): 45 ------------------------------------------------------------------ /(?Pa)...(?P=a)bbb(?P>a)d/BM -Memory allocation (code space): 62 +Memory allocation (code space): 50 ------------------------------------------------------------------ 0 30 Bra 3 7 CBra 1 diff --git a/testdata/testoutput2 b/testdata/testoutput2 index 5fb28d5..85c565d 100644 --- a/testdata/testoutput2 +++ b/testdata/testoutput2 @@ -419,7 +419,7 @@ Need char = '>' /(?U)<.*>/I Capturing subpattern count = 0 -No options +Options: ungreedy First char = '<' Need char = '>' abcghinop @@ -443,7 +443,7 @@ Need char = '=' /(?U)={3,}?/I Capturing subpattern count = 0 -No options +Options: ungreedy First char = '=' Need char = '=' abc========def @@ -477,7 +477,7 @@ Failed: lookbehind assertion is not fixed length at offset 12 /(?i)abc/I Capturing subpattern count = 0 -No options +Options: caseless First char = 'a' (caseless) Need char = 'c' (caseless) @@ -489,7 +489,7 @@ No need char /(?i)^1234/I Capturing subpattern count = 0 -Options: anchored +Options: anchored caseless No first char No need char @@ -502,7 +502,7 @@ No need char /(?s).*/I Capturing subpattern count = 0 May match empty string -Options: anchored +Options: anchored dotall No first char No need char @@ -516,7 +516,7 @@ Starting chars: a b c d /(?i)[abcd]/IS Capturing subpattern count = 0 -No options +Options: caseless No first char No need char Subject length lower bound = 1 @@ -524,7 +524,7 @@ Starting chars: A B C D a b c d /(?m)[xy]|(b|c)/IS Capturing subpattern count = 1 -No options +Options: multiline No first char No need char Subject length lower bound = 1 @@ -538,7 +538,7 @@ No need char /(?i)(^a|^b)/Im Capturing subpattern count = 1 -Options: multiline +Options: caseless multiline First char at start or follows newline No need char @@ -1179,7 +1179,7 @@ No need char End ------------------------------------------------------------------ Capturing subpattern count = 1 -Options: anchored +Options: anchored dotall No first char No need char @@ -2735,7 +2735,7 @@ No match End ------------------------------------------------------------------ Capturing subpattern count = 0 -Options: extended +Options: caseless extended First char = 'a' (caseless) Need char = 'c' (caseless) @@ -2748,7 +2748,7 @@ Need char = 'c' (caseless) End ------------------------------------------------------------------ Capturing subpattern count = 0 -Options: extended +Options: caseless extended First char = 'a' (caseless) Need char = 'c' (caseless) @@ -3095,7 +3095,7 @@ Need char = 'b' End ------------------------------------------------------------------ Capturing subpattern count = 0 -No options +Options: ungreedy First char = 'x' Need char = 'b' xaaaab @@ -3497,7 +3497,7 @@ Need char = 'c' /(?i)[ab]/IS Capturing subpattern count = 0 -No options +Options: caseless No first char No need char Subject length lower bound = 1 @@ -6299,7 +6299,7 @@ Capturing subpattern count = 3 Named capturing subpatterns: A 2 A 3 -Options: anchored +Options: anchored dupnames Duplicate name status changes No first char No need char @@ -14574,80 +14574,4 @@ No match End ------------------------------------------------------------------ -/L(?#(|++)(?J:(?)(?))(?)/ - \O\CC -Matched, but too many substrings -copy substring C failed -7 - -/(?=a\K)/ - ring bpattingbobnd $ 1,oern cou \rb\L -Start of matched string is beyond its end - displaying from end to start. - 0: a - 0L - /-- End of testinput2 --/ diff --git a/testdata/testoutput6 b/testdata/testoutput6 index b64dc0d..beb85aa 100644 --- a/testdata/testoutput6 +++ b/testdata/testoutput6 @@ -2557,20 +2557,4 @@ No match \x{37e} 0: \x{37e} -/[^[:^ascii:]\d]/8W - a - 0: a - ~ - 0: ~ - 0 -No match - \a - 0: \x{07} - \x{7f} - 0: \x{7f} - \x{389} -No match - \x{20ac} -No match - /-- End of testinput6 --/ diff --git a/testdata/testoutput7 b/testdata/testoutput7 index fdfff64..cc9ebdd 100644 --- a/testdata/testoutput7 +++ b/testdata/testoutput7 @@ -2348,24 +2348,4 @@ No match End ------------------------------------------------------------------ -/L(?#(|++