Replace size tokens (%w, %h, %r) in pattern
321973
function replaceSizeTokens(pattern, width, height, next) {
        pattern = pattern
          .replace(/%r/g, '%wx%h')
          .replace(/%w/g, width)
          .replace(/%h/g, height);

        next(null, pattern);
      }
192442
def _compile(pattern, flags):
    """"""

    return re.compile(WcParse(pattern, flags & FLAG_MASK).parse())
1151567
def translate(patterns, *, flags=0):
    """"""

    flags = _flag_transform(flags)
    return _wcparse.translate(_wcparse.split(patterns, flags), flags)
860281
private static void do_replace(GlobData rdata, Context cx,
                                   RegExpImpl regExpImpl)
    {
        StringBuilder charBuf = rdata.charBuf;
        int cp = 0;
        String da = rdata.repstr;
        int dp = rdata.dollar;
        if (dp != -1) {
            int[] skip = new int[1];
            do {
                int len = dp - cp;
                charBuf.append(da.substring(cp, dp));
                cp = dp;
                SubString sub = interpretDollar(cx, regExpImpl, da,
                                                dp, skip);
                if (sub != null) {
                    len = sub.length;
                    if (len > 0) {
                        charBuf.append(sub.str, sub.index, sub.index + len);
                    }
                    cp += skip[0];
                    dp += skip[0];
                } else {
                    ++dp;
                }
                dp = da.indexOf('$', dp);
            } while (dp >= 0);
        }
        int daL = da.length();
        if (daL > cp) {
            charBuf.append(da.substring(cp, daL));
        }
    }
158866
function camelize(str) {
  return str.replace(/(?:^\w|[A-Z]|\b\w)/g, (letter, index) => {
    return index === 0 ? letter.toLowerCase() : letter.toUpperCase()
  }).replace(/[\s\-_]+/g, '')
}
486132
def _make_reserved_tokens_re(reserved_tokens):
  """"""
  if not reserved_tokens:
    return None
  escaped_tokens = [_re_escape(rt) for rt in reserved_tokens]
  pattern = "(%s)" % "|".join(escaped_tokens)
  reserved_tokens_re = _re_compile(pattern)
  return reserved_tokens_re
167694
def build_regex(self, tokens):
        """"""
        patterns = []
        for token in tokens:
            patterns.append(token.pattern_start)
            if token.pattern_end:
                patterns.append(token.pattern_end)
        return re.compile('|'.join(patterns), re.DOTALL)
783409
def reformat_pattern(pattern, compile=False):
    '''
    
    '''
    # User pattern: (w:<name>)  --> Changes to (?P<name>\w)
    rex_pattern = re.sub(r'\(w:<([\w\d_]+)>\)', '(?P<\\1>\w+)', pattern)

    # User pattern: (d:<name>)  --> change to (?P<name>\d)
    rex_pattern = re.sub(r'\(d:<([\w\d_]+)>\)', '(?P<\\1>\d+)', rex_pattern)

    # User pattern: (W:<name>)  --> change to (?P<name>\w)
    rex_pattern = re.sub(r'\(W:<([\w\d_]+)>\)', '(?P<\\1>\W+)', rex_pattern)

    # User pattern: (any:<anyvalue> --> change to (?P<anyvalue>.*)
    rex_pattern = re.sub(r'\(any:<([\w\d_]+)>\)', '(?P<\\1>.*)', rex_pattern)

    # User pattern: (ip:<ipaddr>) --> Change to (?P<ipaddr>\d+\.\d+\.\d+\.\d+)
    rex_pattern = re.sub(r'\(ip:<([\w\d_]+)>\)',
                         '(?P<\\1>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})',
                         rex_pattern)

    # User pattern: (mac:<macaddr>) --> Change to (?P<mac>\w\w:\w\w:\w\w:..)
    rex_pattern = re.sub(r'\(mac:<([\w\d_]+)>\)',
                         '(?P<\\1>\w\w:\w\w:\w\w:\w\w:\w\w:\w\w)', rex_pattern)

    # User pattern: (decimal:<name>) --> Change to (?P<name>\d*\.\d+|\d+)
    rex_pattern = re.sub(r'\(decimal:<([\w\d_]+)>\)',
                         '(?P<\\1>\d*\.\d+|\d+)', rex_pattern)

    # User pattern: (measurement:<name> --> change to \
    # (?P<name>\d*\.\d+|\d+)(\w+|\w+/\w+)
    rex_pattern = re.sub(r'\(measurement:<([\w\d_]+)>\)',
                         '(?P<\\1>\d*\.\d+|\d+)(?P<\\1_unit>\w+|\w+/\w+)',
                         rex_pattern)


    ######################################
    # Timestamp patterns.
    # User pattern: (ts[n]:<timestamp>) -->
    # Converted to: The below options.
    ######################################

    # User pattern: (ts1:<timestamp>)
    # Keystone, nova, libvirt, cinder
    # Example: 2015-11-13 06:38:04.571
    rex_pattern = re.sub(
        r'\(ts1:<([\w\d_]+)>\)',
        '(?P<\\1>(\d{4})-(\d{1,2})-(\d{1,2}) (\d+):(\d+):(\d+)\.(\d+))',
        rex_pattern)

    # User pattern: (ts2:<timestamp>)
    # contrail
    # Example: 2015-11-13 Fri 13:14:51:907.395 PST
    rex_pattern = re.sub(
        r'\(ts2:<([\w\d_]+)>\)',
        '(?P<\\1>(\d{4})-(\d{1,2})-(\d{1,2}) (\w{3}) (\d+):(\d+):(\d+):(\d+)\.(\d+))',
        rex_pattern)

    # User pattern: (ts3:<timestamp>)
    # apache2
    # Example: Thu Aug 20 14:18:34 2015
    rex_pattern = re.sub(
        r'\(ts3:<([\w\d_]+)>\)',
        '(?P<\\1>(\w{3}) (\w{3}) (\d{1,2}) (\d+):(\d+):(\d+) (\d{4}))',
        rex_pattern)

    # User pattern: (ts4:<timestamp>)
    # Example: 02/Nov/2015:09:03:19 -0800
    rex_pattern = re.sub(
        r'\(ts4:<([\w\d_]+)>\)',
        '(?P<\\1>(\d+)\/(\w{3})\/(\d{4}):(\d+):(\d+):(\d+) -(\d+))',
        rex_pattern)

    # User pattern: (ts5:<timestamp>)
    # ceph logs.
    # Example: 2015-11-13 06:25:29.436844
    rex_pattern = re.sub(
        r'\(ts5:<([\w\d_]+)>\)',
        '(?P<\\1>(\d{4})-(\d{1,2})-(\d{1,2}) (\d+):(\d+):(\d+)\.(\d+))',
        rex_pattern)

    # User pattern: (ts6:<timestamp>)
    # cassandra
    # Example:2015-10-23 12:38:15
    rex_pattern = re.sub(
        r'\(ts6:<([\w\d_]+)>\)',
        '(?P<\\1>(\d{4})-(\d{1,2})-(\d{1,2}) (\d+):(\d+):(\d+))',
        rex_pattern)

    # User pattern: (ts7:<timestamp>)
    # haproxy
    # Example: 13/Nov/2015:06:25:05.465
    rex_pattern = re.sub(
        r'\(ts7:<([\w\d_]+)>\)',
        '(?P<\\1>(\d+)\/(\w{3})\/(\d{4}):(\d+):(\d+):(\d+)\.(\d+))',
        rex_pattern)

    # User pattern: (ts8:<timestamp>)
    # mysql
    # Example: 12:03:28
    rex_pattern = re.sub(
        r'\(ts8:<([\w\d_]+)>\)',
        '(?P<\\1>(\d+):(\d+):(\d+))',
        rex_pattern)

    # User pattern: (ts9:<timestamp>)
    # reddis
    # Example: 08 Nov 06:26:05.084
    rex_pattern = re.sub(
        r'\(ts9:<([\w\d_]+)>\)',
        '(?P<\\1>(\d+) (\w{3}) (\d+):(\d+):(\d+)\.(\d+))',
        rex_pattern)

    # user pattern: (ts10:<timestamp>)
    # supervisord, zookeeper
    # Example: 2015-06-30 10:59:18,133
    rex_pattern = re.sub(
        r'\(ts10:<([\w\d_]+)>\)',
        '(?P<\\1>(\d{4})-(\d{1,2})-(\d{1,2}) (\d+):(\d+):(\d+),(\d+))',
        rex_pattern)

    # User pattern: (ts11:<timestamp>)
    # dmesg
    # Example: 11148214.574583
    rex_pattern = re.sub(
        r'\(ts11:<([\w\d_]+)>\)',
        '(?P<\\1>(\d+)\.(\d+))',
        rex_pattern)

    # Finally if no prefix is specified take default action.
    rex_pattern = re.sub(r'\(<([\w\d_]+)>\)', '(?P<\\1>.*)', rex_pattern)

    if compile:
        return re.compile(rex_pattern)

    return rex_pattern
1226595
private static String replaceNewlines(String in, String replaceWith) {
        return in.replaceAll("\r", replaceWith).replaceAll("\n", replaceWith);
    }
93952
function createSizeSuffix(width, height) {
  let result = String(width);
  if (height !== undefined) result += `x${String(height)}`;
  return result;
}
890427
def regex_span_tokenize(s, regex):
    """"""
    left = 0
    for m in re.finditer(regex, s, re.U):
        right, next = m.span()
        if right != 0:
            yield left, right
        left = next
    yield left, len(s)
475696
def _tokenize(bytes, mimetype, extra_tokens=True):
    

    # Start with only the ASCII bytes. Limit it to 8+ character strings.
    try:
        ascii_bytes = b' '.join(re.compile(b'[\x00\x09\x0A\x0D\x20-\x7E]{8,}').findall(bytes))
        ascii_bytes = ascii_bytes.replace(b'\x00', b'')
    except:
        return []

    tokens = []

    # Find anything sandwiched between ( )
    results = re.compile(b'\(([^\)]+)\)').findall(ascii_bytes)
    #results = re.compile(b'\(([^\(\)]+)\)').findall(ascii_bytes)
    tokens += results

    # Find anything sandwiched between < >
    results = re.compile(b'\<([^\>]+)\>').findall(ascii_bytes)
    #results = re.compile(b'\<([^\<\>]+)\>').findall(ascii_bytes)
    tokens += results

    # Find anything sandwiched between [ ]
    results = re.compile(b'\[([^\]]+)\]').findall(ascii_bytes)
    #results = re.compile(b'\[([^\[\]]+)\]').findall(ascii_bytes)
    tokens += results

    # Find anything sandwiched between { }
    results = re.compile(b'\{([^\}]+)\}').findall(ascii_bytes)
    #results = re.compile(b'\{([^\{\}]+)\}').findall(ascii_bytes)
    tokens += results

    if not mimetype == 'data':

        # Find anything sandwiched between ' '
        results = re.compile(b'\'\s*(.*?)\s*\'').findall(ascii_bytes)
        tokens += results

        # Find anything sandwiched between " "
        results = re.compile(b'\"\s*(.*?)\s*\"').findall(ascii_bytes)
        tokens += results

    # Find anything sandwiched between spaces
    results = re.compile(b'\ ([^\ ]+)\ ').findall(ascii_bytes)
    tokens += results

    # Break the bytes apart if we want to generate even more tokens.
    if extra_tokens:

        # Add each newline as a token.
        lines = ascii_bytes.decode('ascii', errors='ignore').splitlines()
        tokens += [l.encode('ascii') for l in lines]

        # Now replace these characters in the ASCII bytes with spaces to create more tokens.
        replace_these = [b'(', b')', b'<', b'>', b'[', b']', b'{', b'}', b"'", b'"', b'\n', b'\r', b'\t', b'\\', b'`', b';']
        for char in replace_these:
            ascii_bytes = ascii_bytes.replace(char, b' ')

        # Split the bytes on spaces and use the results as tokens.
        tokens += ascii_bytes.split(b' ')

    # Decode the tokens as ASCII.
    tokens = [token.decode('ascii', errors='ignore') for token in tokens]

    # Since we want this to find full URLs and not just potential domain names, we want
    # to ensure that the tokens have at least "http" or "/" in them.
    tokens = list(set([t for t in tokens if ('http' in t or 'ftp' in t or ('/' in t and '.' in t)) and len(t) >= 8]))

    # Remove any tokens that look like they're probably bad based on the regex statements above.
    tokens = [t for t in tokens if not all(x in t for x in ['(', ')'])]
    tokens = [t for t in tokens if not all(x in t for x in ['<', '>'])]
    tokens = [t for t in tokens if not all(x in t for x in ['[', ']'])]
    tokens = [t for t in tokens if not all(x in t for x in ['{', '}'])]
    tokens = [t for t in tokens if not any(x in t for x in ['\t', '\n', '\r'])]
    tokens = [t for t in tokens if t[:1].isalpha() or t[:1].isdigit()]
    tokens = [t for t in tokens if not t.count('\'') > 1]
    tokens = [t for t in tokens if not t.count('\"') > 1]

    # Return a list of the tokens in ASCII format.
    return tokens
477638
def replace_wrep(t:str) -> str:
    ""
    def _replace_wrep(m:Collection[str]) -> str:
        c,cc = m.groups()
        return f' {TK_WREP} {len(cc.split())+1} {c} '
    re_wrep = re.compile(r'(\b\w+\W+)(\1{3,})')
    return re_wrep.sub(_replace_wrep, t)
1131597
def _Tokenize(template_str, meta_left, meta_right, whitespace):
    """"""
    
    trimlen = len(meta_left)
    token_re = MakeTokenRegex(meta_left, meta_right)
    do_strip = (whitespace == 'strip-line')  # Do this outside loop
    do_strip_part = False
    
    for line in template_str.splitlines(True):  # retain newlines
        if do_strip or do_strip_part:
            line = line.strip()
        
        tokens = token_re.split(line)
        
        # Check for a special case first.  If a comment or "block" directive is on a
        # line by itself (with only space surrounding it), then the space is
        # omitted.  For simplicity, we don't handle the case where we have 2
        # directives, say '{.end} # {#comment}' on a line.
        if len(tokens) == 3:
            # ''.isspace() == False, so work around that
            if (tokens[0].isspace() or not tokens[0]) and \
                    (tokens[2].isspace() or not tokens[2]):
                token = tokens[1][trimlen: -trimlen]
                
                # Check the ones that begin with ## before #
                if token == COMMENT_BEGIN:
                    yield COMMENT_BEGIN_TOKEN, None
                    continue
                if token == COMMENT_END:
                    yield COMMENT_END_TOKEN, None
                    continue
                if token == OPTION_STRIP_LINE:
                    do_strip_part = True
                    continue
                if token == OPTION_END:
                    do_strip_part = False
                    continue
                
                if token.startswith('#'):
                    continue  # The whole line is omitted
                
                token_type, token = _MatchDirective(token)
                if token_type is not None:
                    yield token_type, token  # Only yield the token, not space
                    continue
        
        # The line isn't special; process it normally.
        for i, token in enumerate(tokens):
            if i % 2 == 0:
                yield LITERAL_TOKEN, token
            
            else:  # It's a "directive" in metachracters
                assert token.startswith(meta_left), repr(token)
                assert token.endswith(meta_right), repr(token)
                token = token[trimlen: -trimlen]
                
                # Check the ones that begin with ## before #
                if token == COMMENT_BEGIN:
                    yield COMMENT_BEGIN_TOKEN, None
                    continue
                if token == COMMENT_END:
                    yield COMMENT_END_TOKEN, None
                    continue
                if token == OPTION_STRIP_LINE:
                    do_strip_part = True
                    continue
                if token == OPTION_END:
                    do_strip_part = False
                    continue
                
                # A single-line comment
                if token.startswith('#'):
                    continue
                
                if token.startswith('.'):
                    literal = {
                        '.meta-left': meta_left,
                        '.meta-right': meta_right,
                        '.space': ' ',
                        '.tab': '\t',
                        '.newline': '\n',
                    }.get(token)
                    
                    if literal is not None:
                        yield META_LITERAL_TOKEN, literal
                        continue
                    
                    token_type, token = _MatchDirective(token)
                    if token_type is not None:
                        yield token_type, token
                
                else:  # Now we know the directive is a substitution.
                    yield SUBST_TOKEN, token
519058
private String replaceVariablesWithPatterns(String pattern) {
        int group = 0;
        Matcher findVariablesMatcher = variablePattern.matcher(pattern);
        while(findVariablesMatcher.find()) {
            String variable = findVariablesMatcher.group(0);
            pattern = pattern.replaceFirst(variable, "(.+)");
            variableToGroupNumber.put(variable, ++group);
        }
        return pattern;
    }
584868
def replace_all_tokens(self, token):
        
        b = self.data_buffer

        for y, row in b.items():
            for x, char in row.items():
                b[y][x] = _CHAR_CACHE[char.char, token]
75431
public void apply(HyphenationPattern pattern) {

    String wordPart = pattern.getWordPart();
    int wordPartLength = wordPart.length();
    int wordPartHash = pattern.getWordPartHash();
    int[] subHashes = getHashes(wordPartLength);
    if (subHashes.length == 0) {
      return;
    }
    int start = 0;
    int end = subHashes.length - 1;
    if (wordPart.charAt(0) == HyphenationPattern.TERMINATOR) {
      end = 0;
    } else if (wordPart.charAt(wordPartLength - 1) == HyphenationPattern.TERMINATOR) {
      start = end;
    }
    for (int i = start; i <= end; i++) {
      if (subHashes[i] == wordPartHash) {
        if (isSubstring(this.normalizedWord, wordPart, i)) {
          apply(pattern, i);
        }
      }
    }
  }