Tokenize a logging format string and extract field names from tokens.

        :param format_string: The logging format string.
        :returns: A generator of :class:`FormatStringToken` objects.
542088
def get_pairs(self, format_string):
        
        for token in self.get_tokens(format_string):
            match = self.name_pattern.search(token)
            name = match.group(1) if match else None
            yield FormatStringToken(name=name, text=token)
847990
def get_formats(
        self, token: dict = None, format_code: str = None, prot: str = "https"
    ) -> dict:
        
        # if specific format
        if isinstance(format_code, str):
            specific_format = "/{}".format(format_code)
        else:
            specific_format = ""

        # search request
        req_url = "{}://v1.{}.isogeo.com/formats{}".format(
            prot, self.api_url, specific_format
        )

        req = self.get(
            req_url, headers=self.header, proxies=self.proxies, verify=self.ssl
        )

        # checking response
        checker.check_api_response(req)

        # end of method
        return req.json()
953743
def GetFormatStringAttributeNames(self):
    
    if self._format_string_attribute_names is None:
      self._format_string_attribute_names = (
          self._FORMAT_STRING_ATTRIBUTE_NAME_RE.findall(
              self.FORMAT_STRING))

    return set(self._format_string_attribute_names)
378761
function getFormatStringFrom (formattedString) {
      var chunks = formattedString.match(R_FORMAT_CHUNKS);

      if (!chunks) {
        // chunks will be null if there were no tokens to parse in
        // formattedString (for example, if formattedString is '2').  Coerce
        // chunks to be useful here.
        chunks = ['', ''];

        // If there is only one chunk, assume that the string is a number
        // followed by a token...
        // NOTE: This may be an unwise assumption.
      } else if (chunks.length === 1 ||
          // ...or if the string starts with a number component (".", "-", or a
          // digit)...
          formattedString[0].match(R_NUMBER_COMPONENT)) {
        // ...prepend an empty string here to make sure that the formatted number
        // is properly replaced by VALUE_PLACEHOLDER
        chunks.unshift('');
      }

      return chunks.join(VALUE_PLACEHOLDER);
    }
863930
def _FormatAttrToken(self, token_data):
    
    return {
        'mode': token_data.file_mode,
        'uid': token_data.user_identifier,
        'gid': token_data.group_identifier,
        'system_id': token_data.file_system_identifier,
        'node_id': token_data.file_identifier,
        'device': token_data.device}
372808
def GetFormatStringAttributeNames(self):
    
    if self._format_string_attribute_names is None:
      self._format_string_attribute_names = []
      for format_string_piece in self.FORMAT_STRING_PIECES:
        attribute_names = self._FORMAT_STRING_ATTRIBUTE_NAME_RE.findall(
            format_string_piece)

        if attribute_names:
          self._format_string_attribute_names.extend(attribute_names)

    return set(self._format_string_attribute_names)
522271
def _get_formatter(self, json):
        '''
        
        '''
        if json:
            return jsonlogger.JsonFormatter()
        else:
            return logging.Formatter(self.format_string)
716690
def formatters(*chained_formatters):
    
    def formatters_chain(input_string):  # pylint:disable=missing-docstring
        for chained_formatter in chained_formatters:
            input_string = chained_formatter(input_string)
        return input_string

    return formatters_chain
376928
def tokenize_words(string):
    
    string = six.text_type(string)
    return re.findall(WORD_TOKENIZATION_RULES, string)
979577
def cleanup_formatted_string(cls, formatted_string):
        
        # Remove whitespace
        result = formatted_string.replace(' ', '')
        # Remove any static token parentheses
        result = re.sub(settings.REGEX_PARENTHESIS, '', result)
        # Remove any multiple separator characters
        multi_character_matches = re.finditer('[%s]{2,}' % settings.SEPARATORS, result)
        for multi_character_match in sorted(multi_character_matches, key=lambda x: len(x.group()), reverse=True):
            match = multi_character_match.group()
            most_common_separator = Counter(list(multi_character_match.group())).most_common(1)[0][0]
            result = result.replace(match, most_common_separator)
        # Remove trailing or preceding non letter characters
        result = re.sub(settings.REGEX_ADJACENT_UNDERSCORE, '', result)
        #  not sure what this one was...but certainly not it.
        result = re.sub(settings.REGEX_SINGLE_PARENTHESIS, '', result)
        return result
1354571
def tokenize(string):
    """"""
    for match in TOKENS_REGEX.finditer(string):
        yield Token(match.lastgroup, match.group().strip(), match.span())
444354
def process_format(self, kind, context):
		""""""
		
		result = None
		
		while True:
			chunk = yield result
			
			if chunk is None:
				return
			
			# We need to split the expression defining the format string from the values to pass when formatting.
			# We want to allow any Python expression, so we'll need to piggyback on Python's own parser in order
			# to exploit the currently available syntax.  Apologies, this is probably the scariest thing in here.
			split = -1
			line = chunk.line
			
			try:
				ast.parse(line)
			except SyntaxError as e:  # We expect this, and catch it.  It'll have exploded after the first expr.
				split = line.rfind(' ', 0, e.offset)
			
			result = chunk.clone(line='_bless(' + line[:split].rstrip() + ').format(' + line[split:].lstrip() + ')')
626742
def _FormatInAddrExToken(self, token_data):
    
    protocol = bsmtoken.BSM_PROTOCOLS.get(token_data.net_type, 'UNKNOWN')
    if token_data.net_type == 4:
      ip_address = self._FormatPackedIPv6Address(token_data.ip_address[:4])
    elif token_data.net_type == 16:
      ip_address = self._FormatPackedIPv6Address(token_data.ip_address)
    return {
        'protocols': protocol,
        'net_type': token_data.net_type,
        'address': ip_address}
698787
def parse_string(self, string=''):
        '''
        
        '''
        self.strings = string.splitlines(1)
        self.strings = [self.__parse(line) for line in self.strings]
        return ''.join(self.strings)
112601
def parse_fields(cls, formatter):
        ""
        if formatter is None: return []
        try:
            parse = list(string.Formatter().parse(formatter))
            return  set(f for f in list(zip(*parse))[1] if f is not None)
        except:
            raise SyntaxError("Could not parse formatter %r" % formatter)
617132
def _parse_string(self):
        """"""
        if self._next_token() in ['{', '(']:
            field = self._parse_field()
            if field:
                self.definitions[field[0]] = field[1]
1371528
def _format_strings(self):
        

        values = self.values
        return [val.strftime('%Y-%m-%d %H:%M:%S') if val is not None else self.nat_rep for val in values]