Normalizes the string using tokenizer.encode.

  Args:
    raw_str: the input string

  Returns:
   A string which is ready to be tokenized using split()
557459
def _normalize_string(raw_str):
  
  return " ".join(
      token.strip()
      for token in tokenizer.encode(text_encoder.native_to_unicode(raw_str)))
230793
def parse(self, string, lexer=None):
        
        if not lexer:
            lexer = Lexer().lexer

        return self.parser.parse(string, lexer=lexer)
968465
def standardize_strings(arg, strtype=settings.MODERNRPC_PY2_STR_TYPE, encoding=settings.MODERNRPC_PY2_STR_ENCODING):
    
    assert six.PY2, "This function should be used with Python 2 only"

    if not strtype:
        return arg

    if strtype == six.binary_type or strtype == 'str':
        # We want to convert from unicode to str
        return _generic_convert_string(arg, six.text_type, six.binary_type, encoding)

    elif strtype == six.text_type or strtype == 'unicode':
        # We want to convert from str to unicode
        return _generic_convert_string(arg, six.binary_type, six.text_type, encoding)

    raise TypeError('standardize_strings() called with an invalid strtype: "{}". Allowed values: str or unicode'
                    .format(repr(strtype)))
1365362
def convert_raw_tuple(value_tuple, format_string):
     
    values = []
    for v, c in zip(value_tuple, format_string):
        if v is None:
            # append None
            values.append(v)
        elif c == u"s":
            # string
            values.append(v)
        elif c == u"S":
            # string, split using space as delimiter
            values.append([s for s in v.split(u" ") if len(s) > 0])
        elif c == u"i":
            # int
            values.append(int(v))
        elif c == u"U":
            # Unicode
            values.append(convert_unicode_field(v))
        elif c == u"A":
            # ASCII
            values.append(convert_ascii_field(v))
        #elif c == u"x":
        #    # ignore
        #    pass
    return tuple(values)
1230723
def _normalize_coerce_to_field_dict(self, v):
        

        def tokenize(s):
            """ Tokenize a string by splitting it by + and -

            >>> tokenize('this + that')
            ['this', '+', 'that']

            >>> tokenize('this+that')
            ['this', '+', 'that']

            >>> tokenize('this+that-other')
            ['this', '+', 'that', '-', 'other]
            """

            # Crude tokenization
            s = s.replace('+', ' + ').replace('-', ' - ') \
                .replace('/', ' / ').replace('*', ' * ')
            words = [w for w in s.split(' ') if w]
            return words

        if isinstance(v, _str_type):
            field_parts = tokenize(v)
            field = field_parts[0]
            d = {'value': field}
            if len(field_parts) > 1:
                # if we need to add and subtract from the field join the field
                # parts into pairs, for instance if field parts is
                #     [MyTable.first, '-', MyTable.second, '+', MyTable.third]
                # we will get two pairs here
                #     [('-', MyTable.second), ('+', MyTable.third)]
                d['operators'] = []
                for operator, other_field in zip(
                    field_parts[1::2], field_parts[2::2]
                ):
                    d['operators'].append({
                        'operator': operator,
                        'field': {
                            'value': other_field
                        }
                    })
            return d
        else:
            return v
486475
def convert(self, string, preprocess = None):
        
        string = unicode(preprocess(string) if preprocess else string, encoding="utf-8")
        if self.regex:
            return self.regex.sub(lambda x: self.substitutes[x.group()], string).encode('utf-8')
        else:
            return string
567046
def _raw_name_string(bufr, strings_offset, str_offset, length):
        
        offset = strings_offset + str_offset
        tmpl = '%ds' % length
        return unpack_from(tmpl, bufr, offset)[0]
399808
def process_tokens(self, tokens):
        
        for tok_type, token, (start_row, start_col), _, _ in tokens:
            if tok_type == tokenize.STRING:
                # 'token' is the whole un-parsed token; we can look at the start
                # of it to see whether it's a raw or unicode string etc.
                self._process_string_token(token, start_row, start_col)
990188
private String normalise(String str) {
		str = str.replaceAll("\\s+", " ");
		str = tc.toSimp(str);
		str = ChineseTrans.toHalfWidth(str);
		return str;
	}
875751
def parse(raw_status, strip_units=False):
    
    lines = split(raw_status)
    if strip_units:
        lines = strip_units_from_lines(lines)
    # Split each line on the SEP character, strip extraneous whitespace and
    # create an OrderedDict out of the keys/values.
    return OrderedDict([[x.strip() for x in x.split(SEP, 1)] for x in lines])
86031
function Tokenizer(str) {

    this.str = (str || '').toString();
    this.operatorCurrent = '';
    this.operatorExpecting = '';
    this.node = null;
    this.escaped = false;

    this.list = [];

}
698787
def parse_string(self, string=''):
        '''
        
        '''
        self.strings = string.splitlines(1)
        self.strings = [self.__parse(line) for line in self.strings]
        return ''.join(self.strings)
158457
def normalize_text(self, value):
        
        if self.ignore_ws:
            value = self.normalize_whitespace(value)
        if self.ignore_case:
            value = self.normalize_case(value)
        if self.unicode_normal:
            value = self.normalize_unf(value)
        return value
385672
def normalize(string: Text) -> Text:
    

    string = string.lower()
    string = unidecode(string)
    string = RE_WHITESPACES.sub(' ', string).strip()

    return string
693912
def tokenize(self, string):
        '''
        
        '''

        new_lexer = ply.lex.lex(module=self, debug=self.debug, errorlog=logger)
        new_lexer.latest_newline = 0
        new_lexer.string_value = None
        new_lexer.input(string)

        while True:
            t = new_lexer.token()
            if t is None: break
            t.col = t.lexpos - new_lexer.latest_newline
            yield t

        if new_lexer.string_value is not None:
            raise JsonPathLexerError('Unexpected EOF in string literal or identifier')
1126772
def parse_string(data, unquote=default_unquote):
    
    if data is None:
        return None

    # We'll soon need to unquote to recover our UTF-8 data.
    # In Python 2, unquote crashes on chars beyond ASCII. So encode functions
    # had better not include anything beyond ASCII in data.
    # In Python 3, unquote crashes on bytes objects, requiring conversion to
    # str objects (unicode) using decode().
    # But in Python 2, the same decode causes unquote to butcher the data.
    # So in that case, just leave the bytes.
    if isinstance(data, bytes):
        if sys.version_info > (3, 0, 0):  # pragma: no cover
            data = data.decode('ascii')
    # Recover URL encoded data
    unquoted = unquote(data)
    # Without this step, Python 2 may have good URL decoded *bytes*,
    # which will therefore not normalize as unicode and not compare to
    # the original.
    if isinstance(unquoted, bytes):
        unquoted = unquoted.decode('utf-8')
    return unquoted
218079
def from_str(string, max_number=9, separator="."):
        
        tokens = string.split(separator)
        tokens = list(reversed(tokens))  # reverse order of importance
        most_important = tokens[-1]  # cannot be parsed like the others
        levels = [
            Level(max_number, int(token)) for token in tokens[:-1]
        ]
        levels.append(
            Level(float("inf"), int(most_important))
        )

        return Subsystem(levels, separator)