Split format into multiple tokens
489773
function createTokens(format) {
		var tokens = [],
			pos = 0,
			match;
			
		while ((match = tokenRE.exec(format))) {
			if (match.index > pos) {
				// doesn't match any token, static string
				tokens.push(angular.extend({
					value: format.substring(pos, match.index)
				}, definedTokens.string));
				pos = match.index;
			} 

			if (match.index == pos) {
				if (match[1]) {
					// sss
					tokens.push(angular.extend({
						value: match[1]
					}, definedTokens.string));
					tokens.push(definedTokens.sss);
				} else if (match[2]) {
					// escaped string
					tokens.push(angular.extend({
						value: match[2].replace("''", "'")
					}, definedTokens.string));
				} else if (definedTokens[match[0]].name == "timezone") {
					// static timezone
					var tz = SYS_TIMEZONE;
					if (definedTokens[match[0]].colon) {
						tz = insertColon(tz);
					}
					tokens.push(angular.extend({
						value: tz
					}, definedTokens[match[0]]));
				} else {
					// other tokens
					tokens.push(definedTokens[match[0]]);
				}
				pos = tokenRE.lastIndex;
			}
		}
		
		if (pos < format.length) {
			tokens.push(angular.extend({
				value: format.substring(pos)
			}, definedTokens.string));
		}
		
		return tokens;
	}
1174044
function formatReportedHTMLToken(token) {
            const valueLines = sourceCode.getText(token).split("\n")
            const value = valueLines[0]
            return value
        }
1175960
def _subtoken_to_tokens(self, subtokens):
        
        concatenated = "".join(subtokens)
        split = concatenated.split("_")
        return [_unescape_token(t + "_") for t in split if t]
526126
def split_elements(value):
    """"""
    l = [v.strip() for v in value.split(',')]
    if len(l) == 1:
        l = value.split()
    return l
122597
function split(separator, data, limit) {
	var data = ""+data,
	    results = data.split(separator);
	if(!limit) return results;
	if(results.length <= limit) return results;
	var separators = data.match(new RegExp(separator.source, "g"));
	results[limit-1] = join(results.slice(limit-1), separators.slice(limit-1));
	results.length = limit;
	return results;
}
17962
function splitCommand(line) {
    // Make sure we get the same results irrespective of leading/trailing spaces
    var match = line.match(TOKEN_WHITESPACE);
    if (!match) {
        return {
            name: line.toUpperCase(),
            rest: ''
        };
    }
    var name = line.substr(0, match.index).toUpperCase();
    var rest = line.substr(match.index + match[0].length);

    return {
        name: name,
        rest: rest
    };
}
917356
def tokenize(self, s):
        
        return [s[start:end] for start, end in self.span_tokenize(s)]
767557
def _handle_splits(_str):
    """"""
    _str = _str.replace('/', '-')
    _tmp_dict = {}

    if '-' in _str:
        start, stop = _str.split('-')
        if _check_number(start):
            start = regex.sub(r'[0-9]+\?*', start, stop)
        elif _check_number(stop):
            stop = regex.sub(r'[0-9]+\?*', stop, start)
    else:
        start = _str
        stop = _str
    _tmp_dict['start_raw'] = start
    _tmp_dict['stop_raw'] = stop

    _tmp_dict['start_epoch'] = _get_epoch(start)
    _tmp_dict['stop_epoch'] = _get_epoch(stop)

    return _tmp_dict
300858
def _split_header(header):
        """"""
        params = {}
        parts = header.split(',')
        for param in parts:
            # Ignore realm parameter.
            if param.lower().startswith('realm='):
                continue
            # Remove whitespace.
            param = param.strip()
            # Split key-value.
            param_parts = param.split('=', 1)
            # Remove quotes and unescape the value.
            params[param_parts[0]] = unquote(param_parts[1].strip('\"'))
        return params
1178571
def split_on(s, sep=" "):
    """"""
    pattern = '''((?:[^%s"']|"[^"]*"|'[^']*')+)''' % sep

    return [_strip_speechmarks(t) for t in re.split(pattern, s)[1::2]]
592362
static String[] splitToken(String token) throws JWTDecodeException {
        String[] parts = token.split("\\.");
        if (parts.length == 2 && token.endsWith(".")) {
            //Tokens with alg='none' have empty String as Signature.
            parts = new String[]{parts[0], parts[1], ""};
        }
        if (parts.length != 3) {
            throw new JWTDecodeException(String.format("The token was expected to have 3 parts, but got %s.", parts.length));
        }
        return parts;
    }
612984
def split_token(output):
    

    output = ensure_tuple(output)

    flags, i, len_output, data_allowed = set(), 0, len(output), True
    while i < len_output and isflag(output[i]):
        if output[i].must_be_first and i:
            raise ValueError("{} flag must be first.".format(output[i]))
        if i and output[i - 1].must_be_last:
            raise ValueError("{} flag must be last.".format(output[i - 1]))
        if output[i] in flags:
            raise ValueError("Duplicate flag {}.".format(output[i]))
        flags.add(output[i])
        data_allowed &= output[i].allows_data
        i += 1

    output = output[i:]
    if not data_allowed and len(output):
        raise ValueError("Output data provided after a flag that does not allow data.")

    return flags, output
1113089
function tokenize (path) {
  const tokens = []

  const parts = path.split('.')

  for (let i = 0; i < parts.length; i++) {
    const part = parts[i].trim()

    if (part.length === 0) {
      continue
    }

    const arrayIndexes = part.split(SPLIT_REG_EXP)

    tokens.push(arrayIndexes[0])

    for (let j = 1; j < arrayIndexes.length; j++) {
      if (arrayIndexes[j].length === 0) {
        continue
      }

      tokens.push(Number(arrayIndexes[j]))
    }
  }
  return tokens
}
105475
static function split($string, $token) {
        $array = [];
        
        $tok = strtok($string, $token);
        
        while ($tok != false) {
            array_push($array, $tok);
            
            $tok = strtok($token);
        }
        
        return $array;
    }
31343
function process_multi(token, last) {
	if (!last) throw new Error("# wildcard must be at the end of the pattern");
	var name = token.slice(1);
	return {
		type: "multi",
		name: name,
		piece: "((?:[^/#+]+/)*)",
		last: "((?:[^/#+]+/?)*)"
	}
}
1203383
def split_sequence(seq, n):
    """"""
    tokens = []
    while seq:
        tokens.append(seq[:n])
        seq = seq[n:]
    return tokens
1276876
def tsplit(string, delimiters):
    """"""
    delimiters = tuple(delimiters)
    if len(delimiters) < 1:
        return [string,]
    final_delimiter = delimiters[0]
    for i in delimiters[1:]:
        string = string.replace(i, final_delimiter)
    return string.split(final_delimiter)