Tokenize the input string

Returns an array of Token structs representing the tokens.

@param string $string
@return Token[]
963900
public function tokenize($string)
    {
        $line     = 1;
        $position = 1;
        $tokens   = array();

        while (strlen($string)) {
            foreach ($this->expressions as $rule) {
                if (!preg_match($rule['match'], $string, $match)) {
                    continue;
                }

                // Remove matched string from input
                $string = substr($string, strlen($match[0]));

                // Update position in file
                $line += substr_count($match[0], "\n");
                if (($pos = strrpos($match[0], "\n")) !== false) {
                    $position  = strrpos($match[0], "\n") + 1;
                } else {
                    $position += strlen($match[0]);
                }

                // Skip irrelevant rules
                if (in_array($rule['type'], $this->ignoreTokens)) {
                    continue 2;
                }

                // Add all other rules including their match to the token array
                $match = $this->removeNumericKeys($match);
                $tokens[] = new Token($rule['type'], $line, $position, $match);
                continue 2;
            }

            throw new \RuntimeException(
                "Could not parse string: '" . substr($string, 0, 20) . "' in line $line at position $position."
            );
        }

        $tokens[] = new Token(self::T_EOF, $line, $position, null);
        return $tokens;
    }
1186778
public static function smartExplode($string)
    {
        if (empty($string))
            return array();

        if (is_array($string))
            return $string;

        if (!is_array($string)) {
            if (strpos($string, ';') !== false)
                return explode(';', trim($string, ';'));

            if (strpos($string, ',') !== false)
                return explode(',', trim($string, ','));
        }

        return (array)$string;
    }
347882
protected static function getStringTypeArray($type, $string, array &$matches)
    {
        return [
            Tokenizer::TOKEN_TYPE => $type,
            Tokenizer::TOKEN_VALUE => \substr($string, 0, \strlen($matches[1])),
        ];
    }
1163588
public function tokenize(string $text) : array
    {
        $pattern =
'/(\[
    \/?
    \w+
    (?:
        =
        (?:
            "[^"]+"
            |\'[^\']+\'
            |[^]]+
        )
    )?
\])/x';

        $flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE;
        $tokens = preg_split($pattern, $text, -1, $flags);
        $tokens = array_map(function ($text) use ($pattern) {
            $matches = [];
            if (preg_match($pattern, $text, $matches)) {
                $token = new TagToken($text);

                // Fallback to a text token if tag is not valid.
                if (!$this->validateTagToken($token)) {
                    return new TextToken($text);
                }

                return $token;
            } else {
                return new TextToken($text);
            }
        }, $tokens);
        return $this->processRawContentTagTokens($tokens);
    }
817028
private function parseStringToArray($string)
    {
        // convert commas to spaces
        $string = str_replace(',', ' ', $string);
        // convert new lines to spaces
        $string = str_replace(PHP_EOL, ' ', $string);
        // split on the spaces and filter out empty elements
        return array_values(
            array_filter(
                array_map(
                    'trim',
                    explode(' ', $string)
                ),
                'strlen'
            )
        );
    }
1367487
public String[] tokenise(String string) {
		String str = string;
		if(str == null || str.length() == 0) {
			return null;
		}
		List<String> list = new ArrayList<String>();
		int length = delimiter.length();
		int idx = str.indexOf(delimiter);
		while(idx != -1) {
			String token = str.substring(0, idx);
			str = str.substring(idx + length);
			if(trimTokens) {
				token = token.trim();
			}			
			if(!skimEmpty || token.length() > 0) {
				list.add(token);
			}			
			idx = str.indexOf(delimiter);
		}
		
		// handle the case where there is only one token
		// or there is a token after the last separator
		if(str.trim().length() > 0) {
			if(trimTokens) {
				str = str.trim();
			}
			list.add(str);
		}
		
		String [] tokens = new String[list.size()];
		list.toArray(tokens);
		return tokens;
	}
386503
private function scanNextType($string)
    {
        foreach ($this->patterns as $type => $pattern) {
            if (preg_match('/^(' . $pattern . ')/sS', $string, $matches) === 1) {
                return [
                    'type'   => $type,
                    'match'  => $matches[1],
                    'length' => strlen($matches[1]),
                ];
            }
        }

        return null;
    }
453912
public static String[] tokenize(String str, String delimiters, boolean trim) {
        if (str == null) {
            return new String[0];
        }
        StringTokenizer st = new StringTokenizer(str, delimiters);
        List<String> tokens = new ArrayList<>();
        while (st.hasMoreTokens()) {
            tokens.add(trim ? st.nextToken().trim() : st.nextToken());
        }
        return tokens.toArray(new String[0]);
    }
335360
public function parseFormat($string) {
        $groups = [];

        if (preg_match_all("#(?<!\\\\)<[/]?(\w+)([\sa-z0-9=_;,]*)>#", $string, $matches, PREG_SET_ORDER) !== false) {
            $groups = $matches;
        }

        return $groups;
    }
483594
public function tokenize(string $string) : array
    {
        $sentences = preg_split(self::SENTENCE_REGEX, $string) ?: [];

        $nGrams = [];

        foreach ($sentences as $sentence) {
            $words = [];

            preg_match_all(self::WORD_REGEX, $sentence, $words);

            $words = $words[0];

            $n = count($words);

            foreach ($words as $i => $word) {
                $p = min($n - $i, $this->max);

                for ($j = $this->min; $j <= $p; $j++) {
                    $nGram = $word;

                    for ($k = 1; $k < $j; $k++) {
                        $nGram .= self::SEPARATOR . $words[$i + $k];
                    }

                    $nGrams[] = $nGram;
                }
            }
        }

        return $nGrams;
    }
739742
private function splitString($string)
    {
        $result = array();
        for ($i = 0; $i < mb_strlen($string, "UTF-8"); ++$i) {
            $result[] = mb_substr($string, $i, 1, "UTF-8");
        }
        return $result;
    }
809009
protected function processTokens($string)
    {
        $this->tokens = [];
        $this->previousToken = '';
        $this->currentStringLength = \strlen($string);
        $this->oldStringLength = \strlen($string) + 1;

        while ($this->currentStringLength >= 0) {
            if ($this->oldStringLength <= $this->currentStringLength) {
                break;
            }
            $string = $this->processOneToken($string);
        }

        return $this->tokens;
    }
560764
public static function fromHeader(string $string): array
    {
        $cookies = \explode(";", $string);
        $result = [];

        try {
            foreach ($cookies as $cookie) {
                $parts = \explode('=', $cookie, 2);

                if (2 !== \count($parts)) {
                    return [];
                }

                list($name, $value) = $parts;

                // We can safely trim quotes, as they're not allowed within cookie values
                $result[] = new self(\trim($name), \trim($value, " \t\""));
            }
        } catch (InvalidCookieException $e) {
            return [];
        }

        return $result;
    }
999529
public static function compress($string)
    {
        $result = '';

        $tokens = self::tokenize($string);

        $whitespace = true;
        foreach ($tokens as $token) {
            // Skip comment tokens
            if ($token[self::TOKEN_TYPE] === self::TOKEN_TYPE_COMMENT || $token[self::TOKEN_TYPE] === self::TOKEN_TYPE_BLOCK_COMMENT) {
                continue;
            }
            // Remove extra whitespace in reserved words (e.g "OUTER     JOIN" becomes "OUTER JOIN")
            elseif ($token[self::TOKEN_TYPE] === self::TOKEN_TYPE_RESERVED || $token[self::TOKEN_TYPE] === self::TOKEN_TYPE_RESERVED_NEWLINE || $token[self::TOKEN_TYPE] === self::TOKEN_TYPE_RESERVED_TOPLEVEL) {
                $token[self::TOKEN_VALUE] = preg_replace('/\s+/',' ',$token[self::TOKEN_VALUE]);
            }

            if ($token[self::TOKEN_TYPE] === self::TOKEN_TYPE_WHITESPACE) {
                // If the last token was whitespace, don't add another one
                if ($whitespace) {
                    continue;
                } else {
                    $whitespace = true;
                    // Convert all whitespace to a single space
                    $token[self::TOKEN_VALUE] = ' ';
                }
            } else {
                $whitespace = false;
            }

            $result .= $token[self::TOKEN_VALUE];
        }

        return rtrim($result);
    }
680317
protected static function tokenize($string)
    {
        self::init();

        $tokens = array();

        // Used for debugging if there is an error while tokenizing the string
        $original_length = strlen($string);

        // Used to make sure the string keeps shrinking on each iteration
        $old_string_len = strlen($string) + 1;

        $token = null;

        $current_length = strlen($string);

        // Keep processing the string until it is empty
        while ($current_length) {
            // If the string stopped shrinking, there was a problem
            if ($old_string_len <= $current_length) {
                $tokens[] = array(
                    self::TOKEN_VALUE=>$string,
                    self::TOKEN_TYPE=>self::TOKEN_TYPE_ERROR
                );

                return $tokens;
            }
            $old_string_len =  $current_length;

            // Determine if we can use caching
            if ($current_length >= self::$max_cachekey_size) {
                $cacheKey = substr($string,0,self::$max_cachekey_size);
            } else {
                $cacheKey = false;
            }

            // See if the token is already cached
            if ($cacheKey && isset(self::$token_cache[$cacheKey])) {
                // Retrieve from cache
                $token = self::$token_cache[$cacheKey];
                $token_length = strlen($token[self::TOKEN_VALUE]);
                self::$cache_hits++;
            } else {
                // Get the next token and the token type
                $token = self::getNextToken($string, $token);
                $token_length = strlen($token[self::TOKEN_VALUE]);
                self::$cache_misses++;

                // If the token is shorter than the max length, store it in cache
                if ($cacheKey && $token_length < self::$max_cachekey_size) {
                    self::$token_cache[$cacheKey] = $token;
                }
            }

            $tokens[] = $token;

            // Advance the string
            $string = substr($string, $token_length);

            $current_length -= $token_length;
        }

        return $tokens;
    }
610276
public function parseTokens($string)
    {
        $this->string = $string;
        $this->cursor = 0;
        $this->current = isset($this->string[0]) ? $this->string[0] : null;
        $this->next = isset($this->string[1]) ? $this->string[1] : null;

        // Unify result of ctype_space() across systems
        $previousLocale = setlocale(LC_CTYPE, 0);
        setlocale(LC_CTYPE, 'C');
        $tokens = $this->doParseTokens();
        setlocale(LC_CTYPE, $previousLocale);

        return $tokens;
    }
873655
protected function parseString($string)
    {
        $array  = array();

        if (strlen($string) < 1) {
            return $array;
        }

        // Separate single string into an array of "key=value" strings
        $keyvalue = explode('&', $query_string);

        // Separate each "key=value" string into an array[key] = value
        foreach ($keyvalue as $pair) {
            list($k, $v) = explode('=', $pair, 2);

            // Handle the case where multiple values map to the same key
            // by pulling those values into an array themselves
            if (isset($query_array[$k])) {
                // If the existing value is a scalar, turn it into an array
                if (is_scalar($query_array[$k])) {
                    $query_array[$k] = array($query_array[$k]);
                }
                array_push($query_array[$k], $v);
            } else {
                $query_array[$k] = $v;
            }
        }

        return $array;
    }