Tokenize a block of text.

@param string $string
@return array
483594
public function tokenize(string $string) : array
    {
        $sentences = preg_split(self::SENTENCE_REGEX, $string) ?: [];

        $nGrams = [];

        foreach ($sentences as $sentence) {
            $words = [];

            preg_match_all(self::WORD_REGEX, $sentence, $words);

            $words = $words[0];

            $n = count($words);

            foreach ($words as $i => $word) {
                $p = min($n - $i, $this->max);

                for ($j = $this->min; $j <= $p; $j++) {
                    $nGram = $word;

                    for ($k = 1; $k < $j; $k++) {
                        $nGram .= self::SEPARATOR . $words[$i + $k];
                    }

                    $nGrams[] = $nGram;
                }
            }
        }

        return $nGrams;
    }
1367487
public String[] tokenise(String string) {
		String str = string;
		if(str == null || str.length() == 0) {
			return null;
		}
		List<String> list = new ArrayList<String>();
		int length = delimiter.length();
		int idx = str.indexOf(delimiter);
		while(idx != -1) {
			String token = str.substring(0, idx);
			str = str.substring(idx + length);
			if(trimTokens) {
				token = token.trim();
			}			
			if(!skimEmpty || token.length() > 0) {
				list.add(token);
			}			
			idx = str.indexOf(delimiter);
		}
		
		// handle the case where there is only one token
		// or there is a token after the last separator
		if(str.trim().length() > 0) {
			if(trimTokens) {
				str = str.trim();
			}
			list.add(str);
		}
		
		String [] tokens = new String[list.size()];
		list.toArray(tokens);
		return tokens;
	}
1176130
public function parseString($string)
    {
        $tokens = new Stack($this->tokenizer->tokenize($string));

        if ($tokens[0]->type === Tokenizer::T_EOF) {
            return new CTX\AccountInfoList();
        }

        $ctx = new CTX();
        $this->reduceStruct($tokens, $ctx);

        return $ctx->accountInfoList[0];
    }
855442
def tokenize_string(cls, string, separator):
        """"""
        results = []
        token = ""
        found_escape = False
        for c in string:
            if found_escape:
                if c == separator:
                    token += separator
                else:
                    token += "\\" + c
                found_escape = False
                continue
            if c == "\\":
                found_escape = True
            elif c == separator:
                results.append(token)
                token = ""
            else:
                token += c

        results.append(token)
        return results
739742
private function splitString($string)
    {
        $result = array();
        for ($i = 0; $i < mb_strlen($string, "UTF-8"); ++$i) {
            $result[] = mb_substr($string, $i, 1, "UTF-8");
        }
        return $result;
    }
1163588
public function tokenize(string $text) : array
    {
        $pattern =
'/(\[
    \/?
    \w+
    (?:
        =
        (?:
            "[^"]+"
            |\'[^\']+\'
            |[^]]+
        )
    )?
\])/x';

        $flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE;
        $tokens = preg_split($pattern, $text, -1, $flags);
        $tokens = array_map(function ($text) use ($pattern) {
            $matches = [];
            if (preg_match($pattern, $text, $matches)) {
                $token = new TagToken($text);

                // Fallback to a text token if tag is not valid.
                if (!$this->validateTagToken($token)) {
                    return new TextToken($text);
                }

                return $token;
            } else {
                return new TextToken($text);
            }
        }, $tokens);
        return $this->processRawContentTagTokens($tokens);
    }
121340
private static function explodeOnCaps(string $string): array
    {
        $string = preg_replace('/\B([A-Z])/', '_\1', $string);
        $string = preg_replace('/([0-9]+)/', '_\1', $string);
        $string = preg_replace('/_+/', '_', $string);
        $string = trim($string, '_');

        return explode('_', $string);
    }
944046
protected function parsePluralizationTags(string $string): string
	{
		return preg_replace_callback('/\<pluralize:([0-9]+)\>(\w*)\<\/pluralize\>/iu', function($matches)
		{
			return $this->pluralize($matches[2], (int) $matches[1]);
		}, $string);
	}
1073717
private static function tokenize($pattern)
	{
		if(!($n=self::$_mbstringAvailable ? mb_strlen($pattern,Yii::app()->charset) : strlen($pattern)))
			return array();
		$tokens=array();
		$c0=self::$_mbstringAvailable ? mb_substr($pattern,0,1,Yii::app()->charset) : substr($pattern,0,1);

		for($start=0,$i=1;$i<$n;++$i)
		{
			$c=self::$_mbstringAvailable ? mb_substr($pattern,$i,1,Yii::app()->charset) : substr($pattern,$i,1);
			if($c!==$c0)
			{
				$tokens[]=self::$_mbstringAvailable ? mb_substr($pattern,$start,$i-$start,Yii::app()->charset) : substr($pattern,$start,$i-$start);
				$c0=$c;
				$start=$i;
			}
		}
		$tokens[]=self::$_mbstringAvailable ? mb_substr($pattern,$start,$n-$start,Yii::app()->charset) : substr($pattern,$start,$n-$start);
		return $tokens;
	}
105475
static function split($string, $token) {
        $array = [];
        
        $tok = strtok($string, $token);
        
        while ($tok != false) {
            array_push($array, $tok);
            
            $tok = strtok($token);
        }
        
        return $array;
    }
1072066
function getTokens (string) {
  let tokens = null
  try { tokens = esprima.tokenize('`'+ string +'`') }
  catch (e) { tokens = [] }
  return tokens
}
1081034
protected static function getCommentString($string)
    {
        $last = \strpos($string, '*/', 2) + 2;
        $type = Tokenizer::TOKEN_TYPE_BLOCK_COMMENT;

        if (!empty($string[0]) && ($string[0] === '-' || $string[0] === '#')) {
            $last = \strpos($string, "\n");
            $type = Tokenizer::TOKEN_TYPE_COMMENT;
        }

        $last = ($last === false) ? \strlen($string) : $last;

        return [
            Tokenizer::TOKEN_VALUE => \substr($string, 0, $last),
            Tokenizer::TOKEN_TYPE => $type,
        ];
    }
2249
public function parse($string) {
        $output = new \SplQueue();
        $operator = new \SplStack();

        // Convert the string to tokens for the lexer.
        $tokens = $this->lexer->tokenize($string);

        // while there are tokens to be read:
        while (count($tokens)) {
            // read a token.
            $token = array_shift($tokens);

            // if the token is a number, then push it to the output queue.
            if ($token->isValue()) {
                $output->push($token);
                continue;
            }

            // if the token is a function, then push it onto the operator stack.
            if ($token->isFunction()) {
                $operator->push($token);
                continue;
            }

            // if the token is a function argument separator:
            if ($token->getValue() === ',') {
                while (($op = $this->getTop($operator)) && $op->getValue() !== '(') {
                    $output->push($operator->pop());
                }

                if ($operator->isEmpty()) {
                    throw new ParserException('Mismatched parentheses or misplaced comma.');
                }

                continue;
            }

            // if the token is an operator, then:
            if ($token->isOperator()) {
                // while (
                //  (there is an operator at the top of the operator stack with greater precedence) or
                //  (the operator at the top of the operator stack has equal precedence and the operator is left associative)) and
                //  (the operator at the top of the stack is not a left bracket):
                //     pop operators from the operator stack, onto the output queue.
                while (($op = $this->getTop($operator)) && $op->isOperator() && $token->hasHigherPriority($op)) {
                    $output->push($operator->pop());
                }

                // push the read operator onto the operator stack.
                $operator->push($token);
                continue;
            }

            // if the token is a left bracket (i.e. "("), then:
            if ($token->getValue() === '(') {
                // push it onto the operator stack.
                $operator->push($token);
                continue;
            }

            // if the token is a right bracket (i.e. ")"), then:
            if ($token->getValue() === ')') {
                // while the operator at the top of the operator stack is not a left bracket:
                while (($op = $this->getTop($operator)) && $op->getValue() !== '(') {
                    // pop operators from the operator stack onto the output queue.
                    $output->push($operator->pop());
                }

                // /* if the stack runs out without finding a left bracket, then there are mismatched parentheses. */
                if ($operator->isEmpty()) {
                    throw new ParserException('Mismatched parentheses');
                }

                // pop the left bracket from the stack.
                $operator->pop();

                // If the token at the top of the stack is a function token, pop it onto the output queue.
                if (($op = $this->getTop($operator)) && $op->isFunction()) {
                    $output->push($operator->pop());
                }
            }
        }

        // if there are no more tokens to read:
        // while there are still operator tokens on the stack:
        while (!$operator->isEmpty()) {
            // /* if the operator token on the top of the stack is a bracket, then there are mismatched parentheses. */
            if (is_null($operator->top()->getOperator())) {
                throw new ParserException('Mismatched parentheses or misplaced comma.');
            }

            // pop the operator onto the output queue.
            $output->push($operator->pop());
        }

        // exit.
        return iterator_to_array($output);
    }
1321089
public function parse($string) {

		$response = new Response();

		// normalize line endings
		$filtered = preg_replace("/(\n|\r){2,}/", PHP_EOL, $string);

		$lines = explode(PHP_EOL, $filtered);

		foreach($lines as $line) {
			$matches = array();
			if(preg_match('/^(\s*)\[(.*)\]$/', $line, $matches)) {
				// entering new section
				$depth = strlen($matches[1]);
				// merge all lower sections
				$this->completeSection($response, $depth);
				$this->stack[$depth] = new Section($matches[2]);
			} else if(preg_match('/^(\s*)(.*)=(.*)$/', $line, $matches)) {
				$depth = strlen($matches[1]);
				if(isset($this->stack[$depth])) {
					$section = $this->stack[$depth];
					$section->__set($matches[2], self::cast($matches[3]));
				}
			}
		}
		// merge remaining children into parents
		$this->completeSection($response, 0);
		
		return $response;
	}
1082658
public static function splitQuery($string)
    {
        $queries = array();
        $current_query = '';
        $empty = true;

        $tokens = self::tokenize($string);

        foreach ($tokens as $token) {
            // If this is a query separator
            if ($token[self::TOKEN_VALUE] === ';') {
                if (!$empty) {
                    $queries[] = $current_query.';';
                }
                $current_query = '';
                $empty = true;
                continue;
            }

            // If this is a non-empty character
            if ($token[self::TOKEN_TYPE] !== self::TOKEN_TYPE_WHITESPACE && $token[self::TOKEN_TYPE] !== self::TOKEN_TYPE_COMMENT && $token[self::TOKEN_TYPE] !== self::TOKEN_TYPE_BLOCK_COMMENT) {
                $empty = false;
            }

            $current_query .= $token[self::TOKEN_VALUE];
        }

        if (!$empty) {
            $queries[] = trim($current_query);
        }

        return $queries;
    }
803976
public static function ucWords($string)
    {
        $words = explode(' ', $string);
        $words = array_map('self::ucFirst', $words);

        return implode(' ', $words);
    }
954544
private function explodeKey(string $string): array
    {
        [$key, $tag] = \explode(self::SEPARATOR_TAG, $string . self::SEPARATOR_TAG);

        if ($key === self::HIERARCHY_SEPARATOR) {
            $parts = ['root'];
        } else {
            $parts = \explode(self::HIERARCHY_SEPARATOR, $key);
            // remove first element since it is always empty and replace it with 'root'
            $parts[0] = 'root';
        }

        foreach ($parts as &$level) {
            $level .= self::SEPARATOR_TAG . $tag;
        }

        return $parts;
    }