Splits a string into tokens in blocks of words which can be encoded quickly.

@param string $string

@return string[]
979595
protected function getEncodableWordTokens($string)
    {
        $tokens = [];

        $encodedToken = '';
        // Split at all whitespace boundaries
        foreach (preg_split('~(?=[\t ])~', $string) as $token) {
            if ($this->tokenNeedsEncoding($token)) {
                $encodedToken .= $token;
            } else {
                if (strlen($encodedToken) > 0) {
                    $tokens[] = $encodedToken;
                    $encodedToken = '';
                }
                $tokens[] = $token;
            }
        }
        if (strlen($encodedToken)) {
            $tokens[] = $encodedToken;
        }

        return $tokens;
    }
23537
public function loadString(&$string): array
    {
        /** @noinspection CallableParameterUseCaseInTypeContextInspection */
        $string = $this->unwrap($string);
        $this->rawSegments = $string;

        return $this->parse($string);
    }
963900
public function tokenize($string)
    {
        $line     = 1;
        $position = 1;
        $tokens   = array();

        while (strlen($string)) {
            foreach ($this->expressions as $rule) {
                if (!preg_match($rule['match'], $string, $match)) {
                    continue;
                }

                // Remove matched string from input
                $string = substr($string, strlen($match[0]));

                // Update position in file
                $line += substr_count($match[0], "\n");
                if (($pos = strrpos($match[0], "\n")) !== false) {
                    $position  = strrpos($match[0], "\n") + 1;
                } else {
                    $position += strlen($match[0]);
                }

                // Skip irrelevant rules
                if (in_array($rule['type'], $this->ignoreTokens)) {
                    continue 2;
                }

                // Add all other rules including their match to the token array
                $match = $this->removeNumericKeys($match);
                $tokens[] = new Token($rule['type'], $line, $position, $match);
                continue 2;
            }

            throw new \RuntimeException(
                "Could not parse string: '" . substr($string, 0, 20) . "' in line $line at position $position."
            );
        }

        $tokens[] = new Token(self::T_EOF, $line, $position, null);
        return $tokens;
    }
1340981
public static function split( $string, $width )
    {
        // str_split is not suitable for multi-byte characters, we should use preg_split to get char array properly.
        // additionally, array_slice() is not enough as some character has doubled width.
        // we need a function to split string not by character count but by string width

        if ( false === ( $_encoding = static::mbSupported( $string ) ) )
        {
            return str_split( $string, $width );
        }

        $_utf8 = mb_convert_encoding( $string, 'utf8', $_encoding );
        $_lines = array();
        $_line = null;

        foreach ( preg_split( '//u', $_utf8 ) as $_character )
        {
            //  Test if $_character could be appended to current line
            if ( mb_strwidth( $_line . $_character, 'utf8' ) <= $width )
            {
                $_line .= $_character;
                continue;
            }

            //  If not, push current line to array and make new line
            $_lines[] = str_pad( $_line, $width );
            $_line = $_character;
        }

        if ( strlen( $_line ) )
        {
            $_lines[] = count( $_lines ) ? str_pad( $_line, $width ) : $_line;
        }

        mb_convert_variables( $_encoding, 'utf8', $_lines );

        return $_lines;
    }
1371793
public static function ordStr($string)
    {
        $codes = array();

        $encoding = mb_detect_encoding($string);
        $size = mb_strlen($string, $encoding);

        for ($i = 0; $i < $size; $i++) {
            $char = mb_substr($string, $i, 1, $encoding);
            $codes[] = static::ord($char);
        }

        return $codes;
    }
239477
protected function processOneToken($string)
    {
        $token = $this->getToken($string, $this->currentStringLength, $this->previousToken);
        $this->tokens[] = $token;
        $this->tokenLength = \strlen($token[self::TOKEN_VALUE]);
        $this->previousToken = $token;

        $this->oldStringLength = $this->currentStringLength;
        $this->currentStringLength -= $this->tokenLength;

        return \substr($string, $this->tokenLength);
    }
1242510
private static String[] tokenize(String string) {
		if (string.indexOf("?") == -1 && string.indexOf("*") == -1) {
			return new String[] { string };
		}

		char[] textArray = string.toCharArray();
		List<String> tokens = new ArrayList<String>();
		StringBuilder tokenBuilder = new StringBuilder();

		for (int i = 0; i < textArray.length; i++) {
			if (textArray[i] != '?' && textArray[i] != '*') {
				// collect non wild card characters
				tokenBuilder.append(textArray[i]);
				continue;
			}

			if (tokenBuilder.length() != 0) {
				tokens.add(tokenBuilder.toString());
				tokenBuilder.setLength(0);
			}
			if (textArray[i] == '?') {
				tokens.add("?");
			} else if (tokens.size() == 0 || (i > 0 && tokens.get(tokens.size() - 1).equals("*") == false)) {
				tokens.add("*");
			}
		}

		if (tokenBuilder.length() != 0) {
			tokens.add(tokenBuilder.toString());
		}

		return (String[]) tokens.toArray(new String[tokens.size()]);
	}
669192
protected function splitLastWord($string)
    {
        // see if we're dealing with a camelCase string
        if ($this->camelCased && preg_match('#^(.*)([A-Z].*)$#', $string, $matches)) {
            return [ $matches[1], '', $matches[2] ];
        }

        // otherwise, attempt to split at anything that might indicate a word separator
        if (preg_match('#^(.*)([_ \s-])([^_ \s-]+)$#', $string, $matches)) {
            return [ $matches[1], $matches[2], $matches[3] ];
        }

        return [ '', '', $string ];
    }
593071
public static function stringToArray(string $string): array
    {
        list($from, $to) = explode(self::SEPARATOR, $string);
        $from = empty($from)
            ? self::ZERO
            : (is_numeric($from)
                ? (int) $from
                : $from);

        $to = empty($to)
            ? self::INFINITE
            : (is_numeric($to)
                ? (int) $to
                : $to);

        return [$from, $to];
    }
1340110
public function parseKeywordText($string)
    {
        $pattern = '/(?P<keywords>[^\s\.]+)(\s+(?:as\s+)?(?P<alias>.+))?/';
        if(preg_match_all($pattern, str_replace(BaseBuilder::DELIMITED_IDENTIFIERS, '', $string), $matches) > 0)
        {
            $alias = end($matches['alias']);
            if(!$alias)
            {
                $alias = null;
            }
            return [
                'keywords'  => $matches['keywords'],
                'alias'     => $alias,
            ];
        }
        else
        {
            return [];
        }
    }
282075
public static function parseStringForSearch($string)
    {
        $words = explode(' ', $string);
        $conditions = [];

        foreach ($words as $word) {
            $word = trim($word);

            if (empty($word)) {
                continue;
            }

            if (is_numeric($word)) {
                $conditions[] = $word;
                continue;
            }

            if (1 === mb_strlen($word)) {
                continue;
            }

            if (mb_strlen($word) > 7) {
                $conditions[] = mb_substr($word, 0, -3, 'UTF-8');
                continue;
            }

            if (mb_strlen($word) > 5) {
                $conditions[] = mb_substr($word, 0, -1, 'UTF-8');
                continue;
            }

            if (in_array(mb_substr($word, -1, null, 'UTF-8'), ['ě', 'š', 'č', 'ř', 'ž', 'ý', 'á', 'í', 'é', 'ú', 'ů'], true)) {
                $conditions[] = mb_substr($word, 0, -1, 'UTF-8');
                continue;
            }

            $conditions[] = $word;
        }

        return $conditions;
    }
747745
public static function explode($string, $characters = ";>|/\\<")
    {
        $string = self::sanitizeString($string);

        if ($string === null) {
            return [];
        }

        // Convert characters into a safe regular expression
        $splitRegex = "";
        for ($i = 0; $i < strlen($characters); $i++) {
            $splitRegex .= "\\" . $characters[$i];
        }
        $splitRegex = "/[" . $splitRegex . "]/";

        $rawElements = preg_split($splitRegex, $string);
        $elements = [];
        foreach($rawElements as $rawElement) {
            $element = self::sanitizeString($rawElement);

            if ($element !== null) {
                $elements[] = $element;
            }
        }

        return $elements;
    }
176664
protected function convertKeywords($string)
    {
        $string = str_replace('\Closure', 'Closure', $string);
        $string = str_replace('Closure', '\Closure', $string);
        $string = str_replace('dynamic', 'mixed', $string);

        return $string;
    }
974633
public static function decode($string)
    {
        $xml = simplexml_load_string($string);
        $array = array($xml->getName() => (array) $xml);
        $array = self::loopDecode($array);

        return $array;
    }
335360
public function parseFormat($string) {
        $groups = [];

        if (preg_match_all("#(?<!\\\\)<[/]?(\w+)([\sa-z0-9=_;,]*)>#", $string, $matches, PREG_SET_ORDER) !== false) {
            $groups = $matches;
        }

        return $groups;
    }
979347
public function tokenize(string $string) : array
    {
        $words = [];

        preg_match_all(self::WORD_REGEX, $string, $words);

        return $words[0];
    }
306619
public function breakParts($string)
	{
		$parts = preg_split('/[\n]+(```|~~~).*[\n]/', $string);
		$string = '';

		foreach ($parts as $key => $part)
		{
			if ($key%2 === 0) {
				$part = $this->removeStars($part);
				$part = $this->parseComments($part);
			} else {
				$part = $this->parseCode($part);
			}

			if ($part != '') {
				$string .= "{$part}";
			}
		}

		return $string;
	}