Tokenize a block of text.

@param string $string
@return array
979347
public function tokenize(string $string) : array
    {
        $words = [];

        preg_match_all(self::WORD_REGEX, $string, $words);

        return $words[0];
    }
483594
public function tokenize(string $string) : array
    {
        $sentences = preg_split(self::SENTENCE_REGEX, $string) ?: [];

        $nGrams = [];

        foreach ($sentences as $sentence) {
            $words = [];

            preg_match_all(self::WORD_REGEX, $sentence, $words);

            $words = $words[0];

            $n = count($words);

            foreach ($words as $i => $word) {
                $p = min($n - $i, $this->max);

                for ($j = $this->min; $j <= $p; $j++) {
                    $nGram = $word;

                    for ($k = 1; $k < $j; $k++) {
                        $nGram .= self::SEPARATOR . $words[$i + $k];
                    }

                    $nGrams[] = $nGram;
                }
            }
        }

        return $nGrams;
    }
252725
public static function removeComments($string)
    {
        $result = '';

        $tokens = self::tokenize($string);

        foreach ($tokens as $token) {
            // Skip comment tokens
            if ($token[self::TOKEN_TYPE] === self::TOKEN_TYPE_COMMENT || $token[self::TOKEN_TYPE] === self::TOKEN_TYPE_BLOCK_COMMENT) {
                continue;
            }

            $result .= $token[self::TOKEN_VALUE];
        }
        $result = self::format( $result,false);

        return $result;
    }
944046
protected function parsePluralizationTags(string $string): string
	{
		return preg_replace_callback('/\<pluralize:([0-9]+)\>(\w*)\<\/pluralize\>/iu', function($matches)
		{
			return $this->pluralize($matches[2], (int) $matches[1]);
		}, $string);
	}
643309
function borkify( $text )
    {
        $textBlocks = preg_split( "/(%[^ ]+)/", $text, -1, PREG_SPLIT_DELIM_CAPTURE );
        $newTextBlocks = array();
        foreach ( $textBlocks as $text )
        {
            if ( $text[0] == '%' )
            {
                $newTextBlocks[] = $text;
                continue;
            }
            $orgtext = $text;
            $text = preg_replace( "/a\B/", "e", $text );
            $text = preg_replace( "/an/", "un", $text );
            $text = preg_replace( "/au/", "oo", $text );
            $text = preg_replace( "/en\b/", "ee", $text );
            $text = preg_replace( "/\Bew/", "oo", $text );
            $text = preg_replace( "/\Bf/", "ff", $text );
            $text = preg_replace( "/\Bi/", "ee", $text );
            $text = preg_replace( "/\Bir/", "ur", $text );
            $text = preg_replace( "/\bo/", "oo", $text );
            $text = preg_replace( "/ow/", "oo", $text );
            $text = preg_replace( "/ph/", "f", $text );
            $text = preg_replace( "/th\b/", "t", $text );
            $text = preg_replace( "/\Btion/", "shun", $text );
            $text = preg_replace( "/\Bu/", "oo", $text );
            $text = preg_replace( "/\bU/", "Oo", $text );
            $text = preg_replace( "/y\b/", "ai", $text );
            $text = preg_replace( "/v/", "f", $text );
            $text = preg_replace( "/w/", "v", $text );
            $text = preg_replace( "/ooo/", "oo", $text );
            if ( $orgtext == $text )
                $text = $text . "-a";
            $newTextBlocks[] = $text;
        }
        $text = implode( '', $newTextBlocks );
        $text = preg_replace( "/([:.?!])(.*)/", "\\2\\1", $text );
        $text = "[" . $text . "]";
        return $text;
    }
301194
public static function toArray($string = '')
    {
        if ($string === '') {
            return [];
        }

        if (self::isJson($string)) {
            return json_decode($string, true);
        }

        if (self::isXml($string)) {
            return self::xmlToArray($string);
        }

        $unserialize = self::unserialize($string);
        if ($unserialize === false) {
            return [];
        }

        if (\is_array($unserialize)) {
            return $unserialize;
        }

        return [];

    }
191502
public function simpleRender($string)
    {
        $paragraphs = '';

        foreach (explode("\n", $string) as $line) {
            if (trim($line)) {
                $paragraphs .= '<p>' . $line . '</p>';
            }
        }

        return $paragraphs;
    }
1163588
public function tokenize(string $text) : array
    {
        $pattern =
'/(\[
    \/?
    \w+
    (?:
        =
        (?:
            "[^"]+"
            |\'[^\']+\'
            |[^]]+
        )
    )?
\])/x';

        $flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE;
        $tokens = preg_split($pattern, $text, -1, $flags);
        $tokens = array_map(function ($text) use ($pattern) {
            $matches = [];
            if (preg_match($pattern, $text, $matches)) {
                $token = new TagToken($text);

                // Fallback to a text token if tag is not valid.
                if (!$this->validateTagToken($token)) {
                    return new TextToken($text);
                }

                return $token;
            } else {
                return new TextToken($text);
            }
        }, $tokens);
        return $this->processRawContentTagTokens($tokens);
    }
487893
static protected function parseProperties( $string ){
		$list	= array();
		foreach( explode( ';', trim( $string ) ) as $line ){
			if( !trim( $line ) )
				continue;
			$parts	= explode( ':', $line );
			$key	= array_shift( $parts );
			$value	= trim( implode( ':', $parts ) );
			$list[]	= new ADT_CSS_Property( $key, $value );
		}
		return $list;
	}
1321089
public function parse($string) {

		$response = new Response();

		// normalize line endings
		$filtered = preg_replace("/(\n|\r){2,}/", PHP_EOL, $string);

		$lines = explode(PHP_EOL, $filtered);

		foreach($lines as $line) {
			$matches = array();
			if(preg_match('/^(\s*)\[(.*)\]$/', $line, $matches)) {
				// entering new section
				$depth = strlen($matches[1]);
				// merge all lower sections
				$this->completeSection($response, $depth);
				$this->stack[$depth] = new Section($matches[2]);
			} else if(preg_match('/^(\s*)(.*)=(.*)$/', $line, $matches)) {
				$depth = strlen($matches[1]);
				if(isset($this->stack[$depth])) {
					$section = $this->stack[$depth];
					$section->__set($matches[2], self::cast($matches[3]));
				}
			}
		}
		// merge remaining children into parents
		$this->completeSection($response, 0);
		
		return $response;
	}
1122796
public static function chunkSplitArr($string, $chunkLen, $padding = '0')
    {
        $count = strlen($string);
        $parts = [];

        for ($i = 0; $i < $count; $i = $i + $chunkLen) {
            if (strlen($string) < $chunkLen) {
                $string .= $padding;
            }
            $parts[] = substr($string, 0, $chunkLen);
            $string = substr($string, $chunkLen);
        }

        return $parts;
    }
855442
def tokenize_string(cls, string, separator):
        """"""
        results = []
        token = ""
        found_escape = False
        for c in string:
            if found_escape:
                if c == separator:
                    token += separator
                else:
                    token += "\\" + c
                found_escape = False
                continue
            if c == "\\":
                found_escape = True
            elif c == separator:
                results.append(token)
                token = ""
            else:
                token += c

        results.append(token)
        return results
1296830
protected function doClassify($string)
    {
        $partsNamespace = explode('\\', $string);
        $return = [];
        foreach ($partsNamespace as $partNamespace) {
            $parts = explode("-", $partNamespace);
            $parts = array_map([$this, "camelize"], $parts);

            $return[] = implode("_", $parts);
        }
        return implode('\\', $return);
    }
292250
def tokenize(text=""):
    
    if text == '':
        return []
    else:
        # split tokens
        mylist = TOKEN_PATTERN.split(text)
        # don't remove newline \n
        mylist = [TOKEN_REPLACE.sub('', x) for x in mylist if x]
        # remove empty substring
        mylist = [x for x in mylist if x]
        return mylist
560764
public static function fromHeader(string $string): array
    {
        $cookies = \explode(";", $string);
        $result = [];

        try {
            foreach ($cookies as $cookie) {
                $parts = \explode('=', $cookie, 2);

                if (2 !== \count($parts)) {
                    return [];
                }

                list($name, $value) = $parts;

                // We can safely trim quotes, as they're not allowed within cookie values
                $result[] = new self(\trim($name), \trim($value, " \t\""));
            }
        } catch (InvalidCookieException $e) {
            return [];
        }

        return $result;
    }
1036165
public static function stringToArray($string)
    {
        if (!preg_match("@^{.*?}$@", $string)) {
            return $string;
        }
        $parts = explode(',', substr($string, 1, -1));
        foreach ($parts as &$part) {
            $part = self::stringToArray($part);
        }
        return $parts;
    }
831644
protected static function getQuotedString($string)
    {
        $tokenType = Tokenizer::TOKEN_TYPE_QUOTE;

        if (!empty($string[0]) && ($string[0] === '`' || $string[0] === '[')) {
            $tokenType = Tokenizer::TOKEN_TYPE_BACK_TICK_QUOTE;
        }

        return [
            Tokenizer::TOKEN_TYPE => $tokenType,
            Tokenizer::TOKEN_VALUE => self::wrapStringWithQuotes($string),
        ];
    }