Apply a stemmer
@param IStemmer $stemmer
@param boolean $removeNulls
@return \TextAnalysis\Documents\TokensDocument
181612
public function applyStemmer(IStemmer $stemmer, $removeNulls = true) 
    {        
        foreach($this->tokens as &$token) 
        { 
            $token = $stemmer->stem($token);
        }
        
        if($removeNulls) {
            //filter null tokens and re-index
            $this->tokens = array_values(array_filter($this->tokens));            
        }
        return $this;
    }
148964
public static function replace(array $tokens, $newTable) {
        HashMap::assert($tokens, 'tokens');

        $operation = SqlOperation::create($tokens);
        $firstKey  = $operation === SqlOperations::DELETE || $operation === SqlOperations::SELECT ? 'FROM' : strtoupper($operation);

        $tokens[$firstKey][$operation === SqlOperations::INSERT ? 1 : 0]['no_quotes']['parts'][0] = $newTable;

        return $tokens;
    }
1200352
public function parse($tokens = null, $additional = [], $embedded = false)
    {
        if (is_string($tokens)) {
            $tokens = $this->tokenize($tokens, $additional, $embedded);
        } elseif (!$tokens instanceof TokenIterator) {
            // Todo: Own Exceptions
            throw new \InvalidArgumentException('$tokens must be string or TokenIterator');
        }

        return $this->_process($tokens);
    }
373134
public function apply( \DOMText $textnode, Settings $settings, $is_title = false ) {
		if ( ! DOM::has_class( $textnode, $this->classes_to_avoid ) ) {
			$this->apply_internal( $textnode, $settings, $is_title );
		}
	}
424868
public function ngettext($singular, $plural, $count) {
		if ($this->reader === null) {
			return $count > 1 ? $plural : $singular;
		}
		return $this->reader->ngettext((string)$singular, (string)$plural, $count);
	}
50853
public function finalize(TokenList $tokenList, $recursive) {
        if ($recursive) {
            return $this;
        }

        $eolCount = 0;
        $pOpened = false;

        for ($i = 0; $i < count($tokenList); $i++) {
            $token = $tokenList->getTokenAt($i);
            $position = $token->getPosition();

            // Count end of line
            if ($token->getName() === EolRule::NAME) {
                $eolCount++;
                continue;
            }

            if ($this->isModeGfm() && $eolCount === 1 && $pOpened) {
                $tokenList->insertTokenAt($this->name . '_EOL', $position - $eolCount, $i - $eolCount);
            }

            // Close paragraph in more then 1 lines are between texts
            if ($eolCount > 1 && $pOpened) {
                $pOpened = false;
                $tokenList->insertTokenAt($this->name . AbstractBlockRule::CLOSE, $position - $eolCount, $i - $eolCount);
                $i++;
            }

            if ($eolCount >= 1 && !$pOpened) {
                $pOpened = true;
                $tokenList->insertTokenAt($this->name . AbstractBlockRule::OPEN, $position, $i);
                $i++;
            }

            $eolCount = 0;

            if (!$pOpened && in_array($token->getName(), $this->openingTokens)) {
                $pOpened = true;
                $tokenList->insertTokenAt($this->name . AbstractBlockRule::OPEN, $position, $i + 1);
                $i++;
            } elseif ($pOpened && in_array($token->getName(), $this->closingTokens)) {
                $pOpened = false;
                $tokenList->insertTokenAt($this->name . AbstractBlockRule::CLOSE, $position, $i);
                $i++;
            }
        }


        // Close p if left opened
        if ($pOpened) {
            $tokenList->insertTokenAt($this->name . AbstractBlockRule::CLOSE, $tokenList->last()
                ->getPosition(), count($tokenList) - $eolCount);
        }

        // Remove empty paragraphs
        $openingIndex = null;
        for ($i = 0; $i < count($tokenList); $i++) {
            if ($tokenList->getTokenAt($i)->getName() === $this->name . AbstractBlockRule::OPEN) {
                $openingIndex = $i;
            }
            if ($tokenList->getTokenAt($i)->getName() === $this->name . AbstractBlockRule::CLOSE && $openingIndex !== null) {
                $tokenList->removeTokenAt($openingIndex);
                $tokenList->removeTokenAt($i - 1);
                $i = $i - 2;
            }
            if ($tokenList->getTokenAt($i)->getName() !== EolRule::NAME && $tokenList->getTokenAt($i)->getName() !== $this->name . AbstractBlockRule::OPEN) {
                $openingIndex = null;
            }
        }

        return $this;
    }
67310
public function apply( \DOMText $textnode, Settings $settings, $is_title = false ) {
		if ( empty( $settings['smartMath'] ) ) {
			return;
		}

		// Cache textnode content.
		$node_data = $textnode->data;

		// First, let's find math equations.
		$node_data = \preg_replace_callback(
			self::MATH_EQUATION,
			function( array $matches ) {
				return \str_replace(
					[
						'-',
						'/',
						'x',
						'*',
					],
					[
						U::MINUS, // @codeCoverageIgnoreStart
						U::DIVISION,
						U::MULTIPLICATION,
						U::MULTIPLICATION, // @codeCoverageIgnoreEnd
					],
					$matches[0]
				);
			},
			$node_data
		);

		// Revert some non-desired changes and restore textnode content.
		$textnode->data = \preg_replace( self::REVERT_MATCHES, self::REVERT_REPLACEMENTS, $node_data );
	}
1301083
public static function pluralize($count = 1, $singular, $plural = null)
    {
        if ($count == 1) {
            return $singular;
        }

        return is_null($plural) ? Inflector::pluralize($singular) : $plural;
    }
1027747
protected function lexDefinition(): NodeInterface
    {
        if ($this->peek(TokenKindEnum::NAME)) {
            $token = $this->lexer->getToken();

            switch ($token->getValue()) {
                case KeywordEnum::QUERY:
                case KeywordEnum::MUTATION:
                case KeywordEnum::SUBSCRIPTION:
                case KeywordEnum::FRAGMENT:
                    return $this->lexExecutableDefinition();
                case KeywordEnum::SCHEMA:
                case KeywordEnum::SCALAR:
                case KeywordEnum::TYPE:
                case KeywordEnum::INTERFACE:
                case KeywordEnum::UNION:
                case KeywordEnum::ENUM:
                case KeywordEnum::INPUT:
                case KeywordEnum::DIRECTIVE:
                    return $this->lexTypeSystemDefinition();
                case KeywordEnum::EXTEND:
                    return $this->lexTypeSystemExtension();

            }
        } elseif ($this->peek(TokenKindEnum::BRACE_L)) {
            return $this->lexExecutableDefinition();
        } elseif ($this->peekDescription()) {
            return $this->lexTypeSystemDefinition();
        }

        throw $this->unexpected();
    }
1082251
private function removeElements(\DOMNodeList $elems, $logMessage = null)
    {
        if (false === $this->hasElements($elems)) {
            return;
        }

        if (null !== $logMessage) {
            $this->logger->info($logMessage, ['length' => $elems->length]);
        }

        for ($i = $elems->length - 1; $i >= 0; --$i) {
            if ($elems->item($i)->parentNode) {
                $elems->item($i)->parentNode->removeChild($elems->item($i));
            }
        }
    }
746050
public function helpers($helpers = null)
    {
        if ($helpers === null) {
            return $this->_helpers;
        }
        $this->_helpers = (array)$helpers;

        return $this;
    }
1118678
public function DeleteClause()
    {
        $this->match(Lexer::T_DELETE);

        if ($this->lexer->isNextToken(Lexer::T_FROM)) {
            $this->match(Lexer::T_FROM);
        }

        $token = $this->lexer->lookahead;
        $deleteClause = new AST\DeleteClause($this->AbstractSchemaName());

        if ($this->lexer->isNextToken(Lexer::T_AS)) {
            $this->match(Lexer::T_AS);
        }

        $aliasIdentificationVariable = $this->AliasIdentificationVariable();

        $deleteClause->aliasIdentificationVariable = $aliasIdentificationVariable;
        $class = $this->em->getClassMetadata($deleteClause->abstractSchemaName);

        // Building queryComponent
        $queryComponent = array(
            'metadata'     => $class,
            'parent'       => null,
            'relation'     => null,
            'map'          => null,
            'nestingLevel' => $this->nestingLevel,
            'token'        => $token,
        );

        $this->queryComponents[$aliasIdentificationVariable] = $queryComponent;

        return $deleteClause;
    }
637218
public function _removeIdxContentContents(array $contentUids)
    {
        if (0 < count($contentUids)) {
            $this->getEntityManager()
                    ->createQuery('DELETE FROM BackBee\ClassContent\Indexes\IdxContentContent i
                        WHERE i.content_uid IN(:content_uids)
                        OR i.subcontent_uid IN(:subcontent_uids)')
                    ->setParameters(array(
                        'content_uids' => $contentUids,
                        'subcontent_uids' => $contentUids, ))
                    ->execute();
        }

        return $this;
    }
1152695
protected function __remove( $tokens )
	{
		$updated = false;

		foreach ( $tokens as $token ) {
			if ( $this->__contains( $token ) ) {

			if ( false !== ( $index = array_search( $token, $this->tokens, true ) ) )
				unset( $this->tokens[ $index ] );
				$updated = true;
			}
		}

		if ( $updated ) {
			$this->__update();
		}

		return $this;
	}
823655
public function apply( \DOMText $textnode, Settings $settings, $is_title = false ) {
		// Update replacement with current non-breaking narrow space.
		$this->replacement = "\$1{$settings->no_break_narrow_space()}\$2";

		// Update regex with custom units.
		$this->regex = "/(\d\.?)\s({$settings->custom_units()}" . self::_STANDARD_UNITS . ')' . self::WORD_BOUNDARY . '/Sxu';

		parent::apply( $textnode, $settings, $is_title );
	}
862866
protected function apply_internal( \DOMText $textnode, Settings $settings, $is_title = false ) {
		if ( empty( $settings[ $this->settings_switch ] ) ) {
			return;
		}

		$textnode->data = \preg_replace( $this->regex, RE::escape_tags( "<span class=\"{$this->css_class}\">\$1</span>" ), $textnode->data );
	}
1269899
public function whereNull($expr)
    {
        foreach ($this->queries as $query) {
            $query->whereNull($expr);
        }

        return $this;
    }