Removes any tokens not allowed by the {@link
edu.ucla.sspace.text.TokenFilter} in the article.
496568
private String filterTokens(String article) {
        Iterator<String> filteredTokens = IteratorFactory.tokenize(article);
        StringBuilder sb = new StringBuilder(article.length());
        while (filteredTokens.hasNext()) {
            sb.append(filteredTokens.next());
            if (filteredTokens.hasNext())
                sb.append(" ");
        }
        return sb.toString();
    }
913820
public void removeToken(String name) {
        if (name == null || name.isEmpty()) {
            return;
        }

        for (Iterator<AntiCsrfParamToken> it = tokens.iterator(); it.hasNext();) {
            AntiCsrfParamToken token = it.next();
            if (name.equals(token.getName())) {
                it.remove();
                if (token.isEnabled()) {
                    this.enabledTokensNames.remove(name);
                }
                break;
            }
        }
    }
1134709
function removeUnwantedCharacters(cleanedString, unwantedCharArray) {
    for (var x = 0; x < unwantedCharArray.length; x++) {
        var unwantedCharRegExp = new RegExp(unwantedCharArray[x], 'g');
        cleanedString = cleanedString.replace(unwantedCharRegExp, '');
    }
    return cleanedString;
}
176763
public function deleteTokens()
    {
        $response = $this
            ->httpClient
            ->get(
                sprintf(
                    '/%s/tokens',
                    $this->getAppUUID()->getId()
                ),
                'delete',
                [],
                [],
                Http::getApisearchHeaders($this)
            );

        self::throwTransportableExceptionIfNeeded($response);
    }
931901
@Override
    public void removeMatching(MetricFilter filter) {
        for (Map.Entry<String, Metric> entry : metrics.entrySet()) {
            if (filter.matches(entry.getKey(), entry.getValue())) {
                remove(entry.getKey());
            }
        }
    }
804107
protected void removeListeners() {
        Editable text = getText();
        if (text != null) {
            TokenSpanWatcher[] spanWatchers = text.getSpans(0, text.length(), TokenSpanWatcher.class);
            for (TokenSpanWatcher watcher : spanWatchers) {
                text.removeSpan(watcher);
            }
            removeTextChangedListener(textWatcher);
        }
    }
1326142
public function clear()
    {
        $count = $this->loadModel('Muffin/Tokenize.Tokens')->deleteAllExpiredOrUsed();
        $this->out(__n('One token deleted', '{0} tokens deleted', $count, $count));
    }
902246
protected final void removeFilter(IoFilterChain filterChain, IoFilter filter) {
        if (filterChain.contains(filter)) {
            filterChain.remove(filter);
        }
    }
96765
def block_anyfilter(parser, token):
    
    bits = token.contents.split()
    nodelist = parser.parse(('endblockanyfilter',))
    parser.delete_first_token()
    return BlockAnyFilterNode(nodelist, bits[1], *bits[2:])
206486
def remove_stopwords(self, tokens):
        """"""
        if isinstance(tokens, (list, tuple)):
            return [word for word in tokens if word.lower() not in self._stopwords]
        else:
            return ' '.join(
                [word for word in tokens.split(' ') if word.lower() not in self._stopwords]
            )
685743
public void trim()
	{
		List<byte[]> chunkList = mChunkList;
		while (chunkList.size() - 1 > mCurrentChunkIndex)
		{
			chunkList.remove(chunkList.size() - 1);
		}
	}
1321027
protected boolean ignoreToken(AnalyzedTokenReadings[] tokens, int idx) throws IOException {
    List<String> words = new ArrayList<>();
    for (AnalyzedTokenReadings token : tokens) {
      words.add(token.getToken());
    }
    return ignoreWord(words, idx);
  }
1302955
public function remove(string ...$tokens)
    {
        if (count($tokens) === 0) {
            return;
        }
        if (count($this->tokens) === 0) {
            return;
        }
        foreach ($tokens as $t) {
            $i = array_search($t, $this->tokens);
            if ($i === false) {
                continue;
            }
            array_splice($this->tokens, $i, 1);
        }
        $this->setAttributeValue();
    }
978118
def cleanString(someText):
    
    ret = ''
    if someText is not None:
        ret = filter(unicode.isalnum, someText.lower())
    return ret
1219173
private function removeUnprocessableTags()
    {
        foreach ($this->unprocessableHtmlTags as $tagName) {
            // Deleting nodes from a 'live' NodeList invalidates iteration on it, so a copy must be made to iterate.
            $nodes = [];
            foreach ($this->domDocument->getElementsByTagName($tagName) as $node) {
                $nodes[] = $node;
            }
            /** @var \DOMNode $node */
            foreach ($nodes as $node) {
                $hasContent = $node->hasChildNodes() || $node->hasChildNodes();
                if (!$hasContent) {
                    $node->parentNode->removeChild($node);
                }
            }
        }
    }
1214491
public void skipAll(final TokenType... tokenTypes) {
        Set<TokenType> tokenTypeSet = Sets.newHashSet(tokenTypes);
        while (tokenTypeSet.contains(lexer.getCurrentToken().getType())) {
            lexer.nextToken();
        }
    }
374248
function markdownToRemarkRemoveTokenizers({ inlineTokenizers }) {
  inlineTokenizers &&
    inlineTokenizers.forEach(tokenizer => {
      delete this.Parser.prototype.inlineTokenizers[tokenizer];
    });
}