Applies the named entity recognizer to the given list of tokens,
            where each token is a [word, tag] list.
330183
def apply(self, tokens):
        
        # Note: we could also scan for patterns, e.g.,
        # "my|his|her name is|was *" => NNP-PERS.
        i = 0
        while i < len(tokens):
            w = tokens[i][0].lower()
            if RE_ENTITY1.match(w) \
            or RE_ENTITY2.match(w) \
            or RE_ENTITY3.match(w):
                tokens[i][1] = self.tag
            if w in self:
                for e in self[w]:
                    # Look ahead to see if successive words match the named entity.
                    e, tag = (e[:-1], "-"+e[-1].upper()) if e[-1] in self._cmd else (e, "")
                    b = True
                    for j, e in enumerate(e):
                        if i + j >= len(tokens) or tokens[i+j][0].lower() != e:
                            b = False; break
                    if b:
                        for token in tokens[i:i+j+1]:
                            token[1] = token[1] if token[1].startswith(self.tag) else self.tag
                            token[1] += tag
                        i += j
                        break
            i += 1
        return tokens
510291
def handle_term(self, _, __, tokens: ParseResults) -> ParseResults:
        """"""
        self.ensure_node(tokens)
        return tokens
305735
def tokenize(s):
    
    return [m.group(m.lastindex) for m in _tdl_re.finditer(s)]
353187
private void addTokens(Queue<String> words,
                           int focusIndex,
                           int distance,
                           Map<Pair<Integer>, Double> matrixEntryToCount) {
        for (String word : words) {
            // skip adding co-occurence values for words that are not
            // accepted by the filter
            if (!word.equals(IteratorFactory.EMPTY_TOKEN)) {
                int index = termToIndex.getDimension(word);
                if (index >= 0) {
                    // Get the current number of times that the focus word has
                    // co-occurred with this word before after it.  Weight the
                    // word appropriately baed on distance
                    Pair<Integer> p = new Pair<Integer>(index, focusIndex);
                    double value = weighting.weight(distance, windowSize);
                    Double curCount = matrixEntryToCount.get(p);
                    matrixEntryToCount.put(p, (curCount == null)
                                           ? value : value + curCount);
                }
            }
            distance++;
        }
    }
91581
def named_entity_labels(self):
        """"""
        if not self.is_tagged(NAMED_ENTITIES):
            self.tag_named_entities()
        return [ne[LABEL] for ne in self[NAMED_ENTITIES]]
795392
def convert_examples_to_features(examples, label_list, max_seq_length,
                                 tokenizer, output_mode):
    """"""

    label_map = {label : i for i, label in enumerate(label_list)}

    features = []
    for (ex_index, example) in enumerate(examples):
        if ex_index % 10000 == 0:
            logger.info("Writing example %d of %d" % (ex_index, len(examples)))

        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = None
        if example.text_b:
            tokens_b = tokenizer.tokenize(example.text_b)
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > max_seq_length - 2:
                tokens_a = tokens_a[:(max_seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids: 0   0  0    0    0     0       0 0    1  1  1  1   1 1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids: 0   0   0   0  0     0 0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambiguously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.
        tokens = ["[CLS]"] + tokens_a + ["[SEP]"]
        segment_ids = [0] * len(tokens)

        if tokens_b:
            tokens += tokens_b + ["[SEP]"]
            segment_ids += [1] * (len(tokens_b) + 1)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask = [1] * len(input_ids)

        # Zero-pad up to the sequence length.
        padding = [0] * (max_seq_length - len(input_ids))
        input_ids += padding
        input_mask += padding
        segment_ids += padding

        assert len(input_ids) == max_seq_length
        assert len(input_mask) == max_seq_length
        assert len(segment_ids) == max_seq_length

        if output_mode == "classification":
            label_id = label_map[example.label]
        elif output_mode == "regression":
            label_id = float(example.label)
        else:
            raise KeyError(output_mode)

        if ex_index < 5:
            logger.info("*** Example ***")
            logger.info("guid: %s" % (example.guid))
            logger.info("tokens: %s" % " ".join(
                    [str(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
            logger.info(
                    "segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
            logger.info("label: %s (id = %d)" % (example.label, label_id))

        features.append(
                InputFeatures(input_ids=input_ids,
                              input_mask=input_mask,
                              segment_ids=segment_ids,
                              label_id=label_id))
    return features
879318
def convert_examples_to_features(examples, seq_length, tokenizer):
    """"""

    features = []
    for (ex_index, example) in enumerate(examples):
        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = None
        if example.text_b:
            tokens_b = tokenizer.tokenize(example.text_b)

        if tokens_b:
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > seq_length - 2:
                tokens_a = tokens_a[0:(seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids:   0   0  0    0    0     0      0   0    1  1  1   1  1   1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids:   0   0   0   0  0     0   0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambigiously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.
        tokens = []
        input_type_ids = []
        tokens.append("[CLS]")
        input_type_ids.append(0)
        for token in tokens_a:
            tokens.append(token)
            input_type_ids.append(0)
        tokens.append("[SEP]")
        input_type_ids.append(0)

        if tokens_b:
            for token in tokens_b:
                tokens.append(token)
                input_type_ids.append(1)
            tokens.append("[SEP]")
            input_type_ids.append(1)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask = [1] * len(input_ids)

        # Zero-pad up to the sequence length.
        while len(input_ids) < seq_length:
            input_ids.append(0)
            input_mask.append(0)
            input_type_ids.append(0)

        assert len(input_ids) == seq_length
        assert len(input_mask) == seq_length
        assert len(input_type_ids) == seq_length

        if ex_index < 5:
            logger.info("*** Example ***")
            logger.info("unique_id: %s" % (example.unique_id))
            logger.info("tokens: %s" % " ".join([str(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
            logger.info(
                "input_type_ids: %s" % " ".join([str(x) for x in input_type_ids]))

        features.append(
            InputFeatures(
                unique_id=example.unique_id,
                tokens=tokens,
                input_ids=input_ids,
                input_mask=input_mask,
                input_type_ids=input_type_ids))
    return features
1330400
def contains_text_tokens(self, text_tokens):
        
        id_run = []
        for token in text_tokens:
            id = self.word_list.get(token, None)
            if id is None:
                return False  # since a token is not in the word list, the phrase is not in the dictionary
            id_run.append(id)
        return self.contains_tokens(id_run)
1356951
def tags(self, *names):
        
        if len(names) == 1 and isinstance(names[0], list):
            names = names[0]

        return TaggedCache(self, TagSet(self, names))
834045
public String[] lemmatize(final String[] tokens) {
    final Span[] origSpans = this.sequenceLabeler.tag(tokens);
    final Span[] seqSpans = SequenceLabelerME.dropOverlappingSpans(origSpans);
    // TODO work with Spans only
    final String[] decodedLemmas = StringUtils.decodeLemmas(tokens, seqSpans);
    return decodedLemmas;
  }
448509
public void setTokenNameList(java.util.Collection<String> tokenNameList) {
        if (tokenNameList == null) {
            this.tokenNameList = null;
            return;
        }

        this.tokenNameList = new com.amazonaws.internal.SdkInternalList<String>(tokenNameList);
    }
287132
def tag(self, tokens):
        """"""
        tags = []
        for token in tokens:
            normalized = self.lexicon[token].normalized
            for regex, tag in self.regexes:
                if regex.match(normalized):
                    tags.append((token, tag))
                    break
            else:
                tags.append((token, None))
        return tags
579313
def convert_tokens_to_ids(self, tokens):
        """"""
        ids = []
        for token in tokens:
            ids.append(self.vocab[token])
        if len(ids) > self.max_len:
            logger.warning(
                "Token indices sequence length is longer than the specified maximum "
                " sequence length for this BERT model ({} > {}). Running this"
                " sequence through BERT will result in indexing errors".format(len(ids), self.max_len)
            )
        return ids
455113
private void addTokens(final Token curToken, final List<Token> tokens) {
    if (curToken.tokenLength() != 0) {
      if (this.unTokenizable) {
        tokens.add(curToken);
      } else if (!this.unTokenizable) {
        if (!replacement.matcher(curToken.getTokenValue()).matches()) {
          tokens.add(curToken);
        }
      }
    }
  }
105764
def tag(self, tokens):
        """"""
        # Combine output from individual taggers
        tags = [None] * len(tokens)
        just_tokens = [t[0] for t in tokens]
        for tagger in self.taggers:
            tag_gen = tagger.tag(tokens) if isinstance(tagger, CrfCemTagger) else tagger.tag(just_tokens)
            for i, (token, newtag) in enumerate(tag_gen):
                if newtag == 'I-CM' and not (i == 0 or tag_gen[i - 1][1] not in {'B-CM', 'I-CM'}):
                    tags[i] = 'I-CM'  # Always overwrite I-CM
                elif newtag == 'B-CM' and tags[i] is None:
                    tags[i] = 'B-CM'  # Only overwrite B-CM over None
        # Postprocess the combined output
        for i, tag in enumerate(tags):
            token, pos = tokens[i]
            lex = self.lexicon[token]
            nexttag = tags[i+1] if i < len(tags) - 1 else None
            # Trim disallowed first tokens
            if tag == 'B-CM' and lex.lower in STRIP_START:
                tags[i] = None
                if nexttag == 'I-CM':
                    tags[i+1] = 'B-CM'
            # Trim disallowed final tokens
            if nexttag is None and lex.lower in STRIP_END:
                tags[i] = None
        # Filter certain entities
        for i, tag in enumerate(tags):
            token, pos = tokens[i]
            if tag == 'B-CM':
                entity_tokens = [self.lexicon[token].lower]
                end_i = i + 1
                for j, subsequent in enumerate(tags[i+1:]):
                    if subsequent == 'I-CM':
                        end_i += 1
                        entity_tokens.append(self.lexicon[tokens[i+j+1][0]].lower)
                    else:
                        break

                # Fix combined '1H NMR' on end  # TODO: Also 13C, etc.?
                if len(entity_tokens) > 2 and entity_tokens[-1] == 'nmr' and entity_tokens[-2] == '1h':
                    tags[end_i-2] = 'B-CM'
                    tags[end_i-1] = None
                    entity_tokens = entity_tokens[:-2]

                entity = ' '.join(entity_tokens)
                if any(e in STOP_TOKENS for e in entity_tokens) or self._in_stoplist(entity):
                    tags[i:end_i] = [None] * (end_i - i)
                else:
                    bl = bracket_level(entity)
                    # Try and add on brackets in neighbouring tokens if they form part of the name
                    # TODO: Check bracket type matches before adding on
                    if bl == 1 and len(tokens) > end_i and bracket_level(tokens[end_i][0]) == -1:
                        #print('BLADJUST: %s - %s' % (entity, tokens[end_i][0]))
                        tags[end_i] = 'I-CM'
                    elif bl == -1 and i > 0 and bracket_level(tokens[i-1][0]) == 1:
                        #print('BLADJUST: %s - %s' % (tokens[i-1][0], entity))
                        tags[i-1] = 'B-CM'
                        tags[i] = 'I-CM'
                    elif not bracket_level(entity) == 0:
                        # Filter entities that overall don't have balanced brackets
                        tags[i:end_i] = [None] * (end_i - i)
                    else:
                        # Remove bracketed alphanumeric from end
                        if len(entity_tokens) >= 4 and entity_tokens[-1] == ')' and entity_tokens[-3] == '(':
                            if re.match('^(\d{1,2}[A-Za-z]?|I|II|III|IV|V|VI|VII|VIII|IX)$', entity_tokens[-2]):
                                log.debug('Removing %s from end of CEM', entity_tokens[-2])
                                tags[end_i-3:end_i] = [None, None, None]
        tokentags = list(six.moves.zip(tokens, tags))
        return tokentags
441680
def tokenize(self):

        

        self.tokens = []
        self.terms = OrderedDict()

        # Generate tokens.
        for token in utils.tokenize(self.text):

            # Ignore stopwords.
            if token['unstemmed'] in self.stopwords:
                self.tokens.append(None)

            else:

                # Token:
                self.tokens.append(token)

                # Term:
                offsets = self.terms.setdefault(token['stemmed'], [])
                offsets.append(token['offset'])
134586
def tokenize_list(self, text):
        
        return [self.get_record_token(record) for record in self.analyze(text)]