Returns sequence of integer ids given a sequence of string ids.

    :param tokens: List of integer tokens.
    :return: List of word ids.
974143
def strids2ids(tokens: Iterable[str]) -> List[int]:
    
    return list(map(int, tokens))
1330400
def contains_text_tokens(self, text_tokens):
        
        id_run = []
        for token in text_tokens:
            id = self.word_list.get(token, None)
            if id is None:
                return False  # since a token is not in the word list, the phrase is not in the dictionary
            id_run.append(id)
        return self.contains_tokens(id_run)
691542
protected void parseSequencesWithColons(TokenList tokens , Sequence sequence ) {

        TokenList.Token t = tokens.getFirst();
        if( t == null )
            return;

        int state = 0;

        TokenList.Token start = null;
        TokenList.Token middle = null;
        TokenList.Token prev = t;

        boolean last = false;
        while( true ) {
            if( state == 0 ) {
                if( isVariableInteger(t) && (t.next != null && t.next.getSymbol() == Symbol.COLON) ) {
                    start = t;
                    state = 1;
                    t = t.next;
                } else if( t != null && t.getSymbol() == Symbol.COLON ) {
                    // If it starts with a colon then it must be 'all'  or a type-o
                    IntegerSequence range = new IntegerSequence.Range(null,null);
                    VariableIntegerSequence varSequence = functions.getManagerTemp().createIntegerSequence(range);
                    TokenList.Token n = new TokenList.Token(varSequence);
                    tokens.insert(t.previous, n);
                    tokens.remove(t);
                    t = n;
                }
            } else if( state == 1 ) {
                // var : ?
                if (isVariableInteger(t)) {
                    state = 2;
                } else {
                    // array range
                    IntegerSequence range = new IntegerSequence.Range(start,null);
                    VariableIntegerSequence varSequence = functions.getManagerTemp().createIntegerSequence(range);
                    replaceSequence(tokens, varSequence, start, prev);
                    state = 0;
                }
            } else if ( state == 2 ) {
                // var:var ?
                if( t != null && t.getSymbol() == Symbol.COLON ) {
                    middle = prev;
                    state = 3;
                } else {
                    // create for sequence with start and stop elements only
                    IntegerSequence numbers = new IntegerSequence.For(start,null,prev);
                    VariableIntegerSequence varSequence = functions.getManagerTemp().createIntegerSequence(numbers);
                    replaceSequence(tokens, varSequence, start, prev );
                    if( t != null )
                        t = t.previous;
                    state = 0;
                }
            } else if ( state == 3 ) {
                // var:var: ?
                if( isVariableInteger(t) ) {
                    // create 'for' sequence with three variables
                    IntegerSequence numbers = new IntegerSequence.For(start,middle,t);
                    VariableIntegerSequence varSequence = functions.getManagerTemp().createIntegerSequence(numbers);
                    t = replaceSequence(tokens, varSequence, start, t);
                } else {
                    // array range with 2 elements
                    IntegerSequence numbers = new IntegerSequence.Range(start,middle);
                    VariableIntegerSequence varSequence = functions.getManagerTemp().createIntegerSequence(numbers);
                    replaceSequence(tokens, varSequence, start, prev);
                }
                state = 0;
            }

            if( last ) {
                break;
            } else if( t.next == null ) {
                // handle the case where it is the last token in the sequence
                last = true;
            }
            prev = t;
            t = t.next;
        }
    }
752756
def get_identifiers(self):
        
        for token in self.tokens:
            if not (token.is_whitespace or token.match(T.Punctuation, ',')):
                yield token
353856
private String getTokenOrBlank(final List<String> tokens, final int reverseIndex) {

		final int length = tokens.size();
		final int index  = length - reverseIndex - 1;

		if (index >= 0 && length > 0 && index < length) {
			return tokens.get(index);
		}

		return "";
	}
1364246
def decode(self, ids, strip_extraneous=False):
    
    if strip_extraneous:
      ids = strip_ids(ids, list(range(self._num_reserved_ids or 0)))
    return unicode_to_native(
        tokenizer.decode(self._subtoken_ids_to_tokens(ids)))
1167657
public final List<SequenceLabel> getSequencesFromSpans(final String[] tokens,
      final Span[] seqSpans) {
    final List<SequenceLabel> sequences = new ArrayList<SequenceLabel>();
    for (final Span seqSpan : seqSpans) {
      final String seqString = seqSpan.getCoveredText(tokens);
      final String seqType = seqSpan.getType();
      final SequenceLabel sequence = this.sequenceFactory
          .createSequence(seqString, seqType, seqSpan);
      sequences.add(sequence);
    }
    return sequences;
  }
1227801
public List<String> getBioDictionaryMatch(final String[] tokens) {

    final List<String> entitiesList = new ArrayList<String>();

    String prefix = "-" + BioCodec.START;
    String gazEntry = null;
    String searchSpan = null;
    // iterative over tokens from the beginning
    for (int i = 0; i < tokens.length; i++) {
      gazEntry = null;
      int j;
      // iterate over tokens from the end
      for (j = tokens.length - 1; j >= i; j--) {
        // create span for search in dictionary Map; the first search takes as
        // span
        // the whole sentence
        searchSpan = createSpan(tokens, i, j);
        gazEntry = lookup(searchSpan.toLowerCase());
        if (gazEntry != null) {
          break;
        }
      }
      prefix = "-" + BioCodec.START;
      // multi-token case
      if (gazEntry != null) {
        while (i < j) {
          entitiesList.add((gazEntry + prefix).intern());
          prefix = "-" + BioCodec.CONTINUE;
          i++;
        }
      }
      // one word case or last member of span
      if (gazEntry != null) {
        entitiesList.add((gazEntry + prefix).intern());
      } else {
        entitiesList.add(BioCodec.OTHER);
      }
    }
    return entitiesList;
  }
741807
def tokens(self):
    
    seq = self.word_tokenizer.transform(Sequence(self.raw))
    return WordList(seq.tokens(), parent=self, language=self.language.code)
952102
def tokens(cls, tokens):
        
        return cls(Lnk.TOKENS, tuple(map(int, tokens)))
509368
def get_token_ids(tree):
        
        def tok2time(token_element):
            '''
            extracts the time (float) of a <tli> element
            (i.e. the absolute position of a token in the document)
            '''
            return float(token_element.attrib['time'])

        timeline = tree.find('//common-timeline')
        return (tok.attrib['id']
                for tok in sorted((tli for tli in timeline.iterchildren()),
                                  key=tok2time))
1022916
def get_ID_text(token):
    
    if isinstance(token, CommonToken):
        text = token.text
    else:
        text = token.getText()

    text = text.lstrip('\\')
    return text
771994
def generate_tokens(self, text):
        """"""
        if text != self.last_text:
            string_io = io.StringIO(text)
            self.last_tokens = list(
                tokenize.generate_tokens(string_io.readline)
            )
            self.last_text = text
        return self.last_tokens
834045
public String[] lemmatize(final String[] tokens) {
    final Span[] origSpans = this.sequenceLabeler.tag(tokens);
    final Span[] seqSpans = SequenceLabelerME.dropOverlappingSpans(origSpans);
    // TODO work with Spans only
    final String[] decodedLemmas = StringUtils.decodeLemmas(tokens, seqSpans);
    return decodedLemmas;
  }
1030864
def token_to_id(self, token):
        
        token = self.process_token(token)
        return self._token2id.get(token, len(self._token2id) - 1)
430200
def _iterate_subsequences(self, tokens):
        
        for start_idx in xrange(len(tokens)):
            for end_idx in xrange(start_idx + 1, len(tokens) + 1):
                yield ' '.join(tokens[start_idx:end_idx]), start_idx
593603
def _subtoken_ids_to_tokens(self, subtokens):
    """"""
    escaped_tokens = "".join([
        self.subtoken_list[s] for s in subtokens
        if s < len(self.subtoken_list)])
    escaped_tokens = escaped_tokens.split("_")

    # All tokens in the vocabulary list have been escaped (see _escape_token())
    # so each token must be unescaped when decoding.
    ret = []
    for token in escaped_tokens:
      if token:
        ret.append(_unescape_token(token))
    return ret