Produces the list of integer indices corresponding to a token list.
496079
def tokenize(self, token_list):
    """"""
    return [
        self._vocab_dict.get(token, self._vocab_dict[self.UNK])
        for token in token_list
    ]
783618
def atIndices(indexable, indices, default=__unique):
    
    """
    if default is __unique:
        return [indexable[i] for i in indices]
    else:
        res = []
        for i in indices:
            try:
                res.append(indexable[i])
            except (IndexError, KeyError):
                res.append(default)
        return res
29775
def token_list_len(tokenlist):
    
    ZeroWidthEscape = Token.ZeroWidthEscape
    return sum(len(item[1]) for item in tokenlist if item[0] != ZeroWidthEscape)
276293
public List<T> tokenize() {
    // System.out.println("tokenize called");
    List<T> result = new ArrayList<T>();
    while (hasNext()) {
      result.add(next());
    }
    return result;
  }
132889
private String[] convertIndsToLabels(int... ind)
	{
		String[] label = new String[ind.length];
		for (int i = 0; i < ind.length; i++)
		{
			if (!hasLabel(ind[i])) throw new IllegalArgumentException(
				"The index " + ind[i] + " does not have a label.");

			label[i] = getLabel(ind[i]);
		}
		return label;
	}
1108173
def index_sets(self, as_set=False):
        """"""
        indexes = frozenset if as_set else tuple
        return [indexes(b.iter_set()) for b in self]
840610
def get_predicate_indices(tags: List[str]) -> List[int]:
    
    return [ind for ind, tag in enumerate(tags) if 'V' in tag]
238620
def tokens2ids(tokens: Iterable[str], vocab: Dict[str, int]) -> List[int]:
    
    return [vocab.get(w, vocab[C.UNK_SYMBOL]) for w in tokens]
1083184
public static int[] shape(INDArrayIndex... indices) {
        int[] ret = new int[indices.length];
        for (int i = 0; i < ret.length; i++) {
            // FIXME: LONG
            ret[i] = (int) indices[i].length();
        }

        List<Integer> nonZeros = new ArrayList<>();
        for (int i = 0; i < ret.length; i++) {
            if (ret[i] > 0)
                nonZeros.add(ret[i]);
        }

        return ArrayUtil.toArray(nonZeros);
    }
970208
def convert_ids_to_tokens(self, ids):
        """"""
        tokens = []
        for i in ids:
            tokens.append(self.ids_to_tokens[i])
        return tokens
25336
def _tokens_to_subtoken_ids(self, tokens):
    
    ret = []
    for token in tokens:
      ret.extend(self._token_to_subtoken_ids(token))
    return ret
1329160
def tokenlist(item, sep, suppress=True):
    """"""
    if suppress:
        sep = sep.suppress()
    return item + ZeroOrMore(sep + item) + Optional(sep)
213562
def tokenize_by_number(s):
    

    r = find_number(s)

    if r == None:
        return [ s ]
    else:
        tokens = []
        if r[0] > 0:
            tokens.append(s[0:r[0]])
        tokens.append( float(s[r[0]:r[1]]) )
        if r[1] < len(s):
            tokens.extend(tokenize_by_number(s[r[1]:]))
        return tokens
    assert False
215658
private int[] getIndexes() {
    int[] res = new int[this.indexes.size()];
    int i = 0;
    for (Integer idx : this.indexes) {
      res[i] = idx;
      i++;
    }
    return res;
  }
579169
def get_tokens(tokens):
    
    tokens = tokens[1:-1]
    ret = []
    start = 0
    stack = 0
    for i in range(len(tokens)):
        if tokens[i] == '(':
            if stack == 0:
                start = i
            stack += 1
        elif tokens[i] == ')':
            stack -= 1
            if stack < 0:
                raise Exception('Bracket mismatch: ' + str(tokens))
            if stack == 0:
                ret.append(get_tokens(tokens[start:i + 1]))
        else:
            if stack == 0:
                ret.append(tokens[i])
    if stack != 0:
        raise Exception('Bracket mismatch: ' + str(tokens))
    return ret
650041
def get_reg_index_value_pairs(self, regIndexList):
        
        str = b''
        regList = self._context.read_core_registers_raw(regIndexList)
        for regIndex, reg in zip(regIndexList, regList):
            str += six.b(conversion.byte_to_hex2(regIndex) + ':' + conversion.u32_to_hex8le(reg) + ';')
        return str
673803
def step_indices(group_idx):
    
    ilen = step_count(group_idx) + 1
    indices = np.empty(ilen, int)
    indices[0] = 0
    indices[-1] = group_idx.size
    inline(c_step_indices, ['group_idx', 'indices'], define_macros=c_macros, extra_compile_args=c_args)
    return indices