Segments a sequence of tokens into a sequence of segments.

        :Parameters:
            tokens : `list` ( :class:`~deltas.Token` )
541193
def segment(self, tokens):
        
        look_ahead = LookAhead(tokens)

        segments = Segment()

        while not look_ahead.empty():

            if look_ahead.peek().type not in self.whitespace:  # Paragraph!
                paragraph = MatchableSegment(look_ahead.i)

                while not look_ahead.empty() and \
                      look_ahead.peek().type not in self.paragraph_end:

                    if look_ahead.peek().type == "tab_open":  # Table
                        tab_depth = 1
                        sentence = MatchableSegment(
                            look_ahead.i, [next(look_ahead)])
                        while not look_ahead.empty() and tab_depth > 0:
                            tab_depth += look_ahead.peek().type == "tab_open"
                            tab_depth -= look_ahead.peek().type == "tab_close"
                            sentence.append(next(look_ahead))

                        paragraph.append(sentence)

                    elif look_ahead.peek().type not in self.whitespace:  # Sentence!
                        sentence = MatchableSegment(
                            look_ahead.i, [next(look_ahead)])
                        sub_depth = int(sentence[0].type in SUB_OPEN)
                        while not look_ahead.empty():

                            sub_depth += look_ahead.peek().type in SUB_OPEN
                            sub_depth -= look_ahead.peek().type in SUB_CLOSE
                            sentence.append(next(look_ahead))

                            if sentence[-1].type in self.sentence_end and sub_depth <= 0:
                                non_whitespace = sum(s.type not in self.whitespace for s in sentence)
                                if non_whitespace >= self.min_sentence:
                                    break

                        paragraph.append(sentence)

                    else:  # look_ahead.peek().type in self.whitespace
                        whitespace = Segment(look_ahead.i, [next(look_ahead)])
                        paragraph.append(whitespace)

                segments.append(paragraph)
            else: # look_ahead.peek().type in self.whitespace
                whitespace = Segment(look_ahead.i, [next(look_ahead)])
                segments.append(whitespace)


        return segments
25308
def tokens(self):
        """"""
        spans = self.word_tokenizer.span_tokenize(self.text)
        toks = [Token(
            text=self.text[span[0]:span[1]],
            start=span[0] + self.start,
            end=span[1] + self.start,
            lexicon=self.lexicon
        ) for span in spans]
        return toks
795392
def convert_examples_to_features(examples, label_list, max_seq_length,
                                 tokenizer, output_mode):
    """"""

    label_map = {label : i for i, label in enumerate(label_list)}

    features = []
    for (ex_index, example) in enumerate(examples):
        if ex_index % 10000 == 0:
            logger.info("Writing example %d of %d" % (ex_index, len(examples)))

        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = None
        if example.text_b:
            tokens_b = tokenizer.tokenize(example.text_b)
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > max_seq_length - 2:
                tokens_a = tokens_a[:(max_seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids: 0   0  0    0    0     0       0 0    1  1  1  1   1 1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids: 0   0   0   0  0     0 0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambiguously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.
        tokens = ["[CLS]"] + tokens_a + ["[SEP]"]
        segment_ids = [0] * len(tokens)

        if tokens_b:
            tokens += tokens_b + ["[SEP]"]
            segment_ids += [1] * (len(tokens_b) + 1)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask = [1] * len(input_ids)

        # Zero-pad up to the sequence length.
        padding = [0] * (max_seq_length - len(input_ids))
        input_ids += padding
        input_mask += padding
        segment_ids += padding

        assert len(input_ids) == max_seq_length
        assert len(input_mask) == max_seq_length
        assert len(segment_ids) == max_seq_length

        if output_mode == "classification":
            label_id = label_map[example.label]
        elif output_mode == "regression":
            label_id = float(example.label)
        else:
            raise KeyError(output_mode)

        if ex_index < 5:
            logger.info("*** Example ***")
            logger.info("guid: %s" % (example.guid))
            logger.info("tokens: %s" % " ".join(
                    [str(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
            logger.info(
                    "segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
            logger.info("label: %s (id = %d)" % (example.label, label_id))

        features.append(
                InputFeatures(input_ids=input_ids,
                              input_mask=input_mask,
                              segment_ids=segment_ids,
                              label_id=label_id))
    return features
102881
public Span[] seqToSpans(final String[] tokens) {
    final Span[] origSpans = this.sequenceLabeler.tag(tokens);
    final Span[] seqSpans = SequenceLabelerME.dropOverlappingSpans(origSpans);
    return seqSpans;
  }
771994
def generate_tokens(self, text):
        """"""
        if text != self.last_text:
            string_io = io.StringIO(text)
            self.last_tokens = list(
                tokenize.generate_tokens(string_io.readline)
            )
            self.last_text = text
        return self.last_tokens
593603
def _subtoken_ids_to_tokens(self, subtokens):
    """"""
    escaped_tokens = "".join([
        self.subtoken_list[s] for s in subtokens
        if s < len(self.subtoken_list)])
    escaped_tokens = escaped_tokens.split("_")

    # All tokens in the vocabulary list have been escaped (see _escape_token())
    # so each token must be unescaped when decoding.
    ret = []
    for token in escaped_tokens:
      if token:
        ret.append(_unescape_token(token))
    return ret
789791
def build(cls, seqs: Iterable[int], uid: bool = False) -> 'SequenceSet':
        
        seqs_list = sorted(set(seqs))
        groups: List[Union[int, Tuple[int, int]]] = []
        group: Union[int, Tuple[int, int]] = seqs_list[0]
        for i in range(1, len(seqs_list)):
            group_i = seqs_list[i]
            if isinstance(group, int):
                if group_i == group + 1:
                    group = (group, group_i)
                else:
                    groups.append(group)
                    group = group_i
            elif isinstance(group, tuple):
                if group_i == group[1] + 1:
                    group = (group[0], group_i)
                else:
                    groups.append(group)
                    group = group_i
        groups.append(group)
        return SequenceSet(groups, uid)
251713
function processInline(tokens) {
  if (tokens[0] && (tokens[0].type === 'header' ||
      tokens[0].type === 'list' || tokens[0].type === 'ordered-list' ||
      tokens[0].type === 'blockquote')) {
    tokens[0].content = processEmphasis(tokens[0].content);
    tokens[0].length = getTokensLength(tokens[0].content);
    delete tokens[0].markup;
  } else if (Array.isArray(tokens)) {
    tokens = processEmphasis(tokens);
  }

  return tokens;
}
777123
def segment(text: str, custom_dict: Trie = None) -> List[str]:
    
    if not text or not isinstance(text, str):
        return []

    if not custom_dict:
        custom_dict = DEFAULT_DICT_TRIE

    return list(_onecut(text, custom_dict))
855004
def tokenize(self, text, context=0, skip_style_tags=False):
        """"""
        split = self.regex.split(text)
        self._text = [segment for segment in split if segment]
        self._head = self._global = self._depth = 0
        self._bad_routes = set()
        self._skip_style_tags = skip_style_tags

        try:
            tokens = self._parse(context)
        except BadRoute:  # pragma: no cover (untestable/exceptional case)
            raise ParserError("Python tokenizer exited with BadRoute")
        if self._stacks:  # pragma: no cover (untestable/exceptional case)
            err = "Python tokenizer exited with non-empty token stack"
            raise ParserError(err)
        return tokens
305735
def tokenize(s):
    
    return [m.group(m.lastindex) for m in _tdl_re.finditer(s)]
579169
def get_tokens(tokens):
    
    tokens = tokens[1:-1]
    ret = []
    start = 0
    stack = 0
    for i in range(len(tokens)):
        if tokens[i] == '(':
            if stack == 0:
                start = i
            stack += 1
        elif tokens[i] == ')':
            stack -= 1
            if stack < 0:
                raise Exception('Bracket mismatch: ' + str(tokens))
            if stack == 0:
                ret.append(get_tokens(tokens[start:i + 1]))
        else:
            if stack == 0:
                ret.append(tokens[i])
    if stack != 0:
        raise Exception('Bracket mismatch: ' + str(tokens))
    return ret
577767
def _formatSequence(tokens, categories, seqID, uniqueID):
    """"""
    record = {"_category":categories,
              "_sequenceId":seqID}
    data = []
    reset = 1
    for t in tokens:
      tokenRecord = record.copy()
      tokenRecord["_token"] = t
      tokenRecord["_reset"] = reset
      tokenRecord["ID"] = uniqueID
      reset = 0
      data.append(tokenRecord)

    return data
912646
def __tokenize_segments(self):
        
        for seg_node_id in self.segments:
            segment_toks = self.node[seg_node_id][self.ns+':text'].split()
            for i, tok in enumerate(segment_toks):
                tok_node_id = '{0}:{1}_{2}'.format(self.ns, seg_node_id, i)
                self.add_node(tok_node_id, layers={self.ns, self.ns+':token'},
                              attr_dict={self.ns+':token': tok, 'label': tok})
                self.tokens.append(tok_node_id)
                self.add_edge(seg_node_id, tok_node_id,
                              layers={'rst', 'rst:token'},
                              edge_type=EdgeTypes.spanning_relation)
725817
def chop_sequence(sequence, limit_length):
        
        return [sequence[i:i + limit_length] for i in range(0, len(sequence), limit_length)]
987135
def token_list_to_text(tokenlist):
    
    ZeroWidthEscape = Token.ZeroWidthEscape
    return ''.join(item[1] for item in tokenlist if item[0] != ZeroWidthEscape)
60439
public final Span[] seqToSpans(final String[] tokens) {
    final Span[] annotatedText = this.sequenceLabeler.tag(tokens);
    final List<Span> probSpans = new ArrayList<Span>(
        Arrays.asList(annotatedText));
    return probSpans.toArray(new Span[probSpans.size()]);
  }