Represent this Sentence as an ASCII tree string. Requires
        the asciitree package. A default token stringifier is provided
        but for custom formatting, specify a str_func which should take
        a single Token and return a string.
556906
def as_asciitree(self, str_func=None):
        """"""
        import asciitree
        from collections import defaultdict
        children = defaultdict(list)
        # since erased nodes may be missing, multiple tokens may have same
        # index (CCprocessed), etc.
        token_to_index = {}
        roots = []
        for token in self:
            children[token.head].append(token)
            token_to_index[token] = token.index
            if token.head == 0:
                roots.append(token)
        assert roots, "Couldn't find root Token(s)"

        if len(roots) > 1:
            # multiple roots so we make a fake one to be their parent
            root = Token(0, 'ROOT', 'ROOT-LEMMA', 'ROOT-CPOS', 'ROOT-POS',
                         None, None, 'ROOT-DEPREL', None, None, None)
            token_to_index[root] = 0
            children[0] = roots
        else:
            root = roots[0]

        def child_func(token):
            index = token_to_index[token]
            return children[index]
        if not str_func:
            def str_func(token):
                return ' %s [%s]' % (token.form, token.deprel)

        return asciitree.draw_tree(root, child_func, str_func)
217196
def ascii2token(self, ascii_code, debug=False):
        
        log.info(repr(ascii_code))
        parts = self.regex.split(ascii_code)
        log.info(repr(parts))
        tokens = []
        for part in parts:
            if not part:
                continue

            if part in self.ascii2token_dict:
                new_token = self.ascii2token_dict[part]
                log.info("\t%r -> %x", part, new_token)
                if new_token > 0xff:
                    tokens.append(new_token >> 8)
                    tokens.append(new_token & 0xff)
                else:
                    tokens.append(new_token)
            else:
                tokens += self.chars2tokens(part)
        return tokens
204415
def to_str(format=List::FORMAT_BINARY,opts={})
      if format != FORMAT_BINARY && format != FORMAT_XML && format != FORMAT_PLAIN
        raise CFFormatError.new("Format #{format} not supported, use List::FORMAT_BINARY or List::FORMAT_XML")
      end

      prsr = @@parsers[format-1].new

      opts[:root] = @value
      opts[:formatted] = @formatted unless opts.has_key?(:formatted)

      return prsr.to_str(opts)
    end
25161
function getWordToken(str) {
    var word = str.split(/\s/g)[0];
    return {text: word, type: 'WORD', length: word.length};
}
604472
def parse(self):
        
        if self.parse_string is not None and self._parse is None:
            self._parse = Tree.parse(self._parse_string)
        return self._parse
800700
def _tree_to_string(cls, root_element, xml_declaration=True, pretty_print=True):
        
        from lxml import etree
        return gf.safe_unicode(etree.tostring(
            root_element,
            encoding="UTF-8",
            method="xml",
            xml_declaration=xml_declaration,
            pretty_print=pretty_print
        ))
412796
def quote_token( str )
      if str.respond_to?(:force_encoding)
        original_encoding = str.encoding
        ascii_str = str.to_s.dup.force_encoding('ASCII-8BIT')
        if token_safe?( ascii_str )
          str
        else
          dquote(ascii_str).force_encoding(original_encoding)
        end
      else
        token_safe?( str ) ? str : dquote(str)
      end
    end
1201727
def tokenize_string(source):
    
    line_reader = StringIO(source).readline
    token_generator = tokenize.generate_tokens(line_reader)

    # Loop over all tokens till a backtick (`) is found.
    # Then, take all tokens till the next backtick to form a backtick quoted
    # string.
    for toknum, tokval, _, _, _ in token_generator:
        if tokval == '`':
            tokval = " ".join(it.takewhile(
                lambda tokval: tokval != '`',
                map(operator.itemgetter(1), token_generator)))
            toknum = _BACKTICK_QUOTED_STRING
        yield toknum, tokval
916336
def tree2text(tree_obj, indent=4):
    # type: (TreeInfo, int) -> str
    
    parts = []

    def _format_node(node, depth=0):
        # type: (NodeInfo, int) -> None
        def p(*args):
            # type: (*str) -> None
            parts.append(" " * depth * indent)
            parts.extend(args)

        if node.is_leaf:
            value_repr = _format_leaf_value(tree_obj, node)
            parts.append("  ---> {}".format(value_repr))
        else:
            assert node.left is not None
            assert node.right is not None
            feat_name = node.feature_name

            if depth > 0:
                parts.append("\n")
            left_samples = node.left.sample_ratio
            p("{feat_name} <= {threshold:0.3f}  ({left_samples:0.1%})".format(
                left_samples=left_samples,
                feat_name=feat_name,
                threshold=node.threshold,
            ))
            _format_node(node.left, depth=depth + 1)

            parts.append("\n")
            right_samples = node.right.sample_ratio
            p("{feat_name} > {threshold:0.3f}  ({right_samples:0.1%})".format(
                right_samples=right_samples,
                feat_name=feat_name,
                threshold=node.threshold,
                ))
            _format_node(node.right, depth=depth + 1)

    _format_node(tree_obj.tree)
    return "".join(parts)
764316
def xml_to_str(tree, encoding=None, xml_declaration=False):
    """"""
    if xml_declaration and not encoding:
        raise ValueError("'xml_declaration' is not supported when 'encoding' is None")
    if encoding:
        return tostring(tree, encoding=encoding, xml_declaration=True)
    return tostring(tree, encoding=text_type, xml_declaration=False)
214853
def auto_string_formatter(func):
    
    @wraps(func)
    def auto_string_formatter_wrapper(function, *args, **kwargs):
        if isinstance(function, string_types):
            function = StringFormatter(function)

        return func(function, *args, **kwargs)

    return auto_string_formatter_wrapper
906998
def t_TEXT(self, token):
        '
        text = token.lexer.lexmatch.group("text").decode("utf8")
        token.value = text
        return token
587597
def etree_to_string(tree):
        
        buff = BytesIO()
        tree.write(buff, xml_declaration=True, encoding='UTF-8')
        return buff.getvalue()
463635
def compose_tree_url(tree, issn_url=False):
    
    url = compose_tree_path(tree, issn_url)

    if WEB_PORT == 80:
        return "%s://%s%s" % (_PROTOCOL, WEB_ADDR, url)

    return "%s://%s:%d%s" % (_PROTOCOL, WEB_ADDR, WEB_PORT, url)
372965
def tree(self, level=0, prefix=''):
        
        text = '%s%s\n' % (prefix, self.value.text)
        prefix = '%s└── ' % ('    ' * level)
        if self._lesser:
            text += self._lesser.tree(level+1, prefix)
        if self._greater:
            text += self._greater.tree(level+1, prefix)
        return text
572451
def make_strslice(lineno, s, lower, upper):
    
    return symbols.STRSLICE.make_node(lineno, s, lower, upper)
313318
function() {
                    if (input.charAt(i) !== '"' && input.charAt(i) !== "'") return;
                    var str = $(/^"((?:[^"\\\r\n]|\\.)*)"|'((?:[^'\\\r\n]|\\.)*)'/);
                    if (str) {
                        return new tree.Quoted(str[1] || str[2]);
                    }
                }