r""" Transform asciidoc text into ASCII text that NL parsers can handle

    TODO:
      Tag lines and words with meta data like italics, underlined, bold, title, heading 1, etc

    >>> clean_asciidoc('**Hello** _world_!')
    '"Hello" "world"!'
972125
def clean_asciidoc(text):
    
    """
    text = re.sub(r'(\b|^)[\[_*]{1,2}([a-zA-Z0-9])', r'"\2', text)
    text = re.sub(r'([a-zA-Z0-9])[\]_*]{1,2}', r'\1"', text)
    return text
1003105
def convert(self, text):
        """"""
        # Main function. The order in which other subs are called here is
        # essential. Link and image substitutions need to happen before
        # _EscapeSpecialChars(), so that any *'s or _'s in the <a>
        # and <img> tags get encoded.

        # Clear the global hashes. If we don't clear these, you get conflicts
        # from other articles when generating a page which contains more than
        # one article (e.g. an index page that shows the N most recent
        # articles):
        self.reset()

        if not isinstance(text, unicode):
            #TODO: perhaps shouldn't presume UTF-8 for string input?
            text = unicode(text, 'utf-8')

        if self.use_file_vars:
            # Look for emacs-style file variable hints.
            emacs_vars = self._get_emacs_vars(text)
            if "markdown-extras" in emacs_vars:
                splitter = re.compile("[ ,]+")
                for e in splitter.split(emacs_vars["markdown-extras"]):
                    if '=' in e:
                        ename, earg = e.split('=', 1)
                        try:
                            earg = int(earg)
                        except ValueError:
                            pass
                    else:
                        ename, earg = e, None
                    self.extras[ename] = earg

        # Standardize line endings:
        text = re.sub("\r\n|\r", "\n", text)

        # Make sure $text ends with a couple of newlines:
        text += "\n\n"

        # Convert all tabs to spaces.
        text = self._detab(text)

        # Strip any lines consisting only of spaces and tabs.
        # This makes subsequent regexen easier to write, because we can
        # match consecutive blank lines with /\n+/ instead of something
        # contorted like /[ \t]*\n+/ .
        text = self._ws_only_line_re.sub("", text)

        # strip metadata from head and extract
        if "metadata" in self.extras:
            text = self._extract_metadata(text)

        text = self.preprocess(text)

        if self.safe_mode:
            text = self._hash_html_spans(text)

        # Turn block-level HTML blocks into hash entries
        text = self._hash_html_blocks(text, raw=True)

        # Strip link definitions, store in hashes.
        if "footnotes" in self.extras:
            # Must do footnotes first because an unlucky footnote defn
            # looks like a link defn:
            #   [^4]: this "looks like a link defn"
            text = self._strip_footnote_definitions(text)
        text = self._strip_link_definitions(text)

        text = self._run_block_gamut(text)

        if "footnotes" in self.extras:
            text = self._add_footnotes(text)

        text = self.postprocess(text)

        text = self._unescape_special_chars(text)

        if self.safe_mode:
            text = self._unhash_html_spans(text)

        if "nofollow" in self.extras:
            text = self._a_nofollow.sub(r'<\1 rel="nofollow"\2', text)

        text += "\n"

        rv = UnicodeWithAttrs(text)
        if "toc" in self.extras:
            rv._toc = self._toc
        if "metadata" in self.extras:
            rv.metadata = self.metadata
        return rv
196318
def metablock(parsed):
    
    parsed = " ".join(parsed.replace("\n", "").split()).replace(" ,", ",")
    return escape(strip_tags(decode_entities(parsed)))
557956
def convertDescriptionToRST(textRoot, node, soupTag, heading):
    '''
    
    '''
    if soupTag.para:
        children = soupTag.findChildren(recursive=False)
        for child in children:
            walk(textRoot, child, 0, None, "\n")
        contents = soupTag.get_text()

        if not heading:
            return contents

        start = textwrap.dedent('''
            {heading}
            {heading_mark}
        '''.format(
            heading=heading,
            heading_mark=utils.heading_mark(
                heading,
                configs.SUB_SECTION_HEADING_CHAR
            )
        ))
        return "{0}{1}".format(start, contents)
    else:
        return ""
637418
def code_to_text(self):
        """"""
        active = is_active(self.ext, self.metadata)
        if self.language != self.default_language and 'active' not in self.metadata:
            active = False

        source = copy(self.source)
        escape_code_start(source, self.ext, self.language)

        if active:
            comment_magic(source, self.language, self.comment_magics)
        else:
            source = [self.comment + ' ' + line if line else self.comment for line in source]

        if self.explicit_start_marker(source):
            self.metadata['endofcell'] = self.cell_marker_end or endofcell_marker(source, self.comment)

        if not self.metadata or not self.use_cell_markers:
            return source

        lines = []
        endofcell = self.metadata['endofcell']
        if endofcell == '-' or self.cell_marker_end:
            del self.metadata['endofcell']

        cell_start = [self.comment, self.cell_marker_start or '+']
        if not self.cell_marker_start:
            cell_start.append(metadata_to_json_options(self.metadata))
        elif self.metadata:
            if 'title' in self.metadata:
                cell_start.append(self.metadata.pop('title'))
            if self.metadata:
                cell_start.append(metadata_to_json_options(self.metadata))

        lines.append(' '.join(cell_start))
        lines.extend(source)
        lines.append(self.comment + ' {}'.format(endofcell))
        return lines
407230
def tlg_plaintext_cleanup(text, rm_punctuation=False, rm_periods=False):
    
    remove_comp = regex.compile(r'-\n|«|»|<|>|\.\.\.|‘|’|_|{.+?}|\(.+?\)|[a-zA-Z0-9]', flags=regex.VERSION1)
    text = remove_comp.sub('', text)

    new_text = None
    if rm_punctuation:
        new_text = ''
        punctuation = [',', '·', ':', '"', "'", '?', '-', '!', '*', '[', ']', '{', '}']
        if rm_periods:
            punctuation += ['.', ';']
        for char in text:
            # second try at rming some punctuation; merge with above regex
            if char in punctuation:
                pass
            else:
                new_text += char
    if new_text:
        text = new_text

    # replace line breaks w/ space
    replace_comp = regex.compile(r'\n')
    text = replace_comp.sub(' ', text)

    comp_space = regex.compile(r'\s+')
    text = comp_space.sub(' ', text)

    return text
766508
def docstring_to_markdown(docstring):
    
    new_docstring_lst = []

    for idx, line in enumerate(docstring.split('\n')):
        line = line.strip()
        if set(line) in ({'-'}, {'='}):
            new_docstring_lst[idx-1] = '**%s**' % new_docstring_lst[idx-1]
        elif line.startswith('>>>'):
            line = '    %s' % line
        new_docstring_lst.append(line)

    for idx, line in enumerate(new_docstring_lst[1:]):
        if line:
            if line.startswith('Description : '):
                new_docstring_lst[idx+1] = (new_docstring_lst[idx+1]
                                            .replace('Description : ', ''))
            elif ' : ' in line:
                line = line.replace(' : ', '` : ')
                new_docstring_lst[idx+1] = '\n- `%s\n' % line
            elif '**' in new_docstring_lst[idx-1] and '**' not in line:
                new_docstring_lst[idx+1] = '\n%s' % line.lstrip()
            elif '**' not in line:
                new_docstring_lst[idx+1] = '    %s' % line.lstrip()

    clean_lst = []
    for line in new_docstring_lst:
        if set(line.strip()) not in ({'-'}, {'='}):
            clean_lst.append(line)
    return clean_lst
1305314
def _clean_text(self, text):
        """"""
        output = []
        for char in text:
            cp = ord(char)
            if cp in (0, 0xfffd) or self._is_control(char):
                continue
            if self._is_whitespace(char):
                output.append(' ')
            else:
                output.append(char)
        return ''.join(output)
767665
def idf2txt(txt):
    """"""
    astr = nocomment(txt)
    objs = astr.split(';')
    objs = [obj.split(',') for obj in objs]
    objs = [[line.strip() for line in obj] for obj in objs]
    objs = [[_tofloat(line) for line in obj] for obj in objs]
    objs = [tuple(obj) for obj in objs]
    objs.sort()

    lst = []
    for obj in objs:
        for field in obj[:-1]:
            lst.append('%s,' % (field, ))
        lst.append('%s;\n' % (obj[-1], ))

    return '\n'.join(lst)
633029
def ascii_native_(s):
    '''
    
    '''
    if isinstance(s, text_type):
        s = s.encode('ascii')

    return str(s, 'ascii', 'strict') if PY3 else s
42553
function stringLiteralize(source) {
        return '"'
            + source
                .replace(/\x5C/g, '\\\\')
                .replace(/"/g, '\\"')
                .replace(/\x0A/g, '\\n')
                .replace(/\x09/g, '\\t')
                .replace(/\x0D/g, '\\r')
                // .replace( /\x08/g, '\\b' )
                // .replace( /\x0C/g, '\\f' )
            + '"';
    }
341704
def latinize_text(text, ascii=False):
    
    if text is None or not isinstance(text, six.string_types) or not len(text):
        return text

    if ascii:
        if not hasattr(latinize_text, '_ascii'):
            # Transform to latin, separate accents, decompose, remove
            # symbols, compose, push to ASCII
            latinize_text._ascii = Transliterator.createInstance('Any-Latin; NFKD; [:Symbol:] Remove; [:Nonspacing Mark:] Remove; NFKC; Accents-Any; Latin-ASCII')  # noqa
        return latinize_text._ascii.transliterate(text)

    if not hasattr(latinize_text, '_tr'):
        latinize_text._tr = Transliterator.createInstance('Any-Latin')
    return latinize_text._tr.transliterate(text)
1128072
def taxonomy(value):
    
    try:
        value.encode('ascii')
    except UnicodeEncodeError:
        raise ValueError('tag %r is not ASCII' % value)
    if re.search(r'\s', value):
        raise ValueError('The taxonomy %r contains whitespace chars' % value)
    return value
1093175
def filter_ascii(lst):
    '''
    
    '''
    return [word for word in lst if all(ord(c) < 128 for c in word)]
706885
def _format_msg(text, width, indent=0, prefix=""):
    
    """
    text = repr(text).replace("`", "\\`").replace("\\n", " ``\\n`` ")
    sindent = " " * indent if not prefix else prefix
    wrapped_text = textwrap.wrap(text, width, subsequent_indent=sindent)
    # [1:-1] eliminates quotes generated by repr in first line
    return ("\n".join(wrapped_text))[1:-1].rstrip()
59905
def unicode2ascii(text, expand=True):
    
    """
    translate = UTF8_TO_ASCII if not expand else UTF8_TO_MULTIASCII
    output = ''
    for c in text:
        if not c or ord(c) < 128:
            output += c
        else:
            output += translate[c] if c in translate else ' '
    return output.strip()
751742
def text_to_data(self, text, elt, ps):
        '''
        '''
        if text is None:
            return None
        
        m = self.lex_pattern.match(text)
        if not m:
            raise EvaluateException('Bad Gregorian: %s' %text, ps.Backtrace(elt))
        try:
            retval = _dict_to_tuple(m.groupdict())
        except ValueError, e:
            #raise EvaluateException(str(e))
            raise
        
        if self.pyclass is not None:
            return self.pyclass(retval)
        return retval