Extract Table of Content of MD
    :param text:
    :return:
482592
def get_toc(text):
    
    mkd.convert(text)
    toc = mkd.toc
    mkd.reset()
    return toc
777123
def segment(text: str, custom_dict: Trie = None) -> List[str]:
    
    if not text or not isinstance(text, str):
        return []

    if not custom_dict:
        custom_dict = DEFAULT_DICT_TRIE

    return list(_onecut(text, custom_dict))
163214
def extract_table(html_string, row_count, column_count):
    
    try:
        from bs4 import BeautifulSoup
        from bs4.element import Tag
    except ImportError:
        print("ERROR: You must have BeautifulSoup to use html2data")
        return

    #html_string = convertRichText(html_string)

    data_table = []
    for row in range(0, row_count):
        data_table.append([])
        for column in range(0, column_count):
            data_table[-1].append(None)

    soup = BeautifulSoup(html_string, 'html.parser')

    table = soup.find('table')

    if not table:
        return ''

    trs = table.findAll('tr')
    if len(trs) == 0:
        return [['']]

    for tr in range(len(trs)):
        ths = trs[tr].findAll('th')
        if len(ths) == 0:
            tds = trs[tr].findAll('td')
        else:
            tds = ths

        if len(tds) == 0:
            tds = []
            for i in range(0, column_count):
                tds.append(Tag("", name=""))

        for i in range(len(tds)):
            td = tds[i]
            row, column = find_unassigned_table_cell(data_table)

            r_span_count = 1
            c_span_count = 1

            if td.has_attr('rowspan'):
                r_span_count = int(td['rowspan'])
            if td.has_attr('colspan'):
                c_span_count = int(td['colspan'])

            for row_prime in range(row, row + r_span_count):
                for column_prime in range(column, column + c_span_count):
                    if row_prime == row and column_prime == column:

                        items = []
                        for item in td.contents:
                            items.append(str(item))
                        string = ''.join(items).strip()

                        text = restructify(string).rstrip()

                        data_table[row_prime][column_prime] = text
                    else:
                        data_table[row_prime][column_prime] = ""

            if i + 1 < column_count and i == len(tds) - 1:
                for x in range(len(tds), column_count):
                    if data_table[row][x] is None:
                        data_table[row][x] = ""
    for row in range(len(data_table)):
        for column in range(len(data_table[row])):
            if not data_table[row][column]:
                data_table[row][column] = ""

    return data_table
873047
def from_text(name, ttl, rdclass, rdtype, *text_rdatas):
    

    return from_text_list(name, ttl, rdclass, rdtype, text_rdatas)
912412
def get_values(text):
    

    res = re.findall(r"\[(.*?)\]", text)
    values = []

    for r in res:
        if "|" in r:
            params = r.split("|")
            for p in params:
                values.append(p)
        else:
            values.append(r)

    values = [str(v.lower()) for v in values]

    return values
1102030
def text_to_data(self, text, elt, ps):
        '''
        '''
        v = text
        if v == 'false': 
            if self.pyclass is None:
                return False
            return self.pyclass(False)

        if v == 'true': 
            if self.pyclass is None:
                return True
            return self.pyclass(True)

        try:
            v = int(v)
        except:
            try:
                v = long(v)
            except:
                raise EvaluateException('Unparseable boolean', 
                        ps.Backtrace(elt))

        if v:
            if self.pyclass is None:
                return True
            return self.pyclass(True)

        if self.pyclass is None:
             return False
        return self.pyclass(False)
291500
def parse(text):
    
    ret = Docstring()
    if not text:
        return ret

    text = inspect.cleandoc(text)
    match = re.search('^:', text, flags=re.M)
    if match:
        desc_chunk = text[:match.start()]
        meta_chunk = text[match.start():]
    else:
        desc_chunk = text
        meta_chunk = ''

    parts = desc_chunk.split('\n', 1)
    ret.short_description = parts[0] or None
    if len(parts) > 1:
        long_desc_chunk = parts[1] or ''
        ret.blank_after_short_description = long_desc_chunk.startswith('\n')
        ret.blank_after_long_description = long_desc_chunk.endswith('\n\n')
        ret.long_description = long_desc_chunk.strip() or None

    for match in re.finditer(
            r'(^:.*?)(?=^:|\Z)', meta_chunk, flags=re.S | re.M
    ):
        chunk = match.group(0)
        if not chunk:
            continue
        try:
            args_chunk, desc_chunk = chunk.lstrip(':').split(':', 1)
        except ValueError:
            raise ParseError(
                'Error parsing meta information near "{}".'.format(chunk)
            )
        args = args_chunk.split()
        desc = desc_chunk.strip()
        if '\n' in desc:
            first_line, rest = desc.split('\n', 1)
            desc = first_line + '\n' + inspect.cleandoc(rest)
        ret.meta.append(DocstringMeta(args, description=desc, type=None))

    return ret
415911
function extractDescription(texts) {
  let document = '';
  texts.forEach(text => {
    document += text.description || '';
  });
  return document.toLowerCase();
}
298460
def of_text(self, text, encoding="utf-8"):
        
        m = self.hash_algo()
        m.update(text.encode(encoding))
        if self.return_int:
            return int(m.hexdigest(), 16)
        else:
            return m.hexdigest()
615312
def get_text(self):
        """"""
        self._load_raw_content()
        if self._text is None:
            assert self._raw_content is not None
            ret_cont = self._raw_content
            if self.compressed:
                ret_cont = zlib.decompress(ret_cont, zlib.MAX_WBITS+16)
            if self.encoded:
                ret_cont = ret_cont.decode('utf-8')
            self._text = ret_cont
        assert self._text is not None
        return self._text
751742
def text_to_data(self, text, elt, ps):
        '''
        '''
        if text is None:
            return None
        
        m = self.lex_pattern.match(text)
        if not m:
            raise EvaluateException('Bad Gregorian: %s' %text, ps.Backtrace(elt))
        try:
            retval = _dict_to_tuple(m.groupdict())
        except ValueError, e:
            #raise EvaluateException(str(e))
            raise
        
        if self.pyclass is not None:
            return self.pyclass(retval)
        return retval
701972
def _parse(self, text):
        '''
        
        '''
        # ensure unicode first
        text = str(text).strip()

        if not text.startswith(DELIMITER):
            return {}, text

        try:
            _, fm, content = BOUNDARY.split(text, 2)
        except ValueError:
            # if we can't split, bail
            return {}, text
        # loader_class = self.loader_factory(self)
        metadata = yaml.load(fm, Loader=self.loader_class)
        metadata = metadata if (isinstance(metadata, dict)) else {}
        return metadata, content
427917
def extract_data item, text_limit: 1200
      link = item.link
      desc = item.description
      title = item.title.force_encoding('UTF-8')

      # Extract main image
      case
      when desc.match(/\|.*\|/m)
        img, terms, text = desc.split('|')
      when desc.include?('|')
        img, text = desc.split('|')
        terms = nil
      when desc.match(/&lt;img.*?src="(.*?)\"/)
        img = desc.match(/&lt;img.*?src="(.*?)\"/)[1]
        text = desc
        terms = nil
      else
        img, terms, text = nil, nil, desc
      end

      # Extract categories
      categories =
      if terms
        terms.split(/term_group: .*\n/).
          select {|g| g.match /taxonomy: category/}.
          map {|c| c.match( /name: (.*)\n/)[1]}
      else
        []
      end

      # Crop text
      short_text = strip_tags(text)[0..text_limit]

      # Normalize newlines
      short_text = short_text.gsub(/\A\s+/,"").gsub(/\n+/,"\n\n")

      # Right-strip to end of last paragraph
      short_text = short_text[0..short_text.rindex(/\.\n/)]

      [title, link, img, short_text, categories]
    end
898494
def text_to_data(self, text, elt, ps):
        '''
        '''
        prefix,localName = SplitQName(text)
        nsdict = ps.GetElementNSdict(elt)
        prefix = prefix or ''
        try:
            namespaceURI = nsdict[prefix]
        except KeyError, ex:
            raise EvaluateException('cannot resolve prefix(%s)'%prefix,
                ps.Backtrace(elt))
                
        v = (namespaceURI,localName)
        if self.pyclass is not None:
            return self.pyclass(v)    
        return v
495878
def extract_table(tabletag):
    # type: (Tag) -> List[Dict]
    
    theadtag = tabletag.find_next('thead')

    headertags = theadtag.find_all('th')
    if len(headertags) == 0:
        headertags = theadtag.find_all('td')
    headers = []
    for tag in headertags:
        headers.append(get_text(tag))

    tbodytag = tabletag.find_next('tbody')
    trtags = tbodytag.find_all('tr')

    table = list()
    for trtag in trtags:
        row = dict()
        tdtags = trtag.find_all('td')
        for i, tag in enumerate(tdtags):
            row[headers[i]] = get_text(tag)
        table.append(row)
    return table
218937
def read_text(forfn, nrows=None, verbose=True):
    
    """
    tqdm_prog = tqdm if verbose else no_tqdm
    nrows = wc(forfn, nrows=nrows)  # not necessary when nrows==None
    lines = np.empty(dtype=object, shape=nrows)
    with ensure_open(forfn) as f:
        for i, line in enumerate(tqdm_prog(f, total=nrows)):
            if i >= len(lines):
                break
            lines[i] = ensure_str(line).rstrip('\n').rstrip('\r')
        if all('\t' in line for line in lines):
            num_tabs = [sum([1 for c in line if c == '\t']) for line in lines]
            del lines
            if all(i == num_tabs[0] for i in num_tabs):
                f.seek(0)
                return read_csv(f, sep='\t', header=None, nrows=nrows)
        elif sum((1 for line in lines if any((tag.lower() in line.lower() for tag in HTML_TAGS)))
                ) / float(len(lines)) > .05:
            return np.array(html2text(EOL.join(lines)).split(EOL))
    return lines
1361313
def extract(self, text: str) -> List[Extraction]:

        
        
        content = BeautifulSoup(text, 'html5lib')
        subject = content.find('h1').text.strip()
        recip = self.mailing_list_name
        
        navbar = content.find(id='navbar')
        if navbar == None:
            info = self.old_format(content)
        else:
            info = self.new_format(navbar, content)
        for i in info[0:3]:
            if i == 'None':
                print('missed something important')
        sender = info[0]
        date   = info[1]
        body   = info[2]
        nxt    = info[3]
        rep_to = info[4]
        pub = 'SeeSat_Obs'
        dRec = datetime.datetime.now().isoformat()
        
        msg_obj = { 
            'url' : self.email_url,
            '@context' : {
                '@vocab' : 'schema.org'
            },
            'subject' : subject,
            'recip' : recip,
            'sender' : sender
        }
        if date != 'None':
            msg_obj['dateReceived'] = date
        if body != 'None':
            msg_obj['body'] = body
        if nxt != 'None':
            msg_obj['nxt'] = nxt
        if rep_to != 'None':
            msg_obj['replyToMessage'] = rep_to
        return Extraction(value=msg_obj, extractor_name=self.name)