extract the text for each page via ocr
488438
function (pdf_file, cb) {
            var quality = 300;
            if (options.hasOwnProperty('quality') && options.quality) {
              quality = options.quality;
            }
            convert(pdf_file.file_path, quality, function (err, tif_path) {
              var zeroBasedNumPages = num_pages-1;
              self.emit('log', 'converted page to intermediate tiff file, page '+ index+ ' (0-based indexing) of '+ zeroBasedNumPages);
              if (err) { return cb(err); }
              var ocr_flags = [
                '-psm 6'
              ];
              if (options.ocr_flags) {
                ocr_flags = options.ocr_flags;
              }
              ocr(tif_path, ocr_flags, function (err, extract) {
                fs.unlink(tif_path, function (tif_cleanup_err, reply) {
                  if (tif_cleanup_err) {
                    err += ', error removing temporary tif file: "'+tif_cleanup_err+'"';
                  }
                  if (err) { return cb(err); }
                  var page_number = index+1
                  self.emit('log', 'raw ocr: page ' + index + ' (0-based indexing) of ' +zeroBasedNumPages + ' complete');
                  single_page_pdf_file_paths.push(pdf_file.file_path);
                  self.emit('page', { hash: hash, text: extract, index: index, num_pages: num_pages, pdf_path: pdf_path, single_page_pdf_path: pdf_file.file_path});
                  text_pages.push(extract);
                  index++;
                  cb();
                });
              });
            });
          }
585833
def _get_names(self):
        
        for line in self.content:
            line = line.strip()
            if line and re.search("^[a-zA-Z0-9]", line):
                yield line
865017
def text(self):
        
        data = self.content
        return data.decode(self.encoding or 'utf-8') if data else ''
1361313
def extract(self, text: str) -> List[Extraction]:

        
        
        content = BeautifulSoup(text, 'html5lib')
        subject = content.find('h1').text.strip()
        recip = self.mailing_list_name
        
        navbar = content.find(id='navbar')
        if navbar == None:
            info = self.old_format(content)
        else:
            info = self.new_format(navbar, content)
        for i in info[0:3]:
            if i == 'None':
                print('missed something important')
        sender = info[0]
        date   = info[1]
        body   = info[2]
        nxt    = info[3]
        rep_to = info[4]
        pub = 'SeeSat_Obs'
        dRec = datetime.datetime.now().isoformat()
        
        msg_obj = { 
            'url' : self.email_url,
            '@context' : {
                '@vocab' : 'schema.org'
            },
            'subject' : subject,
            'recip' : recip,
            'sender' : sender
        }
        if date != 'None':
            msg_obj['dateReceived'] = date
        if body != 'None':
            msg_obj['body'] = body
        if nxt != 'None':
            msg_obj['nxt'] = nxt
        if rep_to != 'None':
            msg_obj['replyToMessage'] = rep_to
        return Extraction(value=msg_obj, extractor_name=self.name)
714387
def get_text(self):
        """"""
        return u''.join(u'{0}'.format(b) for b in self.text)
856581
def decode_page(page):
    
    if hasattr(page, 'read'):  # urllib
        if py3k:
            encoding = page.headers.get_param("charset") or "utf-8"
        else:
            encoding = page.headers.getparam("charset") or "utf-8"
        return text_type(page.read(), encoding=encoding)
    else:  # requests?
        encoding = page.headers.get("charset") or "utf-8"
        return text_type(page.content, encoding=encoding)
212283
def text_to_data(self, text, elt, ps):
        '''
        '''
        val = hexdecode(text)
        if self.pyclass is not None:
            return self.pyclass(val) 
        return val
1306128
private static Page extractNonEmptyPage(PageBuffer pageBuffer)
    {
        Page page = pageBuffer.poll();
        while (page != null && page.getPositionCount() == 0) {
            page = pageBuffer.poll();
        }
        return page;
    }
360212
def page(content, pagecallback=prompt):
    
    width = getwidth()
    height = getheight()
    pagenum = 1

    try:
        try:
            line = content.next().rstrip("\r\n")
        except AttributeError:
            # Python 3 compatibility
            line = content.__next__().rstrip("\r\n")
    except StopIteration:
        pagecallback(pagenum)
        return

    while True:     # page cycle
        linesleft = height-1 # leave the last line for the prompt callback
        while linesleft:
            linelist = [line[i:i+width] for i in range(0, len(line), width)]
            if not linelist:
                linelist = ['']
            lines2print = min(len(linelist), linesleft)
            for i in range(lines2print):
                if WINDOWS and len(line) == width:
                    # avoid extra blank line by skipping linefeed print
                    echo(linelist[i])
                else:
                    print((linelist[i]))
            linesleft -= lines2print
            linelist = linelist[lines2print:]

            if linelist: # prepare symbols left on the line for the next iteration
                line = ''.join(linelist)
                continue
            else:
                try:
                    try:
                        line = content.next().rstrip("\r\n")
                    except AttributeError:
                        # Python 3 compatibility
                        line = content.__next__().rstrip("\r\n")
                except StopIteration:
                    pagecallback(pagenum)
                    return
        if pagecallback(pagenum) == False:
            return
        pagenum += 1
781767
def toc(text):
    
    extensions = ['markdown.extensions.toc']
    mkd = markdown.Markdown(extensions=extensions)
    html = mkd.convert(text)
    return mkd.toc
916946
def get_title(page):
  
  start_pos = page.find("<title>")
  end_pos = page.find("</title>")
  assert start_pos != -1
  assert end_pos != -1
  start_pos += len("<title>")
  return text_encoder.to_unicode_utf8(page[start_pos:end_pos])
415911
function extractDescription(texts) {
  let document = '';
  texts.forEach(text => {
    document += text.description || '';
  });
  return document.toLowerCase();
}
444077
public List<Page> extractText(InputStream src) throws IOException {
        List<Page> pages = Lists.newArrayList();

        PdfReader reader = new PdfReader(src);
        RenderListener listener = new InternalListener();
        PdfContentStreamProcessor processor = new PdfContentStreamProcessor(listener);

        for (int i = 1; i <= reader.getNumberOfPages(); i++) {
            pages.add(currentPage = new Page());
            PdfDictionary pageDic = reader.getPageN(i);
            PdfDictionary resourcesDic = pageDic.getAsDict(PdfName.RESOURCES);
            processor.processContent(ContentByteUtils.getContentBytesForPage(reader, i), resourcesDic);
        }
        reader.close();

        return pages;
    }
353340
def text(self, selector):
        

        result = self.__bs4.select(selector)

        return [r.get_text() for r in result] \
            if result.__len__() > 1 else \
            result[0].get_text() if result.__len__() > 0 else None
339442
def parse_text (url_data):
    """"""
    lineno = 0
    for line in url_data.get_content().splitlines():
        lineno += 1
        line = line.strip()
        if not line or line.startswith('#'):
            continue
        url_data.add_url(line, line=lineno)
621193
function extract_from_line (body) {
    const matches = body.bodytext.match(/\bFrom:[^<\n]*<([^>\n]*)>/);
    if (matches) {
        return matches[1];
    }

    for (let i=0,l=body.children.length; i < l; i++) {
        const from = extract_from_line(body.children[i]);
        if (from) {
            return from;
        }
    }

    return null;
}
167096
def text(self):
        """"""

        s = MLStripper()
        s.feed(self.html)
        return s.get_data()