Generator yields text to be displayed for the current unicode pageview.

        :param data: The current page's data as tuple of ``(ucs, name)``.
        :rtype: generator
422492
def page_view(self, data):
        
        if self.term.is_a_tty:
            yield self.term.move(self.screen.row_begins, 0)
        # sequence clears to end-of-line
        clear_eol = self.term.clear_eol
        # sequence clears to end-of-screen
        clear_eos = self.term.clear_eos

        # track our current column and row, where column is
        # the whole segment of unicode value text, and draw
        # only self.screen.num_columns before end-of-line.
        #
        # use clear_eol at end of each row to erase over any
        # "ghosted" text, and clear_eos at end of screen to
        # clear the same, especially for the final page which
        # is often short.
        col = 0
        for ucs, name in data:
            val = self.text_entry(ucs, name)
            col += 1
            if col == self.screen.num_columns:
                col = 0
                if self.term.is_a_tty:
                    val = u''.join((val, clear_eol, u'\n'))
                else:
                    val = u''.join((val.rstrip(), u'\n'))
            yield val

        if self.term.is_a_tty:
            yield u''.join((clear_eol, u'\n', clear_eos))
970421
def _page_text(page, nowrap=False):
    
    title = page.data['title']
    title = "%s\n%s" % (title, "=" * len(title))

    desc = page.data.get('description')
    if desc:
        desc = "_%s_" % desc

    img = _text_image(page)

    pars = page.data.get('extext')

    if pars:
        # pars = pars.replace(' * ', '\n * ')
        pars = re.sub(r'[ ]+\*[ ]+', '* ', pars)

    if pars and not nowrap:
        parlist = []
        for par in pars.split("\n\n"):
            parlist.append("\n".join(textwrap.wrap(par)))

        disambiguation = page.data.get('disambiguation')
        if disambiguation:
            parlist.append(' * ' + "\n * ".join(page.data.get('links')))

        pars = "\n\n".join(parlist)

    url = '<%s>' % page.data['url']

    txt = []
    txt.append(title)
    txt.append(desc)
    txt.append(url)
    txt.append(pars)
    txt.append(img)

    return "\n\n".join([x for x in txt if x])
792485
def get_unicodedata():
    """"""

    import unicodedata

    fail = False
    uver = unicodedata.unidata_version
    path = os.path.join(os.path.dirname(__file__), 'tools')
    fp, pathname, desc = imp.find_module('unidatadownload', [path])
    try:
        unidatadownload = imp.load_module('unidatadownload', fp, pathname, desc)
        unidatadownload.get_unicodedata(uver, no_zip=True)
    except Exception:
        print(traceback.format_exc())
        fail = True
    finally:
        fp.close()

    assert not fail, "Failed to obtain unicodedata!"
    return uver
1287171
def get_page_text_url(self, page):
        
        template = self.resources.page.get('text')
        url = template.replace("{page}", str(page))
        return url
947830
def generate(self, text):
        
        if not text:
            raise Exception("No text to speak")

        if len(text) >= self.MAX_CHARS:
            raise Exception("Number of characters must be less than 2000")

        params = self.__params.copy()
        params["text"] = text
        self._data = requests.get(self.TTS_URL, params=params,
                                  stream=False).iter_content()
401267
def to_text(data):
    
    try:
        serialized_content = to_json(data, indent=4)
    except Exception, e:
        serialized_content = data
    return serialized_content
1002824
def html(self, data='', py=True):
        

        if py:
            value = self.to_html(data)
        else:
            value = data
        if self.static:
            return str('<span class="value">%s</span>' % safe_str(value))
        else:
            if self.hidden:
                build = Hidden
            else:
                build = self.build
            self._get_http_attrs()
            return str(build(name=self.name, value=value, id=self.id, **self.html_attrs))
634882
def text(self):
        
        t = self.t
        if t is None:
            return u''
        text = t.text
        return to_unicode(text) if text is not None else u''
965691
def get_or_generate_vocabulary(data_dir,
                               tmp_dir,
                               data_prefix,
                               max_page_size_exp,
                               approx_vocab_size=32768,
                               strip=True):
  
  num_pages_for_vocab_generation = approx_vocab_size // 3
  vocab_file = vocab_filename(approx_vocab_size, strip)

  def my_generator(data_prefix):
    """Line generator for vocab."""
    count = 0
    for page in corpus_page_generator(
        all_corpus_files(data_prefix)[::-1], tmp_dir, max_page_size_exp):
      revisions = page["revisions"]
      if revisions:
        text = get_text(revisions[-1], strip=strip)
        yield text
        count += 1
        if count % 100 == 0:
          tf.logging.info("reading pages for vocab %d" % count)
        if count > num_pages_for_vocab_generation:
          break

  return generator_utils.get_or_generate_vocab_inner(data_dir, vocab_file,
                                                     approx_vocab_size,
                                                     my_generator(data_prefix))
684883
def data(self, msg):
        

        (code, repl) = yield self.docmd(b"data")

        if code != 354:
            raise smtplib.SMTPDataError(code, repl)
        else:
            if isinstance(msg, str):
                msg = smtplib._fix_eols(msg).encode('ascii')
            q = smtplib._quote_periods(msg)
            if q[-2:] != CRLF:
                q = q + CRLF
            q = q + b"." + CRLF
            #self.send(q)
            yield self.send(q)
            (code, msg) = yield self.getreply()
            return (code, msg)
823291
def get_paginated_response(self, data):
        
        metadata = {
            'next': self.get_next_link(),
            'previous': self.get_previous_link(),
            'count': self.get_result_count(),
            'num_pages': self.get_num_pages(),
        }
        if isinstance(data, dict):
            if 'results' not in data:
                raise TypeError(u'Malformed result dict')
            data['pagination'] = metadata
        else:
            data = {
                'results': data,
                'pagination': metadata,
            }
        return Response(data)
1081944
def generate_page_title(self, data_slug):
        """"""
        tag = Tag.objects.filter(slug=data_slug)
        return tag[0].word
278365
def render(self, text, auth=None):
        
        # Ensure text is Unicode
        expected = str if sys.version_info[0] >= 3 else unicode  # noqa
        if not isinstance(text, expected):
            raise TypeError(
                'Expected a Unicode string, got {!r}.'.format(text))

        if self.user_content:
            url = '{0}/markdown'.format(self.api_url)
            data = {'text': text, 'mode': 'gfm'}
            if self.context:
                data['context'] = self.context
            data = json.dumps(data, ensure_ascii=False).encode('utf-8')
            headers = {'content-type': 'application/json; charset=UTF-8'}
        else:
            url = '{0}/markdown/raw'.format(self.api_url)
            data = text.encode('utf-8')
            headers = {'content-type': 'text/x-markdown; charset=UTF-8'}

        r = requests.post(url, headers=headers, data=data, auth=auth)
        r.raise_for_status()

        # FUTURE: Remove this once GitHub API properly handles Unicode markdown
        r.encoding = 'utf-8'

        return r.text if self.raw else patch(r.text)
81467
def get_unicode(data):
    
    # if this is a common string, return its unicode representation
    if isinstance(data, str):
        return safe_unicode(data)

    # if this is a list of values, return list of unicode values
    if isinstance(data, list):
        return [get_unicode(item) for item in data]

    # if this is a dictionary, return dictionary of unicode keys and values
    if isinstance(data, dict):
        return {
            get_unicode(key): get_unicode(value)
            for key, value in data.iteritems()
        }
    # if it's anything else, return it in its original form
    return data
1027527
def get_unicode(self):
        
        s = mutf8.decode(self.data)
        if len(s) != self.utf16_size:
            raise ValueError("UTF16 Length does not match!")

        # Return a UTF16 String
        return s
887184
def load_character_images(self, verbose=True):
        
        for dataset in self.character_sets:
            assert self.get_dataset(dataset) is True, "Datasets aren't properly downloaded, " \
                                                      "rerun to try again or download datasets manually."

        for dataset in self.character_sets:
            for image, label in self.load_dataset(dataset, verbose=verbose):
                yield image, label
598135
def to_unicode(text):
    
    if not isinstance(text, str):
        tmpl = 'expected unicode string, got %s value %s'
        raise TypeError(tmpl % (type(text), text))
    return text