Parse the html of the url page.
321487
def parse_html(user_id)
      url = API_URL + user_id
      first_doc = JSON.parse(open(url).read)
      totalsize = first_doc['totalsize']

      url = API_URL + user_id + '&startNum=0&endNum=' + totalsize.to_s
      @doc = JSON.parse(open(url).read)['result1']
    end
170777
def html(content, **kwargs):
    """"""
    if hasattr(content, 'read'):
        return content
    elif hasattr(content, 'render'):
        return content.render().encode('utf8')

    return str(content).encode('utf8')
48448
def _parse_result_page(self, url, payload, only_region=False):
        
        data = []
        try:

            if only_region:
                html = self.scraper._get_html(url)
            else:
                html = self.scraper._post_html(url, payload=payload)

        except RequestException500:

            self.scraper.log.warning(u"Status code 500 on {} with {}".format(url, payload))
            return None


        current_selection = self._get_current_selection(html)

        table = Datatable(html)
        data = []
        for row in table.data:
            region_or_unit_id, region_or_unit_label = row["region_or_unit"]
            if region_or_unit_label in self.regions:
                row["region"] = region_or_unit_label
                row["unit"] = None
            else:
                row["region"] = None
                row["unit"] = region_or_unit_label

            value = row["value"]

            row.pop("value", None)
            row.pop("region_or_unit", None)

            for dim in self.dimensions:
                if dim.id not in row:
                    row[dim.id] = current_selection[dim.id][1] # gets label



            data.append(Result(value, row))
        return data
764939
def parse_chromium (url_data):
    """"""
    from ..bookmarks.chromium import parse_bookmark_data
    for url, name in parse_bookmark_data(url_data.get_content()):
        url_data.add_url(url, name=name)
820264
def parse(self, response):
        """"""

        # Check if page is out of range
        no_more_products = re.search(
            r'No matching products were found',
            response.css('div.paged-results').extract_first(),
            flags=re.I
        )
        if no_more_products:
            pass  # no more pages to populate, stop scraping

        else:
            # Scrape products list
            for product in response.css('article.product-list__item'):
                product_url = product.css('a::attr("href")').extract_first()
                yield response.follow(product_url, self.parse_detail)

            (base, params) = split_url(response.url)
            params.update({'page': int(params.get('page', '1')) + 1})
            next_page_url = join_url(base, params)
            self.logger.info(next_page_url)
            yield response.follow(next_page_url, self.parse)
567597
def parse(self, text, html=True):
        ''''''
        self._urls = []
        self._users = []
        self._lists = []
        self._tags = []

        reply = REPLY_REGEX.match(text)
        reply = reply.groups(0)[0] if reply is not None else None

        parsed_html = self._html(text) if html else self._text(text)
        return ParseResult(self._urls, self._users, reply,
                           self._lists, self._tags, parsed_html)
998411
def _parse_text(self, page_text):
        """"""

        # 1 sanitize: remove leading blank lines
        # 2 separate "config text" from content, store content
        # 3 convert config text + \n to obtain Meta, this is the config.

        lines = page_text.split('\n')
        i = 0
        while lines[i].strip() == '':
            i += 1
        if i > 0: # i points to the first non-blank line.  Else, i is 0, there are no leading blank lines
            lines = lines[i:]  #remove leading blank lines

        i = 0
        while lines[i].strip() != '':
            i += 1
            # i points to the first blank line
        cfg_lines = '\n'.join(lines[0:i + 1])  #config lines, plus the empty line

        md = markdown.Markdown(extensions=['meta','fenced_code', 'codehilite'],output_format="html5")
        md.convert(cfg_lines)   # need to trigger the conversion to obtain md.Meta

        self._config = md.Meta
        self._content =  '\n'.join(lines[i+1:])
1314915
def parse(cls, dom):
        """"""
        handler = cls()
        saxify(dom, handler)
        return handler.content
1020143
def rel_links(cls, page):
    """"""
    for match in cls.REL_RE.finditer(page):
      href, rel = match.group(0), match.group(1)
      if rel not in cls.REL_TYPES:
        continue
      href_match = cls.HREF_RE.search(href)
      if href_match:
        href = cls.href_match_to_url(href_match)
        parsed_href = urlparse(href)
        if any(parsed_href.path.endswith(ext) for ext in cls.REL_SKIP_EXTENSIONS):
          continue
        yield href
48270
def read(self):
        """"""
        try:
            f = urllib.request.urlopen(self.url)
        except urllib.error.HTTPError as err:
            if err.code in (401, 403):
                self.disallow_all = True
            elif err.code >= 400:
                self.allow_all = True
        else:
            raw = f.read()
            self.parse(raw.decode("utf-8").splitlines())
325028
def from_string(self, html_string):
        """"""
        self._html_parser._reset().feed(html_string)
        return self._html_parser.result
1115771
def parse(self, url_data):
        """"""
        self.url_data = url_data
        self.loc = False
        self.url = u""
        data = url_data.get_content()
        isfinal = True
        try:
            self.parser.Parse(data, isfinal)
        except ExpatError as expaterr:
            self.url_data.add_warning(expaterr.message,tag=WARN_XML_PARSE_ERROR)
730296
def getPrevUrl(self, url, data):
        """"""
        mo = compile(r"index(\d+)\.html").search(url)
        num = int(mo.group(1)) - 1
        prefix = url.rsplit('/', 1)[0]
        return "%s/index%d.html" % (prefix, num)
1169863
def parse(self, url):
        
        parsed_url = urlparse.urlparse(url)
        try:
            default_config = self.CONFIG[parsed_url.scheme]
        except KeyError:
            raise ValueError(
                'unrecognised URL scheme for {}: {}'.format(
                    self.__class__.__name__, url))
        handler = self.get_handler_for_scheme(parsed_url.scheme)
        config = copy.deepcopy(default_config)
        return handler(parsed_url, config)
33569
def _get_html_contents(html):
    """"""
    parser = MyHTMLParser()
    parser.feed(html)
    if parser.is_code:
        return ('code', parser.data.strip())
    elif parser.is_math:
        return ('math', parser.data.strip())
    else:
        return '', ''
1301528
def parse_safari (url_data):
    """"""
    from ..bookmarks.safari import parse_bookmark_data
    for url, name in parse_bookmark_data(url_data.get_content()):
        url_data.add_url(url, name=name)
307459
def parse(url):
    """"""

    config = {}

    url = urlparse.urlparse(url)

    # Remove query strings.
    path = url.path[1:]
    path = path.split('?', 2)[0]

    if url.scheme in SCHEMES:
        config["ENGINE"] = SCHEMES[url.scheme]

    if url.scheme in USES_URL:
        config["URL"] = urlparse.urlunparse(("http",) + url[1:])

    if url.scheme in USES_INDEX:
        if path.endswith("/"):
            path = path[:-1]

        split = path.rsplit("/", 1)

        if len(split) > 1:
            path = split[:-1]
            index = split[-1]
        else:
            path = ""
            index = split[0]

        config.update({
            "URL": urlparse.urlunparse(("http",) + url[1:2] + (path,) + url[3:]),
            "INDEX_NAME": index,
        })

    if url.scheme in USES_PATH:
        config.update({
            "PATH": path,
        })

    return config