Takes a url, and returns a dictionary of data with 'bodyLines' lines
487785
def fetch_and_parse(url, bodyLines):
    """"""

    pageHtml = fetch_page(url)
    return parse(url, pageHtml, bodyLines)
650389
def get_request_body_chunk(self, content: bytes, closed: bool,
                               more_content: bool) -> Dict[str, Any]:
        '''
        
        '''
        return {
            'content': content,
            'closed': closed,
            'more_content': more_content
        }
410536
def raw(self):
        
        try:
            return urlopen(str(self.url))
        except HTTPError as error:
            try:
                # parse error body as json and use message property as error message
                parsed = self._parsejson(error)
                exc = RequestError(parsed['message'])
                exc.__cause__ = None
                raise exc
            except ValueError:
                # when error body is not valid json, error might be caused by server
                exc = StatbankError()
                exc.__cause__ = None
                raise exc
798499
def get_urlpatterns(self):
        
        return [
            path(
                _('topic/<str:slug>-<int:pk>/lock/'),
                self.topic_lock_view.as_view(),
                name='topic_lock',
            ),
            path(
                _('topic/<str:slug>-<int:pk>/unlock/'),
                self.topic_unlock_view.as_view(),
                name='topic_unlock',
            ),
            path(
                _('topic/<str:slug>-<int:pk>/delete/'),
                self.topic_delete_view.as_view(),
                name='topic_delete',
            ),
            path(
                _('topic/<str:slug>-<int:pk>/move/'),
                self.topic_move_view.as_view(),
                name='topic_move',
            ),
            path(
                _('topic/<str:slug>-<int:pk>/change/topic/'),
                self.topic_update_to_normal_topic_view.as_view(),
                name='topic_update_to_post',
            ),
            path(
                _('topic/<str:slug>-<int:pk>/change/sticky/'),
                self.topic_update_to_sticky_topic_view.as_view(),
                name='topic_update_to_sticky',
            ),
            path(
                _('topic/<str:slug>-<int:pk>/change/announce/'),
                self.topic_update_to_announce_view.as_view(),
                name='topic_update_to_announce',
            ),
            path(_('queue/'), self.moderation_queue_list_view.as_view(), name='queue'),
            path(
                _('queue/<int:pk>/'),
                self.moderation_queue_detail_view.as_view(),
                name='queued_post',
            ),
            path(
                _('queue/<int:pk>/approve/'),
                self.post_approve_view.as_view(),
                name='approve_queued_post',
            ),
            path(
                _('queue/<int:pk>/disapprove/'),
                self.post_disapprove_view.as_view(),
                name='disapprove_queued_post',
            ),
        ]
1211658
def extractor(url):
    """"""
    response = requester(url, main_url, delay, cook, headers, timeout, host, proxies, user_agents, failed, processed)
    if clone:
        mirror(url, response)
    matches = rhref.findall(response)
    for link in matches:
        # Remove everything after a "#" to deal with in-page anchors
        link = link[1].replace('\'', '').replace('"', '').split('#')[0]
        # Checks if the URLs should be crawled
        if is_link(link, processed, files):
            if link[:4] == 'http':
                if link.startswith(main_url):
                    verb('Internal page', link)
                    internal.add(link)
                else:
                    verb('External page', link)
                    external.add(link)
            elif link[:2] == '//':
                if link.split('/')[2].startswith(host):
                    verb('Internal page', link)
                    internal.add(schema + '://' + link)
                else:
                    verb('External page', link)
                    external.add(link)
            elif link[:1] == '/':
                verb('Internal page', link)
                internal.add(remove_file(url) + link)
            else:
                verb('Internal page', link)
                usable_url = remove_file(url)
                if usable_url.endswith('/'):
                    internal.add(usable_url + link)
                elif link.startswith('/'):
                    internal.add(usable_url + link)
                else:
                    internal.add(usable_url + '/' + link)

    if not only_urls:
        intel_extractor(url, response)
        js_extractor(response)
    if args.regex and not supress_regex:
        regxy(args.regex, response, supress_regex, custom)
    if api:
        matches = rentropy.findall(response)
        for match in matches:
            if entropy(match) >= 4:
                verb('Key', match)
                keys.add(url + ': ' + match)
298614
function getUrls(urls) {
    var options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
    var settings = {
        mimeType: options.mimeType || null,
        onBeforeSend: options.onBeforeSend || Function.prototype,
        onSuccess: options.onSuccess || Function.prototype,
        onError: options.onError || Function.prototype,
        onComplete: options.onComplete || Function.prototype
    };
    var urlArray = Array.isArray(urls) ? urls : [ urls ];
    var urlQueue = Array.apply(null, Array(urlArray.length)).map(function(x) {
        return null;
    });
    function isValidCss() {
        var cssText = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : "";
        var isHTML = cssText.trim().charAt(0) === "<";
        return !isHTML;
    }
    function onError(xhr, urlIndex) {
        settings.onError(xhr, urlArray[urlIndex], urlIndex);
    }
    function onSuccess(responseText, urlIndex) {
        var returnVal = settings.onSuccess(responseText, urlArray[urlIndex], urlIndex);
        responseText = returnVal === false ? "" : returnVal || responseText;
        urlQueue[urlIndex] = responseText;
        if (urlQueue.indexOf(null) === -1) {
            settings.onComplete(urlQueue);
        }
    }
    var parser = document.createElement("a");
    urlArray.forEach(function(url, i) {
        parser.setAttribute("href", url);
        parser.href = String(parser.href);
        var isIElte9 = Boolean(document.all && !window.atob);
        var isIElte9CORS = isIElte9 && parser.host.split(":")[0] !== location.host.split(":")[0];
        if (isIElte9CORS) {
            var isSameProtocol = parser.protocol === location.protocol;
            if (isSameProtocol) {
                var xdr = new XDomainRequest();
                xdr.open("GET", url);
                xdr.timeout = 0;
                xdr.onprogress = Function.prototype;
                xdr.ontimeout = Function.prototype;
                xdr.onload = function() {
                    if (isValidCss(xdr.responseText)) {
                        onSuccess(xdr.responseText, i);
                    } else {
                        onError(xdr, i);
                    }
                };
                xdr.onerror = function(err) {
                    onError(xdr, i);
                };
                setTimeout(function() {
                    xdr.send();
                }, 0);
            } else {
                console.warn("Internet Explorer 9 Cross-Origin (CORS) requests must use the same protocol (".concat(url, ")"));
                onError(null, i);
            }
        } else {
            var xhr = new XMLHttpRequest();
            xhr.open("GET", url);
            if (settings.mimeType && xhr.overrideMimeType) {
                xhr.overrideMimeType(settings.mimeType);
            }
            settings.onBeforeSend(xhr, url, i);
            xhr.onreadystatechange = function() {
                if (xhr.readyState === 4) {
                    if (xhr.status === 200 && isValidCss(xhr.responseText)) {
                        onSuccess(xhr.responseText, i);
                    } else {
                        onError(xhr, i);
                    }
                }
            };
            xhr.send();
        }
    });
}
131896
def get_headers(params, boundary):
    """"""
    headers = {}
    boundary = urllib.quote_plus(boundary)
    headers['Content-Type'] = "multipart/form-data; boundary=%s" % boundary
    headers['Content-Length'] = str(get_body_size(params, boundary))
    return headers
539673
def _json_body_(cls):
        
        json = []
        for series_name, data in six.iteritems(cls._datapoints):
            json.append({'name': series_name,
                         'columns': cls._fields,
                         'points': [[getattr(point, k) for k in cls._fields]
                                    for point in data]
                         })
        return json
559292
def urls(self):
        
        url_bases = self._url_module.url_bases
        unformatted_paths = self._url_module.url_paths

        urls = {}
        for url_base in url_bases:
            for url_path, handler in unformatted_paths.items():
                url = url_path.format(url_base)
                urls[url] = handler

        return urls
447811
def parse_url_rules(urls_fp):
    """"""
    url_rules = []
    for line in urls_fp:
        re_url = line.strip()
        if re_url:
            url_rules.append({'str': re_url, 're': re.compile(re_url)})
    return url_rules
1274242
def get_urlpatterns(self):
        
        urlpatterns = super().get_urlpatterns()

        conversation_urlpatterns = [
            path(
                _('topic/<str:slug>-<int:pk>/'),
                self.topic_view.as_view(),
                name='topic',
            ),
            path(_('topic/create/'), self.topic_create_view.as_view(), name='topic_create'),
            path(
                _('topic/<str:slug>-<int:pk>/update/'),
                self.topic_update_view.as_view(),
                name='topic_update',
            ),
            path(
                _('topic/<str:topic_slug>-<int:topic_pk>/post/create/'),
                self.post_create_view.as_view(),
                name='post_create',
            ),
            path(
                _('topic/<str:topic_slug>-<int:topic_pk>/<int:pk>/post/update/'),
                self.post_update_view.as_view(),
                name='post_update',
            ),
            path(
                _('topic/<str:topic_slug>-<int:topic_pk>/<int:pk>/post/delete/'),
                self.post_delete_view.as_view(),
                name='post_delete',
            ),
        ]

        urlpatterns += [
            path(
                _('forum/<str:forum_slug>-<int:forum_pk>/'),
                include(conversation_urlpatterns),
            ),
        ]

        return urlpatterns
1123584
function bodyToParams(body) {
  var params = {};

  body = body.trim();
  body.split('\n').forEach(function(line) {
    var match = line.trim().match(/^(\w+)=(.*)$/);
    if (match) {
      params[match[1]] = match[2];
    }
  });
  return params;
}
339442
def parse_text (url_data):
    """"""
    lineno = 0
    for line in url_data.get_content().splitlines():
        lineno += 1
        line = line.strip()
        if not line or line.startswith('#'):
            continue
        url_data.add_url(line, line=lineno)
603046
def __getDataFromURL(url):
        
        code = 0

        while code != 200:
            req = Request(url)
            try:
                response = urlopen(req)
                code = response.code
                sleep(0.01)
            except HTTPError as error:
                code = error.code
                if code == 404:
                    break
            except URLError as error:
                sleep(3)

        if code == 404:
            raise Exception("User was not found")
        return response.read().decode('utf-8')
392715
function(line) {
		return {
			'name': String(line[1]),
			'country_code': String(line[0]),
			'gmt_offset': Number(line[2]),
			'dst_offset': Number(line[3]),
			'raw_offset': Number(line[4])
		};
	}
819235
def from_urlencode(self, data, options=None):
        
        qs = dict((k, v if len(v) > 1 else v[0])
                  for k, v in urlparse.parse_qs(data).iteritems())
        return qs
11484
def _extract_line_features(self):
        
        for line in self.line_iterator:
            m = re.search(GENERIC_IPV4_REGEX, line)

            data = {
                'raw': line
            }

            if m:
                data['ip'] = m.group('ip')

            self.parsed_lines.append(data)