\
        Main method to extract an article object from a URL,
        pass in a url and get back a Article
186464
def extract(self, url=None, raw_html=None):
        
        cc = CrawlCandidate(self.config, url, raw_html)
        return self.crawl(cc)
337936
function extractProtocolHost(url) {
    const urlNoQuery = url.split('?')[0];
    const [protocol, hostAndPath] = urlNoQuery.split('//');
    const host = hostAndPath.split('/')[0];
    return `${protocol}//${host}`;
}
554772
def get_article(url):
    
    a = html.parse(url).getroot()
    title = a.cssselect(".firstHeading")[0].text_content()
    date = a.cssselect(".published")[0].text_content()
    date = datetime.datetime.strptime(date, "%A, %B %d, %Y").isoformat()
    paras = a.cssselect("#mw-content-text p")
    paras = paras[1:] # skip first paragraph, which contains date
    text = "\n\n".join(p.text_content().strip() for p in paras)

    return dict(headline=title,
                date=date,
                url=url,
                text=text,
                medium="Wikinews")
385780
function parseURL(s) {
  var url = null;
  try {
    url = new URL(s);
  } catch (error) {
    console.error(error);
  }
  return url;
}
480821
function scrapeMetadatum ($, url, rules) {
  if (!Array.isArray(rules)) rules = [rules]

  return rules.reduce(function (promise, rule) {
    return promise.then(function (value) {
      if (value != null && value !== '') return value
      var next = rule($, url)
      if (next != null && next !== '') return next
      return null
    })
  }, Promise.resolve())
}
1322250
def get_url(self, obj, view_name, request, format):
        
        # Unsaved objects will not yet have a valid URL.
        if obj.pk is None:
            return None

        lookup_value = getattr(obj, self.lookup_field)
        lookup_kwargs = {
            self.lookup_field: lookup_value,
        }

        # Try to lookup parent attr
        parent_lookup_value = None
        for attr in self.parent_lookup_field.split("__"):
            if not hasattr(obj, attr):
                break
            parent_lookup_value = getattr(parent_lookup_value or obj, attr)
        if parent_lookup_value:
            lookup_kwargs.update({self.parent_lookup_field: parent_lookup_value})

        return self.reverse(view_name, kwargs=lookup_kwargs, request=request, format=format)
1346440
def get(self, url, parameters=None):
        
        if self._http == None:
            self._setupHttp()
        uri = url + "?" + self.getParameters(parameters)
        response, content = self._http.request(uri, "GET")
        return content
1014892
def extract_urls(url, data, unescape=HTMLParser.HTMLParser().unescape):
    """"""
    parts = urlparse.urlparse(url)
    prefix = '%s://%s' % (parts.scheme, parts.netloc)

    accessed_dir = os.path.dirname(parts.path)
    if not accessed_dir.endswith('/'):
        accessed_dir += '/'

    for pattern, replacement in REPLACEMENT_REGEXES:
        fixed = replacement % {
            'base': prefix,
            'accessed_dir': accessed_dir,
        }
        data = re.sub(pattern, fixed, data)

    result = set()
    for match in re.finditer(MAYBE_HTML_URL_REGEX, data):
        found_url = unescape(match.groupdict()['absurl'])
        found_url = clean_url(
            found_url,
            force_scheme=parts[0])  # Use the main page's scheme
        result.add(found_url)

    return result
771181
def execute_from_url(self, url, input_url):
        '''
        

        '''
        inputs = self._request(input_url)
        resp = self.execute(url, inputs)
        return resp
1025273
def clean_url(url):
    
    parsed = urlparse(url.strip())
    reconstructed = ParseResult(
        parsed.scheme, parsed.netloc, parsed.path,
        params='', query='', fragment='')
    return reconstructed.geturl()
1346324
def get_article_urls(url):
    
    doc = html.parse(url).getroot()
    for div in doc.cssselect("div.mw-search-result-heading"):
        href = div.cssselect("a")[0].get('href')
        if ":" in href:
            continue # skip Category: links
        href = urljoin(url, href)
        yield href
323535
def get(self, url, params=None):
        ''
        url = self.escapeUrl(url)

        content = six.BytesIO(self.raw(url, params=params))
        # We need to make sure that the xml fits in available memory before we parse
        # with lxml.objectify.fromstring(), or else it will bomb out.
        # If it is too big, we need to buffer it to disk before we run it through objectify. see #87
        #
        # get length of buffer
        content.seek(0,2)
        contentlen = content.tell()
        content.seek(0)

        MAX_BUFFER_SIZE=1024*1024*200 # 200MB. TODO: find a way to compute this
        if contentlen > MAX_BUFFER_SIZE:
            # xml is too big to parse with lxml.objectify.fromstring()
            contentfile = tempfile.NamedTemporaryFile()
            contentfile.write(content.read())
            o = lxml.objectify.parse(contentfile)
        else:
            o = lxml.objectify.fromstring(content.getvalue())
        if o.tag == 'error':
            JFSError.raiseError(o, url)
        return o
172969
def get_pages(url):
    
    while True:
        yield url
        doc = html.parse(url).find("body")
        links = [a for a in doc.findall(".//a") if a.text and a.text.startswith("next ")]
        if not links:
            break
        url = urljoin(url, links[0].get('href'))
613275
def _extractPathFromUrl(url):
        
        if url.netloc != '' and url.netloc != 'localhost':
            raise RuntimeError("The URL '%s' is invalid" % url.geturl())
        return url.netloc + url.path
442866
def get_form(self, request, obj=None, **kwargs):
        
        language = get_language_from_request(request)
        form = super(ArticleAdmin, self).get_form(
            request, obj,
            form=(obj and ArticleForm or ArticleCreateForm),
            **kwargs
        )
        # get_form method operates by overriding initial fields value which
        # may persist across invocation. Code below deepcopies fields definition
        # to avoid leaks
        for field in tuple(form.base_fields.keys()):
            form.base_fields[field] = copy.deepcopy(form.base_fields[field])

        if 'language' in form.base_fields:
            form.base_fields['language'].initial = language

        if obj:
            title_obj = obj.get_title_obj(language=language, fallback=False, force_reload=True)

            if hasattr(title_obj, 'id'):
                for name in ('title', 'description', 'page_title', 'menu_title', 'meta_description', 'image'):
                    if name in form.base_fields:
                        form.base_fields[name].initial = getattr(title_obj, name)
                try:
                    slug = self.SLUG_REGEXP.search(title_obj.slug).groups()[settings.CMS_ARTICLES_SLUG_GROUP_INDEX]
                except AttributeError:
                    warnings.warn('Failed to parse slug from CMS_ARTICLES_SLUG_REGEXP. '
                                  'It probably doesn\'t correspond to CMS_ARTICLES_SLUG_FORMAT.')
                    slug = title_obj.slug
                form.base_fields['slug'].initial = slug

        return form
649524
function(url){
    // this creates json object from url with querystring
    if((typeof url) == 'string'){
        var curURL = url;
        var queryStringIndex = curURL.indexOf("?");
        var qsString = curURL.substr(queryStringIndex + 1, curURL.length);
        return queryString.parse(qsString);
    }else if((typeof url)== 'object'){
        return url;
    }else{
        return {};
    }
}
971286
def getObjectFromUrl(url,type=nil)
      method = 'getObjectFromUrl'
      type = BaseObj if not type
      args = {:url=>url}
      return_object type,method,args,true
    end