Return a list of the urls to download from
333094
def getDownloadUrls(self):
		""""""
		data = self.searchIndex(False)
		fileUrls = []
		for datum in data:
			fileUrl = self.formatDownloadUrl(datum[0])
			fileUrls.append(fileUrl)
		return fileUrls
69686
async def get_links_from_url(url):
    
    response = await httpclient.AsyncHTTPClient().fetch(url)
    print("fetched %s" % url)

    html = response.body.decode(errors="ignore")
    return [urljoin(url, remove_fragment(new_url)) for new_url in get_links(html)]
1108388
def download(self, sub_url):
        """"""
        response = requests.get(sub_url, headers=self.headers).text
        soup = BS(response, 'lxml')
        downlink = self.base_url+soup.select('.download a')[0]['href']
        data = requests.get(downlink, headers=self.headers)
        z = zipfile.ZipFile(cStringIO.StringIO(data.content))
        srt_files = [f.filename for f in z.filelist
                     if f.filename.rsplit('.')[-1].lower() in ['srt', 'ass']]
        z.extract(srt_files[0], '/tmp/')
        return srt_files[0]
75485
def download_links(self, dir_path):
    
    links = self.links
    if not path.exists(dir_path):
      makedirs(dir_path)
    for i, url in enumerate(links):
      if 'start' in self.cseargs:
        i += int(self.cseargs['start'])
      ext = self.cseargs['fileType']
      ext = '.html' if ext == '' else '.' + ext
      file_name = self.cseargs['q'].replace(' ', '_') + '_' + str(i) + ext
      file_path = path.join(dir_path, file_name)
      r = requests.get(url, stream=True)
      if r.status_code == 200:
        with open(file_path, 'wb') as f:
          r.raw.decode_content = True
          shutil.copyfileobj(r.raw, f)
466495
function getUrls (opts) {

    var list    = [];

    var bsLocal = require("url").parse(opts.urls.local);

    list.push([bsLocal.protocol + "//", bsLocal.hostname, ":", opts.port].join(""));

    if (opts.urls.external) {
        var external = require("url").parse(opts.urls.external);
        list.push([bsLocal.protocol + "//", external.hostname, ":", opts.port].join(""));
    }

    return Immutable.List(list);
}
665039
def get_urls(self):
        
        urls = []

        for action in self.get_extra_actions():
            regex = r'^{}/$'.format(self._get_action_href(action))
            view = self.admin_site.admin_view(action)
            urls.append(url(regex, view))

        return urls + super(ExtraActionsMixin, self).get_urls()
1003906
def download(url):
    """"""
    session = requests.Session()
    session.mount('file://', FileAdapter())

    try:
        res = session.get(url)
    except requests.exceptions.ConnectionError as e:
        raise e

    res.raise_for_status()

    return res
18631
def _download(url):
    
    fh = StringIO()

    for line in get(url):
        fh.write(line)

    fh.seek(0)
    return fh
399472
def fetchFiles(self):
        """"""
        print("Fetching files from {}".format(self.baseDir))
        files = glob(path.join(self.baseDir, '*'))
        filesDef = []
        for f in files:
            print("Found SPARQL file {}".format(f))
            relative = f.replace(self.baseDir, '')
            filesDef.append({
                'download_url': relative,
                'name': relative
            })
        return filesDef
628313
def _download_args(options):
    """"""
    return dict(
        version=options.version,
        download_base=options.download_base,
        downloader_factory=options.downloader_factory,
        to_dir=options.to_dir,
    )
338047
def _retrieve_resources(self, url, class_, full):
        ''''''
        objects_to_return = []
        response = self.session.get(url)
        if response.status_code == 200:
            result = response.json()
            resources = result['results']
            objects_to_return.extend([class_(session=self.session, **resource)
                                      for resource in resources])
            while full and result['next'] is not None:
                response = self.session.get(result['next'])
                if response.status_code == 200:
                    result = response.json()
                    resources = result['results']
                    objects_to_return.extend([class_(session=self.session,
                                                     **resource)
                                              for resource in resources])
                else:
                    raise RuntimeError("Failed downloading data with status {}"
                            ". The response was: '{}'"
                            .format(response.status_code, response.text))
            return objects_to_return
        else:
            raise RuntimeError("Failed downloading data with status {}"
                    ". The response was: '{}'"
                    .format(response.status_code, response.text))
622524
def prepend_urls(self):
        
        return [
            url(r"^(?P<resource_name>%s)/(?P<pk>\w[\w/-]*)/generate%s$" %
                (self._meta.resource_name, trailing_slash()),
                self.wrap_view('generate'), name="api_tileset_generate"),
            url(r"^(?P<resource_name>%s)/(?P<pk>\w[\w/-]*)/download%s$" %
                (self._meta.resource_name, trailing_slash()),
                self.wrap_view('download'), name="api_tileset_download"),
            url(r"^(?P<resource_name>%s)/(?P<pk>\w[\w/-]*)/status%s$" %
                (self._meta.resource_name, trailing_slash()),
                self.wrap_view('status'), name="api_tileset_status"),
            url(r"^(?P<resource_name>%s)/(?P<pk>\w[\w/-]*)/stop%s$" %
                (self._meta.resource_name, trailing_slash()),
                self.wrap_view('stop'), name="api_tileset_stop"),
        ]
1283406
def download_images(urls:Collection[str], dest:PathOrStr, max_pics:int=1000, max_workers:int=8, timeout=4):
    ""
    urls = open(urls).read().strip().split("\n")[:max_pics]
    dest = Path(dest)
    dest.mkdir(exist_ok=True)
    parallel(partial(_download_image_inner, dest, timeout=timeout), urls, max_workers=max_workers)
263349
def get_urls(self):
        
        from django.conf.urls import url
        urls = super(AdminModeratorMixin, self).get_urls()
        info = self.model._meta.app_label, self.model._meta.model_name
        return [
            url(r'^(.+)/moderate/$',
                self.admin_site.admin_view(self.moderate_view),
                name='%s_%s_moderate' % info),
        ] + urls
138722
def get_requests(self):
        
        self.download_list = []
        for data_name in [band for band in self.bands if self._band_exists(band)] + self.metafiles:
            if data_name in AwsConstants.TILE_FILES:
                url = self.get_url(data_name)
                filename = self.get_filepath(data_name)
                self.download_list.append(DownloadRequest(url=url, filename=filename,
                                                          data_type=AwsConstants.AWS_FILES[data_name],
                                                          data_name=data_name))
        self.sort_download_list()
        return self.download_list, self.folder_list
478207
def download(url, encoding='utf-8'):
    """"""
    import requests
    response = requests.get(url)
    response.encoding = encoding
    return response.text
21065
def download_urls(cls, urls, force_download=False):
        
        for url in urls:
            file_path = cls.get_path_to_file_from_url(url)

            if os.path.exists(file_path) and not force_download:
                log.info('already downloaded %s to %s', url, file_path)
            else:
                log.info('downloading %s to %s', url, file_path)
                download_timer = time.time()
                urlretrieve(url, file_path)
                log.info('downloaded in %.2f seconds', time.time() - download_timer)