Find entry link and download entry
968047
def download_entry(self, entry):
        
        downloadlinks = {}
        downloaded = False
        ignoreenclosures = self.retrieve_config('ignoreenclosures', 'no')
        notype = self.retrieve_config('notype', 'no')
        if ignoreenclosures == 'no':
            for enclosure in entry.enclosures:
                if notype == 'yes':
                    downloadlinks[urlparse(enclosure["href"]).path.split(
                        "/")[-1]] = enclosure["href"]
                    # preserve original name
                else:
                    try:
                        # We will download all enclosures of the desired
                        # mime-type
                        if any([mimetype in enclosure["type"] for mimetype in
                                self.mime]):
                            downloadlinks[urlparse(
                                enclosure["href"]).path.split(
                                    "/")[-1]] = enclosure["href"]
                            # preserve original name
                    except KeyError:
                        print("This podcast carries no information about "
                              "enclosure types. Try using the notype "
                              "option in your greg.conf", file=sys.stderr,
                              flush=True)
        else:
            downloadlinks[urlparse(entry.link).query.split(
                "/")[-1]] = entry.link
        for podname in downloadlinks:
            if (podname, entry.linkdate) not in zip(self.entrylinks,
                                                    self.linkdates):
                try:
                    title = entry.title
                except:
                    title = podname
                try:
                    sanitizedsummary = aux.html_to_text(entry.summary)
                    if sanitizedsummary == "":
                        sanitizedsummary = "No summary available"
                except:
                    sanitizedsummary = "No summary available"
                try:
                    placeholders = Placeholders(
                        self, entry, downloadlinks[podname], podname, title,
                        sanitizedsummary)
                    placeholders = aux.check_directory(placeholders)
                    condition = aux.filtercond(placeholders)
                    if condition:
                        print("Downloading {} -- {}".format(title, podname))
                        aux.download_handler(self, placeholders)
                        if self.willtag:
                            aux.tag(placeholders)
                        downloaded = True
                    else:
                        print("Skipping {} -- {}".format(title, podname))
                        downloaded = False
                    if self.info:
                        with open(self.info, 'a') as current:
                            # We write to file this often to ensure that
                            # downloaded entries count as downloaded.
                            current.write(''.join([podname, ' ',
                                          str(entry.linkdate), '\n']))
                except URLError:
                    sys.exit(("... something went wrong. "
                             "Are you connected to the internet?"))
        return downloaded
138798
def _ExtractHuntIdFromPath(entry, event):
  """"""
  match = re.match(r".*hunt/([^/]+).*", entry.http_request_path)
  if match:
    event.urn = "aff4:/hunts/{}".format(match.group(1))
126650
def get_entry(self, key, column=None, table=None):
        """"""
        if table is None:  table  = self.main_table
        if column is None: column = "id"
        if isinstance(key, basestring): key = key.replace("'","''")
        query = 'SELECT * from "%s" where "%s"=="%s" LIMIT 1;'
        query = query % (table, column, key)
        self.own_cursor.execute(query)
        return self.own_cursor.fetchone()
991904
def fetch_url(url, dest, parent_to_remove_before_fetch):
    """"""
    logger.debug('Downloading file {} from {}', dest, url)
    try:
        shutil.rmtree(parent_to_remove_before_fetch)
    except FileNotFoundError:
        pass
    os.makedirs(parent_to_remove_before_fetch)
    # TODO(itamar): Better downloading (multi-process-multi-threaded?)
    # Consider offloading this to a "standalone app" invoked with Docker
    resp = requests.get(url, stream=True)
    with open(dest, 'wb') as fetch_file:
        for chunk in resp.iter_content(chunk_size=32 * 1024):
            fetch_file.write(chunk)
1008508
def get_entry_text(self, fname):
        """"""
        if fname.split('.')[-1] == 'gz':
            with gz.open(fname, 'rt') as f:
                filetext = f.read()
        else:
            with codecs.open(fname, 'r') as f:
                filetext = f.read()
        return filetext
20522
def download_from_link(self, link, **kwargs):
        
        # old:new format
        old_arg_map = {'save_path': 'savepath'}  # , 'label': 'category'}

        # convert old option names to new option names
        options = kwargs.copy()
        for old_arg, new_arg in old_arg_map.items():
            if options.get(old_arg) and not options.get(new_arg):
                options[new_arg] = options[old_arg]

        if type(link) is list:
            options['urls'] = "\n".join(link)
        else:
            options['urls'] = link

        # workaround to send multipart/formdata request
        # http://stackoverflow.com/a/23131823/4726598
        dummy_file = {'_dummy': (None, '_dummy')}

        return self._post('command/download', data=options, files=dummy_file)
697733
def entry_links(self):
        
        return {entry['link'] for entry in self.feed.entries if entry and entry.get('link')}
499435
def download_page(url, data=None):
    ''''''
    conn = urllib2.urlopen(url, data)
    resp = conn.read()
    conn.close()
    return resp
929363
def _get_source(link):
    
    if link.startswith("http://") or link.startswith("https://"):
        down = httpkie.Downloader()
        return down.download(link)

    if os.path.exists(link):
        with open(link) as f:
            return f.read()

    raise UserWarning("html: '%s' is neither URL or data!" % link)
980814
def download_handler(feed, placeholders):
    import shlex
    
    value = feed.retrieve_config('downloadhandler', 'greg')
    if value == 'greg':
        while os.path.isfile(placeholders.fullpath):
            placeholders.fullpath = placeholders.fullpath + '_'
            placeholders.filename = placeholders.filename + '_'
        urlretrieve(placeholders.link, placeholders.fullpath)
    else:
        value_list = shlex.split(value)
        instruction_list = [substitute_placeholders(part, placeholders) for
                            part in value_list]
        returncode = subprocess.call(instruction_list)
        if returncode:
            raise URLError
735023
def get_contents_entry(node):
    """"""
    try:
        node = node.disambiguate(must_exist=1)
    except SCons.Errors.UserError:
        # There was nothing on disk with which to disambiguate
        # this entry.  Leave it as an Entry, but return a null
        # string so calls to get_contents() in emitters and the
        # like (e.g. in qt.py) don't have to disambiguate by hand
        # or catch the exception.
        return ''
    else:
        return _get_contents_map[node._func_get_contents](node)
639662
def selective_download(name, oldest, newest):
    """"""
    if six.PY3:
        name = name.encode("utf-8")
    feed = resolve_name(name)
    if six.PY3:
        feed = feed.decode()
    d = feedparser.parse(feed)
    logger.debug(d)
    try:
        d.entries[int(oldest)]
    except IndexError:
        print("Error feed does not contain this many items.")
        print("Hitman thinks there are %d items in this feed." % len(d.entries))
        return
    for url in [q.enclosures[0]['href'] for q in d.entries[int(newest):int(oldest)]]:
        # iterate over urls in feed from newest to oldest feed items.
        url = str(url)
        with Database("downloads") as db:
            if url.split('/')[-1] not in db:
                # download(url, name, feed)
                with Database("settings") as settings:
                    if 'dl' in settings:
                        dl_dir = settings['dl']
                    else:
                        dl_dir = os.path.join(os.path.expanduser("~"), "Downloads")
                requests_get(url, dl_dir)
251495
def get_config_entry(config, url):
    
    if not config:
        return None

    # First, try to search for the best match
    if url in config:
        return config[url]

    # Then search for indirect match
    for item in config:
        if without_http_prefix(item) == without_http_prefix(url):
            return config[item]
475081
function downloadFromClick(page, ph, selector, callback) {
    const HREF_KEY = 'href';
    async.waterfall([
        function (callback) {
            getSelectorAttribute(page, ph, [selector, HREF_KEY], callback);
        },
        function (page, ph, relativeHref, callback) {
            getCurrentUrl(page, ph, function (error, page, ph, currentUrl) {
                if (error) {
                    callback(error, page, ph);
                    return;
                }
                callback(null, page, ph, currentUrl + relativeHref);
            });
        },
        function (page, ph, url, callback) {
            downloadFromUrl(page, ph, url, callback);
        }
    ], callback);
}
1211217
def maybe_download(url, work_directory):
    """"""
    filename = url.split("/")[-1]
    filepath = os.path.join(work_directory, filename)
    if not os.path.exists(filepath):
        logger.info("Downloading to {}...".format(filepath))
        download(url, work_directory)
    return filepath
460221
def get_short_url(self, entry):
        
        try:
            short_url = entry.short_url
        except NoReverseMatch:
            short_url = entry.get_absolute_url()
        return format_html('<a href="{url}" target="blank">{url}</a>',
                           url=short_url)
184637
function getEntryImports(dir, content, arr = []) {
	$C(content.split(eol)).forEach((line) => {
		if (!hasImport.test(line)) {
			return;
		}

		const
			url = RegExp.$2,
			nodeModule = isNodeModule(url);

		if (nodeModule && entriesDir.test(url) || insideEntry.test(url)) {
			const
				d = nodeModule ? lib : dir;

			let
				f = path.join(d, `${url}.js`);

			if (!fs.existsSync(f)) {
				f = path.join(d, url, 'index.js');
			}

			getEntryImports(path.dirname(f), fs.readFileSync(f, 'utf-8'), arr);

		} else {
			arr.push(nodeModule ? url : path.join(dir, url));
		}
	});

	return arr;
}