Download and parse an individual pricing file from AWS

    .. versionadded:: 2015.8.0
963835
def _parse_pricing(url, name):
    '''
    
    '''
    price_js = http.query(url, text=True)

    items = []
    current_item = ''

    price_js = re.sub(JS_COMMENT_RE, '', price_js['text'])
    price_js = price_js.strip().rstrip(');').lstrip('callback(')
    for keyword in (
        'vers',
        'config',
        'rate',
        'valueColumns',
        'currencies',
        'instanceTypes',
        'type',
        'ECU',
        'storageGB',
        'name',
        'vCPU',
        'memoryGiB',
        'storageGiB',
        'USD',
    ):
        price_js = price_js.replace(keyword, '"{0}"'.format(keyword))

    for keyword in ('region', 'price', 'size'):
        price_js = price_js.replace(keyword, '"{0}"'.format(keyword))
        price_js = price_js.replace('"{0}"s'.format(keyword), '"{0}s"'.format(keyword))

    price_js = price_js.replace('""', '"')

    # Turn the data into something that's easier/faster to process
    regions = {}
    price_json = salt.utils.json.loads(price_js)
    for region in price_json['config']['regions']:
        sizes = {}
        for itype in region['instanceTypes']:
            for size in itype['sizes']:
                sizes[size['size']] = size
        regions[region['region']] = sizes

    outfile = os.path.join(
        __opts__['cachedir'], 'ec2-pricing-{0}.p'.format(name)
    )
    with salt.utils.files.fopen(outfile, 'w') as fho:
        salt.utils.msgpack.dump(regions, fho)

    return True
2865
def parse(self):
        
        try:
            if not os.path.getsize(self.ns.pathname):
                # Ignore 0-byte dummy files (Firefox creates these while downloading)
                self.job.LOG.warn("Ignoring 0-byte metafile '%s'" % (self.ns.pathname,))
                return
            self.metadata = metafile.checked_open(self.ns.pathname)
        except EnvironmentError as exc:
            self.job.LOG.error("Can't read metafile '%s' (%s)" % (
                self.ns.pathname, str(exc).replace(": '%s'" % self.ns.pathname, ""),
            ))
            return
        except ValueError as exc:
            self.job.LOG.error("Invalid metafile '%s': %s" % (self.ns.pathname, exc))
            return

        self.ns.info_hash = metafile.info_hash(self.metadata)
        self.ns.info_name = self.metadata["info"]["name"]
        self.job.LOG.info("Loaded '%s' from metafile '%s'" % (self.ns.info_name, self.ns.pathname))

        # Check whether item is already loaded
        try:
            name = self.job.proxy.d.name(self.ns.info_hash, fail_silently=True)
        except xmlrpc.HashNotFound:
            pass
        except xmlrpc.ERRORS as exc:
            if exc.faultString != "Could not find info-hash.":
                self.job.LOG.error("While checking for #%s: %s" % (self.ns.info_hash, exc))
                return
        else:
            self.job.LOG.warn("Item #%s '%s' already added to client" % (self.ns.info_hash, name))
            return

        return True
848121
def acquire(self, **kwargs):
        
        return config.download_data(self.temp_filename, self.url,
                                    self.sha256)
724436
def do_get(self, from_path, to_path):
        
        to_file = open(os.path.expanduser(to_path), "wb")

        self.n.downloadFile(self.current_path + "/" + from_path, to_path)
131275
def web(ticker, field=None, start=None, end=None,
        mrefresh=False, source='yahoo'):
    
    if source == 'yahoo' and field is None:
        field = 'Adj Close'

    tmp = _download_web(ticker, data_source=source,
                        start=start, end=end)

    if tmp is None:
        raise ValueError('failed to retrieve data for %s:%s' % (ticker, field))

    if field:
        return tmp[field]
    else:
        return tmp
448004
def download_from_url(src_url, dst_dir):
    
    dst_path = ""
    if "MD982181" not in src_url:
        dsn = input("Please enter the dataset name for this file (Name.Location.Year) : ")
        dst_path = os.path.join(dst_dir, dsn + ".lpd")
        try:
            print("downloading file from url...")
            urllib.request.urlretrieve(src_url, dst_path)
        except Exception as e:
            print("Error: unable to download from url: {}".format(e))
    else:
        print("Error: That file cannot be download due to server restrictions")
    return dst_path
364478
def download_file(url, path, session=None, params=None):
    

    if url[0:2] == '//':
        url = 'https://' + url[2:]

    # Use a temporary file so that we don't import incomplete files.
    tmp_path = path + '.tmp'

    if session and params:
        r = session.get( url, params=params, stream=True )
    elif session and not params:
        r = session.get( url, stream=True )
    else:
        r = requests.get(url, stream=True)
    with open(tmp_path, 'wb') as f:
        total_length = int(r.headers.get('content-length', 0))
        for chunk in progress.bar(r.iter_content(chunk_size=1024), expected_size=(total_length / 1024) + 1):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
                f.flush()

    os.rename(tmp_path, path)

    return path
773832
public void downloadFileUsingRequesterPays(
      String projectId, String bucketName, String srcFilename, Path destFilePath)
      throws IOException {
    // [START storage_download_file_requester_pays]
    // The project ID to bill
    // String projectId = "my-billable-project-id";

    // The name of the bucket to access
    // String bucketName = "my-bucket";

    // The name of the remote file to download
    // String srcFilename = "file.txt";

    // The path to which the file should be downloaded
    // Path destFilePath = Paths.get("/local/path/to/file.txt");

    // Instantiate a Google Cloud Storage client
    Storage storage = StorageOptions.getDefaultInstance().getService();

    // Get specific file from specified bucket
    Blob blob = storage.get(BlobId.of(bucketName, srcFilename));

    // Download file to specified path
    blob.downloadTo(destFilePath, Blob.BlobSourceOption.userProject(projectId));
    // [END storage_download_file_requester_pays]
  }
797075
def _parse_price(html_chunk):
    
    price = get_first_content(
        html_chunk.find("div", {"class": "prices"})
    )

    if not price:
        return None

    # it is always in format Cena:\n150kč
    price = dhtmlparser.removeTags(price)
    price = price.split("\n")[-1]

    return price
397156
def _download_file(uri, bulk_api):
    """"""
    resp = requests.get(uri, headers=bulk_api.headers(), stream=True)
    with tempfile.TemporaryFile("w+b") as f:
        for chunk in resp.iter_content(chunk_size=None):
            f.write(chunk)
        f.seek(0)
        yield f
916688
def from_file(filename, serverEndpoint=ServerEndpoint):
    '''
    
    '''
    tarOutput = parse1('unpack', filename, serverEndpoint,
                       responseMimeType='application/x-tar',
                       services={'meta': '/meta', 'text': '/tika',
                                 'all': '/rmeta/xml', 'unpack': '/unpack/all'},
                       rawResponse=True)
    return _parse(tarOutput)
1037426
def get_requests(self):
        
        self.download_list = [DownloadRequest(url=self.get_url(metafile), filename=self.get_filepath(metafile),
                                              data_type=AwsConstants.AWS_FILES[metafile], data_name=metafile) for
                              metafile in self.metafiles if metafile in AwsConstants.PRODUCT_FILES]

        tile_parent_folder = os.path.join(self.parent_folder, self.product_id)
        for tile_info in self.product_info['tiles']:
            tile_name, date, aws_index = self.url_to_tile(self.get_tile_url(tile_info))
            if self.tile_list is None or AwsTile.parse_tile_name(tile_name) in self.tile_list:
                tile_downloads, tile_folders = AwsTile(tile_name, date, aws_index, parent_folder=tile_parent_folder,
                                                       bands=self.bands, metafiles=self.metafiles,
                                                       data_source=self.data_source).get_requests()
                self.download_list.extend(tile_downloads)
                self.folder_list.extend(tile_folders)
        self.sort_download_list()
        return self.download_list, self.folder_list
169169
def read_pkg_file(self, file):
    """"""
    msg = message_from_file(file)

    def _read_field(name):
        value = msg[name]
        if value == 'UNKNOWN':
            return None
        return value

    def _read_list(name):
        values = msg.get_all(name, None)
        if values == []:
            return None
        return values

    self.metadata_version = StrictVersion(msg['metadata-version'])
    self.name = _read_field('name')
    self.version = _read_field('version')
    self.description = _read_field('summary')
    # we are filling author only.
    self.author = _read_field('author')
    self.maintainer = None
    self.author_email = _read_field('author-email')
    self.maintainer_email = None
    self.url = _read_field('home-page')
    self.license = _read_field('license')

    if 'download-url' in msg:
        self.download_url = _read_field('download-url')
    else:
        self.download_url = None

    self.long_description = _read_field('description')
    self.description = _read_field('summary')

    if 'keywords' in msg:
        self.keywords = _read_field('keywords').split(',')

    self.platforms = _read_list('platform')
    self.classifiers = _read_list('classifier')

    # PEP 314 - these fields only exist in 1.1
    if self.metadata_version == StrictVersion('1.1'):
        self.requires = _read_list('requires')
        self.provides = _read_list('provides')
        self.obsoletes = _read_list('obsoletes')
    else:
        self.requires = None
        self.provides = None
        self.obsoletes = None
427082
def _download_vswhere():
    
    print('downloading from', _get_latest_release_url())
    try:
        from urllib.request import urlopen
        with urlopen(_get_latest_release_url()) as response, open(DOWNLOAD_PATH, 'wb') as outfile:
            shutil.copyfileobj(response, outfile)
    except ImportError:
        # Python 2
        import urllib
        urllib.urlretrieve(_get_latest_release_url(), DOWNLOAD_PATH)
1002809
def _parse_file():
    
    file_name = path.join(
        path.abspath(path.dirname(path.dirname(__file__))), 'config.json'
    )
    logger.info('loading configuration from file: %r', file_name)
    try:
        data = _read_file(file_name)
    except FileNotFoundError:
        logger.error('no configuration available, set FLASH_CONFIG or '
                     'provide config.json')
        exit()
    for service in data.get('services', []):
        for key, val in service.items():
            if isinstance(val, str) and re.match(r'^\$[A-Z_]+$', val):
                env_val = getenv(val[1:])
                if env_val is None:
                    logger.warning('environment variable %r not found', val[1:])
                service[key] = env_val or val
    return data
255937
def download_source(version):
        
        local_filename = 'v{}.zip'.format(Spark.svm_version_path(version))
        Spark.download(Spark.spark_versions()['v{}'.format(version)], local_filename)
424677
def parse_files(self, req, name, field):
        """"""
        return core.get_value(req.FILES, name, field)