Get a file from hdfs

    :param hdfs_path: Destination (str)
    :param local_path: Source (str)
    :raises: IOError: If unsuccessful
432969
def get(hdfs_path, local_path):
    
    cmd = "hadoop fs -get %s %s" % (hdfs_path, local_path)
    rcode, stdout, stderr = _checked_hadoop_fs_command(cmd)
774495
def get_file(self,
                 path,
                 dest='',
                 makedirs=False,
                 saltenv='base',
                 gzip=None,
                 cachedir=None):
        '''
        
        '''
        path, senv = salt.utils.url.split_env(path)
        if senv:
            saltenv = senv

        if not salt.utils.platform.is_windows():
            hash_server, stat_server = self.hash_and_stat_file(path, saltenv)
            try:
                mode_server = stat_server[0]
            except (IndexError, TypeError):
                mode_server = None
        else:
            hash_server = self.hash_file(path, saltenv)
            mode_server = None

        # Check if file exists on server, before creating files and
        # directories
        if hash_server == '':
            log.debug(
                'Could not find file \'%s\' in saltenv \'%s\'',
                path, saltenv
            )
            return False

        # If dest is a directory, rewrite dest with filename
        if dest is not None \
                and (os.path.isdir(dest) or dest.endswith(('/', '\\'))):
            dest = os.path.join(dest, os.path.basename(path))
            log.debug(
                'In saltenv \'%s\', \'%s\' is a directory. Changing dest to '
                '\'%s\'', saltenv, os.path.dirname(dest), dest
            )

        # Hash compare local copy with master and skip download
        # if no difference found.
        dest2check = dest
        if not dest2check:
            rel_path = self._check_proto(path)

            log.debug(
                'In saltenv \'%s\', looking at rel_path \'%s\' to resolve '
                '\'%s\'', saltenv, rel_path, path
            )
            with self._cache_loc(
                    rel_path, saltenv, cachedir=cachedir) as cache_dest:
                dest2check = cache_dest

        log.debug(
            'In saltenv \'%s\', ** considering ** path \'%s\' to resolve '
            '\'%s\'', saltenv, dest2check, path
        )

        if dest2check and os.path.isfile(dest2check):
            if not salt.utils.platform.is_windows():
                hash_local, stat_local = \
                    self.hash_and_stat_file(dest2check, saltenv)
                try:
                    mode_local = stat_local[0]
                except (IndexError, TypeError):
                    mode_local = None
            else:
                hash_local = self.hash_file(dest2check, saltenv)
                mode_local = None

            if hash_local == hash_server:
                return dest2check

        log.debug(
            'Fetching file from saltenv \'%s\', ** attempting ** \'%s\'',
            saltenv, path
        )
        d_tries = 0
        transport_tries = 0
        path = self._check_proto(path)
        load = {'path': path,
                'saltenv': saltenv,
                'cmd': '_serve_file'}
        if gzip:
            gzip = int(gzip)
            load['gzip'] = gzip

        fn_ = None
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    try:
                        os.makedirs(destdir)
                    except OSError as exc:
                        if exc.errno != errno.EEXIST:  # ignore if it was there already
                            raise
                else:
                    return False
            # We need an open filehandle here, that's why we're not using a
            # with clause:
            fn_ = salt.utils.files.fopen(dest, 'wb+')  # pylint: disable=resource-leakage
        else:
            log.debug('No dest file found')

        while True:
            if not fn_:
                load['loc'] = 0
            else:
                load['loc'] = fn_.tell()
            data = self.channel.send(load, raw=True)
            if six.PY3:
                # Sometimes the source is local (eg when using
                # 'salt.fileserver.FSChan'), in which case the keys are
                # already strings. Sometimes the source is remote, in which
                # case the keys are bytes due to raw mode. Standardize on
                # strings for the top-level keys to simplify things.
                data = decode_dict_keys_to_str(data)
            try:
                if not data['data']:
                    if not fn_ and data['dest']:
                        # This is a 0 byte file on the master
                        with self._cache_loc(
                                data['dest'],
                                saltenv,
                                cachedir=cachedir) as cache_dest:
                            dest = cache_dest
                            with salt.utils.files.fopen(cache_dest, 'wb+') as ofile:
                                ofile.write(data['data'])
                    if 'hsum' in data and d_tries < 3:
                        # Master has prompted a file verification, if the
                        # verification fails, re-download the file. Try 3 times
                        d_tries += 1
                        hsum = salt.utils.hashutils.get_hash(dest, salt.utils.stringutils.to_str(data.get('hash_type', b'md5')))
                        if hsum != data['hsum']:
                            log.warning(
                                'Bad download of file %s, attempt %d of 3',
                                path, d_tries
                            )
                            continue
                    break
                if not fn_:
                    with self._cache_loc(
                            data['dest'],
                            saltenv,
                            cachedir=cachedir) as cache_dest:
                        dest = cache_dest
                        # If a directory was formerly cached at this path, then
                        # remove it to avoid a traceback trying to write the file
                        if os.path.isdir(dest):
                            salt.utils.files.rm_rf(dest)
                        fn_ = salt.utils.atomicfile.atomic_open(dest, 'wb+')
                if data.get('gzip', None):
                    data = salt.utils.gzip_util.uncompress(data['data'])
                else:
                    data = data['data']
                if six.PY3 and isinstance(data, str):
                    data = data.encode()
                fn_.write(data)
            except (TypeError, KeyError) as exc:
                try:
                    data_type = type(data).__name__
                except AttributeError:
                    # Shouldn't happen, but don't let this cause a traceback.
                    data_type = six.text_type(type(data))
                transport_tries += 1
                log.warning(
                    'Data transport is broken, got: %s, type: %s, '
                    'exception: %s, attempt %d of 3',
                    data, data_type, exc, transport_tries
                )
                self._refresh_channel()
                if transport_tries > 3:
                    log.error(
                        'Data transport is broken, got: %s, type: %s, '
                        'exception: %s, retry attempts exhausted',
                        data, data_type, exc
                    )
                    break

        if fn_:
            fn_.close()
            log.info(
                'Fetching file from saltenv \'%s\', ** done ** \'%s\'',
                saltenv, path
            )
        else:
            log.debug(
                'In saltenv \'%s\', we are ** missing ** the file \'%s\'',
                saltenv, path
            )

        return dest
291096
def load_hdf(cls, filename, path=''):
        
        df_long = pd.read_hdf(filename,'{}/long/df'.format(path))
        df_short = pd.read_hdf(filename,'{}/short/df'.format(path))
        return cls.from_df(df_long, df_short)
299703
def upload(self, localpath, href):
        
        localpath = _(localpath)
        href = remote(href)
        if not os.path.exists(localpath):
            logger.info(u("ERROR: localfile: %s not found") % localpath)
            return
        if os.path.islink(localpath):
            return self.upload(os.path.abspath(os.path.realpath(localpath)), href)
            # 3 tryings to upload file
        for iTry in range(TRYINGS):
            try:
                logger.info(u("upload: %s %s") % (localpath, href))
                length = os.path.getsize(localpath)

                if PY3:
                    _open = open(_encode_utf8(localpath), "r", encoding='latin-1')
                else:
                    _open = open(_encode_utf8(localpath), "r")
                with _open as f:
                    return self.write(f, href, length=length)
            except ConnectionException:
                raise
            except Exception:
                e = sys.exc_info()[1]
                logger.exception(e)
558762
def localpath(*args):
    """"""
    plist = [ROOT] + list(args)
    return os.path.abspath(pjoin(*plist))
129344
def realpath(path):
    
    scheme, netloc, path_ = parse(path)
    if scheme == 'file' or hdfs_fs.default_is_local():
        return unparse(scheme, netloc, os.path.realpath(path_))
    return path
118581
def readlocal(path, offset=None, size=-1):
    
    try:
        with open(path, 'rb') as f:
            if offset:
                f.seek(offset)
            buf = f.read(size)
    except IOError as e:
        if e.errno == errno.ENOENT:
            raise FileNotFoundError(e)
        else:
            raise
    return buf
1214364
def find_by_localpath(self, path):
        
        # note that the paths in self.jsondata were already _homepath2real()'d
        # in the class' __init__()
        resolved = _homepath2real(path)
        for row in self.jsondata:
            if resolved == os.path.realpath(row["localpath"]):
                return self._infofromdict(row)
537588
def save(self, path=None, complevel=1, complib='zlib'):
        
        if path is None:
            path = self.hexuid + '.hdf5'
        elif os.path.isdir(path):
            path += os.sep + self.hexuid + '.hdf5'
        elif not (path.endswith('.hdf5') or path.endswith('.hdf')):
            raise ValueError('File path must have a ".hdf5" or ".hdf" extension.')
        with pd.HDFStore(path, 'w', complevel=complevel, complib=complib) as store:
            store['kwargs'] = pd.Series()
            store.get_storer('kwargs').attrs.metadata = self._rel()
            fc = 0    # Field counter (see special handling of fields below)
            for name, data in self._data().items():
                if hasattr(data, '_revert_categories'):
                    data._revert_categories()
                name = name[1:] if name.startswith('_') else name
                if isinstance(data, Field):    # Fields are handled separately
                    fname = 'FIELD{}_'.format(fc) + name + '/'
                    store[fname + 'data'] = pd.DataFrame(data)
                    for i, field in enumerate(data.field_values):
                        ffname = fname + 'values' + str(i)
                        if isinstance(field, pd.Series):
                            store[ffname] = pd.Series(field)
                        else:
                            store[ffname] = pd.DataFrame(field)
                    fc += 1
                elif isinstance(data, Series):
                    s = pd.Series(data)
                    if isinstance(data.dtype, pd.types.dtypes.CategoricalDtype):
                        s = s.astype('O')
                    store[name] = s
                elif isinstance(data, DataFrame):
                    store[name] = pd.DataFrame(data)
                elif isinstance(data, SparseSeries):
                    s = pd.SparseSeries(data)
                    if isinstance(data.dtype, pd.types.dtypes.CategoricalDtype):
                        s = s.astype('O')
                    store[name] = s
                elif isinstance(data, SparseDataFrame):
                    store[name] = pd.SparseDataFrame(data)
                else:
                    if hasattr(data, 'dtype') and isinstance(data.dtype, pd.types.dtypes.CategoricalDtype):
                        data = data.astype('O')
                    else:
                        for col in data:
                            if isinstance(data[col].dtype, pd.types.dtypes.CategoricalDtype):
                                data[col] = data[col].astype('O')
                    store[name] = data
                if hasattr(data, '_set_categories'):
                    data._set_categories()
586269
def get_local_file_list(self):
        
        file_list = []
        for (dirpath, dirnames, filenames) in os.walk(self.build_dir):
            for fname in filenames:
                # relative path, to sync with the S3 key
                local_key = os.path.join(
                    os.path.relpath(dirpath, self.build_dir),
                    fname
                )
                if local_key.startswith('./'):
                    local_key = local_key[2:]
                file_list.append(local_key)
        return file_list
724436
def do_get(self, from_path, to_path):
        
        to_file = open(os.path.expanduser(to_path), "wb")

        self.n.downloadFile(self.current_path + "/" + from_path, to_path)
692857
def downloadTo(self, href, localpath):
        
        for iTry in range(TRYINGS):
            logger.info(u("downloadTo(%s): %s %s") % (iTry, href, localpath))
            try:
                href = remote(href)
                localpath = _(localpath)

                conn = self.getConnection()
                conn.request("GET", _encode_utf8(href), "", self.getHeaders())
                response = conn.getresponse()
                checkResponse(response)
                f = None
                try:
                    while True:
                        data = _decode_utf8(response.read(1024))
                        if not data:
                            break
                        if data == u('resource not found'):
                            return False
                        if not f:
                            f = open(localpath, "w")
                        f.write(data)
                finally:
                    if f:
                        f.close()
                return True
            except ConnectionException:
                raise
            except Exception:
                e = sys.exc_info()[1]
                logger.exception(e)
55445
def _get_local_files(local_dir, pattern=''):
    
    local_files = {}
    
    if pattern:
        cwd = os.getcwd()
        os.chdir(local_dir)
        patterns = pattern.split('|')
        local_list = set([])
        for p in patterns: local_list = local_list | set(glob(p))
        for path in local_list:
            dir, file = os.path.split(path)
            if os.path.isfile(path):
                local_files[dir] = local_files.get(dir,[])+[file]
            elif os.path.isdir(path):
                local_files[file] = local_files.get(dir,[])
        os.chdir(cwd)
    return local_files
655150
def download_file(path, local_drive_path)
      uri = URI.parse(Addressable::URI.escape(@url + path))
      http_request = build_http_object(uri)
      req = build_request(:get, uri, {}, @api_version.to_s)

      http_request.start do |http|
        http.request(req) do |res|
          response_handler(res) unless res.code.to_i.between?(200, 204)
          File.open(local_drive_path, 'wb') do |file|
            res.read_body do |segment|
              file.write(segment)
            end
          end
        end
      end
      true
    end
1342778
def ftp_download(ftp, ftp_path, local_path):
    
    with open(local_path, 'wb') as _f:
        ftp.retrbinary('RETR %s' % ftp_path, _f.write)
1047725
def download_file(self, url, local_path):
        
        response = requests.get(url, stream=True)

        with open(local_path, 'wb') as f:
            for chunk in tqdm(response.iter_content(chunk_size=1024), unit='KB'):
                if chunk: # filter out keep-alive new chunks
                    f.write(chunk)
                    f.flush()
357193
def _parse_path(self, path):
        """"""
        # Support specifying another host via hdfs://host:port/path syntax
        # We ignore the scheme and piece together the query and fragment
        # Note that HDFS URIs are not URL encoded, so a '?' or a '#' in the URI is part of the
        # path
        parts = urlsplit(path, allow_fragments=False)
        if not parts.path.startswith('/'):
            raise ValueError("Path must be absolute, was given {}".format(path))
        if parts.scheme not in ('', 'hdfs', 'hftp', 'webhdfs'):
            warnings.warn("Unexpected scheme {}".format(parts.scheme))
        assert not parts.fragment
        path = parts.path
        if parts.query:
            path += '?' + parts.query
        if parts.netloc:
            hosts = self._parse_hosts(parts.netloc)
        else:
            hosts = self.hosts
        return hosts, path