Copy a non-local partition file to the local build directory
423738
def localize(self, ps=None):
        """"""
        from filelock import FileLock
        from ambry.util import ensure_dir_exists
        from ambry_sources import MPRowsFile
        from fs.errors import ResourceNotFoundError

        if self.is_local:
            return

        local = self._bundle.build_fs

        b = self._bundle.library.bundle(self.identity.as_dataset().vid)

        remote = self._bundle.library.remote(b)

        lock_path = local.getsyspath(self.cache_key + '.lock')

        ensure_dir_exists(lock_path)

        lock = FileLock(lock_path)

        if ps:
            ps.add_update(message='Localizing {}'.format(self.identity.name),
                          partition=self,
                          item_type='bytes',
                          state='downloading')

        if ps:
            def progress(bts):
                if ps.rec.item_total is None:
                    ps.rec.item_count = 0

                if not ps.rec.data:
                    ps.rec.data = {}  # Should not need to do this.
                    return self

                item_count = ps.rec.item_count + bts
                ps.rec.data['updates'] = ps.rec.data.get('updates', 0) + 1

                if ps.rec.data['updates'] % 32 == 1:
                    ps.update(message='Localizing {}'.format(self.identity.name),
                              item_count=item_count)
        else:
            from ambry.bundle.process import call_interval
            @call_interval(5)
            def progress(bts):
                self._bundle.log("Localizing {}. {} bytes downloaded".format(self.vname, bts))

        def exception_cb(e):
            raise e

        with lock:
            # FIXME! This won't work with remote ( http) API, only FS ( s3:, file:)

            if self.is_local:
                return self

            try:
                with remote.fs.open(self.cache_key + MPRowsFile.EXTENSION, 'rb') as f:
                    event = local.setcontents_async(self.cache_key + MPRowsFile.EXTENSION,
                                                    f,
                                                    progress_callback=progress,
                                                    error_callback=exception_cb)
                    event.wait()
                    if ps:
                        ps.update_done()
            except ResourceNotFoundError as e:
                from ambry.orm.exc import NotFoundError
                raise NotFoundError("Failed to get MPRfile '{}' from {}: {} "
                                    .format(self.cache_key, remote.fs, e))

        return self
695088
def build(content_directory=None, out_directory=None):
    """"""
    content_directory = content_directory or '.'
    out_directory = os.path.abspath(out_directory or default_out_directory)
    repo = require_repo(content_directory)

    # Prevent user mistakes
    if out_directory == '.':
        raise ValueError('Output directory must be different than the source directory: ' + repr(out_directory))
    if os.path.basename(os.path.relpath(out_directory, content_directory)) == '..':
        raise ValueError('Output directory must not contain the source directory: ' + repr(out_directory))

    # TODO: read config
    # TODO: use virtualenv
    # TODO: init and run plugins
    # TODO: process with active theme

    # Collect and copy static files
    files = presentation_files(repo)
    remove_directory(out_directory)
    copy_files(files, out_directory, repo)

    return out_directory
720811
def create_local_module_dir(cache_dir, module_name):
  """"""
  tf_v1.gfile.MakeDirs(cache_dir)
  return os.path.join(cache_dir, module_name)
519244
def putfile(self, filepath, buildroot, metahash):
        
        def gen_obj_path(filename):
            filehash = util.hash_file(filepath).hexdigest()
            return filehash, os.path.join(self.obj_cachedir, filehash[0:2],
                                          filehash[2:4], filehash)

        filepath_relative = filepath.split(buildroot)[1][1:]  # Strip leading /
        # Path for the metahashed reference:
        incachepath = self._genpath(filepath_relative, metahash)

        filehash, obj_path = gen_obj_path(filepath)
        if not os.path.exists(obj_path):
            obj_dir = os.path.dirname(obj_path)
            if not os.path.exists(obj_dir):
                os.makedirs(obj_dir)
            log.debug('Adding to obj cache: %s -> %s', filepath, obj_path)
            os.link(filepath, obj_path)

        if os.path.exists(incachepath):
            existingfile_hash = util.hash_file(incachepath).hexdigest()
            if filehash != existingfile_hash:
                log.warn('File found in mh cache, but checksum differs. '
                         'Replacing with this new version. (File: %s)',
                         filepath)
                log.warn('Possible reasons for this:')
                log.warn(' 1. This build is not hermetic, and something '
                         'differs about the build environment compared to the '
                         'previous build.')
                log.warn(' 2. This file has a timestamp or other build-time '
                         'related data encoded into it, which will always '
                         'cause the checksum to differ when built.')
                log.warn(' 3. Everything is terrible and nothing works.')
                os.unlink(incachepath)

        if not os.path.exists(incachepath):
            log.debug('Adding to mh cache: %s -> %s', filepath, incachepath)
            if not os.path.exists(os.path.dirname(incachepath)):
                os.makedirs(os.path.dirname(incachepath))
            os.link(obj_path, incachepath)
1089540
function copyFilesToPack(grunt, buildPath, filesToPack) {
	return function(callback) {
		grunt.util.async.forEach(filesToPack, function(fileConfig, callback) {
			try {
				var filepathDest;
				if (detectDestType(grunt, fileConfig.dest) === 'directory') {
					var dest = (fileConfig.orig.expand) ? fileConfig.dest : path.join(fileConfig.dest, fileConfig.src);
					filepathDest = path.join(buildPath, dest);
				} else {
					filepathDest = path.join(buildPath, fileConfig.dest);
				}

				if (grunt.file.isDir(fileConfig.src)) {
					if (fileConfig.directory) {
						// Copy a whole folder to the destination directory.
						grunt.verbose.writeln('Copying folder "' + fileConfig.src + '" to "' + filepathDest + '"');
						fs.copyRecursive(fileConfig.src, filepathDest, callback);
					} else {
						// Create a folder inside the destination directory.
						grunt.verbose.writeln('Creating folder "' + filepathDest + '"');
						fs.mkdirs(filepathDest, callback);
					}
				} else {
					// Copy a file to the destination directory inside the tmp folder.
					if (fileConfig.link) {
						grunt.verbose.writeln('Copying symlink "' + fileConfig.src + '->' + fileConfig.link + '" to "' + filepathDest + '"');
						//ensure the parent directory exists when making symlinks
						fs.mkdirs(getDirName(filepathDest), function(err) {
							if (err) throw err;
							_fs.symlink(fileConfig.link, filepathDest, 'file', callback);
						});
						
					}
					else {
						grunt.verbose.writeln('Copying file "' + fileConfig.src + '" to "' + filepathDest + '"');
						grunt.file.copy(fileConfig.src, filepathDest);
						fs.lstat(fileConfig.src, function(err, stat) {
							if (err) throw err;
							_fs.chmod(filepathDest, stat.mode, callback);
						});
					}
				}
				
			} catch(e) {
				callback(e);
			}
		}, callback);
	};
}
585922
private static void copyToLocal(Metadata src, Path dst) throws IOException {
    FileChannel dstCh = FileChannel.open(
        dst, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
    ReadableByteChannel srcCh = FileSystems.open(src.resourceId());
    long srcSize = src.sizeBytes();
    long copied = 0;
    do {
      copied += dstCh.transferFrom(srcCh, copied, srcSize - copied);
    } while (copied < srcSize);
    dstCh.close();
    srcCh.close();
    Preconditions.checkState(copied == srcSize);
  }
271332
def put(self, filename):
        
        from . import LocalFile
        target = get_target_path(filename, self.source)
        with self.open('rb') as infile, open(target, 'wb') as outfile:
            shutil.copyfileobj(infile, outfile)
        return LocalFile(target)
119551
def copyfile(source, dest, backup_mode='', cachedir=''):
    '''
    
    '''
    if not os.path.isfile(source):
        raise IOError(
            '[Errno 2] No such file or directory: {0}'.format(source)
        )
    if not os.path.isdir(os.path.dirname(dest)):
        raise IOError(
            '[Errno 2] No such file or directory: {0}'.format(dest)
        )
    bname = os.path.basename(dest)
    dname = os.path.dirname(os.path.abspath(dest))
    tgt = mkstemp(prefix=bname, dir=dname)
    shutil.copyfile(source, tgt)
    bkroot = ''
    if cachedir:
        bkroot = os.path.join(cachedir, 'file_backup')
    if backup_mode == 'minion' or backup_mode == 'both' and bkroot:
        if os.path.exists(dest):
            backup_minion(dest, bkroot)
    if backup_mode == 'master' or backup_mode == 'both' and bkroot:
        # TODO, backup to master
        pass
    # Get current file stats to they can be replicated after the new file is
    # moved to the destination path.
    fstat = None
    if not salt.utils.platform.is_windows():
        try:
            fstat = os.stat(dest)
        except OSError:
            pass

    # The move could fail if the dest has xattr protections, so delete the
    # temp file in this case
    try:
        shutil.move(tgt, dest)
    except Exception:
        __clean_tmp(tgt)
        raise

    if fstat is not None:
        os.chown(dest, fstat.st_uid, fstat.st_gid)
        os.chmod(dest, fstat.st_mode)
    # If SELINUX is available run a restorecon on the file
    rcon = salt.utils.path.which('restorecon')
    if rcon:
        policy = False
        try:
            policy = salt.modules.selinux.getenforce()
        except (ImportError, CommandExecutionError):
            pass
        if policy == 'Enforcing':
            with fopen(os.devnull, 'w') as dev_null:
                cmd = [rcon, dest]
                subprocess.call(cmd, stdout=dev_null, stderr=dev_null)
    if os.path.isfile(tgt):
        # The temp file failed to move
        __clean_tmp(tgt)
631669
public void moveToLocalFile(Path src, Path dst) throws IOException {
    copyToLocalFile(true, false, src, dst);
  }
70144
def copyDirectoryToHdfs(localDirectory, hdfsDirectory, hdfsClient):
    ''''''
    if not os.path.exists(localDirectory):
        raise Exception('Local Directory does not exist!')
    hdfsClient.mkdirs(hdfsDirectory)
    result = True
    for file in os.listdir(localDirectory):
        file_path = os.path.join(localDirectory, file)
        if os.path.isdir(file_path):
            hdfs_directory = os.path.join(hdfsDirectory, file)
            try:
                result = result and copyDirectoryToHdfs(file_path, hdfs_directory, hdfsClient)
            except Exception as exception:
                nni_log(LogType.Error, 'Copy local directory {0} to hdfs directory {1} error: {2}'.format(file_path, hdfs_directory, str(exception)))
                result = False
        else:
            hdfs_file_path = os.path.join(hdfsDirectory, file)
            try:
                result = result and copyFileToHdfs(file_path, hdfs_file_path, hdfsClient)
            except Exception as exception:
                nni_log(LogType.Error, 'Copy local file {0} to hdfs {1} error: {2}'.format(file_path, hdfs_file_path, str(exception)))
                result = False
    return result
100304
def cache_local_file(self, path, **kwargs):
        '''
        
        '''
        dest = os.path.join(self.opts['cachedir'], 'localfiles',
                            path.lstrip('/'))
        destdir = os.path.dirname(dest)

        if not os.path.isdir(destdir):
            os.makedirs(destdir)

        shutil.copyfile(path, dest)
        return dest
1239842
def copy(self, dest):
        
        if os.path.isfile(self.path):
            shutil.copy2(self.path, dest)
        else:
            shutil.copytree(self.path, dest, symlinks=False, ignore=None)
1278627
def put(local_path, hdfs_path):
    
    cmd = "hadoop fs -put %s %s" % (local_path, hdfs_path)
    rcode, stdout, stderr = _checked_hadoop_fs_command(cmd)
106349
def copy_file_tree(src: str,
                   dest: str,
                   ctx: Optional[Dict[str, Any]] = None,
                   option_locations: Optional[List[Tuple[str, List[str]]]] = None):
    
    if os.path.exists(dest):
        shutil.rmtree(dest, ignore_errors=True)
    shutil.copytree(src, dest)

    if option_locations:
        for option, paths in option_locations:
            for path in paths:
                path = os.path.join(dest, path)
                if os.path.isfile(path):
                    os.remove(path)
                else:
                    shutil.rmtree(path, ignore_errors=True)

    if 'app_bundle_module_name' in ctx:
        shutil.move(os.path.join(dest, 'app'),
                    os.path.join(dest, ctx['app_bundle_module_name']))
        shutil.move(os.path.join(dest, 'tests', 'app'),
                    os.path.join(dest, 'tests', ctx['app_bundle_module_name']))

    _render_file_tree(dest, ctx)
1054114
def _copy(src, dst):
    """"""
    try:
        shutil.copy(src, dst)
    except IsADirectoryError:
        if os.path.isdir(dst):
            dst = os.path.join(dst, os.path.basename(src))
        shutil.copytree(src, dst)
1337047
def rsync(local_path, remote_path, exclude=None, extra_opts=None):
    """"""
    if not local_path.endswith('/'):
        local_path += '/'
    exclude = exclude or []
    exclude.extend(['*.egg-info', '*.pyc', '.git', '.gitignore',
                    '.gitmodules', '/build/', '/dist/'])
    with hide('running'):
        run("mkdir -p '{}'".format(remote_path))
        return rsync_project(
            remote_path, local_path, delete=True,
            extra_opts='-i --omit-dir-times -FF ' +
                       (extra_opts if extra_opts else ''),
            ssh_opts='-o StrictHostKeyChecking=no',
            exclude=exclude)
558762
def localpath(*args):
    """"""
    plist = [ROOT] + list(args)
    return os.path.abspath(pjoin(*plist))