Sync source node (file / dir) to destination path using hard links.
487899
def link_node(abs_src: str, abs_dest: str, force: bool=False):
    """"""
    dest_parent_dir = split(abs_dest)[0]
    if not isdir(dest_parent_dir):
        # exist_ok=True in case of concurrent creation of the same
        # parent dir
        os.makedirs(dest_parent_dir, exist_ok=True)
    if isfile(abs_src):
        # sync file by linking it to dest
        link_func(abs_src, abs_dest, force)
    elif isdir(abs_src):
        # sync dir by recursively linking files under it to dest
        shutil.copytree(abs_src, abs_dest,
                        copy_function=functools.partial(link_func,
                                                        force=force),
                        ignore=shutil.ignore_patterns('.git'))
    else:
        raise FileNotFoundError(abs_src)
261915
function getPathsToSync(fs, path, callback) {
  getAttr(fs, path, constants.attributes.pathsToSync, callback);
}
897834
public static void updateNodes(PathDataNode[] target, PathDataNode[] source) {
    for (int i = 0; i < source.length; i ++) {
      target[i].mType = source[i].mType;
      for (int j = 0; j < source[i].mParams.length; j ++) {
        target[i].mParams[j] = source[i].mParams[j];
      }
    }
  }
706970
def rsync(self, gateway, source, notify=None, verbose=False, ignores=None):
        """"""
        # XXX This changes the calling behaviour of
        #     pytest_xdist_rsyncstart and pytest_xdist_rsyncfinish to
        #     be called once per rsync target.
        rsync = HostRSync(source, verbose=verbose, ignores=ignores)
        spec = gateway.spec
        if spec.popen and not spec.chdir:
            # XXX This assumes that sources are python-packages
            #     and that adding the basedir does not hurt.
            gateway.remote_exec(
                """
                import sys ; sys.path.insert(0, %r)
            """
                % os.path.dirname(str(source))
            ).waitclose()
            return
        if (spec, source) in self._rsynced_specs:
            return

        def finished():
            if notify:
                notify("rsyncrootready", spec, source)

        rsync.add_target_host(gateway, finished=finished)
        self._rsynced_specs.add((spec, source))
        self.config.hook.pytest_xdist_rsyncstart(source=source, gateways=[gateway])
        rsync.send()
        self.config.hook.pytest_xdist_rsyncfinish(source=source, gateways=[gateway])
819998
def _sync_folder_to_container(self, folder_path, container, prefix, delete,
            include_hidden, ignore, ignore_timestamps, object_prefix, verbose):
        
        fnames = os.listdir(folder_path)
        ignore = utils.coerce_to_list(ignore)
        log = logging.getLogger("pyrax")
        if not include_hidden:
            ignore.append(".*")
        for fname in fnames:
            if utils.match_pattern(fname, ignore):
                self._sync_summary["ignored"] += 1
                continue
            pth = os.path.join(folder_path, fname)
            if os.path.isdir(pth):
                subprefix = fname
                if prefix:
                    subprefix = os.path.join(prefix, subprefix)
                self._sync_folder_to_container(pth, container, prefix=subprefix,
                        delete=delete, include_hidden=include_hidden,
                        ignore=ignore, ignore_timestamps=ignore_timestamps,
                        object_prefix=object_prefix, verbose=verbose)
                continue
            self._local_files.append(os.path.join(object_prefix, prefix,
                    fname))
            local_etag = utils.get_checksum(pth)
            if object_prefix:
                prefix = os.path.join(object_prefix, prefix)
                object_prefix = ""
            fullname_with_prefix = os.path.join(prefix, fname)
            try:
                obj = self._remote_files[fullname_with_prefix]
                obj_etag = obj.etag
            except KeyError:
                obj = None
                obj_etag = None
            if local_etag != obj_etag:
                if not ignore_timestamps:
                    if obj:
                        obj_time_str = obj.last_modified[:19]
                    else:
                        obj_time_str = EARLY_DATE_STR
                    local_mod = datetime.datetime.utcfromtimestamp(
                            os.stat(pth).st_mtime)
                    local_mod_str = local_mod.isoformat()
                    if obj_time_str >= local_mod_str:
                        # Remote object is newer
                        self._sync_summary["older"] += 1
                        if verbose:
                            log.info("%s NOT UPLOADED because remote object is "
                                    "newer", fullname_with_prefix)
                            log.info("  Local: %s   Remote: %s" % (
                                    local_mod_str, obj_time_str))
                        continue
                try:
                    container.upload_file(pth, obj_name=fullname_with_prefix,
                        etag=local_etag, return_none=True)
                    self._sync_summary["uploaded"] += 1
                    if verbose:
                        log.info("%s UPLOADED", fullname_with_prefix)
                except Exception as e:
                    # Record the failure, and move on
                    self._sync_summary["failed"] += 1
                    self._sync_summary["failure_reasons"].append("%s" % e)
                    if verbose:
                        log.error("%s UPLOAD FAILED. Exception: %s" %
                                (fullname_with_prefix, e))
            else:
                self._sync_summary["duplicate"] += 1
                if verbose:
                    log.info("%s NOT UPLOADED because it already exists",
                            fullname_with_prefix)
        if delete and not prefix:
            self._delete_objects_not_in_list(container, object_prefix)
1149873
def _sync_io(self):
        """"""
        if self._file_epoch == self.file_object.epoch:
            return

        if self._io.binary:
            contents = self.file_object.byte_contents
        else:
            contents = self.file_object.contents

        self._set_stream_contents(contents)
        self._file_epoch = self.file_object.epoch
949921
def copytree_atomic(source_path, dest_path, make_parents=False, backup_suffix=None, symlinks=False):
  
  if os.path.isdir(source_path):
    with atomic_output_file(dest_path, make_parents=make_parents, backup_suffix=backup_suffix) as tmp_path:
      shutil.copytree(source_path, tmp_path, symlinks=symlinks)
  else:
    copyfile_atomic(source_path, dest_path, make_parents=make_parents, backup_suffix=backup_suffix)
1366871
def fast_sync_fetch(working_dir, import_url):
    
    try:
        fd, tmppath = tempfile.mkstemp(prefix='.blockstack-fast-sync-', dir=working_dir)
    except Exception, e:
        log.exception(e)
        return None
    
    log.debug("Fetch {} to {}...".format(import_url, tmppath))

    try:
        path, headers = urllib.urlretrieve(import_url, tmppath)
    except Exception, e:
        os.close(fd)
        log.exception(e)
        return None
    
    os.close(fd)
    return tmppath
1168506
def _sync_download(self, url, destination_path):
    """"""
    proxies = {
        'http': os.environ.get('TFDS_HTTP_PROXY', None),
        'https': os.environ.get('TFDS_HTTPS_PROXY', None),
        'ftp': os.environ.get('TFDS_FTP_PROXY', None)
    }
    if kaggle.KaggleFile.is_kaggle_url(url):
      if proxies['http']:
        os.environ['KAGGLE_PROXY'] = proxies['http']
      return self._sync_kaggle_download(url, destination_path)

    try:
      # If url is on a filesystem that gfile understands, use copy. Otherwise,
      # use requests.
      if not url.startswith('http'):
        return self._sync_file_copy(url, destination_path)
    except tf.errors.UnimplementedError:
      pass

    session = requests.Session()
    session.proxies = proxies
    if _DRIVE_URL.match(url):
      url = self._get_drive_url(url, session)
    use_urllib = url.startswith('ftp')
    if use_urllib:
      if proxies['ftp']:
        proxy = urllib.request.ProxyHandler({'ftp': proxies['ftp']})
        opener = urllib.request.build_opener(proxy)
        urllib.request.install_opener(opener)   # pylint: disable=too-many-function-args
      request = urllib.request.Request(url)
      response = urllib.request.urlopen(request)
    else:
      response = session.get(url, stream=True)
      if response.status_code != 200:
        raise DownloadError('Failed to get url %s. HTTP code: %d.' %
                            (url, response.status_code))
    fname = _get_filename(response)
    path = os.path.join(destination_path, fname)
    size = 0

    size_mb = 0
    unit_mb = units.MiB
    self._pbar_dl_size.update_total(
        int(response.headers.get('Content-length', 0)) // unit_mb)
    with tf.io.gfile.GFile(path, 'wb') as file_:
      checksum = self._checksumer()
      if use_urllib:
        iterator = iter(lambda: response.read(io.DEFAULT_BUFFER_SIZE), b'')
      else:
        iterator = response.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE)

      for block in iterator:
        size += len(block)

        # Update the progress bar
        size_mb += len(block)
        if size_mb > unit_mb:
          self._pbar_dl_size.update(size_mb // unit_mb)
          size_mb %= unit_mb

        checksum.update(block)
        file_.write(block)
    self._pbar_url.update(1)
    return checksum.hexdigest(), size
156488
def do_cat(self, path):
        """"""
        path = path[0]
        tmp_file_path = self.TMP_PATH + 'tmp'

        if not os.path.exists(self.TMP_PATH):
            os.makedirs(self.TMP_PATH)

        f = self.n.downloadFile(self.current_path + path, tmp_file_path)
        f = open(tmp_file_path, 'r')

        self.stdout.write(f.read())
        self.stdout.write("\n")
1266117
def sf(f, dirpath, jottapath):
    """"""
    log.debug('Create SyncFile from %s', repr(f))
    log.debug('Got encoded filename %r, joining with dirpath %r', _encode_filename_to_filesystem(f), dirpath)
    return SyncFile(localpath=os.path.join(dirpath, _encode_filename_to_filesystem(f)),
                  jottapath=posixpath.join(_decode_filename_to_unicode(jottapath), _decode_filename_to_unicode(f)))
217401
def follow_file(self, f, seen, trim):
        """"""
        return (f not in self.graph.nodes and
                f not in seen and
                (not trim or
                 not isinstance(self.provenance[f],
                                (resolve.Builtin, resolve.System))))
813333
def _sync_from_disk(self):
        

        try:
            fobj = self.open_manifest('r')
        except IOError as e:
            if e.errno == errno.ENOENT:
                raise ValueError(
                    "couldn't find manifest file in %s" % (self.path,))
            elif e.errno == errno.ENOTDIR:
                raise ValueError("%s is not a directory" % (self.path,))
            else:
                raise
        with fobj:
            self.contents = json.load(fobj)

        self._set_defaults()
        self._check_refpkg()
329564
def download(self, source, destination):
        """"""
        self._handler.download(posix_path(source), destination)
1236379
def fast_relpath(path, start):
  """"""
  relpath = fast_relpath_optional(path, start)
  if relpath is None:
    raise ValueError('{} is not a directory containing {}'.format(start, path))
  return relpath
1344901
def link(source_path):
    """"""
    if not os.path.isfile(source_path):
        raise SourceNotFound(source_path)
    with open(source_path, 'r') as f:
        content = f.read()
    block_map = BlockMap()  # The map will be populated with the following function call.
    all_block = convert_lines_to_block(
            content.splitlines(), block_map, LinkStack(source_path), source_path)
    return all_block, block_map.get_variables()
900505
public void syncPersistNewDirectory(MutableInodeDirectory dir)
      throws InvalidPathException, FileDoesNotExistException, IOException {
    dir.setPersistenceState(PersistenceState.TO_BE_PERSISTED);
    syncPersistDirectory(dir).ifPresent(status -> {
      // If the directory already exists in the UFS, update our metadata to match the UFS.
      dir.setOwner(status.getOwner())
          .setGroup(status.getGroup())
          .setMode(status.getMode());

      Long lastModificationTime = status.getLastModifiedTime();
      if (lastModificationTime != null) {
        dir.setLastModificationTimeMs(lastModificationTime, true);
      }
    });
    dir.setPersistenceState(PersistenceState.PERSISTED);
  }