Executes RSYNC synchronization job

@throws IOException
196718
public void execute() throws IOException
   {
      // Future todo: Use JNI and librsync library?
      Runtime run = Runtime.getRuntime();
      try
      {
         String command ;
         if(excludeDir != null && !excludeDir.isEmpty())
         {
            command= "rsync -rv --delete --exclude "+ excludeDir + " " + src + " " + dst;
         }
         else
         {
            command= "rsync -rv --delete " + src + " " + dst;
         }

         if (LOG.isDebugEnabled())
         {
            LOG.debug("Rsync job started: " + command);
         }
         if (userName != null && password != null)
         {
            String[] envProperties =
               new String[]{RSYNC_USER_SYSTEM_PROPERTY + "=" + userName,
                  RSYNC_PASSWORD_SYSTEM_PROPERTY + "=" + password};
            process = run.exec(command, envProperties);
         }
         else
         {
            process = run.exec(command);
         }

         // Handle process Standard and Error output
         InputStream stderr = process.getErrorStream();
         InputStreamReader isrErr = new InputStreamReader(stderr);
         BufferedReader brErr = new BufferedReader(isrErr);

         InputStream stdout = process.getInputStream();
         InputStreamReader isrStd = new InputStreamReader(stdout);
         BufferedReader brStd = new BufferedReader(isrStd);

         String val = null;
         StringBuilder stringBuilderErr = new StringBuilder();
         StringBuilder stringBuilderStd = new StringBuilder();
         while ((val = brStd.readLine()) != null)
         {
            stringBuilderStd.append(val);
            stringBuilderStd.append('\n');
         }

         while ((val = brErr.readLine()) != null)
         {
            stringBuilderErr.append(val);
            stringBuilderErr.append('\n');
         }

         Integer returnCode = null;
         // wait for thread
         while (returnCode == null)
         {
            try
            {
               returnCode = process.waitFor();
            }
            catch (InterruptedException e)
            {
               // oops, this can happen sometimes
            }
         }
         if (LOG.isDebugEnabled())
         {
            LOG.debug("Rsync job finished: " + returnCode + ". Error stream output \n"
               + stringBuilderErr.toString() + " Standard stream output \n" + stringBuilderStd.toString());
         }
         if (returnCode != 0)
         {
            throw new IOException("RSync job finished with exit code is " + returnCode + ". Error stream output: \n"
               + stringBuilderErr.toString());
         }
      }
      finally
      {
         process = null;
      }
   }
1328238
def sync_in(self, release):
        """"""
        tree = release['canonical_dir']
        if os.path.exists(tree) and release.get('rsync_in_objs'):
            out = release['output_dir']
            if not os.path.isdir(out):
                self.log.info('Creating %s', out)
                os.makedirs(out)
            self.call(release['rsync_in_objs'])
            self.call(release['rsync_in_rest'])
1200266
public void sync() throws Exception {
        try {
            JedisClusterPipeline pipeline = new JedisClusterPipeline(clusterInfoCache);
            execute(pipeline);
            pipeline.sync();
        } catch (JedisMovedDataException jre) {
            // if MOVED redirection occurred, rebuilds cluster's slot cache,
            // recommended by Redis cluster specification
            connectionHandler.renewSlotCache();
            if (maxAttempts > 0) {
                maxAttempts--;
                sync();
                return;
            }
            throw jre;
        } catch (Exception e) {
            throw e;
        }
    }
728744
public int run() throws IOException {
    Map<String, MountPointInfo> mountTable = mFileSystemMasterClient.getMountTable();
    System.out.println("Alluxio under storage system information:");
    printMountInfo(mountTable);
    return 0;
  }
1144905
public final void run()
   {
      try
      {
         copy( this.is, this.os );
      }
      catch ( IOException ioe )
      {
         log.error(ioe);
      }
      finally
      {
         try
         {
            this.is.close();
         }
         catch (IOException ioe)
         {
            log.error(ioe);
         }
         if (this.closeOsOnExit)
         {
            try
            {
               this.os.flush();
               this.os.close();
            }
            catch (IOException ioe)
            {
               log.error(ioe);
            }
         }
      }
   }
1359148
@Override
    public void execute() throws UpdateExecutionException {
        try {
            sync();
            getMarkLogicClient().sendUpdateQuery(getQueryString(), getBindings(), getIncludeInferred(), getBaseURI());
        }catch(ForbiddenUserException | FailedRequestException e){
            throw new UpdateExecutionException(e);
        } catch (RepositoryException e) {
            throw new UpdateExecutionException(e);
        } catch (MalformedQueryException e) {
            throw new UpdateExecutionException(e);
        } catch (IOException e) {
            throw new UpdateExecutionException(e);
        }
    }
18416
public void sync() {

        if (isClosed) {
            return;
        }

        synchronized (fileStreamOut) {
            if (needsSync) {
                if (busyWriting) {
                    forceSync = true;

                    return;
                }

                try {
                    fileStreamOut.flush();
                    outDescriptor.sync();

                    syncCount++;
                } catch (IOException e) {
                    Error.printSystemOut("flush() or sync() error: "
                                         + e.toString());
                }

                needsSync = false;
                forceSync = false;
            }
        }
    }
792099
@Override
    public void run() {
        do {

            // Scan for directories:
            try {
                log.info("Scanning for directories under {}", root);
                Files.walkFileTree(root, new Visitor());
            } catch (IOException e) {
                log.info("Error in scanning for directories to monitor:");
                e.printStackTrace();
            }

            // Block until notified that a new scan is needed:
            synchronized (Scanner.class) {
                try {
                    Scanner.class.wait();
                } catch (InterruptedException e) {
                    log.info("Scanner interrupted:");
                    e.printStackTrace();
                }
            }

        } while (true);

    }
304878
public void run() {
        synchronized (this) {
            started = true;
        }
        finished = false;
        finish = false;

        try {
            while (true) {
                waitForInput(is);

                if (finish || Thread.interrupted()) {
                    break;
                }

                int availableBytes = is.available();
                byte[] buf = new byte[availableBytes];
                int actualBytesRead = is.read(buf, 0, availableBytes);

                if (actualBytesRead <= 0 || finish || Thread.interrupted()) {
                    break;
                }
                os.write(buf, 0, actualBytesRead);
                if (autoflush) {
                    os.flush();
                }
            }
            os.flush();
        } catch (InterruptedException ie) {
            // likely PumpStreamHandler trying to stop us
        } catch (Exception e) {
            synchronized (this) {
                exception = e;
            }
        } finally {
            if (closeWhenExhausted) {
                FileUtils.close(os);
            }
            finished = true;
            synchronized (this) {
                notifyAll();
            }
        }
    }
781457
private void syncInternal() throws IOException {
        syncCompactor();
        fireBeforePersist();
        
        /* CALLS ORDERED: Need force _segment first and then persist
         * _addressArray. During recovery, the _addressArray can always
         * point to addresses which are valid though may not reflect the
         * most recent address update.
         */
        _segment.force();
        _addressArray.sync();
        _segmentManager.updateMeta();
        
        fireAfterPersist();
    }
1061881
function symlinkOrCopySync(dir, target) {
  try {
    symlinkOrCopy.sync(dir, target);
  } catch (e) {
    if (existsSync(target)) {
      rimraf.sync(target);
    }
    symlinkOrCopy.sync(dir, target);
  }
}
365964
public void sync() {
		if (!validState) {
			throw new InvalidStateException();
		}
		if (useMmap) {
			syncAllMmaps();
		}
		if (fileChannel != null) {
			try {
				fileChannel.force(false);
			} catch (Exception ign) {
			}
		}
		if (callback != null) {
			callback.synched();
		}
	}
1327565
@Override
	public void run() {

		// ----------------------------
		//  Initial State transition
		// ----------------------------
		while (true) {
			ExecutionState current = this.executionState;
			if (current == ExecutionState.CREATED) {
				if (transitionState(ExecutionState.CREATED, ExecutionState.DEPLOYING)) {
					// success, we can start our work
					break;
				}
			}
			else if (current == ExecutionState.FAILED) {
				// we were immediately failed. tell the TaskManager that we reached our final state
				notifyFinalState();
				if (metrics != null) {
					metrics.close();
				}
				return;
			}
			else if (current == ExecutionState.CANCELING) {
				if (transitionState(ExecutionState.CANCELING, ExecutionState.CANCELED)) {
					// we were immediately canceled. tell the TaskManager that we reached our final state
					notifyFinalState();
					if (metrics != null) {
						metrics.close();
					}
					return;
				}
			}
			else {
				if (metrics != null) {
					metrics.close();
				}
				throw new IllegalStateException("Invalid state for beginning of operation of task " + this + '.');
			}
		}

		// all resource acquisitions and registrations from here on
		// need to be undone in the end
		Map<String, Future<Path>> distributedCacheEntries = new HashMap<>();
		AbstractInvokable invokable = null;

		try {
			// ----------------------------
			//  Task Bootstrap - We periodically
			//  check for canceling as a shortcut
			// ----------------------------

			// activate safety net for task thread
			LOG.info("Creating FileSystem stream leak safety net for task {}", this);
			FileSystemSafetyNet.initializeSafetyNetForThread();

			blobService.getPermanentBlobService().registerJob(jobId);

			// first of all, get a user-code classloader
			// this may involve downloading the job's JAR files and/or classes
			LOG.info("Loading JAR files for task {}.", this);

			userCodeClassLoader = createUserCodeClassloader();
			final ExecutionConfig executionConfig = serializedExecutionConfig.deserializeValue(userCodeClassLoader);

			if (executionConfig.getTaskCancellationInterval() >= 0) {
				// override task cancellation interval from Flink config if set in ExecutionConfig
				taskCancellationInterval = executionConfig.getTaskCancellationInterval();
			}

			if (executionConfig.getTaskCancellationTimeout() >= 0) {
				// override task cancellation timeout from Flink config if set in ExecutionConfig
				taskCancellationTimeout = executionConfig.getTaskCancellationTimeout();
			}

			if (isCanceledOrFailed()) {
				throw new CancelTaskException();
			}

			// ----------------------------------------------------------------
			// register the task with the network stack
			// this operation may fail if the system does not have enough
			// memory to run the necessary data exchanges
			// the registration must also strictly be undone
			// ----------------------------------------------------------------

			LOG.info("Registering task at network: {}.", this);

			network.registerTask(this);

			for (ResultPartition partition : producedPartitions) {
				taskEventDispatcher.registerPartition(partition.getPartitionId());
			}

			// add metrics for buffers
			this.metrics.getIOMetricGroup().initializeBufferMetrics(this);

			// register detailed network metrics, if configured
			if (taskManagerConfig.getConfiguration().getBoolean(TaskManagerOptions.NETWORK_DETAILED_METRICS)) {
				// similar to MetricUtils.instantiateNetworkMetrics() but inside this IOMetricGroup
				MetricGroup networkGroup = this.metrics.getIOMetricGroup().addGroup("Network");
				MetricGroup outputGroup = networkGroup.addGroup("Output");
				MetricGroup inputGroup = networkGroup.addGroup("Input");

				// output metrics
				for (int i = 0; i < producedPartitions.length; i++) {
					ResultPartitionMetrics.registerQueueLengthMetrics(
						outputGroup.addGroup(i), producedPartitions[i]);
				}

				for (int i = 0; i < inputGates.length; i++) {
					InputGateMetrics.registerQueueLengthMetrics(
						inputGroup.addGroup(i), inputGates[i]);
				}
			}

			// next, kick off the background copying of files for the distributed cache
			try {
				for (Map.Entry<String, DistributedCache.DistributedCacheEntry> entry :
						DistributedCache.readFileInfoFromConfig(jobConfiguration)) {
					LOG.info("Obtaining local cache file for '{}'.", entry.getKey());
					Future<Path> cp = fileCache.createTmpFile(entry.getKey(), entry.getValue(), jobId, executionId);
					distributedCacheEntries.put(entry.getKey(), cp);
				}
			}
			catch (Exception e) {
				throw new Exception(
					String.format("Exception while adding files to distributed cache of task %s (%s).", taskNameWithSubtask, executionId), e);
			}

			if (isCanceledOrFailed()) {
				throw new CancelTaskException();
			}

			// ----------------------------------------------------------------
			//  call the user code initialization methods
			// ----------------------------------------------------------------

			TaskKvStateRegistry kvStateRegistry = kvStateService.createKvStateTaskRegistry(jobId, getJobVertexId());

			Environment env = new RuntimeEnvironment(
				jobId,
				vertexId,
				executionId,
				executionConfig,
				taskInfo,
				jobConfiguration,
				taskConfiguration,
				userCodeClassLoader,
				memoryManager,
				ioManager,
				broadcastVariableManager,
				taskStateManager,
				aggregateManager,
				accumulatorRegistry,
				kvStateRegistry,
				inputSplitProvider,
				distributedCacheEntries,
				producedPartitions,
				inputGates,
				taskEventDispatcher,
				checkpointResponder,
				taskManagerConfig,
				metrics,
				this);

			// now load and instantiate the task's invokable code
			invokable = loadAndInstantiateInvokable(userCodeClassLoader, nameOfInvokableClass, env);

			// ----------------------------------------------------------------
			//  actual task core work
			// ----------------------------------------------------------------

			// we must make strictly sure that the invokable is accessible to the cancel() call
			// by the time we switched to running.
			this.invokable = invokable;

			// switch to the RUNNING state, if that fails, we have been canceled/failed in the meantime
			if (!transitionState(ExecutionState.DEPLOYING, ExecutionState.RUNNING)) {
				throw new CancelTaskException();
			}

			// notify everyone that we switched to running
			taskManagerActions.updateTaskExecutionState(new TaskExecutionState(jobId, executionId, ExecutionState.RUNNING));

			// make sure the user code classloader is accessible thread-locally
			executingThread.setContextClassLoader(userCodeClassLoader);

			// run the invokable
			invokable.invoke();

			// make sure, we enter the catch block if the task leaves the invoke() method due
			// to the fact that it has been canceled
			if (isCanceledOrFailed()) {
				throw new CancelTaskException();
			}

			// ----------------------------------------------------------------
			//  finalization of a successful execution
			// ----------------------------------------------------------------

			// finish the produced partitions. if this fails, we consider the execution failed.
			for (ResultPartition partition : producedPartitions) {
				if (partition != null) {
					partition.finish();
				}
			}

			// try to mark the task as finished
			// if that fails, the task was canceled/failed in the meantime
			if (!transitionState(ExecutionState.RUNNING, ExecutionState.FINISHED)) {
				throw new CancelTaskException();
			}
		}
		catch (Throwable t) {

			// unwrap wrapped exceptions to make stack traces more compact
			if (t instanceof WrappingRuntimeException) {
				t = ((WrappingRuntimeException) t).unwrap();
			}

			// ----------------------------------------------------------------
			// the execution failed. either the invokable code properly failed, or
			// an exception was thrown as a side effect of cancelling
			// ----------------------------------------------------------------

			try {
				// check if the exception is unrecoverable
				if (ExceptionUtils.isJvmFatalError(t) ||
						(t instanceof OutOfMemoryError && taskManagerConfig.shouldExitJvmOnOutOfMemoryError())) {

					// terminate the JVM immediately
					// don't attempt a clean shutdown, because we cannot expect the clean shutdown to complete
					try {
						LOG.error("Encountered fatal error {} - terminating the JVM", t.getClass().getName(), t);
					} finally {
						Runtime.getRuntime().halt(-1);
					}
				}

				// transition into our final state. we should be either in DEPLOYING, RUNNING, CANCELING, or FAILED
				// loop for multiple retries during concurrent state changes via calls to cancel() or
				// to failExternally()
				while (true) {
					ExecutionState current = this.executionState;

					if (current == ExecutionState.RUNNING || current == ExecutionState.DEPLOYING) {
						if (t instanceof CancelTaskException) {
							if (transitionState(current, ExecutionState.CANCELED)) {
								cancelInvokable(invokable);
								break;
							}
						}
						else {
							if (transitionState(current, ExecutionState.FAILED, t)) {
								// proper failure of the task. record the exception as the root cause
								failureCause = t;
								cancelInvokable(invokable);

								break;
							}
						}
					}
					else if (current == ExecutionState.CANCELING) {
						if (transitionState(current, ExecutionState.CANCELED)) {
							break;
						}
					}
					else if (current == ExecutionState.FAILED) {
						// in state failed already, no transition necessary any more
						break;
					}
					// unexpected state, go to failed
					else if (transitionState(current, ExecutionState.FAILED, t)) {
						LOG.error("Unexpected state in task {} ({}) during an exception: {}.", taskNameWithSubtask, executionId, current);
						break;
					}
					// else fall through the loop and
				}
			}
			catch (Throwable tt) {
				String message = String.format("FATAL - exception in exception handler of task %s (%s).", taskNameWithSubtask, executionId);
				LOG.error(message, tt);
				notifyFatalError(message, tt);
			}
		}
		finally {
			try {
				LOG.info("Freeing task resources for {} ({}).", taskNameWithSubtask, executionId);

				// clear the reference to the invokable. this helps guard against holding references
				// to the invokable and its structures in cases where this Task object is still referenced
				this.invokable = null;

				// stop the async dispatcher.
				// copy dispatcher reference to stack, against concurrent release
				final BlockingCallMonitoringThreadPool dispatcher = this.asyncCallDispatcher;
				if (dispatcher != null && !dispatcher.isShutdown()) {
					dispatcher.shutdownNow();
				}

				// free the network resources
				releaseNetworkResources();

				// free memory resources
				if (invokable != null) {
					memoryManager.releaseAll(invokable);
				}

				// remove all of the tasks library resources
				libraryCache.unregisterTask(jobId, executionId);
				fileCache.releaseJob(jobId, executionId);
				blobService.getPermanentBlobService().releaseJob(jobId);

				// close and de-activate safety net for task thread
				LOG.info("Ensuring all FileSystem streams are closed for task {}", this);
				FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread();

				notifyFinalState();
			}
			catch (Throwable t) {
				// an error in the resource cleanup is fatal
				String message = String.format("FATAL - exception in resource cleanup of task %s (%s).", taskNameWithSubtask, executionId);
				LOG.error(message, t);
				notifyFatalError(message, t);
			}

			// un-register the metrics at the end so that the task may already be
			// counted as finished when this happens
			// errors here will only be logged
			try {
				metrics.close();
			}
			catch (Throwable t) {
				LOG.error("Error during metrics de-registration of task {} ({}).", taskNameWithSubtask, executionId, t);
			}
		}
	}
609945
@Override
    public void execute() throws MojoExecutionException {
        if (wisdomDirectory != null) {
            getLog().info("Skipping the External Assets copy as we are using a remote Wisdom " +
                    "Server");
            removeFromWatching();
            return;
        }
        source = new File(basedir, ASSETS_SRC_DIR);
        destination = new File(getWisdomRootDirectory(), ASSETS_DIR);

        try {
            ResourceCopy.copyExternalAssets(this, filtering);
        } catch (IOException e) {
            throw new MojoExecutionException("Error during asset copy", e);
        }
    }
1333602
function (source, target, callback) {
    source = fss.slashify(source);
    target = fss.slashify(target);

    if (!shell.which('rsync')) {
        log('sync error', 'no rsync command found');
        callback();
    } else {
        log('rsync', fss.baseRelative(source));
        log('remote', target);
        var cmd = [config.rsync, source, target].join(' ');
        var opt = { maxBuffer: 5000*1024 };
        exec(cmd, opt, rsync_end(callback));
    }
}
1185787
@Override
  public void run() {
    long lastUpdate = -1L;
    // Start balancing the clusters
    while (running) {
      try {
        Thread.sleep(updateInterval / 10);
        long now = JobTracker.getClock().getTime();
        if (now - lastUpdate > updateInterval) {
          lastUpdate = now;
          doMoveSlots(clusters);
        }
      } catch (Exception e) {
        LOG.error("Exception while balancing cluster.", e);
      }
    }
  }
396744
private void init() throws IOException {
    ssDir = conf.get("fs.snapshot.dir", "/.SNAPSHOT");
    tempDir = conf.get("fs.snapshot.tempdir", "/tmp/snapshot");

    fileServer = getImageServer();
    dfs = FileSystem.get(conf);

    Path ssPath = new Path(ssDir);
    if (!dfs.exists(ssPath)) {
      dfs.mkdirs(ssPath);
    }

    maxLeaseUpdateThreads = conf.getInt("fs.snapshot.leaseupdatethreads", 100);

    // Waiting room purge thread
    purgeThread = new Daemon((new WaitingRoom(conf)).getPurger());
    purgeThread.start();

    // Get namenode rpc connection
    nameNodeAddr = NameNode.getAddress(conf);
    namenode = (NamenodeProtocol) RPC.waitForProxy(NamenodeProtocol.class,
                               NamenodeProtocol.versionID, nameNodeAddr, conf);

    // Snapshot RPC Server
    InetSocketAddress socAddr = SnapshotNode.getAddress(conf);
    int handlerCount = conf.getInt("fs.snapshot.handler.count", 10);
    server = RPC.getServer(this, socAddr.getHostName(), socAddr.getPort(),
                           handlerCount, false, conf);
    // The rpc-server port can be ephemeral... ensure we have the correct info
    serverAddress = server.getListenerAddress();
    LOG.info("SnapshotNode up at: " + serverAddress);

    server.start(); // start rpc server
  }