Given a rebalance-task info, convert it into the protobuf equivalent

@param stealInfo Rebalance task info
@return Protobuf equivalent of the same
546872
public static RebalanceTaskInfoMap encodeRebalanceTaskInfoMap(RebalanceTaskInfo stealInfo) {
        return RebalanceTaskInfoMap.newBuilder()
                                   .setStealerId(stealInfo.getStealerId())
                                   .setDonorId(stealInfo.getDonorId())
                                   .addAllPerStorePartitionIds(ProtoUtils.encodeStoreToPartitionsTuple(stealInfo.getStoreToPartitionIds()))
                                   .setInitialCluster(new ClusterMapper().writeCluster(stealInfo.getInitialCluster()))
                                   .build();
    }
958181
public static RebalanceTaskInfo create(Map<?, ?> map) {
        int stealerId = (Integer)map.get("stealerId");
        int donorId = (Integer)map.get("donorId");
        List<String> unbalancedStoreList = Utils.uncheckedCast(map.get("unbalancedStores"));
        Cluster initialCluster = new ClusterMapper()
                                     .readCluster(new StringReader((String)map.get("initialCluster")));

        HashMap<String, List<Integer>> storeToPartitionIds = Maps.newHashMap();
        for (String unbalancedStore : unbalancedStoreList) {
            List<Integer> partitionList = Utils.uncheckedCast(map.get(unbalancedStore + "partitionList"));
            if (partitionList.size() > 0)
                storeToPartitionIds.put(unbalancedStore, partitionList);
        }

        return new RebalanceTaskInfo(stealerId, donorId, storeToPartitionIds, initialCluster);
    }
572191
def from_proto(cls, repeated_split_infos):
    """"""
    split_dict = cls()
    for split_info_proto in repeated_split_infos:
      split_info = SplitInfo()
      split_info.CopyFrom(split_info_proto)
      split_dict.add(split_info)
    return split_dict
1010469
public static RebalanceTaskInfo decodeRebalanceTaskInfoMap(VAdminProto.RebalanceTaskInfoMap rebalanceTaskInfoMap) {
        RebalanceTaskInfo rebalanceTaskInfo = new RebalanceTaskInfo(
                                                                    rebalanceTaskInfoMap.getStealerId(),
                                                                    rebalanceTaskInfoMap.getDonorId(),
                                                                    decodeStoreToPartitionIds(rebalanceTaskInfoMap.getPerStorePartitionIdsList()),
                                                                    new ClusterMapper().readCluster(new StringReader(rebalanceTaskInfoMap.getInitialCluster())));
        return rebalanceTaskInfo;
    }
710010
public alluxio.grpc.MasterInfo getMasterInfo() {
    return masterInfo_ == null ? alluxio.grpc.MasterInfo.getDefaultInstance() : masterInfo_;
  }
618653
public static WorkerInfo fromProto(alluxio.grpc.WorkerInfo workerInfo) {
    return new WorkerInfo().setAddress(fromProto(workerInfo.getAddress()))
        .setCapacityBytes(workerInfo.getCapacityBytes())
        .setCapacityBytesOnTiers(workerInfo.getCapacityBytesOnTiers()).setId(workerInfo.getId())
        .setLastContactSec(workerInfo.getLastContactSec())
        .setStartTimeMs(workerInfo.getStartTimeMs()).setState(workerInfo.getState())
        .setUsedBytes(workerInfo.getUsedBytes())
        .setUsedBytesOnTiers(workerInfo.getUsedBytesOnTiersMap());
  }
741098
private Frame reBalance(final Frame fr, boolean local) {
    int chunks = (int)Math.min( 4 * H2O.NUMCPUS * (local ? 1 : H2O.CLOUD.size()), fr.numRows());
    if (fr.anyVec().nChunks() > chunks && !reproducible) {
      Log.info("Dataset already contains " + fr.anyVec().nChunks() + " chunks. No need to rebalance.");
      return fr;
    } else if (reproducible) {
      Log.warn("Reproducibility enforced - using only 1 thread - can be slow.");
      chunks = 1;
    }
    if (!quiet_mode) Log.info("ReBalancing dataset into (at least) " + chunks + " chunks.");
//      return MRUtils.shuffleAndBalance(fr, chunks, seed, local, shuffle_training_data);
    String snewKey = fr._key != null ? (fr._key.toString() + ".balanced") : Key.rand();
    Key newKey = Key.makeSystem(snewKey);
    RebalanceDataSet rb = new RebalanceDataSet(fr, newKey, chunks);
    H2O.submitTask(rb);
    rb.join();
    return UKV.get(newKey);
  }
899016
public static FileInfo fromProto(alluxio.grpc.FileInfo pInfo) {
    FileInfo fileInfo = new FileInfo().setFileId(pInfo.getFileId()).setName(pInfo.getName())
        .setPath(pInfo.getPath()).setUfsPath(pInfo.getUfsPath()).setLength(pInfo.getLength())
        .setBlockSizeBytes(pInfo.getBlockSizeBytes()).setCreationTimeMs(pInfo.getCreationTimeMs())
        .setCompleted(pInfo.getCompleted()).setFolder(pInfo.getFolder())
        .setPinned(pInfo.getPinned()).setCacheable(pInfo.getCacheable())
        .setPersisted(pInfo.getPersisted()).setBlockIds(pInfo.getBlockIdsList())
        .setLastModificationTimeMs(pInfo.getLastModificationTimeMs()).setTtl(pInfo.getTtl())
        .setTtlAction(pInfo.getTtlAction()).setOwner(pInfo.getOwner())
        .setGroup(pInfo.getGroup()).setMode(pInfo.getMode())
        .setPersistenceState(pInfo.getPersistenceState()).setMountPoint(pInfo.getMountPoint())
        .setFileBlockInfos(map(GrpcUtils::fromProto, pInfo.getFileBlockInfosList()))
        .setMountId(pInfo.getMountId()).setInAlluxioPercentage(pInfo.getInAlluxioPercentage())
        .setInMemoryPercentage(pInfo.getInMemoryPercentage())
        .setUfsFingerprint(pInfo.hasUfsFingerprint() ? pInfo.getUfsFingerprint()
            : Constants.INVALID_UFS_FINGERPRINT)
        .setAcl(pInfo.hasAcl() ? (fromProto(pInfo.getAcl())) : AccessControlList.EMPTY_ACL)
        .setDefaultAcl(
            pInfo.hasDefaultAcl() ? ((DefaultAccessControlList) fromProto(pInfo.getDefaultAcl()))
                : DefaultAccessControlList.EMPTY_DEFAULT_ACL)
        .setReplicationMax(pInfo.getReplicationMax()).setReplicationMin(pInfo.getReplicationMin());
    return fileInfo;
  }
1200429
LocatedBlocksWithMetaInfo updateDatanodeInfo(LocatedBlocks locatedBlocks)
      throws IOException {
    if (locatedBlocks.getLocatedBlocks().size() == 0)
      return new LocatedBlocksWithMetaInfo(locatedBlocks.getFileLength(),
          locatedBlocks.getLocatedBlocks(), false,
          DataTransferProtocol.DATA_TRANSFER_VERSION, getNamespaceId(),
          this.nameNode.getClientProtocolMethodsFingerprint());
    List<LocatedBlock> newBlocks = new ArrayList<LocatedBlock>();

    readLock();
    try {
      for (LocatedBlock locBlock: locatedBlocks.getLocatedBlocks()) {
        Block block = locBlock.getBlock();
        int numNodes = blocksMap.numNodes(block);
        int numCorruptNodes = countNodes(block).corruptReplicas();
        int numCorruptReplicas = corruptReplicas.numCorruptReplicas(block); 

        if (numCorruptNodes != numCorruptReplicas) {
          LOG.warn("Inconsistent number of corrupt replicas for " + 
                   block + "blockMap has " + numCorruptNodes + 
                   " but corrupt replicas map has " + numCorruptReplicas);
        }

        boolean blockCorrupt = numCorruptNodes == numNodes;
        int numMachineSet = blockCorrupt ? numNodes : (numNodes - numCorruptNodes);
        DatanodeDescriptor[] machineSet = new DatanodeDescriptor[numMachineSet];

        if (numMachineSet > 0) {
          numNodes = 0;
          for(Iterator<DatanodeDescriptor> it = blocksMap.nodeIterator(block); 
              it.hasNext();) {
            DatanodeDescriptor dn = it.next();
            boolean replicaCorrupt = corruptReplicas.isReplicaCorrupt(block, dn);
            if (blockCorrupt || (!blockCorrupt && !replicaCorrupt))
              machineSet[numNodes++] = dn;
          }
        }

        // We need to make a copy of the block object before releasing the lock
        // to prevent the state of block is changed after that and before the
        // object is serialized to clients, to avoid potential inconsistency.
        // Further optimization is possible to avoid some object copy. Since it
        // is so far not a critical path. We leave a safe approach here.
        //
        Block blockCopy  = null;
        if (block != null) {
          blockCopy = new Block(block);
        }
        LocatedBlock newBlock = new LocatedBlock(blockCopy, machineSet, 0,
            blockCorrupt);
  newBlocks.add(newBlock);
      }
    } finally {
      readUnlock();
    }

    return new LocatedBlocksWithMetaInfo(locatedBlocks.getFileLength(),
        newBlocks, false, DataTransferProtocol.DATA_TRANSFER_VERSION,
        getNamespaceId(), this.nameNode.getClientProtocolMethodsFingerprint());
  }
1273622
public static void printBatchTaskLog(int batchId, int taskId, Logger logger, String message) {
        logger.info("[Rebalance batch/task id " + batchId + "/" + taskId + "] " + message);
    }
27146
public synchronized void updateStorageInfo() {
    Map<String, Long> tierCapacities = mBlockWorker.getStoreMeta().getCapacityBytesOnTiers();
    long lastTierReservedBytes = 0;
    for (int ordinal = 0; ordinal < mStorageTierAssoc.size(); ordinal++) {
      String tierAlias = mStorageTierAssoc.getAlias(ordinal);
      long tierCapacity = tierCapacities.get(tierAlias);
      long reservedSpace;
      PropertyKey tierReservedSpaceProp =
          PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_RESERVED_RATIO.format(ordinal);
      if (ServerConfiguration.isSet(tierReservedSpaceProp)) {
        LOG.warn("The property reserved.ratio is deprecated, use high/low watermark instead.");
        reservedSpace = (long) (tierCapacity * ServerConfiguration
            .getDouble(tierReservedSpaceProp));
      } else {
        // High watermark defines when to start the space reserving process
        PropertyKey tierHighWatermarkProp =
            PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_HIGH_WATERMARK_RATIO.format(ordinal);
        double tierHighWatermarkConf = ServerConfiguration.getDouble(tierHighWatermarkProp);
        Preconditions.checkArgument(tierHighWatermarkConf > 0,
            "The high watermark of tier %s should be positive, but is %s",
            Integer.toString(ordinal), tierHighWatermarkConf);
        Preconditions.checkArgument(tierHighWatermarkConf < 1,
            "The high watermark of tier %s should be less than 1.0, but is %s",
            Integer.toString(ordinal), tierHighWatermarkConf);
        long highWatermark = (long) (tierCapacity * ServerConfiguration
            .getDouble(tierHighWatermarkProp));
        mHighWatermarks.put(tierAlias, highWatermark);

        // Low watermark defines when to stop the space reserving process if started
        PropertyKey tierLowWatermarkProp =
            PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_LOW_WATERMARK_RATIO.format(ordinal);
        double tierLowWatermarkConf = ServerConfiguration.getDouble(tierLowWatermarkProp);
        Preconditions.checkArgument(tierLowWatermarkConf >= 0,
            "The low watermark of tier %s should not be negative, but is %s",
            Integer.toString(ordinal), tierLowWatermarkConf);
        Preconditions.checkArgument(tierLowWatermarkConf < tierHighWatermarkConf,
            "The low watermark (%s) of tier %d should not be smaller than the high watermark (%s)",
            tierLowWatermarkConf, ordinal, tierHighWatermarkConf);
        reservedSpace =
            (long) (tierCapacity - tierCapacity * ServerConfiguration
                .getDouble(tierLowWatermarkProp));
      }
      lastTierReservedBytes += reservedSpace;
      // On each tier, we reserve no more than its capacity
      lastTierReservedBytes =
          (lastTierReservedBytes <= tierCapacity) ? lastTierReservedBytes : tierCapacity;
      mReservedSpaces.put(tierAlias, lastTierReservedBytes);
    }
  }
566911
public int getCrossZonePartitionStoreMoves() {
        int xzonePartitionStoreMoves = 0;
        for (RebalanceTaskInfo info : batchPlan) {
            Node donorNode = finalCluster.getNodeById(info.getDonorId());
            Node stealerNode = finalCluster.getNodeById(info.getStealerId());

            if(donorNode.getZoneId() != stealerNode.getZoneId()) {
                xzonePartitionStoreMoves += info.getPartitionStoreMoves();
            }
        }

        return xzonePartitionStoreMoves;
    }
644125
public static NodeInfo[] generateOrderedNodeInfos(Collection<WorkerInfo> workerInfos) {
    NodeInfo[] ret = new NodeInfo[workerInfos.size()];
    int index = 0;
    for (WorkerInfo workerInfo : workerInfos) {
      ret[index++] = new NodeInfo(workerInfo);
    }
    Arrays.sort(ret);

    return ret;
  }
758557
private static FieldInfo cloneDeprecated(FieldInfo fieldInfo) {
	return new FieldInfo(fieldInfo.getAccess() | Opcodes.ACC_DEPRECATED,
		fieldInfo.getName(), fieldInfo.getDesc(),
		fieldInfo.getSignature(), fieldInfo.getValue());
    }
631282
public static AccountInfo adaptAccountInfo(BitstampBalance bitstampBalance, String userName) {

    // Adapt to XChange DTOs
    List<Balance> balances = new ArrayList<>();
    for (org.knowm.xchange.bitstamp.dto.account.BitstampBalance.Balance b :
        bitstampBalance.getBalances()) {
      Balance xchangeBalance =
          new Balance(
              Currency.getInstance(b.getCurrency().toUpperCase()),
              b.getBalance(),
              b.getAvailable(),
              b.getReserved(),
              ZERO,
              ZERO,
              b.getBalance().subtract(b.getAvailable()).subtract(b.getReserved()),
              ZERO);
      balances.add(xchangeBalance);
    }
    return new AccountInfo(userName, bitstampBalance.getFee(), new Wallet(balances));
  }
591929
@Override
	public Protos.TaskInfo launch(Protos.SlaveID slaveId, MesosResourceAllocation allocation) {

		ContaineredTaskManagerParameters tmParams = params.containeredParameters();

		final Configuration dynamicProperties = new Configuration();

		// incorporate the dynamic properties set by the template
		dynamicProperties.addAll(containerSpec.getDynamicConfiguration());

		// build a TaskInfo with assigned resources, environment variables, etc
		final Protos.TaskInfo.Builder taskInfo = Protos.TaskInfo.newBuilder()
			.setSlaveId(slaveId)
			.setTaskId(taskID)
			.setName(taskID.getValue());

		// take needed resources from the overall allocation, under the assumption of adequate resources
		Set<String> roles = mesosConfiguration.roles();
		taskInfo.addAllResources(allocation.takeScalar("cpus", taskRequest.getCPUs(), roles));
		taskInfo.addAllResources(allocation.takeScalar("gpus", taskRequest.getGPUs(), roles));
		taskInfo.addAllResources(allocation.takeScalar("mem", taskRequest.getMemory(), roles));

		if (taskRequest.getDisk() > 0.0) {
			taskInfo.addAllResources(allocation.takeScalar("disk", taskRequest.getDisk(), roles));
		}

		final Protos.CommandInfo.Builder cmd = taskInfo.getCommandBuilder();
		final Protos.Environment.Builder env = cmd.getEnvironmentBuilder();
		final StringBuilder jvmArgs = new StringBuilder();

		//configure task manager hostname property if hostname override property is supplied
		Option<String> taskManagerHostnameOption = params.getTaskManagerHostname();

		if (taskManagerHostnameOption.isDefined()) {
			// replace the TASK_ID pattern by the actual task id value of the Mesos task
			final String taskManagerHostname = MesosTaskManagerParameters.TASK_ID_PATTERN
				.matcher(taskManagerHostnameOption.get())
				.replaceAll(Matcher.quoteReplacement(taskID.getValue()));

			dynamicProperties.setString(TaskManagerOptions.HOST, taskManagerHostname);
		}

		// take needed ports for the TM
		Set<String> tmPortKeys = extractPortKeys(containerSpec.getDynamicConfiguration());
		List<Protos.Resource> portResources = allocation.takeRanges("ports", tmPortKeys.size(), roles);
		taskInfo.addAllResources(portResources);
		Iterator<String> portsToAssign = tmPortKeys.iterator();
		rangeValues(portResources).forEach(port -> dynamicProperties.setLong(portsToAssign.next(), port));
		if (portsToAssign.hasNext()) {
			throw new IllegalArgumentException("insufficient # of ports assigned");
		}

		// ship additional files
		for (ContainerSpecification.Artifact artifact : containerSpec.getArtifacts()) {
			cmd.addUris(Utils.uri(resolver, artifact));
		}

		// add user-specified URIs
		for (String uri : params.uris()) {
			cmd.addUris(CommandInfo.URI.newBuilder().setValue(uri));
		}

		// propagate environment variables
		for (Map.Entry<String, String> entry : params.containeredParameters().taskManagerEnv().entrySet()) {
			env.addVariables(variable(entry.getKey(), entry.getValue()));
		}
		for (Map.Entry<String, String> entry : containerSpec.getEnvironmentVariables().entrySet()) {
			env.addVariables(variable(entry.getKey(), entry.getValue()));
		}

		// propagate the Mesos task ID to the TM
		env.addVariables(variable(MesosConfigKeys.ENV_FLINK_CONTAINER_ID, taskInfo.getTaskId().getValue()));

		// finalize the memory parameters
		jvmArgs.append(" -Xms").append(tmParams.taskManagerHeapSizeMB()).append("m");
		jvmArgs.append(" -Xmx").append(tmParams.taskManagerHeapSizeMB()).append("m");
		if (tmParams.taskManagerDirectMemoryLimitMB() >= 0) {
			jvmArgs.append(" -XX:MaxDirectMemorySize=").append(tmParams.taskManagerDirectMemoryLimitMB()).append("m");
		}

		// pass dynamic system properties
		jvmArgs.append(' ').append(
			ContainerSpecification.formatSystemProperties(containerSpec.getSystemProperties()));

		// finalize JVM args
		env.addVariables(variable(MesosConfigKeys.ENV_JVM_ARGS, jvmArgs.toString()));

		// populate TASK_NAME and FRAMEWORK_NAME environment variables to the TM container
		env.addVariables(variable(MesosConfigKeys.ENV_TASK_NAME, taskInfo.getTaskId().getValue()));
		env.addVariables(variable(MesosConfigKeys.ENV_FRAMEWORK_NAME, mesosConfiguration.frameworkInfo().getName()));

		// build the launch command w/ dynamic application properties
		StringBuilder launchCommand = new StringBuilder();
		if (params.bootstrapCommand().isDefined()) {
			launchCommand.append(params.bootstrapCommand().get()).append(" && ");
		}
		launchCommand
			.append(params.command())
			.append(" ")
			.append(ContainerSpecification.formatSystemProperties(dynamicProperties));
		cmd.setValue(launchCommand.toString());

		// build the container info
		Protos.ContainerInfo.Builder containerInfo = Protos.ContainerInfo.newBuilder();
		// in event that no docker image or mesos image name is specified, we must still
		// set type to MESOS
		containerInfo.setType(Protos.ContainerInfo.Type.MESOS);
		switch (params.containerType()) {
			case MESOS:
				if (params.containerImageName().isDefined()) {
					containerInfo
						.setMesos(Protos.ContainerInfo.MesosInfo.newBuilder()
							.setImage(Protos.Image.newBuilder()
								.setType(Protos.Image.Type.DOCKER)
								.setDocker(Protos.Image.Docker.newBuilder()
									.setName(params.containerImageName().get()))));
				}
				break;

			case DOCKER:
				assert(params.containerImageName().isDefined());
				containerInfo
					.setType(Protos.ContainerInfo.Type.DOCKER)
					.setDocker(Protos.ContainerInfo.DockerInfo.newBuilder()
						.addAllParameters(params.dockerParameters())
						.setNetwork(Protos.ContainerInfo.DockerInfo.Network.HOST)
						.setImage(params.containerImageName().get())
						.setForcePullImage(params.dockerForcePullImage()));
				break;

			default:
				throw new IllegalStateException("unsupported container type");
		}

		// add any volumes to the containerInfo
		containerInfo.addAllVolumes(params.containerVolumes());
		taskInfo.setContainer(containerInfo);

		return taskInfo.build();
	}
531113
public Protos.TaskInfo constructMesosTaskInfo(Config heronConfig, Config heronRuntime) {
    //String taskIdStr, BaseContainer task, Offer offer
    String taskIdStr = this.taskId;

    Protos.TaskID mesosTaskID = Protos.TaskID.newBuilder().setValue(taskIdStr).build();
    Protos.TaskInfo.Builder taskInfo = Protos.TaskInfo.newBuilder()
        .setName(baseContainer.name)
        .setTaskId(mesosTaskID);
    Protos.Environment.Builder environment = Protos.Environment.newBuilder();

    // If the job defines custom environment variables, add them to the builder
    // Don't add them if they already exist to prevent overwriting the defaults
    Set<String> builtinEnvNames = new HashSet<>();
    for (Protos.Environment.Variable variable : environment.getVariablesList()) {
      builtinEnvNames.add(variable.getName());
    }

    for (BaseContainer.EnvironmentVariable ev : baseContainer.environmentVariables) {
      environment.addVariables(
          Protos.Environment.Variable.newBuilder().setName(ev.name).setValue(ev.value));
    }

    taskInfo
        .addResources(scalarResource(TaskResources.CPUS_RESOURCE_NAME, baseContainer.cpu))
        .addResources(scalarResource(TaskResources.MEM_RESOURCE_NAME, baseContainer.memInMB))
        .addResources(scalarResource(TaskResources.DISK_RESOURCE_NAME, baseContainer.diskInMB))
        .addResources(rangeResource(TaskResources.PORT_RESOURCE_NAME,
            this.freePorts.get(0), this.freePorts.get(this.freePorts.size() - 1))).
        setSlaveId(this.offer.getSlaveId());

    int containerIndex = TaskUtils.getContainerIndexForTaskId(taskIdStr);
    String commandStr = executorCommand(heronConfig, heronRuntime, containerIndex);

    Protos.CommandInfo.Builder command = Protos.CommandInfo.newBuilder();

    List<Protos.CommandInfo.URI> uriProtos = new ArrayList<>();
    for (String uri : baseContainer.dependencies) {
      uriProtos.add(Protos.CommandInfo.URI.newBuilder()
          .setValue(uri)
          .setExtract(true)
          .build());
    }

    command.setValue(commandStr)
        .setShell(baseContainer.shell)
        .setEnvironment(environment)
        .addAllUris(uriProtos);

    if (!baseContainer.runAsUser.isEmpty()) {
      command.setUser(baseContainer.runAsUser);
    }
    taskInfo.setCommand(command);

    return taskInfo.build();
  }