Return a local map spill index file created earlier
@param mapTaskId a map task id
@param spillNumber the number
549761
public Path getSpillIndexFile(TaskAttemptID mapTaskId, int spillNumber)
    throws IOException {
    return lDirAlloc.getLocalPathToRead(TaskTracker.getIntermediateOutputDir(
                       jobId.toString(), mapTaskId.toString())
                       + "/spill" + 
                       spillNumber + ".out.index", conf);
  }
774393
def _spill(self):
        
        global MemoryBytesSpilled, DiskBytesSpilled
        path = self._get_spill_dir(self.spills)
        if not os.path.exists(path):
            os.makedirs(path)

        used_memory = get_used_memory()
        if not self.pdata:
            # The data has not been partitioned, it will iterator the
            # data once, write them into different files, has no
            # additional memory. It only called when the memory goes
            # above limit at the first time.

            # open all the files for writing
            streams = [open(os.path.join(path, str(i)), 'wb')
                       for i in range(self.partitions)]

            # If the number of keys is small, then the overhead of sort is small
            # sort them before dumping into disks
            self._sorted = len(self.data) < self.SORT_KEY_LIMIT
            if self._sorted:
                self.serializer = self.flattened_serializer()
                for k in sorted(self.data.keys()):
                    h = self._partition(k)
                    self.serializer.dump_stream([(k, self.data[k])], streams[h])
            else:
                for k, v in self.data.items():
                    h = self._partition(k)
                    self.serializer.dump_stream([(k, v)], streams[h])

            for s in streams:
                DiskBytesSpilled += s.tell()
                s.close()

            self.data.clear()
            # self.pdata is cached in `mergeValues` and `mergeCombiners`
            self.pdata.extend([{} for i in range(self.partitions)])

        else:
            for i in range(self.partitions):
                p = os.path.join(path, str(i))
                with open(p, "wb") as f:
                    # dump items in batch
                    if self._sorted:
                        # sort by key only (stable)
                        sorted_items = sorted(self.pdata[i].items(), key=operator.itemgetter(0))
                        self.serializer.dump_stream(sorted_items, f)
                    else:
                        self.serializer.dump_stream(self.pdata[i].items(), f)
                self.pdata[i].clear()
                DiskBytesSpilled += os.path.getsize(p)

        self.spills += 1
        gc.collect()  # release the memory as much as possible
        MemoryBytesSpilled += max(used_memory - get_used_memory(), 0) << 20
582075
@java.lang.Deprecated
  public java.util.Map<java.lang.Integer, java.lang.String> getTasks() {
    return getTasksMap();
  }
544047
private String getTaskCacheDirectory(TaskControllerContext context) {
    // In the case of JVM reuse, the task specific directory
    // is different from what is set with respect with
    // env.workDir. Hence building this from the taskId everytime.
    String taskId = context.task.getTaskID().toString();
    File cacheDirForJob = context.env.workDir.getParentFile().getParentFile();
    if(context.task.isTaskCleanupTask()) {
      taskId = taskId + TaskTracker.TASK_CLEANUP_SUFFIX;
    }
    return new File(cacheDirForJob, taskId).getAbsolutePath(); 
  }
727516
public int getThisTaskIndex() {
        List<Integer> tasks = new ArrayList<>(getComponentTasks(getThisComponentId()));
        Collections.sort(tasks);
        for (int i = 0; i < tasks.size(); i++) {
            if (tasks.get(i) == getThisTaskId()) {
                return i;
            }
        }
        throw new RuntimeException("Fatal: could not find this task id in this component");
    }
658040
public synchronized Task obtainNewReduceTask(TaskTrackerStatus tts,
                                               int clusterSize,
                                               int numUniqueHosts
                                              ) throws IOException {
    if (status.getRunState() != JobStatus.RUNNING) {
      LOG.info("Cannot create task split for " + profile.getJobID());
      return null;
    }

    // Ensure we have sufficient map outputs ready to shuffle before
    // scheduling reduces
    if (!scheduleReduces()) {
      return null;
    }

    int  target = findNewReduceTask(tts, clusterSize, numUniqueHosts);
    if (target == -1) {
      return null;
    }

    Task result = reduces[target].getTaskToRun(tts.getTrackerName());
    if (result != null) {
      addRunningTaskToTIP(reduces[target], result.getTaskID(), tts, true);
    }

    return result;
  }
657418
protected Map<Address, Long> getSelectionMap(String wmId, DistributableWork work)
   {
      if (trace)
      {
         log.tracef("getSelectionMap(%s, %s)", wmId, work);
         log.tracef("ShortRunning: %s", shortRunning);
         log.tracef("LongRunning: %s", longRunning);
      }

      Map<Address, Long> sorted = null;

      if (WorkManagerUtil.isLongRunning(work) && longRunning.get(wmId) != null)
      {
         sorted = new HashMap<Address, Long>(longRunning.get(wmId));
      }

      if (sorted == null && shortRunning.get(wmId) != null)
         sorted = new HashMap<Address, Long>(shortRunning.get(wmId));

      return sorted;
   }
448597
public static MapReduceSpecification<Long, Integer, Integer, ArrayList<Integer>,
      GoogleCloudStorageFileSet> createMapReduceSpec(String bucket, long start, long limit,
          int shards) {
    ConsecutiveLongInput input = new ConsecutiveLongInput(start, limit, shards);
    Mapper<Long, Integer, Integer> mapper = new SeedToRandomMapper();
    Marshaller<Integer> intermediateKeyMarshaller = Marshallers.getIntegerMarshaller();
    Marshaller<Integer> intermediateValueMarshaller = Marshallers.getIntegerMarshaller();
    Reducer<Integer, Integer, ArrayList<Integer>> reducer = new CollisionFindingReducer();
    Marshaller<ArrayList<Integer>> outputMarshaller = Marshallers.getSerializationMarshaller();

    Output<ArrayList<Integer>, GoogleCloudStorageFileSet> output = new MarshallingOutput<>(
        new GoogleCloudStorageFileOutput(bucket, "CollidingSeeds-%04d", "integers"),
        outputMarshaller);
    // [START mapReduceSpec]
    MapReduceSpecification<Long, Integer, Integer, ArrayList<Integer>, GoogleCloudStorageFileSet>
        spec = new MapReduceSpecification.Builder<>(input, mapper, reducer, output)
            .setKeyMarshaller(intermediateKeyMarshaller)
            .setValueMarshaller(intermediateValueMarshaller)
            .setJobName("DemoMapreduce")
            .setNumReducers(shards)
            .build();
    // [END mapReduceSpec]
    return spec;
  }
557700
public synchronized String lookupContainer(String lookup) {
        if (aliasToContainerMap.containsKey(lookup)) {
            return aliasToContainerMap.get(lookup);
        }
        return imageToContainerMap.get(lookup);
    }
35399
public synchronized static String getUniqueFile(TaskAttemptContext context,
                                                  String name,
                                                  String extension) {
    TaskID taskId = context.getTaskAttemptID().getTaskID();
    int partition = taskId.getId();
    StringBuilder result = new StringBuilder();
    result.append(name);
    result.append('-');
    result.append(taskId.isMap() ? 'm' : 'r');
    result.append('-');
    result.append(NUMBER_FORMAT.format(partition));
    result.append(extension);
    return result.toString();
  }
1099146
public final V get(K key) {
        if (key == null) {
            throw new NullPointerException("key == null");
        }

        V mapValue;
        synchronized (this) {
            mapValue = map.get(key);
            if (mapValue != null) {
                hitCount++;
                return mapValue;
            }
            missCount++;
        }

        /*
         * Attempt to create a value. This may take a long time, and the map
         * may be different when create() returns. If a conflicting value was
         * added to the map while create() was working, we leave that value in
         * the map and release the created value.
         */

        V createdValue = create(key);
        if (createdValue == null) {
            return null;
        }

        synchronized (this) {
            createCount++;
            mapValue = map.put(key, createdValue);

            if (mapValue != null) {
                // There was a conflict so undo that last put
                map.put(key, mapValue);
            } else {
                size += safeSizeOf(key, createdValue);
            }
        }

        if (mapValue != null) {
            entryRemoved(false, key, createdValue, mapValue);
            return mapValue;
        } else {
            trimToSize(maxSize);
            return createdValue;
        }
    }
297006
def _get_spill_dir(self, n):
        
        return os.path.join(self.localdirs[n % len(self.localdirs)], str(n))
395465
public static int getNumberOfDependentLayers(String imageContent) throws IOException {
        JsonNode history = Utils.mapper().readTree(imageContent).get("history");
        if (history == null) {
            throw new IllegalStateException("Could not find 'history' tag");
        }

        int layersNum = history.size();
        boolean newImageLayers = true;
        for (int i = history.size() - 1; i >= 0; i--) {

            if (newImageLayers) {
                layersNum--;
            }

            JsonNode layer = history.get(i);
            JsonNode emptyLayer = layer.get("empty_layer");
            if (!newImageLayers && emptyLayer != null) {
                layersNum--;
            }

            if (layer.get("created_by") == null) {
                continue;
            }
            String createdBy = layer.get("created_by").textValue();
            if (createdBy.contains("ENTRYPOINT") || createdBy.contains("MAINTAINER")) {
                newImageLayers = false;
            }
        }
        return layersNum;
    }
1153299
@Override
	@Pure
	public RoadPolyline getMapElementAt(int index) {
		final Iterator<RoadSegment> segments = this.roadNetwork.iterator();
		for (int i = 0; i < index - 1 && segments.hasNext(); ++i) {
			segments.next();
		}
		if (segments.hasNext()) {
			return (RoadPolyline) segments.next();
		}
		throw new IndexOutOfBoundsException();
	}
120295
public static String getTaskId(final Configuration config) {
    try {
      return Tang.Factory.getTang().newInjector(config).getNamedInstance(TaskConfigurationOptions.Identifier.class);
    } catch (final InjectionException ex) {
      throw new RuntimeException("Unable to determine task identifier. Giving up.", ex);
    }
  }
482283
public static int getAttemptId(Configuration conf)
      throws IllegalArgumentException {
    if (conf == null) {
      throw new NullPointerException("Conf is null");
    }

    String taskId = conf.get("mapred.task.id");
    if (taskId == null) {
      throw new IllegalArgumentException(
          "Configuration does not contain the property mapred.task.id");
    }

    String[] parts = taskId.split("_");
    if (parts.length != 6 ||
        !parts[0].equals("attempt") ||
        (!"m".equals(parts[3]) && !"r".equals(parts[3]))) {
      throw new IllegalArgumentException(
          "TaskAttemptId string : " + taskId + " is not properly formed");
    }

    return Integer.parseInt(parts[5]);
  }
736164
public Path getInputFile(int mapId, TaskAttemptID reduceTaskId)
    throws IOException {
    // TODO *oom* should use a format here
    return lDirAlloc.getLocalPathToRead(TaskTracker.getIntermediateOutputDir(
                       jobId.toString(), reduceTaskId.toString())
                       + "/map_" + mapId + ".out",
                       conf);
  }