Emit a bunch of logging output at the beginning of the map task.
@throws IOException
@throws InterruptedException
198039
private void emitLogHeader(Context context, String mapredTaskId) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    Text textId = new Text(mapredTaskId);

    for (Map.Entry<String, String> entry: conf) {
      StringBuilder sb = new StringBuilder();
      sb.append(entry.getKey());
      sb.append("=");
      sb.append(entry.getValue());
      context.write(textId, new Text(sb.toString()));
    }

    context.write(textId, new Text("----- Properties -----"));
    String[] plist = {
            "mapred.local.dir",
            "mapred.child.java.opts",
    };
    for (String k : plist) {
      String v = conf.get(k);
      if (v == null) {
        v = "(null)";
      }
      context.write(textId, new Text(k + " " + v));
    }
    String userDir = System.getProperty("user.dir");
    context.write(textId, new Text("user.dir " + userDir));

    try {
      java.net.InetAddress localMachine = java.net.InetAddress.getLocalHost();
      context.write(textId, new Text("hostname " + localMachine.getHostName()));
    }
    catch (java.net.UnknownHostException uhe) {
      // handle exception
    }
  }
1270662
public void startReading() {
    logcat.setListener(new Logcat.Listener() {
      @Override public void onTraceRead(String logcatTrace) {
        try {
          addTraceToTheBuffer(logcatTrace);
        } catch (IllegalTraceException e) {
          return;
        }
        notifyNewTraces();
      }
    });
    boolean logcatWasNotStarted = Thread.State.NEW.equals(logcat.getState());
    if (logcatWasNotStarted) {
      logcat.start();
    }
  }
68756
private void init() 
            throws IOException, InterruptedException {
        
        // get job config properties
        boolean advancedMode = 
            conf.get(INPUT_MODE, BASIC_MODE).equals(ADVANCED_MODE);
        redactionRuleCol = conf.getStrings(REDACTION_RULE_COLLECTION);
        
        // initialize the total length
        float recToFragRatio = conf.getFloat(RECORD_TO_FRAGMENT_RATIO, 
                getDefaultRatio());
        length = mlSplit.getLength() * recToFragRatio;
        
        // generate the query
        String queryLanguage = null;
        String queryText = null;
        long start = mlSplit.getStart() + 1 + count;
        long end = mlSplit.isLastSplit() ? 
                   Long.MAX_VALUE : start + mlSplit.getLength() - count - 1;
        if (!advancedMode) { 
            StringBuilder buf = new StringBuilder();      
            buf.append("xquery version \"1.0-ml\"; \n");
            if (redactionRuleCol != null) {
            	buf.append(
            		"import module namespace rdt = \"http://marklogic.com/xdmp/redaction\" at \"/MarkLogic/redaction.xqy\";\n");
            }
            String indent = conf.get(INDENTED, "FALSE");
            Indentation ind = Indentation.valueOf(indent);
            buf.append(
                    "declare namespace mlmr=\"http://marklogic.com/hadoop\";\n");
            buf.append(
                    "declare variable $mlmr:splitstart as xs:integer external;\n");
            buf.append(
                    "declare variable $mlmr:splitend as xs:integer external;\n");
           
            buf.append(ind.getStatement());
            buf.append("\n");
            String docExpr = conf.get(DOCUMENT_SELECTOR);
            String ctsQuery = conf.get(QUERY_FILTER);
            String subExpr = conf.get(SUBDOCUMENT_EXPRESSION);
            Collection<String> nsCol = docExpr != null || subExpr != null? 
                    conf.getStringCollection(PATH_NAMESPACE) : null;
            if (ctsQuery != null) {
                buildSearchQuery(docExpr, ctsQuery, nsCol, buf);
                queryText = buf.toString();
            } else if (docExpr != null || subExpr != null) {   
                buildDocExprQuery(docExpr, nsCol, subExpr, buf);
                queryText = buf.toString();
            } else {
                LexiconFunction function = null;
                Class<? extends LexiconFunction> lexiconClass = 
                        conf.getClass(INPUT_LEXICON_FUNCTION_CLASS, null,
                                LexiconFunction.class);
                if (lexiconClass != null) {
                    function = ReflectionUtils.newInstance(lexiconClass, 
                            conf);
                    if (nsCol == null) {
                        nsCol = conf.getStringCollection(PATH_NAMESPACE);
                    }
                    queryText = function.getInputQuery(nsCol, start, 
                            mlSplit.isLastSplit() ? 
                                    Long.MAX_VALUE : (long)length);
                }
            } 
            if (queryText == null) {
                buildDocExprQuery(docExpr, nsCol, subExpr, buf);
                queryText = buf.toString();
            }
        } else {
            queryText = conf.get(MarkLogicConstants.INPUT_QUERY);
            queryLanguage = conf.get(MarkLogicConstants.INPUT_QUERY_LANGUAGE);
            if (queryText == null) {
                throw new IllegalStateException(
                  "Input query is required in advanced mode but not defined.");
            }
        }
        
        if (LOG.isDebugEnabled()) {
            LOG.debug(queryText);
        }


        // set up a connection to the server
        while (retry < maxRetries) {
        try {
            if (retry == 1) {
                LOG.info("Retrying connect");
            }
            String curForestName = "";
            String curHostName = "";
            if (curForest == -1) {
                curForestName = mlSplit.getForestId().toString();
                curHostName = hostNames[0];
            } else {
                curForestName = replicas.get(curForest).getForest();
                curHostName = replicas.get(curForest).getHostName();
            }
            ContentSource cs = InternalUtilities.getInputContentSource(conf, 
                    curHostName);
            session = cs.newSession("#"+curForestName);
            if (LOG.isDebugEnabled()) {
                LOG.debug("Connect to forest "
                    + curForestName + " on "
                    + session.getConnectionUri().getHost());
            }
            AdhocQuery query = session.newAdhocQuery(queryText);
            if (advancedMode) {
                boolean bindSplitRange = 
                        conf.getBoolean(BIND_SPLIT_RANGE, false);
                if (bindSplitRange) {
                    query.setNewIntegerVariable(MR_NAMESPACE, 
                            SPLIT_START_VARNAME, start);
                    query.setNewIntegerVariable(MR_NAMESPACE, 
                            SPLIT_END_VARNAME, end);
                } else {
                    query.setPosition(start);
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("split start position: " + start);
                    }        
                    query.setCount(mlSplit.isLastSplit() ?
                            Long.MAX_VALUE : mlSplit.getLength());
                }
            } else {
                query.setNewIntegerVariable(MR_NAMESPACE, 
                        SPLIT_START_VARNAME, start);
                query.setNewIntegerVariable(MR_NAMESPACE, 
                        SPLIT_END_VARNAME, end);
            }
            RequestOptions options = new RequestOptions();
            options.setCacheResult(false);
            if (queryLanguage != null) {
            	options.setQueryLanguage(queryLanguage);
            }
            String ts = conf.get(INPUT_QUERY_TIMESTAMP);
            if (ts != null) {
                options.setEffectivePointInTime(new BigInteger(ts));
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Query timestamp: " + ts);
                }
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug("Input query: " + query.getQuery());
            }
            query.setOptions(options);       
            result = session.submitRequest(query);
        } catch (XccConfigException e) {
            LOG.error("XccConfigException:" + e);
            throw new IOException(e);
        } catch (QueryException e) {
            LOG.error("QueryException:" + e);
            throw new IOException(e);
        } catch (RequestException e) {
            LOG.error("RequestException:" + e.getMessage());
            if (curForest != -1) {
                if (++retry < maxRetries) {
                    // failover
                    try {
                        InternalUtilities.sleep(sleepTime);
                    } catch (Exception e2) {
                    }
                    sleepTime = Math.min(sleepTime * 2,maxSleepTime);

                    curForest = (curForest+1)%replicas.size();
                    continue;
                 }
                 LOG.info("Exceeded max retry");
            }
            LOG.info("Query: " + queryText);
            if (LOG.isDebugEnabled()) {
                LOG.debug("RequestException: " + e);
            }
            throw new IOException(e);
        }
        break;
        } // retry
    }
874600
private void output(final LogEntry logEntry, final Iterable<Writer> writers) {
		if (writingThread == null) {
			for (Writer writer : writers) {
				try {
					writer.write(logEntry);
				} catch (Exception ex) {
					InternalLogger.log(Level.ERROR, ex, "Failed to write log entry '" + logEntry.getMessage() + "'");
				}
			}
		} else {
			for (Writer writer : writers) {
				writingThread.add(writer, logEntry);
			}
		}
	}
779138
private void writeStartSequence(IXADataRecorder dataRecorder)
			throws IOException, InterruptedException {

		LogRecordWriter writer = new LogRecordWriter();
		writer.writeLong(dataRecorder.getXADataRecorderId());
		dataRecorder.writeRollbackData(writer.toByteArray());
	}
859728
private void executeScript() throws IOException, InterruptedException {

        File script = new File(m_scriptPath);
        List<String> params = new LinkedList<String>();
        params.add(script.getAbsolutePath());
        params.addAll(m_writtenFiles);
        ProcessBuilder pb = new ProcessBuilder(params.toArray(new String[params.size()]));
        pb.directory(new File(script.getParent()));
        Process pr = pb.start();
        pr.waitFor();
        BufferedReader buf = new BufferedReader(new InputStreamReader(pr.getInputStream()));
        while (buf.ready()) {
            String line = buf.readLine();
            if (CmsStringUtil.isNotEmptyOrWhitespaceOnly(line)) {
                getReport().println(
                    Messages.get().container(Messages.RPT_OUTPUT_WEBSERVER_1, buf.readLine()),
                    I_CmsReport.FORMAT_OK);
            }
        }
    }
855883
void doAwaitUntil() throws InterruptedException {
        lock.lock();
        try {
            condition.await(DEFAULT_TIMEOUT_MILLISECONDS, TimeUnit.MILLISECONDS);
        } finally {
            lock.unlock();
        }
    }
564812
public void setChannel(@Nonnull InputStream in, @Nonnull OutputStream out,
                           @Nonnull TaskListener taskListener,
                           @CheckForNull Channel.Listener listener) throws IOException, InterruptedException {
        setChannel(in,out,taskListener.getLogger(),listener);
    }
1031580
public synchronized static void setStdout(OutputStream os)
  {
    if (_stdoutStream == null) {
      initStdout();
    }

    if (os == _systemErr || os == _systemOut) {
      return;
    }

    if (os instanceof WriteStream) {
      WriteStream out = (WriteStream) os;

      /*
      if (out.getSource() == StdoutStream.create()
          || out.getSource() == StderrStream.create()) {
        return;
      }
      */
    }
    
    _stdoutStream.setStream(os);
  }
1081667
@SuppressWarnings("deprecation")
  void init() throws InterruptedException, IOException {
    long now = System.currentTimeMillis();

    JobConf jobConf = new JobConf(getConf());
    jobConf.setClass("topology.node.switch.mapping.impl",
        StaticMapping.class, DNSToSwitchMapping.class);
    jobConf.set("fs.default.name", "file:///");
    jobConf.set("mapred.job.tracker", "localhost:8012");
    jobConf.setInt("mapred.jobtracker.job.history.block.size", 512);
    jobConf.setInt("mapred.jobtracker.job.history.buffer.size", 512);
    jobConf.setLong("mapred.tasktracker.expiry.interval", 5000);
    jobConf.setInt("mapred.reduce.copy.backoff", 4);
    jobConf.setLong("mapred.job.reuse.jvm.num.tasks", -1);
    jobConf.setUser("mumak");
    jobConf.set("mapred.system.dir", 
        jobConf.get("hadoop.log.dir", "/tmp/hadoop-"+jobConf.getUser()) + "/mapred/system");
    jobConf.set("mapred.jobtracker.taskScheduler", JobQueueTaskScheduler.class.getName());
    
    FileSystem lfs = FileSystem.getLocal(getConf());
    Path logPath =
      new Path(System.getProperty("hadoop.log.dir")).makeQualified(lfs);
    jobConf.set("mapred.system.dir", logPath.toString());
    jobConf.set("hadoop.job.history.location", (new Path(logPath, "history")
        .toString()));
    
    jt = SimulatorJobTracker.startTracker(jobConf, now, this);
    jt.offerService();
    
    // max Map/Reduce tasks per node
    int maxMaps = getConf().getInt("mapred.tasktracker.map.tasks.maximum",
        DEFAULT_MAP_SLOTS_PER_NODE);
    int maxReduces = getConf().getInt(
        "mapred.tasktracker.reduce.tasks.maximum",
        DEFAULT_REDUCE_SLOTS_PER_NODE);

    MachineNode defaultNode = new MachineNode.Builder("default", 2)
        .setMapSlots(maxMaps).setReduceSlots(maxReduces).build();
    ZombieCluster cluster = new ZombieCluster(new Path(topologyFile), 
        defaultNode, jobConf);
    long firstJobStartTime = now + 60000;
    JobStoryProducer jobStoryProducer = new SimulatorJobStoryProducer(
        new Path(traceFile), cluster, firstJobStartTime, jobConf);
    
    jc = new SimulatorJobClient(jt, jobStoryProducer);
    queue.addAll(jc.init(firstJobStartTime));

    // create TTs based on topology.json     
    startTaskTrackers(cluster, now);
    
    terminateTime = getConf().getLong("mumak.terminate.time", Long.MAX_VALUE);
    if (terminateTime <= 0) {
      throw new IllegalArgumentException("Terminate time must be positive: "
          + terminateTime);
    }
  }
374952
private String write(String logDir) throws IOException, ExecutionException, InterruptedException {
        final File file = new File(logDir, "trace_" + System.currentTimeMillis() + ".json.gz");
        if (file.exists()) {
            throw new IOException("Trace file " + file.getAbsolutePath() + " already exists");
        }
        if (!file.getParentFile().canWrite() || !file.getParentFile().canExecute()) {
            throw new IOException("Trace file " + file.getAbsolutePath() + " is not writable");
        }

        SettableFuture<Future<?>> f = SettableFuture.create();
        m_work.offer(() -> f.set(dumpEvents(file)));
        final Future<?> writeFuture = f.get();
        if (writeFuture != null) {
            writeFuture.get(); // Wait for the write to finish without blocking new events
            return file.getAbsolutePath();
        } else {
            // A write is already in progress, ignore this request
            return null;
        }
    }
278019
protected void throwException() throws InterruptedException, IOException {

        if (this.exception instanceof IOException) {
            throw (IOException) this.exception;
        }
        if (this.exception instanceof InterruptedException) {
            throw (InterruptedException) this.exception;
        }
        if (this.exception instanceof RuntimeException) {
            throw (RuntimeException) this.exception;
        }
        throw new RuntimeException(this.exception);
    }
888669
public static void main(final String[] args) throws IOException, InterruptedException {
        final NamedEntityAnalyser nea = new NamedEntityAnalyser(new Configuration());
        if (args.length >= 1) {
            final String content = new String(Files.readAllBytes(Paths.get(args[0])));
            System.out.println(nea.process(content));
        }
        else {
            try (final Scanner in = new Scanner(System.in)) {
                while (true) {
                    System.out.println("Type in texts, -q for quit.");
                    System.out.print("> ");
                    final String processString = in.nextLine();
                    if ("-q".equalsIgnoreCase(processString)) break;
                    System.out.println(nea.process(processString).getMappedResult());
                }
            }
        }
    }
1088959
private void completedTasklets(final TOutput output, final List<Integer> taskletIds)
      throws InterruptedException {
    final List<TInput> inputs = getInputs(taskletIds);
    final AggregateResult result = new AggregateResult(output, inputs);

    if (callbackHandler != null) {
      executor.execute(new Runnable() {
        @Override
        public void run() {
          callbackHandler.onSuccess(result);
        }
      });
    }

    resultQueue.put(new ImmutablePair<>(taskletIds, result));
  }
1035089
private synchronized void emitRecords() throws IOException {
    for (String recordName : bufferedData.keySet()) {
      RecordMap recordMap = bufferedData.get(recordName);
      synchronized (recordMap) {
        Set<Entry<TagMap, MetricMap>> entrySet = recordMap.entrySet ();
        for (Entry<TagMap, MetricMap> entry : entrySet) {
          OutputRecord outRec = new OutputRecord(entry.getKey(), entry.getValue());
          emitRecord(contextName, recordName, outRec);
        }
      }
    }
    flush();
  }
1356628
private void manage() throws InterruptedException
  {
    log.info("Beginning management in %s.", config.getStartDelay());
    Thread.sleep(config.getStartDelay().getMillis());

    // Ignore return value- we'll get the IDs and futures from getKnownTasks later.
    taskRunner.restore();

    while (active) {
      giant.lock();

      try {
        // Task futures available from the taskRunner
        final Map<String, ListenableFuture<TaskStatus>> runnerTaskFutures = new HashMap<>();
        for (final TaskRunnerWorkItem workItem : taskRunner.getKnownTasks()) {
          runnerTaskFutures.put(workItem.getTaskId(), workItem.getResult());
        }
        // Attain futures for all active tasks (assuming they are ready to run).
        // Copy tasks list, as notifyStatus may modify it.
        for (final Task task : ImmutableList.copyOf(tasks)) {
          if (!taskFutures.containsKey(task.getId())) {
            final ListenableFuture<TaskStatus> runnerTaskFuture;
            if (runnerTaskFutures.containsKey(task.getId())) {
              runnerTaskFuture = runnerTaskFutures.get(task.getId());
            } else {
              // Task should be running, so run it.
              final boolean taskIsReady;
              try {
                taskIsReady = task.isReady(taskActionClientFactory.create(task));
              }
              catch (Exception e) {
                log.warn(e, "Exception thrown during isReady for task: %s", task.getId());
                notifyStatus(task, TaskStatus.failure(task.getId()), "failed because of exception[%s]", e.getClass());
                continue;
              }
              if (taskIsReady) {
                log.info("Asking taskRunner to run: %s", task.getId());
                runnerTaskFuture = taskRunner.run(task);
              } else {
                continue;
              }
            }
            taskFutures.put(task.getId(), attachCallbacks(task, runnerTaskFuture));
          } else if (isTaskPending(task)) {
            // if the taskFutures contain this task and this task is pending, also let the taskRunner
            // to run it to guarantee it will be assigned to run
            // see https://github.com/apache/incubator-druid/pull/6991
            taskRunner.run(task);
          }
        }
        // Kill tasks that shouldn't be running
        final Set<String> tasksToKill = Sets.difference(
            runnerTaskFutures.keySet(),
            ImmutableSet.copyOf(
                Lists.transform(
                    tasks,
                    new Function<Task, Object>()
                    {
                      @Override
                      public String apply(Task task)
                      {
                        return task.getId();
                      }
                    }
                )
            )
        );
        if (!tasksToKill.isEmpty()) {
          log.info("Asking taskRunner to clean up %,d tasks.", tasksToKill.size());
          for (final String taskId : tasksToKill) {
            try {
              taskRunner.shutdown(
                  taskId,
                  "task is not in runnerTaskFutures[%s]",
                  runnerTaskFutures.keySet()
              );
            }
            catch (Exception e) {
              log.warn(e, "TaskRunner failed to clean up task: %s", taskId);
            }
          }
        }
        // awaitNanos because management may become necessary without this condition signalling,
        // due to e.g. tasks becoming ready when other folks mess with the TaskLockbox.
        managementMayBeNecessary.awaitNanos(MANAGEMENT_WAIT_TIMEOUT_NANOS);
      }
      finally {
        giant.unlock();
      }
    }
  }
696672
public void writeTo(final CodedOutputStream output)
                      throws IOException {
    for (int i = 0; i < fields.getNumArrayEntries(); i++) {
      final Map.Entry<FieldDescriptorType, Object> entry =
          fields.getArrayEntryAt(i);
      writeField(entry.getKey(), entry.getValue(), output);
    }
    for (final Map.Entry<FieldDescriptorType, Object> entry :
         fields.getOverflowEntries()) {
      writeField(entry.getKey(), entry.getValue(), output);
    }
  }