/*
Parse and add the job counters
428519
private void parseAndAddJobCounters(Hashtable<Enum, String> job, String counters) throws ParseException {
    Counters cnt = Counters.fromEscapedCompactString(counters);
    for (java.util.Iterator<Counters.Group> grps = cnt.iterator(); grps.hasNext(); ) {
      Counters.Group grp = grps.next();
      //String groupname = "<" + grp.getName() + ">::<" + grp.getDisplayName() + ">";
      for (java.util.Iterator<Counters.Counter> mycounters = grp.iterator(); mycounters.hasNext(); ) {
        Counters.Counter counter = mycounters.next();
        //String countername = "<"+counter.getName()+">::<"+counter.getDisplayName()+">::<"+counter.getValue()+">";
        //System.err.println("groupName:"+groupname+",countername: "+countername);
        String countername = grp.getDisplayName()+"."+counter.getDisplayName();
        String value = (new Long(counter.getValue())).toString();
        String[] parts = {countername,value};
        //System.err.println("part0:<"+parts[0]+">,:part1 <"+parts[1]+">");
        if (parts[0].equals("FileSystemCounters.FILE_BYTES_READ")) {
          job.put(JobKeys.FILE_BYTES_READ, parts[1]);
        } else if (parts[0].equals("FileSystemCounters.FILE_BYTES_WRITTEN")) {
          job.put(JobKeys.FILE_BYTES_WRITTEN, parts[1]);
        } else if (parts[0].equals("FileSystemCounters.HDFS_BYTES_READ")) {
          job.put(JobKeys.HDFS_BYTES_READ, parts[1]);
        } else if (parts[0].equals("FileSystemCounters.HDFS_BYTES_WRITTEN")) {
          job.put(JobKeys.HDFS_BYTES_WRITTEN, parts[1]);
        } else if (parts[0].equals("Job Counters .Launched map tasks")) {
          job.put(JobKeys.LAUNCHED_MAPS, parts[1]);
        } else if (parts[0].equals("Job Counters .Launched reduce tasks")) {
          job.put(JobKeys.LAUNCHED_REDUCES, parts[1]);
        } else if (parts[0].equals("Job Counters .Data-local map tasks")) {
          job.put(JobKeys.DATALOCAL_MAPS, parts[1]);
        } else if (parts[0].equals("Job Counters .Rack-local map tasks")) {
          job.put(JobKeys.RACKLOCAL_MAPS, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Map input records")) {
          job.put(JobKeys.MAP_INPUT_RECORDS, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Map output records")) {
          job.put(JobKeys.MAP_OUTPUT_RECORDS, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Map input bytes")) {
          job.put(JobKeys.MAP_INPUT_BYTES, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Map output bytes")) {
          job.put(JobKeys.MAP_OUTPUT_BYTES, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Combine input records")) {
          job.put(JobKeys.COMBINE_INPUT_RECORDS, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Combine output records")) {
          job.put(JobKeys.COMBINE_OUTPUT_RECORDS, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Reduce input groups")) {
          job.put(JobKeys.REDUCE_INPUT_GROUPS, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Reduce input records")) {
          job.put(JobKeys.REDUCE_INPUT_RECORDS, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Reduce output records")) {
          job.put(JobKeys.REDUCE_OUTPUT_RECORDS, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Spilled Records")) {
          job.put(JobKeys.SPILLED_RECORDS, parts[1]);
        } else if (parts[0].equals("Map-Reduce Framework.Reduce shuffle bytes")) {
          job.put(JobKeys.SHUFFLE_BYTES, parts[1]);
        } else {
          System.err.println("JobCounterKey:<"+parts[0]+"> ==> NOT INCLUDED IN PERFORMANCE ADVISOR");
        }
      }
    }  
  }
1039710
function initJobOverview(jobs, cursor) {
  // Empty body.
  var body = $('#running-list > tbody');
  body.empty();

  if (!jobs || (jobs && jobs.length == 0)) {
    $('<td colspan="8">').text('No job records found.').appendTo(body);
    return;
  }

  // Show header.
  $('#running-list > thead').show();

  // Populate the table.
  $.each(jobs, function(index, job) {
    var row = $('<tr id="row-' + job.mapreduce_id + '">');

    var status = (job.active ? 'running' : job.result_status) || 'unknown';
    row.append($('<td class="status-text">').text(status));

    $('<td>').append(
      $('<a>')
        .attr('href', 'detail?mapreduce_id=' + job.mapreduce_id)
        .text('Detail')).appendTo(row);

    row.append($('<td>').text(job.mapreduce_id))
      .append($('<td>').text(job.name));

    var activity = '' + job.active_shards + ' / ' + job.shards + ' shards';
    row.append($('<td>').text(activity))

    row.append($('<td>').text(getIso8601String(job.start_timestamp_ms)));

    row.append($('<td>').text(getElapsedTimeString(
        job.start_timestamp_ms, job.updated_timestamp_ms)));

    // Controller links for abort, cleanup, etc.
    if (job.active) {
      var control = $('<a href="">').text('Abort')
        .click(function(event) {
          abortJob(job.name, job.mapreduce_id);
          event.stopPropagation();
          return false;
        });
      row.append($('<td>').append(control));
    } else {
      var control = $('<a href="">').text('Cleanup')
        .click(function(event) {
          cleanUpJob(job.name, job.mapreduce_id);
          event.stopPropagation();
          return false;
        });
      row.append($('<td>').append(control));
    }
    row.appendTo(body);
  });

  // Set up the next/first page links.
  $('#running-first-page')
    .show()
    .unbind('click')
    .click(function() {
    listJobs(null, initJobOverview);
    return false;
  });
  $('#running-next-page').unbind('click');
  if (cursor) {
    $('#running-next-page')
      .show()
      .click(function() {
        listJobs(cursor, initJobOverview);
        return false;
      });
  } else {
    $('#running-next-page').hide();
  }
  $('#running-list > tfoot').show();
}
118823
function job_info(jobName, customParams, callback) {
      var _doArgs31 = doArgs(arguments, ['string', ['object', {}], 'function']);

      var _doArgs32 = _slicedToArray(_doArgs31, 3);

      jobName = _doArgs32[0];
      customParams = _doArgs32[1];
      callback = _doArgs32[2];


      doRequest({
        urlPattern: [JOB_INFO, jobName]
      }, customParams, callback);
    }
1262883
def list(self, job_ids, print_array_jobs = False, print_dependencies = False, long = False, print_times = False, status=Status, names=None, ids_only=False):
    """"""
    # configuration for jobs
    fields = ("job-id", "grid-id", "queue", "status", "job-name")
    lengths = (6, 17, 11, 12, 16)
    dependency_length = 0

    if print_dependencies:
      fields += ("dependencies",)
      lengths += (25,)
      dependency_length = lengths[-1]

    if long:
      fields += ("submitted command",)
      lengths += (43,)

    format = "{:^%d}  " * len(lengths)
    format = format % lengths

    # if ids_only:
    #   self.lock()
    #   for job in self.get_jobs():
    #     print(job.unique, end=" ")
    #   self.unlock()
    #   return

    array_format = "{0:^%d}  {1:>%d}  {2:^%d}  {3:^%d}" % lengths[:4]
    delimiter = format.format(*['='*k for k in lengths])
    array_delimiter = array_format.format(*["-"*k for k in lengths[:4]])
    header = [fields[k].center(lengths[k]) for k in range(len(lengths))]

    # print header
    if not ids_only:
      print('  '.join(header))
      print(delimiter)

    self.lock()
    for job in self.get_jobs(job_ids):
      job.refresh()
      if job.status in status and (names is None or job.name in names):
        if ids_only:
          print(job.unique, end=" ")
        else:
          print(job.format(format, dependency_length))
        if print_times:
          print(times(job))

        if (not ids_only) and print_array_jobs and job.array:
          print(array_delimiter)
          for array_job in job.array:
            if array_job.status in status:
              print(array_job.format(array_format))
              if print_times:
                print(times(array_job))
          print(array_delimiter)

    self.unlock()
450042
private static void dumpHadoopCounters(PigStats pigStats) {
    try {
      if (props.getBoolean(PIG_DUMP_HADOOP_COUNTER_PROPERTY, false)) {
        if (pigStats != null) {
          JobGraph jGraph = pigStats.getJobGraph();
          Iterator<JobStats> iter = jGraph.iterator();
          while (iter.hasNext()) {
            JobStats jobStats = iter.next();
            System.out.println("\n === Counters for job: "
                + jobStats.getJobId() + " ===");
            Counters counters = jobStats.getHadoopCounters();
            if (counters != null) {
              for (Counters.Group group : counters) {
                System.out.println(" Counter Group: " + group.getDisplayName()
                    + " (" + group.getName() + ")");
                System.out.println("  number of counters in this group: "
                    + group.size());
                for (Counters.Counter counter : group) {
                  System.out.println("  - " + counter.getDisplayName() + ": "
                      + counter.getCounter());
                }
              }
            } else {
              System.out.println("There are no counters");
            }
          }
        } else {
          System.out.println("pigStats is null, can't dump Hadoop counters");
        }
      }
    } catch (Exception e) {
      System.out.println("Unexpected error: " + e.getMessage());
      e.printStackTrace(System.out);
    }
  }
379052
def parse_job_files(self):
        """"""
        repo_jobs = []
        for rel_job_file_path, job_info in self.job_files.items():
            LOGGER.debug("Checking for job definitions in %s", rel_job_file_path)
            jobs = self.parse_job_definitions(rel_job_file_path, job_info)
            LOGGER.debug("Found %d job definitions in %s", len(jobs), rel_job_file_path)
            repo_jobs.extend(jobs)
        if not repo_jobs:
            LOGGER.info("No job definitions found in repo '%s'", self.repo)
        else:
            LOGGER.info(
                "Found %d job definitions in repo '%s'", len(repo_jobs), self.repo
            )
            # LOGGER.debug(json.dumps(repo_jobs, indent=4))
        return repo_jobs
117327
def main():
    """"""
    parser = getBasicOptionParser()

    parser.add_argument("jobStore", type=str,
                        help="The location of a job store that holds the information about the "
                             "workflow whose status is to be reported on." + jobStoreLocatorHelp)

    parser.add_argument("--failIfNotComplete", action="store_true",
                        help="Return exit value of 1 if toil jobs not all completed. default=%(default)s",
                        default=False)

    parser.add_argument("--noAggStats", dest="stats", action="store_false",
                        help="Do not print overall, aggregate status of workflow.",
                        default=True)

    parser.add_argument("--printDot", action="store_true",
                        help="Print dot formatted description of the graph. If using --jobs will "
                             "restrict to subgraph including only those jobs. default=%(default)s",
                        default=False)

    parser.add_argument("--jobs", nargs='+',
                        help="Restrict reporting to the following jobs (allows subsetting of the report).",
                        default=None)

    parser.add_argument("--printPerJobStats", action="store_true",
                        help="Print info about each job. default=%(default)s",
                        default=False)

    parser.add_argument("--printLogs", action="store_true",
                        help="Print the log files of jobs (if they exist). default=%(default)s",
                        default=False)

    parser.add_argument("--printChildren", action="store_true",
                        help="Print children of each job. default=%(default)s",
                        default=False)

    parser.add_argument("--version", action='version', version=version)

    options = parseBasicOptions(parser)

    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(0)

    config = Config()
    config.setOptions(options)

    try:
        status = ToilStatus(config.jobStore, options.jobs)
    except NoSuchJobStoreException:
        print('No job store found.')
        return
    except JobException:  # Workflow likely complete, user informed in ToilStatus()
        return

    jobStats = status.report_on_jobs()

    # Info to be reported.
    hasChildren = jobStats['hasChildren']
    readyToRun = jobStats['readyToRun']
    zombies = jobStats['zombies']
    hasServices = jobStats['hasServices']
    services = jobStats['services']
    hasLogFile = jobStats['hasLogFile']
    properties = jobStats['properties']
    childNumber = jobStats['childNumber']

    if options.printPerJobStats:
        status.printAggregateJobStats(properties, childNumber)
    if options.printLogs:
        status.printJobLog()
    if options.printChildren:
        status.printJobChildren()
    if options.printDot:
        status.print_dot_chart()
    if options.stats:
        print('Of the %i jobs considered, '
           'there are %i jobs with children, '
           '%i jobs ready to run, '
           '%i zombie jobs, '
           '%i jobs with services, '
           '%i services, '
           'and %i jobs with log files currently in %s.' %
            (len(status.jobsToReport), len(hasChildren), len(readyToRun), len(zombies),
             len(hasServices), len(services), len(hasLogFile), status.jobStore))

    if len(status.jobsToReport) > 0 and options.failIfNotComplete:
        # Upon workflow completion, all jobs will have been removed from job store
        exit(1)
123497
def exon_count(job, job_vars):
    
    input_args, ids = job_vars
    work_dir = job.fileStore.getLocalTempDir()
    uuid = input_args['uuid']
    sudo = input_args['sudo']
    # I/O
    sort_by_ref, normalize_pl, composite_bed = return_input_paths(job, work_dir, ids, 'sort_by_ref.bam',
                                                                  'normalize.pl', 'composite_exons.bed')
    # Command
    tool = 'jvivian/bedtools'
    cmd_1 = ['coverage',
             '-split',
             '-abam', docker_path(sort_by_ref),
             '-b', docker_path(composite_bed)]
    cmd_2 = ['perl',
             os.path.join(work_dir, 'normalize.pl'),
             sort_by_ref,
             composite_bed]

    popen_docker = ['docker', 'run', '-v', '{}:/data'.format(work_dir), tool]
    if sudo:
        popen_docker = ['sudo'] + popen_docker
    p = subprocess.Popen(popen_docker + cmd_1, stdout=subprocess.PIPE)
    with open(os.path.join(work_dir, 'exon_quant'), 'w') as f:
        subprocess.check_call(cmd_2, stdin=p.stdout, stdout=f)

    p1 = subprocess.Popen(['cat', os.path.join(work_dir, 'exon_quant')], stdout=subprocess.PIPE)
    p2 = subprocess.Popen(['tr', '":"', '"\t"'], stdin=p1.stdout, stdout=subprocess.PIPE)
    p3 = subprocess.Popen(['tr', '"-"', '"\t"'], stdin=p2.stdout, stdout=subprocess.PIPE)
    with open(os.path.join(work_dir, 'exon_quant.bed'), 'w') as f:
        subprocess.check_call(['cut', '-f1-4'], stdin=p3.stdout, stdout=f)
    # Create zip, upload to fileStore, and move to output_dir as a backup
    output_files = ['exon_quant.bed', 'exon_quant']
    tarball_files(work_dir, tar_name='exon.tar.gz', uuid=uuid, files=output_files)
    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'exon.tar.gz'))
110773
@Override
    public List<Job> parse(String text) throws ParseException {
        final List<Job> jobs;
        if (StringUtils.isNotBlank(text)) {
            text = StringUtils.replace(text, "\n\t", "");
            jobs = new LinkedList<Job>();
            String separator = "\n";
            if (text.indexOf("\r\n") > 0) {
                separator = "\r\n";
            }
            final String[] lines = text.split(separator);
            Job job = null;
            for (final String line : lines) {
                Matcher matcher = PATTERN_JOB.matcher(line);
                if (matcher.matches()) {
                    if (job != null) {
                        jobs.add(job);
                    }
                    job = new Job();
                    final String id = matcher.group(1).trim();
                    job.setId(id);
                } else if (StringUtils.isNotBlank(line)) {
                    String[] temp = Utils.splitFirst(line, CHAR_EQUALS);
                    if (temp.length == 2) {
                        final String key = temp[0].trim().toLowerCase();
                        final String value = temp[1].trim();
                        if ("job_name".equalsIgnoreCase(key)) {
                            job.setName(value);
                        } else if ("job_owner".equalsIgnoreCase(key)) {
                            job.setOwner(value);
                        } else if (key.startsWith("resources_used.")) {
                            job.getResourcesUsed().put(key, value);
                        } else if ("job_state".equalsIgnoreCase(key)) {
                            job.setState(value);
                        } else if ("queue".equalsIgnoreCase(key)) {
                            job.setQueue(value);
                        } else if ("server".equalsIgnoreCase(key)) {
                            job.setServer(value);
                        } else if ("checkpoint".equalsIgnoreCase(key)) {
                            job.setCheckpoint(value);
                        } else if ("ctime".equalsIgnoreCase(key)) {
                            job.setCtime(value);
                        } else if ("error_path".equalsIgnoreCase(key)) {
                            job.setErrorPath(value);
                        } else if ("exec_host".equalsIgnoreCase(key)) {
                            job.setExecHost(value);
                        } else if ("exec_port".equalsIgnoreCase(key)) {
                            job.setExecPort(value);
                        } else if ("hold_types".equalsIgnoreCase(key)) {
                            job.setHoldTypes(value);
                        } else if ("join_path".equalsIgnoreCase(key)) {
                            job.setJoinPath(value);
                        } else if ("keep_files".equalsIgnoreCase(key)) {
                            job.setKeepFiles(value);
                        } else if ("mail_points".equalsIgnoreCase(key)) {
                            job.setMailPoints(value);
                        } else if ("mail_users".equalsIgnoreCase(key)) {
                            job.setMailUsers(value);
                        } else if ("mtime".equalsIgnoreCase(key)) {
                            job.setMtime(value);
                        } else if ("output_path".equalsIgnoreCase(key)) {
                            job.setOutputPath(value);
                        } else if ("priority".equalsIgnoreCase(key)) {
                            try {
                                job.setPriority(Integer.parseInt(value));
                            } catch (NumberFormatException nfe) {
                                LOGGER.log(Level.WARNING, "Failed parsing job priority: " + nfe.getMessage(), nfe);
                                job.setPriority(-1);
                            }
                        } else if ("qtime".equalsIgnoreCase(key)) {
                            job.setQtime(value);
                        } else if ("rerunable".equalsIgnoreCase(key)) {
                            job.setRerunable(Boolean.parseBoolean(value));
                        } else if (key.startsWith("resource_list.")) {
                            job.getResourceList().put(key, value);
                        } else if ("session_id".equalsIgnoreCase(key)) {
                            try {
                                job.setSessionId(Integer.parseInt(value));
                            } catch (NumberFormatException nfe) {
                                LOGGER.log(Level.WARNING, "Failed parsing job session id: " + nfe.getMessage(), nfe);
                                job.setSessionId(-1);
                            }
                        } else if ("substate".equalsIgnoreCase(key)) {
                            try {
                                job.setSubstate(Integer.parseInt(value));
                            } catch (NumberFormatException nfe) {
                                LOGGER.log(Level.WARNING, "Failed parsing job substate: " + nfe.getMessage(), nfe);
                                job.setSubstate(-1);
                            }
                        } else if (key.startsWith("variable_list")) {
                            job.getVariableList().put(key, value);
                        } else if ("etime".equalsIgnoreCase(key)) {
                            job.setEtime(value);
                        } else if ("euser".equalsIgnoreCase(key)) {
                            job.setEuser(value);
                        } else if ("egroup".equalsIgnoreCase(key)) {
                            job.setEgroup(value);
                        } else if ("hashname".equalsIgnoreCase(key)) {
                            job.setHashName(value);
                        } else if ("queue_rank".equalsIgnoreCase(key)) {
                            try {
                                job.setQueueRank(Integer.parseInt(value));
                            } catch (NumberFormatException nfe) {
                                LOGGER.log(Level.WARNING, "Failed parsing job queue rank: " + nfe.getMessage(), nfe);
                                job.setQueueRank(-1);
                            }
                        } else if ("queue_type".equalsIgnoreCase(key)) {
                            job.setQueueType(value);
                        } else if ("comment".equalsIgnoreCase(key)) {
                            job.setComment(value);
                        } else if ("submit_args".equalsIgnoreCase(key)) {
                            job.setSubmitArgs(value);
                        } else if ("submit_host".equalsIgnoreCase(key)) {
                            job.setSubmitHost(value);
                        } else if ("start_time".equalsIgnoreCase(key)) {
                            job.setStartTime(value);
                        } else if ("start_count".equalsIgnoreCase(key)) {
                            try {
                                job.setStartCount(Integer.parseInt(value));
                            } catch (NumberFormatException nfe) {
                                LOGGER.log(Level.WARNING, "Failed parsing job start count: " + nfe.getMessage(), nfe);
                                job.setStartCount(-1);
                            }
                        } else if ("fault_tolerant".equalsIgnoreCase(key)) {
                            job.setFaultTolerant(Boolean.parseBoolean(value));
                        } else if ("job_array_id".equalsIgnoreCase(key)) {
                            job.setJobArrayId(Integer.parseInt(value));
                        } else if ("job_radix".equalsIgnoreCase(key)) {
                            try {
                                job.setRadix(Integer.parseInt(value));
                            } catch (NumberFormatException nfe) {
                                LOGGER.log(Level.WARNING, "Failed parsing job radix: " + nfe.getMessage(), nfe);
                                job.setRadix(-1);
                            }
                        } else if ("walltime.remaining".equalsIgnoreCase(key)) {
                            try {
                                job.setWalltimeRemaining(Long.parseLong(value));
                            } catch (NumberFormatException nfe) {
                                LOGGER.log(Level.WARNING, "Failed parsing job walltime remaining: " + nfe.getMessage(),
                                        nfe);
                                job.setWalltimeRemaining(-1L);
                            }
                        }
                    }
                }
            }
            if (job != null) {
                jobs.add(job);
            }
            return jobs;
        } else {
            return Collections.emptyList();
        }
    }
409617
def _processReadyJobs(self):
        """"""
        logger.debug('Built the jobs list, currently have %i jobs to update and %i jobs issued',
                     len(self.toilState.updatedJobs), self.getNumberOfJobsIssued())

        updatedJobs = self.toilState.updatedJobs # The updated jobs to consider below
        self.toilState.updatedJobs = set() # Resetting the list for the next set

        for jobGraph, resultStatus in updatedJobs:
            self._processReadyJob(jobGraph, resultStatus)
578362
@SuppressWarnings("unused") // called through reflection by RequestServer
  public JobsV3 list(int version, JobsV3 s) {
    Job[] jobs = Job.jobs();
    // Jobs j = new Jobs();
    // j._jobs = Job.jobs();
    // PojoUtils.copyProperties(s, j, PojoUtils.FieldNaming.ORIGIN_HAS_UNDERSCORES);
    s.jobs = new JobV3[jobs.length];

    int i = 0;
    for (Job j : jobs) {
      try { s.jobs[i] = (JobV3) SchemaServer.schema(version, j).fillFromImpl(j); }
      // no special schema for this job subclass, so fall back to JobV3
      catch (H2ONotFoundArgumentException e) { s.jobs[i] = new JobV3().fillFromImpl(j); }
      i++; // Java does the increment before the function call which throws?!
    }
    return s;
  }
1304245
private List<Job> getJobs(final Jedis jedis, final String queueName, final long jobOffset, final long jobCount) throws Exception {
        final String key = key(QUEUE, queueName);
        final List<Job> jobs = new ArrayList<>();
        if (JedisUtils.isDelayedQueue(jedis, key)) { // If delayed queue, use ZRANGEWITHSCORES
            final Set<Tuple> elements = jedis.zrangeWithScores(key, jobOffset, jobOffset + jobCount - 1);
            for (final Tuple elementWithScore : elements) {
                final Job job = ObjectMapperFactory.get().readValue(elementWithScore.getElement(), Job.class);
                job.setRunAt(elementWithScore.getScore());
                jobs.add(job);
            }
        } else { // Else, use LRANGE
            final List<String> elements = jedis.lrange(key, jobOffset, jobOffset + jobCount - 1);
            for (final String element : elements) {
                jobs.add(ObjectMapperFactory.get().readValue(element, Job.class));
            }
        }
        return jobs;
    }
1148130
public void generateJobTable(JspWriter out, String label, List<JobInProgress> jobs)
      throws IOException {
    if (jobs.size() > 0) {
      for (JobInProgress job : jobs) {
        JobProfile profile = job.getProfile();
        JobStatus status = job.getStatus();
        JobID jobid = profile.getJobID();

        int desiredMaps = job.desiredMaps();
        int desiredReduces = job.desiredReduces();
        int completedMaps = job.finishedMaps();
        int completedReduces = job.finishedReduces();
        String name = profile.getJobName();

        out.print("<" + label + "_job jobid=\"" + jobid + "\">\n");
        out.print("  <jobid>" + jobid + "</jobid>\n");
        out.print("  <user>" + profile.getUser() + "</user>\n");
        out.print("  <name>" + ("".equals(name) ? "&nbsp;" : name) + "</name>\n");
        out.print("  <map_complete>" + StringUtils.formatPercent(status.mapProgress(), 2) + "</map_complete>\n");
        out.print("  <map_total>" + desiredMaps + "</map_total>\n");
        out.print("  <maps_completed>" + completedMaps + "</maps_completed>\n");
        out.print("  <reduce_complete>" + StringUtils.formatPercent(status.reduceProgress(), 2) + "</reduce_complete>\n");
        out.print("  <reduce_total>" + desiredReduces + "</reduce_total>\n");
        out.print("  <reduces_completed>" + completedReduces + "</reduces_completed>\n");
        out.print("</" + label + "_job>\n");
      }
    }
  }
332164
String[] parseArgs(String[] args) {
    int i = 0;
    boolean driverArgs = true;
    while (driverArgs) {
      if (i >= args.length) {
        break;
      }

      String s = args[i];
      if (s.equals("-h") ||
              s.equals("help") ||
              s.equals("-help") ||
              s.equals("--help")) {
        usage();
      }
      else if (s.equals("-n") ||
              s.equals("-nodes")) {
        i++; if (i >= args.length) { usage(); }
        numNodes = Integer.parseInt(args[i]);
      }
      else if (s.equals("-o") ||
              s.equals("-output")) {
        i++; if (i >= args.length) { usage(); }
        outputPath = args[i];
      }
      else if (s.equals("-jobname")) {
        i++; if (i >= args.length) { usage(); }
        jobtrackerName = args[i];
      }
      else if (s.equals("-mapperXmx")) {
        i++; if (i >= args.length) { usage(); }
        mapperXmx = args[i];
      }
      else if (s.equals("-extramempercent")) {
        i++; if (i >= args.length) { usage(); }
        extraMemPercent = Integer.parseInt(args[i]);
      }
      else if (s.equals("-mapperPermSize")) {
        i++; if (i >= args.length) { usage(); }
        mapperPermSize = args[i];
      }
      else if (s.equals("-driverif")) {
        i++; if (i >= args.length) { usage(); }
        driverCallbackIp = args[i];
      }
      else if (s.equals("-driverport")) {
        i++; if (i >= args.length) { usage(); }
        driverCallbackPort = Integer.parseInt(args[i]);
      }
      else if (s.equals("-driverportrange")) {
        i++; if (i >= args.length) { usage(); }
        driverCallbackPortRange = PortRange.parse(args[i]);
      }
      else if (s.equals("-network")) {
        i++; if (i >= args.length) { usage(); }
        network = args[i];
      }
      else if (s.equals("-timeout")) {
        i++; if (i >= args.length) { usage(); }
        cloudFormationTimeoutSeconds = Integer.parseInt(args[i]);
      }
      else if (s.equals("-disown")) {
        disown = true;
      }
      else if (s.equals("-notify")) {
        i++; if (i >= args.length) { usage(); }
        clusterReadyFileName = args[i];
      }
      else if (s.equals("-nthreads")) {
        i++; if (i >= args.length) { usage(); }
        nthreads = Integer.parseInt(args[i]);
      }
      else if (s.equals("-context_path")) {
        i++; if (i >= args.length) { usage(); }
        contextPath = args[i];
      }
      else if (s.equals("-baseport")) {
        i++; if (i >= args.length) { usage(); }
        basePort = Integer.parseInt(args[i]);
        if ((basePort < 0) || (basePort > 65535)) {
            error("Base port must be between 1 and 65535");
        }
      }
      else if (s.equals("-port_offset")) {
        i++; if (i >= args.length) { usage(); }
        portOffset = Integer.parseInt(args[i]);
        if ((portOffset <= 0) || (portOffset > 65534)) {
          error("Port offset must be between 1 and 65534");
        }
      }
      else if (s.equals("-beta")) {
        beta = true;
      }
      else if (s.equals("-random_udp_drop")) {
        enableRandomUdpDrop = true;
      }
      else if (s.equals("-ea")) {
        enableExceptions = true;
      }
      else if (s.equals("-verbose:gc") && !JAVA_VERSION.useUnifiedLogging()) {
        if (!JAVA_VERSION.useUnifiedLogging()) {
          enableVerboseGC = true;
        } else {
          error("Parameter -verbose:gc is unusable, running on JVM with deprecated GC debugging flags.");
        }
      } else if (s.equals("-Xlog:gc=info")) {
        if (JAVA_VERSION.useUnifiedLogging()) {
          enableVerboseGC = true;
        } else {
          error("Parameter -verbose:gc is unusable, running on JVM without unified JVM logging.");
        }
      }
      else if (s.equals("-verbose:class")) {
        enableVerboseClass = true;
      }
      else if (s.equals("-XX:+PrintCompilation")) {
        enablePrintCompilation = true;
      }
      else if (s.equals("-exclude")) {
        enableExcludeMethods = true;
      }
      else if (s.equals("-Dlog4j.defaultInitOverride=true")) {
        enableLog4jDefaultInitOverride = true;
      }
      else if (s.equals("-debug")) {
        enableDebug = true;
      }
      else if (s.equals("-suspend")) {
        enableSuspend = true;
      }
      else if (s.equals("-debugport")) {
        i++; if (i >= args.length) { usage(); }
        debugPort = Integer.parseInt(args[i]);
        if ((debugPort < 0) || (debugPort > 65535)) {
          error("Debug port must be between 1 and 65535");
        }
      } else if (s.equals("-XX:+PrintGCDetails")) {
        if (!JAVA_VERSION.useUnifiedLogging()) {
          enablePrintGCDetails = true;
        } else {
          error("Parameter -XX:+PrintGCDetails is unusable, running on JVM with deprecated GC debugging flags.");
        }
      }
      else if (s.equals("-XX:+PrintGCTimeStamps")) {
        if (!JAVA_VERSION.useUnifiedLogging()) {
          enablePrintGCDetails = true;
        } else {
          error("Parameter -XX:+PrintGCTimeStamps is unusable, running on JVM with deprecated GC debugging flags.");
        }
      }
      else if (s.equals("-gc")) {
        enableVerboseGC = true;
        enablePrintGCDetails = true;
        enablePrintGCTimeStamps = true;
      }
      else if (s.equals("-nogc")) {
        enableVerboseGC = false;
        enablePrintGCDetails = false;
        enablePrintGCTimeStamps = false;
      }
      else if (s.equals("-flow_dir")) {
        i++; if (i >= args.length) { usage(); }
        flowDir = args[i];
      }
      else if (s.equals("-J")) {
        i++; if (i >= args.length) { usage(); }
        extraArguments.add(args[i]);
      }
      else if (s.equals("-JJ")) {
        i++; if (i >= args.length) { usage(); }
        extraJvmArguments.add(args[i]);
      }
      else if (s.equals("-jks")) {
        i++; if (i >= args.length) { usage(); }
        jksFileName = args[i];
      }
      else if (s.equals("-jks_pass")) {
        i++; if (i >= args.length) { usage(); }
        jksPass = args[i];
      }
      else if (s.equals("-internal_secure_connections")) {
        internal_secure_connections = true;
      }
      else if (s.equals("-internal_security_conf") || s.equals("-internal_security")) {
        if(s.equals("-internal_security")){
          System.out.println("The '-internal_security' configuration is deprecated. " +
                  "Please use '-internal_security_conf' instead.");
        }
        i++; if (i >= args.length) { usage(); }
        securityConf = args[i];
      }
      else if (s.equals("-hash_login")) {
        hashLogin = true;
      }
      else if (s.equals("-ldap_login")) {
        ldapLogin = true;
      }
      else if (s.equals("-kerberos_login")) {
        kerberosLogin = true;
      }
      else if (s.equals("-pam_login")) {
        pamLogin = true;
      }
      else if (s.equals("-login_conf")) {
        i++; if (i >= args.length) { usage(); }
        loginConfFileName = args[i];
      }
      else if (s.equals("-form_auth")) {
        formAuth = true;
      }
      else if (s.equals("-session_timeout")) {
        i++; if (i >= args.length) { usage(); }
        sessionTimeout = args[i];
      }
      else if (s.equals("-user_name")) {
        i++; if (i >= args.length) { usage(); }
        userName = args[i];
      }
      else if (s.equals("-client")) {
        client = true;
        driverArgs = false;
      }
      else if (s.equals("-proxy")) {
        proxy = true;
        driverArgs = false;
      } else if (s.equals("-run_as_user")) {
        i++; if (i >= args.length) { usage(); }
        runAsUser = args[i];
      } else if (s.equals("-principal")) {
        i++; if (i >= args.length) { usage(); }
        principal = args[i];
      } else if (s.equals("-keytab")) {
        i++; if (i >= args.length) { usage (); }
        keytabPath = args[i];
      } else if (s.equals("-report_hostname")) {
        reportHostname = true;
      } else if (s.equals("-driver_debug")) {
        driverDebug = true;
      } else if (s.equals("-hiveHost")) {
        i++; if (i >= args.length) { usage (); }
        hiveHost = args[i];
      } else if (s.equals("-hivePrincipal")) {
        i++; if (i >= args.length) { usage (); }
        hivePrincipal = args[i];
      } else if (s.equals("-refreshTokens")) {
        refreshTokens = true;
      } else {
        error("Unrecognized option " + s);
      }

      i++;
    }
    String[] otherArgs = new String[Math.max(args.length - i, 0)];
    for (int j = 0; j < otherArgs.length; j++)
      otherArgs[j] = args[i++];
    return otherArgs;
  }
827869
public static void parseJobTasks(String jobHistoryFile, 
                       JobHistory.JobInfo job, FileSystem fs)
    throws IOException {
    JobHistory.parseHistoryFromFS(jobHistoryFile, 
                            new JobTasksParseListener(job), fs);
  }
990827
def parse_sacct(sacct_stream):
    """"""
    rows = (line.split() for line in sacct_stream)
    # filter rows that begin with a SLURM job id
    relevant_rows = (row for row in rows if row[0].isdigit())
    jobs = [convert_job(row) for row in relevant_rows]
    return jobs
986439
def parse(cls, config_data: dict, sampleinfo_data: dict, sacct_jobs: List[dict],
              jobs: int) -> dict:
        """"""
        analysis_types = [sample['type'] for sample in config_data['samples']]
        run_data = {
            'user': config_data['email'],
            'family': config_data['family'],
            'priority': config_data['priority'],
            'started_at': sampleinfo_data['date'],
            'version': sampleinfo_data['version'],
            'out_dir': config_data['out_dir'],
            'config_path': config_data['config_path'],
            'type': cls._get_analysis_type(analysis_types),
        }

        sacct_data, last_job_end = cls._parse_sacct(sacct_jobs, jobs_count=jobs)
        run_data.update(sacct_data)

        run_data['status'] = cls.get_status(sampleinfo_data['is_finished'],
                                            len(run_data['failed_jobs']))
        if run_data['status'] == 'completed':
            run_data['completed_at'] = last_job_end

        return run_data