Print out debugging information after handling a VFGJob and generating the succeeding jobs.

        :param VFGJob job: The VFGJob instance.
        :param list successors: A list of succeeding states.
        :return: None
541369
def _post_job_handling_debug(self, job, successors):
        

        func = self.project.loader.find_symbol(job.addr)
        function_name = func.name if func is not None else None
        module_name = self.project.loader.find_object_containing(job.addr).provides

        l.debug("VFGJob @ %#08x with callstack [ %s ]", job.addr,
                job.callstack_repr(self.kb),
                )
        l.debug("(Function %s of %s)", function_name, module_name)
        l.debug("-  is call jump: %s", job.is_call_jump)
        for suc in successors:
            if suc not in job.dbg_exit_status:
                l.warning("- %s is not found. FIND OUT WHY.", suc)
                continue

            try:
                l.debug("-  successor: %#08x of %s [%s]", suc.solver.eval_one(suc.ip),
                        suc.history.jumpkind, job.dbg_exit_status[suc])
            except SimValueError:
                l.debug("-  target cannot be concretized. %s [%s]", job.dbg_exit_status[suc], suc.history.jumpkind)
        l.debug("Remaining/pending jobs: %d/%d", len(self._job_info_queue), len(self._pending_returns))
        l.debug("Remaining jobs: %s", [ "%s %d" % (ent.job, id(ent.job)) for ent in self._job_info_queue])
        l.debug("Task stack: %s", self._task_stack)
15929
def insert_jobs(jobs, verify_jobs=True, conn=None):
    
    assert isinstance(jobs, list)
    if verify_jobs \
            and not verify({Jobs.DESCRIPTOR.name: jobs}, Jobs()):
        raise ValueError("Invalid Jobs")
    inserted = RBJ.insert(jobs).run(conn)
    return inserted
8371
def _print_options_help(self):
    
    show_all_help = self._help_request.all_scopes
    if show_all_help:
      help_scopes = list(self._options.known_scope_to_info.keys())
    else:
      # The scopes explicitly mentioned by the user on the cmd line.
      help_scopes = set(self._options.scope_to_flags.keys()) - {GLOBAL_SCOPE}

    scope_infos = list(ScopeInfoIterator(self._options.known_scope_to_info).iterate(help_scopes))
    if scope_infos:
      for scope_info in scope_infos:
        help_str = self._format_help(scope_info)
        if help_str:
          print(help_str)
      return
    else:
      print(pants_release())
      print('\nUsage:')
      print('  ./pants [option ...] [goal ...] [target...]  Attempt the specified goals.')
      print('  ./pants help                                 Get help.')
      print('  ./pants help [goal]                          Get help for a goal.')
      print('  ./pants help-advanced [goal]                 Get help for a goal\'s advanced options.')
      print('  ./pants help-all                             Get help for all goals.')
      print('  ./pants goals                                List all installed goals.')
      print('')
      print('  [target] accepts two special forms:')
      print('    dir:  to include all targets in the specified directory.')
      print('    dir:: to include all targets found recursively under the directory.')
      print('\nFriendly docs:\n  http://pantsbuild.org/')

      print(self._format_help(ScopeInfo(GLOBAL_SCOPE, ScopeInfo.GLOBAL)))
712613
private void printHelpAndExit(String errorMsg) {
        System.out.println(errorMsg);

        HelpFormatter hf = new HelpFormatter();

        hf.printHelp("gobblin-admin.sh jobs [options]", this.options);

        System.exit(1);
    }
464537
def _job_to_text(self, job):
        

        next_run = self._format_date(job.get('next_run', None))

        tasks = ''
        for task in job.get('tasks', []):
            tasks += self._task_to_text(task)
            tasks += '\n\n'

        return '\n'.join(['Job name: %s' % job.get('name', None),
                          'Cron schedule: %s' % job.get('cron_schedule', None),
                          'Next run: %s' % next_run,
                          '',
                          'Parent ID: %s' % job.get('parent_id', None),
                          'Job ID: %s' % job.get('job_id', None),
                          '',
                          'Tasks Detail',
                          '',
                          tasks])
1022301
def _get_job_status(line):
  
  try:
    args = line.strip().split()
    job_name = args[0]

    job = None
    if job_name in _local_jobs:
      job = _local_jobs[job_name]
    else:
      raise Exception('invalid job %s' % job_name)

    if job is not None:
      error = '' if job.fatal_error is None else str(job.fatal_error)
      data = {'exists': True, 'done': job.is_complete, 'error': error}
    else:
      data = {'exists': False}

  except Exception as e:
    google.datalab.utils.print_exception_with_last_stack(e)
    data = {'done': True, 'error': str(e)}

  return IPython.core.display.JSON(data)
808080
def _run_job(self):
        

        args = []

        if isinstance(self.output(), list):
            log_output = os.path.split(self.output()[0].path)
        else:
            log_output = os.path.split(self.output().path)

        args += ["bsub", "-q", self.queue_flag]
        args += ["-n", str(self.n_cpu_flag)]
        args += ["-M", str(self.memory_flag)]
        args += ["-R", "rusage[%s]" % self.resource_flag]
        args += ["-W", str(self.runtime_flag)]
        if self.job_name_flag:
            args += ["-J", str(self.job_name_flag)]
        args += ["-o", os.path.join(log_output[0], "job.out")]
        args += ["-e", os.path.join(log_output[0], "job.err")]
        if self.extra_bsub_args:
            args += self.extra_bsub_args.split()

        # Find where the runner file is
        runner_path = os.path.abspath(lsf_runner.__file__)

        args += [runner_path]
        args += [self.tmp_dir]

        # That should do it. Let the world know what we're doing.
        LOGGER.info("### LSF SUBMISSION ARGS: %s",
                    " ".join([str(a) for a in args]))

        # Submit the job
        run_job_proc = subprocess.Popen(
            [str(a) for a in args],
            stdin=subprocess.PIPE, stdout=subprocess.PIPE, cwd=self.tmp_dir)
        output = run_job_proc.communicate()[0]

        # ASSUMPTION
        # The result will be of the format
        # Job <123> is submitted ot queue <myqueue>
        # So get the number in those first brackets.
        # I cannot think of a better workaround that leaves logic on the Task side of things.
        LOGGER.info("### JOB SUBMISSION OUTPUT: %s", str(output))
        self.job_id = int(output.split("<")[1].split(">")[0])
        LOGGER.info(
            "Job %ssubmitted as job %s",
            self.job_name_flag + ' ',
            str(self.job_id)
        )

        self._track_job()

        # If we want to save the job temporaries, then do so
        # We'll move them to be next to the job output
        if self.save_job_info:
            LOGGER.info("Saving up temporary bits")

            # dest_dir = self.output().path
            shutil.move(self.tmp_dir, "/".join(log_output[0:-1]))

        # Now delete the temporaries, if they're there.
        self._finish()
1262883
def list(self, job_ids, print_array_jobs = False, print_dependencies = False, long = False, print_times = False, status=Status, names=None, ids_only=False):
    """"""
    # configuration for jobs
    fields = ("job-id", "grid-id", "queue", "status", "job-name")
    lengths = (6, 17, 11, 12, 16)
    dependency_length = 0

    if print_dependencies:
      fields += ("dependencies",)
      lengths += (25,)
      dependency_length = lengths[-1]

    if long:
      fields += ("submitted command",)
      lengths += (43,)

    format = "{:^%d}  " * len(lengths)
    format = format % lengths

    # if ids_only:
    #   self.lock()
    #   for job in self.get_jobs():
    #     print(job.unique, end=" ")
    #   self.unlock()
    #   return

    array_format = "{0:^%d}  {1:>%d}  {2:^%d}  {3:^%d}" % lengths[:4]
    delimiter = format.format(*['='*k for k in lengths])
    array_delimiter = array_format.format(*["-"*k for k in lengths[:4]])
    header = [fields[k].center(lengths[k]) for k in range(len(lengths))]

    # print header
    if not ids_only:
      print('  '.join(header))
      print(delimiter)

    self.lock()
    for job in self.get_jobs(job_ids):
      job.refresh()
      if job.status in status and (names is None or job.name in names):
        if ids_only:
          print(job.unique, end=" ")
        else:
          print(job.format(format, dependency_length))
        if print_times:
          print(times(job))

        if (not ids_only) and print_array_jobs and job.array:
          print(array_delimiter)
          for array_job in job.array:
            if array_job.status in status:
              print(array_job.format(array_format))
              if print_times:
                print(times(array_job))
          print(array_delimiter)

    self.unlock()
1370941
def list(args):
  """"""
  jm = setup(args)
  jm.list(job_ids=get_ids(args.job_ids), print_array_jobs=args.print_array_jobs, print_dependencies=args.print_dependencies, status=args.status, long=args.long, print_times=args.print_times, ids_only=args.ids_only, names=args.names)
619841
private <Q, R> void recordFinalStatus(JobIdentifier<Q, R> jobId, JobStatus<Q, R> jobStatus) {
        try {
            _jobStatusDAO.updateJobStatus(jobId, jobStatus);
        } catch (Exception e) {
            _log.error("Failed to record final status for job: [id={}, status={}]", jobId, jobStatus.getStatus(), e);
        }
    }
50788
def process(self, job_id):
        

        self._logger.info(
            '{:.2f}: Process job {}'.format(self._env.now, job_id)
        )

        # log time of commencement of service
        self._observer.notify_service(time=self._env.now, job_id=job_id)

        # draw a new service time
        try:
            service_time = next(self._service_time_generator)
        except StopIteration:
            # ERROR: no more service times
            error_msg = ('Service time generator exhausted')
            self._logger.error(error_msg)

            # raise a different exception, as simpy uses StopIteration to
            # signify end of process (generator)
            raise GGCQServiceTimeStopIteration(error_msg)

        # wait for the service time to pass
        try:
            self._logger.debug('Service time: {:.2f}'.format(service_time))
        except:
            pass

        try:
            yield self._env.timeout(service_time)

        except TypeError:
            # error: service time of wrong type
            error_msg = (
                "service time '{}' has wrong type '{}'".format(
                    service_time, type(service_time).__name__
                )
            )
            self._logger.error(error_msg)

            # trigger exception
            raise GGCQServiceTimeTypeError(error_msg)

        except ValueError as exc:
            if str(exc).startswith('Negative delay'):
                # error: negative service time
                error_msg = (
                    "negative service time {:.2f}".format(
                        service_time
                    )
                )
                self._logger.error(error_msg)

                # trigger exception
                raise GGCQNegativeServiceTimeError(error_msg)
            else:
                raise

        # job finished processing -> departing
        self._logger.info(
            '{:.2f}: Finished processing job {}'.format(self._env.now, job_id)
        )
        # log departure epoch
        self._observer.notify_departure(time=self._env.now, job_id=job_id)
736623
def _makeJobSuccessorReadyToRun(self, jobGraph, jobNode):
        """"""
        successorJobStoreID = jobNode.jobStoreID
        #Build map from successor to predecessors.
        if successorJobStoreID not in self.toilState.successorJobStoreIDToPredecessorJobs:
            self.toilState.successorJobStoreIDToPredecessorJobs[successorJobStoreID] = []
        self.toilState.successorJobStoreIDToPredecessorJobs[successorJobStoreID].append(jobGraph)

        if jobNode.predecessorNumber > 1:
            return self._checkSuccessorReadyToRunMultiplePredecessors(jobGraph, jobNode, successorJobStoreID)
        else:
            return True
1140265
def get_jobs_from_argument(self, raw_job_string):
        
        jobs = []
        if raw_job_string.startswith(":"):
            job_keys = raw_job_string.strip(" :")
            jobs.extend([job for job in self.jobs(job_keys)])
        # we assume a job code
        else:
            assert "/" in raw_job_string, "Job Code {0} is improperly formatted!".format(raw_job_string)
            host, job_name = raw_job_string.rsplit("/", 1)
            host_url = self._config_dict.get(host, {}).get('url', host)
            host = self.get_host(host_url)
            if host.has_job(job_name):
                jobs.append(JenksJob(None, host, job_name,
                                     lambda: self._get_job_api_instance(host_url, job_name)))
            else:
                raise JenksDataException(
                    "Could not find Job {0}/{1}!".format(host, job_name))
        return jobs
1184987
def print_options(self):
        
        options = []

        print("The script is running with the following options:")

        options.append(("dry_run", self.options.dry_run))

        options.append(("worker_config", self.__class__))

        database_config = self.database_config or \
                          self.options.queue_model.database.connection_settings
        options.append(("database", '%s:%s:%s' % (database_config['host'],
                                                  database_config['port'],
                                                  database_config['db'])))

        if self.options.worker_class is not None:
            options.append(("worker-class", self.options.worker_class))

        for name, value in options:
            print(" - %s = %s" % (name.replace('_', '-'), value))

        print("The worker will run with the following options:")
        for name in self.options.worker_class.parameters:
            option = getattr(self.worker, name)
            if name == 'callback' and \
                self.options.worker_class.execute == Worker.execute:
                option = '<jobs "run" method>'
            elif isinstance(option, (list, tuple, set)):
                option = ','.join(option)
            print(" - %s = %s" % (name.replace('_', '-'), option))
486831
def _process_job_and_get_successors(self, job_info):
        

        job = job_info.job

        successors = self._get_successors(job)

        all_new_jobs = [ ]

        for successor in successors:
            new_jobs = self._handle_successor(job, successor, successors)

            if new_jobs:
                all_new_jobs.extend(new_jobs)

                for new_job in new_jobs:
                    self._insert_job(new_job)

        self._post_job_handling(job, all_new_jobs, successors)
841786
void printCounters() {
    System.out.printf("New Job:\n", counters);
    for (String groupName : counters.getGroupNames()) {
      Counters.Group group = counters.getGroup(groupName);
      System.out.printf("\t%s[%s]\n", groupName, group.getDisplayName());

      for (Counters.Counter counter : group) {
        System.out.printf("\t\t%s: %s\n", counter.getDisplayName(),
                                          counter.getCounter());
      }
    }
    System.out.printf("\n");
  }
940970
protected StatusDetail toStatusDetail(final List<JobInfo> jobInfos,
                                          final JobDefinition jobDefinition) {
        final Status status;
        final String message;
        final JobInfo currentJob = jobInfos.get(0);
        final JobInfo lastJob = (!currentJob.getStopped().isPresent() && currentJob.getStatus() == JobStatus.OK && jobInfos.size() > 1) ? jobInfos.get(1) : jobInfos.get(0);
        final JobMeta jobMeta = getJobMeta(jobDefinition.jobType());
        long numFailedJobs = getNumFailedJobs(jobInfos);
        if (!jobMeta.isDisabled()) {
            switch (lastJob.getStatus()) {
                case OK:
                case SKIPPED:
                    if (jobTooOld(lastJob, jobDefinition)) {
                        status = WARNING;
                        message = jobAgeMessage(jobDefinition);
                    } else if (numFailedJobs > maxFailedJobs) {
                        status = WARNING;
                        message = format(TOO_MANY_JOBS_FAILED_MESSAGE, numFailedJobs, jobInfos.size());
                    } else {
                        status = OK;
                        message = SUCCESS_MESSAGE;
                    }
                    break;
                case ERROR:
                    if (numFailedJobs > maxFailedJobs) {
                        status = ERROR;
                    } else {
                        status = WARNING;
                    }
                    if (numberOfJobs == 1 && maxFailedJobs <= 1) {
                        message = ERROR_MESSAGE;
                    } else {
                        message = format(TOO_MANY_JOBS_FAILED_MESSAGE, numFailedJobs, jobInfos.size());
                    }
                    break;

                case DEAD:
                default:
                    status = WARNING;
                    message = DEAD_MESSAGE;
            }
        } else {
            status = OK;
            message = format(JOB_DEACTIVATED_MESSAGE, jobMeta.getDisabledComment());
        }
        return StatusDetail.statusDetail(
                jobDefinition.jobName(),
                status,
                message,
                asList(
                        link(REL_JOB, String.format("%s/jobs/%s", managementContextPath, lastJob.getJobId()), "Details")
                ),
                runningDetailsFor(lastJob)
        );
    }