Trace job xml file..
193344
protected void traceJobXML(String jobXML) {

        if (logger.isLoggable(Level.FINE)) {
            int concatLen = jobXML.length() > 200 ? 200 : jobXML.length();
            logger.fine("Starting job: " + jobXML.substring(0, concatLen) + "... truncated ...");
        }
    }
1312606
def _build_job_dict(self):
        """"""
        if self.args['dry_run']:
            status = JobStatus.unknown
        else:
            status = JobStatus.not_ready

        base_config = self.scatter_link.args

        for jobkey, job_config in sorted(self._job_configs.items()):
            full_job_config = base_config.copy()
            full_job_config.update(job_config)
            ScatterGather._make_scatter_logfile_name(jobkey,
                                                     self.linkname, full_job_config)
            logfile = job_config.get('logfile')
            self._scatter_link._register_job(key=jobkey,
                                             job_config=full_job_config,
                                             logfile=logfile,
                                             status=status)
221927
def get_task_def_files_from_xml(self, sourcefilesTag, base_dir):
        
        sourcefiles = []

        # get included sourcefiles
        for includedFiles in sourcefilesTag.findall("include"):
            sourcefiles += self.expand_filename_pattern(includedFiles.text, base_dir)

        # get sourcefiles from list in file
        for includesFilesFile in sourcefilesTag.findall("includesfile"):

            for file in self.expand_filename_pattern(includesFilesFile.text, base_dir):

                # check for code (if somebody confuses 'include' and 'includesfile')
                if util.is_code(file):
                    logging.error(
                        "'%s' seems to contain code instead of a set of source file names.\n"
                        "Please check your benchmark definition file "
                        "or remove bracket '{' from this file.",
                        file)
                    sys.exit()

                # read files from list
                fileWithList = open(file, 'rt')
                for line in fileWithList:

                    # strip() removes 'newline' behind the line
                    line = line.strip()

                    # ignore comments and empty lines
                    if not util.is_comment(line):
                        sourcefiles += self.expand_filename_pattern(line, os.path.dirname(file))

                fileWithList.close()

        # remove excluded sourcefiles
        for excludedFiles in sourcefilesTag.findall("exclude"):
            excludedFilesList = self.expand_filename_pattern(excludedFiles.text, base_dir)
            for excludedFile in excludedFilesList:
                sourcefiles = util.remove_all(sourcefiles, excludedFile)

        for excludesFilesFile in sourcefilesTag.findall("excludesfile"):
            for file in self.expand_filename_pattern(excludesFilesFile.text, base_dir):
                # read files from list
                fileWithList = open(file, 'rt')
                for line in fileWithList:

                    # strip() removes 'newline' behind the line
                    line = line.strip()

                    # ignore comments and empty lines
                    if not util.is_comment(line):
                        excludedFilesList = self.expand_filename_pattern(line, os.path.dirname(file))
                        for excludedFile in excludedFilesList:
                            sourcefiles = util.remove_all(sourcefiles, excludedFile)

                fileWithList.close()

        return sourcefiles
938580
def _print_task_data(self, task):
    
    print(' {0:s} ({1:s})'.format(task['name'], task['id']))
    paths = task.get('saved_paths', [])
    if not paths:
      return
    for path in paths:
      if path.endswith('worker-log.txt'):
        continue
      if path.endswith('{0:s}.log'.format(task.get('id'))):
        continue
      if path.startswith('/'):
        continue
      print('   ' + path)
464537
def _job_to_text(self, job):
        

        next_run = self._format_date(job.get('next_run', None))

        tasks = ''
        for task in job.get('tasks', []):
            tasks += self._task_to_text(task)
            tasks += '\n\n'

        return '\n'.join(['Job name: %s' % job.get('name', None),
                          'Cron schedule: %s' % job.get('cron_schedule', None),
                          'Next run: %s' % next_run,
                          '',
                          'Parent ID: %s' % job.get('parent_id', None),
                          'Job ID: %s' % job.get('job_id', None),
                          '',
                          'Tasks Detail',
                          '',
                          tasks])
182197
def tail(self, stream='stdout', num_lines=10):
        
        target = self._map_string_to_file(stream)
        if not target:  # no current temp file
            last_run = self.backend.get_latest_run_log(self.parent_job.job_id,
                                                       self.name)
            if not last_run:
                return None
            return self._tail_string(last_run['tasks'][self.name][stream],
                                     num_lines)
        else:
            return self._tail_temp_file(target, num_lines)
81684
def dump_logs(job_queue='run_reach_queue', job_status='RUNNING'):
    """"""
    jobs = get_jobs(job_queue, job_status)
    for job in jobs:
        get_job_log(job, write_file=True)
1256685
def job_details(job_id, connection=None):
    """"""
    if connection is None:
        connection = r
    data = connection.hgetall(job_key(job_id))

    job_data = {'id': job_id, 'schedule_at': int(connection.zscore(REDIS_KEY,
                                                                   job_id))}
    for key, value in data.items():
        try:
            decoded = value.decode('utf-8')
        except UnicodeDecodeError:
            decoded = value
        if decoded.isdigit():
            decoded = int(decoded)
        job_data[key.decode('utf-8')] = decoded
    return job_data
449655
def process_update_job_from_file(file_path, report_email = nil, postback_url = nil, options = {})
      data = options
      data['file'] = file_path
      process_job(:update, data, report_email, postback_url, 'file')
    end
765549
def trace_line_numbers(filename, reload_on_change=False):
    """"""
    fullname = cache_file(filename, reload_on_change)
    if not fullname: return None
    e = file_cache[filename]
    if not e.line_numbers:
        if hasattr(coverage.coverage, 'analyze_morf'):
            e.line_numbers = coverage.the_coverage.analyze_morf(fullname)[1]
        else:
            cov = coverage.coverage()
            cov._warn_no_data = False
            e.line_numbers = cov.analysis(fullname)[1]
            pass
        pass
    return e.line_numbers
78508
def get_job_log_url(self, project, **params):
        
        return self._get_json(self.JOB_LOG_URL_ENDPOINT, project,
                              **params)
1143078
def submit_and_verify(
    xml_str=None, xml_file=None, xml_root=None, config=None, session=None, dry_run=None, **kwargs
):
    """"""
    try:
        config = config or configuration.get_config()
        xml_root = _get_xml_root(xml_root, xml_str, xml_file)
        submit_config = SubmitConfig(xml_root, config, **kwargs)
        session = session or utils.get_session(submit_config.credentials, config)
        submit_response = submit(xml_root, submit_config, session, dry_run=dry_run, **kwargs)
    except Dump2PolarionException as err:
        logger.error(err)
        return None

    valid_response = submit_response.validate_response()
    if not valid_response or kwargs.get("no_verify"):
        return submit_response.response

    response = verify_submit(
        session,
        submit_config.queue_url,
        submit_config.log_url,
        submit_response.job_ids,
        timeout=kwargs.get("verify_timeout"),
        log_file=kwargs.get("log_file"),
    )

    return response
231563
def handle(job_id, log_level='info', log_file=None):
    
    handlers = [LogDatabaseHandler(job_id)]  # log on db always
    if log_file is None:
        # add a StreamHandler if not already there
        if not any(h for h in logging.root.handlers
                   if isinstance(h, logging.StreamHandler)):
            handlers.append(LogStreamHandler(job_id))
    else:
        handlers.append(LogFileHandler(job_id, log_file))
    for handler in handlers:
        logging.root.addHandler(handler)
    init(job_id, LEVELS.get(log_level, logging.WARNING))
    try:
        yield
    finally:
        # sanity check to make sure that the logging on file is working
        if (log_file and log_file != os.devnull and
                os.path.getsize(log_file) == 0):
            logging.root.warn('The log file %s is empty!?' % log_file)
        for handler in handlers:
            logging.root.removeHandler(handler)
42506
def get_job_ids(self):
        """"""
        if not self.parsed_response:
            return None
        try:
            job_ids = self.parsed_response["files"]["results.xml"]["job-ids"]
        except KeyError:
            return None
        if not job_ids or job_ids == [0]:
            return None
        return job_ids
47830
def get_job_logs(id):
    """"""
    crawler_job = models.CrawlerJob.query.filter_by(id=id).one_or_none()
    if crawler_job is None:
        click.secho(
            (
                "CrawlJob %s was not found, maybe it's not a crawl job?" %
                id
            ),
            fg='yellow',
        )
        sys.exit(1)

    if crawler_job.logs is None:
        click.secho(
            (
                "CrawlJob %s has no log, it might be that it has not run "
                "yet, you can try again later." %
                id
            ),
            fg='yellow',
        )
        sys.exit(1)

    _show_file(
        file_path=crawler_job.logs,
        header_name='Log',
    )
104165
def _save_traceback_history(self, status, trace, job_exc):
        
        failure_date = datetime.datetime.utcnow()

        new_history = {
            "date": failure_date,
            "status": status,
            "exceptiontype": job_exc.__name__
        }

        traces = trace.split("---- Original exception: -----")
        if len(traces) > 1:
            new_history["original_traceback"] = traces[1]
        worker = context.get_current_worker()
        if worker:
            new_history["worker"] = worker.id
        new_history["traceback"] = traces[0]
        self.collection.update({
            "_id": self.id
        }, {"$push": {"traceback_history": new_history}})
959199
def trace_module(no_print=True):
    """"""
    with pexdoc.ExDocCxt() as exdoc_obj:
        try:
            docs.support.my_module.func("John")
            obj = docs.support.my_module.MyClass()
            obj.value = 5
            obj.value
        except:
            raise RuntimeError("Tracing did not complete successfully")
    if not no_print:
        module_prefix = "docs.support.my_module."
        callable_names = ["func", "MyClass.value"]
        for callable_name in callable_names:
            callable_name = module_prefix + callable_name
            print("\nCallable: {0}".format(callable_name))
            print(exdoc_obj.get_sphinx_doc(callable_name, width=70))
            print("\n")
    return copy.copy(exdoc_obj)