Post a list of artifacts to a job.
488785
def post_log_artifacts(job_log):
    """"""
    logger.debug("Downloading/parsing log for log %s", job_log.id)

    try:
        artifact_list = extract_text_log_artifacts(job_log)
    except LogSizeException as e:
        job_log.update_status(JobLog.SKIPPED_SIZE)
        logger.warning('Skipping parsing log for %s: %s', job_log.id, e)
        return
    except Exception as e:
        job_log.update_status(JobLog.FAILED)

        # Unrecoverable http error (doesn't exist or permission denied).
        # Apparently this can happen somewhat often with taskcluster if
        # the job fails (bug 1154248), so just warn rather than raising,
        # to prevent the noise/load from retrying.
        if isinstance(e, HTTPError) and e.response.status_code in (403, 404):
            logger.warning("Unable to retrieve log for %s: %s", job_log.id, e)
            return

        logger.error("Failed to download/parse log for %s: %s", job_log.id, e)
        raise

    try:
        serialized_artifacts = serialize_artifact_json_blobs(artifact_list)
        store_job_artifacts(serialized_artifacts)
        job_log.update_status(JobLog.PARSED)
        logger.debug("Stored artifact for %s %s", job_log.job.repository.name,
                     job_log.job.id)
    except Exception as e:
        logger.error("Failed to store parsed artifact for %s: %s", job_log.id, e)
        raise
1108402
def create(self, request, project):
        
        JobNote.objects.create(
            job=Job.objects.get(repository__name=project,
                                id=int(request.data['job_id'])),
            failure_classification_id=int(request.data['failure_classification_id']),
            user=request.user,
            text=request.data.get('text', ''))

        return Response(
            {'message': 'note stored for job {0}'.format(
                request.data['job_id']
            )}
        )
1138028
def _save_artifact(build, data, content_type):
    """"""
    sha1sum = hashlib.sha1(data).hexdigest()
    artifact = models.Artifact.query.filter_by(id=sha1sum).first()

    if artifact:
      logging.debug('Upload already exists: artifact_id=%r', sha1sum)
    else:
      logging.info('Upload received: artifact_id=%r, content_type=%r',
                   sha1sum, content_type)
      artifact = models.Artifact(
          id=sha1sum,
          content_type=content_type,
          data=data)
      _artifact_created(artifact)

    artifact.owners.append(build)
    return artifact
734898
def extract_text_log_artifacts(job_log):
    """"""

    # parse a log given its url
    artifact_bc = ArtifactBuilderCollection(job_log.url)
    artifact_bc.parse()

    artifact_list = []
    for name, artifact in artifact_bc.artifacts.items():
        artifact_list.append({
            "job_guid": job_log.job.guid,
            "name": name,
            "type": 'json',
            "blob": json.dumps(artifact)
        })

    return artifact_list
506245
def list(self, request, project):
        
        job_ids = request.query_params.getlist('job_id')
        if not job_ids:
            raise ParseError(
                detail="The job_id parameter is mandatory for this endpoint")
        try:
            job_ids = [int(job_id) for job_id in job_ids]
        except ValueError:
            raise ParseError(detail="The job_id parameter(s) must be integers")

        logs = JobLog.objects.filter(job__repository__name=project,
                                     job_id__in=job_ids)

        return Response([self._log_as_dict(log) for log in logs])
722376
def search(self, jobs: List[Dict[str, str]]) -> None:
        """"""
        if not isinstance(jobs, list):
            raise Exception("Jobs must be of type list.")
        self._log.info("Project: %s" % self.project)
        self._log.info("Processing jobs: %d", len(jobs))
        for _, job in enumerate(jobs):
            self._unfullfilled.put(job)

        for _ in range(self.PROCESSES):
            proc: Process = Process(target=self._job_handler)
            self._processes.append(proc)
            proc.start()

        for proc in self._processes:
            proc.join()

        while not self._fulfilled.empty():
            output: Dict = self._fulfilled.get()
            output.update({'project': self.project})
            self._processed.append(output['domain'])
            self.results.append(output)

            if output['greedy']:
                bonus_jobs: List = list()
                observed: List = list()
                for item in output['results']['emails']:
                    found: str = item.split('@')[1]
                    if found in self._processed or found in observed:
                        continue
                    observed.append(found)
                    base: Dict = dict()
                    base['limit'] = output['limit']
                    base['modifier'] = output['modifier']
                    base['engine'] = output['engine']
                    base['greedy'] = False
                    base['domain'] = found
                    bonus_jobs.append(base)

                if len(bonus_jobs) > 0:
                    self.search(bonus_jobs)

        self._log.info("All jobs processed")
        if self.output:
            self._save()
1364064
def submit_jobs(job_specs):
    
    gk = get_api_client()
    jobs = []
    try:
        for site, job_spec in job_specs:
            logger.info("Submitting %s on %s" % (job_spec, site))
            jobs.append(gk.sites[site].jobs.create(job_spec))
    except Exception as e:
        logger.error("An error occured during the job submissions")
        logger.error("Cleaning the jobs created")
        for job in jobs:
            job.delete()
        raise(e)

    return jobs
865049
protected function job(): void
    {
        // Get the assets for this character, chunked in a number of blocks
        // that the endpoint will accept.
        CorporationAsset::join('invTypes', 'type_id', '=', 'typeID')
            ->join('invGroups', 'invGroups.groupID', '=', 'invTypes.groupID')
            ->where('corporation_id', $this->getCorporationId())
            ->where('is_singleton', true)// only singleton items may be named
            // it seems only items from that categories can be named
            // 2  : Celestial
            // 6  : Ship
            // 22 : Deployable
            // 23 : Starbase
            // 46 : Orbitals
            // 65 : Structure
            ->whereIn('categoryID', [2, 6, 22, 23, 46, 65])// it seems only items from that categories can be named
            ->select('item_id')
            ->chunk($this->item_id_limit, function ($item_ids) {

                $this->request_body = $item_ids->pluck('item_id')->all();

                $names = $this->retrieve([
                    'corporation_id' => $this->getCorporationId(),
                ]);

                collect($names)->each(function ($name) {

                    // "None" seems to indidate that no name is set.
                    if ($name->name === 'None')
                        return;

                    CorporationAsset::where('corporation_id', $this->getCorporationId())
                        ->where('item_id', $name->item_id)
                        ->update(['name' => $name->name]);
                });
            });
    }
682042
def submit_jobs(self, link, job_dict=None, job_archive=None, stream=sys.stdout):
        
        if link is None:
            return JobStatus.no_job
        if job_dict is None:
            job_keys = link.jobs.keys()
        else:
            job_keys = sorted(job_dict.keys())

        # copy & reverse the keys b/c we will be popping item off the back of
        # the list
        unsubmitted_jobs = job_keys
        unsubmitted_jobs.reverse()

        failed = False
        if unsubmitted_jobs:
            if stream != sys.stdout:
                sys.stdout.write('Submitting jobs (%i): ' %
                                 len(unsubmitted_jobs))
                sys.stdout.flush()
        while unsubmitted_jobs:
            status = get_lsf_status()
            njob_to_submit = min(self._max_jobs - status['NJOB'],
                                 self._jobs_per_cycle,
                                 len(unsubmitted_jobs))

            if self._dry_run:
                njob_to_submit = len(unsubmitted_jobs)

            for i in range(njob_to_submit):
                job_key = unsubmitted_jobs.pop()

                # job_details = job_dict[job_key]
                job_details = link.jobs[job_key]
                job_config = job_details.job_config
                if job_details.status == JobStatus.failed:
                    clean_job(job_details.logfile, {}, self._dry_run)
                    # clean_job(job_details.logfile,
                    #          job_details.outfiles, self.args['dry_run'])

                job_config['logfile'] = job_details.logfile
                new_job_details = self.dispatch_job(
                    link, job_key, job_archive, stream)
                if new_job_details.status == JobStatus.failed:
                    failed = True
                    clean_job(new_job_details.logfile,
                              new_job_details.outfiles, self._dry_run)
                link.jobs[job_key] = new_job_details

            if unsubmitted_jobs:
                if stream != sys.stdout:
                    sys.stdout.write('.')
                    sys.stdout.flush()
                stream.write('Sleeping %.0f seconds between submission cycles\n' %
                             self._time_per_cycle)
                time.sleep(self._time_per_cycle)

        if failed:
            return JobStatus.failed

        if stream != sys.stdout:
            sys.stdout.write('!\n')

        return JobStatus.done
583906
function (jobName, customParams, callback) {
      [jobName, customParams, callback] = doArgs(arguments, ['string', ['object', {}], 'function']);

      doRequest({
        method: 'POST',
        urlPattern: [LAST_SUCCESS, jobName]
      }, customParams, callback);
    }
916702
def list(context, job_id):
    

    result = analytic.list(context, job_id=job_id)
    utils.format_output(result, context.format)
420207
def _archive_self(self, logfile, key=JobDetails.topkey, status=JobStatus.unknown):
        """"""
        self._register_self(logfile, key, status)
        if self._job_archive is None:
            return
        self._job_archive.register_jobs(self.get_jobs())
100728
def post(self):
        """"""
        self.reqparse.add_argument('name', type=str, required=True)
        self.reqparse.add_argument('color', type=str, required=True)
        args = self.reqparse.parse_args()

        role = Role()
        role.name = args['name']
        role.color = args['color']

        db.session.add(role)
        db.session.commit()
        auditlog(event='role.create', actor=session['user'].username, data=args)

        return self.make_response('Role {} has been created'.format(role.role_id), HTTP.CREATED)
1252624
def list(self, request, project):
        
        MAX_JOBS_COUNT = 2000

        # make a mutable copy of these params
        filter_params = request.query_params.copy()

        # various hacks to ensure API backwards compatibility
        for param_key in filter_params.keys():
            # replace `result_set_id` with `push_id`
            if param_key.startswith('result_set_id'):
                new_param_key = param_key.replace('result_set_id', 'push_id')
                filter_params[new_param_key] = filter_params[param_key]
                del filter_params[param_key]
            # convert legacy timestamp parameters to time ones
            elif param_key in ['submit_timestamp', 'start_timestamp',
                               'end_timestamp']:
                new_param_key = param_key.replace('timestamp', 'time')
                filter_params[new_param_key] = datetime.datetime.fromtimestamp(
                    float(filter_params[param_key]))
                del filter_params[param_key]
            # sanity check 'last modified'
            elif param_key.startswith('last_modified'):
                datestr = filter_params[param_key]
                try:
                    parser.parse(datestr)
                except ValueError:
                    return Response(
                        "Invalid date value for `last_modified`: {}".format(datestr),
                        status=HTTP_400_BAD_REQUEST)

        try:
            offset = int(filter_params.get("offset", 0))
            count = int(filter_params.get("count", 10))
        except ValueError:
            return Response(
                "Invalid value for offset or count",
                status=HTTP_400_BAD_REQUEST)
        return_type = filter_params.get("return_type", "dict").lower()

        if count > MAX_JOBS_COUNT:
            msg = "Specified count exceeds API MAX_JOBS_COUNT value: {}".format(MAX_JOBS_COUNT)
            return Response({"detail": msg}, status=HTTP_400_BAD_REQUEST)

        try:
            repository = Repository.objects.get(name=project)
        except Repository.DoesNotExist:
            return Response({
                "detail": "No project with name {}".format(project)
            }, status=HTTP_404_NOT_FOUND)
        jobs = JobFilter({k: v for (k, v) in filter_params.items()},
                         queryset=Job.objects.filter(
                             repository=repository).select_related(
                                 *self._default_select_related)).qs

        response_body = self._get_job_list_response(jobs, offset, count,
                                                    return_type)
        response_body["meta"] = dict(repository=project, offset=offset,
                                     count=count)

        return Response(response_body)
841105
def post(self, object_type, object_id):
        """"""
        if object_id == 0:
            return Response(status=404)

        tagged_objects = []
        for name in request.get_json(force=True):
            if ':' in name:
                type_name = name.split(':', 1)[0]
                type_ = TagTypes[type_name]
            else:
                type_ = TagTypes.custom

            tag = db.session.query(Tag).filter_by(name=name, type=type_).first()
            if not tag:
                tag = Tag(name=name, type=type_)

            tagged_objects.append(
                TaggedObject(
                    object_id=object_id,
                    object_type=object_type,
                    tag=tag,
                ),
            )

        db.session.add_all(tagged_objects)
        db.session.commit()

        return Response(status=201)
1045608
def _CollectArtifact(self, artifact, apply_parsers):
    """"""
    artifact_result = rdf_artifacts.CollectedArtifact(name=artifact.name)

    if apply_parsers:
      parser_factory = parsers.ArtifactParserFactory(str(artifact.name))
    else:
      parser_factory = None

    for source_result_list in self._ProcessSources(artifact.sources,
                                                   parser_factory):
      for response in source_result_list:
        action_result = rdf_artifacts.ClientActionResult()
        action_result.type = response.__class__.__name__
        action_result.value = response
        artifact_result.action_results.append(action_result)
        self.UpdateKnowledgeBase(response, artifact.provides)

    return artifact_result
584909
def UploadArtifactYamlFile(file_content,
                           overwrite=True,
                           overwrite_system_artifacts=False):
  """"""
  loaded_artifacts = []
  registry_obj = artifact_registry.REGISTRY
  # Make sure all artifacts are loaded so we don't accidentally overwrite one.
  registry_obj.GetArtifacts(reload_datastore_artifacts=True)

  new_artifacts = registry_obj.ArtifactsFromYaml(file_content)
  new_artifact_names = set()
  # A quick syntax check before we upload anything.
  for artifact_value in new_artifacts:
    artifact_registry.ValidateSyntax(artifact_value)
    new_artifact_names.add(artifact_value.name)

  # Iterate through each artifact adding it to the collection.
  artifact_coll = artifact_registry.ArtifactCollection(ARTIFACT_STORE_ROOT_URN)
  current_artifacts = list(artifact_coll)

  # We need to remove artifacts we are overwriting.
  filtered_artifacts = [
      art for art in current_artifacts if art.name not in new_artifact_names
  ]

  artifact_coll.Delete()
  with data_store.DB.GetMutationPool() as pool:
    for artifact_value in filtered_artifacts:
      artifact_coll.Add(artifact_value, mutation_pool=pool)

    for artifact_value in new_artifacts:
      registry_obj.RegisterArtifact(
          artifact_value,
          source="datastore:%s" % ARTIFACT_STORE_ROOT_URN,
          overwrite_if_exists=overwrite,
          overwrite_system_artifacts=overwrite_system_artifacts)

      artifact_coll.Add(artifact_value, mutation_pool=pool)
      if data_store.RelationalDBEnabled():
        data_store.REL_DB.WriteArtifact(artifact_value)

      loaded_artifacts.append(artifact_value)

      name = artifact_value.name
      logging.info("Uploaded artifact %s to %s", name, ARTIFACT_STORE_ROOT_URN)

  # Once all artifacts are loaded we can validate dependencies. Note that we do
  # not have to perform a syntax validation because it is already done after
  # YAML is parsed.
  for artifact_value in loaded_artifacts:
    artifact_registry.ValidateDependencies(artifact_value)