Persists the current job metadata to MongoDB. Will be called at each worker report.
434892
def save(self):
        

        if not self.saved and self.data and "progress" in self.data:
            # TODO should we save more fields?
            self.collection.update({"_id": self.id}, {"$set": {
                "progress": self.data["progress"]
            }})
            self.saved = True
113308
def tracked(self):
        ''''''
        results = json.loads(self.client('track'))
        results['jobs'] = [Job(self, **job) for job in results['jobs']]
        return results
303068
def add_job_to_db(self, key, job):
        """"""
        job_msg = self.registry.deep_encode(job)
        prov = prov_key(job_msg)

        def set_link(duplicate_id):
            self.cur.execute(
                'update "jobs" set "link" = ?, "status" = ? where "id" = ?',
                (duplicate_id, Status.DUPLICATE, key))

        with self.lock:
            self.cur.execute(
                'select * from "jobs" where "prov" = ? '
                'and (("result" is not null) or '
                '("session" = ? and "link" is null))',
                (prov, self.session))
            rec = self.cur.fetchone()
            rec = JobEntry(*rec) if rec is not None else None

            self.cur.execute(
                'update "jobs" set "prov" = ?, "version" = ?, "function" = ?, '
                '"arguments" = ?, "status" = ? where "id" = ?',
                (prov, job_msg['data']['hints'].get('version'),
                 json.dumps(job_msg['data']['function']),
                 json.dumps(job_msg['data']['arguments']),
                 Status.WAITING,
                 key))

            if not rec:
                # no duplicate found, go on
                return 'initialized', None

            set_link(rec.id)

            if rec.result is not None and rec.status == Status.WORKFLOW:
                # the found duplicate returned a workflow
                if rec.link is not None:
                    # link is set, so result is fully realized
                    self.cur.execute(
                        'select * from "jobs" where "id" = ?',
                        (rec.link,))
                    rec = self.cur.fetchone()
                    assert rec is not None, "database integrity violation"
                    rec = JobEntry(*rec)

                else:
                    # link is not set, the result is still waited upon
                    assert rec.session == self.session, \
                        "database integrity violation"
                    self.attached[rec.id].append(key)
                    return 'attached', None

            if rec.result is not None:
                # result is found! return it
                result_value = self.registry.from_json(rec.result, deref=True)
                result = ResultMessage(
                    key, 'retrieved', result_value, None)
                return 'retrieved', result

            if rec.session == self.session:
                # still waiting for result, attach
                self.attached[rec.id].append(key)
                return 'attached', None
527039
def persist(self):
        """"""
        if self.hash:
            with open(self.file_name, 'wb') as f:
                pickle.dump(self.object_property, f)
            return True
        return False
250911
def _serialize(self, include_run_logs=False, strict_json=False):
        
        result = {'dagobah_id': self.dagobah_id,
                  'created_jobs': self.created_jobs,
                  'jobs': [job._serialize(include_run_logs=include_run_logs,
                                          strict_json=strict_json)
                           for job in self.jobs]}
        if strict_json:
            result = json.loads(json.dumps(result, cls=StrictJSONEncoder))
        return result
1285876
@Override
    public JobMeta getJobMeta(String jobType) {
        final Map<String, String> document = map.get(jobType);
        if (document != null) {
            final Map<String, String> meta = document.keySet()
                    .stream()
                    .filter(key -> !key.startsWith("_e_"))
                    .collect(toMap(
                            key -> key,
                            document::get
                    ));
            final boolean isRunning = document.containsKey(KEY_RUNNING);
            final boolean isDisabled = document.containsKey(KEY_DISABLED);
            final String comment = document.get(KEY_DISABLED);
            return new JobMeta(jobType, isRunning, isDisabled, comment, meta);
        } else {
            return new JobMeta(jobType, false, false, "", emptyMap());
        }
    }
171985
def job_data(cls, job_id=dummy_job_id, branches=None, **kwargs):
        
        return dict(job_id=job_id, branches=branches or [])
1181845
def job_to_dict(job):
    """"""

    data = OrderedDict()
    data['id'] = job.id
    data['name'] = job.name
    data['func'] = job.func_ref
    data['args'] = job.args
    data['kwargs'] = job.kwargs

    data.update(trigger_to_dict(job.trigger))

    if not job.pending:
        data['misfire_grace_time'] = job.misfire_grace_time
        data['max_instances'] = job.max_instances
        data['next_run_time'] = None if job.next_run_time is None else job.next_run_time

    return data
89335
def save_metadata(self, profile="default"):
        """"""
        if len(self.tables) > 0:
            f = profile_path(DBPY_PROFILE_ID, profile)
            dump_to_json(f, self.to_dict())
605554
def update(self, job_binary_id, name=NotUpdated, is_public=NotUpdated,
               is_protected=NotUpdated):
        """"""

        data = {}
        self._copy_if_updated(data, name=name, is_public=is_public,
                              is_protected=is_protected)

        return self._patch('/job-binary-internals/%s' % job_binary_id, data)
289698
function(data, done) {
      if(!this._key) {
        done('no key found for metadata');
        return;
      }
      var key = data[this._key];
      var self = this;
      this._getCollection(function(err) {
        if(err) {
          done(err);
        }
        var kobj = {};
        kobj[self._key.getName()] = key;
        self._collection.find(kobj).toArray(function(err, docs) {
          if(docs.length > 0) {
            data._id = docs[0]._id;
          }
          self._collection.save(data, function(err, count) {
            self._db.close();
            done(err);
          });
        });
      });
    }
347313
def pass_job(db: JobDB, result_queue: Queue, always_cache=False):
    
    @pull
    def pass_job_stream(job_source):
        """Pull stream instance created by `pass_job`."""
        result_sink = result_queue.sink()

        for message in job_source():
            if message is EndOfQueue:
                return

            key, job = message
            if always_cache or ('store' in job.hints):
                status, retrieved_result = db.add_job_to_db(key, job)

                if status == 'retrieved':
                    result_sink.send(retrieved_result)
                    continue

                elif status == 'attached':
                    continue

            yield message

    return pass_job_stream
157134
protected void aggregateReports() {
    clusterUtil.clear();
    for (String jobId : allJobUtil.keySet()) {
      JobUtilization jobUtil = allJobUtil.get(jobId);
      jobUtil.clear();
      jobUtil.setIsRunning(false);
    }

    for (UtilizationReport report : taskTrackerReports.values()) {
      if (report.isExpired()) {
        continue; // ignore the report if it is older than the timeLimit
      }
      LocalJobUtilization[] localJobUtils = report.getJobUtilization();
      TaskTrackerUtilization ttUtil = report.getTaskTrackerUtilization();

      // Aggregate cluster information
      double cpuUsageGHz = clusterUtil.getCpuUsageGHz();
      double cpuTotalGHz = clusterUtil.getCpuTotalGHz();
      double memUsageGB = clusterUtil.getMemUsageGB();
      double memTotalGB = clusterUtil.getMemTotalGB();
      int numCpu = clusterUtil.getNumCpu();
      int numTaskTrackers = clusterUtil.getNumTaskTrackers();

      cpuUsageGHz += ttUtil.getCpuUsageGHz();
      cpuTotalGHz += ttUtil.getCpuTotalGHz();
      memUsageGB += ttUtil.getMemUsageGB();
      memTotalGB += ttUtil.getMemTotalGB();
      numCpu += ttUtil.getNumCpu();
      numTaskTrackers += 1;

      clusterUtil.setCpuUsageGHz(cpuUsageGHz);
      clusterUtil.setCpuTotalGHz(cpuTotalGHz);
      clusterUtil.setMemUsageGB(memUsageGB);
      clusterUtil.setMemTotalGB(memTotalGB);
      clusterUtil.setNumCpu(numCpu);
      clusterUtil.setNumTaskTrackers(numTaskTrackers);

      // Aggregate Job based information
      if ( localJobUtils == null) {
        continue;
      }

      for (LocalJobUtilization localJobUtil : localJobUtils) {
        String jobId = localJobUtil.getJobId();
        if (jobId == null) {
          continue;
        }

        if (!allJobUtil.containsKey(jobId)) {
          JobUtilization jobUtil = new JobUtilization();
          jobUtil.setJobId(jobId);
          allJobUtil.put(jobId, jobUtil);
        }
        JobUtilization jobUtil = allJobUtil.get(jobId);
        jobUtil.setIsRunning(true);
        double maxCpu = jobUtil.getCpuMaxPercentageOnBox();
        double cpu = jobUtil.getCpuPercentageOnCluster();
        double cpuGigaCycles = jobUtil.getCpuGigaCycles();
        double maxMem = jobUtil.getMemMaxPercentageOnBox();
        double mem = jobUtil.getMemPercentageOnCluster();

        cpu += localJobUtil.getCpuUsageGHz(); // will be normalized later
        mem += localJobUtil.getMemUsageGB(); // will be normalized later
        cpuGigaCycles += localJobUtil.getCpuUsageGHz() *
                         (double)aggregatePeriod * 1e6D / 1e9D;
        // GHz * ms = 1e6. To convert to Giga, divide by 1e9

        double localCpuPercentage =
                localJobUtil.getCpuUsageGHz() / ttUtil.getCpuTotalGHz() * 100;
        double localMemPercentage =
                localJobUtil.getMemUsageGB() / ttUtil.getMemTotalGB() * 100;

        if (maxCpu < localCpuPercentage) {
          maxCpu = localCpuPercentage;
        }
        if (maxMem < localMemPercentage) {
          maxMem = localMemPercentage;
        }
        jobUtil.setCpuMaxPercentageOnBox(maxCpu);
        jobUtil.setCpuGigaCycles(cpuGigaCycles);
        jobUtil.setCpuPercentageOnCluster(cpu); // will be normalized later
        jobUtil.setMemMaxPercentageOnBox(maxMem); // will be normalized later
        jobUtil.setMemPercentageOnCluster(mem);
        if (maxMem > jobUtil.getMemMaxPercentageOnBoxAllTime()) {
          jobUtil.setMemMaxPercentageOnBoxAllTime(maxMem);
        }
      }
    }

    // Normalization and clean up finished jobs
    for (Iterator<String>
            it = allJobUtil.keySet().iterator(); it.hasNext();) {
      String jobId = it.next();
      JobUtilization jobUtil = allJobUtil.get(jobId);
 	 	  if (!jobUtil.getIsRunning()) {
        long stoppedTime = jobUtil.getStoppedTime();
        stoppedTime += aggregatePeriod;
        jobUtil.setStoppedTime(stoppedTime);
        jobUtil.setIsRunning(false);

        if (stoppedTime > stopTimeLimit) {
          // These are finished jobs
          // We may store the information of finished jobs to some place
          double cpuTime = jobUtil.getCpuCumulatedUsageTime() / 1000D;
          double memTime = jobUtil.getMemCumulatedUsageTime() / 1000D;
          double peakMem = jobUtil.getMemMaxPercentageOnBoxAllTime();
          double cpuGigaCycles = jobUtil.getCpuGigaCycles();
          it.remove();
          LOG.info(String.format( 
                "Job done: [JobID,CPU(sec),Mem(sec),Peak Mem(%%),CPU gigacycles]" +
                " = [%s,%f,%f,%f,%f]",
                jobId, cpuTime, memTime, peakMem, cpuGigaCycles));
        }
        continue;
			}

      long runningTime = jobUtil.getRunningTime();
      runningTime += aggregatePeriod; // millisecond to second
      jobUtil.setRunningTime(runningTime);
      jobUtil.setStoppedTime(0);
      jobUtil.setIsRunning(true);

      int numJobs = clusterUtil.getNumRunningJobs();
      numJobs += 1;
      clusterUtil.setNumRunningJobs(numJobs);

      double cpu = jobUtil.getCpuPercentageOnCluster();
      double mem = jobUtil.getMemPercentageOnCluster();
      double cpuTime = jobUtil.getCpuCumulatedUsageTime();
      double memTime = jobUtil.getMemCumulatedUsageTime();
      cpu = cpu / clusterUtil.getCpuTotalGHz() * 100;
      mem = mem / clusterUtil.getMemTotalGB() * 100;
      cpuTime += cpu / 100 * aggregatePeriod;  // in milliseconds
      memTime += mem / 100 * aggregatePeriod;  // in milliseconds
      jobUtil.setCpuPercentageOnCluster(cpu);
      jobUtil.setMemPercentageOnCluster(mem);
      jobUtil.setCpuCumulatedUsageTime(cpuTime);
      jobUtil.setMemCumulatedUsageTime(memTime);
    }
  }
39915
public function snapshot()
    {
        collect($this->measuredJobs())->each(function ($job) {
            $this->storeSnapshotForJob($job);
        });

        collect($this->measuredQueues())->each(function ($queue) {
            $this->storeSnapshotForQueue($queue);
        });

        $this->storeSnapshotTimestamp();
    }
34169
public function save(AbstractJob $job)
    {

        $data = $job->getData();
        if (!$job->isStored()) {
            $job->setId(uniqid());
        }

        $this->redis->set($this->namespace . $job->getId(), json_encode($data));

        return true;
    }
1182775
def store_meta(self,meta):
        ""
        if self.meta is None:
            self.meta = {}
        self.meta.update(meta)
        return self
230226
private function commitPersisted()
    {
        $sortedDocuments = $this->sortObjects($this->scheduledForPersist);
        $client = $this->sm->getClient();

        foreach ($sortedDocuments as $entityName => $documents) {
            $classMetadata = $this->sm->getClassMetadata($entityName);
            $this->updatedIndexes[] = $classMetadata->index;
            $client->addDocuments($classMetadata, $documents);
        }
    }