Insert a job into MongoDB
427295
def insert(cls, jobs_data, queue=None, statuses_no_storage=None, return_jobs=True, w=None, j=None):
        

        now = datetime.datetime.utcnow()
        for data in jobs_data:
            if data["status"] == "started":
                data["datestarted"] = now

        no_storage = (statuses_no_storage is not None) and ("started" in statuses_no_storage)
        if no_storage and return_jobs:
            for data in jobs_data:
                data["_id"] = ObjectId()  # Give the job a temporary ID
        else:
            inserted = context.connections.mongodb_jobs.mrq_jobs.insert(
                jobs_data,
                manipulate=True,
                w=w,
                j=j
            )

        if return_jobs:
            jobs = []
            for data in jobs_data:
                job = cls(data["_id"], queue=queue)
                job.set_data(data)
                job.statuses_no_storage = statuses_no_storage
                job.stored = (not no_storage)
                if data["status"] == "started":
                    job.datestarted = data["datestarted"]
                jobs.append(job)

            return jobs
        else:
            return inserted
233204
def run_job(self, project_id, body, dataset=None):
        

        if dataset and not self.dataset_exists(dataset):
            self.make_dataset(dataset)

        new_job = self.client.jobs().insert(projectId=project_id, body=body).execute()
        job_id = new_job['jobReference']['jobId']
        logger.info('Started import job %s:%s', project_id, job_id)
        while True:
            status = self.client.jobs().get(projectId=project_id, jobId=job_id).execute(num_retries=10)
            if status['status']['state'] == 'DONE':
                if status['status'].get('errorResult'):
                    raise Exception('BigQuery job failed: {}'.format(status['status']['errorResult']))
                return

            logger.info('Waiting for job %s:%s to complete...', project_id, job_id)
            time.sleep(5)
68118
function executeJob (options, opType, callback) {
  var job = client.query(options.namespace, options.set, options.statement)
  var stream = job.execute()
  stream.on('data', function (record) {
    // count the records returned
    intervalData[opType][TPS]++
  })
  stream.on('error', function (error) {
    intervalData[opType][ERROR]++
    if (error.code === status.ERR_TIMEOUT) {
      intervalData[opType][TIMEOUT]++
    }
  })
  stream.on('end', function () {
    // update a stat for number of jobs completed.
    callback(options)
  })
}
566074
function executeWrite(pool, bson, type, opsField, ns, ops, options, callback) {
  if (ops.length === 0) throw new MongoError('insert must contain at least one document');
  if (typeof options === 'function') {
    callback = options;
    options = {};
    options = options || {};
  }

  // Split the ns up to get db and collection
  const p = ns.split('.');
  const d = p.shift();
  // Options
  const ordered = typeof options.ordered === 'boolean' ? options.ordered : true;
  const writeConcern = options.writeConcern;

  // return skeleton
  const writeCommand = {};
  writeCommand[type] = p.join('.');
  writeCommand[opsField] = ops;
  writeCommand.ordered = ordered;

  // Did we specify a write concern
  if (writeConcern && Object.keys(writeConcern).length > 0) {
    writeCommand.writeConcern = writeConcern;
  }

  // If we have collation passed in
  if (options.collation) {
    for (let i = 0; i < writeCommand[opsField].length; i++) {
      if (!writeCommand[opsField][i].collation) {
        writeCommand[opsField][i].collation = options.collation;
      }
    }
  }

  // Do we have bypassDocumentValidation set, then enable it on the write command
  if (options.bypassDocumentValidation === true) {
    writeCommand.bypassDocumentValidation = options.bypassDocumentValidation;
  }

  // optionally decorate command with transactions data
  const err = decorateWithSessionsData(writeCommand, options.session, options, callback);
  if (err) {
    return callback(err, null);
  }

  // Options object
  const opts = { command: true };
  if (typeof options.session !== 'undefined') opts.session = options.session;
  const queryOptions = { checkKeys: false, numberToSkip: 0, numberToReturn: 1 };
  if (type === 'insert') queryOptions.checkKeys = true;
  if (typeof options.checkKeys === 'boolean') queryOptions.checkKeys = options.checkKeys;

  // Ensure we support serialization of functions
  if (options.serializeFunctions) queryOptions.serializeFunctions = options.serializeFunctions;
  // Do not serialize the undefined fields
  if (options.ignoreUndefined) queryOptions.ignoreUndefined = options.ignoreUndefined;

  try {
    // Create write command
    const cmd = new Query(bson, `${d}.$cmd`, writeCommand, queryOptions);
    // Execute command
    pool.write(cmd, opts, callback);
  } catch (err) {
    callback(err);
  }
}
303068
def add_job_to_db(self, key, job):
        """"""
        job_msg = self.registry.deep_encode(job)
        prov = prov_key(job_msg)

        def set_link(duplicate_id):
            self.cur.execute(
                'update "jobs" set "link" = ?, "status" = ? where "id" = ?',
                (duplicate_id, Status.DUPLICATE, key))

        with self.lock:
            self.cur.execute(
                'select * from "jobs" where "prov" = ? '
                'and (("result" is not null) or '
                '("session" = ? and "link" is null))',
                (prov, self.session))
            rec = self.cur.fetchone()
            rec = JobEntry(*rec) if rec is not None else None

            self.cur.execute(
                'update "jobs" set "prov" = ?, "version" = ?, "function" = ?, '
                '"arguments" = ?, "status" = ? where "id" = ?',
                (prov, job_msg['data']['hints'].get('version'),
                 json.dumps(job_msg['data']['function']),
                 json.dumps(job_msg['data']['arguments']),
                 Status.WAITING,
                 key))

            if not rec:
                # no duplicate found, go on
                return 'initialized', None

            set_link(rec.id)

            if rec.result is not None and rec.status == Status.WORKFLOW:
                # the found duplicate returned a workflow
                if rec.link is not None:
                    # link is set, so result is fully realized
                    self.cur.execute(
                        'select * from "jobs" where "id" = ?',
                        (rec.link,))
                    rec = self.cur.fetchone()
                    assert rec is not None, "database integrity violation"
                    rec = JobEntry(*rec)

                else:
                    # link is not set, the result is still waited upon
                    assert rec.session == self.session, \
                        "database integrity violation"
                    self.attached[rec.id].append(key)
                    return 'attached', None

            if rec.result is not None:
                # result is found! return it
                result_value = self.registry.from_json(rec.result, deref=True)
                result = ResultMessage(
                    key, 'retrieved', result_value, None)
                return 'retrieved', result

            if rec.session == self.session:
                # still waiting for result, attach
                self.attached[rec.id].append(key)
                return 'attached', None
925699
@Given("^I insert into MongoDB database '(.+?)' and collection '(.+?)' the document from schema '(.+?)'$")
    public void insertOnMongoTable(String dataBase, String collection, String document) throws Exception {
        String retrievedDoc = commonspec.retrieveData(document, "json");
        commonspec.getMongoDBClient().connectToMongoDBDataBase(dataBase);
        commonspec.getMongoDBClient().insertDocIntoMongoDBCollection(collection, retrievedDoc);
    }
724658
public void insertDocIntoMongoDBCollection(String collection, String document) {

        DBObject dbObject = (DBObject) JSON.parse(document);
        this.dataBase.getCollection(collection).insert(dbObject);

    }
33633
def put(self, job, result):
        ""
        self.job.put(job)
        r = result.get()
        return r
647410
def _insert(self, sock_info, docs, ordered=True, check_keys=True,
                manipulate=False, write_concern=None, op_id=None,
                bypass_doc_val=False):
        """"""
        if isinstance(docs, collections.Mapping):
            return self._insert_one(
                sock_info, docs, ordered,
                check_keys, manipulate, write_concern, op_id, bypass_doc_val)

        ids = []

        if manipulate:
            def gen():
                """Generator that applies SON manipulators to each document
                and adds _id if necessary.
                """
                _db = self.__database
                for doc in docs:
                    # Apply user-configured SON manipulators. This order of
                    # operations is required for backwards compatibility,
                    # see PYTHON-709.
                    doc = _db._apply_incoming_manipulators(doc, self)
                    if not (isinstance(doc, RawBSONDocument) or '_id' in doc):
                        doc['_id'] = ObjectId()

                    doc = _db._apply_incoming_copying_manipulators(doc, self)
                    ids.append(doc['_id'])
                    yield doc
        else:
            def gen():
                """Generator that only tracks existing _ids."""
                for doc in docs:
                    # Don't inflate RawBSONDocument by touching fields.
                    if not isinstance(doc, RawBSONDocument):
                        ids.append(doc.get('_id'))
                    yield doc

        concern = (write_concern or self.write_concern).document
        acknowledged = concern.get("w") != 0

        command = SON([('insert', self.name),
                       ('ordered', ordered)])
        if concern:
            command['writeConcern'] = concern
        if op_id is None:
            op_id = message._randint()
        if bypass_doc_val and sock_info.max_wire_version >= 4:
            command['bypassDocumentValidation'] = True
        bwc = message._BulkWriteContext(
            self.database.name, command, sock_info, op_id,
            self.database.client._event_listeners)
        if sock_info.max_wire_version > 1 and acknowledged:
            # Batched insert command.
            results = message._do_batched_write_command(
                self.database.name + ".$cmd", message._INSERT, command,
                gen(), check_keys, self.__write_response_codec_options, bwc)
            _check_write_command_response(results)
        else:
            # Legacy batched OP_INSERT.
            message._do_batched_insert(self.__full_name, gen(), check_keys,
                                       acknowledged, concern, not ordered,
                                       self.__write_response_codec_options, bwc)
        return ids
211856
def add_job(session, command_line, name = 'job', dependencies = [], array = None, exec_dir=None, log_dir = None, stop_on_failure = False, **kwargs):
  """"""
  job = Job(command_line=command_line, name=name, exec_dir=exec_dir, log_dir=log_dir, array_string=array, stop_on_failure=stop_on_failure, kwargs=kwargs)

  session.add(job)
  session.flush()
  session.refresh(job)

  # by default id and unique id are identical, but the id might be overwritten later on
  job.id = job.unique

  for d in dependencies:
    if d == job.unique:
      logger.warn("Adding self-dependency of job %d is not allowed" % d)
      continue
    depending = list(session.query(Job).filter(Job.unique == d))
    if len(depending):
      session.add(JobDependence(job.unique, depending[0].unique))
    else:
      logger.warn("Could not find dependent job with id %d in database" % d)

  if array:
    (start, stop, step) = array
    # add array jobs
    for i in range(start, stop+1, step):
      session.add(ArrayJob(i, job.unique))

  session.commit()

  return job
713948
def _add_job_from_spec(self, job_json, use_job_id=True):
        

        job_id = (job_json['job_id']
                  if use_job_id
                  else self.backend.get_new_job_id())
        self.add_job(str(job_json['name']), job_id)
        job = self.get_job(job_json['name'])
        if job_json.get('cron_schedule', None):
            job.schedule(job_json['cron_schedule'])

        for task in job_json.get('tasks', []):
            self.add_task_to_job(job,
                                 str(task['command']),
                                 str(task['name']),
                                 soft_timeout=task.get('soft_timeout', 0),
                                 hard_timeout=task.get('hard_timeout', 0),
                                 hostname=task.get('hostname', None))

        dependencies = job_json.get('dependencies', {})
        for from_node, to_nodes in dependencies.iteritems():
            for to_node in to_nodes:
                job.add_dependency(from_node, to_node)

        if job_json.get('notes', None):
            job.update_job_notes(job_json['notes'])
791615
def load_job(self, job):
        
        if not isinstance(job, Job):
            self.log_exc(u"job is not an instance of Job", None, True, ExecuteJobInputError)
        self.job = job
119098
private Document insertDocument(EntityKeyMetadata entityKeyMetadata, Tuple tuple, TupleContext tupleContext ) {
		Document dbObject = objectForInsert( tuple, ( (MongoDBTupleSnapshot) tuple.getSnapshot() ).getDbObject() );
		getCollection( entityKeyMetadata.getTable(), tupleContext.getTupleTypeContext().getOptionsContext() ).insertOne( dbObject );
		return dbObject;
	}
1117250
private void insertJobMetadata() {
    final String baseURL = this.azkabanProps.get(AZKABAN_WEBSERVER_URL);
    if (baseURL != null) {
      final String flowName = this.node.getParentFlow().getFlowId();
      final String projectName = this.node.getParentFlow().getProjectName();

      this.props.put(CommonJobProperties.AZKABAN_URL, baseURL);
      this.props.put(CommonJobProperties.EXECUTION_LINK,
          String.format("%s/executor?execid=%d", baseURL, this.executionId));
      this.props.put(CommonJobProperties.JOBEXEC_LINK, String.format(
          "%s/executor?execid=%d&job=%s", baseURL, this.executionId, this.node.getNestedId()));
      this.props.put(CommonJobProperties.ATTEMPT_LINK, String.format(
          "%s/executor?execid=%d&job=%s&attempt=%d", baseURL, this.executionId,
          this.node.getNestedId(), this.node.getAttempt()));
      this.props.put(CommonJobProperties.WORKFLOW_LINK, String.format(
          "%s/manager?project=%s&flow=%s", baseURL, projectName, flowName));
      this.props.put(CommonJobProperties.JOB_LINK, String.format(
          "%s/manager?project=%s&flow=%s&job=%s", baseURL, projectName,
          flowName, this.jobId));
    } else {
      if (this.logger != null) {
        this.logger.info(AZKABAN_WEBSERVER_URL + " property was not set");
      }
    }
    // out nodes
    this.props.put(CommonJobProperties.OUT_NODES,
        StringUtils.join2(this.node.getOutNodes(), ","));

    // in nodes
    this.props.put(CommonJobProperties.IN_NODES,
        StringUtils.join2(this.node.getInNodes(), ","));
  }
571635
def _insert(self, docs, ordered=True, check_keys=True,
                manipulate=False, write_concern=None, op_id=None,
                bypass_doc_val=False, session=None):
        """"""
        if isinstance(docs, abc.Mapping):
            return self._insert_one(
                docs, ordered, check_keys, manipulate, write_concern, op_id,
                bypass_doc_val, session)

        ids = []

        if manipulate:
            def gen():
                """Generator that applies SON manipulators to each document
                and adds _id if necessary.
                """
                _db = self.__database
                for doc in docs:
                    # Apply user-configured SON manipulators. This order of
                    # operations is required for backwards compatibility,
                    # see PYTHON-709.
                    doc = _db._apply_incoming_manipulators(doc, self)
                    if not (isinstance(doc, RawBSONDocument) or '_id' in doc):
                        doc['_id'] = ObjectId()

                    doc = _db._apply_incoming_copying_manipulators(doc, self)
                    ids.append(doc['_id'])
                    yield doc
        else:
            def gen():
                """Generator that only tracks existing _ids."""
                for doc in docs:
                    # Don't inflate RawBSONDocument by touching fields.
                    if not isinstance(doc, RawBSONDocument):
                        ids.append(doc.get('_id'))
                    yield doc

        write_concern = write_concern or self._write_concern_for(session)
        blk = _Bulk(self, ordered, bypass_doc_val)
        blk.ops = [(message._INSERT, doc) for doc in gen()]
        try:
            blk.execute(write_concern, session=session)
        except BulkWriteError as bwe:
            _raise_last_error(bwe.details)
        return ids
52395
function(name, doc, fromMessage, res) {
            var id   = this.toId(doc._id, true);
            if (typeof id === 'undefined' || id === '') {
                return res.send(400, "invalid id.");
            }
            doc._id  = id;
            doc._time = new Date().getTime();

            var collection = new mongodb.Collection(this.db, name);
            collection.insert(
                doc,
                {safe:true},
                function(err, docs) {
                    if(err) {
                        res.send(400, err);
                    } else {
                        var doc = docs && docs.length > 0 ? docs[0] : null;
                        if (doc) {
                            var msg = {
                                method: 'create',
                                id: doc._id,
                                time: doc._time,
                                data: doc
                            };
                            rest.onSuccess(name, msg, fromMessage);
                            res.send(doc);
                        } else {
                            res.send(400, 'failed to create document.');
                        }
                    }
                }
            );
        }
1041760
def WriteCronJobRun(self, run_object, cursor=None):
    """"""
    query = ("INSERT INTO cron_job_runs "
             "(job_id, run_id, write_time, run) "
             "VALUES (%s, %s, FROM_UNIXTIME(%s), %s) "
             "ON DUPLICATE KEY UPDATE "
             "run=VALUES(run), write_time=VALUES(write_time)")

    write_time_str = mysql_utils.RDFDatetimeToTimestamp(
        rdfvalue.RDFDatetime.Now())
    try:
      cursor.execute(query, [
          run_object.cron_job_id,
          db_utils.CronJobRunIDToInt(run_object.run_id),
          write_time_str,
          run_object.SerializeToString(),
      ])
    except MySQLdb.IntegrityError as e:
      raise db.UnknownCronJobError(
          "CronJob with id %s not found." % run_object.cron_job_id, cause=e)