Returns a batch instance after exporting a batch of txs.
556921
def export_batch(self):
        
        batch = self.batch_cls(
            model=self.model, history_model=self.history_model, using=self.using
        )
        if batch.items:
            try:
                json_file = self.json_file_cls(batch=batch, path=self.path)
                json_file.write()
            except JSONDumpFileError as e:
                raise TransactionExporterError(e)
            batch.close()
            return batch
        return None
967881
def batching(size):
    """"""

    if size < 1:
        raise ValueError("batching() size must be at least 1")

    def batching_transducer(reducer):
        return Batching(reducer, size)

    return batching_transducer
1353914
def batch(items, size):
    """"""
    return [items[x:x + size] for x in xrange(0, len(items), size)]
361743
def encode_batch(self, inputBatch):
        """"""
        X      = inputBatch
        encode = self.encode
        Y      = np.array([ encode(x) for x in X])
        return Y
481496
def _first_batch(sock_info, db, coll, query, ntoreturn,
                 slave_ok, codec_options, read_preference, cmd, listeners):
    """"""
    query = _Query(
        0, db, coll, 0, query, None, codec_options,
        read_preference, ntoreturn, 0, DEFAULT_READ_CONCERN, None)

    name = next(iter(cmd))
    duration = None
    publish = listeners.enabled_for_commands
    if publish:
        start = datetime.datetime.now()

    request_id, msg, max_doc_size = query.get_message(slave_ok,
                                                      sock_info.is_mongos)

    if publish:
        encoding_duration = datetime.datetime.now() - start
        listeners.publish_command_start(
            cmd, db, request_id, sock_info.address)
        start = datetime.datetime.now()

    sock_info.send_message(msg, max_doc_size)
    response = sock_info.receive_message(1, request_id)
    try:
        result = _unpack_response(response, None, codec_options)
    except Exception as exc:
        if publish:
            duration = (datetime.datetime.now() - start) + encoding_duration
            if isinstance(exc, (NotMasterError, OperationFailure)):
                failure = exc.details
            else:
                failure = _convert_exception(exc)
            listeners.publish_command_failure(
                duration, failure, name, request_id, sock_info.address)
        raise
    if publish:
        duration = (datetime.datetime.now() - start) + encoding_duration
        listeners.publish_command_success(
            duration, result, name, request_id, sock_info.address)

    return result
94278
def _replace_batch(self):
        """"""
        if not self._head:
          self._head, self._buffer = self._record(self._buffer[0], 1, 0, None), self._buffer[1:]

        rank = 0.0
        current = self._head

        for b in self._buffer:
            if b < self._head._value:
              self._head = self._record(b, 1, 0, self._head)

            while current._successor and current._value < b:
                rank += current._rank
                current = current._successor

            if not current._successor:
                current._successor = self._record(b, 1, 0, None)

            current._successor = self._record(b, 1, self._invariant(rank, self._observations)-1, current._successor)
1262873
def stub_batch(cls, size, **kwargs):
        
        return [cls.stub(**kwargs) for _ in range(size)]
1003387
def _get_batches(self, mapping, batch_size=10000):
        """"""
        action = mapping.get("action", "insert")
        fields = mapping.get("fields", {}).copy()
        static = mapping.get("static", {})
        lookups = mapping.get("lookups", {})
        record_type = mapping.get("record_type")

        # Skip Id field on insert
        if action == "insert" and "Id" in fields:
            del fields["Id"]

        # Build the list of fields to import
        columns = []
        columns.extend(fields.keys())
        columns.extend(lookups.keys())
        columns.extend(static.keys())

        if record_type:
            columns.append("RecordTypeId")
            # default to the profile assigned recordtype if we can't find any
            # query for the RT by developer name
            query = (
                "SELECT Id FROM RecordType WHERE SObjectType='{0}'"
                "AND DeveloperName = '{1}' LIMIT 1"
            )
            record_type_id = self.sf.query(
                query.format(mapping.get("sf_object"), record_type)
            )["records"][0]["Id"]

        query = self._query_db(mapping)

        total_rows = 0
        batch_num = 1

        def start_batch():
            batch_file = io.BytesIO()
            writer = unicodecsv.writer(batch_file)
            writer.writerow(columns)
            batch_ids = []
            return batch_file, writer, batch_ids

        batch_file, writer, batch_ids = start_batch()
        for row in query.yield_per(batch_size):
            total_rows += 1

            # Add static values to row
            pkey = row[0]
            row = list(row[1:]) + list(static.values())
            if record_type:
                row.append(record_type_id)

            writer.writerow([self._convert(value) for value in row])
            batch_ids.append(pkey)

            # Yield and start a new file every [batch_size] rows
            if not total_rows % batch_size:
                batch_file.seek(0)
                self.logger.info("    Processing batch {}".format(batch_num))
                yield batch_file, batch_ids
                batch_file, writer, batch_ids = start_batch()
                batch_num += 1

        # Yield result file for final batch
        if batch_ids:
            batch_file.seek(0)
            yield batch_file, batch_ids

        self.logger.info(
            "  Prepared {} rows for import to {}".format(
                total_rows, mapping["sf_object"]
            )
        )
2827
def get_batches(qp_pairs, batch_size, need_sort=True):
    '''
    
    '''
    if need_sort:
        qp_pairs = sorted(qp_pairs, key=lambda qp: (
            len(qp['passage_tokens']), qp['id']), reverse=True)
    batches = [{'qp_pairs': qp_pairs[i:(i + batch_size)]}
               for i in range(0, len(qp_pairs), batch_size)]
    shuffle(batches)
    return batches
1370595
public JSONObject batch(List<JSONObject> actions, RequestOptions requestOptions) throws AlgoliaException {
    return postBatch(actions, requestOptions);
  }
17509
private void copyObjectIDsToBatch(BatchResult batchResult, DBObjectBatch dbObjBatch) {
        if (batchResult.getResultObjectCount() < dbObjBatch.getObjectCount()) {
            m_logger.warn("Batch result returned fewer objects ({}) than input batch ({})",
                          batchResult.getResultObjectCount(), dbObjBatch.getObjectCount());
        }
        Iterator<ObjectResult> resultIter = batchResult.getResultObjects().iterator();
        Iterator<DBObject> objectIter = dbObjBatch.getObjects().iterator();
        while (resultIter.hasNext()) {
            if (!objectIter.hasNext()) {
                m_logger.warn("Batch result has more objects ({}) than input batch ({})!",
                              batchResult.getResultObjectCount(), dbObjBatch.getObjectCount());
                break;
            }
            ObjectResult objResult = resultIter.next();
            DBObject dbObj = objectIter.next();
            if (Utils.isEmpty(dbObj.getObjectID())) {
                dbObj.setObjectID(objResult.getObjectID());
            } else if (!dbObj.getObjectID().equals(objResult.getObjectID())) {
                m_logger.warn("Batch results out of order: expected ID '{}', got '{}'",
                              dbObj.getObjectID(), objResult.getObjectID());
            }
        }
    }
850289
def batch(batch_size, items):
    ""
    items = list(items)
    if batch_size is None:
        return [items]
    MISSING = object()
    padded_items = items + [MISSING] * (batch_size - 1)
    groups = zip(*[padded_items[i::batch_size] for i in range(batch_size)])
    return [[item for item in group if item != MISSING] for group in groups]
664416
def get_next_batch(self):
        
        if self.collection_scanner.is_enabled:
            batch = self.collection_scanner.get_new_batch()
            for item in batch:
                base_item = BaseRecord(item)
                self.increase_read()
                self.last_position['last_key'] = item['_key']
                yield base_item
            self.logger.debug('Done reading batch')
        else:
            self.logger.debug('No more batches')
            self.finished = True
413213
def batch(byte_array, funcs):
    
    result = []
    length = bytes_to_int(byte_array[0:4])
    item_size = bytes_to_int(byte_array[4:8])
    for i in range(0, length):
        chunk = byte_array[8+i*item_size:8+(i+1)*item_size]
        for f in funcs:
            f(chunk)
    return result
34126
def populate(self, deserialized_txs=None, filename=None, retry=None):
        
        if not deserialized_txs:
            raise BatchError("Failed to populate batch. There are no objects to add.")
        self.filename = filename
        if not self.filename:
            raise BatchError("Invalid filename. Got None")
        try:
            for deserialized_tx in deserialized_txs:
                self.peek(deserialized_tx)
                self.objects.append(deserialized_tx.object)
                break
            for deserialized_tx in deserialized_txs:
                self.objects.append(deserialized_tx.object)
        except DeserializationError as e:
            raise BatchDeserializationError(e) from e
        except JSONFileError as e:
            raise BatchDeserializationError(e) from e
214427
private Later<JsonNode> getRawBatchResult() {
		if (this.rawBatchResult == null) {
			// Use LaterWrapper to cache the result so we don't fetch over and over
			this.rawBatchResult = new LaterWrapper<JsonNode, JsonNode>(this.createFetcher());
			
			// Also let the master know it's time to kick off any other batches and
			// remove us as a valid batch to add to.
			// This must be called *after* the rawBatchResult is set otherwise we
			// will have endless recursion when the master tries to execute us.
			this.master.execute();
		}
		
		return this.rawBatchResult;
	}
1267331
def _get_batches(self, items):
        """"""
        for i in xrange(0, len(items), self.max_batch_size):
            yield items[i:i + self.max_batch_size]