Build the output, in chunks.

        :return: Number of items processed
        :rtype: int
327015
def _build(self, items, chunk_size=10000):
        
        _log.debug("_build, chunk_size={:d}".format(chunk_size))
        n, i = 0, 0
        for i, item in enumerate(items):
            if i == 0:
                _log.debug("_build, first item")
            if 0 == (i + 1) % chunk_size:
                if self._seq:
                    self._run(0)
                else:
                    self._run_parallel_fn()  # process the chunk
                if self._status.has_failures():
                    break
                n = i + 1
            self._queue.put(item)
        # process final chunk
        if self._seq:
            self._run(0)
        else:
            self._run_parallel_fn()
        if not self._status.has_failures():
            n = i + 1
        return n
195458
def next_chunk(self):
        
        if self._unconsumed:
            data = self._unconsumed.pop()
        else:
            data = self._iterator.next()  # Might raise StopIteration
        self._pos += len(data)
        return data
270159
def chunks(event_list, chunk_size):
    """"""
    for i in range(0, len(event_list), chunk_size):
        yield event_list[i:i + chunk_size]
836341
def chunks(self):
        
        raise NotImplementedError("%s not implemented for %s" % (self.chunks.__func__.__name__,
                                                                 self.__class__.__name__))
1029368
def run(self, num_of_processes=multiprocessing.cpu_count()):
        

        # class + wordification on every example of the main table

        p = multiprocessing.Pool(num_of_processes)

        indices = chunks(list(range(len(self.target_table))), num_of_processes)  # )

        for ex_idxs in indices:
            self.resulting_documents.extend(wordify_examples(self.name_to_table, self.connecting_tables, self.context,
                                                             self.index_by_value, self.target_table.name,
                                                             self.word_att_length, ex_idxs))
        p.close()
        p.join()

        for i, ex in enumerate(self.target_table):
            self.resulting_classes.append(ex.get_class())
1064558
def chunks(container, n):
    
    for i in range(0, len(container), n):
        yield container[i : i + n]
1293968
def build(self, **kwargs):
        
        self.tasks += self.compute_tasks(**kwargs)
        return self.finalize()
670169
def run(self):
        
        self.running = True

        def chunks_producer():
            while self.running:
                self.chunks.put(self.next_chunks())

        t = Thread(target=chunks_producer)
        t.start()

        with self.BackendStream(samplerate=self.sr, channels=1) as stream:
            while self.running:
                try:
                    stream.write(self.chunks.get(timeout=self.timeout))  # timeout so stream.write() thread can exit
                except Empty:
                    self.running = False
331193
def chunks(self):
        

        while True:
            t, v = self.chunk()
            yield t, v
            if t == b'IEND':
                break
1026073
def run(self):
        
        processes = []
        progress_queue = ProgressQueue(Queue())
        num_chunks = ParallelChunkProcessor.determine_num_chunks(self.config.upload_bytes_per_chunk,
                                                                 self.local_file.size)
        work_parcels = ParallelChunkProcessor.make_work_parcels(self.config.upload_workers, num_chunks)
        for (index, num_items) in work_parcels:
            processes.append(self.make_and_start_process(index, num_items, progress_queue))
        wait_for_processes(processes, num_chunks, progress_queue, self.watcher, self.local_file)
266355
def process(self):
        '''
        '''
        self._item_session.request = request = Request(self._item_session.url_record.url)
        verdict = self._fetch_rule.check_ftp_request(self._item_session)[0]

        if not verdict:
            self._item_session.skip()
            return

        self._add_request_password(request)

        dir_name, filename = self._item_session.url_record.url_info.split_path()
        if self._processor.fetch_params.glob and frozenset(filename) & GLOB_CHARS:
            request = self._to_directory_request(request)
            is_file = False
            self._glob_pattern = urllib.parse.unquote(filename)
        else:
            is_file = yield from self._prepare_request_file_vs_dir(request)

            self._file_writer_session.process_request(request)

        wait_time = yield from self._fetch(request, is_file)

        if wait_time:
            _logger.debug('Sleeping {0}.', wait_time)
            yield from asyncio.sleep(wait_time)
16916
def chunk(self, count):
        
        for chunk in self._connection.select_many(
            count, self.to_sql(), self.get_bindings(), not self._use_write_connection
        ):
            yield chunk
621265
def chunked_list(_list, _chunk_size=50):
    
    for i in range(0, len(_list), _chunk_size):
        yield _list[i:i + _chunk_size]
1060994
def output(self):
        
        return luigi.contrib.hdfs.HdfsTarget('data-matrix', format=luigi.format.Gzip)
1015754
def generate_chunks(data, chunk_size=DEFAULT_CHUNK_SIZE):
    """"""
    iterator = iter(repeated.getvalues(data))

    while True:
        chunk = list(itertools.islice(iterator, chunk_size))
        if not chunk:
            return

        yield chunk
202380
def process(self):
        ''''''
        for tag_proc in self.tag_procs:
            before_count = self.entry_count
            self.run_tag_processor(tag_proc)
            after_count = self.entry_count

            if self.verbose:
                print('Inserted %d entries for "%s" tag processor' % (
                    after_count - before_count, tag_proc), file=sys.stderr)

        if self.verbose:
            print('Inserted %d entries overall' % self.entry_count,
                  file=sys.stderr)
865378
def chunks(list_, count_items_in_chunk):
    
    for i in range(0, len(list_), count_items_in_chunk):
        yield list_[i:i + count_items_in_chunk]