Generate randomized batches of data - only sample whole trajectories
335214
def shuffled_batches(self, batch_size):
        
        if batch_size >= self.num_envs * self.num_steps:
            yield self
        else:
            rollouts_in_batch = batch_size // self.num_steps

            batch_splits = math_util.divide_ceiling(self.num_envs, rollouts_in_batch)

            indices = list(range(self.num_envs))
            np.random.shuffle(indices)

            for sub_indices in np.array_split(indices, batch_splits):
                yield Trajectories(
                    num_steps=self.num_steps,
                    num_envs=len(sub_indices),
                    # Dont use it in batches for a moment, can be uncommented later if needed
                    # environment_information=[x[sub_indices.tolist()] for x in self.environment_information],
                    environment_information=None,
                    transition_tensors={k: x[:, sub_indices] for k, x in self.transition_tensors.items()},
                    rollout_tensors={k: x[sub_indices] for k, x in self.rollout_tensors.items()},
                    # extra_data does not go into batches
                )
746404
def _process_batch(self):
        """"""
        if self.block_id == len(self.tgroups):
            return None, None

        tgroup = self.tgroups[self.block_id]
        psize = self.chunk_tsamples[self.block_id]
        mid = self.mids[self.block_id]
        stilde = self.data.overwhitened_data(tgroup[0].delta_f)
        psd = stilde.psd

        valid_end = int(psize - self.data.trim_padding)
        valid_start = int(valid_end - self.data.blocksize * self.data.sample_rate)

        seg = slice(valid_start, valid_end)

        self.corr[self.block_id].execute(stilde)
        self.ifts[mid].execute()

        self.block_id += 1

        snr = numpy.zeros(len(tgroup), dtype=numpy.complex64)
        time = numpy.zeros(len(tgroup), dtype=numpy.float64)
        templates = numpy.zeros(len(tgroup), dtype=numpy.uint64)
        sigmasq = numpy.zeros(len(tgroup), dtype=numpy.float32)

        time[:] = self.data.start_time

        result = {}
        tkeys = tgroup[0].params.dtype.names
        for key in tkeys:
            result[key] = []

        veto_info = []

        # Find the peaks in our SNR times series from the various templates
        i = 0
        for htilde in tgroup:
            l = htilde.out[seg].abs_arg_max()

            sgm = htilde.sigmasq(psd)
            norm = 4.0 * htilde.delta_f / (sgm ** 0.5)

            l += valid_start
            snrv = numpy.array([htilde.out[l]])

            # If nothing is above threshold we can exit this template
            s = abs(snrv[0]) * norm
            if s < self.snr_threshold:
                continue

            time[i] += float(l - valid_start) / self.data.sample_rate

            # We have an SNR so high that we will drop the entire analysis
            # of this chunk of time!
            if self.snr_abort_threshold is not None and s > self.snr_abort_threshold:
                logging.info("We are seeing some *really* high SNRs, lets"
                             " assume they aren't signals and just give up")
                return False, []

            veto_info.append((snrv, norm, l, htilde, stilde))

            snr[i] = snrv[0] * norm
            sigmasq[i] = sgm
            templates[i] = htilde.id
            if not hasattr(htilde, 'dict_params'):
                htilde.dict_params = {}
                for key in tkeys:
                    htilde.dict_params[key] = htilde.params[key]

            for key in tkeys:
                result[key].append(htilde.dict_params[key])
            i += 1


        result['snr'] = abs(snr[0:i])
        result['coa_phase'] = numpy.angle(snr[0:i])
        result['end_time'] = time[0:i]
        result['template_id'] = templates[0:i]
        result['sigmasq'] = sigmasq[0:i]

        for key in tkeys:
            result[key] = numpy.array(result[key])

        return result, veto_info
4897
def samples(self):
        """"""
        names = self.series.dimensions
        for n, offset in enumerate(self.series.offsets):
            dt = datetime.timedelta(microseconds=offset * 1000)
            d = {"ts": self.ts + dt}
            for name in names:
                d[name] = getattr(self.series, name)[n]
            yield d
1077066
def provide_data(self):
        """"""
        return [(k, tuple([self.batch_size] + list(v.shape[1:]))) for k, v in self.data]
90814
def batchify(data, batch_size):
    """"""
    nbatch = data.shape[0] // batch_size
    data = data[:nbatch * batch_size]
    data = data.reshape((batch_size, nbatch)).T
    return data
986580
def _combine_qc_samples(samples):
    
    by_bam = collections.defaultdict(list)
    for data in [utils.to_single_data(x) for x in samples]:
        batch = dd.get_batch(data) or dd.get_sample_name(data)
        if not isinstance(batch, (list, tuple)):
            batch = [batch]
        batch = tuple(batch)
        by_bam[(dd.get_align_bam(data) or dd.get_work_bam(data), batch)].append(data)
    out = []
    for data_group in by_bam.values():
        data = data_group[0]
        alg_qc = []
        qc = {}
        metrics = {}
        for d in data_group:
            qc.update(dd.get_summary_qc(d))
            metrics.update(dd.get_summary_metrics(d))
            alg_qc.extend(dd.get_algorithm_qc(d))
        data["config"]["algorithm"]["qc"] = alg_qc
        data["summary"]["qc"] = qc
        data["summary"]["metrics"] = metrics
        out.append([data])
    return out
678269
def indices(self, data):
        '''
        '''
        duration = self.data_duration(data)

        while True:
            # Generate a sampling interval
            yield self.rng.randint(0, duration - self.min_duration + 1)
353824
def batch_for_sv(samples):
    
    samples = cwlutils.assign_complex_to_samples(samples)
    to_process, extras, background = _batch_split_by_sv(samples, "standard")
    out = [cwlutils.samples_to_records(xs) for xs in to_process.values()] + extras
    return out
751603
def _split_batches(self, data, batch_size):
        """"""
        for i in range(0, len(data), batch_size):
            yield data[i : i + batch_size]
1353914
def batch(items, size):
    """"""
    return [items[x:x + size] for x in xrange(0, len(items), size)]
272836
def batch_gen(data, batch_size):
    '''
    
    '''
    data = data or []
    for i in range(0, len(data), batch_size):
        yield data[i:i + batch_size]
207941
def shuffle_and_batch_data(dataset,
                           target_names,
                           features_info,
                           training,
                           num_devices,
                           shuffle_buffer_size=1024,
                           preprocess_fun=no_preprocess):
  """"""
  def append_targets(example):
    """Append targets to the example dictionary. Needed for Keras."""
    if len(target_names) == 1:
      return (example, example[target_names[0]])
    targets = {}
    for name in target_names:
      targets[name] = example[name]
    return (example, targets)
  dataset = dataset.map(append_targets)
  if training:
    dataset = dataset.repeat()
    # Skip a random fraction at the beginning of the stream.  The skip is
    # essential for synchronous highly-parallel training to avoid multiple
    # replicas reading the same data in lock-step.
    dataset = dataset.skip(random.randint(0, _MAX_SKIP_EXAMPLES))
  dataset = preprocess_fun(dataset, training)
  shapes = {k: features_info[k].shape for k in features_info}
  shapes = (shapes, shapes[target_names[0]])
  dataset = dataset.shuffle(shuffle_buffer_size)
  dataset = batch_fun(dataset, training, shapes, target_names, num_devices)
  return dataset.prefetch(2)
331461
def generate_encoded_samples(self, data_dir, tmp_dir, dataset_split):
    
    generator = self.generate_samples(data_dir, tmp_dir, dataset_split)
    encoder = self.get_or_create_vocab(data_dir, tmp_dir)
    label_encoder = self.get_labels_encoder(data_dir)
    for sample in generator:
      inputs = encoder.encode(sample["inputs"])
      inputs.append(text_encoder.EOS_ID)
      context = encoder.encode(sample["context"])
      context.append(text_encoder.EOS_ID)
      targets = label_encoder.encode(sample["targets"])
      sample["targets"] = targets
      yield {"inputs": inputs, "context": context, "targets": targets}
675861
def sampled_data(self):
        """"""
        if self.args.samples_file:
            if self.args.invert_samples:
                chosen_samples = set(self.hash_to_ind) - self.samples
            else:
                chosen_samples = self.samples
            selected_indices = [self.hash_to_ind[h] for h in chosen_samples]
            return self.train[0][selected_indices], self.train[1][selected_indices]
        else:
            return self.train[0], self.train[1]
378148
def collect_samples(agents, sample_batch_size, num_envs_per_worker,
                    train_batch_size):
    """"""

    num_timesteps_so_far = 0
    trajectories = []
    agent_dict = {}

    for agent in agents:
        fut_sample = agent.sample.remote()
        agent_dict[fut_sample] = agent

    while agent_dict:
        [fut_sample], _ = ray.wait(list(agent_dict))
        agent = agent_dict.pop(fut_sample)
        next_sample = ray_get_and_free(fut_sample)
        assert next_sample.count >= sample_batch_size * num_envs_per_worker
        num_timesteps_so_far += next_sample.count
        trajectories.append(next_sample)

        # Only launch more tasks if we don't already have enough pending
        pending = len(agent_dict) * sample_batch_size * num_envs_per_worker
        if num_timesteps_so_far + pending < train_batch_size:
            fut_sample2 = agent.sample.remote()
            agent_dict[fut_sample2] = agent

    return SampleBatch.concat_samples(trajectories)
1048079
def make_batch(size, graph):
        
        i = iter(graph)
        chunk = list(islice(i, size))
        while chunk:
            yield chunk
            chunk = list(islice(i, size))
1240553
def load_batches(self):
    """"""
    event_iterator = self._directory_watcher.Load()
    while True:
      events = []
      event_bytes = 0
      start = time.time()
      for event_proto in event_iterator:
        events.append(event_proto)
        event_bytes += len(event_proto)
        if len(events) >= self._BATCH_COUNT or event_bytes >= self._BATCH_BYTES:
          break
      elapsed = time.time() - start
      logger.debug('RunLoader.load_batch() yielded in %0.3f sec for %s',
                       elapsed, self._subdir)
      if not events:
        return
      yield _EventBatch(
          events=events,
          experiment_name=self._experiment_name,
          run_name=self._run_name)