Return batch consisting of *consecutive* transitions
480883
def get_trajectories(self, indexes, rollout_length):
        
        # assert indexes.shape[0] > 1, "There must be multiple indexes supplied"
        assert rollout_length > 1, "Rollout length must be greater than 1"

        batch_indexes = (
                indexes.reshape(1, indexes.shape[0]) - np.arange(rollout_length - 1, -1, -1).reshape(rollout_length, 1)
        )

        return self.get_transitions(batch_indexes)
247287
def batch(self):
        
        if self._no_kc:
            raise ValueError("Already in a batch")
        self._no_kc = True
        yield
        self._no_kc = False
1017029
def _threshold_batch(self,
                         vectors,
                         batch_size,
                         threshold,
                         show_progressbar,
                         return_names):
        """"""
        vectors = self.normalize(vectors)

        # Single transpose, makes things faster.
        reference_transposed = self.norm_vectors.T

        for i in tqdm(range(0, len(vectors), batch_size),
                      disable=not show_progressbar):

            distances = vectors[i: i+batch_size].dot(reference_transposed)
            # For safety we clip
            distances = np.clip(distances, a_min=.0, a_max=1.0)
            for lidx, dists in enumerate(distances):
                indices = np.flatnonzero(dists >= threshold)
                sorted_indices = indices[np.argsort(-dists[indices])]
                if return_names:
                    yield [(self.indices[d], dists[d])
                           for d in sorted_indices]
                else:
                    yield list(dists[sorted_indices])
352208
def consecutive_frame(self):
        """"""
        if self._frame.empty:
            return pd.DataFrame(columns=['pids', 'pl', 'cnt', 'is_win'])
        else:
            vals = (self._frame[PC.RET] >= 0).astype(int)
            seq = (vals.shift(1) != vals).astype(int).cumsum()

            def _do_apply(sub):
                return pd.Series({
                    'pids': sub.index.values,
                    'pl': sub[PC.PL].sum(),
                    'cnt': len(sub.index),
                    'is_win': sub[PC.RET].iloc[0] >= 0,
                })

            return self._frame.groupby(seq).apply(_do_apply)
701938
def rolling_window_sequences(X, index, window_size, target_size, target_column):
    """"""
    out_X = list()
    out_y = list()
    X_index = list()
    y_index = list()

    target = X[:, target_column]

    for start in range(len(X) - window_size - target_size + 1):
        end = start + window_size
        out_X.append(X[start:end])
        out_y.append(target[end:end + target_size])
        X_index.append(index[start])
        y_index.append(index[end])

    return np.asarray(out_X), np.asarray(out_y), np.asarray(X_index), np.asarray(y_index)
306100
def next(self):
        """"""
        if self.curr_idx == len(self.idx):
            raise StopIteration
        i, j = self.idx[self.curr_idx]
        self.curr_idx += 1

        if self.major_axis == 1:
            data = self.nddata[i][j:j+self.batch_size].T
            label = self.ndlabel[i][j:j+self.batch_size].T
        else:
            data = self.nddata[i][j:j+self.batch_size]
            label = self.ndlabel[i][j:j+self.batch_size]

        return DataBatch([data], [label], pad=0,
                         bucket_key=self.buckets[i],
                         provide_data=[DataDesc(
                             name=self.data_name, shape=data.shape,
                             layout=self.layout)],
                         provide_label=[DataDesc(
                             name=self.label_name, shape=label.shape,
                             layout=self.layout)])
1060418
def one_batch(self, ds_type:DatasetType=DatasetType.Train, detach:bool=True, denorm:bool=True, cpu:bool=True)->Collection[Tensor]:
        ""
        dl = self.dl(ds_type)
        w = self.num_workers
        self.num_workers = 0
        try:     x,y = next(iter(dl))
        finally: self.num_workers = w
        if detach: x,y = to_detach(x,cpu=cpu),to_detach(y,cpu=cpu)
        norm = getattr(self,'norm',False)
        if denorm and norm:
            x = self.denorm(x)
            if norm.keywords.get('do_y',False): y = self.denorm(y, do_x=True)
        return x,y
1267331
def _get_batches(self, items):
        """"""
        for i in xrange(0, len(items), self.max_batch_size):
            yield items[i:i + self.max_batch_size]
104584
def transitions_to(self, dst):
    '''
    
    '''
    if dst in self._transitions_to:
      for t in self._transitions_to[dst]:
        for s in self._transitions_to[dst][t]:
          yield (s, t)
19521
def sample_forward_transitions(self, batch_size, batch_info, forward_steps: int,
                                   discount_factor: float) -> Transitions:
        
        indexes = self.backend.sample_batch_transitions(batch_size, forward_steps=forward_steps)
        transition_tensors = self.backend.get_transitions_forward_steps(
            indexes, forward_steps=forward_steps, discount_factor=discount_factor
        )

        return Trajectories(
            num_steps=batch_size,
            num_envs=self.backend.num_envs,
            environment_information=None,
            transition_tensors={k: torch.from_numpy(v) for k, v in transition_tensors.items()},
            rollout_tensors={},
            extra_data={
                'forward_steps': forward_steps
            }
        ).to_transitions()
510629
def next_batch(self, n=1):
        """"""
        if len(self.queue) == 0:
            return []
        batch = list(reversed((self.queue[-n:])))
        self.queue = self.queue[:-n]
        return batch
569301
def next(self):
        """"""
        if self.curr_idx == len(self.idx):
            raise StopIteration
        #i = batches index, j = starting record
        i, j = self.idx[self.curr_idx] 
        self.curr_idx += 1

        indices = self.ndindex[i][j:j + self.batch_size]
        sentences = self.ndsent[i][j:j + self.batch_size]
        characters = self.ndchar[i][j:j + self.batch_size]
        label = self.ndlabel[i][j:j + self.batch_size]

        return DataBatch([sentences, characters], [label], pad=0, index = indices, bucket_key=self.buckets[i],
                         provide_data=[DataDesc(name=self.data_names[0], shape=sentences.shape, layout=self.layout),
                                       DataDesc(name=self.data_names[1], shape=characters.shape, layout=self.layout)],
                         provide_label=[DataDesc(name=self.label_name, shape=label.shape, layout=self.layout)])
180816
def sliding_window(seq, lookahead):
    
    it = itertools.chain(iter(seq), ' ' * lookahead)  # Padded iterator
    window_size = lookahead + 1
    result = tuple(itertools.islice(it, window_size))
    if len(result) == window_size:
        yield result
    for elem in it:
        result = result[1:] + (elem,)
        yield result
272836
def batch_gen(data, batch_size):
    '''
    
    '''
    data = data or []
    for i in range(0, len(data), batch_size):
        yield data[i:i + batch_size]
552227
def batch(data, batch_size, batch_size_fn=None):
    """"""
    if batch_size_fn is None:
        def batch_size_fn(new, count, sofar):
            return count
    minibatch, size_so_far = [], 0
    for ex in data:
        minibatch.append(ex)
        size_so_far = batch_size_fn(ex, len(minibatch), size_so_far)
        if size_so_far == batch_size:
            yield minibatch
            minibatch, size_so_far = [], 0
        elif size_so_far > batch_size:
            yield minibatch[:-1]
            minibatch, size_so_far = minibatch[-1:], batch_size_fn(ex, 1, 0)
    if minibatch:
        yield minibatch
11995
def transitions(self):
        """"""
        if self._transitions is not None:
            return self._transitions
        transitions = self.array.astype(np.float)
        transitions /= transitions.sum(1)[:, np.newaxis]
        self._transitions = transitions
        return transitions
1038366
def batch(self):
        """"""
        _batch = []
        yield _batch
        if _batch:
            new_part = super(type(self), type(self)).empty()
            new_part._from_entities(_batch, _batch[0].__dimension__)
            self.concat(new_part, inplace=True)