r"""Creates a sparse representention of ``sequence``.
        Returns a tuple with (indices, values, shape)
970359
def to_sparse_tuple(sequence):
    
    """
    indices = np.asarray(list(zip([0]*len(sequence), range(len(sequence)))), dtype=np.int64)
    shape = np.asarray([1, len(sequence)], dtype=np.int64)
    return indices, sequence, shape
146192
def sequences(self):
        """"""
        lines = Lines()
        lines.add(0, '@cython.final')
        lines.add(0, 'cdef class Sequences(object):')
        for subseqs in self.model.sequences:
            lines.add(1, 'cdef public %s %s'
                         % (objecttools.classname(subseqs), subseqs.name))
        if getattr(self.model.sequences, 'states', None) is not None:
            lines.add(1, 'cdef public StateSequences old_states')
            lines.add(1, 'cdef public StateSequences new_states')
        for subseqs in self.model.sequences:
            print('        - %s' % subseqs.name)
            lines.add(0, '@cython.final')
            lines.add(0, 'cdef class %s(object):'
                         % objecttools.classname(subseqs))
            for seq in subseqs:
                ctype = 'double' + NDIM2STR[seq.NDIM]
                if isinstance(subseqs, sequencetools.LinkSequences):
                    if seq.NDIM == 0:
                        lines.add(1, 'cdef double *%s' % seq.name)
                    elif seq.NDIM == 1:
                        lines.add(1, 'cdef double **%s' % seq.name)
                        lines.add(1, 'cdef public int len_%s' % seq.name)
                else:
                    lines.add(1, 'cdef public %s %s' % (ctype, seq.name))
                lines.add(1, 'cdef public int _%s_ndim' % seq.name)
                lines.add(1, 'cdef public int _%s_length' % seq.name)
                for idx in range(seq.NDIM):
                    lines.add(1, 'cdef public int _%s_length_%d'
                                 % (seq.name, idx))
                if seq.NUMERIC:
                    ctype_numeric = 'double' + NDIM2STR[seq.NDIM+1]
                    lines.add(1, 'cdef public %s _%s_points'
                                 % (ctype_numeric, seq.name))
                    lines.add(1, 'cdef public %s _%s_results'
                                 % (ctype_numeric, seq.name))
                    if isinstance(subseqs, sequencetools.FluxSequences):
                        lines.add(1, 'cdef public %s _%s_integrals'
                                     % (ctype_numeric, seq.name))
                        lines.add(1, 'cdef public %s _%s_sum'
                                     % (ctype, seq.name))
                if isinstance(subseqs, sequencetools.IOSequences):
                    lines.extend(self.iosequence(seq))
            if isinstance(subseqs, sequencetools.InputSequences):
                lines.extend(self.load_data(subseqs))
            if isinstance(subseqs, sequencetools.IOSequences):
                lines.extend(self.open_files(subseqs))
                lines.extend(self.close_files(subseqs))
                if not isinstance(subseqs, sequencetools.InputSequence):
                    lines.extend(self.save_data(subseqs))
            if isinstance(subseqs, sequencetools.LinkSequences):
                lines.extend(self.set_pointer(subseqs))
        return lines
1251246
def pad(cls, sequences, padding, pad_len=None):
        
        max_len = max([len(s) for s in sequences])
        pad_len = pad_len or max_len
        assert pad_len >= max_len, 'pad_len {} must be greater or equal to the longest sequence {}'.format(pad_len, max_len)
        for i, s in enumerate(sequences):
            sequences[i] = [padding] * (pad_len - len(s)) + s
        return np.array(sequences)
1201972
def index_2d(seqs: List[List[Any]], target: Any) -> Tuple[int, int]:
    
    for i in range(len(seqs)):
        for j in range(len(seqs[i])):
            if seqs[i][j] == target:
                return i, j
    raise ValueError('Item not present.')
289783
def write_spmatrix_to_sparse_tensor(file, array, labels=None):
    """"""

    if not issparse(array):
        raise TypeError("Array must be sparse")

    # Validate shape of array and labels, resolve array and label types
    if not len(array.shape) == 2:
        raise ValueError("Array must be a Matrix")
    if labels is not None:
        if not len(labels.shape) == 1:
            raise ValueError("Labels must be a Vector")
        if labels.shape[0] not in array.shape:
            raise ValueError("Label shape {} not compatible with array shape {}".format(
                             labels.shape, array.shape))
        resolved_label_type = _resolve_type(labels.dtype)
    resolved_type = _resolve_type(array.dtype)

    csr_array = array.tocsr()
    n_rows, n_cols = csr_array.shape

    record = Record()
    for row_idx in range(n_rows):
        record.Clear()
        row = csr_array.getrow(row_idx)
        # Write values
        _write_feature_tensor(resolved_type, record, row.data)
        # Write keys
        _write_keys_tensor(resolved_type, record, row.indices.astype(np.uint64))

        # Write labels
        if labels is not None:
            _write_label_tensor(resolved_label_type, record, labels[row_idx])

        # Write shape
        _write_shape(resolved_type, record, n_cols)

        _write_recordio(file, record.SerializeToString())
333251
def partial_fit(self, sequence, y=None):
        
        s = super(MultiSequencePreprocessingMixin, self)
        if hasattr(s, 'fit'):
            return s.fit(sequence)
        return self
339943
def _flatten(n):
    """"""
    if isinstance(n, collections.Sequence):
        for x in n:
            for y in _flatten(x):
                yield y
    else:
        yield n
79753
def _arith_method_SPARSE_SERIES(cls, op, special):
    
    op_name = _get_op_name(op, special)

    def wrapper(self, other):
        if isinstance(other, ABCDataFrame):
            return NotImplemented
        elif isinstance(other, ABCSeries):
            if not isinstance(other, ABCSparseSeries):
                other = other.to_sparse(fill_value=self.fill_value)
            return _sparse_series_op(self, other, op, op_name)
        elif is_scalar(other):
            with np.errstate(all='ignore'):
                new_values = op(self.values, other)
            return self._constructor(new_values,
                                     index=self.index,
                                     name=self.name)
        else:  # pragma: no cover
            raise TypeError('operation with {other} not supported'
                            .format(other=type(other)))

    wrapper.__name__ = op_name
    return wrapper
321704
def encode(self, sequence):
        
        sequence = super().encode(sequence)
        sequence = self.tokenize(sequence)
        vector = [self.stoi.get(token, self.unknown_index) for token in sequence]
        if self.append_eos:
            vector.append(self.eos_index)
        return torch.tensor(vector)
496068
def as_list(x):
    ''''''
    
    if x is None:
        x = []
    elif not isinstance(x, Sequence):
        x = [x]
    return list(x)
44742
def _filtdim(items, shape, dim, nsl):
    """"""
    normshape = tuple(stop - start for start, stop in shape)
    nsl_type = type(nsl)
    newitems = list()
    # Number of groups
    num = reduce(operator.mul, normshape[:dim+1])
    # Size of each group
    size = len(items) // num
    # Size of the dimension
    n = normshape[dim]
    if nsl_type is int:
        for i in range(num):
            if i % n == nsl:
                newitems += items[size*i:size*(i+1)]
        # Collapse dimension
        newshape = shape[:dim] + shape[dim+1:]
    elif nsl_type is slice:
        for i in range(num):
            if nsl.start <= (i % n) < nsl.stop:
                newitems += items[size*i:size*(i+1)]
        # Reshape dimension
        offset = shape[dim][0]
        redim = (offset + nsl.start, offset + nsl.stop)
        newshape = shape[:dim] + (redim, ) + shape[dim+1:]
    # farray
    else:
        if nsl.size < clog2(n):
            fstr = "expected dim {} select to have >= {} bits, got {}"
            raise ValueError(fstr.format(dim, clog2(n), nsl.size))
        groups = [list() for _ in range(n)]
        for i in range(num):
            groups[i % n] += items[size*i:size*(i+1)]
        for muxins in zip(*groups):
            it = boolfunc.iter_terms(nsl._items)
            xs = [reduce(operator.and_, (muxin, ) + next(it))
                  for muxin in muxins]
            newitems.append(reduce(operator.or_, xs))
        # Collapse dimension
        newshape = shape[:dim] + shape[dim+1:]
    return newitems, newshape
1312079
def stack_sparse_frame(frame):
    
    lengths = [s.sp_index.npoints for _, s in frame.items()]
    nobs = sum(lengths)

    # this is pretty fast
    minor_codes = np.repeat(np.arange(len(frame.columns)), lengths)

    inds_to_concat = []
    vals_to_concat = []
    # TODO: Figure out whether this can be reached.
    # I think this currently can't be reached because you can't build a
    # SparseDataFrame with a non-np.NaN fill value (fails earlier).
    for _, series in frame.items():
        if not np.isnan(series.fill_value):
            raise TypeError('This routine assumes NaN fill value')

        int_index = series.sp_index.to_int_index()
        inds_to_concat.append(int_index.indices)
        vals_to_concat.append(series.sp_values)

    major_codes = np.concatenate(inds_to_concat)
    stacked_values = np.concatenate(vals_to_concat)
    index = MultiIndex(levels=[frame.index, frame.columns],
                       codes=[major_codes, minor_codes],
                       verify_integrity=False)

    lp = DataFrame(stacked_values.reshape((nobs, 1)), index=index,
                   columns=['foo'])
    return lp.sort_index(level=0)
1164642
def write_sparse(self, file_path: str):
        """"""

        coo = sparse.coo_matrix(self.X)
        data = OrderedDict([(0, coo.row+1), (1, coo.col+1), (2, coo.data)])
        df = pd.DataFrame(data, columns=data.keys())
        with open(file_path, 'w') as ofh:
            ofh.write('%%MatrixMarket matrix coordinate real general\n')
            ofh.write('%%%s\n' % '\t'.join(self.genes.astype(str)))
            ofh.write('%%%s\n' % '\t'.join(self.cells.astype(str)))
            ofh.write('%\n')
            ofh.write('%d %d %d\n' % (coo.shape[0], coo.shape[1], coo.nnz))
            df.to_csv(ofh, sep=' ', float_format='%.5f',
                      header=None, index=None)
673454
def _arrayize_vectorized_indexer(indexer, shape):
    
    slices = [v for v in indexer.tuple if isinstance(v, slice)]
    if len(slices) == 0:
        return indexer

    arrays = [v for v in indexer.tuple if isinstance(v, np.ndarray)]
    n_dim = arrays[0].ndim if len(arrays) > 0 else 0
    i_dim = 0
    new_key = []
    for v, size in zip(indexer.tuple, shape):
        if isinstance(v, np.ndarray):
            new_key.append(np.reshape(v, v.shape + (1, ) * len(slices)))
        else:  # slice
            shape = ((1,) * (n_dim + i_dim) + (-1,) +
                     (1,) * (len(slices) - i_dim - 1))
            new_key.append(np.arange(*v.indices(size)).reshape(shape))
            i_dim += 1
    return VectorizedIndexer(tuple(new_key))
78754
def build_dataset(filename, sequence_length, alphabet_size, max_lines=-1):
    """"""
    with open(filename) as f:
        content = f.readlines()
    content = content[:max_lines]
    content = [line for line in content if len(line) > 2]   # Remove blank lines
    seqs = np.zeros((sequence_length, len(content), alphabet_size))
    for ix, line in enumerate(content):
        padded_line = (line + " " * sequence_length)[:sequence_length]
        seqs[:, ix, :] = string_to_one_hot(padded_line, alphabet_size)
    return seqs
660983
def sparse_var(X):
    '''
    
    '''
    Xc = X.copy()
    Xc.data **= 2
    return np.array(Xc.mean(axis=0) - np.power(X.mean(axis=0), 2))[0]
495231
def including(self, sequence) -> Generator:
        """"""
        return (element for element in sequence
                if self.indexer(element) in self.predicates)