Compute the Shannon entropy, a measure of uncertainty.
425810
def entropy(data):
    

    if len(data) == 0:
        return None

    n = sum(data)

    _op = lambda f: f * math.log(f)
    return - sum(_op(float(i) / n) for i in data)
771367
def _compare_entropy(start_slice, end_slice, slice, difference):
    
    start_entropy = utils.image_entropy(start_slice)
    end_entropy = utils.image_entropy(end_slice)
    if end_entropy and abs(start_entropy / end_entropy - 1) < 0.01:
        # Less than 1% difference, remove from both sides.
        if difference >= slice * 2:
            return slice, slice
        half_slice = slice // 2
        return half_slice, slice - half_slice
    if start_entropy > end_entropy:
        return 0, slice
    else:
        return slice, 0
623265
def inference(self, kern_r, kern_c, Xr, Xc, Zr, Zc, likelihood, Y, qU_mean ,qU_var_r, qU_var_c, indexD, output_dim):
        

        N, D, Mr, Mc, Qr, Qc = Y.shape[0], output_dim,Zr.shape[0], Zc.shape[0], Zr.shape[1], Zc.shape[1]

        uncertain_inputs_r = isinstance(Xr, VariationalPosterior)
        uncertain_inputs_c = isinstance(Xc, VariationalPosterior)
        uncertain_outputs = isinstance(Y, VariationalPosterior)

        grad_dict = self._init_grad_dict(N,D,Mr,Mc)

        beta = 1./likelihood.variance
        if len(beta)==1:
            beta = np.zeros(D)+beta

        psi0_r, psi1_r, psi2_r = self.gatherPsiStat(kern_r, Xr, Zr, uncertain_inputs_r)
        psi0_c, psi1_c, psi2_c = self.gatherPsiStat(kern_c, Xc, Zc, uncertain_inputs_c)

        #======================================================================
        # Compute Common Components
        #======================================================================

        Kuu_r = kern_r.K(Zr).copy()
        diag.add(Kuu_r, self.const_jitter)
        Lr = jitchol(Kuu_r)

        Kuu_c = kern_c.K(Zc).copy()
        diag.add(Kuu_c, self.const_jitter)
        Lc = jitchol(Kuu_c)

        mu, Sr, Sc = qU_mean, qU_var_r, qU_var_c
        LSr = jitchol(Sr)
        LSc = jitchol(Sc)

        LcInvMLrInvT = dtrtrs(Lc,dtrtrs(Lr,mu.T)[0].T)[0]
        LcInvLSc = dtrtrs(Lc, LSc)[0]
        LrInvLSr = dtrtrs(Lr, LSr)[0]
        LcInvScLcInvT = tdot(LcInvLSc)
        LrInvSrLrInvT = tdot(LrInvLSr)
        tr_LrInvSrLrInvT = np.square(LrInvLSr).sum()
        tr_LcInvScLcInvT = np.square(LcInvLSc).sum()

        mid_res = {
            'psi0_r': psi0_r,
            'psi1_r': psi1_r,
            'psi2_r': psi2_r,
            'psi0_c': psi0_c,
            'psi1_c': psi1_c,
            'psi2_c': psi2_c,
            'Lr':Lr,
            'Lc':Lc,
            'LcInvMLrInvT': LcInvMLrInvT,
            'LcInvScLcInvT': LcInvScLcInvT,
            'LrInvSrLrInvT': LrInvSrLrInvT,
        }

        #======================================================================
        # Compute log-likelihood
        #======================================================================

        logL = 0.
        for d in range(D):
            logL += self.inference_d(d, beta, Y, indexD, grad_dict, mid_res, uncertain_inputs_r, uncertain_inputs_c, Mr, Mc)

        logL += -Mc * (np.log(np.diag(Lr)).sum()-np.log(np.diag(LSr)).sum())  -Mr * (np.log(np.diag(Lc)).sum()-np.log(np.diag(LSc)).sum()) \
               - np.square(LcInvMLrInvT).sum()/2. - tr_LrInvSrLrInvT * tr_LcInvScLcInvT/2. + Mr*Mc/2.

        #======================================================================
        # Compute dL_dKuu
        #======================================================================

        tmp =  tdot(LcInvMLrInvT)/2. + tr_LrInvSrLrInvT/2. * LcInvScLcInvT - Mr/2.*np.eye(Mc)

        dL_dKuu_c = backsub_both_sides(Lc, tmp, 'left')
        dL_dKuu_c += dL_dKuu_c.T
        dL_dKuu_c *= 0.5

        tmp =  tdot(LcInvMLrInvT.T)/2. + tr_LcInvScLcInvT/2. * LrInvSrLrInvT - Mc/2.*np.eye(Mr)

        dL_dKuu_r = backsub_both_sides(Lr, tmp, 'left')
        dL_dKuu_r += dL_dKuu_r.T
        dL_dKuu_r *= 0.5

        #======================================================================
        # Compute dL_dqU
        #======================================================================

        tmp = - LcInvMLrInvT
        dL_dqU_mean = dtrtrs(Lc, dtrtrs(Lr, tmp.T, trans=1)[0].T, trans=1)[0]

        LScInv = dtrtri(LSc)
        tmp = -tr_LrInvSrLrInvT/2.*np.eye(Mc)
        dL_dqU_var_c = backsub_both_sides(Lc, tmp, 'left') + tdot(LScInv.T) * Mr/2.

        LSrInv = dtrtri(LSr)
        tmp =  -tr_LcInvScLcInvT/2.*np.eye(Mr)
        dL_dqU_var_r = backsub_both_sides(Lr, tmp, 'left') + tdot(LSrInv.T) * Mc/2.

        #======================================================================
        # Compute the Posterior distribution of inducing points p(u|Y)
        #======================================================================

        post = PosteriorMultioutput(LcInvMLrInvT=LcInvMLrInvT, LcInvScLcInvT=LcInvScLcInvT,
                LrInvSrLrInvT=LrInvSrLrInvT, Lr=Lr, Lc=Lc, kern_r=kern_r, Xr=Xr, Zr=Zr)

        #======================================================================
        # Compute dL_dpsi
        #======================================================================

        grad_dict['dL_dqU_mean'] += dL_dqU_mean
        grad_dict['dL_dqU_var_c'] += dL_dqU_var_c
        grad_dict['dL_dqU_var_r'] += dL_dqU_var_r
        grad_dict['dL_dKuu_c'] += dL_dKuu_c
        grad_dict['dL_dKuu_r'] += dL_dKuu_r

        if not uncertain_inputs_c:
            grad_dict['dL_dKdiag_c'] = grad_dict['dL_dpsi0_c']
            grad_dict['dL_dKfu_c'] = grad_dict['dL_dpsi1_c']

        if not uncertain_inputs_r:
            grad_dict['dL_dKdiag_r'] = grad_dict['dL_dpsi0_r']
            grad_dict['dL_dKfu_r'] = grad_dict['dL_dpsi1_r']

        return post, logL, grad_dict
636208
def binaryEntropyVectorized(x):
  
  entropy = - x*np.log2(x) - (1-x)*np.log2(1-x)
  entropy[x*(1 - x) == 0] = 0
  return entropy
582471
def mp2q(p, q):
    
    p, q = flatten(p), flatten(q)
    entropy_dist = 1 / len(p)
    return sum(entropy_dist * np.nan_to_num((p ** 2) / q * np.log(p / q)))
122789
def entropy(s):
    
    return -sum(
        p*np.log(p) for i in range(len(s)) for p in [prop(s[i], s)]
    )
469460
def xloss(logits, labels, ignore=None):
    
    return F.cross_entropy(logits, Variable(labels), ignore_index=255)
1327198
def entropy(self, logits):
        
        probs = torch.exp(logits)
        entropy = - torch.sum(probs * logits, dim=-1)
        return entropy
511391
def gpinv(p, k, sigma):
    """"""
    x = np.empty(p.shape)
    x.fill(np.nan)
    if sigma <= 0:
        return x
    ok = (p > 0) & (p < 1)
    if np.all(ok):
        if np.abs(k) < np.finfo(float).eps:
            np.negative(p, out=x)
            np.log1p(x, out=x)
            np.negative(x, out=x)
        else:
            np.negative(p, out=x)
            np.log1p(x, out=x)
            x *= -k
            np.expm1(x, out=x)
            x /= k
        x *= sigma
    else:
        if np.abs(k) < np.finfo(float).eps:
            # x[ok] = - np.log1p(-p[ok])
            temp = p[ok]
            np.negative(temp, out=temp)
            np.log1p(temp, out=temp)
            np.negative(temp, out=temp)
            x[ok] = temp
        else:
            # x[ok] = np.expm1(-k * np.log1p(-p[ok])) / k
            temp = p[ok]
            np.negative(temp, out=temp)
            np.log1p(temp, out=temp)
            temp *= -k
            np.expm1(temp, out=temp)
            temp /= k
            x[ok] = temp
        x *= sigma
        x[p == 0] = 0
        if k >= 0:
            x[p == 1] = np.inf
        else:
            x[p == 1] = -sigma / k
    return x
567100
def system_entropy():
        """"""
        arg = ['cat', '/proc/sys/kernel/random/entropy_avail']
        proc = Popen(arg, stdout=PIPE, stderr=DEVNULL)
        response = proc.communicate()[0]
        return int(response) if response else -1
50283
def binary_cross_entropy_cost( outputs, targets, derivative=False, epsilon=1e-11 ):
    
    # Prevent overflow
    outputs = np.clip(outputs, epsilon, 1 - epsilon)
    divisor = np.maximum(outputs * (1 - outputs), epsilon)
    
    if derivative:
        return (outputs - targets) / divisor
    else:
        return np.mean(-np.sum(targets * np.log( outputs ) + (1 - targets) * np.log(1 - outputs), axis=1))
324454
def black_box_variational_inference(logprob, D, num_samples):
    """"""

    def unpack_params(params):
        # Variational dist is a diagonal Gaussian.
        mean, log_std = params[:D], params[D:]
        return mean, log_std

    def gaussian_entropy(log_std):
        return 0.5 * D * (1.0 + np.log(2*np.pi)) + np.sum(log_std)

    rs = npr.RandomState(0)
    def variational_objective(params, t):
        """Provides a stochastic estimate of the variational lower bound."""
        mean, log_std = unpack_params(params)
        samples = rs.randn(num_samples, D) * np.exp(log_std) + mean
        lower_bound = gaussian_entropy(log_std) + np.mean(logprob(samples, t))
        return -lower_bound

    gradient = grad(variational_objective)

    return variational_objective, gradient, unpack_params
441481
function(len) {
                    if(len === undefined) len=16;

                    var entropy = fs.randomBytes(len);
                    var result = 0;

                    for(var i=0; i<len; i++) {
                        result = result + Number(entropy[i])/Math.pow(256,(i+1));
                    }
                    return result
                }
965805
private static double entropy(double count, double sum) {
        double p = count / sum;
        return Math.log(p) * p;
    }
51868
def P(self):
        """"""
        if self._P is not None:
            return self._P
        sparse_P = sparse.lil_matrix((self.n, self.n))
        alpha_size = len(self.alphabet)
        bitmask = (self.n-1)  # Mask w/ all bits set hi within (k-1)mer range.
        for fr in range(self.n):
            for a in range(alpha_size):
                to = (fr << 2 | a) & bitmask
                sparse_P[fr, to] = self.transitions[fr, a]
        self._P = sparse_P
        return sparse_P
858043
def uncertainty_pi(self):
        
        if self.information_ is None:
            self._build_information()

        sigma_pi = _ratematrix.sigma_pi(
            self.information_, theta=self.theta_, n=self.n_states_)
        return sigma_pi
1172607
def sum_of_entropies(image, mask, binary_image):
    
    mask=mask.copy()
    mask[np.isnan(image)] = False
    if not np.any(mask):
        return 0
    #
    # Clamp the dynamic range of the foreground
    #
    minval = np.max(image[mask])/256
    if minval == 0:
        return 0
    clamped_image = image.copy()
    clamped_image[clamped_image < minval] = minval
    #
    # Smooth image with -8 bits of noise
    #
    image = smooth_with_noise(clamped_image, 8)
    im_min = np.min(image)
    im_max = np.max(image)
    #
    # Figure out the bounds for the histogram
    #
    upper = np.log2(im_max)
    lower = np.log2(im_min)
    if upper == lower:
        # All values are the same, answer is log2 of # of pixels
        return math.log(np.sum(mask),2) 
    #
    # Create log-transformed lists of points in the foreground and background
    # 
    fg = image[binary_image & mask]
    bg = image[(~ binary_image) & mask]
    if len(fg) == 0 or len(bg) == 0:
        return 0
    log_fg = np.log2(fg)
    log_bg = np.log2(bg)
    #
    # Make these into histograms
    hfg = numpy_histogram(log_fg, 256, range=(lower,upper))[0]
    hbg = numpy_histogram(log_bg, 256, range=(lower,upper))[0]
    #hfg = scipy.ndimage.histogram(log_fg,lower,upper,256)
    #hbg = scipy.ndimage.histogram(log_bg,lower,upper,256)
    #
    # Drop empty bins
    #
    hfg = hfg[hfg>0]
    hbg = hbg[hbg>0]
    if np.product(hfg.shape) == 0:
        hfg = np.ones((1,),int)
    if np.product(hbg.shape) == 0:
        hbg = np.ones((1,),int)
    #
    # Normalize
    #
    hfg = hfg.astype(float) / float(np.sum(hfg))
    hbg = hbg.astype(float) / float(np.sum(hbg))
    #
    # Compute sum of entropies
    #
    return np.sum(hfg * np.log2(hfg)) + np.sum(hbg*np.log2(hbg))