watch scan rates across the cluster
423692
def watch(limit):
    """"""
    period = 5.0
    prev = db.db()
    prev_totals = None

    while True:
        click.clear()
        time.sleep(period)
        cur = db.db()
        cur.data['gkrate'] = {}
        progress = []
        prev_buckets = {b.bucket_id: b for b in prev.buckets()}

        totals = {'scanned': 0, 'krate': 0, 'lrate': 0, 'bucket_id': 'totals'}

        for b in cur.buckets():
            if not b.scanned:
                continue

            totals['scanned'] += b.scanned
            totals['krate'] += b.krate
            totals['lrate'] += b.lrate

            if b.bucket_id not in prev_buckets:
                b.data['gkrate'][b.bucket_id] = b.scanned / period
            elif b.scanned == prev_buckets[b.bucket_id].scanned:
                continue
            else:
                b.data['gkrate'][b.bucket_id] = (
                    b.scanned - prev_buckets[b.bucket_id].scanned) / period
            progress.append(b)

        if prev_totals is None:
            totals['gkrate'] = '...'
        else:
            totals['gkrate'] = (totals['scanned'] - prev_totals['scanned']) / period
        prev = cur
        prev_totals = totals

        progress = sorted(progress, key=lambda x: x.gkrate, reverse=True)

        if limit:
            progress = progress[:limit]

        progress.insert(0, Bag(totals))
        format_plain(
            progress, None,
            explicit_only=True,
            keys=['bucket_id', 'scanned', 'gkrate', 'lrate', 'krate'])
427291
protected void updateClusterCenters() {
		// compute the new centers of each cluster
		for (int i = 0; i < clusters.size; i++) {
			double mc = memberCount.get(i);
			double[] w = workClusters.get(i);
			double[] c = clusters.get(i);

			for (int j = 0; j < w.length; j++) {
				c[j] = w[j] / mc;
			}
		}
	}
268993
def _kmeans_run(X, n_clusters, max_iter, tol):
    
    membs = np.empty(shape=X.shape[0], dtype=int)
    centers = _kmeans_init(X, n_clusters)

    sse_last = 9999.9
    n_iter = 0
    for it in range(1,max_iter):
        membs = _assign_clusters(X, centers)
        centers,sse_arr = _update_centers(X, membs, n_clusters)
        sse_total = np.sum(sse_arr)
        if np.abs(sse_total - sse_last) < tol:
            n_iter = it
            break
        sse_last = sse_total

    return(centers, membs, sse_total, sse_arr, n_iter)
174070
private void fullScanLocalDataToClear(Indexes indexes) {
        InternalIndex[] indexesSnapshot = indexes.getIndexes();
        for (Record record : storage.values()) {
            Data key = record.getKey();
            Object value = Records.getValueOrCachedValue(record, serializationService);
            indexes.removeEntry(key, value, Index.OperationSource.SYSTEM);
        }
        Indexes.markPartitionAsUnindexed(partitionId, indexesSnapshot);
    }
793417
def collect(self):
        
        performance_indicators = {
            'ns': 'network_send',
            'nr': 'network_receive',
            'dw': 'disk_write',
            'dr': 'disk_read',
            'al': 'activity_log',
            'bm': 'bit_map',
            'lo': 'local_count',
            'pe': 'pending',
            'ua': 'unacknowledged',
            'ap': 'application_pending',
            'ep': 'epochs',
            'wo': 'write_order',
            'oos': 'out_of_sync',
            'cs': 'connection_state',
            'ro': 'roles',
            'ds': 'disk_states'
        }

        results = dict()
        try:
            statusfile = open('/proc/drbd', 'r')
            current_resource = ''
            for line in statusfile:
                if re.search('version', line) is None:
                    if re.search(r' \d: cs', line):
                        matches = re.match(r' (\d): (cs:\w+) (ro:\w+/\w+) '
                                           '(ds:\w+/\w+) (\w{1}) .*', line)
                        current_resource = matches.group(1)
                        results[current_resource] = dict()
                    elif re.search(r'\sns:', line):
                        metrics = line.strip().split(" ")
                        for metric in metrics:
                            item, value = metric.split(":")
                            results[current_resource][
                                performance_indicators[item]] = value

                else:
                    continue
            statusfile.close()
        except IOError as errormsg:
            self.log.error("Can't read DRBD status file: {}".format(errormsg))
            return

        for resource in results.keys():
            for metric_name, metric_value in results[resource].items():
                if metric_value.isdigit():
                    self.publish(resource + "." + metric_name, metric_value)
                else:
                    continue
972394
def olci(scans_nb, scan_points=None):
    

    if scan_points is None:
        scan_len = 4000  # samples per scan
        scan_points = np.arange(4000)
    else:
        scan_len = len(scan_points)
    # scan_rate = 0.044  # single scan, seconds
    scan_angle_west = 46.5  # swath, degrees
    scan_angle_east = -22.1  # swath, degrees
    # sampling_interval = 18e-3  # single view, seconds
    # build the olci instrument scan line angles
    scanline_angles = np.linspace(np.deg2rad(scan_angle_west),
                                  np.deg2rad(scan_angle_east), scan_len)
    inst = np.vstack((scanline_angles, np.zeros(scan_len,)))

    inst = np.tile(inst[:, np.newaxis, :], [1, np.int(scans_nb), 1])

    # building the corresponding times array
    # times = (np.tile(scan_points * 0.000025 + 0.0025415, [scans_nb, 1])
    #         + np.expand_dims(offset, 1))

    times = np.tile(np.zeros_like(scanline_angles), [np.int(scans_nb), 1])
    # if apply_offset:
    #     offset = np.arange(np.int(scans_nb)) * frequency
    #     times += np.expand_dims(offset, 1)

    return ScanGeometry(inst, times)
1088301
def batch_scan(points, xdist=20, ydist=20, N=5):
    
    chr_pair_points = group_hits(points)

    clusters = []
    for chr_pair in sorted(chr_pair_points.keys()):
        points = chr_pair_points[chr_pair]
        clusters.extend(synteny_scan(points, xdist, ydist, N))

    return clusters
1206479
def _run(self):
        
#        print("Doing APM Clustering...")
        # Start looping for maxIter times
        n_macrostates = 1  # initialized as 1 because no macrostate exist in loop 0
        metaQ = -1.0
        prevQ = -1.0
        global_maxQ = -1.0
        local_maxQ = -1.0

        for iter in range(self.max_iter):
            self.__max_state = -1
            self.__micro_stack = []
            for k in range(n_macrostates):
                self._do_split(micro_state=k, sub_clus=self.sub_clus)
                self._do_time_clustering(macro_state=k)

            # do Lumping
            n_micro_states = np.amax(self.__temp_labels_) + 1
            if n_micro_states > self.n_macrostates:
#                print("PCCA Lumping...", n_micro_states, "microstates")
                self.__temp_MacroAssignments_ = self._do_lumping(
                    n_macrostates=n_macrostates)
                #self.__temp_labels_ = [copy.copy(element) for element in self.__temp_MacroAssignments_]

                #Calculate Metastabilty
                prevQ = metaQ
                metaQ = self.__temp_transmat_.diagonal().sum()
                metaQ /= len(self.__temp_transmat_)
            else:
                self.__temp_MacroAssignments_ = [
                    copy.copy(element) for element in self.__temp_labels_
                ]

            # Optimization / Monte-Carlo
            acceptedMove = False
            MCacc = np.exp(metaQ * metaQ - prevQ * prevQ)

            if MCacc > 1.0:
                MCacc = 1.0
            optLim = 0.95

            if MCacc > optLim:
                acceptedMove = True

            if acceptedMove:
                local_maxQ = metaQ
                if metaQ > global_maxQ:
                    global_maxQ = metaQ
                    self.MacroAssignments_ = [
                        copy.copy(element)
                        for element in self.__temp_MacroAssignments_
                    ]
                    self.labels_ = [copy.copy(element)
                                    for element in self.__temp_labels_]
                    self.transmat_ = self.__temp_transmat_

#            print("Loop:", iter, "AcceptedMove?", acceptedMove, "metaQ:",
#                  metaQ, "prevQ:", prevQ, "global_maxQ:", global_maxQ,
#                  "local_maxQ:", local_maxQ, "macroCount:", n_macrostates)
            #set n_macrostates
            n_macrostates = self.n_macrostates
            self.__temp_labels_ = [copy.copy(element)
                                   for element in self.__temp_MacroAssignments_
                                   ]
641275
def _kmedoids_run(X, n_clusters, distance, max_iter, tol, rng):
    
    membs = np.empty(shape=X.shape[0], dtype=int)
    centers = kmeans._kmeans_init(X, n_clusters, method='', rng=rng)

    sse_last = 9999.9
    n_iter = 0
    for it in range(1,max_iter):
        membs = kmeans._assign_clusters(X, centers)
        centers,sse_arr = _update_centers(X, membs, n_clusters, distance)
        sse_total = np.sum(sse_arr)
        if np.abs(sse_total - sse_last) < tol:
            n_iter = it
            break
        sse_last = sse_total

    return(centers, membs, sse_total, sse_arr, n_iter)
199662
def _collect_monitor_metrics(self, conn, tags):
        
        for entry in conn.entries:
            # Get metrics from monitor backend
            dn = entry.entry_dn.lower()
            if dn.endswith(self.CONNECTIONS_METRICS_DN):
                self._handle_connections_entry(entry, tags)
            elif dn.endswith(self.OPERATIONS_METRICS_DN):
                self._handle_operations_entry(entry, tags)
            elif dn.endswith(self.STATISTICS_METRICS_DN):
                self._handle_statistics_entry(entry, tags)
            elif dn.endswith(self.THREADS_METRICS_DN):
                self._handle_threads_entry(entry, tags)
            elif dn.endswith(self.TIME_METRICS_DN):
                self._handle_time_entry(entry, tags)
            elif dn.endswith(self.WAITERS_METRICS_DN):
                self._handle_waiters_entry(entry, tags)
530197
public double evaluateClusteringPerformance(Clustering clustering, Clustering trueClustering, ArrayList<DataPoint> points) throws Exception{
        long start = System.nanoTime();
        evaluateClustering(clustering, trueClustering, points);
        long duration = System.nanoTime()-start;
        time+=duration;
        duration/=10e5;
        return duration;
     }
1166108
async def _watch(self, node, conn, names):
        ""
        for name in names:
            slot = self._determine_slot('WATCH', name)
            dist_node = self.connection_pool.get_node_by_slot(slot)
            if node.get('name') != dist_node['name']:
                # raise error if commands in a transaction can not hash to same node
                if len(node) > 0:
                    raise ClusterTransactionError("Keys in request don't hash to the same node")
        if self.explicit_transaction:
            raise RedisError('Cannot issue a WATCH after a MULTI')
        await conn.send_command('WATCH', *names)
        return await conn.read_response()
808983
def _advance_cluster_time(self, cluster_time):
        """"""
        if self._cluster_time is None:
            self._cluster_time = cluster_time
        elif cluster_time is not None:
            if cluster_time["clusterTime"] > self._cluster_time["clusterTime"]:
                self._cluster_time = cluster_time
1150091
def aggregate_metrics(latencies_ms, duration_ms, num_bucket):
    """"""
    overall_op_count = 0
    op_counts = {operation : len(latency) for operation,
                 latency in latencies_ms.iteritems()}
    overall_op_count = sum([op_count for op_count in op_counts.itervalues()])

    print('[OVERALL], RunTime(ms), %f' % duration_ms)
    print('[OVERALL], Throughput(ops/sec), %f' % (float(overall_op_count) /
                                                duration_ms * 1000.0))

    for operation in op_counts.keys():
        operation_upper = operation.upper()
        print('[%s], Operations, %d' % (operation_upper, op_counts[operation]))
        print('[%s], AverageLatency(us), %f' % (
            operation_upper, numpy.average(latencies_ms[operation]) * 1000.0))
        print('[%s], LatencyVariance(us), %f' % (
            operation_upper, numpy.var(latencies_ms[operation]) * 1000.0))
        print('[%s], MinLatency(us), %f' % (
            operation_upper, min(latencies_ms[operation]) * 1000.0))
        print('[%s], MaxLatency(us), %f' % (
            operation_upper, max(latencies_ms[operation]) * 1000.0))
        print('[%s], 95thPercentileLatency(us), %f' % (
            operation_upper,
            numpy.percentile(latencies_ms[operation], 95.0) * 1000.0))
        print('[%s], 99thPercentileLatency(us), %f' % (
            operation_upper,
            numpy.percentile(latencies_ms[operation], 99.0) * 1000.0))
        print('[%s], 99.9thPercentileLatency(us), %f' % (
            operation_upper,
            numpy.percentile(latencies_ms[operation], 99.9) * 1000.0))
        print('[%s], Return=OK, %d' % (operation_upper, op_counts[operation]))
        latency_array = numpy.array(latencies_ms[operation])
        for j in range(num_bucket):
            print('[%s], %d, %d' % (
                operation_upper, j,
                ((j <= latency_array) & (latency_array < (j + 1))).sum()))
        print('[%s], >%d, %d' % (operation_upper, num_bucket,
                                 (num_bucket <= latency_array).sum()))
542649
private void calculateGTClusterConnections(){
        for (int c0 = 0; c0 < gt0Clusters.size(); c0++) {
            for (int c1 = 0; c1 < gt0Clusters.size(); c1++) {
                    gt0Clusters.get(c0).calculateClusterConnection(c1, true);
            }
        }

        boolean changedConnection = true;
        while(changedConnection){
            if(debug){
                System.out.println("Cluster Connection");
                for (int c = 0; c < gt0Clusters.size(); c++) {
                    System.out.print("C"+gt0Clusters.get(c).label+" --> ");
                    for (int c1 = 0; c1 < gt0Clusters.get(c).connections.size(); c1++) {
                        System.out.print(" C"+gt0Clusters.get(c1).label+": "+gt0Clusters.get(c).connections.get(c1));
                    }
                    System.out.println("");
                }
                System.out.println("");
            }

            double max = 0;
            int maxIndexI = -1;
            int maxIndexJ = -1;

            changedConnection = false;
            for (int c0 = 0; c0 < gt0Clusters.size(); c0++) {
                for (int c1 = c0+1; c1 < gt0Clusters.size(); c1++) {
                    if(c0==c1) continue;
                        double min =Math.min(gt0Clusters.get(c0).connections.get(c1), gt0Clusters.get(c1).connections.get(c0));
                        if(min > max){
                            max = min;
                            maxIndexI = c0;
                            maxIndexJ = c1;
                        }
                }
            }
            if(maxIndexI!=-1 && max > tauConnection){
                gt0Clusters.get(maxIndexI).mergeCluster(maxIndexJ);
                if(debug)
                    System.out.println("Merging "+maxIndexI+" and "+maxIndexJ+" because of connection "+max);

                changedConnection = true;
            }
        }
        numGT0Classes = gt0Clusters.size();
    }
824490
def gather_data(registry):
    """"""

    # Get the host name of the machine
    host = socket.gethostname()

    # Create our collectors
    ram_metric = Gauge("memory_usage_bytes", "Memory usage in bytes.",
                       {'host': host})
    cpu_metric = Gauge("cpu_usage_percent", "CPU usage percent.",
                       {'host': host})

    # register the metric collectors
    registry.register(ram_metric)
    registry.register(cpu_metric)

    # Start gathering metrics every second
    while True:
        time.sleep(1)

        # Add ram metrics
        ram = psutil.virtual_memory()
        swap = psutil.swap_memory()

        ram_metric.set({'type': "virtual", }, ram.used)
        ram_metric.set({'type': "virtual", 'status': "cached"}, ram.cached)
        ram_metric.set({'type': "swap"}, swap.used)

        # Add cpu metrics
        for c, p in enumerate(psutil.cpu_percent(interval=1, percpu=True)):
            cpu_metric.set({'core': c}, p)
505584
def _group_directional(self, clusters, adj_list, counts):
        ''' '''

        observed = set()
        groups = []
        for cluster in clusters:
            if len(cluster) == 1:
                groups.append(list(cluster))
                observed.update(cluster)
            else:
                cluster = sorted(cluster, key=lambda x: counts[x],
                                 reverse=True)
                # need to remove any node which has already been observed
                temp_cluster = []
                for node in cluster:
                    if node not in observed:
                        temp_cluster.append(node)
                        observed.add(node)
                groups.append(temp_cluster)

        return groups