Extract frequency of occurrences of those cell types that are modeled.
    The data set contains cell types that are not modeled (TCs etc.)
    The returned percentages are renormalized onto modeled cell-types, i.e. they sum up to 1
480721
def get_F_y(fname='binzegger_connectivity_table.json', y=['p23']): 
    '''
     
    '''
    # Load data from json dictionary
    f = open(fname,'r')
    data = json.load(f)
    f.close()
    
    occurr = []
    for cell_type in y:
        occurr += [data['data'][cell_type]['occurrence']]
    return list(np.array(occurr)/np.sum(occurr))
925086
def countRandomBitFrequencies(numTerms = 100000, percentSparsity = 0.01):
  """"""
  # Accumulate counts by inplace-adding sparse matrices
  counts = SparseMatrix()
  size = 128*128
  counts.resize(1, size)

  # Pre-allocate buffer sparse matrix
  sparseBitmap = SparseMatrix()
  sparseBitmap.resize(1, size)

  random.seed(42)

  # Accumulate counts for each bit for each word
  numWords=0
  for term in xrange(numTerms):
    bitmap = random.sample(xrange(size), int(size*percentSparsity))
    bitmap.sort()

    sparseBitmap.setRowFromSparse(0, bitmap, [1]*len(bitmap))
    counts += sparseBitmap
    numWords += 1

  # Compute normalized version of counts as a separate matrix
  frequencies = SparseMatrix()
  frequencies.resize(1, size)
  frequencies.copy(counts)
  frequencies.divide(float(numWords))

  # Wrap up by printing some statistics and then saving the normalized version
  printFrequencyStatistics(counts, frequencies, numWords, size)

  frequencyFilename = "bit_frequencies_random.pkl"
  print "Saving frequency matrix in",frequencyFilename
  with open(frequencyFilename, "wb") as frequencyPickleFile:
    pickle.dump(frequencies, frequencyPickleFile)

  return counts
794151
def object_summary
      nval = count_values(*Daru::MISSING_VALUES)
      summary = "\n  factors: #{factors.to_a.join(',')}" \
                "\n  mode: #{mode.to_a.join(',')}" \
                "\n  Distribution\n"

      data = frequencies.sort.each_with_index.map do |v, k|
        [k, v, '%0.2f%%' % ((nval.zero? ? 1 : v.quo(nval))*100)]
      end

      summary + Formatters::Table.format(data)
    end
34209
def r(self):
        """"""
        if not self._counters_calculated:
            self._counters_calculated = True
            self._extract_counters()

        return self._r
265785
def frequency(self, context):
        
        channels = self._manager.spectral_window_table.getcol(MS.CHAN_FREQ)
        return channels.reshape(context.shape).astype(context.dtype)
810162
def frequencies_plot(self, xmin=0, xmax=200):
        

        helptext = '''
            A possible way to assess the complexity of a library even in
            absence of a reference sequence is to look at the kmer profile of the reads.
            The idea is to count all the kmers (_i.e._, sequence of length `k`) that occur
            in the reads. In this way it is possible to know how many kmers occur
            `1,2,.., N` times and represent this as a plot.
            This plot tell us for each x, how many k-mers (y-axis) are present in the
            dataset in exactly x-copies.

            In an ideal world (no errors in sequencing, no bias, no  repeated regions)
            this plot should be as close as  possible to a gaussian distribution.
            In reality we will always see a peak for `x=1` (_i.e._, the errors)
            and another peak close to the expected coverage. If the genome is highly
            heterozygous a second peak at half of the coverage can be expected.'''

        pconfig = {
            'id': 'Jellyfish_kmer_plot',
            'title': 'Jellyfish: K-mer plot',
            'ylab': 'Counts',
            'xlab': 'k-mer frequency',
            'xDecimals': False,
            'xmin': xmin,
            'xmax': xmax
        }

        self.add_section(
            anchor = 'jellyfish_kmer_plot',
            description = 'The K-mer plot lets you estimate library complexity and coverage from k-mer content.',
            helptext = helptext,
            plot = linegraph.plot(self.jellyfish_data, pconfig)
        )
970857
def predict_cats_from_sigs(self, df_data_ini, df_sig_ini, dist_type='cosine', predict_level='Predict Category',
                             truth_level=1, unknown_thresh=-1):
      '''  '''

      keep_rows = df_sig_ini.index.tolist()
      data_rows = df_data_ini.index.tolist()

      common_rows = list(set(data_rows).intersection(keep_rows))

      df_data = deepcopy(df_data_ini.ix[common_rows])
      df_sig = deepcopy(df_sig_ini.ix[common_rows])

      # calculate sim_mat of df_data and df_sig
      cell_types = df_sig.columns.tolist()
      barcodes = df_data.columns.tolist()
      sim_mat = 1 - pairwise_distances(df_sig.transpose(), df_data.transpose(), metric=dist_type)
      df_sim = pd.DataFrame(data=sim_mat, index=cell_types, columns=barcodes).transpose()

      # get the top column value (most similar signature)
      df_sim_top = df_sim.idxmax(axis=1)

      # get the maximum similarity of a cell to a cell type definition
      max_sim = df_sim.max(axis=1)

      unknown_cells = max_sim[max_sim < unknown_thresh].index.tolist()

      # assign unknown cells (need category of same name)
      df_sim_top[unknown_cells] = 'Unknown'

      # add predicted category name to top list
      top_list = df_sim_top.get_values()
      top_list = [ predict_level + ': ' + x[0] if type(x) is tuple else predict_level + ': ' + x  for x in top_list]

      # add cell type category to input data
      df_cat = deepcopy(df_data)
      cols = df_cat.columns.tolist()
      new_cols = []

      # check whether the columns have the true category available
      has_truth = False
      if type(cols[0]) is tuple:
          has_truth = True

      if has_truth:
          new_cols = [tuple(list(a) + [b]) for a,b in zip(cols, top_list)]
      else:
          new_cols = [tuple([a] + [b]) for a,b in zip(cols, top_list)]

      # transfer new categories
      df_cat.columns = new_cols

      # keep track of true and predicted labels
      y_info = {}
      y_info['true'] = []
      y_info['pred'] = []

      if has_truth:
          y_info['true'] = [x[truth_level].split(': ')[1] for x in cols]
          y_info['pred'] = [x.split(': ')[1] for x in top_list]

      return df_cat, df_sim.transpose(), y_info
529900
def check_cell_type(cell, cell_type):
    '''
    
    '''
    if cell_type == None or cell_type == type(None):
        return cell == None or (isinstance(cell, basestring) and not cell)
    else:
        return isinstance(cell, cell_type)
802646
def _maybe_dt_data(data, feature_names, feature_types):
    
    if not isinstance(data, DataTable):
        return data, feature_names, feature_types

    data_types_names = tuple(lt.name for lt in data.ltypes)
    bad_fields = [data.names[i]
                  for i, type_name in enumerate(data_types_names)
                  if type_name not in DT_TYPE_MAPPER]
    if bad_fields:
        msg = """DataFrame.types for data must be int, float or bool.
                Did not expect the data types in fields """
        raise ValueError(msg + ', '.join(bad_fields))

    if feature_names is None:
        feature_names = data.names

        # always return stypes for dt ingestion
        if feature_types is not None:
            raise ValueError('DataTable has own feature types, cannot pass them in')
        feature_types = np.vectorize(DT_TYPE_MAPPER2.get)(data_types_names)

    return data, feature_names, feature_types
1001618
def get_analysis_intervals(data, vrn_file, base_dir):
    
    from bcbio.bam import callable
    if vrn_file and vcfutils.is_gvcf_file(vrn_file):
        callable_bed = _callable_from_gvcf(data, vrn_file, base_dir)
        if callable_bed:
            return callable_bed

    if data.get("ensemble_bed"):
        return data["ensemble_bed"]
    elif dd.get_sample_callable(data):
        return dd.get_sample_callable(data)
    elif data.get("align_bam"):
        return callable.sample_callable_bed(data["align_bam"], dd.get_ref_file(data), data)[0]
    elif data.get("work_bam"):
        return callable.sample_callable_bed(data["work_bam"], dd.get_ref_file(data), data)[0]
    elif data.get("work_bam_callable"):
        data = utils.deepish_copy(data)
        data["work_bam"] = data.pop("work_bam_callable")
        return callable.sample_callable_bed(data["work_bam"], dd.get_ref_file(data), data)[0]
    elif tz.get_in(["config", "algorithm", "callable_regions"], data):
        return tz.get_in(["config", "algorithm", "callable_regions"], data)
    elif tz.get_in(["config", "algorithm", "variant_regions"], data):
        return tz.get_in(["config", "algorithm", "variant_regions"], data)
829475
def cells(self):
        

        n = 0

        for (order, cells) in self:
            n += len(cells)

        return n
144171
def col_frequencies(col, weights=None, gap_chars='-.'):
    """"""
    counts = col_counts(col, weights, gap_chars)
    # Reduce to frequencies
    scale = 1.0 / sum(counts.values())
    return dict((aa, cnt * scale) for aa, cnt in counts.iteritems())
907656
def find(self,cell_designation,cell_filter=lambda x,c: 'c' in x and x['c'] == c):
        
        res = [i for i,sc in enumerate(self.spike_containers) if cell_filter(sc.meta,cell_designation)]
        if len(res) > 0:
            return res[0]
695300
def cells_from_defaults(clz, jsonobj):
        
        # convert strings to dicts
        if isinstance(jsonobj, (str, unicode)):
            jsonobj = json.loads(jsonobj)
        
        assert 'cells' in jsonobj, "No cells in object"
       
        domain = TaxonomyCell.get_domain()
        cells = []
        for num, cell_dna in enumerate(jsonobj['cells']):
            assert 'kind' in cell_dna, "No type definition"
            classgenerator = domain.node[cell_dna['kind']]['class']
            cell = classgenerator()
            cell['num'].merge(num)
            for attr, val in cell_dna.items():
                if not attr in ['kind']:
                    cell[attr].merge(val)
                    cells.append(cell)
        return cells
1060300
function incidence(from_cells, to_cells) {
  var index = new Array(from_cells.length)
  for(var i=0, il=index.length; i<il; ++i) {
    index[i] = []
  }
  var b = []
  for(var i=0, n=to_cells.length; i<n; ++i) {
    var c = to_cells[i]
    var cl = c.length
    for(var k=1, kn=(1<<cl); k<kn; ++k) {
      b.length = bits.popCount(k)
      var l = 0
      for(var j=0; j<cl; ++j) {
        if(k & (1<<j)) {
          b[l++] = c[j]
        }
      }
      var idx=findCell(from_cells, b)
      if(idx < 0) {
        continue
      }
      while(true) {
        index[idx++].push(i)
        if(idx >= from_cells.length || compareCells(from_cells[idx], b) !== 0) {
          break
        }
      }
    }
  }
  return index
}
339880
def calculate(bam_file, data, sv_bed):
    
    params = {"min": dd.get_coverage_depth_min(data)}
    variant_regions = dd.get_variant_regions_merged(data)
    if not variant_regions:
        variant_regions = _create_genome_regions(data)
    # Back compatible with previous pre-mosdepth callable files
    callable_file = os.path.join(utils.safe_makedir(os.path.join(dd.get_work_dir(data), "align",
                                                                 dd.get_sample_name(data))),
                                 "%s-coverage.callable.bed" % (dd.get_sample_name(data)))
    if not utils.file_uptodate(callable_file, bam_file):
        vr_quantize = ("0:1:%s:" % (params["min"]), ["NO_COVERAGE", "LOW_COVERAGE", "CALLABLE"])
        to_calculate = [("variant_regions", variant_regions,
                         vr_quantize, None, "coverage_perbase" in dd.get_tools_on(data)),
                        ("sv_regions", bedutils.clean_file(sv_bed, data, prefix="svregions-"),
                         None, None, False),
                        ("coverage", bedutils.clean_file(dd.get_coverage(data), data, prefix="cov-"),
                         None, DEPTH_THRESHOLDS, False)]
        depth_files = {}
        for target_name, region_bed, quantize, thresholds, per_base in to_calculate:
            if region_bed:
                cur_depth = {}
                depth_info = run_mosdepth(data, target_name, region_bed, quantize=quantize, thresholds=thresholds,
                                          per_base=per_base)
                for attr in ("dist", "regions", "thresholds", "per_base"):
                    val = getattr(depth_info, attr, None)
                    if val:
                        cur_depth[attr] = val
                depth_files[target_name] = cur_depth
                if target_name == "variant_regions":
                    callable_file = depth_info.quantize
    else:
        depth_files = {}
    final_callable = _subset_to_variant_regions(callable_file, variant_regions, data)
    return final_callable, depth_files
1018496
def preproc_data(data, gene_subset=False, **kwargs):
    
    import uncurl
    from uncurl.preprocessing import log1p, cell_normalize
    from sklearn.decomposition import TruncatedSVD
    data_subset = data
    if gene_subset:
        gene_subset = uncurl.max_variance_genes(data)
        data_subset = data[gene_subset, :]
    tsvd = TruncatedSVD(min(8, data_subset.shape[0] - 1))
    data_tsvd = tsvd.fit_transform(log1p(cell_normalize(data_subset)).T)
    return data_tsvd