Call calculation functions and write stats file.

    This function takes a list of DataFrames,
    and will create a column for each in the tab separated output.
194524
def write_stats(datadfs, outputfile, names=[]):
    
    if outputfile == 'stdout':
        output = sys.stdout
    else:
        output = open(outputfile, 'wt')

    stats = [Stats(df) for df in datadfs]
    features = {
        "Number of reads": "number_of_reads",
        "Total bases": "number_of_bases",
        "Total bases aligned": "number_of_bases_aligned",
        "Median read length": "median_read_length",
        "Mean read length": "mean_read_length",
        "Read length N50": "n50",
        "Average percent identity": "average_identity",
        "Median percent identity": "median_identity",
        "Active channels": "active_channels",
        "Mean read quality": "mean_qual",
        "Median read quality": "median_qual",
    }
    max_len = max([len(k) for k in features.keys()])
    try:
        max_num = max(max([len(str(s.number_of_bases)) for s in stats]),
                      max([len(str(n)) for n in names])) + 6
    except ValueError:
        max_num = max([len(str(s.number_of_bases)) for s in stats]) + 6
    output.write("{:<{}}{}\n".format('General summary:', max_len,
                                     " ".join(['{:>{}}'.format(n, max_num) for n in names])))
    for f in sorted(features.keys()):
        try:
            output.write("{f:{pad}}{v}\n".format(
                f=f + ':',
                pad=max_len,
                v=feature_list(stats, features[f], padding=max_num)))
        except KeyError:
            pass
    if all(["quals" in df for df in datadfs]):
        long_features = {
            "Top 5 longest reads and their mean basecall quality score":
            ["top5_lengths", range(1, 6)],
            "Top 5 highest mean basecall quality scores and their read lengths":
            ["top5_quals", range(1, 6)],
            "Number, percentage and megabases of reads above quality cutoffs":
            ["reads_above_qual", [">Q" + str(q) for q in stats[0].qualgroups]],
        }
        for lf in sorted(long_features.keys()):
            output.write(lf + "\n")
            for i in range(5):
                output.write("{}:\t{}\n".format(
                    long_features[lf][1][i], feature_list(stats, long_features[lf][0], index=i)))
48165
def Write(self, output_writer):
    
    # Round up the column sizes to the nearest tab.
    for column_index, column_size in enumerate(self._column_sizes):
      column_size, _ = divmod(column_size, self._NUMBER_OF_SPACES_IN_TAB)
      column_size = (column_size + 1) * self._NUMBER_OF_SPACES_IN_TAB
      self._column_sizes[column_index] = column_size

    if self._columns:
      self._WriteRow(output_writer, self._columns, in_bold=True)

    for values in self._rows:
      self._WriteRow(output_writer, values)
865565
def _summary(in_file):
    
    data = Counter()
    out_file = in_file + "_size_stats"
    if file_exists(out_file):
        return out_file
    with open(in_file) as in_handle:
        for line in in_handle:
            counts = int(line.strip().split("_x")[1])
            line = next(in_handle)
            l = len(line.strip())
            next(in_handle)
            next(in_handle)
            data[l] += counts
    with file_transaction(out_file) as tx_out_file:
        with open(tx_out_file, 'w') as out_handle:
            for l, c in data.items():
                out_handle.write("%s %s\n" % (l, c))
    return out_file
1198163
def write_mzxml(filename, df, info=None, precision='f'):
    
    for r in df.values:
        df.columns
        pass
1238640
def dtypes_summary(df):
    
    output_df = pd.DataFrame([])
    row_count = df.shape[0]
    row_indexes = ['rows_numerical','rows_string','rows_date_time','category_count','largest_category','rows_na','rows_total']
    for colname in df:
        data = df[colname] # data is the pandas series associated with this column
        # number of numerical values in the column
        rows_numerical = pd.to_numeric(data,errors = 'coerce').count()
        # number of values that can't be coerced to a numerical
        rows_string = row_count - rows_numerical
        # number of values that can be coerced to a date-time object
        rows_date_time = pd.to_datetime(data,errors = 'coerce',infer_datetime_format = True).count()
        # categories in column
        value_counts = data.value_counts().reset_index()
        # number of different values in the dataframe
        categories = len(value_counts)
        # largest category
        largest_category = value_counts.iloc[0,1]
        # number of null/missing values
        rows_na = data.isnull().sum()
        # build the output list
        output_data = [rows_numerical, rows_string, rows_date_time, categories, 
                       largest_category,rows_na,row_count]
        # add to dataframe
        output_df.loc[:,colname] = pd.Series(output_data)

    # row names
    output_df.index = row_indexes
    return output_df
481749
def main():
  """"""
  parser = argparse.ArgumentParser(
      description='Reads in a .xls file and generates a units module for '
                  'OpenHTF.',
      prog='python units_from_xls.py')
  parser.add_argument('xlsfile', type=str,
                      help='the .xls file to parse')
  parser.add_argument(
      '--outfile',
      type=str,
      default=os.path.join(os.path.dirname(__file__), os.path.pardir,
                           'openhtf','util', 'units.py'),
      help='where to put the generated .py file.')
  args = parser.parse_args()

  if not os.path.exists(args.xlsfile):
    print('Unable to locate the file "%s".' % args.xlsfile)
    parser.print_help()
    sys.exit()

  unit_defs = unit_defs_from_sheet(
      xlrd.open_workbook(args.xlsfile).sheet_by_name(SHEET_NAME),
      COLUMN_NAMES)

  _, tmp_path = mkstemp()
  with open(tmp_path, 'w') as new_file:
    new_file.write(PRE)
    new_file.writelines(
        [line.encode('utf8', 'replace') for line in unit_defs])
    new_file.write(POST)
    new_file.flush()

  os.remove(args.outfile)
  shutil.move(tmp_path, args.outfile)
11371
def _construct_df(pdb_lines):
        """"""
        valids = tuple(pdb_records.keys())
        line_lists = {r: [] for r in valids}
        line_lists['OTHERS'] = []
        for line_num, line in enumerate(pdb_lines):
            if line.strip():
                if line.startswith(valids):
                    record = line[:6].rstrip()
                    line_ele = ['' for _ in range(len(
                        pdb_records[record]) + 1)]
                    for idx, ele in enumerate(pdb_records[record]):
                        line_ele[idx] = (line[ele['line'][0]:ele['line'][1]]
                                         .strip())
                    line_ele[-1] = line_num
                    line_lists[record].append(line_ele)
                else:
                    line_lists['OTHERS'].append([line[:6].rstrip(),
                                                line[6:-1].rstrip(), line_num])

        dfs = {}
        for r in line_lists.items():
            df = pd.DataFrame(r[1], columns=[c['id'] for c in
                                             pdb_records[r[0]]] + ['line_idx'])
            for c in pdb_records[r[0]]:
                try:
                    df[c['id']] = df[c['id']].astype(c['type'])
                except ValueError:
                    # expect ValueError if float/int columns are empty strings
                    df[c['id']] = pd.Series(np.nan, index=df.index)

            dfs[r[0]] = df
        return dfs
353994
def save_outputs(outputs, output_fpath):
    
    df = pd.DataFrame(outputs)
    with pd.ExcelWriter(output_fpath) as writer:
        df.to_excel(writer)
299317
def init_stats(output_dir, config):
    

    try:
        os.makedirs(output_dir, 0o755)
    except OSError as e:
        print("ERROR: Can not create output directory: %s\n" % e)
        raise

    db_uri = get_db_uri(config, output_dir)

    set_database(db_uri, db, config)

    tables_to_create = [t for t in [Result, Turret] if not t.table_exists()]

    db.get_conn()
    db.create_tables(tables_to_create)
832535
def write_dataframe(rows, encoding=ENCODING, dialect=DIALECT, **kwargs):
    """"""
    global pandas
    if pandas is None:  # pragma: no cover
        import pandas
    with contextlib.closing(CsvBuffer()) as fd:
        write_csv(fd, rows, encoding, dialect)
        fd.seek(0)
        df = read_csv(pandas, fd, encoding, dialect, kwargs)
    return df
1208448
private function writeColumns(): void
  {
    $content = '';
    foreach ($this->columns as $table)
    {
      $width1 = 0;
      $width2 = 0;
      foreach ($table as $column)
      {
        $width1 = max(mb_strlen($column['column_name']), $width1);
        $width2 = max(mb_strlen((string)$column['length']), $width2);
      }

      foreach ($table as $column)
      {
        if (isset($column['length']))
        {
          if (isset($column['constant_name']))
          {
            $line_format = sprintf("%%s.%%-%ds %%%dd %%s\n", $width1, $width2);
            $content     .= sprintf($line_format,
                                    $column['table_name'],
                                    $column['column_name'],
                                    $column['length'],
                                    $column['constant_name']);
          }
          else
          {
            $line_format = sprintf("%%s.%%-%ds %%%dd\n", $width1, $width2);
            $content     .= sprintf($line_format,
                                    $column['table_name'],
                                    $column['column_name'],
                                    $column['length']);
          }
        }
      }

      $content .= "\n";
    }

    // Save the columns, width and constants to the filesystem.
    $this->writeTwoPhases($this->constantsFilename, $content);
  }
718514
def make_summaries(self):
        """"""
        self.summary_df = save_summaries(self.frames, self.keys,
                                         self.selected_summaries,
                                         self.batch_dir, self.name)
        logger.debug("made and saved summaries")
915718
def write_data_file(self, data, fn, sort_cols=False, data_format=None):
        
        report.saved_raw_data[fn] = data
        util_functions.write_data_file(data, fn, sort_cols, data_format)
707802
def display(self):
        
        data = []
        first_row = ['Stat']
        first_row.extend(self._names)
        data.append(first_row)

        stats = self._stats()

        for stat in stats:
            k, n, f = stat
            # blank row
            if k is None:
                row = [''] * len(data[0])
                data.append(row)
                continue

            row = [n]
            for key in self._names:
                raw = getattr(self[key], k)

                # if rf is a series print nan
                if k == 'rf' and not type(raw) == float:
                    row.append(np.nan)
                elif f is None:
                    row.append(raw)
                elif f == 'p':
                    row.append(fmtp(raw))
                elif f == 'n':
                    row.append(fmtn(raw))
                elif f == 'dt':
                    row.append(raw.strftime('%Y-%m-%d'))
                else:
                    raise NotImplementedError('unsupported format %s' % f)
            data.append(row)

        print(tabulate(data, headers='firstrow'))
1137562
def summarize_topics(f='lsi_topics.json', num_topics=1000, num_tokens=10, column_width=10, do_print=True, justify=True, data_path=DATA_PATH):
    """"""
    if isinstance(f, basestring):
        if os.path.sep not in f:
            f = os.path.expanduser(os.path.join(data_path, f))
        f = open(f, 'rUb')
    if isinstance(f, pd.DataFrame):
        f = list(np.array(f[f.columns[-1]]))
    elif isinstance(f, file):
        f = json.load(f)
    if isinstance(f, Mapping):
        f = [v for k, v in sorted(f.items())]
    topics = list(f)
    s = ''
    digits = num_digits(min(len(topics), num_topics) - 1)
    for i, t in enumerate(topics):
        if i > num_topics:
            break
        t_sorted = sorted(t.items(), key=lambda x: -abs(x[1]))[:num_tokens]
        line = '{:{}d}: {}'.format(i, digits, ' '.join(('-+'[int(v > 0)] + '{:{}s}'.format(k[:column_width], column_width) for (k, v) in t_sorted)))
        if not justify:
            line = ' '.join([col for col in line.split(' \t') if col])
        s += line + '\n'
    if do_print:
        print(s)
    return s.split('\n')[:-1]
271346
def merge(self, frame):
        
        for column_name, _ in self._column_stats.items():
            data_arr = frame[[column_name]].values
            count, min_max_tup, mean, _, _, _ = \
                scistats.describe(data_arr)
            stats_counter = StatCounter()
            stats_counter.n = count
            stats_counter.mu = mean
            stats_counter.m2 = np.sum((data_arr - mean) ** 2)
            stats_counter.minValue, stats_counter.maxValue = min_max_tup
            self._column_stats[column_name] = self._column_stats[
                column_name].mergeStats(stats_counter)
        return self
151877
def main(self):
        
        self.dataframe_setup()
        self.figure_populate(self.outputfolder,
                             self.image_report,
                             self.header_list,
                             self.samples,
                             'genesippr',
                             'report',
                             fail=self.fail)