Create a report combining results from resfinder_assembled and mob_recon_summary reports
548651
def amrsummary(self):
        
        logging.info('Creating AMR summary table from ResFinder and MOB-recon outputs')
        with open(os.path.join(self.reportpath, 'amr_summary.csv'), 'w') as amr:
            data = 'Strain,Gene,Allele,Resistance,PercentIdentity,Contig,Location,PlasmidIncompatibilitySets\n'
            for sample in self.metadata:
                # Initialise a dictionary to store a set of all the incompatibility types listed for a contig.
                # As the inc type will only be located on one of possibly several contigs associated with a predicted
                # plasmid, it is nice to know details about the plasmid
                inc_dict = dict()
                for primarykey, results in sample[self.analysistype].report_dict.items():
                    try:
                        inc = results['cluster_id']
                        # Convert the rep_type field (predicted incompatibilities) into a more a consistent
                        # format - pandas will call empty fields 'nan', which is a float
                        rep = str(results['rep_type']).replace(',', ';') if str(results['rep_type']) != 'nan' else 'ND'
                        # Add the incompatibility to the set
                        try:
                            inc_dict[inc].add(rep)
                        except KeyError:
                            inc_dict[inc] = set()
                            inc_dict[inc].add(rep)
                    except KeyError:
                        pass
                #
                for primarykey, results in sample[self.analysistype].report_dict.items():
                    try:
                        contig = results['contig_id'].split('|')[1]
                        # Unicycler gives contigs names such as: 3_length=187116_depth=1.60x_circular=true - test
                        # to see if the contig name looks unicycler-like, and set the name appropriately (in this
                        # case, it would be 3)
                        if contig.split('_')[1].startswith('length'):
                            contig = contig.split('_')[0]
                        # Use the list of results from the resfinder analyses
                        for amr_result in sample.resfinder_assembled.sampledata:
                            # Ensure that the current contig is the same as the one in the resfinder results. Ensure
                            # that the slice of the amr result is treated as a string. Unicycler contigs seem to be
                            # treated as integers
                            if contig == str(amr_result[-1]):
                                # Set up the output string
                                data += '{sn},'.format(sn=sample.name)
                                # Add the resistance and MOB recon outputs for the strain
                                data += '{amr},{mob}\n'\
                                    .format(amr=','.join(str(res) if str(res) != 'nan' else 'ND' for res in
                                                         amr_result[0:4]),
                                            mob=','.join(str(res) if str(res) != 'nan' else 'ND' for res in
                                                         [contig, results['cluster_id'],
                                                          ';'.join(sorted(inc_dict[str(results['cluster_id'])]))
                                                          ]
                                                         )
                                            )
                    except KeyError:
                        pass
            amr.write(data)
230217
def _generate_report(ret, setup):
    '''
    
    '''

    retdata = ret.get('return', {})

    sorted_data = sorted(
        retdata.items(),
        key=lambda s: s[1].get('__run_num__', 0)
    )

    total = 0
    failed = 0
    changed = 0
    duration = 0.0

    # gather stats
    for _, data in sorted_data:
        if not data.get('result', True):
            failed += 1
        total += 1

        try:
            duration += float(data.get('duration', 0.0))
        except ValueError:
            pass

        if data.get('changes', {}):
            changed += 1

    unchanged = total - failed - changed

    log.debug('highstate total: %s', total)
    log.debug('highstate failed: %s', failed)
    log.debug('highstate unchanged: %s', unchanged)
    log.debug('highstate changed: %s', changed)

    # generate report if required
    if setup.get('report_everything', False) or \
       (setup.get('report_changes', True) and changed != 0) or \
       (setup.get('report_failures', True) and failed != 0):

        report = [
            {'stats': [
                {'total': total},
                {'failed': failed, '__style__': 'failed'},
                {'unchanged': unchanged, '__style__': 'unchanged'},
                {'changed': changed, '__style__': 'changed'},
                {'duration': duration}
            ]},
            {'job': [
                {'function': ret.get('fun', '')},
                {'arguments': ret.get('fun_args', '')},
                {'jid': ret.get('jid', '')},
                {'success': ret.get('success', True)},
                {'retcode': ret.get('retcode', 0)}
            ]},
            {'states': _generate_states_report(sorted_data)}
        ]

        if failed:
            function = setup.get('failure_function', None)
        else:
            function = setup.get('success_function', None)

        if function:
            func_result = __salt__[function]()
            report.insert(
                0,
                {'extra': [{function: _dict_to_name_value(func_result)}]}
            )

    else:
        report = []

    return report, failed
558665
def create_report(self):
        """"""
        data = self.create_data()
        try:
            json_string = json.dumps(data)
        except UnicodeDecodeError as e:
            log.error('ERROR: While preparing JSON:', exc_info=e)
            self.debug_bad_encoding(data)
            raise

        log_string = re.sub(r'"repo_token": "(.+?)"',
                            '"repo_token": "[secure]"', json_string)
        log.debug(log_string)
        log.debug('==\nReporting %s files\n==\n', len(data['source_files']))
        for source_file in data['source_files']:
            log.debug('%s - %s/%s', source_file['name'],
                      sum(filter(None, source_file['coverage'])),
                      len(source_file['coverage']))
        return json_string
17781
def _produce_output(report, failed, setup):
    '''
    
    '''
    report_format = setup.get('report_format', 'yaml')

    log.debug('highstate output format: %s', report_format)

    if report_format == 'json':
        report_text = salt.utils.json.dumps(report)
    elif report_format == 'yaml':
        string_file = StringIO()
        salt.utils.yaml.safe_dump(report, string_file, default_flow_style=False)
        string_file.seek(0)
        report_text = string_file.read()
    else:
        string_file = StringIO()
        _generate_html(report, string_file)
        string_file.seek(0)
        report_text = string_file.read()

    report_delivery = setup.get('report_delivery', 'file')

    log.debug('highstate report_delivery: %s', report_delivery)

    if report_delivery == 'file':
        output_file = _sprinkle(setup.get('file_output', '/tmp/test.rpt'))
        with salt.utils.files.fopen(output_file, 'w') as out:
            out.write(salt.utils.stringutils.to_str(report_text))
    else:
        msg = MIMEText(report_text, report_format)

        sender = setup.get('smtp_sender', '')
        recipients = setup.get('smtp_recipients', '')

        host = setup.get('smtp_server', '')
        port = int(setup.get('smtp_port', 25))
        tls = setup.get('smtp_tls')
        username = setup.get('smtp_username')
        password = setup.get('smtp_password')

        if failed:
            subject = setup.get('smtp_failure_subject', 'Installation failure')
        else:
            subject = setup.get('smtp_success_subject', 'Installation success')

        subject = _sprinkle(subject)

        msg['Subject'] = subject
        msg['From'] = sender
        msg['To'] = recipients

        log.debug('highstate smtp port: %d', port)
        smtp = smtplib.SMTP(host=host, port=port)

        if tls is True:
            smtp.starttls()
            log.debug('highstate smtp tls enabled')

        if username and password:
            smtp.login(username, password)
            log.debug('highstate smtp authenticated')

        smtp.sendmail(
            sender,
            [x.strip() for x in recipients.split(',')], msg.as_string())
        log.debug('highstate message sent.')

        smtp.quit()
694534
def _merge_metrics(samples, out_dir):
    
    logger.info("summarize metrics")
    out_dir = utils.safe_makedir(os.path.join(out_dir, "report", "metrics"))
    sample_metrics = collections.defaultdict(dict)
    for s in samples:
        s = _add_disambiguate(s)
        m = tz.get_in(['summary', 'metrics'], s)
        if isinstance(m, six.string_types):
            m = json.loads(m)
        if m:
            for me in m.keys():
                if isinstance(m[me], list) or isinstance(m[me], dict) or isinstance(m[me], tuple):
                    m.pop(me, None)
            sample_metrics[dd.get_sample_name(s)].update(m)
    out = []
    for sample_name, m in sample_metrics.items():
        sample_file = os.path.join(out_dir, "%s_bcbio.txt" % sample_name)
        with file_transaction(samples[0], sample_file) as tx_out_file:
            dt = pd.DataFrame(m, index=['1'])
            dt.columns = [k.replace(" ", "_").replace("(", "").replace(")", "") for k in dt.columns]
            dt['sample'] = sample_name
            dt['rRNA_rate'] = m.get('rRNA_rate', "NA")
            dt['RiP_pct'] = "%.3f" % (int(m.get("RiP", 0)) / float(m.get("Total_reads", 1)) * 100)
            dt = _fix_duplicated_rate(dt)
            dt.transpose().to_csv(tx_out_file, sep="\t", header=False)
        out.append(sample_file)
    return out
275496
def to_h
      results = {}
      defaults = @config["default_reports"] || {}

      @config["repositories"].map { |repo_config|
        repo = repo_config["repository"]
        config = config_for(repo)

        config["reports"].map { |format, report_config|
          # Sometimes report_data has unused keys, which generates a warning, but
          # we're okay with it, so we wrap it with silence_warnings {}.
          filename = silence_warnings {
            tmp_filename = report_config["filename"] || defaults[format]["filename"]
            tmp_filename % metadata(repo)
          }

          directory = report_config["directory"] || defaults[format]["directory"]
          file = File.join(directory, filename)

          # Export +report+ to the specified +format+ with the specified
          # +frontmatter+.
          frontmatter = report_config["frontmatter"] || {}
          if defaults.has_key?(format) && defaults[format].has_key?("frontmatter")
            frontmatter = defaults[format]["frontmatter"].merge(frontmatter)
          end
          frontmatter = nil if frontmatter == {}

          export = @reports[repo].send("to_#{format}", frontmatter)

          results[file] = export
        }
      }
      results
    end
559130
def split_results(results):
    """"""
    from dask.delayed import Delayed

    def flatten(results):
        out = []
        if isinstance(results, (list, tuple)):
            for itm in results:
                out.extend(flatten(itm))
            return out
        return [results]

    sources = []
    targets = []
    delayeds = []

    for res in flatten(results):
        if isinstance(res, da.Array):
            sources.append(res)
        elif isinstance(res, Delayed):
            delayeds.append(res)
        else:
            targets.append(res)
    return sources, targets, delayeds
273661
def _modify_report(summary_path, out_dir):
    """"""
    summary_path = op.abspath(summary_path)
    template = op.normpath(op.join(op.dirname(op.realpath(template_seqcluster.__file__)), "report.rmd"))
    content = open(template).read()
    out_content = string.Template(content).safe_substitute({'path_abs': summary_path})
    out_file = op.join(out_dir, "srna_report.rmd")
    with open(out_file, 'w') as out_handle:
        out_handle.write(out_content)
    return out_file
881536
def resfinderreporter(self):
        
        # Initialise resistance dictionaries from the notes.txt file
        resistance_classes = ResistanceNotes.classes(self.targetpath)
        # Create a workbook to store the report. Using xlsxwriter rather than a simple csv format, as I want to be
        # able to have appropriately sized, multi-line cells
        workbook = xlsxwriter.Workbook(os.path.join(self.reportpath, '{}.xlsx'.format(self.analysistype)))
        # New worksheet to store the data
        worksheet = workbook.add_worksheet()
        # Add a bold format for header cells. Using a monotype font size 10
        bold = workbook.add_format({'bold': True, 'font_name': 'Courier New', 'font_size': 8})
        # Format for data cells. Monotype, size 10, top vertically justified
        courier = workbook.add_format({'font_name': 'Courier New', 'font_size': 8})
        courier.set_align('top')
        # Initialise the position within the worksheet to be (0,0)
        row = 0
        col = 0
        # A dictionary to store the column widths for every header
        columnwidth = dict()
        extended = False
        headers = ['Strain', 'Gene', 'Allele', 'Resistance', 'PercentIdentity', 'PercentCovered', 'Contig', 'Location',
                   'nt_sequence']
        for sample in self.metadata:
            # Create an attribute to store the string for the eventual pipeline report
            sample[self.analysistype].pipelineresults = list()
            sample[self.analysistype].sampledata = list()
            try:
                blastresults = sample[self.analysistype].blastresults
            except AttributeError:
                blastresults = 'NA'
            # Process the sample only if the script could find targets
            if blastresults != 'NA':
                for result in sample[self.analysistype].blastresults:
                    # Set the name to avoid writing out the dictionary[key] multiple times
                    name = result['subject_id']
                    # Use the ResistanceNotes gene name extraction method to get the necessary variables
                    gname, genename, accession, allele = ResistanceNotes.gene_name(name)
                    # Initialise a list to store all the data for each strain
                    data = list()
                    # Determine resistance phenotype of the gene
                    resistance = ResistanceNotes.resistance(name, resistance_classes)
                    # Append the necessary values to the data list
                    data.append(genename)
                    data.append(allele)
                    data.append(resistance)
                    percentid = result['percentidentity']
                    data.append(percentid)
                    data.append(result['alignment_fraction'])
                    data.append(result['query_id'])
                    data.append('...'.join([str(result['low']), str(result['high'])]))
                    try:
                        # Populate the attribute storing the resfinder results
                        sample[self.analysistype].pipelineresults.append(
                            '{rgene} ({pid}%) {rclass}'.format(rgene=genename,
                                                               pid=percentid,
                                                               rclass=resistance))
                        # Only if the alignment option is selected, for inexact results, add alignments
                        if self.align and percentid != 100.00:

                            # Align the protein (and nucleotide) sequences to the reference
                            self.alignprotein(sample, name)
                            if not extended:
                                # Add the appropriate headers
                                headers.extend(['aa_Identity',
                                                'aa_Alignment',
                                                'aa_SNP_location',
                                                'nt_Alignment',
                                                'nt_SNP_location'
                                                ])
                                extended = True
                            # Create a FASTA-formatted sequence output of the query sequence
                            record = SeqRecord(sample[self.analysistype].dnaseq[name],
                                               id='{}_{}'.format(sample.name, name),
                                               description='')
                            # Add the alignment, and the location of mismatches for both nucleotide and amino
                            # acid sequences
                            data.extend([record.format('fasta'),
                                         sample[self.analysistype].aaidentity[name],
                                         sample[self.analysistype].aaalign[name],
                                         sample[self.analysistype].aaindex[name],
                                         sample[self.analysistype].ntalign[name],
                                         sample[self.analysistype].ntindex[name]
                                         ])
                        else:
                            record = SeqRecord(Seq(result['subject_sequence'], IUPAC.unambiguous_dna),
                                               id='{}_{}'.format(sample.name, name),
                                               description='')
                            data.append(record.format('fasta'))
                            if self.align:
                                # Add '-'s for the empty results, as there are no alignments for exact matches
                                data.extend(['100', '-', '-', '-', '-'])
                    # If there are no blast results for the target, add a '-'
                    except (KeyError, TypeError):
                        data.append('-')
                    sample[self.analysistype].sampledata.append(data)

        if 'nt_sequence' not in headers:
            headers.append('nt_sequence')
        # Write the header to the spreadsheet
        for header in headers:
            worksheet.write(row, col, header, bold)
            # Set the column width based on the longest header
            try:
                columnwidth[col] = len(header) if len(header) > columnwidth[col] else columnwidth[
                    col]
            except KeyError:
                columnwidth[col] = len(header)
            worksheet.set_column(col, col, columnwidth[col])
            col += 1
        # Increment the row and reset the column to zero in preparation of writing results
        row += 1
        col = 0
        # Write out the data to the spreadsheet
        for sample in self.metadata:
            if not sample[self.analysistype].sampledata:
                # Increment the row and reset the column to zero in preparation of writing results
                row += 1
                col = 0
                # Set the width of the row to be the number of lines (number of newline characters) * 12
                worksheet.set_row(row)
                worksheet.set_column(col, col, columnwidth[col])
            for data in sample[self.analysistype].sampledata:
                columnwidth[col] = len(sample.name) + 2
                worksheet.set_column(col, col, columnwidth[col])
                worksheet.write(row, col, sample.name, courier)
                col += 1
                # List of the number of lines for each result
                totallines = list()
                for results in data:
                    #
                    worksheet.write(row, col, results, courier)
                    try:
                        # Counting the length of multi-line strings yields columns that are far too wide, only count
                        # the length of the string up to the first line break
                        alignmentcorrect = len(str(results).split('\n')[1])
                        # Count the number of lines for the data
                        lines = results.count('\n') if results.count('\n') >= 1 else 1
                        # Add the number of lines to the list
                        totallines.append(lines)
                    except IndexError:
                        try:
                            # Counting the length of multi-line strings yields columns that are far too wide, only count
                            # the length of the string up to the first line break
                            alignmentcorrect = len(str(results).split('\n')[0])
                            # Count the number of lines for the data
                            lines = results.count('\n') if results.count('\n') >= 1 else 1
                            # Add the number of lines to the list
                            totallines.append(lines)
                        # If there are no newline characters, set the width to the length of the string
                        except AttributeError:
                            alignmentcorrect = len(str(results))
                            lines = 1
                            # Add the number of lines to the list
                            totallines.append(lines)
                    # Increase the width of the current column, if necessary
                    try:
                        columnwidth[col] = alignmentcorrect if alignmentcorrect > columnwidth[col] else \
                            columnwidth[col]
                    except KeyError:
                        columnwidth[col] = alignmentcorrect
                    worksheet.set_column(col, col, columnwidth[col])
                    col += 1
                # Set the width of the row to be the number of lines (number of newline characters) * 12
                worksheet.set_row(row, max(totallines) * 11)
                # Increase the row counter for the next strain's data
                row += 1
                col = 0
        # Close the workbook
        workbook.close()
202723
function reportReducer (summary, report) {
  summary.errors += report.error.count
  summary.warnings += report.warn.count
  return summary
}
1016947
def generate_reports(subject_list, output_dir, work_dir, run_uuid, config=None,
                     packagename=None):
    
    reports_dir = str(Path(work_dir) / 'reportlets')
    report_errors = [
        run_reports(reports_dir, output_dir, subject_label, run_uuid,
                    config, packagename=packagename)
        for subject_label in subject_list
    ]

    errno = sum(report_errors)
    if errno:
        import logging
        logger = logging.getLogger('cli')
        error_list = ', '.join('%s (%d)' % (subid, err)
                               for subid, err in zip(subject_list, report_errors) if err)
        logger.error(
            'Preprocessing did not finish successfully. Errors occurred while processing '
            'data from participants: %s. Check the HTML reports for details.', error_list)
    return errno
522893
private void createReportMap() {
        logger.entering();

        // Summary Report
        Map<String, List<SummarizedData>> subReportMap = new LinkedHashMap<String, List<SummarizedData>>();
        subReportMap.put("Full Suite Summary", lSuites);
        subReportMap.put("Test Summary", lTests);
        subReportMap.put("Classwise Summary", lClasses);
        subReportMap.put("Groupwise Summary", lGroups);

        ReportMap<SummarizedData> testSummaryReport = new ReportMap<SummarizedData>(
                ReportSheetNames.TESTSUMMARYREPORT.getName(), subReportMap, 0);
        fullReportMap.add(testSummaryReport);

        // Group Detailed Report
        List<SummarizedData> groupsClone = new ArrayList<SummarizedData>(lGroups);
        List<SummarizedData> classData;

        SummarizedData naGroupData = new SummarizedData();
        naGroupData.setsName(TestCaseResult.NA);
        groupsClone.add(naGroupData);
        subReportMap = new LinkedHashMap<String, List<SummarizedData>>();
        for (SummarizedData group : groupsClone) {

            String sGroupName = group.getsName();
            classData = new ArrayList<SummarizedData>();
            for (String sGroupClassName : mpGroupClassData.keySet()) {
                if (sGroupClassName.substring(0, sGroupName.length()).equals(sGroupName)) {
                    classData.add(mpGroupClassData.get(sGroupClassName));
                }
            }
            subReportMap.put(sGroupName, classData);
        }

        ReportMap<SummarizedData> secondReport = new ReportMap<SummarizedData>(
                ReportSheetNames.GROUPSUMMARYREPORT.getName(), subReportMap, 0);
        fullReportMap.add(secondReport);

        // TestCase Status Report
        Map<String, List<List<String>>> subDetailReportMap = new LinkedHashMap<String, List<List<String>>>();
        subDetailReportMap.put("Passed TC List", tcPassedData);
        subDetailReportMap.put("Failed TC List", tcFailedData);
        subDetailReportMap.put("Skipped TC List", tcSkippedData);

        ReportMap<List<String>> thirdReport = new ReportMap<List<String>>(ReportSheetNames.TESTCASEREPORT.getName(),
                subDetailReportMap, 1);
        fullReportMap.add(thirdReport);

        // Defect Report
        Map<String, List<List<String>>> lstDefectReports = new LinkedHashMap<String, List<List<String>>>();
        lstDefectReports.put("Defect Summary", tcDefectData);
        ReportMap<List<String>> fourthReport = new ReportMap<List<String>>(ReportSheetNames.DEFECTREPORT.getName(),
                lstDefectReports, 1);
        fullReportMap.add(fourthReport);

        // Changing the titles of the Defect Report
        BaseReport<List<String>> bR = (BaseReport<List<String>>) fullReportMap.get(fullReportMap.size() - 1)
                .getGeneratedReport().iterator().next();
        List<String> lsTitles = Arrays.asList("Class Name", "Method/Testcase id", "Test Description",
                "Group[s]", "Time taken", "Output Logs", "Error Message", "Error Details");
        bR.setColTitles(lsTitles);

        // TestCase Output Details Report
        Map<String, List<List<String>>> fifthTestOutputSubReportMap = new LinkedHashMap<String, List<List<String>>>();
        fifthTestOutputSubReportMap.put("Test Output", tcOutputData);

        ReportMap<List<String>> fifthReportSheet = new ReportMap<List<String>>(
                ReportSheetNames.TESTOUTPUTDETAILSREPORT.getName(),
                fifthTestOutputSubReportMap, 2);
        fullReportMap.add(fifthReportSheet);
        logger.exiting();
    }
163506
def create_result(self, local_path, container_path, permissions, meta, val, dividers):
        """"""
        if permissions is NotSpecified:
            permissions = 'rw'
        return Mount(local_path, container_path, permissions)
1362575
def create_results_dirs(base_path):
    

    if not os.path.exists(base_path):
        print("Creating directory {} for results data.".format(base_path))
        os.mkdir(base_path)
    if not os.path.exists(os.path.join(base_path, 'results')):
        os.mkdir(os.path.join(base_path, 'results'))
    if not os.path.exists(os.path.join(base_path, 'plots')):
        os.mkdir(os.path.join(base_path, 'plots'))
    if not os.path.exists(os.path.join(base_path, 'info')):
        os.mkdir(os.path.join(base_path, 'info'))
    if not os.path.exists(os.path.join(base_path, 'log')):
        os.mkdir(os.path.join(base_path, 'log'))
336760
def build_report(self, msg=''):
		
		shutit_global.shutit_global_object.yield_to_draw()
		s = '\n'
		s += '################################################################################\n'
		s += '# COMMAND HISTORY BEGIN ' + shutit_global.shutit_global_object.build_id + '\n'
		s += self.get_commands()
		s += '# COMMAND HISTORY END ' + shutit_global.shutit_global_object.build_id + '\n'
		s += '################################################################################\n'
		s += '################################################################################\n'
		s += '# BUILD REPORT FOR BUILD BEGIN ' + shutit_global.shutit_global_object.build_id + '\n'
		s += '# ' + msg + '\n'
		if self.build['report'] != '':
			s += self.build['report'] + '\n'
		else:
			s += '# Nothing to report\n'
		if 'container_id' in self.target:
			s += '# CONTAINER_ID: ' + self.target['container_id'] + '\n'
		s += '# BUILD REPORT FOR BUILD END ' + shutit_global.shutit_global_object.build_id + '\n'
		s += '###############################################################################\n'
		s += '# INVOKING COMMAND WAS: ' + sys.executable
		for arg in sys.argv:
			s += ' ' + arg
		s += '\n'
		s += '###############################################################################\n'
		return s
1278159
def combine_coverage_reports(self, omit, parallel):
        
        tmp_cov = coverage.coverage(omit=omit, data_suffix=parallel)
        tmp_cov.load()
        tmp_cov.combine()
        tmp_cov.save()
1138298
def _add_combine_info(output, combine_map, file_key):
    
    files_per_output = collections.defaultdict(list)
    for part_file, out_file in combine_map.items():
        files_per_output[out_file].append(part_file)
    out_by_file = collections.defaultdict(list)
    out = []
    for data in output:
        # Do not pass along nochrom, noanalysis regions
        if data["region"][0] not in ["nochrom", "noanalysis"]:
            cur_file = data[file_key]
            # If we didn't process, no need to add combine information
            if cur_file in combine_map:
                out_file = combine_map[cur_file]
                if "combine" not in data:
                    data["combine"] = {}
                data["combine"][file_key] = {"out": out_file,
                                             "extras": files_per_output.get(out_file, [])}
                out_by_file[out_file].append(data)
            elif cur_file:
                out_by_file[cur_file].append(data)
            else:
                out.append([data])
    for samples in out_by_file.values():
        regions = [x["region"] for x in samples]
        region_bams = [x["work_bam"] for x in samples]
        assert len(regions) == len(region_bams)
        if len(set(region_bams)) == 1:
            region_bams = [region_bams[0]]
        data = samples[0]
        data["region_bams"] = region_bams
        data["region"] = regions
        data = dd.set_mark_duplicates(data, data["config"]["algorithm"]["orig_markduplicates"])
        del data["config"]["algorithm"]["orig_markduplicates"]
        out.append([data])
    return out