replaces partition_all with a more robust version.
    Workaround for a segfault in pybedtools when using a BedTool as an iterator:
    https://github.com/daler/pybedtools/issues/88 for the discussion
333782
def robust_partition_all(n, iterable):
    
    it = iter(iterable)
    while True:
        x = []
        for _ in range(n):
            try:
                x.append(next(it))
            except StopIteration:
                yield x
                # Omitting this StopIteration results in a segfault!
                raise StopIteration
        yield x
542702
def concat(bed_files, catted=None):
    
    bed_files = [x for x in bed_files if x]
    if len(bed_files) == 0:
        if catted:
            # move to a .bed extension for downstream tools if not already
            sorted_bed = catted.sort()
            if not sorted_bed.fn.endswith(".bed"):
                return sorted_bed.moveto(sorted_bed.fn + ".bed")
            else:
                return sorted_bed
        else:
            return catted

    if not catted:
        bed_files = list(bed_files)
        catted = bt.BedTool(bed_files.pop())
    else:
        catted = catted.cat(bed_files.pop(), postmerge=False,
                            force_truncate=False)

    return concat(bed_files, catted)
1332559
def mergebydepth(args):
    
    p = OptionParser(mergebydepth.__doc__)
    p.add_option("--mindepth", default=3, type="int",
                 help="Minimum depth required")
    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    bedfile, fastafile = args
    mindepth = opts.mindepth
    bedgraph = make_bedgraph(bedfile)

    bedgraphfiltered = bedgraph + ".d{0}".format(mindepth)
    if need_update(bedgraph, bedgraphfiltered):
        filter([bedgraph, "--minaccn={0}".format(mindepth),
                "--outfile={0}".format(bedgraphfiltered)])

    merged = bedgraphfiltered + ".merge.fasta"
    if need_update(bedgraphfiltered, merged):
        mergeBed(bedgraphfiltered, sorted=True)
606355
def _PartitionChunks(chunks):
  """"""
  partitions = [[]]
  partition_size = 0

  for chunk in chunks:
    cursize = len(chunk["blob_chunk"])
    if (cursize + partition_size > BLOB_CHUNK_SIZE or
        len(partitions[-1]) >= CHUNKS_PER_INSERT):
      partitions.append([])
      partition_size = 0
    partitions[-1].append(chunk)
    partition_size += cursize

  return partitions
629415
def get_split_regions(bed_file, data):
    
    out_file = "%s-analysis_blocks.bed" % utils.splitext_plus(bed_file)[0]
    with shared.bedtools_tmpdir(data):
        if not utils.file_uptodate(out_file, bed_file):
            ref_regions = get_ref_bedtool(dd.get_ref_file(data), data["config"])
            nblock_regions = ref_regions.subtract(pybedtools.BedTool(bed_file)).saveas()
            min_n_size = int(tz.get_in(["config", "algorithm", "nomap_split_size"], data, 250))
            block_filter = NBlockRegionPicker(ref_regions, data["config"], min_n_size)
            final_nblock_regions = nblock_regions.filter(
                block_filter.include_block).saveas().each(block_filter.expand_block).saveas()
            with file_transaction(data, out_file) as tx_out_file:
                final_regions = ref_regions.subtract(final_nblock_regions, nonamecheck=True).\
                                saveas().merge(d=min_n_size).saveas(tx_out_file)
        chroms = set([])
        with shared.bedtools_tmpdir(data):
            for r in pybedtools.BedTool(bed_file):
                chroms.add(r.chrom)
        out = []
        for r in pybedtools.BedTool(out_file):
            if r.chrom in chroms:
                out.append((r.chrom, r.start, r.stop))
        return out
1209385
def frombed(args):
    
    from jcvi.formats.fasta import Fasta
    from jcvi.formats.bed import Bed
    from jcvi.utils.cbook import fill

    p = OptionParser(frombed.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 3:
        sys.exit(not p.print_help())

    bedfile, contigfasta, readfasta = args
    prefix = bedfile.rsplit(".", 1)[0]
    contigfile = prefix + ".contig"
    idsfile = prefix + ".ids"

    contigfasta = Fasta(contigfasta)
    readfasta = Fasta(readfasta)

    bed = Bed(bedfile)
    checksum = "00000000 checksum."
    fw_ids = open(idsfile, "w")
    fw = open(contigfile, "w")

    for ctg, reads in bed.sub_beds():
        ctgseq = contigfasta[ctg]
        ctgline = "##{0} {1} {2} bases, {3}".format(\
                ctg, len(reads), len(ctgseq), checksum)

        print(ctg, file=fw_ids)
        print(ctgline, file=fw)
        print(fill(ctgseq.seq), file=fw)

        for b in reads:
            read = b.accn
            strand = b.strand
            readseq = readfasta[read]
            rc = " [RC]" if strand == "-" else ""
            readlen = len(readseq)
            rstart, rend = 1, readlen
            if strand == "-":
                rstart, rend = rend, rstart

            readrange = "{{{0} {1}}}".format(rstart, rend)
            conrange = "<{0} {1}>".format(b.start, b.end)
            readline = "#{0}(0){1} {2} bases, {3} {4} {5}".format(\
                    read, rc, readlen, checksum, readrange, conrange)
            print(readline, file=fw)
            print(fill(readseq.seq), file=fw)

    logging.debug("Mapped contigs written to `{0}`.".format(contigfile))
    logging.debug("Contig IDs written to `{0}`.".format(idsfile))
595140
def __load_partition_entries(self, fd, bs):
        

        fd.seek(self.header.part_lba * bs)
        for p in range(0, self.header.num_partitions):
            data = fd.read(self.header.part_size)
            entry = GptPartitionEntry(data)
            if entry.type_guid != uuid.UUID(
                '{00000000-0000-0000-0000-000000000000}'
            ):
                self.__partition_entries.append(entry)
            else:
                # stop loading on empty partition entry
                break
395408
def pcall(stack_bottom, arg_count, bndng, filename, lineno)

      #err = Lib.lua_pcall(@pointer, 0, 1, 0)
        # When there's only 1 return value.
        # Use LUA_MULTRET (-1) the rest of the time

      err = Lib.lua_pcall(@pointer, arg_count, LUA_MULTRET, @error_handler)
      fail_if_error('eval:pcall', err, bndng, filename, lineno)

      return_result(stack_bottom)
    end
456313
def convert_bam_to_bed(in_bam, out_file):
    
    with file_transaction(out_file) as tx_out_file:
        with open(tx_out_file, "w") as out_handle:
            subprocess.check_call(["bamToBed", "-i", in_bam, "-tag", "NM"],
                                  stdout=out_handle)
    return out_file
559621
def _merge_sorted_items(self, index):
        
        def load_partition(j):
            path = self._get_spill_dir(j)
            p = os.path.join(path, str(index))
            with open(p, 'rb', 65536) as f:
                for v in self.serializer.load_stream(f):
                    yield v

        disk_items = [load_partition(j) for j in range(self.spills)]

        if self._sorted:
            # all the partitions are already sorted
            sorted_items = heapq.merge(disk_items, key=operator.itemgetter(0))

        else:
            # Flatten the combined values, so it will not consume huge
            # memory during merging sort.
            ser = self.flattened_serializer()
            sorter = ExternalSorter(self.memory_limit, ser)
            sorted_items = sorter.sorted(itertools.chain(*disk_items),
                                         key=operator.itemgetter(0))
        return ((k, vs) for k, vs in GroupByKey(sorted_items))
536145
def macro2micro(self, macro_indices):
        
        def from_partition(partition, macro_indices):
            micro_indices = itertools.chain.from_iterable(
                partition[i] for i in macro_indices)
            return tuple(sorted(micro_indices))

        if self.blackbox and self.coarse_grain:
            cg_micro_indices = from_partition(self.coarse_grain.partition,
                                              macro_indices)
            return from_partition(self.blackbox.partition,
                                  reindex(cg_micro_indices))
        elif self.blackbox:
            return from_partition(self.blackbox.partition, macro_indices)
        elif self.coarse_grain:
            return from_partition(self.coarse_grain.partition, macro_indices)
        return macro_indices
472423
def rebuild_token_map(self, partitioner, token_map):
        
        self.partitioner = partitioner
        if partitioner.endswith('RandomPartitioner'):
            token_class = MD5Token
        elif partitioner.endswith('Murmur3Partitioner'):
            token_class = Murmur3Token
        elif partitioner.endswith('ByteOrderedPartitioner'):
            token_class = BytesToken
        else:
            self.token_map = None
            return

        token_to_host_owner = {}
        ring = []
        for host, token_strings in six.iteritems(token_map):
            for token_string in token_strings:
                token = token_class.from_string(token_string)
                ring.append(token)
                token_to_host_owner[token] = host

        all_tokens = sorted(ring)
        self.token_map = TokenMap(
            token_class, token_to_host_owner, all_tokens, self)
876345
def tips(args):
    
    p = OptionParser(tips.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 4:
        sys.exit(not p.print_help())

    pbedfile, cbedfile, sizesfile, bbfasta = args

    pbed = Bed(pbedfile, sorted=False)
    cbed = Bed(cbedfile, sorted=False)

    complements = dict()
    for object, beds in groupby(cbed, key=lambda x: x.seqid):
        beds = list(beds)
        complements[object] = beds

    sizes = Sizes(sizesfile).mapping
    bbsizes = Sizes(bbfasta).mapping
    tbeds = []

    for object, beds in groupby(pbed, key=lambda x: x.accn):
        beds = list(beds)
        startbed, endbed = beds[0], beds[-1]
        start_id, end_id = startbed.seqid, endbed.seqid
        if startbed.start == 1:
            start_id = None
        if endbed.end == sizes[end_id]:
            end_id = None
        print(object, start_id, end_id, file=sys.stderr)
        if start_id:
            b = complements[start_id][0]
            b.accn = object
            tbeds.append(b)
        tbeds.append(BedLine("\t".join(str(x) for x in \
                        (object, 0, bbsizes[object], object, 1000, "+"))))
        if end_id:
            b = complements[end_id][-1]
            b.accn = object
            tbeds.append(b)

    tbed = Bed()
    tbed.extend(tbeds)

    tbedfile = "tips.bed"
    tbed.print_to_file(tbedfile)
1038403
def block_regions(callable_bed, in_bam, ref_file, data):
    
    min_n_size = int(data["config"]["algorithm"].get("nomap_split_size", 250))
    with shared.bedtools_tmpdir(data):
        nblock_bed = "%s-nblocks.bed" % utils.splitext_plus(callable_bed)[0]
        callblock_bed = "%s-callableblocks.bed" % utils.splitext_plus(callable_bed)[0]
        if not utils.file_uptodate(nblock_bed, callable_bed):
            ref_regions = get_ref_bedtool(ref_file, data["config"])
            nblock_regions = _get_nblock_regions(callable_bed, min_n_size, ref_regions)
            nblock_regions = _add_config_regions(nblock_regions, ref_regions, data)
            with file_transaction(data, nblock_bed, callblock_bed) as (tx_nblock_bed, tx_callblock_bed):
                nblock_regions.filter(lambda r: len(r) > min_n_size).saveas(tx_nblock_bed)
                if len(ref_regions.subtract(nblock_regions, nonamecheck=True)) > 0:
                    ref_regions.subtract(tx_nblock_bed, nonamecheck=True).merge(d=min_n_size).saveas(tx_callblock_bed)
                else:
                    raise ValueError("No callable regions found in %s from BAM file %s. Some causes:\n "
                                     " - Alignment regions do not overlap with regions found "
                                     "in your `variant_regions` BED: %s\n"
                                     " - There are no aligned reads in your BAM file that pass sanity checks "
                                     " (mapping score > 1, non-duplicates, both ends of paired reads mapped)"
                                     % (dd.get_sample_name(data), in_bam, dd.get_variant_regions(data)))
    return callblock_bed, nblock_bed
255575
def blackbox_and_coarse_grain(blackbox, coarse_grain):
    
    if blackbox is None:
        return

    for box in blackbox.partition:
        # Outputs of the box
        outputs = set(box) & set(blackbox.output_indices)

        if coarse_grain is None and len(outputs) > 1:
            raise ValueError(
                'A blackboxing with multiple outputs per box must be '
                'coarse-grained.')

        if (coarse_grain and not any(outputs.issubset(part)
                                     for part in coarse_grain.partition)):
            raise ValueError(
                'Multiple outputs from a blackbox must be partitioned into '
                'the same macro-element of the coarse-graining')
438206
def replace_grist (features, new_grist):
    
    assert is_iterable_typed(features, basestring) or isinstance(features, basestring)
    assert isinstance(new_grist, basestring)
    # this function is used a lot in the build phase and the original implementation
    # was extremely slow; thus some of the weird-looking optimizations for this function.
    single_item = False
    if isinstance(features, str):
        features = [features]
        single_item = True

    result = []
    for feature in features:
        # '<feature>value' -> ('<feature', '>', 'value')
        # 'something' -> ('something', '', '')
        # '<toolset>msvc/<feature>value' -> ('<toolset', '>', 'msvc/<feature>value')
        grist, split, value = feature.partition('>')
        # if a partition didn't occur, then grist is just 'something'
        # set the value to be the grist
        if not value and not split:
            value = grist
        result.append(new_grist + value)

    if single_item:
        return result[0]
    return result
1066176
def _segmentation_guts(root, file_paths, max_partition_size):
    
    # Canonicalize root to include the trailing slash, since root is
    # intended to be a directory anyway.
    if not root.endswith(os.path.sep):
        root += os.path.sep
    # Ensure that the root path is a directory before continuing.
    if not os.path.isdir(root):
        raise TarBadRootError(root=root)

    bogus_tar = None

    try:
        # Create a bogus TarFile as a contrivance to be able to run
        # gettarinfo and produce such instances.  Some of the settings
        # on the TarFile are important, like whether to de-reference
        # symlinks.
        bogus_tar = tarfile.TarFile(os.devnull, 'w', dereference=False)

        # Bookkeeping for segmentation of tar members into partitions.
        partition_number = 0
        partition_bytes = 0
        partition_members = 0
        partition = TarPartition(partition_number)

        for file_path in file_paths:

            # Ensure tar members exist within a shared root before
            # continuing.
            if not file_path.startswith(root):
                raise TarBadPathError(root=root, offensive_path=file_path)

            # Create an ExtendedTarInfo to represent the tarfile.
            try:
                et_info = ExtendedTarInfo(
                    tarinfo=bogus_tar.gettarinfo(
                        file_path, arcname=file_path[len(root):]),
                    submitted_path=file_path)

            except EnvironmentError as e:
                if (e.errno == errno.ENOENT and
                    e.filename == file_path):
                    # log a NOTICE/INFO that the file was unlinked.
                    # Ostensibly harmless (such unlinks should be replayed
                    # in the WAL) but good to know.
                    logger.debug(
                        msg='tar member additions skipping an unlinked file',
                        detail='Skipping {0}.'.format(et_info.submitted_path))
                    continue
                else:
                    raise

            # Ensure tar members are within an expected size before
            # continuing.
            if et_info.tarinfo.size > max_partition_size:
                raise TarMemberTooBigError(
                    et_info.tarinfo.name, max_partition_size,
                    et_info.tarinfo.size)

            if (partition_bytes + et_info.tarinfo.size >= max_partition_size
                or partition_members >= PARTITION_MAX_MEMBERS):
                # Partition is full and cannot accept another member,
                # so yield the complete one to the caller.
                yield partition

                # Prepare a fresh partition to accrue additional file
                # paths into.
                partition_number += 1
                partition_bytes = et_info.tarinfo.size
                partition_members = 1
                partition = TarPartition(
                    partition_number, [et_info])
            else:
                # Partition is able to accept this member, so just add
                # it and increment the size counters.
                partition_bytes += et_info.tarinfo.size
                partition_members += 1
                partition.append(et_info)

                # Partition size overflow must not to be possible
                # here.
                assert partition_bytes < max_partition_size

    finally:
        if bogus_tar is not None:
            bogus_tar.close()

    # Flush out the final partition should it be non-empty.
    if partition:
        yield partition