helper for gap encoding.
540109
private static byte[] bitSetToByteArray(BitSet bitSet) {

    while (bitSet.length() % 8 != 0) { bitSet.set(bitSet.length(), true); }
    
    byte[] array = new byte[bitSet.length()/8];

    for (int i = 0; i < array.length; i++) {
      int offset = i * 8;

      int index = 0;
      for (int j = 0; j < 8; j++) {
        index <<= 1;        
        if (bitSet.get(offset+j)) { index++; }
      }

      array[i] = (byte)(index - 128);
    }

    return array;
  }
138611
def normalize_encoding(encoding, default=DEFAULT_ENCODING):
    """"""
    if encoding is None:
        return default
    encoding = encoding.lower().strip()
    if encoding in ['', 'ascii']:
        return default
    try:
        codecs.lookup(encoding)
        return encoding
    except LookupError:
        return default
733002
def tokenize(contents):
    """"""
    tokens = _scan_for_tokens(contents)
    tokens = _compress_tokens(tokens)
    tokens = [token for token in tokens if token.type != TokenType.Whitespace]
    return tokens
949795
def safeunicode(arg, *args, **kwargs):
	""""""
	return arg if isinstance(arg, unicode) else unicode(arg, *args, **kwargs)
59905
def unicode2ascii(text, expand=True):
    
    """
    translate = UTF8_TO_ASCII if not expand else UTF8_TO_MULTIASCII
    output = ''
    for c in text:
        if not c or ord(c) < 128:
            output += c
        else:
            output += translate[c] if c in translate else ' '
    return output.strip()
694678
def dict_encode(in_dict):
    """"""
    if _IS_PY2:
        out_dict = {}
        for k, v in list(in_dict.items()):
            if isinstance(v, unicode):
                v = v.encode('utf8')
            elif isinstance(v, str):
                # Must be encoded in UTF-8
                v.decode('utf8')
            out_dict[k] = v
        return out_dict
    else:
        raise NotImplementedError
832302
def bdecode(text):
    """"""
    text = text.decode('utf-8')

    def bdecode_next(start):
        """bdecode helper function"""
        if text[start] == 'i':
            end = text.find('e', start)
            return int(text[start+1:end], 10), end + 1

        if text[start] == 'l':
            res = []
            start += 1
            while text[start] != 'e':
                elem, start = bdecode_next(start)
                res.append(elem)
            return res, start + 1

        if text[start] == 'd':
            res = {}
            start += 1
            while text[start] != 'e':
                key, start = bdecode_next(start)
                value, start = bdecode_next(start)
                res[key] = value
            return res, start + 1

        lenend = text.find(':', start)
        length = int(text[start:lenend], 10)
        end = lenend + length + 1
        return text[lenend+1:end], end
    return bdecode_next(0)[0]
276466
def encode_simple(d):
    """"""
    if isinstance(d, unicode):
        return d.encode()
    if isinstance(d, list):
        return list(map(encode_simple, d))
    if isinstance(d, dict):
        return dict([(encode_simple(k), encode_simple(v)) for k, v in d.items()])
    return d
692492
def guess_file_encoding(fh, default=DEFAULT_ENCODING):
    """"""
    start = fh.tell()
    detector = chardet.UniversalDetector()
    while True:
        data = fh.read(1024 * 10)
        if not data:
            detector.close()
            break
        detector.feed(data)
        if detector.done:
            break

    fh.seek(start)
    return normalize_result(detector.result, default=default)
270497
def _run_strip_accents(self, text):
        """"""
        text = unicodedata.normalize("NFD", text)
        output = []
        for char in text:
            cat = unicodedata.category(char)
            if cat == "Mn":
                continue
            output.append(char)
        return "".join(output)
539511
def iter_text_fixups(data, encoding):
    """"""

    yield data
    if encoding == Encoding.UTF16BE:
        # wrong termination
        yield data + b"\x00"
    elif encoding == Encoding.UTF16:
        # wrong termination
        yield data + b"\x00"
        # utf-16 is missing BOM, content is usually utf-16-le
        yield codecs.BOM_UTF16_LE + data
        # both cases combined
        yield codecs.BOM_UTF16_LE + data + b"\x00"
908931
def gapsplit(args):
    
    p = OptionParser(gapsplit.__doc__)

    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    gffile, = args

    gff = Gff(gffile)
    for g in gff:
        if re.match("EST_match", g.type):
            """
            hacky implementation:
            since the standard urlparse.parse_qsl() replaces all "+" symbols with spaces
            we will write a regex to check either for a "-" or a " " (space)
            """
            match = re.search(r'\S+ (\d+) \d+ ([\s{1}\-])', g.attributes["Target"][0])
            if match.group(2) == "-":
                strand = match.group(2)
            else:
                strand = "+"
                g.attributes["Target"][0] = " ".join(str(x) \
                        for x in [g.attributes["Target"][0].rstrip(), strand])

            if g.strand == "?":
                g.strand = strand
        else:
            match = re.match(r'\S+ (\d+) \d+', g.attributes["Target"][0])
        target_start = int(match.group(1))

        re_cigar = re.compile(r'(\D+)(\d+)');
        cigar = g.attributes["Gap"][0].split(" ")
        g.attributes["Gap"] = None

        parts = []
        if g.strand == "+":
            for event in cigar:
                match = re_cigar.match(event)
                op, count = match.group(1), int(match.group(2))
                if op in "IHS":
                    target_start += count
                elif op in "DN":
                    g.start += count
                elif op == "P":
                    continue
                else:
                    parts.append([g.start, g.start + count - 1, \
                            target_start, target_start + count - 1])
                    g.start += count
                    target_start += count
        else:
            for event in cigar:
                match = re_cigar.match(event)
                op, count = match.group(1), int(match.group(2))
                if op in "IHS":
                    target_start += count
                elif op in "DN":
                    g.end -= count
                elif op == "P":
                    continue
                else:
                    parts.append([g.end - count + 1, g.end, \
                            target_start, target_start + count - 1])
                    g.end -= count
                    target_start += count

        g.update_attributes()
        print(g)

        parent = g.attributes["Name"][0]
        g.type = "match_part"
        g.attributes.clear()

        for part in parts:
            g.start, g.end = part[0], part[1]
            g.score, g.strand, g.phase = ".", g.strand, "."

            if re.match("EST", g.type):
                target_list = [parent, part[2], part[3], g.strand]
            else:
                target_list = [parent, part[2], part[3]]
            target = " ".join(str(x) for x in target_list)

            g.attributes["Parent"] = [parent]
            g.attributes["Target"] = [target]

            g.update_attributes()
            print(g)
1245419
function deepSplitString (input, separators) {
  if (input === null || typeof input === 'undefined') {
    return input
  }
  if (separators.length === 0) {
    return input
  }

  const sep = separators.shift()
  let output = input

  if (typeof input === 'string') {
    output = splitString(input, sep)
  } else if (Array.isArray(input)) {
    output = input.map(i => splitString(i, sep))
  } else if (typeof input === 'object') {
    output = {}
    Object.keys(input).forEach(key => {
      output[key] = splitString(input[key], sep)
    })
  }

  if (separators.length > 0) {
    return deepSplitString(output, separators)
  } else {
    return output
  }
}
1263938
def force_ascii_values(data):
    """"""
    return {
        k: v.encode('utf8').decode('ascii', 'backslashreplace')
        for k, v in data.items()
    }
1163133
def clipstr(s, dispw):
    ''''''
    w = 0
    ret = ''
    ambig_width = options.disp_ambig_width
    for c in s:
        if c != ' ' and unicodedata.category(c) in ('Cc', 'Zs', 'Zl'):  # control char, space, line sep
            c = options.disp_oddspace

        if c:
            c = c[0]  # multi-char disp_oddspace just uses the first char
            ret += c
            eaw = unicodedata.east_asian_width(c)
            if eaw == 'A':  # ambiguous
                w += ambig_width
            elif eaw in 'WF': # wide/full
                w += 2
            elif not unicodedata.combining(c):
                w += 1

        if w > dispw-len(options.disp_truncator)+1:
            ret = ret[:-2] + options.disp_truncator  # replace final char with ellipsis
            w += len(options.disp_truncator)
            break

    return ret, w
827853
def transcode(infile, outfile=None, incoding="shift-jis", outcoding="utf-8"):
    """"""
    if not outfile:
        outfile = os.path.basename(infile) + '.utf8'
    with codecs.open(infile, "rb", incoding) as fpin:
        with codecs.open(outfile, "wb", outcoding) as fpout:
            fpout.write(fpin.read())
446237
def bounds(filename, start_re, end_re, encoding='utf8'):
    
    start_re, end_re = re.compile(start_re), re.compile(end_re)
    mo, line_start, line_end, byte_start, byte_end = [None]*5
    offset = 0

    with open(filename, 'rb') as f:
        for index, line in enumerate(f):
            line_text = line.decode(encoding)
            start_match = re.match(start_re, line_text)
            if start_match:
                mo, line_start, byte_start = start_match, index, offset

            offset += len(line)

            end_match = re.match(end_re, line_text)
            if end_match:
                yield mo, Bounds(line_start, index, byte_start, offset)
                mo, line_start, line_end, byte_start, byte_end = [None]*5