Read a file into a dictionary of bitsets
974996
def binned_bitsets_proximity( f, chrom_col=0, start_col=1, end_col=2, strand_col=5, upstream=0, downstream=0 ):
    """"""
    last_chrom = None
    last_bitset = None
    bitsets = dict()
    for line in f:
        if line.startswith("#"): continue
#        print "input=%s" % ( line ),
        fields = line.split()
        strand = "+"
        if len(fields) >= strand_col + 1:
            if fields[strand_col] == "-": strand = "-"
        chrom = fields[chrom_col]
        if chrom != last_chrom:
            if chrom not in bitsets:
                bitsets[chrom] = BinnedBitSet( MAX )
            last_chrom = chrom
            last_bitset = bitsets[chrom]
        start, end = int( fields[start_col] ), int( fields[end_col] )
        if strand == "+":
            if upstream: start = max( 0, start - upstream )
            if downstream: end = min( MAX, end + downstream )
        if strand == "-":
            if upstream: end = min( MAX, end + upstream )
            if downstream: start = max( 0, start - downstream )
#        print "set: start=%d\tend=%d" % ( start, end )
        if end-start > 0:
            last_bitset.set_range( start, end-start )
    return bitsets
1059947
def reads(s, filename=None, loader=None, implicit_tuple=True, allow_errors=False):
  """"""
  return ast.reads(s,
      filename=filename or '<input>',
      loader=loader or default_loader,
      implicit_tuple=implicit_tuple,
      allow_errors=allow_errors)
545742
def read_header(filename):
    '''  '''
    header = {}
    in_header = False
    data = nl.universal_read(filename)
    lines = [x.strip() for x in data.split('\n')]
    for line in lines:
        if line=="*** Header Start ***":
            in_header=True
            continue
        if line=="*** Header End ***":
            return header
        fields = line.split(": ")
        if len(fields)==2:
            header[fields[0]] = fields[1]
1341126
def _take_bits(buf, count):
    """"""

    # TODO: Verify output
    bytes_count = (count + 7) // 8
    bytes_mod = count % 8
    data = _unpack_from(buf, 'B', bytes_count)
    values = []
    for i, byte in enumerate(data):
        for _ in range(8 if i != bytes_count - 1 else bytes_mod):
            # TODO: Convert to True / False
            values.append(byte & 0b10000000)
            byte <<= 1
    return values
371656
def _read_dict(cp, dictionary):
    '''
    
    '''
    for section, keys in dictionary.items():
        section = six.text_type(section)

        if _is_defaultsect(section):
            if six.PY2:
                section = configparser.DEFAULTSECT
        else:
            cp.add_section(section)

        for key, value in keys.items():
            key = cp.optionxform(six.text_type(key))
            if value is not None:
                value = six.text_type(value)
            cp.set(section, key, value)
1054457
def readabt(filename, dirs='.'):
    
    # resolve filename
    filename = misc.findfileindirs(filename, dirs)
    f = open(filename, 'rt')
    abt = {'offsetcorrected': False, 'params': {}, 'columns': [], 'data': [], 'title': '<no_title>',
           'offsets': {}, 'filename': filename}
    readingmode = ''
    for l in f:
        l = l.strip()
        if l.startswith('!') or len(l) == 0:
            continue
        elif l.startswith('%c'):
            readingmode = 'comments'
        elif l.startswith('%p'):
            readingmode = 'params'
        elif l.startswith('%d'):
            readingmode = 'data'
        elif readingmode == 'comments':
            m = re.match(
                r'(?P<scantype>\w+)-Scan started at (?P<startdate>\d+-\w+-\d+) (?P<starttime>\d+:\d+:\d+), ended (?P<endtime>\d+:\d+:\d+)', l)
            if m:
                abt.update(m.groupdict())
                continue
            else:
                m = re.match(r'Name: (?P<name>\w+)', l)
                if m:
                    abt.update(m.groupdict())
                    m1 = re.search(r'from (?P<from>\d+(?:.\d+)?)', l)
                    if m1:
                        abt.update(m1.groupdict())
                    m1 = re.search(r'to (?P<to>\d+(?:.\d+)?)', l)
                    if m1:
                        abt.update(m1.groupdict())
                    m1 = re.search(r'by (?P<by>\d+(?:.\d+)?)', l)
                    if m1:
                        abt.update(m1.groupdict())
                    m1 = re.search(r'sampling (?P<sampling>\d+(?:.\d+)?)', l)
                    if m1:
                        abt.update(m1.groupdict())
                    continue
            if l.find('Counter readings are offset corrected') >= 0:
                abt['offsetcorrected'] = True
                readingmode = 'offsets'
                continue
            # if we reach here in 'comments' mode, this is the title line
            abt['title'] = l
            continue
        elif readingmode == 'offsets':
            m = re.findall(r'(\w+)\s(\d+(?:.\d+)?)', l)
            if m:
                abt['offsets'].update(dict(m))
                for k in abt['offsets']:
                    abt['offsets'][k] = float(abt['offsets'][k])
        elif readingmode == 'params':
            abt['params'][l.split('=')[0].strip()] = float(
                l.split('=')[1].strip())
        elif readingmode == 'data':
            if l.startswith('Col'):
                abt['columns'].append(l.split()[2])
            else:
                abt['data'].append([float(x) for x in l.split()])
    f.close()
    # some post-processing
    # remove common prefix from column names
    maxcolnamelen = max(len(c) for c in abt['columns'])
    l = 1
    for l in range(1, maxcolnamelen):
        if len(set([c[:l] for c in abt['columns']])) > 1:
            break
    abt['columns'] = [c[l - 1:] for c in abt['columns']]
    # represent data as a structured array
    dt = np.dtype(list(zip(abt['columns'], itertools.repeat(np.double))))
    abt['data'] = np.array(abt['data'], dtype=np.double).view(dt)
    # dates and times in datetime formats
    monthnames = ['Jan', 'Feb', 'Mar', 'Apr', 'May',
                  'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
    for m, i in zip(monthnames, itertools.count(1)):
        abt['startdate'] = abt['startdate'].replace(m, str(i))
    abt['startdate'] = datetime.date(
        *reversed([int(x) for x in abt['startdate'].split('-')]))
    abt['starttime'] = datetime.time(
        *[int(x) for x in abt['starttime'].split(':')])
    abt['endtime'] = datetime.time(
        *[int(x) for x in abt['endtime'].split(':')])
    abt['start'] = datetime.datetime.combine(
        abt['startdate'], abt['starttime'])
    if abt['endtime'] <= abt['starttime']:
        abt['end'] = datetime.datetime.combine(
            abt['startdate'] + datetime.timedelta(1), abt['endtime'])
    else:
        abt['end'] = datetime.datetime.combine(
            abt['startdate'], abt['endtime'])
    del abt['starttime']
    del abt['startdate']
    del abt['endtime']
    # convert some fields to float
    for k in ['from', 'to', 'by', 'sampling']:
        if k in abt:
            abt[k] = float(abt[k])
    # change space and dash in title to underscore
    abt['title'] = abt['title'].replace('-', '_').replace(' ', '_')
    return abt
844455
def bytes_to_bits(bytes_):
    
    res = []
    for x in bytes_:
        if not isinstance(x, int):
            x = ord(x)
        res += byte_to_bits(x)
    return res
801546
def _read_bits(cls, raw_value):
        
        have_bits = 0
        bits = 0
        byte_source = iter(raw_value)
        result = 0
        while True:
            need_bits = yield result
            while have_bits < need_bits:
                try:
                    bits = (bits << 8) | int(next(byte_source))
                    have_bits += 8
                except StopIteration:
                    return
            result = int(bits >> (have_bits - need_bits))
            bits &= (1 << (have_bits - need_bits)) - 1
            have_bits -= need_bits
755905
def read_plain_byte_array(file_obj, count):
    """"""
    return [file_obj.read(struct.unpack(b"<i", file_obj.read(4))[0]) for i in range(count)]
615527
def read_set_from_file(filename: str) -> Set[str]:
    
    collection = set()
    with open(filename, 'r') as file_:
        for line in file_:
            collection.add(line.rstrip())
    return collection
1288123
def read(parts):
    
    cur_dir = os.path.abspath(os.path.dirname(__file__))
    with codecs.open(os.path.join(cur_dir, *parts), "rb", "utf-8") as f:
        return f.read()
540109
private static byte[] bitSetToByteArray(BitSet bitSet) {

    while (bitSet.length() % 8 != 0) { bitSet.set(bitSet.length(), true); }
    
    byte[] array = new byte[bitSet.length()/8];

    for (int i = 0; i < array.length; i++) {
      int offset = i * 8;

      int index = 0;
      for (int j = 0; j < 8; j++) {
        index <<= 1;        
        if (bitSet.get(offset+j)) { index++; }
      }

      array[i] = (byte)(index - 128);
    }

    return array;
  }
127130
def read_binary_dict(fname,fd,length)
      dict = {}

      # first: read keys
      if(length != 0) then
        buff = fd.read(length * @object_ref_size)
        keys = unpack_with_size(@object_ref_size, buff)

        # second: read object refs
        buff = fd.read(length * @object_ref_size)
        objects = unpack_with_size(@object_ref_size, buff)

        # read real keys and objects
        0.upto(length-1) do |i|
          key = read_binary_object_at(fname,fd,keys[i])
          object = read_binary_object_at(fname,fd,objects[i])
          dict[key.value] = object
        end
      end

      CFDictionary.new(dict)
    end
696296
def read_lexical_file(filename):
    '''
    
    '''
    with open(filename, 'rb') as input_file:
        doc = etree.parse(input_file)

    synsets = []
    assert doc.getroot().tag == 'synsets'
    for synset in doc.getroot():
        if synset.tag != 'synset':
            print('unrecognised child of <synsets>', synset)
            continue
        synset_dict = dict(synset.items())
        synloc = '{0} synset {1},'.format(filename,
                                          synset_dict.get('id', '???'))
        warn_attribs(synloc, synset, SYNSET_ATTRIBS)
        synset_dict['lexunits'] = []
        synsets.append(synset_dict)

        for child in synset:
            if child.tag == 'lexUnit':
                lexunit      = child
                lexunit_dict = dict(lexunit.items())
                lexloc       = synloc + ' lexUnit {0},'.format(
                    lexunit_dict.get('id', '???'))
                warn_attribs(lexloc, lexunit, LEXUNIT_ATTRIBS)
                # convert some properties to booleans
                for key in ['styleMarking', 'artificial', 'namedEntity']:
                    if key in lexunit_dict:
                        if lexunit_dict[key] not in MAP_YESNO_TO_BOOL:
                            print(lexloc, ('lexunit property {0} has '
                                           'non-boolean value').format(key),
                                  lexunit_dict[key])
                            continue
                        lexunit_dict[key] = MAP_YESNO_TO_BOOL[lexunit_dict[key]]
                # convert sense to integer number
                if 'sense' in lexunit_dict:
                    if lexunit_dict['sense'].isdigit():
                        lexunit_dict['sense'] = int(lexunit_dict['sense'], 10)
                    else:
                        print(lexloc,
                              'lexunit property sense has non-numeric value',
                              lexunit_dict['sense'])
                synset_dict['lexunits'].append(lexunit_dict)
                lexunit_dict['examples'] = []
                lexunit_dict['frames']   = []
                for child in lexunit:
                    if child.tag in ['orthForm',
                                     'orthVar',
                                     'oldOrthForm',
                                     'oldOrthVar']:
                        warn_attribs(lexloc, child, set())
                        if not child.text:
                            print(lexloc, '{0} with no text'.format(child.tag))
                            continue
                        if child.tag in lexunit_dict:
                            print(lexloc, 'more than one {0}'.format(child.tag))
                        lexunit_dict[child.tag] = str(child.text)
                    elif child.tag == 'example':
                        example = child
                        text = [child for child in example
                                if child.tag == 'text']
                        if len(text) != 1 or not text[0].text:
                            print(lexloc, '<example> tag without text')
                        example_dict = {'text': str(text[0].text)}
                        for child in example:
                            if child.tag == 'text':
                                continue
                            elif child.tag == 'exframe':
                                if 'exframe' in example_dict:
                                    print(lexloc,
                                          'more than one <exframe> '
                                          'for <example>')
                                warn_attribs(lexloc, child, set())
                                if not child.text:
                                    print(lexloc, '<exframe> with no text')
                                    continue
                                example_dict['exframe'] = str(child.text)
                            else:
                                print(lexloc,
                                      'unrecognised child of <example>',
                                      child)
                        lexunit_dict['examples'].append(example_dict)
                    elif child.tag == 'frame':
                        frame = child
                        warn_attribs(lexloc, frame, set())
                        if 0 < len(frame):
                            print(lexloc, 'unrecognised <frame> children',
                                list(frame))
                        if not frame.text:
                            print(lexloc, '<frame> without text')
                            continue
                        lexunit_dict['frames'].append(str(frame.text))
                    elif child.tag == 'compound':
                        compound = child
                        warn_attribs(lexloc, compound, set())
                        compound_dict = {}
                        for child in compound:
                            if child.tag == 'modifier':
                                modifier_dict = dict(child.items())
                                warn_attribs(lexloc, child,
                                             MODIFIER_ATTRIBS, set())
                                if not child.text:
                                    print(lexloc, 'modifier without text')
                                    continue
                                modifier_dict['text'] = str(child.text)
                                if 'modifier' not in compound_dict:
                                    compound_dict['modifier'] = []
                                compound_dict['modifier'].append(modifier_dict)
                            elif child.tag == 'head':
                                head_dict = dict(child.items())
                                warn_attribs(lexloc, child, HEAD_ATTRIBS, set())
                                if not child.text:
                                    print(lexloc, '<head> without text')
                                    continue
                                head_dict['text'] = str(child.text)
                                if 'head' in compound_dict:
                                    print(lexloc,
                                          'more than one head in <compound>')
                                compound_dict['head'] = head_dict
                            else:
                                print(lexloc,
                                      'unrecognised child of <compound>',
                                      child)
                                continue
                    else:
                        print(lexloc, 'unrecognised child of <lexUnit>', child)
                        continue
            elif child.tag == 'paraphrase':
                paraphrase = child
                warn_attribs(synloc, paraphrase, set())
                paraphrase_text = str(paraphrase.text)
                if not paraphrase_text:
                    print(synloc, 'WARNING: <paraphrase> tag with no text')
            else:
                print(synloc, 'unrecognised child of <synset>', child)
                continue

    return synsets
321124
def readme(fname):
    """"""
    md = open(os.path.join(os.path.dirname(__file__), fname)).read()
    output = md
    try:
        import pypandoc
        output = pypandoc.convert(md, 'rst', format='md')
    except ImportError:
        pass
    return output
977324
def _get_bits(bits=None, target=None):
    

    if bits is not None:
        bits = int(bits)
        if bits in (8, 16, 32, 64):
            return bits
        else:
            raise ValueError('bits needs to be 8, 16, 32 or 64')
    else:
        return int((target if target is not None else pwnypack.target.target).bits)
98606
def read_dictionary_file(dictionary_path):
    """"""
    try:
        return _user_dictionary_cache[dictionary_path]
    except KeyError:
        if dictionary_path and os.path.exists(dictionary_path):
            with open(dictionary_path, "rt") as dict_f:
                words = set(re.findall(r"(\w[\w']*\w|\w)",
                                       " ".join(dict_f.read().splitlines())))
                return words

        return set()