Read hierarchy information from acclaim skeleton file stream.
978549
def read_hierarchy(self, fid):
        """"""

        lin = self.read_line(fid)
                    
        while lin != 'end':
            parts = lin.split()
            if lin != 'begin':
                ind = self.get_index_by_name(parts[0])
                for i in range(1, len(parts)):
                    self.vertices[ind].children.append(self.get_index_by_name(parts[i]))
            lin = self.read_line(fid)
        lin = self.read_line(fid)
        return lin
134804
def read(filepath):
    
    import xml.etree.ElementTree
    root = xml.etree.ElementTree.parse(filepath).getroot()

    # Get the raw data
    recording = []
    strokes = sorted(root.findall('{http://www.w3.org/2003/InkML}trace'),
                     key=lambda child: int(child.attrib['id']))
    time = 0
    for stroke in strokes:
        stroke = stroke.text.strip().split(',')
        stroke = [point.strip().split(' ') for point in stroke]
        if len(stroke[0]) == 3:
            stroke = [{'x': float(x), 'y': float(y), 'time': float(t)}
                      for x, y, t in stroke]
        else:
            stroke = [{'x': float(x), 'y': float(y)} for x, y in stroke]
            new_stroke = []
            for p in stroke:
                new_stroke.append({'x': p['x'], 'y': p['y'], 'time': time})
                time += 20
            stroke = new_stroke
            time += 200
        recording.append(stroke)

    # Get LaTeX
    formula_in_latex = None
    annotations = root.findall('{http://www.w3.org/2003/InkML}annotation')
    for annotation in annotations:
        if annotation.attrib['type'] == 'truth':
            formula_in_latex = annotation.text
    hw = handwritten_data.HandwrittenData(json.dumps(recording),
                                          formula_in_latex=formula_in_latex)
    for annotation in annotations:
        if annotation.attrib['type'] == 'writer':
            hw.writer = annotation.text
        elif annotation.attrib['type'] == 'category':
            hw.category = annotation.text
        elif annotation.attrib['type'] == 'expression':
            hw.expression = annotation.text

    # Get segmentation
    segmentation = []
    trace_groups = root.findall('{http://www.w3.org/2003/InkML}traceGroup')
    if len(trace_groups) != 1:
        raise Exception('Malformed InkML',
                        ('Exactly 1 top level traceGroup expected, found %i. '
                         '(%s) - probably no ground truth?') %
                        (len(trace_groups), filepath))
    trace_group = trace_groups[0]
    symbol_stream = []  # has to be consistent with segmentation
    for tg in trace_group.findall('{http://www.w3.org/2003/InkML}traceGroup'):
        annotations = tg.findall('{http://www.w3.org/2003/InkML}annotation')
        if len(annotations) != 1:
            raise ValueError("%i annotations found for '%s'." %
                             (len(annotations), filepath))
        db_id = formula_to_dbid(normalize_symbol_name(annotations[0].text))
        symbol_stream.append(db_id)
        trace_views = tg.findall('{http://www.w3.org/2003/InkML}traceView')
        symbol = []
        for traceView in trace_views:
            symbol.append(int(traceView.attrib['traceDataRef']))
        segmentation.append(symbol)
    hw.symbol_stream = symbol_stream
    hw.segmentation = segmentation
    _flat_seg = [stroke2 for symbol2 in segmentation for stroke2 in symbol2]
    assert len(_flat_seg) == len(recording), \
        ("Segmentation had length %i, but recording has %i strokes (%s)" %
         (len(_flat_seg), len(recording), filepath))
    assert set(_flat_seg) == set(range(len(_flat_seg)))
    hw.inkml = beautify_xml(filepath)
    hw.filepath = filepath
    return hw
514098
def _contextualise(self):
        ''''''
        # loop through hierarchy identifying unique lineages
        # TODO: gain other contextual information, not just ident
        deja_vues = []
        for rank in reversed(self.taxonomy):
            # return named clades -- '' are ignored
            clades = [e for e in self.hierarchy[rank] if e[1]]
            # print 'Rank: {0} - {1}'.format(rank, len(clades))
            # get unique lineages at this level
            uniques = [e for e in clades if len(e[0]) == 1]
            # removed those already seen
            uniques = [e for e in uniques if e[0][0].ident not in deja_vues]
            # add each to self[ident]['cident']
            for e in uniques:
                ident = e[0][0].ident
                self[ident]['cident'] = e[1]
                deja_vues.append(ident)
713331
def ReadPathInfosHistories(self, client_id, path_type, components_list):
    """"""
    results = {}

    for components in components_list:
      try:
        path_record = self.path_records[(client_id, path_type, components)]
      except KeyError:
        results[components] = []
        continue

      entries_by_ts = {}
      for ts, stat_entry in path_record.GetStatEntries():
        pi = rdf_objects.PathInfo(
            path_type=path_type,
            components=components,
            timestamp=ts,
            stat_entry=stat_entry)
        entries_by_ts[ts] = pi

      for ts, hash_entry in path_record.GetHashEntries():
        try:
          pi = entries_by_ts[ts]
        except KeyError:
          pi = rdf_objects.PathInfo(
              path_type=path_type, components=components, timestamp=ts)
          entries_by_ts[ts] = pi

        pi.hash_entry = hash_entry

      results[components] = [
          entries_by_ts[k] for k in sorted(iterkeys(entries_by_ts))
      ]

    return results
779732
def _deserialize(self, stream):
        """"""
        readline = stream.readline
        self.tree = Tree(self.repo, hex_to_bin(readline().split()[1]), Tree.tree_id << 12, '')

        self.parents = []
        next_line = None
        while True:
            parent_line = readline()
            if not parent_line.startswith(b'parent'):
                next_line = parent_line
                break
            # END abort reading parents
            self.parents.append(type(self)(self.repo, hex_to_bin(parent_line.split()[-1].decode('ascii'))))
        # END for each parent line
        self.parents = tuple(self.parents)

        # we don't know actual author encoding before we have parsed it, so keep the lines around
        author_line = next_line
        committer_line = readline()

        # we might run into one or more mergetag blocks, skip those for now
        next_line = readline()
        while next_line.startswith(b'mergetag '):
            next_line = readline()
            while next_line.startswith(b' '):
                next_line = readline()
        # end skip mergetags

        # now we can have the encoding line, or an empty line followed by the optional
        # message.
        self.encoding = self.default_encoding
        self.gpgsig = None

        # read headers
        enc = next_line
        buf = enc.strip()
        while buf:
            if buf[0:10] == b"encoding ":
                self.encoding = buf[buf.find(' ') + 1:].decode('ascii')
            elif buf[0:7] == b"gpgsig ":
                sig = buf[buf.find(b' ') + 1:] + b"\n"
                is_next_header = False
                while True:
                    sigbuf = readline()
                    if not sigbuf:
                        break
                    if sigbuf[0:1] != b" ":
                        buf = sigbuf.strip()
                        is_next_header = True
                        break
                    sig += sigbuf[1:]
                # end read all signature
                self.gpgsig = sig.rstrip(b"\n").decode('ascii')
                if is_next_header:
                    continue
            buf = readline().strip()
        # decode the authors name

        try:
            self.author, self.authored_date, self.author_tz_offset = \
                parse_actor_and_date(author_line.decode(self.encoding, 'replace'))
        except UnicodeDecodeError:
            log.error("Failed to decode author line '%s' using encoding %s", author_line, self.encoding,
                      exc_info=True)

        try:
            self.committer, self.committed_date, self.committer_tz_offset = \
                parse_actor_and_date(committer_line.decode(self.encoding, 'replace'))
        except UnicodeDecodeError:
            log.error("Failed to decode committer line '%s' using encoding %s", committer_line, self.encoding,
                      exc_info=True)
        # END handle author's encoding

        # a stream from our data simply gives us the plain message
        # The end of our message stream is marked with a newline that we strip
        self.message = stream.read()
        try:
            self.message = self.message.decode(self.encoding, 'replace')
        except UnicodeDecodeError:
            log.error("Failed to decode message '%s' using encoding %s", self.message, self.encoding, exc_info=True)
        # END exception handling

        return self
607020
def read_data(fo, writer_schema, reader_schema=None):
    """"""

    record_type = extract_record_type(writer_schema)
    logical_type = extract_logical_type(writer_schema)

    if reader_schema and record_type in AVRO_TYPES:
        # If the schemas are the same, set the reader schema to None so that no
        # schema resolution is done for this call or future recursive calls
        if writer_schema == reader_schema:
            reader_schema = None
        else:
            match_schemas(writer_schema, reader_schema)

    reader_fn = READERS.get(record_type)
    if reader_fn:
        try:
            data = reader_fn(fo, writer_schema, reader_schema)
        except StructError:
            raise EOFError('cannot read %s from %s' % (record_type, fo))

        if 'logicalType' in writer_schema:
            fn = LOGICAL_READERS.get(logical_type)
            if fn:
                return fn(data, writer_schema, reader_schema)

        if reader_schema is not None:
            return maybe_promote(
                data,
                record_type,
                extract_record_type(reader_schema)
            )
        else:
            return data
    else:
        return read_data(
            fo,
            SCHEMA_DEFS[record_type],
            SCHEMA_DEFS.get(reader_schema)
        )
237846
def _fromfile(self, parent):
        """"""
        fd = parent._fd
        byte_order = parent.byte_order

        self._offset = fd.tell()
        self.value_offset = self._offset + parent.offset_size + 4

        fmt, size = {4: ('HHI4s', 12), 8: ('HHQ8s', 20)}[parent.offset_size]
        data = fd.read(size)
        code, dtype = struct.unpack(byte_order + fmt[:2], data[:4])
        count, value = struct.unpack(byte_order + fmt[2:], data[4:])

        if code in TIFF_TAGS:
            name = TIFF_TAGS[code][0]
        elif code in CUSTOM_TAGS:
            name = CUSTOM_TAGS[code][0]
        else:
            name = str(code)

        try:
            dtype = TIFF_DATA_TYPES[dtype]
        except KeyError:
            raise ValueError("unknown TIFF tag data type %i" % dtype)

        fmt = '%s%i%s' % (byte_order, count*int(dtype[0]), dtype[1])
        size = struct.calcsize(fmt)
        if size > parent.offset_size or code in CUSTOM_TAGS:
            pos = fd.tell()
            tof = {4: 'I', 8: 'Q'}[parent.offset_size]
            self.value_offset = struct.unpack(byte_order+tof, value)[0]
            fd.seek(self.value_offset)
            if code in CUSTOM_TAGS:
                readfunc = CUSTOM_TAGS[code][1]
                value = readfunc(fd, byte_order, dtype, count)
                fd.seek(0, 2)  # bug in numpy/Python 3.x ?
                if isinstance(value, dict):  # numpy.core.records.record
                    value = Record(value)
            elif code in TIFF_TAGS or dtype[-1] == 's':
                value = struct.unpack(fmt, fd.read(size))
            else:
                value = read_numpy(fd, byte_order, dtype, count)
                fd.seek(0, 2)  # bug in numpy/Python 3.x ?
            fd.seek(pos)
        else:
            value = struct.unpack(fmt, value[:size])

        if not code in CUSTOM_TAGS:
            if len(value) == 1:
                value = value[0]

        if dtype.endswith('s'):
            value = stripnull(value)

        self.code = code
        self.name = name
        self.dtype = dtype
        self.count = count
        self.value = value
737845
def read_dir_tree(self, file_hash):
        

        json_d = self.read_index_object(file_hash, 'tree')
        node = {'files' : json_d['files'], 'dirs' : {}}
        for name, hsh in json_d['dirs'].iteritems(): node['dirs'][name] = self.read_dir_tree(hsh)
        return node
314135
def read_legacy(filename):
    """"""
    reader = vtk.vtkDataSetReader()
    reader.SetFileName(filename)
    # Ensure all data is fetched with poorly formated legacy files
    reader.ReadAllScalarsOn()
    reader.ReadAllColorScalarsOn()
    reader.ReadAllNormalsOn()
    reader.ReadAllTCoordsOn()
    reader.ReadAllVectorsOn()
    # Perform the read
    reader.Update()
    output = reader.GetOutputDataObject(0)
    if output is None:
        raise AssertionError('No output when using VTKs legacy reader')
    return vtki.wrap(output)
1084116
def file_tree(start_path):
    
    nested_dirs = {}
    root_dir = start_path.rstrip(os.sep)
    start = root_dir.rfind(os.sep) + 1
    for path, dirs, files in os.walk(root_dir):
        folders = path[start:].split(os.sep)
        subdir = dict.fromkeys(files)
        parent = reduce(dict.get, folders[:-1], nested_dirs)
        parent[folders[-1]] = subdir
    return nested_dirs
880082
def tree_entries_from_data(data):
    """"""
    ord_zero = ord('0')
    space_ord = ord(' ')
    len_data = len(data)
    i = 0
    out = []
    while i < len_data:
        mode = 0

        # read mode
        # Some git versions truncate the leading 0, some don't
        # The type will be extracted from the mode later
        while byte_ord(data[i]) != space_ord:
            # move existing mode integer up one level being 3 bits
            # and add the actual ordinal value of the character
            mode = (mode << 3) + (byte_ord(data[i]) - ord_zero)
            i += 1
        # END while reading mode

        # byte is space now, skip it
        i += 1

        # parse name, it is NULL separated

        ns = i
        while byte_ord(data[i]) != 0:
            i += 1
        # END while not reached NULL

        # default encoding for strings in git is utf8
        # Only use the respective unicode object if the byte stream was encoded
        name = data[ns:i]
        name = safe_decode(name)

        # byte is NULL, get next 20
        i += 1
        sha = data[i:i + 20]
        i = i + 20
        out.append((sha, mode, name))
    # END for each byte in data stream
    return out
229054
def parse_hicpro_stats(self, f, rsection):
        
        s_name = self.clean_s_name(os.path.basename(f['root']), os.path.dirname(f['root']))

        if s_name not in self.hicpro_data.keys():
            self.hicpro_data[s_name] = {}

        self.add_data_source(f, s_name, section=rsection)
        for l in f['f'].splitlines():
            if not l.startswith('#'):
                s = l.split("\t")
                if s[0] in self.hicpro_data[s_name]:
                    log.debug("Duplicated keys found! Overwriting: {}".format(s[0]))
                self.hicpro_data[s_name][s[0]] = int(s[1])
346145
def _get_read_depths(cls, read_depths_file, sequence_name, position):
        ''''''
        assert os.path.exists(read_depths_file)
        assert os.path.exists(read_depths_file + '.tbi')
        tbx = pysam.TabixFile(read_depths_file)
        try:
            rows = [x for x in tbx.fetch(sequence_name, position, position + 1)]
        except:
            return None

        if len(rows) > 1: # which happens with indels, mutiple lines for same base of reference
            test_rows = [x for x in rows if x.rstrip().split()[3] != '.']
            if len(test_rows) != 1:
                rows = [rows[-1]]
            else:
                rows = test_rows

        if len(rows) == 1:
            r, p, ref_base, alt_base, ref_counts, alt_counts = rows[0].rstrip().split()
            bases = ref_base if alt_base == '.' else ref_base + ',' + alt_base
            return bases, int(ref_counts), alt_counts
        else:
            return None
765229
def process(self):
        

        self.index = cindex.Index.create()
        self.headers = {}

        for f in self.files:
            if f in self.processed:
                continue

            print('Processing {0}'.format(os.path.basename(f)))

            tu = self.index.parse(f, self.flags)

            if len(tu.diagnostics) != 0:
                fatal = False

                for d in tu.diagnostics:
                    sys.stderr.write(d.format())
                    sys.stderr.write("\n")

                    if d.severity == cindex.Diagnostic.Fatal or \
                       d.severity == cindex.Diagnostic.Error:
                        fatal = True

                if fatal:
                    sys.stderr.write("\nCould not generate documentation due to parser errors\n")
                    sys.exit(1)

            if not tu:
                sys.stderr.write("Could not parse file %s...\n" % (f,))
                sys.exit(1)

            # Extract comments from files and included files that we are
            # supposed to inspect
            extractfiles = [f]

            for inc in tu.get_includes():
                filename = str(inc.include)
                self.headers[filename] = True

                if filename in self.processed or (not filename in self.files) or filename in extractfiles:
                    continue

                extractfiles.append(filename)

            for e in extractfiles:
                db = comment.CommentsDatabase(e, tu)

                self.add_categories(db.category_names)
                self.commentsdbs[e] = db

            self.visit(tu.cursor.get_children())

            for f in self.processing:
                self.processed[f] = True

            self.processing = {}

        # Construct hierarchy of nodes.
        for node in self.all_nodes:
            q = node.qid

            if node.parent is None:
                par = self.find_parent(node)

                # Lookup categories for things in the root
                if (par is None or par == self.root) and (not node.cursor is None):
                    location = node.cursor.extent.start
                    db = self.commentsdbs[location.file.name]

                    if db:
                        par = self.category_to_node[db.lookup_category(location)]

                if par is None:
                    par = self.root

                par.append(node)

            # Resolve comment
            cm = self.find_node_comment(node)

            if cm:
                node.merge_comment(cm)

        # Keep track of classes to resolve bases and subclasses
        classes = {}

        # Map final qid to node
        for node in self.all_nodes:
            q = node.qid
            self.qid_to_node[q] = node

            if isinstance(node, nodes.Class):
                classes[q] = node

        # Resolve bases and subclasses
        for qid in classes:
            classes[qid].resolve_bases(classes)
681248
def GetTopicsTree(self):
        '''
        '''
        if self.topics is None:
            return None

        if self.topics:
            res, ui = chmlib.chm_resolve_object(self.file, self.topics)
            if (res != chmlib.CHM_RESOLVE_SUCCESS):
                return None

        size, text = chmlib.chm_retrieve_object(self.file, ui, 0l, ui.length)
        if (size == 0):
            sys.stderr.write('GetTopicsTree: file size = 0\n')
            return None
        return text
46647
def tree(path, depth=2, topdown=True, followlinks=False, showhidden=False):
    """"""
    rt = []
    for root, dirs, files in os.walk(path, topdown=topdown, followlinks=followlinks):
        if not showhidden and File.is_hidden(root):
            continue

        current_depth = len(os.path.relpath(root, path).split(os.sep))
        if current_depth > depth:
            continue

        if showhidden:
            _tuple = (
                root,
                [File(os.path.join(root, _dir)) for _dir in dirs],
                [File(os.path.join(root, _file)) for _file in files]
            )
        else:
            _tuple = (
                root,
                [File(os.path.join(root, _dir)) for _dir in dirs if _dir[0] != '.'],
                [File(os.path.join(root, _file)) for _file in files if _file[0] != '.']
            )

        rt.append(_tuple)
    return rt
667868
def read_rels(archive):
    """"""
    xml_source = archive.read(ARC_WORKBOOK_RELS)
    tree = fromstring(xml_source)
    for element in safe_iterator(tree, '{%s}Relationship' % PKG_REL_NS):
        rId = element.get('Id')
        pth = element.get("Target")
        typ = element.get('Type')
        # normalise path
        if pth.startswith("/xl"):
            pth = pth.replace("/xl", "xl")
        elif not pth.startswith("xl") and not pth.startswith(".."):
            pth = "xl/" + pth
        yield rId, {'path':pth, 'type':typ}