Parse the FastQ Screen output into a 3D dict
961149
def parse_fqscreen(self, f):
        
        parsed_data = OrderedDict()
        reads_processed = None
        nohits_pct = None
        for l in f['f']:
            if l.startswith('%Hit_no_genomes:') or l.startswith('%Hit_no_libraries:'):
                nohits_pct = float(l.split(':', 1)[1])
                parsed_data['No hits'] = {'percentages': {'one_hit_one_library': nohits_pct }}
            else:
                fqs = re.search(r"^(\S+)\s+(\d+)\s+(\d+)\s+([\d\.]+)\s+(\d+)\s+([\d\.]+)\s+(\d+)\s+([\d\.]+)\s+(\d+)\s+([\d\.]+)\s+(\d+)\s+([\d\.]+)$", l)
                if fqs:
                    org = fqs.group(1)
                    parsed_data[org] = {'percentages':{}, 'counts':{}}
                    reads_processed = int(fqs.group(2))
                    parsed_data[org]['counts']['reads_processed'] = int(fqs.group(2))
                    parsed_data[org]['counts']['unmapped'] = int(fqs.group(3))
                    parsed_data[org]['percentages']['unmapped'] = float(fqs.group(4))
                    parsed_data[org]['counts']['one_hit_one_library'] = int(fqs.group(5))
                    parsed_data[org]['percentages']['one_hit_one_library'] = float(fqs.group(6))
                    parsed_data[org]['counts']['multiple_hits_one_library'] = int(fqs.group(7))
                    parsed_data[org]['percentages']['multiple_hits_one_library'] = float(fqs.group(8))
                    parsed_data[org]['counts']['one_hit_multiple_libraries'] = int(fqs.group(9))
                    parsed_data[org]['percentages']['one_hit_multiple_libraries'] = float(fqs.group(10))
                    parsed_data[org]['counts']['multiple_hits_multiple_libraries'] = int(fqs.group(11))
                    parsed_data[org]['percentages']['multiple_hits_multiple_libraries'] = float(fqs.group(12))
                    # Can't use #Reads in subset as varies. #Reads_processed should be same for all orgs in a sample
                    parsed_data['total_reads'] = int(fqs.group(2))

        if len(parsed_data) == 0:
            return None

        # Calculate no hits counts
        if reads_processed and nohits_pct:
            parsed_data['No hits']['counts'] = {'one_hit_one_library': int((nohits_pct/100.0) * float(reads_processed)) }
        else:
            log.warn("Couldn't find number of reads with no hits for '{}'".format(f['s_name']))

        self.num_orgs = max(len(parsed_data), self.num_orgs)
        return parsed_data
463453
def read_fastq(filename):
    
    if not filename:
        return itertools.cycle((None,))
    if filename == "-":
        filename_fh = sys.stdin
    elif filename.endswith('gz'):
        if is_python3:
            filename_fh = gzip.open(filename, mode='rt')
        else:
            filename_fh = BufferedReader(gzip.open(filename, mode='rt'))
    else:
        filename_fh = open(filename)
    return stream_fastq(filename_fh)
493403
def display(self):
        '''
        
        '''
        w, h = (0, 0)
        for line in self.shell('dumpsys', 'display').splitlines():
            m = _DISPLAY_RE.search(line, 0)
            if not m:
                continue
            w = int(m.group('width'))
            h = int(m.group('height'))
            o = int(m.group('orientation'))
            w, h = min(w, h), max(w, h)
            return self.Display(w, h, o)

        output = self.shell('LD_LIBRARY_PATH=/data/local/tmp', self.__minicap, '-i')
        try:
            data = json.loads(output)
            (w, h, o) = (data['width'], data['height'], data['rotation']/90)
            return self.Display(w, h, o)            
        except ValueError:
            pass
868884
def get_structured_data(raw_output, platform, command):
    """"""
    template_dir = get_template_dir()
    index_file = os.path.join(template_dir, "index")
    textfsm_obj = clitable.CliTable(index_file, template_dir)
    attrs = {"Command": command, "Platform": platform}
    try:
        # Parse output through template
        textfsm_obj.ParseCmd(raw_output, attrs)
        structured_data = clitable_to_dict(textfsm_obj)
        output = raw_output if structured_data == [] else structured_data
        return output
    except CliTableError:
        return raw_output
381772
def dict_from_qs(qs):
    ''' 
    '''
    entries = qs.split(',') if qs.strip() else []
    entries = [entry.strip() for entry in entries]

    def _dict_from_qs(line, d):
        if '.' in line:
            key, value = line.split('.', 1)
            d.setdefault(key, {})
            return _dict_from_qs(value, d[key])
        else:
            d[line] = {}

    def _default():
        return defaultdict(_default)

    d = defaultdict(_default)
    for line in entries:
        _dict_from_qs(line, d)
    return d
554115
def _read_info_as_dict(fid, values):
    
    output = {}
    for key, fmt in values:
        val = unpack(fmt, fid.read(calcsize(fmt)))
        if len(val) == 1:
            output[key] = val[0]
        else:
            output[key] = val
    return output
1197015
def _transform_fast(self, result, obj, func_nm):
        
        # if there were groups with no observations (Categorical only?)
        # try casting data to original dtype
        cast = self._transform_should_cast(func_nm)

        # for each col, reshape to to size of original frame
        # by take operation
        ids, _, ngroup = self.grouper.group_info
        output = []
        for i, _ in enumerate(result.columns):
            res = algorithms.take_1d(result.iloc[:, i].values, ids)
            if cast:
                res = self._try_cast(res, obj.iloc[:, i])
            output.append(res)

        return DataFrame._from_arrays(output, columns=result.columns,
                                      index=obj.index)
657299
def loadtxt_asdict(fn, **kwargs):
    """"""

    kwargs.update(unpack=True)
    d = loadtxt(fn, **kwargs)
    if len(np.shape(d)) == 2:
        keys = kwargs.get('usecols', None) or range(len(d))
        D = dict([(k, v) for k, v in zip(keys, d)])
    elif len(np.shape(d)) == 1:
        keys = kwargs.get('usecols', None) or [0]
        D = dict([(keys[0], d)])
    else:
        raise Exception('Unknown dimension of loaded data.')

    return D
945378
def parse_TSVline(self, line):
        

        split_row = [token.strip() for token in line.split('\t')]
        _results = {'DefaultResult': 'Conc.'}

        # ID# 1
        if split_row[0] == 'ID#':
            return 0
        # Name	CBDV - cannabidivarin
        elif split_row[0] == 'Name':
            if split_row[1]:
                self._currentanalysiskw = split_row[1]
                return 0
            else:
                self.warn("Analysis Keyword not found or empty",
                          numline=self._numline, line=line)
        # Data Filename	Sample Name	Sample ID	Sample Type	Level#
        elif 'Sample ID' in split_row:
            split_row.insert(0, '#')
            self._currentresultsheader = split_row
            return 0
        # 1	QC PREP A_QC PREP A_009.lcd	QC PREP
        elif split_row[0].isdigit():
            _results.update(dict(zip(self._currentresultsheader, split_row)))

            # 10/17/2016 7:55:06 PM
            try:
                da = datetime.strptime(
                    _results['Date Acquired'], "%m/%d/%Y %I:%M:%S %p")
                self._header['Output Date'] = da
                self._header['Output Time'] = da
            except ValueError:
                self.err("Invalid Output Time format",
                         numline=self._numline, line=line)

            result = _results[_results['DefaultResult']]
            column_name = _results['DefaultResult']
            result = self.zeroValueDefaultInstrumentResults(
                                                    column_name, result, line)
            _results[_results['DefaultResult']] = result

            self._addRawResult(_results['Sample ID'],
                               values={self._currentanalysiskw: _results},
                               override=False)
720076
def read_fastas(input_files):
    
    tumor_file = [y for x, y in input_files.items() if x.startswith('T')][0]
    normal_file = [y for x, y in input_files.items() if x.startswith('N')][0]
    output_files = defaultdict(list)
    output_files = _read_fasta(tumor_file, output_files)
    num_entries = len(output_files)
    output_files = _read_fasta(normal_file, output_files)
    assert len(output_files) == num_entries
    return output_files
651907
def to_dict(self):
        
        final_res = {
            "param": self._param,
            "unused_param": self.unused_param,
            "execution_time": self._exec_time,
            "output": {"accuracy": self.get_accuracy(),
                       "weights": self.get_weights(),
                       "splines": self._splines
                       }
        }
        return final_res
303452
def parse_to_dict(self):
        
        lst = []
        with open(self.fname, 'r') as f:
            hdr = f.readline()
            self.hdrs = hdr.split(',')
            #print("self.hdrs = ", self.hdrs)
            for line in f:
                cols = line.split(',')
                if len(cols) == len(self.hdrs):
                    #print(cols)
                    d = {}
                    for ndx, col_header in enumerate(self.hdrs):
                        #d[self.hdrs[ndx].strip('\n').strip()] = cols[ndx].strip('\n').strip()
                        d[col_header.strip('\n').strip()] = cols[ndx].strip('\n').strip()
                        
                    lst.append(d)
                else:
                    print("Error parsing " + self.fname + " line : " + line)
        return lst
862806
def fastaParserSpectraClusterPy(header):
    
    isUniprot = lambda h: h[0:3] in ['sp|', 'tr|', 'up|']

    if isUniprot(header):
        start = 3
        end = header.find('|', start)
    else:
        start = 0
        breakPositions = [header.find(' '), header.find('|')]
        breakPositions = [i if i > 0 else len(header) for i in breakPositions]
        end = min(breakPositions)

    return {'id': header[start:end]}
404333
def ParseMultiple(self, result_dicts):
    """"""
    for result_dict in result_dicts:
      # Currently we are only grabbing the Identifying Number
      # as the serial number (catches the unique number for VMs).
      # This could be changed to include more information from
      # Win32_ComputerSystemProduct.

      yield rdf_client.HardwareInfo(
          serial_number=result_dict["IdentifyingNumber"],
          system_manufacturer=result_dict["Vendor"])
1174928
def transform_obs(self, obs):
    """"""
    empty = np.array([], dtype=np.int32).reshape((0, 7))
    out = named_array.NamedDict({  # Fill out some that are sometimes empty.
        "single_select": empty,
        "multi_select": empty,
        "build_queue": empty,
        "cargo": empty,
        "cargo_slots_available": np.array([0], dtype=np.int32),
    })

    def or_zeros(layer, size):
      if layer is not None:
        return layer.astype(np.int32, copy=False)
      else:
        return np.zeros((size.y, size.x), dtype=np.int32)

    aif = self._agent_interface_format

    if aif.feature_dimensions:
      out["feature_screen"] = named_array.NamedNumpyArray(
          np.stack(or_zeros(f.unpack(obs.observation),
                            aif.feature_dimensions.screen)
                   for f in SCREEN_FEATURES),
          names=[ScreenFeatures, None, None])
      out["feature_minimap"] = named_array.NamedNumpyArray(
          np.stack(or_zeros(f.unpack(obs.observation),
                            aif.feature_dimensions.minimap)
                   for f in MINIMAP_FEATURES),
          names=[MinimapFeatures, None, None])

    if aif.rgb_dimensions:
      out["rgb_screen"] = Feature.unpack_rgb_image(
          obs.observation.render_data.map).astype(np.int32)
      out["rgb_minimap"] = Feature.unpack_rgb_image(
          obs.observation.render_data.minimap).astype(np.int32)

    out["last_actions"] = np.array(
        [self.reverse_action(a).function for a in obs.actions],
        dtype=np.int32)

    out["action_result"] = np.array([o.result for o in obs.action_errors],
                                    dtype=np.int32)

    out["alerts"] = np.array(obs.observation.alerts, dtype=np.int32)

    out["game_loop"] = np.array([obs.observation.game_loop], dtype=np.int32)

    score_details = obs.observation.score.score_details
    out["score_cumulative"] = named_array.NamedNumpyArray([
        obs.observation.score.score,
        score_details.idle_production_time,
        score_details.idle_worker_time,
        score_details.total_value_units,
        score_details.total_value_structures,
        score_details.killed_value_units,
        score_details.killed_value_structures,
        score_details.collected_minerals,
        score_details.collected_vespene,
        score_details.collection_rate_minerals,
        score_details.collection_rate_vespene,
        score_details.spent_minerals,
        score_details.spent_vespene,
    ], names=ScoreCumulative, dtype=np.int32)

    def get_score_details(key, details, categories):
      row = getattr(details, key.name)
      return [getattr(row, category.name) for category in categories]

    out["score_by_category"] = named_array.NamedNumpyArray([
        get_score_details(key, score_details, ScoreCategories)
        for key in ScoreByCategory
    ], names=[ScoreByCategory, ScoreCategories], dtype=np.int32)

    out["score_by_vital"] = named_array.NamedNumpyArray([
        get_score_details(key, score_details, ScoreVitals)
        for key in ScoreByVital
    ], names=[ScoreByVital, ScoreVitals], dtype=np.int32)

    player = obs.observation.player_common
    out["player"] = named_array.NamedNumpyArray([
        player.player_id,
        player.minerals,
        player.vespene,
        player.food_used,
        player.food_cap,
        player.food_army,
        player.food_workers,
        player.idle_worker_count,
        player.army_count,
        player.warp_gate_count,
        player.larva_count,
    ], names=Player, dtype=np.int32)

    def unit_vec(u):
      return np.array((
          u.unit_type,
          u.player_relative,
          u.health,
          u.shields,
          u.energy,
          u.transport_slots_taken,
          int(u.build_progress * 100),  # discretize
      ), dtype=np.int32)

    ui = obs.observation.ui_data

    with sw("ui"):
      groups = np.zeros((10, 2), dtype=np.int32)
      for g in ui.groups:
        groups[g.control_group_index, :] = (g.leader_unit_type, g.count)
      out["control_groups"] = groups

      if ui.single:
        out["single_select"] = named_array.NamedNumpyArray(
            [unit_vec(ui.single.unit)], [None, UnitLayer])

      if ui.multi and ui.multi.units:
        out["multi_select"] = named_array.NamedNumpyArray(
            [unit_vec(u) for u in ui.multi.units], [None, UnitLayer])

      if ui.cargo and ui.cargo.passengers:
        out["single_select"] = named_array.NamedNumpyArray(
            [unit_vec(ui.single.unit)], [None, UnitLayer])
        out["cargo"] = named_array.NamedNumpyArray(
            [unit_vec(u) for u in ui.cargo.passengers], [None, UnitLayer])
        out["cargo_slots_available"] = np.array([ui.cargo.slots_available],
                                                dtype=np.int32)

      if ui.production and ui.production.build_queue:
        out["single_select"] = named_array.NamedNumpyArray(
            [unit_vec(ui.production.unit)], [None, UnitLayer])
        out["build_queue"] = named_array.NamedNumpyArray(
            [unit_vec(u) for u in ui.production.build_queue],
            [None, UnitLayer])

    def full_unit_vec(u, pos_transform, is_raw=False):
      screen_pos = pos_transform.fwd_pt(
          point.Point.build(u.pos))
      screen_radius = pos_transform.fwd_dist(u.radius)
      return np.array((
          # Match unit_vec order
          u.unit_type,
          u.alliance,  # Self = 1, Ally = 2, Neutral = 3, Enemy = 4
          u.health,
          u.shield,
          u.energy,
          u.cargo_space_taken,
          int(u.build_progress * 100),  # discretize

          # Resume API order
          int(u.health / u.health_max * 255) if u.health_max > 0 else 0,
          int(u.shield / u.shield_max * 255) if u.shield_max > 0 else 0,
          int(u.energy / u.energy_max * 255) if u.energy_max > 0 else 0,
          u.display_type,  # Visible = 1, Snapshot = 2, Hidden = 3
          u.owner,  # 1-15, 16 = neutral
          screen_pos.x,
          screen_pos.y,
          u.facing,
          screen_radius,
          u.cloak,  # Cloaked = 1, CloakedDetected = 2, NotCloaked = 3
          u.is_selected,
          u.is_blip,
          u.is_powered,
          u.mineral_contents,
          u.vespene_contents,

          # Not populated for enemies or neutral
          u.cargo_space_max,
          u.assigned_harvesters,
          u.ideal_harvesters,
          u.weapon_cooldown,
          len(u.orders),
          u.tag if is_raw else 0
      ), dtype=np.int32)

    raw = obs.observation.raw_data

    if aif.use_feature_units:
      with sw("feature_units"):
        # Update the camera location so we can calculate world to screen pos
        self._update_camera(point.Point.build(raw.player.camera))
        feature_units = []
        for u in raw.units:
          if u.is_on_screen and u.display_type != sc_raw.Hidden:
            feature_units.append(
                full_unit_vec(u, self._world_to_feature_screen_px))
        out["feature_units"] = named_array.NamedNumpyArray(
            feature_units, [None, FeatureUnit], dtype=np.int32)

    if aif.use_raw_units:
      with sw("raw_units"):
        raw_units = [full_unit_vec(u, self._world_to_world_tl, is_raw=True)
                     for u in raw.units]
        out["raw_units"] = named_array.NamedNumpyArray(
            raw_units, [None, FeatureUnit], dtype=np.int32)

    if aif.use_unit_counts:
      with sw("unit_counts"):
        unit_counts = collections.defaultdict(int)
        for u in raw.units:
          if u.alliance == sc_raw.Self:
            unit_counts[u.unit_type] += 1
        out["unit_counts"] = named_array.NamedNumpyArray(
            sorted(unit_counts.items()), [None, UnitCounts], dtype=np.int32)

    if aif.use_camera_position:
      camera_position = self._world_to_world_tl.fwd_pt(
          point.Point.build(raw.player.camera))
      out["camera_position"] = np.array((camera_position.x, camera_position.y),
                                        dtype=np.int32)

    out["available_actions"] = np.array(self.available_actions(obs.observation),
                                        dtype=np.int32)

    return out
1114250
def to_dict(self) -> Dict[str, List[int]]:
        
        return {fn: list(lines)
                for (fn, lines) in self.__contents.items()}
518336
def screen(self):
        """"""
        # opengl windows cannot get from it's hwnd, so we use the screen
        hwnd = win32gui.GetDesktopWindow()

        # get window size and offset
        left, top, right, bottom = self.rect
        width, height = right-left, bottom-top

        # the device context of the window 
        hdcwin = win32gui.GetWindowDC(hwnd)
        # make a temporary dc
        hdcmem = win32gui.CreateCompatibleDC(hdcwin)
        # make a temporary bitmap in memory, this is a PyHANDLE object
        hbmp = win32gui.CreateCompatibleBitmap(hdcwin, width, height)
        # select bitmap for temporary dc
        win32gui.SelectObject(hdcmem, hbmp)
        # copy bits to temporary dc
        win32gui.BitBlt(hdcmem, 0, 0, width, height, 
                        hdcwin, left, top, win32con.SRCCOPY)
        # check the bitmap object infomation
        bmp = win32gui.GetObject(hbmp)

        bi = BITMAPINFOHEADER()
        bi.biSize = ctypes.sizeof(BITMAPINFOHEADER)
        bi.biWidth = bmp.bmWidth
        bi.biHeight = bmp.bmHeight
        bi.biPlanes = bmp.bmPlanes
        bi.biBitCount = bmp.bmBitsPixel
        bi.biCompression = 0 # BI_RGB
        bi.biSizeImage = 0
        bi.biXPelsPerMeter = 0
        bi.biYPelsPerMeter = 0
        bi.biClrUsed = 0
        bi.biClrImportant = 0

        # calculate total size for bits
        pixel = bmp.bmBitsPixel
        size = ((bmp.bmWidth * pixel + pixel - 1)/pixel) * 4 * bmp.bmHeight
        buf = (ctypes.c_char * size)()

        # read bits into buffer
        windll.gdi32.GetDIBits(hdcmem, hbmp.handle, 0, bmp.bmHeight, buf, ctypes.byref(bi), win32con.DIB_RGB_COLORS)

        # make a PIL Image
        img = Image.frombuffer('RGB', (bmp.bmWidth, bmp.bmHeight), buf, 'raw', 'BGRX', 0, 1)
        img = img.transpose(Image.FLIP_TOP_BOTTOM)

        # cleanup
        win32gui.DeleteObject(hbmp)
        win32gui.DeleteObject(hdcmem)
        win32gui.ReleaseDC(hwnd, hdcwin)

        return img