Retrieve stats files from pre-existing dictionary or filesystem.
326702
def _get_stats_files(data, out_dir=None):
    
    if not out_dir:
        out_dir = utils.safe_makedir(os.path.join(dd.get_work_dir(data),
                                                  "qc", dd.get_sample_name(data), "samtools"))
    stats_file = tz.get_in(["depth", "samtools", "stats"], data)
    idxstats_file = tz.get_in(["depth", "samtools", "idxstats"], data)
    if not stats_file:
        stats_file = os.path.join(out_dir, "%s.txt" % dd.get_sample_name(data))
    if not idxstats_file:
        idxstats_file = os.path.join(out_dir, "%s-idxstats.txt" % dd.get_sample_name(data))
    return stats_file, idxstats_file
846872
def load_existing_json():
        
        dir = logger.get_logger_dir()
        fname = os.path.join(dir, JSONWriter.FILENAME)
        if tf.gfile.Exists(fname):
            with open(fname) as f:
                stats = json.load(f)
                assert isinstance(stats, list), type(stats)
                return stats
        return None
98624
def stat(filename):
    """"""
    import os
    try:
        # on the host, lstat won't try to follow symlinks
        rstat = os.lstat(filename)
    except:
        rstat = os.stat(filename)
    return rstat[:7] + tuple(tim + TIME_OFFSET for tim in rstat[7:])
1062846
def getProcessStats(self):
        
        info_dict = {}
        try:
            fp = open(cpustatFile, 'r')
            data = fp.read()
            fp.close()
        except:
            raise IOError('Failed reading stats from file: %s' % cpustatFile)
        for line in data.splitlines():
            arr = line.split()
            if len(arr) > 1 and arr[0] in ('ctxt', 'intr', 'softirq',
                                           'processes', 'procs_running', 
                                           'procs_blocked'):
                info_dict[arr[0]] = arr[1]
        return info_dict
687711
def unpickle_stats(stats):
    """"""
    stats = cPickle.loads(stats)
    stats.stream = True
    return stats
172254
def _load_fsstat_data(self, timeout=3):
        """"""

        def stats_thread():
            try:
                cmd = ['fsstat', self.get_raw_path(), '-o', str(self.offset // self.disk.block_size)]

                # Setting the fstype explicitly makes fsstat much faster and more reliable
                # In some versions, the auto-detect yaffs2 check takes ages for large images
                fstype = {
                    "ntfs": "ntfs", "fat": "fat", "ext": "ext", "iso": "iso9660", "hfs+": "hfs",
                    "ufs": "ufs", "swap": "swap", "exfat": "exfat",
                }.get(self.fstype, None)
                if fstype:
                    cmd.extend(["-f", fstype])

                logger.debug('$ {0}'.format(' '.join(cmd)))
                stats_thread.process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

                for line in iter(stats_thread.process.stdout.readline, b''):
                    line = line.decode('utf-8')
                    logger.debug('< {0}'.format(line))
                    if line.startswith("File System Type:"):
                        self.info['statfstype'] = line[line.index(':') + 2:].strip()
                    elif line.startswith("Last Mount Point:") or line.startswith("Last mounted on:"):
                        self.info['lastmountpoint'] = line[line.index(':') + 2:].strip().replace("//", "/")
                    elif line.startswith("Volume Name:") and not self.info.get('label'):
                        self.info['label'] = line[line.index(':') + 2:].strip()
                    elif line.startswith("Version:"):
                        self.info['version'] = line[line.index(':') + 2:].strip()
                    elif line.startswith("Source OS:"):
                        self.info['version'] = line[line.index(':') + 2:].strip()
                    elif 'CYLINDER GROUP INFORMATION' in line or 'BLOCK GROUP INFORMATION' in line:
                        # noinspection PyBroadException
                        try:
                            stats_thread.process.terminate()
                            logger.debug("Terminated fsstat at cylinder/block group information.")
                        except Exception:
                            pass
                        break

                if self.info.get('lastmountpoint') and self.info.get('label'):
                    self.info['label'] = "{0} ({1})".format(self.info['lastmountpoint'], self.info['label'])
                elif self.info.get('lastmountpoint') and not self.info.get('label'):
                    self.info['label'] = self.info['lastmountpoint']
                elif not self.info.get('lastmountpoint') and self.info.get('label') and \
                        self.info['label'].startswith("/"):  # e.g. /boot1
                    if self.info['label'].endswith("1"):
                        self.info['lastmountpoint'] = self.info['label'][:-1]
                    else:
                        self.info['lastmountpoint'] = self.info['label']

            except Exception:  # ignore any exceptions here.
                logger.exception("Error while obtaining stats.")

        stats_thread.process = None

        thread = threading.Thread(target=stats_thread)
        thread.start()
        thread.join(timeout)
        if thread.is_alive():
            # noinspection PyBroadException
            try:
                stats_thread.process.terminate()
            except Exception:
                pass
            thread.join()
            logger.debug("Killed fsstat after {0}s".format(timeout))
68514
def aggregate_stats(stats_files):
  """"""
  all_stats = {}
  for fname in stats_files:
    with tf.gfile.Open(fname) as f:
      stats = json.loads(f.read())
      for k, v in stats.iteritems():
        if k not in all_stats:
          if isinstance(v, list):
            all_stats[k] = []
          else:
            all_stats[k] = 0

        if isinstance(v, list):
          all_stats[k].extend(v)
        else:
          all_stats[k] += v

  stats = all_stats
  ref_coverage = float(stats["total_found_refs"]) / stats["total_original_refs"]
  len_bounds = [0, 2, 10, 100, 1000, 5000, 10000, 20000, 50000, 100000, 1000000]
  len_counts, len_bounds = np.histogram(stats["ref_lengths"], len_bounds)
  len_dist = len_counts.astype(np.float32) / len_counts.sum()
  wiki_coverage = (float(stats["num_wikis_written"]) /
                   stats["total_original_wikis"])
  wikis_skipped_no_ref = (float(stats["wikis_skipped_no_refs"]) /
                          stats["total_original_wikis"])
  wikis_skipped_no_lead = (float(stats["wikis_skipped_short_lead"]) /
                           stats["total_original_wikis"])
  wiki_ref_coverage = [
      float(found) / orig for found, orig
      in zip(stats["wiki_found_refs"], stats["wiki_original_refs"]) if found
  ]
  coverage_bounds = np.arange(21).astype(np.float32) / 20
  coverage_counts, coverage_bounds = np.histogram(wiki_ref_coverage,
                                                  coverage_bounds)
  coverage_dist = coverage_counts.astype(np.float32) / coverage_counts.sum()

  agg_stats = dict(
      total_original_wikis=stats["total_original_wikis"],
      total_original_refs=stats["total_original_refs"],
      wiki_coverage=wiki_coverage,
      wikis_skipped_no_ref=wikis_skipped_no_ref,
      wikis_skipped_no_lead=wikis_skipped_no_lead,
      overall_ref_coverage=ref_coverage,
      per_wiki_ref_coverage_dist=list((coverage_dist * 100).astype(int)),
      per_wiki_ref_coverage_bounds=list((coverage_bounds * 100).astype(int)),
      ref_len_dist=list((len_dist * 100).astype(int)),
      ref_len_bounds=list(len_bounds),
  )
  return agg_stats
68820
def getFilesystemStats(self, fs):
        
        if self._mapFSpathDev is None:
            self._initFilesystemInfo()
        return self._diskStats.get(self._mapFSpathDev.get(fs))
1040776
def store_stats(self):
    """"""
    stats = self._stats()

    # Write stats to user-defined json file.
    stats_json_file_name = self.get_options().stats_local_json_file
    if stats_json_file_name:
      self.write_stats_to_json(stats_json_file_name, stats)

    # Upload to remote stats db.
    stats_upload_urls = copy.copy(self.get_options().stats_upload_urls)
    timeout = self.get_options().stats_upload_timeout
    for stats_url, auth_provider in stats_upload_urls.items():
      self.post_stats(stats_url, stats, timeout=timeout, auth_provider=auth_provider)
970809
def sample_stats_to_xarray(self):
        """"""
        data = self.sample_stats
        if not isinstance(data, dict):
            raise TypeError("DictConverter.sample_stats is not a dictionary")

        return dict_to_dataset(data, library=None, coords=self.coords, dims=self.dims)
243815
def load( self, stats ):
        """"""
        rows = self.rows
        for func, raw in stats.iteritems():
            try:
                rows[func] = row = PStatRow( func,raw )
            except ValueError, err:
                log.info( 'Null row: %s', func )
        for row in rows.itervalues():
            row.weave( rows )
        return self.find_root( rows )
361364
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'documents' in _dict:
            args['documents'] = EnvironmentDocuments._from_dict(
                _dict.get('documents'))
        if 'disk_usage' in _dict:
            args['disk_usage'] = DiskUsage._from_dict(_dict.get('disk_usage'))
        if 'collections' in _dict:
            args['collections'] = CollectionUsage._from_dict(
                _dict.get('collections'))
        return cls(**args)
371186
def _GetStat(self):
    
    stat_object = super(APFSFileEntry, self)._GetStat()

    # File data stat information.
    stat_object.size = self._fsapfs_file_entry.size

    # Ownership and permissions stat information.
    stat_object.mode = self._fsapfs_file_entry.file_mode & 0x0fff
    stat_object.uid = self._fsapfs_file_entry.owner_identifier
    stat_object.gid = self._fsapfs_file_entry.group_identifier

    # File entry type stat information.
    stat_object.type = self.entry_type

    # Other stat information.
    stat_object.ino = self._fsapfs_file_entry.identifier
    stat_object.fs_type = 'APFS'

    stat_object.is_allocated = True

    return stat_object
647767
def get_stats_item(self, item):
        
        if isinstance(self.stats, dict):
            try:
                return self._json_dumps({item: self.stats[item]})
            except KeyError as e:
                logger.error("Cannot get item {} ({})".format(item, e))
                return None
        elif isinstance(self.stats, list):
            try:
                # Source:
                # http://stackoverflow.com/questions/4573875/python-get-index-of-dictionary-item-in-list
                # But https://github.com/nicolargo/glances/issues/1401
                return self._json_dumps({item: list(map(itemgetter(item), self.stats))})
            except (KeyError, ValueError) as e:
                logger.error("Cannot get item {} ({})".format(item, e))
                return None
        else:
            return None
861958
def getSwapStats(self):
        
        info_dict = {}
        try:
            fp = open(swapsFile, 'r')
            data = fp.read()
            fp.close()
        except:
            raise IOError('Failed reading stats from file: %s' % swapsFile)
        lines = data.splitlines()
        if len(lines) > 1:
            colnames = [name.lower() for name in lines[0].split()]
            for line in lines[1:]:
                cols = line.split()
                info_dict[cols[0]] = dict(zip(colnames[1:], cols[1:]))
        return info_dict
495592
def get_used_files():
    """"""
    import psutil
    loop_over_scanpy_processes = (proc for proc in psutil.process_iter()
                                  if proc.name() == 'scanpy')
    filenames = []
    for proc in loop_over_scanpy_processes:
        try:
            flist = proc.open_files()
            for nt in flist:
                filenames.append(nt.path)
        # This catches a race condition where a process ends
        # before we can examine its files
        except psutil.NoSuchProcess as err:
            pass
    return set(filenames)
720632
def update(self):
        """"""
        # Init new stats
        stats = self.get_init_value()

        if self.input_method == 'local':
            # Update stats using the standard system lib

            # Grab the stats using the psutil disk_partitions
            # If 'all'=False return physical devices only (e.g. hard disks, cd-rom drives, USB keys)
            # and ignore all others (e.g. memory partitions such as /dev/shm)
            try:
                fs_stat = psutil.disk_partitions(all=False)
            except UnicodeDecodeError:
                return self.stats

            # Optionnal hack to allow logicals mounts points (issue #448)
            # Ex: Had to put 'allow=zfs' in the [fs] section of the conf file
            #     to allow zfs monitoring
            for fstype in self.get_conf_value('allow'):
                try:
                    fs_stat += [f for f in psutil.disk_partitions(all=True) if f.fstype.find(fstype) >= 0]
                except UnicodeDecodeError:
                    return self.stats

            # Loop over fs
            for fs in fs_stat:
                # Do not take hidden file system into account
                if self.is_hide(fs.mountpoint):
                    continue
                # Grab the disk usage
                try:
                    fs_usage = psutil.disk_usage(fs.mountpoint)
                except OSError:
                    # Correct issue #346
                    # Disk is ejected during the command
                    continue
                fs_current = {
                    'device_name': fs.device,
                    'fs_type': fs.fstype,
                    # Manage non breaking space (see issue #1065)
                    'mnt_point': u(fs.mountpoint).replace(u'\u00A0', ' '),
                    'size': fs_usage.total,
                    'used': fs_usage.used,
                    'free': fs_usage.free,
                    'percent': fs_usage.percent,
                    'key': self.get_key()}
                stats.append(fs_current)

        elif self.input_method == 'snmp':
            # Update stats using SNMP

            # SNMP bulk command to get all file system in one shot
            try:
                fs_stat = self.get_stats_snmp(snmp_oid=snmp_oid[self.short_system_name],
                                              bulk=True)
            except KeyError:
                fs_stat = self.get_stats_snmp(snmp_oid=snmp_oid['default'],
                                              bulk=True)

            # Loop over fs
            if self.short_system_name in ('windows', 'esxi'):
                # Windows or ESXi tips
                for fs in fs_stat:
                    # Memory stats are grabbed in the same OID table (ignore it)
                    if fs == 'Virtual Memory' or fs == 'Physical Memory' or fs == 'Real Memory':
                        continue
                    size = int(fs_stat[fs]['size']) * int(fs_stat[fs]['alloc_unit'])
                    used = int(fs_stat[fs]['used']) * int(fs_stat[fs]['alloc_unit'])
                    percent = float(used * 100 / size)
                    fs_current = {
                        'device_name': '',
                        'mnt_point': fs.partition(' ')[0],
                        'size': size,
                        'used': used,
                        'percent': percent,
                        'key': self.get_key()}
                    stats.append(fs_current)
            else:
                # Default behavior
                for fs in fs_stat:
                    fs_current = {
                        'device_name': fs_stat[fs]['device_name'],
                        'mnt_point': fs,
                        'size': int(fs_stat[fs]['size']) * 1024,
                        'used': int(fs_stat[fs]['used']) * 1024,
                        'percent': float(fs_stat[fs]['percent']),
                        'key': self.get_key()}
                    stats.append(fs_current)

        # Update the stats
        self.stats = stats

        return self.stats