Process HDFS Datanode metrics from given beans
544083
def _hdfs_datanode_metrics(self, beans, tags):
        
        # Only get the first bean
        bean = next(iter(beans))
        bean_name = bean.get('name')

        self.log.debug("Bean name retrieved: {}".format(bean_name))

        for metric, (metric_name, metric_type) in iteritems(self.HDFS_METRICS):
            metric_value = bean.get(metric)
            if metric_value is not None:
                self._set_metric(metric_name, metric_type, metric_value, tags)
444238
private List<HdfsStats> createFromScanResults(String cluster, String path,
      Scan scan, int maxCount, boolean checkPath, long starttime, long endtime)
      throws IOException {
    Map<HdfsStatsKey, HdfsStats> hdfsStats =
        new HashMap<HdfsStatsKey, HdfsStats>();
    ResultScanner scanner = null;
    Stopwatch timer = new Stopwatch().start();
    int rowCount = 0;
    long colCount = 0;
    long resultSize = 0;

    Table hdfsUsageTable = null;

    try {
      hdfsUsageTable = hbaseConnection
          .getTable(TableName.valueOf(HdfsConstants.HDFS_USAGE_TABLE));
      scanner = hdfsUsageTable.getScanner(scan);
      for (Result result : scanner) {
        if (result != null && !result.isEmpty()) {
          colCount += result.size();
          // TODO dogpiledays resultSize += result.getWritableSize();
          rowCount = populateHdfsStats(result, hdfsStats, checkPath, path,
              starttime, endtime, rowCount);
          // return if we've already hit the limit
          if (rowCount >= maxCount) {
            break;
          }
        }
      }

      timer.stop();
      LOG.info("In createFromScanResults For cluster " + cluster
          + " Fetched from hbase " + rowCount + " rows, " + colCount
          + " columns, " + resultSize + " bytes ( " + resultSize / (1024 * 1024)
          + ") MB, in total time of " + timer);
    } finally {
      try {
        if (scanner != null) {
          scanner.close();
        }
      } finally {
        if (hdfsUsageTable != null) {
          hdfsUsageTable.close();
        }
      }
    }

    List<HdfsStats> values = new ArrayList<HdfsStats>(hdfsStats.values());
    // sort so that timestamps are arranged in descending order
    Collections.sort(values);
    return values;
  }
1265426
def get_datasets_in_nodes():
    

    data_dir = os.path.join(scriptdir, "..", "usgs", "data")

    cwic = map(lambda d: d["datasetName"], api.datasets(None, CWIC_LSI_EXPLORER_CATALOG_NODE)['data'])
    ee = map(lambda d: d["datasetName"], api.datasets(None, EARTH_EXPLORER_CATALOG_NODE)['data'])
    hdds = map(lambda d: d["datasetName"], api.datasets(None, HDDS_EXPLORER_CATALOG_NODE)['data'])
    lpcs = map(lambda d: d["datasetName"], api.datasets(None, LPCS_EXPLORER_CATALOG_NODE)['data'])

    # Create mapping from dataset to node
    datasets = {}
    datasets.update( { ds : "CWIC" for ds in cwic } )
    datasets.update( { ds : "EE" for ds in ee } )
    datasets.update( { ds : "HDDS" for ds in hdds } )
    datasets.update( { ds : "LPCS" for ds in lpcs } )

    datasets_path = os.path.join(data_dir, "datasets.json")
    with open(datasets_path, "w") as f:
        f.write(json.dumps(datasets))

    # Find the datasets with ambiguous nodes
    cwic_ee = [ds for ds in cwic if ds in ee]
    cwic_hdds = [ds for ds in cwic if ds in hdds]
    cwic_lpcs = [ds for ds in cwic if ds in lpcs]
    ee_hdds = [ds for ds in ee if ds in hdds]
    ee_lpcs = [ds for ds in ee if ds in lpcs]
    hdds_lpcs = [ds for ds in hdds if ds in lpcs]
325014
private void readMbean() {
        int i = 0;
        for (GarbageCollectorMXBean bean : mxBeans) {
            GCStats stat = (GCStats) iStatsList.get(i);
            stat.update(bean.getCollectionCount(), bean.getCollectionTime());
            i++;
        }
    }
604003
def df(self, src):
        ''''''
        return self._getStdOutCmd([self._hadoop_cmd, 'fs', '-df', self._full_hdfs_path(src)], True)
748638
def serialize(df, measurement, tag_columns=None, **extra_tags) -> bytes:
    """"""
    # Pre-processing
    if measurement is None:
        raise ValueError("Missing 'measurement'")
    if not isinstance(df.index, pd.DatetimeIndex):
        raise ValueError('DataFrame index is not DatetimeIndex')
    tag_columns = set(tag_columns or [])
    isnull = df.isnull().any(axis=1)

    # Make parser function
    tags = []
    fields = []
    for k, v in extra_tags.items():
        tags.append(f"{k}={escape(v, key_escape)}")
    for i, (k, v) in enumerate(df.dtypes.items()):
        k = k.translate(key_escape)
        if k in tag_columns:
            tags.append(f"{k}={{p[{i+1}]}}")
        elif issubclass(v.type, np.integer):
            fields.append(f"{k}={{p[{i+1}]}}i")
        elif issubclass(v.type, (np.float, np.bool_)):
            fields.append(f"{k}={{p[{i+1}]}}")
        else:
            # String escaping is skipped for performance reasons
            # Strings containing double-quotes can cause strange write errors
            # and should be sanitized by the user.
            # e.g., df[k] = df[k].astype('str').str.translate(str_escape)
            fields.append(f"{k}=\"{{p[{i+1}]}}\"")
    fmt = (f'{measurement}', f'{"," if tags else ""}', ','.join(tags),
           ' ', ','.join(fields), ' {p[0].value}')
    f = eval("lambda p: f'{}'".format(''.join(fmt)))

    # Map/concat
    if isnull.any():
        lp = map(f, _itertuples(df[~isnull]))
        rep = _replace(df)
        lp_nan = (reduce(lambda a, b: re.sub(*b, a), rep, f(p))
                  for p in _itertuples(df[isnull]))
        return '\n'.join(chain(lp, lp_nan)).encode('utf-8')
    else:
        return '\n'.join(map(f, _itertuples(df))).encode('utf-8')
105227
def dump(data, hdfs_path, **kwargs):
    
    kwargs["mode"] = "w" if isinstance(data, bintype) else "wt"
    with open(hdfs_path, **kwargs) as fo:
        i = 0
        bufsize = common.BUFSIZE
        while i < len(data):
            fo.write(data[i: i + bufsize])
            i += bufsize
    fo.fs.close()
1053650
def get_config_from_env(cls):
        
        core_path = os.path.join(os.environ['HADOOP_HOME'], 'conf', 'core-site.xml')
        core_configs = cls.read_core_config(core_path)

        hdfs_path = os.path.join(os.environ['HADOOP_HOME'], 'conf', 'hdfs-site.xml')
        hdfs_configs = cls.read_hdfs_config(hdfs_path)

        if (not core_configs) and (not hdfs_configs):
            raise Exception("No config found in %s nor in %s" % (core_path, hdfs_path))

        configs = {
            'use_trash': hdfs_configs.get('use_trash', core_configs.get('use_trash', False)),
            'use_sasl': core_configs.get('use_sasl', False),
            'hdfs_namenode_principal': hdfs_configs.get('hdfs_namenode_principal', None),
            'namenodes': hdfs_configs.get('namenodes', []) or core_configs.get('namenodes', [])
        }

        return configs
769518
@Override
    public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException
    {
        ValidationMetadata validation = (ValidationMetadata) components.get(MetadataType.VALIDATION);
        StatsMetadata stats = (StatsMetadata) components.get(MetadataType.STATS);
        CompactionMetadata compaction = (CompactionMetadata) components.get(MetadataType.COMPACTION);

        assert validation != null && stats != null && compaction != null && validation.partitioner != null;

        EstimatedHistogram.serializer.serialize(stats.estimatedRowSize, out);
        EstimatedHistogram.serializer.serialize(stats.estimatedColumnCount, out);
        ReplayPosition.serializer.serialize(stats.replayPosition, out);
        out.writeLong(stats.minTimestamp);
        out.writeLong(stats.maxTimestamp);
        out.writeInt(stats.maxLocalDeletionTime);
        out.writeDouble(validation.bloomFilterFPChance);
        out.writeDouble(stats.compressionRatio);
        out.writeUTF(validation.partitioner);
        out.writeInt(compaction.ancestors.size());
        for (Integer g : compaction.ancestors)
            out.writeInt(g);
        StreamingHistogram.serializer.serialize(stats.estimatedTombstoneDropTime, out);
        out.writeInt(stats.sstableLevel);
        out.writeInt(stats.minColumnNames.size());
        for (ByteBuffer columnName : stats.minColumnNames)
            ByteBufferUtil.writeWithShortLength(columnName, out);
        out.writeInt(stats.maxColumnNames.size());
        for (ByteBuffer columnName : stats.maxColumnNames)
            ByteBufferUtil.writeWithShortLength(columnName, out);
    }
507106
private static void fillValue2Node(List<MetricInfo> componentMetrics, Map<String, TopologyNode> nodes) {
        String NODE_DIM = MetricDef.EMMITTED_NUM;
        List<String> FILTER = Arrays.asList(MetricDef.EMMITTED_NUM, MetricDef.SEND_TPS, MetricDef.RECV_TPS);
        for (MetricInfo info : componentMetrics) {
            if (info == null) continue;
            for (Map.Entry<String, Map<Integer, MetricSnapshot>> metric : info.get_metrics().entrySet()) {
                String name = metric.getKey();
                String[] split_name = name.split("@");
                String metricName = UIMetricUtils.extractMetricName(split_name);
                String compName = UIMetricUtils.extractComponentName(split_name);

                TopologyNode node = nodes.get(compName);
                if (node != null && FILTER.contains(metricName)) {
                    for (Map.Entry<Integer, MetricSnapshot> winData : metric.getValue().entrySet()) {
                        node.putMapValue(metricName, winData.getKey(),
                                UIMetricUtils.getMetricValue(winData.getValue()));
                    }
                }

                if (metricName == null || !metricName.equals(NODE_DIM)) {
                    continue;
                }

                //get 60 window metric
                MetricSnapshot snapshot = metric.getValue().get(AsmWindow.M1_WINDOW);

                if (node != null) {
                    node.setValue(snapshot.get_longValue());
                    nodes.get(compName).setTitle("Emitted: " + UIMetricUtils.getMetricValue(snapshot));
                }
            }
        }
    }
182280
def convert_trig_to_hdf(workflow, hdfbank, xml_trigger_files, out_dir, tags=None):
    """"""
    if tags is None:
        tags = []
    #FIXME, make me not needed
    logging.info('convert single inspiral trigger files to hdf5')
    make_analysis_dir(out_dir)

    trig_files = FileList()
    for ifo, insp_group in zip(*xml_trigger_files.categorize_by_attr('ifo')):
        trig2hdf_exe = PyCBCTrig2HDFExecutable(workflow.cp, 'trig2hdf',
                                       ifos=ifo, out_dir=out_dir, tags=tags)
        _, insp_bundles = insp_group.categorize_by_attr('segment')
        for insps in insp_bundles:
            trig2hdf_node =  trig2hdf_exe.create_node(insps, hdfbank[0])
            workflow.add_node(trig2hdf_node)
            trig_files += trig2hdf_node.output_files
    return trig_files
201717
def refresh_datasources(self, refreshAll=True):
        """"""
        session = db.session()
        DruidCluster = ConnectorRegistry.sources['druid'].cluster_class
        for cluster in session.query(DruidCluster).all():
            cluster_name = cluster.cluster_name
            valid_cluster = True
            try:
                cluster.refresh_datasources(refreshAll=refreshAll)
            except Exception as e:
                valid_cluster = False
                flash(
                    "Error while processing cluster '{}'\n{}".format(
                        cluster_name, utils.error_msg_from_exception(e)),
                    'danger')
                logging.exception(e)
                pass
            if valid_cluster:
                cluster.metadata_last_refreshed = datetime.now()
                flash(
                    _('Refreshed metadata from cluster [{}]').format(
                        cluster.cluster_name),
                    'info')
        session.commit()
        return redirect('/druiddatasourcemodelview/list/')
140773
function buildNodesStats (opts) {
  // eslint-disable-next-line no-unused-vars
  const { makeRequest, ConfigurationError, handleError, snakeCaseKeys } = opts
  /**
   * Perform a [nodes.stats](http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-stats.html) request
   *
   * @param {list} metric - Limit the information returned to the specified metrics
   * @param {list} index_metric - Limit the information returned for `indices` metric to the specific index metrics. Isn't used if `indices` (or `all`) metric isn't specified.
   * @param {list} node_id - A comma-separated list of node IDs or names to limit the returned information; use `_local` to return information from the node you're connecting to, leave empty to get information from all nodes
   * @param {list} completion_fields - A comma-separated list of fields for `fielddata` and `suggest` index metric (supports wildcards)
   * @param {list} fielddata_fields - A comma-separated list of fields for `fielddata` index metric (supports wildcards)
   * @param {list} fields - A comma-separated list of fields for `fielddata` and `completion` index metric (supports wildcards)
   * @param {boolean} groups - A comma-separated list of search groups for `search` index metric
   * @param {enum} level - Return indices stats aggregated at index, node or shard level
   * @param {list} types - A comma-separated list of document types for the `indexing` index metric
   * @param {time} timeout - Explicit operation timeout
   * @param {boolean} include_segment_file_sizes - Whether to report the aggregated disk usage of each one of the Lucene index files (only applies if segment stats are requested)
   */

  const acceptedQuerystring = [
    'completion_fields',
    'fielddata_fields',
    'fields',
    'groups',
    'level',
    'types',
    'timeout',
    'include_segment_file_sizes',
    'pretty',
    'human',
    'error_trace',
    'source',
    'filter_path'
  ]

  const snakeCase = {
    completionFields: 'completion_fields',
    fielddataFields: 'fielddata_fields',
    includeSegmentFileSizes: 'include_segment_file_sizes',
    errorTrace: 'error_trace',
    filterPath: 'filter_path'
  }

  return function nodesStats (params, options, callback) {
    options = options || {}
    if (typeof options === 'function') {
      callback = options
      options = {}
    }
    if (typeof params === 'function' || params == null) {
      callback = params
      params = {}
      options = {}
    }

    // check required parameters
    if (params.body != null) {
      const err = new ConfigurationError('This API does not require a body')
      return handleError(err, callback)
    }

    // validate headers object
    if (options.headers != null && typeof options.headers !== 'object') {
      const err = new ConfigurationError(`Headers should be an object, instead got: ${typeof options.headers}`)
      return handleError(err, callback)
    }

    var warnings = []
    var { method, body, metric, indexMetric, index_metric, nodeId, node_id, ...querystring } = params
    querystring = snakeCaseKeys(acceptedQuerystring, snakeCase, querystring, warnings)

    if (method == null) {
      method = 'GET'
    }

    var ignore = options.ignore
    if (typeof ignore === 'number') {
      options.ignore = [ignore]
    }

    var path = ''

    if ((node_id || nodeId) != null && (metric) != null && (index_metric || indexMetric) != null) {
      path = '/' + '_nodes' + '/' + encodeURIComponent(node_id || nodeId) + '/' + 'stats' + '/' + encodeURIComponent(metric) + '/' + encodeURIComponent(index_metric || indexMetric)
    } else if ((node_id || nodeId) != null && (metric) != null) {
      path = '/' + '_nodes' + '/' + encodeURIComponent(node_id || nodeId) + '/' + 'stats' + '/' + encodeURIComponent(metric)
    } else if ((metric) != null && (index_metric || indexMetric) != null) {
      path = '/' + '_nodes' + '/' + 'stats' + '/' + encodeURIComponent(metric) + '/' + encodeURIComponent(index_metric || indexMetric)
    } else if ((node_id || nodeId) != null) {
      path = '/' + '_nodes' + '/' + encodeURIComponent(node_id || nodeId) + '/' + 'stats'
    } else if ((metric) != null) {
      path = '/' + '_nodes' + '/' + 'stats' + '/' + encodeURIComponent(metric)
    } else {
      path = '/' + '_nodes' + '/' + 'stats'
    }

    // build request object
    const request = {
      method,
      path,
      body: null,
      querystring
    }

    options.warnings = warnings.length === 0 ? null : warnings
    return makeRequest(request, options, callback)
  }
}
825243
private synchronized void initFsDataSet(Configuration conf,
      AbstractList<File> dataDirs, int numNamespaces) throws IOException {
    if (data != null) { // Already initialized
      return;
    }

    // get version and id info from the name-node
    boolean simulatedFSDataset = 
      conf.getBoolean("dfs.datanode.simulateddatastorage", false);

    if (simulatedFSDataset) {
      storage.createStorageID(selfAddr.getPort());
      // it would have been better to pass storage as a parameter to
      // constructor below - need to augment ReflectionUtils used below.
      conf.set("dfs.datanode.StorageId", storage.getStorageID());
      try {
        data = (FSDatasetInterface) ReflectionUtils.newInstance(
            Class.forName(
            "org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset"),
            conf);
      } catch (ClassNotFoundException e) {
        throw new IOException(StringUtils.stringifyException(e));
      }
    } else {
      data = new FSDataset(this, conf, numNamespaces);
    }
  }
153806
def process_schema(doc, resource, df):
    """"""
    from rowgenerators import SourceError
    from requests.exceptions import ConnectionError

    from metapack.cli.core import extract_path_name, alt_col_name, type_map
    from tableintuit import TypeIntuiter
    from rowgenerators.generator.python import PandasDataframeSource
    from appurl import parse_app_url

    try:
        doc['Schema']
    except KeyError:
        doc.new_section('Schema', ['DataType', 'Altname', 'Description'])

    schema_name = resource.get_value('schema', resource.get_value('name'))

    schema_term = doc.find_first(term='Table', value=schema_name, section='Schema')

    if schema_term:
        logger.info("Found table for '{}'; skipping".format(schema_name))
        return

    path, name = extract_path_name(resource.url)

    logger.info("Processing {}".format(resource.url))

    si = PandasDataframeSource(parse_app_url(resource.url), df, cache=doc._cache, )

    try:
        ti = TypeIntuiter().run(si)
    except SourceError as e:
        logger.warn("Failed to process '{}'; {}".format(path, e))
        return
    except ConnectionError as e:
        logger.warn("Failed to download '{}'; {}".format(path, e))
        return

    table = doc['Schema'].new_term('Table', schema_name)

    logger.info("Adding table '{}' to metatab schema".format(schema_name))

    for i, c in enumerate(ti.to_rows()):
        raw_alt_name = alt_col_name(c['header'], i)
        alt_name = raw_alt_name if raw_alt_name != c['header'] else ''

        t = table.new_child('Column', c['header'],
                            datatype=type_map.get(c['resolved_type'], c['resolved_type']),
                            altname=alt_name,
                            description=df[c['header']].description \
                                        if hasattr(df, 'description') and df[c['header']].description else ''
                            )

    return table
493399
def df(unit = 'GB'):
    ''''''
    details = {}
    headers = ['Filesystem', 'Type', 'Size', 'Used', 'Available', 'Capacity', 'MountedOn']
    n = len(headers)

    unit = df_conversions[unit]
    p = subprocess.Popen(args = ['df', '-TP'], stdout = subprocess.PIPE) # -P prevents line wrapping on long filesystem names
    stdout, stderr = p.communicate()

    lines = stdout.split("\n")
    lines[0] = lines[0].replace("Mounted on", "MountedOn").replace("1K-blocks", "Size").replace("1024-blocks", "Size")
    assert(lines[0].split() == headers)

    lines = [l.strip() for l in lines if l.strip()]
    for line in lines[1:]:
        tokens = line.split()
        if tokens[0] == 'none': # skip uninteresting entries
            continue

        assert(len(tokens) == n)
        d = {}
        for x in range(1, len(headers)):
            d[headers[x]] = tokens[x]
        d['Size'] = float(d['Size']) / unit
        assert(d['Capacity'].endswith("%"))
        d['Use%'] = d['Capacity']
        d['Used'] = float(d['Used']) / unit
        d['Available'] = float(d['Available']) / unit
        d['Using'] = 100*(d['Used']/d['Size']) # same as Use% but with more precision

        if d['Type'].startswith('ext'):
            pass
            d['Using'] += 5 # ext2, ext3, and ext4 reserve 5% by default
        else:
            ext3_filesystems = ['ganon:', 'kortemmelab:', 'albana:']
            for e3fs in ext3_filesystems:
                if tokens[0].find(e3fs) != -1:
                    d['Using'] += 5 # ext3 reserves 5%
                    break

        details[tokens[0]] = d

    return details
133719
def process_raw_data(cls, raw_data):
        """"""
        properties = raw_data["properties"]

        ip_pools = []
        for raw_content in properties.get("ipPools", []):
            raw_content["parentResourceID"] = raw_data["resourceId"]
            raw_content["grandParentResourceID"] = raw_data["parentResourceID"]
            ip_pools.append(IPPools.from_raw_data(raw_content))
        properties["ipPools"] = ip_pools

        ip_configurations = []
        for raw_content in properties.get("ipConfigurations", []):
            resource = Resource.from_raw_data(raw_content)
            ip_configurations.append(resource)
        properties["ipConfigurations"] = ip_configurations

        network_interfaces = []
        for raw_content in properties.get("networkInterfaces", []):
            resource = Resource.from_raw_data(raw_content)
            network_interfaces.append(resource)
        properties["networkInterfaces"] = network_interfaces

        return super(LogicalSubnetworks, cls).process_raw_data(raw_data)