Parse the vulnerability model in NRML 0.4 format.

    :param fname:
        path of the vulnerability file
    :returns:
        a dictionary imt, taxonomy -> vulnerability function + vset
420086
def get_vulnerability_functions_04(fname):
    
    categories = dict(assetCategory=set(), lossCategory=set(),
                      vulnerabilitySetID=set())
    imts = set()
    taxonomies = set()
    vf_dict = {}  # imt, taxonomy -> vulnerability function
    for vset in nrml.read(fname).vulnerabilityModel:
        categories['assetCategory'].add(vset['assetCategory'])
        categories['lossCategory'].add(vset['lossCategory'])
        categories['vulnerabilitySetID'].add(vset['vulnerabilitySetID'])
        IML = vset.IML
        imt_str = IML['IMT']
        imls = ~IML
        imts.add(imt_str)
        for vfun in vset.getnodes('discreteVulnerability'):
            taxonomy = vfun['vulnerabilityFunctionID']
            if taxonomy in taxonomies:
                raise InvalidFile(
                    'Duplicated vulnerabilityFunctionID: %s: %s, line %d' %
                    (taxonomy, fname, vfun.lineno))
            taxonomies.add(taxonomy)
            with context(fname, vfun):
                loss_ratios = ~vfun.lossRatio
                coefficients = ~vfun.coefficientsVariation
            if len(loss_ratios) != len(imls):
                raise InvalidFile(
                    'There are %d loss ratios, but %d imls: %s, line %d' %
                    (len(loss_ratios), len(imls), fname,
                     vfun.lossRatio.lineno))
            if len(coefficients) != len(imls):
                raise InvalidFile(
                    'There are %d coefficients, but %d imls: %s, line %d' %
                    (len(coefficients), len(imls), fname,
                     vfun.coefficientsVariation.lineno))
            with context(fname, vfun):
                vf_dict[imt_str, taxonomy] = scientific.VulnerabilityFunction(
                    taxonomy, imt_str, imls, loss_ratios, coefficients,
                    vfun['probabilisticDistribution'])
    categories['id'] = '_'.join(sorted(categories['vulnerabilitySetID']))
    del categories['vulnerabilitySetID']
    return vf_dict, categories
526045
def parse_file(self, f):
        

        if type(f) is str:
            self.f = open(f, 'rb')
        else:
            self.f = f
        self._parse_header(self.f.read(64))
633603
def from_mmap(cls, fname):
        
        memmaped = joblib.load(fname, mmap_mode="r+")
        return cls(vocab=memmaped.vocab, vectors=memmaped.vectors)
909726
def load_from_file(self, fname):
        
        with open(fname, 'r') as f:
            for line in f:
                k,v = line.split(' = ')
                self._parse_char_line_to_self(k,v)
6400
def defconfig_filename(self):
        
        if self.defconfig_list:
            for filename, cond in self.defconfig_list.defaults:
                if expr_value(cond):
                    try:
                        with self._open_config(filename.str_value) as f:
                            return f.name
                    except IOError:
                        continue

        return None
187259
def tarfile_xml_generator(fname):
    
    tf = tarfile.open(fname, mode='r')
    try:
        tarinf = tf.next()
        while tarinf is not None:
            if tarinf.isfile() and tarinf.name[-4:] == '.xml':
                fbuf = tf.extractfile(tarinf)
                yield TarXML(name=tarinf.name, xml=fbuf.read())
            tarinf = tf.next()
            # Kludge around tarfile memory leak, cf
            # http://blogs.it.ox.ac.uk/inapickle/2011/06/20/high-memory-usage-when-using-pythons-tarfile-module/
            tf.members = []
    finally:
        tf.close()
607136
def load_params_from_file(self, fname: str):
        
        utils.check_condition(os.path.exists(fname), "No model parameter file found under %s. "
                                                     "This is either not a model directory or the first training "
                                                     "checkpoint has not happened yet." % fname)
        self.params, self.aux_params = utils.load_params(fname)
        utils.check_condition(all(name.startswith(self.prefix) for name in self.params.keys()),
                              "Not all parameter names start with model prefix '%s'" % self.prefix)
        utils.check_condition(all(name.startswith(self.prefix) for name in self.aux_params.keys()),
                              "Not all auxiliary parameter names start with model prefix '%s'" % self.prefix)
        logger.info('Loaded params from "%s"', fname)
1191602
def find_version(fname):
    
    version = ""
    with open(fname, "r") as fp:
        reg = re.compile(r'__version__ = [\'"]([^\'"]*)[\'"]')
        for line in fp:
            m = reg.match(line)
            if m:
                version = m.group(1)
                break
    if not version:
        raise RuntimeError("Cannot find version information")
    return version
391108
def parse(name, content, releases, get_head_fn):
    
    changelog = {}
    releases = frozenset(releases)
    head = False
    for line in content.splitlines():
        new_head = get_head_fn(name=name, line=line, releases=releases)
        if new_head:
            head = new_head
            changelog[head] = ""
            continue
        if not head:
            continue
        line = line.replace("@", "")
        line = line.replace("#", "")
        changelog[head] += line + "\n"
    return changelog
571152
def parse_fastp_log(self, f):
        
        try:
            parsed_json = json.load(f['f'])
        except:
            log.warn("Could not parse fastp JSON: '{}'".format(f['fn']))
            return None

        # Fetch a sample name from the command
        s_name = f['s_name']
        cmd = parsed_json['command'].split()
        for i,v in enumerate(cmd):
            if v == '-i':
                s_name = self.clean_s_name(cmd[i+1], f['root'])
        if s_name == 'fastp':
            log.warn('Could not parse sample name from fastp command: {}'.format(f['fn']))

        self.add_data_source(f, s_name)
        self.fastp_data[s_name] = {}
        self.fastp_duplication_plotdata[s_name] = {}
        self.fastp_insert_size_data[s_name] = {}
        self.fastp_all_data[s_name] = parsed_json
        for k in ['read1_before_filtering','read2_before_filtering','read1_after_filtering','read2_after_filtering']:
            self.fastp_qual_plotdata[k][s_name] = {}
            self.fastp_gc_content_data[k][s_name] = {}
            self.fastp_n_content_data[k][s_name] = {}

        # Parse filtering_result
        try:
            for k in parsed_json['filtering_result']:
                self.fastp_data[s_name]['filtering_result_{}'.format(k)] = float(parsed_json['filtering_result'][k])
        except KeyError:
            log.debug("fastp JSON did not have 'filtering_result' key: '{}'".format(f['fn']))

        # Parse duplication
        try:
            self.fastp_data[s_name]['pct_duplication'] = float(parsed_json['duplication']['rate'] * 100.0)
        except KeyError:
            log.debug("fastp JSON did not have a 'duplication' key: '{}'".format(f['fn']))

        # Parse after_filtering
        try:
            for k in parsed_json['summary']['after_filtering']:
                self.fastp_data[s_name]['after_filtering_{}'.format(k)] = float(parsed_json['summary']['after_filtering'][k])
        except KeyError:
            log.debug("fastp JSON did not have a 'summary'-'after_filtering' keys: '{}'".format(f['fn']))


        # Parse data required to calculate Pct reads surviving
        try:
            self.fastp_data[s_name]['before_filtering_total_reads'] = float(parsed_json['summary']['before_filtering']['total_reads'])
        except KeyError:
            log.debug("Could not find pre-filtering # reads: '{}'".format(f['fn']))

        try:
            self.fastp_data[s_name]['pct_surviving'] = (self.fastp_data[s_name]['after_filtering_total_reads'] / self.fastp_data[s_name]['before_filtering_total_reads']) * 100.0
        except KeyError:
            log.debug("Could not calculate 'pct_surviving': {}".format(f['fn']))

        # Parse adapter_cutting
        try:
            for k in parsed_json['adapter_cutting']:
                try:
                    self.fastp_data[s_name]['adapter_cutting_{}'.format(k)] = float(parsed_json['adapter_cutting'][k])
                except (ValueError, TypeError):
                    pass
        except KeyError:
            log.debug("fastp JSON did not have a 'adapter_cutting' key, skipping: '{}'".format(f['fn']))

        try:
            self.fastp_data[s_name]['pct_adapter'] = (self.fastp_data[s_name]['adapter_cutting_adapter_trimmed_reads'] / self.fastp_data[s_name]['before_filtering_total_reads']) * 100.0
        except KeyError:
            log.debug("Could not calculate 'pct_adapter': {}".format(f['fn']))

        # Duplication rate plot data
        try:
            # First count the total read count in the dup analysis
            total_reads = 0
            for v in parsed_json['duplication']['histogram']:
                total_reads += v
            if total_reads == 0:
                raise KeyError
            # Calculate percentages
            for i, v in enumerate(parsed_json['duplication']['histogram']):
                self.fastp_duplication_plotdata[s_name][i+1] = (float(v) / float(total_reads)) * 100.0
        except KeyError:
            log.debug("No duplication rate plot data: {}".format(f['fn']))

        # Insert size plot data
        try:
            # First count the total read count in the insert size analysis
            total_reads = 0
            max_i = 0
            for i, v in enumerate(parsed_json['insert_size']['histogram']):
                total_reads += v
                if float(v) > 0:
                    max_i = i
            if total_reads == 0:
                raise KeyError
            # Calculate percentages
            for i, v in enumerate(parsed_json['insert_size']['histogram']):
                if i <= max_i:
                    self.fastp_insert_size_data[s_name][i+1] = (float(v) / float(total_reads)) * 100.0
        except KeyError:
            log.debug("No insert size plot data: {}".format(f['fn']))

        for k in ['read1_before_filtering','read2_before_filtering','read1_after_filtering','read2_after_filtering']:
            # Read quality data
            try:
                for i, v in enumerate(parsed_json[k]['quality_curves']['mean']):
                    self.fastp_qual_plotdata[k][s_name][i+1] = float(v)
            except KeyError:
                log.debug("Read quality {} not found: {}".format(k, f['fn']))

            # GC and N content plots
            try:
                for i, v in enumerate(parsed_json[k]['content_curves']['GC']):
                    self.fastp_gc_content_data[k][s_name][i+1] = float(v) * 100.0
                for i, v in enumerate(parsed_json[k]['content_curves']['N']):
                    self.fastp_n_content_data[k][s_name][i+1] = float(v) * 100.0
            except KeyError:
                log.debug("Content curve data {} not found: {}".format(k, f['fn']))


        # Remove empty dicts
        if len(self.fastp_data[s_name]) == 0:
            del self.fastp_data[s_name]
        if len(self.fastp_duplication_plotdata[s_name]) == 0:
            del self.fastp_duplication_plotdata[s_name]
        if len(self.fastp_insert_size_data[s_name]) == 0:
            del self.fastp_insert_size_data[s_name]
        if len(self.fastp_all_data[s_name]) == 0:
            del self.fastp_all_data[s_name]
740160
def read(self, nml_fname, nml_patch_in=None, patch_fname=None):
        
        # For switching based on files versus paths
        nml_is_path = not hasattr(nml_fname, 'read')
        patch_is_path = not hasattr(patch_fname, 'read')

        # Convert patch data to a Namelist object
        if nml_patch_in is not None:
            if not isinstance(nml_patch_in, dict):
                raise TypeError('Input patch must be a dict or a Namelist.')

            nml_patch = copy.deepcopy(Namelist(nml_patch_in))

            if not patch_fname and nml_is_path:
                patch_fname = nml_fname + '~'
            elif not patch_fname:
                raise ValueError('f90nml: error: No output file for patch.')
            elif nml_fname == patch_fname:
                raise ValueError('f90nml: error: Patch filepath cannot be the '
                                 'same as the original filepath.')
            if patch_is_path:
                self.pfile = open(patch_fname, 'w')
            else:
                self.pfile = patch_fname
        else:
            nml_patch = Namelist()

        try:
            nml_file = open(nml_fname, 'r') if nml_is_path else nml_fname
            try:
                return self._readstream(nml_file, nml_patch)

            # Close the files we opened on any exceptions within readstream
            finally:
                if nml_is_path:
                    nml_file.close()
        finally:
            if self.pfile and patch_is_path:
                self.pfile.close()
388852
def load_feature(fname, language):
    

    fname = os.path.abspath(fname)
    feat = parse_file(fname, language)
    return feat
660626
def load_model(f, format=None, load_external_data=True):  # type: (Union[IO[bytes], Text], Optional[Any], bool) -> ModelProto
    '''
    
    '''
    s = _load_bytes(f)
    model = load_model_from_string(s, format=format)

    if load_external_data:
        model_filepath = _get_file_path(f)
        if model_filepath:
            base_dir = os.path.dirname(model_filepath)
            load_external_data_for_model(model, base_dir)

    return model
42942
def load(cls, fname):
        
        with open(fname) as f:
            content = f.readlines()
        return Flow.from_json(''.join(content))
1016820
def model_fn(model_dir):
    
    symbol = mx.sym.load('%s/model.json' % model_dir)
    outputs = mx.symbol.softmax(data=symbol, name='softmax_label')
    inputs = mx.sym.var('data')
    param_dict = gluon.ParameterDict('model_')
    net = gluon.SymbolBlock(outputs, inputs, param_dict)
    net.load_params('%s/model.params' % model_dir, ctx=mx.cpu())
    return net
841771
def parse_file(self, name):
        
        with open(name, "rb") as fp:
            return self.parse(fp.read())
1319619
def get_risk_files(inputs):
    
    rfs = {}
    job_ini = inputs['job_ini']
    for key in inputs:
        if key == 'fragility':
            # backward compatibily for .ini files with key fragility_file
            # instead of structural_fragility_file
            rfs['fragility/structural'] = inputs[
                'structural_fragility'] = inputs[key]
            del inputs['fragility']
        elif key.endswith(('_fragility', '_vulnerability', '_consequence')):
            match = RISK_TYPE_REGEX.match(key)
            if match and 'retrofitted' not in key and 'consequence' not in key:
                rfs['%s/%s' % (match.group(2), match.group(1))] = inputs[key]
            elif match is None:
                raise ValueError('Invalid key in %s: %s_file' % (job_ini, key))
    return rfs