Read in parameter values from PSET-like configobj tasks defined for
        source-finding algorithms, and any other PSET-like tasks under this task,
        and merge those values into the input configobj dictionary.
181533
def _managePsets(configobj, section_name, task_name, iparsobj=None, input_dict=None):
    
    # Merge all configobj instances into a single object
    configobj[section_name] = {}

    # Load the default full set of configuration parameters for the PSET:
    iparsobj_cfg = teal.load(task_name)

    # Identify optional parameters in input_dicts that are from this
    # PSET and add it to iparsobj:
    if input_dict is not None:
        for key in list(input_dict.keys()):
            if key in iparsobj_cfg:
                if iparsobj is not None and key in iparsobj:
                    raise DuplicateKeyError("Duplicate parameter '{:s}' "
                        "provided for task {:s}".format(key, task_name))
                iparsobj_cfg[key] = input_dict[key]
                del input_dict[key]

    if iparsobj is not None:
        iparsobj_cfg.update(iparsobj)

    del iparsobj_cfg['_task_name_']

    # merge these parameters into full set
    configobj[section_name].merge(iparsobj_cfg)
689772
def _do_read_config(self, config_file, pommanipext):
        """"""
        parser = InterpolationConfigParser()
        dataset = parser.read(config_file)
        if config_file not in dataset:
            raise IOError("Config file %s not found." % config_file)
        if parser.has_option('common','include'):
            include = parser.get('common', 'include')
            if include is not "":
                sections_ = self.read_and_load(include)
                for section_ in sections_:
                    if parser.has_section(section_):
                        raise DuplicateSectionError( "The config section [%s] is existed in %s and include %s cfg file" % ( section_, config_file, re.split("\\s+", include.strip())[1]))
                parser._sections.update(sections_)

        pom_manipulator_config = {}
        common_section = {}
        package_configs = {}

        if pommanipext and pommanipext != '' and pommanipext != 'None': #TODO ref: remove none check, it is passed over cmd line in jenkins build
            parse_pom_manipulator_ext(pom_manipulator_config, parser, pommanipext)

        if not parser.has_section('common'):
            logging.error('Mandatory common section missing from configuration file.')
            raise NoSectionError('Mandatory common section missing from configuration file.')
        common_section['tag'] = parser.get('common', 'tag')
        common_section['target'] = parser.get('common', 'target')
        common_section['jobprefix'] = parser.get('common', 'jobprefix')
        common_section['jobciprefix'] = parser.get('common', 'jobciprefix')
        common_section['jobjdk'] = parser.get('common', 'jobjdk')
        if parser.has_option('common', 'mvnver'):
            common_section['mvnver'] = parser.get('common', 'mvnver')
        if parser.has_option('common', 'skiptests'):
            common_section['skiptests'] = parser.get('common', 'skiptests')
        if parser.has_option('common', 'base'):
            common_section['base'] = parser.get('common', 'base')
        if parser.has_option('common', 'citemplate'):
            common_section['citemplate'] = parser.get('common', 'citemplate')
        if parser.has_option('common', 'jenkinstemplate'):
            common_section['jenkinstemplate'] = parser.get('common', 'jenkinstemplate')
        if parser.has_option('common', 'product_name'):
            common_section['product_name'] = parser.get('common', 'product_name')

        if parser.has_option('common', 'include'):
            common_section['include'] = parser.get('common', 'include')

        common_section['jobfailureemail'] = parser.get('common', 'jobfailureemail')

        config_dir = utils.get_dir(config_file)

        #Jira
        if parser.has_option('common', 'shared_config') and parser.get('common', 'shared_config') is not "":
            parse_shared_config(common_section, config_dir, parser)

        common_section['jobtimeout'] = parser.getint('common', 'jobtimeout')

        common_section['options'] = {}
        # If the configuration file has global properties insert these into the common properties map.
        # These may be overridden later by particular properties.
        if parser.has_option('common', 'globalproperties'):
            common_section['options']['properties'] = dict(x.strip().split('=') for x in parser.get('common', 'globalproperties').replace(",\n", ",").split(','))
        else:
            # Always ensure properties has a valid dictionary so code below doesn't need multiple checks.
            common_section['options']['properties'] = {}
        # The same for global profiles
        if parser.has_option('common', 'globalprofiles'):
            common_section['options']['profiles'] = [x.strip() for x in parser.get('common', 'globalprofiles').split(',')]
        else:
            # Always ensure profiles has a valid list so code below doesn't need multiple checks.
            common_section['options']['profiles'] = []

        if os.path.dirname(config_file):
            config_path = os.path.dirname(config_file)
        else:
            config_path = os.getcwd()
        logging.info("Configuration file is %s and path %s", os.path.basename(config_file), config_path)

        for section in parser.sections():
            config_type = self.read_config_type(parser, section)
            if section == 'common' or config_type == "bom-builder-meta":
                logging.debug ('Skipping section due to meta-type %s', section)
                continue

            self._do_read_section(config_path, os.path.basename(config_file), package_configs, parser, section)

        return (common_section, package_configs, pom_manipulator_config)
569786
def parser(input_file_path='config.json'):
    '''
    
    '''

    # --- Read .json file
    try:
        with open(input_file_path, 'r') as config_file:
            config_new = json.load(config_file)
            config_file.close()
    except:
        raise Exception('Config file "'+input_file_path+'" not loaded properly. Please check it an try again.')

    import copy
    options = update_config(config_new, copy.deepcopy(default_config))

    return options
538115
def _configobj_factory(self,
                           infile,
                           raise_errors=True,
                           list_values=True,
                           file_error=True,
                           interpolation=False,
                           configspec=None,
                           stringify=True,
                           _inspec=False
                           ):
        

        return configobj.ConfigObj(infile=infile,
                                   raise_errors=raise_errors,
                                   list_values=list_values,
                                   file_error=file_error,
                                   interpolation=interpolation,
                                   configspec=configspec,
                                   stringify=stringify,
                                   _inspec=_inspec
                                   )
1172044
def from_dict(cls, json_object):
        """"""
        config = OpenAIGPTConfig(vocab_size_or_config_json_file=-1)
        for key, value in json_object.items():
            config.__dict__[key] = value
        return config
501275
def from_scf_input(cls, scf_input, qpoints, is_ngqpt=False, tolerance=None,
                       with_becs=False, ddk_tolerance=None, manager=None):
        
        if is_ngqpt:
            qpoints = scf_input.abiget_ibz(ngkpt=qpoints, shiftk=[0, 0, 0], kptopt=1).points

        qpoints = np.reshape(qpoints, (-1, 3))

        new = cls(manager=manager)
        # Create ScfTask
        scf_task = new.register_scf_task(scf_input)

        if with_becs:
            new.add_becs_from_scf_task(scf_task, ddk_tolerance, ph_tolerance=tolerance)

        for qpt in qpoints:
            if with_becs and np.sum(qpt ** 2) < 1e-12: continue
            multi = scf_task.input.make_ph_inputs_qpoint(qpt, tolerance=tolerance)
            for ph_inp in multi:
                new.register_phonon_task(ph_inp, deps={scf_task: "WFK"})

        return new
515508
def slice_config(config, key):
    
    if key:
        keys = key.split('.')
        for k in keys:
            config = config[k]

    return config
694977
def parseReaderConfig(self, confdict):
        
        logger.debug('parseReaderConfig input: %s', confdict)
        conf = {}
        for k, v in confdict.items():
            if not k.startswith('Parameter'):
                continue
            ty = v['Type']
            data = v['Data']
            vendor = None
            subtype = None
            try:
                vendor, subtype = v['Vendor'], v['Subtype']
            except KeyError:
                pass

            if ty == 1023:
                if vendor == 25882 and subtype == 37:
                    tempc = struct.unpack('!H', data)[0]
                    conf.update(temperature=tempc)
            else:
                conf[ty] = data
        return conf
332271
def _map_arguments(self, input_dict):
        
        data = input_dict.get('data')
        comp = input_dict.get('comp')
        library = input_dict.get('library')
        models = input_dict.get('models')
        hpx_order = input_dict.get('hpx_order_fitting')
        dry_run = input_dict.get('dry_run', False)

        self._set_link('init-model', InitModel,
                       comp=comp, data=data,
                       library=library,
                       models=models,
                       hpx_order=hpx_order,
                       dry_run=dry_run)

        self._set_link('assemble-model', AssembleModel_SG,
                       comp=comp, data=data,
                       models=models)
1350182
def merge_tasks(runset_results):
    
    task_list = []
    task_set = set()
    for runset in runset_results:
        index = -1
        currentresult_taskset = set()
        for task in runset.get_tasks():
            if task in currentresult_taskset:
                logging.warning("Task '%s' is present twice, skipping it.", task[0])
            else:
                currentresult_taskset.add(task)
                if task not in task_set:
                    task_list.insert(index+1, task)
                    task_set.add(task)
                    index += 1
                else:
                    index = task_list.index(task)

    merge_task_lists(runset_results, task_list)
206235
def _aggregate_config_values(config_values: typing.List[ConfigValue]) -> dict:
    
    _keys: defaultdict = _nested_default_dict()
    _sorted_values = sorted(config_values, key=lambda x: x.name)
    for value in _sorted_values:
        value_keys = value.path.split(ELIBConfig.config_sep_str)
        this_config_key = _keys
        for sub_key in value_keys[:-1]:
            this_config_key = this_config_key[sub_key]
        this_config_key[value_keys[-1]] = value
    return _default_dict_to_dict(_keys)
37548
def reformat_input(self, **kwargs):
        
        reformatted_input = {}
        needed_formats = []
        for task_cls in self.tasks:
            needed_formats.append(task_cls.data_format)
        self.needed_formats = list(set(needed_formats))

        for output_format in self.needed_formats:
            reformatted_input.update(
                {
                    output_format :
                        {
                        'data' : self.reformat_file(self.input_file, self.input_format, output_format),
                        'target' : self.reformat_file(self.target_file, self.target_format, output_format)
                        }
                }
            )
        return reformatted_input
1130594
def from_dict(cls, context_options_dict):
        """"""
        import copy

        context_options = copy.deepcopy(context_options_dict)

        tasks_inserted = context_options.pop('_tasks_inserted', False)

        insert_tasks = context_options.pop('insert_tasks', None)
        if insert_tasks:
            context_options['insert_tasks'] = path_to_reference(insert_tasks)

        # The constructor expects a reference to the persistence engine.
        persistence_engine = context_options.pop('persistence_engine', None)
        if persistence_engine:
            context_options['persistence_engine'] = path_to_reference(
                persistence_engine)

        # If there are callbacks, reconstitute them.
        callbacks = context_options.pop('callbacks', None)
        if callbacks:
            context_options['callbacks'] = decode_callbacks(callbacks)

        context = cls(**context_options)

        context._tasks_inserted = tasks_inserted

        return context
880152
def load_parameters(self, source):
        """"""
        with open(source) as parameters_source:
            return json.loads(parameters_source.read())
17707
def merge_config(d1, d2):
    '''
    '''
    result = deepcopy(d1)
    elements = deque()
    elements.append((result, d2))

    while elements:
        old, new = elements.popleft()

        new = OrderedDict([(k.lower(), (k, v)) for k, v in new.items()])
        visited_keys = []

        for k, old_value in old.items():
            klow = k.lower()
            if klow in new:
                new_key, new_value = new[klow]
                visited_keys.append(new_key)
                if all(isinstance(e, MutableMapping) for e in (old_value, new_value)):
                    elements.append((old_value, new_value))
                else:
                    old[k] = deepcopy(new_value)

        for k, v in new.values():
            if k not in visited_keys:
                old[k] = deepcopy(v)

    return result
239652
def parse_known_args(self, args=None, namespace=None):
        """"""
        # load the config_manager within the scope of the method that uses it
        # so that we avoid circular references in the outer scope
        from configman.config_manager import ConfigurationManager
        configuration_manager = ConfigurationManager(
            definition_source=[self.get_required_config()],
            values_source_list=self.value_source_list,
            argv_source=args,
            app_name=self.prog,
            app_version=self.version,
            app_description=self.description,
            use_auto_help=False,
        )
        conf = configuration_manager.get_config(
            mapping_class=create_key_translating_dot_dict(
                "HyphenUnderscoreDict",
                (('-', '_'),)
            )
        )
        return conf
842189
def _getParamsFromConfigDict(self, cfgObj, scopePrefix='',
                                 initialPass=False, dumpCfgspcTo=None):
        
        # init
        retval = []
        if initialPass and len(scopePrefix) < 1:
            self._posArgs = [] # positional args [2-tuples]: (index,scopedName)
            # FOR SECURITY: the following 3 chunks of data,
            #     _allTriggers, _allDepdcs, _allExecutes,
            # are collected ONLY from the .cfgspc file
            self._allTriggers = {}
            self._allDepdcs = {}
            self._allExecutes = {}

        # start walking ("tell yer story walkin, buddy")
        # NOTE: this relies on the "in" operator returning keys in the
        # order that they exist in the dict (which depends on ConfigObj keeping
        # the order they were found in the original file)
        for key in cfgObj:
            val = cfgObj[key]

            # Do we need to skip this - if not a par, like a rule or something
            toBeHidden = isHiddenName(key)
            if toBeHidden:
                if key not in self._neverWrite and key != TASK_NAME_KEY:
                    self._neverWrite.append(key)
                    # yes TASK_NAME_KEY is hidden, but it IS output to the .cfg

            # a section
            if isinstance(val, dict):
                if not toBeHidden:
                    if len(list(val.keys()))>0 and len(retval)>0:
                        # Here is where we sneak in the section comment
                        # This is so incredibly kludgy (as the code was), it
                        # MUST be revamped eventually! This is for the epar GUI.
                        prevPar = retval[-1]
                        # Use the key (or its comment?) as the section header
                        prevPar.set(prevPar.get('p_prompt')+'\n\n'+key,
                                    field='p_prompt', check=0)
                    if dumpCfgspcTo:
                        dumpCfgspcTo.write('\n['+key+']\n')
                    # a logical grouping (append its params)
                    pfx = scopePrefix+'.'+key
                    pfx = pfx.strip('.')
                    retval = retval + self._getParamsFromConfigDict(val, pfx,
                                      initialPass, dumpCfgspcTo) # recurse
            else:
                # a param
                fields = []
                choicesOrMin = None
                fields.append(key) # name
                dtype = 's'
                cspc = None
                if cfgObj.configspec:
                    cspc = cfgObj.configspec.get(key) # None if not found
                chk_func_name = ''
                chk_args_dict = {}
                if cspc:
                    chk_func_name = cspc[:cspc.find('(')]
                    chk_args_dict = vtor_checks.sigStrToKwArgsDict(cspc)
                if chk_func_name.find('option') >= 0:
                    dtype = 's'
                    # convert the choices string to a list (to weed out kwds)
                    x = cspc[cspc.find('(')+1:-1] # just the options() args
# cspc e.g.: option_kw("poly5","nearest","linear", default="poly5", comment="Interpolant (poly5,nearest,linear)")
                    x = x.split(',') # tokenize
                    # but! comment value may have commas in it, find it
                    # using it's equal sign, rm all after it
                    has_eq = [i for i in x if i.find('=')>=0]
                    if len(has_eq) > 0:
                        x = x[: x.index(has_eq[0]) ]
                    # rm spaces, extra quotes; rm kywd arg pairs
                    x = [i.strip("' ") for i in x if i.find('=')<0]
                    choicesOrMin = '|'+'|'.join(x)+'|' # IRAF format for enums
                elif chk_func_name.find('boolean') >= 0:     dtype = 'b'
                elif chk_func_name.find('float_or_') >= 0:   dtype = 'r'
                elif chk_func_name.find('float') >= 0:       dtype = 'R'
                elif chk_func_name.find('integer_or_') >= 0: dtype = 'i'
                elif chk_func_name.find('integer') >= 0:     dtype = 'I'
                elif chk_func_name.find('action') >= 0:      dtype = 'z'
                fields.append(dtype)
                fields.append('a')
                if type(val)==bool:
                    if val: fields.append('yes')
                    else:   fields.append('no')
                else:
                    fields.append(val)
                fields.append(choicesOrMin)
                fields.append(None)
                # Primarily use description from .cfgspc file (0). But, allow
                # overrides from .cfg file (1) if different.
                dscrp0 = chk_args_dict.get('comment','').strip() # ok if missing
                dscrp1 = cfgObj.inline_comments[key]
                if dscrp1 is None:
                    dscrp1 = ''

                while len(dscrp1) > 0 and dscrp1[0] in (' ','#'):
                    dscrp1 = dscrp1[1:] # .cfg file comments start with '#'

                dscrp1 = dscrp1.strip()
                # Now, decide what to do/say about the descriptions
                if len(dscrp1) > 0:
                    dscrp = dscrp0
                    if dscrp0 != dscrp1: # allow override if different
                        dscrp = dscrp1+eparoption.DSCRPTN_FLAG # flag it
                        if initialPass:
                            if dscrp0 == '' and cspc is None:
                                # this is a case where this par isn't in the
                                # .cfgspc; ignore, it is caught/error later
                                pass
                            else:
                                self.debug('Description of "'+key+ \
                                    '" overridden, from:  '+repr(dscrp0)+\
                                    '  to:  '+repr(dscrp1))
                    fields.append(dscrp)
                else:
                    # set the field for the GUI
                    fields.append(dscrp0)
                    # ALSO set it in the dict so it is written to file later
                    cfgObj.inline_comments[key] = '# '+dscrp0
                # This little section, while never intended to be used during
                # normal operation, could save a lot of manual work.
                if dumpCfgspcTo:
                    junk = cspc
                    junk = key+' = '+junk.strip()
                    if junk.find(' comment=')<0:
                        junk = junk[:-1]+", comment="+ \
                               repr(irafutils.stripQuotes(dscrp1.strip()))+")"
                    dumpCfgspcTo.write(junk+'\n')
                # Create the par
                if not toBeHidden or chk_func_name.find('action')==0:
                    par = basicpar.parFactory(fields, True)
                    par.setScope(scopePrefix)
                    retval.append(par)
                # else this is a hidden key

                # The next few items require a fully scoped name
                absKeyName = scopePrefix+'.'+key # assumed to be unique
                # Check for pars marked to be positional args
                if initialPass:
                    pos = chk_args_dict.get('pos')
                    if pos:
                        # we'll sort them later, on demand
                        self._posArgs.append( (int(pos), scopePrefix, key) )
                # Check for triggers and/or dependencies
                if initialPass:
                    # What triggers what? (thats why theres an 's' in the kwd)
                    # try "trigger" (old)
                    if chk_args_dict.get('trigger'):
                        print("WARNING: outdated version of .cfgspc!! for "+
                              self.__taskName+", 'trigger' unused for "+
                              absKeyName)
                    # try "triggers"
                    trgs = chk_args_dict.get('triggers')
                    if trgs and len(trgs)>0:
                        # eg. _allTriggers['STEP2.xy'] == ('_rule1_','_rule3_')
                        assert absKeyName not in self._allTriggers, \
                            'More than 1 of these in .cfgspc?: '+absKeyName
                        # we force this to always be a sequence
                        if isinstance(trgs, (list,tuple)):
                            self._allTriggers[absKeyName] = trgs
                        else:
                            self._allTriggers[absKeyName] = (trgs,)
                    # try "executes"
                    excs = chk_args_dict.get('executes')
                    if excs and len(excs)>0:
                        # eg. _allExecutes['STEP2.xy'] == ('_rule1_','_rule3_')
                        assert absKeyName not in self._allExecutes, \
                            'More than 1 of these in .cfgspc?: '+absKeyName
                        # we force this to always be a sequence
                        if isinstance(excs, (list,tuple)):
                            self._allExecutes[absKeyName] = excs
                        else:
                            self._allExecutes[absKeyName] = (excs,)

                    # Dependencies? (besides these used here, may someday
                    # add: 'range_from', 'warn_if', etc.)
                    depName = None
                    if not depName:
                        depType = 'active_if'
                        depName = chk_args_dict.get(depType) # e.g. =='_rule1_'
                    if not depName:
                        depType = 'inactive_if'
                        depName = chk_args_dict.get(depType)
                    if not depName:
                        depType = 'is_set_by'
                        depName = chk_args_dict.get(depType)
                    if not depName:
                        depType = 'set_yes_if'
                        depName = chk_args_dict.get(depType)
                    if not depName:
                        depType = 'set_no_if'
                        depName = chk_args_dict.get(depType)
                    if not depName:
                        depType = 'is_disabled_by'
                        depName = chk_args_dict.get(depType)
                    # NOTE - the above few lines stops at the first dependency
                    # found (depName) for a given par.  If, in the future a
                    # given par can have >1 dependency than we need to revamp!!
                    if depName:
                        # Add to _allDepdcs dict: (val is dict of pars:types)
                        #
                        # e.g. _allDepdcs['_rule1_'] == \
                        #        {'STEP3.ra':      'active_if',
                        #         'STEP3.dec':     'active_if',
                        #         'STEP3.azimuth': 'inactive_if'}
                        if depName in self._allDepdcs:
                            thisRulesDict = self._allDepdcs[depName]
                            assert not absKeyName in thisRulesDict, \
                                'Cant yet handle multiple actions for the '+ \
                                'same par and the same rule.  For "'+depName+ \
                                '" dict was: '+str(thisRulesDict)+ \
                                ' while trying to add to it: '+\
                                str({absKeyName:depType})
                            thisRulesDict[absKeyName] = depType
                        else:
                            self._allDepdcs[depName] = {absKeyName:depType}
                    # else no dependencies found for this chk_args_dict
        return retval