Takes a list of filerefs and returns a merged list
551160
def _merge_extra_filerefs(*args):
    '''
    
    '''
    ret = []
    for arg in args:
        if isinstance(arg, six.string_types):
            if arg:
                ret.extend(arg.split(','))
        elif isinstance(arg, list):
            if arg:
                ret.extend(arg)
    return ','.join(ret)
820593
def files(
            self):
        
        relatedFilesResultsList = []
        relatedFilesResultsList[:] = [dict(l)
                                      for l in self.relatedFilesResultsList]
        return relatedFilesResultsList
118885
def listing(self):
        ""
        return ['.'.join([f,ext]) if ext else f for (f,ext) in self._files.keys()]
366396
def filer(filelist):
    
    # Initialise the dictionary
    filedict = dict()
    for seqfile in filelist:
        # Split off the file extension and remove the path from the name
        strainname = os.path.splitext(os.path.basename(seqfile))[0]
        # Populate the dictionary
        filedict[strainname] = seqfile
    return filedict
362299
def _files_preserve(self):
        
        result = set()
        files = [] if not self.files_preserve else self.files_preserve
        files.extend([self.stdin, self.stdout, self.stderr])
        for item in files:
            if hasattr(item, 'fileno'):
                result.add(item.fileno())
            if isinstance(item, int):
                result.add(item)
        return result
412752
def get_files(files, extnames=['.root']):
    """"""

    files_out = []
    for f in files:

        mime = mimetypes.guess_type(f)
        if os.path.splitext(f)[1] in extnames:
            files_out += [f]
        elif mime[0] == 'text/plain':
            files_out += list(np.loadtxt(f, unpack=True, dtype='str'))
        else:
            raise Exception('Unrecognized input type.')

    return files_out
1099684
def _file_local_list(self, dest):
        '''
        
        '''
        if os.path.isdir(dest):
            destdir = dest
        else:
            destdir = os.path.dirname(dest)

        filelist = set()

        for root, dirs, files in salt.utils.path.os_walk(destdir, followlinks=True):
            for name in files:
                path = os.path.join(root, name)
                filelist.add(path)

        return filelist
443860
def _file_lists(load, form):
    '''
    
    '''
    if 'env' in load:
        # "env" is not supported; Use "saltenv".
        load.pop('env')

    saltenv = load['saltenv']
    if saltenv not in __opts__['file_roots']:
        if '__env__' in __opts__['file_roots']:
            log.debug("salt environment '%s' maps to __env__ file_roots directory", saltenv)
            saltenv = '__env__'
        else:
            return []

    list_cachedir = os.path.join(__opts__['cachedir'], 'file_lists', 'roots')
    if not os.path.isdir(list_cachedir):
        try:
            os.makedirs(list_cachedir)
        except os.error:
            log.critical('Unable to make cachedir %s', list_cachedir)
            return []
    list_cache = os.path.join(list_cachedir, '{0}.p'.format(salt.utils.files.safe_filename_leaf(saltenv)))
    w_lock = os.path.join(list_cachedir, '.{0}.w'.format(salt.utils.files.safe_filename_leaf(saltenv)))
    cache_match, refresh_cache, save_cache = \
        salt.fileserver.check_file_list_cache(
            __opts__, form, list_cache, w_lock
        )
    if cache_match is not None:
        return cache_match
    if refresh_cache:
        ret = {
            'files': set(),
            'dirs': set(),
            'empty_dirs': set(),
            'links': {}
        }

        def _add_to(tgt, fs_root, parent_dir, items):
            '''
            Add the files to the target set
            '''
            def _translate_sep(path):
                '''
                Translate path separators for Windows masterless minions
                '''
                return path.replace('\\', '/') if os.path.sep == '\\' else path

            for item in items:
                abs_path = os.path.join(parent_dir, item)
                log.trace('roots: Processing %s', abs_path)
                is_link = salt.utils.path.islink(abs_path)
                log.trace(
                    'roots: %s is %sa link',
                    abs_path, 'not ' if not is_link else ''
                )
                if is_link and __opts__['fileserver_ignoresymlinks']:
                    continue
                rel_path = _translate_sep(os.path.relpath(abs_path, fs_root))
                log.trace('roots: %s relative path is %s', abs_path, rel_path)
                if salt.fileserver.is_file_ignored(__opts__, rel_path):
                    continue
                tgt.add(rel_path)
                try:
                    if not os.listdir(abs_path):
                        ret['empty_dirs'].add(rel_path)
                except Exception:
                    # Generic exception because running os.listdir() on a
                    # non-directory path raises an OSError on *NIX and a
                    # WindowsError on Windows.
                    pass
                if is_link:
                    link_dest = salt.utils.path.readlink(abs_path)
                    log.trace(
                        'roots: %s symlink destination is %s',
                        abs_path, link_dest
                    )
                    if salt.utils.platform.is_windows() \
                            and link_dest.startswith('\\\\'):
                        # Symlink points to a network path. Since you can't
                        # join UNC and non-UNC paths, just assume the original
                        # path.
                        log.trace(
                            'roots: %s is a UNC path, using %s instead',
                            link_dest, abs_path
                        )
                        link_dest = abs_path
                    if link_dest.startswith('..'):
                        joined = os.path.join(abs_path, link_dest)
                    else:
                        joined = os.path.join(
                            os.path.dirname(abs_path), link_dest
                        )
                    rel_dest = _translate_sep(
                        os.path.relpath(
                            os.path.realpath(os.path.normpath(joined)),
                            fs_root
                        )
                    )
                    log.trace(
                        'roots: %s relative path is %s',
                        abs_path, rel_dest
                    )
                    if not rel_dest.startswith('..'):
                        # Only count the link if it does not point
                        # outside of the root dir of the fileserver
                        # (i.e. the "path" variable)
                        ret['links'][rel_path] = link_dest

        for path in __opts__['file_roots'][saltenv]:
            for root, dirs, files in salt.utils.path.os_walk(
                    path,
                    followlinks=__opts__['fileserver_followsymlinks']):
                _add_to(ret['dirs'], path, root, dirs)
                _add_to(ret['files'], path, root, files)

        ret['files'] = sorted(ret['files'])
        ret['dirs'] = sorted(ret['dirs'])
        ret['empty_dirs'] = sorted(ret['empty_dirs'])

        if save_cache:
            try:
                salt.fileserver.write_file_list_cache(
                    __opts__, ret, list_cache, w_lock
                )
            except NameError:
                # Catch msgpack error in salt-ssh
                pass
        return ret.get(form, [])
    # Shouldn't get here, but if we do, this prevents a TypeError
    return []
676393
def mergeall(filename, snrmin, snrmax, bdfdir):
    

    filename = os.path.abspath(filename)
    bignumber = 500

    if os.path.exists(filename):
        scans = ps.read_scans(filename, bdfdir=bdfdir)
        scanlist = sorted(scans.keys())
    else:
        logger.warn('Could not find file {0}. Estimating scans from available files.'.format(filename))
        filelist = glob.glob(os.path.join(os.path.dirname(filename), '*{0}_sc*pkl'.format(os.path.basename(filename))))
        try:
            scanlist = sorted(set([int(fn.rstrip('.pkl').split('_sc')[1].split('seg')[0]) for fn in filelist]))
        except IndexError:
            logger.warn('Could not parse filenames for scans. Looking over big range.')
            scanlist = range(bignumber)

    logger.info('Merging over scans {0}'.format(scanlist))

    for scan in scanlist:
        pc.merge_segments(filename, scan)
    pc.merge_scans(os.path.dirname(filename), os.path.basename(filename), scanlist, snrmin=snrmin, snrmax=snrmax)
353888
def _glob(filenames):
    
    if isinstance(filenames, string_types):
        filenames = [filenames]
    matches = []
    for name in filenames:
        matched_names = glob(name)
        if not matched_names:
            # use the original string
            matches.append(name)
        else:
            matches.extend(matched_names)
    return matches
449016
def _file_lists(self, load, form):
        '''
        
        '''
        if 'env' in load:
            # "env" is not supported; Use "saltenv".
            load.pop('env')

        if not os.path.isdir(self.file_list_cachedir):
            try:
                os.makedirs(self.file_list_cachedir)
            except os.error:
                log.error('Unable to make cachedir %s', self.file_list_cachedir)
                return []
        list_cache = salt.utils.path.join(
            self.file_list_cachedir,
            '{0}.p'.format(load['saltenv'].replace(os.path.sep, '_|-'))
        )
        w_lock = salt.utils.path.join(
            self.file_list_cachedir,
            '.{0}.w'.format(load['saltenv'].replace(os.path.sep, '_|-'))
        )
        cache_match, refresh_cache, save_cache = \
            salt.fileserver.check_file_list_cache(
                self.opts, form, list_cache, w_lock
            )
        if cache_match is not None:
            return cache_match
        if refresh_cache:
            ret = {'files': set(), 'symlinks': {}, 'dirs': set()}
            if salt.utils.stringutils.is_hex(load['saltenv']) \
                    or load['saltenv'] in self.envs():
                for repo in self.remotes:
                    repo_files, repo_symlinks = repo.file_list(load['saltenv'])
                    ret['files'].update(repo_files)
                    ret['symlinks'].update(repo_symlinks)
                    ret['dirs'].update(repo.dir_list(load['saltenv']))
            ret['files'] = sorted(ret['files'])
            ret['dirs'] = sorted(ret['dirs'])

            if save_cache:
                salt.fileserver.write_file_list_cache(
                    self.opts, ret, list_cache, w_lock
                )
            # NOTE: symlinks are organized in a dict instead of a list, however
            # the 'symlinks' key will be defined above so it will never get to
            # the default value in the call to ret.get() below.
            return ret.get(form, [])
        # Shouldn't get here, but if we do, this prevents a TypeError
        return {} if form == 'symlinks' else []
1072364
def get_list_from_file(file_name):
    """"""
    with open(file_name, mode='r', encoding='utf-8') as f1:
        lst = f1.readlines()
    return lst
1346330
def temp_files(self):
        
        ret_list = []
        for key, val in self.file_dict.items():
            # For temp files we only want files that were marked for removal
            if val & FileFlags.rm_mask:
                ret_list.append(key)
        return ret_list
1171889
def files(self, ftype=None):
        
        a = [self.join(i) for i in self]
        if ftype is not None:
            return [i for i in a if i.isfile() and i.filetype() == ftype]
        return [i for i in a if i.isfile()]
36581
def get_all_filelist(data_dir):
    

    all_meta = []
    all_table = []
    all_element = []
    all_component = []

    special = ['METADATA.json', 'REFERENCES.json']

    for root, dirs, files in os.walk(data_dir):
        for basename in files:
            if basename in special:
                continue

            fpath = os.path.join(root, basename)
            fpath = os.path.relpath(fpath, data_dir)

            if basename.endswith('.metadata.json'):
                all_meta.append(fpath)
            elif basename.endswith('.table.json'):
                all_table.append(fpath)
            elif basename.endswith('.element.json'):
                all_element.append(fpath)
            elif basename.endswith('.json'):
                all_component.append(fpath)

    return (all_meta, all_table, all_element, all_component)
1053270
def fast_combine_pairs(files, force_single, full_name, separators):
    
    files = sort_filenames(files)
    chunks = tz.sliding_window(10, files)
    pairs = [combine_pairs(chunk, force_single, full_name, separators) for chunk in chunks]
    pairs = [y for x in pairs for y in x]
    longest = defaultdict(list)
    # for each file, save the longest pair it is in
    for pair in pairs:
        for file in pair:
            if len(longest[file]) < len(pair):
                longest[file] = pair
    # keep only unique pairs
    longest = {tuple(sort_filenames(x)) for x in longest.values()}
    # ensure filenames are R1 followed by R2
    return [sort_filenames(list(x)) for x in longest]
33278
def file_list(package, **kwargs):
    '''
    
    '''
    ret = file_dict(package)
    files = []
    for pkg_files in six.itervalues(ret['files']):
        files.extend(pkg_files)
    ret['files'] = files
    return ret