Look for files, find authors, sort, write file.
331380
def find_write_contribs() -> None:
    """"""
    map_file_auth = {}  # type: Dict[str, List[str]]
    for filename in scantree('cltk'):
        filepath = filename.path  # type: str
        authors_list = get_authors(filepath)  # type: List[str]
        if authors_list:
            map_file_auth[filepath] = authors_list

    map_auth_file = defaultdict(list)  # type: Dict[str, List[str]]
    for file, authors_file in map_file_auth.items():
        for author in authors_file:
            map_auth_file[author].append(file)
    # now sort the str contents of the list value
    map_auth_file = sort_def_dict(map_auth_file)
    map_auth_file_sorted = sorted(map_auth_file.items())  # type: List[Tuple[str, List[str]]]
    map_auth_file = OrderedDict(map_auth_file_sorted)

    write_contribs(map_auth_file)
1289392
def assemble_phi5_works_filepaths():
    """"""
    plaintext_dir_rel = '~/cltk_data/latin/text/phi5/individual_works/'
    plaintext_dir = os.path.expanduser(plaintext_dir_rel)
    all_filepaths = []
    for author_code in PHI5_WORKS_INDEX:
        author_data = PHI5_WORKS_INDEX[author_code]
        works = author_data['works']
        for work in works:
            f = os.path.join(plaintext_dir, author_code + '.TXT' + '-' + work + '.txt')
            all_filepaths.append(f)
    return all_filepaths
1077847
def __search_files(self, files):
        

        for file in files:
            if self.__interrupt:
                return

            if not foundations.common.path_exists(file):
                continue

            if foundations.io.is_readable(file):
                if foundations.io.is_binary_file(file):
                    continue

            LOGGER.info("{0} | Searching '{1}' file!".format(self.__class__.__name__, file))
            cache_data = self.__container.files_cache.get_content(file)
            if not cache_data:
                reader = foundations.io.File(file)
                content = reader.read()
                if content is None:
                    LOGGER.warning("!> Error occured while reading '{0}' file proceeding to next one!".format(file))
                    continue
                self.__container.files_cache.add_content(**{file: CacheData(content=content, document=None)})
            else:
                content = cache_data.content
            occurrences = self.__search_document(QTextDocument(QString(content)), self.__pattern, self.__settings)
            occurrences and self.__search_results.append(SearchResult(file=file,
                                                                      pattern=self.__pattern,
                                                                      settings=self.__settings,
                                                                      occurrences=occurrences))
1288097
def main_browse():
    
    parser = argparse.ArgumentParser(usage="job_archive.py [options]",
                                     description="Browse a job archive")

    parser.add_argument('--jobs', action='store', dest='job_archive_table',
                        type=str, default='job_archive_temp2.fits', help="Job archive file")
    parser.add_argument('--files', action='store', dest='file_archive_table',
                        type=str, default='file_archive_temp2.fits', help="File archive file")
    parser.add_argument('--base', action='store', dest='base_path',
                        type=str, default=os.path.abspath('.'), help="File archive base path")

    args = parser.parse_args(sys.argv[1:])
    job_ar = JobArchive.build_archive(**args.__dict__)

    job_ar.table.pprint()
1095774
def find_new_posts_and_pages(db):
    """"""

    Q = Query()
    for root, dirs, files in os.walk(CONFIG['content_root']):
        for filename in sorted([f for f in files if
                               f.endswith(('md', 'markdown'))]):
            fullpath = os.path.join(root, filename)
            _p = fullpath.split(CONFIG['content_root'])[-1].lstrip('/')
            new_mtime = int(os.path.getmtime(fullpath))
            e, item = None, None

            for collection in ['posts', 'pages']:
                item = db[collection].get(Q.filename == _p)
                if item:
                    if new_mtime > item['mtime']:
                        db[collection].update({'mtime': new_mtime},
                                              doc_ids=[item.doc_id])
                        e = Entry(fullpath, doc_id=item.doc_id)
                    break

            if not item:
                e = Entry(fullpath)
            if e:
                yield e, e.id
893632
def read_files(filenames):
    """"""
    if isinstance(filenames, list):
        for filename in filenames:
            with open(filename, 'r') as infile:
                return infile.read()
    else:
        with open(filenames, 'r') as infile:
            return infile.read()
306116
function findParsableFile(book, filename) {
    var fs = book.getContentFS();
    var ext = path.extname(filename);
    var basename = path.basename(filename, ext);
    var basedir = path.dirname(filename);

    // Ordered list of extensions to test
    var exts = parsers.extensions;

    return Promise.some(exts, function(ext) {
        var filepath = basename + ext;

        return fs.findFile(basedir, filepath)
        .then(function(found) {
            if (!found || book.isContentFileIgnored(found)) {
                return undefined;
            }

            return fs.statFile(found);
        });
    });
}
450477
protected void searchAndReplace(List<CmsResource> resources) {

        // the file counter
        int counter = 0;
        int resCount = resources.size();
        I_CmsReport report = getReport();
        // iterate over the files in the selected path
        for (CmsResource resource : resources) {

            try {

                // get the content
                CmsFile file = getCms().readFile(resource);
                byte[] contents = file.getContents();

                // report the current resource
                ++counter;
                report(report, counter, resCount, resource);

                // search and replace
                byte[] result = null;
                boolean xpath = false;
                if ((CmsStringUtil.isNotEmptyOrWhitespaceOnly(m_settings.getXpath())
                    || m_settings.isOnlyContentValues()) && CmsResourceTypeXmlContent.isXmlContent(resource)) {
                    xpath = true;
                }
                if (!xpath) {
                    result = replaceInContent(file, contents);
                } else {
                    result = replaceInXml(file);
                }

                if ((result != null) && (contents != null) && !contents.equals(result)) {
                    // rewrite the content
                    writeContent(file, result);
                } else {
                    getReport().println();
                }

            } catch (Exception e) {
                report.print(
                    org.opencms.report.Messages.get().container(Messages.RPT_SOURCESEARCH_COULD_NOT_READ_FILE_0),
                    I_CmsReport.FORMAT_ERROR);
                report.addError(e);
                report.println(
                    org.opencms.report.Messages.get().container(org.opencms.report.Messages.RPT_FAILED_0),
                    I_CmsReport.FORMAT_ERROR);
                m_errorSearch += 1;
                LOG.error(
                    org.opencms.report.Messages.get().container(Messages.RPT_SOURCESEARCH_COULD_NOT_READ_FILE_0),
                    e);
                continue;
            }
        }

        // report results
        reportResults(resources.size());
    }
1322588
def _filter(client, names=None, authors=None, include=None, exclude=None):
    
    if isinstance(authors, str):
        authors = set(authors.split(','))

    if isinstance(authors, list) or isinstance(authors, tuple):
        authors = set(authors)

    records = []
    for path_, dataset in client.datasets.items():
        if not names or dataset.name in names:
            for file_ in dataset.files.values():
                file_.dataset = dataset.name

                path_ = file_.full_path.relative_to(client.path)
                match = _include_exclude(path_, include, exclude)

                if authors:
                    match = match and authors.issubset({
                        author.name
                        for author in file_.authors
                    })

                if match:
                    records.append(file_)

    return sorted(records, key=lambda file_: file_.added)
744885
def main(items=None, printmd=None, printcal=False, found=False,
         save=None, download=None, requestor_pays=False, **kwargs):
    
    
    if items is None:
        ## if there are no items then perform a search
        search = Search.search(**kwargs)
        if found:
            num = search.found()
            print('%s items found' % num)
            return num
        items = search.items()
    else:
        # otherwise, load a search from a file
        items = Items.load(items)

    print('%s items found' % len(items))

    # print metadata
    if printmd is not None:
        print(items.summary(printmd))

    # print calendar
    if printcal:
        print(items.calendar())

    # save all metadata in JSON file
    if save is not None:
        items.save(filename=save)

    # download files given `download` keys
    if download is not None:
        if 'ALL' in download:
            # get complete set of assets
            download = set([k for i in items for k in i.assets])
        for key in download:
            items.download(key=key, path=config.DATADIR, filename=config.FILENAME, requestor_pays=requestor_pays)

    return items
1220882
def _search_for_files(parts):
    
    file_parts = []
    for part in parts:
        if isinstance(part, list):
            file_parts.extend(_search_for_files(part))
        elif isinstance(part, FileToken):
            file_parts.append(part)
    return file_parts
974776
def extract_files(files):
    """"""
    expanded_files = []
    legal_extensions = [".md", ".txt", ".rtf", ".html", ".tex", ".markdown"]

    for f in files:
        # If it's a directory, recursively walk through it and find the files.
        if os.path.isdir(f):
            for dir_, _, filenames in os.walk(f):
                for filename in filenames:
                    fn, file_extension = os.path.splitext(filename)
                    if file_extension in legal_extensions:
                        joined_file = os.path.join(dir_, filename)
                        expanded_files.append(joined_file)

        # Otherwise add the file directly.
        else:
            expanded_files.append(f)

    return expanded_files
25753
def set_authors(data):
    
    if "authors" in data:
        return

    shfile = os.path.join(os.path.dirname(__file__), "get_committers.sh")

    p = subprocess.Popen(["bash", shfile], stdout=subprocess.PIPE)
    out, _ = p.communicate()
    if p.returncode:
        return

    authors = out.strip().split('\n')
    authors = [x.strip() for x in authors]

    data["authors"] = authors
546321
function findFile(root, filename) {
    return Q.nfcall(fs.readdir, root)
    .then(function(files) {
        return _.find(files, function(file) {
            return (file.toLowerCase() == filename.toLowerCase());
        });
    });
}
542280
private void collectFiles(final File folder, final List<File> found) {
		File[] files = folder.listFiles();
		if (files != null) {
			for (File file : files) {
				if (file.isDirectory()) {
					collectFiles(file, found);
				} else if (file.isFile() && file.getPath().endsWith(suffix)) {
					int index = 0;

					for (String text : plainTexts) {
						index = file.getPath().indexOf(text, index);
						if (index == -1) {
							break;
						} else {
							index += text.length();
						}
					}

					if (index >= 0) {
						found.add(file);
					}
				}
			}
		}
	}
315917
function (files, output, name) {
    this.files = files;
    this.outputfile = path.resolve(output || os.tmpdir()) + path.sep + (name || '.check');
    this.data = {};
}
596416
def main(search, query):
    """"""
    url = search.search(query)
    print(url)
    search.open_page(url)