Read all SGFs that match glob

    Parse each game and extract relevant metadata for eval games table.
428697
def read_games(glob, existing_paths):
    

    globbed = sorted(gfile.Glob(glob))

    skipped = 0
    to_parse = []
    for sgf_name in tqdm(globbed):
        assert sgf_name.lower().endswith('.sgf'), sgf_name
        sgf_path = canonical_name(sgf_name)
        sgf_filename = os.path.basename(sgf_path)

        if sgf_path in existing_paths or sgf_filename in existing_paths:
            skipped += 1
            continue

        to_parse.append(sgf_name)

    game_data = []
    with multiprocessing.Pool() as pool:
        game_data = pool.map(process_game, tqdm(to_parse), 100)

    print("Read {} SGFs, {} new, {} existing".format(
        len(globbed), len(game_data), skipped))
    return game_data
1196930
def _ReadStorageMetadata(self):
    """"""
    query = 'SELECT key, value FROM metadata'
    self._cursor.execute(query)

    metadata_values = {row[0]: row[1] for row in self._cursor.fetchall()}

    self._compression_format = metadata_values['compression_format']
1144083
def parse_GSE(filepath):
    
    gpls = {}
    gsms = {}
    series_counter = 0
    database = None
    metadata = {}
    gse_name = None
    with utils.smart_open(filepath) as soft:
        groupper = groupby(soft, lambda x: x.startswith("^"))
        for is_new_entry, group in groupper:
            if is_new_entry:
                entry_type, entry_name = __parse_entry(next(group))
                logger.debug("%s: %s" % (entry_type.upper(), entry_name))
                if entry_type == "SERIES":
                    gse_name = entry_name
                    series_counter += 1
                    if series_counter > 1:
                        raise Exception(
                            "GSE file should contain only one series entry!")
                    is_data, data_group = next(groupper)
                    message = ("The key is not False, probably there is an "
                               "error in the SOFT file")
                    assert not is_data, message
                    metadata = parse_metadata(data_group)
                elif entry_type == "SAMPLE":
                    is_data, data_group = next(groupper)
                    gsms[entry_name] = parse_GSM(data_group, entry_name)
                elif entry_type == "PLATFORM":
                    is_data, data_group = next(groupper)
                    gpls[entry_name] = parse_GPL(data_group, entry_name)
                elif entry_type == "DATABASE":
                    is_data, data_group = next(groupper)
                    database_metadata = parse_metadata(data_group)
                    database = GEODatabase(name=entry_name,
                                           metadata=database_metadata)
                else:
                    logger.error("Cannot recognize type %s" % entry_type)
    gse = GSE(name=gse_name,
              metadata=metadata,
              gpls=gpls,
              gsms=gsms,
              database=database)
    return gse
713306
def parse(self):
		
		c = Collection()
		while self.index < self.datalen:
			g = self.parseOneGame()
			if g:
				c.append(g)
			else:
				break
		return c
552403
def get_metadata(session, games):
    """"""
    for data in games.values():
        parsed = make_scrape_request(session, data['url'])
    return {
        'ladders': get_ladders_metadata(session, parsed)
    }
1198767
def _read_table(self, filename):
        
        filename = pkg_resources.resource_filename(
            __name__, filename)
        segments = []
        with open(filename, 'rb') as f:
            reader = csv.reader(f, encoding='utf-8')
            header = next(reader)
            names = header[1:]
            for row in reader:
                seg = row[0]
                vals = row[1:]
                specs = set(zip(vals, names))
                segments.append((seg, specs))
        seg_dict = dict(segments)
        return segments, seg_dict, names
1109097
def _read(self, directory, filename, session, path, name, extension, spatial, spatialReferenceID, replaceParamFile):
        
        # Set file extension property
        self.fileExtension = extension

        # Dictionary of keywords/cards and parse function names
        KEYWORDS = {'CONNECT': spc.connectChunk,
                    'SJUNC': spc.sjuncChunk,
                    'SLINK': spc.slinkChunk}

        sjuncs = []
        slinks = []
        connections = []

        # Parse file into chunks associated with keywords/cards
        with open(path, 'r') as f:
            chunks = pt.chunk(KEYWORDS, f)

        # Parse chunks associated with each key
        for key, chunkList in iteritems(chunks):
            # Parse each chunk in the chunk list
            for chunk in chunkList:
                # Call chunk specific parsers for each chunk
                result = KEYWORDS[key](key, chunk)

                # Cases
                if key == 'CONNECT':
                    connections.append(result)
                elif key == 'SJUNC':
                    sjuncs.append(result)
                elif key == 'SLINK':
                    slinks.append(result)

        # Create GSSHAPY objects
        self._createConnection(connections)
        self._createSjunc(sjuncs)
        self._createSlink(slinks)
370111
def filer(self):
        
        # Get a list of all the .fastq files in the sequence path
        fastq = sorted(glob(os.path.join(self.sequencepath, '*fastq*')))
        # Initialise a list to store .fasta formatted files
        fasta = list()
        # Get all the files in the sequence path
        for sequencefile in glob(os.path.join(self.sequencepath, '*')):
            # Split the extension from the file
            extension = os.path.splitext(sequencefile)[1]
            # If the extension is one of the acceptable FASTA extensions, add it to the list of FASTA files
            if extension in self.extensions:
                fasta.append(sequencefile)
        # Use filer from accessoryFunctions to find the base file name of each .fastq file/pair
        fastqnames = filer(fastq)
        # Get all the .fastq files into metadata objects
        for fastqname in sorted(fastqnames):
            # Create the metadata object
            sample = MetadataObject()
            sample.name = os.path.basename(fastqname)
            # Create an attribute for the current analysis
            setattr(sample, self.analysistype, GenObject())
            # Set the destination folder
            sample[self.analysistype].outputdir = os.path.join(self.path, fastqname)
            # Make the destination folder
            make_path(sample[self.analysistype].outputdir)
            # Get the fastq files specific to the fastqname
            specificfastq = glob(os.path.join(self.sequencepath, '{fq}*.fastq*'.format(fq=fastqname)))
            # Set the file type for the downstream analysis
            sample[self.analysistype].filetype = 'fastq'
            # Link the files to the output folder
            for fastq in specificfastq:
                relative_symlink(src_file=fastq,
                                 output_dir=sample[self.analysistype].outputdir)
            # Initialise the general and run categories
            sample.general = GenObject()
            # Populate the .fastqfiles category of :self.metadata
            sample.general.fastqfiles = [fastq for fastq in
                                         glob(os.path.join(sample[self.analysistype].outputdir, '{fq}*.fastq*'
                                                           .format(fq=fastqname))) if 'trimmed' not in fastq]
            # Populate certain attributes in order to be compatible with other local software
            sample.general.bestassemblyfile = sample.general.fastqfiles
            sample.general.trimmedcorrectedfastqfiles = sample.general.fastqfiles
            sample.general.outputdirectory = sample[self.analysistype].outputdir
            sample[self.analysistype].baitedfastq = os.path.join(
                sample[self.analysistype].outputdir,
                '{at}_targetMatches.fastq.gz'.format(at=self.analysistype))
            # Append the metadata to the list of samples
            self.metadata.append(sample)
        # Add any .fasta formatted files to the metadata object
        for fastafile in sorted(fasta):
            sample = MetadataObject()
            sample.name = os.path.splitext(os.path.split(fastafile)[-1])[0]
            setattr(sample, self.analysistype, GenObject())
            # Set the destination folder
            sample[self.analysistype].outputdir = os.path.join(self.path, 'detailed_reports', sample.name)
            # Make the destination folder
            make_path(sample[self.analysistype].outputdir)
            # Set the file type for the downstream analysis
            sample[self.analysistype].filetype = 'fasta'
            # Link the .fasta files to :self.path/:filename
            relative_symlink(src_file=fastafile,
                             output_dir=sample[self.analysistype].outputdir)
            # Initialise the general and run categories
            sample.general = GenObject()
            # Populate the .fastqfiles category of :self.metadata
            sample.general.fastqfiles = [fastq for fastq in glob(os.path.join(sample[self.analysistype].outputdir,
                                                                              '{sn}*{ext}*'
                                                                              .format(
                                                                                  sn=sample.name,
                                                                                  ext=os.path.splitext(fastafile)[1])))]
            sample.general.bestassemblyfile = sample.general.fastqfiles
            sample.general.trimmedcorrectedfastqfiles = sample.general.fastqfiles
            sample.general.outputdirectory = sample[self.analysistype].outputdir
            sample[self.analysistype].baitedfastq = os.path.join(
                sample[self.analysistype].outputdir,
                '{at}_targetMatches.fastq.gz'.format(at=self.analysistype))
            sample[self.analysistype].assemblyfile = fastafile
            # Append the metadata to the list of samples
            self.metadata.append(sample)
726792
function collectResults(resultsGlob) {
    var glob = require("glob"),
        fs = require("fs"),
        Q = require('q'),
        JunitConverter = require("./formats/junitConverter"),
        response = {},
        count = 0,
        specs,
        files = [],
        converter,
        result;

    for (var i = 0; i < resultsGlob.length; i++) {
        files = files.concat(glob.sync(resultsGlob[i]));
    }

    var converterCallback = function(output) {
        result = output;
    };

    for (i = 0; i < files.length; i++) {
        if (files[i].indexOf(".xml") !== -1) {
            converter = new JunitConverter(files[i]);
            converter.exec(converterCallback);
        } else {
            try {
                result = JSON.parse(fs.readFileSync(files[i]));
            } catch (err) {
                console.log('Error parsing JSON of ' + files[i]);
                result = {};
            }
        }
        response = mergeResults(result, response);
        count++;
    }
    return Q.fcall(function () {
        return response;
    });
}
202878
def load_matches(self):
        
        if not self.exists():
            return []

        matches = []
        with open(self.location(), 'r') as f:
            for line in f.readlines():
                match_obj = json.loads(line.strip("\n"))
                if match_obj['processed']:
                    continue

                matches.append(match_obj)

        return matches
313920
function getSourceFilesFromGlob(globPattern, ignoreGlobs) {
    return new Promise((fulfil, reject) => {
        glob(globPattern, { ignore: ignoreGlobs }, (err, files) => {
            if (err) {
                reject(err);
            } else {
                fulfil(files);
            }
        });
    });
}
1089359
def parse_all_data(self):
        """"""
        self._master_df.columns = ["domain", "entity", "state", "last_changed"]

        # Check if state is float and store in numericals category.
        self._master_df["numerical"] = self._master_df["state"].apply(
            lambda x: functions.isfloat(x)
        )

        # Multiindexing
        self._master_df.set_index(
            ["domain", "entity", "numerical", "last_changed"], inplace=True
        )
1207364
def _read_gtf(gtf):
    
    if not gtf:
        return gtf
    db = defaultdict(list)
    with open(gtf) as in_handle:
        for line in in_handle:
            if line.startswith("#"):
                continue
            cols = line.strip().split("\t")
            name = [n.split("=")[1] for n in cols[-1].split(";") if n.startswith("Name")]
            chrom, start, end, strand = cols[0], cols[3], cols[4], cols[6]
            if cols[2] == "miRNA_primary_transcript":
                db[name[0]].append([chrom, int(start), int(end), strand])
    return db
1185087
def get_reads(sam, \
        contigs = False, mismatches = False, mm_option = False, \
        sort_sam = True, req_map = False, region = False, sbuffer = False):
    
    tempdir = '%s/' % (os.path.abspath(sam).rsplit('/', 1)[0])
    if sort_sam is True:
        mapping = '%s.sorted.sam' % (sam.rsplit('.', 1)[0])
        if sam != '-':
            if os.path.exists(mapping) is False:
                os.system("\
                    sort -k1 --buffer-size=%sG -T %s -o %s %s\
                    " % (sbuffer, tempdir, mapping, sam))
        else:
            mapping = 'stdin-sam.sorted.sam'
            p = Popen("sort -k1 --buffer-size=%sG -T %s -o %s" \
                    % (sbuffer, tempdir, mapping), stdin = sys.stdin, shell = True)
            p.communicate()
        mapping = open(mapping)
    else:
        if sam == '-':
            mapping = sys.stdin
        else:
            mapping = open(sam)
    for read in reads_from_mapping(mapping, contigs, mismatches, mm_option, req_map, region):
        yield read
1058365
def read_items(group, version='1.1', check=False):
    """"""
    if version == '0.1':
        # parse unicode to strings
        return ''.join(
            [unichr(int(c)) for c in group['files'][...]]
        ).replace('/-', '/').split('/\\')
    elif version == '1.0':
        return Items(list(group['files'][...]), check)
    else:
        return Items(list(group['items'][...]), check)
1370901
def get_train_eval_files(input_dir):
  """"""
  data_dir = _get_latest_data_dir(input_dir)
  train_pattern = os.path.join(data_dir, 'train*.tfrecord.gz')
  eval_pattern = os.path.join(data_dir, 'eval*.tfrecord.gz')
  train_files = file_io.get_matching_files(train_pattern)
  eval_files = file_io.get_matching_files(eval_pattern)
  return train_files, eval_files
274454
def load_and_parse(self, package_name, root_dir, relative_dirs,
                       resource_type, tags=None):
        """"""

        extension = "[!.#~]*.sql"

        if tags is None:
            tags = []

        if dbt.flags.STRICT_MODE:
            dbt.contracts.project.ProjectList(**self.all_projects)

        file_matches = dbt.clients.system.find_matching(
            root_dir,
            relative_dirs,
            extension)

        result = []

        for file_match in file_matches:
            file_contents = dbt.clients.system.load_file_contents(
                file_match.get('absolute_path'))

            parts = dbt.utils.split_path(file_match.get('relative_path', ''))
            name, _ = os.path.splitext(parts[-1])

            path = self.get_compiled_path(name,
                                          file_match.get('relative_path'))

            original_file_path = os.path.join(
                file_match.get('searched_path'),
                path)

            result.append({
                'name': name,
                'root_path': root_dir,
                'resource_type': resource_type,
                'path': path,
                'original_file_path': original_file_path,
                'package_name': package_name,
                'raw_sql': file_contents
            })

        return self.parse_sql_nodes(result, tags)