Checks if there's a suitable SampleSheet.csv present for the run
188715
def run_has_samplesheet(fc_dir, config, require_single=True):
    
    fc_name, _ = flowcell.parse_dirname(fc_dir)
    sheet_dirs = config.get("samplesheet_directories", [])
    fcid_sheet = {}
    for ss_dir in (s for s in sheet_dirs if os.path.exists(s)):
        with utils.chdir(ss_dir):
            for ss in glob.glob("*.csv"):
                fc_ids = _get_flowcell_id(ss, require_single)
                for fcid in fc_ids:
                    if fcid:
                        fcid_sheet[fcid] = os.path.join(ss_dir, ss)
    # difflib handles human errors while entering data on the SampleSheet.
    # Only one best candidate is returned (if any). 0.85 cutoff allows for
    # maximum of 2 mismatches in fcid

    potential_fcids = difflib.get_close_matches(fc_name, fcid_sheet.keys(), 1, 0.85)
    if len(potential_fcids) > 0 and potential_fcids[0] in fcid_sheet:
        return fcid_sheet[potential_fcids[0]]
    else:
        return None
1200085
def _ReadCsvDict(self, file_name, cols, required, deprecated):
    """"""
    assert file_name.endswith(".txt")
    table_name = file_name[0:-4]
    contents = self._GetUtf8Contents(file_name)
    if not contents:
      return

    eol_checker = util.EndOfLineChecker(StringIO.StringIO(contents),
                                   file_name, self._problems)
    # The csv module doesn't provide a way to skip trailing space, but when I
    # checked 15/675 feeds had trailing space in a header row and 120 had spaces
    # after fields. Space after header fields can cause a serious parsing
    # problem, so warn. Space after body fields can cause a problem time,
    # integer and id fields; they will be validated at higher levels.
    reader = csv.reader(eol_checker, skipinitialspace=True)

    raw_header = next(reader)
    header_occurrences = util.defaultdict(lambda: 0)
    header = []
    valid_columns = []  # Index into raw_header and raw_row
    for i, h in enumerate(raw_header):
      h_stripped = h.strip()
      if not h_stripped:
        self._problems.CsvSyntax(
            description="The header row should not contain any blank values. "
                        "The corresponding column will be skipped for the "
                        "entire file.",
            context=(file_name, 1, [''] * len(raw_header), raw_header),
            type=problems.TYPE_ERROR)
        continue
      elif h != h_stripped:
        self._problems.CsvSyntax(
            description="The header row should not contain any "
                        "space characters.",
            context=(file_name, 1, [''] * len(raw_header), raw_header),
            type=problems.TYPE_WARNING)
      header.append(h_stripped)
      valid_columns.append(i)
      header_occurrences[h_stripped] += 1

    for name, count in header_occurrences.items():
      if count > 1:
        self._problems.DuplicateColumn(
            header=name,
            file_name=file_name,
            count=count)

    self._schedule._table_columns[table_name] = header

    # check for unrecognized columns, which are often misspellings
    header_context = (file_name, 1, [''] * len(header), header)
    valid_cols = cols + [deprecated_name for (deprecated_name, _) in deprecated]
    unknown_cols = set(header) - set(valid_cols)
    if len(unknown_cols) == len(header):
      self._problems.CsvSyntax(
            description="The header row did not contain any known column "
                        "names. The file is most likely missing the header row "
                        "or not in the expected CSV format.",
            context=(file_name, 1, [''] * len(raw_header), raw_header),
            type=problems.TYPE_ERROR)
    else:
      for col in unknown_cols:
        # this is provided in order to create a nice colored list of
        # columns in the validator output
        self._problems.UnrecognizedColumn(file_name, col, header_context)

    # check for missing required columns
    missing_cols = set(required) - set(header)
    for col in missing_cols:
      # this is provided in order to create a nice colored list of
      # columns in the validator output
      self._problems.MissingColumn(file_name, col, header_context)

    # check for deprecated columns
    for (deprecated_name, new_name) in deprecated:
      if deprecated_name in header:
        self._problems.DeprecatedColumn(file_name, deprecated_name, new_name,
                                        header_context)

    line_num = 1  # First line read by reader.next() above
    for raw_row in reader:
      line_num += 1
      if len(raw_row) == 0:  # skip extra empty lines in file
        continue

      if len(raw_row) > len(raw_header):
        self._problems.OtherProblem('Found too many cells (commas) in line '
                                    '%d of file "%s".  Every row in the file '
                                    'should have the same number of cells as '
                                    'the header (first line) does.' %
                                    (line_num, file_name),
                                    (file_name, line_num),
                                    type=problems.TYPE_WARNING)

      if len(raw_row) < len(raw_header):
        self._problems.OtherProblem('Found missing cells (commas) in line '
                                    '%d of file "%s".  Every row in the file '
                                    'should have the same number of cells as '
                                    'the header (first line) does.' %
                                    (line_num, file_name),
                                    (file_name, line_num),
                                    type=problems.TYPE_WARNING)

      # raw_row is a list of raw bytes which should be valid utf-8. Convert each
      # valid_columns of raw_row into Unicode.
      valid_values = []
      unicode_error_columns = []  # index of valid_values elements with an error
      for i in valid_columns:
        try:
          valid_values.append(raw_row[i].decode('utf-8'))
        except UnicodeDecodeError:
          # Replace all invalid characters with REPLACEMENT CHARACTER (U+FFFD)
          valid_values.append(codecs.getdecoder("utf8")
                              (raw_row[i], errors="replace")[0])
          unicode_error_columns.append(len(valid_values) - 1)
        except IndexError:
          break

      # The error report may contain a dump of all values in valid_values so
      # problems can not be reported until after converting all of raw_row to
      # Unicode.
      for i in unicode_error_columns:
        self._problems.InvalidValue(header[i], valid_values[i],
                                    'Unicode error',
                                    (file_name, line_num,
                                     valid_values, header))

      # We strip ALL whitespace from around values.  This matches the behavior
      # of both the Google and OneBusAway GTFS parser.
      valid_values = [value.strip() for value in valid_values]

      d = dict(zip(header, valid_values))
      yield (d, line_num, header, valid_values)
354602
def run():
    """"""
    obj = CompareGOsCli()
    obj.write(obj.kws.get('xlsx'), obj.kws.get('ofile'), obj.kws.get('verbose', False))
569023
def _templates_match(t, family_file):
    
    return t.name == family_file.split(os.sep)[-1].split('_detections.csv')[0]
593965
def run_bcl2fastq(run_folder, ss_csv, config):
    
    bc_dir = os.path.join(run_folder, "Data", "Intensities", "BaseCalls")
    output_dir = os.path.join(run_folder, "fastq")

    if not os.path.exists(os.path.join(output_dir, "Makefile")):
        subprocess.check_call(["configureBclToFastq.pl", "--no-eamss",
                               "--input-dir", bc_dir, "--output-dir", output_dir,
                               "--sample-sheet", ss_csv])
    with utils.chdir(output_dir):
        cores = str(utils.get_in(config, ("algorithm", "num_cores"), 1))
        cmd = ["make", "-j", cores]
        if "submit_cmd" in config["process"] and "bcl2fastq_batch" in config["process"]:
            _submit_and_wait(cmd, cores, config, output_dir)
        else:
            subprocess.check_call(cmd)
    return output_dir
982974
def upload_runsummary(csv_filename, dryrun=False):
    """"""
    print("Checking '{}' for consistency.".format(csv_filename))
    if not os.path.exists(csv_filename):
        log.critical("{} -> file not found.".format(csv_filename))
        return
    try:
        df = pd.read_csv(csv_filename, sep='\t')
    except pd.errors.EmptyDataError as e:
        log.error(e)
        return

    cols = set(df.columns)

    if not REQUIRED_COLUMNS.issubset(cols):
        log.error(
            "Missing columns: {}.".format(
                ', '.join(str(c) for c in REQUIRED_COLUMNS - cols)
            )
        )
        return

    parameters = cols - REQUIRED_COLUMNS
    if len(parameters) < 1:
        log.error("No parameter columns found.")
        return

    if len(df) == 0:
        log.critical("Empty dataset.")
        return

    print(
        "Found data for parameters: {}.".format(
            ', '.join(str(c) for c in parameters)
        )
    )
    print("Converting CSV data into JSON")
    if dryrun:
        log.warn("Dryrun: adding 'TEST_' prefix to parameter names")
        prefix = "TEST_"
    else:
        prefix = ""
    data = convert_runsummary_to_json(df, prefix=prefix)
    print("We have {:.3f} MB to upload.".format(len(data) / 1024**2))

    print("Requesting database session.")
    db = kp.db.DBManager()    # noqa
    if kp.db.we_are_in_lyon():
        session_cookie = "sid=_kmcprod_134.158_lyo7783844001343100343mcprod1223user"    # noqa
    else:
        session_cookie = kp.config.Config().get('DB', 'session_cookie')
        if session_cookie is None:
            raise SystemExit("Could not restore DB session.")
    log.debug("Using the session cookie: {}".format(session_cookie))
    cookie_key, sid = session_cookie.split('=')
    print("Uploading the data to the database.")
    r = requests.post(
        RUNSUMMARY_URL, cookies={cookie_key: sid}, files={'datafile': data}
    )
    if r.status_code == 200:
        log.debug("POST request status code: {}".format(r.status_code))
        print("Database response:")
        db_answer = json.loads(r.text)
        for key, value in db_answer.items():
            print("  -> {}: {}".format(key, value))
        if db_answer['Result'] == 'OK':
            print("Upload successful.")
        else:
            log.critical("Something went wrong.")
    else:
        log.error("POST request status code: {}".format(r.status_code))
        log.critical("Something went wrong...")
        return
1371082
def sample_range(range, seed=nil)
      srand seed if seed
      rand * (range.end - range.begin) + range.begin
    end
1033962
private void checkFile() throws IORuntimeException {
		if (false == file.exists()) {
			throw new IORuntimeException("File not exist: " + file);
		}
		if (false == file.isFile()) {
			throw new IORuntimeException("Not a file:" + file);
		}
	}
1125880
def _get_sample(self, mode, encoding):
        
        self._open_file(mode, encoding)
        self._sample = self._file.read(UniversalCsvReader.sample_size)
        self._file.close()
489117
def valid_file(cls, filename):
        
        file_ex = os.path.splitext(filename)[1].replace('.', '', 1)
        return file_ex in SUPPORTED_FILES and is_excel_file(filename)
1232888
def sample(self, n):
    

    row_total_count = 0
    row_counts = []
    for file in self.files:
      with _util.open_local_or_gcs(file, 'r') as f:
        num_lines = sum(1 for line in f)
        row_total_count += num_lines
        row_counts.append(num_lines)

    names = None
    dtype = None
    if self._schema:
      _MAPPINGS = {
        'FLOAT': np.float64,
        'INTEGER': np.int64,
        'TIMESTAMP': np.datetime64,
        'BOOLEAN': np.bool,
      }
      names = [x['name'] for x in self._schema]
      dtype = {x['name']: _MAPPINGS.get(x['type'], object) for x in self._schema}

    skip_count = row_total_count - n
    # Get all skipped indexes. These will be distributed into each file.
    # Note that random.sample will raise Exception if skip_count is greater than rows count.
    skip_all = sorted(random.sample(range(0, row_total_count), skip_count))
    dfs = []
    for file, row_count in zip(self.files, row_counts):
      skip = [x for x in skip_all if x < row_count]
      skip_all = [x - row_count for x in skip_all if x >= row_count]
      with _util.open_local_or_gcs(file, 'r') as f:
        dfs.append(pd.read_csv(f, skiprows=skip, names=names, dtype=dtype, header=None))
    return pd.concat(dfs, axis=0, ignore_index=True)
406028
def matches_sample(output, target, threshold, is_correct, actual_output):
        
        return (bool(output > threshold) == bool(target)) == is_correct and actual_output == bool(output > threshold)
815467
def validate_csv(file):
    """"""
    try:
        # Detect encoding and dialect
        with file.open() as fp:
            encoding = detect_encoding(fp, default='utf-8')
            sample = fp.read(
                current_app.config.get('PREVIEWER_CSV_VALIDATION_BYTES', 1024))
            delimiter = csv.Sniffer().sniff(sample.decode(encoding)).delimiter
            is_valid = True
    except Exception as e:
        current_app.logger.debug(
            'File {0} is not valid CSV: {1}'.format(file.uri, e))
        encoding = ''
        delimiter = ''
        is_valid = False

    return {
        'delimiter': delimiter,
        'encoding': encoding,
        'is_valid': is_valid
    }
1253163
def isample(self, *args, **kwds):
        
        self._exc_info = None
        out = kwds.pop('out', sys.stdout)
        kwds['progress_bar'] = False

        def samp_targ(*args, **kwds):
            try:
                self.sample(*args, **kwds)
            except:
                self._exc_info = sys.exc_info()

        self._sampling_thread = Thread(
            target=samp_targ,
            args=args,
            kwargs=kwds)
        self.status = 'running'
        self._sampling_thread.start()
        self.iprompt(out=out)
1288593
def _check_trim(data):
    
    trim = data["algorithm"].get("trim_reads")
    if trim:
        if trim == "fastp" and data["algorithm"].get("align_split_size") is not False:
            raise ValueError("In sample %s, `trim_reads: fastp` currently requires `align_split_size: false`" %
                             (dd.get_sample_name(data)))
387292
def check_exists(filename, oappend=False):
    
    if op.exists(filename):
        if oappend:
            return oappend
        logging.error("`{0}` found, overwrite (Y/N)?".format(filename))
        overwrite = (raw_input() == 'Y')
    else:
        overwrite = True

    return overwrite
232347
def validate_data(args={})
      args = args.slice(:file_path, :data)
      args = args.compact
      file_path = args[:file_path]
      data = args[:data]

      utf_result = validate_csv(file_path) if file_path
      utf_result = validate_hashes(data) if data
      utf_result
    end