A function to seamlessly run on either scaffold dictionaries or
    info_frags.txt files without having to check the input first.
495806
def format_info_frags(info_frags):
    
    if isinstance(info_frags, dict):
        return info_frags
    else:
        try:
            scaffolds = parse_info_frags(info_frags)
            return scaffolds
        except OSError:
            print("Error when opening info_frags.txt")
            raise
1343396
def input_fn(data_file, num_epochs, shuffle, batch_size):
  """"""
  assert tf.gfile.Exists(data_file), (
      '%s not found. Please make sure you have run census_dataset.py and '
      'set the --data_dir argument to the correct path.' % data_file)

  def parse_csv(value):
    tf.logging.info('Parsing {}'.format(data_file))
    columns = tf.decode_csv(value, record_defaults=_CSV_COLUMN_DEFAULTS)
    features = dict(zip(_CSV_COLUMNS, columns))
    labels = features.pop('income_bracket')
    classes = tf.equal(labels, '>50K')  # binary classification
    return features, classes

  # Extract lines from input files using the Dataset API.
  dataset = tf.data.TextLineDataset(data_file)

  if shuffle:
    dataset = dataset.shuffle(buffer_size=_NUM_EXAMPLES['train'])

  dataset = dataset.map(parse_csv, num_parallel_calls=5)

  # We call repeat after shuffling, rather than before, to prevent separate
  # epochs from blending together.
  dataset = dataset.repeat(num_epochs)
  dataset = dataset.batch(batch_size)
  return dataset
456682
def _parse_and_clean_wikicode(raw_content):
  """"""
  wikicode = tfds.core.lazy_imports.mwparserfromhell.parse(raw_content)

  # Filters for references, tables, and file/image links.
  re_rm_wikilink = re.compile(
      "^(?:File|Image|Media):", flags=re.IGNORECASE | re.UNICODE)
  def rm_wikilink(obj):
    return bool(re_rm_wikilink.match(six.text_type(obj.title)))
  def rm_tag(obj):
    return six.text_type(obj.tag) in {"ref", "table"}
  def rm_template(obj):
    return obj.name.lower() in {
        "reflist", "notelist", "notelist-ua", "notelist-lr", "notelist-ur",
        "notelist-lg"}

  def try_remove_obj(obj, section):
    try:
      section.remove(obj)
    except ValueError:
      # For unknown reasons, objects are sometimes not found.
      pass

  section_text = []
  # Filter individual sections to clean.
  for section in wikicode.get_sections(
      flat=True, include_lead=True, include_headings=True):
    for obj in section.ifilter_wikilinks(matches=rm_wikilink, recursive=True):
      try_remove_obj(obj, section)
    for obj in section.ifilter_templates(matches=rm_template, recursive=True):
      try_remove_obj(obj, section)
    for obj in section.ifilter_tags(matches=rm_tag, recursive=True):
      try_remove_obj(obj, section)

    section_text.append(section.strip_code().strip())
  return "\n\n".join(section_text)
1172360
def main():
    
    params = argparse.ArgumentParser(description="Create or inspect a top-k lexicon for use during decoding.")
    subparams = params.add_subparsers(title="Commands")

    params_create = subparams.add_parser('create', description="Create top-k lexicon for use during decoding. "
                                                               "See sockeye_contrib/fast_align/README.md "
                                                               "for information on creating input lexical tables.")
    arguments.add_lexicon_args(params_create)
    arguments.add_lexicon_create_args(params_create)
    arguments.add_logging_args(params_create)
    params_create.set_defaults(func=create)

    params_inspect = subparams.add_parser('inspect', description="Inspect top-k lexicon for use during decoding.")
    arguments.add_lexicon_inspect_args(params_inspect)
    arguments.add_lexicon_args(params_inspect)
    params_inspect.set_defaults(func=inspect)

    args = params.parse_args()
    if 'func' not in args:
        params.print_help()
        return 1
    else:
        args.func(args)
1141350
def main():
    '''
    
    '''
    args = get_args()
    try:
        utils.make_output_dir(args.outdir)
        utils.init_logs(args)
        args.format = nanoplotter.check_valid_format(args.format)
        settings = vars(args)
        settings["path"] = path.join(args.outdir, args.prefix)
        sources = {
            "fastq": args.fastq,
            "bam": args.bam,
            "cram": args.cram,
            "fastq_rich": args.fastq_rich,
            "fastq_minimal": args.fastq_minimal,
            "summary": args.summary,
            "fasta": args.fasta,
            "ubam": args.ubam,
        }

        if args.pickle:
            datadf = pickle.load(open(args.pickle, 'rb'))
        else:
            datadf = get_input(
                source=[n for n, s in sources.items() if s][0],
                files=[f for f in sources.values() if f][0],
                threads=args.threads,
                readtype=args.readtype,
                combine="simple",
                barcoded=args.barcoded)
        if args.store:
            pickle.dump(
                obj=datadf,
                file=open(settings["path"] + "NanoPlot-data.pickle", 'wb'))
        if args.raw:
            datadf.to_csv(settings["path"] + "NanoPlot-data.tsv.gz",
                          sep="\t",
                          index=False,
                          compression="gzip")

        settings["statsfile"] = [make_stats(datadf, settings, suffix="")]
        datadf, settings = filter_and_transform_data(datadf, settings)
        if settings["filtered"]:  # Bool set when filter was applied in filter_and_transform_data()
            settings["statsfile"].append(make_stats(datadf, settings, suffix="_post_filtering"))

        if args.barcoded:
            barcodes = list(datadf["barcode"].unique())
            plots = []
            for barc in barcodes:
                logging.info("Processing {}".format(barc))
                settings["path"] = path.join(args.outdir, args.prefix + barc + "_")
                dfbarc = datadf[datadf["barcode"] == barc]
                if len(dfbarc) > 5:
                    settings["title"] = barc
                    plots.extend(
                        make_plots(dfbarc, settings)
                    )
                else:
                    sys.stderr.write("Found barcode {} less than 5x, ignoring...\n".format(barc))
                    logging.info("Found barcode {} less than 5 times, ignoring".format(barc))
        else:
            plots = make_plots(datadf, settings)
        make_report(plots, settings)
        logging.info("Finished!")
    except Exception as e:
        logging.error(e, exc_info=True)
        print("\n\n\nIf you read this then NanoPlot has crashed :-(")
        print("Please try updating NanoPlot and see if that helps...\n")
        print("If not, please report this issue at https://github.com/wdecoster/NanoPlot/issues")
        print("If you could include the log file that would be really helpful.")
        print("Thanks!\n\n\n")
        raise
32849
def _standalone(argv):
    """"""
    varprefix = "GEOCODE"
    per_locale = True
    separator = None
    chunks = -1
    try:
        opts, args = getopt.getopt(argv, "hv:fs:c:", ("help", "var=", "flat", "sep=", "chunks="))
    except getopt.GetoptError:
        prnt(__doc__, file=sys.stderr)
        sys.exit(1)
    for opt, arg in opts:
        if opt in ("-h", "--help"):
            prnt(__doc__, file=sys.stderr)
            sys.exit(1)
        elif opt in ("-v", "--var"):
            varprefix = arg
        elif opt in ("-f", "--flat"):
            per_locale = False
        elif opt in ("-s", "--sep"):
            separator = arg
        elif opt in ("-c", "--chunks"):
            chunks = int(arg)
        else:
            prnt("Unknown option %s" % opt, file=sys.stderr)
            prnt(__doc__, file=sys.stderr)
            sys.exit(1)
    if len(args) != 3:
        prnt(__doc__, file=sys.stderr)
        sys.exit(1)
    if per_locale:
        prefixdata = load_locale_prefixdata(args[0], separator=separator)
    else:
        prefixdata = {}
        load_locale_prefixdata_file(prefixdata, args[0], separator=separator)
    output_prefixdata_code(prefixdata, args[1], args[2], varprefix, per_locale, chunks)
825143
def _prep_items_from_base(base, in_files, metadata, separators, force_single=False):
    
    details = []
    in_files = _expand_dirs(in_files, KNOWN_EXTS)
    in_files = _expand_wildcards(in_files)

    ext_groups = collections.defaultdict(list)
    for ext, files in itertools.groupby(
            in_files, lambda x: KNOWN_EXTS.get(utils.splitext_plus(x)[-1].lower())):
        ext_groups[ext].extend(list(files))
    for ext, files in ext_groups.items():
        if ext == "bam":
            for f in files:
                details.append(_prep_bam_input(f, base))
        elif ext in ["fastq", "fq", "fasta"]:
            files, glob_files = _find_glob_matches(files, metadata)
            for fs in glob_files:
                details.append(_prep_fastq_input(fs, base))
            for fs in fastq.combine_pairs(files, force_single, separators=separators):
                details.append(_prep_fastq_input(fs, base))
        elif ext in ["vcf"]:
            for f in files:
                details.append(_prep_vcf_input(f, base))
        else:
            print("Ignoring unexpected input file types %s: %s" % (ext, list(files)))
    return details
1339862
def make_preprocessing_fn(output_dir, features, keep_target):
  
  def preprocessing_fn(inputs):
    """Preprocessing function.

    Args:
      inputs: dictionary of raw input tensors

    Returns:
      A dictionary of transformed tensors
    """
    stats = json.loads(
      file_io.read_file_to_string(
          os.path.join(output_dir, STATS_FILE)).decode())

    result = {}
    for name, transform in six.iteritems(features):
      transform_name = transform['transform']
      source_column = transform['source_column']

      if transform_name == KEY_TRANSFORM:
        transform_name = 'identity'
      elif transform_name == TARGET_TRANSFORM:
        if not keep_target:
          continue
        if file_io.file_exists(os.path.join(output_dir, VOCAB_ANALYSIS_FILE % source_column)):
          transform_name = 'one_hot'
        else:
          transform_name = 'identity'

      if transform_name == 'identity':
        result[name] = inputs[source_column]
      elif transform_name == 'scale':
        result[name] = _scale(
            inputs[name],
            min_x_value=stats['column_stats'][source_column]['min'],
            max_x_value=stats['column_stats'][source_column]['max'],
            output_min=transform.get('value', 1) * (-1),
            output_max=transform.get('value', 1))
      elif transform_name in [ONE_HOT_TRANSFORM, EMBEDDING_TRANSFROM, MULTI_HOT_TRANSFORM,
                              TFIDF_TRANSFORM, BOW_TRANSFORM]:
        vocab, ex_count = read_vocab_file(
            os.path.join(output_dir, VOCAB_ANALYSIS_FILE % source_column))

        if transform_name == TFIDF_TRANSFORM:
          separator = transform.get('separator', ' ')
          tokens = tf.string_split(inputs[source_column], separator)
          ids = _string_to_int(tokens, vocab)
          weights = _tfidf(
              x=ids,
              reduced_term_freq=ex_count + [0],
              vocab_size=len(vocab) + 1,
              corpus_size=stats['num_examples'])

          result[name + '_ids'] = ids
          result[name + '_weights'] = weights
        elif transform_name == BOW_TRANSFORM:
          separator = transform.get('separator', ' ')
          tokens = tf.string_split(inputs[source_column], separator)
          ids = _string_to_int(tokens, vocab)
          weights = _bag_of_words(x=ids)

          result[name + '_ids'] = ids
          result[name + '_weights'] = weights
        elif transform_name == MULTI_HOT_TRANSFORM:
          separator = transform.get('separator', ' ')
          tokens = tf.string_split(inputs[source_column], separator)
          result[name] = _string_to_int(tokens, vocab)
        else:
          # ONE_HOT_TRANSFORM: making a dense vector is done at training
          # EMBEDDING_TRANSFROM: embedding vectors have to be done at training
          result[name] = _string_to_int(inputs[source_column], vocab)
      elif transform_name == IMAGE_TRANSFORM:
        make_image_to_vec_fn = _make_image_to_vec_tito(
            name, checkpoint=transform.get('checkpoint', None))
        result[name] = make_image_to_vec_fn(inputs[source_column])
      else:
        raise ValueError('unknown transform %s' % transform_name)
    return result

  return preprocessing_fn
1240585
def prep_bam_inputs(out_dir, sample, call_file, bam_file):
    
    base = utils.splitext_plus(os.path.basename(bam_file))[0]
    with open(call_file) as in_handle:
        for cur_hla in (x.strip() for x in in_handle):
            out_file = os.path.join(utils.safe_makedir(os.path.join(out_dir, base)),
                                    "%s.type.%s.filtered.bam" % (base, cur_hla))
            if not os.path.exists(out_file):
                cmd = ["samtools", "view", "-b","-o", out_file, bam_file, cur_hla]
                subprocess.check_call(cmd)
269276
def scripts( self, mydict ):
        """"""

        if isinstance( mydict, dict ):
            for src, type in list( mydict.items( ) ):
                self.script( '', src=src, type='text/%s' % type )
        else:
            try:
                for src in mydict:
                    self.script( '', src=src, type='text/javascript' )
            except:
                raise TypeError( "Script should be given a dictionary of src:type pairs or a list of javascript src's." )
549166
def makefile(graph):
    """"""
    from renku.models.provenance.activities import ProcessRun, WorkflowRun

    for activity in graph.activities.values():
        if not isinstance(activity, ProcessRun):
            continue
        elif isinstance(activity, WorkflowRun):
            steps = activity.subprocesses.values()
        else:
            steps = [activity]

        for step in steps:
            click.echo(' '.join(step.outputs) + ': ' + ' '.join(step.inputs))
            tool = step.process
            click.echo(
                '\t@' + ' '.join(tool.to_argv()) + ' ' + ' '.join(
                    tool.STD_STREAMS_REPR[key] + ' ' + str(path)
                    for key, path in tool._std_streams().items()
                )
            )
595189
def _pre_dump(cls):
        """"""
        shutil.rmtree(cls.outdir, ignore_errors=True)
        os.makedirs(cls.outdir)
        super(PlotMetric, cls)._pre_dump()
1061945
def checked_open(filename, log=None, quiet=False):
    
    with open(filename, "rb") as handle:
        raw_data = handle.read()
    data = bencode.bdecode(raw_data)

    try:
        check_meta(data)
        if raw_data != bencode.bencode(data):
            raise ValueError("Bad bencoded data - dict keys out of order?")
    except ValueError as exc:
        if log:
            # Warn about it, unless it's a quiet value query
            if not quiet:
                log.warn("%s: %s" % (filename, exc))
        else:
            raise

    return data
1150540
def _check_file(parameters):
    """"""
    (filename, args) = parameters

    if filename == '-':
        contents = sys.stdin.read()
    else:
        with contextlib.closing(
            docutils.io.FileInput(source_path=filename)
        ) as input_file:
            contents = input_file.read()

    args = load_configuration_from_file(
        os.path.dirname(os.path.realpath(filename)), args)

    ignore_directives_and_roles(args.ignore_directives, args.ignore_roles)

    for substitution in args.ignore_substitutions:
        contents = contents.replace('|{}|'.format(substitution), 'None')

    ignore = {
        'languages': args.ignore_language,
        'messages': args.ignore_messages,
    }
    all_errors = []
    for error in check(contents,
                       filename=filename,
                       report_level=args.report,
                       ignore=ignore,
                       debug=args.debug):
        all_errors.append(error)
    return (filename, all_errors)
794609
def _run_prospector_on(filenames,
                       tools,
                       disabled_linters,
                       show_lint_files,
                       ignore_codes=None):
    
    from prospector.run import Prospector, ProspectorConfig

    assert tools

    tools = list(set(tools) - set(disabled_linters))
    return_dict = dict()
    ignore_codes = ignore_codes or list()

    # Early return if all tools were filtered out
    if not tools:
        return return_dict

    # pylint doesn't like absolute paths, so convert to relative.
    all_argv = (["-F", "-D", "-M", "--no-autodetect", "-s", "veryhigh"] +
                ("-t " + " -t ".join(tools)).split(" "))

    for filename in filenames:
        _debug_linter_status("prospector", filename, show_lint_files)

    with _custom_argv(all_argv + [os.path.relpath(f) for f in filenames]):
        prospector = Prospector(ProspectorConfig())
        prospector.execute()
        messages = prospector.get_messages() or list()
        for message in messages:
            message.to_absolute_path(os.getcwd())
            loc = message.location
            code = message.code

            if code in ignore_codes:
                continue

            key = _Key(loc.path, loc.line, code)
            return_dict[key] = message

    return return_dict
606470
def main():
    '''
    
    '''
    args = get_args()
    try:
        utils.make_output_dir(args.outdir)
        utils.init_logs(args, tool="NanoComp")
        args.format = nanoplotter.check_valid_format(args.format)
        settings = vars(args)
        settings["path"] = path.join(args.outdir, args.prefix)
        sources = [args.fastq, args.bam, args.summary, args.fasta]
        sourcename = ["fastq", "bam", "summary", "fasta"]
        if args.split_runs:
            split_dict = validate_split_runs_file(args.split_runs)
        datadf = nanoget.get_input(
            source=[n for n, s in zip(sourcename, sources) if s][0],
            files=[f for f in sources if f][0],
            threads=args.threads,
            readtype=args.readtype,
            names=args.names,
            barcoded=args.barcoded,
            combine="track")
        datadf, settings = filter_and_transform_data(datadf, vars(args))
        if args.raw:
            datadf.to_csv("NanoComp-data.tsv.gz", sep="\t", index=False, compression="gzip")
        if args.store:
            pickle.dump(
                obj=datadf,
                file=open(settings["path"] + "NanoComp-data.pickle", 'wb'))
        if args.split_runs:
            change_identifiers(datadf, split_dict)
        if args.barcoded:
            datadf["dataset"] = datadf["barcode"]
        identifiers = list(datadf["dataset"].unique())
        write_stats(
            datadfs=[datadf[datadf["dataset"] == i] for i in identifiers],
            outputfile=settings["path"] + "NanoStats.txt",
            names=identifiers)
        if args.plot != 'false':
            plots = make_plots(datadf, settings)
            make_report(plots, path.join(args.outdir, args.prefix))
        logging.info("Succesfully processed all input.")
    except Exception as e:
        logging.error(e, exc_info=True)
        raise
1074391
def pkt_check(*args, func=None):
    """"""
    func = func or inspect.stack()[2][3]
    for var in args:
        dict_check(var, func=func)
        dict_check(var.get('frame'), func=func)
        enum_check(var.get('protocol'), func=func)
        real_check(var.get('timestamp'), func=func)
        ip_check(var.get('src'), var.get('dst'), func=func)
        bool_check(var.get('syn'), var.get('fin'), func=func)
        int_check(var.get('srcport'), var.get('dstport'), var.get('index'), func=func)