Command line parser for dealing with *sick* models.
193321
def parser(input_args=None):
    

    parser = argparse.ArgumentParser(
        description="sick, the spectroscopic inference crank", epilog="Use "
            "'sick-model COMMAND -h' for information on a specific command."
            " Documentation and examples available at "
            "https://github.com/andycasey/sick")

    # Create a parent subparser.
    parent_parser = argparse.ArgumentParser(add_help=False)
    parent_parser.add_argument(
        "-v", "--verbose", dest="verbose", action="store_true", default=False,
        help="Vebose logging mode.")
    parent_parser.add_argument(
        "--clobber", dest="clobber", action="store_true", default=False, 
        help="Overwrite existing files if they already exist.")
    parent_parser.add_argument(
        "--debug", dest="debug", action="store_true", default=False,
        help="Enable debug mode. Any suppressed exception will be re-raised.")
    parent_parser.add_argument(
        "-o", "--output_dir", dest="output_dir", nargs="?", type=str,
        help="Directory for the files that will be created. If not given, this"
        " defaults to the current working directory.", default=os.getcwd())

    # Create subparsers.
    subparsers = parser.add_subparsers(title="command", dest="command",
        description="Specify the action to perform.")

    # Sub-parser for the create model command.
    # create <MODEL_NAME> <GRID_POINTS_FILENAME> <DISPERSION_FILENAME*>
    creator = subparsers.add_parser("create", parents=[parent_parser],
        help="Create a model from existing wavelength and flux files.")
    creator.add_argument("model_name", type=str,
        help="Name for the model to be created. This will form the prefix of "
        "filenames for the model.")
    creator.add_argument("grid_points_filename", type=str,
        help="Filename containing a header with parameter and channel names, "
        "and a list of grid points and associated flux filenames.")
    creator.add_argument("wavelength_filenames", type=str, nargs="+",
        help="Filenames containing the wavelengths for the channels referred "
        "to in the header of `grid_points_filename`.")
    creator.set_defaults(func=create)

    # Sub-parser for the recast model command.
    # recast <MODEL_NAME> <ORIGINAL_MODEL_NAME> <CHANNEL_DESCRIPTION_FILENAME>
    recaster = subparsers.add_parser("recast", parents=[parent_parser],
        help="Take an existing model and create a new model with different "
            "wavelength ranges, binning, and/or resolving powers.")
    recaster.add_argument("model_name", type=str,
        help="Name for the model to be created. This will form the prefix of "
        "filenames for the model.")
    recaster.add_argument("original_model_filename", type=str,
        help="Path of the YAML-formatted model filename to use as a template.")
    recaster.add_argument("channel_description_filename", type=str,
        help="Path to a filename containing a description of the channels to "
        "be cast.")
    recaster.set_defaults(func=recast)

    # Sub-parser for the download model command.
    # download <MODEL_NAME>
    download_parser = subparsers.add_parser("download", parents=[parent_parser],
        help="Download a model from an online repository.")
    download_parser.add_argument("model_name", nargs="?",
        help="The name of the pre-cached model grid to download, or 'list' "
            "(default) to see what models are available.", default="list")
    download_parser.set_defaults(func=download)

    args = parser.parse_args(input_args)
    
    # Setup logging, bro.
    logger.setLevel(logging.DEBUG if args.verbose else logging.INFO)

    return args
514742
def parser(rules=None, **kwargs):
    """"""
    rules = RULES_SPLUNK if rules is None else dict(RULES_SPLUNK, **rules)
    return Parser(rules, **kwargs)
708835
def main():
    """"""

    parser = OptionParser(conflict_handler="resolve")

    parser.add_option('--humanTRA', '--human_T_alpha', action='store_true', dest='humanTRA', default=False, help='use default human TRA model (T cell alpha chain)')
    parser.add_option('--humanTRB', '--human_T_beta', action='store_true', dest='humanTRB', default=False, help='use default human TRB model (T cell beta chain)')
    parser.add_option('--mouseTRB', '--mouse_T_beta', action='store_true', dest='mouseTRB', default=False, help='use default mouse TRB model (T cell beta chain)')
    parser.add_option('--humanIGH', '--human_B_heavy', action='store_true', dest='humanIGH', default=False, help='use default human IGH model (B cell heavy chain)')
    parser.add_option('--set_custom_model_VDJ', dest='vdj_model_folder', metavar='PATH/TO/FOLDER/', help='specify PATH/TO/FOLDER/ for a custom VDJ generative model')
    parser.add_option('--set_custom_model_VJ', dest='vj_model_folder', metavar='PATH/TO/FOLDER/', help='specify PATH/TO/FOLDER/ for a custom VJ generative model')

    parser.add_option('-i', '--infile', dest = 'infile_name',metavar='PATH/TO/FILE', help='read in CDR3 sequences (and optionally V/J masks) from PATH/TO/FILE')
    parser.add_option('-o', '--outfile', dest = 'outfile_name', metavar='PATH/TO/FILE', help='write CDR3 sequences and pgens to PATH/TO/FILE')
    parser.add_option('--seq_in', '--seq_index', type='int', metavar='INDEX', dest='seq_in_index', default = 0, help='specifies sequences to be read in are in column INDEX. Default is index 0 (the first column).')

    parser.add_option('--v_in', '--v_mask_index', type='int', metavar='INDEX', dest='V_mask_index', help='specifies V_masks are found in column INDEX in the input file. Default is no V mask.')
    parser.add_option('--j_in', '--j_mask_index', type='int', metavar='INDEX', dest='J_mask_index', help='specifies J_masks are found in column INDEX in the input file. Default is no J mask.')

    parser.add_option('--v_mask', type='string', dest='V_mask', help='specify V usage to condition Pgen on for seqs read in as arguments.')
    parser.add_option('--j_mask', type='string', dest='J_mask', help='specify J usage to condition Pgen on for seqs read in as arguments.')

    parser.add_option('-m', '--max_number_of_seqs', type='int',metavar='N', dest='max_number_of_seqs', help='compute Pgens for at most N sequences.')
    parser.add_option('--lines_to_skip', type='int',metavar='N', dest='lines_to_skip', default = 0, help='skip the first N lines of the file. Default is 0.')
    parser.add_option('-a', '--alphabet_filename', dest='alphabet_filename', metavar='PATH/TO/FILE', help="specify PATH/TO/FILE defining a custom 'amino acid' alphabet. Default is no custom alphabet.")
    parser.add_option('--seq_type_out', type='choice',metavar='SEQ_TYPE', dest='seq_type_out',  choices=['all', 'ntseq', 'nucleotide', 'aaseq', 'amino_acid'], help="if read in sequences are ntseqs, declare what type of sequence to compute pgen for. Default is all. Choices: 'all', 'ntseq', 'nucleotide', 'aaseq', 'amino_acid'")
    parser.add_option('--skip_off','--skip_empty_off', action='store_true', dest = 'skip_empty', default=True, help='stop skipping empty or blank sequences/lines (if for example you want to keep line index fidelity between the infile and outfile).')

    parser.add_option('--display_off', action='store_false', dest='display_seqs', default=True, help='turn the sequence display off (only applies in write-to-file mode). Default is on.')
    parser.add_option('--num_lines_for_display', type='int', metavar='N', default = 50, dest='num_lines_for_display', help='N lines of the output file are displayed when sequence display is on. Also used to determine the number of sequences to average over for speed and time estimates.')
    parser.add_option('--time_updates_off', action='store_false', dest='time_updates', default=True, help='turn time updates off (only applies when sequence display is disabled).')
    parser.add_option('--seqs_per_time_update', type='float', metavar='N', default = 100, dest='seqs_per_time_update', help='specify the number of sequences between time updates. Default is 1e5.')

    parser.add_option('-d', '--delimiter', type='choice', dest='delimiter',  choices=['tab', 'space', ',', ';', ':'], help="declare infile delimiter. Default is tab for .tsv input files, comma for .csv files, and any whitespace for all others. Choices: 'tab', 'space', ',', ';', ':'")
    parser.add_option('--raw_delimiter', type='str', dest='delimiter', help="declare infile delimiter as a raw string.")
    parser.add_option('--delimiter_out', type='choice', dest='delimiter_out',  choices=['tab', 'space', ',', ';', ':'], help="declare outfile delimiter. Default is tab for .tsv output files, comma for .csv files, and the infile delimiter for all others. Choices: 'tab', 'space', ',', ';', ':'")
    parser.add_option('--raw_delimiter_out', type='str', dest='delimiter_out', help="declare for the delimiter outfile as a raw string.")
    parser.add_option('--gene_mask_delimiter', type='choice', dest='gene_mask_delimiter',  choices=['tab', 'space', ',', ';', ':'], help="declare gene mask delimiter. Default comma unless infile delimiter is comma, then default is a semicolon. Choices: 'tab', 'space', ',', ';', ':'")
    parser.add_option('--raw_gene_mask_delimiter', type='str', dest='gene_mask_delimiter', help="declare delimiter of gene masks as a raw string.")
    parser.add_option('--comment_delimiter', type='str', dest='comment_delimiter', help="character or string to indicate comment or header lines to skip.")


    (options, args) = parser.parse_args()

    #Check that the model is specified properly
    main_folder = os.path.dirname(__file__)

    default_models = {}
    default_models['humanTRA'] = [os.path.join(main_folder, 'default_models', 'human_T_alpha'),  'VJ']
    default_models['humanTRB'] = [os.path.join(main_folder, 'default_models', 'human_T_beta'), 'VDJ']
    default_models['mouseTRB'] = [os.path.join(main_folder, 'default_models', 'mouse_T_beta'), 'VDJ']
    default_models['humanIGH'] = [os.path.join(main_folder, 'default_models', 'human_B_heavy'), 'VDJ']

    num_models_specified = sum([1 for x in default_models.keys() + ['vj_model_folder', 'vdj_model_folder'] if getattr(options, x)])

    if num_models_specified == 1: #exactly one model specified
        try:
            d_model = [x for x in default_models.keys() if getattr(options, x)][0]
            model_folder = default_models[d_model][0]
            recomb_type = default_models[d_model][1]
        except IndexError:
            if options.vdj_model_folder: #custom VDJ model specified
                model_folder = options.vdj_model_folder
                recomb_type = 'VDJ'
            elif options.vj_model_folder: #custom VJ model specified
                model_folder = options.vj_model_folder
                recomb_type = 'VJ'
    elif num_models_specified == 0:
        print 'Need to indicate generative model.'
        print 'Exiting...'
        return -1
    elif num_models_specified > 1:
        print 'Only specify one model'
        print 'Exiting...'
        return -1

    #Check that all model and genomic files exist in the indicated model folder
    if not os.path.isdir(model_folder):
        print 'Check pathing... cannot find the model folder: ' + model_folder
        print 'Exiting...'
        return -1

    params_file_name = os.path.join(model_folder,'model_params.txt')
    marginals_file_name = os.path.join(model_folder,'model_marginals.txt')
    V_anchor_pos_file = os.path.join(model_folder,'V_gene_CDR3_anchors.csv')
    J_anchor_pos_file = os.path.join(model_folder,'J_gene_CDR3_anchors.csv')

    for x in [params_file_name, marginals_file_name, V_anchor_pos_file, J_anchor_pos_file]:
        if not os.path.isfile(x):
            print 'Cannot find: ' + x
            print 'Please check the files (and naming conventions) in the model folder ' + model_folder
            print 'Exiting...'
            return -1

    alphabet_filename = options.alphabet_filename #used if a custom alphabet is to be specified
    if alphabet_filename is not None:
        if not os.path.isfile(alphabet_filename):
            print 'Cannot find custom alphabet file: ' + infile_name
            print 'Exiting...'
            return -1

    #Load up model based on recomb_type
    #VDJ recomb case --- used for TCRB and IGH
    if recomb_type == 'VDJ':
        genomic_data = load_model.GenomicDataVDJ()
        genomic_data.load_igor_genomic_data(params_file_name, V_anchor_pos_file, J_anchor_pos_file)
        generative_model = load_model.GenerativeModelVDJ()
        generative_model.load_and_process_igor_model(marginals_file_name)
        pgen_model = generation_probability.GenerationProbabilityVDJ(generative_model, genomic_data, alphabet_filename)
    #VJ recomb case --- used for TCRA and light chain
    elif recomb_type == 'VJ':
        genomic_data = load_model.GenomicDataVJ()
        genomic_data.load_igor_genomic_data(params_file_name, V_anchor_pos_file, J_anchor_pos_file)
        generative_model = load_model.GenerativeModelVJ()
        generative_model.load_and_process_igor_model(marginals_file_name)
        pgen_model = generation_probability.GenerationProbabilityVJ(generative_model, genomic_data, alphabet_filename)

    aa_alphabet = ''.join(pgen_model.codons_dict.keys())

    if options.infile_name is not None:
        infile_name = options.infile_name

        if not os.path.isfile(infile_name):
            print 'Cannot find input file: ' + infile_name
            print 'Exiting...'
            return -1

    if options.outfile_name is not None:
        outfile_name = options.outfile_name
        if os.path.isfile(outfile_name):
            if not raw_input(outfile_name + ' already exists. Overwrite (y/n)? ').strip().lower() in ['y', 'yes']:
                print 'Exiting...'
                return -1

    #Parse delimiter
    delimiter = options.delimiter
    if delimiter is None: #Default case
        if options.infile_name is None:
            delimiter = '\t'
        elif infile_name.endswith('.tsv'): #parse TAB separated value file
            delimiter = '\t'
        elif infile_name.endswith('.csv'): #parse COMMA separated value file
            delimiter = ','
    else:
        try:
            delimiter = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[delimiter]
        except KeyError:
            pass #Other string passed as the delimiter.

    #Parse delimiter_out
    delimiter_out = options.delimiter_out
    if delimiter_out is None: #Default case
        if delimiter is None:
            delimiter_out = '\t'
        else:
            delimiter_out = delimiter
        if options.outfile_name is None:
            pass
        elif outfile_name.endswith('.tsv'): #output TAB separated value file
            delimiter_out = '\t'
        elif outfile_name.endswith('.csv'): #output COMMA separated value file
            delimiter_out = ','
    else:
        try:
            delimiter_out = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[delimiter_out]
        except KeyError:
            pass #Other string passed as the delimiter.

    #Parse gene_delimiter
    gene_mask_delimiter = options.gene_mask_delimiter
    if gene_mask_delimiter is None: #Default case
        gene_mask_delimiter = ','
        if delimiter == ',':
            gene_mask_delimiter = ';'
    else:
        try:
            gene_mask_delimiter = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[gene_mask_delimiter]
        except KeyError:
            pass #Other string passed as the delimiter.


    #More options
    time_updates = options.time_updates
    display_seqs = options.display_seqs
    num_lines_for_display = options.num_lines_for_display
    seq_in_index = options.seq_in_index #where in the line the sequence is after line.split(delimiter)
    lines_to_skip = options.lines_to_skip #one method of skipping header
    comment_delimiter = options.comment_delimiter #another method of skipping header
    seqs_per_time_update = options.seqs_per_time_update
    max_number_of_seqs = options.max_number_of_seqs
    V_mask_index = options.V_mask_index #Default is not conditioning on V identity
    J_mask_index = options.J_mask_index #Default is not conditioning on J identity
    skip_empty = options.skip_empty

    seq_type_out = options.seq_type_out #type of pgens to be computed. Can be ntseq, aaseq, or both
    if seq_type_out is not None:
        seq_type_out = {'all': None, 'ntseq': 'ntseq', 'nucleotide': 'ntseq', 'aaseq': 'aaseq', 'amino_acid': 'aaseq'}[seq_type_out]

    if options.infile_name is None: #No infile specified -- args should be the input seqs
        print_warnings = True
        seqs = args
        seq_types = [determine_seq_type(seq, aa_alphabet) for seq in seqs]
        unrecognized_seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is None]
        if len(unrecognized_seqs) > 0 and print_warnings:
            print 'The following sequences/arguments were not recognized: ' + ', '.join(unrecognized_seqs)
        seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is not None]
        seq_types = [seq_type for seq_type in seq_types if seq_type is not None]


        #Format V and J masks -- uniform for all argument input sequences
        try:
            V_mask = options.V_mask.split(',')
            unrecognized_v_genes = [v for v in V_mask if v not in pgen_model.V_mask_mapping.keys()]
            V_mask = [v for v in V_mask if v in pgen_model.V_mask_mapping.keys()]
            if len(unrecognized_v_genes) > 0:
                print 'These V genes/alleles are not recognized: ' + ', '.join(unrecognized_v_genes)
            if len(V_mask) == 0:
                print 'No recognized V genes/alleles in the provided V_mask. Continuing without conditioning on V usage.'
                V_mask = None
        except AttributeError:
            V_mask = options.V_mask #Default is None, i.e. not conditioning on V identity

        try:
            J_mask = options.J_mask.split(',')
            unrecognized_j_genes = [j for j in J_mask if j not in pgen_model.J_mask_mapping.keys()]
            J_mask = [j for j in J_mask if j in pgen_model.J_mask_mapping.keys()]
            if len(unrecognized_j_genes) > 0:
                print 'These J genes/alleles are not recognized: ' + ', '.join(unrecognized_j_genes)
            if len(J_mask) == 0:
                print 'No recognized J genes/alleles in the provided J_mask. Continuing without conditioning on J usage.'
                J_mask = None
        except AttributeError:
            J_mask = options.J_mask #Default is None, i.e. not conditioning on J identity

        print ''
        start_time = time.time()
        for seq, seq_type in zip(seqs, seq_types):
            if seq_type == 'aaseq':
                c_pgen = pgen_model.compute_aa_CDR3_pgen(seq, V_mask, J_mask, print_warnings)
                print 'Pgen of the amino acid sequence ' + seq + ': ' + str(c_pgen)
                print ''
            elif seq_type == 'regex':
                c_pgen = pgen_model.compute_regex_CDR3_template_pgen(seq, V_mask, J_mask, print_warnings)
                print 'Pgen of the regular expression sequence ' + seq + ': ' + str(c_pgen)
                print ''
            elif seq_type == 'ntseq':
                if seq_type_out is None or seq_type_out == 'ntseq':
                    c_pgen_nt = pgen_model.compute_nt_CDR3_pgen(seq, V_mask, J_mask, print_warnings)
                    print 'Pgen of the nucleotide sequence ' + seq + ': ' + str(c_pgen_nt)
                if seq_type_out is None or seq_type_out == 'aaseq':
                    c_pgen_aa = pgen_model.compute_aa_CDR3_pgen(nt2aa(seq), V_mask, J_mask, print_warnings)
                    print 'Pgen of the amino acid sequence nt2aa(' + seq + ') = ' + nt2aa(seq) + ': ' + str(c_pgen_aa)
                print ''

        c_time = time.time() - start_time
        if c_time > 86400: #more than a day
            c_time_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(c_time)/86400, (int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
        elif c_time > 3600: #more than an hr
            c_time_str = '%d hours, %d minutes, and %.2f seconds.'%((int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
        elif c_time > 60: #more than a min
            c_time_str = '%d minutes and %.2f seconds.'%((int(c_time)/60)%60, c_time%60)
        else:
            c_time_str = '%.2f seconds.'%(c_time)

        print 'Completed pgen computation in: ' + c_time_str

    else: #Read sequences in from file
        print_warnings = False #Most cases of reading in from file should have warnings disabled
        seqs = []
        seq_types = []
        V_usage_masks = []
        J_usage_masks = []

        infile = open(infile_name, 'r')

        for i, line in enumerate(infile):
            if comment_delimiter is not None: #Default case -- no comments/header delimiter
                if line.startswith(comment_delimiter): #allow comments
                    continue
            if i < lines_to_skip:
                continue

            if delimiter is None: #Default delimiter is any whitespace
                split_line = line.split()
            else:
                split_line = line.split(delimiter)

            #Find the seq
            try:
                seq = split_line[seq_in_index].strip()
                if len(seq.strip()) == 0:
                    if skip_empty:
                        continue
                    else:
                        seqs.append(seq) #keep the blank seq as a placeholder
                        seq_types.append('aaseq')
                else:
                    seqs.append(seq)
                    seq_types.append(determine_seq_type(seq, aa_alphabet))
            except IndexError: #no index match for seq
                if skip_empty and len(line.strip()) == 0:
                    continue
                print 'seq_in_index is out of range'
                print 'Exiting...'
                infile.close()
                return -1

            #Find and format V_usage_mask
            if V_mask_index is None:
                V_usage_masks.append(None) #default mask
            else:
                try:
                    V_usage_mask = split_line[V_mask_index].strip().split(gene_mask_delimiter)
                    #check that all V gene/allele names are recognized
                    if all([v in pgen_model.V_mask_mapping for v in V_usage_mask]):
                        V_usage_masks.append(V_usage_mask)
                    else:
                        print str(V_usage_mask) + " is not a usable V_usage_mask composed exclusively of recognized V gene/allele names"
                        print 'Unrecognized V gene/allele names: ' + ', '.join([v for v in V_usage_mask if not v in pgen_model.V_mask_mapping.keys()])
                        print 'Exiting...'
                        infile.close()
                        return -1
                except IndexError: #no index match for V_mask_index
                    print 'V_mask_index is out of range'
                    print 'Exiting...'
                    infile.close()
                    return -1

            #Find and format J_usage_mask
            if J_mask_index is None:
                J_usage_masks.append(None) #default mask
            else:
                try:
                    J_usage_mask = split_line[J_mask_index].strip().split(gene_mask_delimiter)
                    #check that all V gene/allele names are recognized
                    if all([j in pgen_model.J_mask_mapping for j in J_usage_mask]):
                        J_usage_masks.append(J_usage_mask)
                    else:
                        print str(J_usage_mask) + " is not a usable J_usage_mask composed exclusively of recognized J gene/allele names"
                        print 'Unrecognized J gene/allele names: ' + ', '.join([j for j in J_usage_mask if not j in pgen_model.J_mask_mapping.keys()])
                        print 'Exiting...'
                        infile.close()
                        return -1
                except IndexError: #no index match for J_mask_index
                    print 'J_mask_index is out of range'
                    print 'Exiting...'
                    infile.close()
                    return -1

            if max_number_of_seqs is not None:
                if len(seqs) >= max_number_of_seqs:
                    break


        unrecognized_seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is None]
        if len(unrecognized_seqs) > 0 and len(unrecognized_seqs) < len(seqs):
            if print_warnings or options.outfile_name is not None:
                print 'Some strings read in were not parsed as sequences -- they will be omitted.'
                print 'Examples of improperly read strings: '
                for unrecognized_seq in unrecognized_seqs[:10]:
                    print unrecognized_seq
            seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is not None]
            V_usage_masks = [V_usage_mask for i, V_usage_mask in enumerate(V_usage_masks) if seq_types[i] is not None]
            seq_types = [seq_type for seq_type in seq_types if seq_type is not None]
        elif len(unrecognized_seqs) > 0 and len(unrecognized_seqs) == len(seqs):
            print 'None of the read in strings were parsed as sequences. Check input file.'
            print 'Examples of improperly read strings:'
            for unrecognized_seq in unrecognized_seqs[:10]:
                print unrecognized_seq
            print 'Exiting...'
            return -1

        infile.close()


        if options.outfile_name is not None: #OUTFILE SPECIFIED, allow printed info/display

            print 'Successfully read in and formatted ' + str(len(seqs)) + ' sequences and any V or J usages.'
            if display_seqs:
                sys.stdout.write('\r'+'Continuing to Pgen computation in 3... ')
                sys.stdout.flush()
                time.sleep(0.4)
                sys.stdout.write('\r'+'Continuing to Pgen computation in 2... ')
                sys.stdout.flush()
                time.sleep(0.4)
                sys.stdout.write('\r'+'Continuing to Pgen computation in 1... ')
                sys.stdout.flush()
                time.sleep(0.4)
            else:
                print 'Continuing to Pgen computation.'
                print_warnings = True #Display is off, can print warnings

            if display_seqs:
                lines_for_display = []
                times_for_speed_calc = [time.time()]

            outfile = open(outfile_name, 'w')
            start_time = time.time()
            for i, seq in enumerate(seqs):
                if seq_types[i] == 'aaseq':
                    #Compute Pgen and print out
                    c_pgen_line = seq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                if seq_types[i] == 'regex':
                    #Compute Pgen and print out
                    c_pgen_line = seq + delimiter_out + str(pgen_model.compute_regex_CDR3_template_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                elif seq_types[i] == 'ntseq':
                    ntseq = seq
                    if len(ntseq) % 3 == 0: #inframe sequence
                        aaseq = nt2aa(ntseq)
                        #Compute Pgen and print out based on recomb_type and seq_type_out
                        if seq_type_out is None:
                            c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings)) + delimiter_out + aaseq + delimiter_out +  str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                        elif seq_type_out == 'ntseq':
                            c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                        elif seq_type_out == 'aaseq':
                            c_pgen_line = aaseq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                    else: #out of frame sequence -- Pgens are 0 and use 'out_of_frame' for aaseq
                        if seq_type_out is None:
                            c_pgen_line = ntseq + delimiter_out + '0' + delimiter_out + 'out_of_frame' + delimiter_out + '0'
                        elif seq_type_out == 'ntseq':
                            c_pgen_line = ntseq + delimiter_out + '0'
                        elif seq_type_out == 'aaseq':
                            c_pgen_line = 'out_of_frame' + delimiter_out + '0'

                outfile.write(c_pgen_line + '\n')

                #Print time update
                if display_seqs:
                    cc_time = time.time()
                    c_time = cc_time - start_time
                    times_for_speed_calc = [cc_time] + times_for_speed_calc[:num_lines_for_display]
                    c_avg_speed = (len(times_for_speed_calc)-1)/float(times_for_speed_calc[0] - times_for_speed_calc[-1])

                    #eta = ((len(seqs) - (i+1))/float(i+1))*c_time

                    eta = (len(seqs) - (i+1))/c_avg_speed

                    lines_for_display = [c_pgen_line] + lines_for_display[:num_lines_for_display]


                    c_time_str = '%s hours, %s minutes, and %s seconds.'%(repr(int(c_time)/3600).rjust(3), repr((int(c_time)/60)%60).rjust(2), repr(int(c_time)%60).rjust(2))
                    eta_str = '%s hours, %s minutes, and %s seconds.'%(repr(int(eta)/3600).rjust(3), repr((int(eta)/60)%60).rjust(2), repr(int(eta)%60).rjust(2))
                    time_str = 'Time to compute Pgen on %s seqs: %s \nEst. time for remaining %s seqs: %s'%(repr(i+1).rjust(9), c_time_str, repr(len(seqs) - (i + 1)).rjust(9), eta_str)
                    speed_str = 'Current Pgen computation speed: %s seqs/min'%(repr(round((len(times_for_speed_calc)-1)*60/float(times_for_speed_calc[0] - times_for_speed_calc[-1]), 2)).rjust(8))
                    display_str = '\n'.join(lines_for_display[::-1]) + '\n' + '-'*80 + '\n' + time_str + '\n' + speed_str + '\n' + '-'*80
                    print '\033[2J' + display_str
                elif (i+1)%seqs_per_time_update == 0 and time_updates:
                    c_time = time.time() - start_time
                    eta = ((len(seqs) - (i+1))/float(i+1))*c_time
                    if c_time > 86400: #more than a day
                        c_time_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(c_time)/86400, (int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
                    elif c_time > 3600: #more than an hr
                        c_time_str = '%d hours, %d minutes, and %.2f seconds.'%((int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
                    elif c_time > 60: #more than a min
                        c_time_str = '%d minutes and %.2f seconds.'%((int(c_time)/60)%60, c_time%60)
                    else:
                        c_time_str = '%.2f seconds.'%(c_time)

                    if eta > 86400: #more than a day
                        eta_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(eta)/86400, (int(eta)/3600)%24, (int(eta)/60)%60, eta%60)
                    elif eta > 3600: #more than an hr
                        eta_str = '%d hours, %d minutes, and %.2f seconds.'%((int(eta)/3600)%24, (int(eta)/60)%60, eta%60)
                    elif eta > 60: #more than a min
                        eta_str = '%d minutes and %.2f seconds.'%((int(eta)/60)%60, eta%60)
                    else:
                        eta_str = '%.2f seconds.'%(eta)

                    print 'Pgen computed for %d sequences in: %s Estimated time remaining: %s'%(i+1, c_time_str, eta_str)

            c_time = time.time() - start_time
            if c_time > 86400: #more than a day
                c_time_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(c_time)/86400, (int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
            elif c_time > 3600: #more than an hr
                c_time_str = '%d hours, %d minutes, and %.2f seconds.'%((int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
            elif c_time > 60: #more than a min
                c_time_str = '%d minutes and %.2f seconds.'%((int(c_time)/60)%60, c_time%60)
            else:
                c_time_str = '%.2f seconds.'%(c_time)
            print 'Completed Pgen computation for %d sequences: in %s'%(len(seqs), c_time_str)

            outfile.close()

        else: #NO OUTFILE -- print directly to stdout
            start_time = time.time()
            for i, seq in enumerate(seqs):
                if seq_types[i] == 'aaseq':
                    #Compute Pgen and print out
                    c_pgen_line = seq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                if seq_types[i] == 'regex':
                    #Compute Pgen and print out
                    c_pgen_line = seq + delimiter_out + str(pgen_model.compute_regex_CDR3_template_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                elif seq_types[i] == 'ntseq':
                    ntseq = seq
                    if len(ntseq) % 3 == 0: #inframe sequence
                        aaseq = nt2aa(ntseq)
                        #Compute Pgen and print out based on recomb_type and seq_type_out
                        if seq_type_out is None:
                            c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings)) + delimiter_out + aaseq + delimiter_out +  str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                        elif seq_type_out == 'ntseq':
                            c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                        elif seq_type_out == 'aaseq':
                            c_pgen_line = aaseq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                    else: #out of frame sequence -- Pgens are 0 and use 'out_of_frame' for aaseq
                        if seq_type_out is None:
                            c_pgen_line = ntseq + delimiter_out + '0' + delimiter_out + 'out_of_frame' + delimiter_out + '0'
                        elif seq_type_out == 'ntseq':
                            c_pgen_line = ntseq + delimiter_out + '0'
                        elif seq_type_out == 'aaseq':
                            c_pgen_line = 'out_of_frame' + delimiter_out + '0'

                print c_pgen_line
450676
def parse_args():
  """"""
  parser = argparse.ArgumentParser(
      description='Tool to download dataset images.')
  parser.add_argument('--input_file', required=True,
                      help='Location of dataset.csv')
  parser.add_argument('--output_dir', required=True,
                      help='Output path to download images')
  parser.add_argument('--threads', default=multiprocessing.cpu_count() + 1,
                      help='Number of threads to use')
  args = parser.parse_args()
  return args.input_file, args.output_dir, int(args.threads)
74423
def get_parser():
    """"""
    from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
    parser = ArgumentParser(description=__doc__,
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument("-m", "--model",
                        dest="model",
                        help="where is the model folder (with a info.yml)?",
                        metavar="FOLDER",
                        type=lambda x: utils.is_valid_folder(parser, x),
                        default=utils.default_model())
    return parser
17553
def parse(cmdline = ARGV)
			vals = {}
			required = {}

			opt :version, "Print version and exit" if @version && ! (@specs[:version] || @long["version"])
			opt :help, "Show this message" unless @specs[:help] || @long["help"]

			@specs.each do |sym, opts|
				required[sym] = true if opts.required?
				vals[sym] = opts.default
				vals[sym] = [] if opts.multi && !opts.default # multi arguments default to [], not nil
			end

			resolve_default_short_options!

			## resolve symbols
			given_args = {}
			@leftovers = each_arg cmdline do |arg, params|
				## handle --no- forms
				arg, negative_given = if arg =~ /^--no-([^-]\S*)$/
																["--#{$1}", true]
															else
																[arg, false]
															end

				sym = case arg
							when /^-([^-])$/      then @short[$1]
							when /^--([^-]\S*)$/  then @long[$1] || @long["no-#{$1}"]
							else                       raise CommandlineError, "invalid argument syntax: '#{arg}'"
							end

				sym = nil if arg =~ /--no-/ # explicitly invalidate --no-no- arguments

				next nil if ignore_invalid_options && !sym
				raise CommandlineError, "unknown argument '#{arg}'" unless sym

				if given_args.include?(sym) && !@specs[sym].multi?
					raise CommandlineError, "option '#{arg}' specified multiple times"
				end

				given_args[sym] ||= {}
				given_args[sym][:arg] = arg
				given_args[sym][:negative_given] = negative_given
				given_args[sym][:params] ||= []

				# The block returns the number of parameters taken.
				num_params_taken = 0

				unless params.empty?
					if @specs[sym].single_arg?
						given_args[sym][:params] << params[0, 1]  # take the first parameter
						num_params_taken = 1
					elsif @specs[sym].multi_arg?
						given_args[sym][:params] << params        # take all the parameters
						num_params_taken = params.size
					end
				end

				num_params_taken
			end

			## check for version and help args
			raise VersionNeeded if given_args.include? :version
			raise HelpNeeded if given_args.include? :help

			## check constraint satisfaction
			@constraints.each do |type, syms|
				constraint_sym = syms.find { |sym| given_args[sym] }
				next unless constraint_sym

				case type
				when :depends
					syms.each { |sym| raise CommandlineError, "--#{@specs[constraint_sym].long} requires --#{@specs[sym].long}" unless given_args.include? sym }
				when :conflicts
					syms.each { |sym| raise CommandlineError, "--#{@specs[constraint_sym].long} conflicts with --#{@specs[sym].long}" if given_args.include?(sym) && (sym != constraint_sym) }
				end
			end

			required.each do |sym, val|
				raise CommandlineError, "option --#{@specs[sym].long} must be specified" unless given_args.include? sym
			end

			## parse parameters
			given_args.each do |sym, given_data|
				arg, params, negative_given = given_data.values_at :arg, :params, :negative_given

				opts = @specs[sym]
				if params.empty? && !opts.flag?
					raise CommandlineError, "option '#{arg}' needs a parameter" unless opts.default
					params << (opts.array_default? ? opts.default.clone : [opts.default])
				end

				params[0].each do |p|
					raise CommandlineError, "option '#{arg}' only accepts one of: #{opts.permitted.join(', ')}" unless opts.permitted.include? p
				end unless opts.permitted.nil?

				vals["#{sym}_given".intern] = true # mark argument as specified on the commandline

				vals[sym] = opts.parse(params, negative_given)

				if opts.single_arg?
					if opts.multi?        # multiple options, each with a single parameter
						vals[sym] = vals[sym].map { |p| p[0] }
					else                  # single parameter
						vals[sym] = vals[sym][0][0]
					end
				elsif opts.multi_arg? && !opts.multi?
					vals[sym] = vals[sym][0]  # single option, with multiple parameters
				end
				# else: multiple options, with multiple parameters

				opts.callback.call(vals[sym]) if opts.callback
			end

			## modify input in place with only those
			## arguments we didn't process
			cmdline.clear
			@leftovers.each { |l| cmdline << l }

			## allow openstruct-style accessors
			class << vals
				def method_missing(m, *_args)
					self[m] || self[m.to_s]
				end
			end
			vals
		end
370485
def main(argv=None):
    """"""
    global Verbose
    global EncodeUtf8
    global csvOutput
    if argv is None:
        argv = sys.argv

    if (len(argv) < 3 and not (('-h' in argv) or ('--help' in argv))):
        log.exception('Bad args')
        raise TikaException('Bad args')
    try:
        opts, argv = getopt.getopt(argv[1:], 'hi:s:o:p:v:e:c',
          ['help', 'install=', 'server=', 'output=', 'port=', 'verbose', 'encode', 'csv'])
    except getopt.GetoptError as opt_error:
        msg, bad_opt = opt_error
        log.exception("%s error: Bad option: %s, %s" % (argv[0], bad_opt, msg))
        raise TikaException("%s error: Bad option: %s, %s" % (argv[0], bad_opt, msg))

    tikaServerJar = TikaServerJar
    serverHost = ServerHost
    outDir = '.'
    port = Port
    for opt, val in opts:
        if opt   in ('-h', '--help'):    echo2(USAGE); sys.exit()
        elif opt in ('--install'):       tikaServerJar = val
        elif opt in ('--server'):        serverHost = val
        elif opt in ('-o', '--output'):  outDir = val
        elif opt in ('--port'):          port = val
        elif opt in ('-v', '--verbose'): Verbose = 1
        elif opt in ('-e', '--encode'): EncodeUtf8 = 1
        elif opt in ('-c', '--csv'): csvOutput = 1
        else:
            raise TikaException(USAGE)

    cmd = argv[0]
    option = argv[1]
    try:
        paths = argv[2:]
    except:
        paths = None
    return runCommand(cmd, option, paths, port, outDir, serverHost=serverHost, tikaServerJar=tikaServerJar, verbose=Verbose, encode=EncodeUtf8)
194983
def parser_factory(fake_args=None):
    """"""
    parser = ArgumentParser(description='aomi')
    subparsers = parser.add_subparsers(dest='operation',
                                       help='Specify the data '
                                       ' or extraction operation')
    extract_file_args(subparsers)
    environment_args(subparsers)
    aws_env_args(subparsers)
    seed_args(subparsers)
    render_args(subparsers)
    diff_args(subparsers)
    freeze_args(subparsers)
    thaw_args(subparsers)
    template_args(subparsers)
    password_args(subparsers)
    token_args(subparsers)
    help_args(subparsers)
    export_args(subparsers)

    if fake_args is None:
        return parser, parser.parse_args()

    return parser, parser.parse_args(fake_args)
1172094
def process_args():
    
    parser = argparse.ArgumentParser()

    parser.add_argument('-I', type=str,
                        metavar='<Include directory>',
                        action='append',
                        help='Directory to be searched for included files')
                        
                        
    parser.add_argument('lems_file', type=str, metavar='<LEMS file>', 
                        help='LEMS file to be simulated')
                        
    parser.add_argument('-nogui',
                        action='store_true',
                        help="If this is specified, just parse & simulate the model, but don't show any plots")
                        
    parser.add_argument('-dlems',
                        action='store_true',
                        help="If this is specified, export the LEMS file as "+dlems_info)
    
    return parser.parse_args()
878339
def build_parser():
    """"""

    parser = argparse.ArgumentParser(description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument('firmware_image', nargs="?", help="The firmware image that you wish to load into the emulator")
    parser.add_argument('--gdb', '-g', type=int, help="Start a GDB server on the given port and wait for a connection")
    return parser
747174
def parse(argv, rules=None, config=None, **kwargs):
    """"""
    parser_ = parser(rules, **kwargs)
    if config is not None: parser_.loadrc(config)
    return parser_.parse(argv).result
488966
def get_parser():
    """"""
    parser = ArgumentParser(description='a command-line web scraping tool')
    parser.add_argument('query', metavar='QUERY', type=str, nargs='*',
                        help='URLs/files to scrape')
    parser.add_argument('-a', '--attributes', type=str, nargs='*',
                        help='extract text using tag attributes')
    parser.add_argument('-all', '--crawl-all', help='crawl all pages',
                        action='store_true')
    parser.add_argument('-c', '--crawl', type=str, nargs='*',
                        help='regexp rules for following new pages')
    parser.add_argument('-C', '--clear-cache', help='clear requests cache',
                        action='store_true')
    parser.add_argument('--csv', help='write files as csv',
                        action='store_true')
    parser.add_argument('-cs', '--cache-size', type=int, nargs='?',
                        help='size of page cache (default: 1000)',
                        default=1000)
    parser.add_argument('-f', '--filter', type=str, nargs='*',
                        help='regexp rules for filtering text')
    parser.add_argument('--html', help='write files as HTML',
                        action='store_true')
    parser.add_argument('-i', '--images', action='store_true',
                        help='save page images')
    parser.add_argument('-m', '--multiple', help='save to multiple files',
                        action='store_true')
    parser.add_argument('-max', '--max-crawls', type=int,
                        help='max number of pages to crawl')
    parser.add_argument('-n', '--nonstrict', action='store_true',
                        help='allow crawler to visit any domain')
    parser.add_argument('-ni', '--no-images', action='store_true',
                        help='do not save page images')
    parser.add_argument('-no', '--no-overwrite', action='store_true',
                        help='do not overwrite files if they exist')
    parser.add_argument('-o', '--out', type=str, nargs='*',
                        help='specify outfile names')
    parser.add_argument('-ow', '--overwrite', action='store_true',
                        help='overwrite a file if it exists')
    parser.add_argument('-p', '--pdf', help='write files as pdf',
                        action='store_true')
    parser.add_argument('-pt', '--print', help='print text output',
                        action='store_true')
    parser.add_argument('-q', '--quiet', help='suppress program output',
                        action='store_true')
    parser.add_argument('-s', '--single', help='save to a single file',
                        action='store_true')
    parser.add_argument('-t', '--text', help='write files as text',
                        action='store_true')
    parser.add_argument('-v', '--version', help='display current version',
                        action='store_true')
    parser.add_argument('-x', '--xpath', type=str, nargs='?',
                        help='filter HTML using XPath')
    return parser
787195
def process_args():
    
    parser = argparse.ArgumentParser(
                description=("A script which can be run to tune a NeuroML 2 model against a number of target properties. Work in progress!"))

                        
    parser.add_argument('prefix', 
                        type=str,
                        metavar='<prefix>', 
                        help="Prefix for optimisation run")

    parser.add_argument('neuromlFile', 
                        type=str,
                        metavar='<neuromlFile>', 
                        help="NeuroML2 file containing model")

    parser.add_argument('target', 
                        type=str,
                        metavar='<target>', 
                        help="Target in NeuroML2 model")
                        
    parser.add_argument('parameters', 
                        type=str,
                        metavar='<parameters>', 
                        help="List of parameter to adjust")
                        
    parser.add_argument('maxConstraints', 
                        type=str,
                        metavar='<max_constraints>', 
                        help="Max values for parameters")
                        
    parser.add_argument('minConstraints', 
                        type=str,
                        metavar='<min_constraints>', 
                        help="Min values for parameters")
                        
    parser.add_argument('targetData', 
                        type=str,
                        metavar='<targetData>', 
                        help="List of name/value pairs for properties extracted from data to judge fitness against")
                        
    parser.add_argument('weights', 
                        type=str,
                        metavar='<weights>', 
                        help="Weights to assign to each target name/value pair")
                        
    parser.add_argument('-simTime', 
                        type=float,
                        metavar='<simTime>', 
                        default=DEFAULTS['simTime'],
                        help="Simulation duration")
                        
    parser.add_argument('-dt', 
                        type=float,
                        metavar='<dt>', 
                        default=DEFAULTS['dt'],
                        help="Simulation timestep")
                        
    parser.add_argument('-analysisStartTime', 
                        type=float,
                        metavar='<analysisStartTime>', 
                        default=DEFAULTS['analysisStartTime'],
                        help="Analysis start time")
                        
    parser.add_argument('-populationSize', 
                        type=int,
                        metavar='<populationSize>', 
                        default=DEFAULTS['populationSize'],
                        help="Population size")
                        
    parser.add_argument('-maxEvaluations', 
                        type=int,
                        metavar='<maxEvaluations>', 
                        default=DEFAULTS['maxEvaluations'],
                        help="Maximum evaluations")
                        
    parser.add_argument('-numSelected', 
                        type=int,
                        metavar='<numSelected>', 
                        default=DEFAULTS['numSelected'],
                        help="Number selected")
                        
    parser.add_argument('-numOffspring', 
                        type=int,
                        metavar='<numOffspring>', 
                        default=DEFAULTS['numOffspring'],
                        help="Number offspring")
                        
            
    parser.add_argument('-mutationRate', 
                        type=float,
                        metavar='<mutationRate>', 
                        default=DEFAULTS['mutationRate'],
                        help="Mutation rate")
                        
    parser.add_argument('-numElites', 
                        type=int,
                        metavar='<numElites>', 
                        default=DEFAULTS['numElites'],
                        help="Number of elites")
                        
    parser.add_argument('-numParallelEvaluations', 
                        type=int,
                        metavar='<numParallelEvaluations>', 
                        default=DEFAULTS['numParallelEvaluations'],
                        help="Number of evaluations to run in parallel")
                        
    parser.add_argument('-seed', 
                        type=int,
                        metavar='<seed>', 
                        default=DEFAULTS['seed'],
                        help="Seed for optimiser")
                        
    parser.add_argument('-simulator', 
                        type=str,
                        metavar='<simulator>', 
                        default=DEFAULTS['simulator'],
                        help="Simulator to run")
                        
    parser.add_argument('-knownTargetValues', 
                        type=str,
                        metavar='<knownTargetValues>', 
                        help="List of name/value pairs which represent the known values of the target parameters")
                        
    parser.add_argument('-nogui', 
                        action='store_true',
                        default=DEFAULTS['nogui'],
                        help="Should GUI elements be supressed?")
                        
    parser.add_argument('-showPlotAlready', 
                        action='store_true',
                        default=DEFAULTS['showPlotAlready'],
                        help="Should generated plots be suppressed until show() called?")
                        
    parser.add_argument('-verbose', 
                        action='store_true',
                        default=DEFAULTS['verbose'],
                        help="Verbose mode")
                        
    parser.add_argument('-dryRun', 
                        action='store_true',
                        default=DEFAULTS['dryRun'],
                        help="Dry run; just print setup information")
                        
    parser.add_argument('-extraReportInfo', 
                        type=str,
                        metavar='<extraReportInfo>', 
                        default=DEFAULTS['extraReportInfo'],
                        help='Extra tag/value pairs can be put into the report.json:  -extraReportInfo=["tag":"value"]')
                        
    parser.add_argument('-cleanup', 
                        action='store_true',
                        default=DEFAULTS['cleanup'],
                        help="Should (some) generated files, e.g. *.dat, be deleted as optimisation progresses?")
                        
    return parser.parse_args()
91403
def parse_args():
  """"""
  parser = ArgumentParser(description="ModelBase builder")
  subparsers = parser.add_subparsers()

  sql_parser = subparsers.add_parser(
    "get-query",
    description="Usage: e.g. psql -c \"copy ($(python3 lib/generate_models.py get-query)) to " +
                "stdout with csv header\" DB_NAME postgres")
  sql_parser.set_defaults(func=print_sql_query)

  gen_parser = subparsers.add_parser("generate")
  gen_parser.add_argument("filename", nargs="?", help="Read this file for input, or STDIN if not " \
                                                      "given")
  gen_parser.add_argument("-i", "--indent", default="  ")
  gen_parser.add_argument("-c", "--created-at-col-name", default="created_at")
  gen_parser.add_argument("-u", "--updated-at-col-name", default="updated_at")
  gen_parser.set_defaults(func=generate_models)

  args = parser.parse_args()

  if hasattr(args, "func"):
    return args
  else:
    arg_parser.print_help()
    sys.exit(1)
1009063
def train_model_from_args(args: argparse.Namespace):
    
    train_model_from_file(args.param_path,
                          args.serialization_dir,
                          args.overrides,
                          args.file_friendly_logging,
                          args.recover,
                          args.force,
                          args.cache_directory,
                          args.cache_prefix)
396720
def parse_args():
    """"""

    parser = get_parser()

    if len(sys.argv) < 2:
        parser.print_help()
        warnings.warn('Too few arguments!', UserWarning)
        parser.exit(1)

    # parsing
    try:
        params = parser.parse_args()
    except Exception as exc:
        print(exc)
        raise ValueError('Unable to parse command-line arguments.')

    in_features_path = os.path.abspath(params.in_features_path)
    if not os.path.exists(in_features_path):
        raise IOError("Given features file doesn't exist.")

    groups_path = os.path.abspath(params.groups_path)
    if not os.path.exists(groups_path):
        raise IOError("Given groups file doesn't exist.")

    return in_features_path, groups_path, params.weight_method, params.num_bins, params.edge_range, \
           params.trim_outliers, params.trim_percentile, params.return_networkx_graph, params.out_weights_path
196592
def main(model=None, output_dir=None, n_iter=15):
    """"""
    if model is not None:
        nlp = spacy.load(model)  # load existing spaCy model
        print("Loaded model '%s'" % model)
    else:
        nlp = spacy.blank("en")  # create blank Language class
        print("Created blank 'en' model")

    # We'll use the built-in dependency parser class, but we want to create a
    # fresh instance – just in case.
    if "parser" in nlp.pipe_names:
        nlp.remove_pipe("parser")
    parser = nlp.create_pipe("parser")
    nlp.add_pipe(parser, first=True)

    for text, annotations in TRAIN_DATA:
        for dep in annotations.get("deps", []):
            parser.add_label(dep)

    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != "parser"]
    with nlp.disable_pipes(*other_pipes):  # only train parser
        optimizer = nlp.begin_training()
        for itn in range(n_iter):
            random.shuffle(TRAIN_DATA)
            losses = {}
            # batch up the examples using spaCy's minibatch
            batches = minibatch(TRAIN_DATA, size=compounding(4.0, 32.0, 1.001))
            for batch in batches:
                texts, annotations = zip(*batch)
                nlp.update(texts, annotations, sgd=optimizer, losses=losses)
            print("Losses", losses)

    # test the trained model
    test_model(nlp)

    # save model to output directory
    if output_dir is not None:
        output_dir = Path(output_dir)
        if not output_dir.exists():
            output_dir.mkdir()
        nlp.to_disk(output_dir)
        print("Saved model to", output_dir)

        # test the saved model
        print("Loading from", output_dir)
        nlp2 = spacy.load(output_dir)
        test_model(nlp2)