Main entry point for UNECE code .xls parsing.
481749
def main():
  """"""
  parser = argparse.ArgumentParser(
      description='Reads in a .xls file and generates a units module for '
                  'OpenHTF.',
      prog='python units_from_xls.py')
  parser.add_argument('xlsfile', type=str,
                      help='the .xls file to parse')
  parser.add_argument(
      '--outfile',
      type=str,
      default=os.path.join(os.path.dirname(__file__), os.path.pardir,
                           'openhtf','util', 'units.py'),
      help='where to put the generated .py file.')
  args = parser.parse_args()

  if not os.path.exists(args.xlsfile):
    print('Unable to locate the file "%s".' % args.xlsfile)
    parser.print_help()
    sys.exit()

  unit_defs = unit_defs_from_sheet(
      xlrd.open_workbook(args.xlsfile).sheet_by_name(SHEET_NAME),
      COLUMN_NAMES)

  _, tmp_path = mkstemp()
  with open(tmp_path, 'w') as new_file:
    new_file.write(PRE)
    new_file.writelines(
        [line.encode('utf8', 'replace') for line in unit_defs])
    new_file.write(POST)
    new_file.flush()

  os.remove(args.outfile)
  shutil.move(tmp_path, args.outfile)
1065284
def excel_main(file):
    

    os.chdir(file["dir"])
    name_ext = file["filename_ext"]

    # Filename without extension
    name = file["filename_no_ext"]
    # remove foreign characters to prevent wiki uploading errors
    name = normalize_name(name)
    print("processing: {}".format(name_ext))
    logger_excel.info("processing: {}".format(name_ext))

    pending_csv = []
    final = OrderedDict()
    logger_excel.info("variables initialized")

    # Create a temporary folder and set paths
    dir_tmp = create_tmp_dir()

    """
    EACH DATA TABLE WILL BE STRUCTURED LIKE THIS
    "paleo_chron": "paleo",
    "pc_idx": 1,
    "model_idx": "",
    "table_type": "measurement",
    "table_idx": 1,
    "name": sheet,
    "filename": sheet,
    "data": { column data }
    """

    # Open excel workbook with filename
    try:
        workbook = xlrd.open_workbook(name_ext)
        logger_excel.info("opened XLRD workbook")

    except Exception as e:
        # There was a problem opening a file with XLRD
        print("Failed to open Excel workbook: {}".format(name))
        workbook = None
        logger_excel.debug("excel: xlrd failed to open workbook: {}, {}".format(name, e))

    if workbook:

        # Build sheets, but don't make full filenames yet. Need to parse metadata sheet first to get datasetname.
        sheets, ct_paleo, ct_chron, metadata_str = _get_sheet_metadata(workbook, name)

        # METADATA WORKSHEETS
        # Parse Metadata sheet and add to output dictionary
        if metadata_str:
            logger_excel.info("parsing worksheet: {}".format(metadata_str))
            final = cells_dn_meta(workbook, metadata_str, 0, 0, final)

        # Now that we have the dataSetName, we can use it to build filenames
        dsn = __get_datasetname(final, name)
        filename = str(dsn) + ".lpd"
        sheets = __set_sheet_filenames(sheets, dsn)

        dir_bag = os.path.join(dir_tmp, "bag")
        dir_data = os.path.join(dir_bag, 'data')

        # Make folders in tmp
        os.mkdir(os.path.join(dir_bag))
        os.mkdir(os.path.join(dir_data))

        # PALEO AND CHRON SHEETS
        for sheet in sheets:
            logger_excel.info("parsing data worksheet: {}".format(sheet["new_name"]))
            sheet_meta, sheet_csv = _parse_sheet(workbook, sheet)
            if sheet_csv and sheet_meta:
                pending_csv.append(sheet_csv)
                sheet["data"] = sheet_meta

        # create the metadata skeleton where we will place the tables. dynamically add empty table blocks for data.
        skeleton_paleo, skeleton_chron = _create_skeleton_1(sheets)

        # Reorganize sheet metadata into LiPD structure
        d_paleo, d_chron = _place_tables_main(sheets, skeleton_paleo, skeleton_chron)

        # Add organized metadata into final dictionary
        final['paleoData'] = d_paleo
        final['chronData'] = d_chron

        # OUTPUT

        # Create new files and dump data in dir_data
        os.chdir(dir_data)

        # WRITE CSV
        _write_data_csv(pending_csv)

        # JSON-LD
        # Invoke DOI Resolver Class to update publisher data
        try:
            logger_excel.info("invoking doi resolver")
            final = DOIResolver(file["dir"], name, final).main()
        except Exception as e:
            print("Error: doi resolver failed: {}".format(name))
            logger_excel.debug("excel: doi resolver failed: {}, {}".format(name, e))

        # Dump final_dict to a json file.
        final["lipdVersion"] = 1.2
        final["createdBy"] = "excel"
        write_json_to_file(final)

        # Move files to bag root for re-bagging
        # dir : dir_data -> dir_bag
        logger_excel.info("start cleanup")
        dir_cleanup(dir_bag, dir_data)

        # Create a bag for the 3 files
        finish_bag(dir_bag)

        # dir: dir_tmp -> dir_root
        os.chdir(file["dir"])

        # Check if same lpd file exists. If so, delete so new one can be made
        if os.path.isfile(filename):
            os.remove(filename)

        # Zip dir_bag. Creates in dir_root directory
        logger_excel.info("re-zip and rename")
        zipper(root_dir=dir_tmp, name="bag", path_name_ext=os.path.join(file["dir"], filename))

    # Move back to dir_root for next loop.
    os.chdir(file["dir"])

    # Cleanup and remove tmp directory
    shutil.rmtree(dir_tmp)

    return dsn
726332
def parse(code="", mode="sys"):
    """"""
    if CLI.comp is None:
        setup()
    if mode in PARSERS:
        return PARSERS[mode](CLI.comp)(code)
    else:
        raise CoconutException(
            "invalid parse mode " + ascii(mode),
            extra="valid modes are " + ", ".join(PARSERS),
        )
334691
def main():
    """"""

    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('source', help="location of the downloaded repos")
    parser.add_argument('destination', help="location of the extracted files")
    parser.add_argument(
        '-t', '--nb-test-files', help="number of testing files per language",
        type=int, default=5000)
    parser.add_argument(
        '-l', '--nb-learn-files', help="number of learning files per language",
        type=int, default=10000)
    parser.add_argument(
        '-r', '--remove', help="remove repos that cannot be read",
        action='store_true', default=False)
    parser.add_argument(
        '-d', '--debug', default=False, action='store_true',
        help="show debug messages")

    args = parser.parse_args()
    config_logging(args.debug)

    source = Path(args.source)
    destination = Path(args.destination)
    nb_test = args.nb_test_files
    nb_learn = args.nb_learn_files
    remove = args.remove

    repos = _find_repos(source)
    split_repos = _split_repos(repos, nb_test, nb_learn)
    split_files = _find_files(*split_repos, nb_test, nb_learn, remove)
    _unzip_all(*split_files, destination)
    LOGGER.info("Files saved into %s", destination)
    LOGGER.debug("Exit OK")
725431
def main():
    """"""
    args = parse_arguments()

    ext = os.path.splitext(args.input_file)[-1].lower()
    with gzip.open(args.output_file, mode='wt') as outfile:
        csvwriter = csv.writer(outfile, delimiter=str('\t'), lineterminator='\n')

        try:
            if ext in ('.tab', '.txt', '.tsv'):
                with open(args.input_file) as infile:
                    for line in infile:
                        outline = line.strip().split('\t')
                        csvwriter.writerow(outline)
            elif ext == '.csv':
                with open(args.input_file) as infile:
                    for line in infile:
                        outline = line.strip().split(',')
                        csvwriter.writerow(outline)
            elif ext in ('.xls', '.xlsx'):
                workbook = xlrd.open_workbook(args.input_file)
                worksheet = workbook.sheets()[0]
                for rownum in range(worksheet.nrows):
                    csvwriter.writerow(worksheet.row_values(rownum))
            else:
                print('{"proc.error":"File extension not recognized."}')
        except Exception:
            print('{"proc.error":"Corrupt or unrecognized file."}')
            raise
884457
def parse(options={})
      @sheets.each do |x|
        if !options[:paginate].nil?
          lines = options[:paginate][0]; page = options[:paginate][1]
          sheet = PagSheet.new(lines, page)
        elsif !options[:cellrange].nil?
          range = options[:cellrange]
          sheet = Cellrange.new(range)
        else
          sheet = Xlsheet.new()
        end

        File.open(@destination+"/xl/#{x[:filename]}", 'r') do |f|
          Ox.sax_parse(sheet, f)
        end
        comments = mkcomments(x[:comments])
        sheet.cellarray.each do |sh|
          sh.numformat = @styles.styleary[sh.style.to_i]
          if sh.type=="s"
            sh.value = @sharedstrings[sh.value.to_i]
          end
          if !comments.nil?
            comm=comments.select {|c| c[:ref]==(sh.xlcoords)}
            if comm.size > 0
              sh.comment=comm[0][:comment]
            end
            comments.delete_if{|c| c[:ref]==(sh.xlcoords)}
          end
        end
        x[:cells] = sheet.cellarray
        x[:mergedcells] = sheet.mergedcells
      end
      matrixto options
    end
507663
def main():
  
  args = docopt(__doc__)
  if args['--version']:
    print(__version__)
    sys.exit()

  result = parse(
      args['<source>'],
      segmenter=args['--segmenter'],
      language=args['--language'],
      classname=args['--classname'])
  print(result['html_code'])
  sys.exit()
76050
def main():
    """"""
    import argparse

    # Parse input parameters
    parser = argparse.ArgumentParser(
        prog=__taskname__,
        description=(
            'Run CALACS and standalone acs_destripe script on given post-SM4 '
            'ACS/WFC RAW full-frame or supported subarray image.'))
    parser.add_argument(
        'arg0', metavar='input', type=str, help='Input file')
    parser.add_argument(
        '--suffix', type=str, default='strp',
        help='Output suffix for acs_destripe')
    parser.add_argument(
        '--stat', type=str, default='pmode1', help='Background statistics')
    parser.add_argument(
        '--maxiter', type=int, default=15, help='Max #iterations')
    parser.add_argument(
        '--sigrej', type=float, default=2.0, help='Rejection sigma')
    parser.add_argument(
        '--lower', nargs='?', type=float, default=None,
        help='Lower limit for "good" pixels.')
    parser.add_argument(
        '--upper', nargs='?', type=float, default=None,
        help='Upper limit for "good" pixels.')
    parser.add_argument(
        '--binwidth', type=float, default=0.1,
        help='Bin width for distribution sampling.')
    parser.add_argument(
        '--sci1_mask', nargs=1, type=str, default=None,
        help='Mask image for calibrated [SCI,1]')
    parser.add_argument(
        '--sci2_mask', nargs=1, type=str, default=None,
        help='Mask image for calibrated [SCI,2]')
    parser.add_argument(
        '--dqbits', nargs='?', type=str, default=None,
        help='DQ bits to be considered "good".')
    parser.add_argument(
        '--rpt_clean', type=int, default=0,
        help='Number of *repeated* bias de-stripes to perform.')
    parser.add_argument(
        '--atol', nargs='?', type=float, default=0.01,
        help='Absolute tolerance to stop *repeated* bias de-stripes.')
    parser.add_argument(
        '--nocte', action='store_true', help='Turn off CTE correction.')
    parser.add_argument(
        '--clobber', action='store_true', help='Clobber output')
    parser.add_argument(
        '-q', '--quiet', action="store_true",
        help='Do not print informational messages')
    parser.add_argument(
        '--version', action='version',
        version='{0} v{1} ({2})'.format(__taskname__, __version__, __vdate__))
    options = parser.parse_args()

    if options.sci1_mask:
        mask1 = options.sci1_mask[0]
    else:
        mask1 = options.sci1_mask

    if options.sci2_mask:
        mask2 = options.sci2_mask[0]
    else:
        mask2 = options.sci2_mask

    destripe_plus(options.arg0, suffix=options.suffix,
                  maxiter=options.maxiter, sigrej=options.sigrej,
                  lower=options.lower, upper=options.upper,
                  binwidth=options.binwidth,
                  scimask1=mask1, scimask2=mask2, dqbits=options.dqbits,
                  rpt_clean=options.rpt_clean, atol=options.atol,
                  cte_correct=not options.nocte, clobber=options.clobber,
                  verbose=not options.quiet)
10987
def main():
    
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', dest='in_filename', required=True,
                        help='Zeppelin notebook input file (.json)')
    parser.add_argument('-o', dest='out_filename',
                        help='Markdown output file (.md) (optional)')
    args = parser.parse_args()
    directory = ''

    if args.out_filename:
        directory = os.path.dirname(args.out_filename)
        args.out_filename = os.path.basename(args.out_filename)
        args.out_filename = os.path.splitext(args.out_filename)[0]
        args.out_filename = args.out_filename if args.out_filename else 'knowledge'
    else:
        args.out_filename = 'knowledge'

    with open(args.in_filename, 'rb') as raw:
        try:
            t = json.load(raw)
            full_path = os.path.join(directory, args.out_filename + '.md')
        except ValueError:
            print('ERROR: Invalid JSON format')
            sys.exit(1)

    version = get_version(t)
    if version == '0.7.1':
        zeppelin_converter = NewConverter(args.in_filename, args.out_filename,
                                          directory)
    elif version == '0.6.2':
        zeppelin_converter = LegacyConverter(args.in_filename, args.out_filename,
                                             directory)

    with open(full_path, 'w') as fout:
        zeppelin_converter.convert(t, fout)
808925
def main(argv=None):
    """"""
    parser = argparse.ArgumentParser(description="Process raw HTML to baked"
                                                 " (embedded numbering and"
                                                 " collation)")
    parser.add_argument('-v', '--version', action="version",
                        version=__version__, help='Report the library version')
    parser.add_argument("css_rules",
                        type=argparse.FileType('rb'),
                        help="CSS3 ruleset stylesheet recipe")
    parser.add_argument("html_in", nargs="?",
                        type=argparse.FileType('r'),
                        help="raw HTML file to bake (default stdin)",
                        default=sys.stdin)
    parser.add_argument("html_out", nargs="?",
                        type=argparse.FileType('w'),
                        help="baked HTML file output (default stdout)",
                        default=sys.stdout)
    parser.add_argument('-s', '--stop-at', action='store', metavar='<pass>',
                        help='Stop baking just before given pass name')
    parser.add_argument('-d', '--debug', action='store_true',
                        help='Send debugging info to stderr')
    parser.add_argument('-q', '--quiet', action='store_true',
                        help="Quiet all on stderr except errors")
    parser.add_argument('-c', '--coverage-file', metavar='coverage.lcov',
                        type=FileTypeExt('w'),
                        help="output coverage file (lcov format). If "
                        "filename starts with '+', append coverage info.")
    parser.add_argument('--use-repeatable-ids', action='store_true',
                        help="use repeatable id attributes instead of uuids "
                        "which is useful for diffing")
    args = parser.parse_args(argv)

    formatter = logging.Formatter('%(name)s %(levelname)s %(message)s')
    handler = logging.StreamHandler(sys.stderr)
    handler.setFormatter(formatter)
    logger.addHandler(handler)

    use_quiet_log = (args.quiet and logging.ERROR)
    use_debug_log = (args.debug and logging.DEBUG)

    # Debug option takes higher priority than quiet warnings option
    logger.setLevel(use_debug_log or use_quiet_log or logging.WARNING)

    try:
        easybake(args.css_rules, args.html_in, args.html_out, args.stop_at,
                 args.coverage_file, args.use_repeatable_ids)
    finally:
        if args.css_rules:
            args.css_rules.close()
        if args.html_in:
            args.html_in.close()
        if args.html_out:
            args.html_out.close()
        if args.coverage_file:
            args.coverage_file.close()
945724
def main():
    """"""

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        'reportfile', type=argparse.FileType('r'),
        help="test report file generated by `guesslang --test TESTDIR`")
    parser.add_argument(
        '-d', '--debug', default=False, action='store_true',
        help="show debug messages")

    args = parser.parse_args()
    config_logging(args.debug)

    report = json.load(args.reportfile)
    graph_data = _build_graph(report)
    index_path = _prepare_resources(graph_data)
    webbrowser.open(str(index_path))
1323342
def main():
    """"""

    # Quit when interrupted
    import signal
    signal.signal(signal.SIGINT, signal.SIG_DFL)

    # Arguments:
    # --separator STRING/REGEX - how to split a row into cells (only relevant for CSV parser)
    # --flatten - flatten item hashes. {'a':{'b':'c'}} --> {'a_b':'c'}
    import argparse
    parser = argparse.ArgumentParser(description='View tabulated data via GUI')
    parser.add_argument('-p','--parser',type=str, default='autosplit',help='Type of parser to use') #TODO add possible parsers
    parser.add_argument('--headers',type=str, help='Headers are this comma-delimited names instead of ones supplied in file. Use colons to specify types, like "colname:int"')
    parser.add_argument('--filter',type=str, help='Pre-populate filter box')
    parser.add_argument('-s', '--separator', help='How to seperate columns. Applies only to some parsers')
    parser.add_argument('files', nargs='*', help='Files to show. Each file opens a new window')
    args = parser.parse_args()

    GObject.threads_init()

    default_config = {'format': args.parser}
    if args.filter:
        default_config['filter']=args.filter
    if args.separator:
        default_config['separator']=args.separator
    if args.headers:
        default_config['headers']=map(lambda s: s.strip(),args.headers.split(','))
    inputs = [ (open(f,'r'),default_config) for f in args.files ]
    # Add stdin as input, if it's not a tty
    if not sys.stdin.isatty():
        inputs.append((sys.stdin, default_config))

    global windows
    windows = [Window(i[0],i[1]) for i in inputs]
    for win in windows:
      win.start_read()
      win.connect("destroy",window_closed)
    if windows:
        Gtk.main()
    else:
        print 'No input supplied so no windows are created'
307907
def main(left_path, left_column, right_path, right_column,
         outfile, titles, join, minscore, count, warp):
    """"""
    right_file = csv.reader(open(right_path, 'r'))
    if titles:
        right_header = next(right_file)
    index = NGram((tuple(r) for r in right_file),
                  threshold=minscore,
                  warp=warp, key=lambda x: lowstrip(x[right_column]))
    left_file = csv.reader(open(left_path, 'r'))
    out = csv.writer(open(outfile, 'w'), lineterminator='\n')
    if titles:
        left_header = next(left_file)
        out.writerow(left_header + ["Rank", "Similarity"] + right_header)
    for row in left_file:
        if not row: continue # skip blank lines
        row = tuple(row)
        results = index.search(lowstrip(row[left_column]), threshold=minscore)
        if results:
            if count > 0:
                results = results[:count]
            for rank, result in enumerate(results, 1):
                out.writerow(row + (rank, result[1]) + result[0])
        elif join == "outer":
            out.writerow(row)
394684
def _parse(self) -> None:
        """"""
        self._in_quote = False
        self._in_top_comment = False
        while not self._at_end():
            raw_line = line = self._get_line()
            line = line.replace("from.import ", "from . import ")
            line = line.replace("\t", " ").replace('import*', 'import *')
            line = line.replace(" .import ", " . import ")
            statement_index = self.index
            skip_line = self._skip_line(line)

            if line in self._section_comments and not skip_line:
                if self.import_index == -1:
                    self.import_index = self.index - 1
                continue

            if "isort:imports-" in line and line.startswith("#"):
                section = line.split("isort:imports-")[-1].split()[0].upper()
                self.place_imports[section] = []
                self.import_placements[line] = section

            if ";" in line:
                for part in (part.strip() for part in line.split(";")):
                    if part and not part.startswith("from ") and not part.startswith("import "):
                        skip_line = True

            import_type = self._import_type(line)
            if not import_type or skip_line:
                self.out_lines.append(raw_line)
                continue

            for line in (line.strip() for line in line.split(";")):
                import_type = self._import_type(line)
                if not import_type:
                    self.out_lines.append(line)
                    continue

                if self.import_index == -1:
                    self.import_index = self.index - 1
                nested_comments = {}
                import_string, comments, new_comments = self._strip_comments(line)
                line_parts = [part for part in self._strip_syntax(import_string).strip().split(" ") if part]
                if import_type == "from" and len(line_parts) == 2 and line_parts[1] != "*" and new_comments:
                    nested_comments[line_parts[-1]] = comments[0]

                if "(" in line.split("#")[0] and not self._at_end():
                    while not line.strip().endswith(")") and not self._at_end():
                        line, comments, new_comments = self._strip_comments(self._get_line(), comments)
                        stripped_line = self._strip_syntax(line).strip()
                        if import_type == "from" and stripped_line and " " not in stripped_line and new_comments:
                            nested_comments[stripped_line] = comments[-1]
                        import_string += self.line_separator + line
                else:
                    while line.strip().endswith("\\"):
                        line, comments, new_comments = self._strip_comments(self._get_line(), comments)

                        # Still need to check for parentheses after an escaped line
                        if "(" in line.split("#")[0] and ")" not in line.split("#")[0] and not self._at_end():
                            stripped_line = self._strip_syntax(line).strip()
                            if import_type == "from" and stripped_line and " " not in stripped_line and new_comments:
                                nested_comments[stripped_line] = comments[-1]
                            import_string += self.line_separator + line

                            while not line.strip().endswith(")") and not self._at_end():
                                line, comments, new_comments = self._strip_comments(self._get_line(), comments)
                                stripped_line = self._strip_syntax(line).strip()
                                if import_type == "from" and stripped_line and " " not in stripped_line and new_comments:
                                    nested_comments[stripped_line] = comments[-1]
                                import_string += self.line_separator + line

                        stripped_line = self._strip_syntax(line).strip()
                        if import_type == "from" and stripped_line and " " not in stripped_line and new_comments:
                            nested_comments[stripped_line] = comments[-1]
                        if import_string.strip().endswith(" import") or line.strip().startswith("import "):
                            import_string += self.line_separator + line
                        else:
                            import_string = import_string.rstrip().rstrip("\\") + " " + line.lstrip()

                if import_type == "from":
                    import_string = import_string.replace("import(", "import (")
                    parts = import_string.split(" import ")
                    from_import = parts[0].split(" ")
                    import_string = " import ".join([from_import[0] + " " + "".join(from_import[1:])] + parts[1:])

                imports = [item.replace("{|", "{ ").replace("|}", " }") for item in
                           self._strip_syntax(import_string).split()]
                if "as" in imports and (imports.index('as') + 1) < len(imports):
                    while "as" in imports:
                        index = imports.index('as')
                        if import_type == "from":
                            module = imports[0] + "." + imports[index - 1]
                            self.as_map[module] = imports[index + 1]
                        else:
                            module = imports[index - 1]
                            self.as_map[module] = imports[index + 1]
                        if not self.config['combine_as_imports']:
                            self.comments['straight'][module] = comments
                            comments = []
                        del imports[index:index + 2]
                if import_type == "from":
                    import_from = imports.pop(0)
                    placed_module = self.place_module(import_from)
                    if self.config['verbose']:
                        print("from-type place_module for %s returned %s" % (import_from, placed_module))
                    if placed_module == '':
                        print(
                            "WARNING: could not place module {0} of line {1} --"
                            " Do you need to define a default section?".format(import_from, line)
                        )
                    root = self.imports[placed_module][import_type]
                    for import_name in imports:
                        associated_comment = nested_comments.get(import_name)
                        if associated_comment:
                            self.comments['nested'].setdefault(import_from, {})[import_name] = associated_comment
                            comments.pop(comments.index(associated_comment))
                    if comments:
                        self.comments['from'].setdefault(import_from, []).extend(comments)

                    if len(self.out_lines) > max(self.import_index, self._first_comment_index_end + 1, 1) - 1:
                        last = self.out_lines and self.out_lines[-1].rstrip() or ""
                        while (last.startswith("#") and not last.endswith('"""') and not last.endswith("'''") and
                               'isort:imports-' not in last):
                            self.comments['above']['from'].setdefault(import_from, []).insert(0, self.out_lines.pop(-1))
                            if len(self.out_lines) > max(self.import_index - 1, self._first_comment_index_end + 1, 1) - 1:
                                last = self.out_lines[-1].rstrip()
                            else:
                                last = ""
                        if statement_index - 1 == self.import_index:
                            self.import_index -= len(self.comments['above']['from'].get(import_from, []))

                    if import_from not in root:
                        root[import_from] = OrderedDict()
                    root[import_from].update((module, None) for module in imports)
                else:
                    for module in imports:
                        if comments:
                            self.comments['straight'][module] = comments
                            comments = None

                        if len(self.out_lines) > max(self.import_index, self._first_comment_index_end + 1, 1) - 1:

                            last = self.out_lines and self.out_lines[-1].rstrip() or ""
                            while (last.startswith("#") and not last.endswith('"""') and not last.endswith("'''") and
                                   'isort:imports-' not in last):
                                self.comments['above']['straight'].setdefault(module, []).insert(0,
                                                                                                 self.out_lines.pop(-1))
                                if len(self.out_lines) > 0 and len(self.out_lines) != self._first_comment_index_end:
                                    last = self.out_lines[-1].rstrip()
                                else:
                                    last = ""
                            if self.index - 1 == self.import_index:
                                self.import_index -= len(self.comments['above']['straight'].get(module, []))
                        placed_module = self.place_module(module)
                        if self.config['verbose']:
                            print("else-type place_module for %s returned %s" % (module, placed_module))
                        if placed_module == '':
                            print(
                                "WARNING: could not place module {0} of line {1} --"
                                " Do you need to define a default section?".format(import_from, line)
                            )
                        self.imports[placed_module][import_type][module] = None
660103
def main():
    
    import argparse

    parser = argparse.ArgumentParser(
        description="ftfy (fixes text for you), version %s" % __version__
    )
    parser.add_argument('filename', default='-', nargs='?',
                        help='The file whose Unicode is to be fixed. Defaults '
                             'to -, meaning standard input.')
    parser.add_argument('-o', '--output', type=str, default='-',
                        help='The file to output to. Defaults to -, meaning '
                             'standard output.')
    parser.add_argument('-g', '--guess', action='store_true',
                        help="Ask ftfy to guess the encoding of your input. "
                             "This is risky. Overrides -e.")
    parser.add_argument('-e', '--encoding', type=str, default='utf-8',
                        help='The encoding of the input. Defaults to UTF-8.')
    parser.add_argument('-n', '--normalization', type=str, default='NFC',
                        help='The normalization of Unicode to apply. '
                             'Defaults to NFC. Can be "none".')
    parser.add_argument('--preserve-entities', action='store_true',
                        help="Leave HTML entities as they are. The default "
                             "is to decode them, as long as no HTML tags "
                             "have appeared in the file.")

    args = parser.parse_args()

    encoding = args.encoding
    if args.guess:
        encoding = None

    if args.filename == '-':
        # Get a standard input stream made of bytes, so we can decode it as
        # whatever encoding is necessary.
        file = sys.stdin.buffer
    else:
        file = open(args.filename, 'rb')

    if args.output == '-':
        outfile = sys.stdout
    else:
        if os.path.realpath(args.output) == os.path.realpath(args.filename):
            sys.stderr.write(SAME_FILE_ERROR_TEXT)
            sys.exit(1)
        outfile = open(args.output, 'w', encoding='utf-8')

    normalization = args.normalization
    if normalization.lower() == 'none':
        normalization = None

    if args.preserve_entities:
        fix_entities = False
    else:
        fix_entities = 'auto'

    try:
        for line in fix_file(file, encoding=encoding,
                             fix_entities=fix_entities,
                             normalization=normalization):
            try:
                outfile.write(line)
            except UnicodeEncodeError:
                if sys.platform == 'win32':
                    sys.stderr.write(ENCODE_ERROR_TEXT_WINDOWS)
                else:
                    sys.stderr.write(ENCODE_ERROR_TEXT_UNIX)
                sys.exit(1)
    except UnicodeDecodeError as err:
        sys.stderr.write(DECODE_ERROR_TEXT % (encoding, err))
        sys.exit(1)
145759
def main():
    
    setup_main_logger(file_logging=False)
    params = argparse.ArgumentParser(description='Shows nearest neighbours of input tokens in the embedding space.')
    params.add_argument('--model', '-m', required=True,
                        help='Model folder to load config from.')
    params.add_argument('--checkpoint', '-c', required=False, type=int, default=None,
                        help='Optional specific checkpoint to load parameters from. Best params otherwise.')
    params.add_argument('--side', '-s', required=True, choices=['source', 'target'], help='what embeddings to look at')
    params.add_argument('--norm', '-n', action='store_true', help='normalize embeddings to unit length')
    params.add_argument('-k', type=int, default=5, help='Number of neighbours to print')
    params.add_argument('--gamma', '-g', type=float, default=1.0, help='Softmax distribution steepness.')
    args = params.parse_args()
    embeddings(args)
708835
def main():
    """"""

    parser = OptionParser(conflict_handler="resolve")

    parser.add_option('--humanTRA', '--human_T_alpha', action='store_true', dest='humanTRA', default=False, help='use default human TRA model (T cell alpha chain)')
    parser.add_option('--humanTRB', '--human_T_beta', action='store_true', dest='humanTRB', default=False, help='use default human TRB model (T cell beta chain)')
    parser.add_option('--mouseTRB', '--mouse_T_beta', action='store_true', dest='mouseTRB', default=False, help='use default mouse TRB model (T cell beta chain)')
    parser.add_option('--humanIGH', '--human_B_heavy', action='store_true', dest='humanIGH', default=False, help='use default human IGH model (B cell heavy chain)')
    parser.add_option('--set_custom_model_VDJ', dest='vdj_model_folder', metavar='PATH/TO/FOLDER/', help='specify PATH/TO/FOLDER/ for a custom VDJ generative model')
    parser.add_option('--set_custom_model_VJ', dest='vj_model_folder', metavar='PATH/TO/FOLDER/', help='specify PATH/TO/FOLDER/ for a custom VJ generative model')

    parser.add_option('-i', '--infile', dest = 'infile_name',metavar='PATH/TO/FILE', help='read in CDR3 sequences (and optionally V/J masks) from PATH/TO/FILE')
    parser.add_option('-o', '--outfile', dest = 'outfile_name', metavar='PATH/TO/FILE', help='write CDR3 sequences and pgens to PATH/TO/FILE')
    parser.add_option('--seq_in', '--seq_index', type='int', metavar='INDEX', dest='seq_in_index', default = 0, help='specifies sequences to be read in are in column INDEX. Default is index 0 (the first column).')

    parser.add_option('--v_in', '--v_mask_index', type='int', metavar='INDEX', dest='V_mask_index', help='specifies V_masks are found in column INDEX in the input file. Default is no V mask.')
    parser.add_option('--j_in', '--j_mask_index', type='int', metavar='INDEX', dest='J_mask_index', help='specifies J_masks are found in column INDEX in the input file. Default is no J mask.')

    parser.add_option('--v_mask', type='string', dest='V_mask', help='specify V usage to condition Pgen on for seqs read in as arguments.')
    parser.add_option('--j_mask', type='string', dest='J_mask', help='specify J usage to condition Pgen on for seqs read in as arguments.')

    parser.add_option('-m', '--max_number_of_seqs', type='int',metavar='N', dest='max_number_of_seqs', help='compute Pgens for at most N sequences.')
    parser.add_option('--lines_to_skip', type='int',metavar='N', dest='lines_to_skip', default = 0, help='skip the first N lines of the file. Default is 0.')
    parser.add_option('-a', '--alphabet_filename', dest='alphabet_filename', metavar='PATH/TO/FILE', help="specify PATH/TO/FILE defining a custom 'amino acid' alphabet. Default is no custom alphabet.")
    parser.add_option('--seq_type_out', type='choice',metavar='SEQ_TYPE', dest='seq_type_out',  choices=['all', 'ntseq', 'nucleotide', 'aaseq', 'amino_acid'], help="if read in sequences are ntseqs, declare what type of sequence to compute pgen for. Default is all. Choices: 'all', 'ntseq', 'nucleotide', 'aaseq', 'amino_acid'")
    parser.add_option('--skip_off','--skip_empty_off', action='store_true', dest = 'skip_empty', default=True, help='stop skipping empty or blank sequences/lines (if for example you want to keep line index fidelity between the infile and outfile).')

    parser.add_option('--display_off', action='store_false', dest='display_seqs', default=True, help='turn the sequence display off (only applies in write-to-file mode). Default is on.')
    parser.add_option('--num_lines_for_display', type='int', metavar='N', default = 50, dest='num_lines_for_display', help='N lines of the output file are displayed when sequence display is on. Also used to determine the number of sequences to average over for speed and time estimates.')
    parser.add_option('--time_updates_off', action='store_false', dest='time_updates', default=True, help='turn time updates off (only applies when sequence display is disabled).')
    parser.add_option('--seqs_per_time_update', type='float', metavar='N', default = 100, dest='seqs_per_time_update', help='specify the number of sequences between time updates. Default is 1e5.')

    parser.add_option('-d', '--delimiter', type='choice', dest='delimiter',  choices=['tab', 'space', ',', ';', ':'], help="declare infile delimiter. Default is tab for .tsv input files, comma for .csv files, and any whitespace for all others. Choices: 'tab', 'space', ',', ';', ':'")
    parser.add_option('--raw_delimiter', type='str', dest='delimiter', help="declare infile delimiter as a raw string.")
    parser.add_option('--delimiter_out', type='choice', dest='delimiter_out',  choices=['tab', 'space', ',', ';', ':'], help="declare outfile delimiter. Default is tab for .tsv output files, comma for .csv files, and the infile delimiter for all others. Choices: 'tab', 'space', ',', ';', ':'")
    parser.add_option('--raw_delimiter_out', type='str', dest='delimiter_out', help="declare for the delimiter outfile as a raw string.")
    parser.add_option('--gene_mask_delimiter', type='choice', dest='gene_mask_delimiter',  choices=['tab', 'space', ',', ';', ':'], help="declare gene mask delimiter. Default comma unless infile delimiter is comma, then default is a semicolon. Choices: 'tab', 'space', ',', ';', ':'")
    parser.add_option('--raw_gene_mask_delimiter', type='str', dest='gene_mask_delimiter', help="declare delimiter of gene masks as a raw string.")
    parser.add_option('--comment_delimiter', type='str', dest='comment_delimiter', help="character or string to indicate comment or header lines to skip.")


    (options, args) = parser.parse_args()

    #Check that the model is specified properly
    main_folder = os.path.dirname(__file__)

    default_models = {}
    default_models['humanTRA'] = [os.path.join(main_folder, 'default_models', 'human_T_alpha'),  'VJ']
    default_models['humanTRB'] = [os.path.join(main_folder, 'default_models', 'human_T_beta'), 'VDJ']
    default_models['mouseTRB'] = [os.path.join(main_folder, 'default_models', 'mouse_T_beta'), 'VDJ']
    default_models['humanIGH'] = [os.path.join(main_folder, 'default_models', 'human_B_heavy'), 'VDJ']

    num_models_specified = sum([1 for x in default_models.keys() + ['vj_model_folder', 'vdj_model_folder'] if getattr(options, x)])

    if num_models_specified == 1: #exactly one model specified
        try:
            d_model = [x for x in default_models.keys() if getattr(options, x)][0]
            model_folder = default_models[d_model][0]
            recomb_type = default_models[d_model][1]
        except IndexError:
            if options.vdj_model_folder: #custom VDJ model specified
                model_folder = options.vdj_model_folder
                recomb_type = 'VDJ'
            elif options.vj_model_folder: #custom VJ model specified
                model_folder = options.vj_model_folder
                recomb_type = 'VJ'
    elif num_models_specified == 0:
        print 'Need to indicate generative model.'
        print 'Exiting...'
        return -1
    elif num_models_specified > 1:
        print 'Only specify one model'
        print 'Exiting...'
        return -1

    #Check that all model and genomic files exist in the indicated model folder
    if not os.path.isdir(model_folder):
        print 'Check pathing... cannot find the model folder: ' + model_folder
        print 'Exiting...'
        return -1

    params_file_name = os.path.join(model_folder,'model_params.txt')
    marginals_file_name = os.path.join(model_folder,'model_marginals.txt')
    V_anchor_pos_file = os.path.join(model_folder,'V_gene_CDR3_anchors.csv')
    J_anchor_pos_file = os.path.join(model_folder,'J_gene_CDR3_anchors.csv')

    for x in [params_file_name, marginals_file_name, V_anchor_pos_file, J_anchor_pos_file]:
        if not os.path.isfile(x):
            print 'Cannot find: ' + x
            print 'Please check the files (and naming conventions) in the model folder ' + model_folder
            print 'Exiting...'
            return -1

    alphabet_filename = options.alphabet_filename #used if a custom alphabet is to be specified
    if alphabet_filename is not None:
        if not os.path.isfile(alphabet_filename):
            print 'Cannot find custom alphabet file: ' + infile_name
            print 'Exiting...'
            return -1

    #Load up model based on recomb_type
    #VDJ recomb case --- used for TCRB and IGH
    if recomb_type == 'VDJ':
        genomic_data = load_model.GenomicDataVDJ()
        genomic_data.load_igor_genomic_data(params_file_name, V_anchor_pos_file, J_anchor_pos_file)
        generative_model = load_model.GenerativeModelVDJ()
        generative_model.load_and_process_igor_model(marginals_file_name)
        pgen_model = generation_probability.GenerationProbabilityVDJ(generative_model, genomic_data, alphabet_filename)
    #VJ recomb case --- used for TCRA and light chain
    elif recomb_type == 'VJ':
        genomic_data = load_model.GenomicDataVJ()
        genomic_data.load_igor_genomic_data(params_file_name, V_anchor_pos_file, J_anchor_pos_file)
        generative_model = load_model.GenerativeModelVJ()
        generative_model.load_and_process_igor_model(marginals_file_name)
        pgen_model = generation_probability.GenerationProbabilityVJ(generative_model, genomic_data, alphabet_filename)

    aa_alphabet = ''.join(pgen_model.codons_dict.keys())

    if options.infile_name is not None:
        infile_name = options.infile_name

        if not os.path.isfile(infile_name):
            print 'Cannot find input file: ' + infile_name
            print 'Exiting...'
            return -1

    if options.outfile_name is not None:
        outfile_name = options.outfile_name
        if os.path.isfile(outfile_name):
            if not raw_input(outfile_name + ' already exists. Overwrite (y/n)? ').strip().lower() in ['y', 'yes']:
                print 'Exiting...'
                return -1

    #Parse delimiter
    delimiter = options.delimiter
    if delimiter is None: #Default case
        if options.infile_name is None:
            delimiter = '\t'
        elif infile_name.endswith('.tsv'): #parse TAB separated value file
            delimiter = '\t'
        elif infile_name.endswith('.csv'): #parse COMMA separated value file
            delimiter = ','
    else:
        try:
            delimiter = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[delimiter]
        except KeyError:
            pass #Other string passed as the delimiter.

    #Parse delimiter_out
    delimiter_out = options.delimiter_out
    if delimiter_out is None: #Default case
        if delimiter is None:
            delimiter_out = '\t'
        else:
            delimiter_out = delimiter
        if options.outfile_name is None:
            pass
        elif outfile_name.endswith('.tsv'): #output TAB separated value file
            delimiter_out = '\t'
        elif outfile_name.endswith('.csv'): #output COMMA separated value file
            delimiter_out = ','
    else:
        try:
            delimiter_out = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[delimiter_out]
        except KeyError:
            pass #Other string passed as the delimiter.

    #Parse gene_delimiter
    gene_mask_delimiter = options.gene_mask_delimiter
    if gene_mask_delimiter is None: #Default case
        gene_mask_delimiter = ','
        if delimiter == ',':
            gene_mask_delimiter = ';'
    else:
        try:
            gene_mask_delimiter = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[gene_mask_delimiter]
        except KeyError:
            pass #Other string passed as the delimiter.


    #More options
    time_updates = options.time_updates
    display_seqs = options.display_seqs
    num_lines_for_display = options.num_lines_for_display
    seq_in_index = options.seq_in_index #where in the line the sequence is after line.split(delimiter)
    lines_to_skip = options.lines_to_skip #one method of skipping header
    comment_delimiter = options.comment_delimiter #another method of skipping header
    seqs_per_time_update = options.seqs_per_time_update
    max_number_of_seqs = options.max_number_of_seqs
    V_mask_index = options.V_mask_index #Default is not conditioning on V identity
    J_mask_index = options.J_mask_index #Default is not conditioning on J identity
    skip_empty = options.skip_empty

    seq_type_out = options.seq_type_out #type of pgens to be computed. Can be ntseq, aaseq, or both
    if seq_type_out is not None:
        seq_type_out = {'all': None, 'ntseq': 'ntseq', 'nucleotide': 'ntseq', 'aaseq': 'aaseq', 'amino_acid': 'aaseq'}[seq_type_out]

    if options.infile_name is None: #No infile specified -- args should be the input seqs
        print_warnings = True
        seqs = args
        seq_types = [determine_seq_type(seq, aa_alphabet) for seq in seqs]
        unrecognized_seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is None]
        if len(unrecognized_seqs) > 0 and print_warnings:
            print 'The following sequences/arguments were not recognized: ' + ', '.join(unrecognized_seqs)
        seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is not None]
        seq_types = [seq_type for seq_type in seq_types if seq_type is not None]


        #Format V and J masks -- uniform for all argument input sequences
        try:
            V_mask = options.V_mask.split(',')
            unrecognized_v_genes = [v for v in V_mask if v not in pgen_model.V_mask_mapping.keys()]
            V_mask = [v for v in V_mask if v in pgen_model.V_mask_mapping.keys()]
            if len(unrecognized_v_genes) > 0:
                print 'These V genes/alleles are not recognized: ' + ', '.join(unrecognized_v_genes)
            if len(V_mask) == 0:
                print 'No recognized V genes/alleles in the provided V_mask. Continuing without conditioning on V usage.'
                V_mask = None
        except AttributeError:
            V_mask = options.V_mask #Default is None, i.e. not conditioning on V identity

        try:
            J_mask = options.J_mask.split(',')
            unrecognized_j_genes = [j for j in J_mask if j not in pgen_model.J_mask_mapping.keys()]
            J_mask = [j for j in J_mask if j in pgen_model.J_mask_mapping.keys()]
            if len(unrecognized_j_genes) > 0:
                print 'These J genes/alleles are not recognized: ' + ', '.join(unrecognized_j_genes)
            if len(J_mask) == 0:
                print 'No recognized J genes/alleles in the provided J_mask. Continuing without conditioning on J usage.'
                J_mask = None
        except AttributeError:
            J_mask = options.J_mask #Default is None, i.e. not conditioning on J identity

        print ''
        start_time = time.time()
        for seq, seq_type in zip(seqs, seq_types):
            if seq_type == 'aaseq':
                c_pgen = pgen_model.compute_aa_CDR3_pgen(seq, V_mask, J_mask, print_warnings)
                print 'Pgen of the amino acid sequence ' + seq + ': ' + str(c_pgen)
                print ''
            elif seq_type == 'regex':
                c_pgen = pgen_model.compute_regex_CDR3_template_pgen(seq, V_mask, J_mask, print_warnings)
                print 'Pgen of the regular expression sequence ' + seq + ': ' + str(c_pgen)
                print ''
            elif seq_type == 'ntseq':
                if seq_type_out is None or seq_type_out == 'ntseq':
                    c_pgen_nt = pgen_model.compute_nt_CDR3_pgen(seq, V_mask, J_mask, print_warnings)
                    print 'Pgen of the nucleotide sequence ' + seq + ': ' + str(c_pgen_nt)
                if seq_type_out is None or seq_type_out == 'aaseq':
                    c_pgen_aa = pgen_model.compute_aa_CDR3_pgen(nt2aa(seq), V_mask, J_mask, print_warnings)
                    print 'Pgen of the amino acid sequence nt2aa(' + seq + ') = ' + nt2aa(seq) + ': ' + str(c_pgen_aa)
                print ''

        c_time = time.time() - start_time
        if c_time > 86400: #more than a day
            c_time_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(c_time)/86400, (int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
        elif c_time > 3600: #more than an hr
            c_time_str = '%d hours, %d minutes, and %.2f seconds.'%((int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
        elif c_time > 60: #more than a min
            c_time_str = '%d minutes and %.2f seconds.'%((int(c_time)/60)%60, c_time%60)
        else:
            c_time_str = '%.2f seconds.'%(c_time)

        print 'Completed pgen computation in: ' + c_time_str

    else: #Read sequences in from file
        print_warnings = False #Most cases of reading in from file should have warnings disabled
        seqs = []
        seq_types = []
        V_usage_masks = []
        J_usage_masks = []

        infile = open(infile_name, 'r')

        for i, line in enumerate(infile):
            if comment_delimiter is not None: #Default case -- no comments/header delimiter
                if line.startswith(comment_delimiter): #allow comments
                    continue
            if i < lines_to_skip:
                continue

            if delimiter is None: #Default delimiter is any whitespace
                split_line = line.split()
            else:
                split_line = line.split(delimiter)

            #Find the seq
            try:
                seq = split_line[seq_in_index].strip()
                if len(seq.strip()) == 0:
                    if skip_empty:
                        continue
                    else:
                        seqs.append(seq) #keep the blank seq as a placeholder
                        seq_types.append('aaseq')
                else:
                    seqs.append(seq)
                    seq_types.append(determine_seq_type(seq, aa_alphabet))
            except IndexError: #no index match for seq
                if skip_empty and len(line.strip()) == 0:
                    continue
                print 'seq_in_index is out of range'
                print 'Exiting...'
                infile.close()
                return -1

            #Find and format V_usage_mask
            if V_mask_index is None:
                V_usage_masks.append(None) #default mask
            else:
                try:
                    V_usage_mask = split_line[V_mask_index].strip().split(gene_mask_delimiter)
                    #check that all V gene/allele names are recognized
                    if all([v in pgen_model.V_mask_mapping for v in V_usage_mask]):
                        V_usage_masks.append(V_usage_mask)
                    else:
                        print str(V_usage_mask) + " is not a usable V_usage_mask composed exclusively of recognized V gene/allele names"
                        print 'Unrecognized V gene/allele names: ' + ', '.join([v for v in V_usage_mask if not v in pgen_model.V_mask_mapping.keys()])
                        print 'Exiting...'
                        infile.close()
                        return -1
                except IndexError: #no index match for V_mask_index
                    print 'V_mask_index is out of range'
                    print 'Exiting...'
                    infile.close()
                    return -1

            #Find and format J_usage_mask
            if J_mask_index is None:
                J_usage_masks.append(None) #default mask
            else:
                try:
                    J_usage_mask = split_line[J_mask_index].strip().split(gene_mask_delimiter)
                    #check that all V gene/allele names are recognized
                    if all([j in pgen_model.J_mask_mapping for j in J_usage_mask]):
                        J_usage_masks.append(J_usage_mask)
                    else:
                        print str(J_usage_mask) + " is not a usable J_usage_mask composed exclusively of recognized J gene/allele names"
                        print 'Unrecognized J gene/allele names: ' + ', '.join([j for j in J_usage_mask if not j in pgen_model.J_mask_mapping.keys()])
                        print 'Exiting...'
                        infile.close()
                        return -1
                except IndexError: #no index match for J_mask_index
                    print 'J_mask_index is out of range'
                    print 'Exiting...'
                    infile.close()
                    return -1

            if max_number_of_seqs is not None:
                if len(seqs) >= max_number_of_seqs:
                    break


        unrecognized_seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is None]
        if len(unrecognized_seqs) > 0 and len(unrecognized_seqs) < len(seqs):
            if print_warnings or options.outfile_name is not None:
                print 'Some strings read in were not parsed as sequences -- they will be omitted.'
                print 'Examples of improperly read strings: '
                for unrecognized_seq in unrecognized_seqs[:10]:
                    print unrecognized_seq
            seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is not None]
            V_usage_masks = [V_usage_mask for i, V_usage_mask in enumerate(V_usage_masks) if seq_types[i] is not None]
            seq_types = [seq_type for seq_type in seq_types if seq_type is not None]
        elif len(unrecognized_seqs) > 0 and len(unrecognized_seqs) == len(seqs):
            print 'None of the read in strings were parsed as sequences. Check input file.'
            print 'Examples of improperly read strings:'
            for unrecognized_seq in unrecognized_seqs[:10]:
                print unrecognized_seq
            print 'Exiting...'
            return -1

        infile.close()


        if options.outfile_name is not None: #OUTFILE SPECIFIED, allow printed info/display

            print 'Successfully read in and formatted ' + str(len(seqs)) + ' sequences and any V or J usages.'
            if display_seqs:
                sys.stdout.write('\r'+'Continuing to Pgen computation in 3... ')
                sys.stdout.flush()
                time.sleep(0.4)
                sys.stdout.write('\r'+'Continuing to Pgen computation in 2... ')
                sys.stdout.flush()
                time.sleep(0.4)
                sys.stdout.write('\r'+'Continuing to Pgen computation in 1... ')
                sys.stdout.flush()
                time.sleep(0.4)
            else:
                print 'Continuing to Pgen computation.'
                print_warnings = True #Display is off, can print warnings

            if display_seqs:
                lines_for_display = []
                times_for_speed_calc = [time.time()]

            outfile = open(outfile_name, 'w')
            start_time = time.time()
            for i, seq in enumerate(seqs):
                if seq_types[i] == 'aaseq':
                    #Compute Pgen and print out
                    c_pgen_line = seq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                if seq_types[i] == 'regex':
                    #Compute Pgen and print out
                    c_pgen_line = seq + delimiter_out + str(pgen_model.compute_regex_CDR3_template_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                elif seq_types[i] == 'ntseq':
                    ntseq = seq
                    if len(ntseq) % 3 == 0: #inframe sequence
                        aaseq = nt2aa(ntseq)
                        #Compute Pgen and print out based on recomb_type and seq_type_out
                        if seq_type_out is None:
                            c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings)) + delimiter_out + aaseq + delimiter_out +  str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                        elif seq_type_out == 'ntseq':
                            c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                        elif seq_type_out == 'aaseq':
                            c_pgen_line = aaseq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                    else: #out of frame sequence -- Pgens are 0 and use 'out_of_frame' for aaseq
                        if seq_type_out is None:
                            c_pgen_line = ntseq + delimiter_out + '0' + delimiter_out + 'out_of_frame' + delimiter_out + '0'
                        elif seq_type_out == 'ntseq':
                            c_pgen_line = ntseq + delimiter_out + '0'
                        elif seq_type_out == 'aaseq':
                            c_pgen_line = 'out_of_frame' + delimiter_out + '0'

                outfile.write(c_pgen_line + '\n')

                #Print time update
                if display_seqs:
                    cc_time = time.time()
                    c_time = cc_time - start_time
                    times_for_speed_calc = [cc_time] + times_for_speed_calc[:num_lines_for_display]
                    c_avg_speed = (len(times_for_speed_calc)-1)/float(times_for_speed_calc[0] - times_for_speed_calc[-1])

                    #eta = ((len(seqs) - (i+1))/float(i+1))*c_time

                    eta = (len(seqs) - (i+1))/c_avg_speed

                    lines_for_display = [c_pgen_line] + lines_for_display[:num_lines_for_display]


                    c_time_str = '%s hours, %s minutes, and %s seconds.'%(repr(int(c_time)/3600).rjust(3), repr((int(c_time)/60)%60).rjust(2), repr(int(c_time)%60).rjust(2))
                    eta_str = '%s hours, %s minutes, and %s seconds.'%(repr(int(eta)/3600).rjust(3), repr((int(eta)/60)%60).rjust(2), repr(int(eta)%60).rjust(2))
                    time_str = 'Time to compute Pgen on %s seqs: %s \nEst. time for remaining %s seqs: %s'%(repr(i+1).rjust(9), c_time_str, repr(len(seqs) - (i + 1)).rjust(9), eta_str)
                    speed_str = 'Current Pgen computation speed: %s seqs/min'%(repr(round((len(times_for_speed_calc)-1)*60/float(times_for_speed_calc[0] - times_for_speed_calc[-1]), 2)).rjust(8))
                    display_str = '\n'.join(lines_for_display[::-1]) + '\n' + '-'*80 + '\n' + time_str + '\n' + speed_str + '\n' + '-'*80
                    print '\033[2J' + display_str
                elif (i+1)%seqs_per_time_update == 0 and time_updates:
                    c_time = time.time() - start_time
                    eta = ((len(seqs) - (i+1))/float(i+1))*c_time
                    if c_time > 86400: #more than a day
                        c_time_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(c_time)/86400, (int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
                    elif c_time > 3600: #more than an hr
                        c_time_str = '%d hours, %d minutes, and %.2f seconds.'%((int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
                    elif c_time > 60: #more than a min
                        c_time_str = '%d minutes and %.2f seconds.'%((int(c_time)/60)%60, c_time%60)
                    else:
                        c_time_str = '%.2f seconds.'%(c_time)

                    if eta > 86400: #more than a day
                        eta_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(eta)/86400, (int(eta)/3600)%24, (int(eta)/60)%60, eta%60)
                    elif eta > 3600: #more than an hr
                        eta_str = '%d hours, %d minutes, and %.2f seconds.'%((int(eta)/3600)%24, (int(eta)/60)%60, eta%60)
                    elif eta > 60: #more than a min
                        eta_str = '%d minutes and %.2f seconds.'%((int(eta)/60)%60, eta%60)
                    else:
                        eta_str = '%.2f seconds.'%(eta)

                    print 'Pgen computed for %d sequences in: %s Estimated time remaining: %s'%(i+1, c_time_str, eta_str)

            c_time = time.time() - start_time
            if c_time > 86400: #more than a day
                c_time_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(c_time)/86400, (int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
            elif c_time > 3600: #more than an hr
                c_time_str = '%d hours, %d minutes, and %.2f seconds.'%((int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
            elif c_time > 60: #more than a min
                c_time_str = '%d minutes and %.2f seconds.'%((int(c_time)/60)%60, c_time%60)
            else:
                c_time_str = '%.2f seconds.'%(c_time)
            print 'Completed Pgen computation for %d sequences: in %s'%(len(seqs), c_time_str)

            outfile.close()

        else: #NO OUTFILE -- print directly to stdout
            start_time = time.time()
            for i, seq in enumerate(seqs):
                if seq_types[i] == 'aaseq':
                    #Compute Pgen and print out
                    c_pgen_line = seq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                if seq_types[i] == 'regex':
                    #Compute Pgen and print out
                    c_pgen_line = seq + delimiter_out + str(pgen_model.compute_regex_CDR3_template_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                elif seq_types[i] == 'ntseq':
                    ntseq = seq
                    if len(ntseq) % 3 == 0: #inframe sequence
                        aaseq = nt2aa(ntseq)
                        #Compute Pgen and print out based on recomb_type and seq_type_out
                        if seq_type_out is None:
                            c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings)) + delimiter_out + aaseq + delimiter_out +  str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                        elif seq_type_out == 'ntseq':
                            c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                        elif seq_type_out == 'aaseq':
                            c_pgen_line = aaseq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                    else: #out of frame sequence -- Pgens are 0 and use 'out_of_frame' for aaseq
                        if seq_type_out is None:
                            c_pgen_line = ntseq + delimiter_out + '0' + delimiter_out + 'out_of_frame' + delimiter_out + '0'
                        elif seq_type_out == 'ntseq':
                            c_pgen_line = ntseq + delimiter_out + '0'
                        elif seq_type_out == 'aaseq':
                            c_pgen_line = 'out_of_frame' + delimiter_out + '0'

                print c_pgen_line