Converts the ContigOrdering file into AGP format
553147
def write_agp(self, obj, sizes, fw=sys.stdout, gapsize=100,
                  gaptype="contig", evidence="map"):
        '''
        '''
        contigorder = [(x.contig_name, x.strand) for x in self]
        order_to_agp(obj, contigorder, sizes, fw,
                     gapsize=gapsize, gaptype=gaptype, evidence=evidence)
945580
def format(args):
    
    from jcvi.formats.obo import load_GODag, validate_term

    valid_multiparent_ops = ["split", "merge"]

    p = OptionParser(format.__doc__)

    g1 = OptionGroup(p, "Parameter(s) used to modify GFF attributes (9th column)")
    g1.add_option("--name", help="Add Name attribute from two-column file [default: %default]")
    g1.add_option("--note", help="Add Note attribute from two-column file [default: %default]")
    g1.add_option("--add_attribute", dest="attrib_files", help="Add new attribute(s) " + \
                "from two-column file(s); attribute name comes from filename; " + \
                "accepts comma-separated list of files [default: %default]")
    g1.add_option("--add_dbxref", dest="dbxref_files", help="Add new Dbxref value(s) (DBTAG:ID) " + \
                "from two-column file(s). DBTAG comes from filename, ID comes from 2nd column; " + \
                "accepts comma-separated list of files [default: %default]")
    g1.add_option("--nostrict", default=False, action="store_true",
                 help="Disable strict parsing of GFF file and/or mapping file [default: %default]")
    g1.add_option("--remove_attr", dest="remove_attrs", help="Specify attributes to remove; " + \
                "accepts comma-separated list of attribute names [default: %default]")
    g1.add_option("--copy_id_attr_to_name", default=False, action="store_true",
                 help="Copy `ID` attribute value to `Name`, when `Name` is not defined")
    g1.add_option("--invent_name_attr", default=False, action="store_true",
                 help="Invent `Name` attribute for 2nd level child features; " + \
                "Formatted like PARENT:FEAT_TYPE:FEAT_INDEX")
    g1.add_option("--no_keep_attr_order", default=False, action="store_true",
                 help="Do not maintain attribute order [default: %default]")
    p.add_option_group(g1)

    g2 = OptionGroup(p, "Parameter(s) used to modify content within columns 1-8")
    g2.add_option("--seqid", help="Switch seqid from two-column file. If not" + \
                " a file, value will globally replace GFF seqid [default: %default]")
    g2.add_option("--source", help="Switch GFF source from two-column file. If not" + \
                " a file, value will globally replace GFF source [default: %default]")
    g2.add_option("--type", help="Switch GFF feature type from two-column file. If not" + \
                " a file, value will globally replace GFF type [default: %default]")
    g2.add_option("--fixphase", default=False, action="store_true",
                 help="Change phase 1<->2, 2<->1 [default: %default]")
    p.add_option_group(g2)

    g3 = OptionGroup(p, "Other parameter(s) to perform manipulations to the GFF " + \
                 "file content")
    g3.add_option("--unique", default=False, action="store_true",
                 help="Make IDs unique [default: %default]")
    g3.add_option("--chaindup", default=None, dest="duptype",
                 help="Chain duplicate features of a particular GFF3 `type`," + \
                      " sharing the same ID attribute [default: %default]")
    g3.add_option("--multiparents", default=None, choices=valid_multiparent_ops,
                 help="Split/merge identical features (same `seqid`, `source`, `type` " + \
                 "`coord-range`, `strand`, `phase`) mapping to multiple parents " + \
                 "[default: %default]")
    g3.add_option("--remove_feats", help="Comma separated list of features to remove by type" + \
                " [default: %default]")
    g3.add_option("--remove_feats_by_ID", help="List of features to remove by ID;" + \
                " accepts comma-separated list or list file [default: %default]")
    g3.add_option("--gsac", default=False, action="store_true",
                 help="Fix GSAC GFF3 file attributes [default: %default]")
    g3.add_option("--invent_protein_feat", default=False, action="store_true",
                 help="Invent a protein feature span (chain CDS feats)")
    g3.add_option("--process_ftype", default=None, type="str",
                 help="Specify feature types to process; "
                 "accepts comma-separated list of feature types [default: %default]")
    g3.add_option("--gff3", default=False, action="store_true",
                 help="Print output in GFF3 format [default: %default]")
    g3.add_option("--make_gff_store", default=False, action="store_true",
                 help="Store entire GFF file in memory during first iteration [default: %default]")
    p.add_option_group(g3)

    p.set_outfile()
    p.set_SO_opts()

    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    gffile, = args
    mapfile = opts.seqid
    names = opts.name
    note = opts.note
    source = opts.source
    ftype = opts.type
    attrib_files = opts.attrib_files.split(",") if opts.attrib_files else None
    dbxref_files = opts.dbxref_files.split(",") if opts.dbxref_files else None
    remove_attrs = opts.remove_attrs.split(",") if opts.remove_attrs else None
    process_ftype = opts.process_ftype.split(",") if opts.process_ftype else None
    gsac = opts.gsac
    assert not (opts.unique and opts.duptype), \
        "Cannot use `--unique` and `--chaindup` together"
    assert not(opts.type and opts.duptype), \
        "Cannot use `--type` and `--chaindup` together"
    unique = opts.unique
    duptype = opts.duptype
    fixphase = opts.fixphase
    phaseT = {"1":"2", "2":"1"}
    remove_feats = opts.remove_feats.split(",") if opts.remove_feats else None
    remove_feats_by_ID = None
    if opts.remove_feats_by_ID:
        remove_feats_by_ID = LineFile(opts.remove_feats_by_ID, load=True).lines \
                if op.isfile(opts.remove_feats_by_ID) else \
                opts.remove_feats_by_ID.split(",")
    strict = False if opts.nostrict else True
    make_gff_store = True if gffile in ("-", "stdin") else opts.make_gff_store
    assert not (opts.copy_id_attr_to_name and opts.invent_name_attr), \
        "Cannot use `--copy_id_attr_to_name` and `--invent_name_attr` together"
    copy_id_attr_to_name = opts.copy_id_attr_to_name
    invent_name_attr = opts.invent_name_attr
    invent_protein_feat = opts.invent_protein_feat
    compute_signature = False

    outfile = opts.outfile

    mapping = None
    mod_attrs = set()
    if mapfile and op.isfile(mapfile):
        mapping = DictFile(mapfile, delimiter="\t", strict=strict)
        mod_attrs.add("ID")
    if note:
        note = DictFile(note, delimiter="\t", strict=strict)
        mod_attrs.add("Note")
    if source and op.isfile(source):
        source = DictFile(source, delimiter="\t", strict=strict)
    if ftype and op.isfile(ftype):
        ftype = DictFile(ftype, delimiter="\t", strict=strict)
    if names:
        names = DictFile(names, delimiter="\t", strict=strict)
        mod_attrs.add("Name")

    if attrib_files:
        attr_values = {}
        for fn in attrib_files:
            attr_name = op.basename(fn).rsplit(".", 1)[0]
            if attr_name not in reserved_gff_attributes:
                attr_name = attr_name.lower()
            attr_values[attr_name] = DictFile(fn, delimiter="\t", strict=strict)
            mod_attrs.add(attr_name)
    if dbxref_files:
        dbxref_values = {}
        for fn in dbxref_files:
            dbtag = op.basename(fn).rsplit(".", 1)[0]
            dbxref_values[dbtag] = DictFile(fn, delimiter="\t", strict=strict)
        mod_attrs.add("Dbxref")

    if remove_attrs:
        mod_remove_attrs = []
        for remove_attr in remove_attrs:
            if remove_attr in mod_attrs:
                mod_remove_attrs.append(remove_attr)

        if mod_remove_attrs:
            logging.error("Attributes `{0}` cannot be removed and modified".format( \
                    ",".join(mod_remove_attrs)))
            sys.exit()

    if gsac:  # setting gsac will force IDs to be unique
        unique = True
        notes = {}

    remove = set()
    if unique or duptype or remove_feats or remove_feats_by_ID \
            or opts.multiparents == "merge" or invent_name_attr or make_gff_store \
            or invent_protein_feat:
        if unique:
            dupcounts = defaultdict(int)
            seen = defaultdict(int)
            newparentid = {}
        elif duptype:
            dupranges = AutoVivification()
            skip = defaultdict(int)
        if opts.multiparents == "merge":
            merge_feats = AutoVivification()
        if invent_name_attr:
            ft = GffFeatureTracker()
        elif copy_id_attr_to_name:
            pass
        if invent_protein_feat:
            cds_track = {}
        if opts.multiparents == "merge" or invent_name_attr:
            make_gff_store = compute_signature = True
        gff = Gff(gffile, keep_attr_order=(not opts.no_keep_attr_order), \
                make_gff_store=make_gff_store, compute_signature=compute_signature, \
                strict=strict)
        for g in gff:
            if process_ftype and g.type not in process_ftype:
                continue
            id = g.accn
            if remove_feats and g.type in remove_feats:
                remove.add(id)
            if remove_feats_by_ID and id in remove_feats_by_ID:
                remove.add(id)
            if unique:
                dupcounts[id] += 1
            elif duptype and g.type == duptype:
                dupranges[g.seqid][id][g.idx] = (g.start, g.end)
            if opts.multiparents == "merge" and g.type != "CDS": #don't merge CDS
                pp = g.get_attr("Parent", first=False)
                if pp and len(pp) > 0:
                    for parent in pp:
                        if parent not in remove:
                            sig = g.sign
                            if sig not in merge_feats:
                                merge_feats[sig]['parents'] = []
                            if parent not in merge_feats[sig]['parents']:
                                merge_feats[sig]['parents'].append(parent)
            if invent_name_attr:
                parent, iso = atg_name(g.get_attr("Parent"), retval="locus,iso")
                if not parent:
                    parent = g.get_attr("Parent")
                ft.track(parent, g)
            if invent_protein_feat:
                if g.type == 'CDS':
                    cds_parent = g.get_attr("Parent")
                    if cds_parent not in cds_track:
                        cds_track[cds_parent] = []
                    cds_track[cds_parent].append((g.start, g.end))

    if opts.verifySO:
        so = load_GODag()
        valid_soterm = {}

    fw = must_open(outfile, "w")
    if not make_gff_store:
        gff = Gff(gffile, keep_attr_order=(not opts.no_keep_attr_order), \
                strict=strict)
    for g in gff:
        if process_ftype and g.type not in process_ftype:
            print(g, file=fw)
            continue

        id = g.accn

        if opts.multiparents == "merge" and g.type != "CDS": #don't merge CDS
            sig = g.sign
            if len(merge_feats[sig]['parents']) > 1:
                if 'candidate' not in merge_feats[sig]:
                    merge_feats[sig]['candidate'] = id
                    g.set_attr("Parent", merge_feats[sig]['parents'])
                else:
                    continue

        if remove_feats or remove_feats_by_ID:
            if id in remove:
                continue
            else:
                if "Parent" in g.attributes:
                    keep, parent = [], g.get_attr("Parent", first=False)
                    for i, pid in enumerate(parent):
                        if pid not in remove:
                            keep.append(parent[i])
                        else:
                            remove.add(id)
                    if len(keep) == 0:
                        continue
                    parent = g.set_attr("Parent", keep)

        if remove_attrs:
            for remove_attr in remove_attrs:
                if remove_attr in g.attributes:
                    g.set_attr(remove_attr, None)

        if opts.verifySO:
            if g.type not in valid_soterm:
                valid_soterm[g.type] = validate_term(g.type, so=so, method=opts.verifySO)
            ntype = valid_soterm[g.type]
            if ntype and g.type != ntype:
                g.type = ntype

        origid = g.seqid
        if fixphase:
            phase = g.phase
            g.phase = phaseT.get(phase, phase)

        if mapfile:
            if isinstance(mapping, dict):
                if origid in mapping:
                    g.seqid = mapping[origid]
                else:
                    logging.error("{0} not found in `{1}`. ID unchanged.".\
                            format(origid, mapfile))
            else:
                g.seqid = mapfile

        if source:
            if isinstance(source, dict) and g.source in source:
                g.source = source[g.source]
            else:
                g.source = source

        if names:
            if id in names:
                g.set_attr("Name", names[id])

        if note:
            name = g.get_attr("Name")
            tag = None
            if id in note:
                tag = note[id]
            elif name and name in note:
                tag = note[name]

            if tag:
                g.set_attr("Note", tag, update=False)

        if attrib_files:
            for attr_name in attr_values:
                name = g.get_attr("Name")
                if id in attr_values[attr_name]:
                    g.set_attr(attr_name, attr_values[attr_name][id])
                elif name and name in attr_values[attr_name]:
                    g.set_attr(attr_name, attr_values[attr_name][name])

        if dbxref_files:
            for dbtag in dbxref_values:
                if id in dbxref_values[dbtag]:
                    g.set_attr("Dbxref", dbxref_values[dbtag][id], dbtag=dbtag, append=True)

        if unique:
            if dupcounts[id] > 1:
                seen[id] += 1
                old_id = id
                id = "{0}-{1}".format(old_id, seen[old_id])
                newparentid[old_id] = id
                g.set_attr("ID", id)

            if "Parent" in g.attributes:
                parent = g.attributes["Parent"][0]
                if dupcounts[parent] > 1:
                    g.set_attr("Parent", newparentid[parent])

        if duptype:
            if duptype == g.type and len(dupranges[g.seqid][id]) > 1:
                p = sorted(dupranges[g.seqid][id])
                s, e = dupranges[g.seqid][id][p[0]][0:2]  # get coords of first encountered feature
                if g.start == s and g.end == e and p[0] == g.idx:
                    r = [dupranges[g.seqid][id][x] for x in dupranges[g.seqid][id]]
                    g.start, g.end = range_minmax(r)
                else:
                    skip[(g.seqid, g.idx, id, g.start, g.end)] = 1

        if gsac and g.type == "gene":
            notes[id] = g.attributes["Name"]

        if ftype:
            if isinstance(ftype, dict) and g.type in ftype:
                g.type = ftype[g.type]
            else:
                g.type = ftype

        if invent_name_attr:
            ft.store_symbol(g)
            if re.search(ft.ftype, g.type):
                parent, iso = atg_name(g.get_attr("Parent"), retval="locus,iso")
                if not parent:
                    parent = g.get_attr("Parent")
                if parent in ft.tracker:
                    fidx = ft.feat_index(parent, g.type, g.strand, (g.start, g.end, g.sign))
                    symbol = ft.get_symbol(parent)
                    attr = "ID" if symbol == parent else "Name"
                    g.set_attr(attr, "{0}:{1}:{2}".format(symbol, g.type, fidx + 1))
                    if opts.multiparents == "merge" and attr == "Name":
                        g.set_attr("ID", "{0}:{1}:{2}".format(parent, g.type, fidx + 1))
        elif copy_id_attr_to_name:
            if "Name" not in g.attributes.keys():
                g.set_attr("Name", g.get_attr("ID"))

        protein_feat = None
        if invent_protein_feat:
            if g.type == 'mRNA':
                if id in cds_track:
                    pstart, pstop = range_minmax(cds_track[id])
                    protein_feat = GffLine("\t".join(str(x) for x in [g.seqid, g.source, "protein", pstart, pstop, \
                            ".", g.strand, ".", "ID={0}-Protein;Name={0};Derives_from={0}".format(id)]))
            elif g.type == 'CDS':
                parent = g.get_attr("Parent")
                if parent in cds_track:
                    _parent = [parent, "{0}-Protein".format(parent)]
                    g.set_attr("Parent", _parent)

        pp = g.get_attr("Parent", first=False)
        if opts.multiparents == "split" and (pp and len(pp) > 1) and g.type != "CDS":  # separate features with multiple parents
            id = g.get_attr("ID")
            for i, parent in enumerate(pp):
                if id: g.set_attr("ID", "{0}-{1}".format(id, i + 1))
                g.set_attr("Parent", parent, update=True, urlquote=True)
                if gsac:
                    fix_gsac(g, notes)
                print(g, file=fw)
        else:
            if g.gff3 and not opts.gff3:
                opts.gff3 = True
            g.update_attributes(gff3=opts.gff3)
            if gsac:
                fix_gsac(g, notes)
            if duptype == g.type and skip[(g.seqid, g.idx, id, g.start, g.end)] == 1:
                continue
            print(g, file=fw)
            if g.type == 'mRNA' and invent_protein_feat:
                print(protein_feat, file=fw)

    fw.close()
488814
def agp(args):
    
    from jcvi.formats.agp import OO

    p = OptionParser(agp.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    sizesfile, = args
    sizes = Sizes(sizesfile)
    agpfile = sizes.filename.rsplit(".", 1)[0] + ".agp"
    fw = open(agpfile, "w")
    o = OO()  # Without a filename
    for ctg, size in sizes.iter_sizes():
        o.add(ctg, ctg, size)

    o.write_AGP(fw)
    fw.close()
    logging.debug("AGP file written to `{0}`.".format(agpfile))

    return agpfile
732019
def to_file(format, *args)
      options = extract_options!(args)
      datasource = args[0]
      path = options[:path] || File.join(Dir.tmpdir, "beardley_#{Time.now.to_i.to_s(36)}_#{rand(100_000_000_000).to_s(36)}.#{format}")
      if format == :pdf
        _JasperPrint                 = Rjb.import('net.sf.jasperreports.engine.JasperPrint')
        _JasperExportManager         = Rjb.import('net.sf.jasperreports.engine.JasperExportManager')
        _JasperExportManager._invoke('exportReportToPdfFile', 'Lnet.sf.jasperreports.engine.JasperPrint;Ljava.lang.String;', prepare(datasource), Rjb.import('java.lang.String').new(path.to_s))
      elsif Beardley.exporters[format]
        exporter = Beardley.with_warnings { Rjb.import(Beardley.exporters[format]) }.new
        _JRExporterParameter = Rjb.import('net.sf.jasperreports.engine.JRExporterParameter')
        exporter.setParameter(_JRExporterParameter.JASPER_PRINT, prepare(datasource))
        exporter.setParameter(_JRExporterParameter.OUTPUT_FILE_NAME, path.to_s)
        exporter.exportReport
      else
        raise "Invalid export format: #{format.inspect}"
      end
      path
    end
1132129
function normalizePEMContent(root, file) {
  if (Array.isArray(file)) return file.map(function map(item) {
    return normalizePEMContent(root, item)
  });

  //
  // Assumption that this is a Buffer, a PEM file, or something broken
  //
  if (typeof(file) !== 'string' || pemFormat.test(file)) {
    return file;
  }

  return fs.readFileSync(path.resolve(root, file));
}
278127
def parse_csv(self, infile, delimiter=",", decimal_sep="."):
        ""
        keys = ('name','type','x1','y1','x2','y2','font','size',
            'bold','italic','underline','foreground','background',
            'align','text','priority', 'multiline')
        self.elements = []
        self.pg_no = 0
        if not PY3K:
            f = open(infile, 'rb')
        else:
            f = open(infile)
        for row in csv.reader(f, delimiter=delimiter):
            kargs = {}
            for i,v in enumerate(row):
                if not v.startswith("'") and decimal_sep!=".": 
                    v = v.replace(decimal_sep,".")
                else:
                    v = v
                if v=='':
                    v = None
                else:
                    v = eval(v.strip())
                kargs[keys[i]] = v
            self.elements.append(kargs)
        self.keys = [v['name'].lower() for v in self.elements]
653833
def to_ts(s):
        

        # Strip TZ portion if present
        m = Nginx.DATE_FMT.match(s)
        if m:
            s = m.group(1)
            delta = timedelta(seconds=int(m.group(3)) * (-1 if m.group(2) == '-' else 1)) # Offset from GMT
        else:
            delta = timedelta(seconds=0)
        dt = datetime.strptime(s, "%d/%b/%Y:%H:%M:%S")
        dt += delta
        return dt.strftime('%Y-%m-%dT%H:%M:%SZ')
807395
function toGregorianArguments(aArgs) {
		var oBuddhist, oGregorian;
		oBuddhist = {
			year: aArgs[0],
			month: aArgs[1],
			day: aArgs[2] !== undefined ? aArgs[2] : 1
		};
		oGregorian = toGregorian(oBuddhist);
		aArgs[0] = oGregorian.year;
		return aArgs;
	}
826445
def write_membership(loglike,filename):
    

    ra,dec = gal2cel(loglike.catalog.lon,loglike.catalog.lat)
        
    name_objid = loglike.config['catalog']['objid_field']
    name_mag_1 = loglike.config['catalog']['mag_1_field']
    name_mag_2 = loglike.config['catalog']['mag_2_field']
    name_mag_err_1 = loglike.config['catalog']['mag_err_1_field']
    name_mag_err_2 = loglike.config['catalog']['mag_err_2_field']

    # Angular and isochrone separations
    sep = angsep(loglike.source.lon,loglike.source.lat,
                 loglike.catalog.lon,loglike.catalog.lat)
    isosep = loglike.isochrone.separation(loglike.catalog.mag_1,loglike.catalog.mag_2)

    data = odict()
    data[name_objid] = loglike.catalog.objid
    data['GLON'] = loglike.catalog.lon
    data['GLAT'] = loglike.catalog.lat
    data['RA']   = ra
    data['DEC']  = dec
    data[name_mag_1] = loglike.catalog.mag_1
    data[name_mag_err_1] = loglike.catalog.mag_err_1
    data[name_mag_2] = loglike.catalog.mag_2
    data[name_mag_err_2] = loglike.catalog.mag_err_2
    data['COLOR'] = loglike.catalog.color
    data['ANGSEP'] = sep
    data['ISOSEP'] = isosep
    data['PROB'] = loglike.p

    # HIERARCH allows header keywords longer than 8 characters
    header = []
    for param,value in loglike.source.params.items():
        card = dict(name='HIERARCH %s'%param.upper(),
                    value=value.value,
                    comment=param)
        header.append(card)
    card = dict(name='HIERARCH %s'%'TS',value=loglike.ts(),
                comment='test statistic')
    header.append(card)
    card = dict(name='HIERARCH %s'%'TIMESTAMP',value=time.asctime(),
                comment='creation time')
    header.append(card)
    fitsio.write(filename,data,header=header,clobber=True)
1161423
def _to_file_format(self, iamc_index):
        """"""
        df = self.timeseries(iamc_index=iamc_index).reset_index()
        df = df.rename(columns={c: str(c).title() for c in df.columns})
        return df
598707
function formatMsg(){

        return Array.prototype.slice.call( arguments ) .concat([ " (at ", linen, ":", linec, ")." ]) .join( "" );
    }
781653
function make_bucket_agg(groupby, subagg) {
    var bucket_agg = {};
    groupby.forEach(function(field) {
        bucket_agg[field] = {
            terms: {
                field: field,
                size: _calculate_size_for_aggregation(subagg)
            }
        };
    });

    return _.reduceRight(bucket_agg, function nest(memo, value, key) {
        var obj = {};

        if (!_.isEmpty(memo)) {
            value.aggregations = memo;
        }

        obj[key] = value;

        return obj;
    }, subagg);
}
921331
def _bgzip_file(finput, config, work_dir, needs_bgzip, needs_gunzip, needs_convert, data):
    
    if isinstance(finput, six.string_types):
        in_file = finput
    else:
        assert not needs_convert, "Do not yet handle quality conversion with multiple inputs"
        return _bgzip_multiple_files(finput, work_dir, data)
    out_file = os.path.join(work_dir, os.path.basename(in_file).replace(".bz2", "") +
                            (".gz" if not in_file.endswith(".gz") else ""))
    if not utils.file_exists(out_file):
        with file_transaction(config, out_file) as tx_out_file:
            bgzip = tools.get_bgzip_cmd(config)
            is_remote = objectstore.is_remote(in_file)
            in_file = objectstore.cl_input(in_file, unpack=needs_gunzip or needs_convert or
                                           needs_bgzip or dd.get_trim_ends(data))
            if needs_convert or dd.get_trim_ends(data):
                in_file = fastq_convert_pipe_cl(in_file, data)
            if needs_gunzip and not (needs_convert or dd.get_trim_ends(data)):
                if in_file.endswith(".bz2"):
                    gunzip_cmd = "bunzip2 -c {in_file} |".format(**locals())
                else:
                    gunzip_cmd = "gunzip -c {in_file} |".format(**locals())
                bgzip_in = "/dev/stdin"
            else:
                gunzip_cmd = ""
                bgzip_in = in_file
            if needs_bgzip:
                do.run("{gunzip_cmd} {bgzip} -c {bgzip_in} > {tx_out_file}".format(**locals()),
                       "bgzip input file")
            elif is_remote:
                bgzip = "| bgzip -c" if (needs_convert or dd.get_trim_ends(data)) else ""
                do.run("cat {in_file} {bgzip} > {tx_out_file}".format(**locals()), "Get remote input")
            else:
                raise ValueError("Unexpected inputs: %s %s %s %s" % (in_file, needs_bgzip,
                                                                     needs_gunzip, needs_convert))
    return out_file
721686
function fileCategoryMap() {

        // Go over every file lookup function we export.
        var exports = module.exports(grunt);
        // This list of categories must contain all non-overlapping file categories.
        var categories = ['extLibFiles', 'extLibMapFiles', 'extCssFiles', 'extFontFiles', 'fontFiles',
            'appIndexFiles', 'testIndexFiles', 'configFiles', 'libFiles', 'modelFiles', 'srcDataFiles',
            'codeFiles', 'otherJsFiles', 'otherNonJsFiles', 'taskFiles', 'cssFiles', 'picFiles', 'soundFiles', 'unitTestFiles',
            'commonJsFiles', 'commonOtherFiles', 'ignoredFiles', 'distUncompressedFiles',
            'distLibFiles', 'distIndexFiles', 'distJsFiles', 'distCssFiles',
            'toolsShellFiles', 'unitTestDataFiles', 'picSrcFiles', 'soundSrcFiles',
            'htmlTemplateFiles', 'generatedJsFiles', 'otherMediaFiles', 'unitTestHelperFiles',
            'pythonFiles', 'compiledPythonFiles', 'localizationDataFiles', 'developmentOtherDataFiles', 'developmentTextDataFiles'];

        // Construct the map by calling each function defined above.
        var map = {};
        for (var i = 0; i < categories.length; i++) {
            var files = flatten(exports[categories[i]]());
            for (var j = 0; j < files.length; j++) {
                if (map[files[j]]) {
                    grunt.fail.warn("A file '" + files[j] + "' maps to category '" + categories[i] + "' in addition to '" + map[files[j]] + "' category.");
                } else {
                    map[files[j]] = categories[i];
                }
            }
        }
        return map;
    }
1368480
def format(file_metrics, build_metrics):
    """"""
    def indent(elem, level=0):
        i = "\n" + level*"  "
        if len(elem):
            if not elem.text or not elem.text.strip():
                elem.text = i + "  "
            if not elem.tail or not elem.tail.strip():
                elem.tail = i
            for elem in elem:
                indent(elem, level+1)
            if not elem.tail or not elem.tail.strip():
                elem.tail = i
        else:
            if level and (not elem.tail or not elem.tail.strip()):
                elem.tail = i

    root = ET.Element('metrics')

    # file_metrics
    files = ET.Element('files')
    root.append(files)

    for key in file_metrics.keys():
        tmp_file = ET.SubElement(files, "file",
                                 {'name': key, 'language': file_metrics[key]['language']})
        for name in file_metrics[key].keys():
            if name == 'language':
                continue
            tmp_metric = ET.SubElement(tmp_file, "metric",
                                       {'name': name, 'value': str(file_metrics[key][name])})

    # build_metrics
    if build_metrics:
        build = ET.Element('build')
        root.append(build)
        # TODO

    indent(root)
    if PY3:
        body = ET.tostring(root, encoding='unicode')
    else:
        body = ET.tostring(root)
    return body
490311
function translate(options, contents, copied, filePath) {
  var processed = {};
  for (var lang in dictionaries) {
    if (!processed[lang] && trueOrMatch(options.whitelist, lang) && !trueOrMatch(options.blacklist, lang)) {
      processed[lang] = '';
    }
  }
  if (!Object.keys(processed).length) {
    throw new Error('No translation dictionaries available to create any files!');
  }
  options.verbose && gutil.log('Starting translation for', processed.length, 'languages');
  if (trueOrMatch(options.ignoreTokens, filePath)) {
    options.verbose && gutil.log('Ignoring file', filePath, 'because of ignoreTokens option');
  } else if(isBinary(contents)) {
    options.verbose && gutil.log('Ignoring file', filePath, 'because file is binary');
  } else {
    contents = contents.toString('utf8');
    var i = contents.indexOf(options.delimiter.prefix);
    while ((i !== -1)) {
      var endMatch, length, token, key;
      var tail = contents.substr(i);
      if (options.delimiter.suffix) {
        endMatch = tail.match(options.delimiter.suffix);
        length = endMatch.index + endMatch[0].length;
        token = tail.substr(0, length);
        key = token.substr(options.delimiter.prefix.length, token.length - options.delimiter.prefix.length - options.delimiter.suffix.length);
      }
      else if (options.delimiter.stopCondition) {
        endMatch = tail.match(options.delimiter.stopCondition);
        length = endMatch == null ? tail.length : length = endMatch.index + endMatch[0].length - 1;
        token = tail.substr(0, length);
        key = token.substr(options.delimiter.prefix.length);
      }
      var next = contents.indexOf(options.delimiter.prefix, i + length + 1);

      for (var lang in processed) {
        processed[lang] += contents.substring(copied, i);
        if (dictionaries[lang][key] !== undefined) {
          if (trueOrMatch(options.encodeEntities, filePath)) {
            processed[lang] += he.encode(dictionaries[lang][key], { useNamedReferences: true });
          } else {
            processed[lang] += dictionaries[lang][key];
          }
        } else if (options.verbose || trueOrMatch(options.warn, filePath)) {
          gutil.log('Missing translation of language', lang, 'for key', key, 'in file', filePath);
        }
        processed[lang] += contents.substring(i + length, next == -1 ? contents.length : next);
      }
      copied = next;
      i = next;
    }
  }
  for (var procLang in processed) {
    if (!processed[procLang].length) {
      options.verbose && gutil.log('Copying original content to target language', procLang, 'because no replacements have happened');
      processed[procLang] = contents;
    }
  }
  return processed;
}
571360
function split({url, bucket, key, arn='', maxFiles=0, linesPerFile=500, maxLambdas=20, reverse=false, cb=null}) {

  let fileCounter = 0
  let lineCounter = 0
  const lineBuffer = new Buffer(4096)
  const gunzip = zlib.createGunzip()
  let newStream
  let currentFile
  let lineLength = 0
  let stopSplitting = false
  let header

  switch (url.substr(url.lastIndexOf('.') + 1)) {
    case 'csv':
      newStream = got.stream(url)
      break
    case 'gz':
      newStream = got.stream(url).pipe(gunzip)
      break
  }

  const build = function buildFile(line) {
    // get the csv header
    if (fileCounter === 0 && lineCounter === 0) header = line.toString()

    // create a new file or add to existing
    if (lineCounter === 0) {
      currentFile = new stream.PassThrough()
      currentFile.push(header)
    } else {
      currentFile.push(line.toString())
    }
    lineCounter += 1 // increment the filename
    lineLength = 0 // reset the buffer

    if (lineCounter > linesPerFile) {
      fileCounter += 1
      const fileName = `${key}${fileCounter}.csv`;
      const params = {
        Body: currentFile,
        Bucket: bucket,
        Key: fileName
      }
      currentFile.end()
      s3.upload(params, (e, d) => { if (e) console.log(e) })
      lineCounter = 0 // start counting the lines again
      if ((fileCounter) % 250 === 0 && fileCounter != 0) console.log(`uploaded ${fileCounter} files`)
      // sentinel csv is ordered from old to new so always have to go all the way back
      if ((fileCounter >= maxFiles) && maxFiles != 0 && !reverse) {
        stopSplitting = true
      }
    }
  }

  newStream.on('data', (data) => {
    if (!stopSplitting) {
      const dataLen = data.length
      for (let i = 0; i < dataLen; i++) {
        lineBuffer[lineLength] = data[i] // Buffer new line data.
        lineLength++
        if (data[i] === 10) { // Newline char was found.
          build(lineBuffer.slice(0, lineLength))
        }
      }
    }
  })

  newStream.on('end', () => {
    // write the last records
    if (lineCounter > 0) {
        fileCounter += 1
        const params = {
          Body: currentFile,
          Bucket: bucket,
          Key: `${key}${fileCounter}.csv`
        }
        s3.upload(params, (e, d) => { if (e) console.log(e) })
        currentFile.end()
    }
    console.log(`${fileCounter-1} total files`)
    // determine batches and run lambdas
    if (arn != '') {
      maxFiles = (maxFiles === 0) ? fileCounter : Math.min(maxFiles, fileCounter)
      var numLambdas = Math.min(maxFiles, maxLambdas)
      var batchSize = Math.floor(maxFiles / numLambdas)
      var extra = maxFiles % numLambdas
      var maxEndFile = reverse ? fileCounter : maxFiles
      
      var startFile = reverse ? fileCounter - maxFiles + 1: 1
      var endFile
      console.log(`Invoking ${numLambdas} batches of Lambdas of ${batchSize} files each with ${extra} extra (Files ${startFile}-${maxEndFile})`)
      for (var i = 0; i < numLambdas; i++) {
        endFile = (i < extra) ? startFile + batchSize: startFile + batchSize - 1
        invokeLambda(bucket, key, startFile, Math.min(endFile, maxEndFile), arn)
        startFile = endFile + 1
      }
    }
    cb()
  })
  newStream.on('error', e => cb(e))
}