For STScI GEIS files, need to do extra steps.
199874
def stsci(hdulist):
    """"""

    instrument = hdulist[0].header.get('INSTRUME', '')

    # Update extension header keywords
    if instrument in ("WFPC2", "FOC"):
        rootname = hdulist[0].header.get('ROOTNAME', '')
        filetype = hdulist[0].header.get('FILETYPE', '')
        for i in range(1, len(hdulist)):
            # Add name and extver attributes to match PyFITS data structure
            hdulist[i].name = filetype
            hdulist[i]._extver = i
            # Add extension keywords for this chip to extension
            hdulist[i].header['EXPNAME'] = (rootname, "9 character exposure identifier")
            hdulist[i].header['EXTVER']= (i, "extension version number")
            hdulist[i].header['EXTNAME'] = (filetype, "extension name")
            hdulist[i].header['INHERIT'] = (True, "inherit the primary header")
            hdulist[i].header['ROOTNAME'] = (rootname, "rootname of the observation set")
372445
def _add_secondary_if_exists(secondary, out, get_retriever):
    
    secondary = [_file_local_or_remote(y, get_retriever) for y in secondary]
    secondary = [z for z in secondary if z]
    if secondary:
        out["secondaryFiles"] = [{"class": "File", "path": f} for f in secondary]
    return out
19707
def _find_observable_paths(extra_files=None):
    """"""
    rv = set(
        os.path.dirname(os.path.abspath(x)) if os.path.isfile(x) else os.path.abspath(x)
        for x in sys.path
    )

    for filename in extra_files or ():
        rv.add(os.path.dirname(os.path.abspath(filename)))

    for module in list(sys.modules.values()):
        fn = getattr(module, "__file__", None)
        if fn is None:
            continue
        fn = os.path.abspath(fn)
        rv.add(os.path.dirname(fn))

    return _find_common_roots(rv)
859861
function setupByCopy() {
    let task_dir;
    let required_tasks;

    try {
        const config = require(path.join(process.cwd(), 'project.config'));
        required_tasks = config.required_tasks;
        if (!required_tasks) {
            throw Error('no task required.');
        }
        task_dir = config.dir.task;
        if (!task_dir) {
            throw Error('task directory not defined.');
        }
    } catch (error) {
        console.warn(error);
        process.exit(0);
    }

    const dstTaskDir = path.join(process.cwd(), task_dir);
    if (!fs.existsSync(dstTaskDir)) {
        // create tasks dir
        fs.mkdirSync(dstTaskDir);
    }

    required_tasks.forEach((task) => {
        const src = path.join(__dirname, 'tasks', task);
        const dst = path.join(dstTaskDir, task);
        if (fs.existsSync(src)) {
            if (fs.existsSync(dst)) {
                fs.unlinkSync(dst);
            }
            fs.writeFileSync(dst, fs.readFileSync(src).toString());
        } else {
            console.error('task not found: ' + task);
            process.exit(1);
        }
    });
}
1113167
def do_first(self):
        """"""
        pid = os.getpid()
        self.basename = os.path.join(self.tmpdir, 'iiif_netpbm_' + str(pid))
        outfile = self.basename + '.pnm'
        # Convert source file to pnm
        filetype = self.file_type(self.srcfile)
        if (filetype == 'png'):
            if (self.shell_call(self.pngtopnm + ' ' + self.srcfile + ' > ' + outfile)):
                raise IIIFError(text="Oops... got error from pngtopnm.")
        elif (filetype == 'jpg'):
            if (self.shell_call(self.jpegtopnm + ' ' + self.srcfile + ' > ' + outfile)):
                raise IIIFError(text="Oops... got error from jpegtopnm.")
        else:
            raise IIIFError(code='501',
                            text='bad input file format (only know how to read png/jpeg)')
        self.tmpfile = outfile
        # Get size
        (self.width, self.height) = self.image_size(self.tmpfile)
733641
def update(self):
        """"""
        pids = fileops.readlines(self.paths['cgroup.procs'])
        self.pids = [int(pid) for pid in pids if pid != '']
        self.n_procs = len(pids)
516579
def do_extra_polishing(self):
        '''
        
        '''
        for f in self.EXTRA_POLISH_FUNCTIONS:
            if not hasattr(f, 'polish_commit_indexes'):
                if hasattr(f, 'polish_urls') and self.URL in f.polish_urls:
                    f()

            if not hasattr(f, 'polish_urls'):
                if hasattr(f, 'polish_commit_indexes') and self.CURRENT_COMMIT_INDEX in f.polish_commit_indexes:
                    f()

            if hasattr(f, 'polish_commit_indexes') and hasattr(f, 'polish_urls'):
                if self.URL in f.polish_urls and self.CURRENT_COMMIT_INDEX in f.polish_commit_indexes:
                    f()
462332
def install_indicies
      Utils.logger
           .debug("Downloading index into production dir #{@dest_directory}")

      files = @files
      files.delete @quick_marshal_dir if files.include? @quick_dir

      if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
        files.delete @quick_marshal_dir
        dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
        FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose)
        FileUtils.rm_rf(dst_name, verbose: verbose)
        FileUtils.mv(@quick_marshal_dir, dst_name,
                     verbose: verbose, force: true)
      end

      files.each do |path|
        file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
        src_name = File.join(@directory, file)
        dst_name = File.join(@dest_directory, file)

        if ["#{@specs_index}.gz",
            "#{@latest_specs_index}.gz",
            "#{@prerelease_specs_index}.gz"].include?(path)
          res = build_zlib_file(file, src_name, dst_name, true)
          next unless res
        else
          source_content = download_from_source(file)
          next if source_content.nil?
          MirrorFile.new(dst_name).write(source_content)
        end

        FileUtils.rm_rf(path)
      end
    end
794000
def Move(self, units):
        '  '
        steps = units * self.SPU  # translate units to steps 
        if steps > 0:                                          # look for direction
            spi.SPI_write_byte(self.CS, 0x40 | (~self.Dir & 1))       
        else:
            spi.SPI_write_byte(self.CS, 0x40 | (self.Dir & 1)) 
        steps = int(abs(steps))     
        spi.SPI_write_byte(self.CS, (steps >> 16) & 0xFF)
        spi.SPI_write_byte(self.CS, (steps >> 8) & 0xFF)
        spi.SPI_write_byte(self.CS, steps & 0xFF)
1086845
def Import(context, request, instrumentname='sysmex_xs_500i'):
    
    # I don't really know how this file works, for this reason I added an 'Analysis Service selector'.
    # If non Analysis Service is selected, each 'data' column will be interpreted as a different Analysis Service. In
    # the case that an Analysis Service was selected, all the data columns would be interpreted as different data from
    # an unique Analysis Service.
    formitems = getForm(instrumentname, request)
    infile = formitems['infile']
    fileformat = formitems['fileformat']
    artoapply = formitems['artoapply']
    override = formitems['override']
    instrument = formitems['instrument']
    errors = []
    logs = []
    warns = []

    # Load the most suitable parser according to file extension/options/etc...
    parser = None
    if not hasattr(infile, 'filename'):
        errors.append(_("No file selected"))
    if fileformat == 'csv':
        # Get the Analysis Service selected, if there is one.
        analysis = request.form.get('analysis_service', None)
        if analysis:
            # Get default result key
            defaultresult = request.form.get('default_result', None)
            # Rise an error if default result is missing.
            parser = SysmexXS500iCSVParser(infile, analysis, defaultresult) if defaultresult \
                     else errors.append(t(_("You should introduce a default result key.",
                                             mapping={"fileformat": fileformat})))
        else:
            parser = SysmexXS500iCSVParser(infile)
    else:
        errors.append(t(_("Unrecognized file format ${fileformat}",
                          mapping={"fileformat": fileformat})))

    if parser:
        # Load the importer
        status = ['sample_received', 'attachment_due', 'to_be_verified']
        if artoapply == 'received':
            status = ['sample_received']
        elif artoapply == 'received_tobeverified':
            status = ['sample_received', 'attachment_due', 'to_be_verified']

        over = [False, False]
        if override == 'nooverride':
            over = [False, False]
        elif override == 'override':
            over = [True, False]
        elif override == 'overrideempty':
            over = [True, True]

        importer = SysmexXS500iImporter(parser=parser,
                                        context=context,
                                        allowed_ar_states=status,
                                        allowed_analysis_states=None,
                                        override=over,
                                        instrument_uid=instrument)
        tbex = ''
        try:
            importer.process()
        except:
            tbex = traceback.format_exc()
        errors = importer.errors
        logs = importer.logs
        warns = importer.warns
        if tbex:
            errors.append(tbex)

    results = {'errors': errors, 'log': logs, 'warns': warns}

    return json.dumps(results)
420228
def requires(self, extras=()):
        """"""
        dm = self._dep_map
        deps = []
        deps.extend(dm.get(None, ()))
        for ext in extras:
            try:
                deps.extend(dm[safe_extra(ext)])
            except KeyError:
                raise UnknownExtra(
                    "%s has no such extra feature %r" % (self, ext)
                )
        return deps
1170718
void doInfo(){
		this.infos.add("scanned directories:    " + this.scDir);
		this.infos.add("unreadable directories: " + this.scDirUnreadable);

		this.infos.add("found files:      " + this.scFiles);
		this.infos.add("unreadable files: " + this.scFilesUnreadable);
	}
531884
def _bsecurate_cli_component_file_refs(args):
    ''''''
    data = curate.component_file_refs(args.files)

    s = ''

    for cfile, cdata in data.items():
        s += cfile + '\n'
        rows = []
        for el, refs in cdata:
            rows.append(('    ' + el, ' '.join(refs)))
        s += '\n'.join(format_columns(rows)) + '\n\n'

    return s
460557
def run_steps(args: argparse.Namespace):
    """"""

    logging.basicConfig(level=logging.INFO, format="sockeye.autopilot: %(message)s")

    # (1) Establish task

    logging.info("=== Start Autopilot ===")
    # Listed task
    if args.task:
        task = TASKS[args.task]
        logging.info("Task: %s", task.description)
        logging.info("URL: %s", task.url)

        def report_data(file_sets):
            for file_set in file_sets:
                for fname in file_set[:2]:
                    logging.info("    %s", fname)

        logging.info("  Train:")
        report_data(task.train)
        logging.info("  Dev:")
        report_data(task.dev)
        logging.info("  Test:")
        report_data(task.test)
    # Custom task
    else:
        logging.info("Task: custom")
    # Source and target language codes
    lang_codes = (task.src_lang, task.trg_lang) if args.task else args.custom_lang

    # (2) Establish workspace and task directories

    logging.info("=== Establish working directories ===")
    logging.info("Workspace: %s", args.workspace)
    special_fname = os.path.join(args.workspace, FILE_WORKSPACE)
    if not os.path.exists(args.workspace):
        logging.info("Create: %s", args.workspace)
        os.makedirs(args.workspace)
        touch_file(special_fname)
    else:
        if not os.path.exists(special_fname):
            raise RuntimeError("Directory %s exists but %s does not, stopping to avoid overwriting files in non-workspace directory"
                            % (args.workspace, special_fname))

    dir_third_party = os.path.join(args.workspace, third_party.DIR_THIRD_PARTY)
    dir_cache = os.path.join(args.workspace, DIR_CACHE)
    dir_logs = os.path.join(args.workspace, DIR_LOGS)
    dir_systems = os.path.join(args.workspace, DIR_SYSTEMS)
    task_name = args.task if args.task else args.custom_task
    if args.test:
        task_name += SUFFIX_TEST
    dir_task = os.path.join(dir_systems, task_name)
    for dirname in (dir_third_party, dir_cache, dir_logs, dir_systems, dir_task):
        if os.path.exists(dirname):
            logging.info("Exists: %s", dirname)
        else:
            logging.info("Create: %s", dirname)
            os.makedirs(dirname)

    # (3) Checkout necessary tools

    logging.info("=== Checkout third-party tools ===")
    # Requires tokenization?
    if args.task or args.custom_text_type == CUSTOM_UTF8_RAW:
        third_party.checkout_moses_tokenizer(args.workspace)
    # Requires byte-pair encoding?
    if args.task or args.custom_text_type in (CUSTOM_UTF8_RAW, CUSTOM_UTF8_TOK):
        third_party.checkout_subword_nmt(args.workspace)

    # (4) Populate train/dev/test data

    # This step also normalizes whitespace on data population or copy, ensuring
    # that for all input data, only ASCII newlines are considered line breaks.
    logging.info("=== Populate train/dev/test data ===")
    step_dir_raw = os.path.join(dir_task, DIR_DATA, DIR_RAW)
    complete_fname = os.path.join(step_dir_raw, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_raw)
    else:
        # Listed task
        if args.task:
            raw_files = identify_raw_files(task, test_mode=args.test)
            with tempfile.TemporaryDirectory(prefix="raw.", dir=dir_task) as raw_dir:
                # Download (or locate in cache) and extract raw files to temp directory
                logging.info("=== Download and extract raw files ===")
                download_extract_raw_files(raw_files, dir_cache, raw_dir)
                # Copy required files to train/dev/test
                logging.info("=== Create input data files ===")
                renew_step_dir(step_dir_raw)
                # Test mode uses the full test set as training data and the
                # first line of the test set as dev and test data
                populate_parallel_text(raw_dir,
                                       task.test if args.test else task.train,
                                       os.path.join(step_dir_raw, PREFIX_TRAIN),
                                       False)
                populate_parallel_text(raw_dir,
                                       task.test if args.test else task.dev,
                                       os.path.join(step_dir_raw, PREFIX_DEV),
                                       False,
                                       head_n=1 if args.test else 0)
                populate_parallel_text(raw_dir,
                                       task.test,
                                       os.path.join(step_dir_raw, PREFIX_TEST),
                                       True,
                                       head_n=1 if args.test else 0)
        # Custom task
        else:
            logging.info("=== Copy input data files ===")
            renew_step_dir(step_dir_raw)
            copy_parallel_text(args.custom_train, os.path.join(step_dir_raw, PREFIX_TRAIN))
            copy_parallel_text(args.custom_dev, os.path.join(step_dir_raw, PREFIX_DEV))
            copy_parallel_text(args.custom_test, os.path.join(step_dir_raw, PREFIX_TEST))
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_raw)

    # (5) Tokenize train/dev/test data

    # Task requires tokenization if _any_ raw file is not already tokenized
    requires_tokenization = False
    if args.task:
        for file_sets in (task.train, task.dev, task.test):
            for _, _, text_type in file_sets:
                if text_type in TEXT_REQUIRES_TOKENIZATION:
                    requires_tokenization = True
    else:
        if args.custom_text_type == CUSTOM_UTF8_RAW:
            requires_tokenization = True
    logging.info("=== Tokenize train/dev/test data ===")
    step_dir_tok = os.path.join(dir_task, DIR_DATA, DIR_TOK)
    complete_fname = os.path.join(step_dir_tok, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_tok)
    else:
        renew_step_dir(step_dir_tok)

        # Tokenize each data file using the appropriate language code OR link
        # raw file if already tokenized.
        for fname in os.listdir(step_dir_raw):
            if fname.startswith("."):
                continue
            input_fname = os.path.join(step_dir_raw, fname)
            output_fname = os.path.join(step_dir_tok, fname)
            if requires_tokenization:
                lang_code = lang_codes[0] if fname.endswith(SUFFIX_SRC_GZ) else lang_codes[1]
                logging.info("Tokenize (%s): %s -> %s", lang_code, input_fname, output_fname)
                third_party.call_moses_tokenizer(workspace_dir=args.workspace,
                                                 input_fname=input_fname,
                                                 output_fname=output_fname,
                                                 lang_code=lang_code)
            else:
                logging.info("Link pre-tokenized: %s -> %s", input_fname, output_fname)
                os.symlink(os.path.join("..", DIR_RAW, fname), output_fname)
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_tok)

    # (6) Learn byte-pair encoding model

    # Task requires byte-pair encoding unless using pre-encoded custom data
    skip_bpe = (not args.task) and args.custom_text_type == CUSTOM_UTF8_BPE
    logging.info("=== Learn byte-pair encoding model ===")
    step_dir_bpe_model = os.path.join(dir_task, DIR_BPE_MODEL)
    complete_fname = os.path.join(step_dir_bpe_model, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_bpe_model)
    else:
        renew_step_dir(step_dir_bpe_model)
        if skip_bpe:
            logging.info("BPE model not required for pre-encoded data")
        else:
            source_fname = os.path.join(step_dir_tok, PREFIX_TRAIN + SUFFIX_SRC_GZ)
            target_fname = os.path.join(step_dir_tok, PREFIX_TRAIN + SUFFIX_TRG_GZ)
            codes_fname = os.path.join(step_dir_bpe_model, FILE_BPE_CODES)
            num_ops = task.bpe_op if args.task else args.custom_bpe_op
            if args.test:
                num_ops = TEST_BPE_OPS
            logging.info("BPE Learn (%s): %s + %s -> %s", num_ops, source_fname, target_fname, codes_fname)
            third_party.call_learn_bpe(workspace_dir=args.workspace,
                                       source_fname=source_fname,
                                       target_fname=target_fname,
                                       model_fname=codes_fname,
                                       num_ops=num_ops)
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_bpe_model)

    # (7) Byte-pair encode data

    logging.info("=== Byte-pair encode train/dev/test data ===")
    step_dir_bpe = os.path.join(dir_task, DIR_DATA, DIR_BPE)
    complete_fname = os.path.join(step_dir_bpe, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_bpe)
    else:
        renew_step_dir(step_dir_bpe)
        # Encode each data file
        for fname in os.listdir(step_dir_tok):
            if fname.startswith("."):
                continue
            input_fname = os.path.join(step_dir_tok, fname)
            output_fname = os.path.join(step_dir_bpe, fname)
            if skip_bpe:
                logging.info("Link pre-encoded: %s -> %s", input_fname, output_fname)
                os.symlink(os.path.join("..", DIR_TOK, fname), output_fname)
            else:
                codes_fname = os.path.join(step_dir_bpe_model, FILE_BPE_CODES)
                logging.info("BPE: %s -> %s", input_fname, output_fname)
                third_party.call_apply_bpe(workspace_dir=args.workspace,
                                           input_fname=input_fname,
                                           output_fname=output_fname,
                                           model_fname=codes_fname)
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_bpe)

    # Done if only running data preparation steps
    if args.model == MODEL_NONE:
        return

    # (8) Run Sockeye training

    logging.info("=== Train translation model ===")
    logging.info("Model: %s", args.model)
    if args.model == MODEL_GNMT:
        logging.info("NOTE: This is an 8 layer LSTM model similar (but not exactly identical) to the 'GNMT' architecture.")
    step_dir_model = os.path.join(dir_task, DIR_PREFIX_MODEL + args.model)
    complete_fname = os.path.join(step_dir_model, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_model)
    else:
        log_fname = os.path.join(args.workspace,
                                 DIR_LOGS,
                                 "sockeye.{{}}.{}.{}.{}.log".format(task_name, args.model, os.getpid()))
        call_sockeye_train(args.model,
                           step_dir_bpe,
                           step_dir_model,
                           log_fname.format("train"),
                           args.gpus,
                           test_mode=args.test)
        call_sockeye_average(step_dir_model, log_fname.format("average"))
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_model)

    # (9) Decode test sets

    logging.info("=== Decode test sets ===")
    logging.info("Settings: %s", args.decode_settings)
    step_dir_results = os.path.join(dir_task, DIR_RESULTS)
    if not os.path.exists(step_dir_results):
        logging.info("Create: %s", step_dir_results)
        os.makedirs(step_dir_results)
    # To collect BPE output names
    output_fnames_bpe = []
    # For each test file
    for fname in os.listdir(step_dir_bpe):
        if fname.startswith(PREFIX_TEST) and fname.endswith(SUFFIX_SRC_GZ):
            input_fname = os.path.join(step_dir_bpe, fname)
            # /path/to/results/test[.N].<model>.<settings>
            output_fname = os.path.join(step_dir_results, "{}.{}.{}.{}".format(args.model,
                                                                               args.decode_settings,
                                                                               fname[:-len(SUFFIX_SRC_GZ) - 1],
                                                                               SUFFIX_BPE))
            output_fnames_bpe.append(output_fname)
            # For the shared results directory, a command file indicates that
            # the step has completed successfully.
            command_fname = output_fname + "." + SUFFIX_COMMAND
            if os.path.exists(command_fname):
                logging.info("Re-use output: %s", output_fname)
            else:
                log_fname = os.path.join(args.workspace,
                                 DIR_LOGS,
                                 "sockeye.translate.{}.{}.{}.{}.log".format(task_name,
                                                                            args.model,
                                                                            fname[:-len(SUFFIX_SRC_GZ) - 1],
                                                                            os.getpid()))
                call_sockeye_translate(args=DECODE_ARGS[args.decode_settings],
                                       input_fname=input_fname,
                                       output_fname=output_fname,
                                       model_dir=step_dir_model,
                                       log_fname=log_fname,
                                       use_cpu=(args.gpus == 0))

    # (10) Evaluate test sets (bpe/tok/detok)

    lang_code = lang_codes[1] if lang_codes else None
    logging.info("=== Score outputs ===")
    # For each output file
    for fname_bpe in output_fnames_bpe:
        # Score byte-pair encoded
        fname_base = os.path.basename(fname_bpe)[:-len(SUFFIX_BPE)].split(".", 2)[2]
        fname_ref_bpe = os.path.join(step_dir_bpe, fname_base + SUFFIX_TRG_GZ)
        fname_bleu_bpe = fname_bpe + "." + SUFFIX_BLEU
        if os.path.exists(fname_bleu_bpe):
            logging.info("Re-use output: %s", fname_bleu_bpe)
        else:
            fname_log = os.path.join(args.workspace,
                                     DIR_LOGS,
                                     "sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
                                                                                  args.model,
                                                                                  fname_base + SUFFIX_BPE,
                                                                                  os.getpid()))
            call_sacrebleu(input_fname=fname_bpe,
                           ref_fname=fname_ref_bpe,
                           output_fname=fname_bleu_bpe,
                           log_fname=fname_log,
                           tokenized=True)
        # Score tokenized
        fname_tok = fname_bpe[:-len(SUFFIX_BPE)] + SUFFIX_TOK
        fname_ref_tok = os.path.join(step_dir_tok, fname_base + SUFFIX_TRG_GZ)
        fname_bleu_tok = fname_tok + "." + SUFFIX_BLEU
        if os.path.exists(fname_bleu_tok):
            logging.info("Re-use output: %s", fname_bleu_tok)
        else:
            # Merge BPE
            logging.info("Merge BPE: %s -> %s", fname_bpe, fname_tok)
            third_party.merge_bpe(input_fname=fname_bpe, output_fname=fname_tok)
            fname_log = os.path.join(args.workspace,
                                     DIR_LOGS,
                                     "sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
                                                                                  args.model,
                                                                                  fname_base + SUFFIX_TOK,
                                                                                  os.getpid()))
            call_sacrebleu(input_fname=fname_tok,
                           ref_fname=fname_ref_tok,
                           output_fname=fname_bleu_tok,
                           log_fname=fname_log,
                           tokenized=True)
        # Score detokenized (WMT-compatible BLEU)
        fname_detok = fname_bpe[:-len(SUFFIX_BPE)] + SUFFIX_DETOK
        fname_ref_raw = os.path.join(step_dir_raw, fname_base + SUFFIX_TRG_GZ)
        fname_bleu_detok = fname_detok + "." + SUFFIX_SACREBLEU
        if os.path.exists(fname_bleu_detok):
            logging.info("Re-use output: %s", fname_bleu_detok)
        else:
            if not requires_tokenization:
                logging.info(
                    "WARNING: Task uses pre-tokenized data, cannot reliably detokenize to compute WMT-compatible scores")
                continue
            # Detokenize
            logging.info("Detokenize (%s): %s -> %s", lang_code, fname_tok, fname_detok)
            third_party.call_moses_detokenizer(workspace_dir=args.workspace,
                                               input_fname=fname_tok,
                                               output_fname=fname_detok,
                                               lang_code=lang_code)
            fname_log = os.path.join(args.workspace,
                                     DIR_LOGS,
                                     "sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
                                                                                  args.model,
                                                                                  fname_base + SUFFIX_DETOK,
                                                                                  os.getpid()))
            call_sacrebleu(input_fname=fname_detok,
                           ref_fname=fname_ref_raw,
                           output_fname=fname_bleu_detok,
                           log_fname=fname_log,
                           tokenized=False)
1208841
def _augment_observation_files(self, e):
        
        e.file_records = [self._augment_file(f) for f in e.file_records]
        return e
317549
function bootrapSitesAndComponents() {
  const sites = siteService.sites(),
    sitesArray = _.map(Object.keys(sites), site => sites[site]);

  return highland(sitesArray)
    .flatMap(site => {
      return highland(
        bootstrapComponents(site)
          .then(function () {
            return site;
          })
          .catch(function () {
            log('debug', `Errors bootstrapping components for site: ${site.slug}`);
            return site;
          })
      );
    })
    .flatMap(site => {
      return highland(
        bootstrapPath(site.dir, site)
          .catch(function () {
            log('debug', `No bootstrap file found for site: ${site.slug}`);
            return site;
          })
      );
    })
    .collect()
    .toPromise(bluebird);
}
1126647
def inject_into_files(scheme_file, files):
    """"""
    for file_ in files:
        rec = Recipient(file_)
        colorscheme = rec.get_colorscheme(scheme_file)
        rec.inject_scheme(colorscheme)
        rec.write()