Create all the steps that will be part of this task
431640
public function build() {

        // Define the task contextid (the course one)
        $this->contextid = context_course::instance($this->get_courseid())->id;

        // Executed conditionally if restoring to new course or if overwrite_conf setting is enabled
        if ($this->get_target() == backup::TARGET_NEW_COURSE || $this->get_setting_value('overwrite_conf') == true) {
            $this->add_step(new restore_course_structure_step('course_info', 'course.xml'));

            // Search reindexing (if enabled).
            if (\core_search\manager::is_indexing_enabled()) {
                $this->add_step(new restore_course_search_index('course_search_index'));
            }
        }

        $this->add_step(new restore_course_legacy_files_step('legacy_files'));

        // Deal with enrolment methods and user enrolments.
        if ($this->plan->get_mode() == backup::MODE_IMPORT) {
            // No need to do anything with enrolments.

        } else if (!$this->get_setting_value('users') or $this->plan->get_mode() == backup::MODE_HUB) {
            if ($this->get_setting_value('enrolments') == backup::ENROL_ALWAYS && $this->plan->get_mode() != backup::MODE_HUB) {
                // Restore enrolment methods.
                $this->add_step(new restore_enrolments_structure_step('course_enrolments', 'enrolments.xml'));
            } else if ($this->get_target() == backup::TARGET_CURRENT_ADDING or $this->get_target() == backup::TARGET_EXISTING_ADDING) {
                // Keep current enrolments unchanged.
            } else {
                // If no instances yet add default enrol methods the same way as when creating new course in UI.
                $this->add_step(new restore_default_enrolments_step('default_enrolments'));
            }

        } else {
            // Restore course enrolment data.
            $this->add_step(new restore_enrolments_structure_step('course_enrolments', 'enrolments.xml'));
        }

        // Populate groups, this must be done after enrolments because only enrolled users may be in groups.
        $this->add_step(new restore_groups_members_structure_step('create_groups_members', '../groups.xml'));

        // Restore course role assignments and overrides (internally will observe the role_assignments setting),
        // this must be done after all users are enrolled.
        $this->add_step(new restore_ras_and_caps_structure_step('course_ras_and_caps', 'roles.xml'));

        // Restore course filters (conditionally)
        if ($this->get_setting_value('filters')) {
            $this->add_step(new restore_filters_structure_step('course_filters', 'filters.xml'));
        }

        // Restore course comments (conditionally)
        if ($this->get_setting_value('comments')) {
            $this->add_step(new restore_comments_structure_step('course_comments', 'comments.xml'));
        }

        // Calendar events (conditionally)
        if ($this->get_setting_value('calendarevents')) {
            $this->add_step(new restore_calendarevents_structure_step('course_calendar', 'calendar.xml'));
        }

        // Course competencies.
        $this->add_step(new restore_course_competencies_structure_step('course_competencies', 'competencies.xml'));

        // Activity completion defaults.
        $this->add_step(new restore_completion_defaults_structure_step('course_completion_defaults', 'completiondefaults.xml'));

        // At the end, mark it as built
        $this->built = true;
    }
688274
def run(self, steps=None, resume=False, redo=None):
        

        recipe = {
            "name"      :   self.name,
            "steps"     :   []
        }
        start_at = 0

        if redo:
            recipe = utils.readJson(redo)
            self.log.info('Rerunning recipe {0} from {1}'.format(recipe['name'], redo))
            self.log.info('Recreating recipe instance..')
            self.jobs = []
            for step in recipe['steps']:
                
            #        add I/O folders to the json file
            #        add a string describing the contents of these folders
            #        The user has to ensure that these folders exist, and have the required content
                if step['jtype'] == 'docker':
                    self.log.info('Adding job \'{0}\' to recipe. The container will be named \'{1}\''.format(step['cab'], step['name']))
                    cont = docker.Container(step['cab'], step['name'],
                                          label=step['label'], logger=self.log,
                                          shared_memory=step['shared_memory'])

                    self.log.debug('Adding volumes {0} and environmental variables {1}'.format(step['volumes'], step['environs']))
                    cont.volumes = step['volumes']
                    cont.environs = step['environs']
                    cont.shared_memory = step['shared_memory']
                    cont.input_content = step['input_content']
                    cont.msdir_content = step['msdir_content']
                    cont.logfile = step['logfile']
                    job = StimelaJob(step['name'], recipe=self, label=step['label'])
                    job.job = cont
                    job.jtype = 'docker'


                elif step['jtype'] == 'function':
                    name = step['name']
                    func = inspect.currentframe().f_back.f_locals[step['function']]
                    job = StimelaJob(name, recipe=self, label=step['label'])
                    job.python_job(func, step['parameters'])
                    job.jtype = 'function'

                self.jobs.append(job)

        elif resume:
            self.log.info("Resuming recipe from last run.")
            try:
                recipe = utils.readJson(self.resume_file)
            except IOError:
                raise StimelaRecipeExecutionError("Cannot resume pipeline, resume file '{}' not found".format(self.resume_file))

            steps_ = recipe.pop('steps')
            recipe['steps'] = []
            _steps = []
            for step in steps_:
                if step['status'] == 'completed':
                    recipe['steps'].append(step)
                    continue

                label = step['label']
                number = step['number']
                
                # Check if the recipe flow has changed
                if label == self.jobs[number-1].label:
                    self.log.info('recipe step \'{0}\' is fit for re-execution. Label = {1}'.format(number, label))
                    _steps.append(number)
                else:
                    raise StimelaRecipeExecutionError('Recipe flow, or task scheduling has changed. Cannot resume recipe. Label = {0}'.format(label))

            # Check whether there are steps to resume        
            if len(_steps)==0:
                self.log.info('All the steps were completed. No steps to resume')
                sys.exit(0)
            steps = _steps

        if getattr(steps, '__iter__', False):
            _steps = []
            if isinstance(steps[0], str):
                labels = [ job.label.split('::')[0] for job in self.jobs]

                for step in steps:
                    try:
                        _steps.append(labels.index(step)+1)
                    except ValueError:
                        raise StimelaCabParameterError('Recipe label ID [{0}] doesn\'t exist'.format(step))
                steps = _steps
        else:
            steps = range(1, len(self.jobs)+1)

        jobs = [(step, self.jobs[step-1]) for step in steps]
        
        for i, (step, job) in enumerate(jobs):

            self.log.info('Running job {}'.format(job.name))
            self.log.info('STEP {0} :: {1}'.format(i+1, job.label))
            self.active = job
            try:
                if job.jtype == 'function':
                    job.run_python_job()
                elif job.jtype in ['docker', 'singularity']:
                    with open(job.job.logfile, 'a') as astd:
                        astd.write('\n-----------------------------------\n')
                        astd.write('Stimela version     : {}\n'.format(version.version))
                        astd.write('Cab name            : {}\n'.format(job.job.image))
                        astd.write('-------------------------------------\n')

                    run_job = getattr(job, "run_{0:s}_job".format(job.jtype))
                    run_job()
                    

                self.log2recipe(job, recipe, step, 'completed')

            except (utils.StimelaCabRuntimeError,
                   StimelaRecipeExecutionError,
                   StimelaCabParameterError) as e:
                self.completed = [jb[1] for jb in jobs[:i]]
                self.remaining = [jb[1] for jb in jobs[i+1:]]
                self.failed = job

                self.log.info('Recipe execution failed while running job {}'.format(job.name))
                self.log.info('Completed jobs : {}'.format([c.name for c in self.completed]))
                self.log.info('Remaining jobs : {}'.format([c.name for c in self.remaining]))

                self.log2recipe(job, recipe, step, 'failed')
                for step, jb in jobs[i+1:]:
                    self.log.info('Logging remaining task: {}'.format(jb.label))
                    self.log2recipe(jb, recipe, step, 'remaining')

                self.log.info('Saving pipeline information in {}'.format(self.resume_file))
                utils.writeJson(self.resume_file, recipe)

                pe = PipelineException(e, self.completed, job, self.remaining)
                raise_(pe, None, sys.exc_info()[2])
            except:
                import traceback
                traceback.print_exc()
                raise RuntimeError("An unhandled exception has occured. This is a bug, please report")

            finally:
                if job.jtype == 'docker' and job.created:
                    job.job.stop()
                    job.job.remove()
                if job.jtype == 'singularity' and job.created:
                    job.job.stop()

        self.log.info('Saving pipeline information in {}'.format(self.resume_file))
        utils.writeJson(self.resume_file, recipe)

        self.log.info('Recipe executed successfully')
        
        return 0
1319853
public function build() {
        global $CFG;

        // Set the backup::VAR_CONTEXTID setting to course context as far as next steps require that
        $coursectxid = context_course::instance($this->get_courseid())->id;
        $this->add_setting(new backup_activity_generic_setting(backup::VAR_CONTEXTID, base_setting::IS_INTEGER, $coursectxid));

        // Set the backup::VAR_COURSEID setting to course, we'll need that in some steps
        $courseid = $this->get_courseid();
        $this->add_setting(new backup_activity_generic_setting(backup::VAR_COURSEID, base_setting::IS_INTEGER, $courseid));

        // Generate the groups file with the final annotated groups and groupings
        // including membership based on setting
        $this->add_step(new backup_groups_structure_step('groups', 'groups.xml'));

        // Generate the questions file with the final annotated question_categories
        $this->add_step(new backup_questions_structure_step('questions', 'questions.xml'));

        // Annotate all the question files for the already annotated question
        // categories (this is performed here and not in the structure step because
        // it involves multiple contexts and as far as we are always backup-ing
        // complete question banks we don't need to restrict at all and can be
        // done in a single pass
        $this->add_step(new backup_annotate_all_question_files('question_files'));

        // Annotate all the user files (conditionally) (profile and icon files)
        // Because each user has its own context, we need a separate/specialised step here
        // This step also ensures that the contexts for all the users exist, so next
        // step can be safely executed (join between users and contexts)
        // Not executed if backup is without users of anonymized
        if ($this->get_setting_value('users') && !$this->get_setting_value('anonymize')) {
            $this->add_step(new backup_annotate_all_user_files('user_files'));
        }

        // Generate the users file (conditionally) with the final annotated users
        // including custom profile fields, preferences, tags, role assignments and
        // overrides
        if ($this->get_setting_value('users')) {
            $this->add_step(new backup_users_structure_step('users', 'users.xml'));
        }

        // Generate the top roles file with all the final annotated roles
        // that have been detected along the whole process. It's just
        // the list of role definitions (no assignments nor permissions)
        $this->add_step(new backup_final_roles_structure_step('roleslist', 'roles.xml'));

        // Generate the gradebook file with categories and course grade items. Do it conditionally, using
        // execute_condition() so only will be excuted if ALL module grade_items in course have been exported
        $this->add_step(new backup_gradebook_structure_step('course_gradebook','gradebook.xml'));

        // Generate the grade history file, conditionally.
        $this->add_step(new backup_grade_history_structure_step('course_grade_history','grade_history.xml'));

        // Generate the course completion
        $this->add_step(new backup_course_completion_structure_step('course_completion', 'completion.xml'));

        // Conditionally generate the badges file.
        if ($this->get_setting_value('badges')) {
            $this->add_step(new backup_badges_structure_step('course_badges', 'badges.xml'));
        }

        // Generate the scales file with all the (final) annotated scales
        $this->add_step(new backup_final_scales_structure_step('scaleslist', 'scales.xml'));

        // Generate the outcomes file with all the (final) annotated outcomes
        $this->add_step(new backup_final_outcomes_structure_step('outcomeslist', 'outcomes.xml'));

        // Migrate the pending annotations to final (prev steps may have added some files)
        // This must be executed before backup files
        $this->add_step(new move_inforef_annotations_to_final('migrate_inforef'));

        // Generate the files.xml file with all the (final) annotated files. At the same
        // time copy all the files from moodle storage to backup storage (uses custom
        // backup_nested_element for that)
        $this->add_step(new backup_final_files_structure_step('fileslist', 'files.xml'));

        // Write the main moodle_backup.xml file, with all the information related
        // to the backup, settings, license, versions and other useful information
        $this->add_step(new backup_main_structure_step('mainfile', 'moodle_backup.xml'));

        require_once($CFG->dirroot . '/backup/util/helper/convert_helper.class.php');

        // Look for converter steps only in type course and mode general backup operations.
        $conversion = false;
        if ($this->plan->get_type() == backup::TYPE_1COURSE and $this->plan->get_mode() == backup::MODE_GENERAL) {
            $converters = convert_helper::available_converters(false);
            foreach ($converters as $value) {
                if ($this->get_setting_value($value)) {
                    // Zip class.
                    $zip_contents      = "{$value}_zip_contents";
                    $store_backup_file = "{$value}_store_backup_file";
                    $convert           = "{$value}_backup_convert";

                    $this->add_step(new $convert("package_convert_{$value}"));
                    $this->add_step(new $zip_contents("zip_contents_{$value}"));
                    $this->add_step(new $store_backup_file("save_backupfile_{$value}"));
                    if (!$conversion) {
                        $conversion = true;
                    }
                }
            }
        }

        // On backup::MODE_IMPORT, we don't have to zip nor store the the file, skip these steps
        if (($this->plan->get_mode() != backup::MODE_IMPORT) && !$conversion) {
            // Generate the zip file (mbz extension)
            $this->add_step(new backup_zip_contents('zip_contents'));

            // Copy the generated zip (.mbz) file to final destination
            $this->add_step(new backup_store_backup_file('save_backupfile'));
        }

        // Clean the temp dir (conditionally) and drop temp tables
        $cleanstep = new drop_and_clean_temp_stuff('drop_and_clean_temp_stuff');
        // Decide about to delete the temp dir (based on backup::MODE_IMPORT)
        $cleanstep->skip_cleaning_temp_dir($this->plan->get_mode() == backup::MODE_IMPORT);
        $this->add_step($cleanstep);

        $this->built = true;
    }
1114482
def nf_step_to_process(step, out_handle):
    
    pprint.pprint(step)

    directives = []
    for req in step["task_definition"]["requirements"]:
        if req["requirement_type"] == "docker":
            directives.append("container '%s'" % req["value"])
        elif req["requirement_type"] == "cpu":
            directives.append("cpus %s" % req["value"])
        elif req["requirement_type"] == "memory":
            directives.append("memory '%s'" % req["value"])

    task_id = step["task_id"]
    directives = "\n  ".join(directives)
    inputs = "\n    ".join(nf_io_to_process(step["inputs"], step["task_definition"]["inputs"],
                                        step["scatter"]))
    outputs = "\n    ".join(nf_io_to_process(step["outputs"], step["task_definition"]["outputs"]))
    commandline = (step["task_definition"]["baseCommand"] + " " +
                   " ".join([nf_input_to_cl(i) for i in step["task_definition"]["inputs"]]))
    out_handle.write(_nf_process_tmpl.format(**locals()))
967757
public function build() {

        // Move all the CONTEXT_MODULE question qcats to their
        // final (newly created) module context
        $this->add_step(new restore_move_module_questions_categories('move_module_question_categories'));

        // Create all the question files now that every question is in place
        // and every category has its final contextid associated
        $this->add_step(new restore_create_question_files('create_question_files'));

        // Review all the block_position records in backup_ids in order
        // match them now that all the contexts are created populating DB
        // as needed. Only if we are restoring blocks.
        if ($this->get_setting_value('blocks')) {
            $this->add_step(new restore_review_pending_block_positions('review_block_positions'));
        }

        // Gradebook. Don't restore the gradebook unless activities are being restored.
        if ($this->get_setting_value('activities')) {
            $this->add_step(new restore_gradebook_structure_step('gradebook_step','gradebook.xml'));
            $this->add_step(new restore_grade_history_structure_step('grade_history', 'grade_history.xml'));
        }

        // Course completion.
        $this->add_step(new restore_course_completion_structure_step('course_completion', 'completion.xml'));

        // Conditionally restore course badges.
        if ($this->get_setting_value('badges')) {
            $this->add_step(new restore_badges_structure_step('course_badges', 'badges.xml'));
        }

        // Review all the legacy module_availability records in backup_ids in
        // order to match them with existing modules / grade items and convert
        // into the new system.
        $this->add_step(new restore_process_course_modules_availability('process_modules_availability'));

        // Update restored availability data to account for changes in IDs
        // during backup/restore.
        $this->add_step(new restore_update_availability('update_availability'));

        // Refresh action events conditionally.
        if ($this->get_setting_value('activities')) {
            $this->add_step(new restore_calendar_action_events('restoring_action_events'));
        }

        // Decode all the interlinks
        $this->add_step(new restore_decode_interlinks('decode_interlinks'));

        // Restore course logs (conditionally). They are restored here because we need all
        // the activities to be already restored.
        if ($this->get_setting_value('logs')) {
            // Legacy logs.
            $this->add_step(new restore_course_logs_structure_step('course_logs', 'course/logs.xml'));
            // New log stores.
            $this->add_step(new restore_course_logstores_structure_step('course_logstores', 'course/logstores.xml'));
        }

        // Review all the executed tasks having one after_restore method
        // executing it to perform some final adjustments of information
        // not available when the task was executed.
        // This step is always the last one performing modifications on restored information
        // Don't add any new step after it. Only aliases queue, cache rebuild and clean are allowed.
        $this->add_step(new restore_execute_after_restore('executing_after_restore'));

        // All files were sent to the filepool by now. We need to process
        // the aliases yet as they were not actually created but stashed for us instead.
        // We execute this step after executing_after_restore so that there can't be no
        // more files sent to the filepool after this.
        $this->add_step(new restore_process_file_aliases_queue('process_file_aliases_queue'));

        // Rebuild course cache to see results, whoah!
        $this->add_step(new restore_rebuild_course_cache('rebuild_course_cache'));

        // Clean the temp dir (conditionally) and drop temp table
        $this->add_step(new restore_drop_and_clean_temp_stuff('drop_and_clean_temp_stuff'));

        // If restoring to a new course or overwriting config, reindex the whole course.
        if (\core_search\manager::is_indexing_enabled()) {
            $wholecourse = $this->get_target() == backup::TARGET_NEW_COURSE;
            $wholecourse = $wholecourse || $this->setting_exists('overwrite_conf') && $this->get_setting_value('overwrite_conf');
            if ($wholecourse) {
                $this->add_step(new restore_course_search_index('course_search_index'));
            }
        }

        $this->built = true;
    }
634713
def steps(self):
        """"""
        def _iter(uri, acc):
            acc.appendleft(uri.name if uri.name else '')
            return _iter(uri.parent, acc) if uri.parent else acc
        return _iter(self, acc=deque())
460557
def run_steps(args: argparse.Namespace):
    """"""

    logging.basicConfig(level=logging.INFO, format="sockeye.autopilot: %(message)s")

    # (1) Establish task

    logging.info("=== Start Autopilot ===")
    # Listed task
    if args.task:
        task = TASKS[args.task]
        logging.info("Task: %s", task.description)
        logging.info("URL: %s", task.url)

        def report_data(file_sets):
            for file_set in file_sets:
                for fname in file_set[:2]:
                    logging.info("    %s", fname)

        logging.info("  Train:")
        report_data(task.train)
        logging.info("  Dev:")
        report_data(task.dev)
        logging.info("  Test:")
        report_data(task.test)
    # Custom task
    else:
        logging.info("Task: custom")
    # Source and target language codes
    lang_codes = (task.src_lang, task.trg_lang) if args.task else args.custom_lang

    # (2) Establish workspace and task directories

    logging.info("=== Establish working directories ===")
    logging.info("Workspace: %s", args.workspace)
    special_fname = os.path.join(args.workspace, FILE_WORKSPACE)
    if not os.path.exists(args.workspace):
        logging.info("Create: %s", args.workspace)
        os.makedirs(args.workspace)
        touch_file(special_fname)
    else:
        if not os.path.exists(special_fname):
            raise RuntimeError("Directory %s exists but %s does not, stopping to avoid overwriting files in non-workspace directory"
                            % (args.workspace, special_fname))

    dir_third_party = os.path.join(args.workspace, third_party.DIR_THIRD_PARTY)
    dir_cache = os.path.join(args.workspace, DIR_CACHE)
    dir_logs = os.path.join(args.workspace, DIR_LOGS)
    dir_systems = os.path.join(args.workspace, DIR_SYSTEMS)
    task_name = args.task if args.task else args.custom_task
    if args.test:
        task_name += SUFFIX_TEST
    dir_task = os.path.join(dir_systems, task_name)
    for dirname in (dir_third_party, dir_cache, dir_logs, dir_systems, dir_task):
        if os.path.exists(dirname):
            logging.info("Exists: %s", dirname)
        else:
            logging.info("Create: %s", dirname)
            os.makedirs(dirname)

    # (3) Checkout necessary tools

    logging.info("=== Checkout third-party tools ===")
    # Requires tokenization?
    if args.task or args.custom_text_type == CUSTOM_UTF8_RAW:
        third_party.checkout_moses_tokenizer(args.workspace)
    # Requires byte-pair encoding?
    if args.task or args.custom_text_type in (CUSTOM_UTF8_RAW, CUSTOM_UTF8_TOK):
        third_party.checkout_subword_nmt(args.workspace)

    # (4) Populate train/dev/test data

    # This step also normalizes whitespace on data population or copy, ensuring
    # that for all input data, only ASCII newlines are considered line breaks.
    logging.info("=== Populate train/dev/test data ===")
    step_dir_raw = os.path.join(dir_task, DIR_DATA, DIR_RAW)
    complete_fname = os.path.join(step_dir_raw, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_raw)
    else:
        # Listed task
        if args.task:
            raw_files = identify_raw_files(task, test_mode=args.test)
            with tempfile.TemporaryDirectory(prefix="raw.", dir=dir_task) as raw_dir:
                # Download (or locate in cache) and extract raw files to temp directory
                logging.info("=== Download and extract raw files ===")
                download_extract_raw_files(raw_files, dir_cache, raw_dir)
                # Copy required files to train/dev/test
                logging.info("=== Create input data files ===")
                renew_step_dir(step_dir_raw)
                # Test mode uses the full test set as training data and the
                # first line of the test set as dev and test data
                populate_parallel_text(raw_dir,
                                       task.test if args.test else task.train,
                                       os.path.join(step_dir_raw, PREFIX_TRAIN),
                                       False)
                populate_parallel_text(raw_dir,
                                       task.test if args.test else task.dev,
                                       os.path.join(step_dir_raw, PREFIX_DEV),
                                       False,
                                       head_n=1 if args.test else 0)
                populate_parallel_text(raw_dir,
                                       task.test,
                                       os.path.join(step_dir_raw, PREFIX_TEST),
                                       True,
                                       head_n=1 if args.test else 0)
        # Custom task
        else:
            logging.info("=== Copy input data files ===")
            renew_step_dir(step_dir_raw)
            copy_parallel_text(args.custom_train, os.path.join(step_dir_raw, PREFIX_TRAIN))
            copy_parallel_text(args.custom_dev, os.path.join(step_dir_raw, PREFIX_DEV))
            copy_parallel_text(args.custom_test, os.path.join(step_dir_raw, PREFIX_TEST))
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_raw)

    # (5) Tokenize train/dev/test data

    # Task requires tokenization if _any_ raw file is not already tokenized
    requires_tokenization = False
    if args.task:
        for file_sets in (task.train, task.dev, task.test):
            for _, _, text_type in file_sets:
                if text_type in TEXT_REQUIRES_TOKENIZATION:
                    requires_tokenization = True
    else:
        if args.custom_text_type == CUSTOM_UTF8_RAW:
            requires_tokenization = True
    logging.info("=== Tokenize train/dev/test data ===")
    step_dir_tok = os.path.join(dir_task, DIR_DATA, DIR_TOK)
    complete_fname = os.path.join(step_dir_tok, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_tok)
    else:
        renew_step_dir(step_dir_tok)

        # Tokenize each data file using the appropriate language code OR link
        # raw file if already tokenized.
        for fname in os.listdir(step_dir_raw):
            if fname.startswith("."):
                continue
            input_fname = os.path.join(step_dir_raw, fname)
            output_fname = os.path.join(step_dir_tok, fname)
            if requires_tokenization:
                lang_code = lang_codes[0] if fname.endswith(SUFFIX_SRC_GZ) else lang_codes[1]
                logging.info("Tokenize (%s): %s -> %s", lang_code, input_fname, output_fname)
                third_party.call_moses_tokenizer(workspace_dir=args.workspace,
                                                 input_fname=input_fname,
                                                 output_fname=output_fname,
                                                 lang_code=lang_code)
            else:
                logging.info("Link pre-tokenized: %s -> %s", input_fname, output_fname)
                os.symlink(os.path.join("..", DIR_RAW, fname), output_fname)
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_tok)

    # (6) Learn byte-pair encoding model

    # Task requires byte-pair encoding unless using pre-encoded custom data
    skip_bpe = (not args.task) and args.custom_text_type == CUSTOM_UTF8_BPE
    logging.info("=== Learn byte-pair encoding model ===")
    step_dir_bpe_model = os.path.join(dir_task, DIR_BPE_MODEL)
    complete_fname = os.path.join(step_dir_bpe_model, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_bpe_model)
    else:
        renew_step_dir(step_dir_bpe_model)
        if skip_bpe:
            logging.info("BPE model not required for pre-encoded data")
        else:
            source_fname = os.path.join(step_dir_tok, PREFIX_TRAIN + SUFFIX_SRC_GZ)
            target_fname = os.path.join(step_dir_tok, PREFIX_TRAIN + SUFFIX_TRG_GZ)
            codes_fname = os.path.join(step_dir_bpe_model, FILE_BPE_CODES)
            num_ops = task.bpe_op if args.task else args.custom_bpe_op
            if args.test:
                num_ops = TEST_BPE_OPS
            logging.info("BPE Learn (%s): %s + %s -> %s", num_ops, source_fname, target_fname, codes_fname)
            third_party.call_learn_bpe(workspace_dir=args.workspace,
                                       source_fname=source_fname,
                                       target_fname=target_fname,
                                       model_fname=codes_fname,
                                       num_ops=num_ops)
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_bpe_model)

    # (7) Byte-pair encode data

    logging.info("=== Byte-pair encode train/dev/test data ===")
    step_dir_bpe = os.path.join(dir_task, DIR_DATA, DIR_BPE)
    complete_fname = os.path.join(step_dir_bpe, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_bpe)
    else:
        renew_step_dir(step_dir_bpe)
        # Encode each data file
        for fname in os.listdir(step_dir_tok):
            if fname.startswith("."):
                continue
            input_fname = os.path.join(step_dir_tok, fname)
            output_fname = os.path.join(step_dir_bpe, fname)
            if skip_bpe:
                logging.info("Link pre-encoded: %s -> %s", input_fname, output_fname)
                os.symlink(os.path.join("..", DIR_TOK, fname), output_fname)
            else:
                codes_fname = os.path.join(step_dir_bpe_model, FILE_BPE_CODES)
                logging.info("BPE: %s -> %s", input_fname, output_fname)
                third_party.call_apply_bpe(workspace_dir=args.workspace,
                                           input_fname=input_fname,
                                           output_fname=output_fname,
                                           model_fname=codes_fname)
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_bpe)

    # Done if only running data preparation steps
    if args.model == MODEL_NONE:
        return

    # (8) Run Sockeye training

    logging.info("=== Train translation model ===")
    logging.info("Model: %s", args.model)
    if args.model == MODEL_GNMT:
        logging.info("NOTE: This is an 8 layer LSTM model similar (but not exactly identical) to the 'GNMT' architecture.")
    step_dir_model = os.path.join(dir_task, DIR_PREFIX_MODEL + args.model)
    complete_fname = os.path.join(step_dir_model, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_model)
    else:
        log_fname = os.path.join(args.workspace,
                                 DIR_LOGS,
                                 "sockeye.{{}}.{}.{}.{}.log".format(task_name, args.model, os.getpid()))
        call_sockeye_train(args.model,
                           step_dir_bpe,
                           step_dir_model,
                           log_fname.format("train"),
                           args.gpus,
                           test_mode=args.test)
        call_sockeye_average(step_dir_model, log_fname.format("average"))
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_model)

    # (9) Decode test sets

    logging.info("=== Decode test sets ===")
    logging.info("Settings: %s", args.decode_settings)
    step_dir_results = os.path.join(dir_task, DIR_RESULTS)
    if not os.path.exists(step_dir_results):
        logging.info("Create: %s", step_dir_results)
        os.makedirs(step_dir_results)
    # To collect BPE output names
    output_fnames_bpe = []
    # For each test file
    for fname in os.listdir(step_dir_bpe):
        if fname.startswith(PREFIX_TEST) and fname.endswith(SUFFIX_SRC_GZ):
            input_fname = os.path.join(step_dir_bpe, fname)
            # /path/to/results/test[.N].<model>.<settings>
            output_fname = os.path.join(step_dir_results, "{}.{}.{}.{}".format(args.model,
                                                                               args.decode_settings,
                                                                               fname[:-len(SUFFIX_SRC_GZ) - 1],
                                                                               SUFFIX_BPE))
            output_fnames_bpe.append(output_fname)
            # For the shared results directory, a command file indicates that
            # the step has completed successfully.
            command_fname = output_fname + "." + SUFFIX_COMMAND
            if os.path.exists(command_fname):
                logging.info("Re-use output: %s", output_fname)
            else:
                log_fname = os.path.join(args.workspace,
                                 DIR_LOGS,
                                 "sockeye.translate.{}.{}.{}.{}.log".format(task_name,
                                                                            args.model,
                                                                            fname[:-len(SUFFIX_SRC_GZ) - 1],
                                                                            os.getpid()))
                call_sockeye_translate(args=DECODE_ARGS[args.decode_settings],
                                       input_fname=input_fname,
                                       output_fname=output_fname,
                                       model_dir=step_dir_model,
                                       log_fname=log_fname,
                                       use_cpu=(args.gpus == 0))

    # (10) Evaluate test sets (bpe/tok/detok)

    lang_code = lang_codes[1] if lang_codes else None
    logging.info("=== Score outputs ===")
    # For each output file
    for fname_bpe in output_fnames_bpe:
        # Score byte-pair encoded
        fname_base = os.path.basename(fname_bpe)[:-len(SUFFIX_BPE)].split(".", 2)[2]
        fname_ref_bpe = os.path.join(step_dir_bpe, fname_base + SUFFIX_TRG_GZ)
        fname_bleu_bpe = fname_bpe + "." + SUFFIX_BLEU
        if os.path.exists(fname_bleu_bpe):
            logging.info("Re-use output: %s", fname_bleu_bpe)
        else:
            fname_log = os.path.join(args.workspace,
                                     DIR_LOGS,
                                     "sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
                                                                                  args.model,
                                                                                  fname_base + SUFFIX_BPE,
                                                                                  os.getpid()))
            call_sacrebleu(input_fname=fname_bpe,
                           ref_fname=fname_ref_bpe,
                           output_fname=fname_bleu_bpe,
                           log_fname=fname_log,
                           tokenized=True)
        # Score tokenized
        fname_tok = fname_bpe[:-len(SUFFIX_BPE)] + SUFFIX_TOK
        fname_ref_tok = os.path.join(step_dir_tok, fname_base + SUFFIX_TRG_GZ)
        fname_bleu_tok = fname_tok + "." + SUFFIX_BLEU
        if os.path.exists(fname_bleu_tok):
            logging.info("Re-use output: %s", fname_bleu_tok)
        else:
            # Merge BPE
            logging.info("Merge BPE: %s -> %s", fname_bpe, fname_tok)
            third_party.merge_bpe(input_fname=fname_bpe, output_fname=fname_tok)
            fname_log = os.path.join(args.workspace,
                                     DIR_LOGS,
                                     "sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
                                                                                  args.model,
                                                                                  fname_base + SUFFIX_TOK,
                                                                                  os.getpid()))
            call_sacrebleu(input_fname=fname_tok,
                           ref_fname=fname_ref_tok,
                           output_fname=fname_bleu_tok,
                           log_fname=fname_log,
                           tokenized=True)
        # Score detokenized (WMT-compatible BLEU)
        fname_detok = fname_bpe[:-len(SUFFIX_BPE)] + SUFFIX_DETOK
        fname_ref_raw = os.path.join(step_dir_raw, fname_base + SUFFIX_TRG_GZ)
        fname_bleu_detok = fname_detok + "." + SUFFIX_SACREBLEU
        if os.path.exists(fname_bleu_detok):
            logging.info("Re-use output: %s", fname_bleu_detok)
        else:
            if not requires_tokenization:
                logging.info(
                    "WARNING: Task uses pre-tokenized data, cannot reliably detokenize to compute WMT-compatible scores")
                continue
            # Detokenize
            logging.info("Detokenize (%s): %s -> %s", lang_code, fname_tok, fname_detok)
            third_party.call_moses_detokenizer(workspace_dir=args.workspace,
                                               input_fname=fname_tok,
                                               output_fname=fname_detok,
                                               lang_code=lang_code)
            fname_log = os.path.join(args.workspace,
                                     DIR_LOGS,
                                     "sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
                                                                                  args.model,
                                                                                  fname_base + SUFFIX_DETOK,
                                                                                  os.getpid()))
            call_sacrebleu(input_fname=fname_detok,
                           ref_fname=fname_ref_raw,
                           output_fname=fname_bleu_detok,
                           log_fname=fname_log,
                           tokenized=False)
765653
function main () {
  var taskList = _(arguments).toArray().flatten(true).value()
  gulp.task('default', taskList)
  return this
}
941294
public function build() {

        // If we have decided not to restore activities, prevent anything to be built
        if (!$this->get_setting_value('activities')) {
            $this->built = true;
            return;
        }

        // Load he course_module estructure, generating it (with instance = 0)
        // but allowing the creation of the target context needed in following steps
        $this->add_step(new restore_module_structure_step('module_info', 'module.xml'));

        // Here we add all the common steps for any activity and, in the point of interest
        // we call to define_my_steps() is order to get the particular ones inserted in place.
        $this->define_my_steps();

        // Roles (optionally role assignments and always role overrides)
        $this->add_step(new restore_ras_and_caps_structure_step('course_ras_and_caps', 'roles.xml'));

        // Filters (conditionally)
        if ($this->get_setting_value('filters')) {
            $this->add_step(new restore_filters_structure_step('activity_filters', 'filters.xml'));
        }

        // Comments (conditionally)
        if ($this->get_setting_value('comments')) {
            $this->add_step(new restore_comments_structure_step('activity_comments', 'comments.xml'));
        }

        // Calendar events (conditionally)
        if ($this->get_setting_value('calendarevents')) {
            $this->add_step(new restore_calendarevents_structure_step('activity_calendar', 'calendar.xml'));
        }

        // Grades (module-related, rest of gradebook is restored later if possible: cats, calculations...)
        $this->add_step(new restore_activity_grades_structure_step('activity_grades', 'grades.xml'));

        // Advanced grading methods attached to the module
        $this->add_step(new restore_activity_grading_structure_step('activity_grading', 'grading.xml'));

        // Grade history. The setting 'grade_history' is handled in the step.
        $this->add_step(new restore_activity_grade_history_structure_step('activity_grade_history', 'grade_history.xml'));

        // Userscompletion (conditionally)
        if ($this->get_setting_value('userscompletion')) {
            $this->add_step(new restore_userscompletion_structure_step('activity_userscompletion', 'completion.xml'));
        }

        // Logs (conditionally)
        if ($this->get_setting_value('logs')) {
            // Legacy logs.
            $this->add_step(new restore_activity_logs_structure_step('activity_logs', 'logs.xml'));
            // New log stores.
            $this->add_step(new restore_activity_logstores_structure_step('activity_logstores', 'logstores.xml'));
        }

        // Activity competencies.
        $this->add_step(new restore_activity_competencies_structure_step('activity_competencies', 'competencies.xml'));

        // Search reindexing, if enabled and if not restoring entire course.
        if (\core_search\manager::is_indexing_enabled()) {
            $wholecourse = $this->get_target() == backup::TARGET_NEW_COURSE;
            $wholecourse = $wholecourse || ($this->setting_exists('overwrite_conf') && $this->get_setting_value('overwrite_conf'));
            if (!$wholecourse) {
                $this->add_step(new restore_activity_search_index('activity_search_index'));
            }
        }

        // At the end, mark it as built
        $this->built = true;
    }
683053
def initiate_tasks(self):
        
        self.tasks_classes = TaskLoader().load_tasks(
            paths=self.configuration[Configuration.ALGORITHM][Configuration.TASKS][Configuration.PATHS])
791375
def create_global_steps():
        """"""
        global_step = tf.Variable(0, name="global_step", trainable=False, dtype=tf.int32)
        increment_step = tf.assign(global_step, tf.add(global_step, 1))
        return global_step, increment_step
303361
def step(self, step, total, label='STEP', speed_label='STEPS/S', size=1):
        

        self.lock.acquire()
        try:
            time_diff = time.time() - self.last_step_time

            if self.last_step > step:
                # it restarted
                self.last_step = 0

            made_steps_since_last_call = step - self.last_step
            self.last_step = step

            self.made_steps_since_last_sync += made_steps_since_last_call
            self.made_steps_size_since_last_sync += made_steps_since_last_call * size

            if time_diff >= 1 or step == total:  # only each second or last batch
                self.set_system_info('step', step, True)
                self.set_system_info('steps', total, True)

                steps_per_second = self.made_steps_since_last_sync / time_diff
                samples_per_second = self.made_steps_size_since_last_sync / time_diff
                self.last_step_time = time.time()

                if size:
                    self.report_speed(samples_per_second)

                epochs_per_second = steps_per_second / total  # all batches
                self.set_system_info('epochsPerSecond', epochs_per_second, True)

                current_epochs = self.current_epoch if self.current_epoch else 1
                total_epochs = self.total_epochs if self.total_epochs else 1

                self.made_steps_since_last_sync = 0
                self.made_steps_size_since_last_sync = 0

                eta = 0
                if step < total:
                    # time to end this epoch
                    if steps_per_second != 0:
                        eta = (total - step) / steps_per_second

                # time until all epochs are done
                if total_epochs - current_epochs > 0:
                    if epochs_per_second != 0:
                        eta += (total_epochs - (current_epochs)) / epochs_per_second

                self.git.store_file('aetros/job/times/eta.json', simplejson.dumps(eta))

            if label and self.step_label != label:
                self.set_system_info('stepLabel', label, True)
                self.step_label = label

            if speed_label and self.step_speed_label != speed_label:
                self.set_system_info('stepSpeedLabel', speed_label, True)
                self.step_speed_label = speed_label
        finally:
            self.lock.release()
623850
def _run_step(step_obj, step_declaration, initialized_resources):
    """"""

    start_time = time.time()

    # Open any resources that need to be opened before we run this step
    for res_name in step_declaration.resources.opened:
        initialized_resources[res_name].open()

    # Create a dictionary of all of the resources that are required for this step
    used_resources = {local_name: initialized_resources[global_name] for local_name, global_name in step_declaration.resources.used.items()}

    # Allow steps with no resources to not need a resources keyword parameter
    if len(used_resources) > 0:
        out = step_obj.run(resources=used_resources)
    else:
        out = step_obj.run()

    # Close any resources that need to be closed before we run this step
    for res_name in step_declaration.resources.closed:
        initialized_resources[res_name].close()

    end_time = time.time()

    return (end_time - start_time, out)
636193
def _build(self):
    """"""
    q = tf.FIFOQueue(
        self._queue_capacity, [self._dtype, self._dtype],
        shapes=[[self._num_steps, self._batch_size, self._vocab_size]]*2)
    obs, target = tf.py_func(self._get_batch, [], [tf.int32, tf.int32])
    obs = self._one_hot(obs)
    target = self._one_hot(target)
    enqueue_op = q.enqueue([obs, target])
    obs, target = q.dequeue()
    tf.train.add_queue_runner(tf.train.QueueRunner(q, [enqueue_op]))
    return SequenceDataOpsNoMask(obs, target)
886235
def build_all

      # The first thing we want to do is create a temporary application
      # instance so that we can determine the valid targets.
      app = ::Middleman::Application.new do
        config[:exit_before_ready] = true
      end

      build_list = app.config[:targets].each_key.collect { |item| item.to_s }
      bad_builds = []

      build_list.each do |target|
        bad_builds << target unless Build.start(['--target', target])
      end
      unless bad_builds.count == 0
        say
        say 'These targets produced errors during build:', :red
        bad_builds.each { |item| say "  #{item}", :red}
      end
      bad_builds.count == 0

    end
851334
def add_jobflow_steps(self, jobflow_id, steps):
        
        if type(steps) != types.ListType:
            steps = [steps]
        params = {}
        params['JobFlowId'] = jobflow_id

        # Step args
        step_args = [self._build_step_args(step) for step in steps]
        params.update(self._build_step_list(step_args))

        return self.get_object(
            'AddJobFlowSteps', params, RunJobFlowResponse, verb='POST')
859861
function setupByCopy() {
    let task_dir;
    let required_tasks;

    try {
        const config = require(path.join(process.cwd(), 'project.config'));
        required_tasks = config.required_tasks;
        if (!required_tasks) {
            throw Error('no task required.');
        }
        task_dir = config.dir.task;
        if (!task_dir) {
            throw Error('task directory not defined.');
        }
    } catch (error) {
        console.warn(error);
        process.exit(0);
    }

    const dstTaskDir = path.join(process.cwd(), task_dir);
    if (!fs.existsSync(dstTaskDir)) {
        // create tasks dir
        fs.mkdirSync(dstTaskDir);
    }

    required_tasks.forEach((task) => {
        const src = path.join(__dirname, 'tasks', task);
        const dst = path.join(dstTaskDir, task);
        if (fs.existsSync(src)) {
            if (fs.existsSync(dst)) {
                fs.unlinkSync(dst);
            }
            fs.writeFileSync(dst, fs.readFileSync(src).toString());
        } else {
            console.error('task not found: ' + task);
            process.exit(1);
        }
    });
}