Create all the steps that will be part of this task
967757
public function build() {

        // Move all the CONTEXT_MODULE question qcats to their
        // final (newly created) module context
        $this->add_step(new restore_move_module_questions_categories('move_module_question_categories'));

        // Create all the question files now that every question is in place
        // and every category has its final contextid associated
        $this->add_step(new restore_create_question_files('create_question_files'));

        // Review all the block_position records in backup_ids in order
        // match them now that all the contexts are created populating DB
        // as needed. Only if we are restoring blocks.
        if ($this->get_setting_value('blocks')) {
            $this->add_step(new restore_review_pending_block_positions('review_block_positions'));
        }

        // Gradebook. Don't restore the gradebook unless activities are being restored.
        if ($this->get_setting_value('activities')) {
            $this->add_step(new restore_gradebook_structure_step('gradebook_step','gradebook.xml'));
            $this->add_step(new restore_grade_history_structure_step('grade_history', 'grade_history.xml'));
        }

        // Course completion.
        $this->add_step(new restore_course_completion_structure_step('course_completion', 'completion.xml'));

        // Conditionally restore course badges.
        if ($this->get_setting_value('badges')) {
            $this->add_step(new restore_badges_structure_step('course_badges', 'badges.xml'));
        }

        // Review all the legacy module_availability records in backup_ids in
        // order to match them with existing modules / grade items and convert
        // into the new system.
        $this->add_step(new restore_process_course_modules_availability('process_modules_availability'));

        // Update restored availability data to account for changes in IDs
        // during backup/restore.
        $this->add_step(new restore_update_availability('update_availability'));

        // Refresh action events conditionally.
        if ($this->get_setting_value('activities')) {
            $this->add_step(new restore_calendar_action_events('restoring_action_events'));
        }

        // Decode all the interlinks
        $this->add_step(new restore_decode_interlinks('decode_interlinks'));

        // Restore course logs (conditionally). They are restored here because we need all
        // the activities to be already restored.
        if ($this->get_setting_value('logs')) {
            // Legacy logs.
            $this->add_step(new restore_course_logs_structure_step('course_logs', 'course/logs.xml'));
            // New log stores.
            $this->add_step(new restore_course_logstores_structure_step('course_logstores', 'course/logstores.xml'));
        }

        // Review all the executed tasks having one after_restore method
        // executing it to perform some final adjustments of information
        // not available when the task was executed.
        // This step is always the last one performing modifications on restored information
        // Don't add any new step after it. Only aliases queue, cache rebuild and clean are allowed.
        $this->add_step(new restore_execute_after_restore('executing_after_restore'));

        // All files were sent to the filepool by now. We need to process
        // the aliases yet as they were not actually created but stashed for us instead.
        // We execute this step after executing_after_restore so that there can't be no
        // more files sent to the filepool after this.
        $this->add_step(new restore_process_file_aliases_queue('process_file_aliases_queue'));

        // Rebuild course cache to see results, whoah!
        $this->add_step(new restore_rebuild_course_cache('rebuild_course_cache'));

        // Clean the temp dir (conditionally) and drop temp table
        $this->add_step(new restore_drop_and_clean_temp_stuff('drop_and_clean_temp_stuff'));

        // If restoring to a new course or overwriting config, reindex the whole course.
        if (\core_search\manager::is_indexing_enabled()) {
            $wholecourse = $this->get_target() == backup::TARGET_NEW_COURSE;
            $wholecourse = $wholecourse || $this->setting_exists('overwrite_conf') && $this->get_setting_value('overwrite_conf');
            if ($wholecourse) {
                $this->add_step(new restore_course_search_index('course_search_index'));
            }
        }

        $this->built = true;
    }
1293968
def build(self, **kwargs):
        
        self.tasks += self.compute_tasks(**kwargs)
        return self.finalize()
69938
public function bootstrapTasks()
    {
        $tasks = [
            Check::class,
            Cleanup::class,
            CreateRelease::class,
            CurrentRelease::class,
            Dependencies::class,
            Deploy::class,
            FridayDeploy::class,
            Ignite::class,
            Installer::class,
            Migrate::class,
            Notify::class,
            Primer::class,
            Rollback::class,
            Setup::class,
            SwapSymlink::class,
            Teardown::class,
            Test::class,
            Update::class,
            Updater::class,
        ];

        foreach ($tasks as $task) {
            $task = new $task();
            $task->setContainer($this->container);
            $this->builder->buildTask($task);
        }
    }
750952
def begin(self):
    """"""
    self._global_step_tensor = tf.train.get_global_step()
    if self._global_step_tensor is None:
      raise RuntimeError(
          'Global step should be created to use StepCounterHook.')
487590
function step() {
          const f = files.pop();
          if (f) {
            console.error('Ractive: compiling component ' + path.join(opts.directory, f));
            util
              .readToString(fs.createReadStream(path.join(opts.directory, f)))
              .then(string => {
                return component
                  .build(string, opts, mkReadFile(path.join(opts.directory, f)))
                  .then(string => {
                    const file = path.join(
                      opts.output,
                      f.replace(ext, opts.outputExtension || '.js')
                    );
                    util.mkdirp(path.dirname(file));
                    util.writeToStream(fs.createWriteStream(file), string);
                  });
              })
              .then(step, err => {
                console.error(err && typeof err === 'object' ? err.stack : err);
                process.exit(1);
              });
          }
        }
636193
def _build(self):
    """"""
    q = tf.FIFOQueue(
        self._queue_capacity, [self._dtype, self._dtype],
        shapes=[[self._num_steps, self._batch_size, self._vocab_size]]*2)
    obs, target = tf.py_func(self._get_batch, [], [tf.int32, tf.int32])
    obs = self._one_hot(obs)
    target = self._one_hot(target)
    enqueue_op = q.enqueue([obs, target])
    obs, target = q.dequeue()
    tf.train.add_queue_runner(tf.train.QueueRunner(q, [enqueue_op]))
    return SequenceDataOpsNoMask(obs, target)
1263618
def generate(self, step, params):
        
        subfactory = self.get_factory()
        logger.debug(
            "SubFactory: Instantiating %s.%s(%s), create=%r",
            subfactory.__module__, subfactory.__name__,
            utils.log_pprint(kwargs=params),
            step,
        )
        force_sequence = step.sequence if self.FORCE_SEQUENCE else None
        return step.recurse(subfactory, params, force_sequence=force_sequence)
623850
def _run_step(step_obj, step_declaration, initialized_resources):
    """"""

    start_time = time.time()

    # Open any resources that need to be opened before we run this step
    for res_name in step_declaration.resources.opened:
        initialized_resources[res_name].open()

    # Create a dictionary of all of the resources that are required for this step
    used_resources = {local_name: initialized_resources[global_name] for local_name, global_name in step_declaration.resources.used.items()}

    # Allow steps with no resources to not need a resources keyword parameter
    if len(used_resources) > 0:
        out = step_obj.run(resources=used_resources)
    else:
        out = step_obj.run()

    # Close any resources that need to be closed before we run this step
    for res_name in step_declaration.resources.closed:
        initialized_resources[res_name].close()

    end_time = time.time()

    return (end_time - start_time, out)
1370796
def build(self, parent_step=None, force_sequence=None):
        """"""
        # TODO: Handle "batch build" natively
        pre, post = parse_declarations(
            self.extras,
            base_pre=self.factory_meta.pre_declarations,
            base_post=self.factory_meta.post_declarations,
        )

        if force_sequence is not None:
            sequence = force_sequence
        elif self.force_init_sequence is not None:
            sequence = self.force_init_sequence
        else:
            sequence = self.factory_meta.next_sequence()

        step = BuildStep(
            builder=self,
            sequence=sequence,
            parent_step=parent_step,
        )
        step.resolve(pre)

        args, kwargs = self.factory_meta.prepare_arguments(step.attributes)

        instance = self.factory_meta.instantiate(
            step=step,
            args=args,
            kwargs=kwargs,
        )

        postgen_results = {}
        for declaration_name in post.sorted():
            declaration = post[declaration_name]
            unrolled_context = declaration.declaration.unroll_context(
                instance=instance,
                step=step,
                context=declaration.context,
            )

            postgen_context = PostGenerationContext(
                value_provided='' in unrolled_context,
                value=unrolled_context.get(''),
                extra={k: v for k, v in unrolled_context.items() if k != ''},
            )
            postgen_results[declaration_name] = declaration.declaration.call(
                instance=instance,
                step=step,
                context=postgen_context,
            )
        self.factory_meta.use_postgeneration_results(
            instance=instance,
            step=step,
            results=postgen_results,
        )
        return instance
262251
def create_on_task(self, task_id, file_content, file_name, file_content_type=None, **options):
        """"""
        path = '/tasks/%d/attachments' % (task_id)
        return self.client.request('post', path, files=[('file', (file_name, file_content, file_content_type))], **options)
886235
def build_all

      # The first thing we want to do is create a temporary application
      # instance so that we can determine the valid targets.
      app = ::Middleman::Application.new do
        config[:exit_before_ready] = true
      end

      build_list = app.config[:targets].each_key.collect { |item| item.to_s }
      bad_builds = []

      build_list.each do |target|
        bad_builds << target unless Build.start(['--target', target])
      end
      unless bad_builds.count == 0
        say
        say 'These targets produced errors during build:', :red
        bad_builds.each { |item| say "  #{item}", :red}
      end
      bad_builds.count == 0

    end
129807
function importTasks(options) {
  const opts = parseOptions(options);
  const cwd = process.cwd();
  const dir = path.join(cwd, opts.dir);

  debug(`importing tasks from "${dir}"...`);

  // This synchronously reads the contents within the chosen directory then
  // loops through each item, verifies that it is in fact a file of the correct
  // file type, then creates the tasks object and registers it with gulp.
  fs.readdirSync(dir).forEach(
    handleFile(dir, opts)
  );

  debug(`finished importing tasks`);
}
979425
function taskRunner (ui) {

    return function (item, cb) {

        ui.logger.debug("Starting Step: " + item.step);

        /**
         * Give each step access to the UI Instance
         */
        item.fn(ui, function (err, out) {
            if (err) {
                return cb(err);
            }
            if (out) {
                handleOut(ui, out);
            }
            ui.logger.debug("{green:Step Complete: " + item.step);
            cb();
        });
    };
}
688274
def run(self, steps=None, resume=False, redo=None):
        

        recipe = {
            "name"      :   self.name,
            "steps"     :   []
        }
        start_at = 0

        if redo:
            recipe = utils.readJson(redo)
            self.log.info('Rerunning recipe {0} from {1}'.format(recipe['name'], redo))
            self.log.info('Recreating recipe instance..')
            self.jobs = []
            for step in recipe['steps']:
                
            #        add I/O folders to the json file
            #        add a string describing the contents of these folders
            #        The user has to ensure that these folders exist, and have the required content
                if step['jtype'] == 'docker':
                    self.log.info('Adding job \'{0}\' to recipe. The container will be named \'{1}\''.format(step['cab'], step['name']))
                    cont = docker.Container(step['cab'], step['name'],
                                          label=step['label'], logger=self.log,
                                          shared_memory=step['shared_memory'])

                    self.log.debug('Adding volumes {0} and environmental variables {1}'.format(step['volumes'], step['environs']))
                    cont.volumes = step['volumes']
                    cont.environs = step['environs']
                    cont.shared_memory = step['shared_memory']
                    cont.input_content = step['input_content']
                    cont.msdir_content = step['msdir_content']
                    cont.logfile = step['logfile']
                    job = StimelaJob(step['name'], recipe=self, label=step['label'])
                    job.job = cont
                    job.jtype = 'docker'


                elif step['jtype'] == 'function':
                    name = step['name']
                    func = inspect.currentframe().f_back.f_locals[step['function']]
                    job = StimelaJob(name, recipe=self, label=step['label'])
                    job.python_job(func, step['parameters'])
                    job.jtype = 'function'

                self.jobs.append(job)

        elif resume:
            self.log.info("Resuming recipe from last run.")
            try:
                recipe = utils.readJson(self.resume_file)
            except IOError:
                raise StimelaRecipeExecutionError("Cannot resume pipeline, resume file '{}' not found".format(self.resume_file))

            steps_ = recipe.pop('steps')
            recipe['steps'] = []
            _steps = []
            for step in steps_:
                if step['status'] == 'completed':
                    recipe['steps'].append(step)
                    continue

                label = step['label']
                number = step['number']
                
                # Check if the recipe flow has changed
                if label == self.jobs[number-1].label:
                    self.log.info('recipe step \'{0}\' is fit for re-execution. Label = {1}'.format(number, label))
                    _steps.append(number)
                else:
                    raise StimelaRecipeExecutionError('Recipe flow, or task scheduling has changed. Cannot resume recipe. Label = {0}'.format(label))

            # Check whether there are steps to resume        
            if len(_steps)==0:
                self.log.info('All the steps were completed. No steps to resume')
                sys.exit(0)
            steps = _steps

        if getattr(steps, '__iter__', False):
            _steps = []
            if isinstance(steps[0], str):
                labels = [ job.label.split('::')[0] for job in self.jobs]

                for step in steps:
                    try:
                        _steps.append(labels.index(step)+1)
                    except ValueError:
                        raise StimelaCabParameterError('Recipe label ID [{0}] doesn\'t exist'.format(step))
                steps = _steps
        else:
            steps = range(1, len(self.jobs)+1)

        jobs = [(step, self.jobs[step-1]) for step in steps]
        
        for i, (step, job) in enumerate(jobs):

            self.log.info('Running job {}'.format(job.name))
            self.log.info('STEP {0} :: {1}'.format(i+1, job.label))
            self.active = job
            try:
                if job.jtype == 'function':
                    job.run_python_job()
                elif job.jtype in ['docker', 'singularity']:
                    with open(job.job.logfile, 'a') as astd:
                        astd.write('\n-----------------------------------\n')
                        astd.write('Stimela version     : {}\n'.format(version.version))
                        astd.write('Cab name            : {}\n'.format(job.job.image))
                        astd.write('-------------------------------------\n')

                    run_job = getattr(job, "run_{0:s}_job".format(job.jtype))
                    run_job()
                    

                self.log2recipe(job, recipe, step, 'completed')

            except (utils.StimelaCabRuntimeError,
                   StimelaRecipeExecutionError,
                   StimelaCabParameterError) as e:
                self.completed = [jb[1] for jb in jobs[:i]]
                self.remaining = [jb[1] for jb in jobs[i+1:]]
                self.failed = job

                self.log.info('Recipe execution failed while running job {}'.format(job.name))
                self.log.info('Completed jobs : {}'.format([c.name for c in self.completed]))
                self.log.info('Remaining jobs : {}'.format([c.name for c in self.remaining]))

                self.log2recipe(job, recipe, step, 'failed')
                for step, jb in jobs[i+1:]:
                    self.log.info('Logging remaining task: {}'.format(jb.label))
                    self.log2recipe(jb, recipe, step, 'remaining')

                self.log.info('Saving pipeline information in {}'.format(self.resume_file))
                utils.writeJson(self.resume_file, recipe)

                pe = PipelineException(e, self.completed, job, self.remaining)
                raise_(pe, None, sys.exc_info()[2])
            except:
                import traceback
                traceback.print_exc()
                raise RuntimeError("An unhandled exception has occured. This is a bug, please report")

            finally:
                if job.jtype == 'docker' and job.created:
                    job.job.stop()
                    job.job.remove()
                if job.jtype == 'singularity' and job.created:
                    job.job.stop()

        self.log.info('Saving pipeline information in {}'.format(self.resume_file))
        utils.writeJson(self.resume_file, recipe)

        self.log.info('Recipe executed successfully')
        
        return 0
634713
def steps(self):
        """"""
        def _iter(uri, acc):
            acc.appendleft(uri.name if uri.name else '')
            return _iter(uri.parent, acc) if uri.parent else acc
        return _iter(self, acc=deque())
1018248
def _create_chain_entry(user_id, task_id, task_class, args, kwargs, callbacks, parent=None):
    
    LOGGER.debug(task_class)
    if issubclass(task_class.__class__, UserTaskMixin):
        arguments_dict = task_class.arguments_as_dict(*args, **kwargs)
        name = task_class.generate_name(arguments_dict)
        total_steps = task_class.calculate_total_steps(arguments_dict)
        parent_name = kwargs.get('user_task_name', '')
        with transaction.atomic():
            if parent is None:
                # First task in the chain, create a status record for it
                parent = UserTaskStatus.objects.create(
                    is_container=True, name=parent_name, task_class='celery.chain', task_id=str(uuid4()),
                    total_steps=0, user_id=user_id)
            UserTaskStatus.objects.create(
                name=name, parent=parent, task_class=task_class, task_id=task_id, total_steps=total_steps,
                user_id=user_id)
            parent.increment_total_steps(total_steps)
            if parent_name and not parent.name:
                parent.set_name(parent_name)
    for callback in callbacks:
        links = callback.options.get('link', [])
        callback_class = import_string(callback.task)
        _create_chain_entry(user_id, callback.id, callback_class, callback.args, callback.kwargs, links, parent=parent)
303361
def step(self, step, total, label='STEP', speed_label='STEPS/S', size=1):
        

        self.lock.acquire()
        try:
            time_diff = time.time() - self.last_step_time

            if self.last_step > step:
                # it restarted
                self.last_step = 0

            made_steps_since_last_call = step - self.last_step
            self.last_step = step

            self.made_steps_since_last_sync += made_steps_since_last_call
            self.made_steps_size_since_last_sync += made_steps_since_last_call * size

            if time_diff >= 1 or step == total:  # only each second or last batch
                self.set_system_info('step', step, True)
                self.set_system_info('steps', total, True)

                steps_per_second = self.made_steps_since_last_sync / time_diff
                samples_per_second = self.made_steps_size_since_last_sync / time_diff
                self.last_step_time = time.time()

                if size:
                    self.report_speed(samples_per_second)

                epochs_per_second = steps_per_second / total  # all batches
                self.set_system_info('epochsPerSecond', epochs_per_second, True)

                current_epochs = self.current_epoch if self.current_epoch else 1
                total_epochs = self.total_epochs if self.total_epochs else 1

                self.made_steps_since_last_sync = 0
                self.made_steps_size_since_last_sync = 0

                eta = 0
                if step < total:
                    # time to end this epoch
                    if steps_per_second != 0:
                        eta = (total - step) / steps_per_second

                # time until all epochs are done
                if total_epochs - current_epochs > 0:
                    if epochs_per_second != 0:
                        eta += (total_epochs - (current_epochs)) / epochs_per_second

                self.git.store_file('aetros/job/times/eta.json', simplejson.dumps(eta))

            if label and self.step_label != label:
                self.set_system_info('stepLabel', label, True)
                self.step_label = label

            if speed_label and self.step_speed_label != speed_label:
                self.set_system_info('stepSpeedLabel', speed_label, True)
                self.step_speed_label = speed_label
        finally:
            self.lock.release()