Launch job on ML Engine.
966791
def launch_job(job_spec):
  """"""
  project_id = "projects/{}".format(
      text_encoder.native_to_unicode(default_project()))
  credentials = GoogleCredentials.get_application_default()
  cloudml = discovery.build("ml", "v1", credentials=credentials,
                            cache_discovery=False)
  request = cloudml.projects().jobs().create(body=job_spec, parent=project_id)
  request.execute()
310730
def run(self, *args, **kwargs):
    """"""

    for job in self.jobs:
      job.run(*args, **kwargs)
637349
def launchTask(self, driver, task):
        
        
        log.debug("Asked to launch task %s", repr(task))
        
        def runTask():

            log.debug("Running task %s", task.task_id.value)
            startTime = time.time()
            sendUpdate(task, 'TASK_RUNNING', wallTime=0)

            # try to unpickle the task
            try:
                taskData = pickle.loads(decode_data(task.data))
            except:
                exc_info = sys.exc_info()
                log.error('Exception while unpickling task: ', exc_info=exc_info)
                exc_type, exc_value, exc_trace = exc_info
                sendUpdate(task, 'TASK_FAILED', wallTime=0, msg=''.join(traceback.format_exception_only(exc_type, exc_value)))
                return

            # This is where task.data is first invoked. Using this position to setup cleanupInfo
            if self.workerCleanupInfo is not None:
                assert self.workerCleanupInfo == taskData.workerCleanupInfo
            else:
                self.workerCleanupInfo = taskData.workerCleanupInfo

            # try to invoke a run on the unpickled task
            try:
                process = runJob(taskData)
                self.runningTasks[task.task_id.value] = process.pid
                try:
                    exitStatus = process.wait()
                    wallTime = time.time() - startTime
                    if 0 == exitStatus:
                        sendUpdate(task, 'TASK_FINISHED', wallTime)
                    elif -9 == exitStatus:
                        sendUpdate(task, 'TASK_KILLED', wallTime)
                    else:
                        sendUpdate(task, 'TASK_FAILED', wallTime, msg=str(exitStatus))
                finally:
                    del self.runningTasks[task.task_id.value]
            except:
                wallTime = time.time() - startTime
                exc_info = sys.exc_info()
                log.error('Exception while running task:', exc_info=exc_info)
                exc_type, exc_value, exc_trace = exc_info
                sendUpdate(task, 'TASK_FAILED', wallTime=wallTime, msg=''.join(traceback.format_exception_only(exc_type, exc_value)))

            wallTime = time.time() - startTime
            sendUpdate(task, 'TASK_FINISHED', wallTime)


        def runJob(job):
            """
            :type job: toil.batchSystems.mesos.ToilJob

            :rtype: subprocess.Popen
            """
            if job.userScript:
                job.userScript.register()
            log.debug("Invoking command: '%s'", job.command)
            # Construct the job's environment
            jobEnv = dict(os.environ, **job.environment)
            log.debug('Using environment variables: %s', jobEnv.keys())
            with self.popenLock:
                return subprocess.Popen(job.command,
                                        preexec_fn=lambda: os.setpgrp(),
                                        shell=True, env=jobEnv)

        def sendUpdate(task, taskState, wallTime=None, msg=''):
            update = addict.Dict()
            update.task_id.value = task.task_id.value
            if self.id is not None:
                # Sign our messages as from us, since the driver doesn't do it.
                update.executor_id.value = self.id
            update.state = taskState
            update.timestamp = wallTime
            update.message = msg
            driver.sendStatusUpdate(update)

        thread = threading.Thread(target=runTask)
        thread.start()
184992
def run(self):
        """"""
        # Check already exists #
        if self.status == "READY": return self.launch()
        # Check name conflict #
        if self.status == "DUPLICATE":  message = "Job with same name '%s' already in queue, but we lost the script."
        if self.status == "QUEUED":     message = "Job '%s' already in queue."
        if self.status == "RUNNING":    message = "Job '%s' already running."
        if self.status == "FINISHED":   message = "Job '%s' already ended successfully."
        if self.status == "ABORTED":    message = "Job '%s' was killed without any output file (?)."
        if self.status == "CANCELED":   message = "Job '%s' was canceled or killed while running."
        if self.status == "INTERUPTED": message = "Job '%s' is not running. We don't know why. Look at the log file."
        print Color.i_red + message % (self.name,) + Color.end
        print "Job might have run already (?). Not starting."
1207858
def run(self):
        ""
        while 1:
            # Sleep until there is a job to perform.
            job = self.jobs.get()

            # Yawn. Time to get some work done.
            try:
                job.run()
                self.jobs.task_done()
            except TerminationNotice:
                self.jobs.task_done()
                break
107028
def run(self):
        """"""
        while True:
            self.running = True
            workunit = self._workq.get()
            if is_sentinel(workunit):
                # Got sentinel
                break

            # Run the job / sequence
            workunit.process()
        self.running = False
282879
def launch(self, image, command, **kwargs):
        
        if isinstance(command, PythonCall):
            return PythonJob(self, image, command, **kwargs)
        else:
            return Job(self, image, command, **kwargs)
304003
def train(input_dir, batch_size, max_steps, output_dir, checkpoint, cloud_train_config):
    """"""

    import google.datalab.ml as ml
    if checkpoint is None:
      checkpoint = _util._DEFAULT_CHECKPOINT_GSURL
    staging_package_url = _util.repackage_to_staging(output_dir)
    job_args = {
      'input_dir': input_dir,
      'max_steps': max_steps,
      'batch_size': batch_size,
      'checkpoint': checkpoint
    }
    job_request = {
      'package_uris': [staging_package_url, _TF_GS_URL, _PROTOBUF_GS_URL],
      'python_module': 'mltoolbox.image.classification.task',
      'job_dir': output_dir,
      'args': job_args
    }
    job_request.update(dict(cloud_train_config._asdict()))
    job_id = 'image_classification_train_' + datetime.datetime.now().strftime('%y%m%d_%H%M%S')
    job = ml.Job.submit_training(job_request, job_id)
    if (_util.is_in_IPython()):
      import IPython
      log_url_query_strings = {
        'project': _util.default_project(),
        'resource': 'ml.googleapis.com/job_id/' + job.info['jobId']
      }
      log_url = 'https://console.developers.google.com/logs/viewer?' + \
          urllib.urlencode(log_url_query_strings)
      html = 'Job "%s" submitted.' % job.info['jobId']
      html += '<p>Click <a href="%s" target="_blank">here</a> to view cloud log. <br/>' % log_url
      IPython.display.display_html(html, raw=True)
    return job
430671
def main():
    """"""
    # Initialization variables - update to change your model and execution context
    model_prefix = "FCN8s_VGG16"
    epoch = 19

    # By default, MXNet will run on the CPU. Change to ctx = mx.gpu() to run on GPU.
    ctx = mx.cpu()

    fcnxs, fcnxs_args, fcnxs_auxs = mx.model.load_checkpoint(model_prefix, epoch)
    fcnxs_args["data"] = mx.nd.array(get_data(args.input), ctx)
    data_shape = fcnxs_args["data"].shape
    label_shape = (1, data_shape[2]*data_shape[3])
    fcnxs_args["softmax_label"] = mx.nd.empty(label_shape, ctx)
    exector = fcnxs.bind(ctx, fcnxs_args, args_grad=None, grad_req="null", aux_states=fcnxs_args)
    exector.forward(is_train=False)
    output = exector.outputs[0]
    out_img = np.uint8(np.squeeze(output.asnumpy().argmax(axis=1)))
    out_img = Image.fromarray(out_img)
    out_img.putpalette(get_palette())
    out_img.save(args.output)
287440
def start(self):
        
        self.running = True
        while self.running:
            self.update_prompt()
            try:
                self.cmdloop()
            except KeyboardInterrupt:
                print()
            except botocore.exceptions.BotoCoreError as e:
                print(e)
            except ParseException as e:
                print(self.engine.pformat_exc(e))
            except Exception:
                traceback.print_exc()
            self.engine.reset()
1093942
def start(logger, full_id, fetch=True, env=None, volumes=None, cpus=None, memory=None, gpu_devices=None, offline=False):
    

    owner, name, id = unpack_full_job_id(full_id)

    if isinstance(sys.stdout, GeneralLogger):
        # we don't want to have stuff written to stdout before in job's log
        sys.stdout.clear_buffer()

    job_backend = JobBackend(model_name=owner + '/' + name)

    if fetch:
        job_backend.fetch(id)

    job_backend.restart(id)
    job_backend.start(collect_system=False, offline=offline)
    job_backend.set_status('PREPARE', add_section=False)

    job = job_backend.get_job_model()

    if not cpus:
        cpus = job.get_cpu()

    if not memory:
        memory = job.get_memory()

    if not gpu_devices and job.get_gpu():
        # if requested 2 GPUs and we have 3 GPUs with id [0,1,2], gpus should be [0,1]
        gpu_devices = []
        for i in range(0, job.get_gpu()):
            gpu_devices.append(i)

    start_command(logger, job_backend, env, volumes, cpus=cpus, memory=memory, gpu_devices=gpu_devices, offline=offline)
484191
def run(self):
        ''''''
        # Register our signal handlers
        self.signals()

        with self.listener():
            for job in self.jobs():
                # If there was no job to be had, we should sleep a little bit
                if not job:
                    self.jid = None
                    self.title('Sleeping for %fs' % self.interval)
                    time.sleep(self.interval)
                else:
                    self.jid = job.jid
                    self.title('Working on %s (%s)' % (job.jid, job.klass_name))
                    with Worker.sandbox(self.sandbox):
                        job.sandbox = self.sandbox
                        job.process()
                if self.shutdown:
                    break
1286790
def run(self):
        
        try:
            proxy = config_ini.engine.open()
            self.LOG.info("Stats for %s - up %s, %s" % (
                config_ini.engine.engine_id,
                fmt.human_duration(proxy.system.time() - config_ini.engine.startup, 0, 2, True).strip(),
                proxy
            ))
        except (error.LoggableError, xmlrpc.ERRORS), exc:
            self.LOG.warn(str(exc))
630605
def run(self):
        """"""
        self._make_lib_file_symbolic_links()
        self._copy_each_include_files_to_include_dir()
        self._make_dep_lib_file_sym_links_and_copy_include_files()
        self.setup_py.add_patchs_to_build_without_pkg_config(
            self.rpm.lib_dir, self.rpm.include_dir
        )
        self.setup_py.apply_and_save()
        self._build_and_install()
170703
def run(self):
        """"""
        try:
            while not self.stopped:
                scraperobj = jobs.get(False)
                self.setName(scraperobj.getName())
                try:
                    self.getStrips(scraperobj)
                finally:
                    jobs.task_done()
                    self.setName(self.origname)
        except Empty:
            pass
        except KeyboardInterrupt:
            thread.interrupt_main()
79151
def _run_raw(self, *args, **kwargs):
    """"""
    for job in self.jobs:
      job._run_raw(*args, **kwargs)
241213
def start(self):
		
		if self._thread_running.is_set():
			raise RuntimeError('the JobManager has already been started')
		self._thread.start()
		self._thread_running.wait()
		return