Prepares a task storage for merging.

    Args:
      task (Task): task.

    Raises:
      IOError: if the task storage does not exist.
      OSError: if the task storage does not exist.
541074
def PrepareMergeTaskStorage(self, task):
    
    if task.identifier not in self._task_storage_writers:
      raise IOError('Storage writer for task: {0:s} does not exist.'.format(
          task.identifier))
732773
def set(self, key, val):
        
        data = self.get_data(True)
        if data is not None:
            data[key] = val
        else:
            raise RuntimeError("No task is currently running")
801560
def exec_task(task_path, data):
    
    if not data:
        data = {'data': None, 'path': task_path}

    elif not isinstance(data, (str, bytes)):
        data = {'data': json.dumps(data, cls=RequestJSONEncoder),
                'path': task_path}

    else:
        # Open the data from file, if necessary.
        if data is not None and data.startswith("file://"):
            with open(data[len("file://"):]) as f:
                data = f.read()

        data = {'data': data, 'path': task_path}

    # Prepare the task.
    job = Job(data)
    (task, task_callable) = create_task(task_path)

    with delegating_job_context(job, task, task_callable) as jc:
        return jc.task_callable(jc.task_data)
761858
def execute_task(self, task, parent_result):
        
        task.before_run(parent_result)
        context = task.create_context(self.message_queue)
        pending_result = self.pool.apply_async(execute_task_async, (task.func, task.id, context))
        self.pending_results.append(pending_result)
561312
def put(self, task, args, kwargs):
        
        if self.num_threads == 0:
            return task(*args, **kwargs)
        if self.queue is None:
            self._spawn()
        self.queue.put((task, args, kwargs))
327489
def _task_file_exists(self, task_fs):
        
        for filename in ["task.{}".format(ext) for ext in self.get_available_task_file_extensions()]:
            if task_fs.exists(filename):
                return True
        return False
304858
def run_task(task_cls, task_data):
    
    task = instantiate(task_cls)
    task_callable = get_callable(task)
    return task_callable(TaskData(task_data))
6488
def create_task(self, target: Callable[..., Any], *args: Any)\
            -> asyncio.tasks.Task:
        """"""
        if asyncio.iscoroutine(target):
            task = self._loop.create_task(target)
        elif asyncio.iscoroutinefunction(target):
            task = self._loop.create_task(target(*args))
        else:
            raise ValueError("Expected coroutine as target")
        self._pending_tasks.append(task)
        return task
904561
def copy_file(self, path, prefixed_path, source_storage):
        
        args = (path, prefixed_path, source_storage)
        if settings.threads:
            self.tasks.append(args)
        else:
            self.do_copy_file(args)
146176
def RemoveTask(self, task):
    
    with self._lock:
      if task.identifier not in self._tasks_abandoned:
        raise KeyError('Task {0:s} was not abandoned.'.format(task.identifier))

      if not task.has_retry:
        raise KeyError(
            'Will not remove a task {0:s} without retry task.'.format(
                task.identifier))

      del self._tasks_abandoned[task.identifier]

      logger.debug('Removed task {0:s}.'.format(task.identifier))
245334
def flatten_output(task):
    
    r = flatten(task.output())
    if not r:
        for dep in flatten(task.requires()):
            r += flatten_output(dep)
    return r
334298
def run_task_json(task_cls, task_data):
    
    # TODO what does set_skipping_json do?
    task = instantiate(task_cls)
    task_callable = get_callable(task)
    td = TaskData(task_data)
    td.set_skipping_json(True)
    return task_callable(td)
230187
def save_task(task, broker):
    
    # SAVE LIMIT < 0 : Don't save success
    if not task.get('save', Conf.SAVE_LIMIT >= 0) and task['success']:
        return
    # enqueues next in a chain
    if task.get('chain', None):
        django_q.tasks.async_chain(task['chain'], group=task['group'], cached=task['cached'], sync=task['sync'], broker=broker)
    # SAVE LIMIT > 0: Prune database, SAVE_LIMIT 0: No pruning
    db.close_old_connections()
    try:
        if task['success'] and 0 < Conf.SAVE_LIMIT <= Success.objects.count():
            Success.objects.last().delete()
        # check if this task has previous results
        if Task.objects.filter(id=task['id'], name=task['name']).exists():
            existing_task = Task.objects.get(id=task['id'], name=task['name'])
            # only update the result if it hasn't succeeded yet
            if not existing_task.success:
                existing_task.stopped = task['stopped']
                existing_task.result = task['result']
                existing_task.success = task['success']
                existing_task.save()
        else:
            Task.objects.create(id=task['id'],
                                name=task['name'],
                                func=task['func'],
                                hook=task.get('hook'),
                                args=task['args'],
                                kwargs=task['kwargs'],
                                started=task['started'],
                                stopped=task['stopped'],
                                result=task['result'],
                                group=task.get('group'),
                                success=task['success']
                                )
    except Exception as e:
        logger.error(e)
676190
def _ProcessTask(self, task):
    
    logger.debug('Started processing task: {0:s}.'.format(task.identifier))

    if self._tasks_profiler:
      self._tasks_profiler.Sample(task, 'processing_started')

    self._task = task

    storage_writer = self._storage_writer.CreateTaskStorage(task)

    if self._serializers_profiler:
      storage_writer.SetSerializersProfiler(self._serializers_profiler)

    storage_writer.Open()

    self._parser_mediator.SetStorageWriter(storage_writer)

    storage_writer.WriteTaskStart()

    try:
      # TODO: add support for more task types.
      self._ProcessPathSpec(
          self._extraction_worker, self._parser_mediator, task.path_spec)
      self._number_of_consumed_sources += 1

      if self._guppy_memory_profiler:
        self._guppy_memory_profiler.Sample()

    finally:
      storage_writer.WriteTaskCompletion(aborted=self._abort)

      self._parser_mediator.SetStorageWriter(None)

      storage_writer.Close()

    try:
      self._storage_writer.FinalizeTaskStorage(task)
    except IOError:
      pass

    self._task = None

    if self._tasks_profiler:
      self._tasks_profiler.Sample(task, 'processing_completed')

    logger.debug('Completed processing task: {0:s}.'.format(task.identifier))
167227
async def put(self, data, ttl=None, ttr=None, delay=None):
        
        the_tuple = await self.queue.put(self, data, ttl=ttl, ttr=ttr, delay=delay)
        if the_tuple.rowcount:
            return Task.create_from_tuple(self, the_tuple)
918714
def CreateTaskStorage(self, task):
    
    if task.identifier in self._task_storage_writers:
      raise IOError('Storage writer for task: {0:s} already exists.'.format(
          task.identifier))

    storage_writer = FakeStorageWriter(
        self._session, storage_type=definitions.STORAGE_TYPE_TASK, task=task)
    self._task_storage_writers[task.identifier] = storage_writer
    return storage_writer
884627
def CompleteTask(self, task):
    
    with self._lock:
      if task.identifier not in self._tasks_merging:
        raise KeyError('Task {0:s} was not merging.'.format(task.identifier))

      self.SampleTaskStatus(task, 'completed')

      del self._tasks_merging[task.identifier]

      logger.debug('Completed task {0:s}.'.format(task.identifier))