The main method of the worker. Will ask redis for list items via
        blocking calls, get jobs from them, try to execute these jobs, and end
        when needed.
429104
def run(self):
        
        # if status is not None, we already had a run !
        if self.status:
            self.set_status('aborted')
            raise LimpydJobsException('This worker run is already terminated')

        self.set_status('starting')

        self.start_date = datetime.utcnow()
        if self.max_duration:
            self.wanted_end_date = self.start_date + self.max_duration

        must_stop = self.must_stop()

        if not must_stop:

            # get keys
            while not self.keys and not must_stop:
                self.update_keys()
                if not self.keys:
                    sleep(self.fetch_priorities_delay)
                must_stop = self.must_stop()

            if not must_stop:
                # wait for queues available if no ones are yet
                self.requeue_delayed_jobs()
                self.run_started()
                self._main_loop()

        self.set_status('terminated')
        self.end_date = datetime.utcnow()
        self.run_ended()
        if self.terminate_gracefuly:
            self.stop_handling_end_signal()
201458
def main():
    
    home = os.getcwd()
    options = parse_arguments()
    print(options)
    print()

    #get a list of all directories in the cwd
    dirs = [direc for direc in os.listdir(home) if os.path.isdir(direc)]
    #instantiate all of the classes that we will be using in parallel processing
    instances = []
    for each in dirs:
        thisInstance = meraculousRunAnalyzer(each, options.censor, options.quiet)
        #only process the instances that are meraculous directories
        if thisInstance.isMer:
            instances.append(thisInstance)

    if len(instances) == 0:
        print("There are no meraculous folders in this directory. Exiting")
    elif len(instances) > 0:
        # run the program for each instance
        #process each file sequentially using max number of threads
        #determine the pool size to work with the unique sample names
        pool_size = determine_pool_size(instances)
        pool = ThreadPool(pool_size)
        results = pool.map(run_merReport_dummy, instances)
        pool.close()
        pool.join()
45441
def cmd(send, *_):
    
    thread_names = []
    for x in sorted(threading.enumerate(), key=lambda k: k.name):
        res = re.match(r'Thread-(\d+$)', x.name)
        if res:
            tid = int(res.group(1))
            # Handle the main server thread (permanently listed as _worker)
            if x._target.__name__ == '_worker':
                thread_names.append((tid, "%s running server thread" % x.name))
            # Handle the multiprocessing pool worker threads (they don't have names beyond Thread-x)
            elif x._target.__module__ == 'multiprocessing.pool':
                thread_names.append((tid, "%s running multiprocessing pool worker thread" % x.name))
        # Handle everything else including MainThread and deferred threads
        else:
            res = re.match(r'Thread-(\d+)', x.name)
            tid = 0
            if res:
                tid = int(res.group(1))
            thread_names.append((tid, x.name))
    for x in sorted(thread_names, key=lambda k: k[0]):
        send(x[1])
448035
def handle(self):
        
        while True:
            # TODO: Queue.get() (and anything using thread.allocate_lock
            #       ignores signals :(  so timeout periodically to allow delivery
            try:
                callable = None # ensure no lingering references to past work while blocking
                callable = self._Q.get(True, 1.0)
            except Empty:
                continue  # retry on timeout
            try:
                if callable is self._stopit:
                    break
                callable()
            except:
                _log.exception("Error from WorkQueue")
            finally:
                self._Q.task_done()
67426
def _worker_method(self):
        
        while not self._should_quit():
            try:
                task = self._inq.get(block=True, timeout=0.25)
            except TypeError:
                if self._should_quit():
                    break
                else:
                    raise
            except Empty:
                continue

            try:
                obj = task.object
                if isinstance(obj, RiakObject):
                    rv = task.client.put(obj, **task.options)
                elif isinstance(obj, TsObject):
                    rv = task.client.ts_put(obj, **task.options)
                else:
                    raise ValueError('unknown obj type: %s'.format(type(obj)))
                task.outq.put(rv)
            except KeyboardInterrupt:
                raise
            except Exception as err:
                errdata = (task.object, err)
                task.outq.put(errdata)
            finally:
                self._inq.task_done()
1206764
def _start_worker(self, worker: Worker):
        
        # This function is called from main thread and from worker pools threads to start their children threads
        with self.running_workers_lock:
            self.running_workers.append(worker)
        thread = SchedulerThread(worker, self._worker_ended)
        thread.start()
        # This may or may not be posted to a background thread (see set_callbacks)
        self.worker_start_callback(worker)
1145659
private void mainLoop() {
        for (@AutoreleasePool int i = 0;;) {
            try {
                TimerTask task;
                boolean taskFired;
                synchronized(queue) {
                    // Wait for queue to become non-empty
                    while (queue.isEmpty() && newTasksMayBeScheduled)
                        queue.wait();
                    if (queue.isEmpty())
                        break; // Queue is empty and will forever remain; die

                    // Queue nonempty; look at first evt and do the right thing
                    long currentTime, executionTime;
                    task = queue.getMin();
                    synchronized(task.lock) {
                        if (task.state == TimerTask.CANCELLED) {
                            queue.removeMin();
                            continue;  // No action required, poll queue again
                        }
                        currentTime = System.currentTimeMillis();
                        executionTime = task.nextExecutionTime;
                        if (taskFired = (executionTime<=currentTime)) {
                            if (task.period == 0) { // Non-repeating, remove
                                queue.removeMin();
                                task.state = TimerTask.EXECUTED;
                            } else { // Repeating task, reschedule
                                queue.rescheduleMin(
                                  task.period<0 ? currentTime   - task.period
                                                : executionTime + task.period);
                            }
                        }
                    }
                    if (!taskFired) // Task hasn't yet fired; wait
                        queue.wait(executionTime - currentTime);
                }
                if (taskFired)  // Task fired; run it, holding no locks
                    task.run();
            } catch(InterruptedException e) {
            }
        }
    }
156936
def write_main_jobwrappers(self):
        '''
        
        '''
        main_section = ''

        # toil cannot technically start with multiple jobs, so an empty
        # 'initialize_jobs' function is always called first to get around this
        main_section = main_section + '\n        job0 = Job.wrapJobFn(initialize_jobs)\n'

        # declare each job in main as a wrapped toil function in order of priority
        for wf in self.workflows_dictionary:
            for assignment in self.workflows_dictionary[wf]:
                if assignment.startswith('call'):
                    main_section += '        job0 = job0.encapsulate()\n'
                    main_section += self.write_main_jobwrappers_call(self.workflows_dictionary[wf][assignment])
                if assignment.startswith('scatter'):
                    main_section += '        job0 = job0.encapsulate()\n'
                    main_section += self.write_main_jobwrappers_scatter(self.workflows_dictionary[wf][assignment],
                                                                        assignment)
                if assignment.startswith('if'):
                    main_section += '        if {}:\n'.format(self.workflows_dictionary[wf][assignment]['expression'])
                    main_section += self.write_main_jobwrappers_if(self.workflows_dictionary[wf][assignment]['body'])

        main_section += '\n        fileStore.start(job0)\n'

        return main_section
1370172
def _execute(self, worker):
        
        self._assert_status_is(TaskStatus.RUNNING)
        operation = worker.look_up(self.operation)
        operation.invoke(self, [], worker=worker)
992696
def _get_run_by_other_worker(worker):
    
    task_sets = _get_external_workers(worker).values()
    return functools.reduce(lambda a, b: a | b, task_sets, set())
1315749
def run(self):
        
        read_stdout = partial(self._read_output, stream=self._process.stdout,
                              callback=self._callback_stdout,
                              output_file=self._stdout_file)

        read_stderr = partial(self._read_output, stream=self._process.stderr,
                              callback=self._callback_stderr,
                              output_file=self._stderr_file)

        # capture the process output as long as the process is active
        try:
            while self._process.poll() is None:
                result_stdout = read_stdout()
                result_stderr = read_stderr()

                if not result_stdout and not result_stderr:
                    sleep(self._refresh_time)

            # read remaining lines
            while read_stdout():
                pass

            while read_stderr():
                pass

        except (StopTask, AbortWorkflow) as exc:
            self._exc_obj = exc
545199
def call(*, **, &block)
      pid = fork_and_return_pid(&block)
      write.close
      result = read.read
      Process.wait(pid)

      # rubocop:disable MarshalLoad
      Marshal.load(result)
      # rubocop:enable MarshalLoad
    end
330807
def start(self):
        
        with self._operational_lock:
            if self._thread is not None:
                raise ValueError("Dispatcher is already running.")

            flow_control = self._manager.flow_control
            worker = helper_threads.QueueCallbackWorker(
                self._queue,
                self.dispatch_callback,
                max_items=flow_control.max_request_batch_size,
                max_latency=flow_control.max_request_batch_latency,
            )
            # Create and start the helper thread.
            thread = threading.Thread(name=_CALLBACK_WORKER_NAME, target=worker)
            thread.daemon = True
            thread.start()
            _LOGGER.debug("Started helper thread %s", thread.name)
            self._thread = thread
610592
def main(event_loop=None):
    
    context, credentials = get_context_from_cmdln(sys.argv[1:])
    log.info("Scriptworker starting up at {} UTC".format(arrow.utcnow().format()))
    cleanup(context)
    context.event_loop = event_loop or asyncio.get_event_loop()

    done = False

    async def _handle_sigterm():
        log.info("SIGTERM received; shutting down")
        nonlocal done
        done = True
        if context.running_tasks is not None:
            await context.running_tasks.cancel()

    context.event_loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.ensure_future(_handle_sigterm()))

    while not done:
        try:
            context.event_loop.run_until_complete(async_main(context, credentials))
        except Exception:
            log.critical("Fatal exception", exc_info=1)
            raise
1130389
def run(self):
        

        try:
            self.__options.callbacks.request_in_thread_before_start(self.__queue_item)
        except Exception as e:
            print(e)

        new_requests = []
        failed = False

        try:
            handler = Handler(self.__options, self.__queue_item)
            new_requests = handler.get_new_requests()

            try:
                self.__queue_item.response.raise_for_status()
            except Exception:
                if self.__queue_item.request.parent_raised_error:
                    failed = True
                else:
                    for new_request in new_requests:
                        new_request.parent_raised_error = True

        except Exception as e:
            failed = True

            error_message = "Setting status of '{}' to '{}' because of an HTTP error.".format(
                self.__queue_item.request.url,
                QueueItem.STATUS_ERRORED
            )

            DebugHelper.output(self.__options, error_message)
            DebugHelper.output(self.__options, e)

            try:
                self.__options.callbacks.request_on_error(self.__queue_item, str(e))
            except Exception as e:
                print(e)

        for new_request in new_requests:
            new_request.parent_url = self.__queue_item.request.url

        try:
            self.__options.callbacks.request_in_thread_after_finish(self.__queue_item)
        except Exception as e:
            print(e)

        with self.__callback_lock:
            self.__callback(self.__queue_item, new_requests, failed)
704916
def _execute(self, queue, tasks, log, locks, queue_lock, all_task_ids):
        

        # The tasks must use the same function.
        assert len(tasks)
        task_func = tasks[0].serialized_func
        assert all([task_func == task.serialized_func for task in tasks[1:]])

        # Before executing periodic tasks, queue them for the next period.
        if task_func in self.tiger.periodic_task_funcs:
            tasks[0]._queue_for_next_period()

        with g_fork_lock:
            child_pid = os.fork()

        if child_pid == 0:
            # Child process
            log = log.bind(child_pid=os.getpid())

            # Disconnect the Redis connection inherited from the main process.
            # Note that this doesn't disconnect the socket in the main process.
            self.connection.connection_pool.disconnect()

            random.seed()

            # Ignore Ctrl+C in the child so we don't abort the job -- the main
            # process already takes care of a graceful shutdown.
            signal.signal(signal.SIGINT, signal.SIG_IGN)

            with WorkerContextManagerStack(self.config['CHILD_CONTEXT_MANAGERS']):
                success = self._execute_forked(tasks, log)

            # Wait for any threads that might be running in the child, just
            # like sys.exit() would. Note we don't call sys.exit() directly
            # because it would perform additional cleanup (e.g. calling atexit
            # handlers twice). See also: https://bugs.python.org/issue18966
            threading._shutdown()

            os._exit(int(not success))
        else:
            # Main process
            log = log.bind(child_pid=child_pid)
            for task in tasks:
                log.info('processing', func=task_func, task_id=task.id,
                         params={'args': task.args, 'kwargs': task.kwargs})

            # Attach a signal handler to SIGCHLD (sent when the child process
            # exits) so we can capture it.
            signal.signal(signal.SIGCHLD, sigchld_handler)

            # Since newer Python versions retry interrupted system calls we can't
            # rely on the fact that select() is interrupted with EINTR. Instead,
            # we'll set up a wake-up file descriptor below.

            # Create a new pipe and apply the non-blocking flag (required for
            # set_wakeup_fd).
            pipe_r, pipe_w = os.pipe()
            flags = fcntl.fcntl(pipe_w, fcntl.F_GETFL, 0)
            flags = flags | os.O_NONBLOCK
            fcntl.fcntl(pipe_w, fcntl.F_SETFL, flags)

            # A byte will be written to pipe_w if a signal occurs (and can be
            # read from pipe_r).
            old_wakeup_fd = signal.set_wakeup_fd(pipe_w)

            def check_child_exit():
                """
                Do a non-blocking check to see if the child process exited.
                Returns None if the process is still running, or the exit code
                value of the child process.
                """
                try:
                    pid, return_code = os.waitpid(child_pid, os.WNOHANG)
                    if pid != 0: # The child process is done.
                        return return_code
                except OSError as e:
                    # Of course EINTR can happen if the child process exits
                    # while we're checking whether it exited. In this case it
                    # should be safe to retry.
                    if e.errno == errno.EINTR:
                        return check_child_exit()
                    else:
                        raise

            # Wait for the child to exit and perform a periodic heartbeat.
            # We check for the child twice in this loop so that we avoid
            # unnecessary waiting if the child exited just before entering
            # the while loop or while renewing heartbeat/locks.
            while True:
                return_code = check_child_exit()
                if return_code is not None:
                    break

                # Wait until the timeout or a signal / child exit occurs.
                try:
                    select.select([pipe_r], [], [],
                                  self.config['ACTIVE_TASK_UPDATE_TIMER'])
                except select.error as e:
                    if e.args[0] != errno.EINTR:
                        raise

                return_code = check_child_exit()
                if return_code is not None:
                    break

                try:
                    self._heartbeat(queue, all_task_ids)
                    for lock in locks:
                        lock.renew(self.config['ACTIVE_TASK_UPDATE_TIMEOUT'])
                    if queue_lock:
                        acquired, current_locks = queue_lock.renew()
                        if not acquired:
                            log.debug('queue lock renew failure')
                except OSError as e:
                    # EINTR happens if the task completed. Since we're just
                    # renewing locks/heartbeat it's okay if we get interrupted.
                    if e.errno != errno.EINTR:
                        raise

            # Restore signals / clean up
            signal.signal(signal.SIGCHLD, signal.SIG_DFL)
            signal.set_wakeup_fd(old_wakeup_fd)
            os.close(pipe_r)
            os.close(pipe_w)

            success = (return_code == 0)
            return success
507634
def main(cls):
        
        chain = cls.create()
        args = chain._run_argparser(sys.argv[1:])
        chain._run_chain(sys.stdout, args.dry_run)
        chain._finalize(args.dry_run)