Function for processing data in worker process.
556930
def _worker_fn(samples, batchify_fn, dataset=None):
    """"""
    # pylint: disable=unused-argument
    # it is required that each worker process has to fork a new MXIndexedRecordIO handle
    # preserving dataset as global variable can save tons of overhead and is safe in new process
    global _worker_dataset
    batch = batchify_fn([_worker_dataset[i] for i in samples])
    buf = io.BytesIO()
    ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(batch)
    return buf.getvalue()
614522
def process_data(self, new_data):
        
        self.buffer.feed(new_data)

        # process each non-empty line after logging all lines
        for line in self.buffer:
            log.debug("FROM SERVER: %s", line)
            if not line:
                continue
            self._process_line(line)
992696
def _get_run_by_other_worker(worker):
    
    task_sets = _get_external_workers(worker).values()
    return functools.reduce(lambda a, b: a | b, task_sets, set())
185219
function WorkingQueue(concurrency) {
    // Parallelization degree, if not mentioned use 64
    // Don't forget that we are in a SINGLE-threaded process
    // All we want it to somewhat limit the number of simultaneous
    // I/O requests
    concurrency = concurrency || 64;


    // The queue contains tuples :
    // [job, arg1, arg2, arg3...]
    var queue = [];
    var workers = 0;
    var onceDoneCallbacks = [];
    var held = false;
    var added = false;

    // This function launches a new worker, if possible
    // It returns true if a new worker was launched, false otherwise.
    function _go() {
        if(workers < concurrency) {
            var job = queue.shift();
            if(job !== undefined) {
                var worker = job.shift();
                job.push(over);
                workers++;
                process.nextTick(function nextTickWorker() {
                    worker.apply(null, job);
                });
                return true;
            }
        }
        return false;
    }

    // This function is passed to each job, it must be called
    // when the job is done.
    function over() {
        workers--;

        if(!held) { _go(); }

        checkDone();
    }

    function checkDone() {
        if(queue.length === 0 && workers === 0) {
            var i, l;

            for(i=0, l=onceDoneCallbacks.length; i < l; i++) {
                process.nextTick(onceDoneCallbacks[i]);
            }

            // Reset callbacks & status
            onceDoneCallbacks = [];
            added = false;
        }
    }

    // Enqueue a new job and ends the current one
    over.then = function overThen(/*...*/) {
        // Since it is only possible to get the over function
        // while running a job, no need to call this.perform
        queue.push(Array.prototype.slice.call(arguments, 0));
        added = true;
        return over();
    };

    // Main method : launches a job immediatly or queue it
    // if maximum number of workers has been reached
    this.perform = function perform(/*...*/) {
        queue.push(Array.prototype.slice.call(arguments, 0));
        added = true;
        if(!held) { _go(); }
        return this;
    };

    // Apply the worker function to each element in the list
    // The worker is in the form function(index, element /*, ...*/, over)
    this.processList = function processList(worker, list /*, ... */) {
        var length = list.length;
        var args, extra;

        if(length>0) {
            extra = Array.prototype.slice.call(arguments, 2);

            for(var i=0;i<length;i++) {
                args = [worker, i, list[i]].concat(extra);
                queue.push(args);
            }

            added = true;
            if(!held) { _go(); }
        }

        return this;
    };

    // Apply the worker function to each element in the dictionary
    // The worker is in the form function(key, value, /*, ... */, over)
    this.processObject = function processObject(worker, dict /*, ... */) {
        var args;
        var extra = Array.prototype.slice.call(arguments, 2);

        for(var key in dict) {
            args = [worker, key, dict[key]].concat(extra);
            queue.push(args);
            added = true;
        }

        if(!held) { _go(); }
        return this;
    };

    // Register a callback that will be called when
    // all jobs have been done.
    // If a callback is registered after the jobs are over,
    // it is called immediatly
    this.onceDone = function onceDone(fun) {
        onceDoneCallbacks.push(fun);
        if(added) { checkDone(); }
    };

    this.whenDone = util.deprecate(this.onceDone, "capisce.WorkingQueue.whenDone is deprecated, use onceDone instead.");

    this.doneAddingJobs = function doneAddingJobs() {
        if(!added) { checkDone(); }
    };

    // Holds the WorkingQueue. No job will be executed until
    // the go() method is called
    this.hold = function hold() {
        held = true;
    };

    // Launches the scheduled jobs
    this.go = function go() {
        if(held) {
            held = false;

            // Loop to launch as many workers as possible
            while(_go()) {}
        }
    };

    // This method is used for a little DSL that can build sequence of
    // jobs, as well as concurrent sections within those sequences.
    this.then = function then(block, concurrency2) {
        if(concurrency>1) { throw 'Cannot use then in a concurrent section'; }

        if(block.length === 1) {
            if(arguments.length !== 1) {
                throw 'Cannot specifiy a concurrency level for sequences';
            }
            this.perform(block);
        } else if(block.length === 2) {
            this.perform(function worker(over) {
                var concurrently = new WorkingQueue(concurrency2);
                block(concurrently, function over2() {
                    concurrently.onceDone(over);
                });
            });
        } else {
            throw 'then accepts building function with one (sequence) or two (concurrent) arguments';
        }

        return this;
    };

    function waiter(timeout, job, over) {
       setTimeout(function waiter() {
            if(job.length>0) {
                var func = job.shift();
                job.push(over);
                return func.apply(null, job);
            }
            else {
                return over();
            }
       }, timeout);
    }

    // This method waits the given timeout then executes the facultative
    // job.
    // This can be used in two ways :
    // sequence().wait(5000).then(function(over) {...})
    // OR
    // sequence().wait(5000,function(over) {...})s
    this.wait = function wait(timeout /*...*/) {
        var job = Array.prototype.slice.call(arguments, 1);
        this.perform(waiter, timeout, job);
        return this;
    };
}
60122
def worker_function(
    event_type, assignment_id, participant_id, node_id=None, details=None
):
    """"""
    _config()
    try:
        db.logger.debug(
            "rq: worker_function working on job id: %s", get_current_job().id
        )
        db.logger.debug(
            "rq: Received Queue Length: %d (%s)", len(q), ", ".join(q.job_ids)
        )
    except AttributeError:
        db.logger.debug("Debug worker_function called synchronously")

    exp = Experiment(session)
    key = "-----"

    exp.log(
        "Received an {} notification for assignment {}, participant {}".format(
            event_type, assignment_id, participant_id
        ),
        key,
    )

    if event_type == "TrackingEvent":
        node = None
        if node_id:
            node = models.Node.query.get(node_id)
        if not node:
            participant = None
            if participant_id:
                # Lookup assignment_id to create notifications
                participant = models.Participant.query.get(participant_id)
            elif assignment_id:
                participants = models.Participant.query.filter_by(
                    assignment_id=assignment_id
                ).all()
                # if there are one or more participants select the most recent
                if participants:
                    participant = max(participants, key=attrgetter("creation_time"))
                    participant_id = participant.id
            if not participant:
                exp.log(
                    "Warning: No participant associated with this "
                    "TrackingEvent notification.",
                    key,
                )
                return
            nodes = participant.nodes()
            if not nodes:
                exp.log(
                    "Warning: No node associated with this "
                    "TrackingEvent notification.",
                    key,
                )
                return
            node = max(nodes, key=attrgetter("creation_time"))

        if not details:
            details = {}
        info = information.TrackingEvent(origin=node, details=details)
        session.add(info)
        session.commit()
        return

    runner_cls = WorkerEvent.for_name(event_type)
    if not runner_cls:
        exp.log("Event type {} is not supported... ignoring.".format(event_type))
        return

    if assignment_id is not None:
        # save the notification to the notification table
        notif = models.Notification(assignment_id=assignment_id, event_type=event_type)
        session.add(notif)
        session.commit()

        # try to identify the participant
        participants = models.Participant.query.filter_by(
            assignment_id=assignment_id
        ).all()

        # if there are one or more participants select the most recent
        if participants:
            participant = max(participants, key=attrgetter("creation_time"))

        # if there are none print an error
        else:
            exp.log(
                "Warning: No participants associated with this "
                "assignment_id. Notification will not be processed.",
                key,
            )
            return None

    elif participant_id is not None:
        participant = models.Participant.query.filter_by(id=participant_id).all()[0]
    else:
        raise ValueError(
            "Error: worker_function needs either an assignment_id or a "
            "participant_id, they cannot both be None"
        )

    participant_id = participant.id

    runner = runner_cls(
        participant, assignment_id, exp, session, _config(), datetime.now()
    )
    runner()
    session.commit()
97311
def run_job(self):
        """"""
        try:
            self.load_parsers()
            self.load_filters()
            self.load_outputs()
            self.config_args()
            if self.args.list_parsers:
                self.list_parsers()
            if self.args.verbosemode: print('Loading input files')
            self.load_inputs()
            if self.args.verbosemode: print('Running parsers')
            self.run_parse()
            if self.args.verbosemode: print('Merging data')
            self.data_set['finalized_data'] = \
                    logdissect.utils.merge_logs(
                            self.data_set['data_set'], sort=True)
            if self.args.verbosemode: print('Running filters')
            self.run_filters()
            if self.args.verbosemode: print('Running output')
            self.run_output()
        except KeyboardInterrupt:
            sys.exit(1)
230107
function worker () {
  var indexedDB = self.indexedDB
  var db
  var methodMap = {
    connect: connect,
    disconnect: disconnect,
    create: create,
    update: update,
    delete: remove,
    deleteAll: removeAll
  }

  self.addEventListener('message', function (event) {
    var data = event.data
    var id = data.id
    var method = data.method

    methodMap[method](data, function (error, result, transfer) {
      if (error) {
        self.postMessage({
          id: id, error: '' + error
        })
        return
      }

      self.postMessage({
        id: id, result: result
      }, transfer)
    })
  })


  function connect (data, callback) {
    var request = indexedDB.open(data.name)
    var typesArray = data.typesArray

    request.onerror = errorConnection
    request.onupgradeneeded = handleUpgrade
    request.onsuccess = handleSuccess

    function handleSuccess (event) {
      var i, j

      db = event.target.result

      for (i = 0, j = typesArray.length; i < j; i++)
        if (!~Array.prototype.indexOf.call(
          db.objectStoreNames, typesArray[i])) {
          reconnect()
          return
        }

      loadRecords()
    }

    function handleUpgrade (event) {
      var i, j, type

      db = event.target.result

      for (i = 0, j = typesArray.length; i < j; i++) {
        type = typesArray[i]
        if (!~Array.prototype.indexOf.call(db.objectStoreNames, type))
          db.createObjectStore(type, { keyPath: primaryKey })
      }

      for (i = 0, j = db.objectStoreNames.length; i < j; i++) {
        type = db.objectStoreNames[i]
        if (!~Array.prototype.indexOf.call(typesArray, type))
          db.deleteObjectStore(type)
      }
    }

    function reconnect () {
      var version = (db.version || 1) + 1

      db.close()
      request = indexedDB.open(data.name, version)
      request.onerror = errorReconnection
      request.onupgradeneeded = handleUpgrade
      request.onsuccess = function (event) {
        db = event.target.result
        loadRecords(db)
      }
    }

    function loadRecords () {
      var counter = 0
      var payload = {}
      var transfer = []
      var i, j

      for (i = 0, j = typesArray.length; i < j; i++)
        loadType(typesArray[i])

      function loadType (type) {
        var transaction = db.transaction(type, 'readonly')
        var objectStore = transaction.objectStore(type)
        var cursor = objectStore.openCursor()

        payload[type] = []
        cursor.onsuccess = function (event) {
          var iterator = event.target.result
          if (iterator) {
            payload[type].push(iterator.value[dataKey])
            transfer.push(iterator.value[dataKey].buffer)
            iterator.continue()
            return
          }
          counter++
          if (counter === typesArray.length)
            callback(null, payload, transfer)
        }
        cursor.onerror = errorIteration
      }
    }

    function errorConnection () {
      callback('The database connection could not be established.')
    }

    function errorReconnection () {
      callback('An attempt to reconnect failed.')
    }

    function errorIteration () {
      callback('Failed to read record.')
    }
  }


  function disconnect () {
    db.close()
  }


  function create (data, callback) {
    var recordsLength = Object.keys(data.records).length
    var type = data.type
    var transaction = db.transaction(type, 'readwrite')
    var objectStore = transaction.objectStore(type)
    var id, record, object, request, counter = 0

    for (id in data.records) {
      record = data.records[id]
      object = {}
      object[primaryKey] = type + delimiter + id
      object[dataKey] = record
      request = objectStore.add(object)
      request.onsuccess = check
      request.onerror = error
    }

    function check () {
      counter++
      if (counter === recordsLength) callback()
    }

    function error () {
      callback('A record could not be created.')
    }
  }


  function update (data, callback) {
    var recordsLength = Object.keys(data.records).length
    var type = data.type
    var transaction = db.transaction(type, 'readwrite')
    var objectStore = transaction.objectStore(type)
    var id, record, object, request, counter = 0

    for (id in data.records) {
      record = data.records[id]
      object = {}

      // Here it should be assumed that the type and delimiter are already
      // supplied, rather than needing to add it here.
      object[primaryKey] = id

      object[dataKey] = record
      request = objectStore.put(object)
      request.onsuccess = check
      request.onerror = error
    }

    function check () {
      counter++
      if (counter === recordsLength) callback()
    }

    function error () {
      callback('A record could not be updated.')
    }
  }


  function remove (data, callback) {
    var type = data.type
    var ids = data.ids
    var transaction = db.transaction(type, 'readwrite')
    var objectStore = transaction.objectStore(type)
    var i, j, id, request, counter = 0

    for (i = 0, j = ids.length; i < j; i++) {
      id = ids[i]
      request = objectStore.delete(type + delimiter + id)
      request.onsuccess = check
      request.onerror = error
    }

    function check () {
      counter++
      if (counter === ids.length) callback()
    }

    function error () {
      callback('A record could not be deleted.')
    }
  }


  function removeAll (data, callback) {
    var type = data.type
    var transaction = db.transaction(type, 'readwrite')
    var objectStore = transaction.objectStore(type)
    var request = objectStore.clear()
    request.onsuccess = function () { callback() }
    request.onerror = error

    function error () {
      callback('Not all records could be deleted.')
    }
  }
}
804417
def myFunc(parameter):
    """"""
    print('Hello World from {0}!'.format(scoop.worker))

    # It is possible to get a constant anywhere
    print(shared.getConst('myVar')[2])

    # Parameters are handled as usual
    return parameter + 1
723259
def matrix_worker(data):
    
    matrix = data['matrix']
    Logger.get_logger(__name__ + '.worker').info(
        "Processing pipeline for matrix entry '%s'", matrix['name'])

    env = matrix['env'].copy()
    env.update({'PIPELINE_MATRIX': matrix['name']})

    pipeline = Pipeline(model=data['model'], env=env, options=data['options'])
    pipeline.hooks = data['hooks']
    return pipeline.process(data['pipeline'])
1010678
def process(self, data, **kwargs):
        """"""
        data = super(RequestIdProcessor, self).process(data, **kwargs)
        if g and hasattr(g, 'request_id'):
            tags = data.get('tags', {})
            tags['request_id'] = g.request_id
            data['tags'] = tags
        return data
844291
def _worker_fn(samples, batchify_fn, dataset=None):
    """"""
    # pylint: disable=unused-argument
    # it is required that each worker process has to fork a new MXIndexedRecordIO handle
    # preserving dataset as global variable can save tons of overhead and is safe in new process
    global _worker_dataset
    if isinstance(samples[0], (list, tuple)):
        batch = [batchify_fn([_worker_dataset[i] for i in shard]) for shard in samples]
    else:
        batch = batchify_fn([_worker_dataset[i] for i in samples])
    buf = io.BytesIO()
    ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(batch)
    return buf.getvalue()
222088
def process(ctx, dataset, kwargs):
    ""

    kwargs = parse_kwargs(kwargs)
    data(dataset, **ctx.obj).process(**kwargs)
414637
def apply(self, func, workers=1, job_size=10000):
    
    if workers == 1:
      for lines in self.iter_chunks(job_size):
        yield func(lines)
    else:
      with ProcessPoolExecutor(max_workers=workers) as executor:
        for result in executor.map(func, self.iter_chunks(job_size)):
          yield result
400934
def immutable_worker(worker, state, pre_state, created):
    """"""
    # noinspection PyUnresolvedReferences,PyProtectedMember
    return WorkerData._make(chain(
        (getattr(worker, f) for f in WORKER_OWN_FIELDS),
        (state, pre_state, created),
        (worker.heartbeats[-1] if worker.heartbeats else None,),
    ))
243374
def _start_new_worker_process(self, server_socket):
        """"""
        from multiprocessing import Process
        p = Process(target=forked_child, args=self._get_child_args(server_socket))
        p.start()
        return p
416669
def process_data(self, file_info):
        """"""
        if self._is_already_processed(file_info):
            self.log.debug("Content file already processed '%s'", str(file_info))
            self.fire(events.FilteredFile(file_info))
            self.fire(events.FileInfoAlreadyProcessed(file_info))
        else:
            return file_info
        return None
1050000
def _initialize_processes(self, target, num_workers, description):
        # type: (_MultiprocessOffload, function, int, str) -> None
        
        if num_workers is None or num_workers < 1:
            raise ValueError('invalid num_workers: {}'.format(num_workers))
        logger.debug('initializing {}{} processes'.format(
            num_workers, ' ' + description if not None else ''))
        for _ in range(num_workers):
            proc = multiprocessing.Process(target=target)
            proc.start()
            self._procs.append(proc)