Submit Batch request to ThreatConnect API.
        Args:
            batch_id (string): The batch id of the current job.
973768
def submit_data(self, batch_id, halt_on_error=True):
        
        # check global setting for override
        if self.halt_on_batch_error is not None:
            halt_on_error = self.halt_on_batch_error

        content = self.data
        # store the length of the batch data to use for poll interval calculations
        self._batch_data_count = len(content.get('group')) + len(content.get('indicator'))
        self.tcex.log.info('Batch Size: {:,}'.format(self._batch_data_count))
        if content.get('group') or content.get('indicator'):
            headers = {'Content-Type': 'application/octet-stream'}
            try:
                r = self.tcex.session.post(
                    '/v2/batch/{}'.format(batch_id), headers=headers, json=content
                )
            except Exception as e:
                self.tcex.handle_error(10520, [e], halt_on_error)
            if not r.ok or 'application/json' not in r.headers.get('content-type', ''):
                self.tcex.handle_error(10525, [r.status_code, r.text], halt_on_error)
            return r.json()
        return {}
804
def write_command(self, request_id, msg):
        
        self.send_message(msg, 0)
        reply = self.receive_message(request_id)
        result = reply.command_response()

        # Raises NotMasterError or OperationFailure.
        helpers._check_command_response(result)
        return result
1228458
def submit(self, command_line, name = None, array = None, dependencies = [], exec_dir = None, log_dir = "logs", dry_run = False, verbosity = 0, stop_on_failure = False, **kwargs):
    """"""
    # add job to database
    self.lock()
    job = add_job(self.session, command_line, name, dependencies, array, exec_dir=exec_dir, log_dir=log_dir, stop_on_failure=stop_on_failure, context=self.context, **kwargs)
    logger.info("Added job '%s' to the database." % job)
    if dry_run:
      print("Would have added the Job")
      print(job)
      print("to the database to be executed in the grid with options:", str(kwargs))
      self.session.delete(job)
      logger.info("Deleted job '%s' from the database due to dry-run option" % job)
      job_id = None

    else:
      job_id = self._submit_to_grid(job, name, array, dependencies, log_dir, verbosity, **kwargs)

    self.session.commit()
    self.unlock()

    return job_id
664315
def submit_job(job_ini, username, hazard_job_id=None):
    
    job_id = logs.init('job')
    oq = engine.job_from_file(
        job_ini, job_id, username, hazard_calculation_id=hazard_job_id)
    pik = pickle.dumps(oq, protocol=0)  # human readable protocol
    code = RUNCALC % dict(job_id=job_id, hazard_job_id=hazard_job_id, pik=pik,
                          username=username)
    tmp_py = gettemp(code, suffix='.py')
    # print(code, tmp_py)  # useful when debugging
    devnull = subprocess.DEVNULL
    popen = subprocess.Popen([sys.executable, tmp_py],
                             stdin=devnull, stdout=devnull, stderr=devnull)
    threading.Thread(target=popen.wait).start()
    logs.dbcmd('update_job', job_id, {'pid': popen.pid})
    return job_id, popen.pid
265003
def request(self, method, args, response_cb):
        
        request_id = self._next_request_id
        self._next_request_id = request_id + 1
        self._msgpack_stream.send([0, request_id, method, args])
        self._pending_requests[request_id] = response_cb
455830
def run_attack_work(self, work_id):
    
    adv_batch_id = (
        self.attack_work.work[work_id]['output_adversarial_batch_id'])
    adv_batch = self.adv_batches[adv_batch_id]
    dataset_batch_id = adv_batch['dataset_batch_id']
    submission_id = adv_batch['submission_id']
    epsilon = self.dataset_batches[dataset_batch_id]['epsilon']
    logging.info('Attack work piece: '
                 'dataset_batch_id="%s" submission_id="%s" '
                 'epsilon=%d', dataset_batch_id, submission_id, epsilon)
    if submission_id in self.blacklisted_submissions:
      raise WorkerError('Blacklisted submission')
    # get attack
    attack = AttackSubmission(submission_id, self.submissions,
                              self.storage_bucket)
    attack.download()
    # prepare input
    input_dir = os.path.join(LOCAL_DATASET_DIR, dataset_batch_id)
    if attack.type == TYPE_TARGETED:
      # prepare file with target classes
      target_class_filename = os.path.join(input_dir, 'target_class.csv')
      self.dataset_meta.save_target_classes_for_batch(target_class_filename,
                                                      self.dataset_batches,
                                                      dataset_batch_id)
    # prepare output directory
    if os.path.exists(LOCAL_OUTPUT_DIR):
      sudo_remove_dirtree(LOCAL_OUTPUT_DIR)
    os.mkdir(LOCAL_OUTPUT_DIR)
    if os.path.exists(LOCAL_PROCESSED_OUTPUT_DIR):
      shutil.rmtree(LOCAL_PROCESSED_OUTPUT_DIR)
    os.mkdir(LOCAL_PROCESSED_OUTPUT_DIR)
    if os.path.exists(LOCAL_ZIPPED_OUTPUT_DIR):
      shutil.rmtree(LOCAL_ZIPPED_OUTPUT_DIR)
    os.mkdir(LOCAL_ZIPPED_OUTPUT_DIR)
    # run attack
    elapsed_time_sec = attack.run(input_dir, LOCAL_OUTPUT_DIR, epsilon)
    if attack.type == TYPE_TARGETED:
      # remove target class file
      os.remove(target_class_filename)
    # enforce epsilon and compute hashes
    image_hashes = eval_lib.enforce_epsilon_and_compute_hash(
        input_dir, LOCAL_OUTPUT_DIR, LOCAL_PROCESSED_OUTPUT_DIR, epsilon)
    if not image_hashes:
      logging.warning('No images saved by the attack.')
      return elapsed_time_sec, submission_id
    # write images back to datastore
    # rename images and add information to adversarial batch
    for clean_image_id, hash_val in iteritems(image_hashes):
      # we will use concatenation of batch_id and image_id
      # as adversarial image id and as a filename of adversarial images
      adv_img_id = adv_batch_id + '_' + clean_image_id
      # rename the image
      os.rename(
          os.path.join(LOCAL_PROCESSED_OUTPUT_DIR, clean_image_id + '.png'),
          os.path.join(LOCAL_PROCESSED_OUTPUT_DIR, adv_img_id + '.png'))
      # populate values which will be written to datastore
      image_path = '{0}/adversarial_images/{1}/{1}.zip/{2}.png'.format(
          self.round_name, adv_batch_id, adv_img_id)
      # u'' + foo is a a python 2/3 compatible way of casting foo to unicode
      adv_batch['images'][adv_img_id] = {
          'clean_image_id': u'' + str(clean_image_id),
          'image_path': u'' + str(image_path),
          'image_hash': u'' + str(hash_val),
      }
    # archive all images and copy to storage
    zipped_images_filename = os.path.join(LOCAL_ZIPPED_OUTPUT_DIR,
                                          adv_batch_id + '.zip')
    try:
      logging.debug('Compressing adversarial images to %s',
                    zipped_images_filename)
      shell_call([
          'zip', '-j', '-r', zipped_images_filename,
          LOCAL_PROCESSED_OUTPUT_DIR])
    except subprocess.CalledProcessError as e:
      raise WorkerError('Can''t make archive from adversarial iamges', e)
    # upload archive to storage
    dst_filename = '{0}/adversarial_images/{1}/{1}.zip'.format(
        self.round_name, adv_batch_id)
    logging.debug(
        'Copying archive with adversarial images to %s', dst_filename)
    self.storage_client.new_blob(dst_filename).upload_from_filename(
        zipped_images_filename)
    # writing adv batch to datastore
    logging.debug('Writing adversarial batch to datastore')
    self.adv_batches.write_single_batch_images_to_datastore(adv_batch_id)
    return elapsed_time_sec, submission_id
91629
def submit(self, *args, func=None, monitor=None):
        
        monitor = monitor or self.monitor
        func = func or self.task_func
        if not hasattr(self, 'socket'):  # first time
            self.__class__.running_tasks = self.tasks
            self.socket = Socket(self.receiver, zmq.PULL, 'bind').__enter__()
            monitor.backurl = 'tcp://%s:%s' % (
                config.dbserver.host, self.socket.port)
        assert not isinstance(args[-1], Monitor)  # sanity check
        dist = 'no' if self.num_tasks == 1 else self.distribute
        if dist != 'no':
            args = pickle_sequence(args)
            self.sent += numpy.array([len(p) for p in args])
        res = submit[dist](self, func, args, monitor)
        self.tasks.append(res)
415472
def send_batches(self, batch_list):
        
        if isinstance(batch_list, BaseMessage):
            batch_list = batch_list.SerializeToString()

        return self._post('/batches', batch_list)
18240
def do_batch_status(args):
    
    rest_client = RestClient(args.url, args.user)
    batch_ids = args.batch_ids.split(',')

    if args.wait and args.wait > 0:
        statuses = rest_client.get_statuses(batch_ids, args.wait)
    else:
        statuses = rest_client.get_statuses(batch_ids)

    if args.format == 'yaml':
        fmt.print_yaml(statuses)
    elif args.format == 'json':
        fmt.print_json(statuses)
    else:
        raise AssertionError('Missing handler: {}'.format(args.format))
721005
def _extract_id_from_batch_response(r, name='id'):
        """"""
        names = name + 's'
        if names in r:
            # soon-to-be deprecated batch reponse
            if 'errors' in r and r['errors']:
                raise GeneralException(r['errors'][0]['desc'])
            id = r[names][0]
        else:
            # new-style simplified api response
            id = r[name]
        return int(id)
783864
def markAsSpam(self, thread_id=None):
        
        thread_id, thread_type = self._getThread(thread_id, None)
        r = self._post(self.req_url.MARK_SPAM, {"id": thread_id})
        return r.ok
454659
def handle_batch_requests(request, *args, **kwargs):
    '''
        
    '''
    batch_start_time = datetime.now()
    try:
        # Get the Individual WSGI requests.
        wsgi_requests = get_wsgi_requests(request)
    except BadBatchRequest as brx:
        return HttpResponseBadRequest(content=brx.message)

    # Fire these WSGI requests, and collect the response for the same.
    response = execute_requests(wsgi_requests)

    # Evrything's done, return the response.
    resp = HttpResponse(
        content=json.dumps(response), content_type="application/json")

    if _settings.ADD_DURATION_HEADER:
        resp.__setitem__(_settings.DURATION_HEADER_NAME, str((datetime.now() - batch_start_time).seconds))
    return resp
1040629
def insert_request_to_batch(self, request):
        '''
        
        '''
        self.validate_request_table(request)
        self.validate_request_partition_key(request)
        self.validate_request_row_key(request)
        self.batch_requests.append(request)
989072
def get_submission_batch(submission_batch_id, opts = {})
      data, _status_code, _headers = get_submission_batch_with_http_info(submission_batch_id, opts)
      data
    end
285047
def applet_validate_batch(object_id, input_params={}, always_retry=True, **kwargs):
    
    return DXHTTPRequest('/%s/validateBatch' % object_id, input_params, always_retry=always_retry, **kwargs)
1212193
def wait_on_job(self, job_id):
        """"""

        while True:
            status = self.get_job_status(job_id)
            if status == 'SUCCEEDED':
                logger.info('Batch job {} SUCCEEDED'.format(job_id))
                return True
            elif status == 'FAILED':
                # Raise and notify if job failed
                jobs = self._client.describe_jobs(jobs=[job_id])['jobs']
                job_str = json.dumps(jobs, indent=4)
                logger.debug('Job details:\n' + job_str)

                log_stream_name = jobs[0]['attempts'][0]['container']['logStreamName']
                logs = self.get_logs(log_stream_name)
                raise BatchJobException('Job {} failed: {}'.format(
                    job_id, logs))

            time.sleep(self.poll_time)
            logger.debug('Batch job status for job {0}: {1}'.format(
                job_id, status))
848090
def submit(self, ctx: commands.Context):
        

        self.task_count += 1
        cmdtask = CommandTask(self.task_count, ctx, asyncio.Task.current_task())
        self.tasks.append(cmdtask)

        try:
            yield cmdtask
        finally:
            if cmdtask in self.tasks:
                self.tasks.remove(cmdtask)