TBD: run uploads in parallel using store.submit
559785
async function uploadFiles( store, session, caslib, datadir ) {
    let files = readdir(datadir);
    let executeCmd = session.links('execute');

    for (let i = 0 ; i < files.length ;i++) {
    
        let parms = { caslib: caslib, name: files[i],  quiet : true};
        await store.runAction(session, {action: 'table.dropTable', data: parms})

        parms = { caslib: caslib, source: files[i],quiet : true};
        await store.runAction(session, {action: 'table.deleteSource', data: parms})

        parms = {
            rstore: { name: files[i], caslib: caslib, replace: true},
            store : readFile(files[i], datadir)
        };
        payload = { action: 'aStore.upload', data: parms };
        console.log( `uploading  ${files[i]}`);
        await store.runAction(session, payload);
        
        parms = { rstore: { name: files[i], caslib: caslib } };
        payload = { action: 'aStore.describe', data: parms };
        await store.runAction(session, payload);
       
        parms = {
            caslib : caslib,
            name   : files[i],
            replace: true,
            table: {
                caslib: caslib,
                name  : files[i]
            }
        };
        payload ={action: 'table.save', data: parms};
        await store.runAction(session, payload);

    }
}
589232
function(err) {
                if (err) {
                    // eslint-disable-next-line no-console
                    console.log("\n");
                    return Promise.reject(err);
                }

                // Get 20 files
                let _files = filesNames.splice(0, batch);

                if (_files.length) {
                    return Promise.all(_files.map(performUpload)).then(() => execStack(), execStack);
                } else {
                    return Promise.resolve();
                }
            }
1023600
function mergeSubmissionFiles(params, cb) {
  logger.debug("cacheFiles: mergeSubmissionFiles", params);
  var submission = params.submission;

  //No submission, cannot continue.
  if (!_.isObject(submission)) {
    return cb(buildErrorResponse({
      err: new Error("Expected Submission Object To Cache Files For"),
      msg: "Error Exporting Submission As PDF",
      httpCode: 500
    }));
  }

  var form = submission.formSubmittedAgainst;


  var fileFieldDetails = _.filter(submission.formFields, function(field) {
    return isFileFieldType(field.fieldId.type);
  });
  fileFieldDetails = _.flatten(fileFieldDetails);


  var fieldTypes = _.groupBy(fileFieldDetails, function(field) {
    return field.fieldId.type;
  });

  //Files That Have To Be Loaded From Mbaas
  var mbaasTypes = _.union(fieldTypes.photo, fieldTypes.signature);

  //Only Interested In The File Ids To Download
  var filesToDownload = _.map(mbaasTypes, function(field) {
    return _.map(field.fieldValues, function(fieldValue) {
      return fieldValue.groupId;
    });
  });

  filesToDownload = _.flatten(filesToDownload);

  logger.debug("cacheFiles: mergeSubmissionFiles filesToDownload", filesToDownload);


  //Now download all of the files..
  async.mapSeries(filesToDownload, function(fileIdOrUrl, cb) {
    //Only want the callback to be called once.
    cb = _.once(cb);

    var fileUri = path.join(params.options.pdfExportDir, 'image_binary_' + fileIdOrUrl);
    var localFile = fs.createWriteStream(fileUri);

    //Loading The Submission File From THe Database / MbaaS
    cacheSubmissionFile({
      fileId: fileIdOrUrl,
      connections: params.connections,
      options: params.options,
      submission: submission
    }, localFile, function(err, fileDetails) {
      if (err) {
        logger.error("cacheFiles: cacheSubmissionFile", {error: err});
      }
      fileDetails = fileDetails || {};
      fileDetails.url = "file://" + fileUri;
      fileDetails.fileId = fileIdOrUrl;

      logger.debug("cacheFiles: mergeSubmissionFiles fileDetails", fileDetails);

      return cb(err, _.omit(fileDetails, "stream"));
    });
  }, function(err, cachedFiles) {
    if (err) {
      return cb(buildErrorResponse({
        error: err,
        msg: "Error Cacheing Files For Submission Export",
        code: constants.ERROR_CODES.FH_FORMS_ERR_CODE_PDF_GENERATION,
        httpCode: 500
      }));
    }

    //No Errors, all files are now cached for the submission

    submission.formSubmittedAgainst =  populateSubmissionFileData({
      form: form,
      submissionFiles: cachedFiles,
      downloadUrl: params.options.downloadUrl,
      fileUriPath: params.options.fileUriPath,
      submission: submission
    });

    logger.debug("cacheFiles: mergeSubmissionFiles submission", submission);

    cb(undefined, submission);
  });
}
245728
function(next) {
        var source = config.getSource(asset.source);

        async.parallel([
          getFromSource.bind(null, source, asset, fromVersion, config),
          getFromSource.bind(null, source, asset, toVersion, config)
        ], next);
      }
251121
def submission_post_save(sender, instance, **kwargs):
    ''' '''
    logger.debug("Running post-processing for submission")
    # Make the submitter an author
    if instance.submitter not in instance.authors.all():
        instance.authors.add(instance.submitter)
        instance.save()
    # Mark all existing submissions for this assignment by these authors as invalid.
    # This fixes a race condition with parallel new submissions in multiple browser windows by the same user.
    # Solving this as pre_save security exception does not work, since we have no instance with valid foreign keys to check there.
    # Considering that this runs also on tutor correction backend activities, it also serves as kind-of cleanup functionality
    # for multiplse submissions by the same students for the same assignment - however they got in here.
    if instance.state == instance.get_initial_state():
        for author in instance.authors.all():
            same_author_subm = User.objects.get(pk=author.pk).authored.all().exclude(
                pk=instance.pk).filter(assignment=instance.assignment)
            for subm in same_author_subm:
                subm.state = Submission.WITHDRAWN
                subm.save()
1201493
function writeblobs(data, cb) {
    var ab = asyncbuilder(cb);
    var reqQ = new Queue( { concurrency: 5 } );

    u.each(data, function(file) {
      var append = ab.asyncAppend();
      var blob = { encoding: file.buffer ? 'base64' : 'utf-8',
                   content: file.buffer ? file.buffer.toString('base64') : (file.text || '') };

      reqQ.push(function(reqDone) {
        post(self.endpoint + '/git/blobs', blob, function(err, ghblob) {
          if (ghblob) { file.sha = ghblob.sha; }
          append(err, file);
          reqDone();
        });
      });

    });
    ab.complete();
  }
837533
def submit(self):

        
        futures = []
        while self.submitted < 4 and not self.done():
            part = self.parts.pop(0)
            part_number = part['part']
            part_read_offset = part['offset']
            part_read_limit = part['limit']

            self.fp.seek(part_read_offset)
            part_data = self.fp.read(part_read_limit - part_read_offset)

            futures.append(
                self.pool.submit(
                    _upload_part, self.api, self.session,
                    self._URL['upload_part'], self.upload_id,
                    part_number, part_data, self.retry, self.timeout
                )
            )

            self.submitted += 1
            self.total_submitted += 1

        return futures
457245
CompleteMultipartUploadResult uploadObject(final UploadObjectRequest req)
            throws IOException, InterruptedException, ExecutionException {
        // Set up the pipeline for concurrent encrypt and upload
        // Set up a thread pool for this pipeline
        ExecutorService es = req.getExecutorService();
        final boolean defaultExecutorService = es == null;
        if (es == null)
            es = Executors.newFixedThreadPool(clientConfiguration.getMaxConnections());
        UploadObjectObserver observer = req.getUploadObjectObserver();
        if (observer == null)
            observer = new UploadObjectObserver();
        // initialize the observer
        observer.init(req, this, this, es);
        // Initiate upload
        observer.onUploadInitiation(req);
        final List<PartETag> partETags = new ArrayList<PartETag>();
        MultiFileOutputStream mfos = req.getMultiFileOutputStream();
        if (mfos == null)
            mfos = new MultiFileOutputStream();
        try {
            // initialize the multi-file output stream
            mfos.init(observer, req.getPartSize(), req.getDiskLimit());
            // Kicks off the encryption-upload pipeline;
            // Note mfos is automatically closed upon method completion.
            putLocalObject(req, mfos);
            // block till all part have been uploaded
            for (Future<UploadPartResult> future: observer.getFutures()) {
                UploadPartResult partResult = future.get();
                partETags.add(new PartETag(partResult.getPartNumber(), partResult.getETag()));
            }
        } finally {
            if (defaultExecutorService)
                es.shutdownNow();   // shut down the locally created thread pool
            mfos.cleanup();       // delete left-over temp files
        }
        // Complete upload
        return observer.onCompletion(partETags);
    }
1149918
function fileUploadRetry(fileURI, serverURI, fileUploadOptions, timeout, retries) {
  return fileUpload(fileURI, serverURI, fileUploadOptions)
  .then(function(response) {
    return response;
  }, function() {
    if (retries === 0) {
      throw new Error("Can't upload to " + JSON.stringify(serverURI));
    }
    return q.delay(timeout)
    .then(function() {
      return fileUploadRetry(fileURI, serverURI, fileUploadOptions, timeout, retries - 1);
    });
  });
}
187698
function run (files, callback) {
  let self = this,
    i = 0;

  function next(err, files) {
    let mw = self.middleware[i++];

    if (mw) {
      mw(files, self, next);
    }
    else {
      callback(null, files);
    }

  };

  next(null, files);
}
283489
function(id, params){
        var len = this._queue.push(id);

        var copy = {};        
        qq.extend(copy, params);
        this._params[id] = copy;        
                
        // if too many active uploads, wait...
        if (len <= this._options.maxConnections){               
            this._upload(id, this._params[id]);
        }
    }
77688
def Run(self, request):
    """"""

    self.request = request
    filters = self.BuildChecks(request)

    files_checked = 0
    for f in self.ListDirectory(request.pathspec):
      self.Progress()

      # Ignore this file if any of the checks fail.
      if not any((check(f) for check in filters)):
        self.SendReply(f)

      files_checked += 1
      if files_checked >= self.MAX_FILES_TO_CHECK:
        return
175045
function (data, callback) {
    if (typeof txDataQueue != "undefined" || txInProgress) {
      if (txDataQueue === undefined)
        txDataQueue = "";
      txDataQueue += data;
      return callback();
    } else {
      txDataQueue = data;
    }

    function writeChunk() {
      var chunk;
      var CHUNKSIZE = 20;
      if (txDataQueue.length <= CHUNKSIZE) {
        chunk = txDataQueue;
        txDataQueue = undefined;
      } else {
        chunk = txDataQueue.substr(0, CHUNKSIZE);
        txDataQueue = txDataQueue.substr(CHUNKSIZE);
      }
      txInProgress = true;
      console.log("BT> Sending " + JSON.stringify(chunk));
      try {
        winnus.write(chunk);
        setTimeout(function() {
          txInProgress = false;
          if (txDataQueue)
            writeChunk();
        }, 20);
      } catch (e) {
        console.log("BT> ERROR " + e);
        txDataQueue = undefined;
      }
    }
    writeChunk();
    return callback();
  }
763388
function(req, res, next) {
        async.parallel({
            applications: (fn) => {
                const spy = { stash: {} };
                applications.get(req, spy, () => {
                    if(spy.stash.code !== 200) {
                        return fn(new Error('Error getting applications.'));
                    }

                    return fn(null, spy.stash.body);
                });
            },
            hosts: (fn) => {
                const spy = { stash: {} };
                hosts.get(req, spy, () => {
                    if(spy.stash.code !== 200) {
                        return fn(new Error('Error getting hosts.'));
                    }

                    return fn(null, spy.stash.body);
                });
            }
        }, (err, results) => {
            if(err) {
                res.stash.code = 500;
                return next();
            }

            res.stash.body = {
                applications: results.applications,
                hosts: results.hosts
            };
            res.stash.code = 200;
            return next();
        });
    }
236541
def submit(self, fn, *args, **kwargs):
        """"""
        corofn = asyncio.coroutine(lambda: fn(*args, **kwargs))
        return run_coroutine_threadsafe(corofn(), self.loop)
418437
private void queueOrSubmitUpload(String key, File file) {
        if (mReadyToBroadcast) {
            submitUpload(key, file);
        } else {
            if (VERBOSE) Log.i(TAG, "queueing " + key + " until S3 Credentials available");
            queueUpload(key, file);
        }
    }
1200285
@SuppressWarnings("unchecked")
	private <T extends Resource> T submit(Form form, Parameters parameters) {
		RequestBuilder<Resource> request = doRequest(form.getUri()).set(parameters);
		boolean doPost = form.isDoPost();
		boolean multiPart = form.isMultiPart();
		if(doPost && multiPart) {
			request.multiPart();
			addFiles(request, form);
		}
		return (T) (doPost ? request.post() : request.get()); 
	}