Worker thread transfer
        :param Uploader self: this
549514
def _worker_thread_transfer(self):
        # type: (Uploader) -> None
        
        while not self.termination_check:
            try:
                ud, ase, offsets, data = self._transfer_queue.get(
                    block=False, timeout=0.1)
            except queue.Empty:
                continue
            try:
                self._process_transfer(ud, ase, offsets, data)
            except Exception as e:
                with self._upload_lock:
                    self._exceptions.append(e)
1355389
def run(self):
        
        progress_printer = ProgressPrinter(self.different_items.total_items(), msg_verb='sending')
        upload_settings = UploadSettings(self.config, self.remote_store.data_service, progress_printer,
                                         self.project_name_or_id, self.file_upload_post_processor)
        project_uploader = ProjectUploader(upload_settings)
        project_uploader.run(self.local_project)
        progress_printer.finished()
489787
private <RequestT, ResponseT> ResponseT uploadObject(final InputStream originalStream,
                                                         final File file,
                                                         final ObjectMetadata metadata,
                                                         final ProgressListener listener,
                                                         final Request<RequestT> request,
                                                         final S3DataSource originalRequest,
                                                         final boolean skipServerSideValidation,
                                                         final boolean skipClientSideValidationPerRequest,
                                                         final UploadObjectStrategy<RequestT, ResponseT> uploadStrategy,
                                                         final boolean setContentTypeIfNotProvided) {

        InputStream input = getInputStream(originalStream, file, metadata, request,
                                           skipServerSideValidation, setContentTypeIfNotProvided);

        final ObjectMetadata returnedMetadata;
        MD5DigestCalculatingInputStream md5DigestStream = null;
        try {
            if (metadata.getContentMD5() == null && !skipClientSideValidationPerRequest) {
                /*
                 * If the user hasn't set the content MD5, then we don't want to buffer the whole
                 * stream in memory just to calculate it. Instead, we can calculate it on the fly
                 * and validate it with the returned ETag from the object upload.
                 */
                input = md5DigestStream = new MD5DigestCalculatingInputStream(input);
            }

            populateRequestMetadata(request, metadata);
            request.setContent(input);
            publishProgress(listener, ProgressEventType.TRANSFER_STARTED_EVENT);
            try {
                returnedMetadata = uploadStrategy.invokeServiceCall(request);
            } catch (Throwable t) {
                publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT);
                throw failure(t);
            }
        } finally {
            cleanupDataSource(originalRequest, file, originalStream, input, log);
        }

        String contentMd5 = metadata.getContentMD5();
        if (md5DigestStream != null) {
            contentMd5 = Base64.encodeAsString(md5DigestStream.getMd5Digest());
        }

        final String etag = returnedMetadata.getETag();
        if (contentMd5 != null && !skipMd5CheckStrategy.skipClientSideValidationPerPutResponse(returnedMetadata)) {
            byte[] clientSideHash = BinaryUtils.fromBase64(contentMd5);
            byte[] serverSideHash = BinaryUtils.fromHex(etag);

            if (!Arrays.equals(clientSideHash, serverSideHash)) {
                publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT);
                throw new SdkClientException(
                    "Unable to verify integrity of data upload. Client calculated content hash (contentMD5: " + contentMd5
                    + " in base 64) didn't match hash (etag: " + etag + " in hex) calculated by Amazon S3.  "
                    + "You may need to delete the data stored in Amazon S3. (metadata.contentMD5: " + metadata.getContentMD5()
                    + ", md5DigestStream: " + md5DigestStream
                    + uploadStrategy.md5ValidationErrorSuffix()
                    + ")");
            }
        }

        publishProgress(listener, ProgressEventType.TRANSFER_COMPLETED_EVENT);

        return uploadStrategy.createResult(returnedMetadata, contentMd5);
    }
342877
def transaction(self, func, *watches, **kwargs):
        
        with self.pipeline(True, share_in_threads=kwargs.get('share_in_threads', False)) as pipe:
            while 1:
                try:
                    if watches:
                        pipe.watch(*watches)
                    func(pipe)
                    return pipe.execute()
                except WatchError:
                    continue
1265633
public void upload(IProgressMonitor monitor, File file) {

    final int taskSize = 1024;

    monitor.setTaskName("Upload file " + this.path);

    BufferedInputStream istream = null;
    DataOutputStream ostream = null;

    try {
      istream = new BufferedInputStream(new FileInputStream(file));
      ostream = getDFS().create(this.path);

      int bytes;
      byte[] buffer = new byte[taskSize];

      while ((bytes = istream.read(buffer)) >= 0) {
        if (monitor.isCanceled())
          return;
        ostream.write(buffer, 0, bytes);
        monitor.worked(1);
      }

    } catch (Exception e) {
      ErrorMessageDialog.display(String.format(
          "Unable to uploade file %s to %s", file, this.path), e
          .getLocalizedMessage());

    } finally {
      try {
        if (istream != null)
          istream.close();
      } catch (IOException e) {
        e.printStackTrace();
        // nothing we can do here
      }
      try {
        if (ostream != null)
          ostream.close();
      } catch (IOException e) {
        e.printStackTrace();
        // nothing we can do here
      }
    }
  }
1125121
def upload(self, file_obj):
        
        return self._client.upload_object(
            self._instance, self._bucket, self.name, file_obj)
364950
def set_uploaded(self, files_uploaded):
        
        self.files_uploaded = files_uploaded
        self.__record_progress(Status.UPLOAD_CHANNEL)
330330
public void asyncPut(final byte[] data, final String key, final String token, StringMap params,
                         String mime, boolean checkCrc, UpCompletionHandler handler) throws IOException {
        checkArgs(key, data, null, token);
        if (mime == null) {
            mime = Client.DefaultMime;
        }
        params = filterParam(params);
        new FormUploader(client, token, key, data, params, mime, checkCrc, configuration).asyncUpload(handler);
    }
986410
private void transfer(AtomicReferenceArray src, ResizeContainer resizeContainer)
    {
        AtomicReferenceArray dest = resizeContainer.nextArray;

        for (int j = 0; j < src.length() - 1; )
        {
            Object o = src.get(j);
            if (o == null)
            {
                if (src.compareAndSet(j, null, RESIZED))
                {
                    j++;
                }
            }
            else if (o == RESIZED || o == RESIZING)
            {
                j = (j & ~(ResizeContainer.QUEUE_INCREMENT - 1)) + ResizeContainer.QUEUE_INCREMENT;
                if (resizeContainer.resizers.get() == 1)
                {
                    break;
                }
            }
            else
            {
                Entry<K, V> e = (Entry<K, V>) o;
                if (src.compareAndSet(j, o, RESIZING))
                {
                    while (e != null)
                    {
                        this.unconditionalCopy(dest, e);
                        e = e.getNext();
                    }
                    src.set(j, RESIZED);
                    j++;
                }
            }
        }
        resizeContainer.decrementResizerAndNotify();
        resizeContainer.waitForAllResizers();
    }
664057
def _sendFiles(
        self, files, message=None, thread_id=None, thread_type=ThreadType.USER
    ):
        
        thread_id, thread_type = self._getThread(thread_id, thread_type)
        data = self._getSendData(
            message=self._oldMessage(message),
            thread_id=thread_id,
            thread_type=thread_type,
        )

        data["action_type"] = "ma-type:user-generated-message"
        data["has_attachment"] = True

        for i, (file_id, mimetype) in enumerate(files):
            data["{}s[{}]".format(mimetype_to_key(mimetype), i)] = file_id

        return self._doSendRequest(data)
381692
def set_uploading(self, files_uploaded):
        
        self.files_uploaded = files_uploaded
        self.__record_progress(Status.UPLOADING_FILES)
1041485
def start(self):
        # type: (Uploader) -> None
        
        try:
            blobxfer.operations.progress.output_parameters(
                self._general_options, self._spec)
            self._run()
        except (KeyboardInterrupt, Exception) as ex:
            if isinstance(ex, KeyboardInterrupt):
                logger.info(
                    'KeyboardInterrupt detected, force terminating '
                    'processes and threads (this may take a while)...')
            else:
                logger.exception(ex)
            self._wait_for_transfer_threads(terminate=True)
            self._wait_for_disk_threads(terminate=True)
        finally:
            # shutdown processes
            if self._md5_offload is not None:
                self._md5_offload.finalize_processes()
            if self._crypto_offload is not None:
                self._crypto_offload.finalize_processes()
            # close resume file
            if self._resume is not None:
                self._resume.close()
481578
function Worker(file) {
  if (!(this instanceof Worker))
    return new Worker(file);

  EventEmitter.call(this);

  this.id = -1;
  this.framer = new Framer();
  this.parser = new Parser();
  this.pending = new Map();

  this.child = new Child(file);

  this.init();
}
760153
def _upload_folder_in_background(self, folder_path, container, ignore,
            upload_key, ttl=None):
        """"""
        uploader = FolderUploader(folder_path, container, ignore, upload_key,
                self, ttl=ttl)
        uploader.start()
1288680
def flush(self, multithread=True, **kwargs):
        '''
        
        '''
        if self._write_buf.tell() > 0:
            data = self._write_buf.getvalue()
            self._write_buf = BytesIO()

            if multithread:
                self._async_upload_part_request(data, index=self._cur_part, **kwargs)
            else:
                self.upload_part(data, self._cur_part, **kwargs)

            self._cur_part += 1

        if len(self._http_threadpool_futures) > 0:
            dxpy.utils.wait_for_all_futures(self._http_threadpool_futures)
            try:
                for future in self._http_threadpool_futures:
                    if future.exception() != None:
                        raise future.exception()
            finally:
                self._http_threadpool_futures = set()
545573
function(pack) {
    if (cluster.isMaster) {
      if (pack.toWorkers === true) {
        for (var key in cluster.workers) {
          cluster.workers[key].send(pack)
        }
      }
    }
    else {
      if (pack.toMaster === true || pack.toWorkers === true) {
        process.send(pack)
      }
    }
  }
526763
function (event) {
                // Upload progress events are simpler. Begin building the progress
                // event.
                var progress = {
                    type: HttpEventType.UploadProgress,
                    loaded: event.loaded,
                };
                // If the total number of bytes being uploaded is available, include
                // it.
                if (event.lengthComputable) {
                    progress.total = event.total;
                }
                // Send the event.
                observer.next(progress);
            }