upload metrics sequentially due to thrift frame size limit (15MB)
559388
private void uploadMetrics(TopologyMetric tpMetric) throws Exception {
        long start = System.currentTimeMillis();

        if (tpMetric == null) {
            return;
        }
        try {
            synchronized (lock) {
                if (client == null || !client.isValid()) {
                    client = new NimbusClientWrapper();
                    client.init(conf);
                }
            }

            MetricInfo topologyMetrics = tpMetric.get_topologyMetric();
            MetricInfo componentMetrics = tpMetric.get_componentMetric();
            MetricInfo taskMetrics = tpMetric.get_taskMetric();
            MetricInfo streamMetrics = tpMetric.get_streamMetric();
            MetricInfo workerMetrics = tpMetric.get_workerMetric();
            MetricInfo nettyMetrics = tpMetric.get_nettyMetric();
            MetricInfo compStreamMetrics = tpMetric.get_compStreamMetric();

            int totalSize = topologyMetrics.get_metrics_size()
                    + componentMetrics.get_metrics_size()
                    + taskMetrics.get_metrics_size()
                    + streamMetrics.get_metrics_size()
                    + workerMetrics.get_metrics_size()
                    + nettyMetrics.get_metrics_size()
                    + compStreamMetrics.get_metrics_size();

            // for small topologies, send all metrics together to ease the
            // pressure of nimbus
            if (totalSize < MAX_BATCH_SIZE) {
                TopologyMetric allMetrics = new TopologyMetric(
                        topologyMetrics, componentMetrics, workerMetrics, taskMetrics, streamMetrics, nettyMetrics);
                allMetrics.set_compStreamMetric(compStreamMetrics);

                client.getClient().uploadTopologyMetrics(topologyId, allMetrics);
            } else {
                TopologyMetric partialMetrics = new TopologyMetric(topologyMetrics, componentMetrics,
                        dummy, dummy, dummy, dummy);
                partialMetrics.set_compStreamMetric(compStreamMetrics);
                client.getClient().uploadTopologyMetrics(topologyId, partialMetrics);

                batchUploadMetrics(workerMetrics, MetaType.WORKER);
                batchUploadMetrics(taskMetrics, MetaType.TASK);
                batchUploadMetrics(streamMetrics, MetaType.STREAM);
                batchUploadMetrics(nettyMetrics, MetaType.NETTY);
            }
        } catch (Exception e) {
            String errorInfo = "Failed to upload worker metrics";
            LOG.error(errorInfo, e);
            if (client != null) {
                client.cleanup();
            }
            zkCluster.report_task_error(
                    context.getTopologyId(), context.getThisTaskId(), errorInfo, ErrorConstants.WARN, ErrorConstants.CODE_USER);
        }

        metricLogger.info("upload metrics, cost:{}", System.currentTimeMillis() - start);
    }
941663
def rapid_upload(self):
        '''
        '''
        info = pcs.rapid_upload(self.cookie, self.tokens,
                                self.row[SOURCEPATH_COL], self.row[PATH_COL],
                                self.upload_mode)
        if info and info['md5'] and info['fs_id']:
            self.emit('uploaded', self.row[FID_COL])
        else:
            self.slice_upload()
700354
@Config("thrift.max-frame-size")
    public ThriftServerConfig setMaxFrameSize(DataSize maxFrameSize)
    {
        checkArgument(maxFrameSize.toBytes() <= 0x3FFFFFFF);
        this.maxFrameSize = maxFrameSize;
        return this;
    }
1345914
def stream_upload(self, data, callback):
        
        while True:
            chunk = data.read(self.config.connection.data_block_size)
            if not chunk:
                break
            if callback and callable(callback):
                callback(chunk, response=None)
            yield chunk
663744
function checkRateLimitRemaining(res) {
  var remaining = parseInt(res.headers['x-ratelimit-remaining'], 10);
  if (remaining <= 50) {
    console.warn('github-content-api remote: only %d requests remaining.', remaining);
  }
}
823416
public void setUploadChunkSize(int chunkSize) {
    Preconditions.checkArgument(chunkSize > 0, "Upload chunk size must be great than 0.");
    Preconditions.checkArgument(
        chunkSize % MediaHttpUploader.MINIMUM_CHUNK_SIZE == 0,
        "Upload chunk size must be a multiple of MediaHttpUploader.MINIMUM_CHUNK_SIZE");
    if ((chunkSize > GCS_UPLOAD_GRANULARITY) && (chunkSize % GCS_UPLOAD_GRANULARITY != 0)) {
      logger.atWarning().log(
          "Upload chunk size should be a multiple of %s for best performance, got %s",
          GCS_UPLOAD_GRANULARITY, chunkSize);
    }
    uploadChunkSize = chunkSize;
  }
777735
def upload_chunk(resource, bytes, bytes_start, bytes_read)
      headers = {
        'content-type'   => content_type,
        'content-length' => bytes.size,
        'content-range'  => "bytes #{bytes_start}-#{bytes_read - 1}/#{@file_size}"
      }
      response = TwitterAds::Request.new(
        @client, :put, resource, domain: DEFAULT_DOMAIN, headers: headers, body: bytes).perform

      yield(response)
      response
    end
1059399
def multipart_upload_lister(bucket, key_marker='',
                            upload_id_marker='',
                            headers=None):
    
    more_results = True
    k = None
    while more_results:
        rs = bucket.get_all_multipart_uploads(key_marker=key_marker,
                                              upload_id_marker=upload_id_marker,
                                              headers=headers)
        for k in rs:
            yield k
        key_marker = rs.next_key_marker
        upload_id_marker = rs.next_upload_id_marker
        more_results= rs.is_truncated
1322497
def setUploadRateLimit(infoHash, bytesPerSecond)
      raise "upload rate limit must be an Integer, not a #{bytesPerSecond.class}" if bytesPerSecond && ! bytesPerSecond.is_a?(Integer)
      @handler.setUploadRateLimit(infoHash, bytesPerSecond)
    end
539299
def _flush(self):
        
        # Initialize multipart upload
        if self._upload_id is None:
            with _handle_oss_error():
                self._upload_id = self._bucket.init_multipart_upload(
                    self._key).upload_id

        # Upload part with workers
        response = self._workers.submit(
            self._bucket.upload_part, key=self._key, upload_id=self._upload_id,
            part_number=self._seek, data=self._get_buffer().tobytes())

        # Save part information
        self._write_futures.append(
            dict(response=response, part_number=self._seek))
490323
def add_size_info (self):
        """"""
        maxbytes = self.aggregate.config["maxfilesizedownload"]
        if self.size > maxbytes:
            self.add_warning(
              _("Content size %(size)s is larger than %(maxbytes)s.") %
                  dict(size=strformat.strsize(self.size),
                       maxbytes=strformat.strsize(maxbytes)),
                tag=WARN_URL_CONTENT_SIZE_TOO_LARGE)
177027
def upload_large_items(self):
        
        for local_file, parent in self.large_items:
            if local_file.need_to_send:
                self.process_large_file(local_file, parent)
296073
def estimate_size_in_bytes(cls, key, value, headers):
        
        return (
            cls.HEADER_STRUCT.size + cls.MAX_RECORD_OVERHEAD +
            cls.size_of(key, value, headers)
        )
623323
void updateContentSize(long frameLength, boolean last) {
        if(contentLengthRemaining != -1) {
            contentLengthRemaining -= frameLength;
            if(contentLengthRemaining < 0) {
                UndertowLogger.REQUEST_IO_LOGGER.debugf("Closing stream %s on %s as data length exceeds content size", streamId, getFramedChannel());
                getFramedChannel().sendRstStream(streamId, Http2Channel.ERROR_PROTOCOL_ERROR);
            } else if(last && contentLengthRemaining != 0) {
                UndertowLogger.REQUEST_IO_LOGGER.debugf("Closing stream %s on %s as data length was less than content size", streamId, getFramedChannel());
                getFramedChannel().sendRstStream(streamId, Http2Channel.ERROR_PROTOCOL_ERROR);
            }
        }
    }
88275
def start(self, buffer):
        """"""
        log.debug("In TftpContextServer.start")
        self.metrics.start_time = time.time()
        log.debug("Set metrics.start_time to %s", self.metrics.start_time)
        # And update our last updated time.
        self.last_update = time.time()

        pkt = self.factory.parse(buffer)
        log.debug("TftpContextServer.start() - factory returned a %s", pkt)

        # Call handle once with the initial packet. This should put us into
        # the download or the upload state.
        self.state = self.state.handle(pkt,
                                       self.host,
                                       self.port)
1066872
def upload(
    ctx, files, max_threads, prompt, forward, reverse, tags, metadata, project_id, coerce_ascii
):
    """"""

    appendables = {}
    if tags:
        appendables["tags"] = []
        for tag in tags:
            appendables["tags"].append(tag)

    if metadata:
        appendables["metadata"] = {}
        for metadata_kv in metadata:
            split_metadata = metadata_kv.split("=", 1)
            if len(split_metadata) > 1:
                metadata_value = split_metadata[1]
                appendables["metadata"][split_metadata[0]] = metadata_value

    appendables = validate_appendables(appendables, ctx.obj["API"])

    if (forward or reverse) and not (forward and reverse):
        click.echo("You must specify both forward and reverse files", err=True)
        ctx.exit(1)

    if forward and reverse:
        if len(files) > 0:
            click.echo(
                "You may not pass a FILES argument when using the "
                " --forward and --reverse options.",
                err=True,
            )
            ctx.exit(1)
        files = [(forward, reverse)]
    elif len(files) == 0:
        click.echo(ctx.get_help())
        return
    else:
        files = list(files)

        # "intelligently" find paired files and tuple them
        paired_files = []
        single_files = set(files)

        for filename in files:
            # convert "read 1" filenames into "read 2" and check that they exist; if they do
            # upload the files as a pair, autointerleaving them
            pair = re.sub("[._][Rr]1[._]", lambda x: x.group().replace("1", "2"), filename)

            # we don't necessary need the R2 to have been passed in; we infer it anyways
            if pair != filename and os.path.exists(pair):
                if not prompt and pair not in single_files:
                    # if we're not prompting, don't automatically pull in files
                    # not in the list the user passed in
                    continue

                paired_files.append((filename, pair))

                if pair in single_files:
                    single_files.remove(pair)

                single_files.remove(filename)

        auto_pair = True

        if prompt and len(paired_files) > 0:
            pair_list = ""
            for p in paired_files:
                pair_list += "\n  {}  &  {}".format(os.path.basename(p[0]), os.path.basename(p[1]))

            answer = click.confirm(
                "It appears there are paired files:{}\nInterleave them after upload?".format(
                    pair_list
                ),
                default="Y",
            )

            if not answer:
                auto_pair = False

        if auto_pair:
            files = paired_files + list(single_files)

    total_size = sum(
        [
            (_file_size(x[0], uncompressed=True) + _file_size(x[1], uncompressed=True))
            if isinstance(x, tuple)
            else _file_size(x, uncompressed=False)
            for x in files
        ]
    )

    upload_kwargs = {
        "metadata": appendables["valid_metadata"],
        "tags": appendables["valid_tags"],
        "project": project_id,
        "coerce_ascii": coerce_ascii,
        "progressbar": progressbar(length=total_size, label="Uploading..."),
    }

    run_via_threadpool(
        ctx.obj["API"].Samples.upload,
        files,
        upload_kwargs,
        max_threads=max_threads,
        graceful_exit=False,
    )
56321
def _write(self, data, length, progress_callback=None):
    """"""
    if progress_callback:
      progress = self._handle_progress(length, progress_callback)
      six.next(progress)
    while length:
      tmp = data.read(FASTBOOT_DOWNLOAD_CHUNK_SIZE_KB * 1024)
      length -= len(tmp)
      self.usb.write(tmp)

      if progress_callback:
        progress.send(len(tmp))